├── .editorconfig ├── .env.template ├── .github └── workflows │ ├── build_pages.yml │ ├── ci.yml │ ├── reviewdog.yml │ └── update_logs.yml ├── .gitignore ├── LICENSE.txt ├── Makefile ├── README.md ├── README_ja.md ├── go.mod ├── go.sum ├── internal ├── jsonwriter │ └── jsonwriter.go ├── slackadapter │ ├── common.go │ ├── conversations.go │ ├── conversations_history.go │ ├── cursor_iter.go │ └── users.go └── slacklog │ ├── channel.go │ ├── config.go │ ├── converter.go │ ├── doc.go │ ├── downloader.go │ ├── downloader_test.go │ ├── emoji.go │ ├── filetype.go │ ├── generator.go │ ├── indexer.go │ ├── json.go │ ├── main_test.go │ ├── message.go │ ├── slack.go │ ├── store.go │ ├── testdata │ └── downloader │ │ ├── empty.txt │ │ ├── hoge.txt │ │ └── vim-jp.png │ ├── thread.go │ ├── time.go │ ├── ts.go │ ├── ts_test.go │ └── user.go ├── main.go ├── scripts ├── build.sh ├── config.json ├── download_emoji.sh ├── download_files.sh ├── generate_html.sh └── site_diff.sh ├── static ├── assets │ ├── css │ │ └── site.css │ ├── images │ │ ├── favicon.ico │ │ └── vim2-128.png │ └── javascripts │ │ ├── .gitkeep │ │ ├── search.js │ │ └── slacklog.js └── search.html ├── subcmd ├── buildindex │ └── buildindex.go ├── convert_exported_logs.go ├── download_emoji.go ├── download_files.go ├── fetchchannels │ └── fetchchannels.go ├── fetchmessages │ └── fetchmessages.go ├── fetchusers │ └── fetchusers.go ├── generate_html.go └── serve │ └── serve.go └── templates ├── channel_index.tmpl ├── channel_per_month ├── attachment.tmpl └── index.tmpl └── index.tmpl /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | trim_trailing_whitespace = true 7 | charset = utf-8 8 | indent_style = space 9 | indent_size = tab 10 | tab_width = 2 11 | 12 | [*.{js,json,md,html,css}] 13 | indent_size = 2 14 | indent_style = space 15 | 16 | [*.go] 17 | tab_width = 4 18 | 19 | [Makefile] 20 | tab_width = 8 21 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | # 添付ファイルと絵文字のダウンロードに必要です 2 | SLACK_TOKEN= 3 | -------------------------------------------------------------------------------- /.github/workflows/build_pages.yml: -------------------------------------------------------------------------------- 1 | name: 'Build pages' 2 | on: 3 | push: 4 | branches: 5 | - '**' 6 | workflow_run: 7 | workflows: ['Update logs'] 8 | types: ['completed'] 9 | 10 | jobs: 11 | build-pages: 12 | name: 'Generate htdocs and Update https://vim-jp.org/slacklog/' 13 | if: "${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}" 14 | runs-on: 'ubuntu-latest' 15 | 16 | steps: 17 | - name: 'Checkout generator' 18 | uses: 'actions/checkout@v2' 19 | with: 20 | path: 'generator' 21 | 22 | - name: 'Checkout log-data' 23 | uses: 'actions/checkout@v2' 24 | with: 25 | repository: 'vim-jp/slacklog' 26 | path: 'data' 27 | ref: 'log-data' 28 | 29 | - name: 'Checkout gh-pages' 30 | uses: 'actions/checkout@v2' 31 | with: 32 | repository: 'vim-jp/slacklog' 33 | path: 'pages' 34 | ref: 'gh-pages' 35 | ssh-key: '${{ secrets.SLACKLOG_SSH_KEY }}' 36 | 37 | - name: 'Generate htdocs' 38 | run: | 39 | cp -r data/files/ data/emojis/ generator/static/* pages/ 40 | rm -fr data/files/ data/emojis/ 41 | cd generator 42 | BASEURL=/slacklog go run . generate-html --filesdir ../pages/files/ --indir ../data/slacklog_data/ --outdir ../pages/ 43 | go run . build-index --datadir ../data/slacklog_data --outdir ../pages/index 44 | # create finger print 45 | cd ../pages 46 | find . -type d -name '.git' -prune -o -type f -print0 | xargs -0 md5sum > ../files.txt 47 | 48 | - name: 'Save fingerprint' 49 | uses: actions/upload-artifact@v2 50 | with: 51 | name: fingerprint 52 | path: files.txt 53 | 54 | - name: 'Update https://vim-jp.org/slacklog/' 55 | if: github.ref == 'refs/heads/master' 56 | working-directory: './pages' 57 | run: | 58 | git add --all --intent-to-add --force 59 | if git diff --exit-code --quiet; then 60 | echo 'Nothing to update.' 61 | exit 0 62 | fi 63 | git config user.email "slacklog@vim-jp.org" 64 | git config user.name "Slack Log Generator" 65 | git commit --all --message 'Update pages' --quiet 66 | git push origin gh-pages --quiet 67 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - '**' 7 | 8 | jobs: 9 | 10 | build-tool: 11 | name: 'Build and Compile the Tool' 12 | runs-on: 'ubuntu-latest' 13 | 14 | steps: 15 | - name: Check out code 16 | uses: actions/checkout@v2 17 | 18 | - name: Setup Go into the Go module directory 19 | uses: actions/setup-go@v2 20 | with: 21 | go-version: 1.x 22 | 23 | - name: Build 24 | run: go build 25 | 26 | - name: Test 27 | run: go test . ./internal/... ./subcmd/... 28 | 29 | diff: 30 | name: 'Compare Site' 31 | runs-on: 'ubuntu-latest' 32 | 33 | steps: 34 | - name: 'Check out target branch (commit)' 35 | uses: 'actions/checkout@v2' 36 | with: 37 | # FIXME: 全部取るのはやりすぎ。必要最小限(mereg-base origin/master 38 | # HEAD)までだけ取れたら良いのだが… 39 | fetch-depth: 0 40 | 41 | - name: 'Preparations' 42 | run: | 43 | # origin/master の ref が比較のために要る 44 | git fetch origin master 45 | 46 | # log-data の取得と展開 47 | make logdata 48 | 49 | # 出力用のディレクトリ 50 | mkdir -p tmp 51 | 52 | - name: 'Compare site' 53 | run: | 54 | ./scripts/site_diff.sh -o tmp/site.diff 55 | 56 | - uses: actions/upload-artifact@v2 57 | with: 58 | name: diffs-${{ github.run_id }} 59 | path: tmp/site.diff 60 | 61 | - uses: actions/upload-artifact@v2 62 | with: 63 | name: log-site-diff-${{ github.run_id }} 64 | path: tmp/site_diff/*.log 65 | -------------------------------------------------------------------------------- /.github/workflows/reviewdog.yml: -------------------------------------------------------------------------------- 1 | name: reviewdog 2 | on: [pull_request] 3 | jobs: 4 | # NOTE: golangci-lint doesn't report multiple errors on the same line from 5 | # different linters and just report one of the errors? 6 | 7 | golangci-lint: 8 | name: runner / golangci-lint (pre-build docker image) 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Check out code into the Go module directory 12 | uses: actions/checkout@v2 13 | - name: golangci-lint 14 | # uses: ./ # Build with Dockerfile 15 | uses: docker://reviewdog/action-golangci-lint:v1 # Pre-built image 16 | with: 17 | github_token: ${{ secrets.github_token }} 18 | 19 | golangci-lint-all-in-one: 20 | name: runner / golangci-lint-all-in-one 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Check out code into the Go module directory 24 | uses: actions/checkout@v2 25 | - name: golangci-lint (All-In-One config) 26 | uses: docker://reviewdog/action-golangci-lint:v1 27 | with: 28 | github_token: ${{ secrets.github_token }} 29 | 30 | govet: 31 | name: runner / govet 32 | runs-on: ubuntu-latest 33 | steps: 34 | - name: Check out code into the Go module directory 35 | uses: actions/checkout@v2 36 | - name: govet 37 | uses: docker://reviewdog/action-golangci-lint:v1 38 | with: 39 | github_token: ${{ secrets.github_token }} 40 | tool_name: govet 41 | 42 | staticcheck: 43 | name: runner / staticcheck 44 | runs-on: ubuntu-latest 45 | steps: 46 | - name: Check out code into the Go module directory 47 | uses: actions/checkout@v2 48 | - name: staticcheck 49 | uses: docker://reviewdog/action-golangci-lint:v1 50 | with: 51 | github_token: ${{ secrets.github_token }} 52 | tool_name: staticcheck 53 | 54 | golint: 55 | name: runner / golint 56 | runs-on: ubuntu-latest 57 | steps: 58 | - name: Check out code into the Go module directory 59 | uses: actions/checkout@v2 60 | - name: golint 61 | uses: docker://reviewdog/action-golangci-lint:v1 62 | with: 63 | github_token: ${{ secrets.github_token }} 64 | tool_name: golint 65 | level: warning 66 | 67 | errcheck: 68 | name: runner / errcheck 69 | runs-on: ubuntu-latest 70 | steps: 71 | - name: Check out code into the Go module directory 72 | uses: actions/checkout@v2 73 | - name: errcheck 74 | uses: docker://reviewdog/action-golangci-lint:v1 75 | with: 76 | github_token: ${{ secrets.github_token }} 77 | tool_name: errcheck 78 | level: warning 79 | 80 | misspell: 81 | name: runner / misspell 82 | runs-on: ubuntu-latest 83 | steps: 84 | - name: Check out code into the Go module directory 85 | uses: actions/checkout@v2 86 | - name: misspell 87 | uses: docker://reviewdog/action-golangci-lint:v1 88 | with: 89 | github_token: ${{ secrets.github_token }} 90 | tool_name: misspell 91 | level: info 92 | 93 | shellcheck: 94 | name: runner / shellcheck 95 | runs-on: ubuntu-latest 96 | steps: 97 | - name: Check out code into the Go module directory 98 | uses: actions/checkout@v2 99 | - name: shellcheck 100 | uses: reviewdog/action-shellcheck@v1 101 | with: 102 | github_token: ${{ secrets.github_token }} 103 | level: info 104 | path: "scripts" 105 | pattern: "*.sh" 106 | -------------------------------------------------------------------------------- /.github/workflows/update_logs.yml: -------------------------------------------------------------------------------- 1 | name: 'Update logs' 2 | on: 3 | schedule: 4 | - cron: '0 19 * * *' # Every 4:00 am on JST 5 | workflow_dispatch: 6 | inputs: 7 | date: 8 | description: 'Date to update logs (YYYY-MM-DD) (empty to yesterday)' 9 | required: false 10 | default: '' 11 | 12 | jobs: 13 | update-logs: 14 | name: 'Update log-data of https://github.com/vim-jp/slacklog' 15 | runs-on: 'ubuntu-latest' 16 | 17 | steps: 18 | - name: 'Checkout generator' 19 | uses: 'actions/checkout@v2' 20 | with: 21 | path: 'generator' 22 | 23 | - name: 'Checkout log-data' 24 | uses: 'actions/checkout@v2' 25 | with: 26 | repository: 'vim-jp/slacklog' 27 | path: 'data' 28 | ref: 'log-data' 29 | ssh-key: '${{ secrets.SLACKLOG_SSH_KEY }}' 30 | 31 | - name: 'Update logs' 32 | working-directory: './generator' 33 | id: update-logs 34 | env: 35 | SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} 36 | run: | 37 | go run . fetch-users --datadir ../data/slacklog_data/ 38 | go run . fetch-channels --datadir ../data/slacklog_data/ 39 | date='${{ github.event.inputs.date }}' 40 | : "${date:=$(TZ=Asia/Tokyo date --date '1 day ago' --rfc-3339=date)}" 41 | go run . fetch-messages --datadir ../data/slacklog_data/ --date "${date}" 42 | 43 | go run . download-emoji --outdir ../data/emojis/ --emojiJSON ../data/slacklog_data/emoji.json 44 | go run . download-files --indir ../data/slacklog_data/ --outdir ../data/files/ 45 | 46 | echo "::set-output name=date::${date}" 47 | 48 | - name: 'Push logs' 49 | working-directory: './data' 50 | run: | 51 | git add --all --intent-to-add --force 52 | if git diff --exit-code --quiet; then 53 | echo 'Nothing to update.' 54 | exit 1 # Make fail to avoid triggering 'Build pages' workflow 55 | fi 56 | git config user.email "slacklog@vim-jp.org" 57 | git config user.name "Slack Log Generator" 58 | git commit --all --message 'Log data for ${{ steps.update-logs.outputs.date }}' --quiet 59 | git push origin log-data --quiet 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.swp 3 | /_site/ 4 | /tmp/ 5 | .env 6 | slacklog-generator 7 | *.exe 8 | /_logdata/ 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Attribution 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution 4.0 International Public License 58 | 59 | By exercising the Licensed Rights (defined below), You accept and agree 60 | to be bound by the terms and conditions of this Creative Commons 61 | Attribution 4.0 International Public License ("Public License"). To the 62 | extent this Public License may be interpreted as a contract, You are 63 | granted the Licensed Rights in consideration of Your acceptance of 64 | these terms and conditions, and the Licensor grants You such rights in 65 | consideration of benefits the Licensor receives from making the 66 | Licensed Material available under these terms and conditions. 67 | 68 | 69 | Section 1 -- Definitions. 70 | 71 | a. Adapted Material means material subject to Copyright and Similar 72 | Rights that is derived from or based upon the Licensed Material 73 | and in which the Licensed Material is translated, altered, 74 | arranged, transformed, or otherwise modified in a manner requiring 75 | permission under the Copyright and Similar Rights held by the 76 | Licensor. For purposes of this Public License, where the Licensed 77 | Material is a musical work, performance, or sound recording, 78 | Adapted Material is always produced where the Licensed Material is 79 | synched in timed relation with a moving image. 80 | 81 | b. Adapter's License means the license You apply to Your Copyright 82 | and Similar Rights in Your contributions to Adapted Material in 83 | accordance with the terms and conditions of this Public License. 84 | 85 | c. Copyright and Similar Rights means copyright and/or similar rights 86 | closely related to copyright including, without limitation, 87 | performance, broadcast, sound recording, and Sui Generis Database 88 | Rights, without regard to how the rights are labeled or 89 | categorized. For purposes of this Public License, the rights 90 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 91 | Rights. 92 | 93 | d. Effective Technological Measures means those measures that, in the 94 | absence of proper authority, may not be circumvented under laws 95 | fulfilling obligations under Article 11 of the WIPO Copyright 96 | Treaty adopted on December 20, 1996, and/or similar international 97 | agreements. 98 | 99 | e. Exceptions and Limitations means fair use, fair dealing, and/or 100 | any other exception or limitation to Copyright and Similar Rights 101 | that applies to Your use of the Licensed Material. 102 | 103 | f. Licensed Material means the artistic or literary work, database, 104 | or other material to which the Licensor applied this Public 105 | License. 106 | 107 | g. Licensed Rights means the rights granted to You subject to the 108 | terms and conditions of this Public License, which are limited to 109 | all Copyright and Similar Rights that apply to Your use of the 110 | Licensed Material and that the Licensor has authority to license. 111 | 112 | h. Licensor means the individual(s) or entity(ies) granting rights 113 | under this Public License. 114 | 115 | i. Share means to provide material to the public by any means or 116 | process that requires permission under the Licensed Rights, such 117 | as reproduction, public display, public performance, distribution, 118 | dissemination, communication, or importation, and to make material 119 | available to the public including in ways that members of the 120 | public may access the material from a place and at a time 121 | individually chosen by them. 122 | 123 | j. Sui Generis Database Rights means rights other than copyright 124 | resulting from Directive 96/9/EC of the European Parliament and of 125 | the Council of 11 March 1996 on the legal protection of databases, 126 | as amended and/or succeeded, as well as other essentially 127 | equivalent rights anywhere in the world. 128 | 129 | k. You means the individual or entity exercising the Licensed Rights 130 | under this Public License. Your has a corresponding meaning. 131 | 132 | 133 | Section 2 -- Scope. 134 | 135 | a. License grant. 136 | 137 | 1. Subject to the terms and conditions of this Public License, 138 | the Licensor hereby grants You a worldwide, royalty-free, 139 | non-sublicensable, non-exclusive, irrevocable license to 140 | exercise the Licensed Rights in the Licensed Material to: 141 | 142 | a. reproduce and Share the Licensed Material, in whole or 143 | in part; and 144 | 145 | b. produce, reproduce, and Share Adapted Material. 146 | 147 | 2. Exceptions and Limitations. For the avoidance of doubt, where 148 | Exceptions and Limitations apply to Your use, this Public 149 | License does not apply, and You do not need to comply with 150 | its terms and conditions. 151 | 152 | 3. Term. The term of this Public License is specified in Section 153 | 6(a). 154 | 155 | 4. Media and formats; technical modifications allowed. The 156 | Licensor authorizes You to exercise the Licensed Rights in 157 | all media and formats whether now known or hereafter created, 158 | and to make technical modifications necessary to do so. The 159 | Licensor waives and/or agrees not to assert any right or 160 | authority to forbid You from making technical modifications 161 | necessary to exercise the Licensed Rights, including 162 | technical modifications necessary to circumvent Effective 163 | Technological Measures. For purposes of this Public License, 164 | simply making modifications authorized by this Section 2(a) 165 | (4) never produces Adapted Material. 166 | 167 | 5. Downstream recipients. 168 | 169 | a. Offer from the Licensor -- Licensed Material. Every 170 | recipient of the Licensed Material automatically 171 | receives an offer from the Licensor to exercise the 172 | Licensed Rights under the terms and conditions of this 173 | Public License. 174 | 175 | b. No downstream restrictions. You may not offer or impose 176 | any additional or different terms or conditions on, or 177 | apply any Effective Technological Measures to, the 178 | Licensed Material if doing so restricts exercise of the 179 | Licensed Rights by any recipient of the Licensed 180 | Material. 181 | 182 | 6. No endorsement. Nothing in this Public License constitutes or 183 | may be construed as permission to assert or imply that You 184 | are, or that Your use of the Licensed Material is, connected 185 | with, or sponsored, endorsed, or granted official status by, 186 | the Licensor or others designated to receive attribution as 187 | provided in Section 3(a)(1)(A)(i). 188 | 189 | b. Other rights. 190 | 191 | 1. Moral rights, such as the right of integrity, are not 192 | licensed under this Public License, nor are publicity, 193 | privacy, and/or other similar personality rights; however, to 194 | the extent possible, the Licensor waives and/or agrees not to 195 | assert any such rights held by the Licensor to the limited 196 | extent necessary to allow You to exercise the Licensed 197 | Rights, but not otherwise. 198 | 199 | 2. Patent and trademark rights are not licensed under this 200 | Public License. 201 | 202 | 3. To the extent possible, the Licensor waives any right to 203 | collect royalties from You for the exercise of the Licensed 204 | Rights, whether directly or through a collecting society 205 | under any voluntary or waivable statutory or compulsory 206 | licensing scheme. In all other cases the Licensor expressly 207 | reserves any right to collect such royalties. 208 | 209 | 210 | Section 3 -- License Conditions. 211 | 212 | Your exercise of the Licensed Rights is expressly made subject to the 213 | following conditions. 214 | 215 | a. Attribution. 216 | 217 | 1. If You Share the Licensed Material (including in modified 218 | form), You must: 219 | 220 | a. retain the following if it is supplied by the Licensor 221 | with the Licensed Material: 222 | 223 | i. identification of the creator(s) of the Licensed 224 | Material and any others designated to receive 225 | attribution, in any reasonable manner requested by 226 | the Licensor (including by pseudonym if 227 | designated); 228 | 229 | ii. a copyright notice; 230 | 231 | iii. a notice that refers to this Public License; 232 | 233 | iv. a notice that refers to the disclaimer of 234 | warranties; 235 | 236 | v. a URI or hyperlink to the Licensed Material to the 237 | extent reasonably practicable; 238 | 239 | b. indicate if You modified the Licensed Material and 240 | retain an indication of any previous modifications; and 241 | 242 | c. indicate the Licensed Material is licensed under this 243 | Public License, and include the text of, or the URI or 244 | hyperlink to, this Public License. 245 | 246 | 2. You may satisfy the conditions in Section 3(a)(1) in any 247 | reasonable manner based on the medium, means, and context in 248 | which You Share the Licensed Material. For example, it may be 249 | reasonable to satisfy the conditions by providing a URI or 250 | hyperlink to a resource that includes the required 251 | information. 252 | 253 | 3. If requested by the Licensor, You must remove any of the 254 | information required by Section 3(a)(1)(A) to the extent 255 | reasonably practicable. 256 | 257 | 4. If You Share Adapted Material You produce, the Adapter's 258 | License You apply must not prevent recipients of the Adapted 259 | Material from complying with this Public License. 260 | 261 | 262 | Section 4 -- Sui Generis Database Rights. 263 | 264 | Where the Licensed Rights include Sui Generis Database Rights that 265 | apply to Your use of the Licensed Material: 266 | 267 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 268 | to extract, reuse, reproduce, and Share all or a substantial 269 | portion of the contents of the database; 270 | 271 | b. if You include all or a substantial portion of the database 272 | contents in a database in which You have Sui Generis Database 273 | Rights, then the database in which You have Sui Generis Database 274 | Rights (but not its individual contents) is Adapted Material; and 275 | 276 | c. You must comply with the conditions in Section 3(a) if You Share 277 | all or a substantial portion of the contents of the database. 278 | 279 | For the avoidance of doubt, this Section 4 supplements and does not 280 | replace Your obligations under this Public License where the Licensed 281 | Rights include other Copyright and Similar Rights. 282 | 283 | 284 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 285 | 286 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 287 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 288 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 289 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 290 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 291 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 292 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 293 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 294 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 295 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 296 | 297 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 298 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 299 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 300 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 301 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 302 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 303 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 304 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 305 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 306 | 307 | c. The disclaimer of warranties and limitation of liability provided 308 | above shall be interpreted in a manner that, to the extent 309 | possible, most closely approximates an absolute disclaimer and 310 | waiver of all liability. 311 | 312 | 313 | Section 6 -- Term and Termination. 314 | 315 | a. This Public License applies for the term of the Copyright and 316 | Similar Rights licensed here. However, if You fail to comply with 317 | this Public License, then Your rights under this Public License 318 | terminate automatically. 319 | 320 | b. Where Your right to use the Licensed Material has terminated under 321 | Section 6(a), it reinstates: 322 | 323 | 1. automatically as of the date the violation is cured, provided 324 | it is cured within 30 days of Your discovery of the 325 | violation; or 326 | 327 | 2. upon express reinstatement by the Licensor. 328 | 329 | For the avoidance of doubt, this Section 6(b) does not affect any 330 | right the Licensor may have to seek remedies for Your violations 331 | of this Public License. 332 | 333 | c. For the avoidance of doubt, the Licensor may also offer the 334 | Licensed Material under separate terms or conditions or stop 335 | distributing the Licensed Material at any time; however, doing so 336 | will not terminate this Public License. 337 | 338 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 339 | License. 340 | 341 | 342 | Section 7 -- Other Terms and Conditions. 343 | 344 | a. The Licensor shall not be bound by any additional or different 345 | terms or conditions communicated by You unless expressly agreed. 346 | 347 | b. Any arrangements, understandings, or agreements regarding the 348 | Licensed Material not stated herein are separate from and 349 | independent of the terms and conditions of this Public License. 350 | 351 | 352 | Section 8 -- Interpretation. 353 | 354 | a. For the avoidance of doubt, this Public License does not, and 355 | shall not be interpreted to, reduce, limit, restrict, or impose 356 | conditions on any use of the Licensed Material that could lawfully 357 | be made without permission under this Public License. 358 | 359 | b. To the extent possible, if any provision of this Public License is 360 | deemed unenforceable, it shall be automatically reformed to the 361 | minimum extent necessary to make it enforceable. If the provision 362 | cannot be reformed, it shall be severed from this Public License 363 | without affecting the enforceability of the remaining terms and 364 | conditions. 365 | 366 | c. No term or condition of this Public License will be waived and no 367 | failure to comply consented to unless expressly agreed to by the 368 | Licensor. 369 | 370 | d. Nothing in this Public License constitutes or may be interpreted 371 | as a limitation upon, or waiver of, any privileges and immunities 372 | that apply to the Licensor or You, including from the legal 373 | processes of any jurisdiction or authority. 374 | 375 | 376 | ======================================================================= 377 | 378 | Creative Commons is not a party to its public 379 | licenses. Notwithstanding, Creative Commons may elect to apply one of 380 | its public licenses to material it publishes and in those instances 381 | will be considered the “Licensor.” The text of the Creative Commons 382 | public licenses is dedicated to the public domain under the CC0 Public 383 | Domain Dedication. Except for the limited purpose of indicating that 384 | material is shared under a Creative Commons public license or as 385 | otherwise permitted by the Creative Commons policies published at 386 | creativecommons.org/policies, Creative Commons does not authorize the 387 | use of the trademark "Creative Commons" or any other trademark or logo 388 | of Creative Commons without its prior written consent including, 389 | without limitation, in connection with any unauthorized modifications 390 | to any of its public licenses or any other arrangements, 391 | understandings, or agreements concerning use of licensed material. For 392 | the avoidance of doubt, this paragraph does not form part of the 393 | public licenses. 394 | 395 | Creative Commons may be contacted at creativecommons.org. 396 | 397 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGES=./... 2 | 3 | .PHONY: generate 4 | generate: _site 5 | 6 | _site: _logdata $(wildcard scripts/**) $(wildcard templates/**) 7 | ./scripts/generate_html.sh 8 | ./scripts/build.sh 9 | touch -c _site 10 | 11 | .PHONY: clean 12 | clean: go-clean 13 | rm -rf _site 14 | 15 | .PHONY: distclean 16 | distclean: clean logdata-distclean 17 | 18 | ############################################################################## 19 | # Go 20 | 21 | .PHONY: build 22 | build: 23 | go build 24 | 25 | .PHONY: test 26 | test: 27 | go test ${PACKAGES} 28 | 29 | # cover - テストを実行してカバレッジを計測し結果を tmp/cover.html に出力する 30 | .PHONY: cover 31 | cover: 32 | go test -coverprofile tmp/cover.out ${PACKAGES} 33 | go tool cover -html tmp/cover.out -o tmp/cover.html 34 | 35 | .PHONY: vet 36 | vet: 37 | go vet ${PACKAGES} 38 | 39 | .PHONY: lint 40 | lint: 41 | golint ${PACKAGES} 42 | 43 | .PHONY: go-clean 44 | go-clean: 45 | go clean 46 | 47 | ############################################################################## 48 | # manage logdata 49 | 50 | .PHONY: logdata 51 | logdata: _logdata 52 | 53 | .PHONY: logdata-clean 54 | logdata-clean: 55 | rm -rf _logdata 56 | 57 | .PHONY: logdata-distclean 58 | logdata-distclean: logdata-clean 59 | rm -f tmp/log-data.tar.gz 60 | 61 | .PHONY: logdata-restore 62 | logdata-restore: logdata-clean logdata 63 | 64 | .PHONY: logdata-update 65 | logdata-update: logdata-distclean logdata 66 | 67 | _logdata: tmp/log-data.tar.gz 68 | rm -rf $@ 69 | mkdir -p $@ 70 | tar xz --strip-components=1 --exclude=.github -f tmp/log-data.tar.gz -C $@ 71 | 72 | tmp/log-data.tar.gz: 73 | mkdir -p tmp 74 | curl -Lo $@ https://github.com/vim-jp/slacklog/archive/log-data.tar.gz 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # slacklog 2 | 3 | ## What 4 | 5 | A project to htmlize vim-jp Slack logs. 6 | 7 | This solves the following problems due to using free tier 8 | 9 | * No old messages to see 10 | * Can't see unless you join the workspace, even though you'd like to refer from something such as your blogs 11 | * We lose our knowledge base 12 | 13 | ## How to contribute 14 | 15 | [@tyru](https://twitter.com/_tyru_) and [@thinca](https://twitter.com/thinca) will invite you to slacklog Team if you contact us via vim-jp Slack or Twitter. We'll share Slack token as well. 16 | 17 | How to join vim-jp Slack (Japanese): 18 | 19 | 20 | ## What you need to develop 21 | 22 | - Go 23 | - (Optional) GNU Make 24 | 25 | ## Env vars 26 | 27 | Create `.env` copiying `.env.template`. 28 | See the details for each env vars in the file. 29 | 30 | ## How to develop 31 | 32 | ### Generate HTML 33 | 34 | Unarchive logs 35 | 36 | ```console 37 | $ make logdata 38 | ``` 39 | 40 | Generate HTML 41 | 42 | The following commands will generate them under `_site` dir. 43 | 44 | ```console 45 | scripts/generate_html.sh 46 | scripts/build.sh 47 | ``` 48 | 49 | Or simply run `make` or `gmake` 50 | 51 | ### Download attached files and emojis 52 | 53 | ```console 54 | scripts/download_emoji.sh 55 | scripts/download_files.sh 56 | ``` 57 | 58 | ### Run dev server 59 | 60 | Use your favourite server under `_site` 61 | 62 | e.g. 63 | 64 | ```console 65 | python -m http.server --directory=_site 66 | ``` 67 | 68 | ### How to check the diff from geneate-html subcommands 69 | 70 | The generate-html output diff from your changes can be checked with this: 71 | 72 | ```console 73 | $ ./scripts/site_diff.sh 74 | ``` 75 | 76 | TODO translate the following 77 | 78 | > `site_diff.sh` では現在のHEADでの generate-html の結果と merge-base での 79 | > geneate-html の結果の diff を取得しています。 80 | > 出力先は `./tmp/site_diff/current/` および 81 | > `./tmp/site_diff/{merge-base-commit-id}/` ディレクトリとなっています。 82 | > 83 | > merge-base の算出基準はローカルの origin/master です。そのため origin/master が 84 | > リモート(GitHub)の物よりも古いと出力内容が異なり、差分も異なる場合があります。 85 | > `-u` オプションを使うと merge-base の算出前にローカルの origin/master を更新し 86 | > ます。変更がなくても更新にそれなりに時間がかかるため、デフォルトではオフになっ 87 | > ており明示的に指定するようにしています。 88 | > 89 | > merge-base の出力結果はキャッシュし再利用しています。このキャッシュを無視して強 90 | > 制的に再出力するには `-f` オプションを使ってください。 91 | > 92 | > ```console 93 | > $ ./scripts/site_diff.sh -f 94 | > ``` 95 | > 96 | > 全てのキャッシュを破棄したい場合には `-c` オプションを使ってください。`-c` オプ 97 | > ションでは `./tmp/site_diff/` ディレクトリを消すだけで差分の出力は行いません。 98 | > 99 | > ```console 100 | > $ ./scripts/site_diff.sh -c 101 | > ``` 102 | > 103 | > 差分だけを特定のファイルに出力するには `-o {filename}` オプションを使ってくださ 104 | > い。リダイレクト (` > filename`) では差分以外の動作ログも含まれる場合がありま 105 | > す。 106 | > 107 | > 注意事項: `./scripts/site_diff.sh` は未コミットな変更を stash を用いて保存・復 108 | 帰しているため staged な変更が unstaged に巻き戻ることに留意してください。 109 | 110 | ## How to update log-data 111 | 112 | TODO translate the following 113 | 114 | > log-data ブランチにはSlackからエクスポートしたデータを格納し、それを本番の生成 115 | > に利用しています。log-data ブランチの更新手順は以下の通りです。 116 | > 117 | > 1. Slack からログをエクスポート(今はできる人が限られてる) 118 | > 2. ログをワーキングスペースに展開する 119 | > 3. `convert-exported-logs` サブコマンドを実行する 120 | > 121 | > ```console 122 | > $ go run . convert-exported-logs {indir} {outdir} 123 | > ``` 124 | > 125 | > 4. 更新内容を log-data ブランチに `commit --amend` して `push -f` 126 | 127 | ## How to see the changes at Pull Request 128 | 129 | TODO translate the following 130 | 131 | > 以下の手順で Pull Request への `site_diff.sh` の実行結果を 132 | > Artifacts として Web から取得できます。レビューの際に利用してください。 133 | > 134 | > 1. Pull Request の Checks タブを開く 135 | > 2. CI ワークフロー(右側)を選択 136 | > 3. Compare Pages and Site ジョブ(右側)を選択 137 | > 4. Artifacts ドロワー(左側)を選択 138 | > 5. `diffs-{数値}` アーティファクトをダウンロード 139 | > 140 | > 以下のスクリーンショットは、上記の選択個所をマーキングしたものです。 141 | > (SSには3つのアーティファクトが表示されますが、現在は2つになっています) 142 | > 143 | > ![](https://raw.githubusercontent.com/wiki/vim-jp/slacklog-generator/images/where-are-artifacts.png) 144 | > 145 | > Artifacts はそれぞれ zip としてダウンロードできます。 146 | > `diffs-*.zip` には `sites_diff.sh` の差分が含まれています。 147 | > `log-*.zip` は動作ログが含まれていますが、こちらはCIの動作デバッグ目的のものです。 148 | > 末尾の数値は [`${{ github.run_id }}`](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context) 由来です。 149 | 150 | ### Why we output the diff at Artifacts 151 | 152 | TODO translate the following 153 | 154 | > Artifacts に差分を出力している主な理由は2つあります。1つ目は、小さな変更でも差 155 | > 分をオンライン上のどこかに出力しないと、レビューの負荷が高すぎてそれを解消した 156 | > かったという動機です。 157 | > 158 | > 2つ目は、テストデータとして実際のログを使っているため、差分とはいえログの一部の 159 | > コピーが消せない状態で永続化されるのを避けたい、という動機です。vim-jp slackで 160 | > は参加者の「忘れられる権利」を尊重しています。 161 | > 162 | > 以上の理由から消せる状態でデータ=差分をオンライン上にホストできる GitHub 163 | > Actions の Artifacts を利用しています。 164 | 165 | ## LICNESE 166 | 167 | TODO translate the following 168 | 169 | > クリエイティブ・コモンズ・ライセンス
この 作品 は クリエイティブ・コモンズ 表示 4.0 国際 ライセンスの下に提供されています。 170 | -------------------------------------------------------------------------------- /README_ja.md: -------------------------------------------------------------------------------- 1 | # slacklog 2 | 3 | ## これは何 4 | 5 | vim-jp Slack のログを HTML 化するプロジェクトです。 6 | 7 | Slack 上では無料枠のため 8 | 9 | * 古いメッセージが見れない 10 | * ワークスペースに参加していない人には見えないため、ブログ等から引用などして参照したい 11 | * 知見が消えるのはもったいない 12 | 13 | といった問題があり、それらを解決するため作られました。 14 | 15 | ## 開発に参加するには 16 | 17 | 興味を持った方は vim-jp Slack や [@tyru](https://twitter.com/_tyru_), [@thinca](https://twitter.com/thinca) 等に声をかけて頂ければ GitHub の slacklog Team に招待します (Slack token などもその際共有します)。
18 | vim-jp Slack への参加方法はこちらをどうぞ。
19 | [vim-jp » vim-jpのチャットルームについて](https://vim-jp.org/docs/chat.html) 20 | 21 | ## 開発に必要なもの 22 | 23 | - Go 24 | - (あれば)GNU Make 25 | 26 | ## 環境変数 27 | 28 | `.env.template` からコピーして `.env` ファイルを作成してください。 29 | (各環境変数の説明はファイルを参照) 30 | 31 | ## 開発方法 32 | 33 | ### HTML の生成 34 | 35 | ログを展開 36 | 37 | ```console 38 | $ make logdata 39 | ``` 40 | 41 | HTMLの生成 42 | 43 | 以下のコマンドを実行すると`_site`以下に生成されます 44 | 45 | ```console 46 | scripts/generate_html.sh 47 | scripts/build.sh 48 | ``` 49 | 50 | GNU Makeがあれば`make`もしくは`gmake`を実行するだけで生成されます 51 | 52 | ### 添付ファイルと絵文字のダウンロード 53 | 54 | ```console 55 | scripts/download_emoji.sh 56 | scripts/download_files.sh 57 | ``` 58 | 59 | ### 開発サーバーの起動 60 | 61 | 特定のツールに依存していないので、各自お好きなサーバーを`_site`以下で起動してください 62 | 63 | 開発サーバーの起動(例): 64 | 65 | ```console 66 | python -m http.server --directory=_site 67 | ``` 68 | 69 | ### geneate-html サブコマンドの出力の差分の確認方法 70 | 71 | 以下のコマンドで自分が変更した結果として生じた generate-html の出力内容の差分 72 | を確認できます。 73 | 74 | ```console 75 | $ ./scripts/site_diff.sh 76 | ``` 77 | 78 | `site_diff.sh` では現在のHEADでの generate-html の結果と merge-base での 79 | geneate-html の結果の diff を取得しています。 80 | 出力先は `./tmp/site_diff/current/` および 81 | `./tmp/site_diff/{merge-base-commit-id}/` ディレクトリとなっています。 82 | 83 | merge-base の算出基準はローカルの origin/master です。そのため origin/master が 84 | リモート(GitHub)の物よりも古いと出力内容が異なり、差分も異なる場合があります。 85 | `-u` オプションを使うと merge-base の算出前にローカルの origin/master を更新し 86 | ます。変更がなくても更新にそれなりに時間がかかるため、デフォルトではオフになっ 87 | ており明示的に指定するようにしています。 88 | 89 | merge-base の出力結果はキャッシュし再利用しています。このキャッシュを無視して強 90 | 制的に再出力するには `-f` オプションを使ってください。 91 | 92 | ```console 93 | $ ./scripts/site_diff.sh -f 94 | ``` 95 | 96 | 全てのキャッシュを破棄したい場合には `-c` オプションを使ってください。`-c` オプ 97 | ションでは `./tmp/site_diff/` ディレクトリを消すだけで差分の出力は行いません。 98 | 99 | ```console 100 | $ ./scripts/site_diff.sh -c 101 | ``` 102 | 103 | 差分だけを特定のファイルに出力するには `-o {filename}` オプションを使ってくださ 104 | い。リダイレクト (` > filename`) では差分以外の動作ログも含まれる場合がありま 105 | す。 106 | 107 | 注意事項: `./scripts/site_diff.sh` は未コミットな変更を stash を用いて保存・復 108 | 帰しているため staged な変更が unstaged に巻き戻ることに留意してください。 109 | 110 | ## log-data の更新手順 111 | 112 | log-data ブランチにはSlackからエクスポートしたデータを格納し、それを本番の生成 113 | に利用しています。log-data ブランチの更新手順は以下の通りです。 114 | 115 | 1. Slack からログをエクスポート(今はできる人が限られてる) 116 | 2. ログをワーキングスペースに展開する 117 | 3. `convert-exported-logs` サブコマンドを実行する 118 | 119 | ```console 120 | $ go run . convert-exported-logs {indir} {outdir} 121 | ``` 122 | 123 | 4. 更新内容を log-data ブランチに `commit --amend` して `push -f` 124 | 125 | ## Pull Request の影響の確認の方法 126 | 127 | 以下の手順で Pull Request への `site_diff.sh` の実行結果を 128 | Artifacts として Web から取得できます。レビューの際に利用してください。 129 | 130 | 1. Pull Request の Checks タブを開く 131 | 2. CI ワークフロー(右側)を選択 132 | 3. Compare Pages and Site ジョブ(右側)を選択 133 | 4. Artifacts ドロワー(左側)を選択 134 | 5. `diffs-{数値}` アーティファクトをダウンロード 135 | 136 | 以下のスクリーンショットは、上記の選択個所をマーキングしたものです。 137 | (SSには3つのアーティファクトが表示されますが、現在は2つになっています) 138 | 139 | ![](https://raw.githubusercontent.com/wiki/vim-jp/slacklog-generator/images/where-are-artifacts.png) 140 | 141 | Artifacts はそれぞれ zip としてダウンロードできます。 142 | `diffs-*.zip` には `sites_diff.sh` の差分が含まれています。 143 | `log-*.zip` は動作ログが含まれていますが、こちらはCIの動作デバッグ目的のものです。 144 | 末尾の数値は [`${{ github.run_id }}`](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context) 由来です。 145 | 146 | ### Artifacts に差分を出力している理由 147 | 148 | Artifacts に差分を出力している主な理由は2つあります。1つ目は、小さな変更でも差 149 | 分をオンライン上のどこかに出力しないと、レビューの負荷が高すぎてそれを解消した 150 | かったという動機です。 151 | 152 | 2つ目は、テストデータとして実際のログを使っているため、差分とはいえログの一部の 153 | コピーが消せない状態で永続化されるのを避けたい、という動機です。vim-jp slackで 154 | は参加者の「忘れられる権利」を尊重しています。 155 | 156 | 以上の理由から消せる状態でデータ=差分をオンライン上にホストできる GitHub 157 | Actions の Artifacts を利用しています。 158 | 159 | ## LICNESE 160 | 161 | クリエイティブ・コモンズ・ライセンス
この 作品 は クリエイティブ・コモンズ 表示 4.0 国際 ライセンスの下に提供されています。 162 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vim-jp/slacklog-generator 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/davecgh/go-spew v1.1.1 // indirect 7 | github.com/google/go-cmp v0.4.0 8 | github.com/gorilla/websocket v1.4.2 // indirect 9 | github.com/joho/godotenv v1.3.0 10 | github.com/kyokomi/emoji v2.2.2+incompatible 11 | github.com/pkg/errors v0.9.1 // indirect 12 | github.com/slack-go/slack v0.6.4 13 | github.com/urfave/cli/v2 v2.2.0 14 | ) 15 | 16 | //replace github.com/slack-go/slack => ../slacklog-slack 17 | replace github.com/slack-go/slack => github.com/vim-jp/slacklog-slack v0.0.0-20200516060239-575febb4155b 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= 7 | github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= 8 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 9 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 10 | github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ= 11 | github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 12 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 13 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 14 | github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= 15 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 16 | github.com/kyokomi/emoji v2.2.2+incompatible h1:gaQFbK2+uSxOR4iGZprJAbpmtqTrHhSdgOyIMD6Oidc= 17 | github.com/kyokomi/emoji v2.2.2+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= 18 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 19 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 20 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 21 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 22 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 23 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 24 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 25 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 26 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 27 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 28 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 29 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 30 | github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= 31 | github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= 32 | github.com/vim-jp/slacklog-slack v0.0.0-20200516060239-575febb4155b h1:TpL0ddVABxueZruJj/prrtILmnnipggBrAoEQelmWuI= 33 | github.com/vim-jp/slacklog-slack v0.0.0-20200516060239-575febb4155b/go.mod h1:sGRjv3w+ERAUMMMbldHObQPBcNSyVB7KLKYfnwUFBfw= 34 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 35 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 36 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 37 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 38 | -------------------------------------------------------------------------------- /internal/jsonwriter/jsonwriter.go: -------------------------------------------------------------------------------- 1 | package jsonwriter 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | ) 7 | 8 | // WriteCloser writes objects as JSON array. It provides persistent layer for JSON value. 9 | type WriteCloser interface { 10 | Write(interface{}) error 11 | Close() error 12 | } 13 | 14 | type fileWriter struct { 15 | name string 16 | reverse bool 17 | 18 | // FIXME: いったん全部メモリにためるのであんまよくない 19 | buf []interface{} 20 | } 21 | 22 | func (fw *fileWriter) Write(v interface{}) error { 23 | // XXX: 排他してないのでgoroutineからは使えない 24 | fw.buf = append(fw.buf, v) 25 | return nil 26 | } 27 | 28 | func (fw *fileWriter) Close() error { 29 | if len(fw.buf) == 0 { 30 | return nil 31 | } 32 | // FIXME: ファイルの作成が Close まで遅延している。本来なら CreateFile のタ 33 | // イミングでやるのが好ましいが、いましばらく目を瞑る 34 | f, err := os.Create(fw.name) 35 | if err != nil { 36 | return err 37 | } 38 | defer f.Close() 39 | if fw.reverse { 40 | reverse(fw.buf) 41 | fw.reverse = false 42 | } 43 | err = json.NewEncoder(f).Encode(fw.buf) 44 | if err != nil { 45 | return err 46 | } 47 | fw.buf = nil 48 | return nil 49 | } 50 | 51 | // CreateFile creates a WriteCloser which implemented by file. 52 | func CreateFile(name string, reverse bool) (WriteCloser, error) { 53 | return &fileWriter{name: name}, nil 54 | } 55 | 56 | func reverse(x []interface{}) { 57 | for i, j := 0, len(x)-1; i < j; { 58 | x[i], x[j] = x[j], x[i] 59 | i++ 60 | j-- 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /internal/slackadapter/common.go: -------------------------------------------------------------------------------- 1 | package slackadapter 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // Error represents error response of Slack. 9 | type Error struct { 10 | Ok bool `json:"ok"` 11 | Err string `json:"error"` 12 | } 13 | 14 | // Error returns error message. 15 | func (err *Error) Error() string { 16 | return err.Err 17 | } 18 | 19 | // NextCursor is cursor for next request. 20 | type NextCursor struct { 21 | NextCursor Cursor `json:"next_cursor"` 22 | } 23 | 24 | // Cursor is type of cursor of Slack API. 25 | type Cursor string 26 | 27 | // Timestamp converts time.Time to timestamp formed for 28 | // Slack API (.) 29 | func Timestamp(t *time.Time) string { 30 | if t == nil { 31 | return "" 32 | } 33 | return fmt.Sprintf("%d.%6d", t.Unix(), t.Nanosecond()/1000) 34 | } 35 | -------------------------------------------------------------------------------- /internal/slackadapter/conversations.go: -------------------------------------------------------------------------------- 1 | package slackadapter 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | 7 | "github.com/slack-go/slack" 8 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 9 | ) 10 | 11 | // ConversationsParams is optional parameters for Conversations 12 | type ConversationsParams struct { 13 | Cursor Cursor `json:"cursor,omitempty"` 14 | Limit int `json:"limit,omitempty"` 15 | ExcludeArchived bool `json:"excludeArchived,omitempty"` 16 | Types []string `json:"types,omitempty"` 17 | } 18 | 19 | // ConversationsResponse is response for Conversations 20 | type ConversationsResponse struct { 21 | Ok bool `json:"ok"` 22 | Channels []*slacklog.Channel `json:"channels,omitempty"` 23 | ResponseMetadata *NextCursor `json:"response_metadata"` 24 | } 25 | 26 | // Conversations gets conversation channels in a channel. 27 | func Conversations(ctx context.Context, token string, params ConversationsParams) (*ConversationsResponse, error) { 28 | client := slack.New(token) 29 | channels, nextCursor, err := client.GetConversationsContext(ctx, &slack.GetConversationsParameters{ 30 | Cursor: string(params.Cursor), 31 | Limit: params.Limit, 32 | ExcludeArchived: strconv.FormatBool(params.ExcludeArchived), 33 | Types: params.Types, 34 | }) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | var logChannels []*slacklog.Channel 40 | for _, c := range channels { 41 | logChannels = append(logChannels, &slacklog.Channel{ 42 | Channel: c, 43 | }) 44 | } 45 | 46 | res := &ConversationsResponse{ 47 | Ok: true, 48 | Channels: logChannels, 49 | } 50 | if nextCursor != "" { 51 | res.ResponseMetadata = &NextCursor{ 52 | NextCursor: Cursor(nextCursor), 53 | } 54 | } 55 | return res, nil 56 | } 57 | -------------------------------------------------------------------------------- /internal/slackadapter/conversations_history.go: -------------------------------------------------------------------------------- 1 | package slackadapter 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/slack-go/slack" 8 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 9 | ) 10 | 11 | // ConversationsHistoryParams is optional parameters for ConversationsHistory 12 | type ConversationsHistoryParams struct { 13 | Cursor Cursor `json:"cursor,omitempty"` 14 | Inclusive bool `json:"inclusive,omitempty"` 15 | Latest *time.Time `json:"latest,omitempty"` 16 | Limit int `json:"limit,omitempty"` 17 | Oldest *time.Time `json:"oldest,omitempty"` 18 | } 19 | 20 | // ConversationsHistoryResponse is response for ConversationsHistory 21 | type ConversationsHistoryResponse struct { 22 | Ok bool `json:"ok"` 23 | Messages []*slacklog.Message `json:"messages,omitempty"` 24 | HasMore bool `json:"has_more"` 25 | PinCount int `json:"pin_count"` 26 | ResponseMetadata *NextCursor `json:"response_metadata"` 27 | } 28 | 29 | // ConversationsHistory gets conversation messages in a channel. 30 | func ConversationsHistory(ctx context.Context, token, channel string, params ConversationsHistoryParams) (*ConversationsHistoryResponse, error) { 31 | client := slack.New(token) 32 | res, err := client.GetConversationHistoryContext(ctx, &slack.GetConversationHistoryParameters{ 33 | ChannelID: channel, 34 | Cursor: string(params.Cursor), 35 | Limit: params.Limit, 36 | Oldest: Timestamp(params.Oldest), 37 | Latest: Timestamp(params.Latest), 38 | Inclusive: params.Inclusive, 39 | }) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | var messages []*slacklog.Message 45 | for _, m := range res.Messages { 46 | messages = append(messages, &slacklog.Message{ 47 | Message: m, 48 | }) 49 | } 50 | 51 | return &ConversationsHistoryResponse{ 52 | Ok: true, 53 | Messages: messages, 54 | HasMore: res.HasMore, 55 | PinCount: res.PinCount, 56 | ResponseMetadata: &NextCursor{ 57 | NextCursor: Cursor(res.ResponseMetaData.NextCursor), 58 | }, 59 | }, nil 60 | } 61 | -------------------------------------------------------------------------------- /internal/slackadapter/cursor_iter.go: -------------------------------------------------------------------------------- 1 | package slackadapter 2 | 3 | import "context" 4 | 5 | // CursorIterator is requirements of IterateCursor iterates with cursor. 6 | type CursorIterator interface { 7 | Iterate(context.Context, Cursor) (Cursor, error) 8 | } 9 | 10 | // CursorIteratorFunc is a function which implements CursorIterator. 11 | type CursorIteratorFunc func(context.Context, Cursor) (Cursor, error) 12 | 13 | // Iterate is an implementation for CursorIterator. 14 | func (fn CursorIteratorFunc) Iterate(ctx context.Context, c Cursor) (Cursor, error) { 15 | return fn(ctx, c) 16 | } 17 | 18 | // IterateCursor iterates CursorIterator until returning empty cursor. 19 | func IterateCursor(ctx context.Context, iter CursorIterator) error { 20 | var c Cursor 21 | for { 22 | err := ctx.Err() 23 | if err != nil { 24 | return err 25 | } 26 | next, err := iter.Iterate(ctx, c) 27 | if err != nil { 28 | return err 29 | } 30 | if next == Cursor("") { 31 | return nil 32 | } 33 | c = next 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/slackadapter/users.go: -------------------------------------------------------------------------------- 1 | package slackadapter 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/slack-go/slack" 7 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 8 | ) 9 | 10 | // UsersResponse is response for Conversations 11 | type UsersResponse struct { 12 | Ok bool `json:"ok"` 13 | Users []*slacklog.User `json:"users,omitempty"` 14 | } 15 | 16 | // Users gets users. 17 | func Users(ctx context.Context, token string) ([]*slacklog.User, error) { 18 | client := slack.New(token) 19 | users, err := client.GetUsersContext(ctx) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | var logUsers []*slacklog.User 25 | for _, u := range users { 26 | lu := slacklog.User(u) 27 | logUsers = append(logUsers, &lu) 28 | } 29 | 30 | return logUsers, nil 31 | } 32 | -------------------------------------------------------------------------------- /internal/slacklog/channel.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/slack-go/slack" 7 | ) 8 | 9 | // ChannelTable : チャンネルデータを保持する。 10 | // channelsもchannelMapも保持するチャンネルデータは同じで、channelMapはチャンネ 11 | // ルIDをキーとするmapとなっている。 12 | // ユースケースに応じてchannelsとchannelMapは使い分ける。 13 | type ChannelTable struct { 14 | Channels []Channel 15 | ChannelMap map[string]*Channel 16 | } 17 | 18 | // NewChannelTable : pathに指定したJSON形式のチャンネルデータを読み込み、 19 | // ChannelTable を生成する。 20 | // whitelistに指定したチャンネル名のみを読み込む。 21 | func NewChannelTable(path string, whitelist []string) (*ChannelTable, error) { 22 | var channels []Channel 23 | if err := ReadFileAsJSON(path, true, &channels); err != nil { 24 | return nil, err 25 | } 26 | channels = FilterChannel(channels, whitelist) 27 | sort.Slice(channels, func(i, j int) bool { 28 | return channels[i].Name < channels[j].Name 29 | }) 30 | channelMap := make(map[string]*Channel, len(channels)) 31 | for i, ch := range channels { 32 | channelMap[ch.ID] = &channels[i] 33 | } 34 | return &ChannelTable{ 35 | Channels: channels, 36 | ChannelMap: channelMap, 37 | }, nil 38 | } 39 | 40 | // FilterChannel : whitelistに指定したチャンネル名に該当するチャンネルのみを返 41 | // す。 42 | // whitelistに'*'が含まれる場合はchannelをそのまま返す。 43 | func FilterChannel(channels []Channel, whitelist []string) []Channel { 44 | if len(whitelist) == 0 { 45 | return []Channel{} 46 | } 47 | allowed := map[string]struct{}{} 48 | for _, s := range whitelist { 49 | if s == "*" { 50 | return channels 51 | } 52 | allowed[s] = struct{}{} 53 | } 54 | newChannels := make([]Channel, 0, len(whitelist)) 55 | for _, ch := range channels { 56 | _, ok := allowed[ch.Name] 57 | if ok { 58 | newChannels = append(newChannels, ch) 59 | } 60 | } 61 | return newChannels 62 | } 63 | 64 | // SortChannel sorts []Channel by name. It modify original slice. 65 | func SortChannel(channels []Channel) { 66 | sort.SliceStable(channels, func(i, j int) bool { 67 | return channels[i].Name < channels[j].Name 68 | }) 69 | } 70 | 71 | // Channel represents channel object in Slack. 72 | type Channel struct { 73 | slack.Channel 74 | 75 | Pins []ChannelPin `json:"pins"` 76 | } 77 | 78 | // ChannelPin represents a pinned message for a channel. 79 | type ChannelPin struct { 80 | ID string `json:"id"` 81 | Typ string `json:"type"` 82 | Created int64 `json:"created"` 83 | User string `json:"user"` 84 | Owner string `json:"owner"` 85 | } 86 | -------------------------------------------------------------------------------- /internal/slacklog/config.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | // Config : ログ出力時の設定を保持する。 4 | type Config struct { 5 | EditedSuffix string `json:"edited_suffix"` 6 | Channels []string `json:"channels"` 7 | EmojiJSONPath string `json:"emoji_json_path"` 8 | } 9 | 10 | // ReadConfig : pathに指定したファイルからコンフィグを読み込む。 11 | func ReadConfig(path string) (*Config, error) { 12 | var cfg Config 13 | if err := ReadFileAsJSON(path, true, &cfg); err != nil { 14 | return nil, err 15 | } 16 | return &cfg, nil 17 | } 18 | -------------------------------------------------------------------------------- /internal/slacklog/converter.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "html" 5 | "net/url" 6 | "os" 7 | "regexp" 8 | "strings" 9 | 10 | "github.com/kyokomi/emoji" 11 | ) 12 | 13 | // TextConverter : markdown形式のテキストをHTMLに変換するための構造体。 14 | type TextConverter struct { 15 | // key: emoji name 16 | // value: emoji URL 17 | emojis map[string]string 18 | // key: user ID 19 | // value: display name 20 | users map[string]string 21 | re regexps 22 | // baseURL is root path for public site, configured by `BASEURL` environment variable. 23 | baseURL string 24 | } 25 | 26 | // NewTextConverter : TextConverter を生成する 27 | func NewTextConverter(users, emojis map[string]string) *TextConverter { 28 | re := regexps{} 29 | // TODO tokenize/parse message.Text 30 | re.linkWithTitle = regexp.MustCompile(`<(https?://[^>]+?)\|(.+?)>`) 31 | re.link = regexp.MustCompile(`<(https?://[^>]+?)>`) 32 | // go regexp does not support back reference 33 | re.code = regexp.MustCompile("`{3}|｀{3}") 34 | re.codeShort = regexp.MustCompile("[`｀]([^`]+?)[`｀]") 35 | re.del = regexp.MustCompile(`~([^~]+?)~`) 36 | re.mention = regexp.MustCompile(`<@(\w+?)>`) 37 | re.channel = regexp.MustCompile(`<#([^|]+?)\|([^&]+?)>`) 38 | re.emoji = regexp.MustCompile(`:[^\s!"#$%&()=^/?\\\[\]<>,.;@{}~:]+:`) 39 | re.newLine = regexp.MustCompile(`\n`) 40 | 41 | return &TextConverter{ 42 | emojis: emojis, 43 | users: users, 44 | re: re, 45 | baseURL: os.Getenv("BASEURL"), 46 | } 47 | } 48 | 49 | type regexps struct { 50 | linkWithTitle, link, 51 | code, codeShort, 52 | del, 53 | mention, channel, emoji, 54 | newLine *regexp.Regexp 55 | } 56 | 57 | func (c *TextConverter) escapeSpecialChars(text string) string { 58 | text = html.EscapeString(html.UnescapeString(text)) 59 | text = strings.Replace(text, "{{", "{{", -1) 60 | return strings.Replace(text, "{%", "{%", -1) 61 | } 62 | 63 | func (c *TextConverter) escape(text string) string { 64 | text = html.EscapeString(html.UnescapeString(text)) 65 | text = c.re.newLine.ReplaceAllString(text, " ") 66 | return text 67 | } 68 | 69 | func (c *TextConverter) bindEmoji(emojiExp string) string { 70 | name := emojiExp[1 : len(emojiExp)-1] 71 | extension, ok := c.emojis[name] 72 | if !ok { 73 | char, ok := emoji.CodeMap()[emojiExp] 74 | if ok { 75 | return char 76 | } 77 | return emojiExp 78 | } 79 | for 7 <= len(extension) && extension[:6] == "alias:" { 80 | name = extension[6:] 81 | extension, ok = c.emojis[name] 82 | if !ok { 83 | return emojiExp 84 | } 85 | } 86 | src := c.baseURL + "/emojis/" + url.PathEscape(name) + extension 87 | return "" + emojiExp + "" 88 | } 89 | 90 | func (c *TextConverter) bindUser(userExp string) string { 91 | m := c.re.mention.FindStringSubmatch(userExp) 92 | if name := c.users[m[1]]; name != "" { 93 | return "@" + name 94 | } 95 | return userExp 96 | } 97 | 98 | func (c *TextConverter) bindChannel(channelExp string) string { 99 | matchResult := c.re.channel.FindStringSubmatch(channelExp) 100 | channelID := matchResult[1] 101 | channelName := matchResult[2] 102 | return "#" + channelName + "" 103 | } 104 | 105 | // ToHTML : markdown形式のtextをHTMLに変換する 106 | func (c *TextConverter) ToHTML(text string) string { 107 | text = c.escapeSpecialChars(text) 108 | text = c.re.newLine.ReplaceAllString(text, "
") 109 | chunks := c.re.code.Split(text, -1) 110 | for i, s := range chunks { 111 | if i%2 == 0 { 112 | s = c.re.linkWithTitle.ReplaceAllString(s, "${2}") 113 | s = c.re.link.ReplaceAllString(s, "${1}") 114 | s = c.re.codeShort.ReplaceAllString(s, "${1}") 115 | s = c.re.del.ReplaceAllString(s, "${1}") 116 | s = c.re.emoji.ReplaceAllStringFunc(s, c.bindEmoji) 117 | s = c.re.mention.ReplaceAllStringFunc(s, c.bindUser) 118 | s = c.re.channel.ReplaceAllStringFunc(s, c.bindChannel) 119 | } else { 120 | s = "
" + s + "
" 121 | } 122 | chunks[i] = s 123 | } 124 | return strings.Join(chunks, "") 125 | } 126 | -------------------------------------------------------------------------------- /internal/slacklog/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package slacklog はSlackからエクスポートされた各チャンネルのログの取得、HTMLへ 3 | の変換を行なうためのパッケージである。 4 | 5 | LogStoreはログデータの取得方法を規定し、必要に応じて各種ログテーブルからデータ 6 | を取得する。 7 | 8 | ChannelTable/MessageTable/UserTable/EmojiTableはSlackからエクスポートされたJSON 9 | 形式のログファイルを読み込み、LogStoreが処理しやすい形でデータを保持する。 10 | 11 | TextConverterはログが保持しているテキストのエスケープやHTMLへの変換を行なう。 12 | 13 | HTMLGeneratorはLogStoreから取得し、TextConverterで変換したデータを、 14 | text/templateパッケージを用いてHTMLとして出力する。 15 | */ 16 | package slacklog 17 | -------------------------------------------------------------------------------- /internal/slacklog/downloader.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net" 7 | "net/http" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | // Downloader : ダウンロード処理をするワーカを管理するための構造体。 15 | // TODO: 今のところ、ダウンロード処理中にエラーが発生してもキューに積まれたタス 16 | // クが全て完了するまでは分からない(Wait()の返り値として見るまでは)。 17 | type Downloader struct { 18 | token string 19 | 20 | httpClient *http.Client 21 | targetCh chan downloadTarget 22 | workerWg sync.WaitGroup 23 | 24 | // firstError stores the first error raised in workers. 25 | firstError error 26 | 27 | // errsMu: firstError に触る際は必ずコレでロックを取る 28 | errsMu sync.Mutex 29 | } 30 | 31 | var downloadWorkerNum = 8 32 | 33 | // NewDownloader creates a downloader for Slack with the token. 34 | func NewDownloader(token string) *Downloader { 35 | // http.DefaultTransportの値からMaxConnsPerHostのみ修正 36 | t := &http.Transport{ 37 | Proxy: http.ProxyFromEnvironment, 38 | DialContext: (&net.Dialer{ 39 | Timeout: 30 * time.Second, 40 | KeepAlive: 30 * time.Second, 41 | DualStack: true, 42 | }).DialContext, 43 | ForceAttemptHTTP2: true, 44 | MaxIdleConns: 100, 45 | // ワーカ起動のロジックにバグがあったとしてもこのhttp.Transportを利用してい 46 | // る限りは多量のリクエストが飛ばないように念の為downloadWorkerNumでコネク 47 | // ション数を制限しておく 48 | MaxConnsPerHost: downloadWorkerNum, 49 | IdleConnTimeout: 90 * time.Second, 50 | TLSHandshakeTimeout: 10 * time.Second, 51 | ExpectContinueTimeout: 1 * time.Second, 52 | } 53 | 54 | cli := &http.Client{Transport: t} 55 | // 無効なSlack API tokenを食わせても、リダイレクトされ、200が返ってきてエラー 56 | // かどうか判別できない。 57 | // 一方で、なぜか .svg ファイルのダウンロード時にはリダイレクトが発生する。 58 | // リダイレクト先の URL を見て雰囲気で認証ページに行ってそうだったらエラーにする 59 | cli.CheckRedirect = func(req *http.Request, via []*http.Request) error { 60 | _, redirEixsts := req.URL.Query()["redir"] 61 | if strings.HasSuffix(req.URL.Host, ".slack.com") && 62 | req.URL.Path == "/" && 63 | redirEixsts { 64 | return http.ErrUseLastResponse 65 | } 66 | return nil 67 | } 68 | 69 | d := &Downloader{ 70 | token: token, 71 | httpClient: cli, 72 | targetCh: make(chan downloadTarget), 73 | } 74 | 75 | for i := 0; i < downloadWorkerNum; i++ { 76 | d.workerWg.Add(1) 77 | go func() { 78 | defer d.workerWg.Done() 79 | d.runWorker() 80 | }() 81 | } 82 | 83 | return d 84 | } 85 | 86 | // QueueDownloadRequest : ダウンロード処理をqueueに積む 87 | func (d *Downloader) QueueDownloadRequest(url, outputPath string, withToken bool) { 88 | d.targetCh <- downloadTarget{ 89 | url: url, 90 | outputPath: outputPath, 91 | withToken: withToken, 92 | } 93 | } 94 | 95 | // Wait : ワーカが全て実行終了するまで待つ。 96 | // ダウンロード処理中にエラーが発生していた場合は最初に発生した1つを返す。 97 | // 他のエラーはログに出力している。 98 | func (d *Downloader) Wait() error { 99 | d.workerWg.Wait() 100 | d.errsMu.Lock() 101 | defer d.errsMu.Unlock() 102 | return d.firstError 103 | } 104 | 105 | // CloseQueue : ダウンロードキューへの追加が完了したことをDownloaderに通知する 106 | // ために実行する。 107 | // TODO: 2回実行するとpanicしてしまうのを修正する。Downloaderに状態でも持たせる 108 | // とよいだろうか。 109 | func (d *Downloader) CloseQueue() { 110 | close(d.targetCh) 111 | } 112 | 113 | func (d *Downloader) runWorker() { 114 | for t := range d.targetCh { 115 | err := d.download(t) 116 | if err != nil { 117 | d.errsMu.Lock() 118 | if d.firstError == nil { 119 | d.firstError = err 120 | } 121 | d.errsMu.Unlock() 122 | fmt.Printf("download failed for url=%s: %s\n", t.url, err) 123 | } 124 | } 125 | } 126 | 127 | // downloadTarget : Downloaderにダウンロードする対象を指定するために使う。 128 | type downloadTarget struct { 129 | url string 130 | outputPath string 131 | // ダウンロード時にSlack API tokenを利用するかどうかを指定する 132 | withToken bool 133 | } 134 | 135 | func (d *Downloader) download(t downloadTarget) error { 136 | _, err := os.Stat(t.outputPath) 137 | if err == nil { 138 | // Just skip already downloaded file 139 | fmt.Printf("already exist: %s\n", t.outputPath) 140 | return nil 141 | } 142 | // `err != nil` has two cases at here. first is "not exist" as expected. 143 | // and second is I/O error as unexpected. 144 | if !os.IsNotExist(err) { 145 | return err 146 | } 147 | 148 | fmt.Printf("Downloading: %s\n", t.outputPath) 149 | 150 | req, err := http.NewRequest("GET", t.url, nil) 151 | if err != nil { 152 | return err 153 | } 154 | 155 | if t.withToken { 156 | req.Header.Add("Authorization", "Bearer "+d.token) 157 | } 158 | 159 | resp, err := d.httpClient.Do(req) 160 | if err != nil { 161 | return err 162 | } 163 | defer resp.Body.Close() 164 | 165 | if resp.StatusCode/100 != 2 { 166 | // Some files cannot download by unknown reason. 167 | // Just ignore. 168 | fmt.Fprintf(os.Stderr, "ERROR (ignored): [%s]: %s\n", resp.Status, t.url) 169 | return nil 170 | } 171 | 172 | w, err := os.Create(t.outputPath) 173 | if err != nil { 174 | return err 175 | } 176 | defer w.Close() 177 | 178 | _, err = io.Copy(w, resp.Body) 179 | if err != nil { 180 | return err 181 | } 182 | return nil 183 | } 184 | -------------------------------------------------------------------------------- /internal/slacklog/downloader_test.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | ) 12 | 13 | func TestDownloader(t *testing.T) { 14 | tmpPath := createTmpDir(t) 15 | defer t.Cleanup(func() { 16 | cleanupTmpDir(t, tmpPath) 17 | }) 18 | 19 | ts := httptest.NewServer(http.FileServer(http.Dir("testdata/downloader"))) 20 | defer ts.Close() 21 | 22 | fileInfos, err := ioutil.ReadDir("testdata/downloader") 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | d := NewDownloader("dummyToken") 28 | 29 | for _, fileInfo := range fileInfos { 30 | url := ts.URL + "/" + fileInfo.Name() 31 | path := filepath.Join(tmpPath, fileInfo.Name()) 32 | d.QueueDownloadRequest( 33 | url, 34 | path, 35 | false, 36 | ) 37 | } 38 | d.CloseQueue() 39 | err = d.Wait() 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | err = dirDiff(t, "testdata/downloader", tmpPath) 45 | if err != nil { 46 | t.Fatal(err) 47 | } 48 | } 49 | 50 | func TestDownloader_usingToken(t *testing.T) { 51 | tmpPath := createTmpDir(t) 52 | defer t.Cleanup(func() { 53 | cleanupTmpDir(t, tmpPath) 54 | }) 55 | 56 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 57 | t.Helper() 58 | res := map[string]interface{}{} 59 | res["token"] = r.Header.Get("Authorization")[7:] 60 | res["path"] = r.URL.Path 61 | 62 | err := json.NewEncoder(w).Encode(res) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | })) 67 | defer ts.Close() 68 | 69 | testToken := "dummyToken" 70 | d := NewDownloader(testToken) 71 | 72 | testFileName := "test.json" 73 | url := ts.URL + "/" + testFileName 74 | path := filepath.Join(tmpPath, testFileName) 75 | d.QueueDownloadRequest(url, path, true) 76 | d.CloseQueue() 77 | 78 | err := d.Wait() 79 | if err != nil { 80 | t.Fatal(err) 81 | } 82 | 83 | f, err := os.Open(path) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | var got map[string]string 88 | err = json.NewDecoder(f).Decode(&got) 89 | f.Close() 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | gotPath, ok := got["path"] 94 | if !ok { 95 | t.Fatal("not found got[\"path\"]") 96 | } 97 | if gotPath != "/"+testFileName { 98 | t.Fatalf("want %s, but got %s", "/"+testFileName, gotPath) 99 | } 100 | gotToken, ok := got["token"] 101 | if !ok { 102 | t.Fatal("not found got[\"token\"]") 103 | } 104 | if gotToken != testToken { 105 | t.Fatalf("want %s, but got %s", testToken, gotToken) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /internal/slacklog/emoji.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // EmojiTable : 絵文字データを保持する。 8 | type EmojiTable struct { 9 | // NameToExtは絵文字名をキーとし、画像の拡張子が値である。 10 | // 絵文字は事前に全てダウンロードしている、という前提であり、そのため拡張子のみ 11 | // を保持している。 12 | NameToExt map[string]string 13 | } 14 | 15 | // NewEmojiTable : pathに指定したJSON形式の絵文字データを読み込み、EmojiTableを 16 | // 生成する。 17 | func NewEmojiTable(path string) (*EmojiTable, error) { 18 | emojis := &EmojiTable{ 19 | NameToExt: map[string]string{}, 20 | } 21 | 22 | if info, err := os.Stat(path); err != nil || info.IsDir() { 23 | // pathにディレクトリが存在しても、その場合は無視して、ファイル自体が存在し 24 | // なかったこととする。 25 | return nil, os.ErrNotExist 26 | } 27 | 28 | if err := ReadFileAsJSON(path, true, &emojis.NameToExt); err != nil { 29 | return nil, err 30 | } 31 | 32 | return emojis, nil 33 | } 34 | -------------------------------------------------------------------------------- /internal/slacklog/filetype.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | // FiletypeToExtension is a map which maps filetype to filename extension. 4 | // 5 | // Based on 6 | var FiletypeToExtension = map[string]string{ 7 | "auto": "", // Auto Detect Type, 8 | "text": ".txt", // Plain Text, 9 | "ai": ".ai", // Illustrator File, 10 | "apk": ".apk", // APK, 11 | "applescript": ".applescript", // AppleScript, 12 | "binary": "", // Binary, 13 | "bmp": ".bmp", // Bitmap, 14 | "boxnote": ".boxnote", // BoxNote, 15 | "c": ".c", // C, 16 | "csharp": ".cs", // C#, 17 | "cpp": ".cpp", // C++, 18 | "css": ".css", // CSS, 19 | "csv": ".csv", // CSV, 20 | "clojure": ".clj", // Clojure, 21 | "coffeescript": ".coffee", // CoffeeScript, 22 | "cfm": ".cfm", // ColdFusion, 23 | "d": ".d", // D, 24 | "dart": ".dart", // Dart, 25 | "diff": ".diff", // Diff, 26 | "doc": ".doc", // Word Document, 27 | "docx": ".docx", // Word document, 28 | "dockerfile": ".dockerfile", // Docker, 29 | "dotx": ".dotx", // Word template, 30 | "email": ".eml", // Email, 31 | "eps": ".eps", // EPS, 32 | "epub": ".epub", // EPUB, 33 | "erlang": ".erl", // Erlang, 34 | "fla": ".fla", // Flash FLA, 35 | "flv": ".flv", // Flash video, 36 | "fsharp": ".fs", // F#, 37 | "fortran": ".f90", // Fortran, 38 | "gdoc": ".gdoc", // GDocs Document, 39 | "gdraw": ".gdraw", // GDocs Drawing, 40 | "gif": ".gif", // GIF, 41 | "go": ".go", // Go, 42 | "gpres": ".gpres", // GDocs Presentation, 43 | "groovy": ".groovy", // Groovy, 44 | "gsheet": ".gsheet", // GDocs Spreadsheet, 45 | "gzip": ".gz", // GZip, 46 | "html": ".html", // HTML, 47 | "handlebars": ".handlebars", // Handlebars, 48 | "haskell": ".hs", // Haskell, 49 | "haxe": ".hx", // Haxe, 50 | "indd": ".indd", // InDesign Document, 51 | "java": ".java", // Java, 52 | "javascript": ".js", // JavaScript/JSON, 53 | "jpg": ".jpeg", // JPEG, 54 | "keynote": ".keynote", // Keynote Document, 55 | "kotlin": ".kt", // Kotlin, 56 | "latex": ".tex", // LaTeX/sTeX, 57 | "lisp": ".lisp", // Lisp, 58 | "lua": ".lua", // Lua, 59 | "m4a": ".m4a", // MPEG 4 audio, 60 | "markdown": ".md", // Markdown (raw), 61 | "matlab": ".m", // MATLAB, 62 | "mhtml": ".mhtml", // MHTML, 63 | "mkv": ".mkv", // Matroska video, 64 | "mov": ".mov", // QuickTime video, 65 | "mp3": ".mp3", // mp4, 66 | "mp4": ".mp4", // MPEG 4 video, 67 | "mpg": ".mpeg", // MPEG video, 68 | "mumps": ".m", // MUMPS, 69 | "numbers": ".numbers", // Numbers Document, 70 | "nzb": ".nzb", // NZB, 71 | "objc": ".objc", // Objective-C, 72 | "ocaml": ".ml", // OCaml, 73 | "odg": ".odg", // OpenDocument Drawing, 74 | "odi": ".odi", // OpenDocument Image, 75 | "odp": ".odp", // OpenDocument Presentation, 76 | "ods": ".ods", // OpenDocument Spreadsheet, 77 | "odt": ".odt", // OpenDocument Text, 78 | "ogg": ".ogg", // Ogg Vorbis, 79 | "ogv": ".ogv", // Ogg video, 80 | "pages": ".pages", // Pages Document, 81 | "pascal": ".pp", // Pascal, 82 | "pdf": ".pdf", // PDF, 83 | "perl": ".pl", // Perl, 84 | "php": ".php", // PHP, 85 | "pig": ".pig", // Pig, 86 | "png": ".png", // PNG, 87 | "post": ".post", // Slack Post, 88 | "powershell": ".ps1", // PowerShell, 89 | "ppt": ".ppt", // PowerPoint presentation, 90 | "pptx": ".pptx", // PowerPoint presentation, 91 | "psd": ".psd", // Photoshop Document, 92 | "puppet": ".pp", // Puppet, 93 | "python": ".py", // Python, 94 | "qtz": ".qtz", // Quartz Composer Composition, 95 | "r": ".r", // R, 96 | "rtf": ".rtf", // Rich Text File, 97 | "ruby": ".rb", // Ruby, 98 | "rust": ".rs", // Rust, 99 | "sql": ".sql", // SQL, 100 | "sass": ".sass", // Sass, 101 | "scala": ".scala", // Scala, 102 | "scheme": ".scm", // Scheme, 103 | "sketch": ".sketch", // Sketch File, 104 | "shell": ".sh", // Shell, 105 | "smalltalk": ".st", // Smalltalk, 106 | "svg": ".svg", // SVG, 107 | "swf": ".swf", // Flash SWF, 108 | "swift": ".swift", // Swift, 109 | "tar": ".tar", // Tarball, 110 | "tiff": ".tiff", // TIFF, 111 | "tsv": ".tsv", // TSV, 112 | "vb": ".vb", // VB.NET, 113 | "vbscript": ".vbs", // VBScript, 114 | "vcard": ".vcf", // vCard, 115 | "velocity": ".vm", // Velocity, 116 | "verilog": ".v", // Verilog, 117 | "wav": ".wav", // Waveform audio, 118 | "webm": ".webm", // WebM, 119 | "wmv": ".wmv", // Windows Media Video, 120 | "xls": ".xls", // Excel spreadsheet, 121 | "xlsx": ".xlsx", // Excel spreadsheet, 122 | "xlsb": ".xlsb", // Excel Spreadsheet (Binary, Macro Enabled), 123 | "xlsm": ".xlsm", // Excel Spreadsheet (Macro Enabled), 124 | "xltx": ".xltx", // Excel template, 125 | "xml": ".xml", // XML, 126 | "yaml": ".yaml", // YAML, 127 | "zip": ".zip", // Zip, 128 | } 129 | -------------------------------------------------------------------------------- /internal/slacklog/generator.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "html" 8 | "io/ioutil" 9 | "log" 10 | "net/url" 11 | "os" 12 | "path/filepath" 13 | "regexp" 14 | "sort" 15 | "strings" 16 | "sync" 17 | "text/template" 18 | 19 | "github.com/kyokomi/emoji" 20 | "github.com/slack-go/slack" 21 | ) 22 | 23 | // HTMLGenerator : ログデータからHTMLを生成するための構造体。 24 | type HTMLGenerator struct { 25 | // text/template形式のテンプレートが置いてあるディレクトリ 26 | templateDir string 27 | // files がおいてあるディレクトリ 28 | filesDir string 29 | // ログデータを取得するためのLogStore 30 | s *LogStore 31 | // markdown形式のテキストを変換するためのTextConverter 32 | c *TextConverter 33 | cfg Config 34 | 35 | // baseURL is root path for public site, configured by `BASEURL` environment variable. 36 | baseURL string 37 | 38 | // filesBaseURL is root path for attachment files, configured by `FILES_BASEURL` environment variable. 39 | filesBaseURL string 40 | 41 | // ueMap is a set of unknown emojis. 42 | ueMap map[string]struct{} 43 | ueMu sync.Mutex 44 | } 45 | 46 | // maxEmbeddedFileSize : 添付ファイルの埋め込みを行うファイルサイズ 47 | // これ以下の場合、表示されるようになる 48 | const maxEmbeddedFileSize = 102400 49 | 50 | // NewHTMLGenerator : HTMLGeneratorを生成する。 51 | func NewHTMLGenerator(templateDir string, filesDir string, s *LogStore) *HTMLGenerator { 52 | users := s.GetDisplayNameMap() 53 | emojis := s.GetEmojiMap() 54 | c := NewTextConverter(users, emojis) 55 | 56 | baseURL := os.Getenv("BASEURL") 57 | filesBaseURL := os.Getenv("FILES_BASEURL") 58 | if filesBaseURL == "" { 59 | filesBaseURL = baseURL + "/files" 60 | } 61 | 62 | return &HTMLGenerator{ 63 | templateDir: templateDir, 64 | filesDir: filesDir, 65 | s: s, 66 | c: c, 67 | baseURL: baseURL, 68 | filesBaseURL: filesBaseURL, 69 | } 70 | } 71 | 72 | // Generate はoutDirにログデータの変換結果を生成する。 73 | // 目標とする構造は以下となる: 74 | // - outDir/ 75 | // - index.html // generateIndex() 76 | // - ${channel_id}/ // generateChannelDir() 77 | // - index.html // generateChannelIndex() 78 | // - ${YYYY}/ 79 | // - ${MM}/ 80 | // - index.html // generateMessageDir() 81 | func (g *HTMLGenerator) Generate(outDir string) error { 82 | channels := g.s.GetChannels() 83 | 84 | createdChannels := []Channel{} 85 | var ( 86 | wg sync.WaitGroup 87 | mu sync.Mutex 88 | errs []error 89 | ) 90 | for i := range channels { 91 | wg.Add(1) 92 | go func(i int) { 93 | defer wg.Done() 94 | isCreated, err := g.generateChannelDir( 95 | filepath.Join(outDir, channels[i].ID), 96 | channels[i], 97 | ) 98 | if err != nil { 99 | log.Printf("generateChannelDir(%s) failed: %s", channels[i].ID, err) 100 | mu.Lock() 101 | errs = append(errs, err) 102 | mu.Unlock() 103 | return 104 | } 105 | if isCreated { 106 | mu.Lock() 107 | createdChannels = append(createdChannels, channels[i]) 108 | mu.Unlock() 109 | } 110 | }(i) 111 | } 112 | wg.Wait() 113 | if len(errs) > 0 { 114 | return errs[0] 115 | } 116 | 117 | if err := g.generateIndex(filepath.Join(outDir, "index.html"), createdChannels); err != nil { 118 | return err 119 | } 120 | 121 | return nil 122 | } 123 | 124 | func (g *HTMLGenerator) generateIndex(path string, channels []Channel) error { 125 | params := make(map[string]interface{}) 126 | SortChannel(channels) 127 | params["baseURL"] = g.baseURL 128 | params["channels"] = channels 129 | tmplPath := filepath.Join(g.templateDir, "index.tmpl") 130 | name := filepath.Base(tmplPath) 131 | t, err := template.New(name).ParseFiles(tmplPath) 132 | if err != nil { 133 | return err 134 | } 135 | if err := executeAndWrite(t, params, path); err != nil { 136 | return err 137 | } 138 | return nil 139 | } 140 | 141 | func (g *HTMLGenerator) generateChannelDir(path string, channel Channel) (bool, error) { 142 | msgsMap, err := g.s.GetMessagesPerMonth(channel.ID) 143 | if err != nil { 144 | return false, err 145 | } 146 | if len(msgsMap) == 0 { 147 | return false, nil 148 | } 149 | 150 | if err := os.MkdirAll(path, 0777); err != nil { 151 | return false, fmt.Errorf("could not create %s directory: %w", path, err) 152 | } 153 | 154 | if err := g.generateChannelIndex( 155 | channel, 156 | msgsMap.Keys(), 157 | filepath.Join(path, "index.html"), 158 | ); err != nil { 159 | return true, err 160 | } 161 | 162 | for key, mm := range msgsMap { 163 | if err := g.generateMessageDir( 164 | channel, 165 | key, 166 | mm, 167 | filepath.Join(path, key.Year(), key.Month()), 168 | ); err != nil { 169 | return true, err 170 | } 171 | } 172 | return true, nil 173 | } 174 | 175 | func (g *HTMLGenerator) generateChannelIndex(channel Channel, keys []MessageMonthKey, path string) error { 176 | sort.Slice(keys, func(i, j int) bool { 177 | if keys[i].year < keys[j].year { 178 | return false 179 | } else if keys[i].year > keys[j].year { 180 | return true 181 | } 182 | return keys[i].month > keys[j].month 183 | }) 184 | 185 | params := make(map[string]interface{}) 186 | params["baseURL"] = g.baseURL 187 | params["channel"] = channel 188 | params["keys"] = keys 189 | 190 | tempPath := filepath.Join(g.templateDir, "channel_index.tmpl") 191 | name := filepath.Base(tempPath) 192 | t, err := template.New(name).ParseFiles(tempPath) 193 | if err != nil { 194 | return err 195 | } 196 | if err := executeAndWrite(t, params, path); err != nil { 197 | return err 198 | } 199 | return nil 200 | } 201 | 202 | func (g *HTMLGenerator) generateMessageDir(channel Channel, key MessageMonthKey, msgs Messages, path string) error { 203 | if err := os.MkdirAll(path, 0777); err != nil { 204 | return fmt.Errorf("could not create %s directory: %w", path, err) 205 | } 206 | 207 | params := make(map[string]interface{}) 208 | params["baseURL"] = g.baseURL 209 | params["filesBaseURL"] = g.filesBaseURL 210 | params["channel"] = channel 211 | params["monthKey"] = key 212 | params["msgs"] = msgs 213 | 214 | // TODO check below subtypes work correctly 215 | // TODO support more subtypes 216 | 217 | t, err := template.New(""). 218 | Funcs(map[string]interface{}{ 219 | "visible": g.isVisibleMessage, 220 | "datetime": func(ts string) string { 221 | return TsToDateTime(ts).Format("2日 15:04:05") 222 | }, 223 | "threadMessageTime": func(msgTs, threadTs string) string { 224 | return LevelOfDetailTime(TsToDateTime(msgTs), TsToDateTime(threadTs)) 225 | }, 226 | "slackPermalink": func(ts string) string { 227 | return strings.Replace(ts, ".", "", 1) 228 | }, 229 | "username": func(msg *Message) string { 230 | if msg.Username != "" { 231 | return g.c.escapeSpecialChars(msg.Username) 232 | } 233 | return g.c.escapeSpecialChars(g.s.GetDisplayNameByUserID(msg.User)) 234 | }, 235 | "userIconUrl": func(msg *Message) string { 236 | if msg.Icons != nil && msg.Icons.Image48 != "" { 237 | return msg.Icons.Image48 238 | } 239 | userID := msg.User 240 | if userID == "" && msg.BotID != "" { 241 | userID = msg.BotID 242 | } 243 | user, ok := g.s.GetUserByID(userID) 244 | if !ok { 245 | return "" // TODO show default icon 246 | } 247 | return user.Profile.Image48 248 | }, 249 | "text": g.generateMessageText, 250 | "reactions": g.getReactions, 251 | "attachmentText": g.generateAttachmentText, 252 | "fileHTML": g.generateFileHTML, 253 | "threadMtime": func(ts string) string { 254 | if t, ok := g.s.GetThread(channel.ID, ts); ok { 255 | return LevelOfDetailTime(t.LastReplyTime(), TsToDateTime(ts)) 256 | } 257 | return "" 258 | }, 259 | "threads": func(ts string) Messages { 260 | if t, ok := g.s.GetThread(channel.ID, ts); ok { 261 | return t.Replies() 262 | } 263 | return nil 264 | }, 265 | "threadNum": func(ts string) int { 266 | if t, ok := g.s.GetThread(channel.ID, ts); ok { 267 | return t.ReplyCount() 268 | } 269 | return 0 270 | }, 271 | "threadRootText": func(ts string) string { 272 | thread, ok := g.s.GetThread(channel.ID, ts) 273 | if !ok { 274 | return "" 275 | } 276 | runes := []rune(thread.RootText()) 277 | text := string(runes) 278 | if len(runes) > 20 { 279 | text = string(runes[:20]) + " ..." 280 | } 281 | return g.c.escape(text) 282 | }, 283 | "hasPrevMonth": func(key MessageMonthKey) bool { 284 | return g.s.HasPrevMonth(channel.ID, key) 285 | }, 286 | "hasNextMonth": func(key MessageMonthKey) bool { 287 | return g.s.HasNextMonth(channel.ID, key) 288 | }, 289 | "hostBySlack": HostBySlack, 290 | "localPath": LocalPath, 291 | "topLevelMimetype": TopLevelMimetype, 292 | "thumbImagePath": ThumbImagePath, 293 | "thumbImageWidth": ThumbImageWidth, 294 | "thumbImageHeight": ThumbImageHeight, 295 | "thumbVideoPath": ThumbVideoPath, 296 | "stringsJoin": strings.Join, 297 | "genAttachedURL": func(ts json.Number, fromURL string) string { 298 | timestamp := string(ts) 299 | if len(strings.Split(timestamp, ".")) < 2 { 300 | return "" 301 | } 302 | postTime := TsToDateTime(timestamp) 303 | c := strings.Split(fromURL, "/") 304 | if len(c) < 2 { 305 | return "" 306 | } 307 | channel := c[len(c)-2] 308 | return fmt.Sprintf("/%s/%d/%02d/#ts-%s", channel, postTime.Year(), postTime.Month(), ts) 309 | }, 310 | "isSlackMessage": func(fromURL string) bool { 311 | return len(fromURL) >= 25 && fromURL[0:25] == "https://vim-jp.slack.com/" 312 | }, 313 | "getBaseURL": func() string { 314 | return g.baseURL 315 | }, 316 | }). 317 | ParseGlob(filepath.Join(g.templateDir, "channel_per_month", "*.tmpl")) 318 | if err != nil { 319 | return err 320 | } 321 | tmpl := t.Lookup("index.tmpl") 322 | if tmpl == nil { 323 | return errors.New("no index.tmpl in channel_per_month/ dir") 324 | } 325 | err = executeAndWrite(tmpl, params, filepath.Join(path, "index.html")) 326 | if err != nil { 327 | return err 328 | } 329 | return nil 330 | } 331 | 332 | func (g *HTMLGenerator) isVisibleMessage(msg Message) bool { 333 | return msg.SubType == "" || msg.SubType == "bot_message" || msg.SubType == "slackbot_response" || msg.SubType == "thread_broadcast" 334 | } 335 | 336 | func (g *HTMLGenerator) generateMessageText(msg Message) string { 337 | text := g.c.ToHTML(msg.Text) 338 | if msg.Edited != nil && g.cfg.EditedSuffix != "" { 339 | text += "" + html.EscapeString(g.cfg.EditedSuffix) + "" 340 | } 341 | return text 342 | } 343 | 344 | func (g *HTMLGenerator) generateAttachmentText(attachment slack.Attachment) string { 345 | return g.c.ToHTML(attachment.Text) 346 | } 347 | 348 | // generateFileHTML : 'text/plain' な添付ファイルをHTMLに埋め込む 349 | // 存在しない場合、エラーを表示する 350 | func (g *HTMLGenerator) generateFileHTML(file slack.File) string { 351 | if file.Size > maxEmbeddedFileSize { 352 | return `file size is too big to embed. please download from above link to see.` 353 | } 354 | path := filepath.Join(g.filesDir, file.ID, LocalName(file, file.URLPrivate, "")) 355 | src, err := ioutil.ReadFile(path) 356 | if err != nil { 357 | if os.IsNotExist(err) { 358 | return fmt.Sprintf(`no files found: %s`, err) 359 | } 360 | return fmt.Sprintf(`failed to read a file: %s`, err) 361 | } 362 | ftype := file.Filetype 363 | if file.Filetype == "text" { 364 | ftype = "none" 365 | } 366 | return "" + html.EscapeString(string(src)) + "" 367 | } 368 | 369 | // ReactionInfo is information for a reaction. 370 | type ReactionInfo struct { 371 | EmojiPath string 372 | Name string 373 | Count int 374 | Users []string 375 | Default bool 376 | } 377 | 378 | func (g *HTMLGenerator) getReactions(msg Message) []ReactionInfo { 379 | var info []ReactionInfo 380 | 381 | for _, reaction := range msg.Reactions { 382 | users := make([]string, 0, len(reaction.Users)) 383 | for _, user := range reaction.Users { 384 | n := g.s.GetDisplayNameByUserID(user) 385 | if n == "" { 386 | continue 387 | } 388 | users = append(users, n) 389 | } 390 | 391 | // custom emoji case 392 | emojiExt, ok := g.s.et.NameToExt[reaction.Name] 393 | if ok { 394 | info = append(info, ReactionInfo{ 395 | EmojiPath: url.PathEscape(reaction.Name + emojiExt), 396 | Name: reaction.Name, 397 | Count: reaction.Count, 398 | Users: users, 399 | Default: false, 400 | }) 401 | continue 402 | } 403 | 404 | // fallback to unicode. 405 | emojiStr := ":" + reaction.Name + ":" 406 | unicodeEmojis := g.emojiToString(emojiStr) 407 | if unicodeEmojis == "" { 408 | // This may be a deleted emoji. Show `:emoji:` as is. 409 | unicodeEmojis = emojiStr 410 | } 411 | info = append(info, ReactionInfo{ 412 | Name: unicodeEmojis, 413 | Count: reaction.Count, 414 | Users: users, 415 | Default: true, 416 | }) 417 | } 418 | 419 | return info 420 | } 421 | 422 | var rxEmoji = regexp.MustCompile(`:[^:]+:`) 423 | 424 | func (g *HTMLGenerator) emojiToString(emojiSeq string) string { 425 | b := &strings.Builder{} 426 | for _, s := range rxEmoji.FindAllString(emojiSeq, -1) { 427 | ch, ok := emoji.CodeMap()[s] 428 | if !ok { 429 | g.ueMu.Lock() 430 | if g.ueMap == nil { 431 | g.ueMap = map[string]struct{}{} 432 | } 433 | if _, ok := g.ueMap[s]; !ok { 434 | g.ueMap[s] = struct{}{} 435 | } 436 | g.ueMu.Unlock() 437 | continue 438 | } 439 | b.WriteString(ch) 440 | } 441 | return b.String() 442 | } 443 | 444 | // executeAndWrite executes a template and writes contents to a file. 445 | func executeAndWrite(tmpl *template.Template, data interface{}, filename string) error { 446 | f, err := os.Create(filename) 447 | if err != nil { 448 | return err 449 | } 450 | defer f.Close() 451 | err = tmpl.Execute(f, data) 452 | if err != nil { 453 | return err 454 | } 455 | return nil 456 | } 457 | -------------------------------------------------------------------------------- /internal/slacklog/indexer.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strconv" 10 | "strings" 11 | "unicode/utf16" 12 | ) 13 | 14 | const gramN = 2 15 | 16 | type Indexer struct { 17 | s *LogStore 18 | gramIndex messageIndex 19 | channelNumbers map[int]Channel 20 | } 21 | 22 | func NewIndexer(s *LogStore) *Indexer { 23 | return &Indexer{ 24 | s: s, 25 | gramIndex: messageIndex{}, 26 | channelNumbers: map[int]Channel{}, 27 | } 28 | } 29 | 30 | func (idx *Indexer) Build() error { 31 | channelNumber := 0 32 | for _, c := range idx.s.GetChannels() { 33 | channelNumber++ 34 | idx.channelNumbers[channelNumber] = c 35 | msgs, err := idx.s.GetAllMessages(c.ID) 36 | if err != nil { 37 | return err 38 | } 39 | for _, m := range msgs { 40 | runes := []rune(m.Text) 41 | textLen := len(runes) 42 | for i := range runes { 43 | for n := 1; n <= gramN; n++ { 44 | if textLen <= i+n-1 { 45 | break 46 | } 47 | key := string(runes[i : i+n]) 48 | idx.gramIndex.Add(key, channelNumber, m, i) 49 | } 50 | } 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | func (idx *Indexer) Output(outDir string) error { 57 | channelFilepath := filepath.Join(outDir, "channel") 58 | err := idx.writeChannelFile(channelFilepath, idx.channelNumbers) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | for key, mPositions := range idx.gramIndex { 64 | s := outDir 65 | for _, u := range utf16.Encode([]rune(key)) { 66 | s = filepath.Join(s, fmt.Sprintf("%02x", u>>8), fmt.Sprintf("%02x", u&0xff)) 67 | } 68 | err := idx.writeIndexFile(s+".index", mPositions) 69 | if err != nil { 70 | return err 71 | } 72 | } 73 | return nil 74 | } 75 | 76 | func (idx *Indexer) writeChannelFile(path string, channelNumbers map[int]Channel) error { 77 | err := os.MkdirAll(filepath.Dir(path), 0o777) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | f, err := os.Create(path) 83 | if err != nil { 84 | return err 85 | } 86 | defer f.Close() 87 | 88 | fw := bufio.NewWriter(f) 89 | for channelNumber, channel := range channelNumbers { 90 | _, err := fw.Write([]byte(fmt.Sprintf("%d\t%s\t%s\n", channelNumber, channel.ID, channel.Name))) 91 | if err != nil { 92 | return err 93 | } 94 | } 95 | 96 | err = fw.Flush() 97 | if err != nil { 98 | return err 99 | } 100 | 101 | return nil 102 | } 103 | 104 | func (idx *Indexer) writeIndexFile(path string, mPositions messagePositions) error { 105 | err := os.MkdirAll(filepath.Dir(path), 0o777) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | f, err := os.Create(path) 111 | if err != nil { 112 | return err 113 | } 114 | defer f.Close() 115 | 116 | fw := bufio.NewWriter(f) 117 | for channelNumber, mposMap := range mPositions { 118 | _, err := fw.Write(vintBytes(channelNumber)) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | _, err = fw.Write(vintBytes(len(mposMap))) 124 | if err != nil { 125 | return err 126 | } 127 | 128 | for ts, positions := range mposMap { 129 | tsParts := strings.SplitN(ts, ".", 2) 130 | if len(tsParts) != 2 { 131 | channel := idx.channelNumbers[channelNumber] 132 | return fmt.Errorf("Invalid timestamp %s (%s)", ts, channel.ID) 133 | } 134 | 135 | tsSec, err := strconv.Atoi(tsParts[0]) 136 | if err != nil { 137 | return err 138 | } 139 | err = binary.Write(fw, binary.BigEndian, uint32(tsSec)) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | tsMicrosec, err := strconv.Atoi(tsParts[1]) 145 | if err != nil { 146 | return err 147 | } 148 | _, err = fw.Write(vintBytes(tsMicrosec)) 149 | if err != nil { 150 | return err 151 | } 152 | 153 | if len(positions) == 0 { 154 | channel := idx.channelNumbers[channelNumber] 155 | return fmt.Errorf("Empty positions: %s: %s: %s", path, channel.ID, ts) 156 | } 157 | for _, pos := range positions { 158 | _, err = fw.Write(vintBytes(pos + 1)) 159 | if err != nil { 160 | return err 161 | } 162 | } 163 | err = fw.WriteByte(0) 164 | if err != nil { 165 | return err 166 | } 167 | } 168 | } 169 | 170 | err = fw.Flush() 171 | if err != nil { 172 | return err 173 | } 174 | 175 | return nil 176 | } 177 | 178 | type messageIndex map[string]messagePositions 179 | 180 | func (mi messageIndex) Add(key string, channelNumber int, mes *Message, pos int) { 181 | mp, ok := mi[key] 182 | if !ok { 183 | mp = messagePositions{} 184 | mi[key] = mp 185 | } 186 | mp.Add(channelNumber, mes, pos) 187 | } 188 | 189 | type messagePositions map[int]map[string][]int 190 | 191 | func (mp messagePositions) Add(channelNumber int, mes *Message, pos int) { 192 | mposMap, ok := mp[channelNumber] 193 | if !ok { 194 | mposMap = map[string][]int{} 195 | mp[channelNumber] = mposMap 196 | } 197 | 198 | mposMap[mes.Timestamp] = append(mposMap[mes.Timestamp], pos) 199 | } 200 | 201 | func vintBytes(n int) []byte { 202 | if n == 0 { 203 | return []byte{0} 204 | } 205 | bytes := []byte{} 206 | cont := false 207 | for n != 0 { 208 | b := byte(n & 0b01111111) 209 | if cont { 210 | b |= 0b10000000 211 | } 212 | bytes = append([]byte{b}, bytes...) 213 | n >>= 7 214 | cont = true 215 | } 216 | return bytes 217 | } 218 | -------------------------------------------------------------------------------- /internal/slacklog/json.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | ) 7 | 8 | // ReadFileAsJSON reads a file and unmarshal its contents as JSON to `dst` 9 | // destination object. 10 | func ReadFileAsJSON(filename string, strict bool, dst interface{}) error { 11 | f, err := os.Open(filename) 12 | if err != nil { 13 | return err 14 | } 15 | defer f.Close() 16 | d := json.NewDecoder(f) 17 | if strict { 18 | d.DisallowUnknownFields() 19 | } 20 | err = d.Decode(dst) 21 | if err != nil { 22 | return err 23 | } 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /internal/slacklog/main_test.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "sort" 9 | "testing" 10 | ) 11 | 12 | func cleanupTmpDir(t *testing.T, path string) { 13 | t.Helper() 14 | 15 | err := os.RemoveAll(path) 16 | if err != nil { 17 | t.Fatalf("failed to cleanupTmpDir: %s", err) 18 | } 19 | } 20 | 21 | func createTmpDir(t *testing.T) string { 22 | t.Helper() 23 | 24 | path, err := ioutil.TempDir("", "slacklog") 25 | if err != nil { 26 | t.Fatalf("failed to createTmpDir: %s", err) 27 | } 28 | return path 29 | } 30 | 31 | func dirDiff(t *testing.T, a, b string) (err error) { 32 | t.Helper() 33 | 34 | aInfos, err := ioutil.ReadDir(a) 35 | if err != nil { 36 | err = fmt.Errorf("failed to dirDiff: %w", err) 37 | return err 38 | } 39 | bInfos, err := ioutil.ReadDir(b) 40 | if err != nil { 41 | err = fmt.Errorf("failed to dirDiff: %w", err) 42 | return err 43 | } 44 | 45 | if len(aInfos) != len(bInfos) { 46 | return fmt.Errorf( 47 | "the number of files in the directory is different: (%s: %d) (%s: %d)", 48 | a, len(aInfos), 49 | b, len(bInfos), 50 | ) 51 | } 52 | 53 | sort.Slice(aInfos, func(i, j int) bool { 54 | return aInfos[i].Name() >= aInfos[i].Name() 55 | }) 56 | sort.Slice(bInfos, func(i, j int) bool { 57 | return bInfos[i].Name() >= bInfos[i].Name() 58 | }) 59 | 60 | for i := range aInfos { 61 | if aInfos[i].Name() != bInfos[i].Name() { 62 | err := fmt.Errorf( 63 | "the file name is different: %s != %s", 64 | filepath.Join(a, aInfos[i].Name()), 65 | filepath.Join(b, bInfos[i].Name()), 66 | ) 67 | err = fmt.Errorf("failed to dirDiff: %w", err) 68 | return err 69 | } 70 | if aInfos[i].Size() != bInfos[i].Size() { 71 | err := fmt.Errorf( 72 | "the file size is different: (%s: %d) (%s: %d)", 73 | filepath.Join(a, aInfos[i].Name()), aInfos[i].Size(), 74 | filepath.Join(b, bInfos[i].Name()), bInfos[i].Size(), 75 | ) 76 | err = fmt.Errorf("failed to dirDiff: %w", err) 77 | return err 78 | } 79 | } 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /internal/slacklog/message.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "regexp" 8 | "sort" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/slack-go/slack" 13 | ) 14 | 15 | // Messages is an array of `*Message`. 16 | type Messages []*Message 17 | 18 | // Sort sorts messages by `Message.Ts` ascendant order. 19 | func (msgs Messages) Sort() { 20 | sort.SliceStable(msgs, func(i, j int) bool { 21 | // must be the same digits, so no need to convert the timestamp to a number 22 | return msgs[i].Timestamp < msgs[j].Timestamp 23 | }) 24 | } 25 | 26 | // MessagesMap is a map, maps MessageMonthKey to Messages. 27 | type MessagesMap map[MessageMonthKey]Messages 28 | 29 | // Keys returns all keys in the map. 30 | func (mm MessagesMap) Keys() []MessageMonthKey { 31 | keys := make([]MessageMonthKey, 0, len(mm)) 32 | for key := range mm { 33 | keys = append(keys, key) 34 | } 35 | return keys 36 | } 37 | 38 | // MessageTable : メッセージデータを保持する 39 | // スレッドは投稿時刻からどのスレッドへの返信かが判断できるためThreadMapのキー 40 | // はtsである。 41 | // MsgsMapは月毎にメッセージを保持する。そのためキーは投稿月である。 42 | // loadedFilesはすでに読み込んだファイルパスを保持する。 43 | // loadedFilesは同じファイルを二度読むことを防ぐために用いている。 44 | type MessageTable struct { 45 | // key: timestamp 46 | AllMessageMap map[string]*Message 47 | // key: thread timestamp 48 | ThreadMap map[string]*Thread 49 | MsgsMap MessagesMap 50 | // key: file path 51 | loadedFiles map[string]struct{} 52 | } 53 | 54 | // NewMessageTable : MessageTableを生成する。 55 | // 他のテーブルと違い、メッセージファイルは量が多いため、NewMessageTable()実行 56 | // 時には読み込まず、(*MessageTable).ReadLogDir()/(*MessageTable).ReadLogFile() 57 | // 実行時に読み込ませる。 58 | func NewMessageTable() *MessageTable { 59 | return &MessageTable{ 60 | AllMessageMap: map[string]*Message{}, 61 | ThreadMap: map[string]*Thread{}, 62 | MsgsMap: MessagesMap{}, 63 | loadedFiles: map[string]struct{}{}, 64 | } 65 | } 66 | 67 | // ReadLogDir : pathに指定したディレクトリに存在するJSON形式のメッセージデータ 68 | // を読み込む。 69 | // すでにそのディレクトリが読み込み済みの場合は処理をスキップする。 70 | // デフォルトでは特定のサブタイプを持つメッセージのみをmsgMapに登録するが、 71 | // readAllMessages が true である場合はすべてのメッセージを登録する。 72 | func (m *MessageTable) ReadLogDir(path string, readAllMessages bool) error { 73 | dir, err := os.Open(path) 74 | if err != nil { 75 | return err 76 | } 77 | defer dir.Close() 78 | names, err := dir.Readdirnames(0) 79 | if err != nil { 80 | return err 81 | } 82 | // ReadLogFile()は日付順に処理する必要があり、そのためにファイル名でソート 83 | // している 84 | sort.Strings(names) 85 | for _, name := range names { 86 | if err := m.ReadLogFile(filepath.Join(path, name), readAllMessages); err != nil { 87 | return err 88 | } 89 | } 90 | return nil 91 | } 92 | 93 | // "{year}-{month}-{day}.json" 94 | var reMsgFilename = regexp.MustCompile(`^(\d{4})-(\d{2})-\d{2}\.json$`) 95 | 96 | // ReadLogFile : pathに指定したJSON形式のメッセージデータを読み込む。 97 | // すでにそのファイルが読み込み済みの場合は処理をスキップする。 98 | // readAllMessagesがfalseである場合は特定のサブタイプを持つメッセージのみをmsgMapに登録する。 99 | func (m *MessageTable) ReadLogFile(path string, readAllMessages bool) error { 100 | path, err := filepath.Abs(path) 101 | if err != nil { 102 | return err 103 | } 104 | if _, ok := m.loadedFiles[path]; ok { 105 | return nil 106 | } 107 | 108 | match := reMsgFilename.FindStringSubmatch(filepath.Base(path)) 109 | if len(match) == 0 { 110 | fmt.Fprintf(os.Stderr, "[warning] skipping %s ...\n", path) 111 | return nil 112 | } 113 | 114 | var msgs Messages 115 | err = ReadFileAsJSON(path, true, &msgs) 116 | if err != nil { 117 | return fmt.Errorf("failed to unmarshal %s: %w", path, err) 118 | } 119 | 120 | // assort messages, visible and threaded. 121 | var visibleMsgs Messages 122 | for _, msg := range msgs { 123 | if !readAllMessages && !msg.isVisible() { 124 | continue 125 | } 126 | 127 | // スレッドに所属してるメッセージは ThreadMap へスレッド毎に分別しておく 128 | threadTs := msg.ThreadTimestamp 129 | if threadTs != "" { 130 | thread, ok := m.ThreadMap[threadTs] 131 | if !ok { 132 | thread = &Thread{} 133 | if rootMsg, ok := m.AllMessageMap[threadTs]; ok { 134 | thread.rootMsg = rootMsg 135 | } 136 | m.ThreadMap[threadTs] = thread 137 | } 138 | thread.Put(msg) 139 | } 140 | 141 | if !readAllMessages && msg.isThreadChild() { 142 | continue 143 | } 144 | 145 | m.AllMessageMap[msg.Timestamp] = msg 146 | 147 | visibleMsgs = append(visibleMsgs, msg) 148 | } 149 | 150 | key, err := NewMessageMonthKey(match[1], match[2]) 151 | if err != nil { 152 | return err 153 | } 154 | if len(visibleMsgs) != 0 { 155 | m.MsgsMap[key] = append(m.MsgsMap[key], visibleMsgs...) 156 | } 157 | 158 | for _, msgs := range m.MsgsMap { 159 | msgs.Sort() 160 | var lastUser string 161 | for _, msg := range msgs { 162 | if lastUser == msg.User { 163 | msg.Trail = true 164 | } else { 165 | lastUser = msg.User 166 | } 167 | } 168 | } 169 | 170 | for _, thread := range m.ThreadMap { 171 | thread.replies.Sort() 172 | } 173 | 174 | // loaded marker 175 | m.loadedFiles[path] = struct{}{} 176 | return nil 177 | } 178 | 179 | // MessageMonthKey is a key for messages. 180 | type MessageMonthKey struct { 181 | year int 182 | month int 183 | } 184 | 185 | // NewMessageMonthKey creates MessageMonthKey key from two strings: which 186 | // represents year and month. 187 | func NewMessageMonthKey(year, month string) (MessageMonthKey, error) { 188 | y, err := strconv.Atoi(year) 189 | if err != nil { 190 | return MessageMonthKey{}, err 191 | } 192 | m, err := strconv.Atoi(month) 193 | if err != nil { 194 | return MessageMonthKey{}, err 195 | } 196 | return MessageMonthKey{year: y, month: m}, nil 197 | } 198 | 199 | // Next gets a key for next month. 200 | func (k MessageMonthKey) Next() MessageMonthKey { 201 | if k.month >= 12 { 202 | return MessageMonthKey{year: k.year + 1, month: 1} 203 | } 204 | return MessageMonthKey{year: k.year, month: k.month + 1} 205 | } 206 | 207 | // Prev gets a key for previous month. 208 | func (k MessageMonthKey) Prev() MessageMonthKey { 209 | if k.month <= 1 { 210 | return MessageMonthKey{year: k.year - 1, month: 12} 211 | } 212 | return MessageMonthKey{year: k.year, month: k.month - 1} 213 | } 214 | 215 | // Year returns string represents year. 216 | func (k MessageMonthKey) Year() string { 217 | return fmt.Sprintf("%4d", k.year) 218 | } 219 | 220 | // Month returns string represents month. 221 | func (k MessageMonthKey) Month() string { 222 | return fmt.Sprintf("%02d", k.month) 223 | } 224 | 225 | // NextYear returns a string for next year. 226 | func (k MessageMonthKey) NextYear() string { 227 | if k.month >= 12 { 228 | return fmt.Sprintf("%4d", k.year+1) 229 | } 230 | return fmt.Sprintf("%4d", k.year) 231 | } 232 | 233 | // NextMonth returns a string for next month. 234 | func (k MessageMonthKey) NextMonth() string { 235 | if k.month >= 12 { 236 | return "01" 237 | } 238 | return fmt.Sprintf("%02d", k.month+1) 239 | } 240 | 241 | // PrevYear returns a string for previous year. 242 | func (k MessageMonthKey) PrevYear() string { 243 | if k.month <= 1 { 244 | return fmt.Sprintf("%4d", k.year-1) 245 | } 246 | return fmt.Sprintf("%4d", k.year) 247 | } 248 | 249 | // PrevMonth returns a string for previous month. 250 | func (k MessageMonthKey) PrevMonth() string { 251 | if k.month <= 1 { 252 | return "12" 253 | } 254 | return fmt.Sprintf("%02d", k.month-1) 255 | } 256 | 257 | // Message : メッセージ 258 | // エクスポートしたYYYY-MM-DD.jsonの中身を保持する。 259 | // https://slack.com/intl/ja-jp/help/articles/220556107-Slack-%E3%81%8B%E3%82%89%E3%82%A8%E3%82%AF%E3%82%B9%E3%83%9D%E3%83%BC%E3%83%88%E3%81%97%E3%81%9F%E3%83%87%E3%83%BC%E3%82%BF%E3%81%AE%E8%AA%AD%E3%81%BF%E6%96%B9 260 | type Message struct { 261 | slack.Message 262 | 263 | // Trail shows the user of message is same as the previous one. 264 | // FIXME: 本来はココに書いてはいけない 265 | Trail bool `json:"-"` 266 | } 267 | 268 | // isVisible : 表示すべきメッセージ種別かを判定する。 269 | // 例えばchannel_joinなどは投稿された出力する必要がないため、falseを返す。 270 | func (m *Message) isVisible() bool { 271 | return m.SubType == "" || 272 | m.SubType == "bot_message" || 273 | m.SubType == "slackbot_response" || 274 | m.SubType == "thread_broadcast" 275 | } 276 | 277 | // isBotMessage : メッセージがBotからの物かを判定する。 278 | func (m *Message) isBotMessage() bool { 279 | return m.SubType == "bot_message" || 280 | m.SubType == "slackbot_response" 281 | } 282 | 283 | // IsRootOfThread : メッセージがスレッドの最初のメッセージであるかを判定する。 284 | func (m Message) IsRootOfThread() bool { 285 | return m.Timestamp == m.ThreadTimestamp 286 | } 287 | 288 | // isThreadChild returns true when a message should be shown in a thread only. 289 | func (m *Message) isThreadChild() bool { 290 | return m.ThreadTimestamp != "" && m.Timestamp != m.ThreadTimestamp && m.SubType != "thread_broadcast" 291 | } 292 | 293 | var reToken = regexp.MustCompile(`\?t=xoxe-[-a-f0-9]+$`) 294 | 295 | func removeToken(s string) string { 296 | return reToken.ReplaceAllLiteralString(s, "") 297 | } 298 | 299 | // RemoveTokenFromURLs removes the token from URLs in a message. 300 | func (m *Message) RemoveTokenFromURLs() { 301 | for i, f := range m.Files { 302 | f.URLPrivate = removeToken(f.URLPrivate) 303 | f.URLPrivateDownload = removeToken(f.URLPrivateDownload) 304 | f.Thumb64 = removeToken(f.Thumb64) 305 | f.Thumb80 = removeToken(f.Thumb80) 306 | f.Thumb160 = removeToken(f.Thumb160) 307 | f.Thumb360 = removeToken(f.Thumb360) 308 | f.Thumb480 = removeToken(f.Thumb480) 309 | f.Thumb720 = removeToken(f.Thumb720) 310 | f.Thumb800 = removeToken(f.Thumb800) 311 | f.Thumb960 = removeToken(f.Thumb960) 312 | f.Thumb1024 = removeToken(f.Thumb1024) 313 | f.Thumb360Gif = removeToken(f.Thumb360Gif) 314 | f.Thumb480Gif = removeToken(f.Thumb480Gif) 315 | f.DeanimateGif = removeToken(f.DeanimateGif) 316 | f.ThumbVideo = removeToken(f.ThumbVideo) 317 | m.Files[i] = f 318 | } 319 | } 320 | 321 | var filenameReplacer = strings.NewReplacer( 322 | `\`, "_", 323 | "/", "_", 324 | ":", "_", 325 | "*", "_", 326 | "?", "_", 327 | `"`, "_", 328 | "<", "_", 329 | ">", "_", 330 | "|", "_", 331 | ) 332 | 333 | // RegulateFilename replaces unusable characters as filepath by '_'. 334 | func RegulateFilename(s string) string { 335 | return filenameReplacer.Replace(s) 336 | } 337 | 338 | // MessageIcons represents icon for each message. 339 | type MessageIcons struct { 340 | Image48 string `json:"image_48"` 341 | } 342 | -------------------------------------------------------------------------------- /internal/slacklog/slack.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "net/url" 5 | "path" 6 | "path/filepath" 7 | "strings" 8 | "unicode/utf8" 9 | 10 | "github.com/slack-go/slack" 11 | ) 12 | 13 | // HostBySlack checks a file is hosted by slack or not. 14 | func HostBySlack(f slack.File) bool { 15 | return strings.HasPrefix(f.URLPrivate, "https://files.slack.com/") 16 | } 17 | 18 | // LocalName returns name of local downloaded file. 19 | func LocalName(f slack.File, url, suffix string) string { 20 | ext := filepath.Ext(url) 21 | nameExt := filepath.Ext(f.Name) 22 | name := f.Name[:len(f.Name)-len(nameExt)] 23 | if ext == "" { 24 | ext = nameExt 25 | if ext == "" { 26 | ext = FiletypeToExtension[f.Filetype] 27 | } 28 | } 29 | name = truncateName(name, 200) 30 | return RegulateFilename(name + suffix + ext) 31 | } 32 | 33 | func truncateName(name string, size int) string { 34 | if len(name) < size { 35 | return name 36 | } 37 | name = name[:size] 38 | 39 | if !utf8.ValidString(name) { 40 | v := make([]rune, 0, len(name)) 41 | for i, r := range name { 42 | if r == utf8.RuneError { 43 | _, size := utf8.DecodeRuneInString(name[i:]) 44 | if size == 1 { 45 | continue 46 | } 47 | } 48 | v = append(v, r) 49 | } 50 | name = string(v) 51 | } 52 | return name 53 | } 54 | 55 | // LocalPath returns path of local downloaded file. 56 | func LocalPath(f slack.File) string { 57 | return path.Join(f.ID, url.PathEscape(LocalName(f, f.URLPrivate, ""))) 58 | } 59 | 60 | // TopLevelMimetype extracts top level type from MIME Type. 61 | func TopLevelMimetype(f slack.File) string { 62 | i := strings.Index(f.Mimetype, "/") 63 | if i < 0 { 64 | return "" 65 | } 66 | return f.Mimetype[:i] 67 | } 68 | 69 | // ThumbImagePath returns path of thumbnail image file. 70 | func ThumbImagePath(f slack.File) string { 71 | if f.Thumb1024 == "" { 72 | return LocalPath(f) 73 | } 74 | return path.Join(f.ID, url.PathEscape(LocalName(f, f.Thumb1024, "_1024"))) 75 | } 76 | 77 | // ThumbImageWidth returns width of thumbnail image. 78 | func ThumbImageWidth(f slack.File) int { 79 | if f.Thumb1024 != "" { 80 | return f.Thumb1024W 81 | } 82 | return f.OriginalW 83 | } 84 | 85 | // ThumbImageHeight returns height of thumbnail image. 86 | func ThumbImageHeight(f slack.File) int { 87 | if f.Thumb1024 != "" { 88 | return f.Thumb1024H 89 | } 90 | return f.OriginalH 91 | } 92 | 93 | // ThumbVideoPath returns local path of thumbnail for the video. 94 | func ThumbVideoPath(f slack.File) string { 95 | return path.Join(f.ID, url.PathEscape(LocalName(f, f.ThumbVideo, "_video"))) 96 | } 97 | -------------------------------------------------------------------------------- /internal/slacklog/store.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | // LogStore : ログデータを各種テーブルを介して取得するための構造体。 10 | // MessageTableはチャンネル毎に用意しているためmtsはチャンネルIDをキーとするmap 11 | // となっている。 12 | type LogStore struct { 13 | path string 14 | ut *UserTable 15 | ct *ChannelTable 16 | et *EmojiTable 17 | // key: channel ID 18 | mts map[string]*MessageTable 19 | } 20 | 21 | // NewLogStore : 各テーブルを生成して、LogStoreを生成する。 22 | func NewLogStore(dirPath string, cfg *Config) (*LogStore, error) { 23 | ut, err := NewUserTable(filepath.Join(dirPath, "users.json")) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | ct, err := NewChannelTable(filepath.Join(dirPath, "channels.json"), cfg.Channels) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | et, err := NewEmojiTable(filepath.Join(dirPath, cfg.EmojiJSONPath)) 34 | if err != nil { 35 | if !os.IsNotExist(err) { 36 | return nil, err 37 | } 38 | // EmojiTable is not required, so if the file just doesn't exist, continue 39 | // processing. 40 | } 41 | 42 | mts := make(map[string]*MessageTable, len(ct.Channels)) 43 | for _, ch := range ct.Channels { 44 | mts[ch.ID] = NewMessageTable() 45 | } 46 | 47 | return &LogStore{ 48 | path: dirPath, 49 | ut: ut, 50 | ct: ct, 51 | et: et, 52 | mts: mts, 53 | }, nil 54 | } 55 | 56 | // GetChannels gets all stored channgels. 57 | func (s *LogStore) GetChannels() []Channel { 58 | return s.ct.Channels 59 | } 60 | 61 | // HasNextMonth returns a channel has next key or not. 62 | func (s *LogStore) HasNextMonth(channelID string, key MessageMonthKey) bool { 63 | if mt, ok := s.mts[channelID]; ok && mt != nil { 64 | _, ok := mt.MsgsMap[key.Next()] 65 | return ok 66 | } 67 | return false 68 | } 69 | 70 | // HasPrevMonth returns a channel has previous logs or not. 71 | func (s *LogStore) HasPrevMonth(channelID string, key MessageMonthKey) bool { 72 | if mt, ok := s.mts[channelID]; ok && mt != nil { 73 | _, ok := mt.MsgsMap[key.Prev()] 74 | return ok 75 | } 76 | return false 77 | } 78 | 79 | // GetMessagesPerMonth gets a messages map for the channel. 80 | // Messages map have all message split in per month. 81 | func (s *LogStore) GetMessagesPerMonth(channelID string) (MessagesMap, error) { 82 | mt, ok := s.mts[channelID] 83 | if !ok { 84 | return nil, fmt.Errorf("not found channel: id=%s", channelID) 85 | } 86 | if err := mt.ReadLogDir(filepath.Join(s.path, channelID), false); err != nil { 87 | return nil, err 88 | } 89 | 90 | return mt.MsgsMap, nil 91 | } 92 | 93 | // GetAllMessages returns all messages as array. 94 | func (s *LogStore) GetAllMessages(channelID string) (Messages, error) { 95 | mt, ok := s.mts[channelID] 96 | if !ok { 97 | return nil, fmt.Errorf("not found channel: id=%s", channelID) 98 | } 99 | err := mt.ReadLogDir(filepath.Join(s.path, channelID), true) 100 | if err != nil { 101 | return nil, err 102 | } 103 | var allMsgs Messages 104 | for _, msgs := range mt.MsgsMap { 105 | allMsgs = append(allMsgs, msgs...) 106 | } 107 | return allMsgs, nil 108 | } 109 | 110 | // GetUserByID gets a user by (user) ID. 111 | // Sometimes by bot ID. 112 | func (s *LogStore) GetUserByID(userID string) (*User, bool) { 113 | u, ok := s.ut.UserMap[userID] 114 | return u, ok 115 | } 116 | 117 | // GetDisplayNameByUserID gets display name for the user. 118 | func (s *LogStore) GetDisplayNameByUserID(userID string) string { 119 | if user, ok := s.ut.UserMap[userID]; ok { 120 | if user.Profile.RealName != "" { 121 | return user.Profile.RealName 122 | } 123 | if user.Profile.DisplayName != "" { 124 | return user.Profile.DisplayName 125 | } 126 | } 127 | return "" 128 | } 129 | 130 | // GetDisplayNameMap gets a map from user ID to user's display name. 131 | func (s *LogStore) GetDisplayNameMap() map[string]string { 132 | ret := make(map[string]string, len(s.ut.UserMap)) 133 | for id, u := range s.ut.UserMap { 134 | ret[id] = s.GetDisplayNameByUserID(u.ID) 135 | } 136 | return ret 137 | } 138 | 139 | // GetEmojiMap gets a map from emoji name to its file extension (image type). 140 | func (s *LogStore) GetEmojiMap() map[string]string { 141 | return s.et.NameToExt 142 | } 143 | 144 | // GetThread gets a thread (chain of messages) by channel ID and Ts (thread root's Ts). 145 | func (s *LogStore) GetThread(channelID, ts string) (*Thread, bool) { 146 | mt, ok := s.mts[channelID] 147 | if !ok { 148 | return nil, false 149 | } 150 | if t, ok := mt.ThreadMap[ts]; ok { 151 | return t, true 152 | } 153 | return nil, false 154 | } 155 | -------------------------------------------------------------------------------- /internal/slacklog/testdata/downloader/empty.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vim-jp/slacklog-generator/5915d740320a24900668e9fef1e1b74e66d53423/internal/slacklog/testdata/downloader/empty.txt -------------------------------------------------------------------------------- /internal/slacklog/testdata/downloader/hoge.txt: -------------------------------------------------------------------------------- 1 | hoge 2 | -------------------------------------------------------------------------------- /internal/slacklog/testdata/downloader/vim-jp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vim-jp/slacklog-generator/5915d740320a24900668e9fef1e1b74e66d53423/internal/slacklog/testdata/downloader/vim-jp.png -------------------------------------------------------------------------------- /internal/slacklog/thread.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Thread : スレッド 8 | // rootMsgはスレッドの先頭メッセージを表わす。 9 | // repliesにはそのスレッドへの返信メッセージが入る。先頭メッセージは含まない。 10 | type Thread struct { 11 | rootMsg *Message 12 | replies Messages 13 | } 14 | 15 | // LastReplyTime returns last replied time for the thread. 16 | func (th Thread) LastReplyTime() time.Time { 17 | return TsToDateTime(th.replies[len(th.replies)-1].Timestamp) 18 | } 19 | 20 | // ReplyCount return counts of replied messages. 21 | func (th Thread) ReplyCount() int { 22 | return len(th.replies) 23 | } 24 | 25 | // RootText returns text of root message of the thread. 26 | func (th Thread) RootText() string { 27 | if th.rootMsg != nil { 28 | return th.rootMsg.Text 29 | } 30 | return "" 31 | } 32 | 33 | // Replies returns replied messages for the thread. 34 | func (th Thread) Replies() Messages { 35 | return th.replies 36 | } 37 | 38 | // Put puts a message to the thread as "root" or "reply". 39 | func (th *Thread) Put(m *Message) { 40 | if m.IsRootOfThread() { 41 | th.rootMsg = m 42 | } else { 43 | th.replies = append(th.replies, m) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/slacklog/time.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | // TsToDateTime converts Ts string to time.Time. 12 | func TsToDateTime(ts string) time.Time { 13 | t := strings.Split(ts, ".") 14 | if len(t) != 2 { 15 | fmt.Fprintf(os.Stderr, "[warning] invalid timestamp: %s ...\n", ts) 16 | return time.Time{} 17 | } 18 | sec, err := strconv.ParseInt(t[0], 10, 64) 19 | if err != nil { 20 | fmt.Fprintf(os.Stderr, "[warning] invalid timestamp: %s ...\n", ts) 21 | return time.Time{} 22 | } 23 | nsec, err := strconv.ParseInt(t[1], 10, 64) 24 | if err != nil { 25 | fmt.Fprintf(os.Stderr, "[warning] invalid timestamp: %s ...\n", ts) 26 | return time.Time{} 27 | } 28 | japan, err := time.LoadLocation("Asia/Tokyo") 29 | if err != nil { 30 | fmt.Fprintf(os.Stderr, "[warning] invalid timestamp: %s ...\n", ts) 31 | return time.Time{} 32 | } 33 | return time.Unix(sec, nsec).In(japan) 34 | } 35 | 36 | // LevelOfDetailTime returns a label string which represents time. The 37 | // resolution of the string is determined by differece from base time in 4 38 | // levels. 39 | func LevelOfDetailTime(target, base time.Time) string { 40 | if target.Year() != base.Year() { 41 | return target.Format("2006年1月2日 15:04:05") 42 | } 43 | if target.Month() != base.Month() { 44 | return target.Format("1月2日 15:04:05") 45 | } 46 | if target.Day() != base.Day() { 47 | return target.Format("2日 15:04:05") 48 | } 49 | return target.Format("15:04:05") 50 | } 51 | -------------------------------------------------------------------------------- /internal/slacklog/ts.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strconv" 7 | ) 8 | 9 | // Ts is a type represents "Ts" fields in message.go or so. 10 | type Ts struct { 11 | IsNumber bool 12 | Value string 13 | } 14 | 15 | var _ json.Marshaler = (*Ts)(nil) 16 | var _ json.Unmarshaler = (*Ts)(nil) 17 | 18 | // UnmarshalJSON implements "encoding/json".Unmarshaller interface. 19 | func (ts *Ts) UnmarshalJSON(b []byte) error { 20 | var v interface{} 21 | err := json.Unmarshal(b, &v) 22 | if err != nil { 23 | return nil 24 | } 25 | switch w := v.(type) { 26 | case string: 27 | ts.IsNumber = false 28 | ts.Value = w 29 | case float64: 30 | ts.IsNumber = true 31 | ts.Value = strconv.FormatFloat(w, 'g', -1, 64) 32 | default: 33 | return fmt.Errorf("unsupproted type for Ts: %t", v) 34 | } 35 | return nil 36 | } 37 | 38 | // MarshalJSON implements "encoding/json".Marshaller interface. 39 | func (ts Ts) MarshalJSON() ([]byte, error) { 40 | if ts.IsNumber { 41 | f, err := strconv.ParseFloat(ts.Value, 64) 42 | if err != nil { 43 | f = 0 44 | } 45 | return json.Marshal(f) 46 | } 47 | return json.Marshal(ts.Value) 48 | } 49 | -------------------------------------------------------------------------------- /internal/slacklog/ts_test.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/google/go-cmp/cmp" 8 | ) 9 | 10 | func TestTs_Unmarshal(t *testing.T) { 11 | for _, tc := range []struct { 12 | json string 13 | exp Ts 14 | }{ 15 | {`"1234.5678"`, Ts{false, "1234.5678"}}, 16 | {`1234.5678`, Ts{true, "1234.5678"}}, 17 | } { 18 | var act Ts 19 | err := json.Unmarshal([]byte(tc.json), &act) 20 | if err != nil { 21 | t.Fatalf("unmarshal(%q) failed: %s", tc.json, err) 22 | } 23 | if diff := cmp.Diff(tc.exp, act); diff != "" { 24 | t.Fatalf("unexpected unmarshal Ts: -want +got\n%s", diff) 25 | } 26 | } 27 | } 28 | 29 | func TestTs_Marshal(t *testing.T) { 30 | for _, tc := range []struct { 31 | ts Ts 32 | exp string 33 | }{ 34 | {Ts{false, "1234.5678"}, `"1234.5678"`}, 35 | {Ts{true, "1234.5678"}, `1234.5678`}, 36 | } { 37 | var act string 38 | b, err := json.Marshal(tc.ts) 39 | if err != nil { 40 | t.Fatalf("marshal(%+v) failed: %s", tc.ts, err) 41 | } 42 | act = string(b) 43 | if diff := cmp.Diff(tc.exp, act); diff != "" { 44 | t.Fatalf("unexpected marshal Ts: -want +got\n%s", diff) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/slacklog/user.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import "github.com/slack-go/slack" 4 | 5 | // UserTable : ユーザデータを保持する 6 | // UsersもUserMapも保持するユーザデータは同じで、UserMapはユーザIDをキーとする 7 | // mapとなっている。 8 | // ユースケースに応じてUsersとUserMapは使い分ける。 9 | type UserTable struct { 10 | Users []User 11 | // key: user ID 12 | UserMap map[string]*User 13 | } 14 | 15 | // NewUserTable : pathに指定したJSON形式のユーザデータを読み込み、UserTableを生 16 | // 成する。 17 | func NewUserTable(path string) (*UserTable, error) { 18 | var users []User 19 | err := ReadFileAsJSON(path, true, &users) 20 | if err != nil { 21 | return nil, err 22 | } 23 | userMap := make(map[string]*User, len(users)) 24 | for i, u := range users { 25 | pu := &users[i] 26 | userMap[u.ID] = pu 27 | if u.Profile.BotID != "" { 28 | userMap[u.Profile.BotID] = pu 29 | } 30 | } 31 | return &UserTable{users, userMap}, nil 32 | } 33 | 34 | // User : ユーザ 35 | // エクスポートしたuser.jsonの中身を保持する。 36 | // 公式の情報は以下だがuser.jsonの解説までは書かれていない。 37 | // https://slack.com/intl/ja-jp/help/articles/220556107-Slack-%E3%81%8B%E3%82%89%E3%82%A8%E3%82%AF%E3%82%B9%E3%83%9D%E3%83%BC%E3%83%88%E3%81%97%E3%81%9F%E3%83%87%E3%83%BC%E3%82%BF%E3%81%AE%E8%AA%AD%E3%81%BF%E6%96%B9 38 | type User slack.User 39 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/joho/godotenv" 8 | cli "github.com/urfave/cli/v2" 9 | "github.com/vim-jp/slacklog-generator/subcmd" 10 | "github.com/vim-jp/slacklog-generator/subcmd/buildindex" 11 | "github.com/vim-jp/slacklog-generator/subcmd/fetchchannels" 12 | "github.com/vim-jp/slacklog-generator/subcmd/fetchmessages" 13 | "github.com/vim-jp/slacklog-generator/subcmd/fetchusers" 14 | "github.com/vim-jp/slacklog-generator/subcmd/serve" 15 | ) 16 | 17 | func main() { 18 | err := godotenv.Load() 19 | if err != nil { 20 | log.Printf("[WARN] failed to load .env files") 21 | } 22 | 23 | app := cli.NewApp() 24 | app.EnableBashCompletion = true 25 | app.Name = "slacklog-generator" 26 | app.Usage = "generate slacklog HTML" 27 | app.Version = "0.0.0" 28 | app.Commands = []*cli.Command{ 29 | subcmd.ConvertExportedLogsCommand, // "convert-exported-logs" 30 | subcmd.DownloadEmojiCommand, // "download-emoji" 31 | subcmd.DownloadFilesCommand, // "download-files" 32 | subcmd.GenerateHTMLCommand, // "generate-html" 33 | serve.Command, // "serve" 34 | buildindex.NewCLICommand(), // "build-index" 35 | fetchmessages.NewCLICommand(), // "fetch-messages" 36 | fetchchannels.NewCLICommand(), // "fetch-channels" 37 | fetchusers.NewCLICommand(), // "fetch-users" 38 | } 39 | 40 | err = app.Run(os.Args) 41 | if err != nil { 42 | log.Printf("[ERROR] %s", err) 43 | os.Exit(1) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | outdir="_site" 6 | 7 | while getopts o: OPT ; do 8 | case $OPT in 9 | o) outdir="$OPTARG" ;; 10 | esac 11 | done 12 | 13 | mkdir -p ${outdir} 14 | 15 | cp -a static/* ${outdir} 16 | 17 | for d in emojis files ; do 18 | if [ ! -d _logdata/${d} ] ; then 19 | echo "one of input missing. please run 'make logdata' and retry" 20 | exit 1 21 | fi 22 | cp -a _logdata/${d} ${outdir} 23 | done 24 | -------------------------------------------------------------------------------- /scripts/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "edited_suffix": "", 3 | "channels": [ 4 | "*" 5 | ], 6 | "emoji_json_path": "../slacklog_data/emoji.json" 7 | } 8 | -------------------------------------------------------------------------------- /scripts/download_emoji.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | cd "$(dirname "$0")/.." || exit "$?" 6 | 7 | if [ ! -d _logdata/slacklog_data/ ] ; then 8 | echo "one of input missing. please make assure _logdata/slacklog_data/ directory" 9 | exit 1 10 | fi 11 | 12 | go run . download-emoji 13 | -------------------------------------------------------------------------------- /scripts/download_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | cd "$(dirname "$0")/.." || exit "$?" 6 | 7 | if [ ! -d _logdata/slacklog_data/ ] ; then 8 | echo "one of input missing. please make assure _logdata/slacklog_data/ directory" 9 | exit 1 10 | fi 11 | 12 | go run . download-files 13 | -------------------------------------------------------------------------------- /scripts/generate_html.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | cd "$(dirname "$0")/.." || exit "$?" 6 | 7 | if [ ! -d _logdata/slacklog_data/ ] ; then 8 | echo "one of input missing. please run 'make logdata' and retry" 9 | exit 1 10 | fi 11 | 12 | go run . generate-html 13 | -------------------------------------------------------------------------------- /scripts/site_diff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | force=0 6 | clean=0 7 | update=0 8 | outdiff="" 9 | 10 | while getopts fcuo: OPT ; do 11 | case $OPT in 12 | f) force=1 ;; 13 | c) clean=1 ;; 14 | u) update=1 ;; 15 | o) outdiff="$OPTARG" ;; 16 | esac 17 | done 18 | 19 | cd "$(dirname "$0")/.." || exit "$?" 20 | 21 | outrootdir=tmp/_site_diff 22 | current_pages=${outrootdir}/current 23 | cmd=${outrootdir}/slacklog-tools 24 | 25 | build_tool() { 26 | if [ -f main.go ] ; then 27 | go build -o ${cmd} . 1>&2 28 | else 29 | cd scripts 30 | go build -o ../${cmd} ./main.go 1>&2 31 | cd .. 32 | fi 33 | } 34 | 35 | # generate-html サブコマンドと build.sh を実行して指定ディレクトリに出力する 36 | generate_site() { 37 | id=$1 ; shift 38 | outdir=${outrootdir}/${id} 39 | echo "build to: ${outdir}" 1>&2 40 | rm -rf ${outdir} 41 | build_tool 42 | tmpldir=slacklog_template/ 43 | if [ -d templates ] ; then 44 | tmpldir=templates/ 45 | fi 46 | logdir=slacklog_data/ 47 | if [ -d _logdata ] ; then 48 | logdir=_logdata/slacklog_data/ 49 | fi 50 | filesdir=files/ 51 | if [ -d _logdata ] ; then 52 | filesdir=_logdata/files/ 53 | fi 54 | ${cmd} generate-html --templatedir ${tmpldir} --filesdir ${filesdir} --indir ${logdir} --outdir ${outdir}/ > ${outdir}.generate-html.log 2>&1 || ( cat ${outdir}.generate-html.log && exit 1 ) 55 | rm -f ${cmd} 56 | ./scripts/build.sh -o $outdir > ${outdir}.build.log 2>&1 || ( cat ${outdir}.build.log && exit 1 ) 57 | } 58 | 59 | if [ $clean -ne 0 ] ; then 60 | echo "clean up $outrootdir" 1>&2 61 | rm -rf $outrootdir 62 | exit 0 63 | fi 64 | 65 | generate_site "current" 66 | 67 | if [ $update -ne 0 ] ; then 68 | echo "catching up origin/master" 1>&2 69 | git fetch -q origin master 70 | fi 71 | 72 | base_commit=$(git show-branch --merge-base origin/master HEAD) 73 | base_pages=${outrootdir}/${base_commit} 74 | 75 | if [ $force -ne 0 -o ! \( -d $base_pages \) ] ; then 76 | echo "base commit: $base_commit" 1>&2 77 | 78 | # 現在の変更とHEADの commit ID を退避する 79 | has_changes=$(git status -s -uno | wc -l) 80 | if [ $has_changes -ne 0 ] ; then 81 | git stash push -q 82 | fi 83 | current_commit=$(git rev-parse HEAD) 84 | 85 | # merge-base に巻き戻し generate-html を実行する 86 | git reset -q --hard ${base_commit} 87 | echo "move to base: $(git rev-parse HEAD)" 1>&2 88 | generate_site ${base_commit} 89 | 90 | # 退避したHEADと変更を復帰する 91 | git reset -q --hard ${current_commit} 92 | if [ $has_changes -ne 0 ] ; then 93 | git stash pop -q 94 | fi 95 | 96 | echo "return to current: $(git rev-parse HEAD)" 1>&2 97 | fi 98 | 99 | # 差分を出力 100 | if [ x"$outdiff" = x ] ; then 101 | echo "" 1>&2 102 | diff -uNrw -x sitemap.xml ${base_pages} ${current_pages} || true 103 | else 104 | diff -uNrw -x sitemap.xml ${base_pages} ${current_pages} > "$outdiff" || ( 105 | echo "" 1>&2 106 | echo "have some diff, please check $outdiff" 1>&2 107 | ) 108 | fi 109 | -------------------------------------------------------------------------------- /static/assets/css/site.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding: 1em; 3 | font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; 4 | color: #000; 5 | } 6 | 7 | #content { 8 | float: left; 9 | clear: left; 10 | margin: 0px -400px 0px 0px; 11 | width: 100%; 12 | } 13 | 14 | #footer { 15 | text-align: center; 16 | } 17 | 18 | a, a:link { 19 | color: #6699cc; 20 | text-decoration: none; 21 | } 22 | 23 | a:active, a:hover { 24 | color: #669933; 25 | text-decoration: underline; 26 | } 27 | 28 | a:visited { 29 | color: #8899aa; 30 | } 31 | 32 | code { 33 | margin: 1px; 34 | padding: 2px; 35 | line-height: 1.4em; 36 | border: solid 1px #ccc; 37 | } 38 | 39 | pre > code { 40 | border: 0px; 41 | margin: 0px; 42 | padding: 0px; 43 | } 44 | 45 | pre { 46 | color: white; 47 | background-color: black; 48 | padding: 0.5em; 49 | overflow: scroll; 50 | } 51 | 52 | .slacklog-emoji { 53 | height: 1.0em; 54 | } 55 | 56 | pre[class*="language-"] { 57 | resize: vertical; 58 | min-height: 15vh; 59 | max-height: 80vh; 60 | height: 30vh; 61 | } 62 | 63 | /* vim:set ts=8 sts=2 sw=2 noet: */ 64 | -------------------------------------------------------------------------------- /static/assets/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vim-jp/slacklog-generator/5915d740320a24900668e9fef1e1b74e66d53423/static/assets/images/favicon.ico -------------------------------------------------------------------------------- /static/assets/images/vim2-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vim-jp/slacklog-generator/5915d740320a24900668e9fef1e1b74e66d53423/static/assets/images/vim2-128.png -------------------------------------------------------------------------------- /static/assets/javascripts/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vim-jp/slacklog-generator/5915d740320a24900668e9fef1e1b74e66d53423/static/assets/javascripts/.gitkeep -------------------------------------------------------------------------------- /static/assets/javascripts/search.js: -------------------------------------------------------------------------------- 1 | window.addEventListener('DOMContentLoaded', async () => { 2 | const GRAM_N = 2; 3 | const toHexString = (n) => { 4 | return n.toString(16).padStart(2, "0"); 5 | }; 6 | const to2dString = (n) => { 7 | return n.toString().padStart(2, "0"); 8 | }; 9 | 10 | class Uint8ArrayReader { 11 | constructor(u8ary) { 12 | this.u8ary = u8ary; 13 | this.i = 0; 14 | } 15 | 16 | get length() { 17 | return this.u8ary.length; 18 | } 19 | 20 | readInt() { 21 | const n = (this.u8ary[this.i] << 24) | (this.u8ary[this.i + 1] << 16) | (this.u8ary[this.i + 2] << 8) | this.u8ary[this.i + 3]; 22 | this.i += 4; 23 | return n; 24 | } 25 | 26 | readVInt() { 27 | let n = 0; 28 | let cont = true; 29 | for (; cont; this.i++) { 30 | n <<= 7; 31 | n |= this.u8ary[this.i] & 0b01111111; 32 | cont = (this.u8ary[this.i] & 0b10000000) != 0; 33 | } 34 | return n; 35 | } 36 | 37 | isEOF() { 38 | return this.u8ary.length <= this.i; 39 | } 40 | } 41 | 42 | class Index { 43 | constructor() { 44 | this.indexes = new Map(); 45 | } 46 | 47 | async get(key) { 48 | const cached = this.indexes.get(key); 49 | if (cached) { 50 | return cached; 51 | } 52 | const paths = []; 53 | for (let i = 0; i < key.length; i++) { 54 | const n = key.charCodeAt(i); 55 | paths.push(toHexString(n >> 8)); 56 | paths.push(toHexString(n & 0xff)); 57 | } 58 | const res = await fetch(`./index/${paths.join("/")}.index`); 59 | const index = new Map(); 60 | if (res.ok) { 61 | const blob = await res.blob(); 62 | const reader = new Uint8ArrayReader(new Uint8Array(await blob.arrayBuffer())); 63 | while (!reader.isEOF()) { 64 | const channelNumber = reader.readVInt(); 65 | let mesCount = reader.readVInt(); 66 | while (0 <= --mesCount) { 67 | const tsSec = reader.readInt(); 68 | const tsMicrosec = reader.readVInt(); 69 | const ts = `${tsSec}.${tsMicrosec.toString().padStart(6, "0")}`; 70 | 71 | const key = `${channelNumber}:${ts}`; 72 | let posSet = index.get(key); 73 | if (posSet == null) { 74 | posSet = new Set(); 75 | index.set(key, posSet); 76 | } 77 | for (;;) { 78 | const pos = reader.readVInt(); 79 | if (pos === 0) { 80 | break; 81 | } 82 | posSet.add(pos - 1); 83 | } 84 | } 85 | } 86 | } else { 87 | // TODO: check error type 88 | } 89 | this.indexes.set(key, index); 90 | return index; 91 | } 92 | } 93 | 94 | const index = new Index(); 95 | const sepRegexp = new RegExp(`.{1,${GRAM_N}}`, "g"); 96 | 97 | const numToChannel = await (async () => { 98 | const map = new Map(); 99 | const res = await fetch("./index/channel"); 100 | for (const line of (await res.text()).split("\n")) { 101 | const [n, channelID, channelName] = line.split("\t"); 102 | if (channelID != null) { 103 | map.set(n - 0, {channelID, channelName}); 104 | } 105 | } 106 | return map; 107 | })(); 108 | 109 | const searchByWord = async (word) => { 110 | const indexes = await Promise.all( 111 | [...word.matchAll(sepRegexp)].map(async (chunk) => index.get(chunk[0])) 112 | ); 113 | const result = indexes.reduce((acc, cur) => { 114 | const next = new Map(); 115 | for (const [key, prevPosSet] of acc.entries()) { 116 | const currentPosSet = cur.get(key); 117 | for (const prevPos of prevPosSet.values()) { 118 | if (currentPosSet?.has(prevPos + GRAM_N)) { 119 | let nestPosSet = next.get(key); 120 | if (nestPosSet == null) { 121 | nestPosSet = new Set(); 122 | next.set(key, nestPosSet); 123 | } 124 | nestPosSet.add(prevPos + GRAM_N); 125 | } 126 | } 127 | } 128 | return next; 129 | }); 130 | 131 | return result; 132 | }; 133 | 134 | const parseQuery = (query) => { 135 | // TODO: parse 136 | return query; 137 | }; 138 | 139 | const search = async (query) => { 140 | const word = parseQuery(query); 141 | return searchByWord(word); 142 | }; 143 | 144 | const text = document.getElementById("search-text"); 145 | const resultElement = document.getElementById("result"); 146 | 147 | const execute = async () => { 148 | try { 149 | const startTime = Date.now(); 150 | 151 | const result = await search(text.value); 152 | 153 | const links = 154 | [...result.keys()] 155 | .map((doc) => doc.split(":")) 156 | .map(([channelNumber, ts]) => [channelNumber, ts, parseFloat(ts)]) 157 | .sort(([, , tsA], [, , tsB]) => tsB - tsA) 158 | .map( 159 | ([channelNumber, ts, tsFloat]) => { 160 | const {channelID, channelName} = numToChannel.get(channelNumber - 0); 161 | const date = new Date(tsFloat * 1000); 162 | const link = `${channelID}/${date.getFullYear()}/${(date.getMonth() + 1).toString().padStart(2, "0")}/#ts-${ts}`; 163 | return `#${channelName}: ${date.getFullYear()}-${to2dString(date.getMonth() + 1)}-${to2dString(date.getDate())} ${to2dString(date.getHours())}:${to2dString(date.getMinutes())}:${to2dString(date.getSeconds())}`; 164 | } 165 | ); 166 | const processTime = Date.now() - startTime; 167 | resultElement.innerHTML = `

${links.length} 件ヒットしました (${processTime / 1000} 秒)

${links.join("
")}`; 168 | } catch (e) { 169 | resultElement.innerHTML = `検索中にエラーが発生しました: ${e.name}: ${e.message}`; 170 | } 171 | }; 172 | 173 | const searchFromParams = () => { 174 | const query = new URL(location).searchParams.get("q"); 175 | if (query) { 176 | text.value = query; 177 | execute(); 178 | } else { 179 | text.value = ""; 180 | resultElement.innerHTML = ""; 181 | } 182 | }; 183 | 184 | const moveToResultPage = () => { 185 | const params = new URLSearchParams(); 186 | params.set("q", text.value); 187 | history.pushState({q: text.value}, document.title, `${location.pathname}?${params.toString()}`); 188 | execute(); 189 | }; 190 | 191 | document.getElementById("search-button").addEventListener("click", moveToResultPage); 192 | document.getElementById("search-text").addEventListener("keypress", (event) => { 193 | if (event.key === "Enter") { 194 | moveToResultPage(); 195 | } 196 | }); 197 | window.addEventListener("popstate", (e) => { 198 | console.log(e); 199 | searchFromParams(); 200 | }); 201 | 202 | searchFromParams(); 203 | }); 204 | -------------------------------------------------------------------------------- /static/assets/javascripts/slacklog.js: -------------------------------------------------------------------------------- 1 | const highlightMsg = (function () { 2 | let prevId = ''; 3 | 4 | return (hash) => { 5 | if (hash === '') return; 6 | const id = location.hash.substring(1); 7 | const $msg = $('#' + CSS.escape(id)); 8 | // open thread if inner message is specified by URL fragment 9 | $msg.closest('details.slacklog-thread').attr('open', ''); 10 | // begin highlight animation (see assets/css/slacklog.css) 11 | if (prevId !== '') { 12 | $('#' + CSS.escape(prevId)).removeClass('slacklog-highlight'); 13 | } 14 | $msg.addClass('slacklog-highlight'); 15 | prevId = id; 16 | }; 17 | })(); 18 | 19 | window.addEventListener('hashchange', () => { 20 | highlightMsg(location.hash); 21 | }); 22 | window.addEventListener('DOMContentLoaded', () => { 23 | highlightMsg(location.hash); 24 | }); 25 | -------------------------------------------------------------------------------- /static/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | vim-jp » vim-jp.slack.com log search 7 | 8 | 9 | 10 | 11 | 12 |
13 |

ログ検索

14 | 15 | 16 | 17 |
18 |
19 | 20 | -------------------------------------------------------------------------------- /subcmd/buildindex/buildindex.go: -------------------------------------------------------------------------------- 1 | package buildindex 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | 7 | cli "github.com/urfave/cli/v2" 8 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 9 | ) 10 | 11 | func run(datadir, outdir, config string) error { 12 | configJSONPath := filepath.Clean(config) 13 | cfg, err := slacklog.ReadConfig(configJSONPath) 14 | if err != nil { 15 | return fmt.Errorf("could not read config: %w", err) 16 | } 17 | s, err := slacklog.NewLogStore(datadir, cfg) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | i := slacklog.NewIndexer(s) 23 | err = i.Build() 24 | if err != nil { 25 | return err 26 | } 27 | 28 | err = i.Output(outdir) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | return nil 34 | } 35 | 36 | func NewCLICommand() *cli.Command { 37 | var ( 38 | datadir string 39 | outdir string 40 | config string 41 | ) 42 | return &cli.Command{ 43 | Name: "build-index", 44 | Usage: "build index for searching", 45 | Action: func(c *cli.Context) error { 46 | return run(datadir, outdir, config) 47 | }, 48 | Flags: []cli.Flag{ 49 | &cli.StringFlag{ 50 | Name: "config", 51 | Usage: "config.json path", 52 | Value: filepath.Join("scripts", "config.json"), 53 | Destination: &config, 54 | }, 55 | &cli.StringFlag{ 56 | Name: "datadir", 57 | Usage: "directory to load/save data", 58 | Value: "_logdata", 59 | Destination: &datadir, 60 | }, 61 | &cli.StringFlag{ 62 | Name: "outdir", 63 | Usage: "directory to output result", 64 | Destination: &outdir, 65 | }, 66 | }, 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /subcmd/convert_exported_logs.go: -------------------------------------------------------------------------------- 1 | /* 2 | リファクタリング中 3 | 処理をslacklog packageに移動していく。 4 | 一旦、必要な処理はすべてslacklog packageから一時的にエクスポートするか、このファ 5 | イル内で定義している。 6 | */ 7 | 8 | package subcmd 9 | 10 | import ( 11 | "encoding/json" 12 | "fmt" 13 | "io" 14 | "os" 15 | "path/filepath" 16 | "regexp" 17 | 18 | cli "github.com/urfave/cli/v2" 19 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 20 | ) 21 | 22 | // ConvertExportedLogsCommand provides "convert-exported-logs". It converts log 23 | // which exported from Slack, to generator can treat. 24 | var ConvertExportedLogsCommand = &cli.Command{ 25 | Name: "convert-exported-logs", 26 | Usage: "convert slack exported logs to API download logs", 27 | Action: convertExportedLogs, 28 | Flags: []cli.Flag{ 29 | &cli.StringFlag{ 30 | Name: "indir", 31 | Usage: "exported log dir", 32 | Value: "_old_logs", 33 | }, 34 | &cli.StringFlag{ 35 | Name: "outdir", 36 | Usage: "slacklog_data dir", 37 | Value: filepath.Join("_logdata", "slacklog_data"), 38 | }, 39 | }, 40 | } 41 | 42 | // convertExportedLogs converts log which exported from Slack, to generator can 43 | // treat. 44 | func convertExportedLogs(c *cli.Context) error { 45 | inDir := filepath.Clean(c.String("indir")) 46 | outDir := filepath.Clean(c.String("outdir")) 47 | inChannelsFile := filepath.Join(inDir, "channels.json") 48 | 49 | channels, _, err := readChannels(inChannelsFile, []string{"*"}) 50 | if err != nil { 51 | return fmt.Errorf("could not read channels.json: %w", err) 52 | } 53 | 54 | if err := os.MkdirAll(outDir, 0777); err != nil { 55 | return fmt.Errorf("could not create %s directory: %w", outDir, err) 56 | } 57 | 58 | err = copyFile(filepath.Join(inDir, "channels.json"), filepath.Join(outDir, "channels.json")) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | err = copyFile(filepath.Join(inDir, "users.json"), filepath.Join(outDir, "users.json")) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | for _, channel := range channels { 69 | messages, err := ReadAllMessages(filepath.Join(inDir, channel.Name)) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | for _, message := range messages { 75 | message.RemoveTokenFromURLs() 76 | } 77 | 78 | channelDir := filepath.Join(outDir, channel.ID) 79 | if err := os.MkdirAll(channelDir, 0777); err != nil { 80 | return fmt.Errorf("could not create %s directory: %w", channelDir, err) 81 | } 82 | 83 | messagesPerDay := groupMessagesByDay(messages) 84 | for key, msgs := range messagesPerDay { 85 | err = writeMessages(filepath.Join(channelDir, key+".json"), msgs) 86 | if err != nil { 87 | return err 88 | } 89 | } 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func copyFile(from string, to string) error { 96 | r, err := os.Open(from) 97 | if err != nil { 98 | return err 99 | } 100 | defer r.Close() 101 | 102 | w, err := os.Create(to) 103 | if err != nil { 104 | return err 105 | } 106 | defer w.Close() 107 | 108 | _, err = io.Copy(w, r) 109 | 110 | return err 111 | } 112 | 113 | func readChannels(channelJSONPath string, cfgChannels []string) ([]slacklog.Channel, map[string]*slacklog.Channel, error) { 114 | var channels []slacklog.Channel 115 | err := slacklog.ReadFileAsJSON(channelJSONPath, true, &channels) 116 | if err != nil { 117 | return nil, nil, err 118 | } 119 | 120 | channels = slacklog.FilterChannel(channels, cfgChannels) 121 | slacklog.SortChannel(channels) 122 | channelMap := make(map[string]*slacklog.Channel, len(channels)) 123 | for i, ch := range channels { 124 | channelMap[ch.ID] = &channels[i] 125 | } 126 | return channels, channelMap, err 127 | } 128 | 129 | // ReadAllMessages reads all message files in the directory. 130 | func ReadAllMessages(inDir string) ([]*slacklog.Message, error) { 131 | dir, err := os.Open(inDir) 132 | if err != nil { 133 | return nil, err 134 | } 135 | defer dir.Close() 136 | 137 | names, err := dir.Readdirnames(0) 138 | if err != nil { 139 | return nil, err 140 | } 141 | 142 | r, err := regexp.Compile(`\d{4}-\d{2}-\d{2}\.json`) 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | var messages []*slacklog.Message 148 | for _, name := range names { 149 | if !r.MatchString(name) { 150 | continue 151 | } 152 | var msgs []*slacklog.Message 153 | err := slacklog.ReadFileAsJSON(filepath.Join(inDir, name), false, &msgs) 154 | if err != nil { 155 | return nil, err 156 | } 157 | messages = append(messages, msgs...) 158 | } 159 | 160 | return messages, nil 161 | } 162 | 163 | func groupMessagesByDay(messages []*slacklog.Message) map[string][]*slacklog.Message { 164 | messagesPerDay := map[string][]*slacklog.Message{} 165 | for _, msg := range messages { 166 | time := slacklog.TsToDateTime(msg.Timestamp).Format("2006-01-02") 167 | messagesPerDay[time] = append(messagesPerDay[time], msg) 168 | } 169 | return messagesPerDay 170 | } 171 | 172 | func writeMessages(filename string, messages []*slacklog.Message) error { 173 | file, err := os.Create(filename) 174 | if err != nil { 175 | return err 176 | } 177 | defer file.Close() 178 | encoder := json.NewEncoder(file) 179 | encoder.SetEscapeHTML(false) 180 | err = encoder.Encode(messages) 181 | if err != nil { 182 | return err 183 | } 184 | return nil 185 | } 186 | -------------------------------------------------------------------------------- /subcmd/download_emoji.go: -------------------------------------------------------------------------------- 1 | /* 2 | リファクタリング中 3 | 処理をslacklog packageに移動していく。 4 | 一旦、必要な処理はすべてslacklog packageから一時的にエクスポートするか、このファ 5 | イル内で定義している。 6 | */ 7 | 8 | package subcmd 9 | 10 | import ( 11 | "encoding/json" 12 | "fmt" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | 17 | "github.com/slack-go/slack" 18 | cli "github.com/urfave/cli/v2" 19 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 20 | ) 21 | 22 | // DownloadEmojiCommand provides "download-emoji". It downloads and save emoji 23 | // image files. 24 | var DownloadEmojiCommand = &cli.Command{ 25 | Name: "download-emoji", 26 | Usage: "download customized emoji from slack", 27 | Action: downloadEmoji, 28 | Flags: []cli.Flag{ 29 | &cli.StringFlag{ 30 | Name: "outdir", 31 | Usage: "emojis download target dir", 32 | Value: filepath.Join("_logdata", "emojis"), 33 | }, 34 | &cli.StringFlag{ 35 | Name: "emojiJSON", 36 | Usage: "emoji json path", 37 | Value: filepath.Join("_logdata", "emojis", "emoji.json"), 38 | }, 39 | }, 40 | } 41 | 42 | // downloadEmoji downloads and save emoji image files. 43 | func downloadEmoji(c *cli.Context) error { 44 | slackToken := os.Getenv("SLACK_TOKEN") 45 | if slackToken == "" { 46 | return fmt.Errorf("$SLACK_TOKEN required") 47 | } 48 | 49 | emojisDir := filepath.Clean(c.String("outdir")) 50 | emojiJSONPath := filepath.Clean(c.String("emojiJSON")) 51 | 52 | api := slack.New(slackToken) 53 | 54 | emojis, err := api.GetEmoji() 55 | if err != nil { 56 | return err 57 | } 58 | 59 | d := slacklog.NewDownloader(slackToken) 60 | 61 | go generateEmojiFileTargets(d, emojis, emojisDir) 62 | 63 | err = outputSummary(emojis, emojiJSONPath) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | err = d.Wait() 69 | if err != nil { 70 | return err 71 | } 72 | return nil 73 | } 74 | 75 | func generateEmojiFileTargets(d *slacklog.Downloader, emojis map[string]string, outputDir string) { 76 | defer d.CloseQueue() 77 | err := os.MkdirAll(outputDir, 0777) 78 | if err != nil { 79 | fmt.Fprintf(os.Stderr, "failed to create %s: %s", outputDir, err) 80 | return 81 | } 82 | 83 | for name, url := range emojis { 84 | if strings.HasPrefix(url, "alias:") { 85 | continue 86 | } 87 | ext := filepath.Ext(url) 88 | path := filepath.Join(outputDir, name+ext) 89 | d.QueueDownloadRequest( 90 | url, 91 | path, 92 | false, 93 | ) 94 | } 95 | } 96 | 97 | func outputSummary(emojis map[string]string, path string) error { 98 | exts := make(map[string]string, len(emojis)) 99 | for name, url := range emojis { 100 | if strings.HasPrefix(url, "alias:") { 101 | exts[name] = url 102 | continue 103 | } 104 | ext := filepath.Ext(url) 105 | exts[name] = ext 106 | } 107 | 108 | f, err := os.Create(path) 109 | if err != nil { 110 | return err 111 | } 112 | defer f.Close() 113 | err = json.NewEncoder(f).Encode(exts) 114 | if err != nil { 115 | return err 116 | } 117 | return nil 118 | } 119 | -------------------------------------------------------------------------------- /subcmd/download_files.go: -------------------------------------------------------------------------------- 1 | /* 2 | リファクタリング中 3 | 処理をslacklog packageに移動していく。 4 | 一旦、必要な処理はすべてslacklog packageから一時的にエクスポートするか、このファ 5 | イル内で定義している。 6 | */ 7 | 8 | package subcmd 9 | 10 | import ( 11 | "fmt" 12 | "os" 13 | "path/filepath" 14 | 15 | "github.com/slack-go/slack" 16 | cli "github.com/urfave/cli/v2" 17 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 18 | ) 19 | 20 | // DownloadFilesCommand provides "downloads-files" sub-command, it downloads 21 | // and saves files which attached to message. 22 | var DownloadFilesCommand = &cli.Command{ 23 | Name: "download-files", 24 | Usage: "download files from slack.com", 25 | Action: downloadFiles, 26 | Flags: []cli.Flag{ 27 | &cli.StringFlag{ 28 | Name: "indir", 29 | Usage: "slacklog_data dir", 30 | Value: filepath.Join("_logdata", "slacklog_data"), 31 | }, 32 | &cli.StringFlag{ 33 | Name: "outdir", 34 | Usage: "files download target dir", 35 | Value: filepath.Join("_logdata", "files"), 36 | }, 37 | }, 38 | } 39 | 40 | // downloadFiles downloads and saves files which attached to message. 41 | func downloadFiles(c *cli.Context) error { 42 | slackToken := os.Getenv("SLACK_TOKEN") 43 | if slackToken == "" { 44 | return fmt.Errorf("$SLACK_TOKEN required") 45 | } 46 | 47 | logDir := filepath.Clean(c.String("indir")) 48 | filesDir := filepath.Clean(c.String("outdir")) 49 | 50 | s, err := slacklog.NewLogStore(logDir, &slacklog.Config{Channels: []string{"*"}}) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | d := slacklog.NewDownloader(slackToken) 56 | 57 | go generateMessageFileTargets(d, s, filesDir) 58 | 59 | err = d.Wait() 60 | if err != nil { 61 | return err 62 | } 63 | return nil 64 | } 65 | 66 | func urlAndSuffixes(f slack.File) map[string]string { 67 | return map[string]string{ 68 | f.URLPrivate: "", 69 | f.Thumb64: "_64", 70 | f.Thumb80: "_80", 71 | f.Thumb160: "_160", 72 | f.Thumb360: "_360", 73 | f.Thumb480: "_480", 74 | f.Thumb720: "_720", 75 | f.Thumb800: "_800", 76 | f.Thumb960: "_960", 77 | f.Thumb1024: "_1024", 78 | f.Thumb360Gif: "_360_gif", 79 | f.Thumb480Gif: "_480_gif", 80 | f.DeanimateGif: "_deanimate_gif", 81 | f.ThumbVideo: "_video", 82 | } 83 | } 84 | 85 | func generateMessageFileTargets(d *slacklog.Downloader, s *slacklog.LogStore, outputDir string) { 86 | defer d.CloseQueue() 87 | channels := s.GetChannels() 88 | for _, channel := range channels { 89 | msgs, err := s.GetAllMessages(channel.ID) 90 | if err != nil { 91 | fmt.Fprintf(os.Stderr, "failed to get messages on %s channel: %s", channel.Name, err) 92 | return 93 | } 94 | 95 | for _, msg := range msgs { 96 | for _, f := range msg.Files { 97 | if !slacklog.HostBySlack(f) { 98 | continue 99 | } 100 | // avoid github's file size limit 101 | if f.Size >= 104857600 { 102 | continue 103 | } 104 | 105 | targetDir := filepath.Join(outputDir, f.ID) 106 | err := os.MkdirAll(targetDir, 0777) 107 | if err != nil { 108 | fmt.Fprintf(os.Stderr, "failed to create %s directory: %s", targetDir, err) 109 | return 110 | } 111 | 112 | for url, suffix := range urlAndSuffixes(f) { 113 | if url == "" { 114 | continue 115 | } 116 | d.QueueDownloadRequest( 117 | url, 118 | filepath.Join(targetDir, slacklog.LocalName(f, url, suffix)), 119 | true, 120 | ) 121 | } 122 | } 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /subcmd/fetchchannels/fetchchannels.go: -------------------------------------------------------------------------------- 1 | package fetchchannels 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | 7 | cli "github.com/urfave/cli/v2" 8 | "github.com/vim-jp/slacklog-generator/internal/jsonwriter" 9 | "github.com/vim-jp/slacklog-generator/internal/slackadapter" 10 | ) 11 | 12 | func run(token, datadir string, excludeArchived, verbose bool) error { 13 | outfile := filepath.Join(datadir, "channels.json") 14 | fw, err := jsonwriter.CreateFile(outfile, true) 15 | if err != nil { 16 | return err 17 | } 18 | err = slackadapter.IterateCursor(context.Background(), 19 | slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) { 20 | r, err := slackadapter.Conversations(ctx, token, slackadapter.ConversationsParams{ 21 | Cursor: c, 22 | Limit: 100, 23 | ExcludeArchived: excludeArchived, 24 | Types: []string{"public_channel"}, 25 | }) 26 | if err != nil { 27 | return "", err 28 | } 29 | for _, c := range r.Channels { 30 | err := fw.Write(c) 31 | if err != nil { 32 | return "", err 33 | } 34 | } 35 | if m := r.ResponseMetadata; m != nil { 36 | return m.NextCursor, nil 37 | } 38 | return "", nil 39 | })) 40 | if err != nil { 41 | // ロールバック相当が好ましいが今はまだその時期ではない 42 | fw.Close() 43 | return err 44 | } 45 | if err := fw.Close(); err != nil { 46 | return err 47 | } 48 | 49 | return nil 50 | } 51 | 52 | // NewCLICommand creates a cli.Command, which provides "fetch-channels" 53 | // sub-command. 54 | func NewCLICommand() *cli.Command { 55 | var ( 56 | token string 57 | datadir string 58 | excludeArchived bool 59 | verbose bool 60 | ) 61 | return &cli.Command{ 62 | Name: "fetch-channels", 63 | Usage: "fetch channels in the workspace", 64 | Action: func(c *cli.Context) error { 65 | return run(token, datadir, excludeArchived, verbose) 66 | }, 67 | Flags: []cli.Flag{ 68 | &cli.StringFlag{ 69 | Name: "token", 70 | Usage: "slack token", 71 | EnvVars: []string{"SLACK_TOKEN"}, 72 | Destination: &token, 73 | }, 74 | &cli.StringFlag{ 75 | Name: "datadir", 76 | Usage: "directory to load/save data", 77 | Value: "_logdata", 78 | Destination: &datadir, 79 | }, 80 | &cli.BoolFlag{ 81 | Name: "exclude-archived", 82 | Usage: "exclude archived channesls", 83 | Destination: &excludeArchived, 84 | }, 85 | &cli.BoolFlag{ 86 | Name: "verbose", 87 | Usage: "verbose log", 88 | Destination: &verbose, 89 | }, 90 | }, 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /subcmd/fetchmessages/fetchmessages.go: -------------------------------------------------------------------------------- 1 | package fetchmessages 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | 12 | "github.com/slack-go/slack" 13 | cli "github.com/urfave/cli/v2" 14 | "github.com/vim-jp/slacklog-generator/internal/jsonwriter" 15 | "github.com/vim-jp/slacklog-generator/internal/slackadapter" 16 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 17 | ) 18 | 19 | const dateFormat = "2006-01-02" 20 | 21 | func toDateString(ti time.Time) string { 22 | return ti.Format(dateFormat) 23 | } 24 | 25 | func parseDateString(s string) (time.Time, error) { 26 | l, err := time.LoadLocation("Asia/Tokyo") 27 | if err != nil { 28 | return time.Time{}, err 29 | } 30 | ti, err := time.ParseInLocation(dateFormat, s, l) 31 | if err != nil { 32 | return time.Time{}, err 33 | } 34 | return ti, nil 35 | } 36 | 37 | // Run runs "fetch-messages" sub-command. It fetch messages of a channel by a 38 | // day. 39 | func Run(args []string) error { 40 | var ( 41 | token string 42 | datadir string 43 | date string 44 | verbose bool 45 | ) 46 | fs := flag.NewFlagSet("fetch-messages", flag.ExitOnError) 47 | fs.StringVar(&token, "token", os.Getenv("SLACK_TOKEN"), `slack token. can be set by SLACK_TOKEN env var`) 48 | fs.StringVar(&datadir, "datadir", "_logdata", `directory to load/save data`) 49 | fs.StringVar(&date, "date", toDateString(time.Now()), `target date to get`) 50 | fs.BoolVar(&verbose, "verbose", false, "verbose log") 51 | err := fs.Parse(args) 52 | if err != nil { 53 | return err 54 | } 55 | if token == "" { 56 | return errors.New("SLACK_TOKEN environment variable requied") 57 | } 58 | return run(token, datadir, date, verbose) 59 | } 60 | 61 | func run(token, datadir, date string, verbose bool) error { 62 | oldest, err := parseDateString(date) 63 | if err != nil { 64 | return err 65 | } 66 | latest := oldest.AddDate(0, 0, 1) 67 | 68 | ct, err := slacklog.NewChannelTable(filepath.Join(datadir, "channels.json"), []string{"*"}) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | for _, sch := range ct.Channels { 74 | outdir := filepath.Join(datadir, sch.ID) 75 | if err := os.MkdirAll(outdir, 0755); err != nil { 76 | return fmt.Errorf("making outdir: %w", err) 77 | } 78 | outfile := filepath.Join(outdir, toDateString(oldest)+".json") 79 | fw, err := jsonwriter.CreateFile(outfile, true) 80 | if err != nil { 81 | return err 82 | } 83 | err = slackadapter.IterateCursor(context.Background(), 84 | slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) { 85 | r, err := slackadapter.ConversationsHistory(ctx, token, sch.ID, slackadapter.ConversationsHistoryParams{ 86 | Cursor: c, 87 | Limit: 100, 88 | Oldest: &oldest, 89 | Latest: &latest, 90 | }) 91 | if err != nil { 92 | return "", err 93 | } 94 | for _, message := range r.Messages { 95 | if message.IsRootOfThread() { 96 | client := slack.New(token) 97 | err = slackadapter.IterateCursor(ctx, slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) { 98 | msgs, hasMore, nextCursor, err := client.GetConversationRepliesContext(ctx, &slack.GetConversationRepliesParameters{ 99 | ChannelID: sch.ID, 100 | Cursor: string(c), 101 | Timestamp: message.Timestamp, 102 | }) 103 | if err != nil { 104 | return "", err 105 | } 106 | for _, m := range msgs { 107 | sMes := slacklog.Message{ 108 | Message: m, 109 | } 110 | // スレッドのルートとブロードキャストメッセージは通常のログに含まれるのでここでは弾く 111 | if !sMes.IsRootOfThread() && sMes.SubType != "thread_broadcast" { 112 | r.Messages = append(r.Messages, &sMes) 113 | } 114 | } 115 | if hasMore { 116 | return slackadapter.Cursor(nextCursor), nil 117 | } 118 | return "", nil 119 | })) 120 | if err != nil { 121 | return "", err 122 | } 123 | } 124 | } 125 | for _, m := range r.Messages { 126 | err := fw.Write(m) 127 | if err != nil { 128 | return "", err 129 | } 130 | } 131 | if m := r.ResponseMetadata; r.HasMore && m != nil { 132 | return m.NextCursor, nil 133 | } 134 | // HasMore && ResponseMetadata == nil は明らかにエラーだがいま 135 | // は握りつぶしてる 136 | return "", nil 137 | })) 138 | if err != nil { 139 | // ロールバック相当が好ましいが今はまだその時期ではない 140 | fw.Close() 141 | return err 142 | } 143 | err = fw.Close() 144 | if err != nil { 145 | return err 146 | } 147 | } 148 | 149 | return nil 150 | } 151 | 152 | // NewCLICommand creates a cli.Command, which provides "fetch-messages" 153 | // sub-command. 154 | func NewCLICommand() *cli.Command { 155 | var ( 156 | token string 157 | datadir string 158 | date string 159 | verbose bool 160 | ) 161 | return &cli.Command{ 162 | Name: "fetch-messages", 163 | Usage: "fetch messages of channel by day", 164 | Action: func(c *cli.Context) error { 165 | return run(token, datadir, date, verbose) 166 | }, 167 | Flags: []cli.Flag{ 168 | &cli.StringFlag{ 169 | Name: "token", 170 | Usage: "slack token", 171 | EnvVars: []string{"SLACK_TOKEN"}, 172 | Destination: &token, 173 | }, 174 | &cli.StringFlag{ 175 | Name: "datadir", 176 | Usage: "directory to load/save data", 177 | Value: "_logdata", 178 | Destination: &datadir, 179 | }, 180 | &cli.StringFlag{ 181 | Name: "date", 182 | Usage: "target date to get", 183 | Value: toDateString(time.Now()), 184 | Destination: &date, 185 | }, 186 | &cli.BoolFlag{ 187 | Name: "verbose", 188 | Usage: "verbose log", 189 | Destination: &verbose, 190 | }, 191 | }, 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /subcmd/fetchusers/fetchusers.go: -------------------------------------------------------------------------------- 1 | package fetchusers 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | 7 | cli "github.com/urfave/cli/v2" 8 | "github.com/vim-jp/slacklog-generator/internal/jsonwriter" 9 | "github.com/vim-jp/slacklog-generator/internal/slackadapter" 10 | ) 11 | 12 | func run(token, datadir string, excludeArchived, verbose bool) error { 13 | outfile := filepath.Join(datadir, "users.json") 14 | fw, err := jsonwriter.CreateFile(outfile, true) 15 | if err != nil { 16 | return err 17 | } 18 | err = slackadapter.IterateCursor(context.Background(), 19 | slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) { 20 | users, err := slackadapter.Users(ctx, token) 21 | if err != nil { 22 | return "", err 23 | } 24 | for _, u := range users { 25 | err := fw.Write(u) 26 | if err != nil { 27 | return "", err 28 | } 29 | } 30 | return "", nil 31 | })) 32 | if err != nil { 33 | // ロールバック相当が好ましいが今はまだその時期ではない 34 | fw.Close() 35 | return err 36 | } 37 | if err := fw.Close(); err != nil { 38 | return err 39 | } 40 | 41 | return nil 42 | } 43 | 44 | // NewCLICommand creates a cli.Command, which provides "fetch-users" 45 | // sub-command. 46 | func NewCLICommand() *cli.Command { 47 | var ( 48 | token string 49 | datadir string 50 | excludeArchived bool 51 | verbose bool 52 | ) 53 | return &cli.Command{ 54 | Name: "fetch-users", 55 | Usage: "fetch users in the workspace", 56 | Action: func(c *cli.Context) error { 57 | return run(token, datadir, excludeArchived, verbose) 58 | }, 59 | Flags: []cli.Flag{ 60 | &cli.StringFlag{ 61 | Name: "token", 62 | Usage: "slack token", 63 | EnvVars: []string{"SLACK_TOKEN"}, 64 | Destination: &token, 65 | }, 66 | &cli.StringFlag{ 67 | Name: "datadir", 68 | Usage: "directory to load/save data", 69 | Value: "_logdata", 70 | Destination: &datadir, 71 | }, 72 | &cli.BoolFlag{ 73 | Name: "exclude-archived", 74 | Usage: "exclude archived channesls", 75 | Destination: &excludeArchived, 76 | }, 77 | &cli.BoolFlag{ 78 | Name: "verbose", 79 | Usage: "verbose log", 80 | Destination: &verbose, 81 | }, 82 | }, 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /subcmd/generate_html.go: -------------------------------------------------------------------------------- 1 | package subcmd 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | 7 | cli "github.com/urfave/cli/v2" 8 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 9 | ) 10 | 11 | // GenerateHTMLCommand provoides "generate-html" command. 12 | // It... SlackからエクスポートしたデータをHTMLに変換して出力する。 13 | var GenerateHTMLCommand = &cli.Command{ 14 | Name: "generate-html", 15 | Usage: "generate html from slacklog_data", 16 | Action: generateHTML, 17 | Flags: []cli.Flag{ 18 | &cli.StringFlag{ 19 | Name: "config", 20 | Usage: "config.json path", 21 | Value: filepath.Join("scripts", "config.json"), 22 | }, 23 | &cli.StringFlag{ 24 | Name: "templatedir", 25 | Usage: "templates dir", 26 | Value: "templates", 27 | }, 28 | &cli.StringFlag{ 29 | Name: "filesdir", 30 | Usage: "files downloaded dir", 31 | Value: filepath.Join("_logdata", "files"), 32 | }, 33 | &cli.StringFlag{ 34 | Name: "indir", 35 | Usage: "slacklog_data dir", 36 | Value: filepath.Join("_logdata", "slacklog_data"), 37 | }, 38 | &cli.StringFlag{ 39 | Name: "outdir", 40 | Usage: "generated html target dir", 41 | Value: "_site", 42 | }, 43 | }, 44 | } 45 | 46 | // generateHTML : SlackからエクスポートしたデータをHTMLに変換して出力する。 47 | func generateHTML(c *cli.Context) error { 48 | configJSONPath := filepath.Clean(c.String("config")) 49 | templateDir := filepath.Clean(c.String("templatedir")) 50 | filesDir := filepath.Clean(c.String("filesdir")) 51 | inDir := filepath.Clean(c.String("indir")) 52 | outDir := filepath.Clean(c.String("outdir")) 53 | 54 | cfg, err := slacklog.ReadConfig(configJSONPath) 55 | if err != nil { 56 | return fmt.Errorf("could not read config: %w", err) 57 | } 58 | 59 | s, err := slacklog.NewLogStore(inDir, cfg) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | g := slacklog.NewHTMLGenerator(templateDir, filesDir, s) 65 | return g.Generate(outDir) 66 | } 67 | -------------------------------------------------------------------------------- /subcmd/serve/serve.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httputil" 6 | "net/url" 7 | 8 | cli "github.com/urfave/cli/v2" 9 | ) 10 | 11 | // Command provides "serve" sub-command. It starts HTML server for slacklog for 12 | // development. 13 | var Command = &cli.Command{ 14 | Name: "serve", 15 | Usage: "serve a generated HTML with files proxy", 16 | Action: run, 17 | Flags: []cli.Flag{ 18 | &cli.StringFlag{ 19 | Name: "addr", 20 | Usage: "address for serve", 21 | Value: ":8081", 22 | }, 23 | &cli.StringFlag{ 24 | Name: "htdocs", 25 | Usage: "root of files", 26 | Value: "_site", 27 | }, 28 | &cli.StringFlag{ 29 | Name: "target", 30 | Usage: "proxy target endpoint", 31 | Value: "https://vim-jp.org/slacklog/", 32 | }, 33 | }, 34 | } 35 | 36 | func newReverseProxy(targetURL string) (*httputil.ReverseProxy, error) { 37 | u, err := url.Parse(targetURL) 38 | if err != nil { 39 | return nil, err 40 | } 41 | rp := httputil.NewSingleHostReverseProxy(u) 42 | orig := rp.Director 43 | rp.Director = func(r *http.Request) { 44 | orig(r) 45 | r.Host = "" //u.Host 46 | } 47 | return rp, nil 48 | } 49 | 50 | // run starts combined HTTP server (file + reverse proxy) to serve slacklog for 51 | // development. 52 | func run(c *cli.Context) error { 53 | addr := c.String("addr") 54 | htdocs := c.String("htdocs") 55 | target := c.String("target") 56 | 57 | proxy, err := newReverseProxy(target) 58 | if err != nil { 59 | return err 60 | } 61 | http.Handle("/files/", proxy) 62 | http.Handle("/emojis/", proxy) 63 | http.Handle("/", http.FileServer(http.Dir(htdocs))) 64 | 65 | return http.ListenAndServe(addr, nil) 66 | } 67 | -------------------------------------------------------------------------------- /templates/channel_index.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | vim-jp » vim-jp.slack.com log - #{{ .channel.Name }} 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 | 20 |
21 |

22 | 23 | vim-jp 24 | 25 | / 26 | slacklog 27 |

28 |
29 | 30 |
31 |
32 |

#{{ $.channel.Name }}

33 | 38 |
39 |
40 |
41 | 42 |
43 | 48 | 49 |
50 | 51 | -------------------------------------------------------------------------------- /templates/channel_per_month/attachment.tmpl: -------------------------------------------------------------------------------- 1 | {{- range .Attachments }} 2 | {{- if eq .ServiceName "GitHub" }} 3 |
4 |
5 | {{ html .ServiceName }} 6 |
7 |
8 | 9 | {{ html .Title }} 10 | 11 |
12 |
13 |
14 |
{{ attachmentText . }}
15 |
16 | {{- else if eq .ServiceName "twitter" }} 17 |
18 |
19 | {{ html .ServiceName }} 20 |
21 |
22 | {{ .AuthorName }} {{ .AuthorSubname }} 23 |
24 |
25 |
26 |
{{ attachmentText . }}
27 |
28 |
29 |
30 | 31 |
32 |
33 | {{ html .Footer }} 34 |
35 |
36 | {{- if .VideoHTML }} 37 |
38 | {{ .VideoHTML }} 39 |
40 | {{- end }} 41 | {{- else if eq .ServiceName "Gyazo" }} 42 |
43 |
44 | 45 | 46 | {{ .Fallback }} 47 | 48 |
49 |
50 |
51 |
52 | {{ .Fallback }} 53 |
54 |
55 | {{- else if or .Title .Text }} 56 |
57 | {{- if and .Title .TitleLink }} 58 | {{- if and .ServiceIcon .ServiceName }} 59 |
60 | {{ html .ServiceName }} 61 |
62 | {{- end }} 63 |
64 | 65 | {{ html .Title }} 66 | 67 |
68 | {{- else if .Title }} 69 |
{{ html .Title }}
70 | {{- end }} 71 |
72 | {{- if .Text }} 73 |
74 |
{{ attachmentText . }}
75 |
76 | {{- if isSlackMessage .FromURL}} 77 | 78 | slacklog 79 | 80 | {{- end}} 81 | {{- end }} 82 | 83 | {{- if .ThumbURL }} 84 |
85 |
86 | {{ html .Title }} 87 |
88 |
89 | {{- end }} 90 | {{- end }} 91 | {{- end }} 92 | -------------------------------------------------------------------------------- /templates/channel_per_month/index.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | vim-jp » vim-jp.slack.com log - #{{ .channel.Name }} - {{ .monthKey.Year }}年{{ .monthKey.Month }}月 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 |
22 | 23 |
24 |

25 | 26 | vim-jp 27 | 28 | / 29 | slacklog 30 |

31 |
32 | 33 |
34 |
35 | 41 |

42 |
43 | {{- range .msgs }} 44 | {{- if visible . }} 45 |
46 |
47 |
48 | 49 |
50 |
51 | {{ username . }} 52 | {{ datetime .Timestamp }} 53 | 54 | Slack 55 | 56 |
57 |
58 |
59 | {{ text . }} 60 |
61 | 62 |
63 | {{- $rs := reactions .}} 64 | {{- if $rs }} 65 |
66 |
    67 | {{- range $rs }} 68 |
  • 69 | 70 | {{- if not .Default }} 71 | {{ .Name }}{{ .Count }} 72 | {{- else }} 73 | {{ .Name }}{{ .Count }} 74 | {{- end }} 75 | 76 |
    • 77 | {{- end }} 78 |
79 |
80 | {{- end }} 81 |
82 | 83 | 84 | {{- if and (ne .ThreadTimestamp "") (ne .ThreadTimestamp .Timestamp) }} 85 | 86 | このスレッドに返信しました : {{- threadRootText .ThreadTimestamp }} 87 | 88 | {{- end }} 89 | 90 | {{- if .Attachments }} 91 |
92 | {{ template "attachment.tmpl" . }} 93 |
94 | {{- end }} 95 | 96 | {{- if .Files }} 97 |
98 | {{- range .Files }} 99 |
100 | {{- if hostBySlack . }} 101 | 102 | {{- if eq (topLevelMimetype .) "image" }} 103 | {{ .Title }} 104 | {{- else if eq (topLevelMimetype .) "video" }} 105 | 107 | {{- else }} 108 | [[ダウンロード: {{ .Title }}({{ .PrettyType }})]] 109 | {{- end }} 110 | 111 | {{- if eq .Mimetype "text/plain" }} 112 | 
{{ fileHTML . }}
113 | {{- end }} 114 | {{- else }} 115 | {{- /* TODO: 外部ページへのリンクだということがわかりやすい表示にする */ -}} 116 | {{ .Title }} 117 | {{- end }} 118 |
119 | {{- end }} 120 |
121 | {{- end }} 122 | 123 | {{- if threads .Timestamp }} 124 |
125 | 126 | 127 | {{- threadNum .ThreadTimestamp }} 件の返信 最終返信:{{- threadMtime .ThreadTimestamp }} 128 | 129 | 130 |
131 | {{- range threads .Timestamp }} 132 |
133 | {{- if eq .SubType "thread_broadcast" }} 134 |
135 |
#←
136 |
チャンネルにも投稿済
137 |
138 | {{- end }} 139 |
140 | 141 |
142 |
143 | {{ username . }} 144 | {{ datetime .Timestamp }} 145 |
146 |
147 | {{ text . }} 148 |
149 | 150 | {{- if .Attachments }} 151 | {{ template "attachment.tmpl" . }} 152 | {{- end }} 153 |
154 | {{- end }} 155 |
156 |
157 | {{- end }} 158 |
159 |
160 |
161 | {{- end }} 162 | {{- end }} 163 |
164 |
165 | 166 |
167 | 172 | 173 |
174 | 175 | -------------------------------------------------------------------------------- /templates/index.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | vim-jp » vim-jp.slack.com log 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 | 20 |
21 |

22 | 23 | vim-jp 24 | 25 | / 26 | slacklog 27 |

28 |
29 | 30 |
31 |
32 |
33 | 34 | 40 | 41 | 42 | vim-jp/slacklog について 43 | 44 |
45 |
46 |
47 |

48 | Search 49 |

50 |

Channels

51 | 56 |
57 |
58 |
59 | 60 |
61 | 66 | 67 |
68 | 69 | --------------------------------------------------------------------------------