├── .github └── workflows │ ├── release.yml │ └── scrape.yml ├── .gitignore ├── .goreleaser.yaml ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── twiyou │ ├── dev.go.example │ └── main.go ├── go.mod ├── go.sum ├── grafana └── twitter-statistics.json ├── scraper └── scraper.go ├── store ├── batch.go ├── create_tables.sql ├── db.go ├── iteration.go └── user.go └── twitter └── twitter.go /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | permissions: 9 | contents: write 10 | packages: write 11 | 12 | jobs: 13 | goreleaser: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | with: 18 | fetch-depth: 0 19 | - run: git fetch --force --tags 20 | - uses: actions/setup-go@v3 21 | with: 22 | go-version: '>=1.19.2' 23 | cache: true 24 | - name: Docker Login 25 | uses: docker/login-action@v2 26 | with: 27 | registry: ghcr.io 28 | username: ${{ github.repository_owner }} 29 | password: ${{ secrets.GITHUB_TOKEN }} 30 | - uses: goreleaser/goreleaser-action@v3 31 | with: 32 | distribution: goreleaser 33 | version: latest 34 | args: release --rm-dist 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | -------------------------------------------------------------------------------- /.github/workflows/scrape.yml: -------------------------------------------------------------------------------- 1 | name: Run scraper 2 | 3 | # This workflow can be triggered manually or by an API call. 4 | # https://docs.github.com/en/rest/actions/workflows#create-a-workflow-dispatch-event 5 | # Example: 6 | # curl \ 7 | # -X POST \ 8 | # -H "Accept: application/vnd.github+json" \ 9 | # -H "Authorization: Bearer " \ 10 | # https://api.github.com/repos//twiyou/actions/workflows/scrape.yml/dispatches \ 11 | # -d '{"ref": "master"}' 12 | 13 | on: 14 | workflow_dispatch: 15 | 16 | permissions: {} 17 | 18 | jobs: 19 | scrape: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Download latest release 23 | uses: robinraju/release-downloader@v1.6 24 | with: 25 | repository: disksing/twiyou 26 | latest: true 27 | fileName: "twiyou_*_Linux_x86_64.tar.gz" 28 | - name: Extract tarball 29 | run: tar -xvf twiyou_*_Linux_x86_64.tar.gz 30 | - name: Run scraper 31 | run: ./twiyou 32 | env: 33 | TWITTER_USER_NAME: ${{ secrets.TWITTER_USER_NAME }} 34 | TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }} 35 | DB_HOST: ${{ secrets.DB_HOST }} 36 | DB_PORT: ${{ secrets.DB_PORT }} 37 | DB_USER: ${{ secrets.DB_USER }} 38 | DB_NAME: ${{ secrets.DB_NAME }} 39 | DB_PASSWD: ${{ secrets.DB_PASSWD }} 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /cmd/twiyou/dev.go 2 | .DS_Store 3 | twiyou 4 | 5 | dist/ 6 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # This is an example .goreleaser.yml file with some sensible defaults. 2 | # Make sure to check the documentation at https://goreleaser.com 3 | before: 4 | hooks: 5 | - go mod tidy 6 | builds: 7 | - main: ./cmd/twiyou 8 | env: 9 | - CGO_ENABLED=0 10 | goos: 11 | - linux 12 | - windows 13 | - darwin 14 | goarch: 15 | - amd64 16 | - arm64 17 | flags: 18 | - -v 19 | - -trimpath 20 | archives: 21 | - replacements: 22 | darwin: Darwin 23 | linux: Linux 24 | windows: Windows 25 | 386: i386 26 | amd64: x86_64 27 | checksum: 28 | name_template: 'checksums.txt' 29 | dockers: 30 | - image_templates: 31 | - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .Tag }}" 32 | - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:latest" 33 | goos: linux 34 | goarch: amd64 35 | use: buildx 36 | build_flag_templates: 37 | - "--label=org.opencontainers.image.created={{ .Date }}" 38 | - "--label=org.opencontainers.image.revision={{ .ShortCommit }}" 39 | - "--label=org.opencontainers.image.version={{ .Version }}" 40 | - "--label=org.opencontainers.image.source={{ .GitURL }}" 41 | - "--label=org.opencontainers.image.title={{ .ProjectName }}" 42 | - "--label=org.opencontainers.image.description=Twitter friend monitoring tool" 43 | - "--label=org.opencontainers.image.licenses=BSD 3-Clause" 44 | - "--platform=linux/amd64" 45 | snapshot: 46 | name_template: "{{ incpatch .Version }}-next" 47 | changelog: 48 | sort: asc 49 | filters: 50 | exclude: 51 | - '^docs:' 52 | - '^test:' 53 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.16 2 | 3 | RUN apk --no-cache upgrade && apk --no-cache add ca-certificates 4 | 5 | COPY ./twiyou /app/ 6 | 7 | CMD ["bin/sh", "-c", "echo Running against DB ${DB_NAME} && /app/twiyou"] 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2022, disksing 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # twiyou 2 | 3 | Twitter friend monitoring tool 4 | 5 | ---- 6 | 7 | twiyou（推油）是一款推特好友/推特数据监测工具。 8 | 9 | ![Screenshot 2022-10-07 145315](https://user-images.githubusercontent.com/12077877/194486031-dddb414b-905c-4422-9f47-ada4d2a39545.png) 10 | ![Screenshot 2022-10-07 145528](https://user-images.githubusercontent.com/12077877/194486415-3f53bc70-b82f-42ad-b7c8-f0051a5e7443.png) 11 | 12 | ## 搭建自己的部署 13 | 14 | 运行起来需要部署3个组件：数据库，数据抓取工具，监控面板，目前这3个组件都有免费的白嫖资源，此外需要有推特开发者账号。 15 | 16 | - 数据库我用的是TiDB Cloud DevTier (https://tidbcloud.com/ )的免费资源，理论上兼容MySQL协议的应该都行，不过我没试。 17 | - 数据抓取工具可以从 [Release](https://github.com/disksing/twiyou/releases) 下载，找台机器配置好环境变量定时运行程序就行（推荐5分钟运行一次）。也可以白嫖 GitHub Actions。 18 | - 监控面板用的是Grafana，Grafana Cloud (https://grafana.com/ )也是可以创建免费的账号，添加数据库数据源之后，把监控模板导入就行。 19 | 20 | ## 搭建自己的部署（详细攻略） 21 | 22 | ### 1. 注册 twitter 开发者账号 23 | 24 | 注册地址：https://developer.twitter.com/en 25 | 26 | 申请成功后创建一个项目，然后把如图所示的Bearer Token记录下来备用。 27 | 28 | ![Screenshot 2022-10-07 152412](https://user-images.githubusercontent.com/12077877/194496392-21d8939d-0044-4070-b7a4-272963f5868d.png) 29 | 30 | ### 2. 注册 TiDB Cloud DevTier 31 | 32 | 注册地址：https://tidbcloud.com/ 33 | 34 | 完成后创建一个免费的DevTier集群，然后在集群页面可以看到连接参数。 35 | 36 | ![Screenshot 2022-10-07 152706](https://user-images.githubusercontent.com/12077877/194497136-7d33c809-327d-4d63-9a01-ffc39f1e73f3.png) 37 | 38 | ### 3. 运行数据抓取工具 39 | 40 | #### 方法1：使用 binary 部署 41 | 42 | 下载地址：https://github.com/disksing/twiyou/releases 43 | 44 | 然后配置好crontab和环境变量每5分钟运行一次就行，抓下来的数据都会写到数据库里面，本地不存数据。 45 | 46 | 环境变量设置如下图。 47 | 48 | ![Screenshot 2022-10-07 153706](https://user-images.githubusercontent.com/12077877/194498629-d8a8972f-2545-4469-b26e-c563b242f8b2.png) 49 | 50 | 注意DB_NAME需要提前自己连接数据库后创建，懒得手动创建的话直接填test数据库也行。 51 | 52 | `TWITTER_USER_NAME`填自己的twitter id，当然你如果想监控别人的好友，填别人的id也可以。 53 | 54 | #### 方法2：使用 Github Actions 55 | 56 | 如果自己没有服务器的话，可以使用免费的 GitHub Action 来跑： 57 | 58 | 1. fork 这个项目 59 | 2. 在 Repo Settings - Secrets 里配置好环境变量 60 | 3. 在 https://github.com/settings/tokens 里生成一个新的 Personal access token 61 | 4. 配置一个触发器定时发送类似下面的 HTTP 请求来触发 Github Action： 62 | 63 | ```shell 64 | curl \ 65 | -X POST \ 66 | -H "Accept: application/vnd.github+json" \ 67 | -H "Authorization: Bearer " \ 68 | https://api.github.com/repos//twiyou/actions/workflows/scrape.yml/dispatches \ 69 | -d '{"ref": "master"}' 70 | ``` 71 | 72 | 这类第三方服务比较多，比如 Cloudflare worker，https://cron-job.org 都行。以 cron-job.org 为例： 73 |

74 | image 75 | image 76 |

77 | 78 | #### 方法3：使用 Docker / Kubernetes 部署 79 | 80 | 详细说明参考 [xiaowenz/twiyou](https://hub.docker.com/r/xiaowenz/twiyou) 81 | 82 | ### 4. 配置 Grafana 83 | 84 | 注册地址：https://grafana.com 85 | 86 | 完成后在自己的Grafana实例添加MySQL数据源，参考截图： 87 | 88 | ![image](https://user-images.githubusercontent.com/10510431/209826828-7c14b4bb-648b-4011-aa5f-393f04aafbde.png) 89 | 90 | 这里Timezone不要填，开启 `With CA Cert`，然后将这个根证书 [pem](https://letsencrypt.org/certs/isrgrootx1.pem.txt) 粘贴到 `TLS/SSL Root Certificate` 中。 91 | 92 | 然后导入[预定义模板](https://raw.githubusercontent.com/disksing/twiyou/master/grafana/twitter-statistics.json)，选择之前创建的数据源。在第一轮数据抓完之前，可能不会立即看到数据。 93 | 94 | ## 一些问题和后续计划（请求出谋划策） 95 | 96 | 1. twitter开发者账号申请比较麻烦，而且API的rate limit等限制颇多，可以考虑换成绕过开发者API的方案。例如[twint](https://github.com/twintproject/twint)，不过我还没仔细研究。不好搞的话，或许可以考虑配置多个API key，轮换着用。 97 | 98 | 2. 想把自己的历史推文抓下来分析下，比如统计历史推文的传播表现，或者把发推历史跟follower数量变化做点关联分析之类，但是目前twitter API限制只能抓近一周的历史推文，所以这个功能可能依赖于上一个问题的解决。 99 | 100 | ## 排障 FAQ 101 | 102 | 1. `2022/10/13 07:37:39 Error 1130: Host '4.246.175.211' is blocked by traffic filter. See https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster#connect-via-standard-connection` 103 | 104 | 原因是 TiDB Cloud 中 Cluster 未配置允许外部 IP 访问集群，需要 Create traffic filter，在其中增加 twiyou 二进制访问的出口 IP，如果没有固定 IP 可以允许任意 IP：`0.0.0.0/0`，只需要在 TiCloud 网页 Console 的集群下 Edit 就好了。 105 | 106 | 2. `client has multi-statement capability disabled. Run SET GLOBAL tidb_multi_statement_mode='ON' after you understand the security risk` 107 | 108 | 这个报错是自解释的，不更改 twiyou/client 上的 multi-statement 模式的情况下，可以在 TiDB Cloud WebShell 里执行 `SET GLOBAL tidb_multi_statement_mode='ON'`，只需要在 TiCloud 网页 Console 的集群下点击 Connect 然后点击 WebShell，在 Shell 里输入密码登陆之后执行就好了。 109 | 110 | -------------------------------------------------------------------------------- /cmd/twiyou/dev.go.example: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // `cp dev.go.example dev.go` to make it work. 4 | 5 | import ( 6 | "github.com/disksing/twiyou/store" 7 | "github.com/disksing/twiyou/twitter" 8 | ) 9 | 10 | func init() { 11 | twitter.TWITTER_USER_NAME = "disksing" 12 | twitter.TWITTER_BEARER_TOKEN = "" 13 | store.DB_HOST = "127.0.0.1" 14 | store.DB_PORT = "4000" 15 | store.DB_USER = "root" 16 | store.DB_NAME = "test" 17 | store.DB_PASSWD = "" 18 | } 19 | -------------------------------------------------------------------------------- /cmd/twiyou/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/disksing/twiyou/scraper" 7 | ) 8 | 9 | func main() { 10 | scraper, err := scraper.NewScraper() 11 | if err != nil { 12 | log.Fatal(err) 13 | } 14 | defer scraper.Close() 15 | err = scraper.Run() 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/disksing/twiyou 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/go-sql-driver/mysql v1.6.0 7 | github.com/jmoiron/sqlx v1.3.5 8 | github.com/pkg/errors v0.9.1 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= 2 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 3 | github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= 4 | github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= 5 | github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= 6 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 7 | github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= 8 | github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 9 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 10 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 11 | -------------------------------------------------------------------------------- /grafana/twitter-statistics.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_TIDB-DEVTIER", 5 | "label": "tidb-devtier", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "mysql", 9 | "pluginName": "MySQL" 10 | } 11 | ], 12 | "__elements": {}, 13 | "__requires": [ 14 | { 15 | "type": "panel", 16 | "id": "barchart", 17 | "name": "Bar chart", 18 | "version": "" 19 | }, 20 | { 21 | "type": "grafana", 22 | "id": "grafana", 23 | "name": "Grafana", 24 | "version": "9.1.6-10b38e80" 25 | }, 26 | { 27 | "type": "datasource", 28 | "id": "mysql", 29 | "name": "MySQL", 30 | "version": "1.0.0" 31 | }, 32 | { 33 | "type": "panel", 34 | "id": "piechart", 35 | "name": "Pie chart", 36 | "version": "" 37 | }, 38 | { 39 | "type": "panel", 40 | "id": "stat", 41 | "name": "Stat", 42 | "version": "" 43 | }, 44 | { 45 | "type": "panel", 46 | "id": "table", 47 | "name": "Table", 48 | "version": "" 49 | }, 50 | { 51 | "type": "panel", 52 | "id": "timeseries", 53 | "name": "Time series", 54 | "version": "" 55 | } 56 | ], 57 | "annotations": { 58 | "list": [ 59 | { 60 | "builtIn": 1, 61 | "datasource": { 62 | "type": "grafana", 63 | "uid": "-- Grafana --" 64 | }, 65 | "enable": true, 66 | "hide": true, 67 | "iconColor": "rgba(0, 211, 255, 1)", 68 | "name": "Annotations & Alerts", 69 | "target": { 70 | "limit": 100, 71 | "matchAny": false, 72 | "tags": [], 73 | "type": "dashboard" 74 | }, 75 | "type": "dashboard" 76 | } 77 | ] 78 | }, 79 | "editable": true, 80 | "fiscalYearStartMonth": 0, 81 | "graphTooltip": 0, 82 | "id": null, 83 | "links": [], 84 | "liveNow": false, 85 | "panels": [ 86 | { 87 | "collapsed": false, 88 | "gridPos": { 89 | "h": 1, 90 | "w": 24, 91 | "x": 0, 92 | "y": 0 93 | }, 94 | "id": 7, 95 | "panels": [], 96 | "title": "Stat", 97 | "type": "row" 98 | }, 99 | { 100 | "datasource": { 101 | "type": "mysql", 102 | "uid": "${DS_TIDB-DEVTIER}" 103 | }, 104 | "fieldConfig": { 105 | "defaults": { 106 | "color": { 107 | "mode": "thresholds" 108 | }, 109 | "mappings": [], 110 | "thresholds": { 111 | "mode": "absolute", 112 | "steps": [ 113 | { 114 | "color": "green", 115 | "value": null 116 | } 117 | ] 118 | } 119 | }, 120 | "overrides": [] 121 | }, 122 | "gridPos": { 123 | "h": 4, 124 | "w": 3, 125 | "x": 0, 126 | "y": 1 127 | }, 128 | "id": 36, 129 | "options": { 130 | "colorMode": "background", 131 | "graphMode": "area", 132 | "justifyMode": "auto", 133 | "orientation": "auto", 134 | "reduceOptions": { 135 | "calcs": [ 136 | "lastNotNull" 137 | ], 138 | "fields": "", 139 | "values": false 140 | }, 141 | "textMode": "auto" 142 | }, 143 | "pluginVersion": "9.1.6-10b38e80", 144 | "targets": [ 145 | { 146 | "datasource": { 147 | "type": "mysql", 148 | "uid": "${DS_TIDB-DEVTIER}" 149 | }, 150 | "format": "time_series", 151 | "group": [], 152 | "metricColumn": "none", 153 | "rawQuery": true, 154 | "rawSql": "SELECT\n updated_at AS \"time\",\n follower_count\nFROM stats\nWHERE\n $__timeFilter(updated_at)\nORDER BY updated_at", 155 | "refId": "A", 156 | "select": [ 157 | [ 158 | { 159 | "params": [ 160 | "id" 161 | ], 162 | "type": "column" 163 | } 164 | ] 165 | ], 166 | "table": "iterations", 167 | "timeColumn": "started_at", 168 | "timeColumnType": "timestamp", 169 | "where": [ 170 | { 171 | "name": "$__timeFilter", 172 | "params": [], 173 | "type": "macro" 174 | } 175 | ] 176 | } 177 | ], 178 | "title": "Followers", 179 | "type": "stat" 180 | }, 181 | { 182 | "datasource": { 183 | "type": "mysql", 184 | "uid": "${DS_TIDB-DEVTIER}" 185 | }, 186 | "fieldConfig": { 187 | "defaults": { 188 | "color": { 189 | "mode": "thresholds" 190 | }, 191 | "mappings": [], 192 | "thresholds": { 193 | "mode": "absolute", 194 | "steps": [ 195 | { 196 | "color": "green", 197 | "value": null 198 | } 199 | ] 200 | } 201 | }, 202 | "overrides": [] 203 | }, 204 | "gridPos": { 205 | "h": 4, 206 | "w": 3, 207 | "x": 3, 208 | "y": 1 209 | }, 210 | "id": 37, 211 | "options": { 212 | "colorMode": "background", 213 | "graphMode": "area", 214 | "justifyMode": "auto", 215 | "orientation": "auto", 216 | "reduceOptions": { 217 | "calcs": [ 218 | "lastNotNull" 219 | ], 220 | "fields": "", 221 | "values": false 222 | }, 223 | "textMode": "auto" 224 | }, 225 | "pluginVersion": "9.1.6-10b38e80", 226 | "targets": [ 227 | { 228 | "datasource": { 229 | "type": "mysql", 230 | "uid": "${DS_TIDB-DEVTIER}" 231 | }, 232 | "format": "time_series", 233 | "group": [], 234 | "metricColumn": "none", 235 | "rawQuery": true, 236 | "rawSql": "SELECT\n updated_at AS \"time\",\n tweet_count\nFROM stats\nWHERE\n $__timeFilter(updated_at)\nORDER BY updated_at", 237 | "refId": "A", 238 | "select": [ 239 | [ 240 | { 241 | "params": [ 242 | "id" 243 | ], 244 | "type": "column" 245 | } 246 | ] 247 | ], 248 | "table": "iterations", 249 | "timeColumn": "started_at", 250 | "timeColumnType": "timestamp", 251 | "where": [ 252 | { 253 | "name": "$__timeFilter", 254 | "params": [], 255 | "type": "macro" 256 | } 257 | ] 258 | } 259 | ], 260 | "title": "Tweets", 261 | "type": "stat" 262 | }, 263 | { 264 | "datasource": { 265 | "type": "mysql", 266 | "uid": "${DS_TIDB-DEVTIER}" 267 | }, 268 | "fieldConfig": { 269 | "defaults": { 270 | "color": { 271 | "mode": "palette-classic" 272 | }, 273 | "custom": { 274 | "axisCenteredZero": false, 275 | "axisColorMode": "text", 276 | "axisLabel": "", 277 | "axisPlacement": "auto", 278 | "barAlignment": 0, 279 | "drawStyle": "line", 280 | "fillOpacity": 0, 281 | "gradientMode": "none", 282 | "hideFrom": { 283 | "legend": false, 284 | "tooltip": false, 285 | "viz": false 286 | }, 287 | "lineInterpolation": "linear", 288 | "lineWidth": 1, 289 | "pointSize": 5, 290 | "scaleDistribution": { 291 | "type": "linear" 292 | }, 293 | "showPoints": "auto", 294 | "spanNulls": false, 295 | "stacking": { 296 | "group": "A", 297 | "mode": "none" 298 | }, 299 | "thresholdsStyle": { 300 | "mode": "off" 301 | } 302 | }, 303 | "mappings": [], 304 | "thresholds": { 305 | "mode": "absolute", 306 | "steps": [ 307 | { 308 | "color": "green", 309 | "value": null 310 | } 311 | ] 312 | } 313 | }, 314 | "overrides": [] 315 | }, 316 | "gridPos": { 317 | "h": 8, 318 | "w": 6, 319 | "x": 6, 320 | "y": 1 321 | }, 322 | "id": 58, 323 | "options": { 324 | "legend": { 325 | "calcs": [], 326 | "displayMode": "list", 327 | "placement": "bottom", 328 | "showLegend": true 329 | }, 330 | "tooltip": { 331 | "mode": "single", 332 | "sort": "none" 333 | } 334 | }, 335 | "pluginVersion": "9.1.6-10b38e80", 336 | "targets": [ 337 | { 338 | "datasource": { 339 | "type": "mysql", 340 | "uid": "${DS_TIDB-DEVTIER}" 341 | }, 342 | "format": "time_series", 343 | "group": [], 344 | "metricColumn": "none", 345 | "rawQuery": true, 346 | "rawSql": "SELECT\n updated_at AS \"time\",\n follower_count\nFROM stats\nWHERE\n $__timeFilter(updated_at)\nORDER BY updated_at", 347 | "refId": "A", 348 | "select": [ 349 | [ 350 | { 351 | "params": [ 352 | "id" 353 | ], 354 | "type": "column" 355 | } 356 | ] 357 | ], 358 | "table": "iterations", 359 | "timeColumn": "started_at", 360 | "timeColumnType": "timestamp", 361 | "where": [ 362 | { 363 | "name": "$__timeFilter", 364 | "params": [], 365 | "type": "macro" 366 | } 367 | ] 368 | } 369 | ], 370 | "title": "Followers", 371 | "type": "timeseries" 372 | }, 373 | { 374 | "datasource": { 375 | "type": "mysql", 376 | "uid": "${DS_TIDB-DEVTIER}" 377 | }, 378 | "fieldConfig": { 379 | "defaults": { 380 | "color": { 381 | "mode": "palette-classic" 382 | }, 383 | "custom": { 384 | "hideFrom": { 385 | "legend": false, 386 | "tooltip": false, 387 | "viz": false 388 | } 389 | }, 390 | "mappings": [] 391 | }, 392 | "overrides": [] 393 | }, 394 | "gridPos": { 395 | "h": 8, 396 | "w": 4, 397 | "x": 12, 398 | "y": 1 399 | }, 400 | "id": 49, 401 | "options": { 402 | "displayLabels": [ 403 | "percent" 404 | ], 405 | "legend": { 406 | "displayMode": "list", 407 | "placement": "bottom", 408 | "showLegend": true 409 | }, 410 | "pieType": "pie", 411 | "reduceOptions": { 412 | "calcs": [ 413 | "lastNotNull" 414 | ], 415 | "fields": "", 416 | "values": true 417 | }, 418 | "tooltip": { 419 | "mode": "single", 420 | "sort": "none" 421 | } 422 | }, 423 | "targets": [ 424 | { 425 | "datasource": { 426 | "type": "mysql", 427 | "uid": "${DS_TIDB-DEVTIER}" 428 | }, 429 | "format": "table", 430 | "group": [], 431 | "metricColumn": "none", 432 | "rawQuery": true, 433 | "rawSql": "SELECT\n CASE \n WHEN is_follower=TRUE AND is_following=TRUE THEN 'friend'\n WHEN is_follower=TRUE AND is_following=FALSE THEN 'follower'\n WHEN is_follower=FALSE AND is_following=TRUE THEN 'following'\n ELSE 'unkown' \n END,\n count(*)\nFROM users\nGROUP BY CASE \n WHEN is_follower=TRUE AND is_following=TRUE THEN 'friend'\n WHEN is_follower=TRUE AND is_following=FALSE THEN 'follower'\n WHEN is_follower=FALSE AND is_following=TRUE THEN 'following'\n ELSE 'unkown' \n END\n\n\n\n\n", 434 | "refId": "A", 435 | "select": [ 436 | [ 437 | { 438 | "params": [ 439 | "id" 440 | ], 441 | "type": "column" 442 | } 443 | ] 444 | ], 445 | "table": "iterations", 446 | "timeColumn": "started_at", 447 | "timeColumnType": "timestamp", 448 | "where": [ 449 | { 450 | "name": "$__timeFilter", 451 | "params": [], 452 | "type": "macro" 453 | } 454 | ] 455 | } 456 | ], 457 | "title": "Users", 458 | "type": "piechart" 459 | }, 460 | { 461 | "datasource": { 462 | "type": "mysql", 463 | "uid": "${DS_TIDB-DEVTIER}" 464 | }, 465 | "fieldConfig": { 466 | "defaults": { 467 | "color": { 468 | "mode": "palette-classic" 469 | }, 470 | "custom": { 471 | "hideFrom": { 472 | "legend": false, 473 | "tooltip": false, 474 | "viz": false 475 | } 476 | }, 477 | "mappings": [] 478 | }, 479 | "overrides": [] 480 | }, 481 | "gridPos": { 482 | "h": 8, 483 | "w": 4, 484 | "x": 16, 485 | "y": 1 486 | }, 487 | "id": 50, 488 | "options": { 489 | "displayLabels": [ 490 | "percent" 491 | ], 492 | "legend": { 493 | "displayMode": "list", 494 | "placement": "bottom", 495 | "showLegend": true 496 | }, 497 | "pieType": "pie", 498 | "reduceOptions": { 499 | "calcs": [ 500 | "lastNotNull" 501 | ], 502 | "fields": "", 503 | "values": true 504 | }, 505 | "tooltip": { 506 | "mode": "single", 507 | "sort": "none" 508 | } 509 | }, 510 | "targets": [ 511 | { 512 | "datasource": { 513 | "type": "mysql", 514 | "uid": "${DS_TIDB-DEVTIER}" 515 | }, 516 | "format": "table", 517 | "group": [], 518 | "metricColumn": "none", 519 | "rawQuery": true, 520 | "rawSql": "SELECT\n CASE \n WHEN is_following=TRUE THEN 'Friend'\n WHEN is_following=FALSE THEN 'Follower'\n END,\n count(*)\nFROM users\nWHERE is_follower = TRUE\nGROUP BY CASE \n WHEN is_following=TRUE THEN 'Friend'\n WHEN is_following=FALSE THEN 'Follower'\n END\n\n\n\n\n", 521 | "refId": "A", 522 | "select": [ 523 | [ 524 | { 525 | "params": [ 526 | "id" 527 | ], 528 | "type": "column" 529 | } 530 | ] 531 | ], 532 | "table": "iterations", 533 | "timeColumn": "started_at", 534 | "timeColumnType": "timestamp", 535 | "where": [ 536 | { 537 | "name": "$__timeFilter", 538 | "params": [], 539 | "type": "macro" 540 | } 541 | ] 542 | } 543 | ], 544 | "title": "Friend / Follower", 545 | "type": "piechart" 546 | }, 547 | { 548 | "datasource": { 549 | "type": "mysql", 550 | "uid": "${DS_TIDB-DEVTIER}" 551 | }, 552 | "fieldConfig": { 553 | "defaults": { 554 | "color": { 555 | "mode": "palette-classic" 556 | }, 557 | "custom": { 558 | "hideFrom": { 559 | "legend": false, 560 | "tooltip": false, 561 | "viz": false 562 | } 563 | }, 564 | "mappings": [] 565 | }, 566 | "overrides": [] 567 | }, 568 | "gridPos": { 569 | "h": 8, 570 | "w": 4, 571 | "x": 20, 572 | "y": 1 573 | }, 574 | "id": 51, 575 | "options": { 576 | "displayLabels": [ 577 | "percent" 578 | ], 579 | "legend": { 580 | "displayMode": "list", 581 | "placement": "bottom", 582 | "showLegend": true 583 | }, 584 | "pieType": "pie", 585 | "reduceOptions": { 586 | "calcs": [ 587 | "lastNotNull" 588 | ], 589 | "fields": "", 590 | "values": true 591 | }, 592 | "tooltip": { 593 | "mode": "single", 594 | "sort": "none" 595 | } 596 | }, 597 | "targets": [ 598 | { 599 | "datasource": { 600 | "type": "mysql", 601 | "uid": "${DS_TIDB-DEVTIER}" 602 | }, 603 | "format": "table", 604 | "group": [], 605 | "metricColumn": "none", 606 | "rawQuery": true, 607 | "rawSql": "SELECT\n CASE \n WHEN is_follower=TRUE THEN 'Friend'\n WHEN is_follower=FALSE THEN 'Following'\n END,\n count(*)\nFROM users\nWHERE is_following = TRUE\nGROUP BY CASE \n WHEN is_follower=TRUE THEN 'Friend'\n WHEN is_follower=FALSE THEN 'Following'\n END\n\n\n\n\n", 608 | "refId": "A", 609 | "select": [ 610 | [ 611 | { 612 | "params": [ 613 | "id" 614 | ], 615 | "type": "column" 616 | } 617 | ] 618 | ], 619 | "table": "iterations", 620 | "timeColumn": "started_at", 621 | "timeColumnType": "timestamp", 622 | "where": [ 623 | { 624 | "name": "$__timeFilter", 625 | "params": [], 626 | "type": "macro" 627 | } 628 | ] 629 | } 630 | ], 631 | "title": "Friend / Following", 632 | "type": "piechart" 633 | }, 634 | { 635 | "datasource": { 636 | "type": "mysql", 637 | "uid": "${DS_TIDB-DEVTIER}" 638 | }, 639 | "fieldConfig": { 640 | "defaults": { 641 | "color": { 642 | "mode": "thresholds" 643 | }, 644 | "mappings": [], 645 | "thresholds": { 646 | "mode": "absolute", 647 | "steps": [ 648 | { 649 | "color": "green", 650 | "value": null 651 | } 652 | ] 653 | } 654 | }, 655 | "overrides": [] 656 | }, 657 | "gridPos": { 658 | "h": 4, 659 | "w": 3, 660 | "x": 0, 661 | "y": 5 662 | }, 663 | "id": 38, 664 | "options": { 665 | "colorMode": "background", 666 | "graphMode": "area", 667 | "justifyMode": "auto", 668 | "orientation": "auto", 669 | "reduceOptions": { 670 | "calcs": [ 671 | "lastNotNull" 672 | ], 673 | "fields": "", 674 | "values": false 675 | }, 676 | "textMode": "auto" 677 | }, 678 | "pluginVersion": "9.1.6-10b38e80", 679 | "targets": [ 680 | { 681 | "datasource": { 682 | "type": "mysql", 683 | "uid": "${DS_TIDB-DEVTIER}" 684 | }, 685 | "format": "time_series", 686 | "group": [], 687 | "metricColumn": "none", 688 | "rawQuery": true, 689 | "rawSql": "SELECT\n updated_at AS \"time\",\n following_count\nFROM stats\nWHERE\n $__timeFilter(updated_at)\nORDER BY updated_at", 690 | "refId": "A", 691 | "select": [ 692 | [ 693 | { 694 | "params": [ 695 | "id" 696 | ], 697 | "type": "column" 698 | } 699 | ] 700 | ], 701 | "table": "iterations", 702 | "timeColumn": "started_at", 703 | "timeColumnType": "timestamp", 704 | "where": [ 705 | { 706 | "name": "$__timeFilter", 707 | "params": [], 708 | "type": "macro" 709 | } 710 | ] 711 | } 712 | ], 713 | "title": "Following", 714 | "type": "stat" 715 | }, 716 | { 717 | "datasource": { 718 | "type": "mysql", 719 | "uid": "${DS_TIDB-DEVTIER}" 720 | }, 721 | "fieldConfig": { 722 | "defaults": { 723 | "color": { 724 | "mode": "thresholds" 725 | }, 726 | "mappings": [], 727 | "thresholds": { 728 | "mode": "absolute", 729 | "steps": [ 730 | { 731 | "color": "green", 732 | "value": null 733 | } 734 | ] 735 | } 736 | }, 737 | "overrides": [] 738 | }, 739 | "gridPos": { 740 | "h": 4, 741 | "w": 3, 742 | "x": 3, 743 | "y": 5 744 | }, 745 | "id": 39, 746 | "options": { 747 | "colorMode": "background", 748 | "graphMode": "area", 749 | "justifyMode": "auto", 750 | "orientation": "auto", 751 | "reduceOptions": { 752 | "calcs": [ 753 | "lastNotNull" 754 | ], 755 | "fields": "", 756 | "values": false 757 | }, 758 | "textMode": "auto" 759 | }, 760 | "pluginVersion": "9.1.6-10b38e80", 761 | "targets": [ 762 | { 763 | "datasource": { 764 | "type": "mysql", 765 | "uid": "${DS_TIDB-DEVTIER}" 766 | }, 767 | "format": "time_series", 768 | "group": [], 769 | "metricColumn": "none", 770 | "rawQuery": true, 771 | "rawSql": "SELECT\n updated_at AS \"time\",\n listed_count\nFROM stats\nWHERE\n $__timeFilter(updated_at)\nORDER BY updated_at", 772 | "refId": "A", 773 | "select": [ 774 | [ 775 | { 776 | "params": [ 777 | "id" 778 | ], 779 | "type": "column" 780 | } 781 | ] 782 | ], 783 | "table": "iterations", 784 | "timeColumn": "started_at", 785 | "timeColumnType": "timestamp", 786 | "where": [ 787 | { 788 | "name": "$__timeFilter", 789 | "params": [], 790 | "type": "macro" 791 | } 792 | ] 793 | } 794 | ], 795 | "title": "Listed", 796 | "type": "stat" 797 | }, 798 | { 799 | "collapsed": false, 800 | "gridPos": { 801 | "h": 1, 802 | "w": 24, 803 | "x": 0, 804 | "y": 9 805 | }, 806 | "id": 32, 807 | "panels": [], 808 | "title": "Activity (by selected Interval)", 809 | "type": "row" 810 | }, 811 | { 812 | "datasource": { 813 | "type": "mysql", 814 | "uid": "${DS_TIDB-DEVTIER}" 815 | }, 816 | "fieldConfig": { 817 | "defaults": { 818 | "color": { 819 | "mode": "thresholds" 820 | }, 821 | "mappings": [], 822 | "thresholds": { 823 | "mode": "absolute", 824 | "steps": [ 825 | { 826 | "color": "green", 827 | "value": null 828 | } 829 | ] 830 | } 831 | }, 832 | "overrides": [] 833 | }, 834 | "gridPos": { 835 | "h": 4, 836 | "w": 3, 837 | "x": 0, 838 | "y": 10 839 | }, 840 | "id": 2, 841 | "options": { 842 | "colorMode": "background", 843 | "graphMode": "area", 844 | "justifyMode": "auto", 845 | "orientation": "auto", 846 | "reduceOptions": { 847 | "calcs": [ 848 | "lastNotNull" 849 | ], 850 | "fields": "", 851 | "values": false 852 | }, 853 | "textMode": "auto" 854 | }, 855 | "pluginVersion": "9.1.6-10b38e80", 856 | "targets": [ 857 | { 858 | "datasource": { 859 | "type": "mysql", 860 | "uid": "${DS_TIDB-DEVTIER}" 861 | }, 862 | "format": "table", 863 | "group": [], 864 | "hide": false, 865 | "metricColumn": "none", 866 | "rawQuery": true, 867 | "rawSql": "SELECT\n count(*)\nFROM events\nWHERE\n $__timeFilter(created_at) AND event = 'new_follower'", 868 | "refId": "A", 869 | "select": [ 870 | [ 871 | { 872 | "params": [ 873 | "count(*)" 874 | ], 875 | "type": "column" 876 | } 877 | ] 878 | ], 879 | "table": "events", 880 | "timeColumn": "time", 881 | "timeColumnType": "timestamp", 882 | "where": [ 883 | { 884 | "name": "$__timeFilter", 885 | "params": [], 886 | "type": "macro" 887 | } 888 | ] 889 | } 890 | ], 891 | "title": "New Follower", 892 | "type": "stat" 893 | }, 894 | { 895 | "datasource": { 896 | "type": "mysql", 897 | "uid": "${DS_TIDB-DEVTIER}" 898 | }, 899 | "fieldConfig": { 900 | "defaults": { 901 | "color": { 902 | "mode": "thresholds" 903 | }, 904 | "mappings": [], 905 | "thresholds": { 906 | "mode": "absolute", 907 | "steps": [ 908 | { 909 | "color": "green", 910 | "value": null 911 | }, 912 | { 913 | "color": "#EAB839", 914 | "value": 3 915 | }, 916 | { 917 | "color": "red", 918 | "value": 10 919 | } 920 | ] 921 | } 922 | }, 923 | "overrides": [] 924 | }, 925 | "gridPos": { 926 | "h": 4, 927 | "w": 3, 928 | "x": 3, 929 | "y": 10 930 | }, 931 | "id": 3, 932 | "options": { 933 | "colorMode": "background", 934 | "graphMode": "area", 935 | "justifyMode": "auto", 936 | "orientation": "auto", 937 | "reduceOptions": { 938 | "calcs": [ 939 | "lastNotNull" 940 | ], 941 | "fields": "", 942 | "values": false 943 | }, 944 | "textMode": "auto" 945 | }, 946 | "pluginVersion": "9.1.6-10b38e80", 947 | "targets": [ 948 | { 949 | "datasource": { 950 | "type": "mysql", 951 | "uid": "${DS_TIDB-DEVTIER}" 952 | }, 953 | "format": "table", 954 | "group": [], 955 | "hide": false, 956 | "metricColumn": "none", 957 | "rawQuery": true, 958 | "rawSql": "SELECT\n count(*)\nFROM events\nWHERE\n $__timeFilter(created_at) AND event = 'lost_follower'", 959 | "refId": "A", 960 | "select": [ 961 | [ 962 | { 963 | "params": [ 964 | "count(*)" 965 | ], 966 | "type": "column" 967 | } 968 | ] 969 | ], 970 | "table": "events", 971 | "timeColumn": "time", 972 | "timeColumnType": "timestamp", 973 | "where": [ 974 | { 975 | "name": "$__timeFilter", 976 | "params": [], 977 | "type": "macro" 978 | } 979 | ] 980 | } 981 | ], 982 | "title": "Lost Follower", 983 | "type": "stat" 984 | }, 985 | { 986 | "datasource": { 987 | "type": "mysql", 988 | "uid": "${DS_TIDB-DEVTIER}" 989 | }, 990 | "fieldConfig": { 991 | "defaults": { 992 | "color": { 993 | "mode": "thresholds" 994 | }, 995 | "mappings": [], 996 | "thresholds": { 997 | "mode": "absolute", 998 | "steps": [ 999 | { 1000 | "color": "green", 1001 | "value": null 1002 | } 1003 | ] 1004 | } 1005 | }, 1006 | "overrides": [] 1007 | }, 1008 | "gridPos": { 1009 | "h": 4, 1010 | "w": 3, 1011 | "x": 6, 1012 | "y": 10 1013 | }, 1014 | "id": 5, 1015 | "options": { 1016 | "colorMode": "background", 1017 | "graphMode": "area", 1018 | "justifyMode": "auto", 1019 | "orientation": "auto", 1020 | "reduceOptions": { 1021 | "calcs": [ 1022 | "lastNotNull" 1023 | ], 1024 | "fields": "", 1025 | "values": false 1026 | }, 1027 | "textMode": "auto" 1028 | }, 1029 | "pluginVersion": "9.1.6-10b38e80", 1030 | "targets": [ 1031 | { 1032 | "datasource": { 1033 | "type": "mysql", 1034 | "uid": "${DS_TIDB-DEVTIER}" 1035 | }, 1036 | "format": "table", 1037 | "group": [], 1038 | "hide": false, 1039 | "metricColumn": "none", 1040 | "rawQuery": true, 1041 | "rawSql": "SELECT\n count(*)\nFROM events\nWHERE\n $__timeFilter(created_at) AND event = 'cancel_following'", 1042 | "refId": "A", 1043 | "select": [ 1044 | [ 1045 | { 1046 | "params": [ 1047 | "count(*)" 1048 | ], 1049 | "type": "column" 1050 | } 1051 | ] 1052 | ], 1053 | "table": "events", 1054 | "timeColumn": "time", 1055 | "timeColumnType": "timestamp", 1056 | "where": [ 1057 | { 1058 | "name": "$__timeFilter", 1059 | "params": [], 1060 | "type": "macro" 1061 | } 1062 | ] 1063 | } 1064 | ], 1065 | "title": "Cancel Following", 1066 | "type": "stat" 1067 | }, 1068 | { 1069 | "datasource": { 1070 | "type": "mysql", 1071 | "uid": "${DS_TIDB-DEVTIER}" 1072 | }, 1073 | "fieldConfig": { 1074 | "defaults": { 1075 | "color": { 1076 | "mode": "thresholds" 1077 | }, 1078 | "mappings": [], 1079 | "thresholds": { 1080 | "mode": "absolute", 1081 | "steps": [ 1082 | { 1083 | "color": "green", 1084 | "value": null 1085 | } 1086 | ] 1087 | } 1088 | }, 1089 | "overrides": [] 1090 | }, 1091 | "gridPos": { 1092 | "h": 4, 1093 | "w": 3, 1094 | "x": 9, 1095 | "y": 10 1096 | }, 1097 | "id": 4, 1098 | "options": { 1099 | "colorMode": "background", 1100 | "graphMode": "area", 1101 | "justifyMode": "auto", 1102 | "orientation": "auto", 1103 | "reduceOptions": { 1104 | "calcs": [ 1105 | "lastNotNull" 1106 | ], 1107 | "fields": "", 1108 | "values": false 1109 | }, 1110 | "textMode": "auto" 1111 | }, 1112 | "pluginVersion": "9.1.6-10b38e80", 1113 | "targets": [ 1114 | { 1115 | "datasource": { 1116 | "type": "mysql", 1117 | "uid": "${DS_TIDB-DEVTIER}" 1118 | }, 1119 | "format": "table", 1120 | "group": [], 1121 | "hide": false, 1122 | "metricColumn": "none", 1123 | "rawQuery": true, 1124 | "rawSql": "SELECT\n count(*)\nFROM events\nWHERE\n $__timeFilter(created_at) AND event = 'new_following'", 1125 | "refId": "A", 1126 | "select": [ 1127 | [ 1128 | { 1129 | "params": [ 1130 | "count(*)" 1131 | ], 1132 | "type": "column" 1133 | } 1134 | ] 1135 | ], 1136 | "table": "events", 1137 | "timeColumn": "time", 1138 | "timeColumnType": "timestamp", 1139 | "where": [ 1140 | { 1141 | "name": "$__timeFilter", 1142 | "params": [], 1143 | "type": "macro" 1144 | } 1145 | ] 1146 | } 1147 | ], 1148 | "title": "New Following", 1149 | "type": "stat" 1150 | }, 1151 | { 1152 | "datasource": { 1153 | "type": "mysql", 1154 | "uid": "${DS_TIDB-DEVTIER}" 1155 | }, 1156 | "fieldConfig": { 1157 | "defaults": { 1158 | "color": { 1159 | "mode": "thresholds" 1160 | }, 1161 | "mappings": [], 1162 | "thresholds": { 1163 | "mode": "absolute", 1164 | "steps": [ 1165 | { 1166 | "color": "green", 1167 | "value": null 1168 | } 1169 | ] 1170 | } 1171 | }, 1172 | "overrides": [] 1173 | }, 1174 | "gridPos": { 1175 | "h": 4, 1176 | "w": 3, 1177 | "x": 12, 1178 | "y": 10 1179 | }, 1180 | "id": 43, 1181 | "options": { 1182 | "colorMode": "background", 1183 | "graphMode": "area", 1184 | "justifyMode": "auto", 1185 | "orientation": "auto", 1186 | "reduceOptions": { 1187 | "calcs": [ 1188 | "lastNotNull" 1189 | ], 1190 | "fields": "", 1191 | "values": false 1192 | }, 1193 | "textMode": "auto" 1194 | }, 1195 | "pluginVersion": "9.1.6-10b38e80", 1196 | "targets": [ 1197 | { 1198 | "datasource": { 1199 | "type": "mysql", 1200 | "uid": "${DS_TIDB-DEVTIER}" 1201 | }, 1202 | "format": "table", 1203 | "group": [], 1204 | "hide": false, 1205 | "metricColumn": "none", 1206 | "rawQuery": true, 1207 | "rawSql": "SELECT\n MAX(follower_count) - MIN(follower_count)\nFROM stats\nWHERE\n $__timeFilter(updated_at)", 1208 | "refId": "A", 1209 | "select": [ 1210 | [ 1211 | { 1212 | "params": [ 1213 | "count(*)" 1214 | ], 1215 | "type": "column" 1216 | } 1217 | ] 1218 | ], 1219 | "table": "events", 1220 | "timeColumn": "time", 1221 | "timeColumnType": "timestamp", 1222 | "where": [ 1223 | { 1224 | "name": "$__timeFilter", 1225 | "params": [], 1226 | "type": "macro" 1227 | } 1228 | ] 1229 | } 1230 | ], 1231 | "title": "Follower Increased", 1232 | "type": "stat" 1233 | }, 1234 | { 1235 | "datasource": { 1236 | "type": "mysql", 1237 | "uid": "${DS_TIDB-DEVTIER}" 1238 | }, 1239 | "fieldConfig": { 1240 | "defaults": { 1241 | "color": { 1242 | "mode": "thresholds" 1243 | }, 1244 | "mappings": [], 1245 | "thresholds": { 1246 | "mode": "absolute", 1247 | "steps": [ 1248 | { 1249 | "color": "green", 1250 | "value": null 1251 | } 1252 | ] 1253 | } 1254 | }, 1255 | "overrides": [] 1256 | }, 1257 | "gridPos": { 1258 | "h": 4, 1259 | "w": 3, 1260 | "x": 15, 1261 | "y": 10 1262 | }, 1263 | "id": 45, 1264 | "options": { 1265 | "colorMode": "background", 1266 | "graphMode": "area", 1267 | "justifyMode": "auto", 1268 | "orientation": "auto", 1269 | "reduceOptions": { 1270 | "calcs": [ 1271 | "lastNotNull" 1272 | ], 1273 | "fields": "", 1274 | "values": false 1275 | }, 1276 | "textMode": "auto" 1277 | }, 1278 | "pluginVersion": "9.1.6-10b38e80", 1279 | "targets": [ 1280 | { 1281 | "datasource": { 1282 | "type": "mysql", 1283 | "uid": "${DS_TIDB-DEVTIER}" 1284 | }, 1285 | "format": "table", 1286 | "group": [], 1287 | "hide": false, 1288 | "metricColumn": "none", 1289 | "rawQuery": true, 1290 | "rawSql": "SELECT\n MAX(following_count) - MIN(following_count)\nFROM stats\nWHERE\n $__timeFilter(updated_at)", 1291 | "refId": "A", 1292 | "select": [ 1293 | [ 1294 | { 1295 | "params": [ 1296 | "count(*)" 1297 | ], 1298 | "type": "column" 1299 | } 1300 | ] 1301 | ], 1302 | "table": "events", 1303 | "timeColumn": "time", 1304 | "timeColumnType": "timestamp", 1305 | "where": [ 1306 | { 1307 | "name": "$__timeFilter", 1308 | "params": [], 1309 | "type": "macro" 1310 | } 1311 | ] 1312 | } 1313 | ], 1314 | "title": "Following Increased", 1315 | "type": "stat" 1316 | }, 1317 | { 1318 | "datasource": { 1319 | "type": "mysql", 1320 | "uid": "${DS_TIDB-DEVTIER}" 1321 | }, 1322 | "fieldConfig": { 1323 | "defaults": { 1324 | "color": { 1325 | "mode": "thresholds" 1326 | }, 1327 | "mappings": [], 1328 | "thresholds": { 1329 | "mode": "absolute", 1330 | "steps": [ 1331 | { 1332 | "color": "green", 1333 | "value": null 1334 | } 1335 | ] 1336 | } 1337 | }, 1338 | "overrides": [] 1339 | }, 1340 | "gridPos": { 1341 | "h": 4, 1342 | "w": 3, 1343 | "x": 18, 1344 | "y": 10 1345 | }, 1346 | "id": 44, 1347 | "options": { 1348 | "colorMode": "background", 1349 | "graphMode": "area", 1350 | "justifyMode": "auto", 1351 | "orientation": "auto", 1352 | "reduceOptions": { 1353 | "calcs": [ 1354 | "lastNotNull" 1355 | ], 1356 | "fields": "", 1357 | "values": false 1358 | }, 1359 | "textMode": "auto" 1360 | }, 1361 | "pluginVersion": "9.1.6-10b38e80", 1362 | "targets": [ 1363 | { 1364 | "datasource": { 1365 | "type": "mysql", 1366 | "uid": "${DS_TIDB-DEVTIER}" 1367 | }, 1368 | "format": "table", 1369 | "group": [], 1370 | "hide": false, 1371 | "metricColumn": "none", 1372 | "rawQuery": true, 1373 | "rawSql": "SELECT\n MAX(tweet_count) - MIN(tweet_count)\nFROM stats\nWHERE\n $__timeFilter(updated_at)", 1374 | "refId": "A", 1375 | "select": [ 1376 | [ 1377 | { 1378 | "params": [ 1379 | "count(*)" 1380 | ], 1381 | "type": "column" 1382 | } 1383 | ] 1384 | ], 1385 | "table": "events", 1386 | "timeColumn": "time", 1387 | "timeColumnType": "timestamp", 1388 | "where": [ 1389 | { 1390 | "name": "$__timeFilter", 1391 | "params": [], 1392 | "type": "macro" 1393 | } 1394 | ] 1395 | } 1396 | ], 1397 | "title": "Tweets", 1398 | "type": "stat" 1399 | }, 1400 | { 1401 | "datasource": { 1402 | "type": "mysql", 1403 | "uid": "${DS_TIDB-DEVTIER}" 1404 | }, 1405 | "fieldConfig": { 1406 | "defaults": { 1407 | "color": { 1408 | "mode": "thresholds" 1409 | }, 1410 | "mappings": [], 1411 | "thresholds": { 1412 | "mode": "absolute", 1413 | "steps": [ 1414 | { 1415 | "color": "green", 1416 | "value": null 1417 | } 1418 | ] 1419 | } 1420 | }, 1421 | "overrides": [] 1422 | }, 1423 | "gridPos": { 1424 | "h": 4, 1425 | "w": 3, 1426 | "x": 21, 1427 | "y": 10 1428 | }, 1429 | "id": 42, 1430 | "options": { 1431 | "colorMode": "background", 1432 | "graphMode": "area", 1433 | "justifyMode": "auto", 1434 | "orientation": "auto", 1435 | "reduceOptions": { 1436 | "calcs": [ 1437 | "lastNotNull" 1438 | ], 1439 | "fields": "", 1440 | "values": false 1441 | }, 1442 | "textMode": "auto" 1443 | }, 1444 | "pluginVersion": "9.1.6-10b38e80", 1445 | "targets": [ 1446 | { 1447 | "datasource": { 1448 | "type": "mysql", 1449 | "uid": "${DS_TIDB-DEVTIER}" 1450 | }, 1451 | "format": "table", 1452 | "group": [], 1453 | "hide": false, 1454 | "metricColumn": "none", 1455 | "rawQuery": true, 1456 | "rawSql": "SELECT\n MAX(listed_count) - MIN(listed_count)\nFROM stats\nWHERE\n $__timeFilter(updated_at)", 1457 | "refId": "A", 1458 | "select": [ 1459 | [ 1460 | { 1461 | "params": [ 1462 | "count(*)" 1463 | ], 1464 | "type": "column" 1465 | } 1466 | ] 1467 | ], 1468 | "table": "events", 1469 | "timeColumn": "time", 1470 | "timeColumnType": "timestamp", 1471 | "where": [ 1472 | { 1473 | "name": "$__timeFilter", 1474 | "params": [], 1475 | "type": "macro" 1476 | } 1477 | ] 1478 | } 1479 | ], 1480 | "title": "Listed Increased", 1481 | "type": "stat" 1482 | }, 1483 | { 1484 | "datasource": { 1485 | "type": "mysql", 1486 | "uid": "${DS_TIDB-DEVTIER}" 1487 | }, 1488 | "fieldConfig": { 1489 | "defaults": { 1490 | "color": { 1491 | "mode": "thresholds" 1492 | }, 1493 | "custom": { 1494 | "align": "auto", 1495 | "displayMode": "auto", 1496 | "inspect": false 1497 | }, 1498 | "mappings": [], 1499 | "thresholds": { 1500 | "mode": "absolute", 1501 | "steps": [ 1502 | { 1503 | "color": "green", 1504 | "value": null 1505 | }, 1506 | { 1507 | "color": "red", 1508 | "value": 80 1509 | } 1510 | ] 1511 | } 1512 | }, 1513 | "overrides": [ 1514 | { 1515 | "matcher": { 1516 | "id": "byName", 1517 | "options": "profile" 1518 | }, 1519 | "properties": [ 1520 | { 1521 | "id": "custom.displayMode", 1522 | "value": "image" 1523 | }, 1524 | { 1525 | "id": "custom.width", 1526 | "value": 64 1527 | } 1528 | ] 1529 | }, 1530 | { 1531 | "matcher": { 1532 | "id": "byName", 1533 | "options": "name" 1534 | }, 1535 | "properties": [ 1536 | { 1537 | "id": "links", 1538 | "value": [ 1539 | { 1540 | "targetBlank": true, 1541 | "title": "User Page", 1542 | "url": "https://twitter.com/${__data.fields.user_name}" 1543 | } 1544 | ] 1545 | } 1546 | ] 1547 | }, 1548 | { 1549 | "matcher": { 1550 | "id": "byName", 1551 | "options": "user_name" 1552 | }, 1553 | "properties": [ 1554 | { 1555 | "id": "custom.hidden", 1556 | "value": true 1557 | } 1558 | ] 1559 | }, 1560 | { 1561 | "matcher": { 1562 | "id": "byType", 1563 | "options": "number" 1564 | }, 1565 | "properties": [ 1566 | { 1567 | "id": "custom.width", 1568 | "value": 150 1569 | } 1570 | ] 1571 | }, 1572 | { 1573 | "matcher": { 1574 | "id": "byType", 1575 | "options": "time" 1576 | }, 1577 | "properties": [ 1578 | { 1579 | "id": "custom.width", 1580 | "value": 200 1581 | } 1582 | ] 1583 | } 1584 | ] 1585 | }, 1586 | "gridPos": { 1587 | "h": 8, 1588 | "w": 12, 1589 | "x": 0, 1590 | "y": 14 1591 | }, 1592 | "id": 33, 1593 | "options": { 1594 | "footer": { 1595 | "fields": "", 1596 | "reducer": [ 1597 | "sum" 1598 | ], 1599 | "show": false 1600 | }, 1601 | "showHeader": true, 1602 | "sortBy": [] 1603 | }, 1604 | "pluginVersion": "9.1.6-10b38e80", 1605 | "targets": [ 1606 | { 1607 | "datasource": { 1608 | "type": "mysql", 1609 | "uid": "${DS_TIDB-DEVTIER}" 1610 | }, 1611 | "format": "table", 1612 | "group": [], 1613 | "metricColumn": "none", 1614 | "rawQuery": true, 1615 | "rawSql": "SELECT\n u.profile_image as profile,\n CONCAT(u.name, ' (', u.user_name, ')') as name,\n u.user_name,\n u.follower_count as follower, \n u.tweet_count as tweet,\n u.listed_count as listed,\n u.last_active as 'last tweet'\nFROM events e\nJOIN users u ON u.id = e.user_id\nWHERE e.event='new_follower' AND u.is_follower = true AND $__timeFilter(e.created_at)\nORDER BY created_at desc\nLIMIT 500", 1616 | "refId": "A", 1617 | "select": [ 1618 | [ 1619 | { 1620 | "params": [ 1621 | "follower_count" 1622 | ], 1623 | "type": "column" 1624 | } 1625 | ] 1626 | ], 1627 | "table": "users", 1628 | "timeColumn": "last_active", 1629 | "timeColumnType": "timestamp", 1630 | "where": [ 1631 | { 1632 | "name": "$__timeFilter", 1633 | "params": [], 1634 | "type": "macro" 1635 | } 1636 | ] 1637 | } 1638 | ], 1639 | "title": "New followers", 1640 | "transformations": [], 1641 | "type": "table" 1642 | }, 1643 | { 1644 | "datasource": { 1645 | "type": "mysql", 1646 | "uid": "${DS_TIDB-DEVTIER}" 1647 | }, 1648 | "fieldConfig": { 1649 | "defaults": { 1650 | "color": { 1651 | "mode": "thresholds" 1652 | }, 1653 | "custom": { 1654 | "align": "auto", 1655 | "displayMode": "auto", 1656 | "inspect": false 1657 | }, 1658 | "mappings": [], 1659 | "thresholds": { 1660 | "mode": "absolute", 1661 | "steps": [ 1662 | { 1663 | "color": "green", 1664 | "value": null 1665 | }, 1666 | { 1667 | "color": "red", 1668 | "value": 80 1669 | } 1670 | ] 1671 | } 1672 | }, 1673 | "overrides": [ 1674 | { 1675 | "matcher": { 1676 | "id": "byName", 1677 | "options": "profile" 1678 | }, 1679 | "properties": [ 1680 | { 1681 | "id": "custom.displayMode", 1682 | "value": "image" 1683 | }, 1684 | { 1685 | "id": "custom.width", 1686 | "value": 64 1687 | } 1688 | ] 1689 | }, 1690 | { 1691 | "matcher": { 1692 | "id": "byName", 1693 | "options": "name" 1694 | }, 1695 | "properties": [ 1696 | { 1697 | "id": "links", 1698 | "value": [ 1699 | { 1700 | "targetBlank": true, 1701 | "title": "User Page", 1702 | "url": "https://twitter.com/${__data.fields.user_name}" 1703 | } 1704 | ] 1705 | } 1706 | ] 1707 | }, 1708 | { 1709 | "matcher": { 1710 | "id": "byName", 1711 | "options": "user_name" 1712 | }, 1713 | "properties": [ 1714 | { 1715 | "id": "custom.hidden", 1716 | "value": true 1717 | } 1718 | ] 1719 | }, 1720 | { 1721 | "matcher": { 1722 | "id": "byType", 1723 | "options": "number" 1724 | }, 1725 | "properties": [ 1726 | { 1727 | "id": "custom.width", 1728 | "value": 150 1729 | } 1730 | ] 1731 | } 1732 | ] 1733 | }, 1734 | "gridPos": { 1735 | "h": 8, 1736 | "w": 12, 1737 | "x": 12, 1738 | "y": 14 1739 | }, 1740 | "id": 34, 1741 | "options": { 1742 | "footer": { 1743 | "fields": "", 1744 | "reducer": [ 1745 | "sum" 1746 | ], 1747 | "show": false 1748 | }, 1749 | "showHeader": true, 1750 | "sortBy": [] 1751 | }, 1752 | "pluginVersion": "9.1.6-10b38e80", 1753 | "targets": [ 1754 | { 1755 | "datasource": { 1756 | "type": "mysql", 1757 | "uid": "${DS_TIDB-DEVTIER}" 1758 | }, 1759 | "format": "table", 1760 | "group": [], 1761 | "metricColumn": "none", 1762 | "rawQuery": true, 1763 | "rawSql": "SELECT\n u.profile_image as profile,\n CONCAT(u.name, ' (', u.user_name, ')') as name,\n u.user_name,\n u.follower_count as follower, \n u.tweet_count as tweet,\n u.listed_count as listed\nFROM events e\nJOIN users u ON u.id = e.user_id\nWHERE e.event='lost_follower' AND u.is_follower = false AND $__timeFilter(created_at)\nORDER BY created_at desc\nLIMIT 500", 1764 | "refId": "A", 1765 | "select": [ 1766 | [ 1767 | { 1768 | "params": [ 1769 | "follower_count" 1770 | ], 1771 | "type": "column" 1772 | } 1773 | ] 1774 | ], 1775 | "table": "users", 1776 | "timeColumn": "last_active", 1777 | "timeColumnType": "timestamp", 1778 | "where": [ 1779 | { 1780 | "name": "$__timeFilter", 1781 | "params": [], 1782 | "type": "macro" 1783 | } 1784 | ] 1785 | } 1786 | ], 1787 | "title": "Lost followers", 1788 | "transformations": [], 1789 | "type": "table" 1790 | }, 1791 | { 1792 | "datasource": { 1793 | "type": "mysql", 1794 | "uid": "${DS_TIDB-DEVTIER}" 1795 | }, 1796 | "fieldConfig": { 1797 | "defaults": { 1798 | "color": { 1799 | "mode": "thresholds" 1800 | }, 1801 | "custom": { 1802 | "align": "auto", 1803 | "displayMode": "auto", 1804 | "inspect": false 1805 | }, 1806 | "mappings": [], 1807 | "thresholds": { 1808 | "mode": "absolute", 1809 | "steps": [ 1810 | { 1811 | "color": "green", 1812 | "value": null 1813 | }, 1814 | { 1815 | "color": "red", 1816 | "value": 80 1817 | } 1818 | ] 1819 | } 1820 | }, 1821 | "overrides": [ 1822 | { 1823 | "matcher": { 1824 | "id": "byName", 1825 | "options": "profile" 1826 | }, 1827 | "properties": [ 1828 | { 1829 | "id": "custom.displayMode", 1830 | "value": "image" 1831 | }, 1832 | { 1833 | "id": "custom.width", 1834 | "value": 64 1835 | } 1836 | ] 1837 | }, 1838 | { 1839 | "matcher": { 1840 | "id": "byName", 1841 | "options": "name" 1842 | }, 1843 | "properties": [ 1844 | { 1845 | "id": "links", 1846 | "value": [ 1847 | { 1848 | "targetBlank": true, 1849 | "title": "User Page", 1850 | "url": "https://twitter.com/${__data.fields.user_name}" 1851 | } 1852 | ] 1853 | } 1854 | ] 1855 | }, 1856 | { 1857 | "matcher": { 1858 | "id": "byName", 1859 | "options": "user_name" 1860 | }, 1861 | "properties": [ 1862 | { 1863 | "id": "custom.hidden", 1864 | "value": true 1865 | } 1866 | ] 1867 | }, 1868 | { 1869 | "matcher": { 1870 | "id": "byType", 1871 | "options": "number" 1872 | }, 1873 | "properties": [ 1874 | { 1875 | "id": "custom.width", 1876 | "value": 150 1877 | } 1878 | ] 1879 | } 1880 | ] 1881 | }, 1882 | "gridPos": { 1883 | "h": 8, 1884 | "w": 12, 1885 | "x": 0, 1886 | "y": 22 1887 | }, 1888 | "id": 54, 1889 | "options": { 1890 | "footer": { 1891 | "fields": "", 1892 | "reducer": [ 1893 | "sum" 1894 | ], 1895 | "show": false 1896 | }, 1897 | "showHeader": true, 1898 | "sortBy": [] 1899 | }, 1900 | "pluginVersion": "9.1.6-10b38e80", 1901 | "targets": [ 1902 | { 1903 | "datasource": { 1904 | "type": "mysql", 1905 | "uid": "${DS_TIDB-DEVTIER}" 1906 | }, 1907 | "format": "table", 1908 | "group": [], 1909 | "metricColumn": "none", 1910 | "rawQuery": true, 1911 | "rawSql": "SELECT\n u.profile_image as profile,\n CONCAT(u.name, ' (', u.user_name, ')') as name,\n u.user_name,\n u.follower_count as follower, \n u.tweet_count as tweet,\n u.listed_count as listed\nFROM events e\nJOIN users u ON u.id = e.user_id\nWHERE e.event='new_following' AND u.is_following = true AND $__timeFilter(created_at)\nORDER BY created_at desc\nLIMIT 500", 1912 | "refId": "A", 1913 | "select": [ 1914 | [ 1915 | { 1916 | "params": [ 1917 | "follower_count" 1918 | ], 1919 | "type": "column" 1920 | } 1921 | ] 1922 | ], 1923 | "table": "users", 1924 | "timeColumn": "last_active", 1925 | "timeColumnType": "timestamp", 1926 | "where": [ 1927 | { 1928 | "name": "$__timeFilter", 1929 | "params": [], 1930 | "type": "macro" 1931 | } 1932 | ] 1933 | } 1934 | ], 1935 | "title": "New Following", 1936 | "transformations": [], 1937 | "type": "table" 1938 | }, 1939 | { 1940 | "datasource": { 1941 | "type": "mysql", 1942 | "uid": "${DS_TIDB-DEVTIER}" 1943 | }, 1944 | "fieldConfig": { 1945 | "defaults": { 1946 | "color": { 1947 | "mode": "thresholds" 1948 | }, 1949 | "custom": { 1950 | "align": "auto", 1951 | "displayMode": "auto", 1952 | "inspect": false 1953 | }, 1954 | "mappings": [], 1955 | "thresholds": { 1956 | "mode": "absolute", 1957 | "steps": [ 1958 | { 1959 | "color": "green", 1960 | "value": null 1961 | }, 1962 | { 1963 | "color": "red", 1964 | "value": 80 1965 | } 1966 | ] 1967 | } 1968 | }, 1969 | "overrides": [ 1970 | { 1971 | "matcher": { 1972 | "id": "byName", 1973 | "options": "profile" 1974 | }, 1975 | "properties": [ 1976 | { 1977 | "id": "custom.displayMode", 1978 | "value": "image" 1979 | }, 1980 | { 1981 | "id": "custom.width", 1982 | "value": 64 1983 | } 1984 | ] 1985 | }, 1986 | { 1987 | "matcher": { 1988 | "id": "byName", 1989 | "options": "name" 1990 | }, 1991 | "properties": [ 1992 | { 1993 | "id": "links", 1994 | "value": [ 1995 | { 1996 | "targetBlank": true, 1997 | "title": "User Page", 1998 | "url": "https://twitter.com/${__data.fields.user_name}" 1999 | } 2000 | ] 2001 | } 2002 | ] 2003 | }, 2004 | { 2005 | "matcher": { 2006 | "id": "byName", 2007 | "options": "user_name" 2008 | }, 2009 | "properties": [ 2010 | { 2011 | "id": "custom.hidden", 2012 | "value": true 2013 | } 2014 | ] 2015 | }, 2016 | { 2017 | "matcher": { 2018 | "id": "byType", 2019 | "options": "number" 2020 | }, 2021 | "properties": [ 2022 | { 2023 | "id": "custom.width", 2024 | "value": 150 2025 | } 2026 | ] 2027 | } 2028 | ] 2029 | }, 2030 | "gridPos": { 2031 | "h": 8, 2032 | "w": 12, 2033 | "x": 12, 2034 | "y": 22 2035 | }, 2036 | "id": 55, 2037 | "options": { 2038 | "footer": { 2039 | "fields": "", 2040 | "reducer": [ 2041 | "sum" 2042 | ], 2043 | "show": false 2044 | }, 2045 | "showHeader": true, 2046 | "sortBy": [] 2047 | }, 2048 | "pluginVersion": "9.1.6-10b38e80", 2049 | "targets": [ 2050 | { 2051 | "datasource": { 2052 | "type": "mysql", 2053 | "uid": "${DS_TIDB-DEVTIER}" 2054 | }, 2055 | "format": "table", 2056 | "group": [], 2057 | "metricColumn": "none", 2058 | "rawQuery": true, 2059 | "rawSql": "SELECT\n u.profile_image as profile,\n CONCAT(u.name, ' (', u.user_name, ')') as name,\n u.user_name,\n u.follower_count as follower, \n u.tweet_count as tweet,\n u.listed_count as listed\nFROM events e\nJOIN users u ON u.id = e.user_id\nWHERE e.event='cancel_following' AND u.is_following = false AND $__timeFilter(created_at)\nORDER BY created_at desc\nLIMIT 500", 2060 | "refId": "A", 2061 | "select": [ 2062 | [ 2063 | { 2064 | "params": [ 2065 | "follower_count" 2066 | ], 2067 | "type": "column" 2068 | } 2069 | ] 2070 | ], 2071 | "table": "users", 2072 | "timeColumn": "last_active", 2073 | "timeColumnType": "timestamp", 2074 | "where": [ 2075 | { 2076 | "name": "$__timeFilter", 2077 | "params": [], 2078 | "type": "macro" 2079 | } 2080 | ] 2081 | } 2082 | ], 2083 | "title": "Cancel Following", 2084 | "transformations": [], 2085 | "type": "table" 2086 | }, 2087 | { 2088 | "collapsed": true, 2089 | "gridPos": { 2090 | "h": 1, 2091 | "w": 24, 2092 | "x": 0, 2093 | "y": 30 2094 | }, 2095 | "id": 17, 2096 | "panels": [ 2097 | { 2098 | "datasource": { 2099 | "type": "mysql", 2100 | "uid": "${DS_TIDB-DEVTIER}" 2101 | }, 2102 | "description": "", 2103 | "fieldConfig": { 2104 | "defaults": { 2105 | "color": { 2106 | "fixedColor": "green", 2107 | "mode": "fixed" 2108 | }, 2109 | "custom": { 2110 | "axisCenteredZero": false, 2111 | "axisColorMode": "text", 2112 | "axisLabel": "", 2113 | "axisPlacement": "auto", 2114 | "fillOpacity": 100, 2115 | "gradientMode": "none", 2116 | "hideFrom": { 2117 | "legend": false, 2118 | "tooltip": false, 2119 | "viz": false 2120 | }, 2121 | "lineWidth": 0, 2122 | "scaleDistribution": { 2123 | "type": "linear" 2124 | } 2125 | }, 2126 | "mappings": [], 2127 | "noValue": "0", 2128 | "thresholds": { 2129 | "mode": "absolute", 2130 | "steps": [ 2131 | { 2132 | "color": "green" 2133 | } 2134 | ] 2135 | } 2136 | }, 2137 | "overrides": [ 2138 | { 2139 | "matcher": { 2140 | "id": "byName", 2141 | "options": "count lost_follower" 2142 | }, 2143 | "properties": [ 2144 | { 2145 | "id": "color", 2146 | "value": { 2147 | "fixedColor": "red", 2148 | "mode": "fixed" 2149 | } 2150 | } 2151 | ] 2152 | } 2153 | ] 2154 | }, 2155 | "gridPos": { 2156 | "h": 8, 2157 | "w": 12, 2158 | "x": 0, 2159 | "y": 31 2160 | }, 2161 | "id": 15, 2162 | "options": { 2163 | "barRadius": 0, 2164 | "barWidth": 0.93, 2165 | "groupWidth": 0.56, 2166 | "legend": { 2167 | "calcs": [], 2168 | "displayMode": "list", 2169 | "placement": "bottom", 2170 | "showLegend": true 2171 | }, 2172 | "orientation": "auto", 2173 | "showValue": "auto", 2174 | "stacking": "none", 2175 | "tooltip": { 2176 | "mode": "single", 2177 | "sort": "none" 2178 | }, 2179 | "xField": "Time", 2180 | "xTickLabelRotation": 0, 2181 | "xTickLabelSpacing": 0 2182 | }, 2183 | "pluginVersion": "9.1.5-0100a6a", 2184 | "targets": [ 2185 | { 2186 | "datasource": { 2187 | "type": "mysql", 2188 | "uid": "${DS_TIDB-DEVTIER}" 2189 | }, 2190 | "format": "time_series", 2191 | "group": [], 2192 | "metricColumn": "none", 2193 | "rawQuery": true, 2194 | "rawSql": "SELECT\n DATE(CONVERT_TZ(created_at, \"UTC\", \"$TZ\")) as \"time\",\n COUNT(*) as count,\n event as event\nFROM events\nWHERE (event='new_follower' OR event='lost_follower') AND CONVERT_TZ(created_at, \"UTC\", \"$TZ\") > DATE(CONVERT_TZ($__timeTo(), \"UTC\", \"$TZ\")) - INTERVAL $TREND DAY + INTERVAL 1 DAY\nGROUP BY DATE(CONVERT_TZ(created_at, \"UTC\", \"$TZ\")), event\nORDER BY DATE(CONVERT_TZ(created_at, \"UTC\", \"$TZ\"))", 2195 | "refId": "A", 2196 | "select": [ 2197 | [ 2198 | { 2199 | "params": [ 2200 | "follower_count" 2201 | ], 2202 | "type": "column" 2203 | } 2204 | ] 2205 | ], 2206 | "table": "events", 2207 | "timeColumn": "last_active", 2208 | "timeColumnType": "timestamp", 2209 | "where": [] 2210 | } 2211 | ], 2212 | "title": "Daily Follower Increase/Decrease", 2213 | "transformations": [], 2214 | "type": "barchart" 2215 | }, 2216 | { 2217 | "datasource": { 2218 | "type": "mysql", 2219 | "uid": "${DS_TIDB-DEVTIER}" 2220 | }, 2221 | "description": "", 2222 | "fieldConfig": { 2223 | "defaults": { 2224 | "color": { 2225 | "fixedColor": "green", 2226 | "mode": "fixed" 2227 | }, 2228 | "custom": { 2229 | "axisCenteredZero": false, 2230 | "axisColorMode": "text", 2231 | "axisLabel": "", 2232 | "axisPlacement": "auto", 2233 | "fillOpacity": 100, 2234 | "gradientMode": "none", 2235 | "hideFrom": { 2236 | "legend": false, 2237 | "tooltip": false, 2238 | "viz": false 2239 | }, 2240 | "lineWidth": 0, 2241 | "scaleDistribution": { 2242 | "type": "linear" 2243 | } 2244 | }, 2245 | "mappings": [], 2246 | "noValue": "0", 2247 | "thresholds": { 2248 | "mode": "absolute", 2249 | "steps": [ 2250 | { 2251 | "color": "green" 2252 | } 2253 | ] 2254 | } 2255 | }, 2256 | "overrides": [ 2257 | { 2258 | "matcher": { 2259 | "id": "byName", 2260 | "options": "count lost_follower" 2261 | }, 2262 | "properties": [ 2263 | { 2264 | "id": "color", 2265 | "value": { 2266 | "fixedColor": "red", 2267 | "mode": "fixed" 2268 | } 2269 | } 2270 | ] 2271 | } 2272 | ] 2273 | }, 2274 | "gridPos": { 2275 | "h": 8, 2276 | "w": 12, 2277 | "x": 12, 2278 | "y": 31 2279 | }, 2280 | "id": 61, 2281 | "options": { 2282 | "barRadius": 0, 2283 | "barWidth": 0.27, 2284 | "groupWidth": 0.56, 2285 | "legend": { 2286 | "calcs": [], 2287 | "displayMode": "list", 2288 | "placement": "bottom", 2289 | "showLegend": true 2290 | }, 2291 | "orientation": "auto", 2292 | "showValue": "auto", 2293 | "stacking": "none", 2294 | "tooltip": { 2295 | "mode": "single", 2296 | "sort": "none" 2297 | }, 2298 | "xField": "Time", 2299 | "xTickLabelRotation": 0, 2300 | "xTickLabelSpacing": 0 2301 | }, 2302 | "pluginVersion": "9.1.5-0100a6a", 2303 | "targets": [ 2304 | { 2305 | "datasource": { 2306 | "type": "mysql", 2307 | "uid": "${DS_TIDB-DEVTIER}" 2308 | }, 2309 | "format": "time_series", 2310 | "group": [], 2311 | "metricColumn": "none", 2312 | "rawQuery": true, 2313 | "rawSql": "SELECT s1.time as time, s1.tweet - s2.tweet as tweet FROM\n(SELECT DATE(CONVERT_TZ(updated_at, \"UTC\", \"$TZ\")) as time, MAX(tweet_count) as tweet FROM stats WHERE CONVERT_TZ(updated_at, \"UTC\", \"$TZ\") > DATE(CONVERT_TZ($__timeTo(), \"UTC\", \"$TZ\")) - INTERVAL $TREND DAY GROUP BY time) s1\nJOIN\n(SELECT DATE(CONVERT_TZ(updated_at, \"UTC\", \"$TZ\")) as time, MAX(tweet_count) as tweet FROM stats WHERE CONVERT_TZ(updated_at, \"UTC\", \"$TZ\") > DATE(CONVERT_TZ($__timeTo(), \"UTC\", \"$TZ\")) - INTERVAL $TREND DAY - INTERVAL 1 DAY GROUP BY time) s2\nON s1.time = s2.time + INTERVAL 1 DAY\n\n", 2314 | "refId": "A", 2315 | "select": [ 2316 | [ 2317 | { 2318 | "params": [ 2319 | "follower_count" 2320 | ], 2321 | "type": "column" 2322 | } 2323 | ] 2324 | ], 2325 | "table": "events", 2326 | "timeColumn": "last_active", 2327 | "timeColumnType": "timestamp", 2328 | "where": [] 2329 | } 2330 | ], 2331 | "title": "Daily Tweet", 2332 | "transformations": [], 2333 | "type": "barchart" 2334 | } 2335 | ], 2336 | "repeat": "TREND", 2337 | "repeatDirection": "h", 2338 | "title": "Trend ($TREND Days)", 2339 | "type": "row" 2340 | }, 2341 | { 2342 | "collapsed": true, 2343 | "gridPos": { 2344 | "h": 1, 2345 | "w": 24, 2346 | "x": 0, 2347 | "y": 32 2348 | }, 2349 | "id": 12, 2350 | "panels": [ 2351 | { 2352 | "datasource": { 2353 | "type": "mysql", 2354 | "uid": "${DS_TIDB-DEVTIER}" 2355 | }, 2356 | "fieldConfig": { 2357 | "defaults": { 2358 | "color": { 2359 | "mode": "thresholds" 2360 | }, 2361 | "custom": { 2362 | "align": "auto", 2363 | "displayMode": "auto", 2364 | "inspect": false 2365 | }, 2366 | "mappings": [], 2367 | "thresholds": { 2368 | "mode": "absolute", 2369 | "steps": [ 2370 | { 2371 | "color": "green" 2372 | }, 2373 | { 2374 | "color": "red", 2375 | "value": 80 2376 | } 2377 | ] 2378 | } 2379 | }, 2380 | "overrides": [ 2381 | { 2382 | "matcher": { 2383 | "id": "byName", 2384 | "options": "profile" 2385 | }, 2386 | "properties": [ 2387 | { 2388 | "id": "custom.displayMode", 2389 | "value": "image" 2390 | }, 2391 | { 2392 | "id": "custom.width", 2393 | "value": 64 2394 | } 2395 | ] 2396 | }, 2397 | { 2398 | "matcher": { 2399 | "id": "byName", 2400 | "options": "name" 2401 | }, 2402 | "properties": [ 2403 | { 2404 | "id": "links", 2405 | "value": [ 2406 | { 2407 | "targetBlank": true, 2408 | "title": "User Page", 2409 | "url": "https://twitter.com/${__data.fields.user_name}" 2410 | } 2411 | ] 2412 | } 2413 | ] 2414 | }, 2415 | { 2416 | "matcher": { 2417 | "id": "byName", 2418 | "options": "user_name" 2419 | }, 2420 | "properties": [ 2421 | { 2422 | "id": "custom.hidden", 2423 | "value": true 2424 | } 2425 | ] 2426 | } 2427 | ] 2428 | }, 2429 | "gridPos": { 2430 | "h": 10, 2431 | "w": 24, 2432 | "x": 0, 2433 | "y": 32 2434 | }, 2435 | "id": 46, 2436 | "options": { 2437 | "footer": { 2438 | "enablePagination": true, 2439 | "fields": "", 2440 | "reducer": [ 2441 | "sum" 2442 | ], 2443 | "show": false 2444 | }, 2445 | "showHeader": true, 2446 | "sortBy": [] 2447 | }, 2448 | "pluginVersion": "9.1.6-10b38e80", 2449 | "targets": [ 2450 | { 2451 | "datasource": { 2452 | "type": "mysql", 2453 | "uid": "${DS_TIDB-DEVTIER}" 2454 | }, 2455 | "format": "table", 2456 | "group": [], 2457 | "metricColumn": "none", 2458 | "rawQuery": true, 2459 | "rawSql": "SELECT\n profile_image as profile, CONCAT(name, ' (', user_name, ')') as name, user_name, follower_count as follower, following_count as following, tweet_count as tweet, last_active as 'last tweet', updated_at as 'update time'\nFROM users\nWHERE is_follower=true\nORDER BY follower_count DESC", 2460 | "refId": "A", 2461 | "select": [ 2462 | [ 2463 | { 2464 | "params": [ 2465 | "follower_count" 2466 | ], 2467 | "type": "column" 2468 | } 2469 | ] 2470 | ], 2471 | "table": "users", 2472 | "timeColumn": "last_active", 2473 | "timeColumnType": "timestamp", 2474 | "where": [ 2475 | { 2476 | "name": "$__timeFilter", 2477 | "params": [], 2478 | "type": "macro" 2479 | } 2480 | ] 2481 | } 2482 | ], 2483 | "title": "All Followers", 2484 | "transformations": [], 2485 | "type": "table" 2486 | }, 2487 | { 2488 | "datasource": { 2489 | "type": "mysql", 2490 | "uid": "${DS_TIDB-DEVTIER}" 2491 | }, 2492 | "fieldConfig": { 2493 | "defaults": { 2494 | "color": { 2495 | "mode": "thresholds" 2496 | }, 2497 | "custom": { 2498 | "align": "auto", 2499 | "displayMode": "auto", 2500 | "inspect": false 2501 | }, 2502 | "mappings": [], 2503 | "thresholds": { 2504 | "mode": "absolute", 2505 | "steps": [ 2506 | { 2507 | "color": "green" 2508 | }, 2509 | { 2510 | "color": "red", 2511 | "value": 80 2512 | } 2513 | ] 2514 | } 2515 | }, 2516 | "overrides": [ 2517 | { 2518 | "matcher": { 2519 | "id": "byName", 2520 | "options": "profile" 2521 | }, 2522 | "properties": [ 2523 | { 2524 | "id": "custom.displayMode", 2525 | "value": "image" 2526 | }, 2527 | { 2528 | "id": "custom.width", 2529 | "value": 64 2530 | } 2531 | ] 2532 | }, 2533 | { 2534 | "matcher": { 2535 | "id": "byName", 2536 | "options": "name" 2537 | }, 2538 | "properties": [ 2539 | { 2540 | "id": "links", 2541 | "value": [ 2542 | { 2543 | "targetBlank": true, 2544 | "title": "User Page", 2545 | "url": "https://twitter.com/${__data.fields.user_name}" 2546 | } 2547 | ] 2548 | } 2549 | ] 2550 | }, 2551 | { 2552 | "matcher": { 2553 | "id": "byName", 2554 | "options": "user_name" 2555 | }, 2556 | "properties": [ 2557 | { 2558 | "id": "custom.hidden", 2559 | "value": true 2560 | } 2561 | ] 2562 | } 2563 | ] 2564 | }, 2565 | "gridPos": { 2566 | "h": 10, 2567 | "w": 24, 2568 | "x": 0, 2569 | "y": 42 2570 | }, 2571 | "id": 47, 2572 | "options": { 2573 | "footer": { 2574 | "enablePagination": true, 2575 | "fields": "", 2576 | "reducer": [ 2577 | "sum" 2578 | ], 2579 | "show": false 2580 | }, 2581 | "showHeader": true, 2582 | "sortBy": [ 2583 | { 2584 | "desc": true, 2585 | "displayName": "follower" 2586 | } 2587 | ] 2588 | }, 2589 | "pluginVersion": "9.1.6-10b38e80", 2590 | "targets": [ 2591 | { 2592 | "datasource": { 2593 | "type": "mysql", 2594 | "uid": "${DS_TIDB-DEVTIER}" 2595 | }, 2596 | "format": "table", 2597 | "group": [], 2598 | "metricColumn": "none", 2599 | "rawQuery": true, 2600 | "rawSql": "SELECT\n profile_image as profile, CONCAT(name, ' (', user_name, ')') as name, user_name, follower_count as follower, following_count as following, tweet_count as tweet, last_active as 'last tweet', updated_at as 'update time'\nFROM users\nWHERE is_following=true\nORDER BY follower_count DESC", 2601 | "refId": "A", 2602 | "select": [ 2603 | [ 2604 | { 2605 | "params": [ 2606 | "follower_count" 2607 | ], 2608 | "type": "column" 2609 | } 2610 | ] 2611 | ], 2612 | "table": "users", 2613 | "timeColumn": "last_active", 2614 | "timeColumnType": "timestamp", 2615 | "where": [ 2616 | { 2617 | "name": "$__timeFilter", 2618 | "params": [], 2619 | "type": "macro" 2620 | } 2621 | ] 2622 | } 2623 | ], 2624 | "title": "All Following", 2625 | "transformations": [], 2626 | "type": "table" 2627 | }, 2628 | { 2629 | "datasource": { 2630 | "type": "mysql", 2631 | "uid": "${DS_TIDB-DEVTIER}" 2632 | }, 2633 | "fieldConfig": { 2634 | "defaults": { 2635 | "color": { 2636 | "mode": "thresholds" 2637 | }, 2638 | "custom": { 2639 | "align": "auto", 2640 | "displayMode": "auto", 2641 | "inspect": false 2642 | }, 2643 | "mappings": [], 2644 | "thresholds": { 2645 | "mode": "absolute", 2646 | "steps": [ 2647 | { 2648 | "color": "green" 2649 | }, 2650 | { 2651 | "color": "red", 2652 | "value": 80 2653 | } 2654 | ] 2655 | } 2656 | }, 2657 | "overrides": [ 2658 | { 2659 | "matcher": { 2660 | "id": "byName", 2661 | "options": "profile" 2662 | }, 2663 | "properties": [ 2664 | { 2665 | "id": "custom.displayMode", 2666 | "value": "image" 2667 | }, 2668 | { 2669 | "id": "custom.width", 2670 | "value": 64 2671 | } 2672 | ] 2673 | }, 2674 | { 2675 | "matcher": { 2676 | "id": "byName", 2677 | "options": "name" 2678 | }, 2679 | "properties": [ 2680 | { 2681 | "id": "links", 2682 | "value": [ 2683 | { 2684 | "targetBlank": true, 2685 | "title": "User Page", 2686 | "url": "https://twitter.com/${__data.fields.user_name}" 2687 | } 2688 | ] 2689 | } 2690 | ] 2691 | }, 2692 | { 2693 | "matcher": { 2694 | "id": "byName", 2695 | "options": "user_name" 2696 | }, 2697 | "properties": [ 2698 | { 2699 | "id": "custom.hidden", 2700 | "value": true 2701 | } 2702 | ] 2703 | } 2704 | ] 2705 | }, 2706 | "gridPos": { 2707 | "h": 9, 2708 | "w": 12, 2709 | "x": 0, 2710 | "y": 52 2711 | }, 2712 | "id": 9, 2713 | "options": { 2714 | "footer": { 2715 | "fields": "", 2716 | "reducer": [ 2717 | "sum" 2718 | ], 2719 | "show": false 2720 | }, 2721 | "showHeader": true, 2722 | "sortBy": [] 2723 | }, 2724 | "pluginVersion": "9.1.6-10b38e80", 2725 | "targets": [ 2726 | { 2727 | "datasource": { 2728 | "type": "mysql", 2729 | "uid": "${DS_TIDB-DEVTIER}" 2730 | }, 2731 | "format": "table", 2732 | "group": [], 2733 | "metricColumn": "none", 2734 | "rawQuery": true, 2735 | "rawSql": "SELECT\n profile_image as profile, CONCAT(name, ' (', user_name, ')') as name, user_name, follower_count\nFROM users\nWHERE\n is_follower = TRUE AND is_following = TRUE\nORDER BY follower_count desc\nlimit 100", 2736 | "refId": "A", 2737 | "select": [ 2738 | [ 2739 | { 2740 | "params": [ 2741 | "follower_count" 2742 | ], 2743 | "type": "column" 2744 | } 2745 | ] 2746 | ], 2747 | "table": "users", 2748 | "timeColumn": "last_active", 2749 | "timeColumnType": "timestamp", 2750 | "where": [ 2751 | { 2752 | "name": "$__timeFilter", 2753 | "params": [], 2754 | "type": "macro" 2755 | } 2756 | ] 2757 | } 2758 | ], 2759 | "title": "Most Popular Friend", 2760 | "transformations": [], 2761 | "type": "table" 2762 | }, 2763 | { 2764 | "datasource": { 2765 | "type": "mysql", 2766 | "uid": "${DS_TIDB-DEVTIER}" 2767 | }, 2768 | "fieldConfig": { 2769 | "defaults": { 2770 | "color": { 2771 | "mode": "thresholds" 2772 | }, 2773 | "custom": { 2774 | "align": "auto", 2775 | "displayMode": "auto", 2776 | "inspect": false 2777 | }, 2778 | "mappings": [], 2779 | "thresholds": { 2780 | "mode": "absolute", 2781 | "steps": [ 2782 | { 2783 | "color": "green" 2784 | }, 2785 | { 2786 | "color": "red", 2787 | "value": 80 2788 | } 2789 | ] 2790 | } 2791 | }, 2792 | "overrides": [ 2793 | { 2794 | "matcher": { 2795 | "id": "byName", 2796 | "options": "profile" 2797 | }, 2798 | "properties": [ 2799 | { 2800 | "id": "custom.displayMode", 2801 | "value": "image" 2802 | }, 2803 | { 2804 | "id": "custom.width", 2805 | "value": 64 2806 | } 2807 | ] 2808 | }, 2809 | { 2810 | "matcher": { 2811 | "id": "byName", 2812 | "options": "name" 2813 | }, 2814 | "properties": [ 2815 | { 2816 | "id": "links", 2817 | "value": [ 2818 | { 2819 | "targetBlank": true, 2820 | "title": "User Page", 2821 | "url": "https://twitter.com/${__data.fields.user_name}" 2822 | } 2823 | ] 2824 | } 2825 | ] 2826 | }, 2827 | { 2828 | "matcher": { 2829 | "id": "byName", 2830 | "options": "user_name" 2831 | }, 2832 | "properties": [ 2833 | { 2834 | "id": "custom.hidden", 2835 | "value": true 2836 | } 2837 | ] 2838 | } 2839 | ] 2840 | }, 2841 | "gridPos": { 2842 | "h": 9, 2843 | "w": 12, 2844 | "x": 12, 2845 | "y": 52 2846 | }, 2847 | "id": 24, 2848 | "options": { 2849 | "footer": { 2850 | "fields": "", 2851 | "reducer": [ 2852 | "sum" 2853 | ], 2854 | "show": false 2855 | }, 2856 | "showHeader": true, 2857 | "sortBy": [] 2858 | }, 2859 | "pluginVersion": "9.1.6-10b38e80", 2860 | "targets": [ 2861 | { 2862 | "datasource": { 2863 | "type": "mysql", 2864 | "uid": "${DS_TIDB-DEVTIER}" 2865 | }, 2866 | "format": "table", 2867 | "group": [], 2868 | "metricColumn": "none", 2869 | "rawQuery": true, 2870 | "rawSql": "\nSELECT\n profile_image as profile, CONCAT(name, ' (', user_name, ')') as name, user_name, follower_count as follower, tweet_count as tweet, last_active as 'last tweet'\nFROM users\nWHERE\n is_following = FALSE AND is_follower = TRUE AND last_active = (SELECT MAX(last_active) FROM users) AND follower_count > (SELECT AVG(follower_count) FROM users WHERE is_follower=TRUE)\nORDER BY follower_count/(tweet_count+1)/(tweet_count+1) desc\nlimit 20", 2871 | "refId": "A", 2872 | "select": [ 2873 | [ 2874 | { 2875 | "params": [ 2876 | "follower_count" 2877 | ], 2878 | "type": "column" 2879 | } 2880 | ] 2881 | ], 2882 | "table": "users", 2883 | "timeColumn": "last_active", 2884 | "timeColumnType": "timestamp", 2885 | "where": [ 2886 | { 2887 | "name": "$__timeFilter", 2888 | "params": [], 2889 | "type": "macro" 2890 | } 2891 | ] 2892 | } 2893 | ], 2894 | "title": "Recommend to follow", 2895 | "transformations": [], 2896 | "type": "table" 2897 | } 2898 | ], 2899 | "title": "User", 2900 | "type": "row" 2901 | }, 2902 | { 2903 | "collapsed": true, 2904 | "gridPos": { 2905 | "h": 1, 2906 | "w": 24, 2907 | "x": 0, 2908 | "y": 33 2909 | }, 2910 | "id": 28, 2911 | "panels": [ 2912 | { 2913 | "datasource": { 2914 | "type": "mysql", 2915 | "uid": "${DS_TIDB-DEVTIER}" 2916 | }, 2917 | "fieldConfig": { 2918 | "defaults": { 2919 | "color": { 2920 | "fixedColor": "text", 2921 | "mode": "fixed" 2922 | }, 2923 | "custom": { 2924 | "align": "auto", 2925 | "displayMode": "color-text", 2926 | "inspect": false 2927 | }, 2928 | "mappings": [ 2929 | { 2930 | "options": { 2931 | "error": { 2932 | "color": "red", 2933 | "index": 0 2934 | }, 2935 | "info": { 2936 | "color": "light-blue", 2937 | "index": 1 2938 | } 2939 | }, 2940 | "type": "value" 2941 | }, 2942 | { 2943 | "options": { 2944 | "pattern": "failed.*", 2945 | "result": { 2946 | "color": "red", 2947 | "index": 2 2948 | } 2949 | }, 2950 | "type": "regex" 2951 | } 2952 | ], 2953 | "thresholds": { 2954 | "mode": "absolute", 2955 | "steps": [ 2956 | { 2957 | "color": "green" 2958 | } 2959 | ] 2960 | } 2961 | }, 2962 | "overrides": [ 2963 | { 2964 | "matcher": { 2965 | "id": "byName", 2966 | "options": "time" 2967 | }, 2968 | "properties": [ 2969 | { 2970 | "id": "custom.width", 2971 | "value": 170 2972 | } 2973 | ] 2974 | }, 2975 | { 2976 | "matcher": { 2977 | "id": "byName", 2978 | "options": "level" 2979 | }, 2980 | "properties": [ 2981 | { 2982 | "id": "custom.width", 2983 | "value": 75 2984 | } 2985 | ] 2986 | } 2987 | ] 2988 | }, 2989 | "gridPos": { 2990 | "h": 8, 2991 | "w": 12, 2992 | "x": 0, 2993 | "y": 5 2994 | }, 2995 | "id": 26, 2996 | "options": { 2997 | "footer": { 2998 | "enablePagination": false, 2999 | "fields": "", 3000 | "reducer": [ 3001 | "sum" 3002 | ], 3003 | "show": false 3004 | }, 3005 | "showHeader": true, 3006 | "sortBy": [] 3007 | }, 3008 | "pluginVersion": "9.1.6-10b38e80", 3009 | "targets": [ 3010 | { 3011 | "datasource": { 3012 | "type": "mysql", 3013 | "uid": "${DS_TIDB-DEVTIER}" 3014 | }, 3015 | "format": "table", 3016 | "group": [], 3017 | "metricColumn": "none", 3018 | "rawQuery": true, 3019 | "rawSql": "SELECT\n created_at AS \"time\",\n level, message\nFROM logs\nWHERE\n $__timeFilter(created_at)\nORDER BY created_at desc limit 50", 3020 | "refId": "A", 3021 | "select": [ 3022 | [ 3023 | { 3024 | "params": [ 3025 | "id" 3026 | ], 3027 | "type": "column" 3028 | } 3029 | ] 3030 | ], 3031 | "table": "iterations", 3032 | "timeColumn": "started_at", 3033 | "timeColumnType": "timestamp", 3034 | "where": [ 3035 | { 3036 | "name": "$__timeFilter", 3037 | "params": [], 3038 | "type": "macro" 3039 | } 3040 | ] 3041 | } 3042 | ], 3043 | "title": "System Logs", 3044 | "type": "table" 3045 | }, 3046 | { 3047 | "datasource": { 3048 | "type": "mysql", 3049 | "uid": "${DS_TIDB-DEVTIER}" 3050 | }, 3051 | "fieldConfig": { 3052 | "defaults": { 3053 | "color": { 3054 | "mode": "thresholds" 3055 | }, 3056 | "custom": { 3057 | "align": "auto", 3058 | "displayMode": "auto", 3059 | "inspect": false 3060 | }, 3061 | "mappings": [], 3062 | "thresholds": { 3063 | "mode": "absolute", 3064 | "steps": [ 3065 | { 3066 | "color": "green" 3067 | }, 3068 | { 3069 | "color": "red", 3070 | "value": 80 3071 | } 3072 | ] 3073 | } 3074 | }, 3075 | "overrides": [] 3076 | }, 3077 | "gridPos": { 3078 | "h": 8, 3079 | "w": 12, 3080 | "x": 12, 3081 | "y": 5 3082 | }, 3083 | "id": 30, 3084 | "options": { 3085 | "footer": { 3086 | "fields": "", 3087 | "reducer": [ 3088 | "sum" 3089 | ], 3090 | "show": false 3091 | }, 3092 | "showHeader": true 3093 | }, 3094 | "pluginVersion": "9.1.6-10b38e80", 3095 | "targets": [ 3096 | { 3097 | "datasource": { 3098 | "type": "mysql", 3099 | "uid": "${DS_TIDB-DEVTIER}" 3100 | }, 3101 | "format": "table", 3102 | "group": [], 3103 | "metricColumn": "none", 3104 | "rawQuery": true, 3105 | "rawSql": "SELECT\n *\nFROM system", 3106 | "refId": "A", 3107 | "select": [ 3108 | [ 3109 | { 3110 | "params": [ 3111 | "id" 3112 | ], 3113 | "type": "column" 3114 | } 3115 | ] 3116 | ], 3117 | "table": "iterations", 3118 | "timeColumn": "started_at", 3119 | "timeColumnType": "timestamp", 3120 | "where": [ 3121 | { 3122 | "name": "$__timeFilter", 3123 | "params": [], 3124 | "type": "macro" 3125 | } 3126 | ] 3127 | } 3128 | ], 3129 | "title": "System Variables", 3130 | "type": "table" 3131 | }, 3132 | { 3133 | "datasource": { 3134 | "type": "mysql", 3135 | "uid": "${DS_TIDB-DEVTIER}" 3136 | }, 3137 | "fieldConfig": { 3138 | "defaults": { 3139 | "color": { 3140 | "mode": "palette-classic" 3141 | }, 3142 | "custom": { 3143 | "axisCenteredZero": false, 3144 | "axisColorMode": "text", 3145 | "axisLabel": "", 3146 | "axisPlacement": "auto", 3147 | "fillOpacity": 80, 3148 | "gradientMode": "none", 3149 | "hideFrom": { 3150 | "legend": false, 3151 | "tooltip": false, 3152 | "viz": false 3153 | }, 3154 | "lineWidth": 1, 3155 | "scaleDistribution": { 3156 | "type": "linear" 3157 | } 3158 | }, 3159 | "mappings": [], 3160 | "thresholds": { 3161 | "mode": "absolute", 3162 | "steps": [ 3163 | { 3164 | "color": "green" 3165 | }, 3166 | { 3167 | "color": "red", 3168 | "value": 80 3169 | } 3170 | ] 3171 | }, 3172 | "unit": "s" 3173 | }, 3174 | "overrides": [] 3175 | }, 3176 | "gridPos": { 3177 | "h": 8, 3178 | "w": 12, 3179 | "x": 0, 3180 | "y": 13 3181 | }, 3182 | "id": 41, 3183 | "options": { 3184 | "barRadius": 0, 3185 | "barWidth": 0.97, 3186 | "groupWidth": 0.7, 3187 | "legend": { 3188 | "calcs": [], 3189 | "displayMode": "list", 3190 | "placement": "bottom", 3191 | "showLegend": true 3192 | }, 3193 | "orientation": "auto", 3194 | "showValue": "auto", 3195 | "stacking": "none", 3196 | "tooltip": { 3197 | "mode": "single", 3198 | "sort": "none" 3199 | }, 3200 | "xTickLabelRotation": 75, 3201 | "xTickLabelSpacing": 0 3202 | }, 3203 | "targets": [ 3204 | { 3205 | "datasource": { 3206 | "type": "mysql", 3207 | "uid": "${DS_TIDB-DEVTIER}" 3208 | }, 3209 | "format": "time_series", 3210 | "group": [], 3211 | "metricColumn": "none", 3212 | "rawQuery": true, 3213 | "rawSql": "SELECT\n started_at AS \"time\",\n completed_at - started_at as duration\nFROM iterations\nWHERE\n $__timeFilter(started_at)\nORDER BY started_at", 3214 | "refId": "A", 3215 | "select": [ 3216 | [ 3217 | { 3218 | "params": [ 3219 | "id" 3220 | ], 3221 | "type": "column" 3222 | } 3223 | ] 3224 | ], 3225 | "table": "iterations", 3226 | "timeColumn": "started_at", 3227 | "timeColumnType": "timestamp", 3228 | "where": [ 3229 | { 3230 | "name": "$__timeFilter", 3231 | "params": [], 3232 | "type": "macro" 3233 | } 3234 | ] 3235 | } 3236 | ], 3237 | "title": "Scraper Duration", 3238 | "type": "barchart" 3239 | } 3240 | ], 3241 | "title": "System", 3242 | "type": "row" 3243 | } 3244 | ], 3245 | "refresh": false, 3246 | "schemaVersion": 37, 3247 | "style": "dark", 3248 | "tags": [], 3249 | "templating": { 3250 | "list": [ 3251 | { 3252 | "current": { 3253 | "selected": true, 3254 | "text": "+08:00", 3255 | "value": "+08:00" 3256 | }, 3257 | "description": "Affects how Trend panels calculate dates", 3258 | "hide": 0, 3259 | "includeAll": false, 3260 | "label": "Time Zone", 3261 | "multi": false, 3262 | "name": "TZ", 3263 | "options": [ 3264 | { 3265 | "selected": false, 3266 | "text": "UTC", 3267 | "value": "UTC" 3268 | }, 3269 | { 3270 | "selected": true, 3271 | "text": "+08:00", 3272 | "value": "+08:00" 3273 | } 3274 | ], 3275 | "query": "UTC, +08:00", 3276 | "queryValue": "", 3277 | "skipUrlSync": false, 3278 | "type": "custom" 3279 | }, 3280 | { 3281 | "current": { 3282 | "selected": true, 3283 | "text": [ 3284 | "7", 3285 | "14" 3286 | ], 3287 | "value": [ 3288 | "7", 3289 | "14" 3290 | ] 3291 | }, 3292 | "hide": 0, 3293 | "includeAll": false, 3294 | "label": "Trend days", 3295 | "multi": true, 3296 | "name": "TREND", 3297 | "options": [ 3298 | { 3299 | "selected": true, 3300 | "text": "7", 3301 | "value": "7" 3302 | }, 3303 | { 3304 | "selected": true, 3305 | "text": "14", 3306 | "value": "14" 3307 | }, 3308 | { 3309 | "selected": false, 3310 | "text": "30", 3311 | "value": "30" 3312 | } 3313 | ], 3314 | "query": "7, 14, 30", 3315 | "queryValue": "", 3316 | "skipUrlSync": false, 3317 | "type": "custom" 3318 | } 3319 | ] 3320 | }, 3321 | "time": { 3322 | "from": "now-24h", 3323 | "to": "now" 3324 | }, 3325 | "timepicker": { 3326 | "refresh_intervals": [ 3327 | "5m", 3328 | "15m", 3329 | "30m", 3330 | "1h", 3331 | "2h", 3332 | "1d" 3333 | ] 3334 | }, 3335 | "timezone": "", 3336 | "title": "Twitter Statistics", 3337 | "uid": "3H6MYG4Vz", 3338 | "version": 55, 3339 | "weekStart": "" 3340 | } 3341 | -------------------------------------------------------------------------------- /scraper/scraper.go: -------------------------------------------------------------------------------- 1 | package scraper 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/disksing/twiyou/store" 9 | "github.com/disksing/twiyou/twitter" 10 | ) 11 | 12 | const ( 13 | LErr = "error" 14 | LInfo = "info" 15 | ) 16 | 17 | const ( 18 | InitialState = "initial" 19 | FetchFollowersState = "fetch_followers" 20 | FetchFollowingState = "fetch_following" 21 | PullUsersState = "pull_users" 22 | SumEventsState = "sum_events" 23 | StashUsersState = "stash_users" 24 | CleanUpState = "cleanup" 25 | CompleteState = "complete" 26 | ) 27 | 28 | type Scraper struct { 29 | db *store.DB 30 | self *twitter.User 31 | } 32 | 33 | func NewScraper() (*Scraper, error) { 34 | db, err := store.NewDB() 35 | if err != nil { 36 | return nil, err 37 | } 38 | self, err := twitter.LoadSelf() 39 | if err != nil { 40 | return nil, err 41 | } 42 | return &Scraper{ 43 | db: db, 44 | self: self, 45 | }, nil 46 | } 47 | 48 | func (s *Scraper) Close() { 49 | s.db.Close() 50 | } 51 | 52 | func (s *Scraper) Run() error { 53 | err := s.saveStats() 54 | if err != nil { 55 | return err 56 | } 57 | 58 | it, err := s.db.LoadLastIteration() 59 | if err != nil { 60 | s.db.Log(LErr, fmt.Sprintf("failed to load last iteration: %v", err)) 61 | return err 62 | } 63 | 64 | for { 65 | switch it.State { 66 | case InitialState: 67 | it.State = FetchFollowersState 68 | 69 | case FetchFollowersState: 70 | if it.NextToken == "EOF" { 71 | it.State, it.NextToken = FetchFollowingState, "" 72 | it.CompleteFetchFollowersAt = sql.NullTime{Time: time.Now(), Valid: true} 73 | break 74 | } 75 | 76 | users, token, err := twitter.ListFriends(s.self.ID, "followers", it.NextToken) 77 | if err != nil { 78 | s.db.Log(LErr, fmt.Sprintf("failed to fetch followers: %v", err)) 79 | return err 80 | } 81 | err = s.db.UpdateUserInfo("is_follower", users) 82 | if err != nil { 83 | s.db.Log(LErr, fmt.Sprintf("failed to update followers: %v", err)) 84 | return err 85 | } 86 | if token == "" { 87 | token = "EOF" 88 | } 89 | it.NextToken = token 90 | 91 | case FetchFollowingState: 92 | if it.NextToken == "EOF" { 93 | it.State, it.NextToken = PullUsersState, "" 94 | it.CompleteFetchFollowingAt = sql.NullTime{Time: time.Now(), Valid: true} 95 | if err = s.db.SaveIteration(it); err != nil { 96 | return err 97 | } 98 | break 99 | } 100 | users, token, err := twitter.ListFriends(s.self.ID, "following", it.NextToken) 101 | if err != nil { 102 | s.db.Log(LErr, fmt.Sprintf("failed to fetch following: %v", err)) 103 | return err 104 | } 105 | err = s.db.UpdateUserInfo("is_following", users) 106 | if err != nil { 107 | s.db.Log(LErr, fmt.Sprintf("failed to update following: %v", err)) 108 | return err 109 | } 110 | if token == "" { 111 | token = "EOF" 112 | } 113 | it.NextToken = token 114 | 115 | case PullUsersState: 116 | userIDs, err := s.db.SelectUsersForPull(it.StartedAt, 100) 117 | if err != nil { 118 | s.db.Log(LErr, fmt.Sprintf("failed to pull users: %v", err)) 119 | return err 120 | } 121 | if len(userIDs) > 0 { 122 | users, err := twitter.ListUsers(userIDs) 123 | if err != nil { 124 | s.db.Log(LErr, fmt.Sprintf("failed to list users: %v", err)) 125 | return err 126 | } 127 | err = s.db.UpdateUserInfo("", users) 128 | if err != nil { 129 | s.db.Log(LErr, fmt.Sprintf("failed to update users: %v", err)) 130 | return err 131 | } 132 | err = s.db.UpdateUserUpdateTime(userIDs) 133 | if err != nil { 134 | s.db.Log(LErr, fmt.Sprintf("failed to update user update time: %v", err)) 135 | return err 136 | } 137 | } 138 | it.State = SumEventsState 139 | it.CompletePullUsersAt = sql.NullTime{Time: time.Now(), Valid: true} 140 | 141 | case SumEventsState: 142 | err = s.db.SumUserEvents(it.StartedAt) 143 | if err != nil { 144 | s.db.Log(LErr, fmt.Sprintf("failed to sum events: %v", err)) 145 | return err 146 | } 147 | it.State = StashUsersState 148 | it.CompleteSumEventsAt = sql.NullTime{Time: time.Now(), Valid: true} 149 | 150 | case StashUsersState: 151 | err = s.db.StashUsers() 152 | if err != nil { 153 | s.db.Log(LErr, fmt.Sprintf("failed to stash users: %v", err)) 154 | return err 155 | } 156 | it.State = CleanUpState 157 | it.CompleteStashUsersAt = sql.NullTime{Time: time.Now(), Valid: true} 158 | 159 | case CleanUpState: 160 | err = s.db.CleanUpCache() 161 | if err != nil { 162 | s.db.Log(LErr, fmt.Sprintf("failed to clean up: %v", err)) 163 | return err 164 | } 165 | it.State = CompleteState 166 | it.CompletedAt = sql.NullTime{Time: time.Now(), Valid: true} 167 | if err = s.db.SaveIteration(it); err != nil { 168 | return err 169 | } 170 | s.db.Log(LInfo, fmt.Sprintf("iteration completed: %v", it.ID)) 171 | return nil 172 | 173 | case CompleteState: 174 | it, err = s.db.CreateIteration() 175 | if err != nil { 176 | s.db.Log(LErr, fmt.Sprintf("failed to create iteration: %v", err)) 177 | return err 178 | } 179 | s.db.Log(LInfo, fmt.Sprintf("new iteration: %v", it.ID)) 180 | continue 181 | } 182 | 183 | if err = s.db.SaveIteration(it); err != nil { 184 | return err 185 | } 186 | } 187 | } 188 | 189 | func (s *Scraper) saveIteration(it *store.Iteration) error { 190 | err := s.db.SaveIteration(it) 191 | if err != nil { 192 | s.db.Log(LErr, fmt.Sprintf("failed to save iteration: %v", err)) 193 | return err 194 | } 195 | return nil 196 | } 197 | 198 | func (s *Scraper) saveStats() error { 199 | err := s.db.InsertStats(s.self) 200 | if err != nil { 201 | s.db.Log(LErr, fmt.Sprintf("failed to save stats: %v", err)) 202 | return err 203 | } 204 | s.db.Log(LInfo, fmt.Sprintf("saved stats %v", s.self.Metrics())) 205 | return nil 206 | } 207 | -------------------------------------------------------------------------------- /store/batch.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | func (db *DB) batchInsert(template string, batchSize int, fieldNum int, args []any) error { 10 | var stmtN *sql.Stmt 11 | for len(args) > 0 { 12 | rowNum := len(args) / fieldNum 13 | if rowNum >= batchSize { 14 | if stmtN == nil { 15 | var err error 16 | stmtN, err = db.prepareStmtN(template, fieldNum, batchSize) 17 | if err != nil { 18 | return err 19 | } 20 | } 21 | _, err := stmtN.Exec(args[:batchSize*fieldNum]...) 22 | if err != nil { 23 | return err 24 | } 25 | args = args[batchSize*fieldNum:] 26 | continue 27 | } 28 | stmt, err := db.prepareStmtN(template, fieldNum, rowNum) 29 | if err != nil { 30 | return err 31 | } 32 | _, err = stmt.Exec(args...) 33 | if err != nil { 34 | return err 35 | } 36 | break 37 | } 38 | return nil 39 | } 40 | 41 | func (db *DB) prepareStmtN(template string, fieldNum, rowNum int) (*sql.Stmt, error) { 42 | fields := strings.Repeat("?,", fieldNum) 43 | fields = fields[:len(fields)-1] 44 | rows := strings.Repeat("("+fields+"),", rowNum) 45 | rows = rows[:len(rows)-1] 46 | return db.db.Prepare(fmt.Sprintf(template, rows)) 47 | } 48 | -------------------------------------------------------------------------------- /store/create_tables.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | id varchar(32) NOT NULL, 3 | name varchar(128) NOT NULL, 4 | user_name varchar(255) NOT NULL, 5 | profile_image varchar(255), 6 | 7 | follower_count INT NOT NULL, 8 | following_count INT NOT NULL, 9 | tweet_count INT NOT NULL DEFAULT 0, 10 | listed_count INT NOT NULL, 11 | 12 | is_following BOOLEAN NOT NULL DEFAULT FALSE, 13 | is_follower BOOLEAN NOT NULL DEFAULT FALSE, 14 | 15 | is_following1 BOOLEAN NOT NULL DEFAULT FALSE, 16 | is_follower1 BOOLEAN NOT NULL DEFAULT FALSE, 17 | tweet_count1 INT NOT NULL DEFAULT 0, 18 | 19 | last_active TIMESTAMP, 20 | updated_at TIMESTAMP, 21 | PRIMARY KEY (id), 22 | INDEX (last_active), 23 | INDEX (updated_at) 24 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 25 | 26 | CREATE TABLE stats ( 27 | updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP PRIMARY KEY, 28 | id varchar(32) NOT NULL, 29 | name varchar(128) NOT NULL, 30 | user_name varchar(255) NOT NULL, 31 | profile_image varchar(255), 32 | follower_count INT NOT NULL, 33 | following_count INT NOT NULL, 34 | tweet_count INT NOT NULL, 35 | listed_count INT NOT NULL 36 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 37 | 38 | CREATE TABLE iterations ( 39 | id INT AUTO_INCREMENT PRIMARY KEY, 40 | state varchar(255) NOT NULL, 41 | started_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 42 | complete_fetch_followers_at TIMESTAMP, 43 | complete_fetch_following_at TIMESTAMP, 44 | complete_pull_users_at TIMESTAMP, 45 | complete_sum_events_at TIMESTAMP, 46 | complete_stash_users_at TIMESTAMP, 47 | completed_at TIMESTAMP, 48 | next_token varchar(255) NOT NULL DEFAULT "" 49 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 50 | 51 | INSERT INTO iterations (state) VALUES ('initial'); 52 | 53 | CREATE TABLE events ( 54 | user_id varchar(32) NOT NULL, 55 | created_at TIMESTAMP, 56 | event varchar(32) NOT NULL, 57 | INDEX (created_at) 58 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 59 | 60 | CREATE TABLE logs ( 61 | created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 62 | level varchar(32) NOT NULL, 63 | message varchar(255) NOT NULL, 64 | INDEX (created_at) 65 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 66 | 67 | CREATE TABLE system ( 68 | `key` varchar(255) NOT NULL PRIMARY KEY, 69 | value varchar(255) NOT NULL 70 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 71 | 72 | INSERT INTO system (`key`, value) VALUES ('dbver', '1'); -------------------------------------------------------------------------------- /store/db.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | _ "embed" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | 11 | _ "github.com/go-sql-driver/mysql" 12 | "github.com/jmoiron/sqlx" 13 | ) 14 | 15 | //go:embed create_tables.sql 16 | var createTable string 17 | 18 | var ( 19 | DB_HOST string 20 | DB_PORT string 21 | DB_USER string 22 | DB_NAME string 23 | DB_PASSWD string 24 | ) 25 | 26 | func init() { 27 | DB_HOST = os.Getenv("DB_HOST") 28 | DB_PORT = os.Getenv("DB_PORT") 29 | DB_USER = os.Getenv("DB_USER") 30 | DB_NAME = os.Getenv("DB_NAME") 31 | DB_PASSWD = os.Getenv("DB_PASSWD") 32 | } 33 | 34 | type DB struct { 35 | db *sqlx.DB 36 | } 37 | 38 | func NewDB() (*DB, error) { 39 | if DB_HOST == "" || DB_PORT == "" || DB_USER == "" || DB_NAME == "" { 40 | return nil, errors.New("DB environment variables not set") 41 | } 42 | d, err := sqlx.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true&tls=true", DB_USER, DB_PASSWD, DB_HOST, DB_PORT, DB_NAME)) 43 | if err != nil { 44 | return nil, err 45 | } 46 | db := &DB{db: d} 47 | if err := db.init(); err != nil { 48 | return nil, err 49 | } 50 | return db, nil 51 | } 52 | 53 | func (db *DB) init() error { 54 | _, err := db.db.Exec(`SET GLOBAL tidb_multi_statement_mode='ON'`) 55 | if err != nil { 56 | return err 57 | } 58 | initialized, err := db.initialized() 59 | if err != nil { 60 | return err 61 | } 62 | if !initialized { 63 | if err := db.createTables(); err != nil { 64 | return err 65 | } 66 | } 67 | return nil 68 | } 69 | 70 | func (db *DB) initialized() (bool, error) { 71 | var tables []string 72 | if err := db.db.Select(&tables, "SHOW TABLES"); err != nil { 73 | return false, err 74 | } 75 | for _, t := range tables { 76 | if strings.EqualFold(t, "system") { 77 | return true, nil 78 | } 79 | } 80 | return false, nil 81 | } 82 | 83 | func (db *DB) createTables() error { 84 | _, err := db.db.Exec(createTable) 85 | return err 86 | } 87 | 88 | func (db *DB) Close() { 89 | db.db.Close() 90 | } 91 | 92 | func (db *DB) Log(level, message string) { 93 | log.Println(level, message) 94 | db.db.Exec("INSERT INTO logs (level, message) VALUES (?, ?)", level, message) 95 | } 96 | -------------------------------------------------------------------------------- /store/iteration.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "database/sql" 5 | "time" 6 | ) 7 | 8 | type Iteration struct { 9 | ID int64 `db:"id"` 10 | State string `db:"state"` 11 | StartedAt time.Time `db:"started_at"` 12 | CompleteFetchFollowersAt sql.NullTime `db:"complete_fetch_followers_at"` 13 | CompleteFetchFollowingAt sql.NullTime `db:"complete_fetch_following_at"` 14 | CompletePullUsersAt sql.NullTime `db:"complete_pull_users_at"` 15 | CompleteSumEventsAt sql.NullTime `db:"complete_sum_events_at"` 16 | CompleteStashUsersAt sql.NullTime `db:"complete_stash_users_at"` 17 | CompletedAt sql.NullTime `db:"completed_at"` 18 | NextToken string `db:"next_token"` 19 | } 20 | 21 | func (db *DB) LoadLastIteration() (*Iteration, error) { 22 | var it Iteration 23 | err := db.db.Get(&it, "SELECT * FROM iterations ORDER BY id DESC LIMIT 1") 24 | if err != nil { 25 | return nil, err 26 | } 27 | return &it, nil 28 | } 29 | 30 | func (db *DB) SaveIteration(it *Iteration) error { 31 | _, err := db.db.NamedExec(` 32 | UPDATE iterations SET 33 | state = :state, 34 | started_at = :started_at, 35 | complete_fetch_followers_at = :complete_fetch_followers_at, 36 | complete_fetch_following_at = :complete_fetch_following_at, 37 | complete_pull_users_at = :complete_pull_users_at, 38 | complete_sum_events_at = :complete_sum_events_at, 39 | complete_stash_users_at = :complete_stash_users_at, 40 | completed_at = :completed_at 41 | WHERE id = :id 42 | `, it) 43 | return err 44 | } 45 | 46 | func (db *DB) CreateIteration() (*Iteration, error) { 47 | _, err := db.db.Exec(`INSERT INTO iterations (state) VALUES (?)`, "initial") 48 | if err != nil { 49 | return nil, err 50 | } 51 | return db.LoadLastIteration() 52 | } 53 | -------------------------------------------------------------------------------- /store/user.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/disksing/twiyou/twitter" 8 | "github.com/jmoiron/sqlx" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | func (db *DB) InsertStats(self *twitter.User) error { 13 | _, err := db.db.Exec(` 14 | INSERT INTO stats ( 15 | id, name, user_name, profile_image, 16 | follower_count, following_count, tweet_count, listed_count) 17 | VALUES (?, ?, ?, ?, ?, ?, ?, ?) 18 | `, self.ID, self.Name, self.UserName, self.ProfileImageURL, 19 | self.PublicMetrics.FollowersCount, self.PublicMetrics.FollowingCount, self.PublicMetrics.TweetCount, self.PublicMetrics.ListedCount) 20 | if err != nil { 21 | return errors.New("update self: " + err.Error()) 22 | } 23 | return nil 24 | } 25 | 26 | func (db *DB) UpdateUserInfo(relation string, users []twitter.User) error { 27 | if len(users) == 0 { 28 | return nil 29 | } 30 | 31 | sql := ` 32 | INSERT INTO users ( 33 | id, name, user_name, profile_image, 34 | follower_count, following_count, tweet_count1, listed_count, 35 | updated_at %s) 36 | VALUES %%s 37 | ON DUPLICATE KEY UPDATE 38 | name = VALUES(name), user_name = VALUES(user_name), profile_image = VALUES(profile_image), 39 | follower_count = VALUES(follower_count), following_count = VALUES(following_count), tweet_count1 = VALUES(tweet_count1), listed_count = VALUES(listed_count), 40 | updated_at = VALUES(updated_at) %s 41 | ` 42 | 43 | var param1, param2 string 44 | if relation != "" { 45 | param1 = fmt.Sprintf(",%s1", relation) // `,is_following1` / `,is_follower1` 46 | param2 = fmt.Sprintf(",%[1]s1=VALUES(%[1]s1)", relation) // `,is_following1=VALUES(is_following1)` / `,is_follower1=VALUES(is_follower1)` 47 | } 48 | sql = fmt.Sprintf(sql, param1, param2) 49 | 50 | var args []any 51 | for _, u := range users { 52 | args = append(args, 53 | u.ID, u.Name, u.UserName, u.ProfileImageURL, 54 | u.PublicMetrics.FollowersCount, u.PublicMetrics.FollowingCount, u.PublicMetrics.TweetCount, u.PublicMetrics.ListedCount, 55 | time.Now()) 56 | if relation != "" { 57 | args = append(args, true) 58 | } 59 | } 60 | err := db.batchInsert(sql, 200, len(args)/len(users), args) 61 | if err != nil { 62 | return errors.New("update users: " + err.Error()) 63 | } 64 | return nil 65 | } 66 | 67 | func (db *DB) SelectUsersForPull(t time.Time, limit int) ([]string, error) { 68 | var ids []string 69 | err := db.db.Select(&ids, `SELECT id FROM users WHERE updated_at < ? ORDER BY updated_at LIMIT ?`, t, limit) 70 | return ids, err 71 | } 72 | 73 | func (db *DB) UpdateUserUpdateTime(users []string) error { 74 | if len(users) == 0 { 75 | return nil 76 | } 77 | query, args, err := sqlx.In(`UPDATE users SET updated_at = ? WHERE id IN (?)`, time.Now(), users) 78 | if err != nil { 79 | return err 80 | } 81 | _, err = db.db.Exec(query, args...) 82 | return err 83 | } 84 | 85 | func (db *DB) SumUserEvents(t time.Time) error { 86 | sqls := []string{ 87 | "DELETE FROM events WHERE created_at = ?", 88 | "UPDATE users SET last_active = ? WHERE tweet_count != tweet_count1", 89 | "INSERT INTO events SELECT id, ?, 'new_follower' FROM users WHERE is_follower = FALSE and is_follower1 = TRUE", 90 | "INSERT INTO events SELECT id, ?, 'lost_follower' FROM users WHERE is_follower = TRUE and is_follower1 = FALSE", 91 | "INSERT INTO events SELECT id, ?, 'new_following' FROM users WHERE is_following = FALSE and is_following1 = TRUE", 92 | "INSERT INTO events SELECT id, ?, 'cancel_following' FROM users WHERE is_following = TRUE and is_following1 = FALSE", 93 | } 94 | for _, sql := range sqls { 95 | _, err := db.db.Exec(sql, t) 96 | if err != nil { 97 | return errors.Wrap(err, "sum events failed") 98 | } 99 | } 100 | return nil 101 | } 102 | 103 | func (db *DB) StashUsers() error { 104 | _, err := db.db.Exec(` 105 | UPDATE users SET 106 | tweet_count = tweet_count1, 107 | is_following = is_following1, 108 | is_follower = is_follower1 109 | `) 110 | if err != nil { 111 | return errors.New("failed to stash user relationships:" + err.Error()) 112 | } 113 | return nil 114 | } 115 | 116 | func (db *DB) CleanUpCache() error { 117 | _, err := db.db.Exec(` 118 | UPDATE users SET 119 | is_following1 = FALSE, 120 | is_follower1 = FALSE 121 | `) 122 | if err != nil { 123 | return errors.New("failed to clean up cache:" + err.Error()) 124 | } 125 | return nil 126 | } 127 | -------------------------------------------------------------------------------- /twitter/twitter.go: -------------------------------------------------------------------------------- 1 | package twitter 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "os" 9 | "strings" 10 | ) 11 | 12 | var ( 13 | TWITTER_BEARER_TOKEN string 14 | TWITTER_USER_NAME string 15 | ) 16 | 17 | func init() { 18 | TWITTER_BEARER_TOKEN = os.Getenv("TWITTER_BEARER_TOKEN") 19 | TWITTER_USER_NAME = os.Getenv("TWITTER_USER_NAME") 20 | } 21 | 22 | type User struct { 23 | ID string `json:"id"` 24 | Name string `json:"name"` 25 | UserName string `json:"username"` 26 | ProfileImageURL string `json:"profile_image_url"` 27 | PublicMetrics struct { 28 | FollowersCount int `json:"followers_count"` 29 | FollowingCount int `json:"following_count"` 30 | TweetCount int `json:"tweet_count"` 31 | ListedCount int `json:"listed_count"` 32 | } `json:"public_metrics"` 33 | } 34 | 35 | func (u *User) Metrics() string { 36 | return fmt.Sprintf("{followers=%v, following=%v, tweets=%v, listed=%v}", u.PublicMetrics.FollowersCount, u.PublicMetrics.FollowingCount, u.PublicMetrics.TweetCount, u.PublicMetrics.ListedCount) 37 | } 38 | 39 | func LoadSelf() (*User, error) { 40 | url := fmt.Sprintf("https://api.twitter.com/2/users/by/username/%s?user.fields=id,name,profile_image_url,public_metrics,username", TWITTER_USER_NAME) 41 | req, _ := http.NewRequest("GET", url, nil) 42 | req.Header.Add("Authorization", "Bearer "+TWITTER_BEARER_TOKEN) 43 | res, err := http.DefaultClient.Do(req) 44 | if err != nil { 45 | return nil, err 46 | } 47 | defer res.Body.Close() 48 | type userResult struct { 49 | Data User `json:"data"` 50 | } 51 | var result userResult 52 | if res.StatusCode != http.StatusOK { 53 | b, _ := io.ReadAll(res.Body) 54 | return nil, fmt.Errorf("status code: %d, body: %s", res.StatusCode, string(b)) 55 | } 56 | err = json.NewDecoder(res.Body).Decode(&result) 57 | if err != nil { 58 | return nil, err 59 | } 60 | return &result.Data, nil 61 | } 62 | 63 | // type = followers or following 64 | func ListFriends(selfID string, typ string, token string) ([]User, string, error) { 65 | url := fmt.Sprintf("https://api.twitter.com/2/users/%s/%s?max_results=1000&user.fields=id,name,profile_image_url,public_metrics,username", selfID, typ) 66 | req, _ := http.NewRequest("GET", url, nil) 67 | if token != "" { 68 | p := req.URL.Query() 69 | p.Add("pagination_token", token) 70 | req.URL.RawQuery = p.Encode() 71 | } 72 | req.Header.Add("Authorization", "Bearer "+TWITTER_BEARER_TOKEN) 73 | 74 | res, err := http.DefaultClient.Do(req) 75 | if err != nil { 76 | return nil, "", err 77 | } 78 | defer res.Body.Close() 79 | 80 | type listResult struct { 81 | Data []User `json:"data"` 82 | Meta struct { 83 | NextToken string `json:"next_token"` 84 | } `json:"meta"` 85 | } 86 | var result listResult 87 | if res.StatusCode != http.StatusOK { 88 | b, _ := io.ReadAll(res.Body) 89 | return nil, "", fmt.Errorf("status code: %d, body: %s", res.StatusCode, string(b)) 90 | } 91 | err = json.NewDecoder(res.Body).Decode(&result) 92 | if err != nil { 93 | return nil, "", err 94 | } 95 | return result.Data, result.Meta.NextToken, nil 96 | } 97 | 98 | func ListUsers(ids []string) ([]User, error) { 99 | url := "https://api.twitter.com/2/users?user.fields=id,name,profile_image_url,public_metrics,username&ids=" + strings.Join(ids, ",") 100 | req, _ := http.NewRequest("GET", url, nil) 101 | req.Header.Add("Authorization", "Bearer "+TWITTER_BEARER_TOKEN) 102 | 103 | res, err := http.DefaultClient.Do(req) 104 | if err != nil { 105 | return nil, err 106 | } 107 | defer res.Body.Close() 108 | 109 | type listResult struct { 110 | Data []User `json:"data"` 111 | } 112 | var result listResult 113 | if res.StatusCode != http.StatusOK { 114 | b, _ := io.ReadAll(res.Body) 115 | return nil, fmt.Errorf("status code: %d, body: %s", res.StatusCode, string(b)) 116 | } 117 | err = json.NewDecoder(res.Body).Decode(&result) 118 | if err != nil { 119 | return nil, err 120 | } 121 | return result.Data, nil 122 | } 123 | --------------------------------------------------------------------------------