├── settings.gradle.kts
├── .gitignore
├── gradle
    └── wrapper
    │   ├── gradle-wrapper.jar
    │   └── gradle-wrapper.properties
├── src
    ├── jvmTest
    │   ├── kotlin
    │   │   └── blue
    │   │   │   └── starry
    │   │   │       └── tweetstorm
    │   │   │           ├── Test.kt
    │   │   │           └── TestWebUI.kt
    │   └── resources
    │   │   └── logback-test.xml
    └── jvmMain
    │   ├── kotlin
    │       └── blue
    │       │   └── starry
    │       │       └── tweetstorm
    │       │           ├── task
    │       │               ├── data
    │       │               │   ├── ProduceData.kt
    │       │               │   ├── JsonData.kt
    │       │               │   ├── JsonObjectData.kt
    │       │               │   ├── Heartbeat.kt
    │       │               │   └── JsonModelData.kt
    │       │               ├── producer
    │       │               │   ├── Heartbeat.kt
    │       │               │   ├── Friends.kt
    │       │               │   ├── SampleStream.kt
    │       │               │   ├── FilterStream.kt
    │       │               │   ├── DirectMessage.kt
    │       │               │   └── Timeline.kt
    │       │               ├── Task.kt
    │       │               └── regular
    │       │               │   └── SyncList.kt
    │       │           ├── Main.kt
    │       │           ├── session
    │       │               ├── Stream.kt
    │       │               ├── DemoStream.kt
    │       │               ├── AuthenticatedStream.kt
    │       │               ├── PreAuthenticatedStream.kt
    │       │               └── StreamContent.kt
    │       │           ├── App.kt
    │       │           ├── CLI.kt
    │       │           ├── Logging.kt
    │       │           ├── Auth.kt
    │       │           ├── Layout.kt
    │       │           ├── Config.kt
    │       │           ├── TaskManager.kt
    │       │           └── Routing.kt
    │   └── resources
    │       └── logback.xml
├── gradle.properties
├── .github
    └── workflows
    │   ├── check.yml
    │   └── docker.yml
├── Dockerfile
├── LICENSE
├── README_EN.md
├── gradlew.bat
├── README.md
└── gradlew


/settings.gradle.kts:
--------------------------------------------------------------------------------
1 | rootProject.name = "tweetstorm"
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | .gradle/
3 | build/
4 | 
5 | config.json
6 | 


--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SlashNephy/Tweetstorm/HEAD/gradle/wrapper/gradle-wrapper.jar


--------------------------------------------------------------------------------
/src/jvmTest/kotlin/blue/starry/tweetstorm/Test.kt:
--------------------------------------------------------------------------------
1 | package blue.starry.tweetstorm
2 | 
3 | import kotlinx.coroutines.ObsoleteCoroutinesApi
4 | 
5 | @ObsoleteCoroutinesApi
6 | fun main(args: Array<String>) {
7 |     blue.starry.tweetstorm.main(args)
8 | }
9 | 


--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.properties:
--------------------------------------------------------------------------------
1 | distributionBase=GRADLE_USER_HOME
2 | distributionPath=wrapper/dists
3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip
4 | zipStoreBase=GRADLE_USER_HOME
5 | zipStorePath=wrapper/dists
6 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/data/ProduceData.kt:
--------------------------------------------------------------------------------
1 | package blue.starry.tweetstorm.task.data
2 | 
3 | import blue.starry.tweetstorm.session.StreamContent
4 | 
5 | interface ProduceData<T> {
6 |     val data: T
7 |     suspend fun emit(handler: StreamContent.Handler): Boolean
8 | }
9 | 


--------------------------------------------------------------------------------
/gradle.properties:
--------------------------------------------------------------------------------
 1 | kotlin.code.style=official
 2 | 
 3 | kotlin.mpp.stability.nowarn=true
 4 | 
 5 | kotlin.mpp.enableGranularSourceSetsMetadata=true
 6 | kotlin.native.ignoreDisabledTargets=true
 7 | 
 8 | # workaround for https://github.com/gradle/gradle/issues/11412
 9 | systemProp.org.gradle.internal.publish.checksums.insecure=true
10 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/data/JsonData.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.data
 2 | 
 3 | import blue.starry.tweetstorm.session.StreamContent
 4 | 
 5 | class JsonData(override vararg val data: Pair<String, Any?>): ProduceData<Array<out Pair<String, Any?>>> {
 6 |     override suspend fun emit(handler: StreamContent.Handler): Boolean {
 7 |         return handler.emit(*data)
 8 |     }
 9 | }
10 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/data/JsonObjectData.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.data
 2 | 
 3 | import blue.starry.jsonkt.JsonObject
 4 | import blue.starry.tweetstorm.session.StreamContent
 5 | 
 6 | class JsonObjectData(override val data: JsonObject): ProduceData<JsonObject> {
 7 |     override suspend fun emit(handler: StreamContent.Handler): Boolean {
 8 |         return handler.emit(data)
 9 |     }
10 | }
11 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/data/Heartbeat.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.data
 2 | 
 3 | import blue.starry.tweetstorm.session.StreamContent
 4 | 
 5 | class Heartbeat: ProduceData<Unit> {
 6 |     override val data: Unit
 7 |         get() = throw IllegalArgumentException()
 8 | 
 9 |     override suspend fun emit(handler: StreamContent.Handler): Boolean {
10 |         return handler.heartbeat()
11 |     }
12 | }
13 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/data/JsonModelData.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.data
 2 | 
 3 | import blue.starry.jsonkt.delegation.JsonModel
 4 | import blue.starry.tweetstorm.session.StreamContent
 5 | 
 6 | class JsonModelData(override val data: JsonModel): ProduceData<JsonModel> {
 7 |     override suspend fun emit(handler: StreamContent.Handler): Boolean {
 8 |         return handler.emit(data)
 9 |     }
10 | }
11 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/Main.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm
 2 | 
 3 | import io.ktor.application.Application
 4 | import io.ktor.server.cio.CIO
 5 | import io.ktor.server.engine.embeddedServer
 6 | 
 7 | lateinit var config: Config
 8 |     private set
 9 | 
10 | fun main(args: Array<String>) {
11 |     val cliArguments = parseCommandLine(args)
12 |     config = Config.load(cliArguments.configPath)
13 | 
14 |     embeddedServer(CIO, host = config.wui.host, port = config.wui.port, module = Application::module).start(wait = true)
15 | }
16 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/session/Stream.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.session
 2 | 
 3 | import io.ktor.request.ApplicationRequest
 4 | import io.ktor.utils.io.ByteWriteChannel
 5 | import kotlinx.coroutines.*
 6 | import java.io.Closeable
 7 | 
 8 | abstract class Stream<T>(channel: ByteWriteChannel, val request: ApplicationRequest): Closeable {
 9 |     val job = Job()
10 |     val handler = StreamContent.Handler(channel, request)
11 | 
12 |     abstract suspend fun await(): T
13 | 
14 |     override fun close() {
15 |         runBlocking(Dispatchers.Default) {
16 |             job.cancelChildren()
17 |             job.cancelAndJoin()
18 |         }
19 |     }
20 | }
21 | 


--------------------------------------------------------------------------------
/src/jvmTest/resources/logback-test.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8" ?>
 2 | <configuration>
 3 |     <statusListener class="ch.qos.logback.core.status.NopStatusListener" />
 4 | 
 5 |     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 6 |         <withJansi>true</withJansi>
 7 |         <encoder>
 8 |             <pattern>[%highlight(%-5level)] [%d{HH:mm:ss.SSS}] [%green(%logger) / %thread] %m%n</pattern>
 9 |         </encoder>
10 |     </appender>
11 |     
12 |     <logger name="Tweetstorm" level="ALL" />
13 |     <logger name="Penicillin" level="DEBUG" />
14 |     <root level="INFO">
15 |         <appender-ref ref="STDOUT" />
16 |     </root>
17 | </configuration>
18 | 


--------------------------------------------------------------------------------
/src/jvmTest/kotlin/blue/starry/tweetstorm/TestWebUI.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm
 2 | 
 3 | import io.ktor.application.install
 4 | import io.ktor.features.CallLogging
 5 | import io.ktor.routing.Routing
 6 | import io.ktor.server.cio.CIO
 7 | import io.ktor.server.engine.embeddedServer
 8 | 
 9 | private const val host = "localhost"
10 | private const val port = 8080
11 | 
12 | class TestWebUI {
13 |     companion object {
14 |         @JvmStatic
15 |         fun main(args: Array<String>) {
16 |             embeddedServer(CIO, host = host, port = port) {
17 |                 install(CallLogging)
18 |                 install(Routing) {
19 |                     getTop()
20 |                 }
21 |             }.start(wait = true)
22 |         }
23 |     }
24 | }
25 | 


--------------------------------------------------------------------------------
/src/jvmMain/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8" ?>
 2 | <configuration>
 3 |     <statusListener class="ch.qos.logback.core.status.NopStatusListener" />
 4 |     <timestamp key="timestamp" datePattern="yyyyMMdd_HHmmss" />
 5 | 
 6 |     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 7 |         <withJansi>true</withJansi>
 8 |         <encoder>
 9 |             <pattern>[%highlight(%-5level)] [%d{HH:mm:ss.SSS}] [%green(%logger)] %m%n</pattern>
10 |         </encoder>
11 |     </appender>
12 | 
13 |     <logger name="ktor.application" level="WARN" />
14 |     <logger name="Tweetstorm" level="INFO" />
15 |     <logger name="Penicillin" level="INFO" />
16 | 
17 |     <root level="INFO">
18 |         <appender-ref ref="STDOUT" />
19 |     </root>
20 | </configuration>
21 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/producer/Heartbeat.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.producer
 2 | 
 3 | import blue.starry.tweetstorm.Config
 4 | import blue.starry.tweetstorm.task.ProduceTask
 5 | import blue.starry.tweetstorm.task.data.Heartbeat
 6 | import kotlinx.coroutines.*
 7 | import kotlinx.coroutines.channels.produce
 8 | import kotlin.coroutines.CoroutineContext
 9 | 
10 | class Heartbeat(account: Config.Account): ProduceTask<Heartbeat>(account) {
11 |     @ExperimentalCoroutinesApi
12 |     override fun channel(context: CoroutineContext, parent: Job) = GlobalScope.produce(context + parent) {
13 |         while (isActive) {
14 |             try {
15 |                 delay(10000)
16 |                 send(Heartbeat())
17 |             } catch (e: CancellationException) {
18 |                 break
19 |             }
20 |         }
21 |     }
22 | }
23 | 


--------------------------------------------------------------------------------
/.github/workflows/check.yml:
--------------------------------------------------------------------------------
 1 | name: Check PR
 2 | 
 3 | on:
 4 |   pull_request:
 5 |     types:
 6 |       - opened
 7 |       - synchronize
 8 |     paths-ignore:
 9 |       - '*.md'
10 | 
11 |   workflow_dispatch:
12 | 
13 | jobs:
14 |   check:
15 |     runs-on: ubuntu-latest
16 | 
17 |     steps:
18 |       - name: Checkout Repository
19 |         uses: actions/checkout@v2
20 | 
21 |       - name: Using Caches
22 |         uses: actions/cache@v2.1.3
23 |         with:
24 |           path: ~/.gradle/caches
25 |           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
26 |           restore-keys: |
27 |             ${{ runner.os }}-gradle-
28 |       - name: Setup JDK
29 |         uses: actions/setup-java@v1.4.3
30 |         with:
31 |           java-version: 1.8
32 | 
33 |       - name: Grant Execute Permission to gradlew
34 |         run: chmod +x gradlew
35 | 
36 |       - name: Build with Gradle
37 |         run: ./gradlew build
38 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | # Gradle Cache Dependencies Stage
 2 | # This stage caches plugin/project dependencies from *.gradle.kts and gradle.properties.
 3 | # Gradle image erases GRADLE_USER_HOME each layer. So we need COPY GRADLE_USER_HOME.
 4 | # Refer https://stackoverflow.com/a/59022743
 5 | FROM gradle:jdk8 AS cache
 6 | WORKDIR /app
 7 | ENV GRADLE_USER_HOME /app/gradle
 8 | COPY *.gradle.kts gradle.properties /app/
 9 | # Full build if there are any deps changes
10 | RUN gradle shadowJar --parallel --no-daemon --quiet
11 | 
12 | # Gradle Build Stage
13 | # This stage builds and generates fat jar.
14 | FROM gradle:jdk8 AS build
15 | WORKDIR /app
16 | COPY --from=cache /app/gradle /home/gradle/.gradle
17 | COPY *.gradle.kts gradle.properties /app/
18 | COPY src/jvmMain/ /app/src/jvmMain/
19 | # Stop printing Welcome
20 | RUN gradle -version > /dev/null \
21 |     && gradle shadowJar --parallel --no-daemon
22 | 
23 | # Final Stage
24 | FROM openjdk:17-jdk-alpine
25 | 
26 | COPY --from=build /app/build/libs/tweetstorm-all.jar /app/tweetstorm.jar
27 | 
28 | WORKDIR /app
29 | ENTRYPOINT ["java", "-jar", "/app/tweetstorm.jar"]
30 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 Slash Nephy
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/Task.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task
 2 | 
 3 | import blue.starry.tweetstorm.Config
 4 | import blue.starry.tweetstorm.logger
 5 | import blue.starry.tweetstorm.session.AuthenticatedStream
 6 | import blue.starry.tweetstorm.task.data.ProduceData
 7 | import kotlinx.coroutines.Job
 8 | import kotlinx.coroutines.channels.ReceiveChannel
 9 | import java.util.concurrent.TimeUnit
10 | import kotlin.coroutines.CoroutineContext
11 | 
12 | abstract class Task(val account: Config.Account) {
13 |     val logger by lazy { logger("Tweetstorm.task.${javaClass.simpleName} (@${account.user.screenName})") }
14 | }
15 | 
16 | abstract class ProduceTask<D: ProduceData<*>>(account: Config.Account): Task(account) {
17 |     abstract fun channel(context: CoroutineContext, parent: Job): ReceiveChannel<D>
18 | }
19 | 
20 | abstract class TargetedProduceTask<D: ProduceData<*>>(target: AuthenticatedStream): Task(target.account) {
21 |     abstract fun channel(context: CoroutineContext, parent: Job): ReceiveChannel<D>
22 | }
23 | 
24 | abstract class RegularTask(account: Config.Account, val interval: Long, val unit: TimeUnit): Task(account) {
25 |     abstract suspend fun run()
26 | }
27 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/App.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm
 2 | 
 3 | import io.ktor.application.Application
 4 | import io.ktor.application.call
 5 | import io.ktor.application.install
 6 | import io.ktor.features.DefaultHeaders
 7 | import io.ktor.features.StatusPages
 8 | import io.ktor.features.XForwardedHeaderSupport
 9 | import io.ktor.http.HttpHeaders
10 | import io.ktor.http.HttpStatusCode
11 | import io.ktor.response.respond
12 | import io.ktor.routing.Routing
13 | 
14 | fun Application.module() {
15 |     install(RequestLogging)
16 |     install(XForwardedHeaderSupport)
17 |     install(DefaultHeaders) {
18 |         header(HttpHeaders.Server, "Tweetstorm")
19 |     }
20 |     // install(Compression)
21 |     install(Routing) {
22 |         getTop()
23 |         getUser()
24 |         authByToken()
25 |     }
26 |     install(StatusPages) {
27 |         val logger = logger("Tweetstorm")
28 | 
29 |         exception<Exception> { e ->
30 |             logger.error(e) { "Internal server error occurred." }
31 |             call.respond(HttpStatusCode.InternalServerError)
32 |         }
33 | 
34 |         status(HttpStatusCode.NotFound) {
35 |             notFound()
36 |         }
37 |     }
38 | }
39 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/producer/Friends.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.producer
 2 | 
 3 | import blue.starry.tweetstorm.session.AuthenticatedStream
 4 | import blue.starry.tweetstorm.task.TargetedProduceTask
 5 | import blue.starry.tweetstorm.task.data.JsonData
 6 | import kotlinx.coroutines.ExperimentalCoroutinesApi
 7 | import kotlinx.coroutines.GlobalScope
 8 | import kotlinx.coroutines.Job
 9 | import kotlinx.coroutines.channels.produce
10 | import kotlin.coroutines.CoroutineContext
11 | 
12 | class Friends(private val target: AuthenticatedStream): TargetedProduceTask<JsonData>(target) {
13 |     @ExperimentalCoroutinesApi
14 |     override fun channel(context: CoroutineContext, parent: Job) = GlobalScope.produce(context + parent) {
15 |         val stringifyFriendIds = target.request.queryParameters["stringify_friend_ids"].orEmpty().toBooleanEasy()
16 | 
17 |         if (stringifyFriendIds) {
18 |             send(JsonData("friends_str" to account.friends.map { "$it" }))
19 |         } else {
20 |             send(JsonData("friends" to account.friends))
21 |         }
22 |     }
23 | 
24 |     private fun String.toBooleanEasy(): Boolean {
25 |         return equals("true", true) || equals("1")
26 |     }
27 | }
28 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/session/DemoStream.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.session
 2 | 
 3 | import blue.starry.penicillin.extensions.models.builder.newStatus
 4 | import io.ktor.features.origin
 5 | import io.ktor.request.ApplicationRequest
 6 | import io.ktor.utils.io.ByteWriteChannel
 7 | import kotlinx.coroutines.delay
 8 | 
 9 | private val logger = blue.starry.tweetstorm.logger("Tweetstorm.DemoStream")
10 | 
11 | class DemoStream(channel: ByteWriteChannel, request: ApplicationRequest): Stream<Unit>(channel, request) {
12 |     override suspend fun await() {
13 |         logger.info { "Unknown client: ${request.origin.remoteHost} has connected to DemoStream." }
14 | 
15 |         try {
16 |             handler.emit(newStatus {
17 |                 text { "This is demo stream. Since Tweetstorm could not authenticate you, demo stream has started. Please check your config.json." }
18 |             })
19 | 
20 |             while (handler.isAlive) {
21 |                 if (!handler.heartbeat()) {
22 |                     break
23 |                 }
24 |                 delay(3000)
25 |             }
26 |         } finally {
27 |             logger.info { "Unknown client: ${request.origin.remoteHost} has disconnected from DemoStream." }
28 |         }
29 |     }
30 | }
31 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/producer/SampleStream.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.producer
 2 | 
 3 | import blue.starry.jsonkt.JsonObject
 4 | import blue.starry.penicillin.core.session.ApiClient
 5 | import blue.starry.penicillin.core.streaming.listener.SampleStreamListener
 6 | import blue.starry.penicillin.endpoints.stream
 7 | import blue.starry.penicillin.endpoints.stream.sample
 8 | import blue.starry.tweetstorm.Config
 9 | import blue.starry.tweetstorm.task.ProduceTask
10 | import blue.starry.tweetstorm.task.data.JsonObjectData
11 | import kotlinx.coroutines.ExperimentalCoroutinesApi
12 | import kotlinx.coroutines.GlobalScope
13 | import kotlinx.coroutines.Job
14 | import kotlinx.coroutines.channels.produce
15 | import kotlin.coroutines.CoroutineContext
16 | 
17 | class SampleStream(account: Config.Account, private val client: ApiClient): ProduceTask<JsonObjectData>(account) {
18 |     @ExperimentalCoroutinesApi
19 |     override fun channel(context: CoroutineContext, parent: Job) = GlobalScope.produce<JsonObjectData>(context + parent) {
20 |         client.stream.sample.listen(object: SampleStreamListener {
21 |             override suspend fun onAnyJson(json: JsonObject) {
22 |                 send(JsonObjectData(json))
23 |             }
24 | 
25 |             override suspend fun onConnect() {
26 |                 logger.info { "Connected to SampleStream." }
27 |             }
28 | 
29 |             override suspend fun onDisconnect(cause: Throwable?) {
30 |                 logger.warn { "Disconnected from SampleStream." }
31 |             }
32 |         }, reconnect = true).join()
33 |     }
34 | }
35 | 


--------------------------------------------------------------------------------
/README_EN.md:
--------------------------------------------------------------------------------
 1 | 日本語のREADMEは [こちら](https://github.com/SlashNephy/Tweetstorm/blob/master/README.md) です.  
 2 | 
 3 | # Tweetstorm
 4 | A simple substitute implementation for the Twitter UserStream.  
 5 | Tweetstorm's goal is to simulate the UserStream API which retired on August 23th, 2018 as possible. 
 6 | 
 7 | ```
 8 |                  User Stream API                                          Twitter API
 9 |              +--------------------+     +-------+     +------------+     +------------+
10 |  Client A -> | GET /1.1/user.json |     |       |     |            | <-> |   Tweets   |
11 |              +--------------------+     |       |     |            |     +------------+
12 |              +--------------------+     |       |     |            | <-> | Activities |
13 |  Client B -> | GET /1.1/user.json | <-> | nginx | <-> | Tweetstorm |     +------------+
14 |              +--------------------+     |       |     |            | <-> |   Friends  |
15 |              +--------------------+     |       |     |            |     +------------+
16 |  Client C -> | GET /1.1/user.json |     |       |     |            | <-> |    etc...  |
17 |              +--------------------+     +-------+     +------------+     +------------+
18 |           ~        userstream.twitter.com          ~  127.0.0.1:8080 (Default)
19 | ```
20 | 
21 | Tweetstorm provides following json data.
22 | - Tweets  
23 | - Direct messages    
24 | - Activities / Events
25 | - Friend Ids  
26 | 
27 | ## Wiki
28 | Sections like Setup or Compatibility are moved to [Wiki](https://github.com/SlashNephy/Tweetstorm/wiki).
29 | 
30 | ## License
31 | This project is provided under the MIT license.
32 | 
33 | 
34 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/producer/FilterStream.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.producer
 2 | 
 3 | import blue.starry.jsonkt.JsonObject
 4 | import blue.starry.penicillin.core.session.ApiClient
 5 | import blue.starry.penicillin.core.streaming.listener.FilterStreamListener
 6 | import blue.starry.penicillin.endpoints.stream
 7 | import blue.starry.penicillin.endpoints.stream.filter
 8 | import blue.starry.tweetstorm.Config
 9 | import blue.starry.tweetstorm.task.ProduceTask
10 | import blue.starry.tweetstorm.task.data.JsonObjectData
11 | import kotlinx.coroutines.ExperimentalCoroutinesApi
12 | import kotlinx.coroutines.GlobalScope
13 | import kotlinx.coroutines.Job
14 | import kotlinx.coroutines.channels.produce
15 | import kotlin.coroutines.CoroutineContext
16 | 
17 | class FilterStream(account: Config.Account, private val client: ApiClient): ProduceTask<JsonObjectData>(account) {
18 |     @ExperimentalCoroutinesApi
19 |     override fun channel(context: CoroutineContext, parent: Job) = GlobalScope.produce<JsonObjectData>(context + parent) {
20 |         client.stream.filter(track = account.filterStream.tracks, follow = account.filterStream.follows).listen(object: FilterStreamListener {
21 |             override suspend fun onAnyJson(json: JsonObject) {
22 |                 send(JsonObjectData(json))
23 |             }
24 | 
25 |             override suspend fun onConnect() {
26 |                 logger.info { "Connected to FilterStream." }
27 |             }
28 | 
29 |             override suspend fun onDisconnect(cause: Throwable?) {
30 |                 logger.warn { "Disconnected from FilterStream." }
31 |             }
32 |         }, reconnect = true).join()
33 |     }
34 | }
35 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/session/AuthenticatedStream.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.session
 2 | 
 3 | import blue.starry.tweetstorm.Config
 4 | import blue.starry.tweetstorm.TaskManager
 5 | import blue.starry.tweetstorm.logger
 6 | import io.ktor.features.origin
 7 | import io.ktor.request.ApplicationRequest
 8 | import io.ktor.util.toMap
 9 | import io.ktor.utils.io.ByteWriteChannel
10 | import kotlinx.coroutines.ExperimentalCoroutinesApi
11 | import kotlinx.coroutines.ObsoleteCoroutinesApi
12 | import kotlinx.coroutines.sync.withLock
13 | 
14 | private val logger = logger("Tweetstorm.AuthenticatedStream")
15 | 
16 | class AuthenticatedStream(channel: ByteWriteChannel,request: ApplicationRequest, val account: Config.Account): Stream<Unit>(channel, request) {
17 |     @ExperimentalCoroutinesApi
18 |     @ObsoleteCoroutinesApi
19 |     override suspend fun await() {
20 |         logger.info { "Client: @${account.user.screenName} (${request.origin.remoteHost}) connected to UserStream API with parameter ${request.queryParameters.toMap()}." }
21 | 
22 |         val manager = TaskManager.instances.find { it.account.user.id == account.user.id }?.also {
23 |             it.register(this)
24 |         } ?: TaskManager(this).also {
25 |             TaskManager.mutex.withLock {
26 |                 TaskManager.instances += it
27 |             }
28 | 
29 |             it.start(this)
30 |         }
31 | 
32 |         manager.wait(this)
33 | 
34 |         if (!manager.anyClients()) {
35 |             val removed = TaskManager.mutex.withLock {
36 |                 TaskManager.instances.removeIf { it.account.user.id == account.user.id }
37 |             }
38 |             if (removed) {
39 |                 logger.debug { "Task Manager: @${manager.account.user.screenName} will terminate." }
40 |                 manager.close()
41 |             }
42 |         }
43 | 
44 |         logger.info { "Client: @${account.user.screenName} (${request.origin.remoteHost}) has disconnected from UserStream API." }
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/.github/workflows/docker.yml:
--------------------------------------------------------------------------------
 1 | name: Docker
 2 | 
 3 | on:
 4 |   push:
 5 |     paths-ignore:
 6 |       - .gitignore
 7 |       - LICENSE
 8 |       - '**.md'
 9 |     branches-ignore:
10 |       - 'releases/**'
11 | 
12 |   release:
13 |     types:
14 |       - published
15 | 
16 |   workflow_dispatch:
17 | 
18 | env:
19 |   DOCKER_BASE_NAME: slashnephy/tweetstorm
20 | 
21 | jobs:
22 |   build:
23 |     runs-on: ubuntu-latest
24 | 
25 |     steps:
26 |       - name: Checkout Repository
27 |         uses: actions/checkout@v2
28 | 
29 |       - name: Login to DockerHub
30 |         uses: docker/login-action@v1 
31 |         with:
32 |           username: ${{ secrets.DOCKERHUB_USERNAME }}
33 |           password: ${{ secrets.DOCKERHUB_TOKEN }}
34 | 
35 |       - name: Build & Push (master)
36 |         if: github.event_name == 'push' && github.ref == 'refs/heads/master'
37 |         uses: docker/build-push-action@v2
38 |         with:
39 |           push: true
40 |           tags: ${{ env.DOCKER_BASE_NAME }}:latest
41 | 
42 |       - name: Build & Push (dev)
43 |         if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
44 |         uses: docker/build-push-action@v2
45 |         with:
46 |           push: true
47 |           tags: ${{ env.DOCKER_BASE_NAME }}:dev
48 | 
49 |       - name: Build & Push (Release)
50 |         if: github.event_name == 'release'
51 |         uses: docker/build-push-action@v2
52 |         with:
53 |           push: true
54 |           tags: ${{ env.DOCKER_BASE_NAME }}:${{ github.event.release.tag_name }}
55 | 
56 |       # 2FA を無効化する必要がある
57 |       # https://github.com/peter-evans/dockerhub-description#action-inputs
58 |       # - name: Update Docker Hub description
59 |       #   if: github.event_name == 'push' && github.ref == 'refs/heads/master'
60 |       #   uses: peter-evans/dockerhub-description@v2
61 |       #   with:
62 |       #     username: ${{ secrets.DOCKERHUB_USERNAME }}
63 |       #     password: ${{ secrets.DOCKERHUB_TOKEN }}
64 |       #     repository: ${{ env.DOCKER_BASE_NAME }}
65 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/CLI.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm
 2 | 
 3 | import org.apache.commons.cli.*
 4 | import java.nio.file.Path
 5 | import java.nio.file.Paths
 6 | import kotlin.system.exitProcess
 7 | 
 8 | fun parseCommandLine(args: Array<String>): CLIArguments {
 9 |     val parser = DefaultParser()
10 |     val options = Options()
11 |             .addOption(
12 |                     Option.builder("help").desc("Print this help and exit.").build()
13 |             )
14 |             .addOption(
15 |                     Option.builder("config").longOpt("config-path").hasArg().argName("config_path").numberOfArgs(1).desc("Specify Tweetstorm config.json path. (Default: ./config.json)").build()
16 |             )
17 | 
18 |     val result = try {
19 |         parser.parse(options, args)
20 |     } catch (e: Exception) {
21 |         when (e) {
22 |             is UnrecognizedOptionException -> {
23 |                 System.err.println("Unknown option: ${e.option}")
24 |             }
25 |             is ParseException -> {
26 |                 System.err.println("Failed to parse command line.")
27 |             }
28 |         }
29 | 
30 |         printHelpAndExit(options)
31 |         throw e
32 |     }
33 | 
34 |     if (result.hasOption("help")) {
35 |         printHelpAndExit(options)
36 |     }
37 | 
38 |     val configPath = result.getOptionValue("config")?.let { Paths.get(it) }
39 |     return CLIArguments(configPath)
40 | }
41 | 
42 | private fun printHelpAndExit(options: Options) {
43 |     val header = "\nTweetstorm\n    A simple substitute implementation for the Twitter UserStream.\n\n============================================================\n\nAvailable cli options:"
44 |     val footer = "\nContact:\n    Twitter: @SlashNephy\n    GitHub: https://github.com/SlashNephy/Tweetstorm\n\n    Feel free to report issues anytime."
45 | 
46 |     HelpFormatter().apply {
47 |         leftPadding = 4
48 |         width = 150
49 |     }.printHelp("java -jar tweetstorm-full.jar", header, options, footer, true)
50 | 
51 |     exitProcess(0)
52 | }
53 | 
54 | data class CLIArguments(val configPath: Path?)
55 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/regular/SyncList.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.regular
 2 | 
 3 | import blue.starry.penicillin.core.session.ApiClient
 4 | import blue.starry.penicillin.endpoints.friends
 5 | import blue.starry.penicillin.endpoints.friends.listIds
 6 | import blue.starry.penicillin.endpoints.lists
 7 | import blue.starry.penicillin.endpoints.lists.addMembersByUserIds
 8 | import blue.starry.penicillin.endpoints.lists.members
 9 | import blue.starry.penicillin.endpoints.lists.removeMembersByUserIds
10 | import blue.starry.penicillin.extensions.cursor.allIds
11 | import blue.starry.penicillin.extensions.cursor.allUsers
12 | import blue.starry.penicillin.extensions.cursor.untilLast
13 | import blue.starry.penicillin.extensions.execute
14 | import blue.starry.tweetstorm.Config
15 | import blue.starry.tweetstorm.task.RegularTask
16 | import java.util.concurrent.TimeUnit
17 | 
18 | class SyncList(account: Config.Account, private val client: ApiClient): RegularTask(account, 5, TimeUnit.MINUTES) {
19 |     override suspend fun run() {
20 |         val followingIds = if (account.syncList.includeSelf) {
21 |             client.friends.listIds(count = 5000).untilLast().allIds + account.user.id
22 |         } else {
23 |             client.friends.listIds(count = 5000).untilLast().allIds
24 |         }
25 | 
26 |         if (followingIds.size > 5000) {
27 |             logger.warn { "This list exceeded 5000 members limit." }
28 |             return
29 |         }
30 |         val listMemberIds = client.lists.members(listId = account.listId!!, count = 5000).untilLast().allUsers.map { it.id }
31 |         val willBeRemoved = listMemberIds - followingIds
32 |         if (willBeRemoved.isNotEmpty()) {
33 |             willBeRemoved.chunked(100).forEach {
34 |                 client.lists.removeMembersByUserIds(listId = account.listId!!, userIds = it).execute()
35 |             }
36 |             logger.debug { "Removing ${willBeRemoved.size} user(s)." }
37 |         }
38 |         val willBeAdded = followingIds - listMemberIds
39 |         if (willBeAdded.isNotEmpty()) {
40 |             willBeAdded.chunked(100).forEach {
41 |                 client.lists.addMembersByUserIds(listId = account.listId!!, userIds = it).execute()
42 |             }
43 |             logger.debug { "Adding ${willBeAdded.size} user(s)." }
44 |         }
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/Logging.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm
 2 | 
 3 | import io.ktor.application.*
 4 | import io.ktor.features.origin
 5 | import io.ktor.http.HttpMethod
 6 | import io.ktor.request.httpMethod
 7 | import io.ktor.request.path
 8 | import io.ktor.request.userAgent
 9 | import io.ktor.util.AttributeKey
10 | import io.ktor.util.pipeline.PipelinePhase
11 | import mu.KLogger
12 | import mu.KotlinLogging
13 | 
14 | fun logger(name: String): KLogger {
15 |     return KotlinLogging.logger(name).also {
16 |         (it.underlyingLogger as ch.qos.logback.classic.Logger).level = config.logLevel
17 |     }
18 | }
19 | 
20 | private val logger = logger("Tweetstorm.Routing")
21 | 
22 | internal class RequestLogging private constructor(private val monitor: ApplicationEvents) {
23 |     private val shouldIgnore = { call: ApplicationCall ->
24 |         call.request.httpMethod == HttpMethod.Get && call.request.path() == "/1.1/user.json"
25 |     }
26 | 
27 |     private val onStart: (Application) -> Unit = {
28 |         logger.info("Application is responding at http://${config.wui.host}:${config.wui.port}")
29 |     }
30 |     private var onStop: (Application) -> Unit = {
31 |         logger.info("Application stopped.")
32 |     }
33 | 
34 |     init {
35 |         onStop = {
36 |             onStop(it)
37 |             monitor.unsubscribe(ApplicationStarted, onStart)
38 |             monitor.unsubscribe(ApplicationStopped, onStop)
39 |         }
40 | 
41 |         monitor.subscribe(ApplicationStarted, onStart)
42 |         monitor.subscribe(ApplicationStopped, onStop)
43 |     }
44 | 
45 |     companion object Feature : ApplicationFeature<Application, Unit, RequestLogging> {
46 |         override val key: AttributeKey<RequestLogging> = AttributeKey("RequestLogging")
47 | 
48 |         override fun install(pipeline: Application, configure: Unit.() -> Unit): RequestLogging {
49 |             val phase = PipelinePhase("RequestLogging")
50 |             val feature = RequestLogging(pipeline.environment.monitor)
51 | 
52 |             pipeline.insertPhaseBefore(ApplicationCallPipeline.Monitoring, phase)
53 |             pipeline.intercept(phase) {
54 |                 proceed()
55 |                 if (!feature.shouldIgnore(call)) {
56 |                     logger.info { "[${call.response.status()?.value ?: " - "}] ${call.request.httpMethod.value.toUpperCase()} ${call.request.path()}\nfrom ${call.request.origin.remoteHost} with \"${call.request.userAgent().orEmpty()}\"" }
57 |                 }
58 |             }
59 |             return feature
60 |         }
61 |     }
62 | }
63 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/session/PreAuthenticatedStream.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.session
 2 | 
 3 | import blue.starry.penicillin.extensions.models.builder.newStatus
 4 | import blue.starry.tweetstorm.Config
 5 | import io.ktor.features.origin
 6 | import io.ktor.request.ApplicationRequest
 7 | import io.ktor.utils.io.ByteWriteChannel
 8 | import kotlinx.coroutines.delay
 9 | import java.util.*
10 | import java.util.concurrent.CopyOnWriteArrayList
11 | 
12 | private val logger = blue.starry.tweetstorm.logger("Tweetstorm.PreAuthenticatedStream")
13 | 
14 | class PreAuthenticatedStream(channel: ByteWriteChannel, request: ApplicationRequest, val account: Config.Account): Stream<Boolean>(channel, request) {
15 |     companion object {
16 |         private val streams = CopyOnWriteArrayList<PreAuthenticatedStream>()
17 | 
18 |         @Synchronized
19 |         fun auth(urlToken: String, accountToken: String): Boolean {
20 |             return streams.removeIf { it.urlToken == urlToken && it.account.token == accountToken }
21 |         }
22 | 
23 |         @Synchronized
24 |         fun check(urlToken: String): Boolean {
25 |             return streams.count { it.urlToken == urlToken } > 0
26 |         }
27 | 
28 |         @Synchronized
29 |         private fun contain(stream: PreAuthenticatedStream): Boolean {
30 |             return streams.count { it.urlToken == stream.urlToken && it.account.token == stream.account.token } > 0
31 |         }
32 | 
33 |         @Synchronized
34 |         private fun register(stream: PreAuthenticatedStream) {
35 |             streams += stream
36 |         }
37 |     }
38 | 
39 |     private val urlToken = UUID.randomUUID().toString().toLowerCase().replace("-", "")
40 | 
41 |     override suspend fun await(): Boolean {
42 |         logger.info { "Client: @${account.user.screenName} (${request.origin.remoteHost}) requested account-token authentication." }
43 | 
44 |         register(this)
45 | 
46 |         handler.emit(newStatus {
47 |             user {
48 |                 name("Tweetstorm Authenticator")
49 |             }
50 |             text { "To start streaming, access https://userstream.twitter.com/auth/token/$urlToken" }
51 |             url("https://userstream.twitter.com/auth/token/$urlToken", 27, 101)
52 |         })
53 | 
54 |         repeat(300) {
55 |             if (!contain(this)) {
56 |                 return true
57 |             }
58 |             else if (it % 10 == 0 && !handler.heartbeat()) {
59 |                 return false
60 |             }
61 | 
62 |             delay(1000)
63 |         }
64 | 
65 |         return false
66 |     }
67 | }
68 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/Auth.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm
 2 | 
 3 | import io.ktor.http.*
 4 | import java.net.URLDecoder
 5 | import java.util.*
 6 | import javax.crypto.Mac
 7 | import javax.crypto.spec.SecretKeySpec
 8 | 
 9 | fun Headers.parseAuthorizationHeaderStrict(method: HttpMethod, url: String, query: Parameters): Config.Account? {
10 |     val authorization = get(HttpHeaders.Authorization) ?: return null
11 |     if (!authorization.startsWith("OAuth")) {
12 |         return null
13 |     }
14 | 
15 |     val authorizationData = authorization.removePrefix("OAuth").split(",").asSequence().map {
16 |         it.trim().split("=", limit = 2)
17 |     }.map {
18 |         it.first() to URLDecoder.decode(it.last(), Charsets.UTF_8.name()).removeSurrounding("\"")
19 |     }.toMap()
20 | 
21 |     return config.accounts.asSequence().filter {
22 |         it.ck == authorizationData["oauth_consumer_key"] && it.at == authorizationData["oauth_token"]
23 |     }.find { account ->
24 |         val signatureParam = authorizationData.toSortedMap()
25 |         signatureParam.remove("oauth_signature")
26 |         query.forEach { k, v ->
27 |             signatureParam[k.encodeURLParameter()] = v.last().encodeURLParameter()
28 |         }
29 |         val signatureParamString = signatureParam.toList().joinToString("&") { "${it.first}=${it.second}" }.encodeOAuth()
30 |         val signatureBaseString = "${method.value}&${url.encodeOAuth()}&$signatureParamString"
31 | 
32 |         val signingKey = SecretKeySpec("${account.cs.encodeOAuth()}&${account.ats.encodeOAuth()}".toByteArray(), "HmacSHA1")
33 |         val signature = Mac.getInstance(signingKey.algorithm).apply {
34 |             init(signingKey)
35 |         }.doFinal(signatureBaseString.toByteArray()).let {
36 |             Base64.getEncoder().encodeToString(it)
37 |         }
38 | 
39 |         signature == authorizationData["oauth_signature"]
40 |     }
41 | }
42 | 
43 | fun Headers.parseAuthorizationHeaderSimple(): Config.Account? {
44 |     val authorization = get(HttpHeaders.Authorization) ?: return null
45 |     if (!authorization.startsWith("OAuth")) {
46 |         return null
47 |     }
48 | 
49 |     val authorizationData = authorization.removePrefix("OAuth").split(",").asSequence().map {
50 |         it.trim().split("=", limit = 2)
51 |     }.map {
52 |         it.first() to URLDecoder.decode(it.last(), Charsets.UTF_8.name()).removeSurrounding("\"")
53 |     }.toList().toMap()
54 | 
55 |     val id = authorizationData["oauth_token"]?.split("-")?.firstOrNull()?.toLongOrNull() ?: return null
56 |     return config.accounts.find { it.user.id == id }
57 | }
58 | 


--------------------------------------------------------------------------------
/gradlew.bat:
--------------------------------------------------------------------------------
 1 | @rem
 2 | @rem Copyright 2015 the original author or authors.
 3 | @rem
 4 | @rem Licensed under the Apache License, Version 2.0 (the "License");
 5 | @rem you may not use this file except in compliance with the License.
 6 | @rem You may obtain a copy of the License at
 7 | @rem
 8 | @rem      https://www.apache.org/licenses/LICENSE-2.0
 9 | @rem
10 | @rem Unless required by applicable law or agreed to in writing, software
11 | @rem distributed under the License is distributed on an "AS IS" BASIS,
12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | @rem See the License for the specific language governing permissions and
14 | @rem limitations under the License.
15 | @rem
16 | 
17 | @if "%DEBUG%" == "" @echo off
18 | @rem ##########################################################################
19 | @rem
20 | @rem  Gradle startup script for Windows
21 | @rem
22 | @rem ##########################################################################
23 | 
24 | @rem Set local scope for the variables with windows NT shell
25 | if "%OS%"=="Windows_NT" setlocal
26 | 
27 | set DIRNAME=%~dp0
28 | if "%DIRNAME%" == "" set DIRNAME=.
29 | set APP_BASE_NAME=%~n0
30 | set APP_HOME=%DIRNAME%
31 | 
32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter.
33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
34 | 
35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
37 | 
38 | @rem Find java.exe
39 | if defined JAVA_HOME goto findJavaFromJavaHome
40 | 
41 | set JAVA_EXE=java.exe
42 | %JAVA_EXE% -version >NUL 2>&1
43 | if "%ERRORLEVEL%" == "0" goto execute
44 | 
45 | echo.
46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
47 | echo.
48 | echo Please set the JAVA_HOME variable in your environment to match the
49 | echo location of your Java installation.
50 | 
51 | goto fail
52 | 
53 | :findJavaFromJavaHome
54 | set JAVA_HOME=%JAVA_HOME:"=%
55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe
56 | 
57 | if exist "%JAVA_EXE%" goto execute
58 | 
59 | echo.
60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
61 | echo.
62 | echo Please set the JAVA_HOME variable in your environment to match the
63 | echo location of your Java installation.
64 | 
65 | goto fail
66 | 
67 | :execute
68 | @rem Setup the command line
69 | 
70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
71 | 
72 | 
73 | @rem Execute Gradle
74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
75 | 
76 | :end
77 | @rem End local scope for the variables with windows NT shell
78 | if "%ERRORLEVEL%"=="0" goto mainEnd
79 | 
80 | :fail
81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
82 | rem the _cmd.exe /c_ return code!
83 | if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
84 | exit /b 1
85 | 
86 | :mainEnd
87 | if "%OS%"=="Windows_NT" endlocal
88 | 
89 | :omega
90 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/Layout.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm
 2 | 
 3 | import io.ktor.html.Placeholder
 4 | import io.ktor.html.Template
 5 | import io.ktor.html.insert
 6 | import kotlinx.html.*
 7 | 
 8 | 
 9 | class NavLayout: Template<HTML> {
10 |     val navContent = Placeholder<HtmlBlockTag>()
11 |     override fun HTML.apply() {
12 |         insert(FooterLayout()) {
13 |             footerContent {
14 |                 div("navbar navbar-expand-lg navbar-dark bg-primary") {
15 |                     a("/", "", "navbar-brand") { +"Tweetstorm" }
16 |                 }
17 |                 style {
18 |                     unsafe {
19 |                         +".alert { padding-top: 16px; }"
20 |                     }
21 |                 }
22 |                 div("container") {
23 |                     insert(navContent)
24 |                 }
25 |             }
26 |         }
27 |     }
28 | }
29 | 
30 | class FooterLayout: Template<HTML> {
31 |     val footerContent = Placeholder<HtmlBlockTag>()
32 |     override fun HTML.apply() {
33 |         insert(MainLayout()) {
34 |             content {
35 |                 insert(footerContent)
36 |                 div("container") {
37 |                     hr()
38 |                     div("text-center") {
39 |                         style {
40 |                             unsafe {
41 |                                 +".opacity05 { opacity: 0.5; }"
42 |                             }
43 |                         }
44 |                         p {
45 |                             +"Tweetstorm brought to you by "
46 |                             a("https://github.com/SlashNephy") {
47 |                                 +"@SlashNephy"
48 |                             }
49 |                             +", "
50 |                             a("https://github.com/motitaiyaki") {
51 |                                 +"@motitaiyaki"
52 |                             }
53 |                             +" and "
54 |                             a("https://github.com/suzutan") {
55 |                                 +"@suzutan"
56 |                             }
57 |                             +" with "
58 |                             span("fas fa-heart")
59 |                             br()
60 |                             span("opacity05") {
61 |                                 +"...and "
62 |                                 span("fas fa-angry")
63 |                                 +" to "
64 |                                 span("fab fa-twitter")
65 |                             }
66 |                         }
67 |                     }
68 |                 }
69 |             }
70 |         }
71 |     }
72 | }
73 | 
74 | class MainLayout: Template<HTML> {
75 |     val content = Placeholder<HtmlBlockTag>()
76 |     override fun HTML.apply() {
77 |         head {
78 |             meta(charset = "utf-8")
79 |             meta("viewport", "width=device-width,initial-scale=1.0,minimum-scale=1.0,maximum-scale=1.0,user-scalable=no")
80 |             title { +"Tweetstorm" }
81 |             styleLink("https://cdnjs.cloudflare.com/ajax/libs/bootswatch/4.1.3/cosmo/bootstrap.min.css")
82 |             styleLink("https://use.fontawesome.com/releases/v5.2.0/css/all.css")
83 |         }
84 |         body {
85 |             insert(content)
86 |         }
87 |     }
88 | }
89 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/session/StreamContent.kt:
--------------------------------------------------------------------------------
  1 | package blue.starry.tweetstorm.session
  2 | 
  3 | import blue.starry.jsonkt.JsonObject
  4 | import blue.starry.jsonkt.delegation.JsonModel
  5 | import blue.starry.jsonkt.encodeToString
  6 | import blue.starry.jsonkt.jsonObjectOf
  7 | import blue.starry.penicillin.endpoints.stream.StreamDelimitedBy
  8 | import io.ktor.http.*
  9 | import io.ktor.http.content.OutgoingContent
 10 | import io.ktor.request.ApplicationRequest
 11 | import io.ktor.utils.io.ByteWriteChannel
 12 | import io.ktor.utils.io.writeStringUtf8
 13 | import kotlinx.coroutines.CancellationException
 14 | import kotlinx.coroutines.sync.Mutex
 15 | import kotlinx.coroutines.sync.withLock
 16 | import kotlinx.coroutines.withTimeout
 17 | import java.io.IOException
 18 | 
 19 | private const val delimiter = "\r\n"
 20 | private val logger = blue.starry.tweetstorm.logger("Tweetstorm.StreamContent")
 21 | 
 22 | class StreamContent(private val writer: suspend (channel: ByteWriteChannel) -> Unit): OutgoingContent.WriteChannelContent() {
 23 |     override val status = HttpStatusCode.OK
 24 |     override val headers = headersOf(HttpHeaders.Connection, "keep-alive")
 25 |     override val contentType = ContentType.Application.Json.withCharset(Charsets.UTF_8)
 26 | 
 27 |     override suspend fun writeTo(channel: ByteWriteChannel) {
 28 |         writer(channel)
 29 |     }
 30 | 
 31 |     class Handler(private val channel: ByteWriteChannel, request: ApplicationRequest) {
 32 |         private val delimitedBy = StreamDelimitedBy.valueOf(request.queryParameters["delimited"].orEmpty())
 33 |         private val lock = Mutex()
 34 | 
 35 |         val isAlive: Boolean
 36 |             get() = !channel.isClosedForWrite
 37 | 
 38 |         private suspend fun writeWrap(content: String): Boolean {
 39 |             if (!isAlive){
 40 |                 return false
 41 |             }
 42 | 
 43 |             return try {
 44 |                 lock.withLock {
 45 |                     withTimeout(1000) {
 46 |                         channel.writeStringUtf8(content)
 47 |                         channel.flush()
 48 |                     }
 49 |                 }
 50 |                 true
 51 |             } catch (e: CancellationException) {
 52 |                 false
 53 |             } catch (e: IOException) {
 54 |                 false
 55 |             }
 56 |         }
 57 | 
 58 |         private suspend fun emit(content: String): Boolean {
 59 |             logger.trace { "Payload = $content" }
 60 |             val text = "${content.trim().escapeHtml().escapeUnicode()}$delimiter"
 61 |             return when (delimitedBy) {
 62 |                 StreamDelimitedBy.Length -> {
 63 |                     writeWrap("${text.length}$delimiter$text")
 64 |                 }
 65 |                 else -> {
 66 |                     writeWrap(text)
 67 |                 }
 68 |             }
 69 |         }
 70 | 
 71 |         suspend fun emit(vararg pairs: Pair<String, Any?>): Boolean {
 72 |             return emit(jsonObjectOf(*pairs))
 73 |         }
 74 | 
 75 |         suspend fun emit(json: JsonObject): Boolean {
 76 |             return emit(json.encodeToString())
 77 |         }
 78 | 
 79 |         suspend fun emit(payload: JsonModel): Boolean {
 80 |             return emit(payload.json)
 81 |         }
 82 | 
 83 |         suspend fun heartbeat(): Boolean {
 84 |             return writeWrap(delimiter)
 85 |         }
 86 | 
 87 |         private fun String.escapeHtml(): String {
 88 |             return replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
 89 |         }
 90 | 
 91 |         private fun String.escapeUnicode(): String {
 92 |             return map {
 93 |                 val code = it.toInt()
 94 |                 if (code in 0 until 128) {
 95 |                     "$it"
 96 |                 } else {
 97 |                     String.format("\\u%04x", code)
 98 |                 }
 99 |             }.joinToString("")
100 |         }
101 |     }
102 | }
103 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/producer/DirectMessage.kt:
--------------------------------------------------------------------------------
 1 | package blue.starry.tweetstorm.task.producer
 2 | 
 3 | import blue.starry.penicillin.core.exceptions.PenicillinTwitterApiException
 4 | import blue.starry.penicillin.core.exceptions.TwitterApiError
 5 | import blue.starry.penicillin.core.session.ApiClient
 6 | import blue.starry.penicillin.endpoints.directMessageEvent
 7 | import blue.starry.penicillin.endpoints.directmessages.events.list
 8 | import blue.starry.penicillin.extensions.executeWithTimeout
 9 | import blue.starry.penicillin.extensions.models.builder.newDirectMessage
10 | import blue.starry.penicillin.extensions.rateLimit
11 | import blue.starry.tweetstorm.Config
12 | import blue.starry.tweetstorm.config
13 | import blue.starry.tweetstorm.task.ProduceTask
14 | import blue.starry.tweetstorm.task.data.JsonModelData
15 | import io.ktor.util.date.toJvmDate
16 | import kotlinx.coroutines.*
17 | import kotlinx.coroutines.channels.produce
18 | import kotlinx.coroutines.time.delay
19 | import java.time.Duration
20 | import java.time.Instant
21 | import java.util.concurrent.atomic.AtomicLong
22 | import kotlin.coroutines.CoroutineContext
23 | 
24 | class DirectMessage(account: Config.Account, private val client: ApiClient): ProduceTask<JsonModelData>(account) {
25 |     @ExperimentalCoroutinesApi
26 |     override fun channel(context: CoroutineContext, parent: Job) = GlobalScope.produce(context + parent) {
27 |         val lastId = AtomicLong()
28 |         while (isActive) {
29 |             try {
30 |                 val messages = client.directMessageEvent.list(count = 200).executeWithTimeout(config.app.apiTimeout) ?: continue
31 |                 if (messages.result.events.isNotEmpty()) {
32 |                     val lastIdOrNull = lastId.get().let { if (it > 0) it else null }
33 |                     if (lastIdOrNull != null) {
34 |                         messages.result.events.asSequence().filter { it.type == "message_create" && lastIdOrNull < it.id.toLong() }.toList().reversed().forEach { event ->
35 |                             send(JsonModelData(newDirectMessage {
36 |                                 recipient {
37 |                                     this["id"] = event.messageCreate.target.recipientId.toLong()
38 |                                 }
39 |                                 sender {
40 |                                     this["id"] = event.messageCreate.senderId.toLong()
41 |                                 }
42 |                                 text { event.messageCreate.messageData.text }
43 |                                 entities(event.messageCreate.messageData.entities.json)
44 |                             }))
45 |                         }
46 |                     }
47 | 
48 |                     lastId.set(messages.result.events.first().id.toLong())
49 |                 }
50 | 
51 |                 val rateLimit = messages.rateLimit
52 |                 if (rateLimit != null) {
53 |                     val duration = Duration.between(Instant.now(), rateLimit.resetAt.toJvmDate().toInstant())
54 |                     if (rateLimit.remaining < 2) {
55 |                         logger.warn { "Rate limit: Mostly exceeded. Sleep ${duration.seconds} secs. (Reset at ${rateLimit.resetAt})" }
56 |                         delay(duration)
57 |                     } else if (duration.seconds > 3 && rateLimit.remaining * account.refresh.directMessage.toDouble() / 1000 / duration.seconds < 1) {
58 |                         logger.warn { "Rate limit: API calls (/${messages.request.url}) seem to be frequent than expected so consider adjusting `direct_message_refresh` value in config.json. Sleep 10 secs. (${rateLimit.remaining}/${rateLimit.limit}, Reset at ${rateLimit.resetAt})" }
59 |                         delay(10000)
60 |                     }
61 |                 }
62 | 
63 |                 delay(account.refresh.directMessage)
64 |             } catch (e: CancellationException) {
65 |                 break
66 |             } catch (e: PenicillinTwitterApiException) {
67 |                 if (e.error == TwitterApiError.RateLimitExceeded) {
68 |                     delay(10000)
69 |                 } else {
70 |                     logger.error(e) { "An error occurred while getting direct messages." }
71 |                     delay(account.refresh.directMessage)
72 |                 }
73 |             }
74 |         }
75 |     }
76 | }
77 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/Config.kt:
--------------------------------------------------------------------------------
  1 | package blue.starry.tweetstorm
  2 | 
  3 | import blue.starry.jsonkt.JsonObject
  4 | import blue.starry.jsonkt.delegation.*
  5 | import blue.starry.jsonkt.parseObject
  6 | import blue.starry.jsonkt.stringOrNull
  7 | import blue.starry.penicillin.PenicillinClient
  8 | import blue.starry.penicillin.core.session.ApiClient
  9 | import blue.starry.penicillin.core.session.config.account
 10 | import blue.starry.penicillin.core.session.config.application
 11 | import blue.starry.penicillin.core.session.config.token
 12 | import blue.starry.penicillin.endpoints.account
 13 | import blue.starry.penicillin.endpoints.account.verifyCredentials
 14 | import blue.starry.penicillin.endpoints.friends
 15 | import blue.starry.penicillin.endpoints.friends.listIds
 16 | import blue.starry.penicillin.extensions.complete
 17 | import blue.starry.penicillin.extensions.cursor.allIds
 18 | import blue.starry.penicillin.extensions.cursor.untilLast
 19 | import ch.qos.logback.classic.Level
 20 | import java.nio.file.Path
 21 | import java.nio.file.Paths
 22 | 
 23 | data class Config(override val json: JsonObject): JsonModel {
 24 |     companion object {
 25 |         private val defaultConfigPath = Paths.get("config.json")
 26 | 
 27 |         fun load(configPath: Path?): Config {
 28 |             return try {
 29 |                 (configPath ?: defaultConfigPath).toFile().readText().parseObject { Config(it) }
 30 |             } catch (e: Exception) {
 31 |                 throw IllegalStateException("config.json is invalid.")
 32 |             }
 33 |         }
 34 |     }
 35 | 
 36 |     val wui by lazy { WebUI(json) }
 37 |     data class WebUI(override val json: JsonObject): JsonModel {
 38 |         val host by string { "127.0.0.1" }
 39 |         val port by int { 8080 }
 40 |     }
 41 | 
 42 |     val app by lazy { App(json) }
 43 |     data class App(override val json: JsonObject): JsonModel {
 44 |         val skipAuth by boolean("skip_auth") { false }
 45 |         val apiTimeout by long("api_timeout") { 3000 }
 46 |     }
 47 | 
 48 |     val logLevel: Level by lambda("log_level", Level.INFO) { Level.toLevel(it.stringOrNull, Level.INFO) }
 49 | 
 50 |     val accounts by modelList { Account(it) }
 51 |     data class Account(override val json: JsonObject): JsonModel {
 52 |         val ck by string
 53 |         val cs by string
 54 |         val at by string
 55 |         val ats by string
 56 |         val listId by nullableLong("list_id")
 57 |         val token by nullableString
 58 | 
 59 |         val enableDirectMessage by boolean("enable_direct_message") { true }
 60 |         val enableFriends by boolean("enable_friends") { true }
 61 |         val enableSampleStream by boolean("enable_sample_stream") { false }
 62 | 
 63 |         val filterStream by lazy { FilterStream(json) }
 64 |         data class FilterStream(override val json: JsonObject): JsonModel {
 65 |             val tracks by stringList("filter_stream_tracks")
 66 |             val follows by longList("filter_stream_follows")
 67 |         }
 68 | 
 69 |         val syncList by lazy { SyncList(json) }
 70 |         data class SyncList(override val json: JsonObject): JsonModel {
 71 |             val enabled by boolean("sync_list_following") { false }
 72 |             val includeSelf by boolean("sync_list_include_self") { true }
 73 |         }
 74 | 
 75 |         val refresh by lazy { RefreshTime(json) }
 76 |         data class RefreshTime(override val json: JsonObject): JsonModel {
 77 |             val listTimeline by long("list_timeline_refresh") { 1500 }
 78 |             val userTimeline by long("user_timeline_refresh") { 1500 }
 79 |             val mentionTimeline by long("mention_timeline_refresh") { 30000 }
 80 |             val homeTimeline by long("home_timeline_refresh") { 75000 }
 81 |             val directMessage by long("direct_message_refresh") { 75000 }
 82 |         }
 83 | 
 84 |         val twitter: ApiClient
 85 |             get() = PenicillinClient {
 86 |                 account {
 87 |                     application(ck, cs)
 88 |                     token(at, ats)
 89 |                 }
 90 |             }
 91 | 
 92 |         val user by lazy {
 93 |             twitter.use {
 94 |                 it.account.verifyCredentials.complete().result
 95 |             }
 96 |         }
 97 |         val friends by lazy {
 98 |             twitter.use {
 99 |                 runCatching {
100 |                     it.friends.listIds(count = 5000).untilLast().allIds
101 |                 }.getOrNull().orEmpty()
102 |             }
103 |         }
104 |     }
105 | }
106 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Tweetstorm: UserStream API の簡単な代替実装
  2 | 
  3 | [![Kotlin](https://img.shields.io/badge/Kotlin-1.4.30-blue)](https://kotlinlang.org)
  4 | [![GitHub release (latest by date)](https://img.shields.io/github/v/release/SlashNephy/tweetstorm)](https://github.com/SlashNephy/tweetstorm/releases)
  5 | [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/SlashNephy/tweetstorm/Docker)](https://hub.docker.com/r/slashnephy/tweetstorm)
  6 | [![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/slashnephy/tweetstorm)](https://hub.docker.com/r/slashnephy/tweetstorm)
  7 | [![Docker Pulls](https://img.shields.io/docker/pulls/slashnephy/tweetstorm)](https://hub.docker.com/r/slashnephy/tweetstorm)
  8 | [![license](https://img.shields.io/github/license/SlashNephy/tweetstorm)](https://github.com/SlashNephy/tweetstorm/blob/master/LICENSE)
  9 | [![issues](https://img.shields.io/github/issues/SlashNephy/tweetstorm)](https://github.com/SlashNephy/tweetstorm/issues)
 10 | [![pull requests](https://img.shields.io/github/issues-pr/SlashNephy/tweetstorm)](https://github.com/SlashNephy/tweetstorm/pulls)
 11 | 
 12 | English README is [here](https://github.com/SlashNephy/Tweetstorm/blob/master/README_EN.md).  
 13 | 
 14 | Tweetstorm はクライアントに代わって REST API を呼び出し, 従来の Twitter UserStream API と同等のインターフェイスで配信します。  
 15 | Tweetstorm は 2018/8/23 に完全に廃止された UserStream をできる限り再現することを目標としています。
 16 | 
 17 | ---
 18 | 
 19 | ## Docker
 20 | 
 21 | 環境構築が容易なので Docker で導入することをおすすめします。
 22 | 
 23 | 現在のベースイメージは `openjdk:17-jdk-alpine` です。いくつかフレーバーを用意しています。
 24 | 
 25 | - `slashnephy/tweetstorm:latest`  
 26 |   master ブランチへのプッシュの際にビルドされます。安定しています。
 27 | - `slashnephy/tweetstorm:dev`  
 28 |   dev ブランチへのプッシュの際にビルドされます。開発版のため, 不安定である可能性があります。
 29 | - `slashnephy/tweetstorm:<version>`  
 30 |   GitHub 上のリリースに対応します。
 31 | 
 32 | `config.json`
 33 | 
 34 | ```json
 35 | {
 36 |     "host": "0.0.0.0",
 37 |     "port": 8080,
 38 |     "skip_auth": false,
 39 |     "log_level": "info",
 40 |     "accounts": [
 41 |         {
 42 |             "name": "識別名",
 43 |             "ck": "xxx",
 44 |             "cs": "xxx",
 45 |             "at": "xxx-xxx",
 46 |             "ats": "xxx",
 47 |             "list_id": 123456789000000,
 48 |         }
 49 |     ]
 50 | }
 51 | ```
 52 | 
 53 | `docker-compose.yml`
 54 | 
 55 | ```yaml
 56 | version: '3'
 57 | 
 58 | services:
 59 |   tweetstorm:
 60 |     container_name: Tweetstorm
 61 |     image: slashnephy:tweetstorm:latest
 62 |     restart: always
 63 |     ports:
 64 |       - 8080:8080/tcp
 65 |     volumes:
 66 |       - ./config.json:/app/config.json
 67 | ```
 68 | 
 69 | 別途 nginx でリバースプロキシを設定してください。手順は Wiki にあります。
 70 | 
 71 | ```console
 72 | # イメージ更新
 73 | docker pull slashnephy/saya:latest
 74 | 
 75 | # 起動
 76 | docker-compose up -d
 77 | 
 78 | # ログ表示
 79 | docker-compose logs -f
 80 | 
 81 | # 停止
 82 | docker-compose down
 83 | ```
 84 | 
 85 | ## Demo
 86 | 
 87 | feather で実際に使用しているデモ動画は [YouTube](https://www.youtube.com/watch?v=N_Gf2JK3EeM) にアップロードしてあります。
 88 | 
 89 | ## Structure
 90 | 
 91 | ```
 92 |                  User Stream API                                          Twitter API
 93 |              +--------------------+     +-------+     +------------+     +------------+
 94 |  Client A -> | GET /1.1/user.json |     |       |     |            | <-> |   Tweets   |
 95 |              +--------------------+     |       |     |            |     +------------+
 96 |              +--------------------+     |       |     |            | <-> | Activities |
 97 |  Client B -> | GET /1.1/user.json | <-> | nginx | <-> | Tweetstorm |     +------------+
 98 |              +--------------------+     |       |     |            | <-> |   Friends  |
 99 |              +--------------------+     |       |     |            |     +------------+
100 |  Client C -> | GET /1.1/user.json |     |       |     |            | <-> |    etc...  |
101 |              +--------------------+     +-------+     +------------+     +------------+
102 |           ~        userstream.twitter.com          ~  127.0.0.1:8080 (Default)
103 | ```
104 |  
105 | Tweetstorm は次の JSON データを提供します。  
106 | - ツイート  
107 |   - `list_id` でリストを指定したかどうかや, そのリストに自分のアカウントが含まれているかどうかで挙動が異なります。  
108 |   
109 |     |`list_id` を指定した?|そのリストに自分が含まれている?|取得されるタイムライン|
110 |     |:--:|:--:|--:|
111 |     |Yes|Yes|List, User, Mentions|
112 |     |Yes|No|List, User, Mentions|
113 |     |No|-|Home, User, Mentions|
114 |     
115 |     なお, Tweetstorm にはリストにフォローユーザを同期する機能があります。これはデフォルトでは無効ですが, `sync_list_following` で切り替えることで有効にできます。
116 |     
117 |   - タイムラインによってレートリミットが異なります。そのため取得間隔に違いがあります。  
118 |   
119 |     |タイムライン|API レートリミット|Tweetstorm でのデフォルトの取得間隔|
120 |     |:--:|:--:|--:|
121 |     |Home|15回/15分|75秒|
122 |     |List|900回/15分|1.5秒 (1500 ms)|
123 |     |User|900回/15分|1.5秒 (1500 ms)|
124 |     |Mentions|75回/15分|30秒| 
125 | 
126 | - ダイレクトメッセージ  
127 |   - デフォルトで有効ですが, `enable_direct_message` で切り替えできます。
128 |   - レートリミットは 15回/15分 で, デフォルトの取得間隔は 75秒 です。
129 | - アクティビティ  
130 |   - デフォルトでは無効です。有効にするには `enable_activity` で切り替え, `Twitter for iPhone` のアクセストークンを設定する必要があります。
131 |   - レートリミットは 180回/15分 で, デフォルトの取得間隔は 8秒 です。
132 |   - 自分が関係しているイベントかつポジティブなイベントだけ配信されます。
133 |     - 例えば, 自分のツイートに対するお気に入りイベントは得られますが, お気に入り解除イベントや, 他人による他人のツイートへのお気に入りイベントを得ることはできません。
134 |     - この制約のため, 従来のUserStreamであった, フォローユーザのアクティビティを取得する `include_friends_activity=true` はもう使えません。
135 | - フレンドID  
136 |   - デフォルトで有効ですが, `enable_friends` で切り替えできます。  
137 |   - 従来の UserStream であった, 接続開始直後に流れてきていた `{"friends": [11111, 22222, ...]}` です。  
138 |   - 従来どおり `stringify_friend_ids=true` パラメータで文字列配列で ID を受け取れます。  
139 | - FilterStream  
140 |   - デフォルトでは無効です。有効にするには `filter_stream_tracks` でトラックするワードを, `filter_stream_follows` でトラックするユーザ ID を設定する必要があります。  
141 |   - 指定したワードを含むツイートや, 指定したユーザからのツイート, およびそれらの削除通知が配信されます。  
142 | - SampleStream
143 |   - デフォルトでは無効です。有効にするには `enable_sample_stream` で切り替える必要があります。
144 |   - Twitter に投稿されるツイートの一部とそれらの削除通知が配信されます。
145 | 
146 | これら以外にも従来の UserStream で配信されていたデータもありますが, API の仕様変更により Tweetstorm はそれらを提供できません。
147 | 
148 | ## Wiki
149 | セットアップ, 互換性などのセクションは [Wiki](https://github.com/SlashNephy/Tweetstorm/wiki) に移動しました。  
150 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/task/producer/Timeline.kt:
--------------------------------------------------------------------------------
  1 | package blue.starry.tweetstorm.task.producer
  2 | 
  3 | import blue.starry.jsonkt.copy
  4 | import blue.starry.jsonkt.string
  5 | import blue.starry.penicillin.core.exceptions.PenicillinTwitterApiException
  6 | import blue.starry.penicillin.core.exceptions.TwitterApiError
  7 | import blue.starry.penicillin.core.request.action.JsonArrayApiAction
  8 | import blue.starry.penicillin.core.session.ApiClient
  9 | import blue.starry.penicillin.endpoints.common.TweetMode
 10 | import blue.starry.penicillin.endpoints.timeline
 11 | import blue.starry.penicillin.endpoints.timeline.homeTimeline
 12 | import blue.starry.penicillin.endpoints.timeline.listTimeline
 13 | import blue.starry.penicillin.endpoints.timeline.mentionsTimeline
 14 | import blue.starry.penicillin.endpoints.timeline.userTimeline
 15 | import blue.starry.penicillin.extensions.createdAt
 16 | import blue.starry.penicillin.extensions.executeWithTimeout
 17 | import blue.starry.penicillin.extensions.instant
 18 | import blue.starry.penicillin.extensions.rateLimit
 19 | import blue.starry.penicillin.models.Status
 20 | import blue.starry.tweetstorm.Config
 21 | import blue.starry.tweetstorm.config
 22 | import blue.starry.tweetstorm.task.ProduceTask
 23 | import blue.starry.tweetstorm.task.data.JsonObjectData
 24 | import io.ktor.util.date.toJvmDate
 25 | import kotlinx.coroutines.*
 26 | import kotlinx.coroutines.channels.produce
 27 | import kotlinx.coroutines.time.delay
 28 | import java.time.Duration
 29 | import java.time.Instant
 30 | import java.util.concurrent.atomic.AtomicLong
 31 | import kotlin.coroutines.CoroutineContext
 32 | 
 33 | class ListTimeline(account: Config.Account, client: ApiClient, filter: (Status) -> Boolean = { true }): TimelineTask(account, account.refresh.listTimeline, {
 34 |     client.timeline.listTimeline(listId = account.listId!!, count = 200, sinceId = it, includeEntities = true, includeRTs = true, includeMyRetweet = true, tweetMode = TweetMode.Extended)
 35 | }, filter)
 36 | 
 37 | class HomeTimeline(account: Config.Account, client: ApiClient, filter: (Status) -> Boolean = { true }): TimelineTask(account, account.refresh.homeTimeline, {
 38 |     client.timeline.homeTimeline(count = 200, sinceId = it, includeEntities = true, includeRTs = true, includeMyRetweet = true, tweetMode = TweetMode.Extended)
 39 | }, filter)
 40 | 
 41 | class UserTimeline(account: Config.Account, client: ApiClient, filter: (Status) -> Boolean = { true }): TimelineTask(account, account.refresh.userTimeline, {
 42 |     client.timeline.userTimeline(count = 200, sinceId = it, includeEntities = true, includeRTs = true, includeMyRetweet = true, tweetMode = TweetMode.Extended)
 43 | }, filter)
 44 | 
 45 | class MentionTimeline(account: Config.Account, client: ApiClient, filter: (Status) -> Boolean = { true }): TimelineTask(account, account.refresh.mentionTimeline, {
 46 |     client.timeline.mentionsTimeline(count = 200, sinceId = it, includeEntities = true, includeRTs = true, includeMyRetweet = true, tweetMode = TweetMode.Extended)
 47 | }, filter)
 48 | 
 49 | abstract class TimelineTask(account: Config.Account, private val time: Long, private val source: (lastId: Long?) -> JsonArrayApiAction<Status>, private val filter: (Status) -> Boolean): ProduceTask<JsonObjectData>(account) {
 50 |     @ExperimentalCoroutinesApi
 51 |     override fun channel(context: CoroutineContext, parent: Job) = GlobalScope.produce(context + parent) {
 52 |         val lastId = AtomicLong(0L)
 53 |         while (isActive) {
 54 |             try {
 55 |                 val lastIdOrNull = lastId.get().let { if (it > 0) it else null }
 56 |                 val timeline = source(lastIdOrNull).executeWithTimeout(config.app.apiTimeout) ?: continue
 57 |                 if (timeline.isNotEmpty()) {
 58 |                     if (lastIdOrNull != null) {
 59 |                         timeline.reversed().filter(filter).forEach {
 60 |                             send(JsonObjectData(it.postProcess()))
 61 |                         }
 62 |                     }
 63 | 
 64 |                     lastId.set(timeline.first().id)
 65 |                 }
 66 | 
 67 |                 val rateLimit = timeline.rateLimit
 68 |                 if (rateLimit != null) {
 69 |                     val duration = Duration.between(Instant.now(), rateLimit.resetAt.toJvmDate().toInstant())
 70 |                     if (rateLimit.remaining < 2) {
 71 |                         logger.warn { "Rate limit: Mostly exceeded. Sleep ${duration.seconds} secs. (Reset at ${rateLimit.resetAt})" }
 72 |                         delay(duration)
 73 |                     } else if (duration.seconds > 3 && rateLimit.remaining * time.toDouble() / duration.seconds < 1) {
 74 |                         logger.warn { "Rate limit: API calls (/${timeline.request.url}) seem to be frequent than expected so consider adjusting `*_timeline_refresh` value in config.json. Sleep 10 secs. (${rateLimit.remaining}/${rateLimit.limit}, Reset at ${rateLimit.resetAt})" }
 75 |                         delay(10000)
 76 |                     }
 77 |                     logger.trace { "Rate limit: ${rateLimit.remaining}/${rateLimit.limit}, Reset at ${rateLimit.resetAt}" }
 78 |                 }
 79 | 
 80 |                 delay(time)
 81 |             } catch (e: CancellationException) {
 82 |                 break
 83 |             } catch (e: PenicillinTwitterApiException) {
 84 |                 if (e.error == TwitterApiError.RateLimitExceeded) {
 85 |                     delay(10000)
 86 |                 } else {
 87 |                     logger.error(e) { "An error occurred while getting timeline." }
 88 |                     delay(time)
 89 |                 }
 90 |             }
 91 |         }
 92 |     }
 93 | }
 94 | 
 95 | private fun Status.postProcess() = json.copy {
 96 |     // For compatibility
 97 |     it["text"] = json["full_text"]!!.string
 98 |     it["truncated"] = false
 99 |     it.remove("display_text_range")
100 |     it["quote_count"] = 0
101 |     it["reply_count"] = 0
102 |     it["possibly_sensitive"] = false
103 |     it["filter_level"] = "low"
104 |     it["timestamp_ms"] = createdAt.instant.epochSecond * 1000
105 | }
106 | 


--------------------------------------------------------------------------------
/gradlew:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env sh
  2 | 
  3 | #
  4 | # Copyright 2015 the original author or authors.
  5 | #
  6 | # Licensed under the Apache License, Version 2.0 (the "License");
  7 | # you may not use this file except in compliance with the License.
  8 | # You may obtain a copy of the License at
  9 | #
 10 | #      https://www.apache.org/licenses/LICENSE-2.0
 11 | #
 12 | # Unless required by applicable law or agreed to in writing, software
 13 | # distributed under the License is distributed on an "AS IS" BASIS,
 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 | # See the License for the specific language governing permissions and
 16 | # limitations under the License.
 17 | #
 18 | 
 19 | ##############################################################################
 20 | ##
 21 | ##  Gradle start up script for UN*X
 22 | ##
 23 | ##############################################################################
 24 | 
 25 | # Attempt to set APP_HOME
 26 | # Resolve links: $0 may be a link
 27 | PRG="$0"
 28 | # Need this for relative symlinks.
 29 | while [ -h "$PRG" ] ; do
 30 |     ls=`ls -ld "$PRG"`
 31 |     link=`expr "$ls" : '.*-> \(.*\)$'`
 32 |     if expr "$link" : '/.*' > /dev/null; then
 33 |         PRG="$link"
 34 |     else
 35 |         PRG=`dirname "$PRG"`"/$link"
 36 |     fi
 37 | done
 38 | SAVED="`pwd`"
 39 | cd "`dirname \"$PRG\"`/" >/dev/null
 40 | APP_HOME="`pwd -P`"
 41 | cd "$SAVED" >/dev/null
 42 | 
 43 | APP_NAME="Gradle"
 44 | APP_BASE_NAME=`basename "$0"`
 45 | 
 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 48 | 
 49 | # Use the maximum available, or set MAX_FD != -1 to use that value.
 50 | MAX_FD="maximum"
 51 | 
 52 | warn () {
 53 |     echo "$*"
 54 | }
 55 | 
 56 | die () {
 57 |     echo
 58 |     echo "$*"
 59 |     echo
 60 |     exit 1
 61 | }
 62 | 
 63 | # OS specific support (must be 'true' or 'false').
 64 | cygwin=false
 65 | msys=false
 66 | darwin=false
 67 | nonstop=false
 68 | case "`uname`" in
 69 |   CYGWIN* )
 70 |     cygwin=true
 71 |     ;;
 72 |   Darwin* )
 73 |     darwin=true
 74 |     ;;
 75 |   MINGW* )
 76 |     msys=true
 77 |     ;;
 78 |   NONSTOP* )
 79 |     nonstop=true
 80 |     ;;
 81 | esac
 82 | 
 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 84 | 
 85 | 
 86 | # Determine the Java command to use to start the JVM.
 87 | if [ -n "$JAVA_HOME" ] ; then
 88 |     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
 89 |         # IBM's JDK on AIX uses strange locations for the executables
 90 |         JAVACMD="$JAVA_HOME/jre/sh/java"
 91 |     else
 92 |         JAVACMD="$JAVA_HOME/bin/java"
 93 |     fi
 94 |     if [ ! -x "$JAVACMD" ] ; then
 95 |         die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
 96 | 
 97 | Please set the JAVA_HOME variable in your environment to match the
 98 | location of your Java installation."
 99 |     fi
100 | else
101 |     JAVACMD="java"
102 |     which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
103 | 
104 | Please set the JAVA_HOME variable in your environment to match the
105 | location of your Java installation."
106 | fi
107 | 
108 | # Increase the maximum file descriptors if we can.
109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
110 |     MAX_FD_LIMIT=`ulimit -H -n`
111 |     if [ $? -eq 0 ] ; then
112 |         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
113 |             MAX_FD="$MAX_FD_LIMIT"
114 |         fi
115 |         ulimit -n $MAX_FD
116 |         if [ $? -ne 0 ] ; then
117 |             warn "Could not set maximum file descriptor limit: $MAX_FD"
118 |         fi
119 |     else
120 |         warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
121 |     fi
122 | fi
123 | 
124 | # For Darwin, add options to specify how the application appears in the dock
125 | if $darwin; then
126 |     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
127 | fi
128 | 
129 | # For Cygwin or MSYS, switch paths to Windows format before running java
130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
131 |     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
132 |     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
133 | 
134 |     JAVACMD=`cygpath --unix "$JAVACMD"`
135 | 
136 |     # We build the pattern for arguments to be converted via cygpath
137 |     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
138 |     SEP=""
139 |     for dir in $ROOTDIRSRAW ; do
140 |         ROOTDIRS="$ROOTDIRS$SEP$dir"
141 |         SEP="|"
142 |     done
143 |     OURCYGPATTERN="(^($ROOTDIRS))"
144 |     # Add a user-defined pattern to the cygpath arguments
145 |     if [ "$GRADLE_CYGPATTERN" != "" ] ; then
146 |         OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
147 |     fi
148 |     # Now convert the arguments - kludge to limit ourselves to /bin/sh
149 |     i=0
150 |     for arg in "$@" ; do
151 |         CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
152 |         CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
153 | 
154 |         if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
155 |             eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
156 |         else
157 |             eval `echo args$i`="\"$arg\""
158 |         fi
159 |         i=`expr $i + 1`
160 |     done
161 |     case $i in
162 |         0) set -- ;;
163 |         1) set -- "$args0" ;;
164 |         2) set -- "$args0" "$args1" ;;
165 |         3) set -- "$args0" "$args1" "$args2" ;;
166 |         4) set -- "$args0" "$args1" "$args2" "$args3" ;;
167 |         5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
168 |         6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
169 |         7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
170 |         8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
171 |         9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
172 |     esac
173 | fi
174 | 
175 | # Escape application args
176 | save () {
177 |     for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
178 |     echo " "
179 | }
180 | APP_ARGS=`save "$@"`
181 | 
182 | # Collect all arguments for the java command, following the shell quoting and substitution rules
183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
184 | 
185 | exec "$JAVACMD" "$@"
186 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/TaskManager.kt:
--------------------------------------------------------------------------------
  1 | package blue.starry.tweetstorm
  2 | 
  3 | import blue.starry.penicillin.core.exceptions.PenicillinException
  4 | import blue.starry.penicillin.endpoints.lists
  5 | import blue.starry.penicillin.endpoints.lists.member
  6 | import blue.starry.penicillin.extensions.complete
  7 | import blue.starry.tweetstorm.session.AuthenticatedStream
  8 | import blue.starry.tweetstorm.task.ProduceTask
  9 | import blue.starry.tweetstorm.task.RegularTask
 10 | import blue.starry.tweetstorm.task.producer.*
 11 | import blue.starry.tweetstorm.task.regular.SyncList
 12 | import kotlinx.coroutines.*
 13 | import kotlinx.coroutines.channels.consumeEach
 14 | import kotlinx.coroutines.selects.select
 15 | import kotlinx.coroutines.sync.Mutex
 16 | import kotlinx.coroutines.sync.withLock
 17 | import java.io.Closeable
 18 | import java.util.concurrent.CopyOnWriteArraySet
 19 | 
 20 | class TaskManager(initialStream: AuthenticatedStream): Closeable {
 21 |     companion object {
 22 |         val instances = CopyOnWriteArraySet<TaskManager>()
 23 |         val mutex = Mutex()
 24 |     }
 25 | 
 26 |     private val logger = logger("Tweetstorm.TaskManager (@${initialStream.account.user.screenName})")
 27 |     private val masterJob = Job()
 28 | 
 29 |     val account = initialStream.account
 30 | 
 31 |     private val streams = CopyOnWriteArraySet<AuthenticatedStream>().also {
 32 |         it += initialStream
 33 |     }
 34 |     private val streamsMutex = Mutex()
 35 |     private val twitterClient = account.twitter
 36 | 
 37 |     fun anyClients(): Boolean {
 38 |         return streams.isNotEmpty()
 39 |     }
 40 | 
 41 |     private val tasks = object {
 42 |         val produce = mutableListOf<ProduceTask<*>>().also {
 43 |             if (account.listId != null) {
 44 |                 it += ListTimeline(account, twitterClient)
 45 | 
 46 |                 val listContainsSelf = try {
 47 |                     twitterClient.lists.member(listId = account.listId!!, userId = account.user.id).complete()
 48 |                     true
 49 |                 } catch (e: PenicillinException) {
 50 |                     false
 51 |                 }
 52 | 
 53 |                 if (listContainsSelf && account.syncList.enabled) {
 54 |                     it += UserTimeline(account, twitterClient) { status ->
 55 |                         status.retweetedStatus == null && status.inReplyToUserId != null && status.inReplyToUserId!! !in account.friends
 56 |                     }
 57 |                     it += MentionTimeline(account, twitterClient) { status ->
 58 |                         status.user.id !in account.friends
 59 |                     }
 60 |                 } else {
 61 |                     it += UserTimeline(account, twitterClient)
 62 |                     it += MentionTimeline(account, twitterClient)
 63 |                 }
 64 |             } else {
 65 |                 it += HomeTimeline(account, twitterClient)
 66 |                 it += UserTimeline(account, twitterClient) { status ->
 67 |                     status.retweetedStatus == null && status.inReplyToUserId != null && status.inReplyToUserId!! !in account.friends
 68 |                 }
 69 |                 it += MentionTimeline(account, twitterClient) { status ->
 70 |                     status.user.id !in account.friends
 71 |                 }
 72 |             }
 73 | 
 74 |             if (account.enableDirectMessage) {
 75 |                 it += DirectMessage(account, twitterClient)
 76 |             }
 77 | 
 78 |             if (account.filterStream.tracks.isNotEmpty() || account.filterStream.follows.isNotEmpty()) {
 79 |                 it += FilterStream(account, twitterClient)
 80 |             }
 81 | 
 82 |             if (account.enableSampleStream) {
 83 |                 it += SampleStream(account, twitterClient)
 84 |             }
 85 | 
 86 |             it += Heartbeat(account)
 87 |         }
 88 | 
 89 |         val regular = mutableListOf<RegularTask>().also {
 90 |             if (account.syncList.enabled && account.listId != null) {
 91 |                 it += SyncList(account, twitterClient)
 92 |             }
 93 |         }
 94 |     }
 95 | 
 96 |     @ExperimentalCoroutinesApi
 97 |     suspend fun register(stream: AuthenticatedStream) {
 98 |         if (streamsMutex.withLock { streams.add(stream) }) {
 99 |             stream.startTargetedTasks()
100 |             logger.debug { "A Stream has been registered to @${account.user.screenName}." }
101 |         }
102 |     }
103 | 
104 |     private suspend fun unregister(stream: AuthenticatedStream) {
105 |         if (streamsMutex.withLock { streams.remove(stream) }) {
106 |             stream.close()
107 |             logger.debug { "A Stream has been unregistered from @${account.user.screenName}." }
108 |         }
109 |     }
110 | 
111 |     suspend fun wait(stream: AuthenticatedStream) {
112 |         while (stream.handler.isAlive) {
113 |             delay(1000)
114 |         }
115 | 
116 |         unregister(stream)
117 |     }
118 | 
119 |     @ObsoleteCoroutinesApi
120 |     @ExperimentalCoroutinesApi
121 |     suspend fun start(target: AuthenticatedStream) {
122 |         target.startTargetedTasks()
123 | 
124 |         for (task in tasks.produce) {
125 |             GlobalScope.launch(masterJob) {
126 |                 task.logger.debug { "ProduceTask: ${task.javaClass.simpleName} started." }
127 | 
128 |                 while (isActive) {
129 |                     task.channel(coroutineContext, masterJob).consumeEach { data ->
130 |                         for (stream in streams) {
131 |                             if (data.emit(stream.handler)) {
132 |                                 task.logger.trace { "${data.javaClass.simpleName} (${task.javaClass.simpleName}) emitted successfully." }
133 |                             } else {
134 |                                 task.logger.trace { "${data.javaClass.simpleName} (${task.javaClass.simpleName}) failed to deliver." }
135 |                                 unregister(stream)
136 |                             }
137 |                         }
138 |                     }
139 |                 }
140 |             }
141 |         }
142 | 
143 |         for (task in tasks.regular) {
144 |             GlobalScope.launch(masterJob) {
145 |                 while (isActive) {
146 |                     task.logger.debug { "RegularTask: ${task.javaClass.simpleName} started." }
147 | 
148 |                     try {
149 |                         task.run()
150 |                         task.logger.trace { "RegularTask: ${task.javaClass.simpleName} finished successfully." }
151 |                         delay(task.unit.toMillis(task.interval))
152 |                     } catch (e: CancellationException) {
153 |                         task.logger.debug { "RegularTask: ${task.javaClass.simpleName} will terminate." }
154 |                         break
155 |                     } catch (e: Exception) {
156 |                         task.logger.error(e) { "An error occurred while regular task." }
157 |                         delay(task.unit.toMillis(task.interval))
158 |                     }
159 |                 }
160 |             }
161 |         }
162 |     }
163 | 
164 |     @ExperimentalCoroutinesApi
165 |     private suspend fun AuthenticatedStream.startTargetedTasks() {
166 |         GlobalScope.launch(masterJob) {
167 |             select {
168 |                 if (account.enableFriends) {
169 |                     val task = Friends(this@startTargetedTasks)
170 |                     launch(job) {
171 |                         task.channel(coroutineContext, masterJob).onReceive { data ->
172 |                             if (data.emit(handler)) {
173 |                                 task.logger.trace { "${data.javaClass.simpleName} (${task.javaClass.simpleName}) emitted successfully." }
174 |                             } else {
175 |                                 task.logger.debug { "${data.javaClass.simpleName} (${task.javaClass.simpleName}) failed to deliver." }
176 |                                 unregister(this@startTargetedTasks)
177 |                             }
178 |                         }
179 |                     }
180 | 
181 |                     task.logger.debug { "TargetedProduceTask: ${task.javaClass.simpleName} started." }
182 |                 }
183 |             }
184 |         }
185 |     }
186 | 
187 |     override fun close() {
188 |         twitterClient.close()
189 | 
190 |         runBlocking(Dispatchers.Default) {
191 |             masterJob.cancelChildren()
192 |             masterJob.cancelAndJoin()
193 |         }
194 |     }
195 | }
196 | 


--------------------------------------------------------------------------------
/src/jvmMain/kotlin/blue/starry/tweetstorm/Routing.kt:
--------------------------------------------------------------------------------
  1 | package blue.starry.tweetstorm
  2 | 
  3 | import io.ktor.application.ApplicationCall
  4 | import io.ktor.application.call
  5 | import io.ktor.features.origin
  6 | import io.ktor.html.respondHtmlTemplate
  7 | import io.ktor.http.HttpStatusCode
  8 | import io.ktor.request.httpMethod
  9 | import io.ktor.request.path
 10 | import io.ktor.request.receiveParameters
 11 | import io.ktor.response.respond
 12 | import io.ktor.routing.Route
 13 | import io.ktor.routing.get
 14 | import io.ktor.routing.post
 15 | import io.ktor.routing.route
 16 | import io.ktor.util.pipeline.PipelineContext
 17 | import blue.starry.tweetstorm.session.AuthenticatedStream
 18 | import blue.starry.tweetstorm.session.DemoStream
 19 | import blue.starry.tweetstorm.session.PreAuthenticatedStream
 20 | import blue.starry.tweetstorm.session.StreamContent
 21 | import io.ktor.utils.io.ByteWriteChannel
 22 | import kotlinx.html.*
 23 | 
 24 | private val logger = logger("Tweetstorm.Routing")
 25 | 
 26 | fun Route.getTop() {
 27 |     get("/") {
 28 |         call.respondHtmlTemplate(FooterLayout()) {
 29 |             footerContent {
 30 |                 style {
 31 |                     unsafe {
 32 |                         +".github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}"
 33 |                     }
 34 |                 }
 35 |                 a("https://github.com/SlashNephy/Tweetstorm", "_blank", "github-corner") {
 36 |                     attributes["aria-label"] = "View source on Github"
 37 |                     unsafe { +"""<svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg>""" }
 38 |                 }
 39 |                 div("jumbotron") {
 40 |                     h1 { +"Tweetstorm" }
 41 |                     hr("my-4")
 42 |                     p("lead") { +"A simple substitute implementation for the Twitter UserStream." }
 43 |                 }
 44 |                 div("container") {
 45 |                     p {
 46 |                         +"Tweetstorm is working fine "
 47 |                         span("far fa-smile-wink")
 48 |                         +" Have fun!"
 49 |                     }
 50 |                     div("list-group") {
 51 |                         a("https://github.com/SlashNephy/Tweetstorm", "_blank", "list-group-item list-group-item-action") {
 52 |                             span("fab fa-github")
 53 |                             +" GitHub"
 54 |                         }
 55 |                         a("https://github.com/SlashNephy/Tweetstorm/wiki/Setup-[ja]", "_blank", "list-group-item list-group-item-action") {
 56 |                             span("fas fa-book")
 57 |                             +" Setup [ja]"
 58 |                         }
 59 |                         a("https://twitter.com/SlashNephy", "_blank", "list-group-item list-group-item-action") {
 60 |                             span("fab fa-twitter")
 61 |                             +" Contact"
 62 |                         }
 63 |                     }
 64 |                 }
 65 |             }
 66 |         }
 67 |     }
 68 | }
 69 | 
 70 | fun Route.getUser() {
 71 |     get("/1.1/user.json") {
 72 |         val strict = call.request.headers.parseAuthorizationHeaderStrict(call.request.local.method, "https://userstream.twitter.com/1.1/user.json", call.request.queryParameters)
 73 |         val simple = call.request.headers.parseAuthorizationHeaderSimple()
 74 |         val account = strict ?: simple
 75 |         var authOK = strict != null || (config.app.skipAuth && account != null)
 76 | 
 77 |         call.respondStream { writer ->
 78 |             if (account != null && !config.app.skipAuth && !authOK) {
 79 |                 PreAuthenticatedStream(writer, call.request, account).use { stream ->
 80 |                     if (stream.await()) {
 81 |                         authOK = true
 82 |                         logger.info { "Client: @${account.user.screenName} (${call.request.origin.remoteHost}) has passed account-token authentication." }
 83 |                     } else {
 84 |                         logger.warn { "Client: @${account.user.screenName} (${call.request.origin.remoteHost}) has failed account-token authentication." }
 85 |                     }
 86 |                 }
 87 |             }
 88 | 
 89 |             if (account != null && authOK) {
 90 |                 AuthenticatedStream(writer, call.request, account)
 91 |             } else {
 92 |                 DemoStream(writer, call.request)
 93 |             }.use { stream ->
 94 |                 stream.await()
 95 |             }
 96 |         }
 97 |     }
 98 | }
 99 | 
100 | fun Route.authByToken() {
101 |     route("/auth/token/{urlToken}") {
102 |         get {
103 |             val urlToken = call.parameters["urlToken"]
104 |             if (urlToken == null || !PreAuthenticatedStream.check(urlToken)) {
105 |                 call.respondHtmlTemplate(NavLayout()) {
106 |                     navContent {
107 |                         div("alert alert-dismissible alert-danger") {
108 |                             h4 {
109 |                                 span("far fa-sad-tear")
110 |                                 +" Requested auth token is invalid."
111 |                             }
112 |                             p { +"Try connecting from client you want to use." }
113 |                         }
114 |                     }
115 | 
116 |                 }
117 |             } else {
118 |                 call.respondHtmlTemplate(NavLayout()) {
119 |                     navContent {
120 |                         div("alert alert-dismissible alert-warning") {
121 |                             h4 {
122 |                                 span("far fa-surprise")
123 |                                 +" Authentication Request"
124 |                             }
125 |                             p {
126 |                                 +"Enter your account token which you defined in "
127 |                                 code { +"config.json" }
128 |                                 +"."
129 |                             }
130 |                         }
131 | 
132 |                         form(method = FormMethod.post) {
133 |                             attributes["data-bitwarden-watching"] = "1"
134 |                             div("form-group") {
135 |                                 label {
136 |                                     attributes["for"] = "token"
137 |                                     +"Token"
138 |                                 }
139 |                                 input(type = InputType.password, name = "token", classes = "form-control") {
140 |                                     attributes["id"] = "token"
141 |                                 }
142 |                             }
143 |                             input(type = InputType.submit, classes = "btn btn-primary") {
144 |                                 value = "Submit"
145 |                             }
146 |                         }
147 |                     }
148 |                 }
149 |             }
150 |         }
151 | 
152 |         post {
153 |             val urlToken = call.parameters["urlToken"]
154 |             if (urlToken == null || !PreAuthenticatedStream.check(urlToken)) {
155 |                 return@post call.respond(HttpStatusCode.NotFound)
156 |             }
157 |             val accountToken = call.receiveParameters()["token"]
158 |                     ?: return@post call.respond(HttpStatusCode.Unauthorized)
159 | 
160 |             if (PreAuthenticatedStream.auth(urlToken, accountToken)) {
161 |                 call.respondHtmlTemplate(NavLayout()) {
162 |                     navContent {
163 |                         div("alert alert-dismissible alert-success") {
164 |                             h4 {
165 |                                 span("far fa-grin-squint")
166 |                                 +" Your token is accepted!"
167 |                             }
168 |                             p { +"Streaming starts shortly. Enjoy!" }
169 |                         }
170 |                     }
171 |                 }
172 |             } else {
173 |                 call.respondHtmlTemplate(NavLayout(), HttpStatusCode.Unauthorized) {
174 |                     navContent {
175 |                         div("alert alert-dismissible alert-danger") {
176 |                             h4 {
177 |                                 span("far fa-sad-tear")
178 |                                 +" Your token is invalid!"
179 |                             }
180 |                             p { +"Streaming can't start for this session." }
181 |                         }
182 |                         form(method = FormMethod.post) {
183 |                             attributes["data-bitwarden-watching"] = "1"
184 |                             div("form-group") {
185 |                                 label {
186 |                                     attributes["for"] = "token"
187 |                                     +"Token"
188 |                                 }
189 |                                 input(type = InputType.password, name = "token", classes = "form-control") {
190 |                                     attributes["id"] = "token"
191 |                                 }
192 |                             }
193 |                             input(type = InputType.submit, classes = "btn btn-primary") {
194 |                                 value = "Submit"
195 |                             }
196 | 
197 |                         }
198 |                     }
199 |                 }
200 |             }
201 |         }
202 |     }
203 | }
204 | 
205 | suspend fun PipelineContext<*, ApplicationCall>.notFound() {
206 |     call.respondHtmlTemplate(NavLayout(), HttpStatusCode.NotFound) {
207 |         navContent {
208 |             div("alert alert-dismissible alert-danger") {
209 |                 h4 {
210 |                     span("far fa-sad-tear")
211 |                     +" 404 Page Not Found"
212 |                 }
213 |                 p { +"${call.request.httpMethod.value.toUpperCase()} ${call.request.path()}" }
214 |             }
215 |         }
216 |     }
217 | }
218 | 
219 | private suspend fun ApplicationCall.respondStream(writer: suspend (channel: ByteWriteChannel) -> Unit) {
220 |     respond(StreamContent(writer))
221 | }
222 | 


--------------------------------------------------------------------------------