├── highlighter-sliding-window
    ├── src
    │   ├── main
    │   │   ├── resources
    │   │   │   ├── application.properties
    │   │   │   └── logback.xml
    │   │   └── java
    │   │   │   └── kafkastreams
    │   │   │       └── example
    │   │   │           ├── .DS_Store
    │   │   │           └── KStream
    │   │   │               ├── Consumer.java
    │   │   │               ├── Producer.java
    │   │   │               └── KStreams.java
    │   └── test
    │   │   └── java
    │   │       └── kafkastreams
    │   │           └── example
    │   │               └── ExampleApplicationTests.java
    ├── settings.gradle
    ├── gradle
    │   └── wrapper
    │   │   ├── gradle-wrapper.jar
    │   │   └── gradle-wrapper.properties
    ├── .gitignore
    ├── build.gradle
    ├── gradlew.bat
    └── gradlew
├── .DS_Store
├── youtube-live
    ├── .env.example
    ├── chat
    │   └── tmp.text
    ├── requirements.txt
    ├── Dockerfile
    ├── README.md
    ├── docker-compose.yaml
    └── app.py
├── assets
    ├── logo.png
    ├── influx_01.png
    ├── influx_02.png
    ├── influx_03.png
    ├── influx_04.png
    ├── influx_05.png
    ├── influx_06.png
    ├── influx_07.png
    ├── influx_08.png
    ├── influx_09.png
    ├── influx_10.png
    ├── influx_11.png
    ├── influx_12.png
    ├── influx_13.png
    ├── influx_14.png
    └── architecture.png
├── influx
    ├── requirements.txt
    ├── delete_bucket.py
    ├── delete_measurement.py
    ├── read_binary.py
    ├── insert_text.py
    ├── explanation.md
    ├── insert_binary.py
    ├── Readme.md
    ├── influxDB_video.py
    └── influxDB_range.py
├── spark-streaming
    ├── requirements.txt
    ├── README.md
    └── spark-batch.py
├── terraform
    ├── README.md
    └── main.tf
├── video-stream
    ├── requirements.txt
    ├── README.md
    ├── producer.py
    ├── docker-compose.yaml
    ├── trigger_request.py
    ├── main.py
    ├── sqs_trigger_task.py
    └── sdk-test.ipynb
├── timeframe
    └── normalize_time.py
├── README.md
├── .gitignore
└── video-upload
    └── lambda_videoUpload.py


/highlighter-sliding-window/src/main/resources/application.properties:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/highlighter-sliding-window/settings.gradle:
--------------------------------------------------------------------------------
1 | rootProject.name = 'example'
2 | 


--------------------------------------------------------------------------------
/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/.DS_Store


--------------------------------------------------------------------------------
/youtube-live/.env.example:
--------------------------------------------------------------------------------
1 | GCP_KEY=GCP_KEY
2 | OPENAI_API_KEY=OPENAI_API_KEY
3 | 


--------------------------------------------------------------------------------
/youtube-live/chat/tmp.text:
--------------------------------------------------------------------------------
1 | /* only for upload folder, you can delete for free */


--------------------------------------------------------------------------------
/assets/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/logo.png


--------------------------------------------------------------------------------
/influx/requirements.txt:
--------------------------------------------------------------------------------
1 | confluent_kafka==2.5.0
2 | influxdb_client==1.45.0
3 | python-dotenv==1.0.1


--------------------------------------------------------------------------------
/assets/influx_01.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_01.png


--------------------------------------------------------------------------------
/assets/influx_02.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_02.png


--------------------------------------------------------------------------------
/assets/influx_03.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_03.png


--------------------------------------------------------------------------------
/assets/influx_04.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_04.png


--------------------------------------------------------------------------------
/assets/influx_05.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_05.png


--------------------------------------------------------------------------------
/assets/influx_06.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_06.png


--------------------------------------------------------------------------------
/assets/influx_07.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_07.png


--------------------------------------------------------------------------------
/assets/influx_08.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_08.png


--------------------------------------------------------------------------------
/assets/influx_09.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_09.png


--------------------------------------------------------------------------------
/assets/influx_10.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_10.png


--------------------------------------------------------------------------------
/assets/influx_11.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_11.png


--------------------------------------------------------------------------------
/assets/influx_12.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_12.png


--------------------------------------------------------------------------------
/assets/influx_13.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_13.png


--------------------------------------------------------------------------------
/assets/influx_14.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/influx_14.png


--------------------------------------------------------------------------------
/assets/architecture.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/assets/architecture.png


--------------------------------------------------------------------------------
/spark-streaming/requirements.txt:
--------------------------------------------------------------------------------
1 | pyspark
2 | numpy==1.26.4
3 | pandas
4 | influxdb-client
5 | openai-whisper


--------------------------------------------------------------------------------
/terraform/README.md:
--------------------------------------------------------------------------------
 1 | ## Run
 2 | 
 3 | ```bash
 4 | terraform init
 5 | terraform apply
 6 | ```
 7 | 
 8 | ## Destroy
 9 | ```bash
10 | terraform destroy
11 | ```
12 | 


--------------------------------------------------------------------------------
/youtube-live/requirements.txt:
--------------------------------------------------------------------------------
 1 | youtube-dl
 2 | pytchat
 3 | pafy
 4 | pandas
 5 | python-dotenv
 6 | fastapi
 7 | kafka-python
 8 | uvicorn
 9 | pymongo
10 | DateTime


--------------------------------------------------------------------------------
/highlighter-sliding-window/gradle/wrapper/gradle-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/highlighter-sliding-window/gradle/wrapper/gradle-wrapper.jar


--------------------------------------------------------------------------------
/highlighter-sliding-window/src/main/java/kafkastreams/example/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/YBIGTA/24th-de-highlighter/HEAD/highlighter-sliding-window/src/main/java/kafkastreams/example/.DS_Store


--------------------------------------------------------------------------------
/youtube-live/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.11-slim
 2 | 
 3 | WORKDIR /app
 4 | 
 5 | COPY requirements.txt ./
 6 | RUN pip install --no-cache-dir -r requirements.txt
 7 | 
 8 | COPY . .
 9 | EXPOSE 80
10 | CMD ["uvicorn", "app:app", "--reload"]
11 | 


--------------------------------------------------------------------------------
/highlighter-sliding-window/gradle/wrapper/gradle-wrapper.properties:
--------------------------------------------------------------------------------
1 | distributionBase=GRADLE_USER_HOME
2 | distributionPath=wrapper/dists
3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
4 | zipStoreBase=GRADLE_USER_HOME
5 | zipStorePath=wrapper/dists
6 | 


--------------------------------------------------------------------------------
/youtube-live/README.md:
--------------------------------------------------------------------------------
 1 | ## Youtube-live-chat
 2 | 실시간 유튜브 영상 url이 주어질 때 해당 영상의 livechat을 /chat 에 json형식으로 저장한다.
 3 | 
 4 | ### 실행
 5 | 1. .env 파일을 생성하고 GCP KEY를 넣어준다.
 6 | 2. docker-compose.yaml에 youtube live url을 넣어준다. 
 7 | 3. docker-compose를 실행한다.
 8 |     ```
 9 |     docker-compose up --build
10 |     ```


--------------------------------------------------------------------------------
/youtube-live/docker-compose.yaml:
--------------------------------------------------------------------------------
 1 | version: '3.8'
 2 | services:
 3 |   app:
 4 |     build:
 5 |       context: .
 6 |       dockerfile: Dockerfile
 7 |     image: youtube-live-app
 8 |     volumes:
 9 |       - ./chat:/app/chat
10 |     environment:
11 |        - YOUTUBE_URL="https://www.youtube.com/watch?v=ujB0bnmvCkk"


--------------------------------------------------------------------------------
/highlighter-sliding-window/src/test/java/kafkastreams/example/ExampleApplicationTests.java:
--------------------------------------------------------------------------------
 1 | package kafkastreams.example;
 2 | 
 3 | import org.junit.jupiter.api.Test;
 4 | import org.springframework.boot.test.context.SpringBootTest;
 5 | 
 6 | @SpringBootTest
 7 | class ExampleApplicationTests {
 8 | 
 9 | 	@Test
10 | 	void contextLoads() {
11 | 	}
12 | 
13 | }
14 | 


--------------------------------------------------------------------------------
/spark-streaming/README.md:
--------------------------------------------------------------------------------
1 | ### Spark Streaming
2 | 
3 | 1. Kafka topic에 적재된 video binary를 spark에 저장
4 | 2. spark window를 사용하여 30초 분량의 binary를 적재하고 이를 배치 단위로 전처리
5 |     1) 30초의 binary를 합쳐 video.ts로 저장 → ffmpeg로 video.ts를 video.mp4로 변환
6 |     2) whisper를 사용하여 mp4의 음성을 text로 전사
7 |     3) spark dataframe에 전사된 text를 새로운 column에 추가
8 | 3. spark query를 사용하여 전처리된 배치에 대해 foreachBatch()를 사용하여 influxdb에 적재


--------------------------------------------------------------------------------
/highlighter-sliding-window/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <configuration>
 2 |     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 3 |         <encoder>
 4 |             <pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
 5 |         </encoder>
 6 |     </appender>
 7 | 
 8 |     <root level="INFO">
 9 |         <appender-ref ref="STDOUT" />
10 |     </root>
11 | 
12 |     <logger name="org.apache.kafka" level="WARN"/>
13 | 
14 |     <logger name="kafkastreams.example.KStream" level="INFO"/>
15 | </configuration>


--------------------------------------------------------------------------------
/video-stream/requirements.txt:
--------------------------------------------------------------------------------
 1 | attrs==24.2.0
 2 | certifi==2024.7.4
 3 | charset-normalizer==3.3.2
 4 | confluent-kafka==2.5.0
 5 | h11==0.14.0
 6 | idna==3.7
 7 | isodate==0.6.1
 8 | lxml==5.3.0
 9 | outcome==1.3.0.post0
10 | pycountry==24.6.1
11 | pycryptodome==3.20.0
12 | PySocks==1.7.1
13 | requests==2.32.3
14 | setuptools==72.1.0
15 | six==1.16.0
16 | sniffio==1.3.1
17 | sortedcontainers==2.4.0
18 | streamlink==6.9.0
19 | trio==0.26.2
20 | trio-websocket==0.11.1
21 | typing_extensions==4.12.2
22 | urllib3==2.2.2
23 | websocket-client==1.8.0
24 | wheel==0.43.0
25 | wsproto==1.2.0
26 | 


--------------------------------------------------------------------------------
/highlighter-sliding-window/.gitignore:
--------------------------------------------------------------------------------
 1 | HELP.md
 2 | .gradle
 3 | build/
 4 | !gradle/wrapper/gradle-wrapper.jar
 5 | !**/src/main/**/build/
 6 | !**/src/test/**/build/
 7 | 
 8 | ### STS ###
 9 | .apt_generated
10 | .classpath
11 | .factorypath
12 | .project
13 | .settings
14 | .springBeans
15 | .sts4-cache
16 | bin/
17 | !**/src/main/**/bin/
18 | !**/src/test/**/bin/
19 | 
20 | ### IntelliJ IDEA ###
21 | .idea
22 | *.iws
23 | *.iml
24 | *.ipr
25 | out/
26 | !**/src/main/**/out/
27 | !**/src/test/**/out/
28 | 
29 | ### NetBeans ###
30 | /nbproject/private/
31 | /nbbuild/
32 | /dist/
33 | /nbdist/
34 | /.nb-gradle/
35 | 
36 | ### VS Code ###
37 | .vscode/
38 | 


--------------------------------------------------------------------------------
/video-stream/README.md:
--------------------------------------------------------------------------------
 1 | ## Overview
 2 | <img src="img/flow.svg">
 3 | 
 4 | - [x] Streamlink on EC2
 5 | - [x] EC2에서 SQS로 메시지 전송하기
 6 | - [x] Lambda에서 컨슈머 함수 구현
 7 | - [ ] Lambda 트리거 설정
 8 | - [x] S3에 영상 저장
 9 | 
10 | ## Run
11 | ```bash
12 | python main.py <Livestream URL> <Buffer size> <Time duration>
13 | ffmpeg -i vid.ts -c copy <Video filename>
14 | ```
15 | - 버퍼 크기: 131072 (128KB)
16 | - 시간 제한: 무제한
17 | 
18 | ## Buffer Size
19 | 
20 | 버퍼 크기 별 (총 데이터 크기) / (버퍼 크기 * 읽은 횟수) 값. 5분 단위로 측정.
21 | 
22 | |`BUF_SIZE`|Utilization|
23 | |-|-|
24 | |4 KB|0.9980|
25 | |64 KB|0.9677|
26 | |128 KB|0.9523|
27 | |512 KB|0.8093|
28 | 
29 | 크기를 키우면 SQS에 전송되는 메시지의 개수를 줄일 수 있음. 하지만 크기를 너무 키우면 스트림 받아오는 속도보다 읽는 속도가 빨라져서 빈 공간(?)이 많이 생기는 것 같음. 참고로 SQS의 메시지 크기 제한은 256KB.
30 | 


--------------------------------------------------------------------------------
/influx/delete_bucket.py:
--------------------------------------------------------------------------------
 1 | from influxdb_client import InfluxDBClient
 2 | from dotenv import load_dotenv
 3 | import os
 4 | 
 5 | load_dotenv()
 6 | 
 7 | # InfluxDB 환경 설정
 8 | token = os.getenv("INFLUXDB_TOKEN")
 9 | org = os.getenv("INFLUXDB_ORG")
10 | url = os.getenv("INFLUXDB_URL")
11 | 
12 | # InfluxDB 클라이언트 생성
13 | client = InfluxDBClient(url=url, token=token, org=org)
14 | 
15 | # Bucket API 사용
16 | buckets_api = client.buckets_api()
17 | 
18 | # 삭제할 버킷 이름
19 | bucket_to_delete = "삭제할 버킷 이름"
20 | 
21 | # 버킷 찾기
22 | bucket = buckets_api.find_bucket_by_name(bucket_to_delete)
23 | 
24 | if bucket:
25 |     # 버킷 삭제
26 |     buckets_api.delete_bucket(bucket)
27 |     print(f"Bucket '{bucket_to_delete}' has been deleted.")
28 | else:
29 |     print(f"Bucket '{bucket_to_delete}' not found.")


--------------------------------------------------------------------------------
/highlighter-sliding-window/build.gradle:
--------------------------------------------------------------------------------
 1 | plugins {
 2 | 	id 'java'
 3 | 	id 'org.springframework.boot' version '2.7.8'
 4 | 	id 'io.spring.dependency-management' version '1.0.15.RELEASE'
 5 | }
 6 | 
 7 | group = 'kafkastreams'
 8 | version = '0.0.1-SNAPSHOT'
 9 | sourceCompatibility = '11'
10 | 
11 | repositories {
12 | 	mavenCentral()
13 | }
14 | 
15 | dependencies {
16 | 	implementation 'org.springframework.boot:spring-boot-starter'
17 | 	testImplementation 'org.springframework.boot:spring-boot-starter-test'
18 | 	implementation 'org.apache.kafka:kafka-streams:3.7.0'
19 | 	implementation 'org.apache.kafka:kafka-clients:3.7.0'
20 | 	testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
21 | 	testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
22 | }
23 | 
24 | tasks.named('test') {
25 | 	useJUnitPlatform()
26 | }
27 | 


--------------------------------------------------------------------------------
/influx/delete_measurement.py:
--------------------------------------------------------------------------------
 1 | from influxdb_client import InfluxDBClient
 2 | import os
 3 | from dotenv import load_dotenv
 4 | 
 5 | load_dotenv()
 6 | 
 7 | # InfluxDB 환경 변수 로드
 8 | token = os.getenv("INFLUXDB_TOKEN")
 9 | org = os.getenv("INFLUXDB_ORG")
10 | url = os.getenv("INFLUXDB_URL")
11 | bucket = "HighLighter"  # 데이터를 삭제할 버킷 이름
12 | 
13 | client = InfluxDBClient(url=url, token=token, org=org)
14 | delete_api = client.delete_api()
15 | 
16 | # 삭제할 measurement 이름
17 | measurement_to_delete = "삭제할 measurement 이름"
18 | 
19 | # 모든 데이터를 삭제하는 시간 범위 설정 (전체 범위)
20 | start = "1970-01-01T00:00:00Z"
21 | stop = "2100-01-01T00:00:00Z"
22 | 
23 | # Measurement 데이터 삭제
24 | delete_api.delete(start, stop, f'_measurement="{measurement_to_delete}"', bucket=bucket, org=org)
25 | 
26 | print(f"Measurement '{measurement_to_delete}' has been deleted.")


--------------------------------------------------------------------------------
/video-stream/producer.py:
--------------------------------------------------------------------------------
 1 | import logging
 2 | import time
 3 | 
 4 | import streamlink
 5 | from confluent_kafka import Producer
 6 | 
 7 | # Logging config
 8 | logger = logging.getLogger(__name__)
 9 | logging.basicConfig(
10 |     encoding='utf-8',
11 |     level=logging.INFO,
12 |     format='%(asctime)s [%(levelname)s] %(message)s',
13 |     datefmt='%Y/%m/%d %H:%M:%S'
14 | )
15 | 
16 | # Kafka config
17 | broker_public_ip = None
18 | conf = {
19 |     'bootstrap.servers': f"{broker_public_ip}:9092",
20 |     'security.protocol': 'PLAINTEXT'
21 | }
22 | 
23 | producer = Producer(**conf)
24 | 
25 | def delivery_report(err, msg):
26 |     if err is not None:
27 |         logger.warning(f'Message delivery failed: {err}')
28 |     else:
29 |         logger.info(f'Message delivered to {msg.topic()} [{msg.partition()}]')
30 | 
31 | for i in range(1000):
32 |     producer.poll(0)
33 |     producer.produce('mytopic', f'data {i}'.encode('utf-8'), callback=delivery_report)
34 | producer.flush()
35 | 


--------------------------------------------------------------------------------
/video-stream/docker-compose.yaml:
--------------------------------------------------------------------------------
 1 | services:
 2 |   broker:
 3 |     image: apache/kafka:latest
 4 |     container_name: broker
 5 |     ports:
 6 |       - 9092:9092
 7 |     environment:
 8 |       KAFKA_NODE_ID: 1
 9 |       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
10 |       KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://${PUBLIC_IP}:9092,PLAINTEXT://broker:19092'
11 |       KAFKA_PROCESS_ROLES: 'broker,controller'
12 |       KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
13 |       KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092'
14 |       KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
15 |       KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
16 |       CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
17 |       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
18 |       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
19 |       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
20 |       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
21 |       KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
22 |   


--------------------------------------------------------------------------------
/terraform/main.tf:
--------------------------------------------------------------------------------
 1 | terraform {
 2 |   required_providers {
 3 |     aws = {
 4 |       source  = "hashicorp/aws"
 5 |       version = "~> 4.16"
 6 |     }
 7 |   }
 8 | 
 9 |   required_version = ">= 1.2.0"
10 | }
11 | 
12 | provider "aws" {
13 |   region = "ap-northeast-2"
14 | }
15 | 
16 | variable "vpc_id" {
17 |   type = string
18 | }
19 | 
20 | resource "aws_security_group" "example_sg" {
21 |   name = "example-security-group"
22 |   description = "Example security group generated by Terraform"
23 |   vpc_id = var.vpc_id
24 | 
25 |   ingress {
26 |     from_port = 22
27 |     to_port = 22
28 |     protocol = "tcp"
29 |     cidr_blocks = ["0.0.0.0/0"]
30 |   }
31 | 
32 |   ingress {
33 |     from_port = 80
34 |     to_port = 80
35 |     protocol = "tcp"
36 |     cidr_blocks = ["0.0.0.0/0"]
37 |   }
38 | 
39 |   tags = {
40 |     Name = "example-sg"
41 |   }
42 | }
43 | 
44 | resource "aws_instance" "app_server" {
45 |   ami           = "ami-0c2acfcb2ac4d02a0"
46 |   instance_type = "t2.micro"
47 |   key_name = "key"
48 | 
49 |   vpc_security_group_ids = [aws_security_group.example_sg.id]
50 | 
51 |   tags = {
52 |     Name = "ExampleAppServerInstance"
53 |   }
54 | }
55 | 


--------------------------------------------------------------------------------
/influx/read_binary.py:
--------------------------------------------------------------------------------
 1 | import influxdb_client
 2 | import os
 3 | from dotenv import load_dotenv
 4 | import csv
 5 | 
 6 | load_dotenv()
 7 | 
 8 | # influxDB 세팅
 9 | influxdb_token = os.getenv('INFLUXDB_TOKEN')
10 | influxdb_url = os.getenv('INFLUXDB_URL')
11 | influxdb_org = os.getenv('INFLUXDB_ORG')
12 | bucket = os.getenv('INFLUXDB_BUCKET')
13 | csv.field_size_limit(1048576)
14 | 
15 | # 클라이언트 생성
16 | client = influxdb_client.InfluxDBClient(url=influxdb_url, token=influxdb_token, org=influxdb_org)
17 | 
18 | # Read API 생성
19 | query_api = client.query_api()
20 | 
21 | # 첫 데이터부터 (1970년 00-00), 현재 시점까지읟 데이터를 읽을 거임
22 | start_time = "0"  # Start time
23 | end_time = "now()"  # Current time
24 | 
25 | # influxdb 읽기 (전체 데이터)
26 | # 다른 measurement에 있는 데이터를 읽을 시에, video_text 부분을 바꿔줘야됨
27 | # 타임스탬프 필터링 및 정렬을 추가한 쿼리
28 | query = f'''
29 | from(bucket: "{bucket}")
30 |   |> range(start: {start_time})
31 |   |> filter(fn: (r) => r["_measurement"] == "video_text")
32 |   |> filter(fn: (r) => uint(v: r["_time"]) >= 1724061418579000000 and uint(v: r["_time"]) < 1724061418580000000)
33 |   |> sort(columns: ["_time"], desc: false)
34 | '''
35 | 
36 | # 데이터 출력
37 | tables = query_api.query(query)
38 | 
39 | for table in tables:
40 |     for record in table.records: 
41 |         # print({"time": record.get_time(), "text": record.get_value()})
42 |         print({"value": record.get_value()})


--------------------------------------------------------------------------------
/influx/insert_text.py:
--------------------------------------------------------------------------------
 1 | import influxdb_client
 2 | import os
 3 | from influxdb_client import Point, WritePrecision
 4 | from influxdb_client.client.write_api import SYNCHRONOUS
 5 | from dotenv import load_dotenv
 6 | load_dotenv()
 7 | 
 8 | # InfluxDB 환경 세팅
 9 | token = os.environ.get("INFLUXDB_TOKEN")
10 | org = os.environ.get("INFLUXDB_ORG")
11 | url = os.environ.get("INFLUXDB_URL")
12 | bucket = os.environ.get("INFLUXDB_BUCKET")
13 | 
14 | # Influx Client 생성
15 | client = influxdb_client.InfluxDBClient(url=url, token=token, org=org)
16 | 
17 | # Influx Write API 생성
18 | write_api = client.write_api(write_options=SYNCHRONOUS)
19 | 
20 | #----------------------------------------------------------------------
21 | #----------------------------------------------------------------------
22 | # 임의의 변수
23 | text_data = "model_output" #예시
24 | timestamp = "2024-08-18 00:00:04" #예시, utc기준, 시작시간을 저장
25 | # 데이터 넣기
26 | # -------------------------------------------------------------------
27 | # .time에는 카프카에서 받은 시간 ()
28 | point = (
29 |     Point("video_text")
30 |     .field("text", text_data)
31 |     .time(timestamp, WritePrecision.S)
32 | )
33 | #----------------------------------------------------------------------
34 | #----------------------------------------------------------------------
35 | 
36 | # 데이터 입력
37 | write_api.write(bucket=bucket, org=org, record=point)
38 | 
39 | client.close()


--------------------------------------------------------------------------------
/video-stream/trigger_request.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Enqueue request to SQS, triggered by API Gateway.
 3 | """
 4 | import json
 5 | import boto3
 6 | import time
 7 | import uuid
 8 | from datetime import datetime
 9 | 
10 | 
11 | def lambda_handler(event, context):
12 |     sqs = boto3.client('sqs')
13 |     res = sqs.get_queue_url(QueueName='highlighter-requests.fifo')
14 |     queue_url = res["QueueUrl"]
15 |     
16 |     # Parse timestamp from query parameters
17 |     offset = 60 * 60 * 9
18 |     start_str = event["queryStringParameters"]["start"]
19 |     start_time = time.mktime(
20 |         datetime.strptime(start_str, "%Y-%m-%d %H:%M:%S").timetuple()) - offset
21 |     
22 |     end_str = event["queryStringParameters"]["end"]
23 |     end_time = time.mktime(
24 |         datetime.strptime(end_str, "%Y-%m-%d %H:%M:%S").timetuple()) - offset
25 |     
26 |     try:
27 |         res = sqs.send_message(
28 |             QueueUrl=queue_url,
29 |             MessageBody=json.dumps({
30 |                 "start": start_time,
31 |                 "end": end_time,
32 |             }),
33 |             MessageGroupId="1",
34 |             MessageDeduplicationId=str(uuid.uuid4())
35 |         )
36 |         
37 |         return {
38 |             'statusCode': 200,
39 |             'body': json.dumps("Request successfully enqueued")
40 |         }
41 |         
42 |     except Exception as e:
43 |         return {
44 |             'statusCode': 400,
45 |             'body': json.dumps(f"Request failure: {e}")
46 |         }
47 |     


--------------------------------------------------------------------------------
/timeframe/normalize_time.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | from datetime import timedelta
 3 | 
 4 | # Read the CSV file into a DataFrame
 5 | file_path = 'data/parischim.csv'
 6 | df = pd.read_csv(file_path, header=None, names=['time', 'text'])
 7 | 
 8 | # Function to normalize time format to hh:mm:ss
 9 | def normalize_time_format(time_str):
10 |     parts = time_str.split(':')
11 |     
12 |     if len(parts) == 2:
13 |         # Format is mm:ss, convert to hh:mm:ss
14 |         minutes, seconds = parts
15 |         return f"00:{minutes.zfill(2)}:{seconds.zfill(2)}"
16 |     elif len(parts) == 3:
17 |         # Check if it's mm:ss:00 or hh:mm:ss
18 |         if parts[2] == '00' and int(parts[0]) >= 24:
19 |             # Format is mm:ss:00 for times >= 24 minutes, convert to hh:mm:ss
20 |             minutes, seconds, _ = parts
21 |             return f"00:{minutes.zfill(2)}:{seconds.zfill(2)}"
22 |         else:
23 |             # Format is hh:mm:ss, already correct
24 |             hours, minutes, seconds = parts
25 |             return f"{hours.zfill(2)}:{minutes.zfill(2)}:{seconds.zfill(2)}"
26 |     else:
27 |         raise ValueError(f"Unexpected time format: {time_str}")
28 | 
29 | # Normalize the time column
30 | df['normalized_time'] = df['time'].apply(normalize_time_format)
31 | 
32 | # Display the corrected DataFrame
33 | print(df[['normalized_time', 'text']])
34 | 
35 | # Optionally, save the corrected DataFrame back to a CSV file
36 | df.to_csv('data/normalized_parischim.csv', columns=['normalized_time', 'text'], index=False, header=False)


--------------------------------------------------------------------------------
/influx/explanation.md:
--------------------------------------------------------------------------------
 1 | # InfluxDB vs RDB
 2 | | Database | Databse |
 3 | | --- | --- |
 4 | | Table | Measurement |
 5 | | Column | Key |
 6 | | Indexed Column | Tag Key (String Only) |
 7 | | Unindexed Column | Field Key |
 8 | | Row | Point |
 9 | 
10 | # Dependency
11 | 
12 | `pip3 install influxdb-client`
13 | 
14 | # Initialization
15 | ```python
16 | import influxdb_client, os, time
17 | from influxdb_client import InfluxDBClient, Point, WritePrecision
18 | from dotenv import load_dotenv
19 | from influxdb_client.client.write_api import SYNCHRONOUS
20 | 
21 | load_dotenv()
22 | 
23 | token = os.getenv("INFLUXDB_TOKEN")
24 | org = os.getenv("INFLUXDB_ORG")
25 | url = os.getenv("INFLUXDB_URL")
26 | bucket = os.getenv("INFLUXDB_BUCKET")
27 | 
28 | # 클라이언트 생성
29 | client = influxdb_client.InfluxDBClient(url=url, token=token, org=org)
30 | ```
31 | 
32 | # Write Data
33 | - InfluxDB에서는 Write할 API와 Read할 API를 각각 생성해줘야된다.
34 | 
35 | 예시는 아래와 같다:
36 | ```python
37 | # Write/Insert API 생성
38 | write_api = client.write_api(write_options=SYNCHRONOUS)
39 |    
40 | # Write할 시, point라는 것을 활용해 스키마를 짜줘서, Write해줘야 된다.
41 | # 형식은 아래와 같다.
42 | for value in range(5):
43 |   point = (
44 |     Point("measurement1")
45 |     .tag("tagname1", "tagvalue1")
46 |     .field("field1", value)
47 |   )
48 |   write_api.write(bucket=bucket, org="HighLighter", record=point)
49 |   time.sleep(1) # separate points by 1 second
50 | ```
51 | 
52 | # Query Data
53 | 
54 | ```python
55 | # Read하는 API를 생성해준다
56 | query_api = client.query_api()
57 | 
58 | # 10분 전부터 현재 시간까지의 데이터들을 가져오는 쿼리
59 | query = """from(bucket: "HighLighter")
60 |  |> range(start: -10m)
61 |  |> filter(fn: (r) => r._measurement == "measurement1")"""
62 | tables = query_api.query(query, org="HighLighter")
63 | 
64 | # 출력해보자
65 | for table in tables:
66 |   for record in table.records:
67 |     print(record)
68 | ```
69 | 


--------------------------------------------------------------------------------
/youtube-live/app.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import logging
 3 | import os
 4 | 
 5 | import pafy
 6 | import pytchat
 7 | from confluent_kafka import Producer
 8 | from dotenv import load_dotenv
 9 | 
10 | # Load environment variables
11 | load_dotenv()
12 | 
13 | # Logging config
14 | logger = logging.getLogger(__name__)
15 | logging.basicConfig(
16 |     encoding='utf-8',
17 |     level=logging.INFO,
18 |     format='%(asctime)s [%(levelname)s] %(message)s',
19 |     datefmt='%Y/%m/%d %H:%M:%S'
20 | )
21 | 
22 | 
23 | def delivery_report(err, msg):
24 |     if err is not None:
25 |         logger.warning(f'Message delivery failed: {err}')
26 |     else:
27 |         logger.info(f'Message delivered to {msg.topic()} [{msg.partition()}]')
28 | 
29 | 
30 | def get_chat_from_live(video_id: str, producer: Producer):
31 |     api_key = os.environ.get('GCP_KEY')
32 |     pafy.set_api_key(api_key)
33 | 
34 |     chat = pytchat.create(video_id=video_id)
35 |     while chat.is_alive():
36 |         data = chat.get()
37 |         for c in data.items:
38 |             # send timestamp to kafka
39 |             producer.produce(
40 |                 'stream_filter',
41 |                 value=c.datetime.encode('utf-8'),
42 |                 callback=delivery_report
43 |             )
44 |             producer.poll(0)
45 |     producer.flush()
46 | 
47 | 
48 | if __name__ == "__main__":
49 |     parser = argparse.ArgumentParser(description='Streamlink Kafka Producer')
50 |     parser.add_argument('-u', '--url', type=str, required=True, help='URL link to YouTube live')
51 |     args = parser.parse_args()
52 | 
53 |     # Kafka config
54 |     broker_public_ip = os.getenv('BROKER_IP', 'localhost')
55 |     conf = {
56 |         'bootstrap.servers': f'{broker_public_ip}:9092',
57 |         'security.protocol': 'PLAINTEXT'
58 |     }
59 |     producer = Producer(**conf)
60 | 
61 |     get_chat_from_live(
62 |         video_id=args.url,
63 |         producer=producer
64 |     )
65 | 


--------------------------------------------------------------------------------
/influx/insert_binary.py:
--------------------------------------------------------------------------------
 1 | import influxdb_client
 2 | from confluent_kafka import Consumer
 3 | from influxdb_client import Point, WritePrecision
 4 | from influxdb_client.client.write_api import SYNCHRONOUS
 5 | from dotenv import load_dotenv
 6 | import os
 7 | import base64
 8 | import itertools
 9 | 
10 | load_dotenv()
11 | 
12 | # InfluxDB 환경 세팅
13 | token = os.getenv("INFLUXDB_TOKEN")
14 | org = os.getenv("INFLUXDB_ORG")
15 | url = os.getenv("INFLUXDB_URL")
16 | bucket = os.getenv("INFLUXDB_BUCKET")
17 | 
18 | # Influx Client 생성
19 | client = influxdb_client.InfluxDBClient(url=url, token=token, org=org)
20 | 
21 | # Influx Write API 생성
22 | write_api = client.write_api(write_options=SYNCHRONOUS)
23 | 
24 | # Kafka consumer configuration 세팅
25 | # earliest -> 해당 파티션의 가장 오래된 메시지부터 소비
26 | # latest -> 해당 파티션의 최신 메시지부터 소비
27 | conf = {
28 |     'bootstrap.servers': '52.79.81.77:9092',
29 |     'group.id': 'mygroup',
30 |     'auto.offset.reset': 'latest'
31 | }
32 | 
33 | consumer = Consumer(conf)
34 | consumer.subscribe(['mytopic'])
35 | 
36 | # 나노초 증가값을 생성하기 위한 카운터
37 | nano_increment = itertools.count()
38 | 
39 | try:
40 |     while True:
41 |         msg = consumer.poll(1.0)  # 브로커로부터 메시지를 가져옴
42 |         
43 |         if msg is None:
44 |             continue
45 |         if msg.error():
46 |             print(f"Consumer error: {msg.error()}")
47 |             continue
48 | 
49 |         # 바이너리 메시지 및 타임스탬프 추출
50 |         message_value = base64.b64encode(msg.value()).decode()
51 |         kafka_timestamp = msg.timestamp()[1]
52 |         
53 |         influx_timestamp = (kafka_timestamp * 1_000_000) + next(nano_increment)
54 | 
55 |         # inlfuxDB에 넣기 위해 포인트 생성 
56 |         point = Point("video_binary").field("value", message_value).time(influx_timestamp, WritePrecision.NS)
57 | 
58 |         # influxDB에 데이터 넣기
59 |         write_api.write(bucket=bucket, org=org, record=point)
60 | 
61 |         print(kafka_timestamp)
62 | 
63 | except KeyboardInterrupt:
64 |     pass
65 | 
66 | finally:
67 |     consumer.close()
68 |     client.close()


--------------------------------------------------------------------------------
/video-stream/main.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Read stream, send from EC2 to SQS.
 3 | """
 4 | import argparse
 5 | import logging
 6 | import os
 7 | import time
 8 | 
 9 | import streamlink
10 | from confluent_kafka import Producer
11 | 
12 | 
13 | # Logging config
14 | logger = logging.getLogger(__name__)
15 | logging.basicConfig(
16 |     encoding='utf-8',
17 |     level=logging.INFO,
18 |     format='%(asctime)s [%(levelname)s] %(message)s',
19 |     datefmt='%Y/%m/%d %H:%M:%S'
20 | )
21 | 
22 | # Kafka config
23 | broker_public_ip = os.getenv('BROKER_IP', 'localhost')
24 | conf = {
25 |     'bootstrap.servers': f'{broker_public_ip}:9092',
26 |     'security.protocol': 'PLAINTEXT'
27 | }
28 | 
29 | producer = Producer(**conf)
30 | 
31 | def delivery_report(err, msg):
32 |     if err is not None:
33 |         logger.warning(f'Message delivery failed: {err}')
34 |     else:
35 |         logger.info(f'Message delivered to {msg.topic()} [{msg.partition()}]')
36 | 
37 | 
38 | def get_stream(url: str, duration: int, buf_size: int):
39 |     streams = streamlink.streams(url)
40 |     logger.info(f'Streams found: {streams.keys()}')
41 |     fd = streams['best'].open()
42 | 
43 |     now = time.time()
44 |     while time.time() - now < duration:
45 |         data = fd.read(buf_size)
46 |         producer.produce(
47 |             'mytopic',
48 |             data,
49 |             callback=delivery_report
50 |         )
51 |         producer.poll(0)
52 |     producer.flush()
53 | 
54 | 
55 | if __name__ == "__main__":
56 |     parser = argparse.ArgumentParser(description='Streamlink Kafka Producer')
57 |     parser.add_argument('-u', '--url', type=str, required=True, help='URL link to YouTube live')
58 |     parser.add_argument('-t', '--time', type=int, required=True, help='Time duration to capture streams')
59 |     parser.add_argument('-b', '--bufsize', type=int, required=True, help='Buffer size')
60 |     args = parser.parse_args()
61 | 
62 |     url = args.url
63 |     duration = args.time
64 |     bufsize = args.bufsize
65 | 
66 |     get_stream(
67 |         url=url,
68 |         duration=duration,
69 |         buf_size=bufsize
70 |     )
71 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | <div align="center">
 3 | 
 4 | <img src="assets/logo.png" alt="logo" />
 5 | 
 6 | <!-- <h2>Real-Time Highlight Extraction</h1> -->
 7 | 
 8 | </div>
 9 | 
10 | # Highlighter: Real-Time Highlight Extraction
11 | 
12 | ***`Highlighter`*** is a project designed to automatically extract real-time highlights from YouTube livestreams. Highlight timestamps are initially detected based on chat traffic surges, and these timestamps are then refined using context from video transcripts generated by Speech-to-Text (STT). This approach combines both audience reactions and content analysis for more accurate and meaningful highlight extraction. 
13 | 
14 | After extracting the highlights, ***`Highlighter`*** can automatically upload them back to YouTube for immediate sharing.
15 | 
16 | ## Features
17 | 
18 | - **Real-Time Highlight Detection**: Automatically identifies potential highlight moments during YouTube livestreams by analyzing chat traffic spikes.
19 | - **Contextual Highlight Refinement**: After detecting an initial highlight based on chat activity, ***`Highlighter`*** uses STT to generate transcripts from the video, refining the timestamp range of the highlight based on the content's context.
20 | - **Seamless Automation**: The process of detecting, refining, and saving highlights is fully automated, requiring no manual intervention.
21 | - **Highlight Saving**: Extracted highlights are stored on cloud storage for further review, editing, or sharing.
22 | - **Automatic YouTube Upload**: Once highlights are extracted, videos are automatically uploaded to your YouTube account, making the highlights available for viewers almost instantly.
23 | 
24 | ## Overview
25 | 
26 | ![arch](assets/architecture.png)
27 | 
28 | For more detailed documentation, including advanced usage, configuration examples, and troubleshooting tips, please refer to the [GitHub Wiki](https://github.com/YBIGTA/24th-de-highlighter/wiki) for this project.
29 | 
30 | ## Contributors
31 | 
32 | <a href="https://github.com/YBIGTA/24th-de-highlighter/graphs/contributors">
33 |   <img src="https://contrib.rocks/image?repo=YBIGTA/24th-de-highlighter" />
34 | </a>
35 | 


--------------------------------------------------------------------------------
/influx/Readme.md:
--------------------------------------------------------------------------------
  1 | # EC2 환경 설정
  2 | 
  3 | AMI (Amazon Machine Image):   
  4 | - `Amazon Linux`
  5 |     
  6 | 인스턴스 유형:   
  7 | - `m4.xlarge`
  8 | 
  9 | 
 10 | 보안 그룹:
 11 | - `기존 보안 그룹 선택` $\rightarrow$ DE 세션에 썼던 보안 그룹 (나중에 수정해줘야됨)
 12 | 
 13 | - 또는, `보안 그룹 생성
 14 |     ![](../assets/influx_01.png)
 15 | 
 16 | 스토리지 구성:
 17 | - `40 GiB gp3`
 18 | 
 19 | - TSDB (Time Series Database) 특성상 적재하는 데이터가 많아 40GB로 설정 해 주었다.
 20 | 
 21 | 인스턴스 개수:
 22 | - 1개
 23 | 
 24 | ## 기존 DE 보안 그룹 사용 시
 25 | 1. 인스턴스 $\rightarrow$ 보안 $\rightarrow$ 보안 그룹
 26 | ![](../assets/influx_02.png)
 27 | 
 28 | 2. 인바운드 규칙 편집
 29 | ![](../assets/influx_03.png)
 30 | 
 31 | 3. influxDB를 위한 port를 열어주면 된다.
 32 | ![](../assets/influx_04.png)
 33 | 
 34 | 
 35 | ## 새 보안 그룹 생성 시
 36 | - 위 (기존 DE 보안 그룹 사용 시) 문항의 3번을 참고하여 설정해주면 된다.
 37 | 
 38 | 
 39 | # InfluxDB 환경설정
 40 | 0. InfluxDB documentation:  
 41 |  https://docs.influxdata.com/influxdb/v2/install/?t=Linux
 42 | 
 43 | 1. InfluxDB documentation에 접속하여, `x86_64.rpm`로 끝나는 파일을 `curl`명령어로 다운 받아준다. (Amazon Linux는 `# Red Hat/CentOS/Fedora x86-64 (x64, AMD64)` 이 부분에 해당되겠다)
 44 | 
 45 | 2. 문서는 수시로 업데이트가 되어 있을 수 있으니, 문서를 참고하여 EC2에 InfluxDB를 설치해준다.
 46 | 
 47 | # InfluxDB 연결
 48 | 1. AWS에서 InfluxDB 실행중인 EC2 선택
 49 | 
 50 | 2. `자동 할당된 IP 주소 (Public IP)`를 복사 해준다.
 51 | ![](../assets/influx_05.png)
 52 | 
 53 | 3. 웹 브라우저 (Google Chrome, Safari, ...)에서 복사한 IP주소 + InfluxDB port 번호를 입력하면 InfluxDB 자체에서 제공하는 UI에 접속할 수 있다.
 54 |     - InfluxDB의 default port 번호는 `8086`이다.
 55 |     - 따라서, 2번에서 복사한 IP 주소가 `43.201.148.166`이라고 할 때, `43.201.148.166:8086`에 접속하면 된다.  
 56 | 
 57 |         예시:
 58 |         ![](../assets/influx_06.png)
 59 | 
 60 | ## 추가. Elastic IP Addresses (탄력적 IP 주소)
 61 | >`Elastic IP Addresses`를 설정하고 삭제하기 전까지 인터넷에 접속할 수 있는 고정적인 IP 주소를 할당할 수 있는 서비스.
 62 | 
 63 | 문제점:
 64 | - AWS EC2의 Public IP 주소는 매번 껏다 킬때마다 바뀐다.  
 65 | - 따라서, 매번 새로운 Public IP 주소를 공유해주어야 한다. 
 66 | - 매우 번거롭고 귀찮다.
 67 | 
 68 | 해결:
 69 | - `Elastic IP Addresses (탄력적 IP 주소)`라고 우리가 사용할 IP 주소를 유지하는 것이다 (우리가 제공받은 Elastic IP Address를 할당 해제할 때 까지)
 70 | 
 71 | 단점:
 72 | - 비용이 조금 더 나간다.
 73 | 
 74 | 특징:
 75 | - IPv4만 가능하다.
 76 | 
 77 | ### 1. 탄력적 IP 주소 생성
 78 | 1. AWS 서비스 $\rightarrow$ 네트워크 및 보안 $\rightarrow$ 탄력적 IP
 79 | ![](../assets/influx_07.png)
 80 | 
 81 | 2. `탄력적 IP 주소 할당` 선택
 82 | ![](../assets/influx_08.png)
 83 | 
 84 | 3. 수정 없이, `할당` 선택
 85 | ![](../assets/influx_09.png)
 86 | 
 87 | ### 2. 탄력적 IP 주소 설정
 88 | 이제 생성된 EIA (Elastic IP Addresses)를 EC2 인스턴스와 연결시켜주자.
 89 | 
 90 | 1. 생성된 EIA 선택 $\rightarrow$ 작업 $\rightarrow$ `탄력적 IP 주소 연결` 선택
 91 | ![](../assets/influx_10.png)
 92 | 
 93 | 2. 설정해주기
 94 |     - 인스턴스 유형: `인스턴스`
 95 |     - `인스턴스`와 `프라이빗 IP 주소` 선택 
 96 |     ![](../assets/influx_11.png)
 97 | 
 98 | 3. EIA를 할당해준 인스턴스 확인하기
 99 | ![](../assets/influx_12.png)
100 | 위 사진과 같이, EIA가 할당된 걸 볼 수 있다.
101 | 
102 | 결과:
103 | - ![](../assets/influx_13.png)
104 | 
105 | > 삭제하고 싶으면, 1번의 생성된 EIA 선택 $\rightarrow$ 작업 $\rightarrow$ `탄력적 IP 주소 릴리스` 선택
106 | ![](../assets/influx_14.png)


--------------------------------------------------------------------------------
/highlighter-sliding-window/src/main/java/kafkastreams/example/KStream/Consumer.java:
--------------------------------------------------------------------------------
 1 | package kafkastreams.example.KStream;
 2 | 
 3 | import org.apache.kafka.clients.consumer.ConsumerConfig;
 4 | import org.apache.kafka.clients.consumer.ConsumerRecord;
 5 | import org.apache.kafka.clients.consumer.ConsumerRecords;
 6 | import org.apache.kafka.clients.consumer.KafkaConsumer;
 7 | import org.apache.kafka.common.serialization.StringDeserializer;
 8 | import org.slf4j.Logger;
 9 | import org.slf4j.LoggerFactory;
10 | 
11 | import java.time.Duration;
12 | import java.util.Arrays;
13 | import java.util.Properties;
14 | 
15 | public class Consumer {
16 | 
17 |     private final static Logger log = LoggerFactory.getLogger(Consumer.class);
18 |     private final static String BOOTSTRAP_SERVERS = "43.203.141.74:9092";  /* change ip */
19 |     private final static String GROUP_ID = "kstream-application";  /* this can be anything you want */
20 |     private final static String TOPIC_NAME = "stream_filter_sink";
21 | 
22 |     public Consumer() {}
23 |     public static void main(String[] args) {
24 |         new Consumer().run();
25 |     }
26 | 
27 |     public void run() {
28 |         log.info("Starting Kafka consumer...");
29 |         KafkaConsumer<String, String> consumer = createKafkaConsumer();
30 | 
31 |         // Safe close
32 |         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
33 |             log.info("Stopping application...");
34 |             log.info("Closing consumer...");
35 |             consumer.close();
36 |             log.info("Consumer closed. Done!");
37 |         }));
38 | 
39 |         consumer.subscribe(Arrays.asList(TOPIC_NAME));
40 | 
41 |         log.info("Consumer is ready");
42 |         try {
43 |             while (true) {
44 |                 ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
45 |                 for (ConsumerRecord<String, String> record : records) {
46 |                     log.info("Key (Timestamp): " + record.key() + ", Value (Count): " + record.value());
47 |                     log.info("Partition: " + record.partition() + ", Offset: " + record.offset());
48 |                 }
49 |             }
50 |         } catch (Exception e) {
51 |             log.error("Error in consuming records", e);
52 |         } finally {
53 |             consumer.close();
54 |             log.info("Consumer closed");
55 |         }
56 |     }
57 | 
58 |     public KafkaConsumer<String, String> createKafkaConsumer() {
59 |         Properties properties = new Properties();
60 |         properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
61 |         properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
62 |         properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
63 |         properties.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID);
64 |         properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
65 | 
66 |         return new KafkaConsumer<>(properties);
67 |     }
68 | }


--------------------------------------------------------------------------------
/highlighter-sliding-window/gradlew.bat:
--------------------------------------------------------------------------------
 1 | @rem
 2 | @rem Copyright 2015 the original author or authors.
 3 | @rem
 4 | @rem Licensed under the Apache License, Version 2.0 (the "License");
 5 | @rem you may not use this file except in compliance with the License.
 6 | @rem You may obtain a copy of the License at
 7 | @rem
 8 | @rem      https://www.apache.org/licenses/LICENSE-2.0
 9 | @rem
10 | @rem Unless required by applicable law or agreed to in writing, software
11 | @rem distributed under the License is distributed on an "AS IS" BASIS,
12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | @rem See the License for the specific language governing permissions and
14 | @rem limitations under the License.
15 | @rem
16 | 
17 | @if "%DEBUG%"=="" @echo off
18 | @rem ##########################################################################
19 | @rem
20 | @rem  Gradle startup script for Windows
21 | @rem
22 | @rem ##########################################################################
23 | 
24 | @rem Set local scope for the variables with windows NT shell
25 | if "%OS%"=="Windows_NT" setlocal
26 | 
27 | set DIRNAME=%~dp0
28 | if "%DIRNAME%"=="" set DIRNAME=.
29 | set APP_BASE_NAME=%~n0
30 | set APP_HOME=%DIRNAME%
31 | 
32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter.
33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
34 | 
35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
37 | 
38 | @rem Find java.exe
39 | if defined JAVA_HOME goto findJavaFromJavaHome
40 | 
41 | set JAVA_EXE=java.exe
42 | %JAVA_EXE% -version >NUL 2>&1
43 | if %ERRORLEVEL% equ 0 goto execute
44 | 
45 | echo.
46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
47 | echo.
48 | echo Please set the JAVA_HOME variable in your environment to match the
49 | echo location of your Java installation.
50 | 
51 | goto fail
52 | 
53 | :findJavaFromJavaHome
54 | set JAVA_HOME=%JAVA_HOME:"=%
55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe
56 | 
57 | if exist "%JAVA_EXE%" goto execute
58 | 
59 | echo.
60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
61 | echo.
62 | echo Please set the JAVA_HOME variable in your environment to match the
63 | echo location of your Java installation.
64 | 
65 | goto fail
66 | 
67 | :execute
68 | @rem Setup the command line
69 | 
70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
71 | 
72 | 
73 | @rem Execute Gradle
74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
75 | 
76 | :end
77 | @rem End local scope for the variables with windows NT shell
78 | if %ERRORLEVEL% equ 0 goto mainEnd
79 | 
80 | :fail
81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
82 | rem the _cmd.exe /c_ return code!
83 | set EXIT_CODE=%ERRORLEVEL%
84 | if %EXIT_CODE% equ 0 set EXIT_CODE=1
85 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
86 | exit /b %EXIT_CODE%
87 | 
88 | :mainEnd
89 | if "%OS%"=="Windows_NT" endlocal
90 | 
91 | :omega
92 | 


--------------------------------------------------------------------------------
/highlighter-sliding-window/src/main/java/kafkastreams/example/KStream/Producer.java:
--------------------------------------------------------------------------------
 1 | package kafkastreams.example.KStream;
 2 | 
 3 | import org.apache.kafka.clients.producer.KafkaProducer;
 4 | import org.apache.kafka.clients.producer.ProducerConfig;
 5 | import org.apache.kafka.clients.producer.ProducerRecord;
 6 | import org.apache.kafka.common.serialization.StringSerializer;
 7 | import org.slf4j.Logger;
 8 | import org.slf4j.LoggerFactory;
 9 | import org.springframework.stereotype.Component;
10 | 
11 | import java.io.BufferedReader;
12 | import java.io.IOException;
13 | import java.io.InputStreamReader;
14 | import java.util.Properties;
15 | import java.util.StringTokenizer;
16 | 
17 | public class Producer {
18 |     private final static Logger log = LoggerFactory.getLogger(Consumer.class);
19 |     private final static String BOOTSTRAP_SERVERS = "43.203.141.74:9092";  /* change ip */
20 |     private final static String TOPIC_NAME = "stream_filter";
21 |     static BufferedReader br;
22 |     static StringTokenizer str;
23 | 
24 |     public Producer() {}
25 |     public static void main(String[] args) throws IOException {
26 |         new Producer().run();
27 |     }
28 | 
29 |     public void run() throws IOException {
30 |         log.info("Starting Kafka producer...");
31 |         KafkaProducer<String, String> producer = createKafkaProducer();
32 | 
33 |         // safe close
34 |         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
35 |             log.info("stopping application...");
36 |             log.info("closing producer...");
37 |             producer.close();
38 |             log.info("done!");
39 |         }));
40 | 
41 |         log.info("Producer is ready");
42 |         br = new BufferedReader(new InputStreamReader(System.in));
43 |         boolean keyExist = false;
44 | 
45 |         while(true) {
46 |             String key, value;
47 |             if(keyExist) {
48 |                 str = new StringTokenizer(br.readLine());
49 |                 key = str.nextToken();
50 |                 value = str.nextToken();
51 |                 log.info("K, V: " + key + ", " + value);
52 |             } else {
53 |                 key = null;
54 |                 value = br.readLine();
55 |                 log.info("V: " + value);
56 |             }
57 |             ProducerRecord<String, String> record = new ProducerRecord<>(TOPIC_NAME, key, value);
58 | 
59 |             if(value != null) {
60 |                 producer.send(record, (metadata, e) -> {
61 |                     if (e != null) {
62 |                         log.error("Something went wrong", e);
63 |                     }
64 |                 });
65 |             }
66 |         }
67 |     }
68 | 
69 |     public KafkaProducer<String, String> createKafkaProducer() {
70 |         System.out.println("BOOTSTRAP_SERVERS = " + BOOTSTRAP_SERVERS);
71 |         Properties properties = new Properties();
72 |         properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
73 |         properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
74 |         properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
75 | 
76 |         return new KafkaProducer<>(properties);
77 |     }
78 | }
79 | 
80 | 


--------------------------------------------------------------------------------
/video-stream/sqs_trigger_task.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Save video to S3, triggered by SQS.
  3 | """
  4 | import json
  5 | import uuid
  6 | import boto3
  7 | import subprocess
  8 | import time
  9 | import base64
 10 | import shlex
 11 | from datetime import datetime
 12 | 
 13 | 
 14 | def lambda_handler(event, context):
 15 |     
 16 |     s3 = boto3.client("s3")
 17 |     sqs = boto3.client('sqs')
 18 |     response = sqs.get_queue_url(QueueName='highlighter.fifo')
 19 |     queue_url = response["QueueUrl"]
 20 |     
 21 |     bucket_name = "de-highlighter"
 22 |     file_name = str(uuid.uuid4())
 23 | 
 24 |     # Parse timestamp from query parameters
 25 |     offset = 60 * 60 * 9
 26 |     start_str = event["queryStringParameters"]["start"]
 27 |     start_time = time.mktime(
 28 |         datetime.strptime(start_str, "%Y-%m-%d %H:%M:%S").timetuple()) - offset
 29 |     
 30 |     end_str = event["queryStringParameters"]["end"]
 31 |     end_time = time.mktime(
 32 |         datetime.strptime(end_str, "%Y-%m-%d %H:%M:%S").timetuple()) - offset
 33 |     
 34 |     msg = event["Records"][0]
 35 |     body = json.loads(msg["body"])
 36 |     start_time = body["start"]
 37 |     end_time = body["end"]
 38 |         
 39 |     with open(f"/tmp/vid.ts", "wb") as video_file:
 40 |         while True:
 41 |             response = sqs.receive_message(
 42 |                 QueueUrl=queue_url,
 43 |                 AttributeNames=[
 44 |                     "SentTimestamp"
 45 |                 ],
 46 |                 MaxNumberOfMessages=1,
 47 |                 MessageAttributeNames=[
 48 |                     "All"
 49 |                 ],
 50 |                 VisibilityTimeout=100,
 51 |                 WaitTimeSeconds=0 # Short polling
 52 |             )
 53 |     
 54 |             # Non-empty response
 55 |             if "Messages" in response:
 56 |                 message = response["Messages"][0]
 57 |                 receipt_handle = message["ReceiptHandle"]
 58 |     
 59 |                 sent_timestamp = message['Attributes']['SentTimestamp']
 60 |     
 61 |                 # Delete received message from queue
 62 |                 sqs.delete_message(
 63 |                     QueueUrl=queue_url,
 64 |                     ReceiptHandle=receipt_handle
 65 |                 )
 66 |     
 67 |                 # if time.time() - float(sent_timestamp) / 1000 > 30:
 68 |                 if float(sent_timestamp) / 1000 > end_time:
 69 |                     break
 70 |                 elif float(sent_timestamp) / 1000 < start_time:
 71 |                     pass
 72 |                 else:
 73 |                     body = base64.b64decode(message["Body"].encode())
 74 |                     video_file.write(body)
 75 |             else:
 76 |                 # TODO: Check emptiness of SQS
 77 |                 break
 78 |     
 79 |     try:
 80 |         ffmpeg_cmd = f"/opt/ffmpeg/ffmpeg -i /tmp/vid.ts -c copy /tmp/{file_name}.mp4"
 81 |         command1 = shlex.split(ffmpeg_cmd)
 82 |         p1 = subprocess.run(command1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 83 |         s3.upload_file(Filename=f"/tmp/{file_name}.mp4", Bucket=bucket_name, Key=f"{file_name}.mp4")
 84 |         return {
 85 |             'statusCode': 200,
 86 |             'sent': json.dumps(float(sent_timestamp) / 1000),
 87 |             'start': json.dumps(start_time),
 88 |             'end': json.dumps(end_time),
 89 |             'body': json.dumps("Upload success")
 90 |         }
 91 |     except Exception as e:
 92 |         s3.put_object(Body=f"Error: {e}", Bucket=bucket_name, Key=f"error-{file_name}.txt")
 93 |         return {
 94 |             'statusCode': 400,
 95 |             'sent': json.dumps(float(sent_timestamp) / 1000),
 96 |             'start': json.dumps(start_time),
 97 |             'end': json.dumps(end_time),
 98 |             'body': json.dumps(f"Fail: {e}")
 99 |         }
100 |         
101 | 


--------------------------------------------------------------------------------
/influx/influxDB_video.py:
--------------------------------------------------------------------------------
  1 | import influxdb_client
  2 | import os
  3 | from dotenv import load_dotenv
  4 | import pandas as pd
  5 | from influxdb_client.client.write_api import SYNCHRONOUS  # Import SYNCHRONOUS
  6 | from datetime import timedelta
  7 | import csv
  8 | import sys
  9 | import base64
 10 | import boto3
 11 | from botocore.exceptions import ClientError
 12 | import logging
 13 | import shlex
 14 | import subprocess
 15 | 
 16 | # CSV의 필드 크기 제한을 해제한다
 17 | csv.field_size_limit(sys.maxsize)
 18 | 
 19 | 
 20 | # .env 파일으로부터 API 키를 가져온다
 21 | load_dotenv()
 22 | token = os.getenv('INFLUXDB_TOKEN')
 23 | 
 24 | # InfluxDB 설정
 25 | org = "HighLighter"
 26 | url = "http://13.125.176.29:8086"
 27 | bucket = "HighLighter"
 28 | 
 29 | # InfluxDB client 인스턴스를 생성하고, 쿼리 API를 사용하기 위해 인스턴스를 생성한다
 30 | client = influxdb_client.InfluxDBClient(url=url, token=token, org=org)
 31 | query_api = client.query_api()
 32 | start_time = "-2h"
 33 | 
 34 | # "kafka_data"으로부터 "value" 필드만 가져오는 Flux 쿼리를 작성한다
 35 | query = f'''
 36 | from(bucket: "{bucket}")
 37 |   |> range(start: {start_time})
 38 |   |> filter(fn: (r) => r["_measurement"] == "video_binary_2")
 39 |   |> filter(fn: (r) => r["_field"] == "value")
 40 |   |> keep(columns: ["_time", "_value"])
 41 | '''
 42 | 
 43 | # 쿼리를 실행하고 결과를 가져온다
 44 | tables = query_api.query(query)
 45 | 
 46 | # pandas DataFrame로 변환한다
 47 | results = []
 48 | for table in tables:
 49 |     for record in table.records:
 50 |         results.append({"time": record.get_time(), "video": record.get_value()})
 51 | 
 52 | df = pd.DataFrame(results)
 53 | 
 54 | 
 55 | def decode_byte_to_ts(df):
 56 |   """
 57 |   바이트를 .ts로 디코딩하여 파일로 저장한다.
 58 |   """
 59 |   video_file_path = "obama.ts"
 60 |   if not df.empty:
 61 |       first_entry_time = df['time'].iloc[0]
 62 |       end_entry_time = first_entry_time + timedelta(minutes=30)
 63 |       print(f"First entry time: {first_entry_time}, End time: {end_entry_time}")
 64 |       
 65 |       # 설정한 시간 범위 내의 데이터만 필터링한다
 66 |       df_filtered = df[(df['time'] >= first_entry_time) & (df['time'] <= end_entry_time)]
 67 |       print(df_filtered)
 68 |       
 69 |       print(f"Processing data from {first_entry_time} to {end_entry_time}")
 70 |       
 71 |       # Write the filtered data to a file
 72 |       with open(video_file_path, "wb") as video_file:
 73 |           for _, row in df_filtered.iterrows():
 74 |               decoded_chunk = base64.b64decode(row['video'].encode())
 75 |               video_file.write(decoded_chunk)
 76 |       
 77 |       print(f"Video file created at {video_file_path}")
 78 |   else:
 79 |       print("No data found in the specified time range.")
 80 | 
 81 | 
 82 | 
 83 | def ts_to_mp4(ts_path,mp4_path):
 84 |   """
 85 |   유튜브 업로드를 위해 from .ts to .mp4 변환
 86 |   """
 87 |   ffmpeg_cmd = f"ffmpeg -i {ts_path} -c copy {mp4_path}"
 88 |   command1 = shlex.split(ffmpeg_cmd)
 89 |   try:
 90 |       response=subprocess.run(command1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 91 |   except OSError as e:
 92 |       logging.error(e)
 93 |       return False
 94 |   print("변환 성공")
 95 |   return True
 96 | 
 97 | 
 98 | def upload_file(file_name, bucket, object_name=None):
 99 |   """
100 |   AWS S3 엑세스 키를 가져온다.
101 |   """
102 | 
103 |   # boto3 라이브러리로 s3에 mp4 파일 업로드 
104 |   s3_client = boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
105 |   try:
106 |       response = s3_client.upload_file(file_name, bucket, object_name)
107 |   except ClientError as e:
108 |       logging.error(e)
109 |       return False
110 |   print("업로드 성공!")
111 |   return True
112 | 
113 | 
114 | 
115 | # 환경 변수 불러오기
116 | AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
117 | AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
118 | AWS_DEFAULT_REGION = "ap-northeast-2"
119 | 
120 | 
121 | mp4_path="obama.mp4"
122 | ts_path= "obama.ts"
123 | 
124 | bucket_name="ybigta-highlight"
125 | object_name="obama.mp4"
126 | 
127 | 
128 | 
129 | # 실행코드: 바이트를 .ts로 디코딩하고 .ts를 .mp4로 변환하여 s3에 업로드
130 | decode_byte_to_ts(df)
131 | ts_to_mp4(ts_path,mp4_path)
132 | # upload_file(mp4_path, bucket_name, object_name)
133 | 
134 | 
135 | # 클라이언트를 종료
136 | client.close()


--------------------------------------------------------------------------------
/spark-streaming/spark-batch.py:
--------------------------------------------------------------------------------
  1 | from pyspark.sql import SparkSession
  2 | from pyspark.sql.functions import window, col, collect_list, udf
  3 | from pyspark.sql.types import StringType
  4 | import subprocess
  5 | import whisper
  6 | import uuid
  7 | import shlex
  8 | from pyspark.sql.streaming import StreamingQueryListener
  9 | 
 10 | import os
 11 | import pandas as pd
 12 | import time
 13 | import json
 14 | from dotenv import load_dotenv
 15 | import influxdb_client
 16 | from influxdb_client import Point, WritePrecision
 17 | from influxdb_client.client.write_api import SYNCHRONOUS
 18 | 
 19 | import os
 20 | import shutil
 21 | 
 22 | # create checkpoint folder
 23 | folder_path = 'checkpoint'
 24 | if not os.path.exists(folder_path):
 25 |     os.makedirs(folder_path)
 26 | else:
 27 |     shutil.rmtree(folder_path)
 28 |     os.makedirs(folder_path)
 29 | 
 30 | # load InfluxDB env setting
 31 | load_dotenv()
 32 | 
 33 | token = os.environ.get("INFLUXDB_TOKEN")
 34 | org = os.environ.get("INFLUXDB_ORG")
 35 | url = os.environ.get("INFLUXDB_URL")
 36 | bucket = os.environ.get("INFLUXDB_BUCKET")
 37 | 
 38 | # Load the Whisper model
 39 | stt_model = whisper.load_model("base")
 40 | 
 41 | # Initialize Spark session
 42 | spark = SparkSession.builder.getOrCreate()
 43 | spark.sparkContext.setLogLevel('ERROR')
 44 | 
 45 | # create influx client
 46 | client = influxdb_client.InfluxDBClient(url=url, token=token, org=org)
 47 | write_api = client.write_api(write_options=SYNCHRONOUS)
 48 | 
 49 | # Define custom query listener
 50 | class QueryListener(StreamingQueryListener):
 51 |     def onQueryStarted(self, event):
 52 |         print(f"Query started: id = {event.id}, name = {event.name}")
 53 | 
 54 |     def onQueryProgress(self, event):
 55 |         print(f"Query progress: id = {event}")
 56 | 
 57 |     def onQueryTerminated(self, event):
 58 |         print(f"Query terminated: id = {event}")
 59 | 
 60 | spark.streams.addListener(QueryListener())
 61 | 
 62 | 
 63 | def concat_byte_arrays(byte_array_list):
 64 |     binary_video_frames = b''.join(byte_array_list)
 65 |     u_key = str(uuid.uuid4())
 66 | 
 67 |     #make ts
 68 |     with open(f"video/output_{u_key}.ts", "wb") as video_file:
 69 |         video_file.write(binary_video_frames)
 70 | 
 71 |     try:
 72 |         ffmpeg_cmd = f"ffmpeg -i video/output_{u_key}.ts -c copy video/output_{u_key}.mp4"
 73 |         subprocess.run(shlex.split(ffmpeg_cmd), check=True)
 74 |     except Exception as e:
 75 |         print(f"Error: {e}")
 76 | 
 77 |     result = whisper.transcribe(stt_model, f"video/output_{u_key}.mp4")
 78 |     return result["text"]
 79 | 
 80 | concat_udf = udf(concat_byte_arrays, StringType())
 81 | 
 82 | 
 83 | # Read kafka topic and save in spark
 84 | df = spark \
 85 |     .readStream \
 86 |     .format("kafka") \
 87 |     .option("kafka.bootstrap.servers", "52.79.81.77:9092") \
 88 |     .option("kafka.group.id", "spark") \
 89 |     .option("subscribe", "mytopic") \
 90 |     .load()
 91 | 
 92 | df.selectExpr("value as video_frames", "timestamp as timestamp")
 93 | 
 94 | 
 95 | # aggregate 30 seconds of video byte array by using truncated window (not overlaped)
 96 | # and translate binary array to mp4, do stt after that and save into concatenated_frames
 97 | window_df = df \
 98 |     .withWatermark("timestamp", "30 seconds") \
 99 |     .groupBy(
100 |         window(col("timestamp"), "30 seconds")) \
101 |     .agg(concat_udf(collect_list("value")).alias("concatenated_frames"))
102 | 
103 | 
104 | def write_to_influxdb(batch_df, batch_id):
105 |     
106 |     for row in batch_df.collect():
107 |         text_data = row["concatenated_frames"]
108 |         timestamp = row["window"].start  
109 | 
110 |         print("batch_id", batch_id)
111 |         print("text_data", text_data)
112 |         
113 |         point = (
114 |             Point("video_text")
115 |             .field("text", text_data)
116 |             .time(timestamp, WritePrecision.S)
117 |         )
118 |         
119 |         write_api.write(bucket=bucket, org=org, record=point)
120 |         
121 |         u_key = str(uuid.uuid4())
122 | 
123 |         with open(f"test/{u_key}.json", "w", encoding="utf-8") as txt_file:
124 |             data = {
125 |                     "timestamp": str(timestamp),
126 |                     "text": text_data
127 |                 }
128 |             json.dump(data, txt_file, ensure_ascii=False, indent=4)
129 | 
130 | 
131 | # for check: save data 
132 | query = window_df \
133 |     .writeStream \
134 |     .foreachBatch(write_to_influxdb) \
135 |     .start()
136 | 
137 | query.awaitTermination()
138 | 
139 | client.close()


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | youtube-live/.env
  2 | youtube-live/chat/*
  3 | add-java-docker
  4 | youtube-live/__pychcache__/*
  5 | .idea/*
  6 | .vscode/*
  7 | .DS_Store
  8 | videos/*
  9 | 
 10 | # Terraform
 11 | .terraform*
 12 | terraform.tfstate
 13 | terraform.tfstate.backup
 14 | =======
 15 | 
 16 | # Created by https://www.toptal.com/developers/gitignore/api/python
 17 | # Edit at https://www.toptal.com/developers/gitignore?templates=python
 18 | 
 19 | ### Python ###
 20 | # Byte-compiled / optimized / DLL files
 21 | __pycache__/
 22 | *.py[cod]
 23 | *$py.class
 24 | 
 25 | # C extensions
 26 | *.so
 27 | 
 28 | # Distribution / packaging
 29 | .Python
 30 | build/
 31 | develop-eggs/
 32 | dist/
 33 | downloads/
 34 | eggs/
 35 | .eggs/
 36 | lib/
 37 | lib64/
 38 | parts/
 39 | sdist/
 40 | var/
 41 | wheels/
 42 | share/python-wheels/
 43 | *.egg-info/
 44 | .installed.cfg
 45 | *.egg
 46 | MANIFEST
 47 | 
 48 | # PyInstaller
 49 | #  Usually these files are written by a python script from a template
 50 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 51 | *.manifest
 52 | *.spec
 53 | 
 54 | # Installer logs
 55 | pip-log.txt
 56 | pip-delete-this-directory.txt
 57 | 
 58 | # Unit test / coverage reports
 59 | htmlcov/
 60 | .tox/
 61 | .nox/
 62 | .coverage
 63 | .coverage.*
 64 | .cache
 65 | nosetests.xml
 66 | coverage.xml
 67 | *.cover
 68 | *.py,cover
 69 | .hypothesis/
 70 | .pytest_cache/
 71 | cover/
 72 | 
 73 | # Translations
 74 | *.mo
 75 | *.pot
 76 | 
 77 | # Django stuff:
 78 | *.log
 79 | local_settings.py
 80 | db.sqlite3
 81 | db.sqlite3-journal
 82 | 
 83 | # Flask stuff:
 84 | instance/
 85 | .webassets-cache
 86 | 
 87 | # Scrapy stuff:
 88 | .scrapy
 89 | 
 90 | # Sphinx documentation
 91 | docs/_build/
 92 | 
 93 | # PyBuilder
 94 | .pybuilder/
 95 | target/
 96 | 
 97 | # Jupyter Notebook
 98 | .ipynb_checkpoints
 99 | 
100 | # IPython
101 | profile_default/
102 | ipython_config.py
103 | 
104 | # pyenv
105 | #   For a library or package, you might want to ignore these files since the code is
106 | #   intended to run in multiple environments; otherwise, check them in:
107 | # .python-version
108 | 
109 | # pipenv
110 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
111 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
112 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
113 | #   install all needed dependencies.
114 | #Pipfile.lock
115 | 
116 | # poetry
117 | #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
118 | #   This is especially recommended for binary packages to ensure reproducibility, and is more
119 | #   commonly ignored for libraries.
120 | #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
121 | #poetry.lock
122 | 
123 | # pdm
124 | #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
125 | #pdm.lock
126 | #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
127 | #   in version control.
128 | #   https://pdm.fming.dev/#use-with-ide
129 | .pdm.toml
130 | 
131 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
132 | __pypackages__/
133 | 
134 | # Celery stuff
135 | celerybeat-schedule
136 | celerybeat.pid
137 | 
138 | # SageMath parsed files
139 | *.sage.py
140 | 
141 | # Environments
142 | .env
143 | .venv
144 | env/
145 | venv/
146 | ENV/
147 | env.bak/
148 | venv.bak/
149 | 
150 | # Spyder project settings
151 | .spyderproject
152 | .spyproject
153 | 
154 | # Rope project settings
155 | .ropeproject
156 | 
157 | # mkdocs documentation
158 | /site
159 | 
160 | # mypy
161 | .mypy_cache/
162 | .dmypy.json
163 | dmypy.json
164 | 
165 | # Pyre type checker
166 | .pyre/
167 | 
168 | # pytype static type analyzer
169 | .pytype/
170 | 
171 | # Cython debug symbols
172 | cython_debug/
173 | 
174 | # PyCharm
175 | #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
176 | #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
177 | #  and can be added to the global gitignore or merged into this file.  For a more nuclear
178 | #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
179 | #.idea/
180 | 
181 | ### Python Patch ###
182 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
183 | poetry.toml
184 | 
185 | # ruff
186 | .ruff_cache/
187 | 
188 | # LSP config files
189 | pyrightconfig.json
190 | 
191 | # End of https://www.toptal.com/developers/gitignore/api/python
192 | 
193 | 


--------------------------------------------------------------------------------
/video-upload/lambda_videoUpload.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import random
  3 | import time
  4 | import boto3
  5 | import google.auth
  6 | import google.auth.transport.requests
  7 | import urllib.parse
  8 | from googleapiclient.discovery import build
  9 | from googleapiclient.errors import HttpError
 10 | from googleapiclient.http import MediaFileUpload
 11 | from google.oauth2.credentials import Credentials
 12 | from tempfile import NamedTemporaryFile
 13 | 
 14 | # Secrets Manager 설정
 15 | OAUTH2_SECRET_NAME = "upload_video.py-oauth2.json"
 16 | CLIENT_SECRET_NAME = "client_secrets.json"
 17 | REGION_NAME = "ap-northeast-2"
 18 | 
 19 | # YouTube API 설정
 20 | YOUTUBE_API_SERVICE_NAME = "youtube"
 21 | YOUTUBE_API_VERSION = "v3"
 22 | 
 23 | MAX_RETRIES = 10
 24 | RETRIABLE_STATUS_CODES = [500, 502, 503, 504]
 25 | 
 26 | def get_secret(secret_name):
 27 |     session = boto3.session.Session()
 28 |     client = session.client(service_name='secretsmanager', region_name=REGION_NAME)
 29 |     try:
 30 |         get_secret_value_response = client.get_secret_value(SecretId=secret_name)
 31 |     except boto3.exceptions.Boto3Error as e:
 32 |         raise e
 33 |     return get_secret_value_response['SecretString']
 34 | 
 35 | def save_secret_to_file(secret, file_path):
 36 |     with open(file_path, 'w') as secret_file:
 37 |         secret_file.write(secret)
 38 | 
 39 | def get_authenticated_service():
 40 |     oauth2_credentials_json = get_secret(OAUTH2_SECRET_NAME)
 41 |     client_secret_json = get_secret(CLIENT_SECRET_NAME)
 42 |     
 43 |     oauth2_file_path = "/tmp/upload_video-oauth2.json"
 44 |     client_secret_file_path = "/tmp/client_secrets.json"
 45 |     
 46 |     save_secret_to_file(oauth2_credentials_json, oauth2_file_path)
 47 |     save_secret_to_file(client_secret_json, client_secret_file_path)
 48 | 
 49 |     creds = Credentials.from_authorized_user_file(oauth2_file_path)
 50 |     if not creds.valid:
 51 |         creds.refresh(google.auth.transport.requests.Request())
 52 | 
 53 |     return build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, credentials=creds)
 54 | 
 55 | def initialize_upload(youtube, options):
 56 |     tags = None
 57 |     if 'keywords' in options:
 58 |         tags = options['keywords'].split(",")
 59 | 
 60 |     body = dict(
 61 |         snippet=dict(
 62 |             title=options.get('title', "Test Title"),
 63 |             description=options.get('description', "Test Description"),
 64 |             tags=tags,
 65 |             categoryId=options.get('category', "22")
 66 |         ),
 67 |         status=dict(
 68 |             privacyStatus=options.get('privacyStatus', "private")
 69 |         )
 70 |     )
 71 | 
 72 |     insert_request = youtube.videos().insert(
 73 |         part=",".join(list(body.keys())),
 74 |         body=body,
 75 |         media_body=MediaFileUpload(options['file'], chunksize=-1, resumable=True)
 76 |     )
 77 | 
 78 |     resumable_upload(insert_request)
 79 | 
 80 | def resumable_upload(insert_request):
 81 |     response = None
 82 |     error = None
 83 |     retry = 0
 84 |     while response is None:
 85 |         try:
 86 |             print("Uploading file...")
 87 |             status, response = insert_request.next_chunk()
 88 |             if response is not None:
 89 |                 if 'id' in response:
 90 |                     print(f"Video id '{response['id']}' was successfully uploaded.")
 91 |                 else:
 92 |                     raise Exception(f"The upload failed with an unexpected response: {response}")
 93 |         except HttpError as e:
 94 |             if e.resp.status in RETRIABLE_STATUS_CODES:
 95 |                 error = f"A retriable HTTP error {e.resp.status} occurred:\n{e.content}"
 96 |             else:
 97 |                 raise
 98 |         except Exception as e:
 99 |             error = f"A retriable error occurred: {e}"
100 | 
101 |         if error is not None:
102 |             print(error)
103 |             retry += 1
104 |             if retry > MAX_RETRIES:
105 |                 raise Exception("No longer attempting to retry.")
106 | 
107 |             max_sleep = 2 ** retry
108 |             sleep_seconds = random.random() * max_sleep
109 |             print(f"Sleeping {sleep_seconds} seconds and then retrying...")
110 |             time.sleep(sleep_seconds)
111 | 
112 | 
113 | def lambda_handler(event, context):
114 |     print(f"Event data: {event}")  # 이벤트 데이터 로깅
115 |     
116 |     # S3 버킷 이름과 객체 키 추출
117 |     try:
118 |         s3_bucket = event['Records'][0]['s3']['bucket']['name']
119 |         s3_key = urllib.parse.unquote(event['Records'][0]['s3']['object']['key'])
120 |     except KeyError as e:
121 |         raise ValueError(f"Failed to extract S3 bucket or key: {str(e)}")
122 | 
123 |     # Verify the object exists
124 |     s3 = boto3.client('s3')
125 |     try:
126 |         s3.head_object(Bucket=s3_bucket, Key=s3_key)
127 |     except botocore.exceptions.ClientError as e:
128 |         if e.response['Error']['Code'] == '404':
129 |             raise FileNotFoundError(f"The object {s3_key} does not exist in the bucket {s3_bucket}")
130 |         else:
131 |             raise
132 | 
133 |     with NamedTemporaryFile(delete=False) as temp_file:
134 |         s3.download_fileobj(s3_bucket, s3_key, temp_file)
135 |         temp_file_path = temp_file.name
136 | 
137 |     file_name = os.path.basename(s3_key)
138 |     title = os.path.splitext(file_name)[0]  # 확장자를 제거한 파일 이름을 title로 사용
139 | 
140 |     description = event.get('description', "Test Description")
141 |     category = event.get('category', "22")
142 |     keywords = event.get('keywords', "")
143 |     privacy_status = event.get('privacyStatus', "private")
144 | 
145 |     youtube = get_authenticated_service()
146 | 
147 |     options = {
148 |         "file": temp_file_path,
149 |         "title": title,
150 |         "description": description,
151 |         "category": category,
152 |         "keywords": keywords,
153 |         "privacyStatus": privacy_status
154 |     }
155 | 
156 |     try:
157 |         initialize_upload(youtube, options)
158 |     except HttpError as e:
159 |         print(f"An HTTP error {e.resp.status} occurred:\n{e.content}")
160 |     except Exception as e:
161 |         print(f"An unexpected error occurred: {e}")
162 |     
163 |     os.remove(temp_file_path)  # 임시 파일 정리
164 | 
165 |     return {
166 |         'statusCode': 200,
167 |         'body': 'Video uploaded successfully!'
168 |     }
169 | 


--------------------------------------------------------------------------------
/highlighter-sliding-window/src/main/java/kafkastreams/example/KStream/KStreams.java:
--------------------------------------------------------------------------------
  1 | package kafkastreams.example.KStream;
  2 | 
  3 | import org.apache.kafka.common.serialization.Serdes;
  4 | import org.apache.kafka.streams.KafkaStreams;
  5 | import org.apache.kafka.streams.StreamsBuilder;
  6 | import org.apache.kafka.streams.kstream.*;
  7 | import org.apache.kafka.streams.StreamsConfig;
  8 | import org.apache.kafka.streams.KeyValue;
  9 | import org.apache.kafka.streams.processor.ProcessorContext;
 10 | 
 11 | import java.time.Duration;
 12 | import java.time.Instant;
 13 | import java.time.LocalDateTime;
 14 | import java.time.ZoneId;
 15 | import java.time.format.DateTimeFormatter;
 16 | import java.util.Properties;
 17 | 
 18 | import org.slf4j.Logger;
 19 | import org.slf4j.LoggerFactory;
 20 | 
 21 | public class KStreams {
 22 |     private static final String BOOTSTRAP_SERVERS = "43.203.141.74:9092"; // IP 바꾸세요
 23 |     private static final String APPLICATION_NAME = "timestamp-count-application";
 24 |     private static final String STREAM_SOURCE = "stream_filter";  // Source 바꾸세요
 25 |     private static final String STREAM_SINK = "stream_filter_sink";   // Sink 바꾸세요
 26 |     private static final long THRESHOLD = 3;  // Example threshold
 27 | 
 28 |     public static void main(String[] args) {
 29 |         Properties properties = new Properties();
 30 |         properties.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_NAME);
 31 |         properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
 32 |         properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
 33 |         properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
 34 | 
 35 |         StreamsBuilder builder = new StreamsBuilder();
 36 |         KStream<String, String> streamLog = builder.stream(STREAM_SOURCE);
 37 |         DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
 38 | 
 39 |         KStream<String, Long> timestampsToEpoch = streamLog
 40 |                 .filter((key, value) -> value.matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}"))
 41 |                 .mapValues(value -> {
 42 |                     try {
 43 |                         return LocalDateTime.parse(value, formatter)
 44 |                                 .atZone(ZoneId.systemDefault())
 45 |                                 .toInstant()
 46 |                                 .toEpochMilli();
 47 |                     } catch (Exception e) {
 48 |                         System.err.println("Failed to parse date: " + value);
 49 |                         return null;
 50 |                     }
 51 |                 });
 52 | 
 53 |         // Log after converting to epoch
 54 |         timestampsToEpoch.peek((key, value) -> System.out.println("     [EPOCH VALUE]: " + value));
 55 | 
 56 |         KTable<Windowed<Long>, Long> timestampCounts = timestampsToEpoch
 57 |                 .filter((key, value) -> value != null)
 58 |                 .groupBy((key, value) -> value, Grouped.with(Serdes.Long(), Serdes.Long()))
 59 |                 .windowedBy(TimeWindows.of(Duration.ofSeconds(20)).advanceBy(Duration.ofSeconds(1)))
 60 |                 .count();
 61 | 
 62 |         // Log the windowed counts
 63 | //        timestampCounts.toStream().peek((key, value) -> System.out.println("     [WINDOW COUNT]: - Key: " + key + ", Count: " + value));
 64 | 
 65 |         KStream<String, String> outputStream = timestampCounts
 66 |                 .toStream()
 67 |                 .transform(() -> new ThresholdExceedTransformer(), Named.as("ThresholdExceed"));
 68 | 
 69 |         outputStream.peek((key, value) -> System.out.println("[SEND] Sending to Kafka: Key = " + key + ", Value = " + value));
 70 |         outputStream.to(STREAM_SINK, Produced.with(Serdes.String(), Serdes.String()));
 71 | 
 72 |         KafkaStreams streams = new KafkaStreams(builder.build(), properties);
 73 |         streams.start();
 74 |         Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
 75 |     }
 76 | 
 77 |     static class ThresholdExceedTransformer implements Transformer<Windowed<Long>, Long, KeyValue<String, String>> {
 78 |         private boolean thresholdExceeded = false;
 79 |         private long startTime = 0;
 80 |         private final Logger log = LoggerFactory.getLogger(ThresholdExceedTransformer.class);
 81 | 
 82 |         @Override
 83 |         public void init(ProcessorContext context) {
 84 |         }
 85 | 
 86 |         @Override
 87 |         public KeyValue<String, String> transform(Windowed<Long> key, Long value) {
 88 |             DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
 89 |             long durationThreshold = Duration.ofSeconds(30).toMillis();  // 30초를 밀리초 단위로 변환
 90 |             long oneMinuteMillis = Duration.ofMinutes(1).toMillis();  // 1분을 밀리초 단위로 변환
 91 | 
 92 |             // 윈도우의 시작 시간과 종료 시간을 문자열로 변환
 93 |             String windowStart = Instant.ofEpochMilli(key.window().start()).atZone(ZoneId.systemDefault()).format(formatter);
 94 |             String windowEnd = Instant.ofEpochMilli(key.window().end()).atZone(ZoneId.systemDefault()).format(formatter);
 95 |             log.info("     [WINDOW]: " + windowStart + " to " + windowEnd + ", has count: " + value);
 96 | 
 97 |             if (value > THRESHOLD) {
 98 |                 if (!thresholdExceeded) {
 99 |                     // 임계값을 처음으로 초과하는 윈도우라면, 시작 시간을 설정
100 |                     thresholdExceeded = true;
101 |                     startTime = key.window().start();
102 |                 }
103 |                 return null; // 이벤트가 계속됨을 표시
104 |             } else if (thresholdExceeded) {
105 |                 // 임계값을 넘지 못하고, 이전에 임계값을 넘었던 경우 종료 시간 설정
106 |                 thresholdExceeded = false;
107 |                 long endTime = key.window().end(); // 현재 윈도우의 종료 시점을 이벤트의 종료 시간으로 설정
108 |                 long duration = endTime - startTime; // 이벤트의 지속 시간 계산
109 | 
110 |                 if (duration < durationThreshold) {
111 |                     // 지속 시간이 30초 미만인 경우 아무 것도 반환하지 않음
112 |                     return null;
113 |                 }
114 | 
115 |                 if (duration > oneMinuteMillis) {
116 |                     // 지속 시간이 1분을 초과하는 경우, 종료 시간을 시작 시간에서 1분 뒤로 설정
117 |                     endTime = startTime + oneMinuteMillis;
118 |                 }
119 | 
120 |                 String start = Instant.ofEpochMilli(startTime).atZone(ZoneId.systemDefault()).format(formatter);
121 |                 String end = Instant.ofEpochMilli(endTime).atZone(ZoneId.systemDefault()).format(formatter);
122 | 
123 |                 return new KeyValue<>(start, start + " to " + end);
124 |             }
125 |             return null;
126 |         }
127 | 
128 |         @Override
129 |         public void close() {
130 |         }
131 |     }
132 | }
133 | 


--------------------------------------------------------------------------------
/highlighter-sliding-window/gradlew:
--------------------------------------------------------------------------------
  1 | #!/bin/sh
  2 | 
  3 | #
  4 | # Copyright © 2015-2021 the original authors.
  5 | #
  6 | # Licensed under the Apache License, Version 2.0 (the "License");
  7 | # you may not use this file except in compliance with the License.
  8 | # You may obtain a copy of the License at
  9 | #
 10 | #      https://www.apache.org/licenses/LICENSE-2.0
 11 | #
 12 | # Unless required by applicable law or agreed to in writing, software
 13 | # distributed under the License is distributed on an "AS IS" BASIS,
 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 | # See the License for the specific language governing permissions and
 16 | # limitations under the License.
 17 | #
 18 | 
 19 | ##############################################################################
 20 | #
 21 | #   Gradle start up script for POSIX generated by Gradle.
 22 | #
 23 | #   Important for running:
 24 | #
 25 | #   (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
 26 | #       noncompliant, but you have some other compliant shell such as ksh or
 27 | #       bash, then to run this script, type that shell name before the whole
 28 | #       command line, like:
 29 | #
 30 | #           ksh Gradle
 31 | #
 32 | #       Busybox and similar reduced shells will NOT work, because this script
 33 | #       requires all of these POSIX shell features:
 34 | #         * functions;
 35 | #         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
 36 | #           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
 37 | #         * compound commands having a testable exit status, especially «case»;
 38 | #         * various built-in commands including «command», «set», and «ulimit».
 39 | #
 40 | #   Important for patching:
 41 | #
 42 | #   (2) This script targets any POSIX shell, so it avoids extensions provided
 43 | #       by Bash, Ksh, etc; in particular arrays are avoided.
 44 | #
 45 | #       The "traditional" practice of packing multiple parameters into a
 46 | #       space-separated string is a well documented source of bugs and security
 47 | #       problems, so this is (mostly) avoided, by progressively accumulating
 48 | #       options in "$@", and eventually passing that to Java.
 49 | #
 50 | #       Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
 51 | #       and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
 52 | #       see the in-line comments for details.
 53 | #
 54 | #       There are tweaks for specific operating systems such as AIX, CygWin,
 55 | #       Darwin, MinGW, and NonStop.
 56 | #
 57 | #   (3) This script is generated from the Groovy template
 58 | #       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 59 | #       within the Gradle project.
 60 | #
 61 | #       You can find Gradle at https://github.com/gradle/gradle/.
 62 | #
 63 | ##############################################################################
 64 | 
 65 | # Attempt to set APP_HOME
 66 | 
 67 | # Resolve links: $0 may be a link
 68 | app_path=$0
 69 | 
 70 | # Need this for daisy-chained symlinks.
 71 | while
 72 |     APP_HOME=${app_path%"${app_path##*/}"}  # leaves a trailing /; empty if no leading path
 73 |     [ -h "$app_path" ]
 74 | do
 75 |     ls=$( ls -ld "$app_path" )
 76 |     link=${ls#*' -> '}
 77 |     case $link in             #(
 78 |       /*)   app_path=$link ;; #(
 79 |       *)    app_path=$APP_HOME$link ;;
 80 |     esac
 81 | done
 82 | 
 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 84 | 
 85 | APP_NAME="Gradle"
 86 | APP_BASE_NAME=${0##*/}
 87 | 
 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 90 | 
 91 | # Use the maximum available, or set MAX_FD != -1 to use that value.
 92 | MAX_FD=maximum
 93 | 
 94 | warn () {
 95 |     echo "$*"
 96 | } >&2
 97 | 
 98 | die () {
 99 |     echo
100 |     echo "$*"
101 |     echo
102 |     exit 1
103 | } >&2
104 | 
105 | # OS specific support (must be 'true' or 'false').
106 | cygwin=false
107 | msys=false
108 | darwin=false
109 | nonstop=false
110 | case "$( uname )" in                #(
111 |   CYGWIN* )         cygwin=true  ;; #(
112 |   Darwin* )         darwin=true  ;; #(
113 |   MSYS* | MINGW* )  msys=true    ;; #(
114 |   NONSTOP* )        nonstop=true ;;
115 | esac
116 | 
117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
118 | 
119 | 
120 | # Determine the Java command to use to start the JVM.
121 | if [ -n "$JAVA_HOME" ] ; then
122 |     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
123 |         # IBM's JDK on AIX uses strange locations for the executables
124 |         JAVACMD=$JAVA_HOME/jre/sh/java
125 |     else
126 |         JAVACMD=$JAVA_HOME/bin/java
127 |     fi
128 |     if [ ! -x "$JAVACMD" ] ; then
129 |         die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
130 | 
131 | Please set the JAVA_HOME variable in your environment to match the
132 | location of your Java installation."
133 |     fi
134 | else
135 |     JAVACMD=java
136 |     which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
137 | 
138 | Please set the JAVA_HOME variable in your environment to match the
139 | location of your Java installation."
140 | fi
141 | 
142 | # Increase the maximum file descriptors if we can.
143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
144 |     case $MAX_FD in #(
145 |       max*)
146 |         MAX_FD=$( ulimit -H -n ) ||
147 |             warn "Could not query maximum file descriptor limit"
148 |     esac
149 |     case $MAX_FD in  #(
150 |       '' | soft) :;; #(
151 |       *)
152 |         ulimit -n "$MAX_FD" ||
153 |             warn "Could not set maximum file descriptor limit to $MAX_FD"
154 |     esac
155 | fi
156 | 
157 | # Collect all arguments for the java command, stacking in reverse order:
158 | #   * args from the command line
159 | #   * the main class name
160 | #   * -classpath
161 | #   * -D...appname settings
162 | #   * --module-path (only if needed)
163 | #   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
164 | 
165 | # For Cygwin or MSYS, switch paths to Windows format before running java
166 | if "$cygwin" || "$msys" ; then
167 |     APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
168 |     CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
169 | 
170 |     JAVACMD=$( cygpath --unix "$JAVACMD" )
171 | 
172 |     # Now convert the arguments - kludge to limit ourselves to /bin/sh
173 |     for arg do
174 |         if
175 |             case $arg in                                #(
176 |               -*)   false ;;                            # don't mess with options #(
177 |               /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath
178 |                     [ -e "$t" ] ;;                      #(
179 |               *)    false ;;
180 |             esac
181 |         then
182 |             arg=$( cygpath --path --ignore --mixed "$arg" )
183 |         fi
184 |         # Roll the args list around exactly as many times as the number of
185 |         # args, so each arg winds up back in the position where it started, but
186 |         # possibly modified.
187 |         #
188 |         # NB: a `for` loop captures its iteration list before it begins, so
189 |         # changing the positional parameters here affects neither the number of
190 |         # iterations, nor the values presented in `arg`.
191 |         shift                   # remove old arg
192 |         set -- "$@" "$arg"      # push replacement arg
193 |     done
194 | fi
195 | 
196 | # Collect all arguments for the java command;
197 | #   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
198 | #     shell script including quotes and variable substitutions, so put them in
199 | #     double quotes to make sure that they get re-expanded; and
200 | #   * put everything else in single quotes, so that it's not re-expanded.
201 | 
202 | set -- \
203 |         "-Dorg.gradle.appname=$APP_BASE_NAME" \
204 |         -classpath "$CLASSPATH" \
205 |         org.gradle.wrapper.GradleWrapperMain \
206 |         "$@"
207 | 
208 | # Stop when "xargs" is not available.
209 | if ! command -v xargs >/dev/null 2>&1
210 | then
211 |     die "xargs is not available"
212 | fi
213 | 
214 | # Use "xargs" to parse quoted args.
215 | #
216 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed.
217 | #
218 | # In Bash we could simply go:
219 | #
220 | #   readarray ARGS < <( xargs -n1 <<<"$var" ) &&
221 | #   set -- "${ARGS[@]}" "$@"
222 | #
223 | # but POSIX shell has neither arrays nor command substitution, so instead we
224 | # post-process each arg (as a line of input to sed) to backslash-escape any
225 | # character that might be a shell metacharacter, then use eval to reverse
226 | # that process (while maintaining the separation between arguments), and wrap
227 | # the whole thing up as a single "set" statement.
228 | #
229 | # This will of course break if any of these variables contains a newline or
230 | # an unmatched quote.
231 | #
232 | 
233 | eval "set -- $(
234 |         printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
235 |         xargs -n1 |
236 |         sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
237 |         tr '\n' ' '
238 |     )" '"$@"'
239 | 
240 | exec "$JAVACMD" "$@"
241 | 


--------------------------------------------------------------------------------
/video-stream/sdk-test.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "id": "6e813da8",
  7 |    "metadata": {},
  8 |    "outputs": [],
  9 |    "source": [
 10 |     "import boto3\n",
 11 |     "\n",
 12 |     "sqs = boto3.client('sqs')"
 13 |    ]
 14 |   },
 15 |   {
 16 |    "cell_type": "code",
 17 |    "execution_count": 2,
 18 |    "id": "e9c2c381",
 19 |    "metadata": {},
 20 |    "outputs": [],
 21 |    "source": [
 22 |     "response = sqs.get_queue_url(QueueName='highlighter.fifo')\n",
 23 |     "queue_url = response[\"QueueUrl\"]"
 24 |    ]
 25 |   },
 26 |   {
 27 |    "cell_type": "code",
 28 |    "execution_count": 8,
 29 |    "id": "4f503581",
 30 |    "metadata": {},
 31 |    "outputs": [
 32 |     {
 33 |      "name": "stdout",
 34 |      "output_type": "stream",
 35 |      "text": [
 36 |       "<class 'datetime.datetime'>\n"
 37 |      ]
 38 |     }
 39 |    ],
 40 |    "source": [
 41 |     "import datetime\n",
 42 |     "\n",
 43 |     "print(type(datetime.datetime.now()))"
 44 |    ]
 45 |   },
 46 |   {
 47 |    "cell_type": "code",
 48 |    "execution_count": 9,
 49 |    "id": "525ce046",
 50 |    "metadata": {},
 51 |    "outputs": [],
 52 |    "source": [
 53 |     "response = sqs.send_message(\n",
 54 |     "    QueueUrl=queue_url,\n",
 55 |     "    MessageAttributes={\n",
 56 |     "        \"Timestamp\": {\n",
 57 |     "            \"DataType\": \"String\",\n",
 58 |     "            \"StringValue\": str(datetime.datetime.now())\n",
 59 |     "        }\n",
 60 |     "    },\n",
 61 |     "    MessageBody=(\n",
 62 |     "        \"Information about current NY Times fiction bestseller for \"\n",
 63 |     "        \"week of 12/11/2016.\"\n",
 64 |     "    ),\n",
 65 |     "    MessageGroupId=\"1\",\n",
 66 |     "    MessageDeduplicationId=\"1\",\n",
 67 |     ")"
 68 |    ]
 69 |   },
 70 |   {
 71 |    "cell_type": "code",
 72 |    "execution_count": 10,
 73 |    "id": "74eda5f0",
 74 |    "metadata": {},
 75 |    "outputs": [
 76 |     {
 77 |      "name": "stdout",
 78 |      "output_type": "stream",
 79 |      "text": [
 80 |       "f05218f2-da9f-43ed-8aee-43aeac3889dd\n"
 81 |      ]
 82 |     }
 83 |    ],
 84 |    "source": [
 85 |     "print(response[\"MessageId\"])"
 86 |    ]
 87 |   },
 88 |   {
 89 |    "cell_type": "code",
 90 |    "execution_count": 33,
 91 |    "id": "0ebc2996",
 92 |    "metadata": {
 93 |     "scrolled": true
 94 |    },
 95 |    "outputs": [],
 96 |    "source": [
 97 |     "response = sqs.receive_message(\n",
 98 |     "    QueueUrl=queue_url,\n",
 99 |     "    AttributeNames=[\n",
100 |     "        'All'\n",
101 |     "    ],\n",
102 |     "    MaxNumberOfMessages=1,\n",
103 |     "    MessageAttributeNames=[\n",
104 |     "        'All'\n",
105 |     "    ],\n",
106 |     "    VisibilityTimeout=100,\n",
107 |     "    WaitTimeSeconds=0\n",
108 |     ")"
109 |    ]
110 |   },
111 |   {
112 |    "cell_type": "code",
113 |    "execution_count": 41,
114 |    "id": "84fa568e",
115 |    "metadata": {},
116 |    "outputs": [
117 |     {
118 |      "name": "stdout",
119 |      "output_type": "stream",
120 |      "text": [
121 |       "{'ResponseMetadata': {'RequestId': 'fb7f413f-5d93-55fc-a5fb-1f8dc3ad200e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'fb7f413f-5d93-55fc-a5fb-1f8dc3ad200e', 'date': 'Mon, 06 May 2024 08:59:42 GMT', 'content-type': 'application/x-amz-json-1.0', 'content-length': '2', 'connection': 'keep-alive'}, 'RetryAttempts': 0}}\n"
122 |      ]
123 |     }
124 |    ],
125 |    "source": [
126 |     "print(response)"
127 |    ]
128 |   },
129 |   {
130 |    "cell_type": "code",
131 |    "execution_count": null,
132 |    "id": "5ccb8ee4",
133 |    "metadata": {},
134 |    "outputs": [],
135 |    "source": [
136 |     "message = response['Messages'][0]\n",
137 |     "receipt_handle = message['ReceiptHandle']\n",
138 |     "\n",
139 |     "# Delete received message from queue\n",
140 |     "sqs.delete_message(\n",
141 |     "    QueueUrl=queue_url,\n",
142 |     "    ReceiptHandle=receipt_handle\n",
143 |     ")\n",
144 |     "print('Received and deleted message: %s' % message)"
145 |    ]
146 |   },
147 |   {
148 |    "cell_type": "code",
149 |    "execution_count": 24,
150 |    "id": "2d1af959",
151 |    "metadata": {},
152 |    "outputs": [
153 |     {
154 |      "name": "stdout",
155 |      "output_type": "stream",
156 |      "text": [
157 |       "1714985117254\n"
158 |      ]
159 |     }
160 |    ],
161 |    "source": [
162 |     "print(message['Attributes']['SentTimestamp'])"
163 |    ]
164 |   },
165 |   {
166 |    "cell_type": "code",
167 |    "execution_count": 44,
168 |    "id": "12921acb",
169 |    "metadata": {},
170 |    "outputs": [
171 |     {
172 |      "name": "stdout",
173 |      "output_type": "stream",
174 |      "text": [
175 |       "1714986409.274139\n",
176 |       "1292.0202059745789\n"
177 |      ]
178 |     }
179 |    ],
180 |    "source": [
181 |     "import time\n",
182 |     "\n",
183 |     "# `time.time()`: Epoch time in seconds\n",
184 |     "# `SentTimestamp`: Epoch time in milliseconds\n",
185 |     "print(time.time())\n",
186 |     "print(time.time() - float(message['Attributes']['SentTimestamp']) / 1000)"
187 |    ]
188 |   },
189 |   {
190 |    "cell_type": "code",
191 |    "execution_count": 19,
192 |    "id": "3534b78d",
193 |    "metadata": {},
194 |    "outputs": [
195 |     {
196 |      "name": "stdout",
197 |      "output_type": "stream",
198 |      "text": [
199 |       "2024-05-06 17:45:17.110932\n"
200 |      ]
201 |     }
202 |    ],
203 |    "source": [
204 |     "print(message['MessageAttributes']['Timestamp']['StringValue'])"
205 |    ]
206 |   },
207 |   {
208 |    "cell_type": "code",
209 |    "execution_count": 14,
210 |    "id": "7194a63e",
211 |    "metadata": {},
212 |    "outputs": [
213 |     {
214 |      "name": "stdout",
215 |      "output_type": "stream",
216 |      "text": [
217 |       "Information about current NY Times fiction bestseller for week of 12/11/2016.\n"
218 |      ]
219 |     }
220 |    ],
221 |    "source": [
222 |     "print(message['Body'])"
223 |    ]
224 |   },
225 |   {
226 |    "cell_type": "code",
227 |    "execution_count": 58,
228 |    "id": "4376082d",
229 |    "metadata": {},
230 |    "outputs": [
231 |     {
232 |      "data": {
233 |       "text/plain": [
234 |        "{'ResponseMetadata': {'RequestId': '9e8fc2bc-b1ae-5b4c-ac72-9c120990bc99',\n",
235 |        "  'HTTPStatusCode': 200,\n",
236 |        "  'HTTPHeaders': {'x-amzn-requestid': '9e8fc2bc-b1ae-5b4c-ac72-9c120990bc99',\n",
237 |        "   'date': 'Fri, 03 May 2024 10:01:34 GMT',\n",
238 |        "   'content-type': 'application/x-amz-json-1.0',\n",
239 |        "   'content-length': '0',\n",
240 |        "   'connection': 'keep-alive'},\n",
241 |        "  'RetryAttempts': 0}}"
242 |       ]
243 |      },
244 |      "execution_count": 58,
245 |      "metadata": {},
246 |      "output_type": "execute_result"
247 |     }
248 |    ],
249 |    "source": [
250 |     "sqs.purge_queue(QueueUrl=queue_url)"
251 |    ]
252 |   },
253 |   {
254 |    "cell_type": "code",
255 |    "execution_count": 41,
256 |    "id": "d969b75f",
257 |    "metadata": {
258 |     "scrolled": true
259 |    },
260 |    "outputs": [],
261 |    "source": [
262 |     "import base64\n",
263 |     "a = base64.b64decode(message[\"Body\"].encode())"
264 |    ]
265 |   },
266 |   {
267 |    "cell_type": "code",
268 |    "execution_count": 70,
269 |    "id": "75a107e6",
270 |    "metadata": {},
271 |    "outputs": [],
272 |    "source": [
273 |     "with open(\"vid.ts\", \"wb\") as video_file:\n",
274 |     "    while True:\n",
275 |     "        response = sqs.receive_message(\n",
276 |     "            QueueUrl=queue_url,\n",
277 |     "            AttributeNames=[\n",
278 |     "                \"SentTimestamp\"\n",
279 |     "            ],\n",
280 |     "            MaxNumberOfMessages=1,\n",
281 |     "            MessageAttributeNames=[\n",
282 |     "                \"All\"\n",
283 |     "            ],\n",
284 |     "            VisibilityTimeout=100,\n",
285 |     "            WaitTimeSeconds=0 # Short polling\n",
286 |     "        )\n",
287 |     "\n",
288 |     "        # Non-empty response\n",
289 |     "        if \"Messages\" in response:\n",
290 |     "            message = response[\"Messages\"][0]\n",
291 |     "            receipt_handle = message[\"ReceiptHandle\"]\n",
292 |     "\n",
293 |     "            sent_timestamp = message['Attributes']['SentTimestamp']\n",
294 |     "\n",
295 |     "            # Delete received message from queue\n",
296 |     "            sqs.delete_message(\n",
297 |     "                QueueUrl=queue_url,\n",
298 |     "                ReceiptHandle=receipt_handle\n",
299 |     "            )\n",
300 |     "\n",
301 |     "            # TODO: Replace time.time() to given value from Kafka\n",
302 |     "            # trigger_time = ...\n",
303 |     "            if time.time() - float(sent_timestamp) / 1000 > 30:\n",
304 |     "                pass\n",
305 |     "            else:\n",
306 |     "                body = base64.b64decode(message[\"Body\"].encode())\n",
307 |     "                video_file.write(body)\n",
308 |     "        else:\n",
309 |     "            # TODO: Check emptiness of SQS\n",
310 |     "            break"
311 |    ]
312 |   }
313 |  ],
314 |  "metadata": {
315 |   "kernelspec": {
316 |    "display_name": "Python 3 (ipykernel)",
317 |    "language": "python",
318 |    "name": "python3"
319 |   },
320 |   "language_info": {
321 |    "codemirror_mode": {
322 |     "name": "ipython",
323 |     "version": 3
324 |    },
325 |    "file_extension": ".py",
326 |    "mimetype": "text/x-python",
327 |    "name": "python",
328 |    "nbconvert_exporter": "python",
329 |    "pygments_lexer": "ipython3",
330 |    "version": "3.9.17"
331 |   }
332 |  },
333 |  "nbformat": 4,
334 |  "nbformat_minor": 5
335 | }
336 | 


--------------------------------------------------------------------------------
/influx/influxDB_range.py:
--------------------------------------------------------------------------------
  1 | import influxdb_client
  2 | import os
  3 | from dotenv import load_dotenv
  4 | import pandas as pd
  5 | from influxdb_client import Point, WritePrecision
  6 | from influxdb_client.client.write_api import SYNCHRONOUS  # Import SYNCHRONOUS
  7 | from openai import OpenAI
  8 | from datetime import datetime, timedelta
  9 | import json
 10 | import base64
 11 | import boto3
 12 | from botocore.exceptions import ClientError
 13 | import logging
 14 | import shlex
 15 | import subprocess
 16 | import csv
 17 | import sys
 18 | from confluent_kafka import Consumer
 19 | import time
 20 | 
 21 | 
 22 | # CSV의 필드 크기 제한을 해제한다
 23 | csv.field_size_limit(sys.maxsize)
 24 | 
 25 | # 환경 변수 불러오기
 26 | AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
 27 | AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
 28 | AWS_DEFAULT_REGION = "ap-northeast-2"
 29 | 
 30 | 
 31 | # ----------------------------------------------------------#
 32 | 
 33 | 
 34 | # Kafka consumer configuration 세팅
 35 | # earliest -> 해당 파티션의 가장 오래된 메시지부터 소비
 36 | # latest -> 해당 파티션의 최신 메시지부터 소비
 37 | conf = {
 38 |     'bootstrap.servers': '52.79.81.77:9092',
 39 |     'group.id': 'mygroup',
 40 |     'auto.offset.reset': 'latest'
 41 | }
 42 | 
 43 | consumer = Consumer(conf)
 44 | consumer.subscribe(['stream_filter_sink'])
 45 | 
 46 | 
 47 | 
 48 | 
 49 | 
 50 | # ----------------------------------------------------------#
 51 | 
 52 | 
 53 | def get_timeframe(df, date, target_time, bounds):
 54 |     """
 55 |     주어진 시간 주변 맥락에 맞는 대화를 분석해서 시작 시간, 종료 시간, 주요 주제, 설명을 반환한다.
 56 |     """
 57 |     # .env 파일으로부터 API 키를 가져온다
 58 |     load_dotenv()
 59 |     api_key = os.getenv('GPT_TOKEN')
 60 |     
 61 |     # --------------------------------------------------------#
 62 |     # 필요한 시간 범위를 설정한다
 63 |     print(target_time)
 64 |     # Convert target_time to a timedelta object (time elapsed since midnight)
 65 |     target_datetime = pd.to_datetime(target_time)
 66 |     target_time = target_datetime - target_datetime.normalize()
 67 |     start_time = target_time - timedelta(minutes=bounds)  # 5분 전
 68 |     end_time = target_time + timedelta(minutes=bounds)    # 5분 후
 69 |     print(start_time, end_time)
 70 | 
 71 |     # 필요한 시간 범위에 해당하는 행만 필터링한다
 72 |     filtered_df = df[(df['time'] >= start_time) & (df['time'] <= end_time)]
 73 | 
 74 |     # --------------------------------------------------------#
 75 |     # 타임스탬프가 있는 대화를 분석하기 위한 문자열을 생성한다
 76 |     dialogue_with_timestamps = "\n".join(
 77 |         f"{row['time']} - {row['text']}" for _, row in filtered_df.iterrows()
 78 |     )
 79 | 
 80 |     # 시간을 문자열로 변환한다
 81 |     target_time_str = str(target_time)
 82 |     # 모델에 제공할 프롬프트를 생성한다
 83 |     korean_prompt = (
 84 |         "다음과 같은 타임스탬프가 있는 대화를 제공되었을 때, 지정된 대상 시간 주변의 내용을 분석하십시오. "
 85 |         "대상 시간에서 다뤄진 주요 주제의 시작 및 종료 시간을 결정하고, 주요 주제를 알려주세요.\n\n"
 86 |         f"대상 시간: {target_time_str}\n\n"
 87 |         "대화:\n"
 88 |         f"{dialogue_with_timestamps}\n\n"
 89 |         f"다음 정보를 {date}T[HH:MM:SS]Z 형식의 시간과 함께 JSON 형식으로 제공해주세요:\n"
 90 |         "- start_time: 관련 시작 시간\n"
 91 |         "- end_time: 관련 종료 시간\n"
 92 |         "- main_topic: 주요 주제에 대한 간단한 설명. 이때, 주제 제목에 띄어쓰기 없이 해주세요.\n"
 93 |         "- explanation: 이러한 시간과 주제를 선택한 이유에 대한 간단한 설명\n\n"
 94 |         "JSON 형식으로 직접 응답해주세요. 추가적인 마크다운 형식이나 코드 블록 표시(```)를 사용하지 마세요."
 95 |     )
 96 | 
 97 |     # --------------------------------------------------------#
 98 |     # OpenAI API를 사용하기 위해 API 키를 설정하고 클라이언트를 생성한다
 99 |     client = OpenAI(api_key=api_key)
100 | 
101 |     # API 호출을 통해 대화를 생성한다
102 |     response = client.chat.completions.create(
103 |     model="gpt-4o-mini",
104 |     messages=[
105 |         {"role": "system", "content": "당신은 타임스탬프가 있는 대화를 분석하는 어시스턴트입니다."},
106 |         {"role": "user", "content": korean_prompt}
107 |     ]
108 |     )
109 | 
110 |     # 답변에 해당하는 내용을 출력한다
111 |     return response.choices[0].message.content
112 | 
113 | 
114 | 
115 | 
116 | # .env 파일으로부터 API 키를 가져온다
117 | load_dotenv()
118 | token = os.getenv('INFLUXDB_TOKEN')
119 | 
120 | # InfluxDB connection details
121 | org = "HighLighter"
122 | url = "http://13.125.176.29:8086"
123 | bucket = "HighLighter"
124 | 
125 | # InfluxDB client 인스턴스를 생성하고, 쿼리 API를 사용하기 위해 인스턴스를 생성한다
126 | client = influxdb_client.InfluxDBClient(url=url, token=token, org=org)
127 | query_api = client.query_api()
128 | 
129 | def get_dialogue():
130 | 
131 |     # "video_text_1"으로부터 "text" 필드만 가져오는 Flux 쿼리를 작성한다
132 |     start_time = "-3h"  # One day ago
133 |     query = f'''
134 |     from(bucket: "{bucket}")
135 |     |> range(start: {start_time})
136 |     |> filter(fn: (r) => r["_measurement"] == "video_text")
137 |     |> filter(fn: (r) => r["_field"] == "text")
138 |     |> keep(columns: ["_time", "_value"])
139 |     '''
140 |     tables = query_api.query(query)
141 | 
142 |     # pandas DataFrame로 변환한다
143 |     results = []
144 |     for table in tables:
145 |         for record in table.records:
146 |             results.append({"time": record.get_time(), "text": record.get_value()})
147 | 
148 |     df = pd.DataFrame(results)
149 | 
150 |     # 'time' 필드를 timedelta 형식으로 변환한다
151 |     df['time'] = pd.to_datetime(df['time'], utc=True)
152 |     df['time'] = pd.to_timedelta(df['time'].dt.strftime('%H:%M:%S'))
153 | 
154 |     return df
155 | 
156 | 
157 | 
158 | #----------------------------------------------------------#
159 | #----------------------------------------------------------#
160 | #---------- 영상의 00:00:00을 갖고오고, 글로벌 변수로 사용한다-------#
161 | 
162 | # 첫번째 타임스탬프 데이터를 가져오며, 00:00:00으로 취급한다
163 | # "video_text_1"으로부터 첫번째 타임스탬프를 가져오는 Flux 쿼리를 작성한다
164 | start_time = "-3h"  
165 | first_query = f'''
166 | from(bucket: "{bucket}")
167 |   |> range(start: {start_time})
168 |   |> filter(fn: (r) => r["_measurement"] == "video_binary")
169 |   |> filter(fn: (r) => r["_field"] == "value")
170 |   |> first()
171 |   |> keep(columns: ["_time"])
172 | '''
173 | 
174 | first_time = query_api.query(first_query)
175 | if first_time and len(first_time) > 0 and len(first_time[0].records) > 0:
176 |     time_value = first_time[0].records[0].values["_time"]
177 |     print(time_value)
178 | 
179 |     time_str = time_value.strftime('%H:%M:%S')
180 |     
181 |     first_time_delta = timedelta(hours=int(time_str[:2]), 
182 |                            minutes=int(time_str[3:5]), 
183 |                            seconds=int(time_str[6:]))
184 |     
185 |     print(first_time_delta)
186 | else:
187 |     print("No data found in the specified time range.")
188 | 
189 | # timeframe 함수를 사용하여 주어진 시간 주변의 맥락에 맞는 시간 범위를 가져온다
190 | highlight_time = first_time_delta + timedelta(minutes=15)
191 | date = time_value.date()
192 | 
193 | # Close the client
194 | client.close()
195 | 
196 | 
197 | # ----------------------------------------------------------#
198 | # ----------------------------------------------------------#
199 | # ----------------------------------------------------------#
200 | # 범위를 받았으니, 이제 이를 사용하여 비디오를 추출하고, 변환하고, 업로드한다.#
201 | # ----------------------------------------------------------#
202 | # ----------------------------------------------------------#
203 | # ----------------------------------------------------------#
204 | 
205 | def decode_byte_to_ts(df, title):
206 |   """
207 |   바이트를 .ts로 디코딩하여 파일로 저장한다.
208 |   """
209 |   if not df.empty:
210 |       first_entry_time = df['time'].iloc[0]
211 |       end_entry_time = first_entry_time + timedelta(minutes=30)
212 |       print(f"First entry time: {first_entry_time}, End time: {end_entry_time}")
213 |       
214 |       # 설정한 시간 범위 내의 데이터만 필터링한다
215 |       df_filtered = df[(df['time'] >= first_entry_time) & (df['time'] <= end_entry_time)]
216 |     #   print(df_filtered)
217 |       
218 |       print(f"Processing data from {first_entry_time} to {end_entry_time}")
219 |       
220 |       # Write the filtered data to a file
221 |       with open(title, "wb") as video_file:
222 |           for _, row in df_filtered.iterrows():
223 |               decoded_chunk = base64.b64decode(row['video'].encode())
224 |               video_file.write(decoded_chunk)
225 |         
226 |       print(f"Video file created at {title}")
227 |   else:
228 |       print("No data found in the specified time range.")
229 | 
230 | 
231 | 
232 | def ts_to_mp4(ts_path,mp4_path):
233 |   """
234 |   유튜브 업로드를 위해 from .ts to .mp4 변환
235 |   """
236 |   ffmpeg_cmd = f"ffmpeg -i {ts_path} -c copy {mp4_path}"
237 |   command1 = shlex.split(ffmpeg_cmd)
238 |   try:
239 |       response=subprocess.run(command1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 |   except OSError as e:
241 |       logging.error(e)
242 |       return False
243 |   print("변환 성공")
244 |   return True
245 | 
246 | 
247 | def upload_file(file_name, bucket, object_name=None):
248 |   """
249 |   AWS S3 엑세스 키를 가져온다.
250 |   """
251 | 
252 |   # boto3 라이브러리로 s3에 mp4 파일 업로드 
253 |   s3_client = boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
254 |   try:
255 |       response = s3_client.upload_file(file_name, bucket, object_name)
256 |   except ClientError as e:
257 |       logging.error(e)
258 |       return False
259 |   print("업로드 성공!")
260 |   return True
261 | 
262 | 
263 | def download_video(start, end, title):
264 |     """
265 |     InfluxDB에서 비디오를 다운로드한다.
266 |     """
267 |     
268 |     # InfluxDB client 인스턴스를 생성하고, 쿼리 API를 사용하기 위해 인스턴스를 생성한다
269 |     client = influxdb_client.InfluxDBClient(url=url, token=token, org=org)
270 |     query_api = client.query_api()
271 | 
272 |     
273 | 
274 |     query = f'''
275 |     from(bucket: "{bucket}")
276 |     |> range(start: {start}, stop: {end})
277 |     |> filter(fn: (r) => r["_measurement"] == "video_binary")
278 |     |> filter(fn: (r) => r["_field"] == "value")
279 |     |> keep(columns: ["_time", "_value"])
280 |     '''
281 | 
282 |     # 쿼리를 실행하고 결과를 가져온다
283 |     tables = query_api.query(query)
284 | 
285 |     # pandas DataFrame로 변환한다
286 |     results = []
287 |     for table in tables:
288 |         for record in table.records:
289 |             results.append({"time": record.get_time(), "video": record.get_value()})
290 | 
291 |     df = pd.DataFrame(results)
292 |     
293 |     # 실행코드: 바이트를 .ts로 디코딩하고 .ts를 .mp4로 변환하여 s3에 업로드
294 | 
295 |     mp4_path=f"{title}.mp4"
296 |     ts_path= f"{title}.ts"
297 | 
298 |     decode_byte_to_ts(df, ts_path)
299 |     ts_to_mp4(ts_path,mp4_path)
300 | 
301 | 
302 |     object_name=f"{title}.mp4"
303 |     bucket_name="ybigta-highlight"
304 |     upload_file(mp4_path, bucket_name, object_name)
305 | 
306 | 
307 | 
308 | 
309 | # ----------------------------------------------------------#
310 | # ----------------------------------------------------------#
311 | # ----------------------------------------------------------#
312 | # ----------------------------------------------------------#
313 | # MAIN CODE: Kafka Consumer
314 | 
315 | from confluent_kafka import Consumer
316 | from queue import Queue
317 | 
318 | timestamp_queue = Queue()
319 | 
320 | 
321 | def process_queue():
322 |     if timestamp_queue.empty():
323 |         return None, None, None
324 |     
325 |     current_timestamp = timestamp_queue.get()
326 | 
327 | 
328 |     df = get_dialogue()
329 |     gpt_response = get_timeframe(df, current_timestamp, current_timestamp, 5)
330 |     gpt_data = json.loads(gpt_response)
331 |     start, end, title = gpt_data['start_time'], gpt_data['end_time'], gpt_data['main_topic']
332 | 
333 |     # Check and remove outdated timestamps
334 |     temp_queue = Queue()
335 |     while not timestamp_queue.empty():
336 |         next_timestamp = timestamp_queue.get()
337 |         next_dt = datetime.strptime(next_timestamp, "%Y-%m-%d %H:%M:%S")
338 |         if next_dt > end:
339 |             temp_queue.put(next_timestamp)
340 |         else:
341 |             print(f"Removing outdated timestamp: {next_timestamp}")
342 | 
343 |     # Update the original queue
344 |     timestamp_queue.queue = temp_queue.queue
345 | 
346 | 
347 |     print(f"Processing: {current_timestamp}")
348 |     print(f"Start: {start}, End: {end}")
349 |     return start, end, title
350 | 
351 | 
352 | # Kafka consumer configuration 세팅
353 | # earliest -> 해당 파티션의 가장 오래된 메시지부터 소비
354 | # latest -> 해당 파티션의 최신 메시지부터 소비
355 | conf = {
356 |     'bootstrap.servers': '52.79.81.77:9092',
357 |     'group.id': 'mygroup',
358 |     'auto.offset.reset': 'latest'
359 | }
360 | 
361 | consumer = Consumer(conf)
362 | consumer.subscribe(['stream_filter_sink'])
363 | 
364 | 
365 | 
366 | 
367 | try:
368 |     while True:
369 |         msg = consumer.poll(1.0)  # 브로커로부터 메시지를 가져옴
370 |         
371 |         if msg is None:
372 |             continue
373 |         if msg.error():
374 |             print(f"Consumer error: {msg.error()}")
375 |             continue
376 |             
377 |         try:
378 |             timestamp = msg.value().decode('utf-8').strip()
379 |             timestamp_queue.put(timestamp)
380 |             print(f"Added timestamp: {timestamp}")
381 |             time.sleep(180)
382 |         
383 |             start, end, title = process_queue()
384 |             download_video(start, end, title)
385 | 
386 | 
387 |         except Exception as e:
388 |             print(f"Error processing message: {e}")
389 |             continue
390 | 
391 | except KeyboardInterrupt:
392 |     pass
393 | 
394 | finally:
395 |     consumer.close()
396 |     client.close()


--------------------------------------------------------------------------------