├── .gitignore ├── k8s ├── overlays │ └── production │ │ └── kustomization.yaml ├── base │ ├── github-actions │ │ ├── kustomization.yaml │ │ ├── serviceaccount.yaml │ │ ├── rolebinding.yaml │ │ └── role.yaml │ ├── kafka │ │ ├── kustomization.yaml │ │ ├── service.yaml │ │ └── statefulset.yaml │ ├── redis │ │ ├── kustomization.yaml │ │ ├── service.yaml │ │ └── statefulset.yaml │ ├── video │ │ ├── video-stream │ │ │ ├── kustomization.yaml │ │ │ └── deployment.yaml │ │ ├── video-api │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ └── deployment.yaml │ │ ├── kustomization.yaml │ │ └── video-gateway │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ └── deployment.yaml │ ├── mongodb │ │ ├── kustomization.yaml │ │ ├── service.yaml │ │ └── statefulset.yaml │ ├── postgres │ │ ├── kustomization.yaml │ │ ├── service.yaml │ │ └── statefulset.yaml │ ├── zookeeper │ │ ├── kustomization.yaml │ │ ├── service.yaml │ │ └── statefulset.yaml │ ├── comment │ │ ├── comment-migration │ │ │ ├── kustomization.yaml │ │ │ └── cronjob.yaml │ │ ├── comment-api │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ └── deployment.yaml │ │ ├── comment-gateway │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ └── deployment.yaml │ │ └── kustomization.yaml │ └── kustomization.yaml └── nfs-server-provisioner │ ├── persistentvolume.yaml │ └── values.yaml ├── modules ├── comment │ ├── migration │ │ ├── 20220223035207_create-comment-table.down.sql │ │ └── 20220223035207_create-comment-table.up.sql │ ├── mock │ │ ├── daomock │ │ │ ├── main.go │ │ │ └── mock.go │ │ └── pbmock │ │ │ ├── main.go │ │ │ └── mock.go │ ├── service │ │ ├── error.go │ │ └── service.go │ ├── pb │ │ ├── message.proto │ │ └── rpc.proto │ └── dao │ │ ├── comment.go │ │ ├── main_test.go │ │ ├── comment_redis.go │ │ ├── comment_pg.go │ │ └── comment_redis_test.go └── video │ ├── service │ ├── fixtures │ │ └── big_buck_bunny_240p_1mb.mp4 │ ├── error.go │ └── service.go │ ├── mock │ ├── daomock │ │ ├── main.go │ │ └── mock.go │ └── pbmock │ │ └── main.go │ ├── gateway │ ├── response.go │ └── handler.go │ ├── pb │ ├── stream.proto │ ├── rpc.proto │ ├── message.proto │ ├── stream.pb.sarama.go │ ├── stream_grpc.pb.go │ └── rpc.pb.go │ ├── dao │ ├── main_test.go │ ├── video_redis.go │ ├── video_mongo.go │ ├── video.go │ └── video_redis_test.go │ └── stream │ ├── stream.go │ └── stream_test.go ├── Dockerfile ├── pkg ├── kafkakit │ ├── mock │ │ └── kafkamock │ │ │ ├── main.go │ │ │ └── mock.go │ ├── consumer.go │ └── producer.go ├── storagekit │ ├── mock │ │ └── storagemock │ │ │ ├── main.go │ │ │ └── mock.go │ ├── storage.go │ └── minio.go ├── logkit │ ├── main_test.go │ ├── context.go │ ├── sarama_test.go │ ├── sarama.go │ ├── logger_test.go │ └── logger.go ├── pgkit │ ├── main_test.go │ ├── client_test.go │ └── client.go ├── otelkit │ ├── main_test.go │ ├── prometheus.go │ └── prometheus_test.go ├── mongokit │ ├── main_test.go │ ├── client_test.go │ └── client.go ├── rediskit │ ├── main_test.go │ ├── client_test.go │ └── client.go ├── migrationkit │ ├── main_test.go │ ├── migration_test.go │ └── migration.go ├── runkit │ └── graceful.go ├── pb │ └── google │ │ └── api │ │ ├── annotations.proto │ │ └── httpbody.proto └── grpckit │ └── client_conn.go ├── tools └── tools.go ├── cmd ├── video │ ├── main.go │ ├── stream.go │ ├── gateway.go │ └── api.go ├── comment │ ├── main.go │ ├── migration.go │ ├── gateway.go │ └── api.go └── main.go ├── .github ├── actions │ └── run-migration │ │ └── action.yml └── workflows │ ├── main.yml │ └── deployment.yml ├── prometheus.yml ├── .golangci.yml ├── README.md ├── docker-compose.yml ├── go.mod └── Makefile /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | bin/ 4 | .cache/ 5 | 6 | .idea/ 7 | -------------------------------------------------------------------------------- /k8s/overlays/production/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../../base 3 | 4 | namespace: default 5 | -------------------------------------------------------------------------------- /modules/comment/migration/20220223035207_create-comment-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS comments; 2 | -------------------------------------------------------------------------------- /k8s/base/github-actions/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - role.yaml 3 | - rolebinding.yaml 4 | - serviceaccount.yaml 5 | -------------------------------------------------------------------------------- /k8s/base/github-actions/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: github-actions 5 | -------------------------------------------------------------------------------- /k8s/base/kafka/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - statefulset.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: kafka 7 | -------------------------------------------------------------------------------- /k8s/base/redis/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - statefulset.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: redis 7 | -------------------------------------------------------------------------------- /k8s/base/video/video-stream/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - deployment.yaml 3 | 4 | commonLabels: 5 | app: video-stream 6 | -------------------------------------------------------------------------------- /k8s/base/mongodb/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - statefulset.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: mongodb 7 | -------------------------------------------------------------------------------- /k8s/base/postgres/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - statefulset.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: postgres 7 | -------------------------------------------------------------------------------- /k8s/base/zookeeper/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - statefulset.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: zookeeper 7 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-migration/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - cronjob.yaml 3 | 4 | commonLabels: 5 | app: comment-migration 6 | -------------------------------------------------------------------------------- /k8s/base/video/video-api/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - deployment.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: video-api 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gcr.io/distroless/base-debian12 AS base 2 | 3 | COPY bin/app/cmd /cmd 4 | COPY bin/app/static /static 5 | 6 | CMD ["/cmd"] 7 | -------------------------------------------------------------------------------- /k8s/base/video/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - video-api 3 | - video-gateway 4 | - video-stream 5 | 6 | commonLabels: 7 | module: video 8 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-api/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - deployment.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: comment-api 7 | -------------------------------------------------------------------------------- /k8s/base/video/video-gateway/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - deployment.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: video-gateway 7 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-gateway/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - deployment.yaml 3 | - service.yaml 4 | 5 | commonLabels: 6 | app: comment-gateway 7 | -------------------------------------------------------------------------------- /k8s/base/comment/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - comment-api 3 | - comment-gateway 4 | - comment-migration 5 | 6 | commonLabels: 7 | module: comment 8 | -------------------------------------------------------------------------------- /k8s/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - comment 3 | - kafka 4 | - mongodb 5 | - postgres 6 | - redis 7 | - video 8 | - zookeeper 9 | - github-actions 10 | -------------------------------------------------------------------------------- /modules/video/service/fixtures/big_buck_bunny_240p_1mb.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NTHU-LSALAB/NTHU-Distributed-System/HEAD/modules/video/service/fixtures/big_buck_bunny_240p_1mb.mp4 -------------------------------------------------------------------------------- /pkg/kafkakit/mock/kafkamock/main.go: -------------------------------------------------------------------------------- 1 | package kafkamock 2 | 3 | //go:generate mockgen -destination=mock.go -package=$GOPACKAGE github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit Producer 4 | -------------------------------------------------------------------------------- /modules/video/mock/daomock/main.go: -------------------------------------------------------------------------------- 1 | package daomock 2 | 3 | //go:generate mockgen -destination=mock.go -package=$GOPACKAGE github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao VideoDAO 4 | -------------------------------------------------------------------------------- /modules/comment/mock/daomock/main.go: -------------------------------------------------------------------------------- 1 | package daomock 2 | 3 | //go:generate mockgen -destination=mock.go -package=$GOPACKAGE github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/dao CommentDAO 4 | -------------------------------------------------------------------------------- /modules/comment/mock/pbmock/main.go: -------------------------------------------------------------------------------- 1 | package pbmock 2 | 3 | //go:generate mockgen -destination=mock.go -package=$GOPACKAGE github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb CommentClient 4 | -------------------------------------------------------------------------------- /pkg/storagekit/mock/storagemock/main.go: -------------------------------------------------------------------------------- 1 | package storagemock 2 | 3 | //go:generate mockgen -destination=mock.go -package=$GOPACKAGE github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/storagekit Storage 4 | -------------------------------------------------------------------------------- /k8s/base/kafka/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: kafka 5 | spec: 6 | type: ClusterIP 7 | ports: 8 | - name: kafka 9 | port: 9092 10 | targetPort: 9092 11 | -------------------------------------------------------------------------------- /k8s/base/redis/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redis 5 | spec: 6 | clusterIP: None 7 | ports: 8 | - name: redis 9 | port: 6379 10 | targetPort: 6379 11 | -------------------------------------------------------------------------------- /k8s/base/mongodb/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: mongodb 5 | spec: 6 | clusterIP: None 7 | ports: 8 | - name: mongodb 9 | port: 27017 10 | targetPort: 27017 11 | -------------------------------------------------------------------------------- /k8s/base/postgres/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: postgres 5 | spec: 6 | clusterIP: None 7 | ports: 8 | - name: postgres 9 | port: 5432 10 | targetPort: 5432 11 | -------------------------------------------------------------------------------- /k8s/base/zookeeper/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: zookeeper 5 | spec: 6 | clusterIP: None 7 | ports: 8 | - name: zookeeper 9 | port: 2181 10 | targetPort: 2181 11 | -------------------------------------------------------------------------------- /modules/video/mock/pbmock/main.go: -------------------------------------------------------------------------------- 1 | package pbmock 2 | 3 | //go:generate mockgen -destination=mock.go -package=$GOPACKAGE github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb Video_UploadVideoServer,VideoClient 4 | -------------------------------------------------------------------------------- /k8s/base/video/video-api/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: video-api 5 | spec: 6 | type: ClusterIP 7 | ports: 8 | - name: grpc 9 | port: 8081 10 | targetPort: 8081 11 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-api/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: comment-api 5 | spec: 6 | type: ClusterIP 7 | ports: 8 | - name: grpc 9 | port: 8081 10 | targetPort: 8081 11 | -------------------------------------------------------------------------------- /k8s/base/video/video-gateway/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: video-gateway 5 | spec: 6 | type: ClusterIP 7 | ports: 8 | - name: http 9 | port: 80 10 | targetPort: 8080 11 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-gateway/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: comment-gateway 5 | spec: 6 | type: ClusterIP 7 | ports: 8 | - name: gateway 9 | port: 80 10 | targetPort: 8080 11 | -------------------------------------------------------------------------------- /pkg/logkit/main_test.go: -------------------------------------------------------------------------------- 1 | package logkit 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestLogKit(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Test Log Kit") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/pgkit/main_test.go: -------------------------------------------------------------------------------- 1 | package pgkit 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestPGKit(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Test PG Kit") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/otelkit/main_test.go: -------------------------------------------------------------------------------- 1 | package otelkit 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestOtelKit(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Test Otel Kit") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/mongokit/main_test.go: -------------------------------------------------------------------------------- 1 | package mongokit 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestMongoKit(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Test Mongo Kit") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/rediskit/main_test.go: -------------------------------------------------------------------------------- 1 | package rediskit 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestRedisKit(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Test Redis Kit") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/migrationkit/main_test.go: -------------------------------------------------------------------------------- 1 | package migrationkit 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestMigrationKit(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Test Migration Kit") 13 | } 14 | -------------------------------------------------------------------------------- /k8s/base/github-actions/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | kind: RoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: github-actions 5 | subjects: 6 | - kind: ServiceAccount 7 | name: github-actions 8 | roleRef: 9 | kind: Role 10 | name: github-actions 11 | apiGroup: rbac.authorization.k8s.io 12 | -------------------------------------------------------------------------------- /modules/comment/service/error.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "google.golang.org/grpc/codes" 5 | "google.golang.org/grpc/status" 6 | ) 7 | 8 | var ( 9 | ErrInvalidUUID = status.Errorf(codes.InvalidArgument, "invalid UUID") 10 | ErrCommentNotFound = status.Errorf(codes.NotFound, "comment not found") 11 | ) 12 | -------------------------------------------------------------------------------- /modules/video/service/error.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "google.golang.org/grpc/codes" 5 | "google.golang.org/grpc/status" 6 | ) 7 | 8 | var ( 9 | ErrInvalidObjectID = status.Errorf(codes.InvalidArgument, "invalid objectID") 10 | ErrVideoNotFound = status.Errorf(codes.NotFound, "video not found") 11 | ) 12 | -------------------------------------------------------------------------------- /k8s/base/github-actions/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: github-actions 5 | rules: 6 | - apiGroups: 7 | - apps 8 | - batch 9 | resources: 10 | - "*" 11 | verbs: 12 | - "get" 13 | - "list" 14 | - "watch" 15 | - "create" 16 | - "update" 17 | - "patch" 18 | -------------------------------------------------------------------------------- /modules/comment/migration/20220223035207_create-comment-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS comments ( 2 | id uuid PRIMARY KEY DEFAULT gen_random_uuid(), 3 | video_id TEXT NOT NULL, 4 | content TEXT NOT NULL, 5 | created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 6 | updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP 7 | ); 8 | -------------------------------------------------------------------------------- /tools/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | 3 | package tools 4 | 5 | import ( 6 | _ "github.com/golang/mock/mockgen" 7 | _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway" 8 | _ "github.com/justin0u0/protoc-gen-grpc-sarama" 9 | _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" 10 | _ "google.golang.org/protobuf/cmd/protoc-gen-go" 11 | ) 12 | -------------------------------------------------------------------------------- /cmd/video/main.go: -------------------------------------------------------------------------------- 1 | package video 2 | 3 | import "github.com/spf13/cobra" 4 | 5 | func NewVideoCommand() *cobra.Command { 6 | cmd := &cobra.Command{ 7 | Use: "video [service]", 8 | Short: "start video's service", 9 | } 10 | 11 | cmd.AddCommand(newAPICommand()) 12 | cmd.AddCommand(newGatewayCommand()) 13 | cmd.AddCommand(newStreamCommand()) 14 | 15 | return cmd 16 | } 17 | -------------------------------------------------------------------------------- /cmd/comment/main.go: -------------------------------------------------------------------------------- 1 | package comment 2 | 3 | import "github.com/spf13/cobra" 4 | 5 | func NewCommentCommand() *cobra.Command { 6 | cmd := &cobra.Command{ 7 | Use: "comment [service]", 8 | Short: "start comment's service", 9 | } 10 | 11 | cmd.AddCommand(newAPICommand()) 12 | cmd.AddCommand(newGatewayCommand()) 13 | cmd.AddCommand(newMigrationCommand()) 14 | 15 | return cmd 16 | } 17 | -------------------------------------------------------------------------------- /modules/video/gateway/response.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | type StatusCoder interface { 4 | StatusCode() int 5 | } 6 | 7 | type responseError struct { 8 | Message string `json:"message"` 9 | Err error `json:"error"` 10 | statusCode int `json:"-"` 11 | } 12 | 13 | func NewResponseError(statusCode int, message string, err error) *responseError { 14 | return &responseError{ 15 | Message: message, 16 | Err: err, 17 | statusCode: statusCode, 18 | } 19 | } 20 | 21 | func (re *responseError) StatusCode() int { 22 | return re.statusCode 23 | } 24 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/cmd/comment" 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/cmd/video" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | func main() { 12 | cmd := &cobra.Command{ 13 | Use: "nthu-distributed-system [module]", 14 | Short: "NTHU Distributed System module entrypoints", 15 | } 16 | 17 | cmd.AddCommand(video.NewVideoCommand()) 18 | cmd.AddCommand(comment.NewCommentCommand()) 19 | 20 | if err := cmd.Execute(); err != nil { 21 | log.Fatal(err) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /modules/video/pb/stream.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package video.pb; 4 | 5 | option go_package = "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb"; 6 | 7 | import "google/protobuf/empty.proto"; 8 | import "proto/sarama.proto"; 9 | 10 | service VideoStream { 11 | option (sarama.enabled) = true; 12 | option (sarama.logger_enabled) = true; 13 | 14 | rpc HandleVideoCreated(HandleVideoCreatedRequest) returns (google.protobuf.Empty) {} 15 | } 16 | 17 | message HandleVideoCreatedRequest { 18 | string id = 1; 19 | string url = 2; 20 | int32 scale = 3; 21 | } 22 | -------------------------------------------------------------------------------- /k8s/nfs-server-provisioner/persistentvolume.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolume 3 | metadata: 4 | name: nfs-server-provisioner-pv 5 | spec: 6 | capacity: 7 | storage: 50Gi 8 | volumeMode: Filesystem 9 | accessModes: 10 | - ReadWriteOnce 11 | persistentVolumeReclaimPolicy: Retain 12 | storageClassName: local-storage 13 | local: 14 | path: /mnt/nfs 15 | nodeAffinity: 16 | required: 17 | nodeSelectorTerms: 18 | - matchExpressions: 19 | - key: kubernetes.io/hostname 20 | operator: In 21 | values: 22 | - k8s-master 23 | -------------------------------------------------------------------------------- /pkg/storagekit/storage.go: -------------------------------------------------------------------------------- 1 | package storagekit 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | type PutObjectOptions struct { 9 | ContentType string 10 | } 11 | 12 | // Provide a simplifier interface to upload file 13 | type Storage interface { 14 | // Endpoint returns the endpoint of the object storage 15 | Endpoint() string 16 | // Bucket returns the bucket name in the object storage 17 | Bucket() string 18 | 19 | // PutObject add an object into the storage bucket 20 | PutObject(ctx context.Context, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) error 21 | } 22 | -------------------------------------------------------------------------------- /pkg/logkit/context.go: -------------------------------------------------------------------------------- 1 | package logkit 2 | 3 | import ( 4 | "context" 5 | "log" 6 | ) 7 | 8 | // Inject logger into context to easily carry the logger through everywhere 9 | 10 | type loggerConextKey int8 11 | 12 | const contextKeyLogger loggerConextKey = iota 13 | 14 | func WithContext(ctx context.Context, logger *Logger) context.Context { 15 | return context.WithValue(ctx, contextKeyLogger, logger) 16 | } 17 | 18 | func FromContext(ctx context.Context) *Logger { 19 | logger, ok := ctx.Value(contextKeyLogger).(*Logger) 20 | if !ok { 21 | log.Fatal("logger is not found in context") 22 | } 23 | 24 | return logger 25 | } 26 | -------------------------------------------------------------------------------- /pkg/logkit/sarama_test.go: -------------------------------------------------------------------------------- 1 | package logkit 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | ) 7 | 8 | var _ = Describe("SaramaLogger", func() { 9 | Describe("NewSaramaLogger", func() { 10 | var logger *Logger 11 | var saramaLogger *SaramaLogger 12 | 13 | BeforeEach(func() { 14 | logger = NewLogger(&LoggerConfig{Development: true}) 15 | }) 16 | 17 | JustBeforeEach(func() { 18 | saramaLogger = NewSaramaLogger(logger) 19 | }) 20 | 21 | When("success", func() { 22 | It("returns new SaramaLogger without error", func() { 23 | Expect(saramaLogger).NotTo(BeNil()) 24 | }) 25 | }) 26 | }) 27 | }) 28 | -------------------------------------------------------------------------------- /pkg/logkit/sarama.go: -------------------------------------------------------------------------------- 1 | package logkit 2 | 3 | import ( 4 | "github.com/justin0u0/protoc-gen-grpc-sarama/pkg/saramakit" 5 | "go.uber.org/zap" 6 | ) 7 | 8 | // SaramaLogger implements saramakit.Logger 9 | type SaramaLogger struct { 10 | *Logger 11 | } 12 | 13 | var _ saramakit.Logger = (*SaramaLogger)(nil) 14 | 15 | func (l *SaramaLogger) With(key, value string) saramakit.Logger { 16 | return &SaramaLogger{ 17 | Logger: l.Logger.With(zap.String(key, value)), 18 | } 19 | } 20 | 21 | func (l *SaramaLogger) Error(msg string, err error) { 22 | l.Logger.Error(msg, zap.Error(err)) 23 | } 24 | 25 | func NewSaramaLogger(logger *Logger) *SaramaLogger { 26 | return &SaramaLogger{ 27 | Logger: logger, 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /k8s/base/video/video-gateway/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: video-gateway 5 | spec: 6 | replicas: 1 7 | template: 8 | spec: 9 | containers: 10 | - name: video-gateway 11 | image: ghcr.io/nthu-lsalab/nthu-distributed-system:latest 12 | imagePullPolicy: Always 13 | ports: 14 | - name: http 15 | containerPort: 8080 16 | command: 17 | - /cmd 18 | - video 19 | - gateway 20 | env: 21 | - name: GRPC_SERVER_ADDR 22 | value: video-api:8081 23 | resources: 24 | requests: 25 | memory: 30Mi 26 | cpu: 10m 27 | limits: 28 | memory: 60Mi 29 | cpu: 20m 30 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-gateway/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: comment-gateway 5 | spec: 6 | replicas: 2 7 | template: 8 | spec: 9 | containers: 10 | - name: comment-gateway 11 | image: ghcr.io/nthu-lsalab/nthu-distributed-system:latest 12 | imagePullPolicy: Always 13 | ports: 14 | - name: http 15 | containerPort: 8080 16 | command: 17 | - /cmd 18 | - comment 19 | - gateway 20 | env: 21 | - name: GRPC_SERVER_ADDR 22 | value: comment-api:8081 23 | resources: 24 | requests: 25 | memory: 30Mi 26 | cpu: 10m 27 | limits: 28 | memory: 60Mi 29 | cpu: 20m 30 | -------------------------------------------------------------------------------- /k8s/base/redis/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: redis 5 | spec: 6 | serviceName: redis 7 | replicas: 1 8 | template: 9 | spec: 10 | containers: 11 | - name: redis 12 | image: redis:6.2-alpine 13 | ports: 14 | - name: redis 15 | containerPort: 6379 16 | resources: 17 | requests: 18 | cpu: 100m 19 | memory: 200Mi 20 | limits: 21 | cpu: 200m 22 | memory: 400Mi 23 | volumeMounts: 24 | - name: redis-persistent-storage-claim 25 | mountPath: /data 26 | volumeClaimTemplates: 27 | - metadata: 28 | name: redis-persistent-storage-claim 29 | spec: 30 | storageClassName: nfs 31 | accessModes: 32 | - ReadWriteOnce 33 | resources: 34 | requests: 35 | storage: 500Mi 36 | -------------------------------------------------------------------------------- /k8s/base/mongodb/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: mongodb 5 | spec: 6 | serviceName: mongodb 7 | replicas: 1 8 | template: 9 | spec: 10 | containers: 11 | - name: mongodb 12 | image: mongo:5 13 | ports: 14 | - name: mongodb 15 | containerPort: 27017 16 | resources: 17 | requests: 18 | cpu: 100m 19 | memory: 200Mi 20 | limits: 21 | cpu: 200m 22 | memory: 400Mi 23 | volumeMounts: 24 | - name: mongodb-persistent-storage-claim 25 | mountPath: /data/db 26 | volumeClaimTemplates: 27 | - metadata: 28 | name: mongodb-persistent-storage-claim 29 | spec: 30 | storageClassName: nfs 31 | accessModes: 32 | - ReadWriteOnce 33 | resources: 34 | requests: 35 | storage: 500Mi 36 | -------------------------------------------------------------------------------- /.github/actions/run-migration/action.yml: -------------------------------------------------------------------------------- 1 | name: Run Migration Job 2 | description: Create a migration job from an existing CronJob, wait for the job to finish. 3 | author: Justin Chen 4 | inputs: 5 | migration-cronjob-name: 6 | description: The migration cronjob name in the Kubernetes cluster. 7 | required: true 8 | migration-job-name: 9 | description: The manually run migration job name. 10 | required: true 11 | migration-job-timeout: 12 | description: The timeout duration of the migration job. 13 | required: true 14 | default: 3m 15 | runs: 16 | using: composite 17 | steps: 18 | - name: run migration job 19 | shell: bash 20 | run: | 21 | kubectl create job --from cronjob/${{ inputs.migration-cronjob-name }} ${{ inputs.migration-job-name }} 22 | kubectl wait --for=condition=complete --timeout ${{ inputs.migration-job-timeout }} job/${{ inputs.migration-job-name }} 23 | -------------------------------------------------------------------------------- /pkg/logkit/logger_test.go: -------------------------------------------------------------------------------- 1 | package logkit 2 | 3 | import ( 4 | "context" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var _ = Describe("Logger", func() { 11 | Describe("NewLogger", func() { 12 | var logger *Logger 13 | 14 | JustBeforeEach(func() { 15 | logger = NewLogger(&LoggerConfig{Development: true}) 16 | }) 17 | 18 | When("success", func() { 19 | It("returns new logger without error", func() { 20 | Expect(logger).NotTo(BeNil()) 21 | }) 22 | }) 23 | }) 24 | 25 | Describe("WithContext", func() { 26 | var logger *Logger 27 | var ctx context.Context 28 | 29 | JustBeforeEach(func() { 30 | logger = NewLogger(&LoggerConfig{Development: true}) 31 | ctx = logger.WithContext(context.Background()) 32 | }) 33 | 34 | When("success", func() { 35 | It("inserts logger into context", func() { 36 | Expect(FromContext(ctx)).To(Equal(logger)) 37 | }) 38 | }) 39 | }) 40 | }) 41 | -------------------------------------------------------------------------------- /prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 3 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. 4 | # scrape_timeout is set to the global default (10s). 5 | 6 | # Alertmanager configuration 7 | alerting: 8 | alertmanagers: 9 | - static_configs: 10 | - targets: 11 | - 'localhost:9093' 12 | 13 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 14 | rule_files: 15 | # - "first_rules.yml" 16 | # - "second_rules.yml" 17 | 18 | # A scrape configuration containing exactly one endpoint to scrape: 19 | scrape_configs: 20 | - job_name: 'prometheus' 21 | static_configs: 22 | - targets: 23 | - 'localhost:9090' 24 | 25 | - job_name: video 26 | static_configs: 27 | - targets: 28 | - 'video-api:2222' 29 | 30 | - job_name: comment 31 | static_configs: 32 | - targets: 33 | - 'comment-api:2222' 34 | -------------------------------------------------------------------------------- /k8s/base/postgres/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: postgres 5 | spec: 6 | serviceName: postgres 7 | replicas: 1 8 | template: 9 | spec: 10 | containers: 11 | - name: postgres 12 | image: postgres:14-alpine 13 | ports: 14 | - name: postgres 15 | containerPort: 5432 16 | resources: 17 | requests: 18 | cpu: 100m 19 | memory: 100Mi 20 | limits: 21 | cpu: 200m 22 | memory: 200Mi 23 | env: 24 | - name: POSTGRES_HOST_AUTH_METHOD 25 | value: trust 26 | volumeMounts: 27 | - name: postgres-persistent-storage-claim 28 | mountPath: /var/lib/postgresql/data 29 | volumeClaimTemplates: 30 | - metadata: 31 | name: postgres-persistent-storage-claim 32 | spec: 33 | storageClassName: nfs 34 | accessModes: 35 | - ReadWriteOnce 36 | resources: 37 | requests: 38 | storage: 500Mi 39 | -------------------------------------------------------------------------------- /modules/video/pb/rpc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package video.pb; 4 | 5 | import "google/api/annotations.proto"; 6 | import "modules/video/pb/message.proto"; 7 | 8 | option go_package = "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb"; 9 | 10 | service Video { 11 | rpc Healthz(HealthzRequest) returns (HealthzResponse) { 12 | option (google.api.http) = { 13 | get: "/" 14 | }; 15 | } 16 | 17 | rpc GetVideo(GetVideoRequest) returns (GetVideoResponse) { 18 | option (google.api.http) = { 19 | get: "/v1/videos/{id}" 20 | response_body: "video" 21 | }; 22 | } 23 | 24 | rpc ListVideo(ListVideoRequest) returns (ListVideoResponse) { 25 | option (google.api.http) = { 26 | get: "/v1/videos" 27 | response_body: "*" 28 | }; 29 | } 30 | 31 | rpc UploadVideo(stream UploadVideoRequest) returns (UploadVideoResponse) {} 32 | 33 | rpc DeleteVideo(DeleteVideoRequest) returns (DeleteVideoResponse) { 34 | option (google.api.http) = { 35 | delete: "/v1/videos/{id}" 36 | response_body: "*" 37 | }; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-migration/cronjob.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: CronJob 3 | metadata: 4 | name: comment-migration 5 | spec: 6 | schedule: 0 0 * * * 7 | concurrencyPolicy: Forbid 8 | suspend: true 9 | jobTemplate: 10 | spec: 11 | template: 12 | spec: 13 | restartPolicy: Never 14 | containers: 15 | - name: comment-migration 16 | image: ghcr.io/nthu-lsalab/nthu-distributed-system:latest 17 | imagePullPolicy: Always 18 | command: 19 | - /cmd 20 | - comment 21 | - migration 22 | env: 23 | - name: MIGRATION_SOURCE 24 | value: file:///static/modules/comment/migration 25 | - name: MIGRATION_URL 26 | value: postgres://postgres@postgres:5432/postgres?sslmode=disable 27 | resources: 28 | requests: 29 | memory: 30Mi 30 | cpu: 10m 31 | limits: 32 | memory: 60Mi 33 | cpu: 20m 34 | -------------------------------------------------------------------------------- /pkg/rediskit/client_test.go: -------------------------------------------------------------------------------- 1 | package rediskit 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | var _ = Describe("RedisClient", func() { 13 | Describe("NewRedisClient", func() { 14 | var ( 15 | ctx context.Context 16 | redisClient *RedisClient 17 | redisConf RedisConfig 18 | ) 19 | 20 | BeforeEach(func() { 21 | ctx = logkit.WithContext(context.Background(), logkit.NewNopLogger()) 22 | 23 | redisConf.Addr = "localhost:6379" 24 | if addr := os.Getenv("REDIS_ADDR"); addr != "" { 25 | redisConf.Addr = addr 26 | } 27 | }) 28 | 29 | AfterEach(func() { 30 | Expect(redisClient.Close()).NotTo(HaveOccurred()) 31 | }) 32 | 33 | JustBeforeEach(func() { 34 | redisClient = NewRedisClient(ctx, &redisConf) 35 | }) 36 | 37 | When("success", func() { 38 | It("returns redis client without error", func() { 39 | Expect(redisClient).NotTo(BeNil()) 40 | }) 41 | }) 42 | }) 43 | }) 44 | -------------------------------------------------------------------------------- /k8s/base/video/video-stream/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: video-stream 5 | spec: 6 | replicas: 2 7 | template: 8 | spec: 9 | containers: 10 | - name: video-stream 11 | image: ghcr.io/nthu-lsalab/nthu-distributed-system:latest 12 | imagePullPolicy: Always 13 | command: 14 | - /cmd 15 | - video 16 | - stream 17 | env: 18 | - name: KAFKA_CONSUMER_ADDRS 19 | value: kafka:9092 20 | - name: KAFKA_CONSUMER_GROUP 21 | value: video-stream 22 | - name: KAFKA_CONSUMER_TOPIC 23 | value: video 24 | - name: KAFKA_PRODUCER_ADDRS 25 | value: kafka:9092 26 | - name: KAFKA_PRODUCER_TOPIC 27 | value: video 28 | - name: MONGO_DATABASE 29 | value: nthu_distributed_system 30 | - name: MONGO_URL 31 | value: mongodb://mongodb:27017/ 32 | resources: 33 | requests: 34 | memory: 30Mi 35 | cpu: 10m 36 | limits: 37 | memory: 60Mi 38 | cpu: 20m 39 | -------------------------------------------------------------------------------- /pkg/pgkit/client_test.go: -------------------------------------------------------------------------------- 1 | package pgkit 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | var _ = Describe("PGClient", func() { 13 | Describe("NewPGClient", func() { 14 | var ( 15 | ctx context.Context 16 | pgConf *PGConfig 17 | pgClient *PGClient 18 | ) 19 | 20 | BeforeEach(func() { 21 | ctx = logkit.NewLogger(&logkit.LoggerConfig{ 22 | Development: true, 23 | }).WithContext(context.Background()) 24 | 25 | pgConf = &PGConfig{ 26 | URL: "postgres://postgres@postgres:5432/postgres?sslmode=disable", 27 | } 28 | if url := os.Getenv("POSTGRES_URL"); url != "" { 29 | pgConf.URL = url 30 | } 31 | }) 32 | 33 | JustBeforeEach(func() { 34 | pgClient = NewPGClient(ctx, pgConf) 35 | }) 36 | 37 | AfterEach(func() { 38 | Expect(pgClient.Close()).NotTo(HaveOccurred()) 39 | }) 40 | 41 | When("success", func() { 42 | It("returns new PGClient without error", func() { 43 | Expect(pgClient).NotTo(BeNil()) 44 | }) 45 | }) 46 | }) 47 | }) 48 | -------------------------------------------------------------------------------- /pkg/runkit/graceful.go: -------------------------------------------------------------------------------- 1 | package runkit 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | "time" 10 | ) 11 | 12 | type GracefulConfig struct { 13 | Timeout time.Duration `long:"timeout" description:"gracefully timeout duration" env:"TIMEOUT" default:"10s"` 14 | } 15 | 16 | var ( 17 | ErrGracefullyTimeout = errors.New("gracefully shutdown timeout") 18 | ) 19 | 20 | type GracefulRunFunc func(context.Context) error 21 | 22 | func GracefulRun(fn GracefulRunFunc, conf *GracefulConfig) error { 23 | ctx, cancel := context.WithCancel(context.Background()) 24 | defer cancel() 25 | 26 | done := make(chan error, 1) 27 | go func() { 28 | done <- fn(ctx) 29 | }() 30 | 31 | shutdownCh := make(chan os.Signal, 1) 32 | signal.Notify(shutdownCh, syscall.SIGINT, syscall.SIGTERM) 33 | 34 | select { 35 | case err := <-done: 36 | return err 37 | case <-shutdownCh: 38 | // receive termination signal, cancel manually 39 | cancel() 40 | 41 | select { 42 | case err := <-done: 43 | // gracefully shutdown 44 | return err 45 | case <-time.After(conf.Timeout): 46 | // timeout shutdown 47 | return ErrGracefullyTimeout 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /pkg/pb/google/api/annotations.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/api/http.proto"; 20 | import "google/protobuf/descriptor.proto"; 21 | 22 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "AnnotationsProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | extend google.protobuf.MethodOptions { 29 | // See `HttpRule`. 30 | HttpRule http = 72295728; 31 | } 32 | -------------------------------------------------------------------------------- /pkg/pgkit/client.go: -------------------------------------------------------------------------------- 1 | package pgkit 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | "github.com/go-pg/pg/v10" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | type PGConfig struct { 13 | URL string `long:"url" env:"URL" description:"the URL of PostgreSQL" required:"true"` 14 | } 15 | 16 | type PGClient struct { 17 | *pg.DB 18 | closeFunc func() 19 | } 20 | 21 | func (c *PGClient) Close() error { 22 | if c.closeFunc != nil { 23 | c.closeFunc() 24 | } 25 | return c.DB.Close() 26 | } 27 | 28 | func NewPGClient(ctx context.Context, conf *PGConfig) *PGClient { 29 | if url := os.ExpandEnv(conf.URL); url != "" { 30 | conf.URL = url 31 | } 32 | 33 | logger := logkit.FromContext(ctx).With(zap.String("url", conf.URL)) 34 | opts, err := pg.ParseURL(conf.URL) 35 | if err != nil { 36 | logger.Fatal("failed to parse PostgreSQL url", zap.Error(err)) 37 | } 38 | 39 | db := pg.Connect(opts).WithContext(ctx) 40 | if err := db.Ping(ctx); err != nil { 41 | logger.Fatal("failed to ping PostgreSQL", zap.Error(err)) 42 | } 43 | 44 | logger.Info("create PostgreSQL client successfully") 45 | 46 | return &PGClient{ 47 | DB: db, 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/rediskit/client.go: -------------------------------------------------------------------------------- 1 | package rediskit 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 7 | "github.com/go-redis/redis/v8" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type RedisConfig struct { 12 | Addr string `long:"addr" env:"ADDR" description:"the address of Redis" required:"true"` 13 | Password string `long:"password" env:"PASSWORD" description:"the password of Redis"` 14 | Database int `long:"database" env:"DATABASE" description:"the database of Redis"` 15 | } 16 | 17 | type RedisClient struct { 18 | *redis.Client 19 | closeFunc func() 20 | } 21 | 22 | func (c *RedisClient) Close() error { 23 | if c.closeFunc != nil { 24 | c.closeFunc() 25 | } 26 | 27 | return c.Client.Close() 28 | } 29 | 30 | func NewRedisClient(ctx context.Context, conf *RedisConfig) *RedisClient { 31 | logger := logkit.FromContext(ctx).With( 32 | zap.String("addr", conf.Addr), 33 | zap.Int("database", conf.Database), 34 | ) 35 | 36 | client := redis.NewClient(&redis.Options{ 37 | Addr: conf.Addr, 38 | Password: conf.Password, 39 | DB: conf.Database, 40 | }) 41 | 42 | if err := client.Ping(ctx).Err(); err != nil { 43 | logger.Fatal("failed to ping to Redis", zap.Error(err)) 44 | } 45 | 46 | return &RedisClient{ 47 | Client: client, 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /modules/comment/pb/message.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package comment.pb; 4 | 5 | option go_package = "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb"; 6 | 7 | import "google/protobuf/timestamp.proto"; 8 | 9 | message HealthzRequest {} 10 | 11 | message HealthzResponse { 12 | string status = 1; 13 | } 14 | 15 | message CommentInfo { 16 | string id = 1; 17 | string video_id = 2; 18 | string content = 3; 19 | google.protobuf.Timestamp created_at = 4; 20 | google.protobuf.Timestamp updated_at = 5; 21 | } 22 | 23 | message CreateCommentRequest { 24 | string video_id = 1; 25 | string content = 2; 26 | } 27 | 28 | message CreateCommentResponse { 29 | string id = 1; 30 | } 31 | 32 | message ListCommentRequest { 33 | string video_id = 1; 34 | int32 limit = 2; 35 | int32 offset = 3; 36 | } 37 | 38 | message ListCommentResponse { 39 | repeated CommentInfo comments = 1; 40 | } 41 | 42 | message UpdateCommentRequest { 43 | string id = 1; 44 | string content = 2; 45 | } 46 | 47 | message UpdateCommentResponse { 48 | CommentInfo comment = 1; 49 | } 50 | 51 | message DeleteCommentRequest { 52 | string id = 1; 53 | } 54 | 55 | message DeleteCommentResponse {} 56 | 57 | message DeleteCommentByVideoIDRequest { 58 | string video_id = 1; 59 | } 60 | 61 | message DeleteCommentByVideoIDResponse {} 62 | 63 | -------------------------------------------------------------------------------- /pkg/mongokit/client_test.go: -------------------------------------------------------------------------------- 1 | package mongokit 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | var _ = Describe("MongoClient", func() { 13 | Describe("NewMongoClient", func() { 14 | var ( 15 | mongoClient *MongoClient 16 | ctx context.Context 17 | mongoConfig *MongoConfig 18 | ) 19 | 20 | BeforeEach(func() { 21 | ctx = logkit.NewLogger(&logkit.LoggerConfig{ 22 | Development: true, 23 | }).WithContext(context.Background()) 24 | 25 | mongoConfig = &MongoConfig{ 26 | URL: "mongodb://mongo:27017", 27 | Database: "nthu_distributed_system", 28 | } 29 | 30 | if url := os.Getenv("MONGO_URL"); url != "" { 31 | mongoConfig.URL = url 32 | } 33 | 34 | if database := os.Getenv("MONGO_DATABASE"); database != "" { 35 | mongoConfig.Database = database 36 | } 37 | }) 38 | 39 | JustBeforeEach(func() { 40 | mongoClient = NewMongoClient(ctx, mongoConfig) 41 | }) 42 | 43 | AfterEach(func() { 44 | Expect(mongoClient.Close()).NotTo(HaveOccurred()) 45 | }) 46 | 47 | When("success", func() { 48 | It("returns new MongoClient without error", func() { 49 | Expect(mongoClient).NotTo(BeNil()) 50 | }) 51 | }) 52 | }) 53 | }) 54 | -------------------------------------------------------------------------------- /modules/video/pb/message.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package video.pb; 4 | 5 | option go_package = "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb"; 6 | 7 | import "google/protobuf/timestamp.proto"; 8 | 9 | message HealthzRequest {} 10 | 11 | message HealthzResponse { 12 | string status = 1; 13 | } 14 | 15 | message VideoInfo { 16 | string id = 1; 17 | uint32 width = 2; 18 | uint32 height = 3; 19 | uint64 size = 4; 20 | double duration = 5; 21 | string url = 6; 22 | string status = 7; 23 | map variants = 8; 24 | google.protobuf.Timestamp created_at = 9; 25 | google.protobuf.Timestamp updated_at = 10; 26 | } 27 | 28 | message VideoHeader { 29 | string filename = 1; 30 | uint64 size = 2; 31 | } 32 | 33 | message GetVideoRequest { 34 | string id = 1; 35 | } 36 | 37 | message GetVideoResponse { 38 | VideoInfo video = 1; 39 | } 40 | 41 | message ListVideoRequest { 42 | int64 limit = 1; 43 | int64 skip = 2; 44 | } 45 | 46 | message ListVideoResponse { 47 | repeated VideoInfo videos = 1; 48 | } 49 | 50 | message UploadVideoRequest { 51 | oneof data { 52 | VideoHeader header = 1; 53 | bytes chunk_data = 2; 54 | }; 55 | } 56 | 57 | message UploadVideoResponse { 58 | string id = 1; 59 | } 60 | 61 | message DeleteVideoRequest { 62 | string id = 1; 63 | } 64 | 65 | message DeleteVideoResponse {} 66 | -------------------------------------------------------------------------------- /modules/comment/pb/rpc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package comment.pb; 4 | 5 | import "google/api/annotations.proto"; 6 | import "modules/comment/pb/message.proto"; 7 | 8 | option go_package = "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb"; 9 | 10 | service Comment { 11 | rpc Healthz(HealthzRequest) returns (HealthzResponse) { 12 | option (google.api.http) = { 13 | get: "/" 14 | }; 15 | } 16 | 17 | rpc ListComment(ListCommentRequest) returns (ListCommentResponse) { 18 | option (google.api.http) = { 19 | get: "/v1/comments/{video_id}" 20 | response_body: "*" 21 | }; 22 | } 23 | 24 | rpc CreateComment(CreateCommentRequest) returns (CreateCommentResponse) { 25 | option (google.api.http) = { 26 | post: "/v1/comments" 27 | body: "*" 28 | response_body: "*" 29 | }; 30 | } 31 | 32 | rpc UpdateComment(UpdateCommentRequest) returns (UpdateCommentResponse) { 33 | option (google.api.http) = { 34 | put: "/v1/comments/{id}" 35 | body: "*" 36 | response_body: "comment" 37 | }; 38 | } 39 | 40 | rpc DeleteComment(DeleteCommentRequest) returns (DeleteCommentResponse) { 41 | option (google.api.http) = { 42 | delete: "/v1/comments/{id}" 43 | response_body: "*" 44 | }; 45 | } 46 | 47 | rpc DeleteCommentByVideoID(DeleteCommentByVideoIDRequest) returns (DeleteCommentByVideoIDResponse) {} 48 | } 49 | -------------------------------------------------------------------------------- /pkg/migrationkit/migration_test.go: -------------------------------------------------------------------------------- 1 | package migrationkit 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/pgkit" 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | var _ = Describe("Migration", func() { 14 | Describe("NewMigration", func() { 15 | var ( 16 | ctx context.Context 17 | migration *Migration 18 | migrationConf *MigrationConfig 19 | ) 20 | 21 | BeforeEach(func() { 22 | pgConf := &pgkit.PGConfig{ 23 | URL: "postgres://postgres@postgres:5432/postgres?sslmode=disable", 24 | } 25 | if url := os.Getenv("POSTGRES_URL"); url != "" { 26 | pgConf.URL = url 27 | } 28 | 29 | migrationConf = &MigrationConfig{ 30 | Source: "file://.", 31 | URL: pgConf.URL, 32 | } 33 | 34 | ctx = logkit.NewLogger(&logkit.LoggerConfig{ 35 | Development: true, 36 | }).WithContext(context.Background()) 37 | }) 38 | 39 | JustBeforeEach(func() { 40 | migration = NewMigration(ctx, migrationConf) 41 | }) 42 | 43 | AfterEach(func() { 44 | Expect(migration.Close()).NotTo(HaveOccurred()) 45 | }) 46 | 47 | When("success", func() { 48 | It("returns new Migration without error", func() { 49 | Expect(migration).NotTo(BeNil()) 50 | }) 51 | }) 52 | }) 53 | }) 54 | -------------------------------------------------------------------------------- /modules/video/dao/main_test.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/mongokit" 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/rediskit" 11 | . "github.com/onsi/ginkgo/v2" 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | func TestDAO(t *testing.T) { 16 | RegisterFailHandler(Fail) 17 | RunSpecs(t, "Test DAO") 18 | } 19 | 20 | var ( 21 | mongoClient *mongokit.MongoClient 22 | redisClient *rediskit.RedisClient 23 | ) 24 | 25 | var _ = BeforeSuite(func() { 26 | mongoConf := &mongokit.MongoConfig{ 27 | URL: "mongodb://mongo:27017", 28 | Database: "video", 29 | } 30 | 31 | if url := os.Getenv("MONGO_URL"); url != "" { 32 | mongoConf.URL = url 33 | } 34 | 35 | if database := os.Getenv("MONGO_DATABASE"); database != "" { 36 | mongoConf.Database = database 37 | } 38 | 39 | redisConf := &rediskit.RedisConfig{ 40 | Addr: "redis:6379", 41 | } 42 | 43 | ctx := logkit.NewLogger(&logkit.LoggerConfig{ 44 | Development: true, 45 | }).WithContext(context.Background()) 46 | 47 | mongoClient = mongokit.NewMongoClient(ctx, mongoConf) 48 | redisClient = rediskit.NewRedisClient(ctx, redisConf) 49 | }) 50 | 51 | var _ = AfterSuite(func() { 52 | Expect(mongoClient.Close()).NotTo(HaveOccurred()) 53 | Expect(redisClient.Close()).NotTo(HaveOccurred()) 54 | }) 55 | -------------------------------------------------------------------------------- /k8s/base/comment/comment-api/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: comment-api 5 | spec: 6 | replicas: 2 7 | template: 8 | spec: 9 | containers: 10 | - name: comment-api 11 | image: ghcr.io/nthu-lsalab/nthu-distributed-system:latest 12 | imagePullPolicy: Always 13 | ports: 14 | - name: grpc 15 | containerPort: 8081 16 | - name: prometheus 17 | containerPort: 2222 18 | command: 19 | - /cmd 20 | - comment 21 | - api 22 | env: 23 | - name: METER_HISTOGRAM_BOUNDARIES 24 | value: 10,100,200,500,1000 25 | - name: METER_NAME 26 | value: comment.api 27 | - name: MINIO_BUCKET 28 | value: videos 29 | - name: MINIO_ENDPOINT 30 | value: play.min.io 31 | - name: MINIO_PASSWORD 32 | value: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG 33 | - name: MINIO_USERNAME 34 | value: Q3AM3UQ867SPQQA43P2F 35 | - name: POSTGRES_URL 36 | value: postgres://postgres@postgres:5432/postgres?sslmode=disable 37 | - name: REDIS_ADDR 38 | value: redis:6379 39 | - name: VIDEO_SERVER_ADDR 40 | value: video-api:80 41 | resources: 42 | requests: 43 | memory: 30Mi 44 | cpu: 10m 45 | limits: 46 | memory: 60Mi 47 | cpu: 20m 48 | -------------------------------------------------------------------------------- /pkg/grpckit/client_conn.go: -------------------------------------------------------------------------------- 1 | package grpckit 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | "go.uber.org/zap" 9 | "google.golang.org/grpc" 10 | "google.golang.org/grpc/credentials/insecure" 11 | ) 12 | 13 | type GrpcClientConnConfig struct { 14 | Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s"` 15 | ServerAddr string `long:"server_addr" env:"SERVER_ADDR" required:"true"` 16 | } 17 | 18 | type GrpcClientConn struct { 19 | *grpc.ClientConn 20 | 21 | closeFunc func() 22 | } 23 | 24 | func (c *GrpcClientConn) Close() error { 25 | if c.closeFunc != nil { 26 | c.closeFunc() 27 | } 28 | 29 | return c.ClientConn.Close() 30 | } 31 | 32 | func NewGrpcClientConn(ctx context.Context, conf *GrpcClientConnConfig) *GrpcClientConn { 33 | logger := logkit.FromContext(ctx).With( 34 | zap.String("server_addr", conf.ServerAddr), 35 | ) 36 | 37 | var cancel context.CancelFunc 38 | ctx, cancel = context.WithTimeout(ctx, conf.Timeout) 39 | 40 | conn, err := grpc.DialContext(ctx, conf.ServerAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) 41 | if err != nil { 42 | logger.Fatal("failed to connect to gRPC server", zap.Error(err)) 43 | } 44 | 45 | logger.Info("connect to gRPC server successfully") 46 | 47 | closeFunc := func() { 48 | cancel() 49 | } 50 | 51 | return &GrpcClientConn{ 52 | ClientConn: conn, 53 | closeFunc: closeFunc, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /cmd/comment/migration.go: -------------------------------------------------------------------------------- 1 | package comment 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/migrationkit" 9 | flags "github.com/jessevdk/go-flags" 10 | "github.com/spf13/cobra" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | func newMigrationCommand() *cobra.Command { 15 | return &cobra.Command{ 16 | Use: "migration", 17 | Short: "runs the comment module migration job", 18 | RunE: runMigration, 19 | } 20 | } 21 | 22 | type MigrationArgs struct { 23 | logkit.LoggerConfig `group:"logger" namespace:"logger" env-namespace:"LOGGER"` 24 | migrationkit.MigrationConfig `group:"migration" namespace:"migration" env-namespace:"MIGRATION"` 25 | } 26 | 27 | func runMigration(_ *cobra.Command, _ []string) error { 28 | ctx := context.Background() 29 | 30 | var args MigrationArgs 31 | if _, err := flags.NewParser(&args, flags.Default).Parse(); err != nil { 32 | log.Fatal("failed to parse flag", err.Error()) 33 | } 34 | 35 | logger := logkit.NewLogger(&args.LoggerConfig) 36 | defer func() { 37 | _ = logger.Sync() 38 | }() 39 | 40 | ctx = logger.WithContext(ctx) 41 | 42 | migration := migrationkit.NewMigration(ctx, &args.MigrationConfig) 43 | if err := migration.Up(); err != nil { 44 | logger.Fatal("failed to run migration", zap.Error(err)) 45 | } 46 | 47 | logger.Info("run migration job successfully, terminating ...") 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /k8s/base/zookeeper/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: zookeeper 5 | spec: 6 | serviceName: zookeeper 7 | replicas: 1 8 | template: 9 | spec: 10 | containers: 11 | - name: zookeeper 12 | image: confluentinc/cp-zookeeper:7.0.1 13 | ports: 14 | - name: zookeeper 15 | containerPort: 2181 16 | env: 17 | - name: ZOOKEEPER_CLIENT_PORT 18 | value: "2181" 19 | - name: ZOOKEEPER_TICK_TIME 20 | value: "2000" 21 | resources: 22 | requests: 23 | cpu: 200m 24 | memory: 200Mi 25 | limits: 26 | cpu: 400m 27 | memory: 400Mi 28 | volumeMounts: 29 | - name: zookeeper-persistent-storage-datadir 30 | mountPath: /var/lib/zookeeper/data 31 | - name: zookeeper-persistent-storage-datalogdir 32 | mountPath: /var/lib/zookeeper/log 33 | volumeClaimTemplates: 34 | - metadata: 35 | name: zookeeper-persistent-storage-datadir 36 | spec: 37 | storageClassName: nfs 38 | accessModes: 39 | - ReadWriteOnce 40 | resources: 41 | requests: 42 | storage: 250Mi 43 | - metadata: 44 | name: zookeeper-persistent-storage-datalogdir 45 | spec: 46 | storageClassName: nfs 47 | accessModes: 48 | - ReadWriteOnce 49 | resources: 50 | requests: 51 | storage: 250Mi 52 | -------------------------------------------------------------------------------- /pkg/logkit/logger.go: -------------------------------------------------------------------------------- 1 | package logkit 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | ) 10 | 11 | type LoggerLevel zapcore.Level 12 | 13 | func (l LoggerLevel) MarshalFlag() (string, error) { 14 | return zapcore.Level(l).String(), nil 15 | } 16 | 17 | func (l *LoggerLevel) UnmarshalFlag(value string) error { 18 | zv := (*zapcore.Level)(l) 19 | return zv.UnmarshalText([]byte(value)) 20 | } 21 | 22 | type LoggerConfig struct { 23 | Level LoggerLevel `long:"level" description:"set log level" default:"info" env:"LEVEL"` 24 | Development bool `long:"development" description:"enable development mode" env:"DEVELOPMENT"` 25 | } 26 | 27 | type Logger struct { 28 | *zap.Logger 29 | } 30 | 31 | func NewNopLogger() *Logger { 32 | return &Logger{Logger: zap.NewNop()} 33 | } 34 | 35 | func NewLogger(conf *LoggerConfig) *Logger { 36 | var config zap.Config 37 | 38 | if conf.Development { 39 | config = zap.NewDevelopmentConfig() 40 | } else { 41 | config = zap.NewProductionConfig() 42 | } 43 | 44 | config.Level = zap.NewAtomicLevelAt(zapcore.Level(conf.Level)) 45 | 46 | logger, err := config.Build() 47 | if err != nil { 48 | log.Fatal("failed to build logger") 49 | } 50 | 51 | return &Logger{Logger: logger} 52 | } 53 | 54 | func (l *Logger) WithContext(ctx context.Context) context.Context { 55 | return WithContext(ctx, l) 56 | } 57 | 58 | func (l *Logger) With(fields ...zapcore.Field) *Logger { 59 | return &Logger{ 60 | Logger: l.Logger.With(fields...), 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /k8s/base/kafka/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: kafka 5 | spec: 6 | serviceName: kafka 7 | replicas: 1 8 | template: 9 | spec: 10 | containers: 11 | - name: kafka 12 | image: confluentinc/cp-kafka:7.0.1 13 | ports: 14 | - name: kafka 15 | containerPort: 9092 16 | env: 17 | - name: KAFKA_BOOTSTRAP_SERVERS 18 | value: kafka:9092 19 | - name: KAFKA_BROKER_ID 20 | value: "1" 21 | - name: KAFKA_ZOOKEEPER_CONNECT 22 | value: zookeeper:2181 23 | - name: KAFKA_ADVERTISED_LISTENERS 24 | value: INTERNAL://:29092,EXTERNAL://:9092 25 | - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP 26 | value: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT 27 | - name: KAFKA_INTER_BROKER_LISTENER_NAME 28 | value: INTERNAL 29 | - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR 30 | value: "1" 31 | resources: 32 | requests: 33 | cpu: 300m 34 | memory: 400Mi 35 | limits: 36 | cpu: 600m 37 | memory: 800Mi 38 | volumeMounts: 39 | - name: kafka-persistent-volume-claim 40 | mountPath: /opt/kafka/data-1 41 | enableServiceLinks: false 42 | volumeClaimTemplates: 43 | - metadata: 44 | name: kafka-persistent-volume-claim 45 | spec: 46 | storageClassName: nfs 47 | accessModes: 48 | - ReadWriteOnce 49 | resources: 50 | requests: 51 | storage: 500Mi 52 | -------------------------------------------------------------------------------- /k8s/base/video/video-api/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: video-api 5 | spec: 6 | replicas: 2 7 | template: 8 | spec: 9 | containers: 10 | - name: video-api 11 | image: ghcr.io/nthu-lsalab/nthu-distributed-system:latest 12 | imagePullPolicy: Always 13 | ports: 14 | - name: grpc 15 | containerPort: 8081 16 | - name: prometheus 17 | containerPort: 2222 18 | command: 19 | - /cmd 20 | - video 21 | - api 22 | env: 23 | - name: KAFKA_PRODUCER_ADDRS 24 | value: kafka:9092 25 | - name: KAFKA_PRODUCER_TOPIC 26 | value: video 27 | - name: METER_HISTOGRAM_BOUNDARIES 28 | value: 10,100,200,500,1000 29 | - name: METER_NAME 30 | value: video.api 31 | - name: MINIO_BUCKET 32 | value: videos 33 | - name: MINIO_ENDPOINT 34 | value: play.min.io 35 | - name: MINIO_PASSWORD 36 | value: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG 37 | - name: MINIO_USERNAME 38 | value: Q3AM3UQ867SPQQA43P2F 39 | - name: MONGO_DATABASE 40 | value: nthu_distributed_system 41 | - name: MONGO_URL 42 | value: mongodb://mongodb:27017/ 43 | - name: REDIS_ADDR 44 | value: redis:6379 45 | - name: COMMENT_SERVER_ADDR 46 | value: comment-api:80 47 | resources: 48 | requests: 49 | memory: 30Mi 50 | cpu: 10m 51 | limits: 52 | memory: 60Mi 53 | cpu: 20m 54 | -------------------------------------------------------------------------------- /modules/comment/dao/comment.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb" 10 | "github.com/google/uuid" 11 | "go.mongodb.org/mongo-driver/bson/primitive" 12 | "google.golang.org/protobuf/types/known/timestamppb" 13 | ) 14 | 15 | type Comment struct { 16 | ID uuid.UUID 17 | VideoID string 18 | Content string 19 | CreatedAt time.Time 20 | UpdatedAt time.Time 21 | } 22 | 23 | func (c *Comment) ToProto() *pb.CommentInfo { 24 | return &pb.CommentInfo{ 25 | Id: c.ID.String(), 26 | VideoId: c.VideoID, 27 | Content: c.Content, 28 | CreatedAt: timestamppb.New(c.CreatedAt), 29 | UpdatedAt: timestamppb.New(c.UpdatedAt), 30 | } 31 | } 32 | 33 | type CommentDAO interface { 34 | ListByVideoID(ctx context.Context, videoID string, limit, offset int) ([]*Comment, error) 35 | Create(ctx context.Context, comment *Comment) (uuid.UUID, error) 36 | Update(ctx context.Context, comment *Comment) error 37 | Delete(ctx context.Context, id uuid.UUID) error 38 | DeleteByVideoID(ctx context.Context, videoID string) error 39 | } 40 | 41 | var ( 42 | ErrCommentNotFound = errors.New("comment not found") 43 | ) 44 | 45 | func listCommentKey(videoID string, limit, offset int) string { 46 | return fmt.Sprintf("listComment:%s:%d:%d", videoID, limit, offset) 47 | } 48 | 49 | func NewFakeComment(videoID string) *Comment { 50 | if videoID == "" { 51 | videoID = primitive.NewObjectID().Hex() 52 | } 53 | 54 | return &Comment{ 55 | ID: uuid.New(), 56 | VideoID: videoID, 57 | Content: "comment test", 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/kafkakit/consumer.go: -------------------------------------------------------------------------------- 1 | package kafkakit 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 7 | "github.com/Shopify/sarama" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type KafkaConsumerConfig struct { 12 | Addrs []string `long:"addrs" env:"ADDRS" env-delim:"," description:"the addresses of Kafka servers" required:"true"` 13 | Topic string `long:"topic" env:"TOPIC" description:"the topic for the Kafka consumer group to consume" required:"true"` 14 | Group string `long:"group" env:"GROUP" description:"the ID of the Kafka consumer group" required:"true"` 15 | } 16 | 17 | type KafkaConsumer struct { 18 | sarama.ConsumerGroup 19 | 20 | topic string 21 | } 22 | 23 | func (kc *KafkaConsumer) Consume(ctx context.Context, handler sarama.ConsumerGroupHandler) error { 24 | for { 25 | if err := kc.ConsumerGroup.Consume(ctx, []string{kc.topic}, handler); err != nil { 26 | return err 27 | } 28 | } 29 | } 30 | 31 | func (kc *KafkaConsumer) Close() error { 32 | return kc.ConsumerGroup.Close() 33 | } 34 | 35 | func NewKafkaConsumer(ctx context.Context, conf *KafkaConsumerConfig) *KafkaConsumer { 36 | logger := logkit.FromContext(ctx).With( 37 | zap.Strings("addrs", conf.Addrs), 38 | zap.String("topic", conf.Topic), 39 | zap.String("group", conf.Group), 40 | ) 41 | 42 | config := sarama.NewConfig() 43 | 44 | cg, err := sarama.NewConsumerGroup(conf.Addrs, conf.Group, config) 45 | if err != nil { 46 | logger.Fatal("failed to create Kafka consumer group", zap.Error(err)) 47 | } 48 | 49 | logger.Info("create Kafka consumer successfully") 50 | 51 | return &KafkaConsumer{ 52 | ConsumerGroup: cg, 53 | topic: conf.Topic, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/kafkakit/mock/kafkamock/mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit (interfaces: Producer) 3 | 4 | // Package kafkamock is a generated GoMock package. 5 | package kafkamock 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | kafkakit "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit" 11 | gomock "github.com/golang/mock/gomock" 12 | ) 13 | 14 | // MockProducer is a mock of Producer interface. 15 | type MockProducer struct { 16 | ctrl *gomock.Controller 17 | recorder *MockProducerMockRecorder 18 | } 19 | 20 | // MockProducerMockRecorder is the mock recorder for MockProducer. 21 | type MockProducerMockRecorder struct { 22 | mock *MockProducer 23 | } 24 | 25 | // NewMockProducer creates a new mock instance. 26 | func NewMockProducer(ctrl *gomock.Controller) *MockProducer { 27 | mock := &MockProducer{ctrl: ctrl} 28 | mock.recorder = &MockProducerMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use. 33 | func (m *MockProducer) EXPECT() *MockProducerMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // SendMessages mocks base method. 38 | func (m *MockProducer) SendMessages(arg0 []*kafkakit.ProducerMessage) error { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "SendMessages", arg0) 41 | ret0, _ := ret[0].(error) 42 | return ret0 43 | } 44 | 45 | // SendMessages indicates an expected call of SendMessages. 46 | func (mr *MockProducerMockRecorder) SendMessages(arg0 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockProducer)(nil).SendMessages), arg0) 49 | } 50 | -------------------------------------------------------------------------------- /modules/comment/dao/main_test.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/migrationkit" 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/pgkit" 11 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/rediskit" 12 | . "github.com/onsi/ginkgo/v2" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | func TestDAO(t *testing.T) { 17 | RegisterFailHandler(Fail) 18 | RunSpecs(t, "Test DAO") 19 | } 20 | 21 | var ( 22 | pgClient *pgkit.PGClient 23 | redisClient *rediskit.RedisClient 24 | ) 25 | 26 | var _ = BeforeSuite(func() { 27 | pgConf := &pgkit.PGConfig{ 28 | URL: "postgres://postgres@postgres:5432/postgres?sslmode=disable", 29 | } 30 | if url := os.Getenv("POSTGRES_URL"); url != "" { 31 | pgConf.URL = url 32 | } 33 | 34 | migrationConf := &migrationkit.MigrationConfig{ 35 | Source: "file://../migration", 36 | URL: pgConf.URL, 37 | } 38 | 39 | redisConf := &rediskit.RedisConfig{ 40 | Addr: "redis:6379", 41 | } 42 | 43 | ctx := logkit.NewLogger(&logkit.LoggerConfig{ 44 | Development: true, 45 | }).WithContext(context.Background()) 46 | 47 | migration := migrationkit.NewMigration(ctx, migrationConf) 48 | defer func() { 49 | Expect(migration.Close()).NotTo(HaveOccurred()) 50 | }() 51 | 52 | Expect(migration.Up()).NotTo(HaveOccurred()) 53 | 54 | pgClient = pgkit.NewPGClient(ctx, pgConf) 55 | redisClient = rediskit.NewRedisClient(ctx, redisConf) 56 | }) 57 | 58 | var _ = AfterSuite(func() { 59 | Expect(pgClient.Close()).NotTo(HaveOccurred()) 60 | Expect(redisClient.Close()).NotTo(HaveOccurred()) 61 | }) 62 | 63 | var pgExec = func(query string, params ...interface{}) { 64 | _, err := pgClient.Exec(query, params...) 65 | Expect(err).NotTo(HaveOccurred()) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/mongokit/client.go: -------------------------------------------------------------------------------- 1 | package mongokit 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 8 | "go.mongodb.org/mongo-driver/mongo" 9 | "go.mongodb.org/mongo-driver/mongo/options" 10 | "go.mongodb.org/mongo-driver/mongo/readpref" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | type MongoConfig struct { 15 | URL string `long:"url" env:"URL" description:"the URL of MongoDB" required:"true"` 16 | Database string `long:"database" env:"DATABASE" description:"the database of MongoDB" required:"true"` 17 | } 18 | 19 | type MongoClient struct { 20 | *mongo.Client 21 | database *mongo.Database 22 | closeFunc func() 23 | } 24 | 25 | func (c *MongoClient) Database() *mongo.Database { 26 | return c.database 27 | } 28 | 29 | func (c *MongoClient) Close() error { 30 | if c.closeFunc != nil { 31 | c.closeFunc() 32 | } 33 | 34 | return c.Client.Disconnect(context.Background()) 35 | } 36 | 37 | func NewMongoClient(ctx context.Context, conf *MongoConfig) *MongoClient { 38 | if url := os.ExpandEnv(conf.URL); url != "" { 39 | conf.URL = url 40 | } 41 | 42 | logger := logkit.FromContext(ctx).With( 43 | zap.String("url", conf.URL), 44 | zap.String("database", conf.Database), 45 | ) 46 | 47 | o := options.Client() 48 | o.ApplyURI(conf.URL) 49 | 50 | client, err := mongo.NewClient(o) 51 | if err != nil { 52 | logger.Fatal("failed to create MongoDB client", zap.Error(err)) 53 | } 54 | 55 | if err := client.Connect(ctx); err != nil { 56 | logger.Fatal("failed to connect to MongoDB", zap.Error(err)) 57 | } 58 | 59 | if err := client.Ping(ctx, readpref.Primary()); err != nil { 60 | logger.Fatal("failed to ping to MongoDB", zap.Error(err)) 61 | } 62 | 63 | logger.Info("create MongoDB client successfully") 64 | 65 | return &MongoClient{ 66 | Client: client, 67 | database: client.Database(conf.Database), 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /modules/comment/dao/comment_redis.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/rediskit" 8 | "github.com/go-redis/cache/v8" 9 | "github.com/google/uuid" 10 | ) 11 | 12 | type redisCommentDAO struct { 13 | cache *cache.Cache 14 | baseDAO CommentDAO 15 | } 16 | 17 | var _ CommentDAO = (*redisCommentDAO)(nil) 18 | 19 | const ( 20 | commentDAOLocalCacheSize = 1024 21 | commentDAOLocalCacheDuration = 1 * time.Minute 22 | commentDAORedisCacheDuration = 3 * time.Minute 23 | ) 24 | 25 | func NewRedisCommentDAO(client *rediskit.RedisClient, baseDAO CommentDAO) *redisCommentDAO { 26 | return &redisCommentDAO{ 27 | cache: cache.New(&cache.Options{ 28 | Redis: client, 29 | LocalCache: cache.NewTinyLFU(commentDAOLocalCacheSize, commentDAOLocalCacheDuration), 30 | }), 31 | baseDAO: baseDAO, 32 | } 33 | } 34 | 35 | func (dao *redisCommentDAO) ListByVideoID(ctx context.Context, videoID string, limit, offset int) ([]*Comment, error) { 36 | var comment []*Comment 37 | 38 | if err := dao.cache.Once(&cache.Item{ 39 | Key: listCommentKey(videoID, limit, offset), 40 | Value: &comment, 41 | TTL: commentDAORedisCacheDuration, 42 | Do: func(*cache.Item) (interface{}, error) { 43 | return dao.baseDAO.ListByVideoID(ctx, videoID, limit, offset) 44 | }, 45 | }); err != nil { 46 | return nil, err 47 | } 48 | return comment, nil 49 | } 50 | 51 | // The following operations are not cachable, just pass down to baseDAO 52 | 53 | func (dao *redisCommentDAO) Create(ctx context.Context, comment *Comment) (uuid.UUID, error) { 54 | return dao.baseDAO.Create(ctx, comment) 55 | } 56 | 57 | func (dao *redisCommentDAO) Update(ctx context.Context, comment *Comment) error { 58 | return dao.baseDAO.Update(ctx, comment) 59 | } 60 | 61 | func (dao *redisCommentDAO) Delete(ctx context.Context, id uuid.UUID) error { 62 | return dao.baseDAO.Delete(ctx, id) 63 | } 64 | 65 | func (dao *redisCommentDAO) DeleteByVideoID(ctx context.Context, videoID string) error { 66 | return dao.baseDAO.DeleteByVideoID(ctx, videoID) 67 | } 68 | -------------------------------------------------------------------------------- /modules/comment/dao/comment_pg.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/pgkit" 8 | "github.com/go-pg/pg/v10" 9 | "github.com/google/uuid" 10 | ) 11 | 12 | type pgCommentDAO struct { 13 | client *pgkit.PGClient 14 | } 15 | 16 | var _ CommentDAO = (*pgCommentDAO)(nil) 17 | 18 | func NewPGCommentDAO(pgClient *pgkit.PGClient) *pgCommentDAO { 19 | return &pgCommentDAO{ 20 | client: pgClient, 21 | } 22 | } 23 | 24 | func (dao *pgCommentDAO) ListByVideoID(ctx context.Context, videoID string, limit, offset int) ([]*Comment, error) { 25 | var comments []*Comment 26 | query := dao.client.ModelContext(ctx, &comments). 27 | Where("video_id = ?", videoID). 28 | Limit(limit). 29 | Offset(offset). 30 | Order("updated_at ASC") 31 | 32 | if err := query.Select(); err != nil { 33 | return nil, err 34 | } 35 | 36 | return comments, nil 37 | } 38 | 39 | func (dao *pgCommentDAO) Create(ctx context.Context, comment *Comment) (uuid.UUID, error) { 40 | if _, err := dao.client.ModelContext(ctx, comment).Insert(); err != nil { 41 | return uuid.Nil, err 42 | } 43 | 44 | return comment.ID, nil 45 | } 46 | 47 | func (dao *pgCommentDAO) Update(ctx context.Context, comment *Comment) error { 48 | if _, err := dao.client.ModelContext(ctx, comment).Column("content").WherePK().Returning("*").Update(); err != nil { 49 | if errors.Is(err, pg.ErrNoRows) { 50 | return ErrCommentNotFound 51 | } 52 | 53 | return err 54 | } 55 | 56 | return nil 57 | } 58 | 59 | func (dao *pgCommentDAO) Delete(ctx context.Context, id uuid.UUID) error { 60 | if res, err := dao.client.ModelContext(ctx, &Comment{ID: id}).WherePK().Delete(); err != nil { 61 | return err 62 | } else if res.RowsAffected() == 0 { 63 | return ErrCommentNotFound 64 | } 65 | 66 | return nil 67 | } 68 | 69 | // delete all comments when the video deleted 70 | func (dao *pgCommentDAO) DeleteByVideoID(ctx context.Context, videoID string) error { 71 | if _, err := dao.client.ModelContext(ctx, (*Comment)(nil)).Where("video_id = ?", videoID).Delete(); err != nil { 72 | return err 73 | } 74 | 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /modules/video/pb/stream.pb.sarama.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-grpc-sarama. DO NOT EDIT. 2 | 3 | package pb 4 | 5 | import ( 6 | "errors" 7 | 8 | "github.com/Shopify/sarama" 9 | "google.golang.org/protobuf/proto" 10 | "github.com/justin0u0/protoc-gen-grpc-sarama/pkg/saramakit" 11 | ) 12 | 13 | type VideoStreamHandlers struct { 14 | *HandleVideoCreatedHandler 15 | } 16 | 17 | func NewVideoStreamHandlers(server VideoStreamServer, logger saramakit.Logger) *VideoStreamHandlers { 18 | return &VideoStreamHandlers{ 19 | HandleVideoCreatedHandler: &HandleVideoCreatedHandler{ 20 | server: server, 21 | unmarshaler: &proto.UnmarshalOptions{}, 22 | logger: logger.With("HandlerName", "HandleVideoCreatedHandler"), 23 | }, 24 | } 25 | } 26 | 27 | type HandleVideoCreatedHandler struct { 28 | server VideoStreamServer 29 | unmarshaler *proto.UnmarshalOptions 30 | logger saramakit.Logger 31 | } 32 | 33 | var _ sarama.ConsumerGroupHandler = (*HandleVideoCreatedHandler)(nil) 34 | 35 | func (h *HandleVideoCreatedHandler) Setup(sarama.ConsumerGroupSession) error { 36 | return nil 37 | } 38 | 39 | func (h *HandleVideoCreatedHandler) Cleanup(sarama.ConsumerGroupSession) error { 40 | return nil 41 | } 42 | 43 | func (h *HandleVideoCreatedHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 44 | for msg := range claim.Messages() { 45 | var req HandleVideoCreatedRequest 46 | 47 | if err := h.unmarshaler.Unmarshal(msg.Value, &req); err != nil { 48 | // unretryable failure, skip and consume the message 49 | h.logger.Error("failed to unmarshal message", err) 50 | 51 | continue 52 | } 53 | 54 | if _, err := h.server.HandleVideoCreated(sess.Context(), &req); err != nil { 55 | var e saramakit.HandlerError 56 | 57 | if ok := errors.As(err, &e); ok && e.Retry { 58 | h.logger.Error("failed to handle the message and the error is retryable", err) 59 | 60 | return nil 61 | } 62 | h.logger.Error("failed to handle the message and the error is unretryable", err) 63 | } 64 | 65 | // mark message as completed 66 | sess.MarkMessage(msg, "") 67 | } 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /pkg/migrationkit/migration.go: -------------------------------------------------------------------------------- 1 | package migrationkit 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 9 | "github.com/golang-migrate/migrate/v4" 10 | _ "github.com/golang-migrate/migrate/v4/database/postgres" 11 | _ "github.com/golang-migrate/migrate/v4/source/file" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | type MigrationConfig struct { 16 | // Source is the migration files source directory, 17 | // currently we only accept file system as source. 18 | // 19 | // Register more source types by importing other source packages. 20 | // 21 | // File System: https://github.com/golang-migrate/migrate/tree/master/source/file 22 | Source string `long:"source" env:"SOURCE" description:"the migration files source directory" required:"true"` 23 | // URL is the migration database URL, 24 | // currently we only accept Postgres as database. 25 | // 26 | // Register more database types by importing other database packages. 27 | // 28 | // Postgres: https://github.com/golang-migrate/migrate/tree/master/database/postgres 29 | URL string `long:"url" env:"URL" description:"the database url" required:"true"` 30 | } 31 | 32 | type Migration struct { 33 | *migrate.Migrate 34 | logger *logkit.Logger 35 | } 36 | 37 | func (m *Migration) Up() error { 38 | if err := m.Migrate.Up(); err != nil { 39 | if errors.Is(err, migrate.ErrNoChange) { 40 | m.logger.Info("no change to migrate") 41 | 42 | return nil 43 | } 44 | 45 | return err 46 | } 47 | 48 | return nil 49 | } 50 | 51 | func (m *Migration) Close() error { 52 | serr, derr := m.Migrate.Close() 53 | if serr != nil { 54 | return serr 55 | } 56 | 57 | if derr != nil { 58 | return derr 59 | } 60 | 61 | return nil 62 | } 63 | 64 | func NewMigration(ctx context.Context, conf *MigrationConfig) *Migration { 65 | if url := os.ExpandEnv(conf.URL); url != "" { 66 | conf.URL = url 67 | } 68 | 69 | logger := logkit.FromContext(ctx).With( 70 | zap.String("source", conf.Source), 71 | zap.String("url", conf.URL), 72 | ) 73 | 74 | m, err := migrate.New(conf.Source, conf.URL) 75 | if err != nil { 76 | logger.Fatal("failed to create migration", zap.Error(err)) 77 | } 78 | 79 | logger.Info("create migration successfully") 80 | 81 | return &Migration{ 82 | Migrate: m, 83 | logger: logger, 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /modules/video/dao/video_redis.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/rediskit" 8 | "github.com/go-redis/cache/v8" 9 | "go.mongodb.org/mongo-driver/bson/primitive" 10 | ) 11 | 12 | type redisVideoDAO struct { 13 | cache *cache.Cache 14 | baseDAO VideoDAO 15 | } 16 | 17 | var _ VideoDAO = (*redisVideoDAO)(nil) 18 | 19 | const ( 20 | videoDAOLocalCacheSize = 1024 21 | videoDAOLocalCacheDuration = 1 * time.Minute 22 | videoDAORedisCacheDuration = 3 * time.Minute 23 | ) 24 | 25 | func NewRedisVideoDAO(client *rediskit.RedisClient, baseDAO VideoDAO) *redisVideoDAO { 26 | return &redisVideoDAO{ 27 | cache: cache.New(&cache.Options{ 28 | Redis: client, 29 | LocalCache: cache.NewTinyLFU(videoDAOLocalCacheSize, videoDAOLocalCacheDuration), 30 | }), 31 | baseDAO: baseDAO, 32 | } 33 | } 34 | 35 | func (dao *redisVideoDAO) Get(ctx context.Context, id primitive.ObjectID) (*Video, error) { 36 | var video Video 37 | 38 | if err := dao.cache.Once(&cache.Item{ 39 | Key: getVideoKey(id), 40 | Value: &video, 41 | TTL: videoDAORedisCacheDuration, 42 | Do: func(*cache.Item) (interface{}, error) { 43 | return dao.baseDAO.Get(ctx, id) 44 | }, 45 | }); err != nil { 46 | return nil, err 47 | } 48 | 49 | return &video, nil 50 | } 51 | 52 | func (dao *redisVideoDAO) List(ctx context.Context, limit, skip int64) ([]*Video, error) { 53 | var videos []*Video 54 | 55 | if err := dao.cache.Once(&cache.Item{ 56 | Key: listVideoKey(limit, skip), 57 | Value: &videos, 58 | TTL: videoDAORedisCacheDuration, 59 | Do: func(*cache.Item) (interface{}, error) { 60 | return dao.baseDAO.List(ctx, limit, skip) 61 | }, 62 | }); err != nil { 63 | return nil, err 64 | } 65 | 66 | return videos, nil 67 | } 68 | 69 | // The following operations are not cachable, just pass down to baseDAO. 70 | 71 | func (dao *redisVideoDAO) Create(ctx context.Context, video *Video) error { 72 | return dao.baseDAO.Create(ctx, video) 73 | } 74 | 75 | func (dao *redisVideoDAO) Update(ctx context.Context, video *Video) error { 76 | return dao.baseDAO.Update(ctx, video) 77 | } 78 | 79 | func (dao *redisVideoDAO) UpdateVariant(ctx context.Context, id primitive.ObjectID, variant string, url string) error { 80 | return dao.baseDAO.UpdateVariant(ctx, id, variant, url) 81 | } 82 | 83 | func (dao *redisVideoDAO) Delete(ctx context.Context, id primitive.ObjectID) error { 84 | return dao.baseDAO.Delete(ctx, id) 85 | } 86 | -------------------------------------------------------------------------------- /pkg/kafkakit/producer.go: -------------------------------------------------------------------------------- 1 | package kafkakit 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 7 | "github.com/Shopify/sarama" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type Producer interface { 12 | SendMessages(msgs []*ProducerMessage) error 13 | } 14 | 15 | type ProducerMessage struct { 16 | Key []byte 17 | Value []byte 18 | } 19 | 20 | type KafkaProducerConfig struct { 21 | Addrs []string `long:"addrs" env:"ADDRS" env-delim:"," description:"the addresses of Kafka servers" required:"true"` 22 | Topic string `long:"topic" env:"TOPIC" description:"the topic for the Kafka producer to send" required:"true"` 23 | RequiredAcks int16 `long:"required_acks" env:"REQUIRED_ACKS" description:"number of replica acks the producer must receive before responding, available values are 0, 1 and -1" default:"-1"` 24 | } 25 | 26 | type KafkaProducer struct { 27 | sarama.SyncProducer 28 | 29 | topic string 30 | } 31 | 32 | var _ Producer = (*KafkaProducer)(nil) 33 | 34 | func (kp *KafkaProducer) SendMessages(msgs []*ProducerMessage) error { 35 | smsgs := make([]*sarama.ProducerMessage, 0, len(msgs)) 36 | for _, msg := range msgs { 37 | smsgs = append(smsgs, &sarama.ProducerMessage{ 38 | Topic: kp.topic, 39 | Key: sarama.ByteEncoder(msg.Key), 40 | Value: sarama.ByteEncoder(msg.Value), 41 | }) 42 | } 43 | 44 | return kp.SyncProducer.SendMessages(smsgs) 45 | } 46 | 47 | func (kp *KafkaProducer) Close() error { 48 | return kp.SyncProducer.Close() 49 | } 50 | 51 | func NewKafkaProducer(ctx context.Context, conf *KafkaProducerConfig) *KafkaProducer { 52 | logger := logkit.FromContext(ctx).With( 53 | zap.Strings("addrs", conf.Addrs), 54 | zap.String("topic", conf.Topic), 55 | zap.Int16("required_acks", conf.RequiredAcks), 56 | ) 57 | 58 | config := sarama.NewConfig() 59 | 60 | config.Producer.RequiredAcks = sarama.RequiredAcks(conf.RequiredAcks) 61 | 62 | // If this config is used to create a `SyncProducer`, both must be set 63 | // to true and you shall not read from the channels since the producer 64 | // does this internally. 65 | config.Producer.Return.Successes = true 66 | config.Producer.Return.Errors = true 67 | 68 | producer, err := sarama.NewSyncProducer(conf.Addrs, config) 69 | if err != nil { 70 | logger.Fatal("failed to create Kafka sync producer", zap.Error(err)) 71 | } 72 | 73 | logger.Info("create Kafka producer successfully") 74 | 75 | return &KafkaProducer{ 76 | SyncProducer: producer, 77 | topic: conf.Topic, 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /modules/video/stream/stream.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao" 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit" 11 | "github.com/justin0u0/protoc-gen-grpc-sarama/pkg/saramakit" 12 | "go.mongodb.org/mongo-driver/bson/primitive" 13 | "google.golang.org/protobuf/proto" 14 | "google.golang.org/protobuf/types/known/emptypb" 15 | ) 16 | 17 | type stream struct { 18 | pb.UnimplementedVideoStreamServer 19 | 20 | videoDAO dao.VideoDAO 21 | producer kafkakit.Producer 22 | } 23 | 24 | func NewStream(videoDAO dao.VideoDAO, producer kafkakit.Producer) *stream { 25 | return &stream{ 26 | videoDAO: videoDAO, 27 | producer: producer, 28 | } 29 | } 30 | 31 | func (s *stream) HandleVideoCreated(ctx context.Context, req *pb.HandleVideoCreatedRequest) (*emptypb.Empty, error) { 32 | id, err := primitive.ObjectIDFromHex(req.GetId()) 33 | if err != nil { 34 | return nil, &saramakit.HandlerError{Retry: false, Err: err} 35 | } 36 | 37 | if req.GetScale() != 0 { 38 | variant := strconv.Itoa(int(req.GetScale())) 39 | 40 | if err := s.handleVideoWithVariant(ctx, id, variant, req.GetUrl()); err != nil { 41 | return nil, &saramakit.HandlerError{Retry: true, Err: err} 42 | } 43 | 44 | return &emptypb.Empty{}, nil 45 | } 46 | 47 | // fanout create events to each variant 48 | variants := []int32{1080, 720, 480, 320} 49 | for _, scale := range variants { 50 | if err := s.produceVideoCreatedWithScaleEvent(&pb.HandleVideoCreatedRequest{ 51 | Id: req.GetId(), 52 | Url: req.GetUrl(), 53 | Scale: scale, 54 | }); err != nil { 55 | return nil, &saramakit.HandlerError{Retry: true, Err: err} 56 | } 57 | } 58 | 59 | return &emptypb.Empty{}, nil 60 | } 61 | 62 | func (s *stream) handleVideoWithVariant(ctx context.Context, id primitive.ObjectID, variant string, url string) error { 63 | // we mock the video transcoding only 64 | time.Sleep(3 * time.Second) 65 | 66 | if err := s.videoDAO.UpdateVariant(ctx, id, variant, url); err != nil { 67 | return err 68 | } 69 | 70 | return nil 71 | } 72 | 73 | func (s *stream) produceVideoCreatedWithScaleEvent(req *pb.HandleVideoCreatedRequest) error { 74 | valueBytes, err := proto.Marshal(req) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | msgs := []*kafkakit.ProducerMessage{ 80 | {Value: valueBytes}, 81 | } 82 | 83 | if err := s.producer.SendMessages(msgs); err != nil { 84 | return err 85 | } 86 | 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /pkg/storagekit/mock/storagemock/mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/storagekit (interfaces: Storage) 3 | 4 | // Package storagemock is a generated GoMock package. 5 | package storagemock 6 | 7 | import ( 8 | context "context" 9 | io "io" 10 | reflect "reflect" 11 | 12 | storagekit "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/storagekit" 13 | gomock "github.com/golang/mock/gomock" 14 | ) 15 | 16 | // MockStorage is a mock of Storage interface. 17 | type MockStorage struct { 18 | ctrl *gomock.Controller 19 | recorder *MockStorageMockRecorder 20 | } 21 | 22 | // MockStorageMockRecorder is the mock recorder for MockStorage. 23 | type MockStorageMockRecorder struct { 24 | mock *MockStorage 25 | } 26 | 27 | // NewMockStorage creates a new mock instance. 28 | func NewMockStorage(ctrl *gomock.Controller) *MockStorage { 29 | mock := &MockStorage{ctrl: ctrl} 30 | mock.recorder = &MockStorageMockRecorder{mock} 31 | return mock 32 | } 33 | 34 | // EXPECT returns an object that allows the caller to indicate expected use. 35 | func (m *MockStorage) EXPECT() *MockStorageMockRecorder { 36 | return m.recorder 37 | } 38 | 39 | // Bucket mocks base method. 40 | func (m *MockStorage) Bucket() string { 41 | m.ctrl.T.Helper() 42 | ret := m.ctrl.Call(m, "Bucket") 43 | ret0, _ := ret[0].(string) 44 | return ret0 45 | } 46 | 47 | // Bucket indicates an expected call of Bucket. 48 | func (mr *MockStorageMockRecorder) Bucket() *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockStorage)(nil).Bucket)) 51 | } 52 | 53 | // Endpoint mocks base method. 54 | func (m *MockStorage) Endpoint() string { 55 | m.ctrl.T.Helper() 56 | ret := m.ctrl.Call(m, "Endpoint") 57 | ret0, _ := ret[0].(string) 58 | return ret0 59 | } 60 | 61 | // Endpoint indicates an expected call of Endpoint. 62 | func (mr *MockStorageMockRecorder) Endpoint() *gomock.Call { 63 | mr.mock.ctrl.T.Helper() 64 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Endpoint", reflect.TypeOf((*MockStorage)(nil).Endpoint)) 65 | } 66 | 67 | // PutObject mocks base method. 68 | func (m *MockStorage) PutObject(arg0 context.Context, arg1 string, arg2 io.Reader, arg3 int64, arg4 storagekit.PutObjectOptions) error { 69 | m.ctrl.T.Helper() 70 | ret := m.ctrl.Call(m, "PutObject", arg0, arg1, arg2, arg3, arg4) 71 | ret0, _ := ret[0].(error) 72 | return ret0 73 | } 74 | 75 | // PutObject indicates an expected call of PutObject. 76 | func (mr *MockStorageMockRecorder) PutObject(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*MockStorage)(nil).PutObject), arg0, arg1, arg2, arg3, arg4) 79 | } 80 | -------------------------------------------------------------------------------- /modules/video/dao/video_mongo.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "go.mongodb.org/mongo-driver/bson" 8 | "go.mongodb.org/mongo-driver/bson/primitive" 9 | "go.mongodb.org/mongo-driver/mongo" 10 | "go.mongodb.org/mongo-driver/mongo/options" 11 | ) 12 | 13 | type mongoVideoDAO struct { 14 | collection *mongo.Collection 15 | } 16 | 17 | var _ VideoDAO = (*mongoVideoDAO)(nil) 18 | 19 | func NewMongoVideoDAO(collection *mongo.Collection) *mongoVideoDAO { 20 | return &mongoVideoDAO{ 21 | collection: collection, 22 | } 23 | } 24 | 25 | func (dao *mongoVideoDAO) Get(ctx context.Context, id primitive.ObjectID) (*Video, error) { 26 | var video Video 27 | if err := dao.collection.FindOne(ctx, bson.M{"_id": id}).Decode(&video); err != nil { 28 | if errors.Is(err, mongo.ErrNoDocuments) { 29 | return nil, ErrVideoNotFound 30 | } 31 | return nil, err 32 | } 33 | 34 | return &video, nil 35 | } 36 | 37 | func (dao *mongoVideoDAO) List(ctx context.Context, limit, skip int64) ([]*Video, error) { 38 | o := options.Find().SetLimit(limit).SetSkip(skip) 39 | 40 | cursor, err := dao.collection.Find(ctx, bson.M{}, o) 41 | if err != nil { 42 | return nil, err 43 | } 44 | defer cursor.Close(ctx) 45 | 46 | videos := make([]*Video, 0) 47 | for cursor.Next(ctx) { 48 | var video Video 49 | if err := cursor.Decode(&video); err != nil { 50 | return nil, err 51 | } 52 | 53 | videos = append(videos, &video) 54 | } 55 | 56 | return videos, nil 57 | } 58 | 59 | func (dao *mongoVideoDAO) Create(ctx context.Context, video *Video) error { 60 | result, err := dao.collection.InsertOne(ctx, video) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | video.ID = result.InsertedID.(primitive.ObjectID) 66 | 67 | return nil 68 | } 69 | 70 | func (dao *mongoVideoDAO) Update(ctx context.Context, video *Video) error { 71 | if result, err := dao.collection.ReplaceOne( 72 | ctx, 73 | bson.M{"_id": video.ID}, 74 | video, 75 | ); err != nil { 76 | return err 77 | } else if result.ModifiedCount == 0 { 78 | return ErrVideoNotFound 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func (dao *mongoVideoDAO) UpdateVariant(ctx context.Context, id primitive.ObjectID, variant string, url string) error { 85 | filter := bson.M{"_id": id} 86 | update := bson.D{{Key: "$set", Value: bson.M{"variants." + variant: url}}} 87 | opts := options.Update() 88 | 89 | if result, err := dao.collection.UpdateOne(ctx, filter, update, opts); err != nil { 90 | return err 91 | } else if result.MatchedCount == 0 { 92 | return ErrVideoNotFound 93 | } 94 | 95 | return nil 96 | } 97 | 98 | func (dao *mongoVideoDAO) Delete(ctx context.Context, id primitive.ObjectID) error { 99 | if result, err := dao.collection.DeleteOne(ctx, bson.M{"_id": id}); err != nil { 100 | return err 101 | } else if result.DeletedCount == 0 { 102 | return ErrVideoNotFound 103 | } 104 | 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /pkg/pb/google/api/httpbody.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/protobuf/any.proto"; 20 | 21 | option cc_enable_arenas = true; 22 | option go_package = "google.golang.org/genproto/googleapis/api/httpbody;httpbody"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "HttpBodyProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | // Message that represents an arbitrary HTTP body. It should only be used for 29 | // payload formats that can't be represented as JSON, such as raw binary or 30 | // an HTML page. 31 | // 32 | // 33 | // This message can be used both in streaming and non-streaming API methods in 34 | // the request as well as the response. 35 | // 36 | // It can be used as a top-level request field, which is convenient if one 37 | // wants to extract parameters from either the URL or HTTP template into the 38 | // request fields and also want access to the raw HTTP body. 39 | // 40 | // Example: 41 | // 42 | // message GetResourceRequest { 43 | // // A unique request id. 44 | // string request_id = 1; 45 | // 46 | // // The raw HTTP body is bound to this field. 47 | // google.api.HttpBody http_body = 2; 48 | // 49 | // } 50 | // 51 | // service ResourceService { 52 | // rpc GetResource(GetResourceRequest) 53 | // returns (google.api.HttpBody); 54 | // rpc UpdateResource(google.api.HttpBody) 55 | // returns (google.protobuf.Empty); 56 | // 57 | // } 58 | // 59 | // Example with streaming methods: 60 | // 61 | // service CaldavService { 62 | // rpc GetCalendar(stream google.api.HttpBody) 63 | // returns (stream google.api.HttpBody); 64 | // rpc UpdateCalendar(stream google.api.HttpBody) 65 | // returns (stream google.api.HttpBody); 66 | // 67 | // } 68 | // 69 | // Use of this type only changes how the request and response bodies are 70 | // handled, all other features will continue to work unchanged. 71 | message HttpBody { 72 | // The HTTP Content-Type header value specifying the content type of the body. 73 | string content_type = 1; 74 | 75 | // The HTTP request/response body as raw binary. 76 | bytes data = 2; 77 | 78 | // Application specific response metadata. Must be set in the first response 79 | // for streaming APIs. 80 | repeated google.protobuf.Any extensions = 3; 81 | } 82 | -------------------------------------------------------------------------------- /modules/video/dao/video.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 10 | "go.mongodb.org/mongo-driver/bson/primitive" 11 | "google.golang.org/protobuf/types/known/timestamppb" 12 | ) 13 | 14 | type VideoStatus string 15 | 16 | const ( 17 | VideoStatusUploaded VideoStatus = "uploaded" 18 | VideoStatusEncoding VideoStatus = "encoding" 19 | VideoStatusFailed VideoStatus = "failed" 20 | VideoStatusSuccess VideoStatus = "success" 21 | ) 22 | 23 | func (s VideoStatus) String() string { 24 | return string(s) 25 | } 26 | 27 | type Video struct { 28 | ID primitive.ObjectID `bson:"_id,omitempty"` 29 | Width uint32 `bson:"width,omitempty"` 30 | Height uint32 `bson:"height,omitempty"` 31 | Size uint64 `bson:"size,omitempty"` 32 | Duration float64 `bson:"duration,omitempty"` 33 | URL string `bson:"url,omitempty"` 34 | Status VideoStatus `bson:"status,omitempty"` 35 | Variants map[string]string `bson:"variants,omitempty"` 36 | CreatedAt time.Time `bson:"created_at,omitempty"` 37 | UpdatedAt time.Time `bson:"updated_at,omitempty"` 38 | } 39 | 40 | func (v *Video) ToProto() *pb.VideoInfo { 41 | return &pb.VideoInfo{ 42 | Id: v.ID.Hex(), 43 | Width: v.Width, 44 | Height: v.Height, 45 | Size: v.Size, 46 | Duration: v.Duration, 47 | Url: v.URL, 48 | Status: v.Status.String(), 49 | Variants: v.Variants, 50 | CreatedAt: timestamppb.New(v.CreatedAt), 51 | UpdatedAt: timestamppb.New(v.UpdatedAt), 52 | } 53 | } 54 | 55 | type VideoDAO interface { 56 | Get(ctx context.Context, id primitive.ObjectID) (*Video, error) 57 | List(ctx context.Context, limit, skip int64) ([]*Video, error) 58 | Create(ctx context.Context, video *Video) error 59 | Update(ctx context.Context, video *Video) error 60 | UpdateVariant(ctx context.Context, id primitive.ObjectID, variant string, url string) error 61 | Delete(ctx context.Context, id primitive.ObjectID) error 62 | } 63 | 64 | var ( 65 | ErrVideoNotFound = errors.New("video not found") 66 | ) 67 | 68 | func getVideoKey(id primitive.ObjectID) string { 69 | return "getVideo:" + id.Hex() 70 | } 71 | 72 | func listVideoKey(limit, skip int64) string { 73 | return fmt.Sprintf("listVideo:%d:%d", limit, skip) 74 | } 75 | 76 | // NewFakeVideo returns a fake video instance with random 77 | // id that is useful for testing 78 | func NewFakeVideo() *Video { 79 | id := primitive.NewObjectID() 80 | baseURL := "https://storage.example.com/videos/" + id.Hex() 81 | 82 | // Note that timestamp is hard to test equally, 83 | // so ignore the `createdAt` and `updatedAt` field 84 | 85 | return &Video{ 86 | ID: id, 87 | Width: 800, 88 | Height: 600, 89 | Size: 144000, 90 | Duration: 10.234, 91 | URL: baseURL + ".mp4", 92 | Status: VideoStatusSuccess, 93 | Variants: map[string]string{ 94 | "1080p": baseURL + "-1080p.mp4", 95 | "720p": baseURL + "-720p.mp4", 96 | }, 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /cmd/video/stream.go: -------------------------------------------------------------------------------- 1 | package video 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao" 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/stream" 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit" 11 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 12 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/mongokit" 13 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/runkit" 14 | flags "github.com/jessevdk/go-flags" 15 | "github.com/spf13/cobra" 16 | "go.uber.org/zap" 17 | ) 18 | 19 | func newStreamCommand() *cobra.Command { 20 | return &cobra.Command{ 21 | Use: "stream", 22 | Short: "starts video stream server", 23 | RunE: runStream, 24 | } 25 | } 26 | 27 | type StreamArgs struct { 28 | runkit.GracefulConfig `group:"graceful" namespace:"graceful" env-namespace:"GRACEFUL"` 29 | logkit.LoggerConfig `group:"logger" namespace:"logger" env-namespace:"LOGGER"` 30 | mongokit.MongoConfig `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` 31 | kafkakit.KafkaProducerConfig `group:"kafka_producer" namespace:"kafka_producer" env-namespace:"KAFKA_PRODUCER"` 32 | kafkakit.KafkaConsumerConfig `group:"kafka_consumer" namespace:"kafka_consumer" env-namespace:"KAFKA_CONSUMER"` 33 | } 34 | 35 | func runStream(_ *cobra.Command, _ []string) error { 36 | ctx := context.Background() 37 | 38 | var args StreamArgs 39 | if _, err := flags.NewParser(&args, flags.Default).Parse(); err != nil { 40 | log.Fatal("failed to parse flag", err.Error()) 41 | } 42 | 43 | logger := logkit.NewLogger(&args.LoggerConfig) 44 | defer func() { 45 | _ = logger.Sync() 46 | }() 47 | 48 | ctx = logger.WithContext(ctx) 49 | 50 | mongoClient := mongokit.NewMongoClient(ctx, &args.MongoConfig) 51 | defer func() { 52 | if err := mongoClient.Close(); err != nil { 53 | logger.Fatal("failed to close mongo client", zap.Error(err)) 54 | } 55 | }() 56 | 57 | producer := kafkakit.NewKafkaProducer(ctx, &args.KafkaProducerConfig) 58 | defer func() { 59 | if err := producer.Close(); err != nil { 60 | logger.Fatal("failed to close Kafka producer", zap.Error(err)) 61 | } 62 | }() 63 | 64 | consumer := kafkakit.NewKafkaConsumer(ctx, &args.KafkaConsumerConfig) 65 | defer func() { 66 | if err := consumer.Close(); err != nil { 67 | logger.Fatal("failed to close Kafka consumer", zap.Error(err)) 68 | } 69 | }() 70 | 71 | videoDAO := dao.NewMongoVideoDAO(mongoClient.Database().Collection("videos")) 72 | 73 | svc := stream.NewStream(videoDAO, producer) 74 | 75 | return runkit.GracefulRun(serveConsumer(consumer, svc, logger), &args.GracefulConfig) 76 | } 77 | 78 | func serveConsumer(consumer *kafkakit.KafkaConsumer, svc pb.VideoStreamServer, logger *logkit.Logger) runkit.GracefulRunFunc { 79 | handlers := pb.NewVideoStreamHandlers(svc, logkit.NewSaramaLogger(logger)) 80 | 81 | return func(ctx context.Context) error { 82 | if err := consumer.Consume(ctx, handlers.HandleVideoCreatedHandler); err != nil { 83 | return err 84 | } 85 | 86 | return nil 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | 2 | name: main workflow 3 | 4 | on: push 5 | 6 | jobs: 7 | lint: 8 | runs-on: ubuntu-20.04 9 | steps: 10 | - name: checkout 11 | uses: actions/checkout@v3 12 | 13 | - name: setup go 14 | uses: actions/setup-go@v5 15 | with: 16 | go-version: '1.21' 17 | 18 | - name: golangci-lint 19 | uses: golangci/golangci-lint-action@v4 20 | with: 21 | version: v1.55.2 22 | 23 | # test should be run inside a container to start helper services (ex: mongo) 24 | test: 25 | runs-on: ubuntu-22.04 26 | container: golang:1.21 27 | services: 28 | mongo: 29 | image: mongo:5 30 | postgres: 31 | image: postgres:14-alpine 32 | env: 33 | POSTGRES_HOST_AUTH_METHOD: trust 34 | redis: 35 | image: redis:6.2-alpine 36 | env: 37 | MONGO_URL: mongodb://mongo:27017/ 38 | MONGO_DATABASE: nthu_distributes_system 39 | POSTGRES_URL: postgres://postgres@postgres:5432/postgres?sslmode=disable 40 | REDIS_ADDR: redis:6379 41 | # KAFKA_ADDR: 42 | MINIO_ENDPOINT: play.min.io 43 | MINIO_BUCKET: videos 44 | MINIO_USERNAME: Q3AM3UQ867SPQQA43P2F 45 | MINIO_PASSWORD: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG 46 | steps: 47 | - name: checkout 48 | uses: actions/checkout@v3 49 | 50 | - name: cache 51 | uses: actions/cache@v3 52 | with: 53 | path: | 54 | ~/go/pkg/mod 55 | ~/.cache/go-build 56 | key: ${{ runner.os }}-gocache-${{ hashFiles('**/go.sum') }} 57 | restore-keys: | 58 | ${{ runner.os }}-gocache- 59 | 60 | - name: test 61 | run: go test -v -race ./... 62 | 63 | # build should be run outside container to build docker image 64 | build: 65 | runs-on: ubuntu-20.04 66 | needs: 67 | - lint 68 | - test 69 | steps: 70 | - name: checkout 71 | uses: actions/checkout@v3 72 | 73 | - name: setup go 74 | uses: actions/setup-go@v3 75 | with: 76 | go-version: 1.17 77 | 78 | - name: cache 79 | uses: actions/cache@v3 80 | with: 81 | path: | 82 | ~/go/pkg/mod 83 | ~/.cache/go-build 84 | key: ${{ runner.os }}-gocache-${{ hashFiles('**/go.sum') }} 85 | restore-keys: | 86 | ${{ runner.os }}-gocache- 87 | 88 | - name: build 89 | run: make build 90 | 91 | - name: login ghcr 92 | uses: docker/login-action@v1 93 | with: 94 | registry: ghcr.io 95 | username: ${{ github.actor }} 96 | password: ${{ secrets.GITHUB_TOKEN }} 97 | 98 | - name: setup docker buildx 99 | uses: docker/setup-buildx-action@v1 100 | 101 | - name: set image name 102 | run: | 103 | echo "IMAGE_NAME=ghcr.io/$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> ${{ github.env }} 104 | 105 | - name: docker build and push 106 | uses: docker/build-push-action@v2 107 | with: 108 | context: . 109 | push: true 110 | tags: ${{ env.IMAGE_NAME }}:latest,${{ env.IMAGE_NAME }}:${{ github.sha }} 111 | -------------------------------------------------------------------------------- /cmd/comment/gateway.go: -------------------------------------------------------------------------------- 1 | package comment 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb" 11 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/grpckit" 12 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 13 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/runkit" 14 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 15 | flags "github.com/jessevdk/go-flags" 16 | "github.com/spf13/cobra" 17 | "go.uber.org/zap" 18 | "google.golang.org/grpc" 19 | ) 20 | 21 | func newGatewayCommand() *cobra.Command { 22 | return &cobra.Command{ 23 | Use: "gateway", 24 | Short: "starts comment gateway server", 25 | RunE: runGateway, 26 | } 27 | } 28 | 29 | type GatewayArgs struct { 30 | HTTPAddr string `long:"http_addr" env:"HTTP_ADDR" default:":8080"` 31 | grpckit.GrpcClientConnConfig `group:"grpc" namespace:"grpc" env-namespace:"GRPC"` 32 | runkit.GracefulConfig `group:"graceful" namespace:"graceful" env-namespace:"GRACEFUL"` 33 | logkit.LoggerConfig `group:"logger" namespace:"logger" env-namespace:"LOGGER"` 34 | } 35 | 36 | func runGateway(_ *cobra.Command, _ []string) error { 37 | ctx := context.Background() 38 | 39 | var args GatewayArgs 40 | if _, err := flags.NewParser(&args, flags.Default).Parse(); err != nil { 41 | log.Fatal("failed to parse flag", err.Error()) 42 | } 43 | 44 | logger := logkit.NewLogger(&args.LoggerConfig) 45 | defer func() { 46 | _ = logger.Sync() 47 | }() 48 | 49 | ctx = logger.WithContext(ctx) 50 | 51 | logger.Info("listen to HTTP addr", zap.String("http_addr", args.HTTPAddr)) 52 | lis, err := net.Listen("tcp", args.HTTPAddr) 53 | if err != nil { 54 | logger.Fatal("failed to listen HTTP addr", zap.Error(err)) 55 | } 56 | defer func() { 57 | if err := lis.Close(); err != nil { 58 | logger.Fatal("failed to close HTTP listener", zap.Error(err)) 59 | } 60 | }() 61 | 62 | conn := grpckit.NewGrpcClientConn(ctx, &args.GrpcClientConnConfig) 63 | defer func() { 64 | if err := conn.Close(); err != nil { 65 | logger.Fatal("failed to close gRPC client connection", zap.Error(err)) 66 | } 67 | }() 68 | 69 | return runkit.GracefulRun(serveHTTP(lis, conn.ClientConn, logger), &args.GracefulConfig) 70 | } 71 | 72 | func serveHTTP(lis net.Listener, conn *grpc.ClientConn, logger *logkit.Logger) runkit.GracefulRunFunc { 73 | mux := runtime.NewServeMux() 74 | 75 | httpServer := &http.Server{ 76 | Handler: mux, 77 | ReadHeaderTimeout: 10 * time.Second, 78 | } 79 | 80 | return func(ctx context.Context) error { 81 | if err := pb.RegisterCommentHandler(ctx, mux, conn); err != nil { 82 | logger.Fatal("failed to register handler to HTTP server", zap.Error(err)) 83 | } 84 | 85 | go func() { 86 | if err := httpServer.Serve(lis); err != nil { 87 | logger.Fatal("failed to run HTTP server", zap.Error(err)) 88 | } 89 | }() 90 | 91 | <-ctx.Done() 92 | 93 | if err := httpServer.Shutdown(context.Background()); err != nil { 94 | logger.Fatal("failed to shutdown HTTP server", zap.Error(err)) 95 | } 96 | 97 | return nil 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /k8s/nfs-server-provisioner/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for nfs-provisioner. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | # imagePullSecrets: 8 | 9 | image: 10 | repository: k8s.gcr.io/sig-storage/nfs-provisioner 11 | tag: v3.0.0 12 | # digest: 13 | pullPolicy: IfNotPresent 14 | 15 | # For a list of available arguments 16 | # Please see https://github.com/kubernetes-incubator/external-storage/blob/HEAD/nfs/docs/deployment.md#arguments 17 | extraArgs: {} 18 | # device-based-fsids: false 19 | # grace-period: 0 20 | 21 | service: 22 | type: ClusterIP 23 | 24 | nfsPort: 2049 25 | nlockmgrPort: 32803 26 | mountdPort: 20048 27 | rquotadPort: 875 28 | rpcbindPort: 111 29 | statdPort: 662 30 | # nfsNodePort: 31 | # nlockmgrNodePort: 32 | # mountdNodePort: 33 | # rquotadNodePort: 34 | # rpcbindNodePort: 35 | # statdNodePort: 36 | # clusterIP: 37 | 38 | externalIPs: [] 39 | 40 | persistence: 41 | enabled: true 42 | 43 | ## Persistent Volume Storage Class 44 | ## If defined, storageClassName: 45 | ## If set to "-", storageClassName: "", which disables dynamic provisioning 46 | ## If undefined (the default) or set to null, no storageClassName spec is 47 | ## set, choosing the default provisioner. (gp2 on AWS, standard on 48 | ## GKE, AWS & OpenStack) 49 | ## 50 | storageClass: local-storage 51 | 52 | accessMode: ReadWriteOnce 53 | size: 50Gi 54 | 55 | ## For creating the StorageClass automatically: 56 | storageClass: 57 | create: true 58 | 59 | ## Set a provisioner name. If unset, a name will be generated. 60 | # provisionerName: 61 | 62 | ## Set StorageClass as the default StorageClass 63 | ## Ignored if storageClass.create is false 64 | defaultClass: true 65 | 66 | ## Set a StorageClass name 67 | ## Ignored if storageClass.create is false 68 | name: nfs 69 | 70 | # set to null to prevent expansion 71 | allowVolumeExpansion: null 72 | ## StorageClass parameters 73 | parameters: {} 74 | 75 | # https://www.mongodb.com/docs/manual/administration/production-notes/#remote-filesystems--nfs- 76 | mountOptions: 77 | - bg 78 | - hard 79 | - nolock 80 | - noatime 81 | - nointr 82 | 83 | ## ReclaimPolicy field of the class, which can be either Delete or Retain 84 | reclaimPolicy: Retain 85 | 86 | ## For RBAC support: 87 | rbac: 88 | create: true 89 | 90 | ## Ignored if rbac.create is true 91 | ## 92 | serviceAccountName: default 93 | 94 | ## For creating the PriorityClass automatically: 95 | priorityClass: 96 | ## Enable creation of a PriorityClass resource for this nfs-server-provisioner instance 97 | create: false 98 | 99 | ## Set a PriorityClass name to override the default name 100 | name: "" 101 | 102 | ## PriorityClass value. The higher the value, the higher the scheduling priority 103 | value: 5 104 | 105 | resources: 106 | # limits: 107 | # cpu: 100m 108 | # memory: 128Mi 109 | # requests: 110 | # cpu: 100m 111 | # memory: 128Mi 112 | 113 | nodeSelector: 114 | kubernetes.io/hostname: k8s-master 115 | 116 | tolerations: 117 | - key: node-role.kubernetes.io/master 118 | operator: Exists 119 | effect: NoSchedule 120 | 121 | affinity: {} 122 | -------------------------------------------------------------------------------- /.github/workflows/deployment.yml: -------------------------------------------------------------------------------- 1 | name: deployment workflow 2 | 3 | on: 4 | workflow_run: 5 | workflows: 6 | - main workflow 7 | types: 8 | - completed 9 | branches: 10 | - master 11 | 12 | jobs: 13 | setup: 14 | runs-on: ubuntu-20.04 15 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 16 | outputs: 17 | image-name: ${{ steps.output.outputs.image-name }} 18 | steps: 19 | - name: set image name 20 | run: 21 | echo "IMAGE_NAME=ghcr.io/$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> ${{ github.env }} 22 | - name: set output 23 | id: output 24 | run: echo "::set-output name=image-name::${{ env.IMAGE_NAME }}:${{ github.sha }}" 25 | 26 | deployment-comment: 27 | runs-on: ubuntu-20.04 28 | needs: 29 | - setup 30 | environment: production-comment 31 | steps: 32 | - name: checkout 33 | uses: actions/checkout@v3 34 | 35 | - name: setup kubectl 36 | uses: justin0u0/setup-kubectl@v1 37 | with: 38 | kubectl-version: stable 39 | cluster-certificate-authority-data: ${{ secrets.KUBERNETES_CLUSTER_CLIENT_CERTIFICATE_AUTHORITY_DATA }} 40 | cluster-server: ${{ secrets.KUBERNETES_CLUSTER_SERVER }} 41 | credentials-token: ${{ secrets.KUBERNETES_CREDENTIALS_TOKEN }} 42 | 43 | - name: deploy comment-migration 44 | run: kubectl set image cronjob/comment-migration comment-migration=${{ needs.setup.outputs.image-name }} 45 | 46 | - name: run migration job 47 | uses: ./.github/actions/run-migration 48 | with: 49 | migration-cronjob-name: comment-migration 50 | migration-job-name: comment-migration-${{ github.run_id }} 51 | 52 | - name: deploy comment-api 53 | run: kubectl set image deploy/comment-api comment-api=${{ needs.setup.outputs.image-name }} 54 | 55 | - name: deploy comment-gateway 56 | run: kubectl set image deploy/comment-gateway comment-gateway=${{ needs.setup.outputs.image-name }} 57 | 58 | - name: wait comment-api 59 | run: kubectl rollout status -w deploy/comment-api 60 | 61 | - name: wait comment-gateway 62 | run: kubectl rollout status -w deploy/comment-gateway 63 | 64 | deployment-video: 65 | runs-on: ubuntu-20.04 66 | needs: 67 | - setup 68 | environment: production-video 69 | steps: 70 | - name: checkout 71 | uses: actions/checkout@v3 72 | 73 | - name: setup kubectl 74 | uses: justin0u0/setup-kubectl@v1 75 | with: 76 | kubectl-version: stable 77 | cluster-certificate-authority-data: ${{ secrets.KUBERNETES_CLUSTER_CLIENT_CERTIFICATE_AUTHORITY_DATA }} 78 | cluster-server: ${{ secrets.KUBERNETES_CLUSTER_SERVER }} 79 | credentials-token: ${{ secrets.KUBERNETES_CREDENTIALS_TOKEN }} 80 | 81 | - name: deploy video-api 82 | run: kubectl set image deploy/video-api video-api=${{ needs.setup.outputs.image-name }} 83 | 84 | - name: deploy video-gateway 85 | run: kubectl set image deploy/video-gateway video-gateway=${{ needs.setup.outputs.image-name }} 86 | 87 | - name: deploy video-stream 88 | run: kubectl set image deploy/video-stream video-stream=${{ needs.setup.outputs.image-name }} 89 | 90 | - name: wait video-api 91 | run: kubectl rollout status -w deploy/video-api 92 | 93 | - name: wait video-gateway 94 | run: kubectl rollout status -w deploy/video-gateway 95 | 96 | - name: wait video-stream 97 | run: kubectl rollout status -w deploy/video-stream 98 | -------------------------------------------------------------------------------- /modules/comment/service/service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/dao" 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb" 9 | videopb "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 10 | "github.com/google/uuid" 11 | ) 12 | 13 | type service struct { 14 | pb.UnimplementedCommentServer 15 | 16 | commentDAO dao.CommentDAO 17 | videoClient videopb.VideoClient 18 | } 19 | 20 | func NewService(commentDAO dao.CommentDAO, videoClient videopb.VideoClient) *service { 21 | return &service{ 22 | commentDAO: commentDAO, 23 | videoClient: videoClient, 24 | } 25 | } 26 | 27 | func (s *service) Healthz(ctx context.Context, req *pb.HealthzRequest) (*pb.HealthzResponse, error) { 28 | return &pb.HealthzResponse{Status: "ok"}, nil 29 | } 30 | 31 | func (s *service) ListComment(ctx context.Context, req *pb.ListCommentRequest) (*pb.ListCommentResponse, error) { 32 | comments, err := s.commentDAO.ListByVideoID(ctx, req.GetVideoId(), int(req.GetLimit()), int(req.GetOffset())) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | pbComments := make([]*pb.CommentInfo, 0, len(comments)) 38 | for _, comment := range comments { 39 | pbComments = append(pbComments, comment.ToProto()) 40 | } 41 | 42 | return &pb.ListCommentResponse{Comments: pbComments}, nil 43 | } 44 | 45 | func (s *service) CreateComment(ctx context.Context, req *pb.CreateCommentRequest) (*pb.CreateCommentResponse, error) { 46 | if _, err := s.videoClient.GetVideo(ctx, &videopb.GetVideoRequest{ 47 | Id: req.GetVideoId(), 48 | }); err != nil { 49 | return nil, err 50 | } 51 | 52 | comment := &dao.Comment{ 53 | VideoID: req.GetVideoId(), 54 | Content: req.GetContent(), 55 | } 56 | 57 | commentID, err := s.commentDAO.Create(ctx, comment) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | return &pb.CreateCommentResponse{Id: commentID.String()}, nil 63 | } 64 | 65 | func (s *service) UpdateComment(ctx context.Context, req *pb.UpdateCommentRequest) (*pb.UpdateCommentResponse, error) { 66 | commentID, err := uuid.Parse(req.GetId()) 67 | if err != nil { 68 | return nil, ErrInvalidUUID 69 | } 70 | 71 | comment := &dao.Comment{ 72 | ID: commentID, 73 | Content: req.GetContent(), 74 | } 75 | if err := s.commentDAO.Update(ctx, comment); err != nil { 76 | if errors.Is(err, dao.ErrCommentNotFound) { 77 | return nil, ErrCommentNotFound 78 | } 79 | 80 | return nil, err 81 | } 82 | 83 | return &pb.UpdateCommentResponse{ 84 | Comment: comment.ToProto(), 85 | }, nil 86 | } 87 | 88 | func (s *service) DeleteComment(ctx context.Context, req *pb.DeleteCommentRequest) (*pb.DeleteCommentResponse, error) { 89 | commentID, err := uuid.Parse(req.GetId()) 90 | if err != nil { 91 | return nil, ErrInvalidUUID 92 | } 93 | 94 | if err := s.commentDAO.Delete(ctx, commentID); err != nil { 95 | if errors.Is(err, dao.ErrCommentNotFound) { 96 | return nil, ErrCommentNotFound 97 | } 98 | 99 | return nil, err 100 | } 101 | 102 | return &pb.DeleteCommentResponse{}, nil 103 | } 104 | 105 | func (s *service) DeleteCommentByVideoID(ctx context.Context, req *pb.DeleteCommentByVideoIDRequest) (*pb.DeleteCommentByVideoIDResponse, error) { 106 | if err := s.commentDAO.DeleteByVideoID(ctx, req.GetVideoId()); err != nil { 107 | return nil, err 108 | } 109 | 110 | return &pb.DeleteCommentByVideoIDResponse{}, nil 111 | } 112 | -------------------------------------------------------------------------------- /cmd/video/gateway.go: -------------------------------------------------------------------------------- 1 | package video 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/gateway" 11 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 12 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/grpckit" 13 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 14 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/runkit" 15 | "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 16 | flags "github.com/jessevdk/go-flags" 17 | "github.com/spf13/cobra" 18 | "go.uber.org/zap" 19 | "google.golang.org/grpc" 20 | ) 21 | 22 | func newGatewayCommand() *cobra.Command { 23 | return &cobra.Command{ 24 | Use: "gateway", 25 | Short: "starts video gateway server", 26 | RunE: runGateway, 27 | } 28 | } 29 | 30 | type GatewayArgs struct { 31 | HTTPAddr string `long:"http_addr" env:"HTTP_ADDR" default:":8080"` 32 | GRPCAddr string `long:"grpc_addr" env:"GRPC_ADDR" default:":8081"` 33 | grpckit.GrpcClientConnConfig `group:"grpc" namespace:"grpc" env-namespace:"GRPC"` 34 | runkit.GracefulConfig `group:"graceful" namespace:"graceful" env-namespace:"GRACEFUL"` 35 | logkit.LoggerConfig `group:"logger" namespace:"logger" env-namespace:"LOGGER"` 36 | } 37 | 38 | func runGateway(_ *cobra.Command, _ []string) error { 39 | ctx := context.Background() 40 | 41 | var args GatewayArgs 42 | if _, err := flags.NewParser(&args, flags.Default).Parse(); err != nil { 43 | log.Fatal("failed to parse flag", err.Error()) 44 | } 45 | 46 | logger := logkit.NewLogger(&args.LoggerConfig) 47 | defer func() { 48 | _ = logger.Sync() 49 | }() 50 | 51 | ctx = logger.WithContext(ctx) 52 | 53 | logger.Info("listen to HTTP addr", zap.String("http_addr", args.HTTPAddr)) 54 | lis, err := net.Listen("tcp", args.HTTPAddr) 55 | if err != nil { 56 | logger.Fatal("failed to listen HTTP addr", zap.Error(err)) 57 | } 58 | defer func() { 59 | if err := lis.Close(); err != nil { 60 | logger.Fatal("failed to close HTTP listener", zap.Error(err)) 61 | } 62 | }() 63 | 64 | conn := grpckit.NewGrpcClientConn(ctx, &args.GrpcClientConnConfig) 65 | defer func() { 66 | if err := conn.Close(); err != nil { 67 | logger.Fatal("failed to close gRPC client connection", zap.Error(err)) 68 | } 69 | }() 70 | 71 | return runkit.GracefulRun(serveHTTP(lis, conn.ClientConn, logger), &args.GracefulConfig) 72 | } 73 | 74 | func serveHTTP(lis net.Listener, conn *grpc.ClientConn, logger *logkit.Logger) runkit.GracefulRunFunc { 75 | mux := runtime.NewServeMux() 76 | 77 | // register additional routes 78 | handler := gateway.NewHandler(pb.NewVideoClient(conn), logger) 79 | if err := mux.HandlePath("POST", "/v1/videos", handler.HandleUploadVideo); err != nil { 80 | logger.Fatal("failed to register additional routes", zap.Error(err)) 81 | } 82 | 83 | httpServer := &http.Server{ 84 | Handler: mux, 85 | ReadHeaderTimeout: 10 * time.Second, 86 | } 87 | 88 | return func(ctx context.Context) error { 89 | if err := pb.RegisterVideoHandler(ctx, mux, conn); err != nil { 90 | logger.Fatal("failed to register handler to HTTP server", zap.Error(err)) 91 | } 92 | 93 | go func() { 94 | if err := httpServer.Serve(lis); err != nil { 95 | logger.Fatal("failed to run HTTP server", zap.Error(err)) 96 | } 97 | }() 98 | 99 | <-ctx.Done() 100 | 101 | if err := httpServer.Shutdown(context.Background()); err != nil { 102 | logger.Fatal("failed to shutdown HTTP server", zap.Error(err)) 103 | } 104 | 105 | return nil 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | dupl: 3 | threshold: 100 4 | funlen: 5 | lines: 150 6 | statements: 75 7 | goconst: 8 | min-len: 2 9 | min-occurrences: 3 10 | gocritic: 11 | enabled-tags: 12 | - diagnostic 13 | - experimental 14 | - opinionated 15 | - performance 16 | - style 17 | disabled-checks: 18 | - dupImport # https://github.com/go-critic/go-critic/issues/845 19 | - ifElseChain 20 | - octalLiteral 21 | - whyNoLint 22 | - wrapperFunc 23 | - paramTypeCombine 24 | gocyclo: 25 | min-complexity: 15 26 | goimports: 27 | local-prefixes: '' 28 | gomnd: 29 | # TODO(ldez) must be rewritten after the v1.44.0 release. 30 | settings: 31 | mnd: 32 | # don't include the "operation" and "assign" 33 | checks: 34 | - case 35 | - condition 36 | - return 37 | ignored-numbers: 0,1,2,3 38 | ignored-functions: strings.SplitN 39 | 40 | govet: 41 | check-shadowing: true 42 | settings: 43 | printf: 44 | funcs: 45 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof 46 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf 47 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf 48 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf 49 | lll: 50 | line-length: 200 51 | misspell: 52 | locale: US 53 | nolintlint: 54 | allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) 55 | allow-unused: false # report any unused nolint directives 56 | require-explanation: false # don't require an explanation for nolint directives 57 | require-specific: false # don't require nolint directives to be specific about which linter is being skipped 58 | 59 | linters: 60 | disable-all: true 61 | enable: 62 | - bodyclose 63 | - deadcode 64 | - dogsled 65 | - dupl 66 | - errcheck 67 | - errname 68 | - errorlint 69 | - exportloopref 70 | - funlen 71 | - gochecknoinits 72 | - goconst 73 | - gocritic 74 | - gocyclo 75 | - gofmt 76 | - goimports 77 | - gomnd 78 | - goprintffuncname 79 | - gosec 80 | - gosimple 81 | - govet 82 | - ineffassign 83 | - lll 84 | - misspell 85 | - nakedret 86 | - noctx 87 | - nolintlint 88 | - staticcheck 89 | - structcheck 90 | - stylecheck 91 | - typecheck 92 | - unconvert 93 | - unparam 94 | - unused 95 | - varcheck 96 | - whitespace 97 | 98 | # don't enable: 99 | # - asciicheck 100 | # - depguard 101 | # - scopelint 102 | # - gochecknoglobals 103 | # - gocognit 104 | # - godot 105 | # - godox 106 | # - goerr113 107 | # - interfacer 108 | # - maligned 109 | # - nestif 110 | # - prealloc 111 | # - testpackage 112 | # - revive 113 | # - wsl 114 | 115 | issues: 116 | # Excluding configuration per-path, per-linter, per-text and per-source 117 | exclude-rules: 118 | - path: _test\.go 119 | linters: 120 | - gomnd 121 | 122 | - path: pkg/golinters/errcheck.go 123 | text: "SA1019: errCfg.Exclude is deprecated: use ExcludeFunctions instead" 124 | - path: pkg/commands/run.go 125 | text: "SA1019: lsc.Errcheck.Exclude is deprecated: use ExcludeFunctions instead" 126 | - path: pkg/commands/run.go 127 | text: "SA1019: e.cfg.Run.Deadline is deprecated: Deadline exists for historical compatibility and should not be used." 128 | 129 | run: 130 | timeout: 5m 131 | skip-files: 132 | - modules/.*/.*\.pb\.go 133 | - mock/.*/mock\.go 134 | -------------------------------------------------------------------------------- /modules/video/stream/stream_test.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strconv" 7 | "testing" 8 | 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao" 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/mock/daomock" 11 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 12 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit/mock/kafkamock" 13 | "github.com/golang/mock/gomock" 14 | "github.com/justin0u0/protoc-gen-grpc-sarama/pkg/saramakit" 15 | . "github.com/onsi/ginkgo/v2" 16 | . "github.com/onsi/gomega" 17 | "go.mongodb.org/mongo-driver/bson/primitive" 18 | "google.golang.org/protobuf/types/known/emptypb" 19 | ) 20 | 21 | func TestStream(t *testing.T) { 22 | RegisterFailHandler(Fail) 23 | RunSpecs(t, "Test Stream") 24 | } 25 | 26 | var ( 27 | errSendMessagesUnknown = errors.New("unknown send messages error") 28 | ) 29 | 30 | var _ = Describe("Stream", func() { 31 | var ( 32 | ctx context.Context 33 | controller *gomock.Controller 34 | videoDAO *daomock.MockVideoDAO 35 | producer *kafkamock.MockProducer 36 | stream *stream 37 | ) 38 | 39 | BeforeEach(func() { 40 | ctx = context.Background() 41 | controller = gomock.NewController(GinkgoT()) 42 | videoDAO = daomock.NewMockVideoDAO(controller) 43 | producer = kafkamock.NewMockProducer(controller) 44 | stream = NewStream(videoDAO, producer) 45 | }) 46 | 47 | AfterEach(func() { 48 | controller.Finish() 49 | }) 50 | 51 | Describe("HandleVideoCreated", func() { 52 | var ( 53 | id primitive.ObjectID 54 | url string 55 | resp *emptypb.Empty 56 | err error 57 | scale int32 58 | ) 59 | 60 | BeforeEach(func() { 61 | id = primitive.NewObjectID() 62 | url = "https://www.test.com" 63 | }) 64 | 65 | JustBeforeEach(func() { 66 | resp, err = stream.HandleVideoCreated(ctx, &pb.HandleVideoCreatedRequest{ 67 | Id: id.Hex(), 68 | Url: url, 69 | Scale: scale, 70 | }) 71 | }) 72 | 73 | Context("scale is not presenting", func() { 74 | BeforeEach(func() { scale = 0 }) 75 | 76 | When("producer send messages error", func() { 77 | BeforeEach(func() { 78 | producer.EXPECT().SendMessages(gomock.Any()).Return(errSendMessagesUnknown) 79 | }) 80 | 81 | It("returns the error", func() { 82 | Expect(resp).To(BeNil()) 83 | Expect(err).To(Equal(&saramakit.HandlerError{Retry: true, Err: errSendMessagesUnknown})) 84 | }) 85 | }) 86 | 87 | When("success", func() { 88 | BeforeEach(func() { 89 | producer.EXPECT().SendMessages(gomock.Any()).Times(4).Return(nil) 90 | }) 91 | 92 | It("returns with no error", func() { 93 | Expect(resp).To(Equal(&emptypb.Empty{})) 94 | Expect(err).NotTo(HaveOccurred()) 95 | }) 96 | }) 97 | }) 98 | 99 | Context("scale is presenting", func() { 100 | BeforeEach(func() { scale = 720 }) 101 | 102 | When("video not found", func() { 103 | BeforeEach(func() { 104 | videoDAO.EXPECT().UpdateVariant(ctx, id, strconv.Itoa(int(scale)), url).Return(dao.ErrVideoNotFound) 105 | }) 106 | 107 | It("returns with no error", func() { 108 | Expect(resp).To(BeNil()) 109 | Expect(err).To(Equal(&saramakit.HandlerError{Retry: true, Err: dao.ErrVideoNotFound})) 110 | }) 111 | }) 112 | 113 | When("success", func() { 114 | BeforeEach(func() { 115 | videoDAO.EXPECT().UpdateVariant(ctx, id, strconv.Itoa(int(scale)), url).Return(nil) 116 | }) 117 | 118 | It("returns with no error", func() { 119 | Expect(resp).To(Equal(&emptypb.Empty{})) 120 | Expect(err).NotTo(HaveOccurred()) 121 | }) 122 | }) 123 | }) 124 | }) 125 | }) 126 | -------------------------------------------------------------------------------- /pkg/storagekit/minio.go: -------------------------------------------------------------------------------- 1 | package storagekit 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 9 | "github.com/minio/minio-go/v7" 10 | "github.com/minio/minio-go/v7/pkg/credentials" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | type MinIOConfig struct { 15 | Endpoint string `long:"endpoint" env:"ENDPOINT" description:"the endpoint of MinIO server" required:"true"` 16 | Bucket string `long:"bucket" env:"BUCKET" description:"the bucket name" required:"true"` 17 | Username string `long:"username" env:"USERNAME" description:"the access key id (username) to the MinIO server" required:"true"` 18 | Password string `long:"password" env:"PASSWORD" description:"the secret access key (password) to the MinIO server" required:"true"` 19 | Insecure bool `long:"insecure" env:"INSECURE" description:"disable HTTPS or not"` 20 | Policy string `long:"policy" env:"POLICY" description:"the bucket policy" default:"public"` 21 | } 22 | 23 | type MinIOClient struct { 24 | *minio.Client 25 | bucketName string 26 | } 27 | 28 | var _ Storage = (*MinIOClient)(nil) 29 | 30 | func (c *MinIOClient) Endpoint() string { 31 | return c.Client.EndpointURL().Path 32 | } 33 | 34 | func (c *MinIOClient) Bucket() string { 35 | return c.bucketName 36 | } 37 | 38 | func (c *MinIOClient) PutObject(ctx context.Context, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) error { 39 | if _, err := c.Client.PutObject(ctx, c.bucketName, objectName, reader, objectSize, minio.PutObjectOptions{ 40 | ContentType: opts.ContentType, 41 | }); err != nil { 42 | return err 43 | } 44 | 45 | return nil 46 | } 47 | 48 | func NewMinIOClient(ctx context.Context, conf *MinIOConfig) *MinIOClient { 49 | logger := logkit.FromContext(ctx). 50 | With(zap.String("endpoint", conf.Endpoint)). 51 | With(zap.String("bucket", conf.Bucket)) 52 | 53 | client, err := minio.New(conf.Endpoint, &minio.Options{ 54 | Creds: credentials.NewStaticV4(conf.Username, conf.Password, ""), 55 | Secure: !conf.Insecure, 56 | }) 57 | if err != nil { 58 | logger.Fatal("failed to create MinIO client", zap.Error(err)) 59 | } 60 | 61 | if conf.Bucket != "" { 62 | ok, err := client.BucketExists(ctx, conf.Bucket) 63 | if err != nil { 64 | logger.Fatal("failed to check bucket existence", zap.Error(err)) 65 | } 66 | 67 | if !ok { 68 | if err := client.MakeBucket(ctx, conf.Bucket, minio.MakeBucketOptions{}); err != nil { 69 | logger.Fatal("failed to create bucket", zap.Error(err)) 70 | } 71 | 72 | if policy := generatePolicy(conf.Bucket, conf.Policy); policy != "" { 73 | if err := client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil { 74 | logger.Fatal("failed to set bucket policy") 75 | } 76 | } 77 | } 78 | } 79 | 80 | logger.Info("create MinIO client successfully") 81 | 82 | return &MinIOClient{ 83 | Client: client, 84 | bucketName: conf.Bucket, 85 | } 86 | } 87 | 88 | func generatePolicy(bucketName string, policy string) string { 89 | switch policy { 90 | case "public": 91 | return fmt.Sprintf(` 92 | { 93 | "Version":"2012-10-17", 94 | "Statement": [ 95 | { 96 | "Effect": "Allow", 97 | "Principal": {"AWS": ["*"]}, 98 | "Action": ["s3:GetBucketLocation", "s3:ListBucket", "s3:ListBucketMultipartUploads"], 99 | "Resource": ["arn:aws:s3:::%s"] 100 | }, 101 | { 102 | "Effect": "Allow", 103 | "Principal": {"AWS": ["*"]}, 104 | "Action": ["s3:AbortMultipartUpload", "s3:DeleteObject", "s3:GetObject", "s3:ListMultipartUploadParts", "s3:PutObject"], 105 | "Resource":["arn:aws:s3:::%s/*"] 106 | } 107 | ] 108 | } 109 | `, bucketName, bucketName) 110 | case "private": 111 | return ` 112 | { 113 | "Version": "2012-10-17", 114 | "Statement": [] 115 | } 116 | ` 117 | } 118 | 119 | return "" 120 | } 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NTHU-Distributed-System 2 | 3 | The repository includes microservices for the NTHU Distributed System course lab. The goal of this project is to **introduce a production, realworld microservices backend mono-repo architecture** for teaching purpose. 4 | 5 | Before going through the following parts, make sure your Docker is running since we are generating/testing/building code inside a Docker container to prevent dependencies from conflicting/missing on your host machine. 6 | 7 | ## Features 8 | 9 | The video service serves APIs that accept uploading a video, listing videos, getting a video and deleting a video. 10 | 11 | The comment service serves APIs that accept creating a comment under a video, listing comments under a video, updating a comment and deleting a comment. 12 | 13 | Many popular tools that are used in the realworld applications are adopted in this project too. For example: 14 | 15 | - Use [gRPC](https://grpc.io/) for defining APIs and synchronous communications between microservices. See the [comment module protocol buffer definition](modules/comment/pb/rpc.proto) for example. 16 | - Use [gRPC-gateway](https://github.com/grpc-ecosystem/grpc-gateway) to generate a HTTP gateway server that serves RESTful APIs for the gRPC APIs. The purpose of having a HTTP gateway server is that realworld web applications typically do not use gRPC for communication. 17 | - Use [PostgreSQL](https://www.postgresql.org/) in the comment service and [MongoDB](https://www.mongodb.com/) in the video service as the DBMS. The microservices architecture allows services to use different databases. 18 | - Use [Redis](https://redis.io/) for cache. Realworld backend services use cache to speed up application performance. Redis is one of the most popular caching system and it is easy to learn. 19 | - Use [Kafka](https://kafka.apache.org/) for asynchronous communications between microservices. Realworld backend services typically rely on message queue systems to accomplish asynchronous communications between microservices. 20 | - Use [MinIO](https://min.io/) storing files. Realworld backend services typically store user uploaded files in cloud storage like Google Cloud Storage or AWS S3. MinIO is a AWS S3 compatible storage system that allows the project to upload files without having a real cloud environment. 21 | - Use [OpenTelemetry](https://opentelemetry.io/) to collect telemetry data. 22 | - Use [Prometheus](https://prometheus.io/) as the metrics backend. 23 | - Use [Kubernetes](https://kubernetes.io/) as the container management system for deployment. Deployment yaml files are in the [k8s](k8s/) directory. 24 | 25 | Share libraries are wrapped so that they can be extended easily. For example, logs, traces, and metrics can be easily added to the custom share libraries. Share libraries are in the [`pkg`](./pkg/) directory. 26 | 27 | ## Code Generation 28 | 29 | Some modules use gRPC for communication or use the `mockgen` library for unit testing. 30 | 31 | So there is a need to generate code manually when the code changed. 32 | 33 | For generating code for all modules, run `make dc.generate`. 34 | 35 | For generating code for a single module, run `make dc.{module}.generate`. For example: `make dc.video.generate`. 36 | 37 | ## Unit Testing 38 | 39 | We implements unit testing on the DAO and service layers using the [ginkgo](https://onsi.github.io/ginkgo/) framework. 40 | 41 | To run unit testing for all modules, run `make dc.test`. 42 | 43 | To run unit testing for a single module, run `make dc.{module}.test`. For example: `make dc.video.test`. 44 | 45 | ## Style Check 46 | 47 | We use [golangci-lint](https://github.com/golangci/golangci-lint) for linting. 48 | 49 | To run linting for all modules, run `make dc.lint`. 50 | 51 | To run linting for a single module, run `make dc.{module}.lint`. For example: `make dc.video.lint`. 52 | 53 | ## Build Image 54 | 55 | To build docker image, run `make dc.image`. 56 | 57 | ## CI/CD 58 | 59 | The CI/CD runs in [Github Actions](https://github.com/features/actions). See the [CI workflow spec](.github/workflows/main.yml) and the [CD workflow spec](.github/workflows/deployment.yml) for more details. 60 | -------------------------------------------------------------------------------- /modules/video/gateway/handler.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "errors" 7 | "io" 8 | "net/http" 9 | 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 11 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 12 | "go.uber.org/zap" 13 | "google.golang.org/protobuf/encoding/protojson" 14 | "google.golang.org/protobuf/proto" 15 | ) 16 | 17 | type Handler interface { 18 | HandleUploadVideo(w http.ResponseWriter, r *http.Request, params map[string]string) 19 | } 20 | 21 | type handler struct { 22 | client pb.VideoClient 23 | logger *logkit.Logger 24 | } 25 | 26 | func NewHandler(client pb.VideoClient, logger *logkit.Logger) *handler { 27 | return &handler{ 28 | client: client, 29 | logger: logger, 30 | } 31 | } 32 | 33 | func (h *handler) HandleUploadVideo(w http.ResponseWriter, req *http.Request, params map[string]string) { 34 | if err := req.ParseForm(); err != nil { 35 | h.encodeJSONResponse(w, NewResponseError(http.StatusBadRequest, "failed to parse form", err)) 36 | return 37 | } 38 | 39 | f, header, err := req.FormFile("file") 40 | if err != nil { 41 | h.encodeJSONResponse(w, NewResponseError(http.StatusBadRequest, "failed to get file", err)) 42 | return 43 | } 44 | defer f.Close() 45 | 46 | stream, err := h.client.UploadVideo(req.Context()) 47 | if err != nil { 48 | h.encodeJSONResponse(w, NewResponseError(http.StatusInternalServerError, "failed to create stream client", err)) 49 | } 50 | 51 | // 1. send file header first 52 | if serr := stream.Send(&pb.UploadVideoRequest{ 53 | Data: &pb.UploadVideoRequest_Header{ 54 | Header: &pb.VideoHeader{ 55 | Filename: header.Filename, 56 | Size: uint64(header.Size), 57 | }, 58 | }, 59 | }); serr != nil { 60 | h.encodeJSONResponse(w, NewResponseError(http.StatusInternalServerError, "failed to send file header", serr)) 61 | return 62 | } 63 | 64 | // 2. send file chunk 65 | reader := bufio.NewReader(f) 66 | buffer := make([]byte, 1024) 67 | 68 | for { 69 | n, rerr := reader.Read(buffer) 70 | if rerr != nil { 71 | if errors.Is(rerr, io.EOF) { 72 | break 73 | } 74 | 75 | h.encodeJSONResponse(w, NewResponseError(http.StatusInternalServerError, "failed to read file into buffer", rerr)) 76 | return 77 | } 78 | 79 | if serr := stream.Send(&pb.UploadVideoRequest{ 80 | Data: &pb.UploadVideoRequest_ChunkData{ 81 | ChunkData: buffer[:n], 82 | }, 83 | }); serr != nil { 84 | h.encodeJSONResponse(w, NewResponseError(http.StatusInternalServerError, "failed to send file chuck data", serr)) 85 | return 86 | } 87 | } 88 | 89 | resp, err := stream.CloseAndRecv() 90 | if err != nil { 91 | h.encodeJSONResponse(w, NewResponseError(http.StatusInternalServerError, "failed to receive upload file response", err)) 92 | return 93 | } 94 | 95 | h.encodeJSONResponse(w, resp) 96 | } 97 | 98 | func (h *handler) encodeJSONResponse(w http.ResponseWriter, resp interface{}) { 99 | w.Header().Set("Content-Type", "application/json") 100 | 101 | if message, ok := resp.(proto.Message); ok { 102 | h.encodeProtoJSONResponse(w, message) 103 | return 104 | } 105 | 106 | if err := json.NewEncoder(w).Encode(resp); err != nil { 107 | h.logger.Error("failed to encode JSON response", zap.Error(err)) 108 | w.WriteHeader(http.StatusInternalServerError) 109 | return 110 | } 111 | 112 | if coder, ok := resp.(StatusCoder); ok { 113 | w.WriteHeader(coder.StatusCode()) 114 | return 115 | } 116 | 117 | w.WriteHeader(http.StatusOK) 118 | } 119 | 120 | func (h *handler) encodeProtoJSONResponse(w http.ResponseWriter, resp proto.Message) { 121 | o := &protojson.MarshalOptions{ 122 | EmitUnpopulated: true, 123 | } 124 | 125 | bytes, err := o.Marshal(resp) 126 | if err != nil { 127 | h.logger.Error("failed to encode protobuf JSON response", zap.Error(err)) 128 | w.WriteHeader(http.StatusInternalServerError) 129 | return 130 | } 131 | 132 | if _, err := w.Write(bytes); err != nil { 133 | h.logger.Error("failed to write response", zap.Error(err)) 134 | w.WriteHeader(http.StatusInternalServerError) 135 | return 136 | } 137 | 138 | if coder, ok := resp.(StatusCoder); ok { 139 | w.WriteHeader(coder.StatusCode()) 140 | return 141 | } 142 | 143 | w.WriteHeader(http.StatusOK) 144 | } 145 | -------------------------------------------------------------------------------- /modules/comment/dao/comment_redis_test.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-redis/cache/v8" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "go.mongodb.org/mongo-driver/bson/primitive" 10 | ) 11 | 12 | var _ = Describe("CommentRedisDAO", func() { 13 | var redisCommentDAO *redisCommentDAO 14 | var pgCommentDAO *pgCommentDAO 15 | var ctx context.Context 16 | 17 | BeforeEach(func() { 18 | ctx = context.Background() 19 | pgCommentDAO = NewPGCommentDAO(pgClient) 20 | redisCommentDAO = NewRedisCommentDAO(redisClient, pgCommentDAO) 21 | }) 22 | 23 | Describe("ListByVideoID", func() { 24 | var ( 25 | comments []*Comment 26 | videoID string 27 | limit int 28 | offset int 29 | 30 | resp []*Comment 31 | err error 32 | ) 33 | 34 | BeforeEach(func() { 35 | fakeVideoID := primitive.NewObjectID().Hex() 36 | comments = []*Comment{NewFakeComment(fakeVideoID), NewFakeComment(fakeVideoID), NewFakeComment(fakeVideoID)} 37 | }) 38 | 39 | JustBeforeEach(func() { 40 | resp, err = redisCommentDAO.ListByVideoID(ctx, videoID, limit, offset) 41 | }) 42 | 43 | Context("cache hit", func() { 44 | BeforeEach(func() { 45 | limit, offset = 3, 0 46 | videoID = comments[0].VideoID 47 | insertCommentsInRedis(ctx, redisCommentDAO, comments, videoID, limit, offset) 48 | }) 49 | 50 | AfterEach(func() { 51 | deleteCommentsInRedis(ctx, redisCommentDAO, videoID, limit, offset) 52 | }) 53 | 54 | When("success", func() { 55 | It("returns the comments with no error", func() { 56 | for i := range resp { 57 | Expect(resp[i]).To(matchComment(comments[i])) 58 | } 59 | Expect(err).NotTo(HaveOccurred()) 60 | }) 61 | }) 62 | }) 63 | 64 | Context("cache miss", func() { 65 | BeforeEach(func() { 66 | limit, offset = 2, 0 67 | videoID = comments[0].VideoID 68 | for _, comment := range comments { 69 | insertComment(comment) 70 | } 71 | }) 72 | 73 | AfterEach(func() { 74 | for _, comment := range comments { 75 | deleteComment(comment.ID) 76 | } 77 | deleteCommentsInRedis(ctx, redisCommentDAO, videoID, limit, offset) 78 | }) 79 | 80 | When("comments not found due to non-exist limit", func() { 81 | BeforeEach(func() { offset, limit = 3, 3 }) 82 | 83 | It("returns empty list with no error", func() { 84 | Expect(resp).To(HaveLen(0)) 85 | Expect(err).NotTo(HaveOccurred()) 86 | }) 87 | }) 88 | 89 | When("comments not found due to non-exist offset", func() { 90 | BeforeEach(func() { offset = 3 }) 91 | 92 | It("returns empty list with no error", func() { 93 | Expect(resp).To(HaveLen(0)) 94 | Expect(err).NotTo(HaveOccurred()) 95 | }) 96 | }) 97 | 98 | When("comments not found due to non-exist videoID", func() { 99 | BeforeEach(func() { videoID = primitive.NewObjectID().Hex() }) 100 | 101 | It("returns empty list with no error", func() { 102 | Expect(resp).To(HaveLen(0)) 103 | Expect(err).NotTo(HaveOccurred()) 104 | }) 105 | }) 106 | 107 | When("success", func() { 108 | It("returns the comments with no error", func() { 109 | for i := range resp { 110 | Expect(resp[i]).To(matchComment(comments[i])) 111 | } 112 | Expect(err).NotTo(HaveOccurred()) 113 | }) 114 | 115 | It("insert the comments to cache", func() { 116 | var getComments []*Comment 117 | Expect(redisCommentDAO.cache.Get(ctx, listCommentKey(videoID, limit, offset), &getComments)).NotTo(HaveOccurred()) 118 | for i := range getComments { 119 | Expect(getComments[i]).To(matchComment(comments[i])) 120 | } 121 | }) 122 | }) 123 | }) 124 | }) 125 | }) 126 | 127 | func insertCommentsInRedis(ctx context.Context, commentDAO *redisCommentDAO, comments []*Comment, videoID string, limit, offset int) { 128 | Expect(commentDAO.cache.Set(&cache.Item{ 129 | Ctx: ctx, 130 | Key: listCommentKey(videoID, limit, offset), 131 | Value: comments, 132 | TTL: commentDAORedisCacheDuration, 133 | })).NotTo(HaveOccurred()) 134 | } 135 | 136 | func deleteCommentsInRedis(ctx context.Context, commentDAO *redisCommentDAO, videoID string, limit, offset int) { 137 | Expect(commentDAO.cache.Delete(ctx, listCommentKey(videoID, limit, offset))).NotTo(HaveOccurred()) 138 | } 139 | -------------------------------------------------------------------------------- /modules/comment/mock/daomock/mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/dao (interfaces: CommentDAO) 3 | 4 | // Package daomock is a generated GoMock package. 5 | package daomock 6 | 7 | import ( 8 | context "context" 9 | reflect "reflect" 10 | 11 | dao "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/dao" 12 | gomock "github.com/golang/mock/gomock" 13 | uuid "github.com/google/uuid" 14 | ) 15 | 16 | // MockCommentDAO is a mock of CommentDAO interface. 17 | type MockCommentDAO struct { 18 | ctrl *gomock.Controller 19 | recorder *MockCommentDAOMockRecorder 20 | } 21 | 22 | // MockCommentDAOMockRecorder is the mock recorder for MockCommentDAO. 23 | type MockCommentDAOMockRecorder struct { 24 | mock *MockCommentDAO 25 | } 26 | 27 | // NewMockCommentDAO creates a new mock instance. 28 | func NewMockCommentDAO(ctrl *gomock.Controller) *MockCommentDAO { 29 | mock := &MockCommentDAO{ctrl: ctrl} 30 | mock.recorder = &MockCommentDAOMockRecorder{mock} 31 | return mock 32 | } 33 | 34 | // EXPECT returns an object that allows the caller to indicate expected use. 35 | func (m *MockCommentDAO) EXPECT() *MockCommentDAOMockRecorder { 36 | return m.recorder 37 | } 38 | 39 | // Create mocks base method. 40 | func (m *MockCommentDAO) Create(arg0 context.Context, arg1 *dao.Comment) (uuid.UUID, error) { 41 | m.ctrl.T.Helper() 42 | ret := m.ctrl.Call(m, "Create", arg0, arg1) 43 | ret0, _ := ret[0].(uuid.UUID) 44 | ret1, _ := ret[1].(error) 45 | return ret0, ret1 46 | } 47 | 48 | // Create indicates an expected call of Create. 49 | func (mr *MockCommentDAOMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { 50 | mr.mock.ctrl.T.Helper() 51 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockCommentDAO)(nil).Create), arg0, arg1) 52 | } 53 | 54 | // Delete mocks base method. 55 | func (m *MockCommentDAO) Delete(arg0 context.Context, arg1 uuid.UUID) error { 56 | m.ctrl.T.Helper() 57 | ret := m.ctrl.Call(m, "Delete", arg0, arg1) 58 | ret0, _ := ret[0].(error) 59 | return ret0 60 | } 61 | 62 | // Delete indicates an expected call of Delete. 63 | func (mr *MockCommentDAOMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { 64 | mr.mock.ctrl.T.Helper() 65 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockCommentDAO)(nil).Delete), arg0, arg1) 66 | } 67 | 68 | // DeleteByVideoID mocks base method. 69 | func (m *MockCommentDAO) DeleteByVideoID(arg0 context.Context, arg1 string) error { 70 | m.ctrl.T.Helper() 71 | ret := m.ctrl.Call(m, "DeleteByVideoID", arg0, arg1) 72 | ret0, _ := ret[0].(error) 73 | return ret0 74 | } 75 | 76 | // DeleteByVideoID indicates an expected call of DeleteByVideoID. 77 | func (mr *MockCommentDAOMockRecorder) DeleteByVideoID(arg0, arg1 interface{}) *gomock.Call { 78 | mr.mock.ctrl.T.Helper() 79 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteByVideoID", reflect.TypeOf((*MockCommentDAO)(nil).DeleteByVideoID), arg0, arg1) 80 | } 81 | 82 | // ListByVideoID mocks base method. 83 | func (m *MockCommentDAO) ListByVideoID(arg0 context.Context, arg1 string, arg2, arg3 int) ([]*dao.Comment, error) { 84 | m.ctrl.T.Helper() 85 | ret := m.ctrl.Call(m, "ListByVideoID", arg0, arg1, arg2, arg3) 86 | ret0, _ := ret[0].([]*dao.Comment) 87 | ret1, _ := ret[1].(error) 88 | return ret0, ret1 89 | } 90 | 91 | // ListByVideoID indicates an expected call of ListByVideoID. 92 | func (mr *MockCommentDAOMockRecorder) ListByVideoID(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { 93 | mr.mock.ctrl.T.Helper() 94 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByVideoID", reflect.TypeOf((*MockCommentDAO)(nil).ListByVideoID), arg0, arg1, arg2, arg3) 95 | } 96 | 97 | // Update mocks base method. 98 | func (m *MockCommentDAO) Update(arg0 context.Context, arg1 *dao.Comment) error { 99 | m.ctrl.T.Helper() 100 | ret := m.ctrl.Call(m, "Update", arg0, arg1) 101 | ret0, _ := ret[0].(error) 102 | return ret0 103 | } 104 | 105 | // Update indicates an expected call of Update. 106 | func (mr *MockCommentDAOMockRecorder) Update(arg0, arg1 interface{}) *gomock.Call { 107 | mr.mock.ctrl.T.Helper() 108 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockCommentDAO)(nil).Update), arg0, arg1) 109 | } 110 | -------------------------------------------------------------------------------- /modules/video/pb/stream_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.2.0 4 | // - protoc v3.19.3 5 | // source: modules/video/pb/stream.proto 6 | 7 | package pb 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | emptypb "google.golang.org/protobuf/types/known/emptypb" 15 | ) 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the grpc package it is being compiled against. 19 | // Requires gRPC-Go v1.32.0 or later. 20 | const _ = grpc.SupportPackageIsVersion7 21 | 22 | // VideoStreamClient is the client API for VideoStream service. 23 | // 24 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 25 | type VideoStreamClient interface { 26 | HandleVideoCreated(ctx context.Context, in *HandleVideoCreatedRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) 27 | } 28 | 29 | type videoStreamClient struct { 30 | cc grpc.ClientConnInterface 31 | } 32 | 33 | func NewVideoStreamClient(cc grpc.ClientConnInterface) VideoStreamClient { 34 | return &videoStreamClient{cc} 35 | } 36 | 37 | func (c *videoStreamClient) HandleVideoCreated(ctx context.Context, in *HandleVideoCreatedRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { 38 | out := new(emptypb.Empty) 39 | err := c.cc.Invoke(ctx, "/video.pb.VideoStream/HandleVideoCreated", in, out, opts...) 40 | if err != nil { 41 | return nil, err 42 | } 43 | return out, nil 44 | } 45 | 46 | // VideoStreamServer is the server API for VideoStream service. 47 | // All implementations must embed UnimplementedVideoStreamServer 48 | // for forward compatibility 49 | type VideoStreamServer interface { 50 | HandleVideoCreated(context.Context, *HandleVideoCreatedRequest) (*emptypb.Empty, error) 51 | mustEmbedUnimplementedVideoStreamServer() 52 | } 53 | 54 | // UnimplementedVideoStreamServer must be embedded to have forward compatible implementations. 55 | type UnimplementedVideoStreamServer struct { 56 | } 57 | 58 | func (UnimplementedVideoStreamServer) HandleVideoCreated(context.Context, *HandleVideoCreatedRequest) (*emptypb.Empty, error) { 59 | return nil, status.Errorf(codes.Unimplemented, "method HandleVideoCreated not implemented") 60 | } 61 | func (UnimplementedVideoStreamServer) mustEmbedUnimplementedVideoStreamServer() {} 62 | 63 | // UnsafeVideoStreamServer may be embedded to opt out of forward compatibility for this service. 64 | // Use of this interface is not recommended, as added methods to VideoStreamServer will 65 | // result in compilation errors. 66 | type UnsafeVideoStreamServer interface { 67 | mustEmbedUnimplementedVideoStreamServer() 68 | } 69 | 70 | func RegisterVideoStreamServer(s grpc.ServiceRegistrar, srv VideoStreamServer) { 71 | s.RegisterService(&VideoStream_ServiceDesc, srv) 72 | } 73 | 74 | func _VideoStream_HandleVideoCreated_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 75 | in := new(HandleVideoCreatedRequest) 76 | if err := dec(in); err != nil { 77 | return nil, err 78 | } 79 | if interceptor == nil { 80 | return srv.(VideoStreamServer).HandleVideoCreated(ctx, in) 81 | } 82 | info := &grpc.UnaryServerInfo{ 83 | Server: srv, 84 | FullMethod: "/video.pb.VideoStream/HandleVideoCreated", 85 | } 86 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 87 | return srv.(VideoStreamServer).HandleVideoCreated(ctx, req.(*HandleVideoCreatedRequest)) 88 | } 89 | return interceptor(ctx, in, info, handler) 90 | } 91 | 92 | // VideoStream_ServiceDesc is the grpc.ServiceDesc for VideoStream service. 93 | // It's only intended for direct use with grpc.RegisterService, 94 | // and not to be introspected or modified (even as a copy) 95 | var VideoStream_ServiceDesc = grpc.ServiceDesc{ 96 | ServiceName: "video.pb.VideoStream", 97 | HandlerType: (*VideoStreamServer)(nil), 98 | Methods: []grpc.MethodDesc{ 99 | { 100 | MethodName: "HandleVideoCreated", 101 | Handler: _VideoStream_HandleVideoCreated_Handler, 102 | }, 103 | }, 104 | Streams: []grpc.StreamDesc{}, 105 | Metadata: "modules/video/pb/stream.proto", 106 | } 107 | -------------------------------------------------------------------------------- /cmd/comment/api.go: -------------------------------------------------------------------------------- 1 | package comment 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net" 7 | 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/dao" 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb" 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/service" 11 | videopb "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 12 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/grpckit" 13 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 14 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/otelkit" 15 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/pgkit" 16 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/rediskit" 17 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/runkit" 18 | flags "github.com/jessevdk/go-flags" 19 | "github.com/spf13/cobra" 20 | "go.uber.org/zap" 21 | "google.golang.org/grpc" 22 | ) 23 | 24 | func newAPICommand() *cobra.Command { 25 | return &cobra.Command{ 26 | Use: "api", 27 | Short: "starts comment API server", 28 | RunE: runAPI, 29 | } 30 | } 31 | 32 | type APIArgs struct { 33 | GRPCAddr string `long:"grpc_addr" env:"GRPC_ADDR" default:":8081"` 34 | VideoClientConnConfig grpckit.GrpcClientConnConfig `group:"video" namespace:"video" env-namespace:"VIDEO"` 35 | runkit.GracefulConfig `group:"graceful" namespace:"graceful" env-namespace:"GRACEFUL"` 36 | logkit.LoggerConfig `group:"logger" namespace:"logger" env-namespace:"LOGGER"` 37 | pgkit.PGConfig `group:"postgres" namespace:"postgres" env-namespace:"POSTGRES"` 38 | rediskit.RedisConfig `group:"redis" namespace:"redis" env-namespace:"REDIS"` 39 | otelkit.PrometheusServiceMeterConfig `group:"meter" namespace:"meter" env-namespace:"METER"` 40 | } 41 | 42 | func runAPI(_ *cobra.Command, _ []string) error { 43 | ctx := context.Background() 44 | 45 | var args APIArgs 46 | if _, err := flags.NewParser(&args, flags.Default).Parse(); err != nil { 47 | log.Fatal("failed to parse flag", err.Error()) 48 | } 49 | 50 | logger := logkit.NewLogger(&args.LoggerConfig) 51 | defer func() { 52 | _ = logger.Sync() 53 | }() 54 | 55 | ctx = logger.WithContext(ctx) 56 | 57 | pgClient := pgkit.NewPGClient(ctx, &args.PGConfig) 58 | defer func() { 59 | if err := pgClient.Close(); err != nil { 60 | logger.Fatal("failed to close pg client", zap.Error(err)) 61 | } 62 | }() 63 | 64 | redisClient := rediskit.NewRedisClient(ctx, &args.RedisConfig) 65 | defer func() { 66 | if err := redisClient.Close(); err != nil { 67 | logger.Fatal("failed to close redis client", zap.Error(err)) 68 | } 69 | }() 70 | 71 | videoClientConn := grpckit.NewGrpcClientConn(ctx, &args.VideoClientConnConfig) 72 | defer func() { 73 | if err := videoClientConn.Close(); err != nil { 74 | logger.Fatal("failed to close video gRPC client", zap.Error(err)) 75 | } 76 | }() 77 | 78 | pgCommentDAO := dao.NewPGCommentDAO(pgClient) 79 | commentDAO := dao.NewRedisCommentDAO(redisClient, pgCommentDAO) 80 | videoClient := videopb.NewVideoClient(videoClientConn) 81 | 82 | svc := service.NewService(commentDAO, videoClient) 83 | 84 | logger.Info("listen to gRPC addr", zap.String("grpc_addr", args.GRPCAddr)) 85 | lis, err := net.Listen("tcp", args.GRPCAddr) 86 | if err != nil { 87 | logger.Fatal("failed to listen gRPC addr", zap.Error(err)) 88 | } 89 | defer func() { 90 | if err := lis.Close(); err != nil { 91 | logger.Fatal("failed to close gRPC listener", zap.Error(err)) 92 | } 93 | }() 94 | 95 | meter := otelkit.NewPrometheusServiceMeter(ctx, &args.PrometheusServiceMeterConfig) 96 | defer func() { 97 | if err := meter.Close(); err != nil { 98 | logger.Fatal("failed to close meter", zap.Error(err)) 99 | } 100 | }() 101 | 102 | return runkit.GracefulRun(serveGRPC(lis, svc, logger, grpc.UnaryInterceptor(meter.UnaryServerInterceptor())), &args.GracefulConfig) 103 | } 104 | 105 | func serveGRPC(lis net.Listener, svc pb.CommentServer, logger *logkit.Logger, opt ...grpc.ServerOption) runkit.GracefulRunFunc { 106 | grpcServer := grpc.NewServer(opt...) 107 | pb.RegisterCommentServer(grpcServer, svc) 108 | 109 | return func(ctx context.Context) error { 110 | go func() { 111 | if err := grpcServer.Serve(lis); err != nil { 112 | logger.Error("failed to run gRPC server", zap.Error(err)) 113 | } 114 | }() 115 | 116 | <-ctx.Done() 117 | 118 | grpcServer.GracefulStop() 119 | 120 | return nil 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /modules/video/mock/daomock/mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao (interfaces: VideoDAO) 3 | 4 | // Package daomock is a generated GoMock package. 5 | package daomock 6 | 7 | import ( 8 | context "context" 9 | reflect "reflect" 10 | 11 | dao "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao" 12 | gomock "github.com/golang/mock/gomock" 13 | primitive "go.mongodb.org/mongo-driver/bson/primitive" 14 | ) 15 | 16 | // MockVideoDAO is a mock of VideoDAO interface. 17 | type MockVideoDAO struct { 18 | ctrl *gomock.Controller 19 | recorder *MockVideoDAOMockRecorder 20 | } 21 | 22 | // MockVideoDAOMockRecorder is the mock recorder for MockVideoDAO. 23 | type MockVideoDAOMockRecorder struct { 24 | mock *MockVideoDAO 25 | } 26 | 27 | // NewMockVideoDAO creates a new mock instance. 28 | func NewMockVideoDAO(ctrl *gomock.Controller) *MockVideoDAO { 29 | mock := &MockVideoDAO{ctrl: ctrl} 30 | mock.recorder = &MockVideoDAOMockRecorder{mock} 31 | return mock 32 | } 33 | 34 | // EXPECT returns an object that allows the caller to indicate expected use. 35 | func (m *MockVideoDAO) EXPECT() *MockVideoDAOMockRecorder { 36 | return m.recorder 37 | } 38 | 39 | // Create mocks base method. 40 | func (m *MockVideoDAO) Create(arg0 context.Context, arg1 *dao.Video) error { 41 | m.ctrl.T.Helper() 42 | ret := m.ctrl.Call(m, "Create", arg0, arg1) 43 | ret0, _ := ret[0].(error) 44 | return ret0 45 | } 46 | 47 | // Create indicates an expected call of Create. 48 | func (mr *MockVideoDAOMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockVideoDAO)(nil).Create), arg0, arg1) 51 | } 52 | 53 | // Delete mocks base method. 54 | func (m *MockVideoDAO) Delete(arg0 context.Context, arg1 primitive.ObjectID) error { 55 | m.ctrl.T.Helper() 56 | ret := m.ctrl.Call(m, "Delete", arg0, arg1) 57 | ret0, _ := ret[0].(error) 58 | return ret0 59 | } 60 | 61 | // Delete indicates an expected call of Delete. 62 | func (mr *MockVideoDAOMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { 63 | mr.mock.ctrl.T.Helper() 64 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockVideoDAO)(nil).Delete), arg0, arg1) 65 | } 66 | 67 | // Get mocks base method. 68 | func (m *MockVideoDAO) Get(arg0 context.Context, arg1 primitive.ObjectID) (*dao.Video, error) { 69 | m.ctrl.T.Helper() 70 | ret := m.ctrl.Call(m, "Get", arg0, arg1) 71 | ret0, _ := ret[0].(*dao.Video) 72 | ret1, _ := ret[1].(error) 73 | return ret0, ret1 74 | } 75 | 76 | // Get indicates an expected call of Get. 77 | func (mr *MockVideoDAOMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { 78 | mr.mock.ctrl.T.Helper() 79 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVideoDAO)(nil).Get), arg0, arg1) 80 | } 81 | 82 | // List mocks base method. 83 | func (m *MockVideoDAO) List(arg0 context.Context, arg1, arg2 int64) ([]*dao.Video, error) { 84 | m.ctrl.T.Helper() 85 | ret := m.ctrl.Call(m, "List", arg0, arg1, arg2) 86 | ret0, _ := ret[0].([]*dao.Video) 87 | ret1, _ := ret[1].(error) 88 | return ret0, ret1 89 | } 90 | 91 | // List indicates an expected call of List. 92 | func (mr *MockVideoDAOMockRecorder) List(arg0, arg1, arg2 interface{}) *gomock.Call { 93 | mr.mock.ctrl.T.Helper() 94 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockVideoDAO)(nil).List), arg0, arg1, arg2) 95 | } 96 | 97 | // Update mocks base method. 98 | func (m *MockVideoDAO) Update(arg0 context.Context, arg1 *dao.Video) error { 99 | m.ctrl.T.Helper() 100 | ret := m.ctrl.Call(m, "Update", arg0, arg1) 101 | ret0, _ := ret[0].(error) 102 | return ret0 103 | } 104 | 105 | // Update indicates an expected call of Update. 106 | func (mr *MockVideoDAOMockRecorder) Update(arg0, arg1 interface{}) *gomock.Call { 107 | mr.mock.ctrl.T.Helper() 108 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockVideoDAO)(nil).Update), arg0, arg1) 109 | } 110 | 111 | // UpdateVariant mocks base method. 112 | func (m *MockVideoDAO) UpdateVariant(arg0 context.Context, arg1 primitive.ObjectID, arg2, arg3 string) error { 113 | m.ctrl.T.Helper() 114 | ret := m.ctrl.Call(m, "UpdateVariant", arg0, arg1, arg2, arg3) 115 | ret0, _ := ret[0].(error) 116 | return ret0 117 | } 118 | 119 | // UpdateVariant indicates an expected call of UpdateVariant. 120 | func (mr *MockVideoDAOMockRecorder) UpdateVariant(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { 121 | mr.mock.ctrl.T.Helper() 122 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVariant", reflect.TypeOf((*MockVideoDAO)(nil).UpdateVariant), arg0, arg1, arg2, arg3) 123 | } 124 | -------------------------------------------------------------------------------- /modules/video/service/service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "errors" 8 | "io" 9 | "path" 10 | 11 | commentpb "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb" 12 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao" 13 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 14 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit" 15 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/storagekit" 16 | "go.mongodb.org/mongo-driver/bson/primitive" 17 | "google.golang.org/protobuf/proto" 18 | ) 19 | 20 | type service struct { 21 | pb.UnimplementedVideoServer 22 | 23 | videoDAO dao.VideoDAO 24 | storage storagekit.Storage 25 | commentClient commentpb.CommentClient 26 | producer kafkakit.Producer 27 | } 28 | 29 | func NewService(videoDAO dao.VideoDAO, storage storagekit.Storage, commentClient commentpb.CommentClient, producer kafkakit.Producer) *service { 30 | return &service{ 31 | videoDAO: videoDAO, 32 | storage: storage, 33 | commentClient: commentClient, 34 | producer: producer, 35 | } 36 | } 37 | 38 | func (s *service) Healthz(ctx context.Context, req *pb.HealthzRequest) (*pb.HealthzResponse, error) { 39 | return &pb.HealthzResponse{Status: "ok"}, nil 40 | } 41 | 42 | func (s *service) GetVideo(ctx context.Context, req *pb.GetVideoRequest) (*pb.GetVideoResponse, error) { 43 | id, err := primitive.ObjectIDFromHex(req.GetId()) 44 | if err != nil { 45 | return nil, ErrInvalidObjectID 46 | } 47 | 48 | video, err := s.videoDAO.Get(ctx, id) 49 | if err != nil { 50 | if errors.Is(err, dao.ErrVideoNotFound) { 51 | return nil, ErrVideoNotFound 52 | } 53 | 54 | return nil, err 55 | } 56 | 57 | return &pb.GetVideoResponse{Video: video.ToProto()}, nil 58 | } 59 | 60 | func (s *service) ListVideo(ctx context.Context, req *pb.ListVideoRequest) (*pb.ListVideoResponse, error) { 61 | videos, err := s.videoDAO.List(ctx, req.GetLimit(), req.GetSkip()) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | pbVideos := make([]*pb.VideoInfo, 0, len(videos)) 67 | for _, video := range videos { 68 | pbVideos = append(pbVideos, video.ToProto()) 69 | } 70 | 71 | return &pb.ListVideoResponse{Videos: pbVideos}, nil 72 | } 73 | 74 | func (s *service) UploadVideo(stream pb.Video_UploadVideoServer) error { 75 | ctx := stream.Context() 76 | 77 | req, err := stream.Recv() 78 | if err != nil { 79 | return err 80 | } 81 | 82 | filename := req.GetHeader().GetFilename() 83 | size := req.GetHeader().GetSize() 84 | 85 | var buf bytes.Buffer 86 | 87 | for { 88 | req, err := stream.Recv() 89 | if err != nil { 90 | if errors.Is(err, io.EOF) { 91 | break 92 | } 93 | 94 | return err 95 | } 96 | 97 | chunk := req.GetChunkData() 98 | if _, err := buf.Write(chunk); err != nil { 99 | return err 100 | } 101 | } 102 | 103 | id := primitive.NewObjectID() 104 | objectName := id.Hex() + "-" + filename 105 | 106 | if err := s.storage.PutObject(ctx, objectName, bufio.NewReader(&buf), int64(size), storagekit.PutObjectOptions{ 107 | ContentType: "application/octet-stream", 108 | }); err != nil { 109 | return err 110 | } 111 | 112 | video := &dao.Video{ 113 | ID: id, 114 | Size: size, 115 | URL: path.Join(s.storage.Endpoint(), s.storage.Bucket(), objectName), 116 | Status: dao.VideoStatusUploaded, 117 | } 118 | 119 | if err := s.videoDAO.Create(ctx, video); err != nil { 120 | return err 121 | } 122 | 123 | if err := s.produceVideoCreatedEvent(&pb.HandleVideoCreatedRequest{ 124 | Id: id.Hex(), 125 | Url: path.Join(s.storage.Endpoint(), s.storage.Bucket(), objectName), 126 | }); err != nil { 127 | return err 128 | } 129 | 130 | if err := stream.SendAndClose(&pb.UploadVideoResponse{ 131 | Id: id.Hex(), 132 | }); err != nil { 133 | return err 134 | } 135 | 136 | return nil 137 | } 138 | 139 | func (s *service) DeleteVideo(ctx context.Context, req *pb.DeleteVideoRequest) (*pb.DeleteVideoResponse, error) { 140 | id, err := primitive.ObjectIDFromHex(req.GetId()) 141 | if err != nil { 142 | return nil, ErrInvalidObjectID 143 | } 144 | 145 | if err := s.videoDAO.Delete(ctx, id); err != nil { 146 | if errors.Is(err, dao.ErrVideoNotFound) { 147 | return nil, ErrVideoNotFound 148 | } 149 | 150 | return nil, err 151 | } 152 | 153 | if _, err := s.commentClient.DeleteCommentByVideoID(ctx, &commentpb.DeleteCommentByVideoIDRequest{ 154 | VideoId: id.Hex(), 155 | }); err != nil { 156 | return nil, err 157 | } 158 | 159 | return &pb.DeleteVideoResponse{}, nil 160 | } 161 | 162 | func (s *service) produceVideoCreatedEvent(req *pb.HandleVideoCreatedRequest) error { 163 | valueBytes, err := proto.Marshal(req) 164 | if err != nil { 165 | return err 166 | } 167 | 168 | msgs := []*kafkakit.ProducerMessage{ 169 | {Value: valueBytes}, 170 | } 171 | 172 | if err := s.producer.SendMessages(msgs); err != nil { 173 | return err 174 | } 175 | 176 | return nil 177 | } 178 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | x-common-env: &common-env 4 | GOPATH: /go 5 | GOCACHE: /src/.cache/gocache 6 | MONGO_URL: mongodb://mongo:27017/ 7 | MONGO_DATABASE: nthu_distributed_system 8 | POSTGRES_URL: postgres://postgres@postgres:5432/postgres?sslmode=disable 9 | REDIS_ADDR: redis:6379 10 | KAFKA_PRODUCER_ADDRS: kafka:29092 11 | KAFKA_PRODUCER_TOPIC: video 12 | KAFKA_CONSUMER_ADDRS: kafka:29092 13 | KAFKA_CONSUMER_TOPIC: video 14 | KAFKA_CONSUMER_GROUP: video-stream 15 | MINIO_ENDPOINT: play.min.io 16 | MINIO_BUCKET: videos 17 | MINIO_USERNAME: Q3AM3UQ867SPQQA43P2F 18 | MINIO_PASSWORD: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG 19 | 20 | x-common-build: &common-build 21 | image: justin0u0/golang-protoc:protoc-25.3-golang-1.21.7 22 | working_dir: /src 23 | environment: 24 | <<: *common-env 25 | volumes: 26 | - .:/src 27 | - ~/go/pkg/mod/cache:/go/pkg/mod/cache 28 | 29 | services: 30 | mongo: 31 | image: mongo:5 32 | 33 | postgres: 34 | image: postgres:14-alpine 35 | environment: 36 | - POSTGRES_HOST_AUTH_METHOD=trust 37 | 38 | redis: 39 | image: redis:6.2-alpine 40 | 41 | zookeeper: 42 | image: confluentinc/cp-zookeeper:7.0.1 43 | environment: 44 | ZOOKEEPER_CLIENT_PORT: 2181 45 | ZOOKEEPER_TICK_TIME: 2000 46 | 47 | kafka: 48 | image: confluentinc/cp-kafka:7.0.1 49 | environment: 50 | KAFKA_BOOTSTRAP_SERVERS: kafka:29092 51 | KAFKA_BROKER_ID: 1 52 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 53 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 54 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 55 | KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT 56 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 57 | ports: 58 | - 9092:9092 59 | depends_on: 60 | - zookeeper 61 | 62 | prometheus: 63 | image: prom/prometheus 64 | volumes: 65 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 66 | command: 67 | - '--config.file=/etc/prometheus/prometheus.yml' 68 | - '--storage.tsdb.path=/prometheus' 69 | - '--web.console.libraries=/usr/share/prometheus/console_libraries' 70 | - '--web.console.templates=/usr/share/prometheus/consoles' 71 | ports: 72 | - 9090:9090 73 | 74 | generate: 75 | <<: *common-build 76 | command: 77 | - make 78 | - generate 79 | 80 | lint: 81 | image: golangci/golangci-lint:v1.44.2 82 | working_dir: /src 83 | environment: 84 | GOLANGCI_LINT_CACHE: /src/.cache/golangci-lint-cache 85 | volumes: 86 | - .:/src 87 | command: 88 | - make 89 | - lint 90 | 91 | test: 92 | <<: *common-build 93 | command: 94 | - make 95 | - test 96 | depends_on: 97 | - mongo 98 | - redis 99 | - postgres 100 | 101 | build: 102 | <<: *common-build 103 | command: 104 | - make 105 | - build 106 | 107 | image: 108 | image: nthu-distributed-system:latest 109 | build: 110 | context: . 111 | environment: 112 | <<: *common-env 113 | 114 | video-api: 115 | image: nthu-distributed-system:latest 116 | environment: 117 | <<: *common-env 118 | COMMENT_SERVER_ADDR: comment-api:8081 119 | METER_NAME: video.api 120 | METER_HISTOGRAM_BOUNDARIES: "10,100,200,500,1000" 121 | command: 122 | - /cmd 123 | - video 124 | - api 125 | depends_on: 126 | - mongo 127 | - redis 128 | - kafka 129 | 130 | video-gateway: 131 | image: nthu-distributed-system:latest 132 | environment: 133 | <<: *common-env 134 | GRPC_SERVER_ADDR: video-api:8081 135 | command: 136 | - /cmd 137 | - video 138 | - gateway 139 | ports: 140 | - 10080:8080 141 | 142 | video-stream: 143 | image: nthu-distributed-system:latest 144 | environment: 145 | <<: *common-env 146 | command: 147 | - /cmd 148 | - video 149 | - stream 150 | depends_on: 151 | - mongo 152 | - kafka 153 | 154 | comment-api: 155 | image: nthu-distributed-system:latest 156 | environment: 157 | <<: *common-env 158 | VIDEO_SERVER_ADDR: video-api:8081 159 | METER_NAME: comment.api 160 | METER_HISTOGRAM_BOUNDARIES: "10,100,200,500,1000" 161 | command: 162 | - /cmd 163 | - comment 164 | - api 165 | depends_on: 166 | - postgres 167 | - redis 168 | 169 | comment-gateway: 170 | image: nthu-distributed-system:latest 171 | environment: 172 | <<: *common-env 173 | GRPC_SERVER_ADDR: comment-api:8081 174 | command: 175 | - /cmd 176 | - comment 177 | - gateway 178 | ports: 179 | - 10081:8080 180 | 181 | comment-migration: 182 | image: nthu-distributed-system:latest 183 | environment: 184 | MIGRATION_SOURCE: file:///static/modules/comment/migration 185 | MIGRATION_URL: postgres://postgres@postgres:5432/postgres?sslmode=disable 186 | command: 187 | - /cmd 188 | - comment 189 | - migration 190 | depends_on: 191 | - postgres 192 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/NTHU-LSALAB/NTHU-Distributed-System 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.33.0 7 | github.com/go-pg/pg/v10 v10.10.6 8 | github.com/go-redis/cache/v8 v8.4.3 9 | github.com/go-redis/redis/v8 v8.11.5 10 | github.com/golang-migrate/migrate/v4 v4.15.2 11 | github.com/golang/mock v1.6.0 12 | github.com/google/uuid v1.3.0 13 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 14 | github.com/jessevdk/go-flags v1.5.0 15 | github.com/justin0u0/protoc-gen-grpc-sarama v0.0.1 16 | github.com/minio/minio-go/v7 v7.0.26 17 | github.com/onsi/ginkgo/v2 v2.1.4 18 | github.com/onsi/gomega v1.19.0 19 | github.com/prometheus/client_model v0.2.0 20 | github.com/prometheus/common v0.34.0 21 | github.com/spf13/cobra v1.4.0 22 | go.mongodb.org/mongo-driver v1.9.1 23 | go.opentelemetry.io/otel v1.7.0 24 | go.opentelemetry.io/otel/exporters/prometheus v0.30.0 25 | go.opentelemetry.io/otel/metric v0.30.0 26 | go.opentelemetry.io/otel/sdk/metric v0.30.0 27 | go.uber.org/zap v1.21.0 28 | google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 29 | google.golang.org/grpc v1.46.2 30 | google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 31 | google.golang.org/protobuf v1.28.0 32 | ) 33 | 34 | require ( 35 | github.com/beorn7/perks v1.0.1 // indirect 36 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 37 | github.com/davecgh/go-spew v1.1.1 // indirect 38 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 39 | github.com/dustin/go-humanize v1.0.0 // indirect 40 | github.com/eapache/go-resiliency v1.2.0 // indirect 41 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 42 | github.com/eapache/queue v1.1.0 // indirect 43 | github.com/go-logr/logr v1.2.3 // indirect 44 | github.com/go-logr/stdr v1.2.2 // indirect 45 | github.com/go-pg/zerochecker v0.2.0 // indirect 46 | github.com/go-stack/stack v1.8.1 // indirect 47 | github.com/golang/glog v1.0.0 // indirect 48 | github.com/golang/protobuf v1.5.2 // indirect 49 | github.com/golang/snappy v0.0.4 // indirect 50 | github.com/hashicorp/errwrap v1.1.0 // indirect 51 | github.com/hashicorp/go-multierror v1.1.1 // indirect 52 | github.com/hashicorp/go-uuid v1.0.3 // indirect 53 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 54 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 55 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 56 | github.com/jcmturner/gofork v1.0.0 // indirect 57 | github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect 58 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 59 | github.com/jinzhu/inflection v1.0.0 // indirect 60 | github.com/json-iterator/go v1.1.12 // indirect 61 | github.com/klauspost/compress v1.15.4 // indirect 62 | github.com/klauspost/cpuid/v2 v2.0.12 // indirect 63 | github.com/lib/pq v1.10.4 // indirect 64 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 65 | github.com/minio/md5-simd v1.1.2 // indirect 66 | github.com/minio/sha256-simd v1.0.0 // indirect 67 | github.com/mitchellh/go-homedir v1.1.0 // indirect 68 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 69 | github.com/modern-go/reflect2 v1.0.2 // indirect 70 | github.com/pierrec/lz4 v2.6.1+incompatible // indirect 71 | github.com/pkg/errors v0.9.1 // indirect 72 | github.com/prometheus/client_golang v1.12.2 // indirect 73 | github.com/prometheus/procfs v0.7.3 // indirect 74 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 75 | github.com/rs/xid v1.4.0 // indirect 76 | github.com/sirupsen/logrus v1.8.1 // indirect 77 | github.com/spf13/pflag v1.0.5 // indirect 78 | github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect 79 | github.com/vmihailenco/bufpool v0.1.11 // indirect 80 | github.com/vmihailenco/go-tinylfu v0.2.2 // indirect 81 | github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect 82 | github.com/vmihailenco/tagparser v0.1.2 // indirect 83 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 84 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 85 | github.com/xdg-go/scram v1.1.1 // indirect 86 | github.com/xdg-go/stringprep v1.0.3 // indirect 87 | github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect 88 | go.opentelemetry.io/otel/sdk v1.7.0 // indirect 89 | go.opentelemetry.io/otel/trace v1.7.0 // indirect 90 | go.uber.org/atomic v1.9.0 // indirect 91 | go.uber.org/multierr v1.8.0 // indirect 92 | golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 // indirect 93 | golang.org/x/exp v0.0.0-20220303002715-f922e1b6e9ab // indirect 94 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect 95 | golang.org/x/net v0.0.0-20220513224357-95641704303c // indirect 96 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 97 | golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a // indirect 98 | golang.org/x/text v0.3.7 // indirect 99 | golang.org/x/tools v0.1.10 // indirect 100 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 101 | gopkg.in/ini.v1 v1.66.4 // indirect 102 | gopkg.in/yaml.v2 v2.4.0 // indirect 103 | mellium.im/sasl v0.2.1 // indirect 104 | sigs.k8s.io/yaml v1.3.0 // indirect 105 | ) 106 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PATH := $(CURDIR)/bin:$(PATH) 2 | 3 | MODULES := video comment 4 | 5 | BUILD_DIR := bin/app 6 | BUILD_STATIC_DIR := $(BUILD_DIR)/static 7 | 8 | STATIC_DIRS := $(wildcard modules/*/migration) 9 | 10 | DOCKER_COMPOSE := $(or $(DOCKER_COMPOSE),$(DOCKER_COMPOSE),docker compose) 11 | 12 | #################################################################################################### 13 | ### Automatically include components' extensions and ad-hoc rules (makefile.mk) 14 | ### 15 | -include */makefile.mk 16 | 17 | .PHONY: clean 18 | clean: 19 | rm -rf bin/* 20 | 21 | #################################################################################################### 22 | ### Rule for the `generate` command 23 | ### 24 | 25 | define make-dc-generate-rules 26 | 27 | .PHONY: dc.$1.generate 28 | 29 | # to generate individual module, override the command defined in the docker-compose.yml file 30 | dc.$1.generate: 31 | $(DOCKER_COMPOSE) run --rm generate make $1.generate 32 | 33 | endef 34 | $(foreach module,$(MODULES),$(eval $(call make-dc-generate-rules,$(module)))) 35 | 36 | .PHONY: dc.pkg.generate 37 | dc.pkg.generate: 38 | $(DOCKER_COMPOSE) run --rm generate make pkg.generate 39 | 40 | .PHONY: dc.generate 41 | dc.generate: 42 | $(DOCKER_COMPOSE) run --rm generate 43 | 44 | define make-generate-rules 45 | 46 | .PHONY: $1.generate 47 | $1.generate: bin/protoc-gen-go bin/protoc-gen-go-grpc bin/protoc-gen-grpc-gateway bin/protoc-gen-grpc-sarama bin/mockgen 48 | protoc \ 49 | -I . \ 50 | -I ./pkg/pb \ 51 | -I $(dir $(shell (go list -f '{{ .Dir }}' github.com/justin0u0/protoc-gen-grpc-sarama/proto))) \ 52 | --go_out=paths=source_relative:. \ 53 | --go-grpc_out=paths=source_relative:. \ 54 | --grpc-gateway_out=paths=source_relative:. \ 55 | --grpc-sarama_out=paths=source_relative:. \ 56 | ./modules/$1/pb/*.proto 57 | 58 | go generate ./modules/$1/... 59 | 60 | endef 61 | $(foreach module,$(MODULES),$(eval $(call make-generate-rules,$(module)))) 62 | 63 | .PHONY: pkg.generate 64 | pkg.generate: bin/protoc-gen-go bin/protoc-gen-go-grpc bin/protoc-gen-grpc-gateway bin/protoc-gen-grpc-sarama bin/mockgen 65 | go generate ./pkg/... 66 | 67 | .PHONY: generate 68 | generate: pkg.generate $(addsuffix .generate,$(MODULES)) 69 | 70 | bin/protoc-gen-go: go.mod 71 | go build -o $@ google.golang.org/protobuf/cmd/protoc-gen-go 72 | 73 | bin/protoc-gen-go-grpc: go.mod 74 | go build -o $@ google.golang.org/grpc/cmd/protoc-gen-go-grpc 75 | 76 | bin/protoc-gen-grpc-gateway: go.mod 77 | go build -o $@ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway 78 | 79 | bin/protoc-gen-grpc-sarama: go.mod 80 | go build -o $@ github.com/justin0u0/protoc-gen-grpc-sarama 81 | 82 | bin/mockgen: go.mod 83 | go build -o $@ github.com/golang/mock/mockgen 84 | 85 | #################################################################################################### 86 | ### Rule for the `lint` command 87 | ### 88 | 89 | define make-dc-lint-rules 90 | 91 | .PHONY: dc.$1.lint 92 | dc.$1.lint: 93 | $(DOCKER_COMPOSE) run --rm lint make $1.lint 94 | 95 | endef 96 | $(foreach module,$(MODULES),$(eval $(call make-dc-lint-rules,$(module)))) 97 | 98 | .PHONY: dc.pkg.lint 99 | dc.pkg.lint: 100 | $(DOCKER_COMPOSE) run --rm lint make pkg.lint 101 | 102 | .PHONY: dc.lint 103 | dc.lint: 104 | $(DOCKER_COMPOSE) run --rm lint 105 | 106 | define make-lint-rules 107 | 108 | .PHONY: $1.lint 109 | $1.lint: 110 | golangci-lint run ./modules/$1/... 111 | 112 | endef 113 | $(foreach module,$(MODULES),$(eval $(call make-lint-rules,$(module)))) 114 | 115 | .PHONY: pkg.lint 116 | pkg.lint: 117 | golangci-lint run ./pkg/... 118 | 119 | .PHONY: lint 120 | lint: 121 | golangci-lint run ./... 122 | 123 | #################################################################################################### 124 | ### Rule for the `test` command 125 | ### 126 | 127 | define make-dc-test-rules 128 | 129 | .PHONY: dc.$1.test 130 | dc.$1.test: 131 | $(DOCKER_COMPOSE) run --rm test make $1.test 132 | 133 | endef 134 | $(foreach module,$(MODULES),$(eval $(call make-dc-test-rules,$(module)))) 135 | 136 | .PHONY: dc.pkg.test 137 | dc.pkg.test: 138 | $(DOCKER_COMPOSE) run --rm test make pkg.test 139 | 140 | .PHONY: dc.test 141 | dc.test: 142 | $(DOCKER_COMPOSE) run --rm test 143 | 144 | define make-test-rules 145 | 146 | .PHONY: $1.test 147 | $1.test: 148 | go test -v -race ./modules/$1/... 149 | 150 | endef 151 | $(foreach module,$(MODULES),$(eval $(call make-test-rules,$(module)))) 152 | 153 | .PHONY: pkg.test 154 | pkg.test: 155 | go test -v -race ./pkg/... 156 | 157 | .PHONY: test 158 | test: pkg.test $(addsuffix .test,$(MODULES)) 159 | 160 | #################################################################################################### 161 | ### Rule for the `build` command 162 | ### 163 | 164 | .PHONY: dc.image 165 | dc.image: dc.build 166 | $(DOCKER_COMPOSE) build --force-rm image 167 | 168 | .PHONY: dc.build 169 | dc.build: 170 | $(DOCKER_COMPOSE) run --rm build 171 | 172 | .PHONY: build 173 | build: $(STATIC_DIRS) 174 | @mkdir -p $(BUILD_DIR) 175 | go build -o $(BUILD_DIR)/cmd ./cmd/main.go 176 | 177 | .PHONY: $(STATIC_DIRS) 178 | .SECONDEXPANSION: 179 | $(STATIC_DIRS): %: $$(wildcard %/*) 180 | @mkdir -p $(BUILD_STATIC_DIR)/$@ 181 | cp -R $@/. $(BUILD_STATIC_DIR)/$@ 182 | -------------------------------------------------------------------------------- /cmd/video/api.go: -------------------------------------------------------------------------------- 1 | package video 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net" 7 | 8 | commentpb "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb" 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/dao" 10 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/pb" 11 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/video/service" 12 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/grpckit" 13 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/kafkakit" 14 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 15 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/mongokit" 16 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/otelkit" 17 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/rediskit" 18 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/runkit" 19 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/storagekit" 20 | flags "github.com/jessevdk/go-flags" 21 | "github.com/spf13/cobra" 22 | "go.uber.org/zap" 23 | "google.golang.org/grpc" 24 | ) 25 | 26 | func newAPICommand() *cobra.Command { 27 | return &cobra.Command{ 28 | Use: "api", 29 | Short: "starts video API server", 30 | RunE: runAPI, 31 | } 32 | } 33 | 34 | type APIArgs struct { 35 | GRPCAddr string `long:"grpc_addr" env:"GRPC_ADDR" default:":8081"` 36 | CommentClientConnConfig grpckit.GrpcClientConnConfig `group:"comment" namespace:"comment" env-namespace:"COMMENT"` 37 | runkit.GracefulConfig `group:"graceful" namespace:"graceful" env-namespace:"GRACEFUL"` 38 | logkit.LoggerConfig `group:"logger" namespace:"logger" env-namespace:"LOGGER"` 39 | mongokit.MongoConfig `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` 40 | storagekit.MinIOConfig `group:"minio" namespace:"minio" env-namespace:"MINIO"` 41 | rediskit.RedisConfig `group:"redis" namespace:"redis" env-namespace:"REDIS"` 42 | otelkit.PrometheusServiceMeterConfig `group:"meter" namespace:"meter" env-namespace:"METER"` 43 | kafkakit.KafkaProducerConfig `group:"kafka_producer" namespace:"kafka_producer" env-namespace:"KAFKA_PRODUCER"` 44 | } 45 | 46 | func runAPI(_ *cobra.Command, _ []string) error { 47 | ctx := context.Background() 48 | 49 | var args APIArgs 50 | if _, err := flags.NewParser(&args, flags.Default).Parse(); err != nil { 51 | log.Fatal("failed to parse flag", err.Error()) 52 | } 53 | 54 | logger := logkit.NewLogger(&args.LoggerConfig) 55 | defer func() { 56 | _ = logger.Sync() 57 | }() 58 | 59 | ctx = logger.WithContext(ctx) 60 | 61 | mongoClient := mongokit.NewMongoClient(ctx, &args.MongoConfig) 62 | defer func() { 63 | if err := mongoClient.Close(); err != nil { 64 | logger.Fatal("failed to close mongo client", zap.Error(err)) 65 | } 66 | }() 67 | 68 | redisClient := rediskit.NewRedisClient(ctx, &args.RedisConfig) 69 | defer func() { 70 | if err := redisClient.Close(); err != nil { 71 | logger.Fatal("failed to close redis client", zap.Error(err)) 72 | } 73 | }() 74 | 75 | commentClientConn := grpckit.NewGrpcClientConn(ctx, &args.CommentClientConnConfig) 76 | defer func() { 77 | if err := commentClientConn.Close(); err != nil { 78 | logger.Fatal("failed to close comment gRPC client", zap.Error(err)) 79 | } 80 | }() 81 | 82 | producer := kafkakit.NewKafkaProducer(ctx, &args.KafkaProducerConfig) 83 | defer func() { 84 | if err := producer.Close(); err != nil { 85 | logger.Fatal("failed to close Kafka producer", zap.Error(err)) 86 | } 87 | }() 88 | 89 | mongoVideoDAO := dao.NewMongoVideoDAO(mongoClient.Database().Collection("videos")) 90 | videoDAO := dao.NewRedisVideoDAO(redisClient, mongoVideoDAO) 91 | storage := storagekit.NewMinIOClient(ctx, &args.MinIOConfig) 92 | commentClient := commentpb.NewCommentClient(commentClientConn) 93 | 94 | svc := service.NewService(videoDAO, storage, commentClient, producer) 95 | 96 | logger.Info("listen to gRPC addr", zap.String("grpc_addr", args.GRPCAddr)) 97 | lis, err := net.Listen("tcp", args.GRPCAddr) 98 | if err != nil { 99 | logger.Fatal("failed to listen gRPC addr", zap.Error(err)) 100 | } 101 | defer func() { 102 | if err := lis.Close(); err != nil { 103 | logger.Fatal("failed to close gRPC listener", zap.Error(err)) 104 | } 105 | }() 106 | 107 | meter := otelkit.NewPrometheusServiceMeter(ctx, &args.PrometheusServiceMeterConfig) 108 | defer func() { 109 | if err := meter.Close(); err != nil { 110 | logger.Fatal("failed to close meter", zap.Error(err)) 111 | } 112 | }() 113 | 114 | return runkit.GracefulRun(serveGRPC(lis, svc, logger, grpc.UnaryInterceptor(meter.UnaryServerInterceptor())), &args.GracefulConfig) 115 | } 116 | 117 | func serveGRPC(lis net.Listener, svc pb.VideoServer, logger *logkit.Logger, opt ...grpc.ServerOption) runkit.GracefulRunFunc { 118 | grpcServer := grpc.NewServer(opt...) 119 | pb.RegisterVideoServer(grpcServer, svc) 120 | 121 | return func(ctx context.Context) error { 122 | go func() { 123 | if err := grpcServer.Serve(lis); err != nil { 124 | logger.Error("failed to run gRPC server", zap.Error(err)) 125 | } 126 | }() 127 | 128 | <-ctx.Done() 129 | 130 | grpcServer.GracefulStop() 131 | 132 | return nil 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /pkg/otelkit/prometheus.go: -------------------------------------------------------------------------------- 1 | package otelkit 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 9 | "go.opentelemetry.io/otel/attribute" 10 | "go.opentelemetry.io/otel/exporters/prometheus" 11 | "go.opentelemetry.io/otel/metric" 12 | "go.opentelemetry.io/otel/metric/instrument" 13 | "go.opentelemetry.io/otel/metric/instrument/syncint64" 14 | "go.opentelemetry.io/otel/sdk/metric/aggregator/histogram" 15 | controller "go.opentelemetry.io/otel/sdk/metric/controller/basic" 16 | "go.opentelemetry.io/otel/sdk/metric/export/aggregation" 17 | processor "go.opentelemetry.io/otel/sdk/metric/processor/basic" 18 | selector "go.opentelemetry.io/otel/sdk/metric/selector/simple" 19 | "go.uber.org/zap" 20 | "google.golang.org/grpc" 21 | ) 22 | 23 | type PrometheusServiceMeterConfig struct { 24 | Addr string `long:"addr" env:"ADDR" description:"the prometheus exporter address" default:":2222"` 25 | Path string `long:"path" env:"PATH" description:"the prometheus exporter path" default:"/metrics"` 26 | Name string `long:"name" env:"NAME" description:"the unique instrumentation name" required:"true"` 27 | HistogramBoundaries []float64 `long:"histogram_boundaries" env:"HISTOGRAM_BOUNDARIES" env-delim:"," description:"the default histogram boundaries of prometheus" required:"true"` 28 | } 29 | 30 | // PrometheusServiceMeter provides 3 meters to measure: 31 | // 1. Count number of requests 32 | // 2. Measure response time 33 | // 3. Count number of error requests 34 | type PrometheusServiceMeter struct { 35 | metric.Meter 36 | 37 | server *http.Server 38 | requestCounter syncint64.Counter 39 | requestErrorCounter syncint64.Counter 40 | responseTimeHistogram syncint64.Histogram 41 | } 42 | 43 | // UnaryServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Unary RPCs. 44 | func (m *PrometheusServiceMeter) UnaryServerInterceptor() func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 45 | return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 46 | attributes := []attribute.KeyValue{ 47 | attribute.String("FullMethod", info.FullMethod), 48 | } 49 | 50 | // count request 51 | m.requestCounter.Add(ctx, 1, attributes...) 52 | 53 | start := time.Now() 54 | 55 | resp, err := handler(ctx, req) 56 | 57 | // error count request 58 | if err != nil { 59 | m.requestErrorCounter.Add(ctx, 1, attributes...) 60 | } 61 | 62 | // measure response time 63 | responseTime := time.Since(start) 64 | m.responseTimeHistogram.Record(ctx, responseTime.Milliseconds(), attributes...) 65 | 66 | return resp, err 67 | } 68 | } 69 | 70 | func (m *PrometheusServiceMeter) Close() error { 71 | return m.server.Close() 72 | } 73 | 74 | func NewPrometheusServiceMeter(ctx context.Context, conf *PrometheusServiceMeterConfig) *PrometheusServiceMeter { 75 | logger := logkit.FromContext(ctx).With( 76 | zap.String("path", conf.Path), 77 | zap.String("port", conf.Addr), 78 | zap.String("name", conf.Name), 79 | ) 80 | 81 | exporter := newPrometheusExporter(conf, logger) 82 | server := newPrometheusServer(exporter, conf, logger) 83 | 84 | meter := exporter.MeterProvider().Meter(conf.Name) 85 | 86 | requestCounter, err := meter.SyncInt64().Counter("request", instrument.WithDescription("count number of requests")) 87 | if err != nil { 88 | logger.Fatal("failed to create requests counter", zap.Error(err)) 89 | } 90 | 91 | requestErrorCounter, err := meter.SyncInt64().Counter("error_request", instrument.WithDescription("count number of error requests")) 92 | if err != nil { 93 | logger.Fatal("failed to create error requests counter", zap.Error(err)) 94 | } 95 | 96 | responseTimeHistogram, err := meter.SyncInt64().Histogram("response_time", instrument.WithDescription("measure response time")) 97 | if err != nil { 98 | logger.Fatal("failed to create response time histogram", zap.Error(err)) 99 | } 100 | 101 | return &PrometheusServiceMeter{ 102 | server: server, 103 | requestCounter: requestCounter, 104 | requestErrorCounter: requestErrorCounter, 105 | responseTimeHistogram: responseTimeHistogram, 106 | } 107 | } 108 | 109 | func newPrometheusExporter(conf *PrometheusServiceMeterConfig, logger *logkit.Logger) *prometheus.Exporter { 110 | config := prometheus.Config{ 111 | DefaultHistogramBoundaries: conf.HistogramBoundaries, 112 | } 113 | 114 | c := controller.New( 115 | processor.NewFactory( 116 | selector.NewWithHistogramDistribution( 117 | histogram.WithExplicitBoundaries(config.DefaultHistogramBoundaries), 118 | ), 119 | aggregation.CumulativeTemporalitySelector(), 120 | processor.WithMemory(true), 121 | ), 122 | ) 123 | 124 | exporter, err := prometheus.New(config, c) 125 | if err != nil { 126 | logger.Fatal("failed to create prometheus exporter", zap.Error(err)) 127 | } 128 | 129 | return exporter 130 | } 131 | 132 | func newPrometheusServer(exporter *prometheus.Exporter, conf *PrometheusServiceMeterConfig, logger *logkit.Logger) *http.Server { 133 | server := &http.Server{ 134 | Addr: conf.Addr, 135 | Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 136 | if r.Method == http.MethodGet && r.URL.Path == conf.Path { 137 | exporter.ServeHTTP(w, r) 138 | } else { 139 | http.NotFound(w, r) 140 | } 141 | }), 142 | ReadHeaderTimeout: 10 * time.Second, 143 | } 144 | 145 | go func() { 146 | if err := server.ListenAndServe(); err != nil { 147 | logger.Error("failed to serve prometheus exporter", zap.Error(err)) 148 | } 149 | }() 150 | 151 | logger.Info("serve prometheus exporter successfully") 152 | 153 | return server 154 | } 155 | -------------------------------------------------------------------------------- /modules/video/dao/video_redis_test.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-redis/cache/v8" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | . "github.com/onsi/gomega/gstruct" 10 | "github.com/onsi/gomega/types" 11 | "go.mongodb.org/mongo-driver/bson/primitive" 12 | ) 13 | 14 | var _ = Describe("VideoRedisDAO", func() { 15 | var redisVideoDAO *redisVideoDAO 16 | var mongoVideoDAO *mongoVideoDAO 17 | var ctx context.Context 18 | 19 | BeforeEach(func() { 20 | ctx = context.Background() 21 | mongoVideoDAO = NewMongoVideoDAO(mongoClient.Database().Collection("videos")) 22 | redisVideoDAO = NewRedisVideoDAO(redisClient, mongoVideoDAO) 23 | }) 24 | 25 | Describe("Get", func() { 26 | var ( 27 | video *Video 28 | id primitive.ObjectID 29 | 30 | resp *Video 31 | err error 32 | ) 33 | 34 | BeforeEach(func() { 35 | video = NewFakeVideo() 36 | id = video.ID 37 | }) 38 | 39 | JustBeforeEach(func() { 40 | resp, err = redisVideoDAO.Get(ctx, id) 41 | }) 42 | 43 | Context("cache hit", func() { 44 | BeforeEach(func() { 45 | insertVideoInRedis(ctx, redisVideoDAO, video) 46 | }) 47 | 48 | AfterEach(func() { 49 | deleteVideoInRedis(ctx, redisVideoDAO, getVideoKey(video.ID)) 50 | }) 51 | 52 | When("success", func() { 53 | BeforeEach(func() { id = video.ID }) 54 | 55 | It("returns the video with no error", func() { 56 | Expect(resp).To(matchVideo(video)) 57 | Expect(err).NotTo(HaveOccurred()) 58 | }) 59 | }) 60 | }) 61 | 62 | Context("cache miss", func() { 63 | BeforeEach(func() { 64 | insertVideo(ctx, mongoVideoDAO, video) 65 | }) 66 | 67 | AfterEach(func() { 68 | deleteVideo(ctx, mongoVideoDAO, video.ID) 69 | deleteVideoInRedis(ctx, redisVideoDAO, getVideoKey(video.ID)) 70 | }) 71 | 72 | When("video not found", func() { 73 | BeforeEach(func() { id = primitive.NewObjectID() }) 74 | 75 | It("returns video not found error", func() { 76 | Expect(resp).To(BeNil()) 77 | Expect(err).To(MatchError(ErrVideoNotFound)) 78 | }) 79 | }) 80 | 81 | When("success", func() { 82 | BeforeEach(func() { id = video.ID }) 83 | 84 | It("returns the video with no error", func() { 85 | Expect(resp).To(matchVideo(video)) 86 | Expect(err).NotTo(HaveOccurred()) 87 | }) 88 | 89 | It("insert the video to cache", func() { 90 | var getVideo Video 91 | 92 | Expect( 93 | redisVideoDAO.cache.Get(ctx, getVideoKey(id), &getVideo), 94 | ).NotTo(HaveOccurred()) 95 | Expect(resp).To(matchVideo(video)) 96 | }) 97 | }) 98 | }) 99 | }) 100 | 101 | Describe("List", func() { 102 | var ( 103 | videos []*Video 104 | limit int64 105 | skip int64 106 | 107 | resp []*Video 108 | err error 109 | ) 110 | 111 | BeforeEach(func() { 112 | videos = []*Video{NewFakeVideo(), NewFakeVideo(), NewFakeVideo()} 113 | }) 114 | 115 | JustBeforeEach(func() { 116 | resp, err = redisVideoDAO.List(ctx, limit, skip) 117 | }) 118 | 119 | Context("cache hit", func() { 120 | BeforeEach(func() { 121 | limit, skip = 3, 0 122 | insertVideosInRedis(ctx, redisVideoDAO, videos, limit, skip) 123 | }) 124 | 125 | AfterEach(func() { 126 | deleteVideosInRedis(ctx, redisVideoDAO, limit, skip) 127 | }) 128 | 129 | When("success", func() { 130 | It("returns the videos with no error", func() { 131 | for i := range resp { 132 | Expect(resp[i]).To(matchVideo(videos[i])) 133 | } 134 | Expect(err).NotTo(HaveOccurred()) 135 | }) 136 | }) 137 | }) 138 | 139 | Context("cache miss", func() { 140 | BeforeEach(func() { 141 | limit, skip = 3, 0 142 | for _, video := range videos { 143 | insertVideo(ctx, mongoVideoDAO, video) 144 | } 145 | }) 146 | 147 | AfterEach(func() { 148 | for _, video := range videos { 149 | deleteVideo(ctx, mongoVideoDAO, video.ID) 150 | } 151 | deleteVideoInRedis(ctx, redisVideoDAO, listVideoKey(limit, skip)) 152 | }) 153 | 154 | When("videos not found", func() { 155 | BeforeEach(func() { limit, skip = 4, 4 }) 156 | 157 | It("returns empty list with no error", func() { 158 | Expect(resp).To(HaveLen(0)) 159 | Expect(err).NotTo(HaveOccurred()) 160 | }) 161 | }) 162 | 163 | When("success", func() { 164 | It("returns the videos with no error", func() { 165 | for i := range resp { 166 | Expect(resp[i]).To(matchVideo(videos[i])) 167 | } 168 | Expect(err).NotTo(HaveOccurred()) 169 | }) 170 | 171 | It("insert the videos to cache", func() { 172 | var getVideos []*Video 173 | Expect(redisVideoDAO.cache.Get(ctx, listVideoKey(limit, skip), &getVideos)).NotTo(HaveOccurred()) 174 | for i := range getVideos { 175 | Expect(getVideos[i]).To(matchVideo(videos[i])) 176 | } 177 | }) 178 | }) 179 | }) 180 | }) 181 | }) 182 | 183 | func insertVideoInRedis(ctx context.Context, videoDAO *redisVideoDAO, video *Video) { 184 | Expect(videoDAO.cache.Set(&cache.Item{ 185 | Ctx: ctx, 186 | Key: getVideoKey(video.ID), 187 | Value: video, 188 | TTL: videoDAORedisCacheDuration, 189 | })).NotTo(HaveOccurred()) 190 | } 191 | 192 | func deleteVideoInRedis(ctx context.Context, videoDAO *redisVideoDAO, key string) { 193 | Expect(videoDAO.cache.Delete(ctx, key)).NotTo(HaveOccurred()) 194 | } 195 | 196 | func insertVideosInRedis(ctx context.Context, videoDAO *redisVideoDAO, videos []*Video, limit int64, skip int64) { 197 | Expect(videoDAO.cache.Set(&cache.Item{ 198 | Ctx: ctx, 199 | Key: listVideoKey(limit, skip), 200 | Value: videos, 201 | TTL: videoDAORedisCacheDuration, 202 | })).NotTo(HaveOccurred()) 203 | } 204 | 205 | func deleteVideosInRedis(ctx context.Context, videoDAO *redisVideoDAO, limit int64, skip int64) { 206 | Expect(videoDAO.cache.Delete(ctx, listVideoKey(limit, skip))).NotTo(HaveOccurred()) 207 | } 208 | 209 | func matchVideo(video *Video) types.GomegaMatcher { 210 | return PointTo(MatchFields(IgnoreExtras, Fields{ 211 | "ID": Equal(video.ID), 212 | "Width": Equal(video.Width), 213 | "Height": Equal(video.Height), 214 | "Size": Equal(video.Size), 215 | "Duration": Equal(video.Duration), 216 | "URL": Equal(video.URL), 217 | "Status": Equal(video.Status), 218 | "Variants": Equal(video.Variants), 219 | })) 220 | } 221 | -------------------------------------------------------------------------------- /pkg/otelkit/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package otelkit 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/NTHU-LSALAB/NTHU-Distributed-System/pkg/logkit" 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | . "github.com/onsi/gomega/gstruct" 13 | "github.com/onsi/gomega/types" 14 | prompb "github.com/prometheus/client_model/go" 15 | "github.com/prometheus/common/expfmt" 16 | "google.golang.org/grpc" 17 | ) 18 | 19 | var errUnknown = errors.New("unknown error") 20 | 21 | var _ = Describe("PrometheusServiceMeter", func() { 22 | var ( 23 | ctx context.Context 24 | conf *PrometheusServiceMeterConfig 25 | meter *PrometheusServiceMeter 26 | interceptor grpc.UnaryServerInterceptor 27 | ) 28 | 29 | BeforeEach(func() { 30 | ctx = context.Background() 31 | ctx = logkit.NewNopLogger().WithContext(ctx) 32 | 33 | conf = &PrometheusServiceMeterConfig{ 34 | Addr: ":52222", 35 | Path: "/metrics", 36 | Name: "test_prometheus_service_meter", 37 | HistogramBoundaries: []float64{10, 100}, 38 | } 39 | 40 | meter = NewPrometheusServiceMeter(ctx, conf) 41 | time.Sleep(50 * time.Millisecond) // wait prometheus exporter server to start 42 | 43 | interceptor = meter.UnaryServerInterceptor() 44 | }) 45 | 46 | AfterEach(func() { 47 | Expect(meter.Close()).NotTo(HaveOccurred()) 48 | }) 49 | 50 | Context("single handler success", func() { 51 | var ( 52 | handler grpc.UnaryHandler 53 | responseTime time.Duration 54 | handlerResp interface{} 55 | resp interface{} 56 | err error 57 | ) 58 | 59 | BeforeEach(func() { 60 | handlerResp = "fake response" 61 | }) 62 | 63 | JustBeforeEach(func() { 64 | handler = func(ctx context.Context, req interface{}) (interface{}, error) { 65 | time.Sleep(responseTime) 66 | return handlerResp, nil 67 | } 68 | 69 | resp, err = interceptor(ctx, nil, &grpc.UnaryServerInfo{ 70 | FullMethod: "test_handler_success", 71 | }, handler) 72 | }) 73 | 74 | for _, t := range []time.Duration{5 * time.Millisecond, 50 * time.Millisecond, 150 * time.Millisecond} { 75 | t := t 76 | 77 | When("handler takes "+responseTime.String()+" to finish", func() { 78 | BeforeEach(func() { responseTime = t }) 79 | 80 | It("record metrics correctly", func() { 81 | validateCounter(ctx, conf, "request", 1) 82 | validateHistogram(ctx, conf, "response_time", []float64{float64(responseTime.Milliseconds())}) 83 | }) 84 | 85 | It("does not change handler response", func() { 86 | Expect(resp).To(Equal(handlerResp)) 87 | Expect(err).NotTo(HaveOccurred()) 88 | }) 89 | }) 90 | } 91 | }) 92 | 93 | Context("single handler fail", func() { 94 | var ( 95 | handler grpc.UnaryHandler 96 | responseTime time.Duration 97 | handlerResp interface{} 98 | resp interface{} 99 | err error 100 | ) 101 | 102 | BeforeEach(func() { 103 | handlerResp = "fake response" 104 | }) 105 | 106 | JustBeforeEach(func() { 107 | handler = func(ctx context.Context, req interface{}) (interface{}, error) { 108 | time.Sleep(responseTime) 109 | return handlerResp, errUnknown 110 | } 111 | 112 | resp, err = interceptor(ctx, nil, &grpc.UnaryServerInfo{ 113 | FullMethod: "test_handler_fail_and_success", 114 | }, handler) 115 | }) 116 | 117 | for _, t := range []time.Duration{5 * time.Millisecond, 50 * time.Millisecond, 150 * time.Millisecond} { 118 | t := t 119 | 120 | When("handler takes "+responseTime.String()+" to finish", func() { 121 | BeforeEach(func() { responseTime = t }) 122 | 123 | It("record metrics correctly", func() { 124 | validateCounter(ctx, conf, "request", 1) 125 | validateCounter(ctx, conf, "error_request", 1) 126 | validateHistogram(ctx, conf, "response_time", []float64{float64(responseTime.Milliseconds())}) 127 | }) 128 | 129 | It("does not change handler response", func() { 130 | Expect(resp).To(Equal(handlerResp)) 131 | Expect(err).To(MatchError(errUnknown)) 132 | }) 133 | }) 134 | } 135 | }) 136 | }) 137 | 138 | func validateCounter(ctx context.Context, conf *PrometheusServiceMeterConfig, name string, count int) { 139 | mf := parseMetric(ctx, conf, name) 140 | 141 | Expect(mf.GetName()).To(Equal(name)) 142 | Expect(mf.GetType().String()).To(Equal("COUNTER")) 143 | Expect(mf.GetMetric()).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ 144 | "Counter": PointTo(MatchFields(IgnoreExtras, Fields{ 145 | "Value": PointTo(Equal(float64(count))), 146 | })), 147 | })))) 148 | } 149 | 150 | func validateHistogram(ctx context.Context, conf *PrometheusServiceMeterConfig, name string, values []float64) { 151 | mf := parseMetric(ctx, conf, name) 152 | 153 | Expect(mf.GetName()).To(Equal(name)) 154 | Expect(mf.GetType().String()).To(Equal("HISTOGRAM")) 155 | Expect(mf.GetMetric()).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ 156 | "Histogram": PointTo(MatchFields(IgnoreExtras, Fields{ 157 | "SampleCount": PointTo(Equal(uint64(len(values)))), 158 | "Bucket": matchBucket(conf, values), 159 | })), 160 | })))) 161 | } 162 | 163 | func parseMetric(ctx context.Context, conf *PrometheusServiceMeterConfig, name string) *prompb.MetricFamily { 164 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+conf.Addr+conf.Path, http.NoBody) 165 | Expect(err).NotTo(HaveOccurred()) 166 | 167 | resp, err := http.DefaultClient.Do(req) 168 | Expect(err).NotTo(HaveOccurred()) 169 | 170 | defer func() { 171 | Expect(resp.Body.Close()).NotTo(HaveOccurred()) 172 | }() 173 | 174 | var parser expfmt.TextParser 175 | mfs, err := parser.TextToMetricFamilies(resp.Body) 176 | Expect(err).NotTo(HaveOccurred()) 177 | 178 | return mfs[name] 179 | } 180 | 181 | func matchBucket(conf *PrometheusServiceMeterConfig, values []float64) types.GomegaMatcher { 182 | matcher := make([]types.GomegaMatcher, 0, len(conf.HistogramBoundaries)) 183 | 184 | for _, bound := range conf.HistogramBoundaries { 185 | count := uint64(0) 186 | for _, value := range values { 187 | if value <= bound { 188 | count++ 189 | } 190 | } 191 | 192 | matcher = append(matcher, PointTo(MatchFields(IgnoreExtras, Fields{ 193 | "CumulativeCount": PointTo(Equal(count)), 194 | "UpperBound": PointTo(Equal(bound)), 195 | }))) 196 | } 197 | 198 | return ContainElements(matcher) 199 | } 200 | -------------------------------------------------------------------------------- /modules/comment/mock/pbmock/mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb (interfaces: CommentClient) 3 | 4 | // Package pbmock is a generated GoMock package. 5 | package pbmock 6 | 7 | import ( 8 | context "context" 9 | reflect "reflect" 10 | 11 | pb "github.com/NTHU-LSALAB/NTHU-Distributed-System/modules/comment/pb" 12 | gomock "github.com/golang/mock/gomock" 13 | grpc "google.golang.org/grpc" 14 | ) 15 | 16 | // MockCommentClient is a mock of CommentClient interface. 17 | type MockCommentClient struct { 18 | ctrl *gomock.Controller 19 | recorder *MockCommentClientMockRecorder 20 | } 21 | 22 | // MockCommentClientMockRecorder is the mock recorder for MockCommentClient. 23 | type MockCommentClientMockRecorder struct { 24 | mock *MockCommentClient 25 | } 26 | 27 | // NewMockCommentClient creates a new mock instance. 28 | func NewMockCommentClient(ctrl *gomock.Controller) *MockCommentClient { 29 | mock := &MockCommentClient{ctrl: ctrl} 30 | mock.recorder = &MockCommentClientMockRecorder{mock} 31 | return mock 32 | } 33 | 34 | // EXPECT returns an object that allows the caller to indicate expected use. 35 | func (m *MockCommentClient) EXPECT() *MockCommentClientMockRecorder { 36 | return m.recorder 37 | } 38 | 39 | // CreateComment mocks base method. 40 | func (m *MockCommentClient) CreateComment(arg0 context.Context, arg1 *pb.CreateCommentRequest, arg2 ...grpc.CallOption) (*pb.CreateCommentResponse, error) { 41 | m.ctrl.T.Helper() 42 | varargs := []interface{}{arg0, arg1} 43 | for _, a := range arg2 { 44 | varargs = append(varargs, a) 45 | } 46 | ret := m.ctrl.Call(m, "CreateComment", varargs...) 47 | ret0, _ := ret[0].(*pb.CreateCommentResponse) 48 | ret1, _ := ret[1].(error) 49 | return ret0, ret1 50 | } 51 | 52 | // CreateComment indicates an expected call of CreateComment. 53 | func (mr *MockCommentClientMockRecorder) CreateComment(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 54 | mr.mock.ctrl.T.Helper() 55 | varargs := append([]interface{}{arg0, arg1}, arg2...) 56 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateComment", reflect.TypeOf((*MockCommentClient)(nil).CreateComment), varargs...) 57 | } 58 | 59 | // DeleteComment mocks base method. 60 | func (m *MockCommentClient) DeleteComment(arg0 context.Context, arg1 *pb.DeleteCommentRequest, arg2 ...grpc.CallOption) (*pb.DeleteCommentResponse, error) { 61 | m.ctrl.T.Helper() 62 | varargs := []interface{}{arg0, arg1} 63 | for _, a := range arg2 { 64 | varargs = append(varargs, a) 65 | } 66 | ret := m.ctrl.Call(m, "DeleteComment", varargs...) 67 | ret0, _ := ret[0].(*pb.DeleteCommentResponse) 68 | ret1, _ := ret[1].(error) 69 | return ret0, ret1 70 | } 71 | 72 | // DeleteComment indicates an expected call of DeleteComment. 73 | func (mr *MockCommentClientMockRecorder) DeleteComment(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 74 | mr.mock.ctrl.T.Helper() 75 | varargs := append([]interface{}{arg0, arg1}, arg2...) 76 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteComment", reflect.TypeOf((*MockCommentClient)(nil).DeleteComment), varargs...) 77 | } 78 | 79 | // DeleteCommentByVideoID mocks base method. 80 | func (m *MockCommentClient) DeleteCommentByVideoID(arg0 context.Context, arg1 *pb.DeleteCommentByVideoIDRequest, arg2 ...grpc.CallOption) (*pb.DeleteCommentByVideoIDResponse, error) { 81 | m.ctrl.T.Helper() 82 | varargs := []interface{}{arg0, arg1} 83 | for _, a := range arg2 { 84 | varargs = append(varargs, a) 85 | } 86 | ret := m.ctrl.Call(m, "DeleteCommentByVideoID", varargs...) 87 | ret0, _ := ret[0].(*pb.DeleteCommentByVideoIDResponse) 88 | ret1, _ := ret[1].(error) 89 | return ret0, ret1 90 | } 91 | 92 | // DeleteCommentByVideoID indicates an expected call of DeleteCommentByVideoID. 93 | func (mr *MockCommentClientMockRecorder) DeleteCommentByVideoID(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 94 | mr.mock.ctrl.T.Helper() 95 | varargs := append([]interface{}{arg0, arg1}, arg2...) 96 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCommentByVideoID", reflect.TypeOf((*MockCommentClient)(nil).DeleteCommentByVideoID), varargs...) 97 | } 98 | 99 | // Healthz mocks base method. 100 | func (m *MockCommentClient) Healthz(arg0 context.Context, arg1 *pb.HealthzRequest, arg2 ...grpc.CallOption) (*pb.HealthzResponse, error) { 101 | m.ctrl.T.Helper() 102 | varargs := []interface{}{arg0, arg1} 103 | for _, a := range arg2 { 104 | varargs = append(varargs, a) 105 | } 106 | ret := m.ctrl.Call(m, "Healthz", varargs...) 107 | ret0, _ := ret[0].(*pb.HealthzResponse) 108 | ret1, _ := ret[1].(error) 109 | return ret0, ret1 110 | } 111 | 112 | // Healthz indicates an expected call of Healthz. 113 | func (mr *MockCommentClientMockRecorder) Healthz(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 114 | mr.mock.ctrl.T.Helper() 115 | varargs := append([]interface{}{arg0, arg1}, arg2...) 116 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Healthz", reflect.TypeOf((*MockCommentClient)(nil).Healthz), varargs...) 117 | } 118 | 119 | // ListComment mocks base method. 120 | func (m *MockCommentClient) ListComment(arg0 context.Context, arg1 *pb.ListCommentRequest, arg2 ...grpc.CallOption) (*pb.ListCommentResponse, error) { 121 | m.ctrl.T.Helper() 122 | varargs := []interface{}{arg0, arg1} 123 | for _, a := range arg2 { 124 | varargs = append(varargs, a) 125 | } 126 | ret := m.ctrl.Call(m, "ListComment", varargs...) 127 | ret0, _ := ret[0].(*pb.ListCommentResponse) 128 | ret1, _ := ret[1].(error) 129 | return ret0, ret1 130 | } 131 | 132 | // ListComment indicates an expected call of ListComment. 133 | func (mr *MockCommentClientMockRecorder) ListComment(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 134 | mr.mock.ctrl.T.Helper() 135 | varargs := append([]interface{}{arg0, arg1}, arg2...) 136 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListComment", reflect.TypeOf((*MockCommentClient)(nil).ListComment), varargs...) 137 | } 138 | 139 | // UpdateComment mocks base method. 140 | func (m *MockCommentClient) UpdateComment(arg0 context.Context, arg1 *pb.UpdateCommentRequest, arg2 ...grpc.CallOption) (*pb.UpdateCommentResponse, error) { 141 | m.ctrl.T.Helper() 142 | varargs := []interface{}{arg0, arg1} 143 | for _, a := range arg2 { 144 | varargs = append(varargs, a) 145 | } 146 | ret := m.ctrl.Call(m, "UpdateComment", varargs...) 147 | ret0, _ := ret[0].(*pb.UpdateCommentResponse) 148 | ret1, _ := ret[1].(error) 149 | return ret0, ret1 150 | } 151 | 152 | // UpdateComment indicates an expected call of UpdateComment. 153 | func (mr *MockCommentClientMockRecorder) UpdateComment(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 154 | mr.mock.ctrl.T.Helper() 155 | varargs := append([]interface{}{arg0, arg1}, arg2...) 156 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateComment", reflect.TypeOf((*MockCommentClient)(nil).UpdateComment), varargs...) 157 | } 158 | -------------------------------------------------------------------------------- /modules/video/pb/rpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.27.1 4 | // protoc v3.19.3 5 | // source: modules/video/pb/rpc.proto 6 | 7 | package pb 8 | 9 | import ( 10 | _ "google.golang.org/genproto/googleapis/api/annotations" 11 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 12 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 13 | reflect "reflect" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | var File_modules_video_pb_rpc_proto protoreflect.FileDescriptor 24 | 25 | var file_modules_video_pb_rpc_proto_rawDesc = []byte{ 26 | 0x0a, 0x1a, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x2f, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x2f, 27 | 0x70, 0x62, 0x2f, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x76, 0x69, 28 | 0x64, 0x65, 0x6f, 0x2e, 0x70, 0x62, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 29 | 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 30 | 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x2f, 0x76, 0x69, 31 | 0x64, 0x65, 0x6f, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 32 | 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xca, 0x03, 0x0a, 0x05, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x49, 33 | 0x0a, 0x07, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x7a, 0x12, 0x18, 0x2e, 0x76, 0x69, 0x64, 0x65, 34 | 0x6f, 0x2e, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x7a, 0x52, 0x65, 0x71, 0x75, 35 | 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x2e, 0x70, 0x62, 0x2e, 0x48, 36 | 0x65, 0x61, 0x6c, 0x74, 0x68, 0x7a, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x09, 37 | 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x03, 0x12, 0x01, 0x2f, 0x12, 0x61, 0x0a, 0x08, 0x47, 0x65, 0x74, 38 | 0x56, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x19, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x2e, 0x70, 0x62, 39 | 0x2e, 0x47, 0x65, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 40 | 0x1a, 0x1a, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x56, 41 | 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1e, 0x82, 0xd3, 42 | 0xe4, 0x93, 0x02, 0x18, 0x12, 0x0f, 0x2f, 0x76, 0x31, 0x2f, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x73, 43 | 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x62, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x5b, 0x0a, 0x09, 44 | 0x4c, 0x69, 0x73, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x1a, 0x2e, 0x76, 0x69, 0x64, 0x65, 45 | 0x6f, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 46 | 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x2e, 0x70, 0x62, 47 | 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 48 | 0x73, 0x65, 0x22, 0x15, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0f, 0x12, 0x0a, 0x2f, 0x76, 0x31, 0x2f, 49 | 0x76, 0x69, 0x64, 0x65, 0x6f, 0x73, 0x62, 0x01, 0x2a, 0x12, 0x4e, 0x0a, 0x0b, 0x55, 0x70, 0x6c, 50 | 0x6f, 0x61, 0x64, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x1c, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 51 | 0x2e, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 52 | 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x2e, 0x70, 53 | 0x62, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x73, 54 | 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x66, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 55 | 0x65, 0x74, 0x65, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x1c, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 56 | 0x2e, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 57 | 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x2e, 0x70, 58 | 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x52, 0x65, 0x73, 59 | 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x14, 0x2a, 0x0f, 0x2f, 60 | 0x76, 0x31, 0x2f, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x62, 0x01, 61 | 0x2a, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 62 | 0x4e, 0x54, 0x48, 0x55, 0x2d, 0x4c, 0x53, 0x41, 0x4c, 0x41, 0x42, 0x2f, 0x4e, 0x54, 0x48, 0x55, 63 | 0x2d, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x2d, 0x53, 0x79, 0x73, 64 | 0x74, 0x65, 0x6d, 0x2f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x2f, 0x76, 0x69, 0x64, 0x65, 65 | 0x6f, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 66 | } 67 | 68 | var file_modules_video_pb_rpc_proto_goTypes = []interface{}{ 69 | (*HealthzRequest)(nil), // 0: video.pb.HealthzRequest 70 | (*GetVideoRequest)(nil), // 1: video.pb.GetVideoRequest 71 | (*ListVideoRequest)(nil), // 2: video.pb.ListVideoRequest 72 | (*UploadVideoRequest)(nil), // 3: video.pb.UploadVideoRequest 73 | (*DeleteVideoRequest)(nil), // 4: video.pb.DeleteVideoRequest 74 | (*HealthzResponse)(nil), // 5: video.pb.HealthzResponse 75 | (*GetVideoResponse)(nil), // 6: video.pb.GetVideoResponse 76 | (*ListVideoResponse)(nil), // 7: video.pb.ListVideoResponse 77 | (*UploadVideoResponse)(nil), // 8: video.pb.UploadVideoResponse 78 | (*DeleteVideoResponse)(nil), // 9: video.pb.DeleteVideoResponse 79 | } 80 | var file_modules_video_pb_rpc_proto_depIdxs = []int32{ 81 | 0, // 0: video.pb.Video.Healthz:input_type -> video.pb.HealthzRequest 82 | 1, // 1: video.pb.Video.GetVideo:input_type -> video.pb.GetVideoRequest 83 | 2, // 2: video.pb.Video.ListVideo:input_type -> video.pb.ListVideoRequest 84 | 3, // 3: video.pb.Video.UploadVideo:input_type -> video.pb.UploadVideoRequest 85 | 4, // 4: video.pb.Video.DeleteVideo:input_type -> video.pb.DeleteVideoRequest 86 | 5, // 5: video.pb.Video.Healthz:output_type -> video.pb.HealthzResponse 87 | 6, // 6: video.pb.Video.GetVideo:output_type -> video.pb.GetVideoResponse 88 | 7, // 7: video.pb.Video.ListVideo:output_type -> video.pb.ListVideoResponse 89 | 8, // 8: video.pb.Video.UploadVideo:output_type -> video.pb.UploadVideoResponse 90 | 9, // 9: video.pb.Video.DeleteVideo:output_type -> video.pb.DeleteVideoResponse 91 | 5, // [5:10] is the sub-list for method output_type 92 | 0, // [0:5] is the sub-list for method input_type 93 | 0, // [0:0] is the sub-list for extension type_name 94 | 0, // [0:0] is the sub-list for extension extendee 95 | 0, // [0:0] is the sub-list for field type_name 96 | } 97 | 98 | func init() { file_modules_video_pb_rpc_proto_init() } 99 | func file_modules_video_pb_rpc_proto_init() { 100 | if File_modules_video_pb_rpc_proto != nil { 101 | return 102 | } 103 | file_modules_video_pb_message_proto_init() 104 | type x struct{} 105 | out := protoimpl.TypeBuilder{ 106 | File: protoimpl.DescBuilder{ 107 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 108 | RawDescriptor: file_modules_video_pb_rpc_proto_rawDesc, 109 | NumEnums: 0, 110 | NumMessages: 0, 111 | NumExtensions: 0, 112 | NumServices: 1, 113 | }, 114 | GoTypes: file_modules_video_pb_rpc_proto_goTypes, 115 | DependencyIndexes: file_modules_video_pb_rpc_proto_depIdxs, 116 | }.Build() 117 | File_modules_video_pb_rpc_proto = out.File 118 | file_modules_video_pb_rpc_proto_rawDesc = nil 119 | file_modules_video_pb_rpc_proto_goTypes = nil 120 | file_modules_video_pb_rpc_proto_depIdxs = nil 121 | } 122 | --------------------------------------------------------------------------------