├── .github └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── README.md ├── go.mod ├── lexer.go └── markut.go /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | go-build: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | - uses: actions/setup-go@v2 10 | with: 11 | go-version: '^1.13.1' 12 | - run: go build 13 | - run: | 14 | go fmt markut.go 15 | git diff --exit-code 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.mp4 2 | *.csv 3 | *.txt 4 | markut 5 | markut.exe 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2021 Alexey Kutepov 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Markut 2 | 3 | You describe how you want to edit your video in a `MARKUT` file using a simple [Stack-Based Language](https://en.wikipedia.org/wiki/Stack-oriented_programming) and Markut translates it to a sequence of ffmpeg command and assembles the final video. I'm using this tools to edit my VODs that I upload at [Tsoding Daily](https://youtube.com/@TsodingDaily) YouTube channel. 4 | 5 | ## Quick Start 6 | 7 | Install [Go](https://golang.org/) and [ffmpeg](https://www.ffmpeg.org/). 8 | 9 | ```console 10 | $ go build 11 | ``` 12 | 13 | To get the list of markut subcommands do 14 | 15 | ```console 16 | $ ./markut help 17 | ``` 18 | 19 | To get the list of functions of the stack language do 20 | 21 | ```console 22 | $ ./markut funcs 23 | ``` 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tsoding/markut 2 | 3 | go 1.21 4 | -------------------------------------------------------------------------------- /lexer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "unicode" 6 | "strings" 7 | "strconv" 8 | ) 9 | 10 | type Millis int64 11 | 12 | // TODO: we really need more precise error reported for timestamp tokens. 13 | // Like the diagnostics should point out at specific characters that are 14 | // incorrect. 15 | 16 | func parseSsAndMs(s string) (ss int64, ms Millis, err error) { 17 | switch comps := strings.Split(s, "."); len(comps) { 18 | case 2: 19 | ss, err = strconv.ParseInt(comps[0], 10, 64); 20 | if err != nil { 21 | return 22 | } 23 | runes := []rune(comps[1]) 24 | ms = 0 25 | for i := 0; i < 3; i += 1 { 26 | ms = ms*10 27 | if i < len(runes) { 28 | ms += Millis(runes[i] - '0') 29 | } 30 | } 31 | return 32 | case 1: 33 | ss, err = strconv.ParseInt(comps[0], 10, 64); 34 | if err != nil { 35 | return 36 | } 37 | ms = 0 38 | return 39 | default: 40 | err = fmt.Errorf("Unexpected amount of components in the seconds (%d): %s", len(comps), s) 41 | return 42 | } 43 | } 44 | 45 | func tsToMillis(ts string) (Millis, error) { 46 | var err error = nil 47 | var mm, hh, ss int64 = 0, 0, 0 48 | var ms Millis = 0 49 | index := 0 50 | switch comps := strings.Split(ts, ":"); len(comps) { 51 | case 3: 52 | hh, err = strconv.ParseInt(comps[index], 10, 64) 53 | if err != nil { 54 | return 0, err 55 | } 56 | index += 1 57 | fallthrough 58 | case 2: 59 | mm, err = strconv.ParseInt(comps[index], 10, 64) 60 | if err != nil { 61 | return 0, err 62 | } 63 | index += 1 64 | fallthrough 65 | case 1: 66 | ss, ms, err = parseSsAndMs(comps[index]) 67 | if err != nil { 68 | return 0, err 69 | } 70 | 71 | return 60*60*1000*Millis(hh) + 60*1000*Millis(mm) + Millis(ss)*1000 + ms, nil 72 | default: 73 | return 0, fmt.Errorf("Unexpected amount of components in the timestamp (%d)", len(comps)) 74 | } 75 | } 76 | 77 | type Loc struct { 78 | FilePath string 79 | Row int 80 | Col int 81 | } 82 | 83 | type DiagErr struct { 84 | Loc Loc 85 | Err error 86 | } 87 | 88 | func (err *DiagErr) Error() string { 89 | return fmt.Sprintf("%s: ERROR: %s", err.Loc, err.Err) 90 | } 91 | 92 | func (loc Loc) String() string { 93 | return fmt.Sprintf("%s:%d:%d", loc.FilePath, loc.Row+1, loc.Col+1) 94 | } 95 | 96 | type Lexer struct { 97 | Content []rune 98 | FilePath string 99 | Row int 100 | Cur int 101 | Bol int 102 | PeekBuf Token 103 | PeekFull bool 104 | } 105 | 106 | func NewLexer(content string, filePath string) Lexer { 107 | return Lexer{ 108 | Content: []rune(content), 109 | FilePath: filePath, 110 | } 111 | } 112 | 113 | type TokenKind int 114 | 115 | const ( 116 | TokenEOF TokenKind = iota 117 | TokenSymbol 118 | TokenString 119 | TokenBracketOpen 120 | TokenBracketClose 121 | TokenCurlyOpen 122 | TokenCurlyClose 123 | TokenParenOpen 124 | TokenParenClose 125 | TokenEllipsis 126 | TokenAsterisk 127 | TokenTimestamp 128 | TokenDash 129 | TokenPlus 130 | ) 131 | 132 | var TokenKindName = map[TokenKind]string{ 133 | TokenEOF: "end of file", 134 | TokenSymbol: "symbol", 135 | TokenString: "string literal", 136 | TokenBracketOpen: "open bracket", 137 | TokenBracketClose: "close bracket", 138 | TokenCurlyOpen: "open curly", 139 | TokenCurlyClose: "close curly", 140 | TokenParenOpen: "open paren", 141 | TokenParenClose: "close paren", 142 | TokenEllipsis: "ellipsis", 143 | TokenAsterisk: "asterisk", 144 | TokenTimestamp: "timestamp", 145 | TokenDash: "dash", 146 | TokenPlus: "plus", 147 | } 148 | 149 | type LiteralToken struct { 150 | Text string 151 | Kind TokenKind 152 | } 153 | 154 | var LiteralTokens = []LiteralToken{ 155 | {Text: "[", Kind: TokenBracketOpen}, 156 | {Text: "]", Kind: TokenBracketClose}, 157 | {Text: "{", Kind: TokenCurlyOpen}, 158 | {Text: "}", Kind: TokenCurlyClose}, 159 | {Text: "(", Kind: TokenParenOpen}, 160 | {Text: ")", Kind: TokenParenClose}, 161 | {Text: "...", Kind: TokenEllipsis}, 162 | {Text: "*", Kind: TokenAsterisk}, 163 | {Text: "-", Kind: TokenDash}, 164 | {Text: "+", Kind: TokenPlus}, 165 | } 166 | 167 | type Token struct { 168 | Kind TokenKind 169 | Text []rune 170 | Timestamp Millis 171 | Loc Loc 172 | } 173 | 174 | func (lexer *Lexer) ChopChar() { 175 | if lexer.Cur >= len(lexer.Content) { 176 | return 177 | } 178 | x := lexer.Content[lexer.Cur]; 179 | lexer.Cur += 1; 180 | if x == '\n' { 181 | lexer.Row += 1; 182 | lexer.Bol = lexer.Cur; 183 | } 184 | } 185 | 186 | func (lexer *Lexer) ChopChars(n int) { 187 | for lexer.Cur < len(lexer.Content) && n > 0 { 188 | lexer.ChopChar() 189 | n -= 1 190 | } 191 | } 192 | 193 | func (lexer *Lexer) DropLine() { 194 | for lexer.Cur < len(lexer.Content) && lexer.Content[lexer.Cur] != '\n' { 195 | lexer.ChopChar() 196 | } 197 | if lexer.Cur < len(lexer.Content) { 198 | lexer.ChopChar() 199 | } 200 | } 201 | 202 | func (lexer *Lexer) TrimLeft() { 203 | for lexer.Cur < len(lexer.Content) && unicode.IsSpace(lexer.Content[lexer.Cur]) { 204 | lexer.ChopChar() 205 | } 206 | } 207 | 208 | func (lexer *Lexer) Prefix(prefix []rune) bool { 209 | for i := range prefix { 210 | if lexer.Cur+i >= len(lexer.Content) { 211 | return false 212 | } 213 | if lexer.Content[lexer.Cur+i] != prefix[i] { 214 | return false 215 | } 216 | } 217 | return true 218 | } 219 | 220 | func (lexer *Lexer) Loc() Loc { 221 | return Loc{ 222 | FilePath: lexer.FilePath, 223 | Row: lexer.Row, 224 | Col: lexer.Cur - lexer.Bol, 225 | } 226 | } 227 | 228 | func (lexer *Lexer) ChopHexByteValue() (result rune, err error) { 229 | for i := 0; i < 2; i += 1 { 230 | if lexer.Cur >= len(lexer.Content) { 231 | err = &DiagErr{ 232 | Loc: lexer.Loc(), 233 | Err: fmt.Errorf("Unfinished hexadecimal value of a byte. Expected 2 hex digits, but got %d.", i), 234 | } 235 | return 236 | } 237 | x := lexer.Content[lexer.Cur] 238 | if '0' <= x && x <= '9' { 239 | result = result*0x10 + x - '0' 240 | } else if 'a' <= x && x <= 'f' { 241 | result = result*0x10 + x - 'a' + 10 242 | } else if 'A' <= x && x <= 'F' { 243 | result = result*0x10 + x - 'A' + 10 244 | } else { 245 | err = &DiagErr{ 246 | Loc: lexer.Loc(), 247 | Err: fmt.Errorf("Expected hex digit, but got `%c`", x), 248 | } 249 | return 250 | } 251 | lexer.ChopChar() 252 | } 253 | return 254 | } 255 | 256 | func (lexer *Lexer) ChopStrLit() (lit []rune, err error) { 257 | if lexer.Cur >= len(lexer.Content) { 258 | return 259 | } 260 | 261 | quote := lexer.Content[lexer.Cur] 262 | lexer.ChopChar() 263 | begin := lexer.Cur 264 | 265 | loop: 266 | for lexer.Cur < len(lexer.Content) { 267 | if lexer.Content[lexer.Cur] == '\\' { 268 | lexer.ChopChar() 269 | if lexer.Cur >= len(lexer.Content) { 270 | err = &DiagErr{ 271 | Loc: lexer.Loc(), 272 | Err: fmt.Errorf("Unfinished escape sequence"), 273 | } 274 | return 275 | } 276 | 277 | switch lexer.Content[lexer.Cur] { 278 | case '0': 279 | lit = append(lit, 0) 280 | lexer.ChopChar(); 281 | case 'n': 282 | lit = append(lit, '\n') 283 | lexer.ChopChar(); 284 | case 'r': 285 | lit = append(lit, '\r') 286 | lexer.ChopChar() 287 | case '\\': 288 | lit = append(lit, '\\') 289 | lexer.ChopChar() 290 | case 'x': 291 | lexer.ChopChar() 292 | var value rune 293 | value, err = lexer.ChopHexByteValue() 294 | if err != nil { 295 | return 296 | } 297 | lit = append(lit, value) 298 | default: 299 | if lexer.Content[lexer.Cur] == quote { 300 | lit = append(lit, quote) 301 | lexer.ChopChar() 302 | } else { 303 | err = &DiagErr{ 304 | Loc: lexer.Loc(), 305 | Err: fmt.Errorf("Unknown escape sequence starting with %c", lexer.Content[lexer.Cur]), 306 | } 307 | return 308 | } 309 | } 310 | } else { 311 | if lexer.Content[lexer.Cur] == quote { 312 | break loop 313 | } 314 | lit = append(lit, lexer.Content[lexer.Cur]) 315 | lexer.ChopChar() 316 | } 317 | } 318 | 319 | if lexer.Cur >= len(lexer.Content) || lexer.Content[lexer.Cur] != quote { 320 | err = &DiagErr{ 321 | Loc: Loc{ 322 | FilePath: lexer.FilePath, 323 | Row: lexer.Row, 324 | Col: begin, 325 | }, 326 | Err: fmt.Errorf("Expected '%c' at the end of this string literal", quote), 327 | } 328 | return 329 | } 330 | lexer.ChopChar() 331 | 332 | return 333 | } 334 | 335 | func IsSymbolStart(ch rune) bool { 336 | return unicode.IsLetter(ch) || ch == '_' 337 | } 338 | 339 | func IsSymbol(ch rune) bool { 340 | return unicode.IsLetter(ch) || unicode.IsNumber(ch) || ch == '_' 341 | } 342 | 343 | func IsTimestamp(ch rune) bool { 344 | return unicode.IsNumber(ch) || ch == ':' || ch == '.' 345 | } 346 | 347 | func (lexer *Lexer) ChopToken() (token Token, err error) { 348 | for lexer.Cur < len(lexer.Content) { 349 | lexer.TrimLeft() 350 | 351 | if lexer.Prefix([]rune("//")) { 352 | lexer.DropLine() 353 | continue 354 | } 355 | 356 | if lexer.Prefix([]rune("/*")) { 357 | for lexer.Cur < len(lexer.Content) && !lexer.Prefix([]rune("*/")) { 358 | lexer.ChopChar() 359 | } 360 | if lexer.Prefix([]rune("*/")) { 361 | lexer.ChopChars(2) 362 | } 363 | continue 364 | } 365 | 366 | break 367 | } 368 | 369 | token.Loc = lexer.Loc() 370 | 371 | if lexer.Cur >= len(lexer.Content) { 372 | return 373 | } 374 | 375 | if unicode.IsNumber(lexer.Content[lexer.Cur]) { 376 | token.Kind = TokenTimestamp 377 | begin := lexer.Cur 378 | 379 | for lexer.Cur < len(lexer.Content) && IsTimestamp(lexer.Content[lexer.Cur]) { 380 | lexer.ChopChar() 381 | } 382 | 383 | token.Text = lexer.Content[begin:lexer.Cur] 384 | token.Timestamp, err = tsToMillis(string(token.Text)) 385 | if err != nil { 386 | err = &DiagErr{ 387 | Loc: token.Loc, 388 | Err: fmt.Errorf("Invalid timestamp symbol: %w", err), 389 | } 390 | } 391 | return 392 | } 393 | 394 | if IsSymbolStart(lexer.Content[lexer.Cur]) { 395 | begin := lexer.Cur 396 | 397 | for lexer.Cur < len(lexer.Content) && IsSymbol(lexer.Content[lexer.Cur]) { 398 | lexer.ChopChar() 399 | } 400 | 401 | token.Kind = TokenSymbol 402 | token.Text = lexer.Content[begin:lexer.Cur] 403 | return 404 | } 405 | 406 | if lexer.Content[lexer.Cur] == '"' || lexer.Content[lexer.Cur] == '\'' { 407 | var lit []rune 408 | lit, err = lexer.ChopStrLit() 409 | if err != nil { 410 | return 411 | } 412 | token.Kind = TokenString 413 | token.Text = lit 414 | return 415 | } 416 | 417 | for i := range LiteralTokens { 418 | runeName := []rune(LiteralTokens[i].Text) 419 | if lexer.Prefix(runeName) { 420 | token.Kind = LiteralTokens[i].Kind 421 | token.Text = runeName 422 | lexer.ChopChars(len(runeName)) 423 | return 424 | } 425 | } 426 | 427 | err = &DiagErr{ 428 | Loc: lexer.Loc(), 429 | Err: fmt.Errorf("Invalid token"), 430 | } 431 | return 432 | } 433 | 434 | func (lexer *Lexer) Peek() (token Token, err error) { 435 | if !lexer.PeekFull { 436 | token, err = lexer.ChopToken() 437 | if err != nil { 438 | return 439 | } 440 | lexer.PeekFull = true 441 | lexer.PeekBuf = token 442 | } else { 443 | token = lexer.PeekBuf 444 | } 445 | return 446 | } 447 | 448 | func (lexer *Lexer) Next() (token Token, err error) { 449 | if lexer.PeekFull { 450 | token = lexer.PeekBuf 451 | lexer.PeekFull = false 452 | return 453 | } 454 | 455 | token, err = lexer.ChopToken() 456 | return 457 | } 458 | -------------------------------------------------------------------------------- /markut.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "os" 11 | "os/exec" 12 | "path" 13 | "regexp" 14 | "slices" 15 | "sort" 16 | "strconv" 17 | "strings" 18 | "time" 19 | ) 20 | 21 | func decomposeMillis(millis Millis) (hh int64, mm int64, ss int64, ms int64, sign string) { 22 | sign = "" 23 | if millis < 0 { 24 | sign = "-" 25 | millis = -millis 26 | } 27 | hh = int64(millis / 1000 / 60 / 60) 28 | mm = int64(millis / 1000 / 60 % 60) 29 | ss = int64(millis / 1000 % 60) 30 | ms = int64(millis % 1000) 31 | return 32 | } 33 | 34 | // Timestamp format used by Markut Language 35 | func millisToTs(millis Millis) string { 36 | hh, mm, ss, ms, sign := decomposeMillis(millis) 37 | return fmt.Sprintf("%s%02d:%02d:%02d.%03d", sign, hh, mm, ss, ms) 38 | } 39 | 40 | // Timestamp format used on YouTube 41 | func millisToYouTubeTs(millis Millis) string { 42 | hh, mm, ss, _, sign := decomposeMillis(millis) 43 | return fmt.Sprintf("%s%02d:%02d:%02d", sign, hh, mm, ss) 44 | } 45 | 46 | // Timestamp format used by SubRip https://en.wikipedia.org/wiki/SubRip that we 47 | // use for generating the chat in subtitles on YouTube 48 | func millisToSubRipTs(millis Millis) string { 49 | hh, mm, ss, ms, sign := decomposeMillis(millis) 50 | return fmt.Sprintf("%s%02d:%02d:%02d,%03d", sign, hh, mm, ss, ms) 51 | } 52 | 53 | type ChatMessage struct { 54 | Nickname string 55 | Color string 56 | Text string 57 | } 58 | 59 | type ChatMessageGroup struct { 60 | TimeOffset Millis 61 | Messages []ChatMessage 62 | } 63 | 64 | type Chunk struct { 65 | Start Millis 66 | End Millis 67 | Loc Loc 68 | InputPath string 69 | ChatLog []ChatMessageGroup 70 | Blur bool 71 | Unfinished bool 72 | ExtraOutFlags []Token 73 | } 74 | 75 | const ChunksFolder = "chunks" 76 | const TwitchChatDownloaderCSVHeader = "time,user_name,user_color,message" 77 | 78 | func (chunk Chunk) Name() string { 79 | inputPath := strings.ReplaceAll(chunk.InputPath, "/", "_") 80 | sb := strings.Builder{} 81 | fmt.Fprintf(&sb, "%s/%s-%09d-%09d", ChunksFolder, inputPath, chunk.Start, chunk.End) 82 | if chunk.Blur { 83 | sb.WriteString("-blur") 84 | } 85 | for _, outFlag := range chunk.ExtraOutFlags { 86 | sb.WriteString(strings.ReplaceAll(string(outFlag.Text), "/", "_")) 87 | } 88 | sb.WriteString(".mp4") 89 | return sb.String() 90 | } 91 | 92 | func (chunk Chunk) Duration() Millis { 93 | return chunk.End - chunk.Start 94 | } 95 | 96 | func (chunk Chunk) Rendered() (bool, error) { 97 | _, err := os.Stat(chunk.Name()) 98 | if err == nil { 99 | return true, nil 100 | } 101 | 102 | if errors.Is(err, os.ErrNotExist) { 103 | return false, nil 104 | } 105 | 106 | return false, err 107 | } 108 | 109 | type Chapter struct { 110 | Loc Loc 111 | Timestamp Millis 112 | Label string 113 | } 114 | 115 | const MinYouTubeChapterDuration Millis = 10 * 1000 116 | 117 | func (context *EvalContext) typeCheckArgs(loc Loc, signature ...TokenKind) (args []Token, err error) { 118 | if len(context.argsStack) < len(signature) { 119 | err = &DiagErr{ 120 | Loc: loc, 121 | Err: fmt.Errorf("Expected %d arguments but got %d", len(signature), len(context.argsStack)), 122 | } 123 | return 124 | } 125 | 126 | for _, kind := range signature { 127 | n := len(context.argsStack) 128 | arg := context.argsStack[n-1] 129 | context.argsStack = context.argsStack[:n-1] 130 | if kind != arg.Kind { 131 | err = &DiagErr{ 132 | Loc: arg.Loc, 133 | Err: fmt.Errorf("Expected %s but got %s", TokenKindName[kind], TokenKindName[arg.Kind]), 134 | } 135 | return 136 | } 137 | args = append(args, arg) 138 | } 139 | 140 | return 141 | } 142 | 143 | type Cut struct { 144 | chunk int 145 | pad Millis 146 | } 147 | 148 | type EvalContext struct { 149 | inputPath string 150 | inputPathLog []Token 151 | outputPath string 152 | chatLog []ChatMessageGroup 153 | chunks []Chunk 154 | chapters []Chapter 155 | cuts []Cut 156 | 157 | argsStack []Token 158 | chapStack []Chapter 159 | chapOffset Millis 160 | 161 | VideoCodec *Token 162 | VideoBitrate *Token 163 | AudioCodec *Token 164 | AudioBitrate *Token 165 | 166 | ExtraOutFlags []Token 167 | ExtraInFlags []Token 168 | } 169 | 170 | const ( 171 | DefaultVideoCodec = "libx264" 172 | DefaultVideoBitrate = "4000k" 173 | DefaultAudioCodec = "aac" 174 | DefaultAudioBitrate = "300k" 175 | ) 176 | 177 | func defaultContext() (EvalContext, bool) { 178 | context := EvalContext{ 179 | outputPath: "output.mp4", 180 | } 181 | 182 | if home, ok := os.LookupEnv("HOME"); ok { 183 | path := path.Join(home, ".markut") 184 | content, err := ioutil.ReadFile(path) 185 | if err != nil { 186 | if os.IsNotExist(err) { 187 | return context, true 188 | } 189 | fmt.Printf("ERROR: Could not open %s to read as a config: %s\n", path, err) 190 | return context, false 191 | } 192 | if !context.evalMarkutContent(string(content), path) { 193 | return context, false 194 | } 195 | } 196 | return context, true 197 | } 198 | 199 | func MaxChunksLocWidthPlusOne(chunks []Chunk) int { 200 | locWidth := 0 201 | for _, chunk := range chunks { 202 | // TODO: Loc.String() should include the extra ":", but that requires a huge refactoring in all the places where call it explicitly or implicitly 203 | locWidth = max(locWidth, len(chunk.Loc.String()) + 1) 204 | } 205 | return locWidth 206 | } 207 | 208 | func MaxTokensLocWidthPlusOne(tokens []Token) int { 209 | locWidth := 0 210 | for _, token := range tokens { 211 | // TODO: Loc.String() should include the extra ":", but that requires a huge refactoring in all the places where call it explicitly or implicitly 212 | locWidth = max(locWidth, len(token.Loc.String()) + 1) 213 | } 214 | return locWidth 215 | } 216 | 217 | func PrintFlagsSummary(flags []Token) { 218 | locWidth := MaxTokensLocWidthPlusOne(flags) 219 | // TODO: merge together parameters defined on the same line 220 | for _, flag := range flags { 221 | fmt.Printf("%-*s %s\n", locWidth, flag.Loc.String() + ":", string(flag.Text)) 222 | } 223 | } 224 | 225 | func (context EvalContext) PrintSummary() error { 226 | fmt.Printf(">>> Main Output Parameters:\n") 227 | if context.VideoCodec != nil { 228 | fmt.Printf("Video Codec: %s (Defined at %s)\n", string(context.VideoCodec.Text), context.VideoCodec.Loc) 229 | } else { 230 | fmt.Printf("Video Codec: %s (Default)\n", DefaultVideoCodec) 231 | } 232 | if context.VideoBitrate != nil { 233 | fmt.Printf("Video Bitrate: %s (Defined at %s)\n", string(context.VideoBitrate.Text), context.VideoBitrate.Loc) 234 | } else { 235 | fmt.Printf("Video Bitrate: %s (Default)\n", DefaultVideoBitrate) 236 | } 237 | if context.AudioCodec != nil { 238 | fmt.Printf("Audio Codec: %s (Defined at %s)\n", string(context.AudioCodec.Text), context.AudioCodec.Loc) 239 | } else { 240 | fmt.Printf("Audio Codec: %s (Default)\n", DefaultAudioCodec) 241 | } 242 | if context.AudioBitrate != nil { 243 | fmt.Printf("Audio Bitrate: %s (Defined at %s)\n", string(context.AudioBitrate.Text), context.AudioBitrate.Loc) 244 | } else { 245 | fmt.Printf("Audio Bitrate: %s (Default)\n", DefaultAudioBitrate) 246 | } 247 | fmt.Println() 248 | if len(context.ExtraInFlags) > 0 { 249 | fmt.Printf(">>> Extra Input Parameters:\n") 250 | PrintFlagsSummary(context.ExtraInFlags) 251 | fmt.Println() 252 | } 253 | if len(context.ExtraOutFlags) > 0 { 254 | fmt.Printf(">>> Extra Output Parameters:\n") 255 | PrintFlagsSummary(context.ExtraOutFlags) 256 | fmt.Println() 257 | } 258 | TwitchVodFileRegexp := "([0-9]+)-[0-9a-f\\-]+\\.mp4" 259 | re := regexp.MustCompile(TwitchVodFileRegexp) 260 | fmt.Printf(">>> Twitch Chat Logs (Detected by regex `%s`)\n", TwitchVodFileRegexp) 261 | locWidth := MaxTokensLocWidthPlusOne(context.inputPathLog) 262 | for _, inputPath := range context.inputPathLog { 263 | match := re.FindStringSubmatch(string(inputPath.Text)) 264 | if len(match) > 0 { 265 | fmt.Printf("%-*s https://www.twitchchatdownloader.com/video/%s\n", locWidth, inputPath.Loc.String() + ":", match[1]) 266 | } else { 267 | fmt.Printf("%-*s NO MATCH\n", locWidth, inputPath.Loc.String() + ":") 268 | } 269 | } 270 | fmt.Println() 271 | locWidth = MaxChunksLocWidthPlusOne(context.chunks) 272 | fmt.Printf(">>> Cuts (%d):\n", max(len(context.chunks)-1, 0)) 273 | var fullLength Millis = 0 274 | var finishedLength Millis = 0 275 | var renderedLength Millis = 0 276 | for i, chunk := range context.chunks { 277 | if i < len(context.chunks)-1 { 278 | fmt.Printf("%-*s Cut %d - %s\n", locWidth, chunk.Loc.String() + ":", i, millisToTs(fullLength+chunk.Duration())) 279 | } 280 | fullLength += chunk.Duration() 281 | if !chunk.Unfinished { 282 | finishedLength += chunk.Duration() 283 | } 284 | if _, err := os.Stat(chunk.Name()); err == nil { 285 | renderedLength += chunk.Duration() 286 | } 287 | } 288 | fmt.Println() 289 | fmt.Printf(">>> Chunks (%d):\n", len(context.chunks)) 290 | for index, chunk := range context.chunks { 291 | rendered, err := chunk.Rendered() 292 | if err != nil { 293 | return nil 294 | } 295 | checkMark := "[ ]" 296 | if rendered { 297 | checkMark = "[x]" 298 | } 299 | fmt.Printf("%-*s %s Chunk %d - %s -> %s (Duration: %s)\n", locWidth, chunk.Loc.String() + ":", checkMark, index, millisToTs(chunk.Start), millisToTs(chunk.End), millisToTs(chunk.Duration())) 300 | // TODO: Print extra output flags of the chunk 301 | } 302 | fmt.Println() 303 | fmt.Printf(">>> YouTube Chapters (%d):\n", len(context.chapters)) 304 | for _, chapter := range context.chapters { 305 | fmt.Printf("- %s - %s\n", millisToYouTubeTs(chapter.Timestamp), chapter.Label) 306 | } 307 | fmt.Println() 308 | fmt.Printf(">>> Length:\n") 309 | fmt.Printf("Rendered Length: %s\n", millisToTs(renderedLength)) 310 | fmt.Printf("Finished Length: %s\n", millisToTs(finishedLength)) 311 | fmt.Printf("Full Length: %s\n", millisToTs(fullLength)) 312 | return nil 313 | } 314 | 315 | func (context EvalContext) containsChunkWithName(filePath string) bool { 316 | for _, chunk := range context.chunks { 317 | if chunk.Name() == filePath { 318 | return true 319 | } 320 | } 321 | return false 322 | } 323 | 324 | // IMPORTANT! chatLog is assumed to be sorted by TimeOffset. 325 | func sliceChatLog(chatLog []ChatMessageGroup, start, end Millis) []ChatMessageGroup { 326 | // TODO: use Binary Search for a speed up on big chat logs 327 | lower := 0 328 | for lower < len(chatLog) && chatLog[lower].TimeOffset < start { 329 | lower += 1 330 | } 331 | upper := lower 332 | for upper < len(chatLog) && chatLog[upper].TimeOffset <= end { 333 | upper += 1 334 | } 335 | if lower < len(chatLog) { 336 | return chatLog[lower:upper] 337 | } 338 | return []ChatMessageGroup{} 339 | } 340 | 341 | // IMPORTANT! chatLog is assumed to be sorted by TimeOffset. 342 | func compressChatLog(chatLog []ChatMessageGroup) []ChatMessageGroup { 343 | result := []ChatMessageGroup{} 344 | for i := range chatLog { 345 | if len(result) > 0 && result[len(result)-1].TimeOffset == chatLog[i].TimeOffset { 346 | result[len(result)-1].Messages = append(result[len(result)-1].Messages, chatLog[i].Messages...) 347 | } else { 348 | result = append(result, chatLog[i]) 349 | } 350 | } 351 | return result 352 | } 353 | 354 | type Func struct { 355 | Description string 356 | Signature string 357 | Category string 358 | Run func(context *EvalContext, command string, token Token) bool 359 | } 360 | 361 | var funcs map[string]Func 362 | 363 | // This function is compatible with the format https://www.twitchchatdownloader.com/ generates. 364 | // It does not use encoding/csv because that website somehow generates unparsable garbage. 365 | func loadTwitchChatDownloaderCSVButParseManually(path string) ([]ChatMessageGroup, error) { 366 | chatLog := []ChatMessageGroup{} 367 | f, err := os.Open(path) 368 | if err != nil { 369 | return chatLog, err 370 | } 371 | bytes, err := ioutil.ReadAll(f) 372 | if err != nil { 373 | return chatLog, err 374 | } 375 | 376 | content := string(bytes) 377 | for i, line := range strings.Split(content, "\n") { 378 | if i == 0 && line == TwitchChatDownloaderCSVHeader { 379 | // If first line contains the TwitchChatDownloader's stupid header, just ignore it. Just let people have it. 380 | continue 381 | } 382 | if len(line) == 0 { 383 | // We encounter empty line usually at the end of the file. So it should be safe to break. 384 | break 385 | } 386 | pair := strings.SplitN(line, ",", 2) 387 | secs, err := strconv.Atoi(pair[0]) 388 | if err != nil { 389 | return chatLog, fmt.Errorf("%s:%d: invalid timestamp: %w", path, i, err) 390 | } 391 | 392 | pair = strings.SplitN(pair[1], ",", 2) 393 | nickname := pair[0] 394 | 395 | pair = strings.SplitN(pair[1], ",", 2) 396 | color := pair[0] 397 | text := pair[1] 398 | 399 | if len(text) >= 2 && text[0] == '"' && text[len(text)-1] == '"' { 400 | text = text[1 : len(text)-1] 401 | } 402 | 403 | chatLog = append(chatLog, ChatMessageGroup{ 404 | TimeOffset: Millis(secs * 1000), 405 | Messages: []ChatMessage{ 406 | {Color: color, Nickname: nickname, Text: text}, 407 | }, 408 | }) 409 | } 410 | 411 | sort.Slice(chatLog, func(i, j int) bool { 412 | return chatLog[i].TimeOffset < chatLog[j].TimeOffset 413 | }) 414 | 415 | return compressChatLog(chatLog), nil 416 | } 417 | 418 | func (context *EvalContext) evalMarkutContent(content string, path string) bool { 419 | lexer := NewLexer(content, path) 420 | token := Token{} 421 | var err error 422 | for { 423 | token, err = lexer.Next() 424 | if err != nil { 425 | fmt.Printf("%s\n", err) 426 | return false 427 | } 428 | 429 | if token.Kind == TokenEOF { 430 | break 431 | } 432 | 433 | var args []Token 434 | switch token.Kind { 435 | case TokenDash: 436 | args, err = context.typeCheckArgs(token.Loc, TokenTimestamp, TokenTimestamp) 437 | if err != nil { 438 | fmt.Printf("%s: ERROR: type check failed for subtraction\n", token.Loc) 439 | fmt.Printf("%s\n", err) 440 | return false 441 | } 442 | context.argsStack = append(context.argsStack, Token{ 443 | Loc: token.Loc, 444 | Kind: TokenTimestamp, 445 | Timestamp: args[1].Timestamp - args[0].Timestamp, 446 | }) 447 | case TokenPlus: 448 | args, err = context.typeCheckArgs(token.Loc, TokenTimestamp, TokenTimestamp) 449 | if err != nil { 450 | fmt.Printf("%s: ERROR: type check failed for addition\n", token.Loc) 451 | fmt.Printf("%s\n", err) 452 | return false 453 | } 454 | context.argsStack = append(context.argsStack, Token{ 455 | Loc: token.Loc, 456 | Kind: TokenTimestamp, 457 | Timestamp: args[1].Timestamp + args[0].Timestamp, 458 | }) 459 | case TokenString: 460 | fallthrough 461 | case TokenTimestamp: 462 | context.argsStack = append(context.argsStack, token) 463 | case TokenSymbol: 464 | command := string(token.Text) 465 | f, ok := funcs[command] 466 | if !ok { 467 | fmt.Printf("%s: ERROR: Unknown command %s\n", token.Loc, command) 468 | return false 469 | } 470 | if !f.Run(context, command, token) { 471 | return false 472 | } 473 | default: 474 | fmt.Printf("%s: ERROR: Unexpected token %s\n", token.Loc, TokenKindName[token.Kind]) 475 | return false 476 | } 477 | } 478 | 479 | return true 480 | } 481 | 482 | func (context *EvalContext) evalMarkutFile(loc *Loc, path string, ignoreIfMissing bool) bool { 483 | content, err := ioutil.ReadFile(path) 484 | if err != nil { 485 | sb := strings.Builder{} 486 | if loc != nil { 487 | sb.WriteString(fmt.Sprintf("%s: ", *loc)) 488 | } 489 | if ignoreIfMissing { 490 | sb.WriteString("WARNING: ") 491 | } else { 492 | sb.WriteString("ERROR: ") 493 | } 494 | sb.WriteString(fmt.Sprintf("%s", err)) 495 | fmt.Fprintf(os.Stderr, "%s\n", sb.String()) 496 | return ignoreIfMissing 497 | } 498 | 499 | return context.evalMarkutContent(string(content), path) 500 | } 501 | 502 | func (context *EvalContext) finishEval() bool { 503 | for i := 0; i+1 < len(context.chapters); i += 1 { 504 | duration := context.chapters[i+1].Timestamp - context.chapters[i].Timestamp 505 | // TODO: angled brackets are not allowed on YouTube. Let's make `chapters` check for that too. 506 | if duration < MinYouTubeChapterDuration { 507 | fmt.Printf("%s: ERROR: the chapter \"%s\" has duration %s which is shorter than the minimal allowed YouTube chapter duration which is %s (See https://support.google.com/youtube/answer/9884579)\n", context.chapters[i].Loc, context.chapters[i].Label, millisToTs(duration), millisToTs(MinYouTubeChapterDuration)) 508 | fmt.Printf("%s: NOTE: the chapter ends here\n", context.chapters[i+1].Loc) 509 | return false 510 | } 511 | } 512 | 513 | if len(context.argsStack) > 0 || len(context.chapStack) > 0 { 514 | for i := range context.argsStack { 515 | fmt.Printf("%s: ERROR: unused argument\n", context.argsStack[i].Loc) 516 | } 517 | for i := range context.chapStack { 518 | fmt.Printf("%s: ERROR: unused chapter\n", context.chapStack[i].Loc) 519 | } 520 | return false 521 | } 522 | 523 | return true 524 | } 525 | 526 | func ffmpegPathToBin() (ffmpegPath string) { 527 | ffmpegPath = "ffmpeg" 528 | // TODO: replace FFMPEG_PREFIX envar in favor of a func `ffmpeg_prefix` that you have to call in $HOME/.markut 529 | ffmpegPrefix, ok := os.LookupEnv("FFMPEG_PREFIX") 530 | if ok { 531 | ffmpegPath = path.Join(ffmpegPrefix, "bin", "ffmpeg") 532 | } 533 | return 534 | } 535 | 536 | func logCmd(name string, args ...string) { 537 | chunks := []string{} 538 | chunks = append(chunks, name) 539 | for _, arg := range args { 540 | if strings.Contains(arg, " ") { 541 | // TODO: use proper shell escaping instead of just wrapping with double quotes 542 | chunks = append(chunks, "\""+arg+"\"") 543 | } else { 544 | chunks = append(chunks, arg) 545 | } 546 | } 547 | fmt.Printf("[CMD] %s\n", strings.Join(chunks, " ")) 548 | } 549 | 550 | func millisToSecsForFFmpeg(millis Millis) string { 551 | return fmt.Sprintf("%d.%03d", millis/1000, millis%1000) 552 | } 553 | 554 | func ffmpegCutChunk(context EvalContext, chunk Chunk) error { 555 | rendered, err := chunk.Rendered() 556 | if err != nil { 557 | return err 558 | } 559 | 560 | if rendered { 561 | fmt.Printf("INFO: %s is already rendered\n", chunk.Name()) 562 | return nil 563 | } 564 | 565 | err = os.MkdirAll(ChunksFolder, 0755) 566 | if err != nil { 567 | return err 568 | } 569 | 570 | ffmpeg := ffmpegPathToBin() 571 | args := []string{} 572 | 573 | // We always rerender unfinished-chunk.mp4, because it might still 574 | // exist due to the rendering erroring out or canceling. It's a 575 | // temporary file that is copied and renamed to the chunks/ folder 576 | // after the rendering has finished successfully. The successfully 577 | // rendered chunks are not being rerendered due to the check at 578 | // the beginning of the function. 579 | args = append(args, "-y") 580 | 581 | args = append(args, "-ss", millisToSecsForFFmpeg(chunk.Start)) 582 | for _, inFlag := range context.ExtraInFlags { 583 | args = append(args, string(inFlag.Text)) 584 | } 585 | args = append(args, "-i", chunk.InputPath) 586 | 587 | if context.VideoCodec != nil { 588 | args = append(args, "-c:v", string(context.VideoCodec.Text)) 589 | } else { 590 | args = append(args, "-c:v", DefaultVideoCodec) 591 | } 592 | if context.VideoBitrate != nil { 593 | args = append(args, "-vb", string(context.VideoBitrate.Text)) 594 | } else { 595 | args = append(args, "-vb", DefaultVideoBitrate) 596 | } 597 | if context.AudioCodec != nil { 598 | args = append(args, "-c:a", string(context.AudioCodec.Text)) 599 | } else { 600 | args = append(args, "-c:a", DefaultAudioCodec) 601 | } 602 | if context.AudioBitrate != nil { 603 | args = append(args, "-ab", string(context.AudioBitrate.Text)) 604 | } else { 605 | args = append(args, "-ab", DefaultAudioBitrate) 606 | } 607 | args = append(args, "-t", millisToSecsForFFmpeg(chunk.Duration())) 608 | if chunk.Blur { 609 | args = append(args, "-vf", "boxblur=50:5") 610 | } 611 | for _, outFlag := range context.ExtraOutFlags { 612 | args = append(args, string(outFlag.Text)) 613 | } 614 | for _, outFlag := range chunk.ExtraOutFlags { 615 | args = append(args, string(outFlag.Text)) 616 | } 617 | unfinishedChunkName := "unfinished-chunk.mp4" 618 | args = append(args, unfinishedChunkName) 619 | 620 | logCmd(ffmpeg, args...) 621 | cmd := exec.Command(ffmpeg, args...) 622 | cmd.Stdin = os.Stdin 623 | cmd.Stdout = os.Stdout 624 | cmd.Stderr = os.Stderr 625 | err = cmd.Run() 626 | if err != nil { 627 | return err 628 | } 629 | 630 | fmt.Printf("INFO: Rename %s -> %s\n", unfinishedChunkName, chunk.Name()) 631 | return os.Rename(unfinishedChunkName, chunk.Name()) 632 | } 633 | 634 | func ffmpegConcatChunks(listPath string, outputPath string) error { 635 | ffmpeg := ffmpegPathToBin() 636 | args := []string{} 637 | 638 | // Unlike ffmpegCutChunk(), concatinating chunks is really 639 | // cheap. So we can just allow ourselves to always do that no 640 | // matter what. 641 | args = append(args, "-y") 642 | 643 | args = append(args, "-f", "concat") 644 | args = append(args, "-safe", "0") 645 | args = append(args, "-i", listPath) 646 | args = append(args, "-c", "copy") 647 | args = append(args, outputPath) 648 | 649 | logCmd(ffmpeg, args...) 650 | cmd := exec.Command(ffmpeg, args...) 651 | cmd.Stdin = os.Stdin 652 | cmd.Stdout = os.Stdout 653 | cmd.Stderr = os.Stderr 654 | return cmd.Run() 655 | } 656 | 657 | func ffmpegFixupInput(inputPath, outputPath string, y bool) error { 658 | ffmpeg := ffmpegPathToBin() 659 | args := []string{} 660 | 661 | if y { 662 | args = append(args, "-y") 663 | } 664 | 665 | // ffmpeg -y -i {{ morning_input }} -codec copy -bsf:v h264_mp4toannexb {{ morning_input }}.fixed.ts 666 | args = append(args, "-i", inputPath) 667 | args = append(args, "-codec", "copy") 668 | args = append(args, "-bsf:v", "h264_mp4toannexb") 669 | args = append(args, outputPath) 670 | logCmd(ffmpeg, args...) 671 | cmd := exec.Command(ffmpeg, args...) 672 | cmd.Stdin = os.Stdin 673 | cmd.Stdout = os.Stdout 674 | cmd.Stderr = os.Stderr 675 | return cmd.Run() 676 | } 677 | 678 | func ffmpegGenerateConcatList(chunks []Chunk, outputPath string) error { 679 | f, err := os.Create(outputPath) 680 | if err != nil { 681 | return err 682 | } 683 | defer f.Close() 684 | 685 | for _, chunk := range chunks { 686 | fmt.Fprintf(f, "file '%s'\n", chunk.Name()) 687 | } 688 | 689 | return nil 690 | } 691 | 692 | func captionsRingPush(ring []ChatMessageGroup, message ChatMessageGroup, capacity int) []ChatMessageGroup { 693 | if len(ring) < capacity { 694 | return append(ring, message) 695 | } 696 | return append(ring[1:], message) 697 | } 698 | 699 | type Subcommand struct { 700 | Run func(name string, args []string) bool 701 | Description string 702 | } 703 | 704 | var Subcommands = map[string]Subcommand{ 705 | "fixup": { 706 | Description: "Fixup the initial footage", 707 | Run: func(name string, args []string) bool { 708 | subFlag := flag.NewFlagSet(name, flag.ExitOnError) 709 | inputPtr := subFlag.String("input", "", "Path to the input video file (mandatory)") 710 | outputPtr := subFlag.String("output", "input.ts", "Path to the output video file") 711 | yPtr := subFlag.Bool("y", false, "Pass -y to ffmpeg") 712 | 713 | err := subFlag.Parse(args) 714 | if err == flag.ErrHelp { 715 | return true 716 | } 717 | 718 | if err != nil { 719 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 720 | return false 721 | } 722 | 723 | if *inputPtr == "" { 724 | subFlag.Usage() 725 | fmt.Printf("ERROR: No -input file is provided\n") 726 | return false 727 | } 728 | 729 | err = ffmpegFixupInput(*inputPtr, *outputPtr, *yPtr) 730 | if err != nil { 731 | fmt.Printf("ERROR: Could not fixup input file %s: %s\n", *inputPtr, err) 732 | return false 733 | } 734 | fmt.Printf("Generated %s\n", *outputPtr) 735 | 736 | return true 737 | }, 738 | }, 739 | "cut": { 740 | Description: "Render specific cut of the final video", 741 | Run: func(name string, args []string) bool { 742 | subFlag := flag.NewFlagSet(name, flag.ContinueOnError) 743 | markutPtr := subFlag.String("markut", "MARKUT", "Path to the MARKUT file") 744 | 745 | err := subFlag.Parse(args) 746 | if err == flag.ErrHelp { 747 | return true 748 | } 749 | 750 | if err != nil { 751 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 752 | return false 753 | } 754 | 755 | context, ok := defaultContext() 756 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 757 | if !ok { 758 | return false 759 | } 760 | 761 | if len(context.cuts) == 0 { 762 | fmt.Printf("ERROR: No cuts are provided. Use `cut` command after a `chunk` command to define a cut\n") 763 | return false 764 | } 765 | 766 | for _, cut := range context.cuts { 767 | if cut.chunk+1 >= len(context.chunks) { 768 | fmt.Printf("ERROR: %d is an invalid cut number. There is only %d of them.\n", cut.chunk, len(context.chunks)-1) 769 | return false 770 | } 771 | 772 | cutChunks := []Chunk{ 773 | { 774 | Start: context.chunks[cut.chunk].End - cut.pad, 775 | End: context.chunks[cut.chunk].End, 776 | InputPath: context.chunks[cut.chunk].InputPath, 777 | }, 778 | { 779 | Start: context.chunks[cut.chunk+1].Start, 780 | End: context.chunks[cut.chunk+1].Start + cut.pad, 781 | InputPath: context.chunks[cut.chunk+1].InputPath, 782 | }, 783 | } 784 | 785 | for _, chunk := range cutChunks { 786 | err := ffmpegCutChunk(context, chunk) 787 | if err != nil { 788 | fmt.Printf("WARNING: Failed to cut chunk %s: %s\n", chunk.Name(), err) 789 | } 790 | } 791 | 792 | cutListPath := "cut-%02d-list.txt" 793 | listPath := fmt.Sprintf(cutListPath, cut.chunk) 794 | err = ffmpegGenerateConcatList(cutChunks, listPath) 795 | if err != nil { 796 | fmt.Printf("ERROR: Could not generate not generate cut concat list %s: %s\n", cutListPath, err) 797 | return false 798 | } 799 | 800 | cutOutputPath := fmt.Sprintf("cut-%02d.mp4", cut.chunk) 801 | err = ffmpegConcatChunks(listPath, cutOutputPath) 802 | if err != nil { 803 | fmt.Printf("ERROR: Could not generate cut output file %s: %s\n", cutOutputPath, err) 804 | return false 805 | } 806 | 807 | fmt.Printf("Generated %s\n", cutOutputPath) 808 | fmt.Printf("%s: NOTE: cut is defined in here\n", context.chunks[cut.chunk].Loc) 809 | } 810 | 811 | return true 812 | }, 813 | }, 814 | "chunk": { 815 | Description: "Render specific chunk of the final video", 816 | Run: func(name string, args []string) bool { 817 | subFlag := flag.NewFlagSet(name, flag.ContinueOnError) 818 | markutPtr := subFlag.String("markut", "MARKUT", "Path to the MARKUT file") 819 | chunkPtr := subFlag.Int("chunk", 0, "Chunk number to render") 820 | 821 | err := subFlag.Parse(args) 822 | 823 | if err == flag.ErrHelp { 824 | return true 825 | } 826 | 827 | if err != nil { 828 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 829 | return false 830 | } 831 | 832 | context, ok := defaultContext() 833 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 834 | if !ok { 835 | return false 836 | } 837 | 838 | if *chunkPtr > len(context.chunks) { 839 | fmt.Printf("ERROR: %d is an incorrect chunk number. There is only %d of them.\n", *chunkPtr, len(context.chunks)) 840 | return false 841 | } 842 | 843 | chunk := context.chunks[*chunkPtr] 844 | 845 | err = ffmpegCutChunk(context, chunk) 846 | if err != nil { 847 | fmt.Printf("ERROR: Could not cut the chunk %s: %s\n", chunk.Name(), err) 848 | return false 849 | } 850 | 851 | fmt.Printf("%s is rendered!\n", chunk.Name()) 852 | return true 853 | }, 854 | }, 855 | "final": { 856 | Description: "Render the final video", 857 | Run: func(name string, args []string) bool { 858 | subFlag := flag.NewFlagSet(name, flag.ContinueOnError) 859 | markutPtr := subFlag.String("markut", "MARKUT", "Path to the MARKUT file") 860 | 861 | err := subFlag.Parse(args) 862 | if err == flag.ErrHelp { 863 | return true 864 | } 865 | 866 | if err != nil { 867 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 868 | return false 869 | } 870 | 871 | context, ok := defaultContext() 872 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 873 | if !ok { 874 | return false 875 | } 876 | 877 | for _, chunk := range context.chunks { 878 | err := ffmpegCutChunk(context, chunk) 879 | if err != nil { 880 | fmt.Printf("WARNING: Failed to cut chunk %s: %s\n", chunk.Name(), err) 881 | } 882 | } 883 | 884 | listPath := "final-list.txt" 885 | err = ffmpegGenerateConcatList(context.chunks, listPath) 886 | if err != nil { 887 | fmt.Printf("ERROR: Could not generate final concat list %s: %s\n", listPath, err) 888 | return false 889 | } 890 | 891 | err = ffmpegConcatChunks(listPath, context.outputPath) 892 | if err != nil { 893 | fmt.Printf("ERROR: Could not generated final output %s: %s\n", context.outputPath, err) 894 | return false 895 | } 896 | 897 | err = context.PrintSummary() 898 | if err != nil { 899 | fmt.Printf("ERROR: Could not print summary: %s\n", err) 900 | return false 901 | } 902 | 903 | return true 904 | }, 905 | }, 906 | "summary": { 907 | Description: "Print the summary of the video", 908 | Run: func(name string, args []string) bool { 909 | summFlag := flag.NewFlagSet(name, flag.ContinueOnError) 910 | markutPtr := summFlag.String("markut", "MARKUT", "Path to the MARKUT file") 911 | 912 | err := summFlag.Parse(args) 913 | 914 | if err == flag.ErrHelp { 915 | return true 916 | } 917 | 918 | if err != nil { 919 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 920 | return false 921 | } 922 | 923 | context, ok := defaultContext() 924 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 925 | if !ok { 926 | return false 927 | } 928 | 929 | err = context.PrintSummary() 930 | if err != nil { 931 | fmt.Printf("ERROR: Could not print summary: %s\n", err) 932 | return false 933 | } 934 | 935 | return true 936 | }, 937 | }, 938 | "chat": { 939 | Description: "Generate chat captions", 940 | Run: func(name string, args []string) bool { 941 | chatFlag := flag.NewFlagSet(name, flag.ContinueOnError) 942 | markutPtr := chatFlag.String("markut", "MARKUT", "Path to the MARKUT file") 943 | csvPtr := chatFlag.Bool("csv", false, "Generate the chat using the stupid Twich Chat Downloader CSV format. You can then feed this output to tools like SubChat https://github.com/Kam1k4dze/SubChat") 944 | 945 | err := chatFlag.Parse(args) 946 | 947 | if err == flag.ErrHelp { 948 | return true 949 | } 950 | 951 | if err != nil { 952 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 953 | return false 954 | } 955 | 956 | context, ok := defaultContext() 957 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 958 | if !ok { 959 | return false 960 | } 961 | 962 | if *csvPtr { 963 | fmt.Printf("%s\n", TwitchChatDownloaderCSVHeader) 964 | var cursor Millis = 0 965 | for _, chunk := range context.chunks { 966 | for _, messageGroup := range chunk.ChatLog { 967 | timestamp := cursor + messageGroup.TimeOffset - chunk.Start 968 | for _, message := range messageGroup.Messages { 969 | fmt.Printf("%d,%s,%s,\"%s\"\n", timestamp, message.Nickname, message.Color, message.Text) 970 | } 971 | } 972 | cursor += chunk.End - chunk.Start 973 | } 974 | } else { 975 | capacity := 1 976 | ring := []ChatMessageGroup{} 977 | timeCursor := Millis(0) 978 | subRipCounter := 0 979 | sb := strings.Builder{} 980 | for _, chunk := range context.chunks { 981 | prevTime := chunk.Start 982 | for _, message := range chunk.ChatLog { 983 | deltaTime := message.TimeOffset - prevTime 984 | prevTime = message.TimeOffset 985 | if len(ring) > 0 { 986 | subRipCounter += 1 987 | fmt.Printf("%d\n", subRipCounter) 988 | fmt.Printf("%s --> %s\n", millisToSubRipTs(timeCursor), millisToSubRipTs(timeCursor+deltaTime)) 989 | for _, ringMessageGroup := range ring { 990 | sb.Reset() 991 | for _, message := range ringMessageGroup.Messages { 992 | sb.WriteString(fmt.Sprintf("[%s] %s\n", message.Nickname, message.Text)) 993 | } 994 | fmt.Printf("%s", sb.String()) 995 | } 996 | fmt.Printf("\n") 997 | } 998 | timeCursor += deltaTime 999 | ring = captionsRingPush(ring, message, capacity) 1000 | } 1001 | timeCursor += chunk.End - prevTime 1002 | } 1003 | } 1004 | 1005 | return true 1006 | }, 1007 | }, 1008 | "prune": { 1009 | Description: "Prune unused chunks", 1010 | Run: func(name string, args []string) bool { 1011 | subFlag := flag.NewFlagSet(name, flag.ContinueOnError) 1012 | markutPtr := subFlag.String("markut", "MARKUT", "Path to the MARKUT file") 1013 | 1014 | err := subFlag.Parse(args) 1015 | 1016 | if err == flag.ErrHelp { 1017 | return true 1018 | } 1019 | 1020 | if err != nil { 1021 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 1022 | return false 1023 | } 1024 | 1025 | context, ok := defaultContext() 1026 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 1027 | if !ok { 1028 | return false 1029 | } 1030 | 1031 | files, err := ioutil.ReadDir(ChunksFolder) 1032 | if err != nil { 1033 | fmt.Printf("ERROR: could not read %s folder: %s\n", ChunksFolder, err) 1034 | return false 1035 | } 1036 | 1037 | for _, file := range files { 1038 | if !file.IsDir() { 1039 | filePath := fmt.Sprintf("%s/%s", ChunksFolder, file.Name()) 1040 | if !context.containsChunkWithName(filePath) { 1041 | fmt.Printf("INFO: deleting chunk file %s\n", filePath) 1042 | err = os.Remove(filePath) 1043 | if err != nil { 1044 | fmt.Printf("ERROR: could not remove file %s: %s\n", filePath, err) 1045 | return false 1046 | } 1047 | } 1048 | } 1049 | } 1050 | 1051 | fmt.Printf("DONE\n") 1052 | 1053 | return true 1054 | }, 1055 | }, 1056 | // TODO: Maybe watch mode should just be a flag for the `final` subcommand 1057 | "watch": { 1058 | Description: "Render finished chunks in watch mode every time MARKUT file is modified", 1059 | Run: func(name string, args []string) bool { 1060 | subFlag := flag.NewFlagSet(name, flag.ContinueOnError) 1061 | markutPtr := subFlag.String("markut", "MARKUT", "Path to the MARKUT file") 1062 | skipcatPtr := subFlag.Bool("skipcat", false, "Skip concatenation step") 1063 | 1064 | err := subFlag.Parse(args) 1065 | 1066 | if err == flag.ErrHelp { 1067 | return true 1068 | } 1069 | 1070 | if err != nil { 1071 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 1072 | return false 1073 | } 1074 | 1075 | fmt.Printf("INFO: Waiting for updates to %s\n", *markutPtr) 1076 | for { 1077 | // NOTE: always use rsync(1) for updating the MARKUT file remotely. 1078 | // This kind of crappy modification checking needs at least some sort of atomicity. 1079 | // rsync(1) is as atomic as rename(2). So it's alright for majority of the cases. 1080 | 1081 | context, ok := defaultContext() 1082 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 1083 | if !ok { 1084 | return false 1085 | } 1086 | 1087 | done := true 1088 | for _, chunk := range context.chunks { 1089 | if chunk.Unfinished { 1090 | done = false 1091 | continue 1092 | } 1093 | 1094 | if _, err := os.Stat(chunk.Name()); errors.Is(err, os.ErrNotExist) { 1095 | err = ffmpegCutChunk(context, chunk) 1096 | if err != nil { 1097 | fmt.Printf("ERROR: Could not cut the chunk %s: %s\n", chunk.Name(), err) 1098 | return false 1099 | } 1100 | fmt.Printf("INFO: Waiting for more updates to %s\n", *markutPtr) 1101 | done = false 1102 | break 1103 | } 1104 | } 1105 | 1106 | if done { 1107 | break 1108 | } 1109 | 1110 | time.Sleep(1 * time.Second) 1111 | } 1112 | 1113 | context, ok := defaultContext() 1114 | ok = ok && context.evalMarkutFile(nil, *markutPtr, false) && context.finishEval() 1115 | if !ok { 1116 | return false 1117 | } 1118 | 1119 | if !*skipcatPtr { 1120 | 1121 | listPath := "final-list.txt" 1122 | err = ffmpegGenerateConcatList(context.chunks, listPath) 1123 | if err != nil { 1124 | fmt.Printf("ERROR: Could not generate final concat list %s: %s\n", listPath, err) 1125 | return false 1126 | } 1127 | 1128 | err = ffmpegConcatChunks(listPath, context.outputPath) 1129 | if err != nil { 1130 | fmt.Printf("ERROR: Could not generated final output %s: %s\n", context.outputPath, err) 1131 | return false 1132 | } 1133 | } 1134 | 1135 | err = context.PrintSummary() 1136 | if err != nil { 1137 | fmt.Printf("ERROR: Could not print summary: %s\n", err) 1138 | return false 1139 | } 1140 | 1141 | return true 1142 | }, 1143 | }, 1144 | "funcs": { 1145 | Description: "Print info about all the available funcs of the Markut Language", 1146 | Run: func(commandName string, args []string) bool { 1147 | if len(args) > 0 { 1148 | name := args[0] 1149 | funk, ok := funcs[name] 1150 | if !ok { 1151 | fmt.Printf("ERROR: no func named %s is found\n", name) 1152 | return false 1153 | } 1154 | fmt.Printf("%s : %s\n", name, funk.Signature) 1155 | fmt.Printf(" %s\n", strings.ReplaceAll(funk.Description, "$SPOILER$", "")) 1156 | return true 1157 | } 1158 | 1159 | names := []string{} 1160 | for name, _ := range funcs { 1161 | names = append(names, name) 1162 | } 1163 | sort.Slice(names, func(i, j int) bool { 1164 | return names[i] < names[j] 1165 | }) 1166 | sort.SliceStable(names, func(i, j int) bool { // Rare moment in my boring dev life when I actually need a stable sort 1167 | return funcs[names[i]].Category < funcs[names[j]].Category 1168 | }) 1169 | if len(names) > 0 { 1170 | category := "" 1171 | for _, name := range names { 1172 | if category != funcs[name].Category { 1173 | category = funcs[name].Category 1174 | fmt.Printf("%s:\n", category) 1175 | } 1176 | fmt.Printf(" %s - %s\n", name, strings.Split(funcs[name].Description, "$SPOILER$")[0]) 1177 | } 1178 | } 1179 | return true 1180 | }, 1181 | }, 1182 | "twitch-chat-download": { 1183 | Description: "Download Twitch Chat of a VOD and print it in the stupid format https://twitchchatdownloader.com/ uses to maintain compatibility with our existing chat parser", 1184 | Run: func(commandName string, args []string) bool { 1185 | subFlag := flag.NewFlagSet(commandName, flag.ContinueOnError) 1186 | videoIdPtr := subFlag.String("videoID", "", "Video ID of the Twitch VOD to download") 1187 | 1188 | err := subFlag.Parse(args) 1189 | 1190 | if err == flag.ErrHelp { 1191 | return true 1192 | } 1193 | 1194 | if err != nil { 1195 | fmt.Printf("ERROR: Could not parse command line arguments: %s\n", err) 1196 | return false 1197 | } 1198 | 1199 | if *videoIdPtr == "" { 1200 | subFlag.Usage() 1201 | fmt.Printf("ERROR: No -videoID is provided\n") 1202 | return false 1203 | } 1204 | 1205 | client := &http.Client{} 1206 | 1207 | queryMessagesByOffset := func(videoId, gqlCursorId string) (string, bool) { 1208 | // twitchClientId := "kimne78kx3ncx6brgo4mv6wki5h1ko" // This is the Client ID of the Twitch Web App itself 1209 | twitchClientId := "kd1unb4b3q4t58fwlpcbzcbnm76a8fp" // https://github.com/ihabunek/twitch-dl/issues/124#issuecomment-1537030937 1210 | gqlUrl := "https://gql.twitch.tv/gql" 1211 | var body string 1212 | if gqlCursorId == "" { 1213 | body = fmt.Sprintf("[{\"operationName\":\"VideoCommentsByOffsetOrCursor\",\"variables\":{\"videoID\":\"%s\",\"contentOffsetSeconds\":0},\"extensions\":{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"b70a3591ff0f4e0313d126c6a1502d79a1c02baebb288227c582044aa76adf6a\"}}}]", videoId) 1214 | } else { 1215 | body = fmt.Sprintf("[{\"operationName\":\"VideoCommentsByOffsetOrCursor\",\"variables\":{\"videoID\":\"%s\",\"cursor\":\"%s\"},\"extensions\":{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"b70a3591ff0f4e0313d126c6a1502d79a1c02baebb288227c582044aa76adf6a\"}}}]", videoId, gqlCursorId) 1216 | } 1217 | req, err := http.NewRequest("POST", gqlUrl, strings.NewReader(body)) 1218 | if err != nil { 1219 | fmt.Printf("ERROR: could not create request for url %s: %s\n", gqlUrl, err) 1220 | return "", false 1221 | } 1222 | req.Header.Add("Client-Id", twitchClientId) 1223 | resp, err := client.Do(req) 1224 | if err != nil { 1225 | fmt.Printf("ERROR: could not perform POST request to %s: %s\n", gqlUrl, err) 1226 | return "", false 1227 | } 1228 | defer resp.Body.Close() 1229 | 1230 | var root interface{} 1231 | decoder := json.NewDecoder(resp.Body) 1232 | decoder.Decode(&root) 1233 | 1234 | type Object = map[string]interface{} 1235 | type Array = []interface{} 1236 | 1237 | cursor := root 1238 | cursor = cursor.(Array)[0] 1239 | cursor = cursor.(Object)["data"] 1240 | cursor = cursor.(Object)["video"] 1241 | cursor = cursor.(Object)["comments"] 1242 | cursor = cursor.(Object)["edges"] 1243 | edges := cursor.(Array) 1244 | for _, edge := range edges { 1245 | cursor = edge 1246 | cursor = cursor.(Object)["cursor"] 1247 | gqlCursorId = cursor.(string) 1248 | 1249 | cursor = edge 1250 | cursor = cursor.(Object)["node"] 1251 | cursor = cursor.(Object)["contentOffsetSeconds"] 1252 | fmt.Printf("%d,", int(cursor.(float64))) 1253 | 1254 | cursor = edge 1255 | cursor = cursor.(Object)["node"] 1256 | cursor = cursor.(Object)["commenter"] 1257 | var commenter string 1258 | if cursor != nil { 1259 | cursor = cursor.(Object)["login"] 1260 | commenter = cursor.(string) 1261 | } else { 1262 | // Apparent this may happen if the account got deleted after the stream 1263 | commenter = "" 1264 | } 1265 | fmt.Printf("%s,", commenter) 1266 | 1267 | cursor = edge 1268 | cursor = cursor.(Object)["node"] 1269 | cursor = cursor.(Object)["message"] 1270 | cursor = cursor.(Object)["userColor"] 1271 | var color string 1272 | if cursor != nil { 1273 | color = cursor.(string) 1274 | } else { 1275 | // Taken from https://discuss.dev.twitch.com/t/default-user-color-in-chat/385 1276 | // I don't know if it's still accurate, but I don't care, we just need some sort of 1277 | // default color 1278 | defaultColors := []string{ 1279 | "#FF0000", "#0000FF", "#00FF00", 1280 | "#B22222", "#FF7F50", "#9ACD32", 1281 | "#FF4500", "#2E8B57", "#DAA520", 1282 | "#D2691E", "#5F9EA0", "#1E90FF", 1283 | "#FF69B4", "#8A2BE2", "#00FF7F", 1284 | } 1285 | index := int(commenter[0] + commenter[len(commenter)-1]) 1286 | color = defaultColors[index%len(defaultColors)] 1287 | } 1288 | fmt.Printf("%s,", color) 1289 | 1290 | cursor = edge 1291 | cursor = cursor.(Object)["node"] 1292 | cursor = cursor.(Object)["message"] 1293 | cursor = cursor.(Object)["fragments"] 1294 | fragments := cursor.(Array) 1295 | sb := strings.Builder{} 1296 | for _, fragment := range fragments { 1297 | cursor = fragment.(Object)["text"] 1298 | sb.WriteString(cursor.(string)) 1299 | } 1300 | fmt.Printf("\"%s\"\n", sb.String()) 1301 | } 1302 | return gqlCursorId, true 1303 | } 1304 | 1305 | fmt.Printf("%s\n", TwitchChatDownloaderCSVHeader) 1306 | gqlCursorId, ok := queryMessagesByOffset(*videoIdPtr, "") 1307 | if !ok { 1308 | return false 1309 | } 1310 | for gqlCursorId != "" { 1311 | gqlCursorId, ok = queryMessagesByOffset(*videoIdPtr, gqlCursorId) 1312 | if !ok { 1313 | return false 1314 | } 1315 | } 1316 | return true 1317 | }, 1318 | }, 1319 | } 1320 | 1321 | func usage() { 1322 | names := []string{} 1323 | for name, _ := range Subcommands { 1324 | names = append(names, name) 1325 | } 1326 | sort.Strings(names) 1327 | fmt.Printf("Usage: markut [OPTIONS]\n") 1328 | fmt.Printf("SUBCOMMANDS:\n") 1329 | for _, name := range names { 1330 | fmt.Printf(" %s - %s\n", name, Subcommands[name].Description) 1331 | } 1332 | fmt.Printf("ENVARS:\n") 1333 | fmt.Printf(" FFMPEG_PREFIX Prefix path for a custom ffmpeg distribution\n") 1334 | fmt.Printf("FILES:\n") 1335 | fmt.Printf(" $HOME/.markut File that is always evaluated automatically before the MARKUT file\n") 1336 | } 1337 | 1338 | func main() { 1339 | if len(os.Args) < 2 { 1340 | usage() 1341 | fmt.Printf("ERROR: No subcommand is provided\n") 1342 | os.Exit(1) 1343 | } 1344 | 1345 | funcs = map[string]Func{ 1346 | "chat": { 1347 | Description: "Load a chat log file generated by https://www.twitchchatdownloader.com/$SPOILER$ which is going to be used by the subsequent `chunk` func calls to include certain messages into the subtitles generated by the `markut chat` subcommand. There could be only one loaded chat log at a time. Repeated calls to the `chat` func replace the currently loaded chat log with another one. The already defined chunks keep the copy of the logs that were loaded at the time of their definition.", 1348 | Signature: " --", 1349 | Category: "Chat", 1350 | Run: func(context *EvalContext, command string, token Token) bool { 1351 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1352 | if err != nil { 1353 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1354 | fmt.Printf("%s\n", err) 1355 | return false 1356 | } 1357 | path := args[0] 1358 | context.chatLog, err = loadTwitchChatDownloaderCSVButParseManually(string(path.Text)) 1359 | if err != nil { 1360 | fmt.Printf("%s: ERROR: could not load the chat logs: %s\n", path.Loc, err) 1361 | return false 1362 | } 1363 | return true 1364 | }, 1365 | }, 1366 | "chat_offset": { 1367 | Description: "Offsets the timestamps of the currently loaded chat log$SPOILER$ by removing all the messages between `start` and `end` Timestamps", 1368 | Category: "Chat", 1369 | Signature: " --", 1370 | Run: func(context *EvalContext, command string, token Token) bool { 1371 | // // TODO: this check does not make any sense when there are several chat commands 1372 | // // But I still want to have some sort of sanity check for chat_offsets 1373 | // if len(context.chunks) > 0 { 1374 | // fmt.Printf("%s: ERROR: chat offset should be applied after `chat` commands but before any `chunks` commands. This is due to `chunk` commands making copies of the chat slices that are not affected by the consequent chat offsets\n", token.Loc); 1375 | // return false; 1376 | // } 1377 | 1378 | args, err := context.typeCheckArgs(token.Loc, TokenTimestamp, TokenTimestamp) 1379 | if err != nil { 1380 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1381 | fmt.Printf("%s\n", err) 1382 | return false 1383 | } 1384 | 1385 | start := args[1] 1386 | end := args[0] 1387 | 1388 | if start.Timestamp < 0 { 1389 | fmt.Printf("%s: ERROR: the start of the chat offset is negative %s\n", start.Loc, millisToTs(start.Timestamp)) 1390 | return false 1391 | } 1392 | 1393 | if end.Timestamp < 0 { 1394 | fmt.Printf("%s: ERROR: the end of the chat offset is negative %s\n", end.Loc, millisToTs(end.Timestamp)) 1395 | return false 1396 | } 1397 | 1398 | if start.Timestamp > end.Timestamp { 1399 | fmt.Printf("%s: ERROR: the end of the chat offset %s is earlier than its start %s\n", end.Loc, millisToTs(end.Timestamp), millisToTs(start.Timestamp)) 1400 | fmt.Printf("%s: NOTE: the start is located here\n", start.Loc) 1401 | return false 1402 | } 1403 | 1404 | chatLen := len(context.chatLog) 1405 | if chatLen > 0 { 1406 | last := context.chatLog[chatLen-1].TimeOffset 1407 | before := sliceChatLog(context.chatLog, 0, start.Timestamp) 1408 | after := sliceChatLog(context.chatLog, end.Timestamp, last) 1409 | delta := end.Timestamp - start.Timestamp 1410 | for i := range after { 1411 | after[i].TimeOffset -= delta 1412 | } 1413 | context.chatLog = append(before, after...) 1414 | } 1415 | 1416 | return true 1417 | }, 1418 | }, 1419 | "no_chat": { 1420 | Description: "Clears out the current loaded chat log$SPOILER$ as if nothing is loaded", 1421 | Category: "Chat", 1422 | Signature: "--", 1423 | Run: func(context *EvalContext, command string, token Token) bool { 1424 | context.chatLog = []ChatMessageGroup{} 1425 | return true 1426 | }, 1427 | }, 1428 | "chunk": { 1429 | Description: "Define a chunk$SPOILER$ between `start` and `end` timestamp for the current input defined by the `input` func", 1430 | Category: "Chunk", 1431 | Signature: " --", 1432 | Run: func(context *EvalContext, command string, token Token) bool { 1433 | args, err := context.typeCheckArgs(token.Loc, TokenTimestamp, TokenTimestamp) 1434 | if err != nil { 1435 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1436 | fmt.Printf("%s\n", err) 1437 | return false 1438 | } 1439 | 1440 | start := args[1] 1441 | end := args[0] 1442 | 1443 | if start.Timestamp < 0 { 1444 | fmt.Printf("%s: ERROR: the start of the chunk is negative %s\n", start.Loc, millisToTs(start.Timestamp)) 1445 | return false 1446 | } 1447 | 1448 | if end.Timestamp < 0 { 1449 | fmt.Printf("%s: ERROR: the end of the chunk is negative %s\n", end.Loc, millisToTs(end.Timestamp)) 1450 | return false 1451 | } 1452 | 1453 | if start.Timestamp > end.Timestamp { 1454 | fmt.Printf("%s: ERROR: the end of the chunk %s is earlier than its start %s\n", end.Loc, millisToTs(end.Timestamp), millisToTs(start.Timestamp)) 1455 | fmt.Printf("%s: NOTE: the start is located here\n", start.Loc) 1456 | return false 1457 | } 1458 | 1459 | chunk := Chunk{ 1460 | Loc: token.Loc, 1461 | Start: start.Timestamp, 1462 | End: end.Timestamp, 1463 | InputPath: context.inputPath, 1464 | ChatLog: sliceChatLog(context.chatLog, start.Timestamp, end.Timestamp), 1465 | } 1466 | 1467 | context.chunks = append(context.chunks, chunk) 1468 | 1469 | for _, chapter := range context.chapStack { 1470 | if chapter.Timestamp < chunk.Start || chunk.End < chapter.Timestamp { 1471 | fmt.Printf("%s: ERROR: the timestamp %s of chapter \"%s\" is outside of the the current chunk\n", chapter.Loc, millisToTs(chapter.Timestamp), chapter.Label) 1472 | fmt.Printf("%s: NOTE: which starts at %s\n", start.Loc, millisToTs(start.Timestamp)) 1473 | fmt.Printf("%s: NOTE: and ends at %s\n", end.Loc, millisToTs(end.Timestamp)) 1474 | return false 1475 | } 1476 | 1477 | context.chapters = append(context.chapters, Chapter{ 1478 | Loc: chapter.Loc, 1479 | Timestamp: chapter.Timestamp - chunk.Start + context.chapOffset, 1480 | Label: chapter.Label, 1481 | }) 1482 | } 1483 | 1484 | context.chapOffset += chunk.End - chunk.Start 1485 | 1486 | context.chapStack = []Chapter{} 1487 | return true 1488 | }, 1489 | }, 1490 | "blur": { 1491 | Description: "Blur the last defined chunk$SPOILER$. Useful for bluring out sensitive information.", 1492 | Signature: "--", 1493 | Category: "Chunk", 1494 | Run: func(context *EvalContext, command string, token Token) bool { 1495 | if len(context.chunks) == 0 { 1496 | fmt.Printf("%s: ERROR: no chunks defined for a blur\n", token.Loc) 1497 | return false 1498 | } 1499 | context.chunks[len(context.chunks)-1].Blur = true 1500 | return true 1501 | }, 1502 | }, 1503 | "removed": { 1504 | Description: "Remove the last defined chunk$SPOILER$. Useful for disabling a certain chunk, so you can reenable it later if needed.", 1505 | Signature: "--", 1506 | Category: "Chunk", 1507 | Run: func(context *EvalContext, command string, token Token) bool { 1508 | if len(context.chunks) == 0 { 1509 | fmt.Printf("%s: ERROR: no chunks defined for removal\n", token.Loc) 1510 | return false 1511 | } 1512 | context.chunks = context.chunks[:len(context.chunks)-1] 1513 | return true 1514 | }, 1515 | }, 1516 | "unfinished": { 1517 | Description: "Mark the last defined chunk as unfinished$SPOILER$. This is used by the `markut watch` subcommand. `markut watch` does not render any unfinished chunks and keeps monitoring the MARKUT file until there is no unfinished chunks.", 1518 | Signature: "--", 1519 | Category: "Chunk", 1520 | Run: func(context *EvalContext, command string, token Token) bool { 1521 | if len(context.chunks) == 0 { 1522 | fmt.Printf("%s: ERROR: no chunks defined for marking as unfinished\n", token.Loc) 1523 | return false 1524 | } 1525 | context.chunks[len(context.chunks)-1].Unfinished = true 1526 | return true 1527 | }, 1528 | }, 1529 | "video_codec": { 1530 | Description: "Set the value of the output video codec flag (-c:v). Default is \"" + DefaultVideoCodec + "\".", 1531 | Signature: " --", 1532 | Category: "FFmpeg Arguments", 1533 | Run: func(context *EvalContext, command string, token Token) bool { 1534 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1535 | if err != nil { 1536 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1537 | fmt.Printf("%s\n", err) 1538 | return false 1539 | } 1540 | context.VideoCodec = &args[0] 1541 | return true 1542 | }, 1543 | }, 1544 | "video_bitrate": { 1545 | Description: "Set the value of the output video bitrate flag (-vb). Default is \"" + DefaultVideoBitrate + "\".", 1546 | Signature: " --", 1547 | Category: "FFmpeg Arguments", 1548 | Run: func(context *EvalContext, command string, token Token) bool { 1549 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1550 | if err != nil { 1551 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1552 | fmt.Printf("%s\n", err) 1553 | return false 1554 | } 1555 | context.VideoBitrate = &args[0] 1556 | return true 1557 | }, 1558 | }, 1559 | "audio_codec": { 1560 | Description: "Set the value of the output audio codec flag (-c:a). Default is \"" + DefaultAudioCodec + "\".", 1561 | Signature: " --", 1562 | Category: "FFmpeg Arguments", 1563 | Run: func(context *EvalContext, command string, token Token) bool { 1564 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1565 | if err != nil { 1566 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1567 | fmt.Printf("%s\n", err) 1568 | return false 1569 | } 1570 | context.AudioCodec = &args[0] 1571 | return true 1572 | }, 1573 | }, 1574 | "audio_bitrate": { 1575 | Description: "Set the value of the output audio bitrate flag (-ab). Default is \"" + DefaultAudioBitrate + "\".", 1576 | Signature: " --", 1577 | Category: "FFmpeg Arguments", 1578 | Run: func(context *EvalContext, command string, token Token) bool { 1579 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1580 | if err != nil { 1581 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1582 | fmt.Printf("%s\n", err) 1583 | return false 1584 | } 1585 | context.AudioBitrate = &args[0] 1586 | return true 1587 | }, 1588 | }, 1589 | "chunk_outf": { 1590 | Description: "Append extra output flag to the last defined chunk", 1591 | Signature: " --", 1592 | Category: "FFmpeg Arguments", 1593 | Run: func(context *EvalContext, command string, token Token) bool { 1594 | if len(context.chunks) == 0 { 1595 | fmt.Printf("%s: ERROR: no chunks defined to add extra output flag to\n", token.Loc) 1596 | return false 1597 | } 1598 | 1599 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1600 | if err != nil { 1601 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1602 | fmt.Printf("%s\n", err) 1603 | return false 1604 | } 1605 | outFlag := args[0] 1606 | 1607 | chunk := &context.chunks[len(context.chunks)-1]; 1608 | chunk.ExtraOutFlags = append(chunk.ExtraOutFlags, outFlag) 1609 | 1610 | return true 1611 | }, 1612 | }, 1613 | "outf": { 1614 | Description: "Append extra output flag for every chunk", 1615 | Signature: " --", 1616 | Category: "FFmpeg Arguments", 1617 | Run: func(context *EvalContext, command string, token Token) bool { 1618 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1619 | if err != nil { 1620 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1621 | fmt.Printf("%s\n", err) 1622 | return false 1623 | } 1624 | outFlag := args[0] 1625 | context.ExtraOutFlags = append(context.ExtraOutFlags, outFlag) 1626 | return true 1627 | }, 1628 | }, 1629 | "inf": { 1630 | Description: "Append extra input flag for every chunk", 1631 | Signature: " --", 1632 | Category: "FFmpeg Arguments", 1633 | Run: func(context *EvalContext, command string, token Token) bool { 1634 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1635 | if err != nil { 1636 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1637 | fmt.Printf("%s\n", err) 1638 | return false 1639 | } 1640 | inFlag := args[0] 1641 | context.ExtraInFlags = append(context.ExtraInFlags, inFlag) 1642 | return true 1643 | }, 1644 | }, 1645 | "over": { 1646 | Description: "Copy the argument below the top of the stack on top", 1647 | Signature: " -- ", 1648 | Category: "Stack", 1649 | Run: func(context *EvalContext, command string, token Token) bool { 1650 | arity := 2 1651 | if len(context.argsStack) < arity { 1652 | fmt.Printf("%s: Expected %d arguments but got %d", token.Loc, arity, len(context.argsStack)) 1653 | return false 1654 | } 1655 | n := len(context.argsStack) 1656 | context.argsStack = append(context.argsStack, context.argsStack[n-2]) 1657 | return true 1658 | }, 1659 | }, 1660 | "swap": { 1661 | Description: "Swap two argument on top of the stack", 1662 | Signature: " -- ", 1663 | Category: "Stack", 1664 | Run: func(context *EvalContext, command string, token Token) bool { 1665 | arity := 2 1666 | if len(context.argsStack) < arity { 1667 | fmt.Printf("%s: Expected %d arguments but got %d", token.Loc, arity, len(context.argsStack)) 1668 | return false 1669 | } 1670 | n := len(context.argsStack) 1671 | context.argsStack[n-1], context.argsStack[n-2] = context.argsStack[n-2], context.argsStack[n-1]; 1672 | return true 1673 | }, 1674 | }, 1675 | "drop": { 1676 | Description: "Drop the argument on top of the stack", 1677 | Signature: " --", 1678 | Category: "Stack", 1679 | Run: func(context *EvalContext, command string, token Token) bool { 1680 | arity := 1 1681 | if len(context.argsStack) < arity { 1682 | fmt.Printf("%s: Expected %d arguments but got %d", token.Loc, arity, len(context.argsStack)) 1683 | return false 1684 | } 1685 | n := len(context.argsStack) 1686 | context.argsStack = context.argsStack[:n-1]; 1687 | return true 1688 | }, 1689 | }, 1690 | "dup": { 1691 | Description: "Duplicate the argument on top of the stack", 1692 | Signature: " -- ", 1693 | Category: "Stack", 1694 | Run: func(context *EvalContext, command string, token Token) bool { 1695 | arity := 1 1696 | if len(context.argsStack) < arity { 1697 | fmt.Printf("%s: Expected %d arguments but got %d", token.Loc, arity, len(context.argsStack)) 1698 | return false 1699 | } 1700 | n := len(context.argsStack) 1701 | // TODO: the location of the dupped value should be the location of the "dup" token 1702 | context.argsStack = append(context.argsStack, context.argsStack[n-1]) 1703 | return true 1704 | }, 1705 | }, 1706 | "input": { 1707 | Description: "Set the current input for the consequent chunks.", 1708 | Category: "Misc", 1709 | Signature: " --", 1710 | Run: func(context *EvalContext, command string, token Token) bool { 1711 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1712 | if err != nil { 1713 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1714 | fmt.Printf("%s\n", err) 1715 | return false 1716 | } 1717 | path := args[0] 1718 | if len(path.Text) == 0 { 1719 | fmt.Printf("%s: ERROR: cannot set empty input path\n", path.Loc) 1720 | return false 1721 | } 1722 | context.inputPath = string(path.Text) 1723 | context.inputPathLog = append(context.inputPathLog, path) 1724 | return true 1725 | }, 1726 | }, 1727 | "chapter": { 1728 | Description: "Define a new YouTube chapter for within a chunk for `markut summary` command.", 1729 | Category: "Misc", 1730 | Signature: " --", 1731 | Run: func(context *EvalContext, command string, token Token) bool { 1732 | args, err := context.typeCheckArgs(token.Loc, TokenString, TokenTimestamp) 1733 | if err != nil { 1734 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1735 | fmt.Printf("%s\n", err) 1736 | return false 1737 | } 1738 | context.chapStack = append(context.chapStack, Chapter{ 1739 | Loc: args[1].Loc, 1740 | Label: string(args[0].Text), 1741 | Timestamp: args[1].Timestamp, 1742 | }) 1743 | return true 1744 | }, 1745 | }, 1746 | "cut": { 1747 | Description: "Define a new cut for `markut cut` command.", 1748 | Category: "Misc", 1749 | Signature: " --", 1750 | Run: func(context *EvalContext, command string, token Token) bool { 1751 | args, err := context.typeCheckArgs(token.Loc, TokenTimestamp) 1752 | if err != nil { 1753 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1754 | fmt.Printf("%s\n", err) 1755 | return false 1756 | } 1757 | pad := args[0] 1758 | if len(context.chunks) == 0 { 1759 | fmt.Printf("%s: ERROR: no chunks defined for a cut\n", token.Loc) 1760 | return false 1761 | } 1762 | context.cuts = append(context.cuts, Cut{ 1763 | chunk: len(context.chunks) - 1, 1764 | pad: pad.Timestamp, 1765 | }) 1766 | return true 1767 | }, 1768 | }, 1769 | "include": { 1770 | Description: "Include another MARKUT file and fail if it does not exist.", 1771 | Category: "Misc", 1772 | Signature: " --", 1773 | Run: func(context *EvalContext, command string, token Token) bool { 1774 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1775 | if err != nil { 1776 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1777 | fmt.Printf("%s\n", err) 1778 | return false 1779 | } 1780 | path := args[0] 1781 | return context.evalMarkutFile(&path.Loc, string(path.Text), false) 1782 | }, 1783 | }, 1784 | "include_if_exists": { 1785 | Description: "Try to include another MARKUT file but do not fail if it does not exist.", 1786 | Category: "Misc", 1787 | Signature: " --", 1788 | Run: func(context *EvalContext, command string, token Token) bool { 1789 | args, err := context.typeCheckArgs(token.Loc, TokenString) 1790 | if err != nil { 1791 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1792 | fmt.Printf("%s\n", err) 1793 | return false 1794 | } 1795 | path := args[0] 1796 | return context.evalMarkutFile(&path.Loc, string(path.Text), true) 1797 | }, 1798 | }, 1799 | "home": { 1800 | Description: "Path to the home folder.", 1801 | Category: "Misc", 1802 | Signature: "-- ", 1803 | Run: func(context *EvalContext, command string, token Token) bool { 1804 | context.argsStack = append(context.argsStack, Token{ 1805 | Kind: TokenString, 1806 | Text: []rune(os.Getenv("HOME")), 1807 | Loc: token.Loc, 1808 | }) 1809 | return true 1810 | }, 1811 | }, 1812 | "concat": { 1813 | Description: "Concatenate two strings.", 1814 | Category: "Misc", 1815 | Signature: " -- ", 1816 | Run: func(context *EvalContext, command string, token Token) bool { 1817 | args, err := context.typeCheckArgs(token.Loc, TokenString, TokenString) 1818 | if err != nil { 1819 | fmt.Printf("%s: ERROR: type check failed for %s\n", token.Loc, command) 1820 | fmt.Printf("%s\n", err) 1821 | return false 1822 | } 1823 | context.argsStack = append(context.argsStack, Token{ 1824 | Kind: TokenString, 1825 | Text: slices.Concat(args[1].Text, args[0].Text), 1826 | Loc: token.Loc, 1827 | }) 1828 | return true 1829 | }, 1830 | }, 1831 | } 1832 | 1833 | name := os.Args[1] 1834 | args := os.Args[2:] 1835 | subcommand, ok := Subcommands[name] 1836 | if !ok { 1837 | usage() 1838 | fmt.Printf("ERROR: Unknown subcommand %s\n", name) 1839 | os.Exit(1) 1840 | } 1841 | if !subcommand.Run(name, args) { 1842 | os.Exit(1) 1843 | } 1844 | } 1845 | 1846 | // TODO: Consider rewritting Markut in C with nob.h 1847 | // There is no reason for it to be written in go at this point. C+nob.h can do all the tricks. 1848 | // For the lexing part we can even use https://github.com/tsoding/alexer 1849 | // TODO: Embed git hash into the executable and display it on `markut version` 1850 | --------------------------------------------------------------------------------