├── PythonRpcServer ├── .gitignore ├── transcribe_hellohellohello.wav ├── mediaprovider.py ├── unused │ ├── tesseract_test.py │ └── scenedetector_test.py ├── hasher.py ├── youtube_test.py ├── requirements.txt ├── ffmpeg.py └── titledetector_test.py ├── UnitTests ├── Assets │ ├── test.txt │ ├── no-captions.vtt │ ├── test.mov │ ├── test.mp4 │ ├── test.png │ ├── example.srt │ └── subtitles.xml ├── Utils │ ├── TestGlobals.cs │ ├── Common.cs │ └── MockServices.cs ├── ClassTranscribeServer │ └── ControllerTests │ │ ├── BaseControllerTest.cs │ │ ├── StaticFileConrollerTest.cs │ │ └── AdminControllerTest.cs ├── UnitTests.csproj ├── ClassTranscribeDatabase │ └── ModelsTest.cs └── README.md ├── TaskEngine ├── global.json ├── TaskEngine.sln ├── Tasks │ ├── CreateBoxTokenTask.cs │ ├── PythonCrawlerTask.cs │ ├── UpdateBoxTokenTask.cs │ └── ExampleTask.cs └── TaskEngine.csproj ├── ClassTranscribeServer ├── global.json ├── launchscript.sh ├── appsettings.Development.json ├── Connected Services │ └── Application Insights │ │ └── ConnectedService.json ├── Controllers │ ├── BaseController.cs │ ├── CaptionsSearchController.cs │ ├── RolesController.cs │ └── TermsController.cs ├── Program.cs └── Utils │ ├── UIUCSeed.cs │ └── SwaggerSchemaFilter.cs ├── ClassTranscribeDatabase ├── global.json ├── Migrations │ ├── 20201005192717_EPubFix.cs │ ├── 20200306150525_ChangedUser.cs │ ├── 20221116223342_LightweightVideo.cs │ ├── 20191025194553_Drop_MediaId_From_Video.cs │ ├── 20190910053352_FTS.cs │ ├── 20200212000529_Media-Name.cs │ ├── 20190624223657_two.cs │ ├── 20190709031430_AudioPath.cs │ ├── 20191025190603_Add_VideoId.cs │ ├── 20221129095657_GlossaryDataId.cs │ ├── 20190711220930_Status2.cs │ ├── 20200205192815_Playlist_JsonMetaData.cs │ ├── 20200326193756_Offering-JsonMetadata.cs │ ├── 20230105193204_GlossaryTimestamp.cs │ ├── 20190809193644_TranscriptionStatus.cs │ ├── 20191025215955_Transcription_Refactor.cs │ ├── 20190722215432_Caption_Table3.cs │ ├── 20200127214916_CaptionType.cs │ ├── 20230215055550_GlossaryOneParagraphExplanation.cs │ ├── 20191121171237_TranscribingAttempts.cs │ ├── 20220901042623_AddGlossaryToVideo.cs │ ├── 20190701185525_EndDate.cs │ ├── 20221229041355_ASLVideoGlossaryMapPublished.cs │ ├── 20211116200810_FilePath.cs │ ├── 20200119114020_Video.SceneData.cs │ ├── 20231211224116_PlaylistMediaOptions.cs │ ├── 20191024223759_CourseName-Desc.cs │ ├── 20231012191610_TaskLogString.cs │ ├── 20191024234713_DropCourseName-Desc-From-Courses.cs │ ├── 20190719214917_not_mapped.cs │ ├── 20200331223215_Indexing.cs │ ├── 20201008171703_AddDurationFileMediaInfo.cs │ ├── 20190903201631_Metadata.cs │ ├── 20190831191804_UpVote.cs │ ├── 20221122030110_PlaylistAddDateTimes.cs │ ├── 20190707011746_Media_modification.cs │ ├── 20191025195537_Add_Medias_to_Video.cs │ ├── 20190705215327_MediaJon.cs │ ├── 20201201145853_PublishControlForOfferings.cs │ ├── 20190717220252_FileRecords.cs │ ├── 20191025195048_Drop_MediaId_From_Video2.cs │ ├── 20221116231920_LightweightVideo2.cs │ ├── 20191026021015_SrtFile.cs │ ├── 20220930075448_ASLGlossaryMap.cs │ ├── 20200612211632_TaskItems.cs │ ├── 20200414071209_Epubchapter.cs │ ├── 20191025232219_Transcription_Refactorv2.cs │ ├── 20210908174659_NonnullVideoJObjects.cs │ ├── 20211013142047_TranscriptionsUpdate.cs │ ├── 20191112212156_EPub.cs │ ├── 20190717222124_FileRecordsForeignKey.cs │ ├── 20200326194432_Add_WatchHistory_Table.cs │ ├── 20190722204241_Caption_Table2.cs │ ├── 20190918214544_Logs-RemoveFK.cs │ ├── 20190722203911_Caption_Table.cs │ ├── 20190922221537_Add_Processed_Video.cs │ ├── 20220930063430_ASLVideo.cs │ ├── 20220722044401_Glossary.cs │ ├── 20231011192212_aslvideo3.cs │ ├── 20201113200736_Disable_Soft_Delete_UserOffering.cs │ ├── 20221116162318_PhraseHintJsonTable.cs │ ├── 20190831192420_Log.cs │ ├── 20200205033251_Dictionary.cs │ └── 20190707013307_Add_Playlist.cs ├── Services │ ├── Grpc.cs │ ├── MSTranscription │ │ └── KeyProvider.cs │ └── Notification.cs ├── migration_instruction.md ├── CaptionQueries.cs └── ClassTranscribeDatabase.csproj ├── TestRemoteLLM ├── dieselsubmarine.jpg ├── ClassTranscribeStudentsUse2020.png └── TestRemoteLLM.csproj ├── TestAzureCognitiveServices ├── shortwav.wav ├── shortvideo.mp4 ├── .dockerignore ├── TestAzureCognitiveServices.csproj ├── install-libssl1.sh └── TestAzure.Dockerfile ├── randomvoice_16kHz.json ├── .dockerignore ├── .editorconfig ├── gotchas.md ├── install-speech-hack-libssl1.sh ├── .github └── workflows │ └── ci.yaml ├── DevExperiments ├── DevExperiments.csproj └── Program.cs ├── LICENSE ├── TaskEngine.Dockerfile ├── ct.proto ├── vs_appsettings-example.txt ├── API.Dockerfile ├── pythonrpcserver.Dockerfile ├── .gitignore └── README.md /PythonRpcServer/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | -------------------------------------------------------------------------------- /UnitTests/Assets/test.txt: -------------------------------------------------------------------------------- 1 | This is a test file. -------------------------------------------------------------------------------- /TaskEngine/global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "8.0" 4 | } 5 | } -------------------------------------------------------------------------------- /ClassTranscribeServer/global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "8.0" 4 | } 5 | } -------------------------------------------------------------------------------- /ClassTranscribeDatabase/global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "8.0" 4 | } 5 | } -------------------------------------------------------------------------------- /UnitTests/Assets/no-captions.vtt: -------------------------------------------------------------------------------- 1 | WEBVTT 2 | 3 | NOTE 4 | duration:"00:00:00" 5 | language:En-US -------------------------------------------------------------------------------- /UnitTests/Assets/test.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/UnitTests/Assets/test.mov -------------------------------------------------------------------------------- /UnitTests/Assets/test.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/UnitTests/Assets/test.mp4 -------------------------------------------------------------------------------- /UnitTests/Assets/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/UnitTests/Assets/test.png -------------------------------------------------------------------------------- /TestRemoteLLM/dieselsubmarine.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/TestRemoteLLM/dieselsubmarine.jpg -------------------------------------------------------------------------------- /TestAzureCognitiveServices/shortwav.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/TestAzureCognitiveServices/shortwav.wav -------------------------------------------------------------------------------- /TestAzureCognitiveServices/shortvideo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/TestAzureCognitiveServices/shortvideo.mp4 -------------------------------------------------------------------------------- /PythonRpcServer/transcribe_hellohellohello.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/PythonRpcServer/transcribe_hellohellohello.wav -------------------------------------------------------------------------------- /TestRemoteLLM/ClassTranscribeStudentsUse2020.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/classtranscribe/WebAPI/HEAD/TestRemoteLLM/ClassTranscribeStudentsUse2020.png -------------------------------------------------------------------------------- /ClassTranscribeServer/launchscript.sh: -------------------------------------------------------------------------------- 1 | dotnet ef -p /src/ClassTranscribeDatabase/ClassTranscribeDatabase.csproj database update 2 | dotnet /app/ClassTranscribeServer.dll 3 | -------------------------------------------------------------------------------- /TestAzureCognitiveServices/.dockerignore: -------------------------------------------------------------------------------- 1 | **/.dockerignore 2 | **/.env 3 | **/.git 4 | **/.gitignore 5 | **/.vs 6 | **/.vscode 7 | **/*.*proj.user 8 | **/charts 9 | **/bin 10 | **/obj 11 | *Dockerfile 12 | *cproj 13 | -------------------------------------------------------------------------------- /ClassTranscribeServer/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "System": "Information", 6 | "Microsoft": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /ClassTranscribeServer/Connected Services/Application Insights/ConnectedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "ProviderId": "Microsoft.ApplicationInsights.ConnectedService.ConnectedServiceProvider", 3 | "Version": "16.0.0.0", 4 | "GettingStartedDocument": { 5 | "Uri": "https://go.microsoft.com/fwlink/?LinkID=798432" 6 | } 7 | } -------------------------------------------------------------------------------- /randomvoice_16kHz.json: -------------------------------------------------------------------------------- 1 | {"text": " Hello? Hello? Hello?", "segments": [{"id": 0, "seek": 0, "start": 0.0, "end": 3.0, "text": " Hello? Hello? Hello?", "tokens": [50363, 18435, 30, 18435, 30, 18435, 30, 50513], "temperature": 0.0, "avg_logprob": -0.636968559688992, "compression_ratio": 1.1764705882352942, "no_speech_prob": 0.22877301275730133}], "language": "en"} -------------------------------------------------------------------------------- /TestAzureCognitiveServices/TestAzureCognitiveServices.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.dockerignore 2 | **/.env 3 | **/.git 4 | **/.gitignore 5 | **/.vs 6 | **/.vscode 7 | **/*.*proj.user 8 | **/azds.yaml 9 | **/charts 10 | **/bin 11 | **/obj 12 | **/Dockerfile 13 | **/Dockerfile.develop 14 | **/docker-compose.yml 15 | **/docker-compose.*.yml 16 | **/*.dbmdl 17 | **/*.jfm 18 | **/secrets.dev.yaml 19 | **/values.dev.yaml 20 | **/.toolstarget 21 | **/node_modules 22 | whisper.cpp 23 | -------------------------------------------------------------------------------- /PythonRpcServer/mediaprovider.py: -------------------------------------------------------------------------------- 1 | class MediaProvider: 2 | def getPlaylistItems(self, request): 3 | raise NotImplementedError("To be implemented") 4 | 5 | def getMedia(self, request): 6 | raise NotImplementedError("To be implemented") 7 | 8 | class InvalidPlaylistInfoException(Exception): 9 | def __init__(self, message = 'INVALID_PLAYLIST_IDENTIFIER'): 10 | self.message = message 11 | 12 | -------------------------------------------------------------------------------- /UnitTests/Utils/TestGlobals.cs: -------------------------------------------------------------------------------- 1 | namespace UnitTests 2 | { 3 | public static class TestGlobals 4 | { 5 | public const string TEST_USER_ID = "TestUser"; 6 | // This is an example of a valid JWT token from https://jwt.io/ 7 | public const string TEST_JWT_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20201005192717_EPubFix.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class EPubFix : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | 10 | } 11 | 12 | protected override void Down(MigrationBuilder migrationBuilder) 13 | { 14 | 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200306150525_ChangedUser.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class ChangedUser : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | 10 | } 11 | 12 | protected override void Down(MigrationBuilder migrationBuilder) 13 | { 14 | 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20221116223342_LightweightVideo.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class LightweightVideo : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | 10 | } 11 | 12 | protected override void Down(MigrationBuilder migrationBuilder) 13 | { 14 | 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191025194553_Drop_MediaId_From_Video.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Drop_MediaId_From_Video : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | 10 | } 11 | 12 | protected override void Down(MigrationBuilder migrationBuilder) 13 | { 14 | 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /TestRemoteLLM/TestRemoteLLM.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.cs] 2 | 3 | # CA2007: Consider calling ConfigureAwait on the awaited task 4 | dotnet_diagnostic.CA2007.severity = none 5 | 6 | # CA1303: Do not pass literals as localized parameters 7 | dotnet_diagnostic.CA1303.severity = none 8 | 9 | # CA1307: Specify StringComparison 10 | dotnet_diagnostic.CA1307.severity = none 11 | 12 | # CA1034: Nested types should not be visible 13 | dotnet_diagnostic.CA1034.severity = none 14 | 15 | # CA2227: Collection properties should be read only 16 | dotnet_diagnostic.CA2227.severity = none 17 | 18 | # CA1305: Specify IFormatProvider 19 | dotnet_diagnostic.CA1305.severity = none 20 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190910053352_FTS.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class FTS : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.Sql(@"CREATE INDEX fts_idx ON ""Captions"" USING GIN (to_tsvector('english', ""Text""));"); 10 | } 11 | 12 | protected override void Down(MigrationBuilder migrationBuilder) 13 | { 14 | migrationBuilder.Sql(@"DROP INDEX fts_idx;"); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /TestAzureCognitiveServices/install-libssl1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Temporary 2024 Hack for MS SpeechSDK to run on dotnet8 3 | ARCH=$(dpkg --print-architecture) 4 | 5 | if [ "$ARCH" = "arm64" ] ; then 6 | BASE="http://ports.ubuntu.com/ubuntu-ports/pool/main/o/openssl/" 7 | else 8 | BASE="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/" 9 | fi 10 | 11 | wget $BASE/libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb 12 | wget $BASE/libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb 13 | dpkg -i libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb 14 | dpkg -i libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb 15 | rm libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb 16 | 17 | 18 | -------------------------------------------------------------------------------- /ClassTranscribeServer/Controllers/BaseController.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using Microsoft.AspNetCore.Mvc; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace ClassTranscribeServer.Controllers 6 | { 7 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1051:Do not declare visible instance fields")] 8 | public class BaseController : ControllerBase 9 | { 10 | protected readonly CTDbContext _context; 11 | protected readonly ILogger _logger; 12 | 13 | public BaseController(CTDbContext context, ILogger logger) 14 | { 15 | _context = context; 16 | _logger = logger; 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /gotchas.md: -------------------------------------------------------------------------------- 1 | ## Gotchas 2 | 3 | This document lists known coding mistakes and other gotchas. 4 | 5 | 1. Lazy loading error. Please refer [this](https://docs.microsoft.com/en-us/ef/ef6/querying/related-data) to know the best practices on loading related entities of a model. 6 | 2. An update from Microsoft prevents EntityFrameworkCore's Add-Migration from completing. Solution: 7 | In ClassTranscribeDatabase.csproj remove or xml-comment "\all\" from \ 8 | See [this](https://stackoverflow.com/questions/52536588/your-startup-project-doesnt-reference-microsoft-entityframeworkcore-design) 9 | 10 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200212000529_Media-Name.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class MediaName : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Name", 11 | table: "Medias", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "Name", 19 | table: "Medias"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190624223657_two.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class two : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "CourseName", 11 | table: "Courses", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "CourseName", 19 | table: "Courses"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190709031430_AudioPath.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class AudioPath : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "AudioPath", 11 | table: "Videos", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "AudioPath", 19 | table: "Videos"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191025190603_Add_VideoId.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Add_VideoId : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "VideoId", 11 | table: "Medias", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "VideoId", 19 | table: "Medias"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20221129095657_GlossaryDataId.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class GlossaryDataId : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "GlossaryDataId", 11 | table: "Videos", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "GlossaryDataId", 19 | table: "Videos"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190711220930_Status2.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Status2 : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Status", 11 | table: "AspNetUsers", 12 | nullable: false, 13 | defaultValue: 0); 14 | } 15 | 16 | protected override void Down(MigrationBuilder migrationBuilder) 17 | { 18 | migrationBuilder.DropColumn( 19 | name: "Status", 20 | table: "AspNetUsers"); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200205192815_Playlist_JsonMetaData.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Playlist_JsonMetaData : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "JsonMetadata", 11 | table: "Playlists", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "JsonMetadata", 19 | table: "Playlists"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200326193756_Offering-JsonMetadata.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class OfferingJsonMetadata : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "JsonMetadata", 11 | table: "Offerings", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "JsonMetadata", 19 | table: "Offerings"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20230105193204_GlossaryTimestamp.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class GlossaryTimestamp : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "GlossaryTimestampId", 11 | table: "Videos", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "GlossaryTimestampId", 19 | table: "Videos"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190809193644_TranscriptionStatus.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class TranscriptionStatus : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "TranscriptionStatus", 11 | table: "Videos", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "TranscriptionStatus", 19 | table: "Videos"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191025215955_Transcription_Refactor.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Transcription_Refactor : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "VideoId", 11 | table: "Transcriptions", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "VideoId", 19 | table: "Transcriptions"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190722215432_Caption_Table3.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Caption_Table3 : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Index", 11 | table: "Captions", 12 | nullable: false, 13 | defaultValue: 0); 14 | } 15 | 16 | protected override void Down(MigrationBuilder migrationBuilder) 17 | { 18 | migrationBuilder.DropColumn( 19 | name: "Index", 20 | table: "Captions"); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200127214916_CaptionType.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class CaptionType : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "CaptionType", 11 | table: "Captions", 12 | nullable: false, 13 | defaultValue: 0); 14 | } 15 | 16 | protected override void Down(MigrationBuilder migrationBuilder) 17 | { 18 | migrationBuilder.DropColumn( 19 | name: "CaptionType", 20 | table: "Captions"); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20230215055550_GlossaryOneParagraphExplanation.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class GlossaryOneParagraphExplanation : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Explanation", 11 | table: "Glossaries", 12 | nullable: true); 13 | } 14 | 15 | protected override void Down(MigrationBuilder migrationBuilder) 16 | { 17 | migrationBuilder.DropColumn( 18 | name: "Explanation", 19 | table: "Glossaries"); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /PythonRpcServer/unused/tesseract_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytesseract 3 | import cv2 4 | 5 | def get_test_image(): 6 | font = cv2.FONT_HERSHEY_SIMPLEX 7 | org = (50,300) 8 | fontScale = 1 9 | color = (255,255,255) 10 | thickness = 2 11 | text = 'Test' 12 | width,height = (1024,720) 13 | image = np.zeros((height,width,3), np.uint8) 14 | image = cv2.putText(image,text,org,font,fontScale,color,thickness,cv2.LINE_AA) 15 | return image 16 | 17 | def tesseract_test(): 18 | frame = get_test_image() 19 | gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 20 | ocr_result = pytesseract.image_to_data(gray_frame, output_type='dict') 21 | print(ocr_result) 22 | assert( 'Test' == ocr_result['text'][0]) 23 | 24 | if __name__ == '__main__' : 25 | tesseract_test() -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191121171237_TranscribingAttempts.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class TranscribingAttempts : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "TranscribingAttempts", 11 | table: "Videos", 12 | nullable: false, 13 | defaultValue: 0); 14 | } 15 | 16 | protected override void Down(MigrationBuilder migrationBuilder) 17 | { 18 | migrationBuilder.DropColumn( 19 | name: "TranscribingAttempts", 20 | table: "Videos"); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20220901042623_AddGlossaryToVideo.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class AddGlossaryToVideo : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Glossary", 11 | table: "Videos", 12 | nullable: false, 13 | defaultValue: ""); 14 | 15 | } 16 | 17 | protected override void Down(MigrationBuilder migrationBuilder) 18 | { 19 | migrationBuilder.DropColumn( 20 | name: "Glossary", 21 | table: "Videos"); 22 | 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190701185525_EndDate.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | using System; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class EndDate : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.AddColumn( 11 | name: "EndDate", 12 | table: "Terms", 13 | nullable: false, 14 | defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); 15 | } 16 | 17 | protected override void Down(MigrationBuilder migrationBuilder) 18 | { 19 | migrationBuilder.DropColumn( 20 | name: "EndDate", 21 | table: "Terms"); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20221229041355_ASLVideoGlossaryMapPublished.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class ASLVideoGlossaryMapPublished : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Published", 11 | table: "ASLVideoGlossaryMaps", 12 | nullable: false, 13 | defaultValue: false); 14 | } 15 | 16 | protected override void Down(MigrationBuilder migrationBuilder) 17 | { 18 | migrationBuilder.DropColumn( 19 | name: "Published", 20 | table: "ASLVideoGlossaryMaps"); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /install-speech-hack-libssl1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Temporary 2024 Hack for MS SpeechSDK to run on dotnet8 4 | # https://github.com/Azure-Samples/cognitive-services-speech-sdk/issues/2204 5 | 6 | SSLVERSION="1.1.1f" 7 | 8 | ARCH=$(dpkg --print-architecture) 9 | if [ "$ARCH" = "arm64" ] ; then 10 | BASE="http://ports.ubuntu.com/pool/main/o/openssl" 11 | RELEASE="1ubuntu2" 12 | else 13 | BASE="http://security.ubuntu.com/ubuntu/pool/main/o/openssl" 14 | RELEASE="1ubuntu2" 15 | fi 16 | 17 | wget ${BASE}/libssl1.1_${SSLVERSION}-${RELEASE}_${ARCH}.deb 18 | wget ${BASE}/libssl-dev_${SSLVERSION}-${RELEASE}_${ARCH}.deb 19 | dpkg -i libssl1.1_${SSLVERSION}-${RELEASE}_${ARCH}.deb 20 | dpkg -i libssl-dev_${SSLVERSION}-${RELEASE}_${ARCH}.deb 21 | rm libssl1.1_${SSLVERSION}-${RELEASE}_${ARCH}.deb libssl-dev_${SSLVERSION}-${RELEASE}_${ARCH}.deb 22 | 23 | 24 | -------------------------------------------------------------------------------- /PythonRpcServer/hasher.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | #eventually this may replace the C# method 4 | # However today the C# file hash is calculated inside the Database project, which does not depende on the RPC projec 5 | 6 | # This implementation can be extended to create multiple digests using a single read through the file 7 | # A major advantage of calculating this in python is that we can perform this under ionice and nice(cpu) constraints 8 | def hashFile(filepath, algorithms): 9 | 10 | if algorithms != "sha256" : 11 | raise Exception(f"digest not yet implemented: alg=({algorithms})") 12 | 13 | sha256 = hashlib.sha256() 14 | 15 | blocksize = 64 * 1024 16 | 17 | with open(filepath, 'rb') as f: 18 | while True: 19 | block = f.read(blocksize) 20 | if not block: 21 | break 22 | sha256.update(block) 23 | 24 | return sha256.hexdigest() 25 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Services/Grpc.cs: -------------------------------------------------------------------------------- 1 | using CTGrpc; 2 | using Grpc.Core; 3 | using System; 4 | 5 | namespace ClassTranscribeDatabase.Services 6 | { 7 | public class RpcClient 8 | { 9 | AppSettings _appSettings; 10 | public PythonServer.PythonServerClient PythonServerClient; 11 | public RpcClient() 12 | { 13 | _appSettings = Globals.appSettings; 14 | AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); 15 | var channel = new Channel(_appSettings.PYTHON_RPC_SERVER, ChannelCredentials.Insecure, new[]{ 16 | new ChannelOption(ChannelOptions.MaxSendMessageLength , 2*1024*1024), 17 | new ChannelOption(ChannelOptions.MaxReceiveMessageLength , 5 *1024*1024) 18 | }); 19 | PythonServerClient = new PythonServer.PythonServerClient(channel); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - staging 8 | - expt 9 | pull_request: 10 | 11 | jobs: 12 | build: 13 | name: Build 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - uses: actions/setup-dotnet@v1 19 | with: 20 | dotnet-version: '8.0.x' # SDK Version to use; x will use the latest version of the channel 21 | 22 | - uses: actions/cache@v4 23 | with: 24 | path: ~/.nuget/packages 25 | key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }} 26 | restore-keys: | 27 | ${{ runner.os }}-nuget- 28 | 29 | - name: Compile application 30 | run: | 31 | dotnet build ClassTranscribeServer.sln --configuration Release 32 | 33 | - name: Run unit tests 34 | run: | 35 | dotnet test UnitTests --configuration Release -v n 36 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20211116200810_FilePath.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class FilePath : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "FilePath", 11 | table: "Courses", 12 | nullable: true); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "FilePath", 16 | table: "CourseOfferings", 17 | nullable: true); 18 | } 19 | 20 | protected override void Down(MigrationBuilder migrationBuilder) 21 | { 22 | migrationBuilder.DropColumn( 23 | name: "FilePath", 24 | table: "Courses"); 25 | 26 | migrationBuilder.DropColumn( 27 | name: "FilePath", 28 | table: "CourseOfferings"); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200119114020_Video.SceneData.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class VideoSceneData : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "JsonMetadata", 11 | table: "Videos", 12 | nullable: true); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "SceneData", 16 | table: "Videos", 17 | nullable: true); 18 | } 19 | 20 | protected override void Down(MigrationBuilder migrationBuilder) 21 | { 22 | migrationBuilder.DropColumn( 23 | name: "JsonMetadata", 24 | table: "Videos"); 25 | 26 | migrationBuilder.DropColumn( 27 | name: "SceneData", 28 | table: "Videos"); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20231211224116_PlaylistMediaOptions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class PlaylistMediaOptions : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Options", 11 | table: "Playlists", 12 | nullable: true); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "Options", 16 | table: "Medias", 17 | nullable: true); 18 | } 19 | 20 | protected override void Down(MigrationBuilder migrationBuilder) 21 | { 22 | migrationBuilder.DropColumn( 23 | name: "Options", 24 | table: "Playlists"); 25 | 26 | migrationBuilder.DropColumn( 27 | name: "Options", 28 | table: "Medias"); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191024223759_CourseName-Desc.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class CourseNameDesc : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "CourseName", 11 | table: "Offerings", 12 | nullable: true); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "Description", 16 | table: "Offerings", 17 | nullable: true); 18 | } 19 | 20 | protected override void Down(MigrationBuilder migrationBuilder) 21 | { 22 | migrationBuilder.DropColumn( 23 | name: "CourseName", 24 | table: "Offerings"); 25 | 26 | migrationBuilder.DropColumn( 27 | name: "Description", 28 | table: "Offerings"); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20231012191610_TaskLogString.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class TaskLogString : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropColumn( 10 | name: "ProcessingLog", 11 | table: "Videos"); 12 | 13 | migrationBuilder.AddColumn( 14 | name: "TaskLog", 15 | table: "Videos", 16 | nullable: true); 17 | } 18 | 19 | protected override void Down(MigrationBuilder migrationBuilder) 20 | { 21 | migrationBuilder.DropColumn( 22 | name: "TaskLog", 23 | table: "Videos"); 24 | 25 | migrationBuilder.AddColumn( 26 | name: "ProcessingLog", 27 | table: "Videos", 28 | type: "text", 29 | nullable: true); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191024234713_DropCourseName-Desc-From-Courses.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class DropCourseNameDescFromCourses : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropColumn( 10 | name: "CourseName", 11 | table: "Courses"); 12 | 13 | migrationBuilder.DropColumn( 14 | name: "Description", 15 | table: "Courses"); 16 | } 17 | 18 | protected override void Down(MigrationBuilder migrationBuilder) 19 | { 20 | migrationBuilder.AddColumn( 21 | name: "CourseName", 22 | table: "Courses", 23 | nullable: true); 24 | 25 | migrationBuilder.AddColumn( 26 | name: "Description", 27 | table: "Courses", 28 | nullable: true); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190719214917_not_mapped.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class not_mapped : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.RenameColumn( 10 | name: "Path", 11 | table: "FileRecords", 12 | newName: "PrivatePath"); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "FileName", 16 | table: "FileRecords", 17 | nullable: true); 18 | } 19 | 20 | protected override void Down(MigrationBuilder migrationBuilder) 21 | { 22 | migrationBuilder.DropColumn( 23 | name: "FileName", 24 | table: "FileRecords"); 25 | 26 | migrationBuilder.RenameColumn( 27 | name: "PrivatePath", 28 | table: "FileRecords", 29 | newName: "Path"); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200331223215_Indexing.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Indexing : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Index", 11 | table: "Playlists", 12 | nullable: false, 13 | defaultValue: 0); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "Index", 17 | table: "Medias", 18 | nullable: false, 19 | defaultValue: 0); 20 | } 21 | 22 | protected override void Down(MigrationBuilder migrationBuilder) 23 | { 24 | migrationBuilder.DropColumn( 25 | name: "Index", 26 | table: "Playlists"); 27 | 28 | migrationBuilder.DropColumn( 29 | name: "Index", 30 | table: "Medias"); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20201008171703_AddDurationFileMediaInfo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class AddDurationFileMediaInfo : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.AddColumn( 11 | name: "Duration", 12 | table: "Videos", 13 | nullable: true); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "FileMediaInfo", 17 | table: "Videos", 18 | nullable: true); 19 | } 20 | 21 | protected override void Down(MigrationBuilder migrationBuilder) 22 | { 23 | migrationBuilder.DropColumn( 24 | name: "Duration", 25 | table: "Videos"); 26 | 27 | migrationBuilder.DropColumn( 28 | name: "FileMediaInfo", 29 | table: "Videos"); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190903201631_Metadata.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Metadata : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "LogEventsFlag", 11 | table: "Offerings", 12 | nullable: false, 13 | defaultValue: false); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "Metadata", 17 | table: "AspNetUsers", 18 | nullable: true); 19 | } 20 | 21 | protected override void Down(MigrationBuilder migrationBuilder) 22 | { 23 | migrationBuilder.DropColumn( 24 | name: "LogEventsFlag", 25 | table: "Offerings"); 26 | 27 | migrationBuilder.DropColumn( 28 | name: "Metadata", 29 | table: "AspNetUsers"); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190831191804_UpVote.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class UpVote : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "DownVote", 11 | table: "Captions", 12 | nullable: false, 13 | defaultValue: 0); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "UpVote", 17 | table: "Captions", 18 | nullable: false, 19 | defaultValue: 0); 20 | } 21 | 22 | protected override void Down(MigrationBuilder migrationBuilder) 23 | { 24 | migrationBuilder.DropColumn( 25 | name: "DownVote", 26 | table: "Captions"); 27 | 28 | migrationBuilder.DropColumn( 29 | name: "UpVote", 30 | table: "Captions"); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20221122030110_PlaylistAddDateTimes.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class PlaylistAddDateTimes : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.AddColumn( 11 | name: "ListCheckedAt", 12 | table: "Playlists", 13 | nullable: true); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "ListUpdatedAt", 17 | table: "Playlists", 18 | nullable: true); 19 | } 20 | 21 | protected override void Down(MigrationBuilder migrationBuilder) 22 | { 23 | migrationBuilder.DropColumn( 24 | name: "ListCheckedAt", 25 | table: "Playlists"); 26 | 27 | migrationBuilder.DropColumn( 28 | name: "ListUpdatedAt", 29 | table: "Playlists"); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190707011746_Media_modification.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Media_modification : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropColumn( 10 | name: "MediaSource", 11 | table: "Medias"); 12 | 13 | migrationBuilder.RenameColumn( 14 | name: "MediaUrl", 15 | table: "Medias", 16 | newName: "UniqueMediaIdentifier"); 17 | } 18 | 19 | protected override void Down(MigrationBuilder migrationBuilder) 20 | { 21 | migrationBuilder.RenameColumn( 22 | name: "UniqueMediaIdentifier", 23 | table: "Medias", 24 | newName: "MediaUrl"); 25 | 26 | migrationBuilder.AddColumn( 27 | name: "MediaSource", 28 | table: "Medias", 29 | nullable: true); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /TaskEngine/TaskEngine.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.002.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TaskEngine", "TaskEngine.csproj", "{09720F43-452A-43D0-BE7E-AAE2CF822AA0}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {09720F43-452A-43D0-BE7E-AAE2CF822AA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {09720F43-452A-43D0-BE7E-AAE2CF822AA0}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {09720F43-452A-43D0-BE7E-AAE2CF822AA0}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {09720F43-452A-43D0-BE7E-AAE2CF822AA0}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {83861AB8-4CDE-41B0-8309-63BD076F3EA6} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/migration_instruction.md: -------------------------------------------------------------------------------- 1 | ## How to create migrations in EF Core 2 | 1. Install dotnet with version 3.1.201. Please be advised that ClassTranscribe Database may not run properly on newer version. Install instruction can be found at https://github.com/dotnet/core/blob/main/release-notes/3.1/3.1.3/3.1.201-download.md 3 | 4 | 2. Install dotnet-ef 3.1.4 by running 5 | ``` 6 | dotnet tool install --global dotnet-ef --version 3.1.4 7 | ``` 8 | 9 | 3. Go to the ClassTranscribeDatabase directory 10 | ``` 11 | cd WebAPI/ClassTranscribeDatabase 12 | ``` 13 | 14 | 4. Create a new migration by running 15 | ``` 16 | (also try dotnet ef may work for you) 17 | dotnet-ef migrations add 18 | ``` 19 | 20 | ## How to apply migration in the local database 21 | ``` 22 | dotnet-ef database update 23 | ``` 24 | 25 | To remove, 26 | ``` 27 | dotnet-ef migrations remove 28 | ``` 29 | 30 | ## How to apply migration in Docker container 31 | 1. Rebuild the solution 32 | ``` 33 | dotnet build --no-restore 34 | ``` 35 | 36 | 2. Rebuild API image 37 | ``` 38 | docker build -t api -f API.Dockerfile . 39 | ``` 40 | 41 | 3. Run Docker compose to see the changes 42 | ``` 43 | docker compose up api 44 | ``` -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191025195537_Add_Medias_to_Video.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Add_Medias_to_Video : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.CreateIndex( 10 | name: "IX_Medias_VideoId", 11 | table: "Medias", 12 | column: "VideoId"); 13 | 14 | migrationBuilder.AddForeignKey( 15 | name: "FK_Medias_Videos_VideoId", 16 | table: "Medias", 17 | column: "VideoId", 18 | principalTable: "Videos", 19 | principalColumn: "Id", 20 | onDelete: ReferentialAction.Restrict); 21 | } 22 | 23 | protected override void Down(MigrationBuilder migrationBuilder) 24 | { 25 | migrationBuilder.DropForeignKey( 26 | name: "FK_Medias_Videos_VideoId", 27 | table: "Medias"); 28 | 29 | migrationBuilder.DropIndex( 30 | name: "IX_Medias_VideoId", 31 | table: "Medias"); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /TaskEngine/Tasks/CreateBoxTokenTask.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Services; 2 | using Microsoft.Extensions.Logging; 3 | using System.Diagnostics.CodeAnalysis; 4 | using System.Threading.Tasks; 5 | using static ClassTranscribeDatabase.CommonUtils; 6 | 7 | #pragma warning disable CA2007 8 | // https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca2007 9 | // We are okay awaiting on a task in the same thread 10 | 11 | namespace TaskEngine.Tasks 12 | { 13 | [SuppressMessage("Microsoft.Performance", "CA1812:MarkMembersAsStatic")] // This class is never directly instantiated 14 | class CreateBoxTokenTask : RabbitMQTask 15 | { 16 | private readonly BoxAPI _box; 17 | public CreateBoxTokenTask(RabbitMQConnection rabbitMQ, BoxAPI box, ILogger logger) 18 | : base(rabbitMQ, TaskType.CreateBoxToken, logger) 19 | { 20 | _box = box; 21 | } 22 | 23 | protected async override Task OnConsume(string authCode, TaskParameters taskParameters, ClientActiveTasks cleanup) 24 | { 25 | RegisterTask(cleanup, "CreateAccessTokenAsync"); 26 | await _box.CreateAccessTokenAsync(authCode); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /UnitTests/ClassTranscribeServer/ControllerTests/BaseControllerTest.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeServer.Utils; 3 | using Microsoft.EntityFrameworkCore; 4 | using System; 5 | using Xunit; 6 | 7 | namespace UnitTests.ClassTranscribeServer.ControllerTests 8 | { 9 | [Collection("Global")] 10 | public class BaseControllerTest 11 | { 12 | protected readonly CTDbContext _context; 13 | protected readonly UserUtils _userUtils; 14 | 15 | // This constructor is run before every test, ensuring a new context and in-memory DB for each test case 16 | // https://xunit.net/docs/shared-context 17 | public BaseControllerTest(GlobalFixture fixture) 18 | { 19 | var optionsBuilder = new DbContextOptionsBuilder() 20 | .UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString()) 21 | .UseInternalServiceProvider(fixture._serviceProvider); 22 | 23 | _context = new CTDbContext(optionsBuilder.Options, null); 24 | _userUtils = new UserUtils( 25 | (MockUserManager) fixture._serviceProvider.GetService(typeof(MockUserManager)), 26 | _context 27 | ); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /DevExperiments/DevExperiments.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Always 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /ClassTranscribeServer/Program.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using Microsoft.AspNetCore; 3 | using Microsoft.AspNetCore.Hosting; 4 | using Microsoft.Extensions.DependencyInjection; 5 | using Microsoft.Extensions.Logging; 6 | using System; 7 | 8 | namespace ClassTranscribeServer 9 | { 10 | public static class Program 11 | { 12 | public static void Main(string[] args) 13 | { 14 | CreateWebHostBuilder(args).Build().Run(); 15 | } 16 | 17 | public static IWebHostBuilder CreateWebHostBuilder(string[] args) 18 | { 19 | var v = WebHost.CreateDefaultBuilder(args) 20 | .ConfigureServices(c => c.AddOptions().Configure(CTDbContext.GetConfigurations())); 21 | 22 | // TTODO better code would use AppSettings 23 | 24 | string viewSQL = Environment.GetEnvironmentVariable("LogEntityFrameworkSQL") ?? "false"; 25 | 26 | if( viewSQL.Trim().ToUpperInvariant() != "TRUE") { 27 | 28 | v.ConfigureLogging((context, logging) => { 29 | logging.AddFilter("Microsoft.EntityFrameworkCore.Database.Command", LogLevel.Warning); 30 | }); 31 | } 32 | return v.UseStartup(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /TaskEngine/Tasks/PythonCrawlerTask.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Models; 3 | using ClassTranscribeDatabase.Services; 4 | using Microsoft.Extensions.Logging; 5 | using Newtonsoft.Json.Linq; 6 | using System.Collections.Generic; 7 | using System.Diagnostics.CodeAnalysis; 8 | using System.IO; 9 | using System.Threading.Tasks; 10 | using static ClassTranscribeDatabase.CommonUtils; 11 | 12 | #pragma warning disable CA2007 13 | // https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca2007 14 | // We are okay awaiting on a task in the same thread 15 | 16 | namespace TaskEngine.Tasks 17 | { 18 | [SuppressMessage("Microsoft.Performance", "CA1812:MarkMembersAsStatic")] // This class is never directly instantiated 19 | class PythonCrawlerTask : RabbitMQTask 20 | { 21 | public PythonCrawlerTask(RabbitMQConnection rabbitMQ, ILogger logger) 22 | : base(rabbitMQ, TaskType.PythonCrawler, logger) 23 | { 24 | 25 | } 26 | /// Extracts ASL videos from online sources 27 | 28 | protected async override Task OnConsume(string sourceId, TaskParameters taskParameters, ClientActiveTasks cleanup) 29 | { 30 | // To suppress CS1998 warning 31 | await Task.CompletedTask; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190705215327_MediaJon.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class MediaJon : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.RenameColumn( 10 | name: "Path", 11 | table: "Videos", 12 | newName: "Video2Path"); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "Video1Path", 16 | table: "Videos", 17 | nullable: true); 18 | 19 | migrationBuilder.AddColumn( 20 | name: "SourceType", 21 | table: "Medias", 22 | nullable: false, 23 | defaultValue: 0); 24 | } 25 | 26 | protected override void Down(MigrationBuilder migrationBuilder) 27 | { 28 | migrationBuilder.DropColumn( 29 | name: "Video1Path", 30 | table: "Videos"); 31 | 32 | migrationBuilder.DropColumn( 33 | name: "SourceType", 34 | table: "Medias"); 35 | 36 | migrationBuilder.RenameColumn( 37 | name: "Video2Path", 38 | table: "Videos", 39 | newName: "Path"); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20201201145853_PublishControlForOfferings.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class PublishControlForOfferings : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "IsPublished", 11 | table: "Playlists", 12 | nullable: false, 13 | defaultValue: true); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "IsPublished", 17 | table: "Offerings", 18 | nullable: false, 19 | defaultValue: true); 20 | 21 | migrationBuilder.AddColumn( 22 | name: "IsPublished", 23 | table: "Medias", 24 | nullable: false, 25 | defaultValue: true); 26 | } 27 | 28 | protected override void Down(MigrationBuilder migrationBuilder) 29 | { 30 | migrationBuilder.DropColumn( 31 | name: "IsPublished", 32 | table: "Playlists"); 33 | 34 | migrationBuilder.DropColumn( 35 | name: "IsPublished", 36 | table: "Offerings"); 37 | 38 | migrationBuilder.DropColumn( 39 | name: "IsPublished", 40 | table: "Medias"); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /UnitTests/UnitTests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | all 13 | runtime; build; native; contentfiles; analyzers; buildtransitive 14 | 15 | 16 | all 17 | runtime; build; native; contentfiles; analyzers; buildtransitive 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | Always 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190717220252_FileRecords.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | using System; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class FileRecords : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.CreateTable( 11 | name: "FileRecords", 12 | columns: table => new 13 | { 14 | Id = table.Column(nullable: false), 15 | CreatedAt = table.Column(nullable: false), 16 | CreatedBy = table.Column(nullable: true), 17 | LastUpdatedAt = table.Column(nullable: false), 18 | LastUpdatedBy = table.Column(nullable: true), 19 | IsDeletedStatus = table.Column(nullable: false), 20 | FileName = table.Column(nullable: true), 21 | Path = table.Column(nullable: true), 22 | Hash = table.Column(nullable: true) 23 | }, 24 | constraints: table => 25 | { 26 | table.PrimaryKey("PK_FileRecords", x => x.Id); 27 | }); 28 | } 29 | 30 | protected override void Down(MigrationBuilder migrationBuilder) 31 | { 32 | migrationBuilder.DropTable( 33 | name: "FileRecords"); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191025195048_Drop_MediaId_From_Video2.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Drop_MediaId_From_Video2 : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropForeignKey( 10 | name: "FK_Videos_Medias_MediaId", 11 | table: "Videos"); 12 | 13 | migrationBuilder.DropIndex( 14 | name: "IX_Videos_MediaId", 15 | table: "Videos"); 16 | 17 | migrationBuilder.DropColumn( 18 | name: "MediaId", 19 | table: "Videos"); 20 | } 21 | 22 | protected override void Down(MigrationBuilder migrationBuilder) 23 | { 24 | migrationBuilder.AddColumn( 25 | name: "MediaId", 26 | table: "Videos", 27 | nullable: true); 28 | 29 | migrationBuilder.CreateIndex( 30 | name: "IX_Videos_MediaId", 31 | table: "Videos", 32 | column: "MediaId"); 33 | 34 | migrationBuilder.AddForeignKey( 35 | name: "FK_Videos_Medias_MediaId", 36 | table: "Videos", 37 | column: "MediaId", 38 | principalTable: "Medias", 39 | principalColumn: "Id", 40 | onDelete: ReferentialAction.Restrict); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /TaskEngine/Tasks/UpdateBoxTokenTask.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Services; 2 | using Microsoft.Extensions.Logging; 3 | using System.Diagnostics.CodeAnalysis; 4 | using System.Threading.Tasks; 5 | using static ClassTranscribeDatabase.CommonUtils; 6 | 7 | #pragma warning disable CA2007 8 | // https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca2007 9 | // We are okay awaiting on a task in the same thread 10 | 11 | 12 | // !!! May be removed in the future 13 | namespace TaskEngine.Tasks 14 | { 15 | [SuppressMessage("Microsoft.Performance", "CA1812:MarkMembersAsStatic")] // This class is never directly instantiated 16 | class UpdateBoxTokenTask : RabbitMQTask 17 | { 18 | private BoxAPI _box; 19 | public UpdateBoxTokenTask(RabbitMQConnection rabbitMQ, BoxAPI box, ILogger logger) 20 | : base(rabbitMQ, TaskType.UpdateBoxToken, logger) 21 | { 22 | _box = box; 23 | } 24 | #pragma warning disable 1998 25 | protected async override Task OnConsume(string emptyString, TaskParameters taskParameters, ClientActiveTasks cleanup) 26 | { 27 | // Maybe in the future if we use this task again: registerTask(cleanup, "RefreshAccessTokenAsync"); // may throw AlreadyInProgress exception 28 | // no. xx nope await _box.RefreshAccessTokenAsync(); 29 | // refreshing the Box access token caused the token to go stale 30 | // We've had a better experience not refreshing it 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /PythonRpcServer/youtube_test.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | if 'DATA_DIRECTORY' not in os.environ: 4 | os.environ['DATA_DIRECTORY']='.' 5 | 6 | import youtube 7 | 8 | def test_youtube1(): 9 | print("Test 1/2: Download playlist") 10 | yt=youtube.YoutubeProvider() 11 | pl=yt.get_youtube_playlist('PLBgxzZMu3GpPb35BDIU5eeopR4MhBOZw_') 12 | print(pl) 13 | #[{'channelId': 'UC4JRDwrS2QC4XNZnSex0Udw', 'playlistId': 'PLBgxzZMu3GpPb35BDIU5eeopR4MhBOZw_', 'title': 'STAT 385 /// Welcome', 'description': 'Course: https://stat385.org/', 'publishedAt': '2021/08/24', 'videoUrl': 'https://youtube.com/watch?v=DqHMh8nqCPw', 'videoId': 'DqHMh8nqCPw', 'createdAt': '2021/08/24'}] 14 | assert len(pl) >0 15 | for k in [ 'playlistId', 'title','description','videoUrl','videoId']: 16 | assert k in pl[0].keys(), f"Expected key {k} in playlist entries" 17 | 18 | assert 'STAT 385' in pl[0]['title'] 19 | 20 | def test_youtube2(): 21 | print("Test 2/2: Download video") 22 | yt=youtube.YoutubeProvider() 23 | onevid = yt.download_youtube_video('https://youtube.com/watch?v=DqHMh8nqCPw') # 24-72 seconds typical 24 | print(onevid) 25 | assert len(onevid) == 2 26 | 27 | path, filetype = onevid 28 | assert filetype == '.mp4' 29 | # Typical result: ('/PythonRpcServer/./OLYHLMQZ', '.mp4') 30 | assert os.path.exists(path) 31 | assert os.stat(path).st_size == 166619691 32 | 33 | print(f"Cleaning up. Removing file {path}") 34 | os.remove(path) 35 | 36 | print("All tests completed") 37 | 38 | if __name__ == "__main__": 39 | test_youtube1() 40 | test_youtube2() 41 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20221116231920_LightweightVideo2.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class LightweightVideo2 : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropForeignKey( 10 | name: "FK_Videos_TextData_PhraseHintsDataId", 11 | table: "Videos"); 12 | 13 | migrationBuilder.DropIndex( 14 | name: "IX_Videos_PhraseHintsDataId", 15 | table: "Videos"); 16 | 17 | migrationBuilder.AddColumn( 18 | name: "SceneObjectDataId", 19 | table: "Videos", 20 | nullable: true); 21 | } 22 | 23 | protected override void Down(MigrationBuilder migrationBuilder) 24 | { 25 | migrationBuilder.DropColumn( 26 | name: "SceneObjectDataId", 27 | table: "Videos"); 28 | 29 | migrationBuilder.CreateIndex( 30 | name: "IX_Videos_PhraseHintsDataId", 31 | table: "Videos", 32 | column: "PhraseHintsDataId"); 33 | 34 | migrationBuilder.AddForeignKey( 35 | name: "FK_Videos_TextData_PhraseHintsDataId", 36 | table: "Videos", 37 | column: "PhraseHintsDataId", 38 | principalTable: "TextData", 39 | principalColumn: "Id", 40 | onDelete: ReferentialAction.Restrict); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191026021015_SrtFile.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class SrtFile : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "SrtFileId", 11 | table: "Transcriptions", 12 | nullable: true); 13 | 14 | migrationBuilder.CreateIndex( 15 | name: "IX_Transcriptions_SrtFileId", 16 | table: "Transcriptions", 17 | column: "SrtFileId"); 18 | 19 | migrationBuilder.AddForeignKey( 20 | name: "FK_Transcriptions_FileRecords_SrtFileId", 21 | table: "Transcriptions", 22 | column: "SrtFileId", 23 | principalTable: "FileRecords", 24 | principalColumn: "Id", 25 | onDelete: ReferentialAction.Restrict); 26 | } 27 | 28 | protected override void Down(MigrationBuilder migrationBuilder) 29 | { 30 | migrationBuilder.DropForeignKey( 31 | name: "FK_Transcriptions_FileRecords_SrtFileId", 32 | table: "Transcriptions"); 33 | 34 | migrationBuilder.DropIndex( 35 | name: "IX_Transcriptions_SrtFileId", 36 | table: "Transcriptions"); 37 | 38 | migrationBuilder.DropColumn( 39 | name: "SrtFileId", 40 | table: "Transcriptions"); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20220930075448_ASLGlossaryMap.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class ASLGlossaryMap : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.CreateTable( 11 | name: "ASLVideoGlossaryMaps", 12 | columns: table => new 13 | { 14 | Id = table.Column(nullable: false), 15 | CreatedAt = table.Column(nullable: false), 16 | CreatedBy = table.Column(nullable: true), 17 | LastUpdatedAt = table.Column(nullable: false), 18 | LastUpdatedBy = table.Column(nullable: true), 19 | IsDeletedStatus = table.Column(nullable: false), 20 | DeletedAt = table.Column(nullable: true), 21 | DeletedBy = table.Column(nullable: true), 22 | GlossaryId = table.Column(nullable: true), 23 | ASLVideoId = table.Column(nullable: true) 24 | }, 25 | constraints: table => 26 | { 27 | table.PrimaryKey("PK_ASLVideoGlossaryMaps", x => x.Id); 28 | }); 29 | } 30 | 31 | protected override void Down(MigrationBuilder migrationBuilder) 32 | { 33 | migrationBuilder.DropTable( 34 | name: "ASLVideoGlossaryMaps"); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /TestAzureCognitiveServices/TestAzure.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/sdk:8.0.100-1-bookworm-slim as build1 2 | 3 | WORKDIR /src/TestAzureCognitiveServices 4 | 5 | COPY . . 6 | RUN dotnet restore ./TestAzureCognitiveServices.csproj 7 | RUN dotnet publish ./TestAzureCognitiveServices.csproj -c Release -o /app --no-restore 8 | 9 | #FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim as publish_base1 10 | # FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim-arm64v8 as publish_base1 11 | FROM mcr.microsoft.com/dotnet/aspnet:8.0 as publish_base1 12 | 13 | #COPY ./Program.cs / 14 | 15 | # Grrr AzureServices does not work on dotnet8 on Debian 12 because it wont link to libssl3 - fix below is needed for short-term 16 | 17 | # Install prerequisites for Azure Speech Services 18 | # See https://docs.microsoft.com/en-us/azure/cognitive-services/speech-service/quickstarts/setup-platform 19 | RUN apt-get update 20 | RUN apt-get -y install build-essential libssl-dev libasound2 wget 21 | 22 | # Microsoft 8.0 issue: https://github.com/Azure-Samples/cognitive-services-speech-sdk/issues/2204 23 | # This will install OpenSSL 1.1.1 because it is needed by the Speech SDK. 24 | #RUN ARCH=$(dpkg --print-architecture) 25 | COPY ./install-libssl1.sh / 26 | RUN /install-libssl1.sh 27 | 28 | FROM publish_base1 as publish1 29 | WORKDIR /app 30 | COPY --from=build1 /app . 31 | COPY shortwav.wav / 32 | CMD ["dotnet", "/app/TestAzureCognitiveServices.dll"] 33 | 34 | # Example 35 | #docker build -t azuretest -f TestAzure.Dockerfile . 36 | #docker run -t azuretest ls 37 | # [690371]: 31ms SPX_TRACE_ERROR: AZ_LOG_ERROR: shim_openssl.c:55 libssl could not be loaded 38 | # [690371]: 31ms SPX_TRACE_ERROR: AZ_LOG_ERROR: tlsio_openssl.c:2175 Could not load libssl -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019-2024 University of Illinois. All rights reserved. 2 | 3 | Developed by: ClassTranscribe Group 4 | University of Illinois 5 | classtranscribe.illinois.edu 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy of 8 | this software and associated documentation files (the "Software"), to deal with 9 | the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 11 | of the Software, and to permit persons to whom the Software is furnished to 12 | do so, subject to the following conditions: 13 | * Redistributions of source code must retain the above copyright notice, 14 | this list of conditions and the following disclaimers. 15 | * Redistributions in binary form must reproduce the above copyright notice, 16 | this list of conditions and the following disclaimers in the documentation 17 | and/or other materials provided with the distribution. 18 | * Neither the names of ClassTranscribe Group, University of Illinois, 19 | nor the names of its contributors may be used to endorse or promote products 20 | derived from this Software without specific prior written permission. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE 28 | SOFTWARE. 29 | -------------------------------------------------------------------------------- /UnitTests/ClassTranscribeServer/ControllerTests/StaticFileConrollerTest.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeServer.Controllers; 2 | using Microsoft.AspNetCore.Mvc; 3 | using System.IO; 4 | using System.Text; 5 | using Xunit; 6 | 7 | namespace UnitTests.ClassTranscribeServer.ControllerTests 8 | { 9 | public class StaticFileControllerTest : BaseControllerTest 10 | { 11 | private readonly StaticFileController _controller; 12 | 13 | public StaticFileControllerTest(GlobalFixture fixture) : base(fixture) 14 | { 15 | _controller = new StaticFileController( 16 | fixture._authorizationService, 17 | _context, 18 | _userUtils, 19 | fixture._physicalFileProvider, 20 | null 21 | ) 22 | { 23 | ControllerContext = fixture._controllerContext 24 | }; 25 | } 26 | 27 | [Fact] 28 | public void Get_File() 29 | { 30 | var result = _controller.GetFile(null); 31 | Assert.IsType(result); 32 | 33 | result = _controller.GetFile("non-existent.txt"); 34 | Assert.IsType(result); 35 | 36 | result = _controller.GetFile("test.txt"); 37 | Assert.IsType(result); 38 | 39 | var fileStream = ((FileStreamResult)result).FileStream; 40 | using var sr1 = new StreamReader(fileStream, Encoding.UTF8); 41 | string content1 = sr1.ReadToEnd(); 42 | 43 | using var stream = File.OpenRead("Assets/test.txt"); 44 | using var sr2 = new StreamReader(stream, Encoding.UTF8); 45 | string content2 = sr2.ReadToEnd(); 46 | 47 | Assert.Equal(content1, content2); 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /DevExperiments/Program.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Services; 3 | using ClassTranscribeDatabase.Services.MSTranscription; 4 | using Microsoft.EntityFrameworkCore; 5 | using Microsoft.Extensions.Configuration; 6 | using Microsoft.Extensions.DependencyInjection; 7 | using Microsoft.Extensions.Logging; 8 | using Microsoft.Extensions.Options; 9 | 10 | namespace DevExperiments 11 | { 12 | class Program 13 | { 14 | static void Main(string[] args) 15 | { 16 | var configuration = CTDbContext.GetConfigurations(); 17 | //setup our DI 18 | var serviceProvider = new ServiceCollection() 19 | .AddLogging(builder => 20 | { 21 | builder.AddConsole(); 22 | builder.AddFilter 23 | ("", LogLevel.Warning); 24 | //builder.AddApplicationInsights(configuration.GetValue("APPLICATION_INSIGHTS_KEY")); 25 | }) 26 | .AddOptions() 27 | .Configure(configuration) 28 | //.AddDbContext(options => options.UseNpgsql(CTDbContext.ConnectionStringBuilder())) 29 | .AddDbContext(options => options.UseLazyLoadingProxies().UseNpgsql(CTDbContext.ConnectionStringBuilder())) 30 | .AddScoped() 31 | .AddSingleton() 32 | .AddSingleton() 33 | .AddSingleton() 34 | .BuildServiceProvider(); 35 | 36 | Globals.appSettings = serviceProvider.GetService>().Value; 37 | 38 | TempCode tempCode = serviceProvider.GetService(); 39 | tempCode.Temp(); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /TaskEngine/TaskEngine.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | 8 | 9 | ../TaskEngineDebug/bin 10 | ../TaskEngineDebug/obj 11 | 12 | 13 | 14 | ../TaskEngineRelease/bin 15 | ../TaskEngineRelease/obj 16 | 17 | 18 | 19 | 20 | 21 | all 22 | runtime; build; native; contentfiles; analyzers; buildtransitive 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | Always 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200612211632_TaskItems.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class TaskItems : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.CreateTable( 11 | name: "TaskItems", 12 | columns: table => new 13 | { 14 | Id = table.Column(nullable: false), 15 | CreatedAt = table.Column(nullable: false), 16 | CreatedBy = table.Column(nullable: true), 17 | LastUpdatedAt = table.Column(nullable: false), 18 | LastUpdatedBy = table.Column(nullable: true), 19 | IsDeletedStatus = table.Column(nullable: false), 20 | UniqueId = table.Column(nullable: false), 21 | TaskType = table.Column(nullable: false), 22 | Attempts = table.Column(nullable: false), 23 | TaskParameters = table.Column(nullable: true), 24 | Result = table.Column(nullable: false), 25 | Retry = table.Column(nullable: false), 26 | ResultData = table.Column(nullable: true) 27 | }, 28 | constraints: table => 29 | { 30 | table.PrimaryKey("PK_TaskItems", x => x.Id); 31 | table.UniqueConstraint("AK_TaskItems_UniqueId_TaskType", x => new { x.UniqueId, x.TaskType }); 32 | }); 33 | } 34 | 35 | protected override void Down(MigrationBuilder migrationBuilder) 36 | { 37 | migrationBuilder.DropTable( 38 | name: "TaskItems"); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /ClassTranscribeServer/Utils/UIUCSeed.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Models; 3 | using CsvHelper; 4 | using System.Collections.Generic; 5 | using System.IO; 6 | using System.Linq; 7 | 8 | namespace TaskEngine.Utils 9 | { 10 | public static class UIUCSeed 11 | { 12 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1707:Identifiers should not contain underscores")] 13 | public class CSVCourse 14 | { 15 | public string TERM_DESC { get; set; } 16 | public string CRN { get; set; } 17 | 18 | public string SUBJ { get; set; } 19 | public string NBR { get; set; } 20 | 21 | public string SEC { get; set; } 22 | public string CRS_TITLE { get; set; } 23 | public string SCHED_TYPE { get; set; } 24 | } 25 | 26 | public static void SeedCourses() 27 | { 28 | // Dry run code before using it on Production. 29 | string file = Path.Combine(Globals.appSettings.DATA_DIRECTORY, "seed", "Fall2019InstructorList.csv"); 30 | TextReader reader = new StreamReader(file); 31 | var csvReader = new CsvReader(reader, System.Globalization.CultureInfo.CurrentCulture); 32 | var records = csvReader.GetRecords(); 33 | List csvCourses = new List(records); 34 | 35 | using (var _context = CTDbContext.CreateDbContext()) 36 | { 37 | Department eceDept = _context.Departments.Where(d => d.Acronym == "ECE" && d.UniversityId == "1001").FirstOrDefault(); 38 | List courses = csvCourses.Where(c => c.SUBJ == "ECE").GroupBy(c => new { c.CRS_TITLE, c.NBR, c.SUBJ }).Select(c => new Course 39 | { 40 | CourseNumber = c.First().NBR, 41 | Department = eceDept 42 | }).ToList(); 43 | _context.Courses.AddRange(courses); 44 | _context.SaveChanges(); 45 | } 46 | 47 | csvReader.Dispose(); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /PythonRpcServer/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2025-4-30 2 | protobuf 3 | certifi==2025.4.26 4 | backcall==0.2.0 5 | chardet==5.2.0 6 | click==8.1.8 7 | decorator==5.2.1 8 | ffmpy==0.3.1 9 | grpcio 10 | grpcio-tools 11 | idna==3.10 12 | KalturaApiClient==19.3.0 13 | lxml==5.1.0 14 | parso==0.8.3 15 | pexpect==4.9.0 16 | pickleshare==0.7.5 17 | ptyprocess==0.7.0 18 | requests==2.32.2 19 | requests-toolbelt==1.0.0 20 | six==1.16.0 21 | tqdm==4.66.3 22 | traitlets==4.3.3 23 | urllib3==2.2.2 24 | wcwidth==0.2.13 25 | numpy 26 | yt-dlp 27 | 28 | # 2024-1-17 Removed Pygments,python-genutils and ipython 29 | # Also removed jedi. why autocomplete?? jedi==0.19.1 30 | #Why? prompt-toolkit==3.0.43 31 | 32 | #Did not try updating (maybe nexttime) 33 | #protobuf==3.15.0 #4.25.2 34 | # protobuf==4.25.2 35 | 36 | # #Floowing Updated to latest 2024-1-17: 37 | # certifi==2024.7.4 38 | # backcall==0.2.0 39 | # chardet==5.2.0 40 | # click==8.1.7 41 | # decorator==5.1.1 42 | # ffmpy==0.3.1 43 | # grpcio==1.60.0 44 | # grpcio-tools==1.60.0 45 | # idna==3.7 46 | # KalturaApiClient==19.3.0 47 | # lxml==5.1.0 48 | # parso==0.8.3 49 | # pexpect==4.9.0 50 | # pickleshare==0.7.5 51 | # ptyprocess==0.7.0 52 | # requests==2.32.2 53 | # requests-toolbelt==1.0.0 54 | # six==1.16.0 55 | # tqdm==4.66.3 56 | # traitlets==4.3.3 57 | # urllib3==2.2.2 58 | # wcwidth==0.2.13 59 | 60 | # # Not versioned 61 | # numpy 62 | # # No longer maintained pytube # if not available, use the tar.gz package (see Dockerfile) 63 | # yt-dlp 64 | #Always get latest 65 | 66 | # protobuf version 3.18.3 causes NotImplementedError("To be implemented") in PythonRpcServer/mediaprovider.py 67 | # Likely need to coordinate updating the C# version too 68 | 69 | 70 | ############# 71 | # Gone 72 | #ipython==7.16.3 73 | #ipython-genutils==0.2.0 74 | # Pygments==2.7.4 75 | 76 | # No longer needed for remaning pythonrpcserver tasks 77 | #scenedetect==0.5.2 78 | #scikit-image==0.17.2 79 | #nltk==3.6.6 80 | #pytesseract==0.3.7 81 | #prefixspan==0.5.2 82 | #opencv-contrib-python==4.5.3.56 83 | #mtcnn-opencv==1.0.2 84 | #decord==0.6.0 85 | #opencv-python==4.2.0.34; replaced by opencv-contrib-python 86 | # 87 | -------------------------------------------------------------------------------- /UnitTests/ClassTranscribeDatabase/ModelsTest.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Models; 2 | using Xunit; 3 | 4 | namespace UnitTests.ClassTranscribeDatabase 5 | { 6 | public class ModelsTest 7 | { 8 | [Fact] 9 | public void Model_JObjects_Not_Null() 10 | { 11 | var video = new Video(); 12 | var applicationUser = new ApplicationUser(); 13 | var offering = new Offering(); 14 | var playlist = new Playlist(); 15 | var media = new Media(); 16 | var log = new Log(); 17 | var ePub = new EPub(); 18 | var watchHistory = new WatchHistory(); 19 | var message = new Message(); 20 | var taskItem = new TaskItem(); 21 | 22 | Assert.NotNull(video.JsonMetadata); 23 | Assert.NotNull(video.SceneData); 24 | Assert.NotNull(video.FileMediaInfo); 25 | Assert.NotNull(applicationUser.Metadata); 26 | Assert.NotNull(offering.JsonMetadata); 27 | Assert.NotNull(playlist.JsonMetadata); 28 | Assert.NotNull(media.JsonMetadata); 29 | Assert.NotNull(log.Json); 30 | Assert.NotNull(ePub.Cover); 31 | Assert.NotNull(watchHistory.Json); 32 | Assert.NotNull(message.Payload); 33 | Assert.NotNull(taskItem.TaskParameters); 34 | Assert.NotNull(taskItem.ResultData); 35 | Assert.NotNull(taskItem.RemoteResultData); 36 | 37 | Assert.Empty(video.JsonMetadata); 38 | Assert.Empty(video.SceneData); 39 | Assert.Empty(video.FileMediaInfo); 40 | Assert.Empty(applicationUser.Metadata); 41 | Assert.Empty(offering.JsonMetadata); 42 | Assert.Empty(playlist.JsonMetadata); 43 | Assert.Empty(media.JsonMetadata); 44 | Assert.Empty(log.Json); 45 | Assert.Empty(ePub.Cover); 46 | Assert.Empty(watchHistory.Json); 47 | Assert.Empty(message.Payload); 48 | Assert.Empty(taskItem.TaskParameters); 49 | Assert.Empty(taskItem.ResultData); 50 | Assert.Empty(taskItem.RemoteResultData); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /UnitTests/Utils/Common.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Models; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | 7 | namespace UnitTests.Utils 8 | { 9 | public class Common 10 | { 11 | public static bool IsValidFilePath(Entity entity) 12 | { 13 | switch (entity) 14 | { 15 | case Course c: 16 | return c.FilePath.Length == 9 17 | && c.FilePath.Substring(0, 5) == $"{c.CreatedAt:yyMM}-" 18 | && Directory.Exists(Path.Combine(Globals.appSettings.DATA_DIRECTORY, c.FilePath)) 19 | && c.FilePath[5..].ToCharArray().All(char.IsLetterOrDigit); 20 | 21 | case CourseOffering co: 22 | return co.FilePath.Length == 19 23 | && co.FilePath.Substring(0, 9) == co.Course.FilePath 24 | && co.FilePath.ToCharArray()[9] == Path.DirectorySeparatorChar 25 | && co.FilePath.Substring(10, 5) == $"{co.CreatedAt:yyMM}-" 26 | && Directory.Exists(Path.Combine(Globals.appSettings.DATA_DIRECTORY, co.FilePath)) 27 | && co.FilePath[15..].ToCharArray().All(char.IsLetterOrDigit); 28 | 29 | default: 30 | return false; 31 | } 32 | } 33 | 34 | public static async Task GetCourseOfferingForFileRecord(CTDbContext context) 35 | { 36 | var c = new Course { Id = "c_filerecord" }; 37 | var o = new Offering { Id = "o_filerecord" }; 38 | var co = new CourseOffering { CourseId = c.Id, OfferingId = o.Id }; 39 | var p = new Playlist { OfferingId = o.Id }; 40 | 41 | context.Courses.Add(c); 42 | context.CourseOfferings.Add(co); 43 | context.Offerings.Add(o); 44 | context.Playlists.Add(p); 45 | 46 | await FileRecord.SetFilePath(context, c); 47 | await FileRecord.SetFilePath(context, co); 48 | 49 | context.SaveChanges(); 50 | 51 | return co; 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200414071209_Epubchapter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class Epubchapter : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.AddColumn( 11 | name: "Json", 12 | table: "EPubs", 13 | nullable: true); 14 | 15 | migrationBuilder.CreateTable( 16 | name: "EPubChapters", 17 | columns: table => new 18 | { 19 | Id = table.Column(nullable: false), 20 | CreatedAt = table.Column(nullable: false), 21 | CreatedBy = table.Column(nullable: true), 22 | LastUpdatedAt = table.Column(nullable: false), 23 | LastUpdatedBy = table.Column(nullable: true), 24 | IsDeletedStatus = table.Column(nullable: false), 25 | EPubId = table.Column(nullable: true), 26 | Data = table.Column(nullable: true) 27 | }, 28 | constraints: table => 29 | { 30 | table.PrimaryKey("PK_EPubChapters", x => x.Id); 31 | table.ForeignKey( 32 | name: "FK_EPubChapters_EPubs_EPubId", 33 | column: x => x.EPubId, 34 | principalTable: "EPubs", 35 | principalColumn: "Id", 36 | onDelete: ReferentialAction.Restrict); 37 | }); 38 | 39 | migrationBuilder.CreateIndex( 40 | name: "IX_EPubChapters_EPubId", 41 | table: "EPubChapters", 42 | column: "EPubId"); 43 | } 44 | 45 | protected override void Down(MigrationBuilder migrationBuilder) 46 | { 47 | migrationBuilder.DropTable( 48 | name: "EPubChapters"); 49 | 50 | migrationBuilder.DropColumn( 51 | name: "Json", 52 | table: "EPubs"); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /TaskEngine/Tasks/ExampleTask.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Services; 3 | using Microsoft.EntityFrameworkCore; 4 | using Microsoft.Extensions.Logging; 5 | using System.Diagnostics.CodeAnalysis; 6 | using System.Linq; 7 | using System.Threading.Tasks; 8 | using static ClassTranscribeDatabase.CommonUtils; 9 | 10 | #pragma warning disable CA2007 11 | // https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca2007 12 | // We are okay awaiting on a task in the same thread 13 | 14 | 15 | namespace TaskEngine.Tasks 16 | { 17 | [SuppressMessage("Microsoft.Performance", "CA1812:MarkMembersAsStatic")] // This class is never directly instantiated 18 | class ExampleTask : RabbitMQTask 19 | { 20 | public ExampleTask(RabbitMQConnection rabbitMQ, 21 | ILogger logger) 22 | : base(rabbitMQ, TaskType.ExampleTask, logger) 23 | { 24 | } 25 | protected async override Task OnConsume(string example, TaskParameters taskParameters, ClientActiveTasks cleanup) 26 | { 27 | RegisterTask(cleanup, "ExampleTask"); // may throw AlreadyInProgress exception 28 | GetLogger().LogInformation("Example Task Starting"); 29 | int captionCount = 0; 30 | int transcriptionCount = 0; 31 | 32 | using (var _context = CTDbContext.CreateDbContext()) 33 | { 34 | CaptionQueries captionQueries = new CaptionQueries(_context); 35 | 36 | var transcriptions = await _context.Transcriptions.Take(30).ToListAsync(); 37 | 38 | foreach (var transcription in transcriptions) 39 | { 40 | 41 | var transcriptionId = transcription.Id; 42 | var videoID = transcription.VideoId; 43 | var captions = await captionQueries.GetCaptionsAsync(transcriptionId); 44 | 45 | GetLogger().LogInformation($"{transcription.Id}: Caption count= {captions.Count}"); 46 | transcriptionCount++; 47 | } 48 | } 49 | 50 | GetLogger().LogInformation($"Example Task Done. transcriptionCount={transcriptionCount} captionCount={captionCount}"); 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /TaskEngine.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim as build 2 | # See https://mcr.microsoft.com/en-us/product/dotnet/sdk/tags 3 | #See more comments in API.Dockerfile 4 | # RUN ls 5 | RUN dotnet --list-sdks 6 | 7 | WORKDIR / 8 | RUN git clone https://github.com/eficode/wait-for.git 9 | 10 | WORKDIR /src 11 | COPY ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj 12 | # --verbosity normal|diagnostic 13 | 14 | 15 | RUN dotnet restore --verbosity diagnostic ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj 16 | 17 | COPY ./TaskEngine/TaskEngine.csproj ./TaskEngine/TaskEngine.csproj 18 | RUN dotnet restore ./TaskEngine/TaskEngine.csproj 19 | 20 | COPY ./world_universities_and_domains.json ./world_universities_and_domains.json 21 | COPY ./ct.proto ./ct.proto 22 | COPY ./ClassTranscribeDatabase ./ClassTranscribeDatabase 23 | COPY ./TaskEngine ./TaskEngine 24 | WORKDIR /src/TaskEngine 25 | RUN dotnet publish TaskEngine.csproj -c Release -o /app --no-restore 26 | 27 | #FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim as publish_base 28 | FROM mcr.microsoft.com/dotnet/aspnet:8.0 as publish_base 29 | # https://hub.docker.com/_/microsoft-dotnet-aspnet/ 30 | 31 | # force AMD64 build here: the ssl1.1.1 workaround below assumes amd64 32 | # Install prerequisites for Azure Speech Services: build-essential libssl-dev ca-certificates libasound2 wget 33 | # See https://docs.microsoft.com/en-us/azure/cognitive-services/speech-service/quickstarts/setup-platform 34 | 35 | RUN apt-get update && apt-get install -y build-essential libssl-dev ca-certificates libasound2 wget && \ 36 | apt-get install -y netcat-traditional && apt-get -q update 37 | 38 | # Microsoft 8.0 issue: https://github.com/Azure-Samples/cognitive-services-speech-sdk/issues/2204 39 | # This will install OpenSSL 1.1.1 because it is needed by the Speech SDK. 40 | # RUN ARCH=$(dpkg --print-architecture) 41 | # COPY ./install-speech-hack-libssl1.sh / 42 | # RUN wget /install-speech-hack-libssl1.sh 43 | 44 | RUN wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb 45 | RUN dpkg -i libssl1.1_1.1.0g-2ubuntu4_amd64.deb 46 | 47 | 48 | FROM publish_base as publish 49 | WORKDIR / 50 | COPY --from=build /wait-for . 51 | WORKDIR /app 52 | COPY --from=build /app . 53 | CMD ["dotnet", "/app/TaskEngine.dll"] -------------------------------------------------------------------------------- /ClassTranscribeServer/Utils/SwaggerSchemaFilter.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Models; 2 | using Microsoft.OpenApi.Models; 3 | using Newtonsoft.Json; 4 | using Swashbuckle.AspNetCore.SwaggerGen; 5 | using System.Linq; 6 | using System.Reflection; 7 | 8 | namespace ClassTranscribeServer.Utils 9 | { 10 | 11 | 12 | internal static class StringExtensions 13 | { 14 | internal static string ToCamelCase(this string value) 15 | { 16 | if (string.IsNullOrEmpty(value)) return value; 17 | return char.ToLowerInvariant(value[0]) + value.Substring(1); 18 | } 19 | } 20 | 21 | public class SwaggerSchemaFilter : ISchemaFilter 22 | { 23 | public void Apply(OpenApiSchema schema, SchemaFilterContext context) 24 | { 25 | 26 | if (schema == null || context == null || schema.Properties.Count == 0) 27 | { 28 | return; 29 | } 30 | 31 | const BindingFlags bindingFlags = BindingFlags.Public | 32 | BindingFlags.NonPublic | 33 | BindingFlags.Instance; 34 | 35 | var memberList = context.Type 36 | .GetFields(bindingFlags).Cast() 37 | .Concat(context.Type 38 | .GetProperties(bindingFlags)); 39 | 40 | var excludedList = memberList.Where(m => 41 | m.GetCustomAttribute() 42 | != null) 43 | .Select(m => 44 | (m.GetCustomAttribute() 45 | ?.PropertyName 46 | ?? m.Name.ToCamelCase())); 47 | 48 | foreach (var excludedName in excludedList) 49 | { 50 | if (schema.Properties.ContainsKey(excludedName)) 51 | { 52 | System.Diagnostics.Debug.WriteLine("????"); 53 | System.Diagnostics.Debug.WriteLine(excludedName); 54 | schema.Properties.Remove(excludedName); 55 | } 56 | 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /ct.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package CTGrpc; 4 | 5 | service PythonServer { 6 | rpc GetScenesRPC (File) returns (JsonString) {} 7 | rpc ToPhraseHintsRPC(PhraseHintRequest) returns (PhraseHintResponse) {} 8 | 9 | rpc GetKalturaChannelEntriesRPC (PlaylistRequest) returns (JsonString) {} 10 | rpc DownloadKalturaVideoRPC (MediaRequest) returns (File) {} 11 | 12 | rpc GetEchoPlaylistRPC (PlaylistRequest) returns (JsonString) {} 13 | rpc DownloadEchoVideoRPC (MediaRequest) returns (File) {} 14 | 15 | rpc GetYoutubePlaylistRPC (PlaylistRequest) returns (JsonString) {} 16 | rpc DownloadYoutubeVideoRPC (MediaRequest) returns (File) {} 17 | 18 | rpc ConvertVideoToWavRPCWithOffset (FileForConversion) returns (File) {} 19 | rpc ProcessVideoRPC (File) returns (File) {} 20 | 21 | rpc ComputeFileHash (FileHashRequest) returns (FileHashResponse) {} 22 | rpc GetMediaInfoRPC(File) returns (JsonString) {} 23 | 24 | rpc TranscribeAudioRPC (TranscriptionRequest) returns (JsonString) {} 25 | } 26 | 27 | message TranscriptionRequest { 28 | string filePath = 1; // Path to the audio/video file to be transcribed 29 | string model = 2; // Whisper model to use (e.g., 'base-en', 'tiny-en') 30 | string language = 3; // Language in audio. 31 | string logId = 4; 32 | bool testing = 5; 33 | } 34 | 35 | 36 | // The request message containing the user's name. 37 | message JsonString { 38 | string json = 1; 39 | } 40 | 41 | // The response message containing the greetings. 42 | message PlaylistRequest { 43 | string Url = 1; 44 | int32 stream = 2; 45 | JsonString metadata = 3; 46 | } 47 | 48 | message MediaRequest { 49 | string videoUrl = 1; 50 | string additionalInfo = 2; 51 | } 52 | message PhraseHintRequest { 53 | string rawPhraseData = 1; 54 | } 55 | 56 | message File { 57 | string filePath = 1; 58 | string ext = 2; 59 | } 60 | 61 | message FileForConversion { 62 | File file = 1; 63 | float offset = 2; 64 | } 65 | 66 | message EPubData { 67 | string title = 1; 68 | string author = 2; 69 | string publisher = 3; 70 | repeated EPubChapter chapters = 4; 71 | string file = 5; 72 | } 73 | 74 | message EPubChapter { 75 | string title = 1; 76 | string text = 2; 77 | File image = 3; 78 | } 79 | 80 | message FileHashRequest { 81 | string file = 1; 82 | string algorithms = 2; 83 | } 84 | 85 | message FileHashResponse { 86 | string result = 1; 87 | } 88 | 89 | message PhraseHintResponse { 90 | string result = 1; 91 | } -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191025232219_Transcription_Refactorv2.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Transcription_Refactorv2 : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropForeignKey( 10 | name: "FK_Transcriptions_Medias_MediaId", 11 | table: "Transcriptions"); 12 | 13 | migrationBuilder.DropIndex( 14 | name: "IX_Transcriptions_MediaId", 15 | table: "Transcriptions"); 16 | 17 | migrationBuilder.DropColumn( 18 | name: "MediaId", 19 | table: "Transcriptions"); 20 | 21 | migrationBuilder.CreateIndex( 22 | name: "IX_Transcriptions_VideoId", 23 | table: "Transcriptions", 24 | column: "VideoId"); 25 | 26 | migrationBuilder.AddForeignKey( 27 | name: "FK_Transcriptions_Videos_VideoId", 28 | table: "Transcriptions", 29 | column: "VideoId", 30 | principalTable: "Videos", 31 | principalColumn: "Id", 32 | onDelete: ReferentialAction.Restrict); 33 | } 34 | 35 | protected override void Down(MigrationBuilder migrationBuilder) 36 | { 37 | migrationBuilder.DropForeignKey( 38 | name: "FK_Transcriptions_Videos_VideoId", 39 | table: "Transcriptions"); 40 | 41 | migrationBuilder.DropIndex( 42 | name: "IX_Transcriptions_VideoId", 43 | table: "Transcriptions"); 44 | 45 | migrationBuilder.AddColumn( 46 | name: "MediaId", 47 | table: "Transcriptions", 48 | nullable: true); 49 | 50 | migrationBuilder.CreateIndex( 51 | name: "IX_Transcriptions_MediaId", 52 | table: "Transcriptions", 53 | column: "MediaId"); 54 | 55 | migrationBuilder.AddForeignKey( 56 | name: "FK_Transcriptions_Medias_MediaId", 57 | table: "Transcriptions", 58 | column: "MediaId", 59 | principalTable: "Medias", 60 | principalColumn: "Id", 61 | onDelete: ReferentialAction.Restrict); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20210908174659_NonnullVideoJObjects.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class NonnullVideoJObjects : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.Sql("UPDATE \"Videos\" SET \"SceneData\" = '{}' WHERE \"SceneData\" IS NULL"); 10 | migrationBuilder.Sql("UPDATE \"Videos\" SET \"JsonMetadata\" = '{}' WHERE \"JsonMetadata\" IS NULL"); 11 | migrationBuilder.Sql("UPDATE \"Videos\" SET \"FileMediaInfo\" = '{}' WHERE \"FileMediaInfo\" IS NULL"); 12 | 13 | migrationBuilder.AlterColumn( 14 | name: "SceneData", 15 | table: "Videos", 16 | nullable: false, 17 | oldClrType: typeof(string), 18 | oldType: "text", 19 | oldNullable: true); 20 | 21 | migrationBuilder.AlterColumn( 22 | name: "JsonMetadata", 23 | table: "Videos", 24 | nullable: false, 25 | oldClrType: typeof(string), 26 | oldType: "text", 27 | oldNullable: true); 28 | 29 | migrationBuilder.AlterColumn( 30 | name: "FileMediaInfo", 31 | table: "Videos", 32 | nullable: false, 33 | oldClrType: typeof(string), 34 | oldType: "text", 35 | oldNullable: true); 36 | } 37 | 38 | protected override void Down(MigrationBuilder migrationBuilder) 39 | { 40 | migrationBuilder.AlterColumn( 41 | name: "SceneData", 42 | table: "Videos", 43 | type: "text", 44 | nullable: true, 45 | oldClrType: typeof(string)); 46 | 47 | migrationBuilder.AlterColumn( 48 | name: "JsonMetadata", 49 | table: "Videos", 50 | type: "text", 51 | nullable: true, 52 | oldClrType: typeof(string)); 53 | 54 | migrationBuilder.AlterColumn( 55 | name: "FileMediaInfo", 56 | table: "Videos", 57 | type: "text", 58 | nullable: true, 59 | oldClrType: typeof(string)); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20211013142047_TranscriptionsUpdate.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class TranscriptionsUpdate : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Editable", 11 | table: "Transcriptions", 12 | nullable: false, 13 | defaultValue: 0); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "Label", 17 | table: "Transcriptions", 18 | nullable: true); 19 | 20 | migrationBuilder.AddColumn( 21 | name: "PublishStatus", 22 | table: "Transcriptions", 23 | nullable: false, 24 | defaultValue: 0); 25 | 26 | migrationBuilder.AddColumn( 27 | name: "SourceInternalRef", 28 | table: "Transcriptions", 29 | nullable: true); 30 | 31 | migrationBuilder.AddColumn( 32 | name: "SourceLabel", 33 | table: "Transcriptions", 34 | nullable: true); 35 | 36 | migrationBuilder.AddColumn( 37 | name: "TranscriptionType", 38 | table: "Transcriptions", 39 | nullable: false, 40 | defaultValue: 0); 41 | } 42 | 43 | protected override void Down(MigrationBuilder migrationBuilder) 44 | { 45 | migrationBuilder.DropColumn( 46 | name: "Editable", 47 | table: "Transcriptions"); 48 | 49 | migrationBuilder.DropColumn( 50 | name: "Label", 51 | table: "Transcriptions"); 52 | 53 | migrationBuilder.DropColumn( 54 | name: "PublishStatus", 55 | table: "Transcriptions"); 56 | 57 | migrationBuilder.DropColumn( 58 | name: "SourceInternalRef", 59 | table: "Transcriptions"); 60 | 61 | migrationBuilder.DropColumn( 62 | name: "SourceLabel", 63 | table: "Transcriptions"); 64 | 65 | migrationBuilder.DropColumn( 66 | name: "TranscriptionType", 67 | table: "Transcriptions"); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20191112212156_EPub.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | using System; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class EPub : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.CreateTable( 11 | name: "EPub", 12 | columns: table => new 13 | { 14 | Id = table.Column(nullable: false), 15 | CreatedAt = table.Column(nullable: false), 16 | CreatedBy = table.Column(nullable: true), 17 | LastUpdatedAt = table.Column(nullable: false), 18 | LastUpdatedBy = table.Column(nullable: true), 19 | IsDeletedStatus = table.Column(nullable: false), 20 | Language = table.Column(nullable: true), 21 | FileId = table.Column(nullable: true), 22 | VideoId = table.Column(nullable: true) 23 | }, 24 | constraints: table => 25 | { 26 | table.PrimaryKey("PK_EPub", x => x.Id); 27 | table.ForeignKey( 28 | name: "FK_EPub_FileRecords_FileId", 29 | column: x => x.FileId, 30 | principalTable: "FileRecords", 31 | principalColumn: "Id", 32 | onDelete: ReferentialAction.Restrict); 33 | table.ForeignKey( 34 | name: "FK_EPub_Videos_VideoId", 35 | column: x => x.VideoId, 36 | principalTable: "Videos", 37 | principalColumn: "Id", 38 | onDelete: ReferentialAction.Restrict); 39 | }); 40 | 41 | migrationBuilder.CreateIndex( 42 | name: "IX_EPub_FileId", 43 | table: "EPub", 44 | column: "FileId"); 45 | 46 | migrationBuilder.CreateIndex( 47 | name: "IX_EPub_VideoId", 48 | table: "EPub", 49 | column: "VideoId"); 50 | } 51 | 52 | protected override void Down(MigrationBuilder migrationBuilder) 53 | { 54 | migrationBuilder.DropTable( 55 | name: "EPub"); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /UnitTests/README.md: -------------------------------------------------------------------------------- 1 | # ClassTranscribeServer Unit Tests 2 | Our unit tests use the xUnit testing framework (https://xunit.net/). 3 | 4 | To remove dependencies on a real database, we use Entity Framework's In-Memory database that they offer for unit testing. You can read more about the in-memory DB (and other options for unit testing) here: https://docs.microsoft.com/en-us/ef/core/miscellaneous/testing/. 5 | 6 | ## Running 7 | 8 | From the `WebAPI` directory, you can run these tests on the command line with: 9 | ``` 10 | dotnet test UnitTests 11 | ``` 12 | 13 | You can also use the CLI to run specific unit tests (https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests?pivots=xunit). 14 | 15 | You can also run the tests in Visual Studio. 16 | 17 | ## Unit Tests vs Integration Tests 18 | Unit tests aim to test small parts of business logic (BL) without any dependencies on other parts of the system (such as a database or data access layer). This is why we use an in-memory DB for mocking the real database and data access layer. 19 | 20 | To learn more about unit testing compared to integration testing, look here: https://stackoverflow.com/a/5357837. 21 | 22 | Look here for EF core unit testing guidelines: https://docs.microsoft.com/en-us/ef/core/miscellaneous/testing/#unit-testing. 23 | 24 | ## Architecture 25 | 26 | Each controller has its own test class which is a subclass of the `BaseControllerTest`. This base class sets up a new context and In-Memory DB in its constructor. 27 | 28 | Following xUnit's built-in parallelism (https://xunit.net/docs/running-tests-in-parallel.html), test cases are run sequentially within any given test class but all test classes are run in parallel. Following xUnit's shared contexts (https://xunit.net/docs/shared-context), the parent test class's constructor is run before every single test case, which ensures that every single test case gets a completely fresh DB (so make sure to not rely on previous test cases when writing a new test). 29 | 30 | The `BaseControllerTest` class is a part of the `Global` test collection, which is denoted by the annotation `[Collection("Global")]`. Following xUnit's shared contexts (https://xunit.net/docs/shared-context), all test classes in a given collection share a single constructor that is only run _once_ (in our case, this constructor is in `GlobalFixture.cs`). The `Global` collection is meant to be a global setup for all unit tests, so make sure that all unit test classes are in this collection (which means that they need to be added to the `Global` collection directly or their parent class needs to be). -------------------------------------------------------------------------------- /vs_appsettings-example.txt: -------------------------------------------------------------------------------- 1 | Steps to create vs_appsettings.json for local development: 2 | 1. Copy the object below (including the { and }) into a file named "vs_appsettings.json": 3 | { 4 | "JWT_EXPIRE_DAYS": 30, 5 | "JWT_KEY": "GXVM7aJOEL6fB2ME6kRXI8JA6+0PeLEnwf8hcl0an20=", 6 | "ALLOWED_HOSTS": "*", 7 | "POSTGRES_DB": "ct2019db", 8 | "RabbitMQServer": "localhost", 9 | "PYTHON_RPC_SERVER": "localhost:50051", 10 | "TEST_SIGN_IN": "true", 11 | "ADMIN_USER_ID": "mahipal2@illinois.edu", 12 | "ADMIN_PASSWORD": "Test123", 13 | "POSTGRES_SERVER_NAME": "localhost", 14 | "AZURE_SUBSCRIPTION_KEYS": "", 15 | "DATA_DIRECTORY": "", 16 | "AUTH0_DOMAIN": "classtranscribe.auth0.com", 17 | "AUTH0_CLIENT_ID": "GsBj4oR32ys2g2p4iGKWaaqkQRhTHr0d", 18 | "APPLICATION_INSIGHTS_KEY": "d66dd2dd-6b0c-4b9a-8f63-d91350ea9ce7", 19 | "SLACK_WEBHOOK_URL": "", 20 | 21 | "MAX_CONCURRENT_TRANSCRIPTIONS":"1", 22 | "MAX_CONCURRENT_VIDEO_TASKS" : "1", 23 | "MAX_CONCURRENT_SYNC_TASKS" : "1", 24 | 25 | "HOST_NAME": "localhost", 26 | "RABBITMQ_REFCOUNT_CHANNELS" : "N" 27 | } 28 | 29 | 2. For the "AZURE_SUBSCRIPTION_KEYS" field, you need to create a free resource in the Azure portal. Follow these steps: 30 | - Go to https://azure.microsoft.com/en-us/services/cognitive-services/speech-to-text/ and click on "Start free" 31 | - Follow the steps to either sign up for an account or sign into an existing one 32 | - Follow the steps to deploy your free Cognitive Services resource, giving it an appropriate name and selecting a region near your location 33 | - After the resource is successfully deployed, navigate to the resource's menu and click on the "Keys and Endpoint" sub-menu 34 | - Copy and paste the keys on that page into the field like the following example. Use a comma as separator between key and region and semi-colon to add multiple. 35 | - "AZURE_SUBSCRIPTION_KEYS": "ccea4c3982b54a3b82dsffs32a07,eastus;7741185bfc2b475c80a47w,eastus;" 36 | 37 | 3. For the "DATA_DIRECTORY", paste the absolute path of the /test_data/data directory. For example, it might look like this on a unix machine: 38 | - "DATA_DIRECTORY": "/Users//code/Deployment/test_data/data" (make sure to use backslaches if using Windows) 39 | 40 | There's a script in Deployments repo to create the subdirectories. Without these the database container will not start. 41 | 42 | 4. For the "SLACK_WEBHOOK_URL", you can leave it blank or create your own webhook with these instructions: https://slack.com/help/articles/115005265063-Incoming-webhooks-for-Slack -------------------------------------------------------------------------------- /API.Dockerfile: -------------------------------------------------------------------------------- 1 | # Also remove platform from docker-compose.override.yml for api and taskengine 2 | # Uncomment build context in docker-compose.override.yml for api and taskengine 3 | 4 | # e.g., 5 | # taskengine: 6 | # image: classtranscribe/taskengine:staging 7 | # #xx platform: linux/amd64 # Nope - Causes SDK "dotnet restore" to hang on M1 Mac 8 | # build: 9 | # context: ../../WebAPI 10 | # target: publish 11 | # dockerfile: ./TaskEngine.Dockerfile 12 | # 13 | 14 | 15 | #FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim-amd64 as build 16 | FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim as build 17 | # See https://mcr.microsoft.com/en-us/product/dotnet/sdk/tags 18 | 19 | # Running the AMD64 version is of the SDK is broken 20 | # https://github.com/dotnet/dotnet-docker/discussions/4285 21 | # https://github.com/NuGet/Home/issues/13062 22 | 23 | RUN apt-get -q update && apt-get -qy install git 24 | WORKDIR / 25 | RUN git clone https://github.com/eficode/wait-for.git 26 | 27 | WORKDIR /src 28 | COPY ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj 29 | # Did not help ENV DOTNET_NUGET_SIGNATURE_VERIFICATION=false 30 | # Add --verbosity normal|diagnostic 31 | RUN dotnet --list-sdks 32 | RUN dotnet restore --verbosity diagnostic ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj 33 | 34 | COPY ./ClassTranscribeServer/ClassTranscribeServer.csproj ./ClassTranscribeServer/ClassTranscribeServer.csproj 35 | RUN dotnet restore ./ClassTranscribeServer/ClassTranscribeServer.csproj 36 | 37 | COPY ./world_universities_and_domains.json ./world_universities_and_domains.json 38 | COPY ./ct.proto ./ct.proto 39 | COPY ./ClassTranscribeServer ./ClassTranscribeServer 40 | COPY ./ClassTranscribeDatabase ./ClassTranscribeDatabase 41 | WORKDIR /src/ClassTranscribeServer 42 | RUN dotnet publish ClassTranscribeServer.csproj -c Release -o /app --no-restore 43 | 44 | FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim as publish_base 45 | # FROM mcr.microsoft.com/dotnet/aspnet:7.0.14-bookworm-slim as publish_base 46 | 47 | # FROM mcr.microsoft.com/dotnet/core/aspnet:3.1.3-bionic as publish_base 48 | RUN apt-get -q update && apt-get -qy install netcat-traditional 49 | 50 | FROM publish_base as publish 51 | WORKDIR / 52 | COPY --from=build /wait-for . 53 | WORKDIR /app 54 | COPY --from=build /app . 55 | EXPOSE 80 56 | EXPOSE 443 57 | 58 | ARG GITSHA1=unspecified 59 | ENV GITSHA1=$GITSHA1 60 | 61 | ARG BUILDNUMBER=unspecified 62 | ENV BUILDNUMBER=$BUILDNUMBER 63 | 64 | LABEL git_commit_hash=$GITSHA1 65 | 66 | CMD ["dotnet", "/app/ClassTranscribeServer.dll"] -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190717222124_FileRecordsForeignKey.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class FileRecordsForeignKey : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "Video1Id", 11 | table: "Videos", 12 | nullable: true); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "Video2Id", 16 | table: "Videos", 17 | nullable: true); 18 | 19 | migrationBuilder.CreateIndex( 20 | name: "IX_Videos_Video1Id", 21 | table: "Videos", 22 | column: "Video1Id"); 23 | 24 | migrationBuilder.CreateIndex( 25 | name: "IX_Videos_Video2Id", 26 | table: "Videos", 27 | column: "Video2Id"); 28 | 29 | migrationBuilder.AddForeignKey( 30 | name: "FK_Videos_FileRecords_Video1Id", 31 | table: "Videos", 32 | column: "Video1Id", 33 | principalTable: "FileRecords", 34 | principalColumn: "Id", 35 | onDelete: ReferentialAction.Restrict); 36 | 37 | migrationBuilder.AddForeignKey( 38 | name: "FK_Videos_FileRecords_Video2Id", 39 | table: "Videos", 40 | column: "Video2Id", 41 | principalTable: "FileRecords", 42 | principalColumn: "Id", 43 | onDelete: ReferentialAction.Restrict); 44 | } 45 | 46 | protected override void Down(MigrationBuilder migrationBuilder) 47 | { 48 | migrationBuilder.DropForeignKey( 49 | name: "FK_Videos_FileRecords_Video1Id", 50 | table: "Videos"); 51 | 52 | migrationBuilder.DropForeignKey( 53 | name: "FK_Videos_FileRecords_Video2Id", 54 | table: "Videos"); 55 | 56 | migrationBuilder.DropIndex( 57 | name: "IX_Videos_Video1Id", 58 | table: "Videos"); 59 | 60 | migrationBuilder.DropIndex( 61 | name: "IX_Videos_Video2Id", 62 | table: "Videos"); 63 | 64 | migrationBuilder.DropColumn( 65 | name: "Video1Id", 66 | table: "Videos"); 67 | 68 | migrationBuilder.DropColumn( 69 | name: "Video2Id", 70 | table: "Videos"); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200326194432_Add_WatchHistory_Table.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class Add_WatchHistory_Table : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.CreateTable( 11 | name: "WatchHistories", 12 | columns: table => new 13 | { 14 | Id = table.Column(nullable: false), 15 | CreatedAt = table.Column(nullable: false), 16 | CreatedBy = table.Column(nullable: true), 17 | LastUpdatedAt = table.Column(nullable: false), 18 | LastUpdatedBy = table.Column(nullable: true), 19 | IsDeletedStatus = table.Column(nullable: false), 20 | MediaId = table.Column(nullable: true), 21 | ApplicationUserId = table.Column(nullable: true), 22 | Json = table.Column(nullable: true) 23 | }, 24 | constraints: table => 25 | { 26 | table.PrimaryKey("PK_WatchHistories", x => x.Id); 27 | table.ForeignKey( 28 | name: "FK_WatchHistories_AspNetUsers_ApplicationUserId", 29 | column: x => x.ApplicationUserId, 30 | principalTable: "AspNetUsers", 31 | principalColumn: "Id", 32 | onDelete: ReferentialAction.Restrict); 33 | table.ForeignKey( 34 | name: "FK_WatchHistories_Medias_MediaId", 35 | column: x => x.MediaId, 36 | principalTable: "Medias", 37 | principalColumn: "Id", 38 | onDelete: ReferentialAction.Restrict); 39 | }); 40 | 41 | migrationBuilder.CreateIndex( 42 | name: "IX_WatchHistories_ApplicationUserId", 43 | table: "WatchHistories", 44 | column: "ApplicationUserId"); 45 | 46 | migrationBuilder.CreateIndex( 47 | name: "IX_WatchHistories_MediaId", 48 | table: "WatchHistories", 49 | column: "MediaId"); 50 | } 51 | 52 | protected override void Down(MigrationBuilder migrationBuilder) 53 | { 54 | migrationBuilder.DropTable( 55 | name: "WatchHistories"); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pythonrpcserver.Dockerfile: -------------------------------------------------------------------------------- 1 | # ------------------------------ 2 | # Stage 1: Build Whisper.cpp 3 | # ------------------------------ 4 | FROM --platform=linux/amd64 python:3.13.3-bookworm AS whisperbuild 5 | RUN apt-get update && \ 6 | apt-get install -y curl gcc g++ make libglib2.0-0 libsm6 libxext6 libxrender-dev ffmpeg git wget && \ 7 | wget https://github.com/Kitware/CMake/releases/download/v3.27.7/cmake-3.27.7-linux-x86_64.sh -O /tmp/cmake-install.sh && \ 8 | chmod +x /tmp/cmake-install.sh && \ 9 | /tmp/cmake-install.sh --skip-license --prefix=/usr/local && \ 10 | rm /tmp/cmake-install.sh 11 | 12 | WORKDIR /whisper.cpp 13 | RUN git clone https://github.com/ggml-org/whisper.cpp . && \ 14 | cmake -B build -DWHISPER_BUILD_EXAMPLES=ON -DBUILD_SHARED_LIBS=OFF && \ 15 | cmake --build build --parallel $(nproc) 16 | RUN bash ./models/download-ggml-model.sh base.en 17 | RUN bash ./models/download-ggml-model.sh tiny.en 18 | RUN bash ./models/download-ggml-model.sh large-v3 19 | 20 | # ------------------------------ 21 | # Stage 2: Build Python Dependencies 22 | # ------------------------------ 23 | FROM --platform=linux/amd64 python:3.13.3-bookworm AS builder 24 | RUN apt-get update && apt-get install -y build-essential libxml2-dev libxslt1-dev zlib1g-dev libssl-dev libffi-dev python3-dev 25 | 26 | COPY ./PythonRpcServer/requirements.txt . 27 | RUN pip install --no-cache-dir --upgrade pip && \ 28 | pip wheel --wheel-dir=/wheels -r requirements.txt 29 | 30 | # ------------------------------ 31 | # Stage 3: Setup Python RPC Server 32 | # ------------------------------ 33 | FROM --platform=linux/amd64 python:3.13.3-slim-bookworm AS rpcserver 34 | RUN apt-get update && \ 35 | apt-get install -y curl gcc g++ make libglib2.0-0 libsm6 libxext6 libxrender-dev ffmpeg libxml2-dev libxslt1-dev zlib1g-dev libssl-dev libffi-dev python3-dev 36 | 37 | ENV OMP_THREAD_LIMIT=1 38 | COPY --from=whisperbuild /whisper.cpp/build/bin/whisper-cli /usr/local/bin/whisper 39 | COPY --from=whisperbuild /whisper.cpp/models /PythonRpcServer/models 40 | 41 | # copy pre-built wheels from builder stage 42 | COPY --from=builder /wheels /wheels 43 | COPY ./PythonRpcServer/requirements.txt . 44 | RUN pip install --no-cache-dir --upgrade pip && \ 45 | pip install --no-cache-dir --find-links=/wheels -r requirements.txt 46 | 47 | WORKDIR /PythonRpcServer 48 | COPY ./PythonRpcServer/transcribe_hellohellohello.wav . 49 | RUN whisper -ojf -f transcribe_hellohellohello.wav 50 | 51 | COPY ct.proto ct.proto 52 | RUN python -m grpc_tools.protoc -I . --python_out=./ --grpc_python_out=./ ct.proto 53 | 54 | COPY ./PythonRpcServer . 55 | CMD ["nice", "-n", "18", "ionice", "-c", "2", "-n", "6", "python3", "-u", "/PythonRpcServer/server.py"] 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Manually added - 2 | vs_appsettings.json 3 | *vs_appsettings.json 4 | 5 | 6 | ## Ignore Visual Studio temporary files, build results, and 7 | ## files generated by popular Visual Studio add-ons. 8 | 9 | #personal vs setting 10 | vs_appsettings.json 11 | 12 | #ENV Variable files 13 | *.list 14 | 15 | # VS Things 16 | app/ 17 | 18 | 19 | # User-specific files 20 | *.suo 21 | *.user 22 | *.userosscache 23 | *.sln.docstates 24 | 25 | # User-specific files (MonoDevelop/Xamarin Studio) 26 | *.userprefs 27 | 28 | # Build results 29 | [Dd]ebug/ 30 | [Dd]ebugPublic/ 31 | [Rr]elease/ 32 | [Rr]eleases/ 33 | x64/ 34 | x86/ 35 | build/ 36 | bld/ 37 | [Bb]in/ 38 | [Oo]bj/ 39 | 40 | # Visual Studio 2015 cache/options directory 41 | .vs/ 42 | .vscode/ 43 | # Uncomment if you have tasks that create the project's static files in wwwroot 44 | #wwwroot/ 45 | 46 | # MSTest test Results 47 | [Tt]est[Rr]esult*/ 48 | [Bb]uild[Ll]og.* 49 | 50 | # NodeThings 51 | # Logs 52 | logs 53 | *.log 54 | npm-debug.log* 55 | yarn-debug.log* 56 | yarn-error.log* 57 | lerna-debug.log* 58 | 59 | # Diagnostic reports (https://nodejs.org/api/report.html) 60 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 61 | 62 | # Runtime data 63 | pids 64 | *.pid 65 | *.seed 66 | *.pid.lock 67 | 68 | # Directory for instrumented libs generated by jscoverage/JSCover 69 | lib-cov 70 | 71 | # Coverage directory used by tools like istanbul 72 | coverage 73 | *.lcov 74 | 75 | # nyc test coverage 76 | .nyc_output 77 | 78 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 79 | .grunt 80 | 81 | # Bower dependency directory (https://bower.io/) 82 | bower_components 83 | 84 | # node-waf configuration 85 | .lock-wscript 86 | 87 | # Compiled binary addons (https://nodejs.org/api/addons.html) 88 | build/Release 89 | 90 | # Dependency directories 91 | node_modules/ 92 | jspm_packages/ 93 | 94 | # TypeScript v1 declaration files 95 | typings/ 96 | 97 | # TypeScript cache 98 | *.tsbuildinfo 99 | 100 | # Optional npm cache directory 101 | .npm 102 | 103 | # NUNIT 104 | *.VisualState.xml 105 | TestResult.xml 106 | Data/ 107 | data/ 108 | /NodeRpcServer/cookies.txt 109 | nginx/app.conf 110 | NodeRpcServer/ct.proto 111 | /NodeRpcServer 112 | /ClassTranscribeServer/Properties 113 | __pycache__/ 114 | PythonRpcServer/ct_pb2_grpc.py 115 | PythonRpcServer/ct_pb2.py 116 | 117 | *.pyc 118 | .ipynb_checkpoints/ 119 | /old--dot.env 120 | /docker-compose.yml 121 | /PythonRpcServer/corpus_count.json 122 | DevExperiments/LocalEnvironmentVariables.txt 123 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190722204241_Caption_Table2.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Caption_Table2 : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropForeignKey( 10 | name: "FK_Caption_Transcriptions_TranscriptionId", 11 | table: "Caption"); 12 | 13 | migrationBuilder.DropPrimaryKey( 14 | name: "PK_Caption", 15 | table: "Caption"); 16 | 17 | migrationBuilder.RenameTable( 18 | name: "Caption", 19 | newName: "Captions"); 20 | 21 | migrationBuilder.RenameIndex( 22 | name: "IX_Caption_TranscriptionId", 23 | table: "Captions", 24 | newName: "IX_Captions_TranscriptionId"); 25 | 26 | migrationBuilder.AddPrimaryKey( 27 | name: "PK_Captions", 28 | table: "Captions", 29 | column: "Id"); 30 | 31 | migrationBuilder.AddForeignKey( 32 | name: "FK_Captions_Transcriptions_TranscriptionId", 33 | table: "Captions", 34 | column: "TranscriptionId", 35 | principalTable: "Transcriptions", 36 | principalColumn: "Id", 37 | onDelete: ReferentialAction.Restrict); 38 | } 39 | 40 | protected override void Down(MigrationBuilder migrationBuilder) 41 | { 42 | migrationBuilder.DropForeignKey( 43 | name: "FK_Captions_Transcriptions_TranscriptionId", 44 | table: "Captions"); 45 | 46 | migrationBuilder.DropPrimaryKey( 47 | name: "PK_Captions", 48 | table: "Captions"); 49 | 50 | migrationBuilder.RenameTable( 51 | name: "Captions", 52 | newName: "Caption"); 53 | 54 | migrationBuilder.RenameIndex( 55 | name: "IX_Captions_TranscriptionId", 56 | table: "Caption", 57 | newName: "IX_Caption_TranscriptionId"); 58 | 59 | migrationBuilder.AddPrimaryKey( 60 | name: "PK_Caption", 61 | table: "Caption", 62 | column: "Id"); 63 | 64 | migrationBuilder.AddForeignKey( 65 | name: "FK_Caption_Transcriptions_TranscriptionId", 66 | table: "Caption", 67 | column: "TranscriptionId", 68 | principalTable: "Transcriptions", 69 | principalColumn: "Id", 70 | onDelete: ReferentialAction.Restrict); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190918214544_Logs-RemoveFK.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class LogsRemoveFK : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropForeignKey( 10 | name: "FK_Logs_Medias_MediaId", 11 | table: "Logs"); 12 | 13 | migrationBuilder.DropForeignKey( 14 | name: "FK_Logs_Offerings_OfferingId", 15 | table: "Logs"); 16 | 17 | migrationBuilder.DropForeignKey( 18 | name: "FK_Logs_AspNetUsers_UserId", 19 | table: "Logs"); 20 | 21 | migrationBuilder.DropIndex( 22 | name: "IX_Logs_MediaId", 23 | table: "Logs"); 24 | 25 | migrationBuilder.DropIndex( 26 | name: "IX_Logs_OfferingId", 27 | table: "Logs"); 28 | 29 | migrationBuilder.DropIndex( 30 | name: "IX_Logs_UserId", 31 | table: "Logs"); 32 | } 33 | 34 | protected override void Down(MigrationBuilder migrationBuilder) 35 | { 36 | migrationBuilder.CreateIndex( 37 | name: "IX_Logs_MediaId", 38 | table: "Logs", 39 | column: "MediaId"); 40 | 41 | migrationBuilder.CreateIndex( 42 | name: "IX_Logs_OfferingId", 43 | table: "Logs", 44 | column: "OfferingId"); 45 | 46 | migrationBuilder.CreateIndex( 47 | name: "IX_Logs_UserId", 48 | table: "Logs", 49 | column: "UserId"); 50 | 51 | migrationBuilder.AddForeignKey( 52 | name: "FK_Logs_Medias_MediaId", 53 | table: "Logs", 54 | column: "MediaId", 55 | principalTable: "Medias", 56 | principalColumn: "Id", 57 | onDelete: ReferentialAction.Restrict); 58 | 59 | migrationBuilder.AddForeignKey( 60 | name: "FK_Logs_Offerings_OfferingId", 61 | table: "Logs", 62 | column: "OfferingId", 63 | principalTable: "Offerings", 64 | principalColumn: "Id", 65 | onDelete: ReferentialAction.Restrict); 66 | 67 | migrationBuilder.AddForeignKey( 68 | name: "FK_Logs_AspNetUsers_UserId", 69 | table: "Logs", 70 | column: "UserId", 71 | principalTable: "AspNetUsers", 72 | principalColumn: "Id", 73 | onDelete: ReferentialAction.Restrict); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190722203911_Caption_Table.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | using System; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class Caption_Table : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.AddColumn( 11 | name: "Description", 12 | table: "Transcriptions", 13 | nullable: true); 14 | 15 | migrationBuilder.AddColumn( 16 | name: "Language", 17 | table: "Transcriptions", 18 | nullable: true); 19 | 20 | migrationBuilder.CreateTable( 21 | name: "Caption", 22 | columns: table => new 23 | { 24 | Id = table.Column(nullable: false), 25 | CreatedAt = table.Column(nullable: false), 26 | CreatedBy = table.Column(nullable: true), 27 | LastUpdatedAt = table.Column(nullable: false), 28 | LastUpdatedBy = table.Column(nullable: true), 29 | IsDeletedStatus = table.Column(nullable: false), 30 | Begin = table.Column(nullable: false), 31 | End = table.Column(nullable: false), 32 | Text = table.Column(nullable: true), 33 | TranscriptionId = table.Column(nullable: true) 34 | }, 35 | constraints: table => 36 | { 37 | table.PrimaryKey("PK_Caption", x => x.Id); 38 | table.ForeignKey( 39 | name: "FK_Caption_Transcriptions_TranscriptionId", 40 | column: x => x.TranscriptionId, 41 | principalTable: "Transcriptions", 42 | principalColumn: "Id", 43 | onDelete: ReferentialAction.Restrict); 44 | }); 45 | 46 | migrationBuilder.CreateIndex( 47 | name: "IX_Caption_TranscriptionId", 48 | table: "Caption", 49 | column: "TranscriptionId"); 50 | } 51 | 52 | protected override void Down(MigrationBuilder migrationBuilder) 53 | { 54 | migrationBuilder.DropTable( 55 | name: "Caption"); 56 | 57 | migrationBuilder.DropColumn( 58 | name: "Description", 59 | table: "Transcriptions"); 60 | 61 | migrationBuilder.DropColumn( 62 | name: "Language", 63 | table: "Transcriptions"); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190922221537_Add_Processed_Video.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Add_Processed_Video : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "ProcessedVideo1Id", 11 | table: "Videos", 12 | nullable: true); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "ProcessedVideo2Id", 16 | table: "Videos", 17 | nullable: true); 18 | 19 | migrationBuilder.CreateIndex( 20 | name: "IX_Videos_ProcessedVideo1Id", 21 | table: "Videos", 22 | column: "ProcessedVideo1Id"); 23 | 24 | migrationBuilder.CreateIndex( 25 | name: "IX_Videos_ProcessedVideo2Id", 26 | table: "Videos", 27 | column: "ProcessedVideo2Id"); 28 | 29 | migrationBuilder.AddForeignKey( 30 | name: "FK_Videos_FileRecords_ProcessedVideo1Id", 31 | table: "Videos", 32 | column: "ProcessedVideo1Id", 33 | principalTable: "FileRecords", 34 | principalColumn: "Id", 35 | onDelete: ReferentialAction.Restrict); 36 | 37 | migrationBuilder.AddForeignKey( 38 | name: "FK_Videos_FileRecords_ProcessedVideo2Id", 39 | table: "Videos", 40 | column: "ProcessedVideo2Id", 41 | principalTable: "FileRecords", 42 | principalColumn: "Id", 43 | onDelete: ReferentialAction.Restrict); 44 | } 45 | 46 | protected override void Down(MigrationBuilder migrationBuilder) 47 | { 48 | migrationBuilder.DropForeignKey( 49 | name: "FK_Videos_FileRecords_ProcessedVideo1Id", 50 | table: "Videos"); 51 | 52 | migrationBuilder.DropForeignKey( 53 | name: "FK_Videos_FileRecords_ProcessedVideo2Id", 54 | table: "Videos"); 55 | 56 | migrationBuilder.DropIndex( 57 | name: "IX_Videos_ProcessedVideo1Id", 58 | table: "Videos"); 59 | 60 | migrationBuilder.DropIndex( 61 | name: "IX_Videos_ProcessedVideo2Id", 62 | table: "Videos"); 63 | 64 | migrationBuilder.DropColumn( 65 | name: "ProcessedVideo1Id", 66 | table: "Videos"); 67 | 68 | migrationBuilder.DropColumn( 69 | name: "ProcessedVideo2Id", 70 | table: "Videos"); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /ClassTranscribeServer/Controllers/CaptionsSearchController.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Models; 3 | using Microsoft.AspNetCore.Mvc; 4 | using Microsoft.Extensions.Logging; 5 | using Nest; 6 | using Elasticsearch.Net; 7 | using System.Collections.Generic; 8 | using System.Threading.Tasks; 9 | 10 | namespace ClassTranscribeServer.Controllers 11 | { 12 | [Route("api/[controller]")] 13 | [ApiController] 14 | public class CaptionsSearchController : BaseController 15 | { 16 | private readonly IElasticClient _elasticClient; 17 | 18 | public CaptionsSearchController(CTDbContext context, 19 | IElasticClient client, 20 | ILogger logger) : base(context, logger) 21 | { 22 | _elasticClient = client; 23 | } 24 | 25 | [HttpPost] 26 | public async Task>> Search([FromBody] string[] ids, string query, int page = 1, int pageSize = 10) 27 | { 28 | // TODO: add authentication 29 | if (ids == null || ids.Length == 0) 30 | { 31 | 32 | return NotFound(); 33 | } 34 | var result = await _elasticClient.SearchAsync(s => s 35 | .SearchType(SearchType.DfsQueryThenFetch) 36 | .Index(ids) 37 | .From((page - 1)*pageSize) 38 | .Size(pageSize) 39 | .Query(q => q 40 | .Bool(b => b 41 | .Must(m => m 42 | .Match(m1 => m1 43 | .Field(f => f.Text) 44 | .Query(query) 45 | .Fuzziness(Fuzziness.Auto) 46 | ) 47 | ) 48 | ) 49 | ) 50 | ); 51 | 52 | if (result.Total == 0) 53 | { 54 | return NotFound(); 55 | } 56 | return Ok(new SearchResult 57 | { 58 | Total = result.Total, 59 | Page = page, 60 | Results = result.Documents, 61 | ElapsedMilliseconds = result.Took 62 | }); 63 | } 64 | } 65 | 66 | public class SearchResult 67 | { 68 | public long Total { get; set; } 69 | 70 | public int Page { get; set; } 71 | 72 | public IEnumerable Results { get; set; } 73 | 74 | public long ElapsedMilliseconds { get; set; } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20220930063430_ASLVideo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 4 | 5 | namespace ClassTranscribeDatabase.Migrations 6 | { 7 | public partial class ASLVideo : Migration 8 | { 9 | protected override void Up(MigrationBuilder migrationBuilder) 10 | { 11 | migrationBuilder.CreateTable( 12 | name: "ASLVideos", 13 | columns: table => new 14 | { 15 | Id = table.Column(nullable: false), 16 | CreatedAt = table.Column(nullable: false), 17 | CreatedBy = table.Column(nullable: true), 18 | LastUpdatedAt = table.Column(nullable: false), 19 | LastUpdatedBy = table.Column(nullable: true), 20 | IsDeletedStatus = table.Column(nullable: false), 21 | DeletedAt = table.Column(nullable: true), 22 | DeletedBy = table.Column(nullable: true), 23 | Term = table.Column(nullable: true), 24 | Kind = table.Column(nullable: false), 25 | Text = table.Column(nullable: true), 26 | Link = table.Column(nullable: true), 27 | Source = table.Column(nullable: true), 28 | LicenseTag = table.Column(nullable: true), 29 | Shared = table.Column(nullable: false), 30 | Editable = table.Column(nullable: false), 31 | Domain = table.Column(nullable: true), 32 | Likes = table.Column(nullable: false), 33 | CourseId = table.Column(nullable: true), 34 | OfferingId = table.Column(nullable: true) 35 | }, 36 | constraints: table => 37 | { 38 | table.PrimaryKey("PK_ASLVideos", x => x.Id); 39 | table.ForeignKey( 40 | name: "FK_ASLVideos_CourseOfferings_CourseId_OfferingId", 41 | columns: x => new { x.CourseId, x.OfferingId }, 42 | principalTable: "CourseOfferings", 43 | principalColumns: new[] { "CourseId", "OfferingId" }, 44 | onDelete: ReferentialAction.Restrict); 45 | }); 46 | 47 | migrationBuilder.CreateIndex( 48 | name: "IX_ASLVideos_CourseId_OfferingId", 49 | table: "ASLVideos", 50 | columns: new[] { "CourseId", "OfferingId" }); 51 | } 52 | 53 | protected override void Down(MigrationBuilder migrationBuilder) 54 | { 55 | 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20220722044401_Glossary.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 4 | 5 | namespace ClassTranscribeDatabase.Migrations 6 | { 7 | public partial class Glossary : Migration 8 | { 9 | protected override void Up(MigrationBuilder migrationBuilder) 10 | { 11 | migrationBuilder.CreateTable( 12 | name: "Glossaries", 13 | columns: table => new 14 | { 15 | Id = table.Column(nullable: false), 16 | CreatedAt = table.Column(nullable: false), 17 | CreatedBy = table.Column(nullable: true), 18 | LastUpdatedAt = table.Column(nullable: false), 19 | LastUpdatedBy = table.Column(nullable: true), 20 | IsDeletedStatus = table.Column(nullable: false), 21 | DeletedAt = table.Column(nullable: true), 22 | DeletedBy = table.Column(nullable: true), 23 | Term = table.Column(nullable: true), 24 | Link = table.Column(nullable: true), 25 | Description = table.Column(nullable: true), 26 | Source = table.Column(nullable: true), 27 | LicenseTag = table.Column(nullable: true), 28 | Domain = table.Column(nullable: true), 29 | Likes = table.Column(nullable: true), 30 | Shared = table.Column(nullable: false), 31 | Editable = table.Column(nullable: false), 32 | CourseId = table.Column(nullable: true), 33 | OfferingId = table.Column(nullable: true) 34 | }, 35 | constraints: table => 36 | { 37 | table.PrimaryKey("PK_Glossaries", x => x.Id); 38 | table.ForeignKey( 39 | name: "FK_Glossaries_CourseOfferings_CourseId_OfferingId", 40 | columns: x => new { x.CourseId, x.OfferingId }, 41 | principalTable: "CourseOfferings", 42 | principalColumns: new[] { "CourseId", "OfferingId" }, 43 | onDelete: ReferentialAction.Restrict); 44 | }); 45 | 46 | migrationBuilder.CreateIndex( 47 | name: "IX_Glossaries_CourseId_OfferingId", 48 | table: "Glossaries", 49 | columns: new[] { "CourseId", "OfferingId" }); 50 | } 51 | 52 | protected override void Down(MigrationBuilder migrationBuilder) 53 | { 54 | migrationBuilder.DropTable( 55 | name: "Glossaries"); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /UnitTests/Assets/example.srt: -------------------------------------------------------------------------------- 1 | 1 2 | 00:00:03,400 --> 00:00:06,177 3 | In this lesson, we're going to 4 | be talking about finance. And 5 | 6 | 2 7 | 00:00:06,177 --> 00:00:10,009 8 | one of the most important aspects 9 | of finance is interest. 10 | 11 | 3 12 | 00:00:10,009 --> 00:00:13,655 13 | When I go to a bank or some 14 | other lending institution 15 | 16 | 4 17 | 00:00:13,655 --> 00:00:17,720 18 | to borrow money, the bank is happy 19 | to give me that money. But then I'm 20 | 21 | 5 22 | 00:00:17,900 --> 00:00:21,480 23 | going to be paying the bank for the 24 | privilege of using their money. And that 25 | 26 | 6 27 | 00:00:21,660 --> 00:00:26,440 28 | amount of money that I pay the bank is 29 | called interest. Likewise, if I put money 30 | 31 | 7 32 | 00:00:26,620 --> 00:00:31,220 33 | in a savings account or I purchase a 34 | certificate of deposit, the bank just 35 | 36 | 8 37 | 00:00:31,300 --> 00:00:35,800 38 | doesn't put my money in a little box 39 | and leave it there until later. They take 40 | 41 | 9 42 | 00:00:35,800 --> 00:00:40,822 43 | my money and lend it to someone 44 | else. So they are using my money. 45 | 46 | 10 47 | 00:00:40,822 --> 00:00:44,400 48 | The bank has to pay me for the privilege 49 | of using my money. 50 | 51 | 11 52 | 00:00:44,400 --> 00:00:48,700 53 | Now what makes banks 54 | profitable is the rate 55 | 56 | 12 57 | 00:00:48,700 --> 00:00:53,330 58 | that they charge people to use the bank's 59 | money is higher than the rate that they 60 | 61 | 13 62 | 00:00:53,510 --> 00:01:00,720 63 | pay people like me to use my money. The 64 | amount of interest that a person pays or 65 | 66 | 14 67 | 00:01:00,800 --> 00:01:06,640 68 | earns is dependent on three things. It's 69 | dependent on how much money is involved. 70 | 71 | 15 72 | 00:01:06,820 --> 00:01:11,300 73 | It's dependent upon the rate of interest 74 | being paid or the rate of interest being 75 | 76 | 16 77 | 00:01:11,480 --> 00:01:17,898 78 | charged. And it's also dependent upon 79 | how much time is involved. If I have 80 | 81 | 17 82 | 00:01:17,898 --> 00:01:22,730 83 | a loan and I want to decrease the amount 84 | of interest that I'm going to pay, then 85 | 86 | 18 87 | 00:01:22,800 --> 00:01:28,040 88 | I'm either going to have to decrease how 89 | much money I borrow, I'm going to have 90 | 91 | 19 92 | 00:01:28,220 --> 00:01:32,420 93 | to borrow the money over a shorter period 94 | of time, or I'm going to have to find a 95 | 96 | 20 97 | 00:01:32,600 --> 00:01:37,279 98 | lending institution that charges a lower 99 | interest rate. On the other hand, if I 100 | 101 | 21 102 | 00:01:37,279 --> 00:01:41,480 103 | want to earn more interest on my 104 | investment, I'm going to have to invest 105 | 106 | 22 107 | 00:01:41,480 --> 00:01:46,860 108 | more money, leave the money in the 109 | account for a longer period of time, or 110 | 111 | 23 112 | 00:01:46,860 --> 00:01:49,970 113 | find an institution that will pay 114 | me a higher interest rate. -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20231011192212_aslvideo3.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class aslvideo3 : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.AddColumn( 10 | name: "ASLVideoId", 11 | table: "Videos", 12 | nullable: true); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "ProcessedASLVideoId", 16 | table: "Videos", 17 | nullable: true); 18 | 19 | migrationBuilder.AddColumn( 20 | name: "ProcessingLog", 21 | table: "Videos", 22 | nullable: true); 23 | 24 | migrationBuilder.CreateIndex( 25 | name: "IX_Videos_ASLVideoId", 26 | table: "Videos", 27 | column: "ASLVideoId"); 28 | 29 | migrationBuilder.CreateIndex( 30 | name: "IX_Videos_ProcessedASLVideoId", 31 | table: "Videos", 32 | column: "ProcessedASLVideoId"); 33 | 34 | migrationBuilder.AddForeignKey( 35 | name: "FK_Videos_FileRecords_ASLVideoId", 36 | table: "Videos", 37 | column: "ASLVideoId", 38 | principalTable: "FileRecords", 39 | principalColumn: "Id", 40 | onDelete: ReferentialAction.Restrict); 41 | 42 | migrationBuilder.AddForeignKey( 43 | name: "FK_Videos_FileRecords_ProcessedASLVideoId", 44 | table: "Videos", 45 | column: "ProcessedASLVideoId", 46 | principalTable: "FileRecords", 47 | principalColumn: "Id", 48 | onDelete: ReferentialAction.Restrict); 49 | } 50 | 51 | protected override void Down(MigrationBuilder migrationBuilder) 52 | { 53 | migrationBuilder.DropForeignKey( 54 | name: "FK_Videos_FileRecords_ASLVideoId", 55 | table: "Videos"); 56 | 57 | migrationBuilder.DropForeignKey( 58 | name: "FK_Videos_FileRecords_ProcessedASLVideoId", 59 | table: "Videos"); 60 | 61 | migrationBuilder.DropIndex( 62 | name: "IX_Videos_ASLVideoId", 63 | table: "Videos"); 64 | 65 | migrationBuilder.DropIndex( 66 | name: "IX_Videos_ProcessedASLVideoId", 67 | table: "Videos"); 68 | 69 | migrationBuilder.DropColumn( 70 | name: "ASLVideoId", 71 | table: "Videos"); 72 | 73 | migrationBuilder.DropColumn( 74 | name: "ProcessedASLVideoId", 75 | table: "Videos"); 76 | 77 | migrationBuilder.DropColumn( 78 | name: "ProcessingLog", 79 | table: "Videos"); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20201113200736_Disable_Soft_Delete_UserOffering.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Disable_Soft_Delete_UserOffering : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DeleteData( 10 | table: "UserOfferings", 11 | keyColumn: "IsDeletedStatus", 12 | keyValue: 1 13 | ); 14 | 15 | migrationBuilder.DropForeignKey( 16 | name: "FK_UserOfferings_AspNetRoles_IdentityRoleId", 17 | table: "UserOfferings"); 18 | 19 | migrationBuilder.DropPrimaryKey( 20 | name: "PK_UserOfferings", 21 | table: "UserOfferings"); 22 | 23 | migrationBuilder.AlterColumn( 24 | name: "IdentityRoleId", 25 | table: "UserOfferings", 26 | nullable: false, 27 | oldClrType: typeof(string), 28 | oldType: "text", 29 | oldNullable: true); 30 | 31 | migrationBuilder.AddPrimaryKey( 32 | name: "PK_UserOfferings", 33 | table: "UserOfferings", 34 | columns: new[] { "ApplicationUserId", "OfferingId", "IdentityRoleId" }); 35 | 36 | migrationBuilder.AddForeignKey( 37 | name: "FK_UserOfferings_AspNetRoles_IdentityRoleId", 38 | table: "UserOfferings", 39 | column: "IdentityRoleId", 40 | principalTable: "AspNetRoles", 41 | principalColumn: "Id", 42 | onDelete: ReferentialAction.Cascade); 43 | } 44 | 45 | protected override void Down(MigrationBuilder migrationBuilder) 46 | { 47 | migrationBuilder.DropForeignKey( 48 | name: "FK_UserOfferings_AspNetRoles_IdentityRoleId", 49 | table: "UserOfferings"); 50 | 51 | migrationBuilder.DropPrimaryKey( 52 | name: "PK_UserOfferings", 53 | table: "UserOfferings"); 54 | 55 | migrationBuilder.AlterColumn( 56 | name: "IdentityRoleId", 57 | table: "UserOfferings", 58 | type: "text", 59 | nullable: true, 60 | oldClrType: typeof(string)); 61 | 62 | migrationBuilder.AddPrimaryKey( 63 | name: "PK_UserOfferings", 64 | table: "UserOfferings", 65 | columns: new[] { "ApplicationUserId", "OfferingId" }); 66 | 67 | migrationBuilder.AddForeignKey( 68 | name: "FK_UserOfferings_AspNetRoles_IdentityRoleId", 69 | table: "UserOfferings", 70 | column: "IdentityRoleId", 71 | principalTable: "AspNetRoles", 72 | principalColumn: "Id", 73 | onDelete: ReferentialAction.Restrict); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /ClassTranscribeServer/Controllers/RolesController.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Models; 3 | using ClassTranscribeServer.Utils; 4 | using Microsoft.AspNetCore.Authorization; 5 | using Microsoft.AspNetCore.Identity; 6 | using Microsoft.AspNetCore.Mvc; 7 | using Microsoft.Extensions.Logging; 8 | using System.Collections.Generic; 9 | using System.Linq; 10 | using System.Threading.Tasks; 11 | 12 | namespace ClassTranscribeServer.Controllers 13 | { 14 | [Route("api/[controller]")] 15 | [ApiController] 16 | public class RolesController : BaseController 17 | { 18 | private readonly RoleManager _roleManager; 19 | private readonly UserManager _userManager; 20 | private readonly UserUtils _userutils; 21 | 22 | public RolesController(RoleManager roleManager, UserManager userManager, 23 | CTDbContext context, ILogger logger, UserUtils userUtils) : base(context, logger) 24 | { 25 | _roleManager = roleManager; 26 | _userManager = userManager; 27 | _userutils = userUtils; 28 | } 29 | 30 | // POST: api/Roles 31 | [HttpPost] 32 | [Authorize(Roles = Globals.ROLE_ADMIN)] 33 | public async Task AddUserToRole(string mailId, string role) 34 | { 35 | ApplicationUser user = await _userManager.FindByEmailAsync(mailId); 36 | if (user == null) 37 | { 38 | user = await _userutils.CreateNonExistentUser(mailId); 39 | } 40 | if (await _roleManager.RoleExistsAsync(role) && !(await _userManager.IsInRoleAsync(user, role))) 41 | { 42 | await _userManager.AddToRoleAsync(user, role); 43 | } 44 | return Ok(); 45 | } 46 | 47 | // POST: api/Roles 48 | [HttpDelete] 49 | [Authorize(Roles = Globals.ROLE_ADMIN)] 50 | public async Task RemoveInstructor(string mailId) 51 | { 52 | ApplicationUser user = await _userManager.FindByEmailAsync(mailId); 53 | await _userManager.RemoveFromRoleAsync(user, Globals.ROLE_INSTRUCTOR); 54 | return Ok(); 55 | } 56 | 57 | // POST: api/Roles 58 | [HttpGet] 59 | [Authorize(Roles = Globals.ROLE_ADMIN)] 60 | public List GetInstructors(string universityId) 61 | { 62 | var instructorRoleId = _context.Roles.Where(r => r.Name == Globals.ROLE_INSTRUCTOR).First().Id; 63 | 64 | var userIds = (from user in _context.Users 65 | join ur in _context.UserRoles on user.Id equals ur.UserId 66 | where user.UniversityId == universityId && ur.RoleId == instructorRoleId 67 | select new ApplicationUser { Id = user.Id, Email = user.Email, FirstName = user.FirstName, LastName = user.LastName }).ToList(); 68 | return userIds; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Services/MSTranscription/KeyProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | 5 | namespace ClassTranscribeDatabase.Services.MSTranscription 6 | { 7 | /// 8 | /// Each object of this class represents an azure subscription key. 9 | /// 10 | public class Key 11 | { 12 | public string ApiKey { get; set; } 13 | public string Region { get; set; } 14 | /// 15 | /// This value indicates the number of tasks that are already using this key currently. 16 | /// 17 | public int Load { get; set; } 18 | } 19 | 20 | /// 21 | /// This class "provides" azure subscription keys for transcription tasks. 22 | /// It reads its key values from configuration variable "AZURE_SUBSCRIPTION_KEYS" 23 | /// 24 | public class KeyProvider 25 | { 26 | private readonly AppSettings _appSettings; 27 | private readonly List Keys; 28 | private readonly HashSet CurrentVideoIds; 29 | 30 | public KeyProvider(AppSettings appSettings) 31 | { 32 | _appSettings = appSettings; 33 | string subscriptionKeys = _appSettings.AZURE_SUBSCRIPTION_KEYS ?? ""; 34 | Keys = new List(); 35 | CurrentVideoIds = new HashSet(); 36 | 37 | foreach (string subscriptionKey in subscriptionKeys.Split(';')) 38 | { 39 | if (!subscriptionKeys.Contains(",")) 40 | { 41 | continue; 42 | } 43 | string[] keyregion = subscriptionKey.Split(','); 44 | if (keyregion.Length != 2) 45 | { 46 | throw new Exception("AZURE_SUBSCRIPTION_KEYS should be in the form key,region;key,region"); 47 | } 48 | Keys.Add(new Key 49 | { 50 | ApiKey = keyregion[0].Trim(), 51 | Region = keyregion[1].Trim(), 52 | Load = 0 53 | }); 54 | } 55 | 56 | } 57 | 58 | /// 59 | /// To obtain a new key for a task 60 | /// 61 | public Key GetKey(string videoId) 62 | { 63 | if (!CurrentVideoIds.Contains(videoId)) 64 | { 65 | Key key = Keys.OrderBy(k => k.Load).First(); 66 | key.Load += 1; 67 | CurrentVideoIds.Add(videoId); 68 | return key; 69 | } 70 | else 71 | { 72 | throw new Exception("Video already being transcribed"); 73 | } 74 | } 75 | 76 | /// 77 | /// To release a key that was being used. 78 | /// 79 | public void ReleaseKey(Key key, string videoId) 80 | { 81 | Keys.Find(k => k.ApiKey == key.ApiKey).Load -= 1; 82 | CurrentVideoIds.Remove(videoId); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20221116162318_PhraseHintJsonTable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class PhraseHintJsonTable : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | // migrationBuilder.DropColumn( 11 | // name: "ASLVideo", 12 | // table: "Videos"); 13 | 14 | migrationBuilder.AddColumn( 15 | name: "PhraseHintDataId", 16 | table: "Videos", 17 | nullable: true); 18 | 19 | migrationBuilder.CreateTable( 20 | name: "JsonData", 21 | columns: table => new 22 | { 23 | Id = table.Column(nullable: false), 24 | CreatedAt = table.Column(nullable: false), 25 | CreatedBy = table.Column(nullable: true), 26 | LastUpdatedAt = table.Column(nullable: false), 27 | LastUpdatedBy = table.Column(nullable: true), 28 | IsDeletedStatus = table.Column(nullable: false), 29 | DeletedAt = table.Column(nullable: true), 30 | DeletedBy = table.Column(nullable: true), 31 | Json = table.Column(nullable: true) 32 | }, 33 | constraints: table => 34 | { 35 | table.PrimaryKey("PK_JsonData", x => x.Id); 36 | }); 37 | 38 | migrationBuilder.CreateIndex( 39 | name: "IX_Videos_PhraseHintDataId", 40 | table: "Videos", 41 | column: "PhraseHintDataId"); 42 | 43 | migrationBuilder.AddForeignKey( 44 | name: "FK_Videos_JsonData_PhraseHintDataId", 45 | table: "Videos", 46 | column: "PhraseHintDataId", 47 | principalTable: "JsonData", 48 | principalColumn: "Id", 49 | onDelete: ReferentialAction.Restrict); 50 | } 51 | 52 | protected override void Down(MigrationBuilder migrationBuilder) 53 | { 54 | migrationBuilder.DropForeignKey( 55 | name: "FK_Videos_JsonData_PhraseHintDataId", 56 | table: "Videos"); 57 | 58 | migrationBuilder.DropTable( 59 | name: "JsonData"); 60 | 61 | // migrationBuilder.DropIndex( 62 | // name: "IX_Videos_PhraseHintDataId", 63 | // table: "Videos"); 64 | 65 | migrationBuilder.DropColumn( 66 | name: "PhraseHintDataId", 67 | table: "Videos"); 68 | 69 | // migrationBuilder.AddColumn( 70 | // name: "ASLVideo", 71 | // table: "Videos", 72 | // type: "text", 73 | // nullable: false, 74 | // defaultValue: ""); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190831192420_Log.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | using System; 3 | 4 | namespace ClassTranscribeDatabase.Migrations 5 | { 6 | public partial class Log : Migration 7 | { 8 | protected override void Up(MigrationBuilder migrationBuilder) 9 | { 10 | migrationBuilder.CreateTable( 11 | name: "Logs", 12 | columns: table => new 13 | { 14 | Id = table.Column(nullable: false), 15 | CreatedAt = table.Column(nullable: false), 16 | CreatedBy = table.Column(nullable: true), 17 | LastUpdatedAt = table.Column(nullable: false), 18 | LastUpdatedBy = table.Column(nullable: true), 19 | IsDeletedStatus = table.Column(nullable: false), 20 | UserId = table.Column(nullable: true), 21 | OfferingId = table.Column(nullable: true), 22 | MediaId = table.Column(nullable: true), 23 | EventType = table.Column(nullable: true), 24 | Json = table.Column(nullable: true) 25 | }, 26 | constraints: table => 27 | { 28 | table.PrimaryKey("PK_Logs", x => x.Id); 29 | table.ForeignKey( 30 | name: "FK_Logs_Medias_MediaId", 31 | column: x => x.MediaId, 32 | principalTable: "Medias", 33 | principalColumn: "Id", 34 | onDelete: ReferentialAction.Restrict); 35 | table.ForeignKey( 36 | name: "FK_Logs_Offerings_OfferingId", 37 | column: x => x.OfferingId, 38 | principalTable: "Offerings", 39 | principalColumn: "Id", 40 | onDelete: ReferentialAction.Restrict); 41 | table.ForeignKey( 42 | name: "FK_Logs_AspNetUsers_UserId", 43 | column: x => x.UserId, 44 | principalTable: "AspNetUsers", 45 | principalColumn: "Id", 46 | onDelete: ReferentialAction.Restrict); 47 | }); 48 | 49 | migrationBuilder.CreateIndex( 50 | name: "IX_Logs_MediaId", 51 | table: "Logs", 52 | column: "MediaId"); 53 | 54 | migrationBuilder.CreateIndex( 55 | name: "IX_Logs_OfferingId", 56 | table: "Logs", 57 | column: "OfferingId"); 58 | 59 | migrationBuilder.CreateIndex( 60 | name: "IX_Logs_UserId", 61 | table: "Logs", 62 | column: "UserId"); 63 | } 64 | 65 | protected override void Down(MigrationBuilder migrationBuilder) 66 | { 67 | migrationBuilder.DropTable( 68 | name: "Logs"); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /UnitTests/Assets/subtitles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Example subtitles 4 | Example subtitles 5 | Example subtitles 6 | Example subtitles 7 | Example subtitles 8 | Example subtitles 9 | Example subtitles 10 | Example subtitles 11 | Example subtitles 12 | Example subtitles 13 | Example subtitles 14 | Example subtitles 15 | Example subtitles 16 | Example subtitles 17 | Example subtitles 18 | Example subtitles 19 | Example subtitles 20 | Example subtitles 21 | Example subtitles 22 | Example subtitles 23 | Example subtitles 24 | Example subtitles 25 | Example subtitles 26 | Example subtitles 27 | Example subtitles 28 | Example subtitles 29 | Example subtitles 30 | Example subtitles 31 | Example subtitles 32 | Example subtitles 33 | Example subtitles 34 | Example subtitles 35 | Example subtitles 36 | Example subtitles 37 | Example subtitles 38 | Example subtitles 39 | Example subtitles 40 | Example subtitles 41 | Example subtitles 42 | Example subtitles 43 | Example subtitles 44 | Example subtitles 45 | Example subtitles 46 | Example subtitles 47 | Example subtitles 48 | Example subtitles 49 | Example subtitles 50 | Example subtitles 51 | Example subtitles 52 | Example subtitles 53 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/CaptionQueries.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Models; 2 | using Microsoft.EntityFrameworkCore; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | 7 | namespace ClassTranscribeDatabase 8 | { 9 | /// 10 | /// Some commonly used queries to get the captions for a given videoId. 11 | /// 12 | public class CaptionQueries 13 | { 14 | private readonly CTDbContext _context; 15 | public CaptionQueries(CTDbContext context) 16 | { 17 | _context = context; 18 | } 19 | 20 | /// 21 | /// Get the captions for a given videoId 22 | /// 23 | /// Language of the captions to fetch. 24 | public async Task> GetCaptionsAsync(string videoId, string sourceInternalRef, string language) // = "en-US" 25 | { 26 | try 27 | { 28 | var transcriptionId = _context.Transcriptions.Where(t => t.Language == language && t.VideoId == videoId && t.SourceInternalRef == sourceInternalRef 29 | && t.TranscriptionType == TranscriptionType.Caption).First().Id; 30 | return await GetCaptionsAsync(transcriptionId); 31 | } 32 | catch (System.InvalidOperationException) 33 | { 34 | // If Transcriptions do not exist then First() will throw InvalidOperationException 35 | 36 | return new List(); 37 | } 38 | } 39 | 40 | /// 41 | /// Get the text descriptions for a given videoId 42 | /// 43 | /// Language of the captions to fetch. 44 | public async Task> GetDescriptionsAsync(string videoId, string language) // = "en-US" 45 | { 46 | try 47 | { 48 | var transcriptionId = _context.Transcriptions.Where(t => t.Language == language && t.VideoId == videoId 49 | && t.TranscriptionType == TranscriptionType.TextDescription).First().Id; 50 | return await GetCaptionsAsync(transcriptionId); 51 | } 52 | catch (System.InvalidOperationException) 53 | { 54 | // If Transcriptions do not exist then First() will throw InvalidOperationException 55 | 56 | return new List(); 57 | } 58 | } 59 | 60 | /// 61 | /// Get the captions for a given transcriptionId 62 | /// 63 | public async Task> GetCaptionsAsync(string transcriptionId) 64 | { 65 | // This has to be split in two because of https://docs.microsoft.com/en-us/ef/core/querying/client-eval 66 | var allCaptions = await _context.Captions.Where(c => c.TranscriptionId == transcriptionId).ToListAsync(); 67 | var captions = allCaptions.GroupBy(c => c.Index).Select(g => g.OrderByDescending(c => c.CreatedAt).First()) 68 | .OrderBy(c => c.Index).ToList(); 69 | return captions; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /PythonRpcServer/ffmpeg.py: -------------------------------------------------------------------------------- 1 | from ffmpy import FFmpeg 2 | import subprocess 3 | from time import perf_counter 4 | import utils 5 | import json 6 | 7 | default_max_threads = 3 8 | 9 | def convertVideoToWavWithOffset(input_filepath, offset): 10 | try: 11 | start_time = perf_counter() 12 | if offset is None: 13 | offset = 0.0 14 | 15 | nthreads = utils.getMaxThreads() 16 | 17 | print(f"convertVideoToWavWithOffset('{input_filepath}',{offset}) using {nthreads} thread(s).") 18 | output_filepath = utils.getTmpFile() 19 | # For less verbosity try, global_options= '-hide_banner -loglevel error -nostats' 20 | # See https://github.com/Ch00k/ffmpy/blob/master/ffmpy.py 21 | 22 | ext = '.wav' 23 | ff = FFmpeg( 24 | global_options=f"-hide_banner -loglevel error -nostats -threads {nthreads}", 25 | inputs={ 26 | input_filepath: '-ss {}'.format(offset)}, 27 | outputs={output_filepath: '-c:a pcm_s16le -ac 1 -y -ar 16000 -f wav'} 28 | ) 29 | print(f"Starting. Audio output will be saved in {output_filepath}") 30 | ff.run() 31 | end_time = perf_counter() 32 | print(f"convertVideoToWavWithOffset('{input_filepath}',{offset}) Complete. Duration {int(end_time - start_time)} seconds") 33 | return output_filepath, ext 34 | except Exception as e: 35 | print("Exception:" + str(e)) 36 | raise e 37 | 38 | # Creates a low res mp4 39 | def processVideo(input_filepath): 40 | try: 41 | start_time = perf_counter() 42 | 43 | nthreads = utils.getMaxThreads() 44 | 45 | print(f"processVideo('{input_filepath}') using {nthreads} threads") 46 | output_filepath = utils.getTmpFile() 47 | ext = '.mp4' 48 | ff = FFmpeg( 49 | global_options= f"-hide_banner -loglevel error -nostats -threads {nthreads}", 50 | inputs={input_filepath: None}, 51 | outputs={ 52 | output_filepath: '-c:v libx264 -f mp4 -b:v 500K -s 768x432 -movflags faststart -ar 48000 -preset medium'} 53 | ) 54 | ff.run() 55 | end_time = perf_counter() 56 | print(f"processVideo('{input_filepath}') Complete. Duration {int(end_time - start_time)} seconds") 57 | return output_filepath, ext 58 | except Exception as e: 59 | print("Exception:" + str(e)) 60 | raise e 61 | 62 | def getMediaInfo(input_filepath): 63 | #Exception printing and timing is now handled by caller -see LogWorker 64 | # In seconds 65 | #https://gist.github.com/nrk/2286511 66 | staticargs = "-hide_banner -loglevel fatal -show_error -show_format -show_streams -show_programs -show_chapters -show_private_data -print_format json" 67 | jsonresult = subprocess.check_output( 68 | ['ffprobe','-i', input_filepath] + staticargs.split(' '), 69 | encoding='utf-8' 70 | ) 71 | print(f'{input_filepath}: {jsonresult}') 72 | # Check if is a valid json object 73 | try: 74 | json.loads(jsonresult) 75 | except json.JSONDecodeError: 76 | jsonresult = '{}' 77 | 78 | return jsonresult 79 | 80 | # example r = ffmpeg.getMediaInfo('/data/5ff44cac-fbfe-4745-bcae-9dbb181cf0f2.mp4') -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20200205033251_Dictionary.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 3 | using System; 4 | 5 | namespace ClassTranscribeDatabase.Migrations 6 | { 7 | public partial class Dictionary : Migration 8 | { 9 | protected override void Up(MigrationBuilder migrationBuilder) 10 | { 11 | migrationBuilder.AlterColumn( 12 | name: "Id", 13 | table: "AspNetUserClaims", 14 | nullable: false, 15 | oldClrType: typeof(int)) 16 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn) 17 | .OldAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn); 18 | 19 | migrationBuilder.AlterColumn( 20 | name: "Id", 21 | table: "AspNetRoleClaims", 22 | nullable: false, 23 | oldClrType: typeof(int)) 24 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn) 25 | .OldAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn); 26 | 27 | migrationBuilder.CreateTable( 28 | name: "Dictionaries", 29 | columns: table => new 30 | { 31 | Id = table.Column(nullable: false), 32 | CreatedAt = table.Column(nullable: false), 33 | CreatedBy = table.Column(nullable: true), 34 | LastUpdatedAt = table.Column(nullable: false), 35 | LastUpdatedBy = table.Column(nullable: true), 36 | IsDeletedStatus = table.Column(nullable: false), 37 | Key = table.Column(nullable: true), 38 | Value = table.Column(nullable: true) 39 | }, 40 | constraints: table => 41 | { 42 | table.PrimaryKey("PK_Dictionaries", x => x.Id); 43 | }); 44 | } 45 | 46 | protected override void Down(MigrationBuilder migrationBuilder) 47 | { 48 | migrationBuilder.DropTable( 49 | name: "Dictionaries"); 50 | 51 | migrationBuilder.AlterColumn( 52 | name: "Id", 53 | table: "AspNetUserClaims", 54 | nullable: false, 55 | oldClrType: typeof(int)) 56 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn) 57 | .OldAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 58 | 59 | migrationBuilder.AlterColumn( 60 | name: "Id", 61 | table: "AspNetRoleClaims", 62 | nullable: false, 63 | oldClrType: typeof(int)) 64 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn) 65 | .OldAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Migrations/20190707013307_Add_Playlist.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace ClassTranscribeDatabase.Migrations 4 | { 5 | public partial class Add_Playlist : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.DropForeignKey( 10 | name: "FK_Medias_Playlist_PlaylistId", 11 | table: "Medias"); 12 | 13 | migrationBuilder.DropForeignKey( 14 | name: "FK_OfferingPlaylists_Playlist_PlaylistId", 15 | table: "OfferingPlaylists"); 16 | 17 | migrationBuilder.DropPrimaryKey( 18 | name: "PK_Playlist", 19 | table: "Playlist"); 20 | 21 | migrationBuilder.RenameTable( 22 | name: "Playlist", 23 | newName: "Playlists"); 24 | 25 | migrationBuilder.AddPrimaryKey( 26 | name: "PK_Playlists", 27 | table: "Playlists", 28 | column: "Id"); 29 | 30 | migrationBuilder.AddForeignKey( 31 | name: "FK_Medias_Playlists_PlaylistId", 32 | table: "Medias", 33 | column: "PlaylistId", 34 | principalTable: "Playlists", 35 | principalColumn: "Id", 36 | onDelete: ReferentialAction.Restrict); 37 | 38 | migrationBuilder.AddForeignKey( 39 | name: "FK_OfferingPlaylists_Playlists_PlaylistId", 40 | table: "OfferingPlaylists", 41 | column: "PlaylistId", 42 | principalTable: "Playlists", 43 | principalColumn: "Id", 44 | onDelete: ReferentialAction.Cascade); 45 | } 46 | 47 | protected override void Down(MigrationBuilder migrationBuilder) 48 | { 49 | migrationBuilder.DropForeignKey( 50 | name: "FK_Medias_Playlists_PlaylistId", 51 | table: "Medias"); 52 | 53 | migrationBuilder.DropForeignKey( 54 | name: "FK_OfferingPlaylists_Playlists_PlaylistId", 55 | table: "OfferingPlaylists"); 56 | 57 | migrationBuilder.DropPrimaryKey( 58 | name: "PK_Playlists", 59 | table: "Playlists"); 60 | 61 | migrationBuilder.RenameTable( 62 | name: "Playlists", 63 | newName: "Playlist"); 64 | 65 | migrationBuilder.AddPrimaryKey( 66 | name: "PK_Playlist", 67 | table: "Playlist", 68 | column: "Id"); 69 | 70 | migrationBuilder.AddForeignKey( 71 | name: "FK_Medias_Playlist_PlaylistId", 72 | table: "Medias", 73 | column: "PlaylistId", 74 | principalTable: "Playlist", 75 | principalColumn: "Id", 76 | onDelete: ReferentialAction.Restrict); 77 | 78 | migrationBuilder.AddForeignKey( 79 | name: "FK_OfferingPlaylists_Playlist_PlaylistId", 80 | table: "OfferingPlaylists", 81 | column: "PlaylistId", 82 | principalTable: "Playlist", 83 | principalColumn: "Id", 84 | onDelete: ReferentialAction.Cascade); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/ClassTranscribeDatabase.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | 6 | 7 | 8 | ../ClassTranscribeDatabaseDebug/bin 9 | ../ClassTranscribeDatabaseDebug/obj 10 | 11 | 12 | 13 | ../ClassTranscribeDatabaseRelease/bin 14 | ../ClassTranscribeDatabaseRelease/obj 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | Always 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | all 36 | runtime; build; native; contentfiles; analyzers; buildtransitive 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | runtime; build; native; contentfiles; analyzers; buildtransitive 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /UnitTests/ClassTranscribeServer/ControllerTests/AdminControllerTest.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Models; 2 | using ClassTranscribeServer; 3 | using ClassTranscribeServer.Controllers; 4 | using System.Collections.Generic; 5 | using System.Threading.Tasks; 6 | using UnitTests.Utils; 7 | using Xunit; 8 | 9 | namespace UnitTests.ClassTranscribeServer.ControllerTests 10 | { 11 | public class AdminControllerTest : BaseControllerTest 12 | { 13 | private readonly AdminController _controller; 14 | 15 | public AdminControllerTest(GlobalFixture fixture) : base(fixture) 16 | { 17 | _controller = new AdminController( 18 | fixture._authorizationService, 19 | (WakeDownloader)fixture._serviceProvider.GetService(typeof(WakeDownloader)), 20 | _context, 21 | null 22 | ); 23 | } 24 | 25 | [Fact] 26 | public async Task Generate_File_Paths() 27 | { 28 | var result = await _controller.GenerateFilePaths(); 29 | Assert.Equal(0, result.Value); 30 | 31 | var courses = new List { 32 | new Course { Id = "001" }, 33 | new Course { Id = "002" }, 34 | new Course { Id = "003", FilePath = "aaa" }, 35 | 36 | new Course { Id = "004", FilePath = "bbb" }, 37 | }; 38 | var courseOfferings = new List { 39 | new CourseOffering { Id = "co001", CourseId = "001", OfferingId= "Oid3" }, 40 | new CourseOffering { Id = "co002", CourseId = "002", FilePath = "ccc", OfferingId = "Oid4" }, 41 | }; 42 | _context.Courses.AddRange(courses); 43 | _context.CourseOfferings.AddRange(courseOfferings); 44 | _context.SaveChanges(); 45 | 46 | result = await _controller.GenerateFilePaths(); 47 | Assert.Equal(3, result.Value); 48 | 49 | Assert.True(Common.IsValidFilePath(courses[0])); 50 | Assert.True(Common.IsValidFilePath(courses[1])); 51 | Assert.True(Common.IsValidFilePath(courseOfferings[0])); 52 | 53 | result = await _controller.GenerateFilePaths(); 54 | Assert.Equal(0, result.Value); 55 | 56 | // only course offerings with valid CourseId fields should be processed 57 | var newCourse = new Course { }; 58 | var newCo = new CourseOffering { CourseId = "002" , OfferingId = "cc0"}; 59 | var newOff0 = new Offering { Id = "cc0" }; 60 | var newOff1 = new Offering { Id = "cc1" }; 61 | var newOff2 = new Offering { Id = "cc2" }; 62 | _context.Offerings.AddRange(newOff0, newOff1, newOff2); 63 | _context.Courses.Add(newCourse); 64 | _context.CourseOfferings.AddRange( 65 | new CourseOffering { CourseId = "blah", OfferingId = "cc1" }, 66 | new CourseOffering { CourseId = "non-existing" , OfferingId = "cc2" }, 67 | newCo 68 | ); 69 | 70 | _context.SaveChanges(); 71 | 72 | result = await _controller.GenerateFilePaths(); 73 | Assert.Equal(2, result.Value); 74 | 75 | 76 | Assert.True(Common.IsValidFilePath(newCourse)); 77 | Assert.True(Common.IsValidFilePath(newCo)); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## WebAPI 2 | 3 | This repository provides the source code primary API endpoint of the ClassTranscribe Server. Copyright (C) University of Illinois, USA. 2019-2022 4 | 5 | The source code in this repository is licensed [here](https://github.com/classtranscribe/WebAPI/blob/staging/LICENSE). Please email angrave at Illinois if you are interested in alternative licenses of this code and related intellectual property. 6 | 7 | ## Build Status 8 | 9 | | Branch | Status | 10 | |-------:|--------| 11 | | Production | ![Production](https://github.com/classtranscribe/WebAPI/actions/workflows/docker.yaml/badge.svg?event=push&branch=main) | 12 | | Staging | ![Staging](https://github.com/classtranscribe/WebAPI/actions/workflows/docker.yaml/badge.svg?event=push&branch=staging) | 13 | | Experimental | ![Experimental](https://github.com/classtranscribe/WebAPI/actions/workflows/docker.yaml/badge.svg?event=push&branch=expt) | 14 | 15 | # Pull requests, Submitting code and copyright. 16 | 17 | In submitting code to this repository - for example by issuing a git pull-request, or working directly with ClassTranscribe developers to merge or add code - you agree to re-assign copyright of the code to the University of Illinois. 18 | 19 | # Build Instructions 20 | 21 | This repository is in the form of 3 docker services, 22 | 1. The ClassTranscribeServer (service name - classtranscribeserver, accessible on port 8080) 23 | 2. A postgresQL database server (service name - db, accessible on port 5432) 24 | 3. A pgadmin server, which allows to interact with the db using a GUI interface (service name - pgadmin, accessible on port 5050) 25 | 26 | To build this repository, 27 | 1. Install docker 28 | 2. Obtain all the environment variable files from an admin 29 | 3. To start all the services 30 | ``` 31 | docker-compose up 32 | ``` 33 | 34 | To start a single service 35 | ``` 36 | docker-compose up [service_name] 37 | ``` 38 | Eg. 39 | ``` 40 | docker-compose up -d api 41 | ``` 42 | "-d" option allows running the service in a detached mode 43 | 44 | 4. That's it. 45 | 46 | ## Building Docker Images Manually 47 | You can also build any of the three services individually for testing using one or more of the following commands: 48 | ``` 49 | docker build -t classtranscribe/api:latest -f API.Dockerfile . 50 | docker build -t classtranscribe/pythonrpcserver -f pythonrpcserver.Dockerfile . 51 | docker build -t classtranscribe/taskengine -f TaskEngine.Dockerfile . 52 | ``` 53 | 54 | #### Overriding PYTUBE_VERSION 55 | By default, the `pythonrpcserver` is built with the latest version of pytube available from pypi. 56 | 57 | To build `pythonrpcserver` with a specific version of [pytube](https://pypi.org/project/pytube/) (for testing purposes): 58 | ```bash 59 | docker build -t classtranscribe/pythonrpcserver -f pythonrpcserver.Dockerfile . --build-arg=PYTUBE_VERSION=12.1.0 60 | ``` 61 | 62 | This will run an optional build step to override the installed version of `pytube` with the target version downloaded directly from Github. 63 | 64 | # Notes 65 | 1. If there are dependency changes made to classtranscribeserver, you are required to rebuild using docker-compose 66 | ```docker-compose up --build api``` 67 | 2. After making any code changes to rebuild code, just re-run the service, 68 | ```docker-compose up api``` 69 | 3. Refer [gotchas.md](./gotchas.md) for known coding blunders. 70 | 71 | -------------------------------------------------------------------------------- /PythonRpcServer/unused/scenedetector_test.py: -------------------------------------------------------------------------------- 1 | import scenedetector as sd 2 | import time 3 | import json 4 | import os 5 | import urllib.request 6 | import shutil 7 | import cv2 8 | from skimage.metrics import structural_similarity as ssim 9 | 10 | DATA_DIR = os.getcwd() 11 | 12 | # General Testing Scheme For All Test Cases 13 | def test_scheme(folder_name, url, expected_phrases): 14 | # General Testing Scheme 15 | print("----------" + folder_name + "---STARTED----------") 16 | 17 | video_name = folder_name + '.mp4' 18 | 19 | video_path = DATA_DIR + '/' + video_name 20 | folder_path = DATA_DIR + '/frames/' + folder_name 21 | 22 | # Download the video file 23 | urllib.request.urlretrieve(url, video_name) 24 | 25 | # Run SceneDetector on the video 26 | toy_lecture_json = sd.find_scenes(video_name) 27 | 28 | scenes = json.loads(toy_lecture_json) 29 | 30 | corpus = [] 31 | for scene in scenes: 32 | corpus.append(scene['phrases']) 33 | 34 | raw_phrases = '\n'.join( ['\n'.join(words) for words in corpus] ) 35 | raw_phrases = raw_phrases.replace('``','') 36 | 37 | # Check phrase occurance 38 | for phrase in expected_phrases: 39 | assert(phrase in raw_phrases) 40 | print("Phrase " + phrase + " was in the OCR output") 41 | 42 | # Check frame number 43 | frame_list = os.listdir(folder_path) 44 | for frame_name in frame_list: 45 | frame_path = folder_path + '/' + frame_name 46 | output_frame = cv2.imread(frame_path) 47 | frame_number = frame_name[frame_name.rfind('-')+1 : frame_name.find('.')] 48 | 49 | cap = cv2.VideoCapture(video_path) 50 | cap.set(cv2.CAP_PROP_POS_FRAMES, int(frame_number) ) 51 | ret, frame = cap.read() 52 | 53 | rezied_frame = cv2.cvtColor(cv2.resize(frame, (320,240)), cv2.COLOR_BGR2GRAY) 54 | resized_output_frame = cv2.cvtColor(cv2.resize(output_frame, (320,240)), cv2.COLOR_BGR2GRAY) 55 | 56 | sim = ssim(rezied_frame, resized_output_frame) 57 | print('Frame ' + frame_number + " Simlarity: " + str(sim)) 58 | #assert(sim > 0.99) 59 | 60 | # Delete files 61 | os.remove(video_path) 62 | shutil.rmtree(folder_path) 63 | 64 | print("----------" + folder_name + "---Passed----------") 65 | 66 | def run_scenedetector_tests(): 67 | video_names = ['test_1_toy_lecture', 'test_2_241_thread', 'test_3_adv582_w3'] 68 | urls = [ 69 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=pec6m3vbjzu2l4d2m1gv9588npq9nw7o&file_id=f_827175578540', 70 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=xbs7aqcsnrjdglplc6neh3fxdhkqz2fi&file_id=f_827616401395', 71 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=t8lnoxyrmiff0g2xtvg9t5tlak0npnyl&file_id=f_828444986436' 72 | ] 73 | expected_phrases_list = [ 74 | ['Slide One', 'float', 'char'], 75 | ['pthread_create', 'Compile', 'stacks', 'printf', 'nothing happens'], 76 | ['Merchants of Cool', 'Barak Goodman', 'New York Times'] 77 | ] 78 | for i in range(len(video_names)): 79 | test_scheme(video_names[i], urls[i], expected_phrases_list[i]) 80 | 81 | 82 | if __name__ == '__main__': 83 | if 'DATA_DIRECTORY' not in os.environ.keys(): 84 | os.environ['DATA_DIRECTORY'] = str(os.getcwd()) 85 | run_scenedetector_tests() 86 | print('done') 87 | -------------------------------------------------------------------------------- /UnitTests/Utils/MockServices.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Models; 2 | using Microsoft.AspNetCore.Authentication; 3 | using Microsoft.AspNetCore.Http; 4 | using Microsoft.AspNetCore.Identity; 5 | using Microsoft.Extensions.Logging; 6 | using Microsoft.Extensions.Options; 7 | using Moq; 8 | using System; 9 | using System.Collections.Generic; 10 | using System.Threading.Tasks; 11 | 12 | namespace UnitTests 13 | { 14 | public class MockUserManager : UserManager 15 | { 16 | public MockUserManager() 17 | : base(new Mock>().Object, 18 | new Mock>().Object, 19 | new Mock>().Object, 20 | new IUserValidator[0], 21 | new IPasswordValidator[0], 22 | new Mock().Object, 23 | new Mock().Object, 24 | new Mock().Object, 25 | new Mock>>().Object) 26 | { } 27 | 28 | public override Task FindByEmailAsync(string email) 29 | { 30 | if (string.IsNullOrEmpty(email)) 31 | { 32 | return null; 33 | } 34 | 35 | return Task.FromResult(new ApplicationUser { Id = email, Email = email }); 36 | } 37 | 38 | public override Task> GetRolesAsync(ApplicationUser user) 39 | { 40 | return Task.FromResult((IList) new List()); 41 | } 42 | 43 | public override Task RemoveFromRoleAsync(ApplicationUser user, string role) 44 | { 45 | return Task.FromResult(new IdentityResult()); 46 | } 47 | 48 | public override Task IsInRoleAsync(ApplicationUser user, string role) 49 | { 50 | return Task.FromResult(true); 51 | } 52 | } 53 | 54 | public class MockRoleManager : RoleManager 55 | { 56 | public MockRoleManager() 57 | : base(new Mock>().Object, 58 | new IRoleValidator[0], 59 | new Mock().Object, 60 | new Mock().Object, 61 | new Mock>>().Object) 62 | { } 63 | 64 | public override Task RoleExistsAsync(string roleName) 65 | { 66 | return Task.FromResult(true); 67 | } 68 | } 69 | 70 | public class MockSignInManager : SignInManager 71 | { 72 | public MockSignInManager() 73 | : base(new MockUserManager(), 74 | new Mock().Object, 75 | new Mock>().Object, 76 | new Mock>().Object, 77 | new Mock>>().Object, 78 | new Mock().Object, 79 | new Mock>().Object) 80 | { } 81 | 82 | public override Task SignInAsync(ApplicationUser user, bool isPersistent, string authenticationMethod = null) 83 | { 84 | return Task.CompletedTask; 85 | } 86 | } 87 | } -------------------------------------------------------------------------------- /PythonRpcServer/titledetector_test.py: -------------------------------------------------------------------------------- 1 | import titledetector as td 2 | import pytesseract 3 | import cv2 4 | import urllib.request 5 | import os 6 | 7 | # General Testing Scheme For All Test Cases 8 | def test_scheme(image_name, url, expected_result): 9 | # Download the sample slide from Box 10 | urllib.request.urlretrieve(url, image_name) 11 | 12 | # Change the sample slide into gray scale 13 | img = cv2.imread(image_name, cv2.IMREAD_COLOR) 14 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 15 | 16 | # Perform the OCR for the sample slide 17 | str_text = pytesseract.image_to_data(gray, output_type='dict') 18 | 19 | # Run TitleDetector upon the pytesseract result 20 | frame_height, frame_width, frame_channels = img.shape 21 | title = td.title_detection(str_text, frame_height, frame_width) 22 | 23 | # Delete the downloaded image file 24 | os.remove(image_name) 25 | 26 | assert(title == expected_result) 27 | print('Result for ' + image_name + ' is correct!' ) 28 | 29 | def run_titledetector_tests(): 30 | image_names = [ 31 | 'two_line_test.jpeg', 32 | 'chem_102_test.jpeg', 33 | 'three_line_test.jpeg', 34 | 'words_limit_test.jpeg', 35 | 'untitled_test.jpeg', 36 | 'cs_418_test.jpeg', 37 | 'text_in_background_test.jpeg', 38 | 'middle_right_title_test.jpeg', 39 | 'text_with_number_test.jpeg', 40 | 'middle_title_test.jpeg' 41 | ] 42 | 43 | urls = [ 44 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=1phs3sikitxdw0vjaryiu6isu3oq5yii&file_id=f_839325860753', 45 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=jmac3vq3n7p1fg71suhwa1h1xadhual4&file_id=f_839327923696', 46 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=41271vrfzgzrtqjnsktb2fm7t9xdm68t&file_id=f_839327603328', 47 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=00abfapnqimrmk3q6l5sliqdgn3gaug1&file_id=f_839327514553', 48 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=v01hqc4jb3eghh3ht88lita0zpk72vka&file_id=f_839325009914', 49 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=l8e5yhz0vxl7u1g3mfyte4y8vye4sfpa&file_id=f_839324975094', 50 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=xnrfm1aoei3f61k3jq8w7l48s2w1reax&file_id=f_839327396585', 51 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=41f4jba1xn3dlokxaaz8aijw4e76700u&file_id=f_839326174324', 52 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=46akcl4psp9j5szsgn0ch5h14q31wcki&file_id=f_839324672130', 53 | 'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=o3osn2appwflfqqz6k9lq1f31337bhmz&file_id=f_839326678905' 54 | ] 55 | 56 | expected_results = [ 57 | 'BUILDING A BETTER VIDEO PLAYER', 58 | '4.1 IONIC BONDING (CONT. FROM CH. 03)', 59 | '2020 ASEE: How Introduction of ClassTranscribe is Changing Engineering Education at the University of Illinois', 60 | 'Mining extra slide from the so called thing which is in Quantitative Association kill Predicate Sets Mining extraordinary Redundancy Filtering', 61 | '', 62 | 'Gamma Correction in sRGB', 63 | 'Data Visualization and Storytelling', 64 | '2019 ClassTranscribe...', 65 | 'Nav1.1 Mutations and Epilepsy', 66 | 'Title Detection For Videos' 67 | ] 68 | 69 | for i in range(len(urls)): 70 | test_scheme(image_names[i], urls[i], expected_results[i]) 71 | 72 | if __name__ == '__main__': 73 | run_titledetector_tests(); 74 | print('done'); -------------------------------------------------------------------------------- /ClassTranscribeServer/Controllers/TermsController.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase; 2 | using ClassTranscribeDatabase.Models; 3 | using Microsoft.AspNetCore.Authorization; 4 | using Microsoft.AspNetCore.Mvc; 5 | using Microsoft.EntityFrameworkCore; 6 | using Microsoft.Extensions.Logging; 7 | using System.Collections.Generic; 8 | using System.Linq; 9 | using System.Threading.Tasks; 10 | 11 | namespace ClassTranscribeServer.Controllers 12 | { 13 | [Route("api/[controller]")] 14 | [ApiController] 15 | public class TermsController : BaseController 16 | { 17 | public TermsController(CTDbContext context, ILogger logger) : base(context, logger) { } 18 | 19 | /// 20 | /// Gets all Terms for universityId 21 | /// 22 | [HttpGet("ByUniversity/{universityId}")] 23 | public async Task>> GetTerms(string universityId) 24 | { 25 | return await _context.Terms.Where(t => t.UniversityId == universityId).OrderBy(t => t.StartDate).ToListAsync(); 26 | } 27 | 28 | // GET: api/Terms/5 29 | [HttpGet("{id}")] 30 | public async Task> GetTerm(string id) 31 | { 32 | var term = await _context.Terms.FindAsync(id); 33 | 34 | if (term == null) 35 | { 36 | return NotFound(); 37 | } 38 | 39 | return term; 40 | } 41 | 42 | // PUT: api/Terms/5 43 | [HttpPut("{id}")] 44 | [Authorize(Roles = Globals.ROLE_ADMIN)] 45 | public async Task PutTerm(string id, Term term) 46 | { 47 | if (term == null || id == null || id != term.Id) 48 | { 49 | return BadRequest(); 50 | } 51 | 52 | _context.Entry(term).State = EntityState.Modified; 53 | 54 | try 55 | { 56 | await _context.SaveChangesAsync(); 57 | } 58 | catch (DbUpdateConcurrencyException) 59 | { 60 | if (!TermExists(id)) 61 | { 62 | return NotFound(); 63 | } 64 | else 65 | { 66 | throw; 67 | } 68 | } 69 | 70 | return NoContent(); 71 | } 72 | 73 | // POST: api/Terms 74 | [HttpPost] 75 | [Authorize(Roles = Globals.ROLE_ADMIN)] 76 | public async Task> PostTerm(Term term) 77 | { 78 | if (term == null) 79 | { 80 | return BadRequest(); 81 | } 82 | 83 | _context.Terms.Add(term); 84 | await _context.SaveChangesAsync(); 85 | 86 | return CreatedAtAction("GetTerm", new { id = term.Id }, term); 87 | } 88 | 89 | // DELETE: api/Terms/5 90 | [HttpDelete("{id}")] 91 | [Authorize(Roles = Globals.ROLE_ADMIN)] 92 | public async Task> DeleteTerm(string id) 93 | { 94 | var term = await _context.Terms.FindAsync(id); 95 | if (term == null) 96 | { 97 | return NotFound(); 98 | } 99 | 100 | _context.Terms.Remove(term); 101 | await _context.SaveChangesAsync(); 102 | 103 | return term; 104 | } 105 | 106 | private bool TermExists(string id) 107 | { 108 | return _context.Terms.Any(e => e.Id == id); 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /ClassTranscribeDatabase/Services/Notification.cs: -------------------------------------------------------------------------------- 1 | using ClassTranscribeDatabase.Models; 2 | using Microsoft.EntityFrameworkCore; 3 | using Microsoft.Extensions.Logging; 4 | using Newtonsoft.Json.Linq; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Threading.Tasks; 8 | 9 | namespace ClassTranscribeDatabase.Services 10 | { 11 | public class SubscriptionManager 12 | { 13 | private readonly CTDbContext _context; 14 | 15 | public SubscriptionManager(CTDbContext context) 16 | { 17 | _context = context; 18 | } 19 | 20 | public async Task Subscribe(Entity entity, ApplicationUser user) 21 | { 22 | Subscription subscription = new Subscription 23 | { ApplicationUserId = user.Id, 24 | ResourceType = entity.GetResourceType(), 25 | ResourceId = entity.Id 26 | }; 27 | if(!(await _context.Subscriptions.AnyAsync(s => s.ResourceType == subscription.ResourceType && 28 | s.ResourceId == subscription.ResourceId && 29 | s.ApplicationUserId == subscription.ApplicationUserId))) 30 | { 31 | _context.Subscriptions.Add(subscription); 32 | } 33 | await _context.SaveChangesAsync(); 34 | } 35 | 36 | public async Task Unsubscribe(Entity entity, ApplicationUser user) 37 | { 38 | Subscription subscription = new Subscription 39 | { 40 | ApplicationUserId = user.Id, 41 | ResourceType = entity.GetResourceType(), 42 | ResourceId = entity.Id 43 | }; 44 | if (await _context.Subscriptions.AnyAsync(s => s.ResourceType == subscription.ResourceType && 45 | s.ResourceId == subscription.ResourceId && 46 | s.ApplicationUserId == subscription.ApplicationUserId)) 47 | { 48 | _context.Subscriptions.Remove(subscription); 49 | } 50 | await _context.SaveChangesAsync(); 51 | } 52 | 53 | public async Task AddMessageToSubscription(Entity entity, JObject payload, LogLevel logLevel) 54 | { 55 | var userIds = await _context.Subscriptions 56 | .Where(s => s.ResourceType == entity.GetResourceType() && s.Id == entity.Id) 57 | .Select(s => s.ApplicationUserId) 58 | .ToListAsync(); 59 | 60 | var messages = userIds.Select(uId => new Message 61 | { 62 | LogLevel = logLevel, 63 | ApplicationUserId = uId, 64 | Payload = payload ?? new JObject() 65 | }).ToList(); 66 | 67 | await _context.Messages.AddRangeAsync(messages); 68 | } 69 | 70 | public async Task> GetNotifications(ApplicationUser user) 71 | { 72 | return await _context.Messages 73 | .Where(m => m.Ack == Ack.Pending && m.ApplicationUserId == user.Id) 74 | .ToListAsync(); 75 | } 76 | 77 | public async Task AcknowledgeMessage(string messageId) 78 | { 79 | Message m = await _context.Messages.Where(m => m.Id == messageId).FirstOrDefaultAsync(); 80 | m.Ack = Ack.Seen; 81 | await _context.SaveChangesAsync(); 82 | } 83 | 84 | public async Task AcknowledgeMessages(List messageIds) 85 | { 86 | List messages = await _context.Messages.Where(m => messageIds.Contains(m.Id)).ToListAsync(); 87 | messages.ForEach(m => 88 | { 89 | m.Ack = Ack.Seen; 90 | }); 91 | await _context.SaveChangesAsync(); 92 | } 93 | } 94 | } 95 | --------------------------------------------------------------------------------