├── .github └── workflows │ └── build.yml ├── .gitignore ├── LICENSE ├── README.md ├── SharpGrabber.sln ├── assets ├── SharpGrabberDesktop-ScreenShot-4.0-2.png ├── blackwidow-logo-text-sm.png ├── blackwidow-logo-text.png ├── blackwidow-logo-text.psd ├── blackwidow-logo.png ├── blackwidow-logo.psd └── icon.png ├── blackwidow ├── README.md ├── repo │ ├── feed.json │ └── scripts │ │ ├── pornhub.js │ │ └── vimeo.js └── schema │ └── feed.json ├── src ├── SharpGrabber.Adult │ ├── AdultGrabberBuilderExtensions.cs │ ├── Internal │ │ ├── JsonHelper.cs │ │ └── StringHelper.cs │ ├── PornHubGrabber.cs │ ├── SharpGrabber.Adult.csproj │ ├── XVideosGrabber.cs │ └── XnxxGrabber.cs ├── SharpGrabber.BlackWidow │ ├── BlackWidowConstants.cs │ ├── BlackWidowInitializer.cs │ ├── BlackWidowService.cs │ ├── Builder │ │ ├── BlackWidowBuilder.cs │ │ ├── BlackWidowRepositoryConfigurator.cs │ │ ├── BuilderExtensions.cs │ │ ├── GrabberScriptInterpreterActivationContext.cs │ │ ├── GrabberScriptInterpreterApiServiceActivationContext.cs │ │ ├── GrabberScriptInterpreterServiceConfigurator.cs │ │ ├── IBlackWidowBuilder.cs │ │ ├── IBlackWidowRepositoryConfigurator.cs │ │ └── IGrabberScriptInterpreterServiceConfigurator.cs │ ├── Definitions │ │ ├── GrabberScriptSource.cs │ │ ├── GrabberScriptType.cs │ │ ├── GrabberScriptTypeAttribute.cs │ │ └── IGrabberScriptSource.cs │ ├── Exceptions │ │ ├── BlackWidowException.cs │ │ ├── ScriptApiVersionMismatchException.cs │ │ └── ScriptInterpretException.cs │ ├── Host │ │ ├── ConsoleLog.cs │ │ ├── ConsoleLogLevel.cs │ │ ├── IScriptHost.cs │ │ └── ScriptHost.cs │ ├── IBlackWidowGrabber.cs │ ├── IBlackWidowService.cs │ ├── Internal │ │ ├── ConcurrentHashSet.cs │ │ ├── EqualityUtils.cs │ │ ├── HashCodeUtils.cs │ │ ├── StringExtensions.cs │ │ └── TypeExtensions.cs │ ├── Interpreter │ │ ├── Api │ │ │ ├── DefaultInterpreterApiService.cs │ │ │ ├── Delegates.cs │ │ │ ├── IInterpreterApiService.cs │ │ │ ├── ProcessedGrabScript.cs │ │ │ └── v1 │ │ │ │ ├── ApiGrabRequest.cs │ │ │ │ ├── ApiGrabResponse.cs │ │ │ │ ├── ApiGrabberContext.cs │ │ │ │ ├── ApiHostObject.cs │ │ │ │ ├── ApiMimeContext.cs │ │ │ │ ├── Delegates.cs │ │ │ │ ├── Html │ │ │ │ ├── ApiHtmlContext.cs │ │ │ │ └── ApiHtmlElement.cs │ │ │ │ └── Http │ │ │ │ ├── ApiHttpClient.cs │ │ │ │ ├── ApiHttpContext.cs │ │ │ │ ├── ApiHttpRequest.cs │ │ │ │ └── ApiHttpResponse.cs │ │ ├── GrabberScriptInterpretOptions.cs │ │ ├── GrabberScriptInterpreterService.cs │ │ ├── IGrabberScriptInterpreter.cs │ │ ├── IGrabberScriptInterpreterService.cs │ │ └── JavaScript │ │ │ ├── Conversion │ │ │ ├── JintConvertExTypeConverter.cs │ │ │ └── JintMultiTypeConverter.cs │ │ │ ├── JintInterpreterServiceExtensions.cs │ │ │ ├── JintJavaScriptHost.cs │ │ │ └── JintJavaScriptInterpreter.cs │ ├── Repository │ │ ├── GitHub │ │ │ └── GitHubGrabberRepository.cs │ │ ├── GrabberRepositoryBase.cs │ │ ├── GrabberRepositoryChangeDetector.cs │ │ ├── GrabberRepositoryChangeEventHandler.cs │ │ ├── GrabberRepositoryConfigurationExtensions.cs │ │ ├── GrabberRepositoryFeed.cs │ │ ├── GrabberRepositoryFeedDifference.cs │ │ ├── GrabberRepositoryScript.cs │ │ ├── IGrabberRepository.cs │ │ ├── IGrabberRepositoryChangeDetector.cs │ │ ├── IGrabberRepositoryFeed.cs │ │ ├── IGrabberRepositoryScript.cs │ │ ├── IGrabberRepositorySubscription.cs │ │ ├── Local │ │ │ └── PhysicalGrabberRepository.cs │ │ ├── Memory │ │ │ └── InMemoryRepository.cs │ │ └── OfficialGrabberRepository.cs │ ├── SharpGrabber.BlackWidow.csproj │ ├── SharpGrabber.BlackWidow.csproj.DotSettings │ └── TypeConversion │ │ ├── ToBoolConverter.cs │ │ └── TypeConverters.cs ├── SharpGrabber.Converter │ ├── FFMpegHelper.cs │ ├── IOContext.cs │ ├── MediaConcatenator.cs │ ├── MediaDecoder.cs │ ├── MediaFrame.cs │ ├── MediaHelper.cs │ ├── MediaLibrary.cs │ ├── MediaMerger.cs │ ├── MediaMuxer.cs │ ├── MediaPacket.cs │ ├── MediaStreamSource.cs │ ├── Reference.cs │ ├── SharpGrabber.Converter.csproj │ └── VideoFrameConverter.cs ├── SharpGrabber.Desktop │ ├── App.xaml │ ├── App.xaml.cs │ ├── Auth │ │ └── InstagramAuthenticationHandlerInterface.cs │ ├── Components │ │ ├── LoadingSpinner.xaml │ │ ├── LoadingSpinner.xaml.cs │ │ ├── MediaResourceView.xaml │ │ ├── MediaResourceView.xaml.cs │ │ ├── StreamReferenceView.xaml │ │ ├── StreamReferenceView.xaml.cs │ │ ├── StreamResourceView.xaml │ │ └── StreamResourceView.xaml.cs │ ├── Constants.cs │ ├── ConvertHelper.cs │ ├── Downloading │ │ ├── MediaDownloader.cs │ │ └── StreamDownloader.cs │ ├── IOHelper.cs │ ├── MainWindow.xaml │ ├── MainWindow.xaml.cs │ ├── Program.cs │ ├── SharpGrabber.Desktop.csproj │ ├── UI │ │ ├── ButtonStyle.xaml │ │ ├── Icons.xaml │ │ ├── StringHelpers.cs │ │ └── UIHelpers.cs │ ├── Utils │ │ └── ExceptionHelpers.cs │ └── ViewModel │ │ ├── BaseViewModel.cs │ │ ├── GrabbedMediaViewModel.cs │ │ ├── GrabbedStreamRefViewModel.cs │ │ └── GrabbedStreamViewModel.cs ├── SharpGrabber.Hls │ ├── AssemblyInitializer.cs │ ├── Def │ │ ├── HlsKey.cs │ │ ├── HlsKeyMethod.cs │ │ ├── HlsMediaSegment.cs │ │ ├── HlsPlaylistType.cs │ │ ├── HlsStreamInfo.cs │ │ ├── PlaylistDocument.cs │ │ ├── PlaylistDocumentLoadException.cs │ │ ├── PlaylistTagValue.cs │ │ ├── PlaylistToken.cs │ │ ├── PlaylistTokenType.cs │ │ └── PlaylistTokenizer.cs │ ├── Grabbed │ │ ├── GrabbedHlsStreamMetadata.cs │ │ └── GrabbedHlsStreamReference.cs │ ├── HlsGrabber.cs │ ├── HlsGrabberBuilderExtensions.cs │ ├── Internal │ │ ├── HlsAes128Decryptor.cs │ │ └── ListWrapper.cs │ └── SharpGrabber.Hls.csproj ├── SharpGrabber.Instagram │ ├── IInstagramAuthenticationInterface.cs │ ├── InstagramAuthenticationHandler.cs │ ├── InstagramAuthenticationHandlerExtensions.cs │ ├── InstagramAuthenticationRequestState.cs │ ├── InstagramGrabber.cs │ ├── InstagramGrabberBuilderExtensions.cs │ └── SharpGrabber.Instagram.csproj ├── SharpGrabber.Odysee │ ├── OdyseeGrabber.cs │ ├── OdyseeGrabberBuilderExtensions.cs │ └── SharpGrabber.Odysee.csproj ├── SharpGrabber.Vimeo │ ├── Internal │ │ ├── VimeoConfiguration.cs │ │ ├── VimeoFile.cs │ │ ├── VimeoProgressive.cs │ │ ├── VimeoRequest.cs │ │ └── VimeoVideoInfo.cs │ ├── SharpGrabber.Vimeo.csproj │ ├── VimeoGrabber.cs │ └── VimeoGrabberBuilderExtensions.cs ├── SharpGrabber.YouTube │ ├── SharpGrabber.YouTube.csproj │ ├── StringHelper.cs │ ├── YouTubeAdaptiveStream.cs │ ├── YouTubeGrabber.cs │ ├── YouTubeGrabberBase.cs │ ├── YouTubeGrabberBuilderExtensions.cs │ ├── YouTubeImageType.cs │ ├── YouTubeMetadata.cs │ ├── YouTubeMuxedStream.cs │ ├── YouTubePageInfo.cs │ ├── YouTubePlayerResponse.cs │ ├── YouTubeScript.cs │ ├── YouTubeStreamInfo.cs │ ├── YouTubeTagInfo.cs │ ├── YouTubeUtils.cs │ └── YouTubeWatchPageData.cs └── SharpGrabber │ ├── Auth │ ├── FileAuthenticationStore.cs │ ├── GrabberAuthenticationRequest.cs │ ├── GrabberAuthenticationResult.cs │ ├── GrabberAuthenticationService.cs │ ├── GrabberBasicCredentials.cs │ ├── IGrabberAuthenticationHandler.cs │ ├── IGrabberAuthenticationService.cs │ ├── IGrabberAuthenticationStore.cs │ └── InMemoryAuthenticationStore.cs │ ├── Builder │ ├── GrabberAuthenticationServiceBuilder.cs │ ├── GrabberBuilder.cs │ ├── GrabberServicesBuilder.cs │ ├── IGrabberAuthenticationServiceBuilder.cs │ ├── IGrabberBuilder.cs │ └── IGrabberServicesBuilder.cs │ ├── Exceptions │ ├── GrabAuthenticationException.cs │ ├── GrabException.cs │ ├── GrabParseException.cs │ ├── SharpGrabberException.cs │ └── UnsupportedGrabException.cs │ ├── Extensions │ ├── GrabResultExtensions.cs │ └── GrabbedMediaExtensions.cs │ ├── GrabOptions.cs │ ├── GrabResult.cs │ ├── Grabbed │ ├── GrabbedComment.cs │ ├── GrabbedComments.cs │ ├── GrabbedHlsStream.cs │ ├── GrabbedImage.cs │ ├── GrabbedImageSize.cs │ ├── GrabbedImageType.cs │ ├── GrabbedInfo.cs │ ├── GrabbedMedia.cs │ ├── GrabbedTypeAttribute.cs │ ├── MediaChannels.cs │ ├── MediaFormat.cs │ └── MediaSegment.cs │ ├── GrabbedResourceBase.cs │ ├── GrabbedTypeCollection.cs │ ├── GrabberBase.cs │ ├── GrabberServices.cs │ ├── GrabberServicesAssemblyRegistry.cs │ ├── IGrabResult.cs │ ├── IGrabbed.cs │ ├── IGrabbedResource.cs │ ├── IGrabbedTypeCollection.cs │ ├── IGrabber.cs │ ├── IGrabberServices.cs │ ├── IMultiGrabber.cs │ ├── Internal │ ├── ConcurrentHashSet.cs │ ├── DefaultGlobalHttpProvider.cs │ ├── DictionaryHelper.cs │ ├── StringHelper.cs │ └── WorkStatus.cs │ ├── Mime │ ├── DefaultMimeService.cs │ ├── IMimeService.cs │ ├── MimeInfo.cs │ └── MimeType.cs │ ├── MultiGrabber.cs │ ├── MultiGrabberBase.cs │ ├── RectSize.cs │ ├── SharpGrabber.csproj │ ├── State │ ├── GrabberStateBag.cs │ └── IGrabberStateBag.cs │ ├── StreamWrappingDelegate.cs │ └── Utils │ ├── ResolutionUtils.cs │ └── VideoResolutionDescriptor.cs └── tests ├── DotNetTools.SharpGrabber.Tests ├── GrabTests.cs ├── MultiGrabberTests.cs └── SharpGrabber.Tests.csproj ├── SharpGrabber.BlackWidow.Tests ├── Assets │ ├── AssetsAccessor.cs │ └── Html │ │ └── SamplePage.html ├── BlackWidowServiceTests.cs ├── Interpreter │ └── JavaScript │ │ ├── JintGrabberTests.cs │ │ ├── JintMultiTypeConverterTests.cs │ │ └── JintUrlTests.cs ├── Repository │ ├── ChangeDetectorTests.cs │ ├── GetDifferencesTests.cs │ ├── GitHubRepositoryTests.cs │ └── LocalRepositoryTests.cs └── SharpGrabber.BlackWidow.Tests.csproj ├── SharpGrabber.Tests └── SharpGrabber.Tests │ ├── Adult │ └── PornHubTests.cs │ ├── Odysee │ └── OdyseeTests.cs │ ├── SharpGrabber.Tests.csproj │ └── YouTube │ └── YouTubeTests.cs └── assets └── blackwidow └── repo ├── emptyfeed.json ├── feed.json └── scripts └── youtube.js /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout source code 16 | uses: actions/checkout@v3 17 | - name: Setup .NET 18 | uses: actions/setup-dotnet@v2 19 | with: 20 | dotnet-version: 3.1.x 21 | - name: Restore dependencies 22 | run: dotnet restore 23 | - name: Build 24 | run: dotnet build --no-restore 25 | - name: Test 26 | run: dotnet test --no-build --verbosity normal 27 | -------------------------------------------------------------------------------- /assets/SharpGrabberDesktop-ScreenShot-4.0-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnettools/SharpGrabber/96743e630a4ff37f39ed3f11cabb0b5c773d058a/assets/SharpGrabberDesktop-ScreenShot-4.0-2.png -------------------------------------------------------------------------------- /assets/blackwidow-logo-text-sm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnettools/SharpGrabber/96743e630a4ff37f39ed3f11cabb0b5c773d058a/assets/blackwidow-logo-text-sm.png -------------------------------------------------------------------------------- /assets/blackwidow-logo-text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnettools/SharpGrabber/96743e630a4ff37f39ed3f11cabb0b5c773d058a/assets/blackwidow-logo-text.png -------------------------------------------------------------------------------- /assets/blackwidow-logo-text.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnettools/SharpGrabber/96743e630a4ff37f39ed3f11cabb0b5c773d058a/assets/blackwidow-logo-text.psd -------------------------------------------------------------------------------- /assets/blackwidow-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnettools/SharpGrabber/96743e630a4ff37f39ed3f11cabb0b5c773d058a/assets/blackwidow-logo.png -------------------------------------------------------------------------------- /assets/blackwidow-logo.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnettools/SharpGrabber/96743e630a4ff37f39ed3f11cabb0b5c773d058a/assets/blackwidow-logo.psd -------------------------------------------------------------------------------- /assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnettools/SharpGrabber/96743e630a4ff37f39ed3f11cabb0b5c773d058a/assets/icon.png -------------------------------------------------------------------------------- /blackwidow/README.md: -------------------------------------------------------------------------------- 1 | SharpGrabber 2 | 3 | # BlackWidow 4 | 5 | BlackWidow is a .NET library based on SharpGrabber. Rather than relying on .NET assemblies, BlackWidow executes scripts written specifically for grabbing. 6 | 7 | ## Why use BlackWidow? 8 | BlackWidow gives you the following advantages over the traditional NuGet package approach: 9 | 10 | - **Always Up-to-date:** The scripts are always kept up-to-date at runtime; so the functionality of the host application won't break as the sources change - at least not for long! 11 | - **ECMAScript Support:** Supports JavaScript/ECMAScript out of the box. 12 | - **Easy Maintenance:** *JavaScript* is quick to code and welcomes many developers. This helps contributors to quickly write new grabbers or fix the existing ones. 13 | - **Secure**: The scripts are executed in a sandbox environment, and they only have access to what the BlackWidow API exposes to them. 14 | - **Highly Customizable:** Almost everything is open for extension or replacement. Make new script interpreters, custom grabber repositories, or roll out your own interpreter APIs 15 | 16 | ## How does it work? 17 | 18 | BlackWidow keeps a collection of scripts locally - called the local repository. 19 | Each script gets interpreted as an object implementing `IGrabber`. 20 | To keep the scripts up-to-date, a remote repository is constantly monitored as the single source of truth. 21 | 22 | **🗐 Installation / Documentation** 23 | 24 |
25 | <- Back to Home Page 26 | -------------------------------------------------------------------------------- /blackwidow/repo/feed.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": [ 3 | { 4 | "id": "vimeo.com", 5 | "name": "Vimeo", 6 | "version": "1.0", 7 | "type": "JavaScript", 8 | "apiVersion": 1, 9 | "supportedRegularExpressions": [ "^https?://(www\\.|player\\.)?vimeo\\.com/(video/)?([0-9]+)" ], 10 | "file": "scripts/vimeo.js" 11 | }, 12 | { 13 | "id": "pornhub.com", 14 | "name": "PornHub", 15 | "version": "1.0", 16 | "type": "JavaScript", 17 | "apiVersion": 1, 18 | "supportedRegularExpressions": [ "^(https?:\\/\\/)?(www\\.)?pornhub\\.com\\/([^\\/]+)viewkey=(\\w+).*$" ], 19 | "file": "scripts/pornhub.js" 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /src/SharpGrabber.Adult/AdultGrabberBuilderExtensions.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.Adult; 2 | 3 | namespace DotNetTools.SharpGrabber 4 | { 5 | /// 6 | /// Provides extension methods for . 7 | /// 8 | public static class AdultGrabberBuilderExtensions 9 | { 10 | /// 11 | /// Includes the PornHub grabber. 12 | /// 13 | public static IGrabberBuilder AddPornHub(this IGrabberBuilder builder) 14 | => builder.Add(); 15 | 16 | /// 17 | /// Includes the XNXX grabber. 18 | /// 19 | public static IGrabberBuilder AddXnxx(this IGrabberBuilder builder) 20 | => builder.Add(); 21 | 22 | /// 23 | /// Includes the xvideos grabber. 24 | /// 25 | public static IGrabberBuilder AddXVideos(this IGrabberBuilder builder) 26 | => builder.Add(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/SharpGrabber.Adult/Internal/JsonHelper.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | 3 | namespace DotNetTools.SharpGrabber.Adult.Internal 4 | { 5 | internal static class JsonHelper 6 | { 7 | /// 8 | /// Adopted from https://stackoverflow.com/a/51428508/492352 with minor modifications 9 | /// 10 | public static bool TryParseJson(string text, out T result) 11 | { 12 | bool success = true; 13 | var settings = new JsonSerializerSettings 14 | { 15 | Error = (sender, args) => { success = false; args.ErrorContext.Handled = true; }, 16 | MissingMemberHandling = MissingMemberHandling.Error 17 | }; 18 | result = JsonConvert.DeserializeObject(text, settings); 19 | return success; 20 | } 21 | 22 | public static bool TryParseJson(string text, out object result) 23 | { 24 | bool success = true; 25 | var settings = new JsonSerializerSettings 26 | { 27 | Error = (sender, args) => { success = false; args.ErrorContext.Handled = true; }, 28 | MissingMemberHandling = MissingMemberHandling.Error 29 | }; 30 | result = JsonConvert.DeserializeObject(text, settings); 31 | return success; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/SharpGrabber.Adult/Internal/StringHelper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetTools.SharpGrabber.Internal 4 | { 5 | internal static class StringHelper 6 | { 7 | public static int ForceParseInt(string s) 8 | { 9 | var num = 0; 10 | foreach (var ch in s) 11 | { 12 | if (!char.IsDigit(ch)) 13 | continue; 14 | var digit = (int)char.GetNumericValue(ch); 15 | num = num * 10 + digit; 16 | } 17 | return num; 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/SharpGrabber.Adult/SharpGrabber.Adult.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.0 5 | DotNetTools.SharpGrabber.Adult 6 | true 7 | Javid Shoaei 8 | github.com/dotnettools 9 | Adds support for grabbing information and media from top adult services. 10 | LICENSE 11 | https://github.com/dotnettools/SharpGrabber 12 | icon.png 13 | 14 | 9 15 | https://github.com/dotnettools/SharpGrabber 16 | adult,porn,video,download,grab,scrape,pornhub,xvideos,xnxx 17 | 1701;1702;1591 18 | 1.0.2 19 | AnyCPU;x64 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | True 30 | 31 | 32 | 33 | True 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /src/SharpGrabber.Adult/XVideosGrabber.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text.RegularExpressions; 3 | 4 | namespace DotNetTools.SharpGrabber.Adult 5 | { 6 | public class XVideosGrabber : XnxxGrabber 7 | { 8 | private static readonly Regex HostRegex = new(@"^(https?://)?(www\.)?xvideos.com/video([^/]+)/", RegexOptions.Compiled | RegexOptions.IgnoreCase); 9 | 10 | public XVideosGrabber(IGrabberServices services) : base(services) 11 | { 12 | } 13 | 14 | public override string StringId { get; } = "xvideos.com"; 15 | 16 | public override string Name { get; } = "xvideos"; 17 | 18 | public override bool Supports(Uri uri) 19 | { 20 | return GetVideoId(uri.ToString()) != null; 21 | } 22 | 23 | public static new string GetVideoId(string url) 24 | { 25 | var match = HostRegex.Match(url); 26 | if (match.Success) 27 | return match.Groups[3].Value; 28 | return null; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/BlackWidowConstants.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow 6 | { 7 | /// 8 | /// Defines BlackWidow-related constants. 9 | /// 10 | public static class BlackWidowConstants 11 | { 12 | public static class GitHub 13 | { 14 | public static class OfficialRepository 15 | { 16 | /// 17 | /// The offical repository name 18 | /// 19 | public const string RepositoryAddress = "dotnettools/SharpGrabber"; 20 | 21 | /// 22 | /// Name of the main branch 23 | /// 24 | public const string MasterBranch = "master"; 25 | 26 | /// 27 | /// Path to the directory that contains the feed file and the scripts 28 | /// 29 | public const string RootPath = "blackwidow/repo"; 30 | 31 | /// 32 | /// Name of the feed JSON file 33 | /// 34 | public const string FeedFileName = "feed.json"; 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/BlackWidowInitializer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Reflection; 4 | using System.Text; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow 7 | { 8 | internal static class BlackWidowInitializer 9 | { 10 | static BlackWidowInitializer() 11 | { 12 | EnsureLoaded(Hls.HlsGrabber.Initializer); 13 | } 14 | 15 | public static void Test() 16 | { 17 | // nothing should be done here, 18 | // the static constructor would run once. 19 | } 20 | 21 | private static void EnsureLoaded(params Type[] types) 22 | { 23 | foreach (var type in types) 24 | { 25 | // create a dummy instance just to ensure the type is loaded 26 | var o = Activator.CreateInstance(type); 27 | if (o is IDisposable disposable) 28 | disposable.Dispose(); 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Builder/BlackWidowRepositoryConfigurator.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.BlackWidow.Repository; 2 | 3 | namespace DotNetTools.SharpGrabber.BlackWidow 4 | { 5 | /// 6 | /// Builds BlackWidow repositories. 7 | /// 8 | internal class BlackWidowRepositoryConfigurator : IBlackWidowRepositoryConfigurator 9 | { 10 | public IGrabberRepository Repository { get; private set; } 11 | 12 | public IBlackWidowRepositoryConfigurator Use(IGrabberRepository repository) 13 | { 14 | Repository?.Dispose(); 15 | Repository = repository; 16 | return this; 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Builder/BuilderExtensions.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.BlackWidow.Definitions; 2 | using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api; 3 | using DotNetTools.SharpGrabber.BlackWidow.Interpreter.JavaScript; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow 6 | { 7 | /// 8 | /// Defines extension methods for builder and configurator interfaces to work with built-in implementations. 9 | /// 10 | public static class BuilderExtensions 11 | { 12 | /// 13 | /// Registers Jint as the JavaScript interpreter. 14 | /// 15 | public static IGrabberScriptInterpreterServiceConfigurator AddJint(this IGrabberScriptInterpreterServiceConfigurator interpreterService) 16 | { 17 | return interpreterService.AddInterpreter(GrabberScriptType.JavaScript, context => 18 | { 19 | return new JintJavaScriptInterpreter(context.ApiService, context.GrabberServices, context.ScriptHost); 20 | }); 21 | } 22 | 23 | /// 24 | /// Configures to use the official API service. 25 | /// 26 | public static IGrabberScriptInterpreterServiceConfigurator SetDefaultApiService(this IGrabberScriptInterpreterServiceConfigurator interpreterService) 27 | { 28 | return interpreterService.SetApiService(context => new DefaultInterpreterApiService(context.GrabberServices, context.GrabbedTypes, context.TypeConverter)); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterActivationContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using DotNetTools.SharpGrabber.BlackWidow.Host; 3 | using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow 6 | { 7 | /// 8 | /// Provides references to services used when activating an interpreter. 9 | /// 10 | public class GrabberScriptInterpreterActivationContext 11 | { 12 | public GrabberScriptInterpreterActivationContext(IInterpreterApiService apiService, IGrabberServices grabberServices, IScriptHost scripHost) 13 | { 14 | ApiService = apiService ?? throw new ArgumentNullException(nameof(apiService)); 15 | GrabberServices = grabberServices ?? throw new ArgumentNullException(nameof(grabberServices)); 16 | ScriptHost = scripHost ?? throw new ArgumentNullException(nameof(scripHost)); 17 | } 18 | 19 | /// 20 | /// Gets the interpreter API service. 21 | /// 22 | public IInterpreterApiService ApiService { get; } 23 | 24 | /// 25 | /// Gets the grabber services. 26 | /// 27 | public IGrabberServices GrabberServices { get; } 28 | 29 | /// 30 | /// Gets the script host. 31 | /// 32 | public IScriptHost ScriptHost { get; } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterApiServiceActivationContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using DotNetTools.ConvertEx; 3 | using DotNetTools.SharpGrabber.BlackWidow.Host; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow 6 | { 7 | /// 8 | /// Provides references to services used when activating an interpreter API service. 9 | /// 10 | public class GrabberScriptInterpreterApiServiceActivationContext 11 | { 12 | public GrabberScriptInterpreterApiServiceActivationContext(IGrabberServices grabberServices, IScriptHost scriptHost, 13 | IGrabbedTypeCollection grabbedTypes, ITypeConverter typeConverter) 14 | { 15 | GrabberServices = grabberServices ?? throw new ArgumentNullException(nameof(grabberServices)); 16 | ScriptHost = scriptHost ?? throw new ArgumentNullException(nameof(scriptHost)); 17 | GrabbedTypes = grabbedTypes ?? throw new ArgumentNullException(nameof(grabbedTypes)); 18 | TypeConverter = typeConverter ?? throw new ArgumentNullException(nameof(typeConverter)); 19 | } 20 | 21 | /// 22 | /// Gets the grabber services. 23 | /// 24 | public IGrabberServices GrabberServices { get; } 25 | 26 | /// 27 | /// Gets the script host. 28 | /// 29 | public IScriptHost ScriptHost { get; } 30 | 31 | /// 32 | /// Gets the collection of grabbed types. 33 | /// 34 | public IGrabbedTypeCollection GrabbedTypes { get; } 35 | 36 | /// 37 | /// Gets the type converter. 38 | /// 39 | public ITypeConverter TypeConverter { get; } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Builder/IBlackWidowBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using DotNetTools.SharpGrabber.BlackWidow.Host; 4 | using DotNetTools.SharpGrabber.BlackWidow.Interpreter; 5 | using DotNetTools.SharpGrabber.BlackWidow.Repository; 6 | 7 | namespace DotNetTools.SharpGrabber.BlackWidow 8 | { 9 | /// 10 | /// Builds a . 11 | /// 12 | public interface IBlackWidowBuilder 13 | { 14 | /// 15 | /// Configures the local repository. 16 | /// 17 | IBlackWidowBuilder ConfigureLocalRepository(Action configure); 18 | 19 | /// 20 | /// Configures the remote repository. 21 | /// 22 | IBlackWidowBuilder ConfigureRemoteRepository(Action configure); 23 | 24 | /// 25 | /// Sets the grabber services. 26 | /// 27 | IBlackWidowBuilder SetGrabberServices(IGrabberServices grabberServices); 28 | 29 | /// 30 | /// Sets the script host. 31 | /// 32 | IBlackWidowBuilder SetScriptHost(IScriptHost scriptHost); 33 | 34 | /// 35 | /// Sets the change detector. 36 | /// 37 | IBlackWidowBuilder SetChangeDetector(IGrabberRepositoryChangeDetector changeDetector); 38 | 39 | /// 40 | /// Sets to be used. 41 | /// 42 | IBlackWidowBuilder UseInterpreterService(IGrabberScriptInterpreterService interpreterService); 43 | 44 | /// 45 | /// Configures the interpreter service. 46 | /// 47 | IBlackWidowBuilder ConfigureInterpreterService(Action configure); 48 | 49 | /// 50 | /// Builds the service. 51 | /// 52 | /// Thrown in case of misconfiguration. 53 | Task BuildAsync(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Builder/IBlackWidowRepositoryConfigurator.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.BlackWidow.Repository; 2 | 3 | namespace DotNetTools.SharpGrabber.BlackWidow 4 | { 5 | /// 6 | /// Configures a repository on a builder. 7 | /// 8 | public interface IBlackWidowRepositoryConfigurator 9 | { 10 | /// 11 | /// Gets the configured repository. 12 | /// 13 | IGrabberRepository Repository { get; } 14 | 15 | /// 16 | /// Uses a repository instance. 17 | /// 18 | IBlackWidowRepositoryConfigurator Use(IGrabberRepository repository); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Definitions/GrabberScriptSource.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Threading.Tasks; 3 | 4 | namespace DotNetTools.SharpGrabber.BlackWidow.Definitions 5 | { 6 | /// 7 | /// Default implementation for 8 | /// 9 | public class GrabberScriptSource : IGrabberScriptSource 10 | { 11 | /// 12 | /// Refers to a static empty source. 13 | /// 14 | public static readonly GrabberScriptSource Empty = new(string.Empty); 15 | 16 | private readonly string _source; 17 | 18 | public GrabberScriptSource(string source) 19 | { 20 | _source = source; 21 | } 22 | 23 | /// 24 | /// Creates a by reading all the source code from a file. 25 | /// 26 | public static GrabberScriptSource FromFile(string fileName) 27 | { 28 | var src = File.ReadAllText(fileName); 29 | return new GrabberScriptSource(src); 30 | } 31 | 32 | public string GetSource() 33 | => _source; 34 | 35 | public Task GetSourceAsync() 36 | => Task.FromResult(_source); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Definitions/GrabberScriptType.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Reflection; 3 | 4 | namespace DotNetTools.SharpGrabber.BlackWidow.Definitions 5 | { 6 | /// 7 | /// Defines all possible script types. 8 | /// 9 | public enum GrabberScriptType 10 | { 11 | /// 12 | /// ECMAScript 13 | /// 14 | [GrabberScriptType(FileExtension = "js")] 15 | JavaScript = 1, 16 | } 17 | 18 | public static class GrabberScriptTypeExtensions 19 | { 20 | /// 21 | /// Gets the associated with the value. 22 | /// 23 | public static GrabberScriptTypeAttribute GetScriptTypeAttribute(this GrabberScriptType value, bool orDefault = true) 24 | { 25 | GrabberScriptTypeAttribute GetDefault() 26 | => orDefault ? GrabberScriptTypeAttribute.Default : null; 27 | 28 | var enumType = typeof(GrabberScriptType); 29 | var member = enumType.GetMember(value.ToString()).FirstOrDefault(m => m.DeclaringType == enumType); 30 | return member.GetCustomAttribute() ?? GetDefault(); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Definitions/GrabberScriptTypeAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetTools.SharpGrabber.BlackWidow.Definitions 4 | { 5 | [AttributeUsage(AttributeTargets.Field)] 6 | public class GrabberScriptTypeAttribute : Attribute 7 | { 8 | /// 9 | /// Gets the default value. 10 | /// 11 | public static GrabberScriptTypeAttribute Default => new(); 12 | 13 | /// 14 | /// Gets or sets the file extension associated with this script type. 15 | /// 16 | public string FileExtension { get; set; } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Definitions/IGrabberScriptSource.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace DotNetTools.SharpGrabber.BlackWidow.Definitions 4 | { 5 | /// 6 | /// Provides access to the source of a grabber script. 7 | /// 8 | public interface IGrabberScriptSource 9 | { 10 | /// 11 | /// Gets the source code of the grabber script. 12 | /// 13 | string GetSource(); 14 | 15 | /// 16 | /// Gets the source code of the grabber script. 17 | /// 18 | Task GetSourceAsync(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Exceptions/BlackWidowException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Runtime.Serialization; 4 | using System.Text; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Exceptions 7 | { 8 | public class BlackWidowException : Exception 9 | { 10 | public BlackWidowException() 11 | { 12 | } 13 | 14 | public BlackWidowException(string message) : base(message) 15 | { 16 | } 17 | 18 | public BlackWidowException(string message, Exception innerException) : base(message, innerException) 19 | { 20 | } 21 | 22 | protected BlackWidowException(SerializationInfo info, StreamingContext context) : base(info, context) 23 | { 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Exceptions/ScriptApiVersionMismatchException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Runtime.Serialization; 4 | using System.Text; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Exceptions 7 | { 8 | public class ScriptApiVersionMismatchException : BlackWidowException 9 | { 10 | public ScriptApiVersionMismatchException() 11 | { 12 | } 13 | 14 | public ScriptApiVersionMismatchException(string message) : base(message) 15 | { 16 | } 17 | 18 | public ScriptApiVersionMismatchException(string message, Exception innerException) : base(message, innerException) 19 | { 20 | } 21 | 22 | protected ScriptApiVersionMismatchException(SerializationInfo info, StreamingContext context) : base(info, context) 23 | { 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Exceptions/ScriptInterpretException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Runtime.Serialization; 4 | using System.Text; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Exceptions 7 | { 8 | public class ScriptInterpretException : BlackWidowException 9 | { 10 | public ScriptInterpretException() : this("Script interpret error.") 11 | { 12 | } 13 | 14 | public ScriptInterpretException(string message) : base(message) 15 | { 16 | } 17 | 18 | public ScriptInterpretException(string message, Exception innerException) : base(message, innerException) 19 | { 20 | } 21 | 22 | protected ScriptInterpretException(SerializationInfo info, StreamingContext context) : base(info, context) 23 | { 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Host/ConsoleLog.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Host 6 | { 7 | /// 8 | /// Describes a log entry. 9 | /// 10 | public class ConsoleLog 11 | { 12 | public ConsoleLog(ConsoleLogLevel level, params object[] objects) 13 | { 14 | Level = level; 15 | Objects = objects; 16 | } 17 | 18 | /// 19 | /// Gets the level. 20 | /// 21 | public ConsoleLogLevel Level { get; } 22 | 23 | /// 24 | /// Gets the logged objects. 25 | /// 26 | public object[] Objects { get; } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Host/ConsoleLogLevel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Host 6 | { 7 | public enum ConsoleLogLevel 8 | { 9 | Log, 10 | Debug, 11 | Error, 12 | Info, 13 | Warning, 14 | Trace 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Host/IScriptHost.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Host 6 | { 7 | /// 8 | /// Defines handlers for various operations on the script host. 9 | /// 10 | public interface IScriptHost 11 | { 12 | void Alert(object input); 13 | 14 | void Log(ConsoleLog log); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Host/ScriptHost.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Host 6 | { 7 | /// 8 | /// Implements with events. 9 | /// 10 | public class ScriptHost : IScriptHost 11 | { 12 | public ScriptHost() 13 | { 14 | } 15 | 16 | public event Action OnAlert; 17 | public event Action OnLog; 18 | 19 | public void Alert(object input) 20 | { 21 | OnAlert?.Invoke(input); 22 | } 23 | 24 | public void Log(ConsoleLog log) 25 | { 26 | OnLog?.Invoke(log); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/IBlackWidowGrabber.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow 6 | { 7 | /// 8 | /// Represents a BlackWidow grabber. 9 | /// 10 | public interface IBlackWidowGrabber : IGrabber 11 | { 12 | /// 13 | /// Enumerates internal grabbers, each representing a single grabber script. 14 | /// 15 | IEnumerable GetScriptGrabbers(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Internal/EqualityUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Internal 6 | { 7 | internal static class EqualityUtils 8 | { 9 | public static bool Equals(object object1, object object2, params Func[] getValues) 10 | where T : class 11 | { 12 | // test references 13 | if (ReferenceEquals(object1, object2)) 14 | return true; 15 | 16 | // test null values 17 | if (object1 == null || object2 == null) 18 | return false; 19 | 20 | // test different types 21 | var o1 = object1 as T; 22 | var o2 = object2 as T; 23 | if (o1 != null ^ o2 != null) 24 | return false; 25 | if (o1 == null && o2 == null) 26 | throw new ArgumentException($"Invalid type argument: {typeof(T)}", nameof(T)); 27 | 28 | foreach (var getValue in getValues) 29 | { 30 | var val1 = getValue(o1); 31 | var val2 = getValue(o2); 32 | var areEqual = val1 == null ? val2 == null : val1.Equals(val2); 33 | if (!areEqual) 34 | return false; 35 | } 36 | 37 | return true; 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Internal/HashCodeUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Internal 6 | { 7 | internal static class HashCodeUtils 8 | { 9 | public static int Compute(params object[] values) 10 | { 11 | return ComputeCustom(values); 12 | } 13 | 14 | private static int ComputeCustom(object[] values) 15 | { 16 | unchecked 17 | { 18 | var hash = 17; 19 | foreach (var value in values) 20 | { 21 | var ohash = value?.GetHashCode() ?? 0; 22 | hash = hash * 23 + ohash; 23 | } 24 | return hash; 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Internal/StringExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Internal 6 | { 7 | internal static class StringExtension 8 | { 9 | public static string ToCamelCase(this string str) 10 | { 11 | if (string.IsNullOrEmpty(str)) 12 | return str; 13 | 14 | if (str.Length == 1) 15 | return str.ToLowerInvariant(); 16 | 17 | return char.ToLowerInvariant(str[0]) + str.Substring(1); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Internal/TypeExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Internal 6 | { 7 | internal static class TypeExtensions 8 | { 9 | public static bool IsNumericType(this object o) 10 | { 11 | switch (Type.GetTypeCode(o.GetType())) 12 | { 13 | case TypeCode.Byte: 14 | case TypeCode.SByte: 15 | case TypeCode.UInt16: 16 | case TypeCode.UInt32: 17 | case TypeCode.UInt64: 18 | case TypeCode.Int16: 19 | case TypeCode.Int32: 20 | case TypeCode.Int64: 21 | case TypeCode.Decimal: 22 | case TypeCode.Double: 23 | case TypeCode.Single: 24 | return true; 25 | default: 26 | return false; 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/Delegates.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | using System.Threading; 6 | 7 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api 8 | { 9 | public delegate bool SupportsDelegate(Uri uri); 10 | 11 | public delegate Task GrabDelegate(Uri uri, CancellationToken cancellationToken, GrabOptions options, IProgress progress); 12 | } 13 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/IInterpreterApiService.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.ConvertEx; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api 7 | { 8 | /// 9 | /// Interpreter API service 10 | /// 11 | public interface IInterpreterApiService 12 | { 13 | /// 14 | /// Gets the host object, which will be exposed to the script. 15 | /// 16 | object GetHostObject(int apiVersion, IGrabberServices grabberServices); 17 | 18 | /// 19 | /// Processes the result of the call by processing the . 20 | /// 21 | ProcessedGrabScript ProcessResult(int apiVersion, object hostObject); 22 | 23 | /// 24 | /// Gets the proper type converter that should be used for the specified API version. 25 | /// 26 | ITypeConverter GetTypeConverter(int apiVersion); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/ProcessedGrabScript.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api 6 | { 7 | public class ProcessedGrabScript 8 | { 9 | public ProcessedGrabScript(SupportsDelegate supports, GrabDelegate grabAsync) 10 | { 11 | Supports = supports; 12 | GrabAsync = grabAsync; 13 | } 14 | 15 | public SupportsDelegate Supports { get; } 16 | 17 | public GrabDelegate GrabAsync { get; } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/ApiGrabRequest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1 7 | { 8 | public class ApiGrabRequest 9 | { 10 | private readonly Uri _uri; 11 | private readonly CancellationToken _cancellationToken; 12 | private readonly GrabOptions _options; 13 | private readonly IProgress _progress; 14 | private double _progressValue; 15 | 16 | public ApiGrabRequest(Uri uri, CancellationToken cancellationToken, GrabOptions options, IProgress progress) 17 | { 18 | _uri = uri; 19 | _cancellationToken = cancellationToken; 20 | _options = options; 21 | _progress = progress; 22 | } 23 | 24 | public string Url => _uri.ToString(); 25 | 26 | public GrabOptions Options => _options; 27 | 28 | public bool IsCanceled => _cancellationToken.IsCancellationRequested; 29 | 30 | public double Progress 31 | { 32 | get => _progressValue; 33 | set => _progress.Report(_progressValue = value); 34 | } 35 | 36 | public void TestCanceled() 37 | => _cancellationToken.ThrowIfCancellationRequested(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/ApiGrabberContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1 7 | { 8 | public class ApiGrabberContext 9 | { 10 | public GrabberSupportsDelegate Supports { get; set; } 11 | 12 | public GrabberGrabDelegate Grab { get; set; } 13 | 14 | public GrabberGrabAsyncDelegate GrabAsync { get; set; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/ApiHostObject.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1.Html; 2 | using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1.Http; 3 | using DotNetTools.SharpGrabber.Exceptions; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1 9 | { 10 | public class ApiHostObject 11 | { 12 | public ApiHostObject(IGrabberServices grabberServices) 13 | { 14 | Http = new ApiHttpContext(grabberServices); 15 | Mime = new ApiMimeContext(grabberServices.Mime); 16 | } 17 | 18 | public ApiGrabberContext Grabber { get; } = new ApiGrabberContext(); 19 | 20 | public ApiHttpContext Http { get; } 21 | 22 | public ApiHtmlContext Html { get; } = new ApiHtmlContext(); 23 | 24 | public ApiMimeContext Mime { get; } 25 | } 26 | } -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/ApiMimeContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1 4 | { 5 | public class ApiMimeContext 6 | { 7 | private readonly IMimeService _mimeService; 8 | 9 | public ApiMimeContext(IMimeService mimeService) 10 | { 11 | _mimeService = mimeService; 12 | } 13 | 14 | public string GetExtension(string mime) 15 | { 16 | return _mimeService.ExtractMimeExtension(mime); 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/Delegates.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1 7 | { 8 | public delegate bool GrabberSupportsDelegate(string url); 9 | 10 | public delegate bool GrabberGrabDelegate(ApiGrabRequest request, ApiGrabResponse response); 11 | 12 | public delegate Task GrabberGrabAsyncDelegate(ApiGrabRequest request, ApiGrabResponse response); 13 | } 14 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/Html/ApiHtmlContext.cs: -------------------------------------------------------------------------------- 1 | using AngleSharp.Html.Parser; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Web; 6 | 7 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1.Html 8 | { 9 | public class ApiHtmlContext 10 | { 11 | public ApiHtmlElement Parse(string source) 12 | { 13 | var parser = new HtmlParser(); 14 | var doc = parser.ParseDocument(source); 15 | return new ApiHtmlElement(doc.DocumentElement); 16 | } 17 | 18 | public string Encode(string str) 19 | { 20 | return HttpUtility.HtmlEncode(str); 21 | } 22 | 23 | public string Decode(string str) 24 | { 25 | return HttpUtility.HtmlDecode(str); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/Html/ApiHtmlElement.cs: -------------------------------------------------------------------------------- 1 | using AngleSharp.Common; 2 | using AngleSharp.Dom; 3 | using AngleSharp.Html.Dom; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Text; 8 | 9 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1.Html 10 | { 11 | public class ApiHtmlElement 12 | { 13 | private readonly IElement _element; 14 | private Dictionary _attributes; 15 | 16 | public ApiHtmlElement(IElement element) 17 | { 18 | _element = element; 19 | } 20 | 21 | public string TagName => _element.TagName; 22 | 23 | public string InnerHTML => _element.InnerHtml; 24 | 25 | public string OuterHTML => _element.OuterHtml; 26 | 27 | public string InnerText => _element.TextContent; 28 | 29 | public int ChildrenCount => _element.Children.Length; 30 | 31 | public IDictionary Attributes 32 | { 33 | get 34 | { 35 | if (_attributes != null) 36 | return _attributes; 37 | 38 | _attributes = _element.Attributes.AsEnumerable().ToDictionary(k => k.Name, k => k.Value); 39 | return _attributes; 40 | } 41 | } 42 | 43 | public string GetAttribute(string name) 44 | { 45 | return Attributes.GetOrDefault(name); 46 | } 47 | 48 | public ApiHtmlElement ChildAt(int index) 49 | { 50 | return new ApiHtmlElement(_element.Children[index]); 51 | } 52 | 53 | public ApiHtmlElement Select(string cssSelector) 54 | { 55 | return new ApiHtmlElement(_element.QuerySelector(cssSelector)); 56 | } 57 | 58 | public ApiHtmlElement[] SelectAll(string cssSelector) 59 | { 60 | return _element 61 | .QuerySelectorAll(cssSelector) 62 | .Select(n => new ApiHtmlElement(n)) 63 | .ToArray(); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/Http/ApiHttpContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1.Http 6 | { 7 | public class ApiHttpContext 8 | { 9 | private IGrabberServices _grabberServices; 10 | 11 | public ApiHttpContext(IGrabberServices grabberServices) 12 | { 13 | _grabberServices = grabberServices; 14 | var client = _grabberServices.GetClient(); 15 | Client = new ApiHttpClient(client); 16 | } 17 | 18 | public ApiHttpClient Client { get; } 19 | 20 | public ApiHttpRequest Request(ApiHttpRequest request) 21 | { 22 | return request; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/Http/ApiHttpRequest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1.Http 6 | { 7 | public class ApiHttpRequest 8 | { 9 | public string Url { get; set; } 10 | 11 | public string Method { get; set; } 12 | 13 | public object BodyText { get; set; } 14 | 15 | public bool ExpectText { get; set; } 16 | 17 | public Dictionary> Headers { get; set; } = new Dictionary>(); 18 | 19 | public ApiHttpRequest AddHeader(string name, string value) 20 | { 21 | if (!Headers.TryGetValue(name, out var list)) 22 | Headers.Add(name, list = new List()); 23 | list.Add(value); 24 | return this; 25 | } 26 | 27 | public ApiHttpRequest SetHeader(string name, string value) 28 | { 29 | if (!Headers.TryGetValue(name, out var list)) 30 | Headers.Add(name, list = new List()); 31 | list.Clear(); 32 | list.Add(value); 33 | return this; 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/Api/v1/Http/ApiHttpResponse.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net.Http; 5 | using System.Text; 6 | 7 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api.v1.Http 8 | { 9 | public class ApiHttpResponse 10 | { 11 | public ApiHttpResponse(HttpResponseMessage response, string bodyText) 12 | { 13 | Success = response.IsSuccessStatusCode; 14 | StatusCode = (int)response.StatusCode; 15 | StatusText = response.ReasonPhrase; 16 | Headers = new Dictionary(); 17 | foreach (var header in response.Headers) 18 | Headers[header.Key] = header.Value.ToArray(); 19 | BodyText = bodyText; 20 | } 21 | 22 | public int StatusCode { get; } 23 | 24 | public string StatusText { get; } 25 | 26 | public IDictionary Headers { get; } 27 | 28 | public string BodyText { get; } 29 | 30 | public bool Success { get; } 31 | 32 | public void AssertSuccess() 33 | { 34 | if (!Success) 35 | throw new HttpRequestException($"The status code {StatusCode} does not indicate success."); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/GrabberScriptInterpretOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter 6 | { 7 | /// 8 | /// Defines different options for . 9 | /// 10 | public struct GrabberScriptInterpretOptions 11 | { 12 | /// 13 | /// Gets the default instance of options. 14 | /// 15 | public static readonly GrabberScriptInterpretOptions Default = new(); 16 | 17 | /// 18 | /// Gets or sets the additional data exposed to the script, besides the host object. 19 | /// 20 | public IDictionary ExposedData { get; set; } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/GrabberScriptInterpreterService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using DotNetTools.SharpGrabber.BlackWidow.Definitions; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter 7 | { 8 | /// 9 | /// Default implementation for 10 | /// 11 | public class GrabberScriptInterpreterService : IGrabberScriptInterpreterService 12 | { 13 | private readonly Dictionary _interpreters = new(); 14 | 15 | public void Register(GrabberScriptType scriptType, IGrabberScriptInterpreter interpreter) 16 | { 17 | _interpreters[scriptType] = interpreter; 18 | } 19 | 20 | public IGrabberScriptInterpreter GetInterpreter(GrabberScriptType scriptType) 21 | { 22 | return _interpreters.GetOrDefault(scriptType); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/IGrabberScriptInterpreter.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.BlackWidow.Repository; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using DotNetTools.SharpGrabber.BlackWidow.Definitions; 8 | 9 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter 10 | { 11 | /// 12 | /// Interprets a script as a grabber. 13 | /// 14 | public interface IGrabberScriptInterpreter 15 | { 16 | /// 17 | /// Loads the specified script source and interprets it as a grabber. 18 | /// 19 | Task InterpretAsync(IGrabberRepositoryScript script, IGrabberScriptSource source, int apiVersion, 20 | GrabberScriptInterpretOptions options, CancellationToken cancellationToken); 21 | } 22 | 23 | public static class GrabberScriptInterpreterExtensions 24 | { 25 | /// 26 | /// Loads the specified script source and interprets it as a grabber. 27 | /// 28 | public static Task InterpretAsync(this IGrabberScriptInterpreter interpreter, IGrabberRepositoryScript script, 29 | IGrabberScriptSource source, int apiVersion, GrabberScriptInterpretOptions options) 30 | { 31 | return interpreter.InterpretAsync(script, source, apiVersion, options, CancellationToken.None); 32 | } 33 | 34 | /// 35 | /// Loads the specified script source and interprets it as a grabber. 36 | /// 37 | public static Task InterpretAsync(this IGrabberScriptInterpreter interpreter, IGrabberRepositoryScript script, 38 | IGrabberScriptSource source, int apiVersion) 39 | { 40 | return interpreter.InterpretAsync(script, source, apiVersion, GrabberScriptInterpretOptions.Default); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/IGrabberScriptInterpreterService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using DotNetTools.SharpGrabber.BlackWidow.Definitions; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter 7 | { 8 | /// 9 | /// Manages grabber script interpreters. 10 | /// 11 | public interface IGrabberScriptInterpreterService 12 | { 13 | /// 14 | /// Registers for . 15 | /// 16 | void Register(GrabberScriptType scriptType, IGrabberScriptInterpreter interpreter); 17 | 18 | /// 19 | /// Gets the proper interpreter for , if available. 20 | /// 21 | IGrabberScriptInterpreter GetInterpreter(GrabberScriptType scriptType); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/JavaScript/Conversion/JintConvertExTypeConverter.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.ConvertEx; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.JavaScript 7 | { 8 | internal class JintConvertExTypeConverter : Jint.Runtime.Interop.ITypeConverter 9 | { 10 | private readonly ITypeConverter _converter; 11 | 12 | public JintConvertExTypeConverter(ITypeConverter converter) 13 | { 14 | _converter = converter; 15 | } 16 | 17 | public object Convert(object value, Type type, IFormatProvider formatProvider) 18 | { 19 | return _converter.Convert(value, type, formatProvider); 20 | } 21 | 22 | public bool TryConvert(object value, Type type, IFormatProvider formatProvider, out object converted) 23 | { 24 | return _converter.TryConvert(value, type, formatProvider, out converted); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Interpreter/JavaScript/JintInterpreterServiceExtensions.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.BlackWidow.Host; 2 | using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | using DotNetTools.SharpGrabber.BlackWidow.Definitions; 7 | 8 | namespace DotNetTools.SharpGrabber.BlackWidow.Interpreter.JavaScript 9 | { 10 | /// 11 | /// Provides extension methods for . 12 | /// 13 | public static class JintInterpreterServiceExtensions 14 | { 15 | /// 16 | /// Registers Jint as the JavaScript interpreter. 17 | /// 18 | public static void RegisterJint(this IGrabberScriptInterpreterService interpreterService, IInterpreterApiService apiService, 19 | IGrabberServices grabberServices, IScriptHost scriptHost) 20 | { 21 | interpreterService.Register(GrabberScriptType.JavaScript, new JintJavaScriptInterpreter(apiService, grabberServices, scriptHost)); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Repository/GrabberRepositoryChangeEventHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Repository 6 | { 7 | /// 8 | /// Represents a method that handles changes of a grabber repository. 9 | /// 10 | /// The updated repository 11 | /// The new feed 12 | /// The previous feed 13 | public delegate void GrabberRepositoryChangeEventHandler(IGrabberRepository repository, IGrabberRepositoryFeed feed, IGrabberRepositoryFeed previousFeed); 14 | } 15 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Repository/GrabberRepositoryFeedDifference.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Repository 6 | { 7 | /// 8 | /// Defines all possible types of difference between two objects. 9 | /// 10 | public enum GrabberRepositoryFeedDifferenceType 11 | { 12 | /// 13 | /// Indicates that a script has been added. 14 | /// 15 | ScriptAdded, 16 | 17 | /// 18 | /// Indicates that a script has been removed. 19 | /// 20 | ScriptRemoved, 21 | 22 | /// 23 | /// Indicates that a script has been changed. 24 | /// 25 | ScriptChanged 26 | } 27 | 28 | /// 29 | /// Describes a difference between two objects. 30 | /// 31 | public class GrabberRepositoryFeedDifference 32 | { 33 | public GrabberRepositoryFeedDifference(GrabberRepositoryFeedDifferenceType type, IGrabberRepositoryScript ownScript, IGrabberRepositoryScript otherScript) 34 | { 35 | Type = type; 36 | OwnScript = ownScript; 37 | OtherScript = otherScript; 38 | } 39 | 40 | /// 41 | /// Gets the type of this difference. 42 | /// 43 | public GrabberRepositoryFeedDifferenceType Type { get; } 44 | 45 | /// 46 | /// Gets the descriptor for the own script. 47 | /// This value is NULL if is . 48 | /// 49 | public IGrabberRepositoryScript OwnScript { get; } 50 | 51 | /// 52 | /// Gets the descriptor for the other script. 53 | /// This value is NULL if is . 54 | /// 55 | public IGrabberRepositoryScript OtherScript { get; } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Repository/IGrabberRepositoryChangeDetector.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | 6 | namespace DotNetTools.SharpGrabber.BlackWidow.Repository 7 | { 8 | /// 9 | /// Detects changes in a grabber repository. 10 | /// 11 | public interface IGrabberRepositoryChangeDetector : IDisposable 12 | { 13 | /// 14 | /// Invoked when a change in a repository gets detected. 15 | /// 16 | event GrabberRepositoryChangeEventHandler RepositoryChanged; 17 | 18 | /// 19 | /// Forces a manual update of all repository feeds. 20 | /// 21 | /// Whether or not to only update feeds of repositories that don't support change notification. 22 | Task ForceUpdateFeedAsync(bool pollableOnly = true); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Repository/IGrabberRepositorySubscription.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DotNetTools.SharpGrabber.BlackWidow.Repository 6 | { 7 | /// 8 | /// Describes a subscription to changes of a grabber repository. 9 | /// 10 | /// 11 | /// To unsubscribe, the instance should be disposed. 12 | /// 13 | public interface IGrabberRepositorySubscription : IDisposable 14 | { 15 | /// 16 | /// Invoked when the feed gets updated. 17 | /// 18 | event Action FeedUpdated; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/Repository/OfficialGrabberRepository.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.SharpGrabber.BlackWidow.Repository.GitHub; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Net.Http; 5 | using System.Text; 6 | 7 | namespace DotNetTools.SharpGrabber.BlackWidow.Repository 8 | { 9 | /// 10 | /// Defines the official SharpGrabber grabber repository. 11 | /// 12 | public class OfficialGrabberRepository : GitHubGrabberRepository 13 | { 14 | public OfficialGrabberRepository() 15 | { 16 | Setup(); 17 | } 18 | 19 | public OfficialGrabberRepository(HttpClient httpClient, bool ownClient = true) : base(httpClient, ownClient) 20 | { 21 | Setup(); 22 | } 23 | 24 | private void Setup() 25 | { 26 | Repository = BlackWidowConstants.GitHub.OfficialRepository.RepositoryAddress; 27 | BranchName = BlackWidowConstants.GitHub.OfficialRepository.MasterBranch; 28 | RepoRootPath = BlackWidowConstants.GitHub.OfficialRepository.RootPath; 29 | FeedFileName = BlackWidowConstants.GitHub.OfficialRepository.FeedFileName; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/SharpGrabber.BlackWidow.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.0 5 | DotNetTools.SharpGrabber.BlackWidow 6 | true 7 | Javid Shoaei 8 | github.com/dotnettools 9 | Adds support for executing grabber scripts written in ECMAScript (also known as JavaScript); and introduces grabber repositories to always keep the grabbers updated at runtime. 10 | LICENSE 11 | https://github.com/dotnettools/SharpGrabber 12 | blackwidow-logo.png 13 | 14 | 9 15 | https://github.com/dotnettools/SharpGrabber 16 | blackwidow,grab,scrape,script,ecmascript,ecma,javascript,js 17 | 1701;1702;1591 18 | true 19 | 1.1.0 20 | AnyCPU;x64 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | True 35 | 36 | 37 | 38 | True 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/SharpGrabber.BlackWidow.csproj.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | True 3 | True -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/TypeConversion/ToBoolConverter.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.ConvertEx; 2 | using DotNetTools.SharpGrabber.BlackWidow.Internal; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | 7 | namespace DotNetTools.SharpGrabber.BlackWidow.TypeConversion 8 | { 9 | internal class ToBoolConverter : ITypeConverter 10 | { 11 | public bool TryConvert(object value, Type targetType, IFormatProvider formatProvider, out object convertedValue) 12 | { 13 | if (targetType != typeof(bool)) 14 | { 15 | convertedValue = null; 16 | return false; 17 | } 18 | 19 | if (value is string) 20 | { 21 | var str = value?.ToString(); 22 | convertedValue = !string.IsNullOrEmpty(str); 23 | } 24 | else if (value.GetType().IsNumericType()) 25 | { 26 | var num = (double)Convert.ChangeType(value, typeof(double)); 27 | convertedValue = num != 0; 28 | return true; 29 | } 30 | else 31 | { 32 | convertedValue = null; 33 | return false; 34 | } 35 | return true; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/SharpGrabber.BlackWidow/TypeConversion/TypeConverters.cs: -------------------------------------------------------------------------------- 1 | using DotNetTools.ConvertEx; 2 | using DotNetTools.ConvertEx.Converters; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | 7 | namespace DotNetTools.SharpGrabber.BlackWidow.TypeConversion 8 | { 9 | internal static class TypeConverters 10 | { 11 | public static readonly ITypeConverter Default; 12 | 13 | static TypeConverters() 14 | { 15 | var converter = new TypeConverter(); 16 | converter 17 | .AddDigester() 18 | .AddConverter() 19 | .AddConverter() 20 | .AddConverter() 21 | .AddConverter() 22 | .AddConverter() 23 | .AddConverter() 24 | .AddConverter(); 25 | Default = converter; 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/FFMpegHelper.cs: -------------------------------------------------------------------------------- 1 | using FFmpeg.AutoGen; 2 | using System; 3 | using System.Runtime.InteropServices; 4 | 5 | namespace DotNetTools.SharpGrabber.Converter 6 | { 7 | internal static class FFMpegHelper 8 | { 9 | public static unsafe string av_strerror(int error) 10 | { 11 | var bufferSize = 1024; 12 | var buffer = stackalloc byte[bufferSize]; 13 | ffmpeg.av_strerror(error, buffer, (ulong)bufferSize); 14 | var message = Marshal.PtrToStringAnsi((IntPtr)buffer); 15 | return message; 16 | } 17 | 18 | public static int ThrowOnError(this int result) 19 | { 20 | if (result < 0) 21 | throw new ApplicationException(av_strerror(result)); 22 | return result; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/IOContext.cs: -------------------------------------------------------------------------------- 1 | using FFmpeg.AutoGen; 2 | using System; 3 | 4 | namespace DotNetTools.SharpGrabber.Converter 5 | { 6 | unsafe sealed class IOContext : IDisposable 7 | { 8 | #region Fields 9 | private AVIOContext* _ioContext; 10 | private bool usedAvioOpen; 11 | #endregion 12 | 13 | #region Properties 14 | public AVIOContext* Pointer => _ioContext; 15 | #endregion 16 | 17 | #region Constructor 18 | public IOContext(AVIOContext* ioContext) 19 | { 20 | _ioContext = ioContext; 21 | } 22 | 23 | public IOContext(string path, int flags) 24 | { 25 | usedAvioOpen = true; 26 | AVIOContext* ioContext = null; 27 | ffmpeg.avio_open2(&ioContext, path, flags, null, null).ThrowOnError(); 28 | _ioContext = ioContext; 29 | } 30 | 31 | public IOContext(Uri uri, int flags) : this(uri.IsFile ? uri.LocalPath : uri.ToString(), flags) { } 32 | #endregion 33 | 34 | #region Methods 35 | public void Dispose() 36 | { 37 | if (_ioContext != null) 38 | { 39 | var ioContext = _ioContext; 40 | if (usedAvioOpen) 41 | ffmpeg.avio_close(ioContext); 42 | else 43 | ffmpeg.avio_context_free(&ioContext); 44 | _ioContext = null; 45 | } 46 | } 47 | #endregion 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/MediaFrame.cs: -------------------------------------------------------------------------------- 1 | using FFmpeg.AutoGen; 2 | using System; 3 | 4 | namespace DotNetTools.SharpGrabber.Converter 5 | { 6 | /// 7 | /// Wrapper for . 8 | /// 9 | public sealed unsafe class MediaFrame : IDisposable 10 | { 11 | #region Properties 12 | public AVFrame* Pointer { get; private set; } 13 | #endregion 14 | 15 | #region Constructor 16 | public MediaFrame(AVFrame* frame) 17 | { 18 | Pointer = frame; 19 | } 20 | #endregion 21 | 22 | #region Methods 23 | public void Dispose() 24 | { 25 | if (Pointer != null) 26 | { 27 | var frame = Pointer; 28 | ffmpeg.av_frame_unref(frame); 29 | Pointer = null; 30 | } 31 | } 32 | #endregion 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/MediaHelper.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using FFmpeg.AutoGen; 4 | 5 | namespace DotNetTools.SharpGrabber.Converter 6 | { 7 | public static class MediaHelper 8 | { 9 | public static AVHWDeviceType[] GetHardwareDeviceTypes() 10 | { 11 | var set = new HashSet(); 12 | var type = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; 13 | while ((type = ffmpeg.av_hwdevice_iterate_types(type)) != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE) 14 | set.Add(type); 15 | return set.ToArray(); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/MediaLibrary.cs: -------------------------------------------------------------------------------- 1 | using FFmpeg.AutoGen; 2 | 3 | namespace DotNetTools.SharpGrabber.Converter 4 | { 5 | public static class MediaLibrary 6 | { 7 | public static void Load(string dir) 8 | { 9 | ffmpeg.RootPath = dir; 10 | } 11 | 12 | public static string FFMpegVersion() => ffmpeg.av_version_info(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/MediaPacket.cs: -------------------------------------------------------------------------------- 1 | using FFmpeg.AutoGen; 2 | using System; 3 | 4 | namespace DotNetTools.SharpGrabber.Converter 5 | { 6 | /// 7 | /// Wrapper for / 8 | /// 9 | public sealed unsafe class MediaPacket : IDisposable 10 | { 11 | #region Properties 12 | public AVPacket* Pointer { get; private set; } 13 | #endregion 14 | 15 | #region Constructor 16 | public MediaPacket(AVPacket* packet) 17 | { 18 | Pointer = packet; 19 | } 20 | #endregion 21 | 22 | #region Methods 23 | public void Dispose() 24 | { 25 | if (Pointer != null) 26 | { 27 | var packet = Pointer; 28 | ffmpeg.av_packet_unref(packet); 29 | Pointer = null; 30 | } 31 | } 32 | #endregion 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/MediaStreamSource.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetTools.SharpGrabber.Converter 4 | { 5 | public enum MediaStreamType { Audio, Video } 6 | 7 | class MediaStreamSource 8 | { 9 | #region Properties 10 | public Uri Path { get; } 11 | 12 | public MediaStreamType StreamType { get; } 13 | #endregion 14 | 15 | #region Constructors 16 | public MediaStreamSource(Uri path, MediaStreamType streamType) 17 | { 18 | Path = path; 19 | StreamType = streamType; 20 | } 21 | #endregion 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/SharpGrabber.Converter/Reference.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetTools.SharpGrabber.Converter 2 | { 3 | public class Reference where T : struct 4 | { 5 | public static implicit operator T(Reference val) 6 | { 7 | return val.Value; 8 | } 9 | 10 | private T _value; 11 | 12 | public Reference(T value) 13 | { 14 | _value = value; 15 | } 16 | 17 | public Reference() : this(default) { } 18 | 19 | public T Value 20 | { 21 | get 22 | { 23 | return _value; 24 | } 25 | set 26 | { 27 | _value = value; 28 | } 29 | } 30 | 31 | public override string ToString() 32 | { 33 | return _value.ToString(); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/SharpGrabber.Desktop/App.xaml: -------------------------------------------------------------------------------- 1 |  4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /src/SharpGrabber.Desktop/App.xaml.cs: -------------------------------------------------------------------------------- 1 | using Avalonia; 2 | using Avalonia.Markup.Xaml; 3 | 4 | namespace SharpGrabber.Desktop 5 | { 6 | public class App : Application 7 | { 8 | public override void Initialize() 9 | { 10 | AvaloniaXamlLoader.Load(this); 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/SharpGrabber.Desktop/Auth/InstagramAuthenticationHandlerInterface.cs: -------------------------------------------------------------------------------- 1 | using Avalonia.Threading; 2 | using DotNetTools.SharpGrabber.Auth; 3 | using DotNetTools.SharpGrabber.Instagram; 4 | using InstagramApiSharp.Classes; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | using System.Threading.Tasks; 9 | 10 | namespace SharpGrabber.Desktop.Auth 11 | { 12 | internal class InstagramAuthenticationHandlerInterface : IInstagramAuthenticationInterface 13 | { 14 | private readonly MainWindow _mainWindow; 15 | 16 | public InstagramAuthenticationHandlerInterface(MainWindow mainWindow) 17 | { 18 | _mainWindow = mainWindow; 19 | } 20 | 21 | public async Task ProvideCredentialsAsync(GrabberAuthenticationRequest request) 22 | { 23 | GrabberBasicCredentials credentials = null; 24 | await Dispatcher.UIThread.InvokeAsync(async () => 25 | { 26 | credentials = await _mainWindow.ShowBasicAuthDialog(string.Format("Authenticate {0}", request.Grabber.Name)); 27 | }); 28 | return credentials; 29 | } 30 | 31 | public async Task GetTwoFactorVerificationCodeAsync(GrabberAuthenticationRequest request, IResult twoFactorInfo, 32 | IResult smsResult) 33 | { 34 | string code = null; 35 | await Dispatcher.UIThread.InvokeAsync(async () => 36 | { 37 | code = await _mainWindow.ShowTwoFactorAuthDialog(text: "Two Factor Code (leave empty if approved from phone)"); 38 | }); 39 | return code; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/SharpGrabber.Desktop/Components/LoadingSpinner.xaml: -------------------------------------------------------------------------------- 1 |  9 | 10 | 24 | 25 | 26 | 27 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/SharpGrabber.Desktop/Components/LoadingSpinner.xaml.cs: -------------------------------------------------------------------------------- 1 | using Avalonia.Controls; 2 | using Avalonia.Markup.Xaml; 3 | 4 | namespace SharpGrabber.Desktop.Components 5 | { 6 | public class LoadingSpinner : UserControl 7 | { 8 | public LoadingSpinner() 9 | { 10 | this.InitializeComponent(); 11 | } 12 | 13 | private void InitializeComponent() 14 | { 15 | AvaloniaXamlLoader.Load(this); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/SharpGrabber.Desktop/Components/StreamReferenceView.xaml.cs: -------------------------------------------------------------------------------- 1 | using Avalonia; 2 | using Avalonia.Controls; 3 | using Avalonia.Markup.Xaml; 4 | using SharpGrabber.Desktop.ViewModel; 5 | using System; 6 | 7 | namespace SharpGrabber.Desktop.Components 8 | { 9 | public class StreamReferenceView : UserControl 10 | { 11 | #region Fields 12 | private Button btnDownload, btnCopyLink; 13 | private DrawingPresenter iconCheck, iconVideo, iconAudio, iconCreate; 14 | #endregion 15 | 16 | #region Properties 17 | public GrabbedStreamRefViewModel Ref => DataContext as GrabbedStreamRefViewModel; 18 | 19 | public MainWindow MainWindow => Program.MainWindow; 20 | #endregion 21 | 22 | public StreamReferenceView() { } 23 | 24 | public StreamReferenceView(GrabbedStreamRefViewModel viewModel) 25 | { 26 | DataContext = viewModel; 27 | InitializeComponent(); 28 | 29 | iconCheck.IsVisible = true; 30 | } 31 | 32 | private void InitializeComponent() 33 | { 34 | AvaloniaXamlLoader.Load(this); 35 | 36 | btnDownload = this.Find