├── VOffline
    ├── Models
    │   ├── Vk
    │   │   ├── VkToken.cs
    │   │   └── VkCredentials.cs
    │   ├── Google
    │   │   ├── GoogleCheckIn.cs
    │   │   ├── ProtobufField.cs
    │   │   └── GoogleCredentials.cs
    │   ├── Mode.cs
    │   ├── Storage
    │   │   ├── NetworkException.cs
    │   │   ├── IDownload.cs
    │   │   ├── PlaylistWithAudio.cs
    │   │   ├── Download.cs
    │   │   └── AlbumWithPhoto.cs
    │   └── Settings.cs
    ├── Services
    │   ├── Storage
    │   │   ├── CreateMode.cs
    │   │   ├── DownloadQueueProvider.cs
    │   │   ├── BackgroundDownloader.cs
    │   │   ├── AttachmentProcessor.cs
    │   │   └── FilesystemTools.cs
    │   ├── Handlers
    │   │   ├── IHandler.cs
    │   │   ├── CommentHandler.cs
    │   │   ├── WallHandler.cs
    │   │   ├── HandlerBase.cs
    │   │   ├── CommentsHandler.cs
    │   │   ├── PlaylistHandler.cs
    │   │   ├── AlbumHandler.cs
    │   │   ├── PostHandler.cs
    │   │   ├── PhotoHandler.cs
    │   │   └── AudioHandler.cs
    │   ├── VkNetHacks
    │   │   ├── LimitedSizeStack.cs
    │   │   ├── SimpleLoggerProvider.cs
    │   │   ├── SimpleLogger.cs
    │   │   ├── CancellableConstraint.cs
    │   │   └── CustomRestClient.cs
    │   ├── Token
    │   │   ├── TokenMagic.cs
    │   │   ├── Vk
    │   │   │   ├── VkTokenReceiver.cs
    │   │   │   └── VkHttpRequests.cs
    │   │   └── Google
    │   │   │   ├── AndroidAuth.cs
    │   │   │   ├── VarInt.cs
    │   │   │   ├── ProtobufParser.cs
    │   │   │   ├── MTalk.cs
    │   │   │   └── GoogleHttpRequests.cs
    │   ├── Extensions.cs
    │   ├── FileCache.cs
    │   ├── Logic.cs
    │   └── Vk
    │   │   ├── AttachmentExtensions.cs
    │   │   └── VkApiUtils.cs
    ├── appsettings.json
    ├── log4net.config
    ├── VOffline.csproj
    └── Program.cs
├── README.md
├── VOffline.sln
└── .gitignore


/VOffline/Models/Vk/VkToken.cs:
--------------------------------------------------------------------------------
1 | namespace VOffline.Models.Vk
2 | {
3 |     public class VkToken
4 |     {
5 |         public string Token { get; set; }
6 |     }
7 | }


--------------------------------------------------------------------------------
/VOffline/Models/Google/GoogleCheckIn.cs:
--------------------------------------------------------------------------------
1 | namespace VOffline.Models.Google
2 | {
3 |     public class GoogleCheckIn
4 |     {
5 |         public byte[] Response { get; set; }
6 |     }
7 | }


--------------------------------------------------------------------------------
/VOffline/Models/Mode.cs:
--------------------------------------------------------------------------------
 1 | namespace VOffline.Models
 2 | {
 3 |     public enum Mode
 4 |     {
 5 |         Wall,
 6 |         Audio,
 7 |         Photos,
 8 |         Video,
 9 |         Docs,
10 |         All
11 |     }
12 | }


--------------------------------------------------------------------------------
/VOffline/Models/Vk/VkCredentials.cs:
--------------------------------------------------------------------------------
1 | namespace VOffline.Models.Vk
2 | {
3 |     public class VkCredentials
4 |     {
5 |         public string Login { get; set; }
6 |         public string Password { get; set; }
7 |     }
8 | }


--------------------------------------------------------------------------------
/VOffline/Services/Storage/CreateMode.cs:
--------------------------------------------------------------------------------
1 | namespace VOffline.Services.Storage
2 | {
3 |     public enum CreateMode
4 |     {
5 |         AutoRenameCollisions,
6 |         ThrowIfExists,
7 |         OverwriteExisting
8 |     }
9 | }


--------------------------------------------------------------------------------
/VOffline/Models/Google/ProtobufField.cs:
--------------------------------------------------------------------------------
 1 | namespace VOffline.Models.Google
 2 | {
 3 |     public class ProtobufField
 4 |     {
 5 |         public ProtobufField(int value)
 6 |         {
 7 |             Type = value & 0x7;  // last three bits, 0b00000111
 8 |             FieldNumber = value >> 3;
 9 |         }
10 | 
11 |         public int Type { get; }
12 |         public int FieldNumber { get; }
13 |     }
14 | }


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/IHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Threading;
 3 | using System.Threading.Tasks;
 4 | using log4net;
 5 | 
 6 | namespace VOffline.Services.Handlers
 7 | {
 8 |     public interface IHandler<T>
 9 |     {
10 |         Task Process(T data, DirectoryInfo parentDir, CancellationToken token, ILog log);
11 |         DirectoryInfo GetWorkingDirectory(T data, DirectoryInfo parentDir);
12 |     }
13 | }


--------------------------------------------------------------------------------
/VOffline/Models/Storage/NetworkException.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | 
 3 | namespace VOffline.Models.Storage
 4 | {
 5 |     public class NetworkException : ApplicationException
 6 |     {
 7 |         public NetworkException(string message, Exception innerException) : base(message, innerException)
 8 |         {
 9 |         }
10 | 
11 |         public NetworkException(string message) : base(message)
12 |         {
13 |         }
14 |     }
15 | }


--------------------------------------------------------------------------------
/VOffline/appsettings.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "Settings": {
 3 |     //"Targets": [ "id1", "durov", "public147415323", "tech" ],
 4 |     //"Targets": [ "retrowave" ],
 5 |     //"Targets": ["rast1234", "retrowave"],
 6 |     //"Modes": [ "Audio", "Wall" ],
 7 |     //"Modes": [ "Photos" ],
 8 |     //"Modes": [ "All" ],
 9 |     "OutputPath": "."
10 |   },
11 |   "VkCredentials": {
12 |     //"Login": "username@gmail.com",
13 |     //"Password": "your_vk_password"
14 |   }
15 | }


--------------------------------------------------------------------------------
/VOffline/Models/Google/GoogleCredentials.cs:
--------------------------------------------------------------------------------
 1 | using System.Collections.Generic;
 2 | 
 3 | namespace VOffline.Models.Google
 4 | {
 5 |     public class GoogleCredentials
 6 |     {
 7 |         public long Id { get; set; }
 8 |         public long Token { get; set; }
 9 |         public List<byte> RawId { get; set; }
10 | 
11 |         public override string ToString()
12 |         {
13 |             return $"{nameof(GoogleCredentials)}(id {Id}; token {Token})";
14 |         }
15 |     }
16 | }


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # VOffline
 2 | Evacuate from VK! 🚚
 3 | 
 4 | #### This tool can save almost everything from public VK content (wall, posts, comments, photos, polls). Audio and playlists too!
 5 | 
 6 | 
 7 | It works fine but needs some polishing and documentation is not yet done.
 8 | To start, look at `appsettings.json`, it is pretty self explanatory. For now you will need Visual Studio to buld and run the thing.
 9 | 
10 | ___
11 | All interested in token for Audio API have look at https://github.com/Rast1234/VkNet.TokenMagic
12 | 


--------------------------------------------------------------------------------
/VOffline/Models/Storage/IDownload.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.IO;
 4 | using System.Threading;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace VOffline.Models.Storage
 8 | {
 9 |     public interface IDownload
10 |     {
11 |         DirectoryInfo Location { get; }
12 |         string DesiredName { get; }
13 |         int RetryCount { get; }
14 |         IReadOnlyList<Exception> Errors { get; }
15 |         void AddError(Exception e);
16 |         Task<byte[]> GetContent(CancellationToken token);
17 |     }
18 | }


--------------------------------------------------------------------------------
/VOffline/Services/VkNetHacks/LimitedSizeStack.cs:
--------------------------------------------------------------------------------
 1 | using System.Collections.Generic;
 2 | 
 3 | namespace VOffline.Services.VkNetHacks
 4 | {
 5 |     public class LimitedSizeStack<T> : LinkedList<T>
 6 |     {
 7 |         private readonly int _maxSize;
 8 | 
 9 |         public LimitedSizeStack(int maxSize)
10 |         {
11 |             this._maxSize = maxSize;
12 |         }
13 | 
14 |         public void Push(T item)
15 |         {
16 |             this.AddFirst(item);
17 |             if (this.Count <= this._maxSize)
18 |                 return;
19 |             this.RemoveLast();
20 |         }
21 |     }
22 | }


--------------------------------------------------------------------------------
/VOffline/Services/VkNetHacks/SimpleLoggerProvider.cs:
--------------------------------------------------------------------------------
 1 | using log4net;
 2 | using Microsoft.Extensions.Logging;
 3 | 
 4 | namespace VOffline.Services.VkNetHacks
 5 | {
 6 |     public class SimpleLoggerProvider : ILoggerProvider
 7 |     {
 8 |         private readonly SimpleLogger logger;
 9 | 
10 |         public SimpleLoggerProvider(ILog log)
11 |         {
12 |             logger = new SimpleLogger(log);
13 |         }
14 | 
15 |         public void Dispose()
16 |         {
17 |         }
18 | 
19 |         public ILogger CreateLogger(string categoryName)
20 |         {
21 |             return logger;
22 |         }
23 |     }
24 | }


--------------------------------------------------------------------------------
/VOffline/Services/Storage/DownloadQueueProvider.cs:
--------------------------------------------------------------------------------
 1 | using System.Collections.Generic;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using Microsoft.Extensions.Options;
 6 | using Nito.AsyncEx;
 7 | using VOffline.Models;
 8 | using VOffline.Models.Storage;
 9 | 
10 | namespace VOffline.Services.Storage
11 | {
12 |     public class DownloadQueueProvider
13 |     {
14 |         public DownloadQueueProvider(IOptionsSnapshot<Settings> settings)
15 |         {
16 |             var queueSizeLimit = settings.Value.DownloadQueueLimit;
17 |             Pending = queueSizeLimit > 0
18 |                 ? new AsyncProducerConsumerQueue<IDownload>(settings.Value.DownloadQueueLimit)
19 |                 : new AsyncProducerConsumerQueue<IDownload>();
20 |         }
21 | 
22 |         public async Task EnqueueAll(IEnumerable<IDownload> items, CancellationToken token)
23 |         {
24 |             await Task.WhenAll(items.Select(async x => await Pending.EnqueueAsync(x, token)));
25 |         }
26 | 
27 |         public AsyncProducerConsumerQueue<IDownload> Pending { get; }
28 |     }
29 | }


--------------------------------------------------------------------------------
/VOffline/Models/Storage/PlaylistWithAudio.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using VkNet.Model.Attachments;
 5 | 
 6 | namespace VOffline.Models.Storage
 7 | {
 8 |     public class PlaylistWithAudio
 9 |     {
10 |         public AudioPlaylist Playlist { get; }
11 |         public IReadOnlyList<Audio> Audio { get; }
12 | 
13 |         public PlaylistWithAudio(AudioPlaylist playlist, IReadOnlyList<Audio> audio)
14 |         {
15 |             Playlist = playlist;
16 |             Audio = audio
17 |                 .OrderBy(a => a.Date)
18 |                 .ToList();
19 |         }
20 | 
21 |         public PlaylistWithAudio(IReadOnlyList<Audio> audio)
22 |         {
23 | 
24 |             this.Playlist = new AudioPlaylist
25 |             {
26 |                 Id = -1,
27 |                 Title = "__default",
28 |                 Description = "Tracks without playlist",
29 |                 CreateTime = DateTime.MinValue
30 |             };
31 |             this.Audio = audio
32 |                 .OrderBy(a => a.Date)
33 |                 .ToList();
34 |         }
35 |     }
36 | }
37 | 


--------------------------------------------------------------------------------
/VOffline.sln:
--------------------------------------------------------------------------------
 1 | 
 2 | Microsoft Visual Studio Solution File, Format Version 12.00
 3 | # Visual Studio 15
 4 | VisualStudioVersion = 15.0.28010.2036
 5 | MinimumVisualStudioVersion = 10.0.40219.1
 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VOffline", "VOffline\VOffline.csproj", "{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}"
 7 | EndProject
 8 | Global
 9 | 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
10 | 		Debug|Any CPU = Debug|Any CPU
11 | 		Release|Any CPU = Release|Any CPU
12 | 	EndGlobalSection
13 | 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
14 | 		{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
15 | 		{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}.Debug|Any CPU.Build.0 = Debug|Any CPU
16 | 		{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}.Release|Any CPU.ActiveCfg = Release|Any CPU
17 | 		{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}.Release|Any CPU.Build.0 = Release|Any CPU
18 | 	EndGlobalSection
19 | 	GlobalSection(SolutionProperties) = preSolution
20 | 		HideSolutionNode = FALSE
21 | 	EndGlobalSection
22 | 	GlobalSection(ExtensibilityGlobals) = postSolution
23 | 		SolutionGuid = {BE4D7888-9108-4D6F-92D6-03417FC2A1DA}
24 | 	EndGlobalSection
25 | EndGlobal
26 | 


--------------------------------------------------------------------------------
/VOffline/Services/Token/TokenMagic.cs:
--------------------------------------------------------------------------------
 1 | using log4net;
 2 | using VOffline.Models.Vk;
 3 | using VOffline.Services.Token.Google;
 4 | using VOffline.Services.Token.Vk;
 5 | 
 6 | namespace VOffline.Services.Token
 7 | {
 8 |     public class TokenMagic
 9 |     {
10 |         private AndroidAuth AndroidAuth { get; }
11 |         private VkTokenReceiver VkTokenReceiver { get; }
12 |         public FileCache<VkToken> VkToken { get; }
13 | 
14 |         public TokenMagic(AndroidAuth androidAuth, VkTokenReceiver vkTokenReceiver, FileCache<VkToken> vkToken)
15 |         {
16 |             AndroidAuth = androidAuth;
17 |             VkTokenReceiver = vkTokenReceiver;
18 |             VkToken = vkToken;
19 |         }
20 | 
21 |         public VkToken GetTokenFromScratch(ILog log)
22 |         {
23 |             if (VkToken.Value != null)
24 |             {
25 |                 return VkToken.Value;
26 |             }
27 | 
28 |             AndroidAuth.GetCredentials(log);
29 |             var token = VkTokenReceiver.GetToken(log);
30 |             VkToken.Value = new VkToken
31 |             {
32 |                 Token = token
33 |             };
34 |             return VkToken.Value;
35 |         }
36 |     }
37 | }


--------------------------------------------------------------------------------
/VOffline/log4net.config:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="utf-8" ?>
 2 | <log4net>
 3 |   <appender name="ConsoleAppender" type="log4net.Appender.ConsoleAppender" >
 4 |     <layout type="log4net.Layout.PatternLayout">
 5 |       <conversionPattern value="%date [%3thread] %-5level  %message%newline" />
 6 |     </layout>
 7 |   </appender>
 8 |   <appender name="RollingLogFileAppender" type="log4net.Appender.RollingFileAppender">
 9 |     <file type="log4net.Util.PatternString" value="logs\%date{yyyy-MM-dd}.txt" />
10 |     <!--
11 |     <file value="logs/" />
12 |     <datePattern value="yyyy-MM-dd.'txt'"/>
13 |     -->
14 |     <staticLogFileName value="false"/>
15 |     <appendToFile value="false"/>
16 |     <rollingStyle value="Once"/>
17 |     <preserveLogFileNameExtension  value="true"/>
18 |     <maxSizeRollBackups value="100"/>
19 |     <maximumFileSize value="15MB"/>
20 |     <layout type="log4net.Layout.PatternLayout">
21 |       <conversionPattern value="%date [%3thread] %-5level  %message%newline"/>
22 |     </layout>
23 |   </appender>
24 |   <root>
25 |     <level value="ALL"/>
26 |     <appender-ref ref="RollingLogFileAppender"/>
27 |     <appender-ref ref="ConsoleAppender"/>
28 |   </root>
29 | </log4net>


--------------------------------------------------------------------------------
/VOffline/Services/Token/Vk/VkTokenReceiver.cs:
--------------------------------------------------------------------------------
 1 | using log4net;
 2 | using VOffline.Models.Google;
 3 | using VOffline.Services.Token.Google;
 4 | 
 5 | namespace VOffline.Services.Token.Vk
 6 | {
 7 |     public class VkTokenReceiver
 8 |     {
 9 |         private FileCache<GoogleCredentials> CredentialsCache { get; }
10 |         private VkHttpRequests VkHttpRequests { get; }
11 |         private GoogleHttpRequests GoogleHttpRequests { get; }
12 | 
13 |         public VkTokenReceiver(FileCache<GoogleCredentials> credentialsCache, VkHttpRequests vkHttpRequests, GoogleHttpRequests googleHttpRequests)
14 |         {
15 |             CredentialsCache = credentialsCache;
16 |             VkHttpRequests = vkHttpRequests;
17 |             GoogleHttpRequests = googleHttpRequests;
18 |         }
19 | 
20 |         public string GetToken(ILog log)
21 |         {
22 |             var receipt = GoogleHttpRequests.GetReceipt(log);
23 |             var token = VkHttpRequests.GetNonRefreshedToken(log);
24 |             var newToken = VkHttpRequests.RefreshToken(token, receipt, log);
25 |             log.Info($"success! fresh token [{newToken}]");
26 |             return newToken;
27 |         }
28 | 
29 |         
30 |     }
31 | 
32 | }
33 | 


--------------------------------------------------------------------------------
/VOffline/Models/Settings.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | 
 5 | namespace VOffline.Models
 6 | {
 7 |     public class Settings
 8 |     {
 9 |         public List<string> Targets { get; set; }
10 |         public List<Mode> Modes { get; set; }
11 |         public string OutputPath { get; set; }
12 | 
13 |         public IReadOnlyList<Mode> GetWorkingModes()
14 |         {
15 |             var unique = Modes.Distinct().ToList();
16 |             if (!unique.Contains(Mode.All))
17 |             {
18 |                 return unique
19 |                     .ToList(); 
20 |             }
21 | 
22 |             return Enum.GetValues(typeof(Mode))
23 |                 .Cast<Mode>()
24 |                 .Except(Enumerable.Repeat(Mode.All, 1))
25 |                 .OrderBy(x => x.ToString())
26 |                 .ToList();
27 | 
28 |         }
29 | 
30 |         public string UserAgent { get; set; } = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
31 |         public int RequestRetryCount { get; set; } = 3;
32 |         public TimeSpan RequestRetryDelay { get; set; } = TimeSpan.FromSeconds(3);
33 |         public int DownloadQueueLimit { get; set; } = 0;
34 |     }
35 | }
36 | 


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/CommentHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VkNet.Model;
 7 | using VkNet.Model.Attachments;
 8 | using VOffline.Services.Storage;
 9 | 
10 | namespace VOffline.Services.Handlers
11 | {
12 |     public class CommentHandler : HandlerBase<Comment>
13 |     {
14 |         private readonly AttachmentProcessor attachmentProcessor;
15 | 
16 |         public CommentHandler(FilesystemTools filesystemTools, AttachmentProcessor attachmentProcessor) : base(filesystemTools)
17 |         {
18 |             this.attachmentProcessor = attachmentProcessor;
19 |         }
20 | 
21 |         public override async Task ProcessInternal(Comment comment, DirectoryInfo workDir, CancellationToken token, ILog log)
22 |         {
23 |             var attachmentTasks = comment.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
24 |             await Task.WhenAll(attachmentTasks);
25 |         }
26 | 
27 |         public override DirectoryInfo GetWorkingDirectory(Comment comment, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{comment.Date.Value:s} {comment.Id}", CreateMode.OverwriteExisting);
28 |     }
29 | }


--------------------------------------------------------------------------------
/VOffline/Services/Extensions.cs:
--------------------------------------------------------------------------------
 1 | using System.Collections.Generic;
 2 | using System.Net;
 3 | using log4net;
 4 | using RestSharp;
 5 | using VOffline.Models.Storage;
 6 | 
 7 | namespace VOffline.Services
 8 | {
 9 |     public static class Extensions
10 |     {
11 |         public static void ThrowIfSomethingWrong(this IRestResponse response)
12 |         {
13 |             if (response.ErrorException != null)
14 |             {
15 |                 throw new NetworkException("RestSharp failed", response.ErrorException);
16 |             }
17 | 
18 |             if (response.StatusCode != HttpStatusCode.OK)
19 |             {
20 |                 throw new NetworkException($"Bad response code: [{response.StatusCode}]");
21 |             }
22 | 
23 |             if (response.RawBytes == null)
24 |             {
25 |                 throw new NetworkException($"Null response bytes");
26 |             }
27 |         }
28 | 
29 |         public static void AddUnique<T>(this HashSet<T> result, IEnumerable<T> items, ILog log)
30 |         {
31 |             foreach (var item in items)
32 |             {
33 |                 if (!result.Add(item))
34 |                 {
35 |                     log.Warn($"Duplicate item [{item}]");
36 |                 }
37 |             }
38 |         }
39 |     }
40 | }


--------------------------------------------------------------------------------
/VOffline/Services/Token/Google/AndroidAuth.cs:
--------------------------------------------------------------------------------
 1 | using log4net;
 2 | using VOffline.Models.Google;
 3 | 
 4 | namespace VOffline.Services.Token.Google
 5 | {
 6 |     public class AndroidAuth
 7 |     {
 8 |         private FileCache<GoogleCredentials> CredentialsCache { get; }
 9 |         private MTalk MTalk { get; }
10 |         private GoogleHttpRequests GoogleHttpRequests { get; }
11 | 
12 |         public AndroidAuth(FileCache<GoogleCredentials> credentialsCache, MTalk mTalk, GoogleHttpRequests googleHttpRequests)
13 |         {
14 |             CredentialsCache = credentialsCache;
15 |             MTalk = mTalk;
16 |             GoogleHttpRequests = googleHttpRequests;
17 |         }
18 | 
19 |         public GoogleCredentials GetCredentials(ILog log)
20 |         {
21 |             if (CredentialsCache.Value != null)
22 |             {
23 |                 return CredentialsCache.Value;
24 |             }
25 | 
26 |             var protobuf = GoogleHttpRequests.GetCheckIn(log);
27 |             log.Debug($"parse protobuf response");
28 |             var googleCredentials = new ProtobufParser(protobuf).Parse();
29 |             MTalk.SendRequest(googleCredentials, log);
30 |             log.Info($"success! {googleCredentials}");
31 |             CredentialsCache.Value = googleCredentials;
32 |             return CredentialsCache.Value;
33 |         }
34 |     }
35 | }
36 | 


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/WallHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VkNet.Model.Attachments;
 7 | using VOffline.Services.Storage;
 8 | using VOffline.Services.Vk;
 9 | 
10 | namespace VOffline.Services.Handlers
11 | {
12 |     public class WallHandler : HandlerBase<long>
13 |     {
14 |         private readonly VkApiUtils vkApiUtils;
15 |         private readonly IHandler<Post> postHandler;
16 | 
17 |         public WallHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<Post> postHandler):base(filesystemTools)
18 |         {
19 |             this.vkApiUtils = vkApiUtils;
20 |             this.postHandler = postHandler;
21 |         }
22 | 
23 |         public override async Task ProcessInternal(long id, DirectoryInfo workDir, CancellationToken token, ILog log)
24 |         {
25 |             var allPosts = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Posts(id), 100, token, log);
26 |             log.Debug($"Wall has {allPosts.Count} posts");
27 |             var allTasks = allPosts
28 |                 .OrderBy(x => x.Date)
29 |                 .Select(p => postHandler.Process(p, workDir, token, log));
30 |             await Task.WhenAll(allTasks);
31 |         }
32 | 
33 |         public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Wall", CreateMode.OverwriteExisting);
34 |     }
35 | }


--------------------------------------------------------------------------------
/VOffline/Services/FileCache.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using log4net;
 3 | using Newtonsoft.Json;
 4 | 
 5 | namespace VOffline.Services
 6 | {
 7 |     public class FileCache<T>
 8 |         where T : class, new()
 9 |     {
10 |         private readonly ILog log;
11 | 
12 |         public FileCache(ILog log)
13 |         {
14 |             this.log = log;
15 |         }
16 |         //log.Debug($"found cached value");
17 | 
18 |         private T value;
19 | 
20 |         public string Filename => $"FileCache.{typeof(T).Name}.json";
21 | 
22 |         public T Value
23 |         {
24 |             get
25 |             {
26 |                 if (value != null)
27 |                 {
28 |                     return value;
29 |                 }
30 | 
31 |                 if (File.Exists(Filename))
32 |                 {
33 |                     var content = File.ReadAllText(Filename);
34 |                     if (!string.IsNullOrEmpty(content))
35 |                     {
36 |                         value = JsonConvert.DeserializeObject<T>(content);
37 |                         log.Debug($"{Filename} loaded value");
38 |                     }
39 |                 }
40 | 
41 |                 return value;
42 |             }
43 |             set
44 |             {
45 |                 this.value = value;
46 |                 File.WriteAllText(Filename, JsonConvert.SerializeObject(value, Formatting.Indented));
47 |                 log.Debug($"{Filename} saved value");
48 |             }
49 |         }
50 |     }
51 | }
52 | 


--------------------------------------------------------------------------------
/VOffline/Services/Token/Google/VarInt.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | 
 4 | namespace VOffline.Services.Token.Google
 5 | {
 6 |     public static class VarInt
 7 |     {
 8 |         public static (int value, int length) Read(byte[] data, int offset)
 9 |         {
10 |             var i = 0;
11 |             var result = 0;
12 |             while (i + offset < data.Length)
13 |             {
14 |                 var current = data[i + offset];
15 |                 if ((current & 0x80) != 0)
16 |                 {
17 |                     result |= (current ^ 0x80) << (i * 7);
18 |                     i++;
19 |                 }
20 |                 else
21 |                 {
22 |                     result |= current << (i * 7);
23 |                     i++;
24 |                     break;
25 |                 }
26 |             }
27 | 
28 |             if (i + offset == data.Length)
29 |             {
30 |                 throw new InvalidOperationException($"{nameof(VarInt)} failed to read varint");
31 |             }
32 | 
33 |             return (result, i);
34 |         }
35 | 
36 |         public static IEnumerable<byte> Write(int value)
37 |         {
38 |             while (value != 0)
39 |             {
40 |                 var current = value & 0x7F;
41 |                 value >>= 7;
42 |                 if (value != 0)
43 |                 {
44 |                     yield return (byte)(current | 0x80);
45 |                 }
46 |                 else
47 |                 {
48 |                     yield return (byte)current;
49 |                 }
50 |             }
51 |         }
52 |     }
53 | }


--------------------------------------------------------------------------------
/VOffline/Models/Storage/Download.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.IO;
 4 | using System.Threading;
 5 | using System.Threading.Tasks;
 6 | using RestSharp;
 7 | using VOffline.Services;
 8 | 
 9 | namespace VOffline.Models.Storage
10 | {
11 |     public class Download : IDownload
12 |     {
13 |         private readonly List<Exception> errors;
14 | 
15 |         public Download(Uri uri, DirectoryInfo location, string desiredName)
16 |         {
17 |             errors = new List<Exception>();
18 |             Uri = uri ?? throw new ArgumentNullException(nameof(uri));
19 |             Location = location ?? throw new ArgumentNullException(nameof(location));
20 |             DesiredName = desiredName ?? throw new ArgumentNullException(nameof(desiredName));
21 |         }
22 | 
23 |         public Uri Uri { get; }
24 |         public DirectoryInfo Location { get; }
25 |         public string DesiredName { get; }
26 | 
27 |         public int RetryCount => errors.Count;
28 | 
29 |         public IReadOnlyList<Exception> Errors => errors;
30 | 
31 |         public void AddError(Exception e)
32 |         {
33 |             errors.Add(e);
34 |         }
35 | 
36 |         public async Task<byte[]> GetContent(CancellationToken token)
37 |         {
38 |             var client = new RestClient($"{Uri.Scheme}://{Uri.Authority}");
39 |             var response = await client.ExecuteGetTaskAsync(new RestRequest(Uri.PathAndQuery), token);
40 |             response.ThrowIfSomethingWrong();
41 |             return response.RawBytes;
42 |         }
43 | 
44 |         public override string ToString()
45 |         {
46 |             return $"[{Location.FullName}][{DesiredName}] ({Uri})";
47 |         }
48 |     }
49 | }


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/HandlerBase.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.IO;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using Newtonsoft.Json;
 7 | using VOffline.Services.Storage;
 8 | 
 9 | namespace VOffline.Services.Handlers
10 | {
11 |     public abstract class HandlerBase<T> : IHandler<T>
12 |     {
13 |         protected readonly FilesystemTools filesystemTools;
14 | 
15 |         protected HandlerBase(FilesystemTools filesystemTools)
16 |         {
17 |             this.filesystemTools = filesystemTools;
18 |         }
19 | 
20 |         public async Task Process(T data, DirectoryInfo parentDir, CancellationToken token, ILog log)
21 |         {
22 |             DirectoryInfo workDir = null;
23 |             try
24 |             {
25 |                 token.ThrowIfCancellationRequested();  // this helps stop synchronous stuff inside long nested loops
26 |                 workDir = GetWorkingDirectory(data, parentDir);
27 |                 await ProcessInternal(data, workDir, token, log);
28 |                 log.Info($"Completed crawling data for [{workDir.FullName}]");
29 | 
30 |             }
31 |             catch (OperationCanceledException)
32 |             {
33 |                 throw;
34 |             }
35 |             catch (Exception e)
36 |             {
37 |                 log.Error($"Failed {GetType().FullName} while working in [{workDir?.FullName}]. Data is:\n{JsonConvert.SerializeObject(data)}", e);
38 |             }
39 |         }
40 | 
41 |         public abstract Task ProcessInternal(T data, DirectoryInfo workDir, CancellationToken token, ILog log);
42 | 
43 |         public abstract DirectoryInfo GetWorkingDirectory(T data, DirectoryInfo parentDir);
44 |     }
45 | }


--------------------------------------------------------------------------------
/VOffline/Models/Storage/AlbumWithPhoto.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using VkNet.Model;
 5 | using VkNet.Model.Attachments;
 6 | 
 7 | namespace VOffline.Models.Storage
 8 | {
 9 |     public class AlbumWithPhoto
10 |     {
11 |         public PhotoAlbum Album { get; }
12 |         public IReadOnlyList<Photo> Photo { get; }
13 | 
14 |         public AlbumWithPhoto(PhotoAlbum album, IReadOnlyList<Photo> photo)
15 |         {
16 |             Album = album;
17 |             Photo = photo
18 |                 .OrderBy(a => a.CreateTime)
19 |                 .ToList();
20 |         }
21 | 
22 |         public AlbumWithPhoto(Album album, IReadOnlyList<Photo> photo)
23 |         {
24 |             this.Album = new PhotoAlbum
25 |             {
26 |                 Id = album.Id.Value,
27 |                 Title = album.Title,
28 |                 Description = album.Description,
29 |                 Created = album.CreateTime,
30 |                 ThumbId = album.Thumb?.Id,
31 |                 OwnerId = album.OwnerId,
32 |                 Updated = album.UpdateTime
33 |             };
34 |             this.Photo = photo
35 |                 .OrderBy(a => a.CreateTime)
36 |                 .ToList();
37 |         }
38 | 
39 |         public AlbumWithPhoto(IReadOnlyList<Photo> photo)
40 |         {
41 |             this.Album = new PhotoAlbum
42 |             {
43 |                 Id = long.MinValue,
44 |                 Title = "__default",
45 |                 Description = "Photos without album",
46 |                 Created = DateTime.MinValue
47 |             };
48 |             this.Photo = photo
49 |                 .OrderBy(a => a.CreateTime)
50 |                 .ToList();
51 |         }
52 | 
53 |         
54 |     }
55 | }


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/CommentsHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VkNet.Model;
 7 | using VkNet.Model.Attachments;
 8 | using VOffline.Services.Storage;
 9 | using VOffline.Services.Vk;
10 | 
11 | namespace VOffline.Services.Handlers
12 | {
13 |     public class CommentsHandler : HandlerBase<Post>
14 |     {
15 |         private readonly VkApiUtils vkApiUtils;
16 |         private readonly IHandler<Comment> commentHandler;
17 | 
18 |         public CommentsHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<Comment> commentHandler) : base(filesystemTools)
19 |         {
20 |             this.vkApiUtils = vkApiUtils;
21 |             this.commentHandler = commentHandler;
22 |         }
23 | 
24 |         public override async Task ProcessInternal(Post post, DirectoryInfo workDir, CancellationToken token, ILog log)
25 |         {
26 |             var allComments = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Comments(post), 100, token, log);
27 |             log.Debug($"Post {post.Id} has {allComments.Count} comments");
28 |             var byDate = allComments
29 |                 .OrderBy(c => c.Date)
30 |                 .ToList();
31 |             await byDate.SaveHumanReadableText(filesystemTools, workDir, token, log);
32 |             var commentTasks = byDate
33 |                 .Where(c => c.Attachments.Count > 0)
34 |                 .Select(c => commentHandler.Process(c, workDir, token, log));
35 |             await Task.WhenAll(commentTasks);
36 |         }
37 | 
38 |         public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Comments", CreateMode.OverwriteExisting);
39 |     }
40 | }


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/PlaylistHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VOffline.Models.Storage;
 7 | using VOffline.Services.Storage;
 8 | 
 9 | namespace VOffline.Services.Handlers
10 | {
11 |     public class PlaylistHandler : HandlerBase<PlaylistWithAudio>
12 |     {
13 |         private readonly AttachmentProcessor attachmentProcessor;
14 | 
15 |         public PlaylistHandler(FilesystemTools filesystemTools, AttachmentProcessor attachmentProcessor) : base(filesystemTools)
16 |         {
17 |             this.attachmentProcessor = attachmentProcessor;
18 |         }
19 | 
20 |         public override async Task ProcessInternal(PlaylistWithAudio playlistWithAudio, DirectoryInfo workDir, CancellationToken token, ILog log)
21 |         {
22 |             if (!string.IsNullOrWhiteSpace(playlistWithAudio.Playlist.Description))
23 |             {
24 |                 var text = filesystemTools.CreateFile(workDir, $"__description.txt", CreateMode.OverwriteExisting);
25 |                 await File.WriteAllTextAsync(text.FullName, playlistWithAudio.Playlist.Description, token);
26 |             }
27 | 
28 |             if (playlistWithAudio.Playlist.Cover != null)
29 |             {
30 |                 await attachmentProcessor.ProcessAttachment(playlistWithAudio.Playlist.Cover, workDir, token, log);
31 |             }
32 |             
33 |             var attachmentTasks = playlistWithAudio.Audio.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
34 |             await Task.WhenAll(attachmentTasks);
35 |         }
36 | 
37 |         public override DirectoryInfo GetWorkingDirectory(PlaylistWithAudio playlistWithAudio, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{playlistWithAudio.Playlist.Title}", CreateMode.OverwriteExisting);
38 |     }
39 | }


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/AlbumHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VOffline.Models.Storage;
 7 | using VOffline.Services.Storage;
 8 | 
 9 | namespace VOffline.Services.Handlers
10 | {
11 |     public class AlbumHandler : HandlerBase<AlbumWithPhoto>
12 |     {
13 |         private readonly AttachmentProcessor attachmentProcessor;
14 | 
15 |         public AlbumHandler(FilesystemTools filesystemTools, AttachmentProcessor attachmentProcessor) : base(filesystemTools)
16 |         {
17 |             this.attachmentProcessor = attachmentProcessor;
18 |         }
19 | 
20 |         public override async Task ProcessInternal(AlbumWithPhoto albumWithPhoto, DirectoryInfo workDir, CancellationToken token, ILog log)
21 |         {
22 |             if (!string.IsNullOrWhiteSpace(albumWithPhoto.Album.Description))
23 |             {
24 |                 var text = filesystemTools.CreateFile(workDir, $"__description.txt", CreateMode.OverwriteExisting);
25 |                 await File.WriteAllTextAsync(text.FullName, albumWithPhoto.Album.Description, token);
26 |             }
27 | 
28 |             if (albumWithPhoto.Album.ThumbId != null)
29 |             {
30 |                 var cover = albumWithPhoto.Photo.FirstOrDefault(x => x.Id == albumWithPhoto.Album.ThumbId);
31 |                 if (cover != null)
32 |                 {
33 |                     await attachmentProcessor.ProcessAttachment(cover, -1, workDir, token, log);
34 |                 }
35 |             }
36 | 
37 |             var attachmentTasks = albumWithPhoto.Photo.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
38 |             await Task.WhenAll(attachmentTasks);
39 |         }
40 | 
41 |         public override DirectoryInfo GetWorkingDirectory(AlbumWithPhoto albumWithPhoto, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{albumWithPhoto.Album.Title}", CreateMode.OverwriteExisting);
42 |     }
43 | }


--------------------------------------------------------------------------------
/VOffline/VOffline.csproj:
--------------------------------------------------------------------------------
 1 | <Project Sdk="Microsoft.NET.Sdk">
 2 | 
 3 |   <PropertyGroup>
 4 |     <OutputType>Exe</OutputType>
 5 |     <TargetFramework>netcoreapp2.1</TargetFramework>
 6 |     <LangVersion>latest</LangVersion>
 7 |     <RootNamespace>VOffline</RootNamespace>
 8 |     <UserSecretsId>93680df2-87fc-4326-b623-001d5725f6ae</UserSecretsId>
 9 |   </PropertyGroup>
10 | 
11 |   <ItemGroup>
12 |     <None Remove="appsettings.json" />
13 |     <None Remove="log4net.config" />
14 |   </ItemGroup>
15 | 
16 |   <ItemGroup>
17 |     <EmbeddedResource Include="appsettings.json">
18 |       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
19 |     </EmbeddedResource>
20 |     <EmbeddedResource Include="log4net.config">
21 |       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
22 |     </EmbeddedResource>
23 |   </ItemGroup>
24 | 
25 |   <ItemGroup>
26 |     <PackageReference Include="log4net" Version="2.0.8" />
27 |     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
28 |     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
29 |     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />
30 |     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.2.0" />
31 |     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.2.0" />
32 |     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.2.0" />
33 |     <PackageReference Include="Microsoft.Extensions.Logging.Log4Net.AspNetCore" Version="2.2.9" />
34 |     <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.2.0" />
35 |     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
36 |     <PackageReference Include="Nito.AsyncEx" Version="5.0.0-pre-05" />
37 |     <PackageReference Include="RestSharp" Version="106.6.5" />
38 |     <PackageReference Include="VkNet" Version="1.41.0" />
39 |   </ItemGroup>
40 | 
41 | </Project>
42 | 


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/PostHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VkNet.Model;
 7 | using VkNet.Model.Attachments;
 8 | using VOffline.Services.Storage;
 9 | 
10 | namespace VOffline.Services.Handlers
11 | {
12 |     public class PostHandler : HandlerBase<Post>
13 |     {
14 |         private readonly AttachmentProcessor attachmentProcessor;
15 |         private readonly CommentsHandler commentsHandler;
16 | 
17 |         public PostHandler(FilesystemTools filesystemTools, AttachmentProcessor attachmentProcessor, CommentsHandler commentsHandler) : base(filesystemTools)
18 |         {
19 |             this.attachmentProcessor = attachmentProcessor;
20 |             this.commentsHandler = commentsHandler;
21 |         }
22 | 
23 |         public override async Task ProcessInternal(Post post, DirectoryInfo workDir, CancellationToken token, ILog log)
24 |         {
25 |             if (!string.IsNullOrWhiteSpace(post.Text))
26 |             {
27 |                 var postText = filesystemTools.CreateFile(workDir, $"text.txt", CreateMode.OverwriteExisting);
28 |                 await File.WriteAllTextAsync(postText.FullName, post.Text, token);
29 |             }
30 | 
31 |             var attachmentTasks = post.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
32 |             await Task.WhenAll(attachmentTasks);
33 | 
34 |             if (post.Comments?.Count > 0)
35 |             {
36 |                 await commentsHandler.Process(post, workDir, token, log);
37 |             }
38 |             
39 |             // recursively walk reposts
40 |             var allRepostTasks = post.CopyHistory
41 |                 .OrderBy(x => x.Date)
42 |                 .Select(repost => Process(repost, workDir, token, log));
43 |             await Task.WhenAll(allRepostTasks);
44 |         }
45 | 
46 |         public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir)
47 |         {
48 |             var dateMaybe = post.Date != null
49 |                 ? $"{post.Date.Value:s}"
50 |                 : "no_date";
51 |             return filesystemTools.CreateSubdir(parentDir, $"{dateMaybe} {post.Id}", CreateMode.OverwriteExisting);
52 |         }
53 |     }
54 | }


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/PhotoHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VOffline.Models.Storage;
 7 | using VOffline.Services.Storage;
 8 | using VOffline.Services.Vk;
 9 | 
10 | namespace VOffline.Services.Handlers
11 | {
12 |     public class PhotoHandler : HandlerBase<long>
13 |     {
14 |         private readonly VkApiUtils vkApiUtils;
15 |         private readonly IHandler<AlbumWithPhoto> albumHandler;
16 | 
17 |         public PhotoHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<AlbumWithPhoto> albumHandler) : base(filesystemTools)
18 |         {
19 |             this.vkApiUtils = vkApiUtils;
20 |             this.albumHandler = albumHandler;
21 |         }
22 | 
23 |         public override async Task ProcessInternal(long id, DirectoryInfo workDir, CancellationToken token, ILog log)
24 |         {
25 |             var vkAlbums = await vkApiUtils.GetAllPagesAsync(vkApiUtils.PhotoAlbums(id), int.MaxValue, token, log);
26 |             var expandTasks = vkAlbums.Select(album => vkApiUtils.ExpandAlbum(album, token, log));
27 |             var albums = await Task.WhenAll(expandTasks);
28 | 
29 |             var allPhotos = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Photos(id), 100, token, log);
30 |             var photosInAlbums = albums
31 |                 .SelectMany(awp => awp.Photo.Select(photo => photo.Id.Value))
32 |                 .ToHashSet();
33 |             var uniquePhotos = allPhotos
34 |                 .Where(photo => !photosInAlbums.Contains(photo.Id.Value))
35 |                 .ToList();
36 |             var defaultAlbum = new AlbumWithPhoto(uniquePhotos);
37 | 
38 |             var allAlbums = albums.ToList();
39 |             if (defaultAlbum.Photo.Any())
40 |             {
41 |                 allAlbums.Add(defaultAlbum);
42 |             }
43 | 
44 |             var allTasks = allAlbums
45 |                 .OrderBy(p => p.Album.Created)
46 |                 .Select(p => albumHandler.Process(p, workDir, token, log));
47 |             await Task.WhenAll(allTasks);
48 |         }
49 | 
50 |         public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Photo", CreateMode.OverwriteExisting);
51 |     }
52 | }


--------------------------------------------------------------------------------
/VOffline/Services/Handlers/AudioHandler.cs:
--------------------------------------------------------------------------------
 1 | using System.IO;
 2 | using System.Linq;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using log4net;
 6 | using VOffline.Models.Storage;
 7 | using VOffline.Services.Storage;
 8 | using VOffline.Services.Vk;
 9 | 
10 | namespace VOffline.Services.Handlers
11 | {
12 |     public class AudioHandler : HandlerBase<long>
13 |     {
14 |         private readonly VkApiUtils vkApiUtils;
15 |         private readonly IHandler<PlaylistWithAudio> playlistHandler;
16 | 
17 |         public AudioHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<PlaylistWithAudio> playlistHandler) : base(filesystemTools)
18 |         {
19 |             this.vkApiUtils = vkApiUtils;
20 |             this.playlistHandler = playlistHandler;
21 |         }
22 | 
23 |         public override async Task ProcessInternal(long id, DirectoryInfo workDir, CancellationToken token, ILog log)
24 |         {
25 |             var vkPlaylists = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Playlists(id), 200, token, log);
26 |             log.Debug($"Audio: {vkPlaylists.Count} playlists");
27 |             var expandTasks = vkPlaylists.Select(p => vkApiUtils.ExpandPlaylist(p, token, log));
28 |             var playlists = await Task.WhenAll(expandTasks);
29 |             log.Debug($"Audio: {playlists.Sum(p => p.Audio.Count)} in {playlists.Length} playlists");
30 | 
31 |             var allAudios = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Audios(id), long.MaxValue, token, log);
32 |             var audioInPlaylists = playlists
33 |                 .SelectMany(p => p.Audio.Select(t => t.Id))
34 |                 .ToHashSet();
35 |             var uniqueAudios = allAudios
36 |                 .Where(a => !audioInPlaylists.Contains(a.Id.Value))
37 |                 .ToList();
38 |             var defaultPlaylist = new PlaylistWithAudio(uniqueAudios);
39 |             log.Debug($"Audio: {allAudios.Count} total, {defaultPlaylist.Audio.Count} unique in default playlist");
40 | 
41 |             var allPlaylists = playlists.ToList();
42 |             allPlaylists.Add(defaultPlaylist);
43 | 
44 |             var allTasks = allPlaylists
45 |                 .OrderBy(p => p.Playlist.CreateTime)
46 |                 .Select(p => playlistHandler.Process(p, workDir, token, log));
47 |             await Task.WhenAll(allTasks);
48 |         }
49 | 
50 |         public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Audio", CreateMode.OverwriteExisting);
51 |     }
52 | }
53 | 


--------------------------------------------------------------------------------
/VOffline/Services/Storage/BackgroundDownloader.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Threading;
 5 | using System.Threading.Tasks;
 6 | using log4net;
 7 | using Nito.AsyncEx;
 8 | using VOffline.Models.Storage;
 9 | 
10 | namespace VOffline.Services.Storage
11 | {
12 |     public class BackgroundDownloader
13 |     {
14 |         private readonly DownloadQueueProvider queueProvider;
15 |         private readonly FilesystemTools filesystemTools;
16 | 
17 |         public BackgroundDownloader(DownloadQueueProvider queueProvider, FilesystemTools filesystemTools)
18 |         {
19 |             this.queueProvider = queueProvider;
20 |             this.filesystemTools = filesystemTools;
21 |         }
22 | 
23 |         public async Task<List<IDownload>> Process(CancellationToken token, ILog log)
24 |         {
25 |             log.Debug($"{nameof(BackgroundDownloader)} started");
26 |             var errors = new AsyncProducerConsumerQueue<IDownload>();
27 |             IDownload item;
28 |             int success;
29 | 
30 |             for (success = 0; (item = await GetItem(token)) != null;)
31 |             {
32 |                 // TODO: add second queue, semaphore and support for retries
33 |                 try
34 |                 {
35 |                     await filesystemTools.WriteFileWithCompletionMark(item.Location, item.DesiredName, async () => await item.GetContent(token), token, log);
36 |                     success++;
37 |                 }
38 |                 catch (OperationCanceledException)
39 |                 {
40 |                     throw;
41 |                 }
42 |                 catch (Exception e)
43 |                 {
44 |                     item.AddError(e);
45 |                     await errors.EnqueueAsync(item, token);
46 |                     log.Warn($"Error downloading {item}", e);
47 |                 }
48 |             }
49 | 
50 |             errors.CompleteAdding();
51 |             var errorList = errors.GetConsumingEnumerable(token).ToList();
52 |             log.Info($"All downloads completed. {success} successful, {errorList.Count} failed");
53 |             return errorList;
54 |         }
55 | 
56 |         private async Task<IDownload> GetItem(CancellationToken token)
57 |         {
58 |             try
59 |             {
60 |                 return await queueProvider.Pending.DequeueAsync(token);
61 |             }
62 |             catch (InvalidOperationException)
63 |             {
64 |             }
65 | 
66 |             return null;
67 |         }
68 |     }
69 | }


--------------------------------------------------------------------------------
/VOffline/Services/Token/Google/ProtobufParser.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using VOffline.Models.Google;
 5 | 
 6 | namespace VOffline.Services.Token.Google
 7 | {
 8 |     public class ProtobufParser
 9 |     {
10 |         private const int IdFieldNumber = 7;
11 |         private const int TokenFieldNumber = 8;
12 | 
13 |         private readonly byte[] data;
14 |         private int offset;
15 | 
16 |         public ProtobufParser(byte[] data)
17 |         {
18 |             this.data = data;
19 |             this.offset = 0;
20 |         }
21 | 
22 |         public GoogleCredentials Parse()
23 |         {
24 |             List<byte> rawId = null;
25 |             long? id = null;
26 |             long? token = null;
27 |             while (offset < data.Length)
28 |             {
29 |                 var (value, length) = VarInt.Read(data, offset);
30 |                 offset += length;
31 |                 var field = new ProtobufField(value);
32 |                 switch (field.Type)
33 |                 {
34 |                     case 0:
35 |                         var (_, i) = VarInt.Read(data, offset);
36 |                         offset += i;
37 |                         break;
38 |                     case 1 when field.FieldNumber == IdFieldNumber:
39 |                         rawId = data.Skip(offset).Take(8).ToList();
40 |                         id = BitConverter.ToInt64(data, offset);
41 |                         offset += 8;
42 |                         break;
43 |                     case 1 when field.FieldNumber == TokenFieldNumber:
44 |                         token = BitConverter.ToInt64(data, offset);
45 |                         offset += 8;
46 |                         break;
47 |                     case 1:
48 |                         offset += 8;
49 |                         break;
50 |                     case 2:
51 |                         var (skip, j) = VarInt.Read(data, offset);
52 |                         offset += skip + j;
53 |                         break;
54 |                     default:
55 |                         throw new InvalidOperationException($"{nameof(ProtobufParser)} unexpected code [{field.Type}]");
56 |                 }
57 |             }
58 | 
59 |             if (offset == data.Length && id == null && token == null)
60 |             {
61 |                 throw new InvalidOperationException($"{nameof(ProtobufParser)} reached end of data, id and token not found");
62 |             }
63 | 
64 |             if (id == null)
65 |             {
66 |                 throw new InvalidOperationException($"{nameof(ProtobufParser)} id not found");
67 |             }
68 | 
69 |             if (token == null)
70 |             {
71 |                 throw new InvalidOperationException($"{nameof(ProtobufParser)}  token not found");
72 |             }
73 | 
74 |             return new GoogleCredentials()
75 |             {
76 |                 Id = id.Value,
77 |                 Token = token.Value,
78 |                 RawId = rawId
79 |             };
80 |         }
81 | 
82 |         
83 |     }
84 | }


--------------------------------------------------------------------------------
/VOffline/Services/VkNetHacks/SimpleLogger.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using log4net;
 3 | using Microsoft.Extensions.Logging;
 4 | using Microsoft.Extensions.Logging.Log4Net.AspNetCore.Extensions;
 5 | 
 6 | namespace VOffline.Services.VkNetHacks
 7 | {
 8 |     public class SimpleLogger : ILogger
 9 |     {
10 |         private readonly ILog log;
11 | 
12 |         public SimpleLogger(ILog log)
13 |         {
14 |             this.log = log;
15 |         }
16 | 
17 |         public string Name => log.Logger.Name;
18 | 
19 |         public IDisposable BeginScope<TState>(TState state)
20 |         {
21 |             return null;
22 |         }
23 | 
24 |         public bool IsEnabled(LogLevel logLevel)
25 |         {
26 |             switch (logLevel)
27 |             {
28 |                 case LogLevel.Trace:
29 |                 case LogLevel.Debug:
30 |                     return this.log.IsDebugEnabled;
31 |                 case LogLevel.Information:
32 |                     return this.log.IsInfoEnabled;
33 |                 case LogLevel.Warning:
34 |                     return this.log.IsWarnEnabled;
35 |                 case LogLevel.Error:
36 |                     return this.log.IsErrorEnabled;
37 |                 case LogLevel.Critical:
38 |                     return this.log.IsFatalEnabled;
39 |                 default:
40 |                     throw new ArgumentOutOfRangeException(nameof(logLevel));
41 |             }
42 |         }
43 | 
44 |         public void Log<TState>(LogLevel logLevel,EventId eventId,TState state,Exception exception,Func<TState, Exception, string> formatter)
45 |         {
46 |             if (!this.IsEnabled(logLevel))
47 |                 return;
48 |             if (formatter == null)
49 |                 throw new ArgumentNullException(nameof(formatter));
50 |             string str = formatter(state, exception);
51 |             if (string.IsNullOrEmpty(str) && exception == null)
52 |                 return;
53 |             switch (logLevel)
54 |             {
55 |                 case LogLevel.Trace:
56 |                     this.log.Trace((object)str, exception);
57 |                     break;
58 |                 case LogLevel.Debug:
59 |                     this.log.Debug((object)str, exception);
60 |                     break;
61 |                 case LogLevel.Information:
62 |                     this.log.Info((object)str, exception);
63 |                     break;
64 |                 case LogLevel.Warning:
65 |                     this.log.Warn((object)str, exception);
66 |                     break;
67 |                 case LogLevel.Error:
68 |                     this.log.Error((object)str, exception);
69 |                     break;
70 |                 case LogLevel.Critical:
71 |                     this.log.Fatal((object)str, exception);
72 |                     break;
73 |                 default:
74 |                     this.log.Warn((object)string.Format("Encountered unknown log level {0}, writing out as Info.", (object)logLevel));
75 |                     this.log.Info((object)str, exception);
76 |                     break;
77 |             }
78 |         }
79 |     }
80 | }


--------------------------------------------------------------------------------
/VOffline/Services/VkNetHacks/CancellableConstraint.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Threading;
 4 | using System.Threading.Tasks;
 5 | using VkNet.Abstractions.Core;
 6 | using VkNet.Utils;
 7 | 
 8 | namespace VOffline.Services.VkNetHacks
 9 | {
10 |     public class CancellableConstraint : IAwaitableConstraint
11 |     {
12 |         private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1);
13 |         private readonly object _sync = new object();
14 |         private readonly LimitedSizeStack<DateTime> _timeStamps;
15 |         private int _count;
16 |         private TimeSpan _timeSpan;
17 |         private readonly CancellationToken token;
18 | 
19 |         public CancellableConstraint(int number, TimeSpan timeSpan, CancellationToken token)
20 |         {
21 |             if (number <= 0)
22 |                 throw new ArgumentException("count should be strictly positive", nameof(number));
23 |             if (timeSpan.TotalMilliseconds <= 0.0)
24 |                 throw new ArgumentException("timeSpan should be strictly positive", nameof(timeSpan));
25 |             this._count = number;
26 |             this._timeSpan = timeSpan;
27 |             this.token = token;
28 |             this._timeStamps = new LimitedSizeStack<DateTime>(this._count);
29 |         }
30 | 
31 |         public async Task<IDisposable> WaitForReadiness(CancellationToken cancellationToken)
32 |         {
33 | 
34 |             using (var cts = CancellationTokenSource.CreateLinkedTokenSource(token, cancellationToken))
35 |             {
36 | 
37 |                 CancellableConstraint awaitableConstraint = this;
38 |                 await awaitableConstraint._semaphore.WaitAsync(cts.Token);
39 |                 int num = 0;
40 |                 DateTime now = DateTime.Now;
41 |                 DateTime dateTime = now - awaitableConstraint._timeSpan;
42 |                 LinkedListNode<DateTime> linkedListNode1 = awaitableConstraint._timeStamps.First;
43 |                 LinkedListNode<DateTime> linkedListNode2 = (LinkedListNode<DateTime>) null;
44 |                 while (linkedListNode1 != null && linkedListNode1.Value > dateTime)
45 |                 {
46 |                     linkedListNode2 = linkedListNode1;
47 |                     linkedListNode1 = linkedListNode1.Next;
48 |                     ++num;
49 |                 }
50 | 
51 |                 if (num < awaitableConstraint._count)
52 |                     return (IDisposable) new DisposableAction(new Action(awaitableConstraint.OnEnded));
53 |                 TimeSpan delay = linkedListNode2.Value.Add(awaitableConstraint._timeSpan) - now;
54 |                 try
55 |                 {
56 |                     await Task.Delay(delay, cts.Token);
57 |                 }
58 |                 catch (Exception)
59 |                 {
60 |                     awaitableConstraint._semaphore.Release();
61 |                     throw;
62 |                 }
63 | 
64 |                 return (IDisposable) new DisposableAction(new Action(awaitableConstraint.OnEnded));
65 |             }
66 |         }
67 | 
68 |         /// <inheritdoc />
69 |         public void SetRate(int number, TimeSpan timeSpan)
70 |         {
71 |             lock (this._sync)
72 |             {
73 |                 this._count = number;
74 |                 this._timeSpan = timeSpan;
75 |             }
76 |         }
77 | 
78 |         private void OnEnded()
79 |         {
80 |             this._timeStamps.Push(DateTime.Now);
81 |             this._semaphore.Release();
82 |         }
83 |     }
84 | }


--------------------------------------------------------------------------------
/VOffline/Services/Token/Google/MTalk.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Linq;
 3 | using System.Net.Security;
 4 | using System.Net.Sockets;
 5 | using System.Text;
 6 | using log4net;
 7 | using VOffline.Models.Google;
 8 | 
 9 | namespace VOffline.Services.Token.Google
10 | {
11 |     public class MTalk
12 |     {
13 |         public void SendRequest(GoogleCredentials googleCredentials, ILog log)
14 |         {
15 |             log.Debug($"mtalk build request");
16 |             var request = BuildMTalkRequest(googleCredentials);
17 |             log.Debug($"mtalk tcp connect");
18 |             using (var tcpClient = new TcpClient(Host, Port))
19 |             {
20 |                 using (var sslStream = new SslStream(tcpClient.GetStream(), false, (sender, certificate, chain, errors) => true, null))
21 |                 {
22 |                     log.Debug($"mtalk ssl authenticate");
23 |                     sslStream.AuthenticateAsClient(Host);
24 |                     log.Debug($"mtalk send request");
25 |                     sslStream.Write(request);
26 |                     sslStream.Flush();
27 |                     log.Debug($"mtalk read response");
28 |                     sslStream.ReadByte();
29 |                     var responseCode = sslStream.ReadByte();
30 |                     if (responseCode != SuccessCode)
31 |                     {
32 |                         throw new InvalidOperationException($"MTalk expected response code [{SuccessCode}], got [{responseCode}]");
33 |                     }
34 |                 }
35 |             }
36 |         }
37 | 
38 |         private byte[] BuildMTalkRequest(GoogleCredentials googleCredentials)
39 |         {
40 |             var idStringBytes = Encoding.ASCII.GetBytes(googleCredentials.Id.ToString());
41 |             var idLen = VarInt.Write(idStringBytes.Length).ToList();
42 |             
43 |             var tokenStringByes = Encoding.ASCII.GetBytes(googleCredentials.Token.ToString());
44 |             var tokenLen = VarInt.Write(tokenStringByes.Length).ToList();
45 | 
46 |             var hexId = "android-" + BitConverter.ToString(googleCredentials.RawId.ToArray()).Replace("-", string.Empty).ToLowerInvariant();
47 |             var hexIdBytes = Encoding.ASCII.GetBytes(hexId);
48 |             var hexIdLen = VarInt.Write(hexIdBytes.Length);
49 | 
50 |             var body = message1
51 |                 .Concat(idLen)
52 |                 .Concat(idStringBytes)
53 |                 .Concat(message2)
54 |                 .Concat(idLen)
55 |                 .Concat(idStringBytes)
56 |                 .Concat(message3)
57 |                 .Concat(tokenLen)
58 |                 .Concat(tokenStringByes)
59 |                 .Concat(message4)
60 |                 .Concat(hexIdLen)
61 |                 .Concat(hexIdBytes)
62 |                 .Concat(message5)
63 |                 .ToList();
64 | 
65 |             var bodyLen = VarInt.Write(body.Count);
66 |             return message6
67 |                 .Concat(bodyLen)
68 |                 .Concat(body)
69 |                 .ToArray();
70 |         }
71 | 
72 |         
73 | 
74 |         private byte[] message1 = { 0x0a, 0x0a, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2d, 0x31, 0x39, 0x12, 0x0f, 0x6d, 0x63, 0x73, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x63, 0x6f, 0x6d, 0x1a };
75 |         private byte[] message2 = { 0x22 };
76 |         private byte[] message3 = { 0x2a };
77 |         private byte[] message4 = { 0x32 };
78 |         private byte[] message5 = { 0x42, 0x0b, 0x0a, 0x06, 0x6e, 0x65, 0x77, 0x5f, 0x76, 0x63, 0x12, 0x01, 0x31, 0x60, 0x00, 0x70, 0x01, 0x80, 0x01, 0x02, 0x88, 0x01, 0x01 };
79 |         private byte[] message6 = { 0x29, 0x02 };
80 | 
81 |         private const int SuccessCode = 3;
82 |         private const string Host = "mtalk.google.com";
83 |         private const int Port = 5228;
84 |     }
85 | }
86 | 


--------------------------------------------------------------------------------
/VOffline/Services/Token/Vk/VkHttpRequests.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using log4net;
 3 | using Microsoft.Extensions.Options;
 4 | using Newtonsoft.Json;
 5 | using RestSharp;
 6 | using VOffline.Models;
 7 | using VOffline.Models.Vk;
 8 | using RestClient = RestSharp.RestClient;
 9 | 
10 | namespace VOffline.Services.Token.Vk
11 | {
12 |     public class VkHttpRequests
13 |     {
14 |         private readonly string userAgent;
15 |         private VkCredentials VkCredentials { get; }
16 | 
17 |         public VkHttpRequests(IOptionsSnapshot<VkCredentials> vkCredentials, IOptionsSnapshot<Settings> settings)
18 |         {
19 |             userAgent = settings.Value.UserAgent;
20 |             VkCredentials = vkCredentials.Value;
21 |         }
22 | 
23 |         public string GetNonRefreshedToken(ILog log)
24 |         {
25 |             var client = GetClient("https://oauth.vk.com/token");
26 |             var request = new RestRequest(Method.GET);
27 |             request.AddQueryParameter("grant_type", "password");
28 |             request.AddQueryParameter("client_id", ClientId.ToString());
29 |             request.AddQueryParameter("client_secret", "lxhD8OD7dMsqtXIm5IUY");
30 |             request.AddQueryParameter("username", VkCredentials.Login);
31 |             request.AddQueryParameter("password", VkCredentials.Password);
32 |             request.AddQueryParameter("v", VkApiVersion);
33 |             request.AddQueryParameter("scope", Scope);
34 |             log.Debug($"request non-refreshed token");
35 |             var response = client.Execute(request);
36 |             if (!response.IsSuccessful)
37 |             {
38 |                 throw new InvalidOperationException($"{nameof(GetNonRefreshedToken)} bad response: {response.StatusCode}\n{response.Content}");
39 |             }
40 | 
41 |             var json = JsonConvert.DeserializeObject<dynamic>(response.Content);
42 |             if (json.user_id == null)
43 |             {
44 |                 throw new InvalidOperationException($"{nameof(GetNonRefreshedToken)} bad response: user_id is null\n{response.Content}");
45 |             }
46 | 
47 |             return (string) json.access_token;
48 |         }
49 | 
50 |         public string RefreshToken(string token, string receipt, ILog log)
51 |         {
52 |             var client = GetClient("https://api.vk.com/method/auth.refreshToken");
53 |             var request = new RestRequest(Method.GET);
54 |             request.AddQueryParameter("access_token", token);
55 |             request.AddQueryParameter("receipt", receipt);
56 |             request.AddQueryParameter("v", VkApiVersion);
57 |             log.Debug($"refresh token");
58 |             var response = client.Execute(request);
59 |             if (!response.IsSuccessful)
60 |             {
61 |                 throw new InvalidOperationException($"{nameof(RefreshToken)} bad response: {response.StatusCode}\n{response.Content}");
62 |             }
63 | 
64 |             var json = JsonConvert.DeserializeObject<dynamic>(response.Content);
65 |             var newToken = (string)json.response.token;
66 |             if (newToken == token)
67 |             {
68 |                 throw new InvalidOperationException($"{nameof(RefreshToken)} bad response: token not refreshed\n{response.Content}");
69 |             }
70 | 
71 |             return newToken;
72 |         }
73 | 
74 |         private RestClient GetClient(string url)
75 |         {
76 |             var client = new RestClient(url)
77 |             {
78 |                 UserAgent = userAgent
79 |             };
80 |             client.RemoteCertificateValidationCallback += (sender, certificate, chain, errors) => true;
81 |             return client;
82 |         }
83 | 
84 |         
85 | 
86 |         public const string VkApiVersion = "5.72";
87 |         public const ulong ClientId = 2685278;
88 |         public const string Scope = "audio,offline,notify,friends,photos,video,stories,pages,status,notes,messages,wall,ads,docs,groups,notifications,stats,email,market";
89 |     }
90 | }
91 | 


--------------------------------------------------------------------------------
/VOffline/Services/Logic.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Immutable;
  3 | using System.IO;
  4 | using System.Linq;
  5 | using System.Threading;
  6 | using System.Threading.Tasks;
  7 | using log4net;
  8 | using Microsoft.Extensions.Options;
  9 | using Newtonsoft.Json;
 10 | using VkNet;
 11 | using VkNet.Model;
 12 | using VOffline.Models;
 13 | using VOffline.Services.Handlers;
 14 | using VOffline.Services.Storage;
 15 | using VOffline.Services.Token;
 16 | using VOffline.Services.Vk;
 17 | 
 18 | namespace VOffline.Services
 19 | {
 20 |     public class Logic
 21 |     {
 22 |         private readonly Settings settings;
 23 |         private readonly TokenMagic tokenMagic;
 24 |         private readonly VkApi vkApi;
 25 |         private readonly VkApiUtils vkApiUtils;
 26 |         private readonly BackgroundDownloader downloader;
 27 |         private readonly FilesystemTools filesystemTools;
 28 |         private readonly DownloadQueueProvider queueProvider;
 29 |         private readonly WallHandler wallHandler;
 30 |         private readonly AudioHandler audioHandler;
 31 |         private readonly PhotoHandler photoHandler;
 32 | 
 33 |         public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, BackgroundDownloader downloader, FilesystemTools filesystemTools, DownloadQueueProvider queueProvider, WallHandler wallHandler, AudioHandler audioHandler, PhotoHandler photoHandler, IOptionsSnapshot<Settings> settings)
 34 |         {
 35 |             this.settings = settings.Value;
 36 |             this.tokenMagic = tokenMagic;
 37 |             this.vkApi = vkApi;
 38 |             this.vkApiUtils = vkApiUtils;
 39 |             this.downloader = downloader;
 40 |             this.filesystemTools = filesystemTools;
 41 |             this.queueProvider = queueProvider;
 42 |             this.wallHandler = wallHandler;
 43 |             this.audioHandler = audioHandler;
 44 |             this.photoHandler = photoHandler;
 45 |         }
 46 | 
 47 |         public async Task Run(CancellationToken token, ILog log)
 48 |         {
 49 |             log.Debug("Started");
 50 | 
 51 |             // TODO: async token retrieval
 52 |             var vkToken = tokenMagic.GetTokenFromScratch(log);
 53 | 
 54 |             await vkApi.AuthorizeAsync(new ApiAuthParams()
 55 |             {
 56 |                 AccessToken = vkToken.Token
 57 |             });
 58 | 
 59 | 
 60 |             var modes = settings.GetWorkingModes();
 61 |             var idTasks = settings.Targets
 62 |                 .Select(async x =>
 63 |                 {
 64 |                     var id = await vkApiUtils.ResolveId(x);
 65 |                     return (name: x, id: id);
 66 |                 });
 67 |             var allIds = await Task.WhenAll(idTasks);
 68 |             var identifiers = allIds
 69 |                 .Distinct()
 70 |                 .ToImmutableList();
 71 |             log.Debug($"Processing {JsonConvert.SerializeObject(modes)} for {string.Join(", ", identifiers.Select(x => $"[{x.name} {x.id}]"))}");
 72 |             filesystemTools.LoadCache(log);
 73 |             var downloaderTask = downloader.Process(token, log);
 74 | 
 75 |             try
 76 |             {
 77 |                 foreach (var identifier in identifiers)
 78 |                 {
 79 |                     var name = await vkApiUtils.GetName(identifier.id);
 80 |                     var workDir = filesystemTools.CreateSubdir(filesystemTools.RootDir, name, CreateMode.OverwriteExisting);
 81 |                     log.Info($"id [{identifier.id}], name [{name}], path [{workDir.FullName}]");
 82 |                     foreach (var mode in modes)
 83 |                     {
 84 |                         await ProcessTarget(identifier.id, workDir, mode, token, log);
 85 |                     }
 86 |                 }
 87 | 
 88 |                 queueProvider.Pending.CompleteAdding();
 89 |                 var downloadErrors = await downloaderTask;
 90 |                 foreach (var downloadError in downloadErrors)
 91 |                 {
 92 |                     log.Warn($"Failed {downloadError.DesiredName}", downloadError.Errors.LastOrDefault());
 93 |                 }
 94 |             }
 95 |             catch (TaskCanceledException)
 96 |             {
 97 |                 queueProvider.Pending.CompleteAdding();
 98 |                 log.Warn($"Abandoned {queueProvider.Pending.GetConsumingEnumerable().Count()} pending downloads");
 99 |             }
100 |             finally
101 |             {
102 |                 filesystemTools.SaveCache(log);
103 |             }
104 |         }
105 | 
106 |         private async Task ProcessTarget(long id, DirectoryInfo dir, Mode mode, CancellationToken token, ILog log)
107 |         {
108 |             // TODO: save raw requests/responses for future use?
109 |             switch (mode)
110 |             {
111 |                 case Mode.Wall:
112 |                     await wallHandler.Process(id, dir, token, log);
113 |                     break;
114 |                 case Mode.Audio:
115 |                     await audioHandler.Process(id, dir, token, log);
116 |                     break;
117 |                 case Mode.Photos:
118 |                     await photoHandler.Process(id, dir, token, log);
119 |                     break;
120 |                 case Mode.All:
121 |                     throw new ArgumentOutOfRangeException(nameof(mode), mode, "This mode should have been replaced before processing");
122 |                 default:
123 |                     throw new NotImplementedException();
124 |             }
125 |         }
126 | 
127 |        
128 |     }
129 | }


--------------------------------------------------------------------------------
/VOffline/Services/Storage/AttachmentProcessor.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.IO;
  3 | using System.Threading;
  4 | using System.Threading.Tasks;
  5 | using log4net;
  6 | using Microsoft.Extensions.DependencyInjection;
  7 | using Newtonsoft.Json;
  8 | using VkNet;
  9 | using VkNet.Model;
 10 | using VkNet.Model.Attachments;
 11 | using VOffline.Models.Storage;
 12 | using VOffline.Services.Handlers;
 13 | using VOffline.Services.Vk;
 14 | 
 15 | namespace VOffline.Services.Storage
 16 | {
 17 |     public class AttachmentProcessor
 18 |     {
 19 |         public IServiceProvider ServiceProvider { get; }
 20 |         private readonly VkApi vkApi;
 21 |         private readonly VkApiUtils vkApiUtils;
 22 |         private readonly FilesystemTools filesystemTools;
 23 |         private readonly DownloadQueueProvider downloadQueueProvider;
 24 | 
 25 |         public AttachmentProcessor(VkApi vkApi, VkApiUtils vkApiUtils, FilesystemTools filesystemTools, DownloadQueueProvider downloadQueueProvider, IServiceProvider serviceProvider)
 26 |         {
 27 |             ServiceProvider = serviceProvider;
 28 |             this.vkApi = vkApi;
 29 |             this.vkApiUtils = vkApiUtils;
 30 |             this.filesystemTools = filesystemTools;
 31 |             this.downloadQueueProvider = downloadQueueProvider;
 32 |         }
 33 | 
 34 |         public async Task ProcessAttachment(Attachment attachment, int number, DirectoryInfo workDir, CancellationToken token, ILog log)
 35 |         {
 36 |             await ProcessAttachment(attachment.Instance, number, workDir, token, log);
 37 |         }
 38 | 
 39 |         public async Task ProcessAttachment(MediaAttachment mediaAttachment, int number, DirectoryInfo workDir, CancellationToken token, ILog log)
 40 |         {
 41 |             switch (mediaAttachment)
 42 |             {
 43 |                 case VkNet.Model.Attachments.Photo photo:
 44 |                     await downloadQueueProvider.EnqueueAll(photo.ToDownloads(number, filesystemTools, workDir, log), token);
 45 |                     await photo.SaveText(number, filesystemTools, workDir, token, log);
 46 |                     break;
 47 |                 case VkNet.Model.Attachments.Audio audio:
 48 |                     await downloadQueueProvider.EnqueueAll(audio.ToDownloads(number, filesystemTools, workDir, log), token);
 49 |                     await audio.SaveLyrics(number, vkApi, filesystemTools, workDir, token, log);
 50 |                     break;
 51 |                 case VkNet.Model.Attachments.Document document:
 52 |                     await downloadQueueProvider.EnqueueAll(document.ToDownloads(number, filesystemTools, workDir, log), token);
 53 |                     break;
 54 |                 case VkNet.Model.Attachments.Poll poll:
 55 |                     await downloadQueueProvider.EnqueueAll(poll.ToDownloads(number, filesystemTools, workDir, log), token);
 56 |                     await poll.SaveHumanReadableText(number, filesystemTools, workDir, token, log);
 57 |                     break;
 58 |                 case VkNet.Model.Attachments.Link link:
 59 |                     await downloadQueueProvider.EnqueueAll(link.ToDownloads(number, filesystemTools, workDir, log), token);
 60 |                     await link.SaveHumanReadableText(number, filesystemTools, workDir, token, log);
 61 |                     break;
 62 |                 case VkNet.Model.Attachments.AudioPlaylist audioPlaylist:
 63 |                     var playlistWithAudio = await vkApiUtils.ExpandPlaylist(audioPlaylist, token, log);
 64 |                     var playlistHandler = ServiceProvider.GetRequiredService<IHandler<PlaylistWithAudio>>();
 65 |                     await playlistHandler.Process(playlistWithAudio, workDir, token, log);
 66 |                     break;
 67 |                 case VkNet.Model.Attachments.Album album:
 68 |                     var albumWithPhoto = await vkApiUtils.ExpandAlbum(album, token, log);
 69 |                     var albumHandler = ServiceProvider.GetRequiredService<IHandler<AlbumWithPhoto>>();
 70 |                     await albumHandler.Process(albumWithPhoto, workDir, token, log);
 71 |                     break;
 72 |                 case VkNet.Model.Attachments.Video video:
 73 |                     await downloadQueueProvider.EnqueueAll(video.ToDownloads(number, filesystemTools, workDir, log), token);
 74 |                     break;
 75 | 
 76 |                 case VkNet.Model.Attachments.Note note:  // note и page похожи
 77 |                 case VkNet.Model.Attachments.Page page:
 78 | 
 79 |                 
 80 | 
 81 |                 // остальное похоже на хлам
 82 |                 case VkNet.Model.Attachments.ApplicationContent applicationContent:
 83 |                 case VkNet.Model.Attachments.AudioMessage audioMessage:
 84 |                 case VkNet.Model.Attachments.Gift gift:
 85 |                 case VkNet.Model.Attachments.Graffiti graffiti:
 86 |                 case VkNet.Model.Attachments.MarketAlbum marketAlbum:
 87 |                 case VkNet.Model.Attachments.PrettyCards prettyCards:
 88 |                 case VkNet.Model.Attachments.Sticker sticker:
 89 |                 case VkNet.Model.Attachments.StringLink stringLink:
 90 |                 case VkNet.Model.Attachments.WallReply wallReply:
 91 |                 default:
 92 |                     log.Warn($"Not yet supported: attachment [{mediaAttachment.GetType().FullName}] {JsonConvert.SerializeObject(mediaAttachment)}");
 93 |                     break;
 94 |             }
 95 |         }
 96 | 
 97 |         public async Task ProcessAttachment(AudioCover audioCover, DirectoryInfo workDir, CancellationToken token, ILog log)
 98 |         {
 99 |             await downloadQueueProvider.EnqueueAll(audioCover.ToDownloads(filesystemTools, workDir, log), token);
100 |         }
101 |     }
102 | }


--------------------------------------------------------------------------------
/VOffline/Services/VkNetHacks/CustomRestClient.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.Linq;
  4 | using System.Net;
  5 | using System.Net.Http;
  6 | using System.Threading;
  7 | using System.Threading.Tasks;
  8 | using Microsoft.Extensions.Logging;
  9 | using Microsoft.Extensions.Options;
 10 | using Newtonsoft.Json;
 11 | using VkNet.Abstractions.Utils;
 12 | using VkNet.Utils;
 13 | using VOffline.Models;
 14 | 
 15 | namespace VOffline.Services.VkNetHacks
 16 | {
 17 |     /// <summary>
 18 |     /// Retries failed requests, supports cancellation token and custom user agent.
 19 |     /// </summary>
 20 |     public class CustomRestClient : IRestClient
 21 |     {
 22 |         private TimeSpan timeoutSeconds;
 23 |         private readonly ILogger<RestClient> logger;
 24 |         private readonly CancellationToken token;
 25 |         private readonly string userAgent;
 26 |         private readonly int retryMaxCount;
 27 |         private readonly TimeSpan retryDelay;
 28 | 
 29 |         public CustomRestClient(ILogger<RestClient> logger, IWebProxy proxy, IOptionsSnapshot<Settings> settings, CancellationTokenSource cancellationTokenSource)
 30 |         {
 31 |             this.logger = logger;
 32 |             this.token = cancellationTokenSource.Token;
 33 |             Proxy = proxy;
 34 |             userAgent = settings.Value.UserAgent;
 35 |             retryMaxCount = settings.Value.RequestRetryCount;
 36 |             retryDelay = settings.Value.RequestRetryDelay;
 37 |         }
 38 | 
 39 |         public IWebProxy Proxy { get; set; }
 40 | 
 41 |         public TimeSpan Timeout
 42 |         {
 43 |             get => timeoutSeconds == TimeSpan.Zero ? TimeSpan.FromSeconds(300) : timeoutSeconds;
 44 |             set => timeoutSeconds = value;
 45 |         }
 46 | 
 47 |         public Task<HttpResponse<string>> GetAsync(Uri uri,IEnumerable<KeyValuePair<string, string>> parameters)
 48 |         {
 49 |             var queries = parameters
 50 |                 .Where(parameter => !string.IsNullOrWhiteSpace(parameter.Value))
 51 |                 .Select(parameter => $"{parameter.Key.ToLowerInvariant()}={parameter.Value}");
 52 |             var url = new UriBuilder(uri)
 53 |             {
 54 |                 Query = string.Join("&", queries)
 55 |             };
 56 |             logger?.LogDebug($"GET request: {url.Uri}");
 57 |             Task<HttpResponseMessage> HttpFunc(HttpClient httpClient)
 58 |             {
 59 |                 var request = new HttpRequestMessage(HttpMethod.Get, uri);
 60 |                 return httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
 61 |             }
 62 | 
 63 |             return CallWithRetry(HttpFunc);
 64 |         }
 65 | 
 66 |         public Task<HttpResponse<string>> PostAsync(Uri uri,IEnumerable<KeyValuePair<string, string>> parameters)
 67 |         {
 68 |             var parametersList = parameters.ToList();
 69 |             logger?.LogDebug($"POST request: {uri}{Environment.NewLine}{Utilities.PrettyPrintJson(JsonConvert.SerializeObject(parametersList))}");
 70 |             Task<HttpResponseMessage> HttpFunc(HttpClient httpClient)
 71 |             {
 72 |                 var content = new FormUrlEncodedContent(parametersList);
 73 |                 var request = new HttpRequestMessage(HttpMethod.Post, uri) {Content = content};
 74 |                 return httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
 75 |             }
 76 | 
 77 |             return CallWithRetry(HttpFunc);
 78 |         }
 79 | 
 80 |         private async Task<HttpResponse<string>> CallWithRetry(Func<HttpClient, Task<HttpResponseMessage>> method)
 81 |         {
 82 |             var errors = new Lazy<List<Exception>>();
 83 |             for (var i = 1; i <= retryMaxCount; i++)
 84 |             {
 85 |                 try
 86 |                 {
 87 |                     var result = await Call(method);
 88 |                     // inspect for bad responses. yes this is wrong to do it here but so convenient!
 89 |                     var json = result.Message ?? result.Value;
 90 |                     VkErrors.IfErrorThrowException(json);
 91 |                     return result;
 92 |                 }
 93 |                 catch (OperationCanceledException)
 94 |                 {
 95 |                     throw;
 96 |                 }
 97 |                 catch (Exception e)
 98 |                 {
 99 |                     errors.Value.Add(e);
100 |                     logger?.LogWarning($"Failed vk request, attempt {i}/{retryMaxCount}", e);
101 |                 }
102 | 
103 |                 await Task.Delay(retryDelay, token);
104 |             }
105 |             logger?.LogError($"Failed vk request after {retryMaxCount} attempts");
106 |             throw new Exception(string.Join("\n---------------------------------\n", errors.Value.Select(x => x.ToString())));
107 |         }
108 | 
109 |         private async Task<HttpResponse<string>> Call(Func<HttpClient, Task<HttpResponseMessage>> method)
110 |         {
111 |             var useProxyCondition = Proxy != null;
112 | 
113 |             if (useProxyCondition)
114 |             {
115 |                 logger?.LogDebug($"Use Proxy: {Proxy}");
116 |             }
117 |             var handler = new HttpClientHandler
118 |             {
119 |                 Proxy = Proxy,
120 |                 UseProxy = useProxyCondition
121 |             };
122 | 
123 |             using (var client = new HttpClient(handler) { Timeout = Timeout })
124 |             {
125 |                 client.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
126 |                 var response = await method(client).ConfigureAwait(false);
127 |                 var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
128 |                 logger?.LogDebug($"Response:{Environment.NewLine}{Utilities.PrettyPrintJson(content)}");
129 |                 var url = response.RequestMessage.RequestUri.ToString();
130 |                 return response.IsSuccessStatusCode
131 |                     ? HttpResponse<string>.Success(response.StatusCode, content, url)
132 |                     : HttpResponse<string>.Fail(response.StatusCode, content, url);
133 |             }
134 |         }
135 |     }
136 | }
137 | 


--------------------------------------------------------------------------------
/VOffline/Program.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.IO;
  4 | using System.Reflection;
  5 | using System.Threading;
  6 | using System.Threading.Tasks;
  7 | using log4net;
  8 | using log4net.Config;
  9 | using Microsoft.Extensions.Configuration;
 10 | using Microsoft.Extensions.DependencyInjection;
 11 | using Microsoft.Extensions.Logging;
 12 | using Microsoft.Extensions.Options;
 13 | using Newtonsoft.Json;
 14 | using Newtonsoft.Json.Converters;
 15 | using VkNet;
 16 | using VkNet.Abstractions.Core;
 17 | using VkNet.Abstractions.Utils;
 18 | using VkNet.Model;
 19 | using VkNet.Model.Attachments;
 20 | using VOffline.Models;
 21 | using VOffline.Models.Google;
 22 | using VOffline.Models.Storage;
 23 | using VOffline.Models.Vk;
 24 | using VOffline.Services;
 25 | using VOffline.Services.Handlers;
 26 | using VOffline.Services.Storage;
 27 | using VOffline.Services.Token;
 28 | using VOffline.Services.Token.Google;
 29 | using VOffline.Services.Token.Vk;
 30 | using VOffline.Services.Vk;
 31 | using VOffline.Services.VkNetHacks;
 32 | 
 33 | namespace VOffline
 34 | {
 35 | 
 36 |     public class Program
 37 |     {
 38 |         public static async Task<int> Main(string[] args)
 39 |         {
 40 |             var log = ConfigureLog4Net();
 41 | 
 42 |             try
 43 |             {
 44 |                 var cts = CreateCancellationTokenSource();
 45 |                 ConfigureJsonSerializer();
 46 | 
 47 |                 var serviceCollection = new ServiceCollection();
 48 | 
 49 |                 var configuration = new ConfigurationBuilder()
 50 |                     .AddJsonFile("appsettings.json")
 51 |                     .AddUserSecrets<Settings>()
 52 |                     .AddUserSecrets<VkCredentials>()
 53 |                     .Build();
 54 |                 serviceCollection.Configure<Settings>(configuration.GetSection("Settings"));
 55 |                 serviceCollection.Configure<VkCredentials>(configuration.GetSection("VkCredentials"));
 56 | 
 57 |                 serviceCollection.AddSingleton<FileCache<VkToken>>();
 58 |                 serviceCollection.AddSingleton<FileCache<GoogleCredentials>>();
 59 |                 serviceCollection.AddSingleton<FileCache<GoogleCheckIn>>();
 60 |                 serviceCollection.AddSingleton<Random>();
 61 |                 serviceCollection.AddSingleton<GoogleHttpRequests>();
 62 |                 serviceCollection.AddSingleton<VkHttpRequests>();
 63 |                 serviceCollection.AddSingleton<VkApiUtils>();
 64 |                 serviceCollection.AddSingleton<VkApi>(s => CreateVkApi(s, cts, log));
 65 |                 serviceCollection.AddSingleton<FilesystemTools>();
 66 |                 serviceCollection.AddSingleton<DownloadQueueProvider>();
 67 |                 serviceCollection.AddSingleton<BackgroundDownloader>();
 68 | 
 69 |                 serviceCollection.AddSingleton<AudioHandler>();
 70 |                 serviceCollection.AddSingleton<PhotoHandler>();
 71 |                 serviceCollection.AddSingleton<WallHandler>();
 72 |                 serviceCollection.AddSingleton<CommentsHandler>();
 73 |                 serviceCollection.AddSingleton<IHandler<Post>, PostHandler>();
 74 |                 serviceCollection.AddSingleton<IHandler<Comment>, CommentHandler>();
 75 |                 serviceCollection.AddSingleton<IHandler<PlaylistWithAudio>, PlaylistHandler>();
 76 |                 serviceCollection.AddSingleton<IHandler<AlbumWithPhoto>, AlbumHandler>();
 77 |                 serviceCollection.AddSingleton<AttachmentProcessor>();
 78 |                 serviceCollection.AddSingleton<IServiceProvider>(s => s);  // hack to avoid circular deps between AttachmentProcessor and Handlers
 79 | 
 80 |                 serviceCollection.AddTransient(provider => LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program)));
 81 |                 serviceCollection.AddTransient<AndroidAuth>();
 82 |                 serviceCollection.AddTransient<VkTokenReceiver>();
 83 |                 serviceCollection.AddTransient<MTalk>();
 84 |                 serviceCollection.AddTransient<TokenMagic>();
 85 |                 serviceCollection.AddTransient<Logic>();
 86 | 
 87 |                 var services = serviceCollection.BuildServiceProvider();
 88 |                 await services.GetRequiredService<Logic>().Run(cts.Token, log);
 89 |                 return 0;
 90 |             }
 91 |             catch (Exception e)
 92 |             {
 93 |                 log.Fatal(e);
 94 |                 return -1;
 95 |             }
 96 |         }
 97 | 
 98 |         private static VkApi CreateVkApi(IServiceProvider services, CancellationTokenSource cancellationTokenSource, ILog log)
 99 |         {
100 |             // VkNet uses its own DI for internal services
101 |             var sc = new ServiceCollection();
102 |             
103 |             sc.AddSingleton<CancellationTokenSource>(cancellationTokenSource);
104 |             sc.AddSingleton<IRestClient, CustomRestClient>();
105 |             sc.AddSingleton<IAwaitableConstraint, CancellableConstraint>(_ => new CancellableConstraint(3, TimeSpan.FromSeconds(1), cancellationTokenSource.Token));
106 |             sc.AddSingleton<ILoggerFactory, LoggerFactory>();
107 |             sc.AddSingleton(typeof(ILogger<>), typeof(Logger<>));
108 |             sc.AddLogging(builder =>
109 |             {
110 |                 builder.ClearProviders();
111 |                 builder.SetMinimumLevel(LogLevel.Warning);
112 |                 //builder.AddLog4Net();
113 |                 //builder.Services.AddSingleton<ILoggerProvider>((ILoggerProvider)new Log4NetProvider(options));
114 |                 builder.AddProvider(new SimpleLoggerProvider(log));
115 |             });
116 | 
117 |             sc.AddTransient<Settings>(_ => services.GetRequiredService<IOptionsSnapshot<Settings>>().Value);
118 | 
119 |             var vkApi = new VkApi(sc);
120 |             vkApi.VkApiVersion.SetVersion(5,90);  // hack for linear comments without threads
121 |             return vkApi;
122 |         }
123 | 
124 |         private static CancellationTokenSource CreateCancellationTokenSource()
125 |         {
126 |             var cts = new CancellationTokenSource();
127 |             Console.CancelKeyPress += (sender, a) =>
128 |             {
129 |                 a.Cancel = true;
130 |                 Console.WriteLine("Ctrl-C pressed, trying to stop gracefully...");
131 |                 cts.Cancel(true);
132 |             };
133 |             return cts;
134 |         }
135 | 
136 |         private static void ConfigureJsonSerializer()
137 |         {
138 |             JsonConvert.DefaultSettings = () => new JsonSerializerSettings()
139 |             {
140 |                 Converters = new List<JsonConverter>() {new StringEnumConverter()}
141 |             };
142 |         }
143 |         
144 |         private static ILog ConfigureLog4Net()
145 |         {
146 |             var logRepository = LogManager.GetRepository(Assembly.GetEntryAssembly());
147 |             XmlConfigurator.Configure(logRepository, new FileInfo("log4net.config"));
148 |             return LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program));
149 |         }
150 |     }
151 | }
152 | 
153 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | ## Ignore Visual Studio temporary files, build results, and
  2 | ## files generated by popular Visual Studio add-ons.
  3 | ##
  4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
  5 | 
  6 | # User-specific files
  7 | *.rsuser
  8 | *.suo
  9 | *.user
 10 | *.userosscache
 11 | *.sln.docstates
 12 | 
 13 | # User-specific files (MonoDevelop/Xamarin Studio)
 14 | *.userprefs
 15 | 
 16 | # Build results
 17 | [Dd]ebug/
 18 | [Dd]ebugPublic/
 19 | [Rr]elease/
 20 | [Rr]eleases/
 21 | x64/
 22 | x86/
 23 | bld/
 24 | [Bb]in/
 25 | [Oo]bj/
 26 | [Ll]og/
 27 | 
 28 | # Visual Studio 2015/2017 cache/options directory
 29 | .vs/
 30 | # Uncomment if you have tasks that create the project's static files in wwwroot
 31 | #wwwroot/
 32 | 
 33 | # Visual Studio 2017 auto generated files
 34 | Generated\ Files/
 35 | 
 36 | # MSTest test Results
 37 | [Tt]est[Rr]esult*/
 38 | [Bb]uild[Ll]og.*
 39 | 
 40 | # NUNIT
 41 | *.VisualState.xml
 42 | TestResult.xml
 43 | 
 44 | # Build Results of an ATL Project
 45 | [Dd]ebugPS/
 46 | [Rr]eleasePS/
 47 | dlldata.c
 48 | 
 49 | # Benchmark Results
 50 | BenchmarkDotNet.Artifacts/
 51 | 
 52 | # .NET Core
 53 | project.lock.json
 54 | project.fragment.lock.json
 55 | artifacts/
 56 | 
 57 | # StyleCop
 58 | StyleCopReport.xml
 59 | 
 60 | # Files built by Visual Studio
 61 | *_i.c
 62 | *_p.c
 63 | *_h.h
 64 | *.ilk
 65 | *.meta
 66 | *.obj
 67 | *.iobj
 68 | *.pch
 69 | *.pdb
 70 | *.ipdb
 71 | *.pgc
 72 | *.pgd
 73 | *.rsp
 74 | *.sbr
 75 | *.tlb
 76 | *.tli
 77 | *.tlh
 78 | *.tmp
 79 | *.tmp_proj
 80 | *_wpftmp.csproj
 81 | *.log
 82 | *.vspscc
 83 | *.vssscc
 84 | .builds
 85 | *.pidb
 86 | *.svclog
 87 | *.scc
 88 | 
 89 | # Chutzpah Test files
 90 | _Chutzpah*
 91 | 
 92 | # Visual C++ cache files
 93 | ipch/
 94 | *.aps
 95 | *.ncb
 96 | *.opendb
 97 | *.opensdf
 98 | *.sdf
 99 | *.cachefile
100 | *.VC.db
101 | *.VC.VC.opendb
102 | 
103 | # Visual Studio profiler
104 | *.psess
105 | *.vsp
106 | *.vspx
107 | *.sap
108 | 
109 | # Visual Studio Trace Files
110 | *.e2e
111 | 
112 | # TFS 2012 Local Workspace
113 | $tf/
114 | 
115 | # Guidance Automation Toolkit
116 | *.gpState
117 | 
118 | # ReSharper is a .NET coding add-in
119 | _ReSharper*/
120 | *.[Rr]e[Ss]harper
121 | *.DotSettings.user
122 | 
123 | # JustCode is a .NET coding add-in
124 | .JustCode
125 | 
126 | # TeamCity is a build add-in
127 | _TeamCity*
128 | 
129 | # DotCover is a Code Coverage Tool
130 | *.dotCover
131 | 
132 | # AxoCover is a Code Coverage Tool
133 | .axoCover/*
134 | !.axoCover/settings.json
135 | 
136 | # Visual Studio code coverage results
137 | *.coverage
138 | *.coveragexml
139 | 
140 | # NCrunch
141 | _NCrunch_*
142 | .*crunch*.local.xml
143 | nCrunchTemp_*
144 | 
145 | # MightyMoose
146 | *.mm.*
147 | AutoTest.Net/
148 | 
149 | # Web workbench (sass)
150 | .sass-cache/
151 | 
152 | # Installshield output folder
153 | [Ee]xpress/
154 | 
155 | # DocProject is a documentation generator add-in
156 | DocProject/buildhelp/
157 | DocProject/Help/*.HxT
158 | DocProject/Help/*.HxC
159 | DocProject/Help/*.hhc
160 | DocProject/Help/*.hhk
161 | DocProject/Help/*.hhp
162 | DocProject/Help/Html2
163 | DocProject/Help/html
164 | 
165 | # Click-Once directory
166 | publish/
167 | 
168 | # Publish Web Output
169 | *.[Pp]ublish.xml
170 | *.azurePubxml
171 | # Note: Comment the next line if you want to checkin your web deploy settings,
172 | # but database connection strings (with potential passwords) will be unencrypted
173 | *.pubxml
174 | *.publishproj
175 | 
176 | # Microsoft Azure Web App publish settings. Comment the next line if you want to
177 | # checkin your Azure Web App publish settings, but sensitive information contained
178 | # in these scripts will be unencrypted
179 | PublishScripts/
180 | 
181 | # NuGet Packages
182 | *.nupkg
183 | # The packages folder can be ignored because of Package Restore
184 | **/[Pp]ackages/*
185 | # except build/, which is used as an MSBuild target.
186 | !**/[Pp]ackages/build/
187 | # Uncomment if necessary however generally it will be regenerated when needed
188 | #!**/[Pp]ackages/repositories.config
189 | # NuGet v3's project.json files produces more ignorable files
190 | *.nuget.props
191 | *.nuget.targets
192 | 
193 | # Microsoft Azure Build Output
194 | csx/
195 | *.build.csdef
196 | 
197 | # Microsoft Azure Emulator
198 | ecf/
199 | rcf/
200 | 
201 | # Windows Store app package directories and files
202 | AppPackages/
203 | BundleArtifacts/
204 | Package.StoreAssociation.xml
205 | _pkginfo.txt
206 | *.appx
207 | 
208 | # Visual Studio cache files
209 | # files ending in .cache can be ignored
210 | *.[Cc]ache
211 | # but keep track of directories ending in .cache
212 | !*.[Cc]ache/
213 | 
214 | # Others
215 | ClientBin/
216 | ~$*
217 | *~
218 | *.dbmdl
219 | *.dbproj.schemaview
220 | *.jfm
221 | *.pfx
222 | *.publishsettings
223 | orleans.codegen.cs
224 | 
225 | # Including strong name files can present a security risk
226 | # (https://github.com/github/gitignore/pull/2483#issue-259490424)
227 | #*.snk
228 | 
229 | # Since there are multiple workflows, uncomment next line to ignore bower_components
230 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
231 | #bower_components/
232 | 
233 | # RIA/Silverlight projects
234 | Generated_Code/
235 | 
236 | # Backup & report files from converting an old project file
237 | # to a newer Visual Studio version. Backup files are not needed,
238 | # because we have git ;-)
239 | _UpgradeReport_Files/
240 | Backup*/
241 | UpgradeLog*.XML
242 | UpgradeLog*.htm
243 | ServiceFabricBackup/
244 | *.rptproj.bak
245 | 
246 | # SQL Server files
247 | *.mdf
248 | *.ldf
249 | *.ndf
250 | 
251 | # Business Intelligence projects
252 | *.rdl.data
253 | *.bim.layout
254 | *.bim_*.settings
255 | *.rptproj.rsuser
256 | 
257 | # Microsoft Fakes
258 | FakesAssemblies/
259 | 
260 | # GhostDoc plugin setting file
261 | *.GhostDoc.xml
262 | 
263 | # Node.js Tools for Visual Studio
264 | .ntvs_analysis.dat
265 | node_modules/
266 | 
267 | # Visual Studio 6 build log
268 | *.plg
269 | 
270 | # Visual Studio 6 workspace options file
271 | *.opt
272 | 
273 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
274 | *.vbw
275 | 
276 | # Visual Studio LightSwitch build output
277 | **/*.HTMLClient/GeneratedArtifacts
278 | **/*.DesktopClient/GeneratedArtifacts
279 | **/*.DesktopClient/ModelManifest.xml
280 | **/*.Server/GeneratedArtifacts
281 | **/*.Server/ModelManifest.xml
282 | _Pvt_Extensions
283 | 
284 | # Paket dependency manager
285 | .paket/paket.exe
286 | paket-files/
287 | 
288 | # FAKE - F# Make
289 | .fake/
290 | 
291 | # JetBrains Rider
292 | .idea/
293 | *.sln.iml
294 | 
295 | # CodeRush personal settings
296 | .cr/personal
297 | 
298 | # Python Tools for Visual Studio (PTVS)
299 | __pycache__/
300 | *.pyc
301 | 
302 | # Cake - Uncomment if you are using it
303 | # tools/**
304 | # !tools/packages.config
305 | 
306 | # Tabs Studio
307 | *.tss
308 | 
309 | # Telerik's JustMock configuration file
310 | *.jmconfig
311 | 
312 | # BizTalk build output
313 | *.btp.cs
314 | *.btm.cs
315 | *.odx.cs
316 | *.xsd.cs
317 | 
318 | # OpenCover UI analysis results
319 | OpenCover/
320 | 
321 | # Azure Stream Analytics local run output
322 | ASALocalRun/
323 | 
324 | # MSBuild Binary and Structured Log
325 | *.binlog
326 | 
327 | # NVidia Nsight GPU debugger configuration file
328 | *.nvuser
329 | 
330 | # MFractors (Xamarin productivity tool) working folder
331 | .mfractor/
332 | 
333 | # Local History for Visual Studio
334 | .localhistory/
335 | 
336 | **/FileCache.*.json
337 | **logs/


--------------------------------------------------------------------------------
/VOffline/Services/Token/Google/GoogleHttpRequests.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.Text;
  4 | using log4net;
  5 | using RestSharp;
  6 | using VOffline.Models.Google;
  7 | 
  8 | namespace VOffline.Services.Token.Google
  9 | {
 10 |     public class GoogleHttpRequests
 11 |     {
 12 |         private FileCache<GoogleCredentials> CredentialsCache { get; }
 13 |         private FileCache<GoogleCheckIn> CheckInCache { get; }
 14 |         public string RandomAppId { get; }
 15 | 
 16 |         public GoogleHttpRequests(FileCache<GoogleCredentials> credentialsCache, FileCache<GoogleCheckIn> checkInCache, Random random)
 17 |         {
 18 |             CredentialsCache = credentialsCache;
 19 |             CheckInCache = checkInCache;
 20 |             RandomAppId = GenerateRandomString(11, random);
 21 |         }
 22 | 
 23 |         public byte[] GetCheckIn(ILog log)
 24 |         {
 25 |             if (CheckInCache.Value != null)
 26 |             {
 27 |                 return CheckInCache.Value.Response;
 28 |             }
 29 | 
 30 |             log.Debug($"request check in");
 31 |             var response = RequestCheckIn();
 32 |             CheckInCache.Value = new GoogleCheckIn()
 33 |             {
 34 |                 Response = response
 35 |             };
 36 |             return CheckInCache.Value.Response;
 37 |         }
 38 | 
 39 |         public string GetReceipt(ILog log)
 40 |         {
 41 |             var client = GetClient("https://android.clients.google.com/c2dm/register3");
 42 |             log.Debug($"request receipt 1");
 43 |             RequestReceipt1(client);
 44 |             log.Debug($"request receipt 2");
 45 |             return RequestReceipt2(client);
 46 |         }
 47 | 
 48 |         private byte[] RequestCheckIn()
 49 |         {
 50 |             var request = new RestRequest(Method.POST);
 51 |             request.AddHeader("Expect", "");
 52 |             request.AddParameter("application/x-protobuffer", queryMessage, ParameterType.RequestBody);
 53 |             var client = GetClient("https://android.clients.google.com/checkin");
 54 |             var response = client.Execute(request);
 55 |             if (!response.IsSuccessful)
 56 |             {
 57 |                 throw new InvalidOperationException($"{nameof(RequestCheckIn)} bad response: {response.StatusCode}\n{response.Content}");
 58 |             }
 59 |             return response.RawBytes;
 60 |         }
 61 | 
 62 |         private void RequestReceipt1(RestClient client)
 63 |         {
 64 |             var request = new RestRequest(Method.POST);
 65 |             request.AddHeader("Authorization", $"AidLogin {CredentialsCache.Value.Id}:{CredentialsCache.Value.Token}");
 66 |             foreach (var requestParam in GetRequestParams())
 67 |             {
 68 |                 request.AddParameter(requestParam.Key, requestParam.Value, ParameterType.GetOrPost);
 69 |             }
 70 | 
 71 |             var response = client.Execute(request);  // result ignored?
 72 |             if (!response.IsSuccessful)
 73 |             {
 74 |                 throw new InvalidOperationException($"{nameof(RequestReceipt1)} bad response: {response.StatusCode}\n{response.Content}");
 75 |             }
 76 |         }
 77 | 
 78 |         private string RequestReceipt2(RestClient client)
 79 |         {
 80 |             var request = new RestRequest(Method.POST);
 81 |             request.AddHeader("Authorization", $"AidLogin {CredentialsCache.Value.Id}:{CredentialsCache.Value.Token}");
 82 |             var requestParams = GetRequestParams();
 83 |             requestParams["X-scope"] = $"id{string.Empty}";  // id is always empty here?
 84 |             requestParams["X-kid"] = "|ID|2|";
 85 |             requestParams["X-X-kid"] = "|ID|2|";
 86 |             foreach (var requestParam in requestParams)
 87 |             {
 88 |                 request.AddParameter(requestParam.Key, requestParam.Value, ParameterType.GetOrPost);
 89 |             }
 90 |             var response = client.Execute(request);
 91 |             if (!response.IsSuccessful)
 92 |             {
 93 |                 throw new InvalidOperationException($"{nameof(RequestReceipt2)} bad response: {response.StatusCode}\n{response.Content}");
 94 |             }
 95 | 
 96 |             var result = response.Content.Split("|ID|2|:")[1];
 97 |             if (result == "PHONE_REGISTRATION_ERROR")
 98 |             {
 99 |                 throw new InvalidOperationException($"{nameof(RequestReceipt2)} bad response: {result}\n{response.Content}");
100 |             }
101 | 
102 |             return result;
103 |         }
104 | 
105 |         private Dictionary<string, string> GetRequestParams()
106 |         {
107 |             return new Dictionary<string, string>
108 |             {
109 |                 {"X-scope", "GCM"},
110 |                 {"X-osv", "23"},
111 |                 {"X-subtype", "54740537194"},
112 |                 {"X-app_ver", "443"},
113 |                 {"X-kid", "|ID|1|"},
114 |                 {"X-appid", RandomAppId},
115 |                 {"X-gmsv", "13283005"},
116 |                 {"X-cliv", "iid-10084000"},
117 |                 {"X-app_ver_name", "51.2 lite"},
118 |                 {"X-X-kid", "|ID|1|"},
119 |                 {"X-subscription", "54740537194"},
120 |                 {"X-X-subscription", "54740537194"},
121 |                 {"X-X-subtype", "54740537194"},
122 |                 {"app", "com.perm.kate_new_6"},
123 |                 {"sender", "54740537194"},
124 |                 {"device", CredentialsCache.Value.Id.ToString()},
125 |                 {"cert", "966882ba564c2619d55d0a9afd4327a38c327456"},
126 |                 {"app_ver", "443"},
127 |                 {"info", "g57d5w1C4CcRUO6eTSP7b7VoT8yTYhY"},
128 |                 {"gcm_ver", "13283005"},
129 |                 {"plat", "0"},
130 |                 {"X-messenger2", "1"}
131 |             };
132 |         }
133 | 
134 |         private RestClient GetClient(string url)
135 |         {
136 |             var client = new RestClient(url)
137 |             {
138 |                 UserAgent = GcmUserAgent
139 |             };
140 |             client.RemoteCertificateValidationCallback += (sender, certificate, chain, errors) => true;
141 |             return client;
142 |         }
143 | 
144 |         private string GenerateRandomString(int length, Random random)
145 |         {
146 |             var sb = new StringBuilder(length);
147 |             for (var i = 0; i < length; i++)
148 |             {
149 |                 sb.Append(alphabet[random.Next(alphabet.Length)]);
150 |             }
151 |             return sb.ToString();
152 |         }
153 | 
154 |         private const string GcmUserAgent = "Android-GCM/1.5 (generic_x86 KK)";
155 | 
156 |         private static byte[] queryMessage = {
157 |             0x10, 0x00, 0x1a, 0x2a, 0x31, 0x2d, 0x39, 0x32, 0x39, 0x61, 0x30, 0x64, 0x63, 0x61, 0x30, 0x65, 0x65, 0x65, 0x35, 0x35, 0x35, 0x31, 0x33, 0x32, 0x38, 0x30, 0x31, 0x37, 0x31, 0x61, 0x38, 0x35, 0x38, 0x35, 0x64, 0x61, 0x37, 0x64, 0x63, 0x64, 0x33, 0x37, 0x30, 0x30, 0x66, 0x38, 0x22, 0xe3, 0x01, 0x0a, 0xbf, 0x01, 0x0a, 0x45, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x78, 0x38, 0x36, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x73, 0x64, 0x6b, 0x5f, 0x78, 0x38, 0x36, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x78, 0x38, 0x36, 0x3a, 0x34, 0x2e, 0x34, 0x2e, 0x32, 0x2f, 0x4b, 0x4b, 0x2f, 0x33, 0x30, 0x37, 0x39, 0x31, 0x38, 0x33, 0x3a, 0x65, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2d, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x06, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x75, 0x1a, 0x0b, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x78, 0x38, 0x36, 0x2a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x32, 0x0e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x40, 0x85, 0xb5, 0x86, 0x06, 0x4a, 0x0b, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x78, 0x38, 0x36, 0x50, 0x13, 0x5a, 0x19, 0x41, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x20, 0x53, 0x44, 0x4b, 0x20, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x78, 0x38, 0x36, 0x62, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x6a, 0x0e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x73, 0x64, 0x6b, 0x5f, 0x78, 0x38, 0x36, 0x70, 0x00, 0x10, 0x00, 0x32, 0x06, 0x33, 0x31, 0x30, 0x32, 0x36, 0x30, 0x3a, 0x06, 0x33, 0x31, 0x30, 0x32, 0x36, 0x30, 0x42, 0x0b, 0x6d, 0x6f, 0x62, 0x69, 0x6c, 0x65, 0x3a, 0x4c, 0x54, 0x45, 0x3a, 0x48, 0x00, 0x32, 0x05, 0x65, 0x6e, 0x5f, 0x55, 0x53, 0x38, 0xf0, 0xb4, 0xdf, 0xa6, 0xb9, 0x9a, 0xb8, 0x83, 0x8e, 0x01, 0x52, 0x0f, 0x33, 0x35, 0x38, 0x32, 0x34, 0x30, 0x30, 0x35, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x30, 0x5a, 0x00, 0x62, 0x10, 0x41, 0x6d, 0x65, 0x72, 0x69, 0x63, 0x61, 0x2f, 0x4e, 0x65, 0x77, 0x5f, 0x59, 0x6f, 0x72, 0x6b, 0x70, 0x03, 0x7a, 0x1c, 0x37, 0x31, 0x51, 0x36, 0x52, 0x6e, 0x32, 0x44, 0x44, 0x5a, 0x6c, 0x31, 0x7a, 0x50, 0x44, 0x56, 0x61, 0x61, 0x65, 0x45, 0x48, 0x49, 0x74, 0x64, 0x2b, 0x59, 0x67, 0x3d, 0xa0, 0x01, 0x00, 0xb0, 0x01, 0x00
158 |         };
159 | 
160 |         private const string alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-";
161 |     }
162 | }
163 | 


--------------------------------------------------------------------------------
/VOffline/Services/Storage/FilesystemTools.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Concurrent;
  3 | using System.Collections.Generic;
  4 | using System.Collections.Specialized;
  5 | using System.Globalization;
  6 | using System.IO;
  7 | using System.Linq;
  8 | using System.Threading;
  9 | using System.Threading.Tasks;
 10 | using log4net;
 11 | using Microsoft.Extensions.Options;
 12 | using Newtonsoft.Json;
 13 | using VOffline.Models;
 14 | 
 15 | namespace VOffline.Services.Storage
 16 | {
 17 |     public class FilesystemTools
 18 |     {
 19 |         private readonly FileInfo cacheFile;
 20 | 
 21 |         private readonly ConcurrentDictionary<string, DateTime> cache;
 22 | 
 23 |         public DirectoryInfo RootDir { get; }
 24 | 
 25 |         public FilesystemTools(IOptionsSnapshot<Settings> settings)
 26 |         {
 27 |             RootDir = MkDir(settings.Value.OutputPath);
 28 |             cacheFile = new FileInfo(CombineCutPath(RootDir, CacheFilename));
 29 |             cache = new ConcurrentDictionary<string, DateTime>();
 30 | 
 31 |             if (!cacheFile.Exists)
 32 |             {
 33 |                 cacheFile.Create()?.Dispose();
 34 |             }
 35 |         }
 36 | 
 37 |         public void LoadCache(ILog log)
 38 |         {
 39 |             lock (LockObject)
 40 |             {
 41 |                 var lines = File.ReadAllLines(cacheFile.FullName);
 42 |                 var separator = new[] {' '};
 43 |                 foreach (var line in lines)
 44 |                 {
 45 |                     var items = line.Split(separator, 2);
 46 |                     var datetime = DateTime.ParseExact(items[0], "O", CultureInfo.InvariantCulture);
 47 |                     var path = items[1];
 48 |                     cache[path] = datetime;
 49 |                 }
 50 |             }
 51 | 
 52 |             log.Info($"Cache loaded: {cache.Count} items, {cacheFile.FullName}");
 53 |         }
 54 | 
 55 |         public void SaveCache(ILog log)
 56 |         {
 57 |             lock (LockObject)
 58 |             {
 59 |                 var data = cache
 60 |                     .OrderBy(kv => kv.Value)
 61 |                     .Select(kv => $"{kv.Value:O} {kv.Key}");
 62 |                 File.WriteAllLines(cacheFile.FullName, data);
 63 |             }
 64 | 
 65 |             log.Info($"Cache saved: {cache.Count} items, {cacheFile.FullName}");
 66 |         }
 67 | 
 68 |         public DirectoryInfo CreateSubdir(DirectoryInfo parent, string desiredName, CreateMode mode)
 69 |         {
 70 |             if (!parent.Exists)
 71 |             {
 72 |                 throw new DirectoryNotFoundException($"Parent {parent.FullName} does not exist, can not create dir {desiredName}");
 73 |             }
 74 | 
 75 |             var validName = MakeValidName(desiredName);
 76 |             lock (LockObject)
 77 |             {
 78 |                 DirectoryInfo directory;
 79 |                 switch (mode)
 80 |                 {
 81 |                     case CreateMode.AutoRenameCollisions:
 82 |                         directory = GetUniqueDirectory(parent, validName);
 83 |                         break;
 84 |                     case CreateMode.ThrowIfExists:
 85 |                         directory = new DirectoryInfo(CombineCutPath(parent, validName));
 86 |                         if (directory.Exists)
 87 |                         {
 88 |                             throw new InvalidOperationException($"Dir already exists: {directory.FullName}");
 89 |                         }
 90 | 
 91 |                         break;
 92 |                     case CreateMode.OverwriteExisting:
 93 |                         directory = new DirectoryInfo(CombineCutPath(parent, validName));
 94 |                         break;
 95 |                     default:
 96 |                         throw new ArgumentOutOfRangeException(nameof(mode), mode, null);
 97 |                 }
 98 | 
 99 |                 directory.Create();
100 |                 directory.Refresh();
101 |                 return directory;
102 |             }
103 |         }
104 | 
105 |         public FileInfo CreateFile(DirectoryInfo parent, string desiredName, CreateMode mode)
106 |         {
107 |             if (!parent.Exists)
108 |             {
109 |                 throw new DirectoryNotFoundException($"Parent {parent.FullName} does not exist, can not create file {desiredName}");
110 |             }
111 | 
112 |             var validName = MakeValidName(desiredName);
113 |             lock (LockObject)
114 |             {
115 |                 FileInfo file;
116 |                 switch (mode)
117 |                 {
118 |                     case CreateMode.AutoRenameCollisions:
119 |                         file = GetUniqueFile(parent, validName);
120 |                         break;
121 |                     case CreateMode.ThrowIfExists:
122 |                         file = new FileInfo(CombineCutPath(parent, validName));
123 |                         if (file.Exists)
124 |                         {
125 |                             throw new InvalidOperationException($"File already exists: {file.FullName}");
126 |                         }
127 | 
128 |                         break;
129 |                     case CreateMode.OverwriteExisting:
130 |                         file = new FileInfo(CombineCutPath(parent, validName));
131 |                         break;
132 |                     default:
133 |                         throw new ArgumentOutOfRangeException(nameof(mode), mode, null);
134 |                 }
135 | 
136 |                 file.Create().Close();
137 |                 file.Refresh();
138 |                 return file;
139 |             }
140 |         }
141 | 
142 |         public async Task WriteFileWithCompletionMark(DirectoryInfo parent, string desiredName, Func<Task<string>> contentTaskFunc, CancellationToken token, ILog log)
143 |         {
144 |             var validName = MakeValidName(desiredName);
145 |             var file = new FileInfo(CombineCutPath(parent, validName));
146 |             if (cache.ContainsKey(file.FullName))
147 |             {
148 |                 log.Debug($"Skipping [{file.FullName}] because marked as competed");
149 |                 return;
150 |             }
151 | 
152 |             var content = await contentTaskFunc();
153 |             if (string.IsNullOrEmpty(content))
154 |             {
155 |                 log.Warn($"Content for [{file.FullName}] is empty");
156 |             }
157 |             else
158 |             {
159 |                 await File.WriteAllTextAsync(file.FullName, content, token);
160 |                 log.Info($"Saved [{desiredName}] as [{file.FullName}] with [{content.Length}] chars");
161 |             }
162 |             cache[file.FullName] = DateTime.Now;
163 |         }
164 | 
165 |         public async Task WriteFileWithCompletionMark(DirectoryInfo parent, string desiredName, Func<Task<byte[]>> contentTaskFunc, CancellationToken token, ILog log)
166 |         {
167 |             var validName = MakeValidName(desiredName);
168 |             var file = new FileInfo(CombineCutPath(parent, validName));
169 |             if (cache.ContainsKey(file.FullName))
170 |             {
171 |                 log.Debug($"Skipping [{file.FullName}] because marked as competed");
172 |                 return;
173 |             }
174 | 
175 |             var content = await contentTaskFunc();
176 |             if (content == null || content.Length == 0)
177 |             {
178 |                 log.Warn($"Content for [{file.FullName}] is empty");
179 |             }
180 |             else
181 |             {
182 |                 await File.WriteAllBytesAsync(file.FullName, content, token);
183 |                 log.Info($"Saved [{desiredName}] as [{file.FullName}] with [{content.Length}] bytes");
184 |             }
185 | 
186 |             cache[file.FullName] = DateTime.Now;
187 |         }
188 | 
189 |         private DirectoryInfo MkDir(string path)
190 |         {
191 |             var dir = new DirectoryInfo(path);
192 |             dir.Create();
193 |             dir.Refresh();
194 |             return dir;
195 |         }
196 | 
197 |         /// <summary>
198 |         /// Appends number to name in case of collision. Has no side effects. Should be used under lock.
199 |         /// </summary>
200 |         /// <param name="parent"></param>
201 |         /// <param name="validName"></param>
202 |         /// <returns></returns>
203 |         private static FileInfo GetUniqueFile(DirectoryInfo parent, string validName)
204 |         {
205 |             var name = Path.GetFileNameWithoutExtension(validName);
206 |             var extension = Path.GetExtension(validName);
207 |             var fileInfo = new FileInfo(CombineCutPath(parent, validName));
208 |             for (var i = 1;; i++)
209 |             {
210 |                 if (!fileInfo.Exists)
211 |                 {
212 |                     return fileInfo;
213 |                 }
214 | 
215 |                 fileInfo = new FileInfo(CombineCutPath(parent, $"{name} ({i}){extension}"));
216 |             }
217 |         }
218 | 
219 |         /// <summary>
220 |         /// Appends number to name in case of collision. Has no side effects. Should be used under lock. 
221 |         /// </summary>
222 |         /// <param name="parent"></param>
223 |         /// <param name="validName"></param>
224 |         /// <returns></returns>
225 |         private static DirectoryInfo GetUniqueDirectory(DirectoryInfo parent, string validName)
226 |         {
227 |             var directoryInfo = new DirectoryInfo(CombineCutPath(parent, validName));
228 |             for (var i = 1;; i++)
229 |             {
230 |                 if (!directoryInfo.Exists)
231 |                 {
232 |                     return directoryInfo;
233 |                 }
234 | 
235 |                 directoryInfo = new DirectoryInfo(CombineCutPath(parent, $"{validName} ({i})"));
236 |             }
237 |         }
238 | 
239 |         private static string CombineCutPath(DirectoryInfo parentDir, string name)
240 |         {
241 |             var namePart = Path.GetFileNameWithoutExtension(name);
242 |             var extensionPart = Path.GetExtension(name);
243 |             while (namePart.Length > 1)
244 |             {
245 |                 var testName = $"{namePart} (NNN){extensionPart}"; // extra filler for possible " (NNN)"
246 |                 var testPath = Path.Combine(parentDir.FullName, testName);
247 |                 if (testPath.Length < 248)
248 |                 {
249 |                     return Path.Combine(parentDir.FullName, $"{namePart}{extensionPart}");
250 |                 }
251 | 
252 |                 //Path.GetFullPath(testPath);  // should throw on long paths but does not work on net core?
253 |                 namePart = namePart.Substring(0, Math.Max(1, namePart.Length - 1));
254 |             }
255 | 
256 |             throw new PathTooLongException($"Tried to shorten [{name}] to [{namePart}{extensionPart}] but path [{parentDir.FullName}] is still too long");
257 |         }
258 | 
259 |         private static string MakeValidName(string value)
260 |         {
261 |             // fix stuff like "image.jpg_foo=bar&blah=blah" without losing extension
262 |             var namePart = string.Join("_", Path.GetFileNameWithoutExtension(value).Split(AllBadChars)).Trim();
263 |             var extension = Path.GetExtension(value)
264 |                 .Split('?').First()
265 |                 .Split('_').First();
266 |             var extensionPart = string.Join("_", extension.Split(AllBadChars)).Trim();
267 |             return $"{namePart}{extensionPart}";
268 |         }
269 | 
270 |         private static readonly char[] AllBadChars =
271 |             Path.GetInvalidFileNameChars()
272 |                 .Concat(Path.GetInvalidPathChars())
273 |                 .Concat(new[] {'\\', '/', ':'})
274 |                 .Distinct()
275 |                 .ToArray();
276 | 
277 |         private static readonly object LockObject = new object();
278 | 
279 |         private static readonly string CacheFilename = MakeValidName(".cache.voffline");
280 |     }
281 | }
282 | 


--------------------------------------------------------------------------------
/VOffline/Services/Vk/AttachmentExtensions.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.IO;
  4 | using System.Linq;
  5 | using System.Text;
  6 | using System.Threading;
  7 | using System.Threading.Tasks;
  8 | using log4net;
  9 | using Newtonsoft.Json;
 10 | using VkNet;
 11 | using VkNet.Enums.SafetyEnums;
 12 | using VkNet.Model;
 13 | using VkNet.Model.Attachments;
 14 | using VOffline.Models.Storage;
 15 | using VOffline.Services.Storage;
 16 | 
 17 | namespace VOffline.Services.Vk
 18 | {
 19 |     public static class AttachmentExtensions
 20 |     {
 21 |         public static IEnumerable<IDownload> ToDownloads(this Audio audio, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
 22 |         {
 23 |             var trackName = audio.GetName();
 24 |             if (audio.Url != null)
 25 |             {
 26 |                 yield return new Download(audio.Url, dir, $"{i} {trackName}.mp3");
 27 |             }
 28 |             else
 29 |             {
 30 |                 filesystemTools.CreateFile(dir, $"{i} {trackName}.mp3.deleted", CreateMode.OverwriteExisting);
 31 |             }
 32 |         }
 33 | 
 34 |         public static IEnumerable<IDownload> ToDownloads(this Photo photo, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
 35 |         {
 36 |             // TODO: not sure about ANYTHING here
 37 |             var url = photo.BigPhotoSrc
 38 |                       ?? photo.Photo2560
 39 |                       ?? photo.PhotoSrc
 40 |                       ?? photo.Photo1280
 41 |                       ?? photo.Photo807
 42 |                       ?? photo.Photo604
 43 |                       ?? photo.SmallPhotoSrc
 44 |                       ?? photo.Photo200
 45 |                       ?? photo.Photo130
 46 |                       ?? photo.Photo100
 47 |                       ?? photo.Photo75
 48 |                       ?? photo.Photo50
 49 |                 ?? photo.Sizes
 50 |                           .Select(s => (square:s.Width*s.Height, size:s))
 51 |                           .OrderByDescending(x => x.square)
 52 |                           .FirstOrDefault(s => s.square > 0)   // width/height can be null
 53 |                           .size?.Url
 54 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.W)?.Url
 55 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.Z)?.Url
 56 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.Y)?.Url
 57 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.X)?.Url
 58 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.R)?.Url
 59 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.Q)?.Url
 60 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.P)?.Url
 61 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.O)?.Url
 62 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.M)?.Url
 63 |                 ?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.S)?.Url
 64 |                 ;
 65 | 
 66 |             // TODO: i guess it's always jpeg?
 67 |             var ext = Path.HasExtension(url?.AbsoluteUri) ? Path.GetExtension(url?.AbsoluteUri) : ".jpg";
 68 |             if (url != null)
 69 |             {
 70 |                 var file = $"{i}_{photo.Id}{ext}";
 71 |                 yield return new Download(url, dir, file);
 72 |             }
 73 |             else
 74 |             {
 75 |                 log.Warn($"Photo with no url! {JsonConvert.SerializeObject(photo)}");
 76 |             }
 77 |         }
 78 | 
 79 |         public static IEnumerable<IDownload> ToDownloads(this Video video, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
 80 |         {
 81 |             var vkUrl = video.Files?.Mp4_1080
 82 |                         ?? video.Files?.Mp4_720
 83 |                         ?? video.Files?.Mp4_480
 84 |                         ?? video.Files?.Mp4_360
 85 |                         ?? video.Files?.Mp4_240;
 86 |             if (vkUrl != null)
 87 |             {
 88 |                 yield return new Download(vkUrl, dir, video.Title);
 89 |             }
 90 |             else if(video.Files?.External != null)
 91 |             {
 92 |                 log.Warn($"Video {video.Id} [{video.Title}] is external");
 93 |                 yield return new Download(video.Files.External, dir, video.Title);
 94 |             }
 95 |             else
 96 |             {
 97 |                 log.Warn($"Video {video.Id} [{video.Title}] has no links. Trying to compose vk url");
 98 |                 var file = filesystemTools.CreateFile(dir, $"{i} {video.Id}.txt", CreateMode.OverwriteExisting);
 99 |                 File.WriteAllText(file.FullName, $"https://vk.com/video{video.OwnerId}_{video.Id}");
100 |             }
101 |         }
102 | 
103 |         public static IEnumerable<IDownload> ToDownloads(this Document document, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
104 |         {
105 |             // TODO: looks like Title already has Extension
106 |             yield return new Download(new Uri(document.Uri), dir, $"{i} {document.Title}");
107 |         }
108 | 
109 |         public static IEnumerable<IDownload> ToDownloads(this Poll poll, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
110 |         {
111 |             var photos = new List<Photo>();
112 |             if (poll.Photo != null)
113 |             {
114 |                 photos.Add(poll.Photo);
115 |             }
116 |             if (poll?.Background?.Images != null)
117 |             {
118 |                 photos.AddRange(poll.Background.Images);
119 |             }
120 | 
121 |             if (!photos.Any())
122 |             {
123 |                 yield break;
124 |             }
125 |             var pollDir = filesystemTools.CreateSubdir(dir, $"{i} {poll.GetName()}", CreateMode.OverwriteExisting);
126 |             var imageDownloads = photos.SelectMany((p, j) => p.ToDownloads(j, filesystemTools, pollDir, log));
127 |             foreach (var imageDownload in imageDownloads)
128 |             {
129 |                 yield return imageDownload;
130 |             }
131 |         }
132 | 
133 |         public static IEnumerable<IDownload> ToDownloads(this Link link, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
134 |         {
135 |             yield return new Download(link.Uri, dir, link.Title);
136 |         }
137 | 
138 |         public static IEnumerable<IDownload> ToDownloads(this AudioCover cover, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
139 |         {
140 |             var best = cover.Photo600
141 |                         ?? cover.Photo300
142 |                         ?? cover.Photo270
143 |                         ?? cover.Photo135
144 |                         ?? cover.Photo68
145 |                         ?? cover.Photo34;
146 |             var image = best == null
147 |                 ? null
148 |                 : new Uri(best);
149 | 
150 |             // TODO: i guess it's always jpeg?
151 |             var ext = Path.HasExtension(image?.AbsoluteUri) ? Path.GetExtension(image?.AbsoluteUri) : ".jpg";
152 |             if (image == null)
153 |             {
154 |                 yield break;
155 |             }
156 |             var file = $"__cover{ext}";
157 |             yield return new Download(image, dir, file);
158 |         }
159 | 
160 |         public static string Serialize(this Poll poll)
161 |         {
162 |             var sb = new StringBuilder();
163 |             sb.AppendLine(poll.Question);
164 | 
165 |             sb.Append($"{poll.Created.Value:O}");
166 |             if (poll.EndDate.HasValue)
167 |             {
168 |                 sb.Append($" - {poll.Closed.Value:O}");
169 |             }
170 |             sb.AppendLine();
171 |             
172 |             sb.AppendLine($"votes: {poll.Votes}");
173 |             sb.AppendLine($"flags: alonymous={poll.Anonymous},");
174 |             foreach (var answer in poll.Answers.OrderByDescending(a => a.Rate))
175 |             {
176 |                 var voteState = poll.AnswerId == answer.Id.Value
177 |                                 || poll.AnswerIds.Contains(answer.Id.Value)
178 |                     ? "[v]"
179 |                     : "[ ]";
180 |                 sb.AppendLine($"{voteState} {answer.Text} ({answer.Rate:F}%, {answer.Votes})");
181 |             }
182 | 
183 |             return sb.ToString();
184 |         }
185 | 
186 |         public static string Serialize(this Link link)
187 |         {
188 |             var sb = new StringBuilder();
189 |             sb.AppendLine(link.Title);
190 |             sb.AppendLine(link.Caption);
191 |             sb.AppendLine(link.Description);
192 |             sb.AppendLine($"{link.Uri}");
193 |             return sb.ToString();
194 |         }
195 | 
196 |         public static string Serialize(this Comment comment)
197 |         {
198 |             var sb = new StringBuilder();
199 |             sb.AppendLine($"{comment.Date} {comment.FromId}");
200 |             sb.AppendLine(comment.Text);
201 |             sb.Append($"likes={comment.Likes?.Count}, attachments={comment.Attachments?.Count}, id={comment.Id}");
202 |             if (comment.ReplyToCommentId != null || comment.ReplyToUserId != null)
203 |             {
204 |                 sb.Append($", reply to user {comment.ReplyToUserId} comment {comment.ReplyToCommentId}");
205 |             }
206 | 
207 |             return sb.ToString();
208 |         }
209 | 
210 |         public static string GetName(this Audio audio) => string.Join(" - ", new[] { audio.Artist, audio.Title }.Where(x => !string.IsNullOrEmpty(x)));
211 | 
212 |         public static string GetName(this Poll poll) => $"poll {poll.Id}";
213 | 
214 |         public static async Task SaveText(this Photo photo, int i, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
215 |         {
216 |             if (!string.IsNullOrWhiteSpace(photo.Text))
217 |             {
218 |                 var textFile = filesystemTools.CreateFile(dir, $"{i} {photo.Id}.txt", CreateMode.OverwriteExisting);
219 |                 await File.WriteAllTextAsync(textFile.FullName, photo.Text, token);
220 |             }
221 |         }
222 | 
223 |         public static async Task SaveLyrics(this Audio audio, int i, VkApi vkApi, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
224 |         {
225 |             if (audio.LyricsId != null)
226 |             {
227 |                 var filename = $"{i} {audio.GetName()}.txt";
228 |                 await filesystemTools.WriteFileWithCompletionMark(dir, filename, async () =>
229 |                 {
230 |                     var lyrics = await vkApi.Audio.GetLyricsAsync(audio.LyricsId.Value);
231 |                     return lyrics.Text;
232 |                 }, token, log);
233 |             }
234 |         }
235 | 
236 |         public static async Task SaveHumanReadableText(this Poll poll, int i, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
237 |         {
238 |             var textFile = filesystemTools.CreateFile(dir, $"{i} {poll.GetName()}.txt", CreateMode.OverwriteExisting);
239 |             await File.WriteAllTextAsync(textFile.FullName, poll.Serialize(), token);
240 |         }
241 | 
242 |         public static async Task SaveHumanReadableText(this Link link, int i, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
243 |         {
244 |             var textFile = filesystemTools.CreateFile(dir, $"{i} {link.Title}.txt", CreateMode.OverwriteExisting);
245 |             await File.WriteAllTextAsync(textFile.FullName, link.Serialize(), token);
246 |         }
247 | 
248 |         public static async Task SaveHumanReadableText(this IReadOnlyList<Comment> comments, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
249 |         {
250 |             var data = string.Join("\n\n", comments.Select(c => c.Serialize()));
251 |             var textFile = filesystemTools.CreateFile(dir, $"comments.txt", CreateMode.OverwriteExisting);
252 |             await File.WriteAllTextAsync(textFile.FullName, data, token);
253 |         }
254 |     }
255 | }


--------------------------------------------------------------------------------
/VOffline/Services/Vk/VkApiUtils.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.Linq;
  4 | using System.Text.RegularExpressions;
  5 | using System.Threading;
  6 | using System.Threading.Tasks;
  7 | using log4net;
  8 | using VkNet;
  9 | using VkNet.Enums;
 10 | using VkNet.Enums.Filters;
 11 | using VkNet.Enums.SafetyEnums;
 12 | using VkNet.Model;
 13 | using VkNet.Model.Attachments;
 14 | using VkNet.Model.RequestParams;
 15 | using VkNet.Utils;
 16 | using VOffline.Models.Storage;
 17 | 
 18 | namespace VOffline.Services.Vk
 19 | {
 20 |     public class VkApiUtils
 21 |     {
 22 |         private readonly VkApi vkApi;
 23 | 
 24 |         public VkApiUtils(VkApi vkApi)
 25 |         {
 26 |             this.vkApi = vkApi;
 27 |         }
 28 | 
 29 |         public async Task<long> ResolveId(string target)
 30 |         {
 31 |             // any community type with id, eg. club123 or event123
 32 |             var communityMatch = CommunityPattern.Match(target);
 33 |             if (communityMatch.Success)
 34 |             {
 35 |                 return -1 * Int64.Parse(communityMatch.Groups[2].Value);
 36 |             }
 37 | 
 38 |             // any user eg. id123
 39 |             var personalMatch = PersonalPattern.Match(target);
 40 |             if (personalMatch.Success)
 41 |             {
 42 |                 return Int64.Parse(personalMatch.Groups[2].Value);
 43 |             }
 44 | 
 45 |             // any id eg. 123 or -123
 46 |             var digitalMatch = DigitalPattern.Match(target);
 47 |             if (digitalMatch.Success)
 48 |             {
 49 |                 return Int64.Parse(digitalMatch.Groups[1].Value);
 50 |             }
 51 | 
 52 |             // any screen name
 53 |             var vkObj = await vkApi.Utils.ResolveScreenNameAsync(target);
 54 |             switch (vkObj.Type)
 55 |             {
 56 |                 case VkObjectType.User:
 57 |                     return vkObj.Id.Value;
 58 |                 case VkObjectType.Group:
 59 |                     return -1 * vkObj.Id.Value;
 60 |                 default:
 61 |                     throw new ArgumentOutOfRangeException();
 62 |             }
 63 |         }
 64 | 
 65 |         public async Task<string> GetName(long id)
 66 |         {
 67 |             if (id >= 0)
 68 |             {
 69 |                 var users = await vkApi.Users.GetAsync(new []{id}, ProfileFields.All);
 70 |                 var user = users[0];
 71 |                 return String.Join(" ", user.LastName, user.FirstName, GroupOrEmpty(" - ", user.Id.ToString(), user.ScreenName, user.Domain));
 72 |             }
 73 |             var groups = await vkApi.Groups.GetByIdAsync(null, (-1*id).ToString(), GroupsFields.All);
 74 |             var group = groups[0];
 75 |             return String.Join(" ", group.Name, GroupOrEmpty(" - ", group.Id.ToString(), group.ScreenName, group.Type.ToString()));
 76 |         }
 77 | 
 78 |         public PageGetter<PhotoAlbum> PhotoAlbums(long id) => async (count, offset) => await vkApi.Photo.GetAlbumsAsync(new PhotoGetAlbumsParams()
 79 |         {
 80 |             OwnerId = id,
 81 |             NeedCovers = true,
 82 |             NeedSystem = true,
 83 |             Count = (uint)count,
 84 |             Offset = (uint)offset
 85 |         });
 86 | 
 87 |         public PageGetter<Photo> Photos(long id) => async (count, offset) => await vkApi.Photo.GetAllAsync(new PhotoGetAllParams()
 88 |         {
 89 |             OwnerId = id,
 90 |             Extended = true,
 91 |             SkipHidden = false,
 92 |             Count = (ulong)count,
 93 |             Offset = (ulong)offset
 94 |         });
 95 | 
 96 |         public PageGetter<Photo> PhotosInAlbum(long id, PhotoAlbumType albumIdType) => async (count, offset) => await vkApi.Photo.GetAsync(new PhotoGetParams()
 97 |         {
 98 |             OwnerId = id,
 99 |             AlbumId = albumIdType,
100 |             Extended = true,
101 |             PhotoSizes = true,
102 |             Count = (ulong)count,
103 |             Offset = (ulong)offset
104 |         });
105 | 
106 |         public PageGetter<Post> Posts(long id) => async (count, offset) =>
107 |         {
108 |             var response = await vkApi.Wall.GetAsync(new WallGetParams
109 |             {
110 |                 OwnerId = id,
111 |                 Count = (ulong)count,
112 |                 Offset = (ulong)offset
113 |             });
114 |             return new VkCollection<Post>(response.TotalCount, response.WallPosts);
115 |         };
116 | 
117 |         public PageGetter<AudioPlaylist> Playlists(long id) => async (count, offset) => await vkApi.Audio.GetPlaylistsAsync(id, (uint)count, (uint)offset);
118 | 
119 |         public PageGetter<Audio> Audios(long id) => async (count, offset) => await vkApi.Audio.GetAsync(new AudioGetParams()
120 |         {
121 |             OwnerId = id,
122 |             Count = (long)count,
123 |             Offset = (long)offset
124 |         });
125 | 
126 |         public PageGetter<Audio> AudiosInPlaylist(AudioPlaylist playlist) => async (count, offset) => await vkApi.Audio.GetAsync(new AudioGetParams()
127 |         {
128 |             OwnerId = playlist.OwnerId.Value,
129 |             AlbumId = playlist.Id.Value,
130 |             Count = (long)count,
131 |             Offset = (long)offset
132 |         });
133 | 
134 |         public PageGetter<Comment> Comments(Post post) => async (count, offset) => await vkApi.Wall.GetCommentsAsync(new WallGetCommentsParams()
135 |         {
136 |             Count = (uint)count,
137 |             Offset = (uint)offset,
138 |             OwnerId = post.OwnerId.Value,
139 |             PostId = post.Id.Value,
140 |         });
141 | 
142 |         public async Task<IReadOnlyList<T>> GetAllPagesAsync<T>(PageGetter<T> pageGetter, decimal pageSize, CancellationToken token, ILog log, bool throwIfCountMismatch=false)
143 |         {
144 |             var firstPage = await pageGetter(pageSize, 0);
145 |             var total = firstPage.TotalCount;
146 |             log.Debug($"{0}/{total}, {firstPage.Count} items");
147 | 
148 |             var result = new HashSet<T>((int)total);
149 |             result.AddUnique(firstPage, log);
150 | 
151 |             var remainingPages = total / pageSize;
152 |             var pageTasks = Enumerable.Range(1, (int)remainingPages)
153 |                 .Select(pageNumber => (ulong)pageNumber * pageSize)
154 |                 .Select(async offset =>
155 |                 {
156 |                     VkCollection<T> page = null;
157 |                     try
158 |                     {
159 |                         page = await pageGetter(pageSize, offset);
160 |                         log.Debug($"{offset}/{total}, {page.Count} items");
161 |                     }
162 |                     catch (Exception e)
163 |                     {
164 |                         log.Warn($"Error getting page {offset}/{total}. Getting page by smaller chunks...", e);
165 |                         var items = await GetSinglePage(pageGetter, offset, pageSize, token, log);
166 |                         page = new VkCollection<T>(0, items);
167 |                     }
168 | 
169 |                     token.ThrowIfCancellationRequested();
170 |                     return page ?? new VkCollection<T>(0, new List<T>());
171 |                 });
172 |             var pages = await Task.WhenAll(pageTasks);
173 |             foreach (var page in pages)
174 |             {
175 |                 result.AddUnique(page, log);
176 |             }
177 | 
178 |             if ((int)total != result.Count)
179 |             {
180 |                 var message = $"Expected {total} items, got {result.Count}. Maybe they were created/deleted, or it's VK bugs again.";
181 |                 log.Warn(message);
182 |                 if (throwIfCountMismatch)
183 |                 {
184 |                     throw new InvalidOperationException(message);
185 |                 }
186 |             }
187 | 
188 |             return result.ToList();
189 |         }
190 | 
191 |         private async Task<IEnumerable<T>> GetSinglePage<T>(PageGetter<T> pageGetter, decimal offset, decimal length, CancellationToken token, ILog log)
192 |         {
193 |             var pageTasks = Enumerable.Range((int)offset, (int)length)
194 |                 .Select(async x =>
195 |                 {
196 |                     VkCollection<T> page = null;
197 |                     try
198 |                     {
199 |                         page = await pageGetter(1, offset);
200 |                         log.Debug($"{x}/{length} item from bad page");
201 |                     }
202 |                     catch (Exception e)
203 |                     {
204 |                         log.Warn($"Error getting item {x}", e);
205 |                     }
206 | 
207 |                     token.ThrowIfCancellationRequested();
208 |                     return page ?? new VkCollection<T>(0, new List<T>());
209 |                 });
210 |             var pages = await Task.WhenAll(pageTasks);
211 |             return pages.SelectMany(x => x);
212 |         }
213 |         
214 |         public async Task<PlaylistWithAudio> ExpandPlaylist(AudioPlaylist playlist, CancellationToken token, ILog log)
215 |         {
216 |             var audios = await GetAllPagesAsync(AudiosInPlaylist(playlist), long.MaxValue, token, log, true);
217 |             log.Debug($"Expanded playlist {playlist.Title}: {audios.Count} audios");
218 |             return new PlaylistWithAudio(playlist, audios);
219 |         }
220 | 
221 |         public async Task<int> GetPhotoAlbumsSimpleCountAsync(long id, CancellationToken token, ILog log)
222 |         {
223 |             throw new NotImplementedException();
224 |             var errors = new Lazy<List<Exception>>();
225 |             try
226 |             {
227 |                 return await vkApi.Photo.GetAlbumsCountAsync(id, null);
228 |             }
229 |             catch (Exception e)
230 |             {
231 |                 errors.Value.Add(e);
232 | 
233 |             }
234 |             token.ThrowIfCancellationRequested();
235 |             try
236 |             {
237 |                 return await vkApi.Photo.GetAlbumsCountAsync(null, id);
238 |             }
239 |             catch (Exception e)
240 |             {
241 |                 errors.Value.Add(e);
242 | 
243 |             }
244 |             throw new Exception(string.Join("\n---------------------------------\n", errors.Value.Select(x => x.ToString())));
245 |         }
246 | 
247 |         public async Task<AlbumWithPhoto> ExpandAlbum(Album album, CancellationToken token, ILog log)
248 |         {
249 |             var photos = await GetAllPagesAsync(PhotosInAlbum(album.OwnerId.Value, GetAlbumType(album.Id.Value)), 1000, token, log, true);
250 |             return new AlbumWithPhoto(album, photos);
251 |         }
252 | 
253 |         public async Task<AlbumWithPhoto> ExpandAlbum(PhotoAlbum album, CancellationToken token, ILog log)
254 |         {
255 |             var photos = await GetAllPagesAsync(PhotosInAlbum(album.OwnerId.Value, GetAlbumType(album.Id)), 1000, token, log, true);
256 |             return new AlbumWithPhoto(album, photos);
257 |         }
258 | 
259 |         private static PhotoAlbumType GetAlbumType(long albumId)
260 |         {
261 |             switch (albumId)
262 |             {
263 |                 case -6:
264 |                     return PhotoAlbumType.Profile;
265 |                 case -7:
266 |                     return PhotoAlbumType.Wall;
267 |                 case -15:
268 |                     return PhotoAlbumType.Saved;
269 |                 default:
270 |                     return PhotoAlbumType.Id(albumId);
271 |             }
272 |         }
273 | 
274 |         private static string GroupOrEmpty(string separator, params string[] parts)
275 |         {
276 |             var all = String.Join(separator, parts);
277 |             return String.IsNullOrEmpty(all)
278 |                 ? String.Empty
279 |                 : $"({all})";
280 |         }
281 | 
282 |         private static readonly Regex CommunityPattern = new Regex(@"^(public|club|event)(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
283 |         private static readonly Regex PersonalPattern = new Regex(@"^(id)(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
284 |         private static readonly Regex DigitalPattern = new Regex(@"^(-?\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
285 |     }
286 | 
287 |     public delegate Task<VkCollection<T>> PageGetter<T>(decimal count, decimal offset);
288 | }


--------------------------------------------------------------------------------