├── .DS_Store ├── .dockerignore ├── .gitignore ├── DbArchiver.Core.Unit.Tests ├── DatabaseArchiverTests.cs ├── DbArchiver.Core.Unit.Tests.csproj └── Stub │ ├── SourceSettings.cs │ └── TargetSettings.cs ├── DbArchiver.Core ├── Config │ ├── ArchiverConfiguration.cs │ ├── ArchiverConfigurationFactory.cs │ ├── JobSchedulerSettings.cs │ └── TransferSettings.cs ├── Contract │ ├── IArchiverConfigurationFactory.cs │ └── IDatabaseArchiverFactory.cs ├── DatabaseArchiver.cs ├── DatabaseArchiverFactory.cs ├── DbArchiver.Core.csproj ├── Helper │ ├── AssemblyTypeResolver.cs │ ├── Constants.cs │ └── IpPinger.cs └── ServiceCollectionExtension.cs ├── DbArchiver.Provider.Common ├── Config │ ├── ISourceSettings.cs │ └── ITargetSettings.cs ├── DbArchiver.Provider.Common.csproj ├── IDatabaseProviderIterator.cs ├── IDatabaseProviderSource.cs └── IDatabaseProviderTarget.cs ├── DbArchiver.Provider.MSSQL ├── Config │ ├── SourceSettings.cs │ └── TargetSettings.cs ├── DbArchiver.Provider.MSSQL.csproj ├── MSSQLProvider.cs ├── MSSQLProviderIterator.cs └── ServiceCollectionExtension.cs ├── DbArchiver.Provider.MongoDB ├── Config │ ├── SourceSettings.cs │ └── TargetSettings.cs ├── DbArchiver.Provider.MongoDB.csproj ├── MongoDBProvider.cs ├── MongoDBProviderIterator.cs └── ServiceCollectionExtension.cs ├── DbArchiver.Provider.MySQL ├── Config │ ├── SourceSettings.cs │ └── TargetSettings.cs ├── DbArchiver.Provider.MySQL.csproj ├── MySQLProvider.cs ├── MySQLProviderIterator.cs └── ServiceCollectionExtension.cs ├── DbArchiver.Provider.PostgreSQL ├── Config │ ├── SourceSettings.cs │ └── TargetSettings.cs ├── DbArchiver.Provider.PostgreSQL.csproj ├── PostgreSQLProvider.cs ├── PostgreSQLProviderIterator.cs └── ServiceCollectionExtension.cs ├── DbArchiver.Provider.SQLite ├── Config │ ├── SourceSettings.cs │ └── TargetSettings.cs ├── DbArchiver.Provider.SQLite.csproj ├── SQLiteProvider.cs ├── SQLiteProviderIterator.cs └── ServiceCollectionExtension.cs ├── DbArchiver.WService ├── DataTransferJob.cs ├── DbArchiver.WService.csproj ├── Dockerfile ├── Program.cs ├── Properties │ └── launchSettings.json ├── appsettings.Development.json └── appsettings.json ├── DbArchiver.sln ├── LICENSE ├── README.md └── tests └── IntegrationTests ├── DbArchiver.Provider.Integration.Tests.csproj ├── MSSQLProviderTests.cs ├── MySQLProviderTests.cs ├── PostgreSQLProviderTests.cs └── SQLiteProviderTests.cs /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasulhsn/Database-Archiver/c897e0fce02baaab5ca1611a726d3f083e9997be/.DS_Store -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.classpath 2 | **/.dockerignore 3 | **/.env 4 | **/.git 5 | **/.gitignore 6 | **/.project 7 | **/.settings 8 | **/.toolstarget 9 | **/.vs 10 | **/.vscode 11 | **/*.*proj.user 12 | **/*.dbmdl 13 | **/*.jfm 14 | **/azds.yaml 15 | **/bin 16 | **/charts 17 | **/docker-compose* 18 | **/Dockerfile* 19 | **/node_modules 20 | **/npm-debug.log 21 | **/obj 22 | **/secrets.dev.yaml 23 | **/values.dev.yaml 24 | LICENSE 25 | README.md 26 | !**/.gitignore 27 | !.git/HEAD 28 | !.git/config 29 | !.git/packed-refs 30 | !.git/refs/heads/** -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | .idea/* 13 | 14 | # User-specific files (MonoDevelop/Xamarin Studio) 15 | *.userprefs 16 | 17 | # Mono auto generated files 18 | mono_crash.* 19 | 20 | # Build results 21 | [Dd]ebug/ 22 | [Dd]ebugPublic/ 23 | [Rr]elease/ 24 | [Rr]eleases/ 25 | x64/ 26 | x86/ 27 | [Ww][Ii][Nn]32/ 28 | [Aa][Rr][Mm]/ 29 | [Aa][Rr][Mm]64/ 30 | bld/ 31 | [Bb]in/ 32 | [Oo]bj/ 33 | [Ll]og/ 34 | [Ll]ogs/ 35 | 36 | # Visual Studio 2015/2017 cache/options directory 37 | .vs/ 38 | # Uncomment if you have tasks that create the project's static files in wwwroot 39 | #wwwroot/ 40 | 41 | # Visual Studio 2017 auto generated files 42 | Generated\ Files/ 43 | 44 | # MSTest test Results 45 | [Tt]est[Rr]esult*/ 46 | [Bb]uild[Ll]og.* 47 | 48 | # NUnit 49 | *.VisualState.xml 50 | TestResult.xml 51 | nunit-*.xml 52 | 53 | # Build Results of an ATL Project 54 | [Dd]ebugPS/ 55 | [Rr]eleasePS/ 56 | dlldata.c 57 | 58 | # Benchmark Results 59 | BenchmarkDotNet.Artifacts/ 60 | 61 | # .NET Core 62 | project.lock.json 63 | project.fragment.lock.json 64 | artifacts/ 65 | 66 | # ASP.NET Scaffolding 67 | ScaffoldingReadMe.txt 68 | 69 | # StyleCop 70 | StyleCopReport.xml 71 | 72 | # Files built by Visual Studio 73 | *_i.c 74 | *_p.c 75 | *_h.h 76 | *.ilk 77 | *.meta 78 | *.obj 79 | *.iobj 80 | *.pch 81 | *.pdb 82 | *.ipdb 83 | *.pgc 84 | *.pgd 85 | *.rsp 86 | # but not Directory.Build.rsp, as it configures directory-level build defaults 87 | !Directory.Build.rsp 88 | *.sbr 89 | *.tlb 90 | *.tli 91 | *.tlh 92 | *.tmp 93 | *.tmp_proj 94 | *_wpftmp.csproj 95 | *.log 96 | *.tlog 97 | *.vspscc 98 | *.vssscc 99 | .builds 100 | *.pidb 101 | *.svclog 102 | *.scc 103 | 104 | # Chutzpah Test files 105 | _Chutzpah* 106 | 107 | # Visual C++ cache files 108 | ipch/ 109 | *.aps 110 | *.ncb 111 | *.opendb 112 | *.opensdf 113 | *.sdf 114 | *.cachefile 115 | *.VC.db 116 | *.VC.VC.opendb 117 | 118 | # Visual Studio profiler 119 | *.psess 120 | *.vsp 121 | *.vspx 122 | *.sap 123 | 124 | # Visual Studio Trace Files 125 | *.e2e 126 | 127 | # TFS 2012 Local Workspace 128 | $tf/ 129 | 130 | # Guidance Automation Toolkit 131 | *.gpState 132 | 133 | # ReSharper is a .NET coding add-in 134 | _ReSharper*/ 135 | *.[Rr]e[Ss]harper 136 | *.DotSettings.user 137 | 138 | # TeamCity is a build add-in 139 | _TeamCity* 140 | 141 | # DotCover is a Code Coverage Tool 142 | *.dotCover 143 | 144 | # AxoCover is a Code Coverage Tool 145 | .axoCover/* 146 | !.axoCover/settings.json 147 | 148 | # Coverlet is a free, cross platform Code Coverage Tool 149 | coverage*.json 150 | coverage*.xml 151 | coverage*.info 152 | 153 | # Visual Studio code coverage results 154 | *.coverage 155 | *.coveragexml 156 | 157 | # NCrunch 158 | _NCrunch_* 159 | .*crunch*.local.xml 160 | nCrunchTemp_* 161 | 162 | # MightyMoose 163 | *.mm.* 164 | AutoTest.Net/ 165 | 166 | # Web workbench (sass) 167 | .sass-cache/ 168 | 169 | # Installshield output folder 170 | [Ee]xpress/ 171 | 172 | # DocProject is a documentation generator add-in 173 | DocProject/buildhelp/ 174 | DocProject/Help/*.HxT 175 | DocProject/Help/*.HxC 176 | DocProject/Help/*.hhc 177 | DocProject/Help/*.hhk 178 | DocProject/Help/*.hhp 179 | DocProject/Help/Html2 180 | DocProject/Help/html 181 | 182 | # Click-Once directory 183 | publish/ 184 | 185 | # Publish Web Output 186 | *.[Pp]ublish.xml 187 | *.azurePubxml 188 | # Note: Comment the next line if you want to checkin your web deploy settings, 189 | # but database connection strings (with potential passwords) will be unencrypted 190 | *.pubxml 191 | *.publishproj 192 | 193 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 194 | # checkin your Azure Web App publish settings, but sensitive information contained 195 | # in these scripts will be unencrypted 196 | PublishScripts/ 197 | 198 | # NuGet Packages 199 | *.nupkg 200 | # NuGet Symbol Packages 201 | *.snupkg 202 | # The packages folder can be ignored because of Package Restore 203 | **/[Pp]ackages/* 204 | # except build/, which is used as an MSBuild target. 205 | !**/[Pp]ackages/build/ 206 | # Uncomment if necessary however generally it will be regenerated when needed 207 | #!**/[Pp]ackages/repositories.config 208 | # NuGet v3's project.json files produces more ignorable files 209 | *.nuget.props 210 | *.nuget.targets 211 | 212 | # Microsoft Azure Build Output 213 | csx/ 214 | *.build.csdef 215 | 216 | # Microsoft Azure Emulator 217 | ecf/ 218 | rcf/ 219 | 220 | # Windows Store app package directories and files 221 | AppPackages/ 222 | BundleArtifacts/ 223 | Package.StoreAssociation.xml 224 | _pkginfo.txt 225 | *.appx 226 | *.appxbundle 227 | *.appxupload 228 | 229 | # Visual Studio cache files 230 | # files ending in .cache can be ignored 231 | *.[Cc]ache 232 | # but keep track of directories ending in .cache 233 | !?*.[Cc]ache/ 234 | 235 | # Others 236 | ClientBin/ 237 | ~$* 238 | *~ 239 | *.dbmdl 240 | *.dbproj.schemaview 241 | *.jfm 242 | *.pfx 243 | *.publishsettings 244 | orleans.codegen.cs 245 | 246 | # Including strong name files can present a security risk 247 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 248 | #*.snk 249 | 250 | # Since there are multiple workflows, uncomment next line to ignore bower_components 251 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 252 | #bower_components/ 253 | 254 | # RIA/Silverlight projects 255 | Generated_Code/ 256 | 257 | # Backup & report files from converting an old project file 258 | # to a newer Visual Studio version. Backup files are not needed, 259 | # because we have git ;-) 260 | _UpgradeReport_Files/ 261 | Backup*/ 262 | UpgradeLog*.XML 263 | UpgradeLog*.htm 264 | ServiceFabricBackup/ 265 | *.rptproj.bak 266 | 267 | # SQL Server files 268 | *.mdf 269 | *.ldf 270 | *.ndf 271 | 272 | # Business Intelligence projects 273 | *.rdl.data 274 | *.bim.layout 275 | *.bim_*.settings 276 | *.rptproj.rsuser 277 | *- [Bb]ackup.rdl 278 | *- [Bb]ackup ([0-9]).rdl 279 | *- [Bb]ackup ([0-9][0-9]).rdl 280 | 281 | # Microsoft Fakes 282 | FakesAssemblies/ 283 | 284 | # GhostDoc plugin setting file 285 | *.GhostDoc.xml 286 | 287 | # Node.js Tools for Visual Studio 288 | .ntvs_analysis.dat 289 | node_modules/ 290 | 291 | # Visual Studio 6 build log 292 | *.plg 293 | 294 | # Visual Studio 6 workspace options file 295 | *.opt 296 | 297 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 298 | *.vbw 299 | 300 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 301 | *.vbp 302 | 303 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 304 | *.dsw 305 | *.dsp 306 | 307 | # Visual Studio 6 technical files 308 | *.ncb 309 | *.aps 310 | 311 | # Visual Studio LightSwitch build output 312 | **/*.HTMLClient/GeneratedArtifacts 313 | **/*.DesktopClient/GeneratedArtifacts 314 | **/*.DesktopClient/ModelManifest.xml 315 | **/*.Server/GeneratedArtifacts 316 | **/*.Server/ModelManifest.xml 317 | _Pvt_Extensions 318 | 319 | # Paket dependency manager 320 | .paket/paket.exe 321 | paket-files/ 322 | 323 | # FAKE - F# Make 324 | .fake/ 325 | 326 | # CodeRush personal settings 327 | .cr/personal 328 | 329 | # Python Tools for Visual Studio (PTVS) 330 | __pycache__/ 331 | *.pyc 332 | 333 | # Cake - Uncomment if you are using it 334 | # tools/** 335 | # !tools/packages.config 336 | 337 | # Tabs Studio 338 | *.tss 339 | 340 | # Telerik's JustMock configuration file 341 | *.jmconfig 342 | 343 | # BizTalk build output 344 | *.btp.cs 345 | *.btm.cs 346 | *.odx.cs 347 | *.xsd.cs 348 | 349 | # OpenCover UI analysis results 350 | OpenCover/ 351 | 352 | # Azure Stream Analytics local run output 353 | ASALocalRun/ 354 | 355 | # MSBuild Binary and Structured Log 356 | *.binlog 357 | 358 | # NVidia Nsight GPU debugger configuration file 359 | *.nvuser 360 | 361 | # MFractors (Xamarin productivity tool) working folder 362 | .mfractor/ 363 | 364 | # Local History for Visual Studio 365 | .localhistory/ 366 | 367 | # Visual Studio History (VSHistory) files 368 | .vshistory/ 369 | 370 | # BeatPulse healthcheck temp database 371 | healthchecksdb 372 | 373 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 374 | MigrationBackup/ 375 | 376 | # Ionide (cross platform F# VS Code tools) working folder 377 | .ionide/ 378 | 379 | # Fody - auto-generated XML schema 380 | FodyWeavers.xsd 381 | 382 | # VS Code files for those working on multiple tools 383 | .vscode/* 384 | !.vscode/settings.json 385 | !.vscode/tasks.json 386 | !.vscode/launch.json 387 | !.vscode/extensions.json 388 | *.code-workspace 389 | 390 | # Local History for Visual Studio Code 391 | .history/ 392 | 393 | # Windows Installer files from build outputs 394 | *.cab 395 | *.msi 396 | *.msix 397 | *.msm 398 | *.msp 399 | 400 | # JetBrains Rider 401 | *.sln.iml 402 | -------------------------------------------------------------------------------- /DbArchiver.Core.Unit.Tests/DatabaseArchiverTests.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core.Config; 2 | using DbArchiver.Provider.Common; 3 | using Microsoft.Extensions.Logging; 4 | using Moq; 5 | 6 | namespace DbArchiver.Core.Unit.Tests 7 | { 8 | [TestFixture] 9 | public class DatabaseArchiverTests 10 | { 11 | private Mock _mockSource; 12 | private Mock _mockTarget; 13 | private Mock> _mockLogger; 14 | private Mock _mockIterator; 15 | private TransferSettings _transferSettings; 16 | private DatabaseArchiver _archiver; 17 | 18 | [SetUp] 19 | public void SetUp() 20 | { 21 | _mockSource = new Mock(); 22 | _mockTarget = new Mock(); 23 | _mockLogger = new Mock>(); 24 | _mockIterator = new Mock(); 25 | 26 | _transferSettings = new TransferSettings 27 | { 28 | Source = new SourceProviderSettings { Host = "localhost", TransferQuantity = 10, DeleteAfterArchived = true, Settings = new SourceSettings() }, 29 | Target = new TargetProviderSettings { Host = "localhost", PreScript = "PreScript", Settings = new TargetSettings() } 30 | }; 31 | 32 | _archiver = new DatabaseArchiver(_transferSettings, _mockSource.Object, _mockTarget.Object, _mockLogger.Object); 33 | } 34 | 35 | [Test] 36 | public async Task ArchiveAsync_ExecutesPreScript_WhenPreScriptExists() 37 | { 38 | // Arrange 39 | _mockSource.Setup(s => s.GetIteratorAsync(_transferSettings.Source.Settings, It.IsAny())) 40 | .ReturnsAsync(_mockIterator.Object); 41 | _mockIterator.Setup(i => i.NextAsync()).ReturnsAsync(false); 42 | 43 | // Act 44 | await _archiver.ArchiveAsync(CancellationToken.None); 45 | 46 | // Assert 47 | _mockTarget.Verify(t => t.ExecuteScriptAsync(_transferSettings.Target.Settings, _transferSettings.Target.PreScript), Times.Once); 48 | } 49 | 50 | [Test] 51 | public async Task ArchiveAsync_ArchivesData_WhenIteratorHasData() 52 | { 53 | // Arrange 54 | var sampleData = new List { new object() }; 55 | _mockIterator.Setup(i => i.NextAsync()).ReturnsAsync(true); 56 | _mockIterator.Setup(i => i.Data).Returns(sampleData); 57 | _mockSource.Setup(s => s.GetIteratorAsync(_transferSettings.Source.Settings, It.IsAny())) 58 | .ReturnsAsync(_mockIterator.Object); 59 | _mockIterator.SetupSequence(i => i.NextAsync()) 60 | .ReturnsAsync(true) 61 | .ReturnsAsync(false); 62 | 63 | // Act 64 | await _archiver.ArchiveAsync(CancellationToken.None); 65 | 66 | // Assert 67 | _mockTarget.Verify(t => t.InsertAsync(_transferSettings.Target.Settings, sampleData), Times.Once); 68 | } 69 | 70 | [Test] 71 | public async Task ArchiveAsync_DeletesData_WhenDeleteAfterArchivedIsTrue() 72 | { 73 | // Arrange 74 | var sampleData = new List { new object() }; 75 | _mockIterator.Setup(i => i.NextAsync()).ReturnsAsync(true); 76 | _mockIterator.Setup(i => i.Data).Returns(sampleData); 77 | _mockSource.Setup(s => s.GetIteratorAsync(_transferSettings.Source.Settings, It.IsAny())) 78 | .ReturnsAsync(_mockIterator.Object); 79 | _mockIterator.SetupSequence(i => i.NextAsync()) 80 | .ReturnsAsync(true) 81 | .ReturnsAsync(false); 82 | 83 | // Act 84 | await _archiver.ArchiveAsync(CancellationToken.None); 85 | 86 | // Assert 87 | _mockSource.Verify(s => s.DeleteAsync(_transferSettings.Source.Settings, sampleData), Times.Once); 88 | } 89 | 90 | [Test] 91 | public async Task ArchiveAsync_LogsError_WhenExceptionOccurs() 92 | { 93 | // Arrange 94 | var occurException = new Exception("Test Exception"); 95 | _mockSource.Setup(s => s.GetIteratorAsync(_transferSettings.Source.Settings, It.IsAny())) 96 | .ThrowsAsync(occurException); 97 | 98 | // Act & Assert 99 | Assert.ThrowsAsync(async () => await _archiver.ArchiveAsync(CancellationToken.None)); 100 | 101 | _mockLogger.Verify( 102 | l => l.Log( 103 | It.Is(logLevel => logLevel == LogLevel.Error), 104 | It.IsAny(), 105 | It.Is((v, t) => v.ToString().Contains(occurException.Message)), 106 | It.IsAny(), 107 | It.IsAny>()), 108 | Times.AtLeastOnce); 109 | } 110 | 111 | [Test] 112 | public async Task ArchiveAsync_DisposesIterator_WhenCompleted() 113 | { 114 | // Arrange 115 | _mockSource.Setup(s => s.GetIteratorAsync(_transferSettings.Source.Settings, It.IsAny())) 116 | .ReturnsAsync(_mockIterator.Object); 117 | _mockIterator.Setup(i => i.NextAsync()).ReturnsAsync(false); 118 | 119 | // Act 120 | await _archiver.ArchiveAsync(CancellationToken.None); 121 | 122 | // Assert 123 | _mockIterator.Verify(i => i.Dispose(), Times.Once); 124 | } 125 | } 126 | } -------------------------------------------------------------------------------- /DbArchiver.Core.Unit.Tests/DbArchiver.Core.Unit.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | false 9 | true 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /DbArchiver.Core.Unit.Tests/Stub/SourceSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Core.Unit.Tests 4 | { 5 | public class SourceSettings : ISourceSettings 6 | { 7 | } 8 | } -------------------------------------------------------------------------------- /DbArchiver.Core.Unit.Tests/Stub/TargetSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Core.Unit.Tests 4 | { 5 | public class TargetSettings : ITargetSettings 6 | { 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /DbArchiver.Core/Config/ArchiverConfiguration.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace DbArchiver.Core.Config 3 | { 4 | public class ArchiverConfiguration 5 | { 6 | public IEnumerable Items { get; set; } 7 | } 8 | 9 | public class ArchiverConfigurationItem 10 | { 11 | public JobSchedulerSettings JobSchedulerSettings { get; set; } 12 | public TransferSettings TransferSettings { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /DbArchiver.Core/Config/ArchiverConfigurationFactory.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core.Contract; 2 | using DbArchiver.Core.Helper; 3 | using DbArchiver.Provider.Common.Config; 4 | using Microsoft.Extensions.Configuration; 5 | 6 | namespace DbArchiver.Core.Config 7 | { 8 | public class ArchiverConfigurationFactory : IArchiverConfigurationFactory 9 | { 10 | private readonly IConfiguration _configuration; 11 | 12 | public ArchiverConfigurationFactory(IConfiguration configuration) 13 | { 14 | _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); 15 | } 16 | 17 | public ArchiverConfiguration Create() 18 | { 19 | ArchiverConfiguration instance = new ArchiverConfiguration(); 20 | 21 | var section = _configuration.GetSection($"{nameof(ArchiverConfiguration)}"); 22 | 23 | var items = new List(); 24 | 25 | foreach (var child in section.GetChildren()) 26 | { 27 | var item = new ArchiverConfigurationItem 28 | { 29 | JobSchedulerSettings = new JobSchedulerSettings 30 | { 31 | JobName = child.GetSection($"{nameof(ArchiverConfigurationItem.JobSchedulerSettings)}:{nameof(JobSchedulerSettings.JobName)}") 32 | .Value, 33 | Cron = child.GetSection($"{nameof(ArchiverConfigurationItem.JobSchedulerSettings)}:{nameof(JobSchedulerSettings.Cron)}") 34 | .Value 35 | }, 36 | TransferSettings = new TransferSettings 37 | { 38 | Source = new SourceProviderSettings 39 | { 40 | Provider = child.GetSection($"{nameof(TransferSettings)}:{nameof(TransferSettings.Source)}:{nameof(SourceProviderSettings.Provider)}") 41 | .Value, 42 | Host = child.GetSection($"{nameof(TransferSettings)}:{nameof(TransferSettings.Source)}:{nameof(SourceProviderSettings.Host)}") 43 | .Value, 44 | TransferQuantity = child.GetSection($"{nameof(TransferSettings)}:{nameof(TransferSettings.Source)}:{nameof(SourceProviderSettings.TransferQuantity)}") 45 | .Get(), 46 | DeleteAfterArchived = child.GetSection($"{nameof(TransferSettings)}:{nameof(TransferSettings.Source)}:{nameof(SourceProviderSettings.DeleteAfterArchived)}") 47 | .Get(), 48 | }, 49 | Target = new TargetProviderSettings 50 | { 51 | Provider = child.GetSection($"{nameof(TransferSettings)}:{nameof(TransferSettings.Target)}:{nameof(TargetProviderSettings.Provider)}") 52 | .Value, 53 | Host = child.GetSection($"{nameof(TransferSettings)}:{nameof(TransferSettings.Target)}:{nameof(TargetProviderSettings.Host)}") 54 | .Value, 55 | PreScript = child.GetSection($"{nameof(TransferSettings)}:{nameof(TransferSettings.Target)}:{nameof(TargetProviderSettings.PreScript)}") 56 | .Value 57 | } 58 | } 59 | }; 60 | 61 | item.TransferSettings.Source.Settings = CreateSettings(item.TransferSettings.Source.Provider, 62 | child.GetSection($"{nameof(TransferSettings)}:Source:Settings") 63 | ); 64 | 65 | item.TransferSettings.Target.Settings = CreateSettings(item.TransferSettings.Target.Provider, 66 | child.GetSection($"{nameof(TransferSettings)}:Target:Settings") 67 | ); 68 | 69 | items.Add(item); 70 | } 71 | 72 | instance.Items = items; 73 | 74 | return instance; 75 | } 76 | 77 | private TInterface CreateSettings(string providerName, IConfigurationSection section) where TInterface : class 78 | { 79 | if (string.IsNullOrWhiteSpace(providerName)) 80 | { 81 | throw new ArgumentException("Source provider name cannot be null or empty."); 82 | } 83 | 84 | var sourceType = ResolveType(providerName); 85 | return BindSettings(sourceType, section); 86 | } 87 | 88 | private static Type ResolveType(string providerName) 89 | { 90 | var assemblyName = $"{Constants.PROVIDER_ASSEMBLY_PREFIX}{providerName}"; 91 | var resolvedType = AssemblyTypeResolver.ResolveByInterface(assemblyName); 92 | 93 | if (resolvedType == null) 94 | { 95 | throw new InvalidOperationException($"Could not resolve type for provider '{providerName}'."); 96 | } 97 | 98 | return resolvedType; 99 | } 100 | 101 | private static T BindSettings(Type type, IConfigurationSection section) where T : class 102 | { 103 | var instance = Activator.CreateInstance(type); 104 | 105 | if (instance == null) 106 | { 107 | throw new InvalidOperationException($"Could not create instance of type '{type.FullName}'."); 108 | } 109 | 110 | section.Bind(instance); 111 | return instance as T ?? throw new InvalidCastException($"Instance of type '{type.FullName}' could not be cast to '{typeof(T).FullName}'."); 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /DbArchiver.Core/Config/JobSchedulerSettings.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace DbArchiver.Core.Config 3 | { 4 | public class JobSchedulerSettings 5 | { 6 | public string JobName { get; set; } 7 | public string Cron { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /DbArchiver.Core/Config/TransferSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Core.Config 4 | { 5 | public class TransferSettings 6 | { 7 | public SourceProviderSettings Source { get; set; } 8 | public TargetProviderSettings Target { get; set; } 9 | } 10 | 11 | public class SourceProviderSettings 12 | { 13 | public string Provider { get; set; } 14 | public string Host { get; set; } 15 | public int TransferQuantity { get; set; } 16 | public bool DeleteAfterArchived { get; set; } 17 | public ISourceSettings Settings { get; set; } 18 | } 19 | 20 | public class TargetProviderSettings 21 | { 22 | public string Provider { get; set; } 23 | public string Host { get; set; } 24 | public string PreScript { get; set; } 25 | public bool HasPreScript => !string.IsNullOrEmpty(PreScript); 26 | public ITargetSettings Settings { get; set; } 27 | } 28 | } -------------------------------------------------------------------------------- /DbArchiver.Core/Contract/IArchiverConfigurationFactory.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core.Config; 2 | 3 | namespace DbArchiver.Core.Contract 4 | { 5 | public interface IArchiverConfigurationFactory 6 | { 7 | ArchiverConfiguration Create(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /DbArchiver.Core/Contract/IDatabaseArchiverFactory.cs: -------------------------------------------------------------------------------- 1 |  2 | using DbArchiver.Core.Config; 3 | 4 | namespace DbArchiver.Core.Contract 5 | { 6 | public interface IDatabaseArchiverFactory 7 | { 8 | DatabaseArchiver Create(TransferSettings TransferSettings); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /DbArchiver.Core/DatabaseArchiver.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core.Config; 2 | using DbArchiver.Core.Helper; 3 | using DbArchiver.Provider.Common; 4 | using Microsoft.Extensions.Logging; 5 | 6 | namespace DbArchiver.Core 7 | { 8 | public class DatabaseArchiver 9 | { 10 | private readonly IDatabaseProviderSource _providerSource; 11 | private readonly IDatabaseProviderTarget _providerTarget; 12 | private readonly TransferSettings _transferSettings; 13 | 14 | private readonly ILogger _logger; 15 | 16 | public DatabaseArchiver(TransferSettings transferSettings, 17 | IDatabaseProviderSource source, 18 | IDatabaseProviderTarget target, 19 | ILogger logger) 20 | { 21 | _transferSettings = transferSettings; 22 | _providerSource = source; 23 | _providerTarget = target; 24 | 25 | _logger = logger; 26 | } 27 | 28 | public async Task ArchiveAsync(CancellationToken cancellationToken) 29 | { 30 | _logger.LogDebug($"{nameof(DatabaseArchiver.ArchiveAsync)} started!"); 31 | 32 | PingHosts(); 33 | 34 | IDatabaseProviderIterator iterator = null; 35 | 36 | try 37 | { 38 | // execute pre-script 39 | if (_transferSettings.Target.HasPreScript) 40 | await _providerTarget.ExecuteScriptAsync(_transferSettings.Target.Settings, 41 | _transferSettings.Target.PreScript); 42 | 43 | iterator = await _providerSource.GetIteratorAsync(_transferSettings.Source.Settings, 44 | _transferSettings.Source.TransferQuantity); 45 | while ((await iterator.NextAsync())) 46 | { 47 | // should be transactional! 48 | await _providerTarget.InsertAsync(_transferSettings.Target.Settings, iterator.Data); 49 | 50 | if (_transferSettings.Source.DeleteAfterArchived) 51 | await _providerSource.DeleteAsync(_transferSettings.Source.Settings, iterator.Data); 52 | 53 | _logger.LogDebug($"{nameof(DatabaseArchiver.ArchiveAsync)} archived {_transferSettings.Source.TransferQuantity} data!"); 54 | } 55 | } 56 | catch (Exception ex) 57 | { 58 | _logger.LogError(ex.Message); 59 | throw; 60 | } 61 | finally 62 | { 63 | iterator?.Dispose(); 64 | } 65 | 66 | _logger.LogDebug($"{nameof(DatabaseArchiver.ArchiveAsync)} Finished!"); 67 | } 68 | 69 | private void PingHosts() 70 | { 71 | IpPinger.Ping(_transferSettings.Source.Host); 72 | IpPinger.Ping(_transferSettings.Target.Host); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /DbArchiver.Core/DatabaseArchiverFactory.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core.Contract; 2 | using DbArchiver.Core.Config; 3 | using DbArchiver.Core.Helper; 4 | using DbArchiver.Provider.Common; 5 | using Microsoft.Extensions.Configuration; 6 | using Microsoft.Extensions.DependencyInjection; 7 | using Microsoft.Extensions.Logging; 8 | 9 | namespace DbArchiver.Core 10 | { 11 | public class DatabaseArchiverFactory : IDatabaseArchiverFactory 12 | { 13 | private readonly IServiceProvider _serviceProvider; 14 | private readonly IConfiguration _configuration; 15 | 16 | private readonly ILogger _logger; 17 | 18 | public DatabaseArchiverFactory(IServiceProvider serviceProvider, 19 | IConfiguration configuration, 20 | ILogger logger) 21 | { 22 | _serviceProvider = serviceProvider; 23 | _configuration = configuration; 24 | 25 | _logger = logger; 26 | } 27 | 28 | public DatabaseArchiver Create(TransferSettings TransferSettings) 29 | { 30 | string sourceAssemblyName = $"{Constants.PROVIDER_ASSEMBLY_PREFIX}{TransferSettings.Source.Provider}"; 31 | Type sourceType = AssemblyTypeResolver.ResolveByInterface(sourceAssemblyName); 32 | 33 | string targetAssemblyName = $"{Constants.PROVIDER_ASSEMBLY_PREFIX}{TransferSettings.Target.Provider}"; 34 | Type targetType = AssemblyTypeResolver.ResolveByInterface(targetAssemblyName); 35 | 36 | var providerSource = _serviceProvider.GetService(sourceType) as IDatabaseProviderSource; 37 | var providerTarget = _serviceProvider.GetService(targetType) as IDatabaseProviderTarget; 38 | 39 | if (providerSource == null) 40 | { 41 | string errorMessage = "Database Source Provider is invalid!"; 42 | _logger.LogError(errorMessage); 43 | throw new Exception(errorMessage); 44 | } 45 | 46 | if (providerTarget == null) 47 | { 48 | string errorMessage = "Database Target Provider is invalid!"; 49 | _logger.LogError(errorMessage); 50 | throw new Exception(errorMessage); 51 | } 52 | 53 | var loggerInstance = _serviceProvider.GetService>(); 54 | 55 | return new DatabaseArchiver(TransferSettings, 56 | providerSource, 57 | providerTarget, 58 | loggerInstance!); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /DbArchiver.Core/DbArchiver.Core.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /DbArchiver.Core/Helper/AssemblyTypeResolver.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | 3 | namespace DbArchiver.Core.Helper 4 | { 5 | public static class AssemblyTypeResolver 6 | { 7 | public static Type ResolveByInterface(string assemblyName) 8 | { 9 | AssemblyName? assemblyNameInstance = Assembly.GetEntryAssembly().GetReferencedAssemblies() 10 | .Single(x => x.Name.Equals(assemblyName, StringComparison.OrdinalIgnoreCase)); 11 | 12 | Assembly providerAssembly = Assembly.Load(assemblyNameInstance); 13 | 14 | Type? resolvedType = providerAssembly.GetTypes() 15 | .Single(x => x.IsClass && !x.IsAbstract 16 | && x.IsAssignableTo(typeof(IInterface))); 17 | 18 | return resolvedType!; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /DbArchiver.Core/Helper/Constants.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace DbArchiver.Core.Helper 3 | { 4 | public static class Constants 5 | { 6 | public const string PROVIDER_ASSEMBLY_PREFIX = "DbArchiver.Provider."; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /DbArchiver.Core/Helper/IpPinger.cs: -------------------------------------------------------------------------------- 1 | using System.Net.NetworkInformation; 2 | 3 | namespace DbArchiver.Core.Helper 4 | { 5 | public static class IpPinger 6 | { 7 | public static void Ping(string hostIp, int timeout = 3000) 8 | { 9 | var ping = new Ping(); 10 | 11 | var pingReply = ping.Send(hostIp, timeout); 12 | if (pingReply?.Status != IPStatus.Success) 13 | { 14 | throw new Exception($"{hostIp} is unreachable"); 15 | } 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /DbArchiver.Core/ServiceCollectionExtension.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core.Contract; 2 | using DbArchiver.Core.Config; 3 | using Microsoft.Extensions.DependencyInjection; 4 | 5 | namespace DbArchiver.Core 6 | { 7 | public static class ServiceCollectionExtension 8 | { 9 | public static IServiceCollection AddCoreServices(this IServiceCollection services) { 10 | 11 | services.AddSingleton(); 12 | services.AddSingleton(); 13 | 14 | return services; 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /DbArchiver.Provider.Common/Config/ISourceSettings.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace DbArchiver.Provider.Common.Config 3 | { 4 | public interface ISourceSettings 5 | { 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /DbArchiver.Provider.Common/Config/ITargetSettings.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace DbArchiver.Provider.Common.Config 3 | { 4 | public interface ITargetSettings 5 | { 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /DbArchiver.Provider.Common/DbArchiver.Provider.Common.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /DbArchiver.Provider.Common/IDatabaseProviderIterator.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace DbArchiver.Provider.Common 3 | { 4 | public interface IDatabaseProviderIterator : IDisposable 5 | { 6 | T Data { get; } 7 | Task NextAsync(); 8 | } 9 | 10 | public interface IDatabaseProviderIterator : IDatabaseProviderIterator> 11 | { 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /DbArchiver.Provider.Common/IDatabaseProviderSource.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.Common 4 | { 5 | public interface IDatabaseProviderSource 6 | { 7 | Task GetIteratorAsync(ISourceSettings settings, int transferQuantity); 8 | Task DeleteAsync(ISourceSettings settings, IEnumerable data); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /DbArchiver.Provider.Common/IDatabaseProviderTarget.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.Common 4 | { 5 | public interface IDatabaseProviderTarget 6 | { 7 | Task ExecuteScriptAsync(ITargetSettings settings, string script); 8 | Task InsertAsync(ITargetSettings settings, IEnumerable data); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MSSQL/Config/SourceSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.MSSQL.Config 4 | { 5 | public class SourceSettings : ISourceSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MSSQL/Config/TargetSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.MSSQL.Config 4 | { 5 | public class TargetSettings : ITargetSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MSSQL/DbArchiver.Provider.MSSQL.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MSSQL/MSSQLProvider.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.Common; 3 | using DbArchiver.Provider.Common.Config; 4 | using DbArchiver.Provider.MSSQL.Config; 5 | using Microsoft.Data.SqlClient; 6 | using Microsoft.Extensions.Logging; 7 | using System.Text; 8 | using static Dapper.SqlMapper; 9 | 10 | namespace DbArchiver.Provider.MSSQL 11 | { 12 | public class MSSQLProvider : IDatabaseProviderSource, 13 | IDatabaseProviderTarget 14 | { 15 | private readonly ILogger _logger; 16 | 17 | public MSSQLProvider(ILogger logger) 18 | { 19 | _logger = logger; 20 | } 21 | 22 | public async Task DeleteAsync(ISourceSettings settings, IEnumerable data) 23 | { 24 | if (data == null || !data.Any()) 25 | return; 26 | 27 | var sourceSettings = ResolveSourceSettings(settings); 28 | 29 | using (var connection = new SqlConnection(sourceSettings.ConnectionString)) 30 | { 31 | try 32 | { 33 | var dataDictionaries = data.Select(item => 34 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 35 | 36 | await connection.OpenAsync(); 37 | 38 | foreach (var record in dataDictionaries) 39 | { 40 | if (!record.ContainsKey(sourceSettings.IdColumn)) 41 | throw new InvalidOperationException($"Primary key '{sourceSettings.IdColumn}' not found in record."); 42 | 43 | var query = $"DELETE FROM {sourceSettings.Schema}.{sourceSettings.Table} WHERE {sourceSettings.IdColumn} = @Id"; 44 | 45 | await connection.ExecuteAsync(query, new { Id = record[sourceSettings.IdColumn] }); 46 | } 47 | } 48 | catch (Exception ex) 49 | { 50 | _logger.LogError(ex, "Error deleting data."); 51 | throw; 52 | } 53 | } 54 | } 55 | 56 | public async Task InsertAsync(ITargetSettings settings, IEnumerable data) 57 | { 58 | if (data == null || !data.Any()) 59 | return; 60 | 61 | var targetSettings = ResolveTargetSettings(settings); 62 | 63 | using (var connection = new SqlConnection(targetSettings.ConnectionString)) 64 | { 65 | try 66 | { 67 | var dataDictionaries = data.Select(item => 68 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 69 | 70 | await connection.OpenAsync(); 71 | 72 | foreach (var record in dataDictionaries) 73 | { 74 | var columns = string.Join(", ", record.Keys); 75 | var parameters = string.Join(", ", record.Keys.Select(k => $"@{k}")); 76 | 77 | StringBuilder queryStrBuilder = new StringBuilder(); 78 | queryStrBuilder.Append($"IF NOT EXISTS (SELECT {targetSettings.IdColumn} FROM {targetSettings.Schema}.{targetSettings.Table} WHERE {targetSettings.IdColumn} = @{targetSettings.IdColumn}) "); 79 | queryStrBuilder.Append($"INSERT INTO {targetSettings.Schema}.{targetSettings.Table} ({columns}) VALUES ({parameters})"); 80 | 81 | await connection.ExecuteAsync(queryStrBuilder.ToString(), record); 82 | } 83 | } 84 | catch (Exception ex) 85 | { 86 | _logger.LogError(ex, "Error inserting data."); 87 | throw; 88 | } 89 | } 90 | } 91 | 92 | public async Task ExecuteScriptAsync(ITargetSettings settings, string script) 93 | { 94 | if (string.IsNullOrEmpty(script)) 95 | throw new ArgumentNullException(nameof(script)); 96 | 97 | var sourceSettings = ResolveTargetSettings(settings); 98 | 99 | using (var connection = new SqlConnection(sourceSettings.ConnectionString)) 100 | { 101 | try 102 | { 103 | await connection.OpenAsync(); 104 | await connection.ExecuteAsync(script); 105 | await connection.CloseAsync(); 106 | _logger.LogInformation("Script executed successfully."); 107 | } 108 | catch (Exception ex) 109 | { 110 | _logger.LogError(ex, "Error executing script."); 111 | throw; 112 | } 113 | } 114 | } 115 | 116 | public async Task GetIteratorAsync(ISourceSettings settings, int transferQuantity) 117 | { 118 | var sourceSettings = ResolveSourceSettings(settings); 119 | 120 | StringBuilder queryStrBuilder = new StringBuilder(); 121 | queryStrBuilder.AppendLine($"SELECT * FROM {sourceSettings.Schema}.{sourceSettings.Table}"); 122 | 123 | if (sourceSettings.HasCondition) 124 | queryStrBuilder.AppendLine($"WHERE {sourceSettings.Condition}"); 125 | 126 | var connection = new SqlConnection(sourceSettings.ConnectionString); 127 | await connection.OpenAsync(); 128 | 129 | var iterator = new MSSQLProviderIterator(connection, 130 | queryStrBuilder.ToString(), 131 | sourceSettings.IdColumn, 132 | transferQuantity); 133 | return iterator; 134 | } 135 | 136 | /// 137 | /// Down-cast to mssql source settings 138 | /// 139 | private SourceSettings ResolveSourceSettings(ISourceSettings sourceSettings) 140 | => sourceSettings as SourceSettings; 141 | 142 | /// 143 | /// Down-cast to mssql target settings 144 | /// 145 | private TargetSettings ResolveTargetSettings(ITargetSettings targetSettings) 146 | => targetSettings as TargetSettings; 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MSSQL/MSSQLProviderIterator.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.Common; 3 | using System.Data; 4 | 5 | namespace DbArchiver.Provider.MSSQL 6 | { 7 | public class MSSQLProviderIterator : IDatabaseProviderIterator 8 | { 9 | private readonly IDbConnection _connection; 10 | private readonly string _queryStr; 11 | private readonly string _orderByColumn; 12 | private readonly int _batchSize; 13 | private int _currentOffset; 14 | 15 | private bool _disposed; 16 | 17 | public IEnumerable Data { get; private set; } 18 | 19 | public MSSQLProviderIterator(IDbConnection connection, 20 | string query, string orderByColumn,int batchSize) 21 | { 22 | _connection = connection; 23 | _queryStr = query; 24 | _orderByColumn = orderByColumn; 25 | _batchSize = batchSize; 26 | _currentOffset = 0; 27 | _disposed = false; 28 | } 29 | 30 | public async Task NextAsync() 31 | { 32 | if (_disposed) return false; 33 | 34 | var paginatedQuery = $@"{_queryStr} 35 | ORDER BY {_orderByColumn} 36 | OFFSET @currentOffset ROWS FETCH NEXT @batchSize ROWS ONLY"; 37 | 38 | var dynamicParams = new DynamicParameters(); 39 | dynamicParams.Add("currentOffset", _currentOffset); 40 | dynamicParams.Add("batchSize", _batchSize); 41 | 42 | Data = (await _connection.QueryAsync(paginatedQuery, dynamicParams)).ToList(); 43 | _currentOffset += _batchSize; 44 | 45 | return Data.Any(); 46 | } 47 | 48 | public void Dispose() 49 | { 50 | if (_disposed) return; 51 | 52 | _connection?.Dispose(); 53 | _disposed = true; 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MSSQL/ServiceCollectionExtension.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common; 2 | using Microsoft.Extensions.DependencyInjection; 3 | 4 | namespace DbArchiver.Provider.MSSQL 5 | { 6 | public static class ServiceCollectionExtension 7 | { 8 | public static IServiceCollection AddMSSQLProviderServices(this IServiceCollection services) { 9 | 10 | services.AddTransient(); 11 | 12 | return services; 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MongoDB/Config/SourceSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.MongoDB.Config 4 | { 5 | public class SourceSettings : ISourceSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string DatabaseName { get; set; } 9 | public string Collection { get; set; } 10 | public string IdColumn { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MongoDB/Config/TargetSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.MongoDB.Config 4 | { 5 | public class TargetSettings : ITargetSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string DatabaseName { get; set; } 9 | public string Collection { get; set; } 10 | public string IdColumn { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MongoDB/DbArchiver.Provider.MongoDB.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MongoDB/MongoDBProvider.cs: -------------------------------------------------------------------------------- 1 | using MongoDB.Bson; 2 | using MongoDB.Driver; 3 | using Microsoft.Extensions.Logging; 4 | using DbArchiver.Provider.Common; 5 | using DbArchiver.Provider.Common.Config; 6 | using DbArchiver.Provider.MongoDB.Config; 7 | 8 | namespace DbArchiver.Provider.MongoDB 9 | { 10 | public class MongoDBProvider : IDatabaseProviderSource, 11 | IDatabaseProviderTarget 12 | { 13 | private readonly ILogger _logger; 14 | private IMongoDatabase _database; 15 | 16 | public MongoDBProvider(ILogger logger) 17 | { 18 | _logger = logger; 19 | } 20 | 21 | public async Task DeleteAsync(ISourceSettings settings, IEnumerable data) 22 | { 23 | if (data == null || !data.Any()) 24 | return; 25 | 26 | try 27 | { 28 | var sourceSettings = ResolveSourceSettings(settings); 29 | await InitializeAsync(sourceSettings.ConnectionString, sourceSettings.DatabaseName); 30 | 31 | var collection = _database.GetCollection(sourceSettings.Collection); 32 | 33 | var ids = data.Select(d => ((IDictionary)d)[sourceSettings.IdColumn]); 34 | var filter = Builders.Filter.In(sourceSettings.IdColumn, ids); 35 | 36 | var result = await collection.DeleteManyAsync(filter); 37 | _logger.LogInformation($"Deleted {result.DeletedCount} documents."); 38 | } 39 | catch (Exception ex) 40 | { 41 | _logger.LogError(ex, "Error deleting data."); 42 | throw; 43 | } 44 | } 45 | 46 | public async Task InsertAsync(ITargetSettings settings, IEnumerable data) 47 | { 48 | throw new NotImplementedException(); 49 | } 50 | 51 | public Task ExecuteScriptAsync(ITargetSettings settings, string script) 52 | { 53 | throw new NotImplementedException(); 54 | } 55 | 56 | public async Task GetIteratorAsync(ISourceSettings settings, int transferQuantity) 57 | { 58 | var sourceSettings = ResolveSourceSettings(settings); 59 | await InitializeAsync(sourceSettings.ConnectionString, sourceSettings.DatabaseName); 60 | 61 | var collection = _database.GetCollection(sourceSettings.Collection); 62 | 63 | var iterator = new MongoDBProviderIterator(collection, sourceSettings.IdColumn, transferQuantity); 64 | return iterator; 65 | } 66 | 67 | private async Task InitializeAsync(string connectionString, string databaseName) 68 | { 69 | if (_database == null) 70 | { 71 | var client = new MongoClient(connectionString); 72 | _database = client.GetDatabase(databaseName); 73 | } 74 | } 75 | 76 | private SourceSettings ResolveSourceSettings(ISourceSettings sourceSettings) 77 | => sourceSettings as SourceSettings; 78 | 79 | private TargetSettings ResolveTargetSettings(ITargetSettings targetSettings) 80 | => targetSettings as TargetSettings; 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MongoDB/MongoDBProviderIterator.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common; 2 | using MongoDB.Bson; 3 | using MongoDB.Driver; 4 | 5 | namespace DbArchiver.Provider.MongoDB 6 | { 7 | public class MongoDBProviderIterator : IDatabaseProviderIterator 8 | { 9 | private readonly IMongoCollection _collection; 10 | private readonly string _orderByColumn; 11 | private readonly int _batchSize; 12 | private int _currentOffset; 13 | 14 | private bool _disposed; 15 | public IEnumerable Data { get; private set; } 16 | 17 | public MongoDBProviderIterator(IMongoCollection collection, string orderByColumn, int batchSize) 18 | { 19 | _collection = collection; 20 | _orderByColumn = orderByColumn; 21 | _batchSize = batchSize; 22 | _currentOffset = 0; 23 | _disposed = false; 24 | } 25 | 26 | public async Task NextAsync() 27 | { 28 | if (_disposed) return false; 29 | 30 | var sort = Builders.Sort.Ascending(_orderByColumn); 31 | var filteredData = await _collection.Find(new BsonDocument()) 32 | .Sort(sort) 33 | .Skip(_currentOffset) 34 | .Limit(_batchSize) 35 | .ToListAsync(); 36 | 37 | Data = filteredData.Select(d => 38 | { 39 | var dictionary = d.ToDictionary(); 40 | if (dictionary.ContainsKey("_id") && dictionary["_id"] is ObjectId objectId) 41 | { 42 | dictionary["_id"] = objectId.ToString(); 43 | } 44 | return FlattenDictionary(dictionary); 45 | }); 46 | 47 | _currentOffset += _batchSize; 48 | 49 | return Data.Any(); 50 | } 51 | 52 | private Dictionary FlattenDictionary(Dictionary source, string parentKey = "") 53 | { 54 | var result = new Dictionary(); 55 | 56 | foreach (var kvp in source) 57 | { 58 | var key = string.IsNullOrEmpty(parentKey) ? kvp.Key : $"{parentKey}_{kvp.Key}"; 59 | 60 | if (kvp.Value is Dictionary nestedDictionary) 61 | { 62 | var flattenedNested = FlattenDictionary(nestedDictionary, key); 63 | foreach (var nestedKvp in flattenedNested) 64 | { 65 | result[nestedKvp.Key] = nestedKvp.Value; 66 | } 67 | } 68 | else if (kvp.Value is BsonDocument bsonDocument) 69 | { 70 | var nestedFromBson = FlattenDictionary(bsonDocument.ToDictionary(), key); 71 | foreach (var nestedKvp in nestedFromBson) 72 | { 73 | result[nestedKvp.Key] = nestedKvp.Value; 74 | } 75 | } 76 | else 77 | { 78 | result[key] = kvp.Value; 79 | } 80 | } 81 | 82 | return result; 83 | } 84 | 85 | public void Dispose() 86 | { 87 | if (_disposed) return; 88 | 89 | _disposed = true; 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MongoDB/ServiceCollectionExtension.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.DependencyInjection; 2 | 3 | namespace DbArchiver.Provider.MongoDB 4 | { 5 | public static class ServiceCollectionExtension 6 | { 7 | public static IServiceCollection AddMongoDBProviderServices(this IServiceCollection services) { 8 | 9 | services.AddTransient(); 10 | 11 | return services; 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MySQL/Config/SourceSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.MySQL.Config; 4 | 5 | public class SourceSettings : ISourceSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } -------------------------------------------------------------------------------- /DbArchiver.Provider.MySQL/Config/TargetSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.MySQL.Config; 4 | 5 | public class TargetSettings : ITargetSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } -------------------------------------------------------------------------------- /DbArchiver.Provider.MySQL/DbArchiver.Provider.MySQL.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /DbArchiver.Provider.MySQL/MySQLProvider.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using DbArchiver.Provider.Common; 3 | using DbArchiver.Provider.Common.Config; 4 | using DbArchiver.Provider.MySQL.Config; 5 | using MySql.Data.MySqlClient; 6 | using Microsoft.Extensions.Logging; 7 | using Dapper; 8 | namespace DbArchiver.Provider.MySQL; 9 | 10 | public class MySQLProvider : IDatabaseProviderSource, IDatabaseProviderTarget 11 | { 12 | private readonly ILogger _logger; 13 | 14 | public MySQLProvider(ILogger logger) 15 | { 16 | _logger = logger; 17 | } 18 | 19 | public async Task DeleteAsync(ISourceSettings settings, IEnumerable data) 20 | { 21 | if (data == null || !data.Any()) 22 | return; 23 | 24 | var sourceSettings = ResolveSourceSettings(settings); 25 | 26 | using (var connection = new MySqlConnection(sourceSettings.ConnectionString)) 27 | { 28 | try 29 | { 30 | var dataDictionaries = data.Select(item => 31 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 32 | 33 | await connection.OpenAsync(); 34 | 35 | foreach (var record in dataDictionaries) 36 | { 37 | if (!record.ContainsKey(sourceSettings.IdColumn)) 38 | throw new InvalidOperationException($"Primary key '{sourceSettings.IdColumn}' not found in record."); 39 | 40 | var query = $"DELETE FROM `{sourceSettings.Schema}`.`{sourceSettings.Table}` WHERE `{sourceSettings.IdColumn}` = @Id"; 41 | 42 | await connection.ExecuteAsync(query, new { Id = record[sourceSettings.IdColumn] }); 43 | } 44 | } 45 | catch (Exception ex) 46 | { 47 | _logger.LogError(ex, "Error deleting data."); 48 | throw; 49 | } 50 | } 51 | } 52 | 53 | public async Task InsertAsync(ITargetSettings settings, IEnumerable data) 54 | { 55 | if (data == null || !data.Any()) 56 | return; 57 | 58 | var targetSettings = ResolveTargetSettings(settings); 59 | 60 | using (var connection = new MySqlConnection(targetSettings.ConnectionString)) 61 | { 62 | try 63 | { 64 | var dataDictionaries = data.Select(item => 65 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 66 | 67 | await connection.OpenAsync(); 68 | 69 | foreach (var record in dataDictionaries) 70 | { 71 | var columns = string.Join(", ", record.Keys.Select(k => $"`{k}`")); 72 | var parameters = string.Join(", ", record.Keys.Select(k => $"@{k}")); 73 | 74 | StringBuilder queryStrBuilder = new StringBuilder(); 75 | queryStrBuilder.Append($"INSERT INTO `{targetSettings.Schema}`.`{targetSettings.Table}` ({columns}) "); 76 | queryStrBuilder.Append($"VALUES ({parameters}) "); 77 | queryStrBuilder.Append($"ON DUPLICATE KEY UPDATE "); 78 | 79 | var updateColumns = string.Join(", ", record.Keys.Where(k => k != targetSettings.IdColumn) 80 | .Select(k => $"`{k}` = @{k}")); 81 | queryStrBuilder.Append(updateColumns); 82 | 83 | await connection.ExecuteAsync(queryStrBuilder.ToString(), record); 84 | } 85 | } 86 | catch (Exception ex) 87 | { 88 | _logger.LogError(ex, "Error inserting data."); 89 | throw; 90 | } 91 | } 92 | } 93 | 94 | public async Task ExecuteScriptAsync(ITargetSettings settings, string script) 95 | { 96 | if (string.IsNullOrEmpty(script)) 97 | throw new ArgumentNullException(nameof(script)); 98 | 99 | var sourceSettings = ResolveTargetSettings(settings); 100 | 101 | using (var connection = new MySqlConnection(sourceSettings.ConnectionString)) 102 | { 103 | try 104 | { 105 | await connection.OpenAsync(); 106 | await connection.ExecuteAsync(script); 107 | await connection.CloseAsync(); 108 | _logger.LogInformation("Script executed successfully."); 109 | } 110 | catch (Exception ex) 111 | { 112 | _logger.LogError(ex, "Error executing script."); 113 | throw; 114 | } 115 | } 116 | } 117 | 118 | public async Task GetIteratorAsync(ISourceSettings settings, int transferQuantity) 119 | { 120 | var sourceSettings = ResolveSourceSettings(settings); 121 | 122 | StringBuilder queryStrBuilder = new StringBuilder(); 123 | queryStrBuilder.AppendLine($"SELECT * FROM `{sourceSettings.Schema}`.`{sourceSettings.Table}`"); 124 | 125 | if (sourceSettings.HasCondition) 126 | queryStrBuilder.AppendLine($"WHERE {sourceSettings.Condition}"); 127 | 128 | var connection = new MySqlConnection(sourceSettings.ConnectionString); 129 | await connection.OpenAsync(); 130 | 131 | var iterator = new MySQLProviderIterator(connection, 132 | queryStrBuilder.ToString(), 133 | sourceSettings.IdColumn, 134 | transferQuantity); 135 | return iterator; 136 | } 137 | 138 | /// 139 | /// Down-cast to MySQL source settings 140 | /// 141 | private SourceSettings ResolveSourceSettings(ISourceSettings sourceSettings) 142 | => sourceSettings as SourceSettings; 143 | 144 | /// 145 | /// Down-cast to MySQL target settings 146 | /// 147 | private TargetSettings ResolveTargetSettings(ITargetSettings targetSettings) 148 | => targetSettings as TargetSettings; 149 | } -------------------------------------------------------------------------------- /DbArchiver.Provider.MySQL/MySQLProviderIterator.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using System.Data; 3 | using DbArchiver.Provider.Common; 4 | 5 | namespace DbArchiver.Provider.MySQL; 6 | 7 | public class MySQLProviderIterator : IDatabaseProviderIterator 8 | { 9 | private readonly IDbConnection _connection; 10 | private readonly string _queryStr; 11 | private readonly string _orderByColumn; 12 | private readonly int _batchSize; 13 | private int _currentOffset; 14 | 15 | private bool _disposed; 16 | 17 | public IEnumerable Data { get; private set; } 18 | 19 | public MySQLProviderIterator(IDbConnection connection, string query, string orderByColumn, int batchSize) 20 | { 21 | _connection = connection; 22 | _queryStr = query; 23 | _orderByColumn = orderByColumn; 24 | _batchSize = batchSize; 25 | _currentOffset = 0; 26 | _disposed = false; 27 | } 28 | 29 | public async Task NextAsync() 30 | { 31 | if (_disposed) return false; 32 | 33 | // MySQL uses LIMIT with OFFSET for pagination 34 | var paginatedQuery = $@"{_queryStr} 35 | ORDER BY `{_orderByColumn}` 36 | LIMIT @batchSize OFFSET @currentOffset"; 37 | 38 | var dynamicParams = new DynamicParameters(); 39 | dynamicParams.Add("batchSize", _batchSize); 40 | dynamicParams.Add("currentOffset", _currentOffset); 41 | 42 | Data = (await _connection.QueryAsync(paginatedQuery, dynamicParams)).ToList(); 43 | _currentOffset += _batchSize; 44 | 45 | return Data.Any(); 46 | } 47 | 48 | public void Dispose() 49 | { 50 | if (_disposed) return; 51 | 52 | _connection?.Dispose(); 53 | _disposed = true; 54 | } 55 | } -------------------------------------------------------------------------------- /DbArchiver.Provider.MySQL/ServiceCollectionExtension.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.DependencyInjection; 2 | 3 | namespace DbArchiver.Provider.MySQL; 4 | 5 | public static class ServiceCollectionExtension 6 | { 7 | public static IServiceCollection AddMySQLProviderServices(this IServiceCollection services) { 8 | 9 | services.AddTransient(); 10 | 11 | return services; 12 | } 13 | } -------------------------------------------------------------------------------- /DbArchiver.Provider.PostgreSQL/Config/SourceSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.PostgreSQL.Config 4 | { 5 | public class SourceSettings : ISourceSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /DbArchiver.Provider.PostgreSQL/Config/TargetSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.PostgreSQL.Config 4 | { 5 | public class TargetSettings : ITargetSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /DbArchiver.Provider.PostgreSQL/DbArchiver.Provider.PostgreSQL.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /DbArchiver.Provider.PostgreSQL/PostgreSQLProvider.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.Common; 3 | using DbArchiver.Provider.Common.Config; 4 | using DbArchiver.Provider.PostgreSQL.Config; 5 | using Microsoft.Extensions.Logging; 6 | using Npgsql; 7 | using System.Text; 8 | 9 | namespace DbArchiver.Provider.PostgreSQL 10 | { 11 | public class PostgreSQLProvider(ILogger logger) : IDatabaseProviderSource, 12 | IDatabaseProviderTarget 13 | { 14 | public async Task DeleteAsync(ISourceSettings settings, IEnumerable data) 15 | { 16 | if (data == null || !data.Any()) 17 | return; 18 | 19 | var sourceSettings = ResolveSourceSettings(settings); 20 | 21 | using (var connection = new NpgsqlConnection(sourceSettings.ConnectionString)) 22 | { 23 | try 24 | { 25 | var dataDictionaries = data.Select(item => 26 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 27 | 28 | await connection.OpenAsync(); 29 | 30 | foreach (var record in dataDictionaries) 31 | { 32 | if (!record.ContainsKey(sourceSettings.IdColumn)) 33 | throw new InvalidOperationException($"Primary key '{sourceSettings.IdColumn}' not found in record."); 34 | 35 | var query = $"DELETE FROM {sourceSettings.Schema}.{sourceSettings.Table} WHERE {sourceSettings.IdColumn} = @Id"; 36 | 37 | await connection.ExecuteAsync(query, new { Id = record[sourceSettings.IdColumn] }); 38 | } 39 | } 40 | catch (Exception ex) 41 | { 42 | logger.LogError(ex, "Error deleting data."); 43 | throw; 44 | } 45 | } 46 | } 47 | 48 | public async Task InsertAsync(ITargetSettings settings, IEnumerable data) 49 | { 50 | if (data == null || !data.Any()) 51 | return; 52 | 53 | var targetSettings = ResolveTargetSettings(settings); 54 | 55 | using (var connection = new NpgsqlConnection(targetSettings.ConnectionString)) 56 | { 57 | try 58 | { 59 | var dataDictionaries = data.Select(item => 60 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 61 | 62 | await connection.OpenAsync(); 63 | 64 | foreach (var record in dataDictionaries) 65 | { 66 | var columns = string.Join(", ", record.Keys); 67 | var parameters = string.Join(", ", record.Keys.Select(k => $"@{k}")); 68 | 69 | StringBuilder queryStrBuilder = new StringBuilder(); 70 | queryStrBuilder.Append($"INSERT INTO {targetSettings.Schema}.{targetSettings.Table} ({columns}) VALUES ({parameters}) "); 71 | queryStrBuilder.Append($"ON CONFLICT ({targetSettings.IdColumn}) DO NOTHING"); 72 | 73 | await connection.ExecuteAsync(queryStrBuilder.ToString(), record); 74 | } 75 | } 76 | catch (Exception ex) 77 | { 78 | logger.LogError(ex, "Error inserting data."); 79 | throw; 80 | } 81 | } 82 | } 83 | 84 | public async Task ExecuteScriptAsync(ITargetSettings settings, string script) 85 | { 86 | if (string.IsNullOrEmpty(script)) 87 | throw new ArgumentNullException(nameof(script)); 88 | 89 | var targetSettings = ResolveTargetSettings(settings); 90 | 91 | using (var connection = new NpgsqlConnection(targetSettings.ConnectionString)) 92 | { 93 | try 94 | { 95 | await connection.OpenAsync(); 96 | await connection.ExecuteAsync(script); 97 | await connection.CloseAsync(); 98 | logger.LogInformation("Script executed successfully."); 99 | } 100 | catch (Exception ex) 101 | { 102 | logger.LogError(ex, "Error executing script."); 103 | throw; 104 | } 105 | } 106 | } 107 | 108 | public async Task GetIteratorAsync(ISourceSettings settings, int transferQuantity) 109 | { 110 | var sourceSettings = ResolveSourceSettings(settings); 111 | 112 | StringBuilder queryStrBuilder = new StringBuilder(); 113 | queryStrBuilder.AppendLine($"SELECT * FROM {sourceSettings.Schema}.{sourceSettings.Table}"); 114 | 115 | if (sourceSettings.HasCondition) 116 | queryStrBuilder.AppendLine($"WHERE {sourceSettings.Condition}"); 117 | 118 | var connection = new NpgsqlConnection(sourceSettings.ConnectionString); 119 | await connection.OpenAsync(); 120 | 121 | var iterator = new PostgreSQLProviderIterator(connection, 122 | queryStrBuilder.ToString(), 123 | sourceSettings.IdColumn, 124 | transferQuantity); 125 | return iterator; 126 | } 127 | 128 | /// 129 | /// Down-cast to PostgreSQL source settings 130 | /// 131 | private SourceSettings ResolveSourceSettings(ISourceSettings sourceSettings) 132 | => sourceSettings as SourceSettings; 133 | 134 | /// 135 | /// Down-cast to PostgreSQL target settings 136 | /// 137 | private TargetSettings ResolveTargetSettings(ITargetSettings targetSettings) 138 | => targetSettings as TargetSettings; 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /DbArchiver.Provider.PostgreSQL/PostgreSQLProviderIterator.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.Common; 3 | using System.Data; 4 | 5 | namespace DbArchiver.Provider.PostgreSQL 6 | { 7 | public class PostgreSQLProviderIterator : IDatabaseProviderIterator 8 | { 9 | private readonly IDbConnection _connection; 10 | private readonly string _queryStr; 11 | private readonly string _orderByColumn; 12 | private readonly int _batchSize; 13 | private int _currentOffset; 14 | 15 | private bool _disposed; 16 | 17 | public IEnumerable Data { get; private set; } 18 | 19 | public PostgreSQLProviderIterator(IDbConnection connection, 20 | string query, string orderByColumn, int batchSize) 21 | { 22 | _connection = connection; 23 | _queryStr = query; 24 | _orderByColumn = orderByColumn; 25 | _batchSize = batchSize; 26 | _currentOffset = 0; 27 | _disposed = false; 28 | } 29 | 30 | public async Task NextAsync() 31 | { 32 | if (_disposed) return false; 33 | 34 | var paginatedQuery = $@" 35 | {_queryStr} 36 | ORDER BY {_orderByColumn} 37 | LIMIT @batchSize OFFSET @currentOffset"; 38 | 39 | var dynamicParams = new DynamicParameters(); 40 | dynamicParams.Add("currentOffset", _currentOffset); 41 | dynamicParams.Add("batchSize", _batchSize); 42 | 43 | Data = (await _connection.QueryAsync(paginatedQuery, dynamicParams)).ToList(); 44 | _currentOffset += _batchSize; 45 | 46 | return Data.Any(); 47 | } 48 | 49 | public void Dispose() 50 | { 51 | if (_disposed) return; 52 | 53 | _connection?.Dispose(); 54 | _disposed = true; 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /DbArchiver.Provider.PostgreSQL/ServiceCollectionExtension.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.DependencyInjection; 2 | 3 | namespace DbArchiver.Provider.PostgreSQL 4 | { 5 | public static class ServiceCollectionExtension 6 | { 7 | public static IServiceCollection AddPostgreSQLProviderServices(this IServiceCollection services) { 8 | 9 | services.AddTransient(); 10 | 11 | return services; 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /DbArchiver.Provider.SQLite/Config/SourceSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.SQLite.Config 4 | { 5 | public class SourceSettings : ISourceSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /DbArchiver.Provider.SQLite/Config/TargetSettings.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Provider.Common.Config; 2 | 3 | namespace DbArchiver.Provider.SQLite.Config 4 | { 5 | public class TargetSettings : ITargetSettings 6 | { 7 | public string ConnectionString { get; set; } 8 | public string Schema { get; set; } 9 | public string Table { get; set; } 10 | public string IdColumn { get; set; } 11 | public string Condition { get; set; } 12 | 13 | public bool HasCondition => !string.IsNullOrEmpty(Condition); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /DbArchiver.Provider.SQLite/DbArchiver.Provider.SQLite.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /DbArchiver.Provider.SQLite/SQLiteProvider.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.Common; 3 | using DbArchiver.Provider.Common.Config; 4 | using DbArchiver.Provider.SQLite.Config; 5 | using Microsoft.Data.Sqlite; 6 | using Microsoft.Extensions.Logging; 7 | using System.Text; 8 | 9 | namespace DbArchiver.Provider.SQLite 10 | { 11 | public class SQLiteProvider(ILogger logger) : IDatabaseProviderSource, 12 | IDatabaseProviderTarget 13 | { 14 | public async Task DeleteAsync(ISourceSettings settings, IEnumerable data) 15 | { 16 | if (data == null || !data.Any()) 17 | return; 18 | 19 | var sourceSettings = ResolveSourceSettings(settings); 20 | 21 | using (var connection = new SqliteConnection(sourceSettings.ConnectionString)) 22 | { 23 | try 24 | { 25 | var dataDictionaries = data.Select(item => 26 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 27 | 28 | await connection.OpenAsync(); 29 | 30 | foreach (var record in dataDictionaries) 31 | { 32 | if (!record.ContainsKey(sourceSettings.IdColumn)) 33 | throw new InvalidOperationException($"Primary key '{sourceSettings.IdColumn}' not found in record."); 34 | 35 | var query = $"DELETE FROM {sourceSettings.Table} WHERE {sourceSettings.IdColumn} = @Id"; 36 | 37 | await connection.ExecuteAsync(query, new { Id = record[sourceSettings.IdColumn] }); 38 | } 39 | } 40 | catch (Exception ex) 41 | { 42 | logger.LogError(ex, "Error deleting data."); 43 | throw; 44 | } 45 | } 46 | } 47 | 48 | public async Task InsertAsync(ITargetSettings settings, IEnumerable data) 49 | { 50 | if (data == null || !data.Any()) 51 | return; 52 | 53 | var targetSettings = ResolveTargetSettings(settings); 54 | 55 | using (var connection = new SqliteConnection(targetSettings.ConnectionString)) 56 | { 57 | try 58 | { 59 | var dataDictionaries = data.Select(item => 60 | ((IDictionary)item).ToDictionary(k => k.Key, v => v.Value)); 61 | 62 | await connection.OpenAsync(); 63 | 64 | foreach (var record in dataDictionaries) 65 | { 66 | var columns = string.Join(", ", record.Keys); 67 | var parameters = string.Join(", ", record.Keys.Select(k => $"@{k}")); 68 | 69 | StringBuilder queryStrBuilder = new StringBuilder(); 70 | queryStrBuilder.Append($"INSERT OR REPLACE INTO {targetSettings.Table} ({columns}) VALUES ({parameters})"); 71 | 72 | await connection.ExecuteAsync(queryStrBuilder.ToString(), record); 73 | } 74 | } 75 | catch (Exception ex) 76 | { 77 | logger.LogError(ex, "Error inserting data."); 78 | throw; 79 | } 80 | } 81 | } 82 | 83 | public async Task ExecuteScriptAsync(ITargetSettings settings, string script) 84 | { 85 | if (string.IsNullOrEmpty(script)) 86 | throw new ArgumentNullException(nameof(script)); 87 | 88 | var sourceSettings = ResolveTargetSettings(settings); 89 | 90 | using (var connection = new SqliteConnection(sourceSettings.ConnectionString)) 91 | { 92 | try 93 | { 94 | await connection.OpenAsync(); 95 | await connection.ExecuteAsync(script); 96 | logger.LogInformation("Script executed successfully."); 97 | } 98 | catch (Exception ex) 99 | { 100 | logger.LogError(ex, "Error executing script."); 101 | throw; 102 | } 103 | } 104 | } 105 | 106 | public async Task GetIteratorAsync(ISourceSettings settings, int transferQuantity) 107 | { 108 | var sourceSettings = ResolveSourceSettings(settings); 109 | 110 | StringBuilder queryStrBuilder = new StringBuilder(); 111 | queryStrBuilder.AppendLine($"SELECT * FROM {sourceSettings.Table}"); 112 | 113 | if (sourceSettings.HasCondition) 114 | queryStrBuilder.AppendLine($"WHERE {sourceSettings.Condition}"); 115 | 116 | var connection = new SqliteConnection(sourceSettings.ConnectionString); 117 | await connection.OpenAsync(); 118 | 119 | var iterator = new SQLiteProviderIterator(connection, 120 | queryStrBuilder.ToString(), 121 | sourceSettings.IdColumn, 122 | transferQuantity); 123 | return iterator; 124 | } 125 | 126 | /// 127 | /// Down-cast to SQLite source settings 128 | /// 129 | private SourceSettings ResolveSourceSettings(ISourceSettings sourceSettings) 130 | => sourceSettings as SourceSettings; 131 | 132 | /// 133 | /// Down-cast to SQLite target settings 134 | /// 135 | private TargetSettings ResolveTargetSettings(ITargetSettings targetSettings) 136 | => targetSettings as TargetSettings; 137 | } 138 | 139 | } 140 | -------------------------------------------------------------------------------- /DbArchiver.Provider.SQLite/SQLiteProviderIterator.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.Common; 3 | using System.Data; 4 | 5 | namespace DbArchiver.Provider.SQLite 6 | { 7 | public class SQLiteProviderIterator( 8 | IDbConnection connection, 9 | string query, 10 | string orderByColumn, 11 | int batchSize) 12 | : IDatabaseProviderIterator 13 | { 14 | private int _currentOffset = 0; 15 | 16 | private bool _disposed = false; 17 | 18 | public IEnumerable Data { get; private set; } 19 | 20 | public async Task NextAsync() 21 | { 22 | if (_disposed) return false; 23 | 24 | var paginatedQuery = $@" 25 | {query} 26 | ORDER BY {orderByColumn} 27 | LIMIT @batchSize OFFSET @currentOffset"; 28 | 29 | var dynamicParams = new DynamicParameters(); 30 | dynamicParams.Add("currentOffset", _currentOffset); 31 | dynamicParams.Add("batchSize", batchSize); 32 | 33 | Data = (await connection.QueryAsync(paginatedQuery, dynamicParams)).ToList(); 34 | _currentOffset += batchSize; 35 | 36 | return Data.Any(); 37 | } 38 | 39 | public void Dispose() 40 | { 41 | if (_disposed) return; 42 | 43 | connection?.Dispose(); 44 | _disposed = true; 45 | } 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /DbArchiver.Provider.SQLite/ServiceCollectionExtension.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.DependencyInjection; 2 | 3 | namespace DbArchiver.Provider.SQLite 4 | { 5 | public static class ServiceCollectionExtension 6 | { 7 | public static IServiceCollection AddSQLiteProviderServices(this IServiceCollection services) { 8 | 9 | services.AddTransient(); 10 | 11 | return services; 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /DbArchiver.WService/DataTransferJob.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core.Contract; 2 | using DbArchiver.Core.Config; 3 | using Quartz; 4 | 5 | namespace DbArchiver.WService 6 | { 7 | public class DataTransferJob : IJob 8 | { 9 | private readonly IDatabaseArchiverFactory _archiverFactory; 10 | private readonly ILogger _logger; 11 | 12 | public DataTransferJob(IDatabaseArchiverFactory dbArchiverFactory, ILogger logger) 13 | { 14 | _archiverFactory = dbArchiverFactory ?? throw new ArgumentNullException(nameof(dbArchiverFactory)); 15 | _logger = logger; 16 | } 17 | 18 | public async Task Execute(IJobExecutionContext context) 19 | { 20 | _logger.LogInformation("DataTransferJob started at {Time}", DateTimeOffset.Now); 21 | 22 | try 23 | { 24 | var transferSettings = context.MergedJobDataMap.Get("TransferSettings") as TransferSettings; 25 | 26 | if (transferSettings == null) 27 | { 28 | _logger.LogError("TransferSettings not provided to job."); 29 | return; 30 | } 31 | 32 | var archiver = _archiverFactory.Create(transferSettings); 33 | await archiver.ArchiveAsync(CancellationToken.None); 34 | 35 | _logger.LogInformation("DataTransferJob completed successfully at {Time}", DateTimeOffset.Now); 36 | } 37 | catch (Exception ex) 38 | { 39 | _logger.LogError(ex, "An error occurred while executing DataTransferJob."); 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /DbArchiver.WService/DbArchiver.WService.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | dotnet-DbArchiver.WService-0d01545c-3af7-4884-9a19-4499e32b9803 8 | Linux 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /DbArchiver.WService/Dockerfile: -------------------------------------------------------------------------------- 1 | # See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. 2 | 3 | # This stage is used when running from VS in fast mode (Default for Debug configuration) 4 | FROM mcr.microsoft.com/dotnet/runtime:8.0 AS base 5 | USER app 6 | WORKDIR /app 7 | 8 | 9 | # This stage is used to build the service project 10 | FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build 11 | ARG BUILD_CONFIGURATION=Release 12 | WORKDIR /src 13 | COPY ["DbArchiver.WService/DbArchiver.WService.csproj", "DbArchiver.WService/"] 14 | RUN dotnet restore "./DbArchiver.WService/DbArchiver.WService.csproj" 15 | COPY . . 16 | WORKDIR "/src/DbArchiver.WService" 17 | RUN dotnet build "./DbArchiver.WService.csproj" -c $BUILD_CONFIGURATION -o /app/build 18 | 19 | # This stage is used to publish the service project to be copied to the final stage 20 | FROM build AS publish 21 | ARG BUILD_CONFIGURATION=Release 22 | RUN dotnet publish "./DbArchiver.WService.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false 23 | 24 | # This stage is used in production or when running from VS in regular mode (Default when not using the Debug configuration) 25 | FROM base AS final 26 | WORKDIR /app 27 | COPY --from=publish /app/publish . 28 | ENTRYPOINT ["dotnet", "DbArchiver.WService.dll"] -------------------------------------------------------------------------------- /DbArchiver.WService/Program.cs: -------------------------------------------------------------------------------- 1 | using DbArchiver.Core; 2 | using DbArchiver.Provider.MSSQL; 3 | using DbArchiver.Provider.SQLite; 4 | using DbArchiver.Provider.PostgreSQL; 5 | using DbArchiver.Provider.MySQL; 6 | using Quartz; 7 | using DbArchiver.Core.Contract; 8 | using DbArchiver.Provider.MongoDB; 9 | 10 | namespace DbArchiver.WService 11 | { 12 | public class Program 13 | { 14 | public static void Main(string[] args) 15 | { 16 | var builder = Host.CreateApplicationBuilder(args); 17 | 18 | builder.Services.AddMongoDBProviderServices(); 19 | builder.Services.AddMSSQLProviderServices(); 20 | builder.Services.AddPostgreSQLProviderServices(); 21 | builder.Services.AddSQLiteProviderServices(); 22 | builder.Services.AddMySQLProviderServices(); 23 | builder.Services.AddCoreServices(); 24 | 25 | builder.Services.AddQuartz(q => 26 | { 27 | q.UseMicrosoftDependencyInjectionJobFactory(); 28 | 29 | using var serviceProvider = builder.Services.BuildServiceProvider(); 30 | var factory = serviceProvider.GetRequiredService(); 31 | 32 | var archiverConfig = factory.Create(); 33 | 34 | if (archiverConfig?.Items != null) 35 | { 36 | foreach (var item in archiverConfig.Items) 37 | { 38 | var jobKey = new JobKey(item.JobSchedulerSettings.JobName); 39 | 40 | q.AddJob(opts => opts.WithIdentity(jobKey)); 41 | 42 | q.AddTrigger(opts => opts 43 | .ForJob(jobKey) 44 | .UsingJobData(new JobDataMap 45 | { 46 | { "TransferSettings", item.TransferSettings } 47 | }) 48 | .WithIdentity($"{item.JobSchedulerSettings.JobName}_Trigger") 49 | .WithCronSchedule(item.JobSchedulerSettings.Cron)); 50 | } 51 | } 52 | }); 53 | 54 | builder.Services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true); 55 | 56 | var host = builder.Build(); 57 | host.Run(); 58 | } 59 | } 60 | } -------------------------------------------------------------------------------- /DbArchiver.WService/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "DbArchiver.WService": { 4 | "commandName": "Project", 5 | "environmentVariables": { 6 | "DOTNET_ENVIRONMENT": "Development" 7 | }, 8 | "dotnetRunMessages": true 9 | }, 10 | "Container (Dockerfile)": { 11 | "commandName": "Docker" 12 | } 13 | }, 14 | "$schema": "http://json.schemastore.org/launchsettings.json" 15 | } -------------------------------------------------------------------------------- /DbArchiver.WService/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.Hosting.Lifetime": "Information" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /DbArchiver.WService/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.Hosting.Lifetime": "Information" 6 | } 7 | }, 8 | "ArchiverConfiguration": [ 9 | { 10 | "JobSchedulerSettings": { 11 | "JobName": "DbArchivingJob", 12 | "Cron": "0/20 * * ? * *" 13 | }, 14 | "TransferSettings": { 15 | "Source": { 16 | "Provider": "MongoDB", 17 | "Host": "localhost", 18 | "TransferQuantity": 10, 19 | "DeleteAfterArchived": false, 20 | "Settings": { 21 | "ConnectionString": "mongodb://localhost:27017", 22 | "DatabaseName": "account-microservice", 23 | "Collection": "persons", 24 | "IdColumn": "_Id" 25 | } 26 | }, 27 | "Target": { 28 | "Provider": "MSSQL", 29 | "Host": "localhost", 30 | "Settings": { 31 | "ConnectionString": "Server=localhost;Database=AdventureWorks2022;Trusted_Connection=True;TrustServerCertificate=True", 32 | "Schema": "dbo", 33 | "Table": "Persons", 34 | "IdColumn": "_Id" 35 | } 36 | } 37 | } 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /DbArchiver.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.11.35222.181 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Core", "DbArchiver.Core\DbArchiver.Core.csproj", "{8C2BCA9C-A2CE-4BBF-BEEA-78D7032AFCAC}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Provider.MSSQL", "DbArchiver.Provider.MSSQL\DbArchiver.Provider.MSSQL.csproj", "{72C83068-F69C-43EC-A3B3-DC5BD748B117}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Provider.Common", "DbArchiver.Provider.Common\DbArchiver.Provider.Common.csproj", "{DE1D71FE-BB40-4FED-BC90-6A75A9451D18}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.WService", "DbArchiver.WService\DbArchiver.WService.csproj", "{2D0EA228-DAA1-406C-A3F0-B83EB4D7386B}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Provider.PostgreSQL", "DbArchiver.Provider.PostgreSQL\DbArchiver.Provider.PostgreSQL.csproj", "{F49AD8DC-E24F-4489-9FC3-B2FE6BFD53D5}" 15 | EndProject 16 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Provider.SQLite", "DbArchiver.Provider.SQLite\DbArchiver.Provider.SQLite.csproj", "{8419E32C-6B00-407A-9043-EC430F458BFD}" 17 | EndProject 18 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Provider.MySQL", "DbArchiver.Provider.MySQL\DbArchiver.Provider.MySQL.csproj", "{D0ED3C7F-D1F4-4B5D-8664-57C6A41AFE40}" 19 | EndProject 20 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Provider.Integration.Tests", "tests\IntegrationTests\DbArchiver.Provider.Integration.Tests.csproj", "{7570BAE6-EFFC-4EA7-BDF3-5AC0F983E98B}" 21 | EndProject 22 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DbArchiver.Provider.MongoDB", "DbArchiver.Provider.MongoDB\DbArchiver.Provider.MongoDB.csproj", "{3D9F954A-2A65-4BF2-B5CE-26AB3CA3F9DE}" 23 | EndProject 24 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{8FD93E9F-750A-4B35-B229-7094520A17B7}" 25 | EndProject 26 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{6DBBF278-31BB-4BC8-8817-327A542C360D}" 27 | EndProject 28 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "providers", "providers", "{1633089B-BF62-44E5-8B21-1049BE6C8184}" 29 | EndProject 30 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DbArchiver.Core.Unit.Tests", "DbArchiver.Core.Unit.Tests\DbArchiver.Core.Unit.Tests.csproj", "{34AFC8A7-4BC0-4F96-BA49-1E5906FA0533}" 31 | EndProject 32 | Global 33 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 34 | Debug|Any CPU = Debug|Any CPU 35 | Release|Any CPU = Release|Any CPU 36 | EndGlobalSection 37 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 38 | {8C2BCA9C-A2CE-4BBF-BEEA-78D7032AFCAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 39 | {8C2BCA9C-A2CE-4BBF-BEEA-78D7032AFCAC}.Debug|Any CPU.Build.0 = Debug|Any CPU 40 | {8C2BCA9C-A2CE-4BBF-BEEA-78D7032AFCAC}.Release|Any CPU.ActiveCfg = Release|Any CPU 41 | {8C2BCA9C-A2CE-4BBF-BEEA-78D7032AFCAC}.Release|Any CPU.Build.0 = Release|Any CPU 42 | {72C83068-F69C-43EC-A3B3-DC5BD748B117}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 43 | {72C83068-F69C-43EC-A3B3-DC5BD748B117}.Debug|Any CPU.Build.0 = Debug|Any CPU 44 | {72C83068-F69C-43EC-A3B3-DC5BD748B117}.Release|Any CPU.ActiveCfg = Release|Any CPU 45 | {72C83068-F69C-43EC-A3B3-DC5BD748B117}.Release|Any CPU.Build.0 = Release|Any CPU 46 | {DE1D71FE-BB40-4FED-BC90-6A75A9451D18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 47 | {DE1D71FE-BB40-4FED-BC90-6A75A9451D18}.Debug|Any CPU.Build.0 = Debug|Any CPU 48 | {DE1D71FE-BB40-4FED-BC90-6A75A9451D18}.Release|Any CPU.ActiveCfg = Release|Any CPU 49 | {DE1D71FE-BB40-4FED-BC90-6A75A9451D18}.Release|Any CPU.Build.0 = Release|Any CPU 50 | {2D0EA228-DAA1-406C-A3F0-B83EB4D7386B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 51 | {2D0EA228-DAA1-406C-A3F0-B83EB4D7386B}.Debug|Any CPU.Build.0 = Debug|Any CPU 52 | {2D0EA228-DAA1-406C-A3F0-B83EB4D7386B}.Release|Any CPU.ActiveCfg = Release|Any CPU 53 | {2D0EA228-DAA1-406C-A3F0-B83EB4D7386B}.Release|Any CPU.Build.0 = Release|Any CPU 54 | {F49AD8DC-E24F-4489-9FC3-B2FE6BFD53D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 55 | {F49AD8DC-E24F-4489-9FC3-B2FE6BFD53D5}.Debug|Any CPU.Build.0 = Debug|Any CPU 56 | {F49AD8DC-E24F-4489-9FC3-B2FE6BFD53D5}.Release|Any CPU.ActiveCfg = Release|Any CPU 57 | {F49AD8DC-E24F-4489-9FC3-B2FE6BFD53D5}.Release|Any CPU.Build.0 = Release|Any CPU 58 | {8419E32C-6B00-407A-9043-EC430F458BFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 59 | {8419E32C-6B00-407A-9043-EC430F458BFD}.Debug|Any CPU.Build.0 = Debug|Any CPU 60 | {8419E32C-6B00-407A-9043-EC430F458BFD}.Release|Any CPU.ActiveCfg = Release|Any CPU 61 | {8419E32C-6B00-407A-9043-EC430F458BFD}.Release|Any CPU.Build.0 = Release|Any CPU 62 | {D0ED3C7F-D1F4-4B5D-8664-57C6A41AFE40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 63 | {D0ED3C7F-D1F4-4B5D-8664-57C6A41AFE40}.Debug|Any CPU.Build.0 = Debug|Any CPU 64 | {D0ED3C7F-D1F4-4B5D-8664-57C6A41AFE40}.Release|Any CPU.ActiveCfg = Release|Any CPU 65 | {D0ED3C7F-D1F4-4B5D-8664-57C6A41AFE40}.Release|Any CPU.Build.0 = Release|Any CPU 66 | {7570BAE6-EFFC-4EA7-BDF3-5AC0F983E98B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 67 | {7570BAE6-EFFC-4EA7-BDF3-5AC0F983E98B}.Debug|Any CPU.Build.0 = Debug|Any CPU 68 | {7570BAE6-EFFC-4EA7-BDF3-5AC0F983E98B}.Release|Any CPU.ActiveCfg = Release|Any CPU 69 | {7570BAE6-EFFC-4EA7-BDF3-5AC0F983E98B}.Release|Any CPU.Build.0 = Release|Any CPU 70 | {3D9F954A-2A65-4BF2-B5CE-26AB3CA3F9DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 71 | {3D9F954A-2A65-4BF2-B5CE-26AB3CA3F9DE}.Debug|Any CPU.Build.0 = Debug|Any CPU 72 | {3D9F954A-2A65-4BF2-B5CE-26AB3CA3F9DE}.Release|Any CPU.ActiveCfg = Release|Any CPU 73 | {3D9F954A-2A65-4BF2-B5CE-26AB3CA3F9DE}.Release|Any CPU.Build.0 = Release|Any CPU 74 | {34AFC8A7-4BC0-4F96-BA49-1E5906FA0533}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 75 | {34AFC8A7-4BC0-4F96-BA49-1E5906FA0533}.Debug|Any CPU.Build.0 = Debug|Any CPU 76 | {34AFC8A7-4BC0-4F96-BA49-1E5906FA0533}.Release|Any CPU.ActiveCfg = Release|Any CPU 77 | {34AFC8A7-4BC0-4F96-BA49-1E5906FA0533}.Release|Any CPU.Build.0 = Release|Any CPU 78 | EndGlobalSection 79 | GlobalSection(SolutionProperties) = preSolution 80 | HideSolutionNode = FALSE 81 | EndGlobalSection 82 | GlobalSection(NestedProjects) = preSolution 83 | {8C2BCA9C-A2CE-4BBF-BEEA-78D7032AFCAC} = {8FD93E9F-750A-4B35-B229-7094520A17B7} 84 | {72C83068-F69C-43EC-A3B3-DC5BD748B117} = {1633089B-BF62-44E5-8B21-1049BE6C8184} 85 | {DE1D71FE-BB40-4FED-BC90-6A75A9451D18} = {8FD93E9F-750A-4B35-B229-7094520A17B7} 86 | {2D0EA228-DAA1-406C-A3F0-B83EB4D7386B} = {8FD93E9F-750A-4B35-B229-7094520A17B7} 87 | {F49AD8DC-E24F-4489-9FC3-B2FE6BFD53D5} = {1633089B-BF62-44E5-8B21-1049BE6C8184} 88 | {8419E32C-6B00-407A-9043-EC430F458BFD} = {1633089B-BF62-44E5-8B21-1049BE6C8184} 89 | {D0ED3C7F-D1F4-4B5D-8664-57C6A41AFE40} = {1633089B-BF62-44E5-8B21-1049BE6C8184} 90 | {7570BAE6-EFFC-4EA7-BDF3-5AC0F983E98B} = {6DBBF278-31BB-4BC8-8817-327A542C360D} 91 | {3D9F954A-2A65-4BF2-B5CE-26AB3CA3F9DE} = {1633089B-BF62-44E5-8B21-1049BE6C8184} 92 | {6DBBF278-31BB-4BC8-8817-327A542C360D} = {8FD93E9F-750A-4B35-B229-7094520A17B7} 93 | {1633089B-BF62-44E5-8B21-1049BE6C8184} = {8FD93E9F-750A-4B35-B229-7094520A17B7} 94 | {34AFC8A7-4BC0-4F96-BA49-1E5906FA0533} = {6DBBF278-31BB-4BC8-8817-327A542C360D} 95 | EndGlobalSection 96 | GlobalSection(ExtensibilityGlobals) = postSolution 97 | SolutionGuid = {7510C1D4-D6F0-4158-9897-0B8EE8B01461} 98 | EndGlobalSection 99 | EndGlobal 100 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Rasul Huseynov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Db.Archiver 2 | Simple tool for transfering/Archiving data between databases. 3 | 4 | ![Data Transfer](https://github.com/user-attachments/assets/d986dbc4-02d7-46b8-b949-d8df7895fcc5) 5 | -------------------------------------------------------------------------------- /tests/IntegrationTests/DbArchiver.Provider.Integration.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | false 9 | true 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /tests/IntegrationTests/MSSQLProviderTests.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.MSSQL; 3 | using DbArchiver.Provider.MSSQL.Config; 4 | using Microsoft.Data.SqlClient; 5 | using Microsoft.Extensions.Logging; 6 | using Moq; 7 | 8 | namespace DbArchiver.Provider.Integration.Tests; 9 | 10 | public class MSSQLProviderTests 11 | { 12 | private readonly string _connectionString = "Server=(localdb)\\MSSQLLocalDB;Database=TestDbArchiver;Trusted_Connection=True;"; 13 | private readonly ILogger _logger; 14 | 15 | public MSSQLProviderTests() 16 | { 17 | var loggerMock = new Mock>(); 18 | _logger = loggerMock.Object; 19 | 20 | // Ensure test database is created 21 | EnsureDatabase(); 22 | } 23 | 24 | [Fact] 25 | public async Task InsertAsync_ShouldInsertData() 26 | { 27 | // Arrange 28 | var provider = new MSSQLProvider(_logger); 29 | var settings = new TargetSettings 30 | { 31 | ConnectionString = _connectionString, 32 | Schema = "dbo", 33 | Table = "TestTable", 34 | IdColumn = "Id" 35 | }; 36 | 37 | var testData = new List 38 | { 39 | new Dictionary { { "Id", 1 }, { "Name", "TestName1" } }, 40 | new Dictionary { { "Id", 2 }, { "Name", "TestName2" } } 41 | }; 42 | 43 | // Act 44 | await provider.InsertAsync(settings, testData); 45 | 46 | // Assert 47 | using (var connection = new SqlConnection(_connectionString)) 48 | { 49 | var result = (await connection.QueryAsync("SELECT * FROM dbo.TestTable")).ToList(); 50 | Assert.Equal(2, result.Count); 51 | } 52 | } 53 | 54 | [Fact] 55 | public async Task DeleteAsync_ShouldDeleteData() 56 | { 57 | // Arrange 58 | var provider = new MSSQLProvider(_logger); 59 | var settings = new SourceSettings 60 | { 61 | ConnectionString = _connectionString, 62 | Schema = "dbo", 63 | Table = "TestTable", 64 | IdColumn = "Id" 65 | }; 66 | 67 | var testData = new List 68 | { 69 | new Dictionary { { "Id", 1 } } 70 | }; 71 | 72 | // Act 73 | await provider.DeleteAsync(settings, testData); 74 | 75 | // Assert 76 | using (var connection = new SqlConnection(_connectionString)) 77 | { 78 | var result = (await connection.QueryAsync("SELECT * FROM dbo.TestTable")).ToList(); 79 | Assert.Single(result); 80 | Assert.Equal(2, result.First()["Id"]); 81 | } 82 | } 83 | 84 | private void EnsureDatabase() 85 | { 86 | using (var connection = new SqlConnection("Server=(localdb)\\MSSQLLocalDB;Trusted_Connection=True;")) 87 | { 88 | connection.Open(); 89 | 90 | connection.Execute(@" 91 | IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'TestDbArchiver') 92 | BEGIN 93 | CREATE DATABASE TestDbArchiver; 94 | END"); 95 | 96 | connection.ChangeDatabase("TestDbArchiver"); 97 | 98 | connection.Execute(@" 99 | IF OBJECT_ID('dbo.TestTable', 'U') IS NULL 100 | BEGIN 101 | CREATE TABLE dbo.TestTable ( 102 | Id INT PRIMARY KEY, 103 | Name NVARCHAR(100) NOT NULL 104 | ); 105 | END"); 106 | } 107 | } 108 | 109 | } -------------------------------------------------------------------------------- /tests/IntegrationTests/MySQLProviderTests.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.MySQL; 3 | using DbArchiver.Provider.MySQL.Config; 4 | using Microsoft.Extensions.Logging; 5 | using Moq; 6 | using MySql.Data.MySqlClient; 7 | 8 | namespace DbArchiver.Provider.Integration.Tests; 9 | 10 | public class MySQLProviderTests 11 | { 12 | private readonly Mock> _loggerMock; 13 | private readonly MySQLProvider _provider; 14 | private readonly string _connectionString = "Server=localhost;Database=TestDb;Uid=root;Pwd=yourpassword;"; 15 | private readonly string _tableName = "TestTable"; 16 | 17 | public MySQLProviderTests() 18 | { 19 | _loggerMock = new Mock>(); 20 | _provider = new MySQLProvider(_loggerMock.Object); 21 | } 22 | 23 | public async Task InitializeAsync() 24 | { 25 | using var connection = new MySqlConnection(_connectionString); 26 | await connection.OpenAsync(); 27 | 28 | var createTableScript = $@" 29 | CREATE TABLE IF NOT EXISTS `{_tableName}` ( 30 | `Id` INT PRIMARY KEY AUTO_INCREMENT, 31 | `Name` VARCHAR(255), 32 | `Value` VARCHAR(255) 33 | );"; 34 | await connection.ExecuteAsync(createTableScript); 35 | } 36 | 37 | public async Task DisposeAsync() 38 | { 39 | using var connection = new MySqlConnection(_connectionString); 40 | await connection.OpenAsync(); 41 | 42 | var dropTableScript = $@"DROP TABLE IF EXISTS `{_tableName}`;"; 43 | await connection.ExecuteAsync(dropTableScript); 44 | } 45 | 46 | [Fact] 47 | public async Task InsertAsync_ShouldInsertDataCorrectly() 48 | { 49 | // Arrange 50 | var targetSettings = new TargetSettings 51 | { 52 | ConnectionString = _connectionString, 53 | Schema = "TestDb", 54 | Table = _tableName, 55 | IdColumn = "Id" 56 | }; 57 | 58 | var testData = new List 59 | { 60 | new Dictionary { { "Id", 1 }, { "Name", "Test1" }, { "Value", "Value1" } }, 61 | new Dictionary { { "Id", 2 }, { "Name", "Test2" }, { "Value", "Value2" } } 62 | }; 63 | 64 | // Act 65 | await _provider.InsertAsync(targetSettings, testData); 66 | 67 | // Assert 68 | using var connection = new MySqlConnection(_connectionString); 69 | var insertedData = (await connection.QueryAsync($"SELECT * FROM `{_tableName}`")).ToList(); 70 | 71 | Assert.Equal(2, insertedData.Count); 72 | Assert.Contains(insertedData, row => row.Name == "Test1" && row.Value == "Value1"); 73 | Assert.Contains(insertedData, row => row.Name == "Test2" && row.Value == "Value2"); 74 | } 75 | 76 | [Fact] 77 | public async Task DeleteAsync_ShouldDeleteDataCorrectly() 78 | { 79 | // Arrange 80 | var sourceSettings = new SourceSettings 81 | { 82 | ConnectionString = _connectionString, 83 | Schema = "TestDb", 84 | Table = _tableName, 85 | IdColumn = "Id" 86 | }; 87 | 88 | var testData = new List 89 | { 90 | new Dictionary { { "Id", 1 }, { "Name", "Test1" }, { "Value", "Value1" } }, 91 | new Dictionary { { "Id", 2 }, { "Name", "Test2" }, { "Value", "Value2" } } 92 | }; 93 | 94 | using var connection = new MySqlConnection(_connectionString); 95 | 96 | // Insert data for testing deletion 97 | await connection.ExecuteAsync($"INSERT INTO `{_tableName}` (`Id`, `Name`, `Value`) VALUES (@Id, @Name, @Value);", testData); 98 | 99 | // Assert initial data count 100 | var initialData = (await connection.QueryAsync($"SELECT * FROM `{_tableName}`")).ToList(); 101 | Assert.Equal(2, initialData.Count); 102 | 103 | // Act 104 | await _provider.DeleteAsync(sourceSettings, testData); 105 | 106 | // Assert 107 | var remainingData = (await connection.QueryAsync($"SELECT * FROM `{_tableName}`")).ToList(); 108 | Assert.Empty(remainingData); 109 | } 110 | 111 | [Fact] 112 | public async Task ExecuteScriptAsync_ShouldExecuteCorrectly() 113 | { 114 | // Arrange 115 | var targetSettings = new TargetSettings 116 | { 117 | ConnectionString = _connectionString, 118 | Schema = "TestDb", 119 | Table = _tableName 120 | }; 121 | 122 | var insertScript = $@" 123 | INSERT INTO `{_tableName}` (`Name`, `Value`) 124 | VALUES ('ScriptTest', 'ScriptValue');"; 125 | 126 | // Act 127 | await _provider.ExecuteScriptAsync(targetSettings, insertScript); 128 | 129 | // Assert 130 | using var connection = new MySqlConnection(_connectionString); 131 | var data = await connection.QueryFirstOrDefaultAsync($"SELECT * FROM `{_tableName}` WHERE `Name` = 'ScriptTest'"); 132 | Assert.NotNull(data); 133 | Assert.Equal("ScriptValue", data.Value); 134 | } 135 | 136 | [Fact] 137 | public async Task GetIteratorAsync_ShouldReturnBatchedData() 138 | { 139 | // Arrange 140 | var sourceSettings = new SourceSettings 141 | { 142 | ConnectionString = _connectionString, 143 | Schema = "TestDb", 144 | Table = _tableName, 145 | IdColumn = "Id" 146 | }; 147 | 148 | var testData = Enumerable.Range(1, 10).Select(i => new 149 | { 150 | Id = i, 151 | Name = $"Name{i}", 152 | Value = $"Value{i}" 153 | }); 154 | 155 | using var connection = new MySqlConnection(_connectionString); 156 | await connection.ExecuteAsync($"INSERT INTO `{_tableName}` (`Id`, `Name`, `Value`) VALUES (@Id, @Name, @Value);", testData); 157 | 158 | // Act 159 | var iterator = await _provider.GetIteratorAsync(sourceSettings, 5); 160 | 161 | // Assert 162 | var batchCount = 0; 163 | while (await iterator.NextAsync()) 164 | { 165 | batchCount++; 166 | Assert.Equal(5, iterator.Data.Count()); 167 | } 168 | Assert.Equal(2, batchCount); 169 | 170 | iterator.Dispose(); 171 | } 172 | 173 | } -------------------------------------------------------------------------------- /tests/IntegrationTests/PostgreSQLProviderTests.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.PostgreSQL; 3 | using DbArchiver.Provider.PostgreSQL.Config; 4 | using Microsoft.Extensions.Logging.Abstractions; 5 | using Npgsql; 6 | 7 | namespace DbArchiver.Provider.Integration.Tests; 8 | 9 | public class PostgreSQLProviderTests 10 | { 11 | private readonly string _connectionString = "Host=localhost;Port=5432;Database=TestDbArchiver;Username=postgres;Password=postgres"; 12 | 13 | public PostgreSQLProviderTests() 14 | { 15 | EnsureDatabase(); 16 | } 17 | 18 | [Fact] 19 | public async Task InsertAsync_ShouldInsertData() 20 | { 21 | // Arrange 22 | var provider = new PostgreSQLProvider(new NullLogger()); 23 | var settings = new TargetSettings 24 | { 25 | ConnectionString = _connectionString, 26 | Schema = "public", 27 | Table = "test_table", 28 | IdColumn = "id" 29 | }; 30 | 31 | var testData = new List 32 | { 33 | new Dictionary { { "id", 1 }, { "name", "TestName1" } }, 34 | new Dictionary { { "id", 2 }, { "name", "TestName2" } } 35 | }; 36 | 37 | // Act 38 | await provider.InsertAsync(settings, testData); 39 | 40 | // Assert 41 | using (var connection = new NpgsqlConnection(_connectionString)) 42 | { 43 | var result = await connection.QueryAsync("SELECT * FROM public.test_table"); 44 | Assert.Equal(2, result.Count()); 45 | } 46 | } 47 | 48 | [Fact] 49 | public async Task DeleteAsync_ShouldDeleteData() 50 | { 51 | // Arrange 52 | var provider = new PostgreSQLProvider(new NullLogger()); 53 | var settings = new SourceSettings 54 | { 55 | ConnectionString = _connectionString, 56 | Schema = "public", 57 | Table = "test_table", 58 | IdColumn = "id" 59 | }; 60 | 61 | var testData = new List 62 | { 63 | new Dictionary { { "id", 1 } } 64 | }; 65 | 66 | // Act 67 | await provider.DeleteAsync(settings, testData); 68 | 69 | // Assert 70 | using (var connection = new NpgsqlConnection(_connectionString)) 71 | { 72 | var result = await connection.QueryAsync("SELECT * FROM public.test_table"); 73 | Assert.Single(result); 74 | Assert.Equal(2, result.First().id); 75 | } 76 | } 77 | 78 | [Fact] 79 | public async Task ExecuteScriptAsync_ShouldExecuteScript() 80 | { 81 | // Arrange 82 | var provider = new PostgreSQLProvider(new NullLogger()); 83 | var settings = new TargetSettings 84 | { 85 | ConnectionString = _connectionString, 86 | Schema = "public", 87 | Table = "test_table", 88 | IdColumn = "id" 89 | }; 90 | 91 | var script = "INSERT INTO public.test_table (id, name) VALUES (3, 'TestName3')"; 92 | 93 | // Act 94 | await provider.ExecuteScriptAsync(settings, script); 95 | 96 | // Assert 97 | using (var connection = new NpgsqlConnection(_connectionString)) 98 | { 99 | var result = await connection.QueryAsync("SELECT * FROM public.test_table WHERE id = 3"); 100 | Assert.Single(result); 101 | Assert.Equal("TestName3", result.First().name); 102 | } 103 | } 104 | 105 | [Fact] 106 | public async Task GetIteratorAsync_ShouldReturnDataInBatches() 107 | { 108 | // Arrange 109 | var provider = new PostgreSQLProvider(new NullLogger()); 110 | var settings = new SourceSettings 111 | { 112 | ConnectionString = _connectionString, 113 | Schema = "public", 114 | Table = "test_table", 115 | IdColumn = "id" 116 | }; 117 | 118 | var transferQuantity = 1; 119 | 120 | // Act 121 | var iterator = await provider.GetIteratorAsync(settings, transferQuantity); 122 | 123 | // Assert 124 | Assert.NotNull(iterator); 125 | 126 | var hasNext = await iterator.NextAsync(); 127 | Assert.True(hasNext); 128 | Assert.Single(iterator.Data); 129 | 130 | hasNext = await iterator.NextAsync(); 131 | Assert.True(hasNext); 132 | Assert.Single(iterator.Data); 133 | 134 | hasNext = await iterator.NextAsync(); 135 | Assert.False(hasNext); 136 | } 137 | 138 | private void EnsureDatabase() 139 | { 140 | using (var connection = new NpgsqlConnection(_connectionString)) 141 | { 142 | connection.Open(); 143 | 144 | connection.Execute(@" 145 | CREATE TABLE IF NOT EXISTS public.test_table ( 146 | id INT PRIMARY KEY, 147 | name VARCHAR(100) NOT NULL 148 | ); 149 | 150 | DELETE FROM public.test_table; 151 | 152 | INSERT INTO public.test_table (id, name) 153 | VALUES 154 | (1, 'InitialName1'), 155 | (2, 'InitialName2');"); 156 | } 157 | } 158 | } -------------------------------------------------------------------------------- /tests/IntegrationTests/SQLiteProviderTests.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using DbArchiver.Provider.SQLite; 3 | using DbArchiver.Provider.SQLite.Config; 4 | using Microsoft.Data.Sqlite; 5 | using Microsoft.Extensions.Logging.Abstractions; 6 | 7 | namespace DbArchiver.Provider.Integration.Tests; 8 | 9 | public class SQLiteProviderTests 10 | { 11 | private const string ConnectionString = "Data Source=:memory:"; 12 | 13 | private void InitializeDatabase() 14 | { 15 | using var connection = new SqliteConnection(ConnectionString); 16 | connection.Open(); 17 | connection.Execute(@" 18 | CREATE TABLE test_table ( 19 | id INTEGER PRIMARY KEY, 20 | name TEXT NOT NULL 21 | ); 22 | 23 | INSERT INTO test_table (id, name) VALUES (1, 'InitialName1'); 24 | INSERT INTO test_table (id, name) VALUES (2, 'InitialName2'); 25 | "); 26 | } 27 | 28 | [Fact] 29 | public async Task InsertAsync_ShouldInsertData() 30 | { 31 | // Arrange 32 | InitializeDatabase(); 33 | var provider = new SQLiteProvider(new NullLogger()); 34 | var settings = new TargetSettings 35 | { 36 | ConnectionString = ConnectionString, 37 | Table = "test_table" 38 | }; 39 | 40 | var testData = new List 41 | { 42 | new Dictionary { { "id", 3 }, { "name", "TestName3" } }, 43 | new Dictionary { { "id", 4 }, { "name", "TestName4" } } 44 | }; 45 | 46 | // Act 47 | await provider.InsertAsync(settings, testData); 48 | 49 | // Assert 50 | using var connection = new SqliteConnection(ConnectionString); 51 | var result = connection.Query("SELECT * FROM test_table WHERE id IN (3, 4)"); 52 | Assert.Equal(2, result.Count()); 53 | } 54 | 55 | [Fact] 56 | public async Task DeleteAsync_ShouldDeleteData() 57 | { 58 | // Arrange 59 | InitializeDatabase(); 60 | var provider = new SQLiteProvider(new NullLogger()); 61 | var settings = new SourceSettings 62 | { 63 | ConnectionString = ConnectionString, 64 | Table = "test_table", 65 | IdColumn = "id" 66 | }; 67 | 68 | var testData = new List 69 | { 70 | new Dictionary { { "id", 1 } } 71 | }; 72 | 73 | // Act 74 | await provider.DeleteAsync(settings, testData); 75 | 76 | // Assert 77 | using var connection = new SqliteConnection(ConnectionString); 78 | var result = connection.Query("SELECT * FROM test_table WHERE id = 1"); 79 | Assert.Empty(result); 80 | } 81 | 82 | [Fact] 83 | public async Task ExecuteScriptAsync_ShouldExecuteScript() 84 | { 85 | // Arrange 86 | InitializeDatabase(); 87 | var provider = new SQLiteProvider(new NullLogger()); 88 | var settings = new TargetSettings 89 | { 90 | ConnectionString = ConnectionString, 91 | Table = "test_table" 92 | }; 93 | 94 | var script = "INSERT INTO test_table (id, name) VALUES (5, 'ScriptName5')"; 95 | 96 | // Act 97 | await provider.ExecuteScriptAsync(settings, script); 98 | 99 | // Assert 100 | using var connection = new SqliteConnection(ConnectionString); 101 | var result = connection.Query("SELECT * FROM test_table WHERE id = 5"); 102 | Assert.Single(result); 103 | } 104 | 105 | [Fact] 106 | public async Task GetIteratorAsync_ShouldReturnDataInBatches() 107 | { 108 | // Arrange 109 | InitializeDatabase(); 110 | var provider = new SQLiteProvider(new NullLogger()); 111 | var settings = new SourceSettings 112 | { 113 | ConnectionString = ConnectionString, 114 | Table = "test_table", 115 | IdColumn = "id" 116 | }; 117 | 118 | var batchSize = 1; 119 | 120 | // Act 121 | var iterator = await provider.GetIteratorAsync(settings, batchSize); 122 | 123 | // Assert 124 | Assert.NotNull(iterator); 125 | 126 | var hasNext = await iterator.NextAsync(); 127 | Assert.True(hasNext); 128 | Assert.Single(iterator.Data); 129 | Assert.Equal(1, ((dynamic)iterator.Data.First()).id); 130 | 131 | hasNext = await iterator.NextAsync(); 132 | Assert.True(hasNext); 133 | Assert.Single(iterator.Data); 134 | Assert.Equal(2, ((dynamic)iterator.Data.First()).id); 135 | 136 | hasNext = await iterator.NextAsync(); 137 | Assert.False(hasNext); 138 | } 139 | 140 | } --------------------------------------------------------------------------------