├── .gitattributes ├── .gitignore ├── .nuget └── NuGet.Config ├── Directory.Build.props ├── README.md ├── Sitecore.DataBlaster.sln ├── license.txt └── src ├── Sitecore.DataBlaster ├── BulkField.cs ├── BulkItem.cs ├── ItemReference.cs ├── Load │ ├── BulkLoadAction.cs │ ├── BulkLoadContext.cs │ ├── BulkLoadItem.cs │ ├── BulkLoader.cs │ ├── FieldRule.cs │ ├── ItemChange.cs │ ├── Links │ │ ├── BulkItemLink.cs │ │ └── BulkItemLinkParser.cs │ ├── Paths │ │ └── AncestorGenerator.cs │ ├── Processors │ │ ├── ChangeCacheClearer.cs │ │ ├── ChangeIndexer.cs │ │ ├── ChangeLogger.cs │ │ ├── IChangeProcessor.cs │ │ ├── IItemProcessor.cs │ │ ├── ISyncInTransaction.cs │ │ ├── IValidateStagedData.cs │ │ ├── ItemBucketer.cs │ │ ├── ItemLinker.cs │ │ ├── ItemValidator.cs │ │ ├── ItemVersionEnsurer.cs │ │ ├── SyncHistoryTable.cs │ │ ├── SyncPublishQueue.cs │ │ ├── ValidateDataIntegrity.cs │ │ └── ValidateNoDuplicates.cs │ ├── Sql │ │ ├── 01.CreateTempTable.sql │ │ ├── 02.LookupBlobs.sql │ │ ├── 03.LookupItems.sql │ │ ├── 04.SplitTempTable.sql │ │ ├── 05.CheckDuplicates.sql │ │ ├── 06.CreateIndexes.sql │ │ ├── 07.CheckTempData.sql │ │ ├── 08.MergeTempData.sql │ │ ├── 09.UpdateHistory.sql │ │ ├── 10.UpdatePublishQueue.sql │ │ ├── 20.CreateLinkTempTable.sql │ │ ├── 21.MergeLinkTempData.sql │ │ └── BulkLoadSqlContext.cs │ ├── Stage.cs │ └── StageResult.cs ├── Read │ ├── BulkReader.cs │ ├── ItemHeader.cs │ ├── ItemVersionHeader.cs │ └── Sql │ │ ├── GetDescendantHeaders.sql │ │ ├── GetDescendantVersionHeaders.sql │ │ └── GetDescendants.sql ├── SharedBulkField.cs ├── Sitecore.DataBlaster.csproj ├── UnversionedBulkField.cs ├── Util │ ├── CacheUtil.cs │ ├── Chain.cs │ ├── GuidUtility.cs │ ├── ISearchIndexExtensions.cs │ ├── LogExtensions.cs │ └── Sql │ │ ├── AbstractEnumeratorReader.cs │ │ ├── SqlContext.cs │ │ ├── SqlContextExtensions.cs │ │ └── SqlLine.cs └── VersionedBulkField.cs └── Unicorn.DataBlaster ├── App_Config └── Unicorn │ └── Unicorn.DataBlaster.config ├── Logging └── SitecoreAndUnicornLog.cs ├── Sync ├── DataBlasterParameters.cs ├── ItemExtractor.cs ├── ItemMapper.cs └── UnicornDataBlaster.cs └── Unicorn.DataBlaster.csproj /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | bld/ 21 | [Bb]in/ 22 | [Oo]bj/ 23 | [Ll]og/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | project.fragment.lock.json 46 | artifacts/ 47 | 48 | *_i.c 49 | *_p.c 50 | *_i.h 51 | *.ilk 52 | *.meta 53 | *.obj 54 | *.pch 55 | *.pdb 56 | *.pgc 57 | *.pgd 58 | *.rsp 59 | *.sbr 60 | *.tlb 61 | *.tli 62 | *.tlh 63 | *.tmp 64 | *.tmp_proj 65 | *.log 66 | *.vspscc 67 | *.vssscc 68 | .builds 69 | *.pidb 70 | *.svclog 71 | *.scc 72 | 73 | # Chutzpah Test files 74 | _Chutzpah* 75 | 76 | # Visual C++ cache files 77 | ipch/ 78 | *.aps 79 | *.ncb 80 | *.opendb 81 | *.opensdf 82 | *.sdf 83 | *.cachefile 84 | *.VC.db 85 | *.VC.VC.opendb 86 | 87 | # Visual Studio profiler 88 | *.psess 89 | *.vsp 90 | *.vspx 91 | *.sap 92 | 93 | # TFS 2012 Local Workspace 94 | $tf/ 95 | 96 | # Guidance Automation Toolkit 97 | *.gpState 98 | 99 | # ReSharper is a .NET coding add-in 100 | _ReSharper*/ 101 | *.[Rr]e[Ss]harper 102 | *.DotSettings.user 103 | 104 | # JustCode is a .NET coding add-in 105 | .JustCode 106 | 107 | # TeamCity is a build add-in 108 | _TeamCity* 109 | 110 | # DotCover is a Code Coverage Tool 111 | *.dotCover 112 | 113 | # NCrunch 114 | _NCrunch_* 115 | .*crunch*.local.xml 116 | nCrunchTemp_* 117 | 118 | # MightyMoose 119 | *.mm.* 120 | AutoTest.Net/ 121 | 122 | # Web workbench (sass) 123 | .sass-cache/ 124 | 125 | # Installshield output folder 126 | [Ee]xpress/ 127 | 128 | # DocProject is a documentation generator add-in 129 | DocProject/buildhelp/ 130 | DocProject/Help/*.HxT 131 | DocProject/Help/*.HxC 132 | DocProject/Help/*.hhc 133 | DocProject/Help/*.hhk 134 | DocProject/Help/*.hhp 135 | DocProject/Help/Html2 136 | DocProject/Help/html 137 | 138 | # Click-Once directory 139 | publish/ 140 | 141 | # Publish Web Output 142 | *.[Pp]ublish.xml 143 | *.azurePubxml 144 | # TODO: Comment the next line if you want to checkin your web deploy settings 145 | # but database connection strings (with potential passwords) will be unencrypted 146 | #*.pubxml 147 | *.publishproj 148 | 149 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 150 | # checkin your Azure Web App publish settings, but sensitive information contained 151 | # in these scripts will be unencrypted 152 | PublishScripts/ 153 | 154 | # NuGet Packages 155 | *.nupkg 156 | # The packages folder can be ignored because of Package Restore 157 | **/packages/* 158 | # except build/, which is used as an MSBuild target. 159 | !**/packages/build/ 160 | # Uncomment if necessary however generally it will be regenerated when needed 161 | #!**/packages/repositories.config 162 | # NuGet v3's project.json files produces more ignoreable files 163 | *.nuget.props 164 | *.nuget.targets 165 | 166 | # Microsoft Azure Build Output 167 | csx/ 168 | *.build.csdef 169 | 170 | # Microsoft Azure Emulator 171 | ecf/ 172 | rcf/ 173 | 174 | # Windows Store app package directories and files 175 | AppPackages/ 176 | BundleArtifacts/ 177 | Package.StoreAssociation.xml 178 | _pkginfo.txt 179 | 180 | # Visual Studio cache files 181 | # files ending in .cache can be ignored 182 | *.[Cc]ache 183 | # but keep track of directories ending in .cache 184 | !*.[Cc]ache/ 185 | 186 | # Others 187 | ClientBin/ 188 | ~$* 189 | *~ 190 | *.dbmdl 191 | *.dbproj.schemaview 192 | *.jfm 193 | *.pfx 194 | *.publishsettings 195 | node_modules/ 196 | orleans.codegen.cs 197 | 198 | # Since there are multiple workflows, uncomment next line to ignore bower_components 199 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 200 | #bower_components/ 201 | 202 | # RIA/Silverlight projects 203 | Generated_Code/ 204 | 205 | # Backup & report files from converting an old project file 206 | # to a newer Visual Studio version. Backup files are not needed, 207 | # because we have git ;-) 208 | _UpgradeReport_Files/ 209 | Backup*/ 210 | UpgradeLog*.XML 211 | UpgradeLog*.htm 212 | 213 | # SQL Server files 214 | *.mdf 215 | *.ldf 216 | 217 | # Business Intelligence projects 218 | *.rdl.data 219 | *.bim.layout 220 | *.bim_*.settings 221 | 222 | # Microsoft Fakes 223 | FakesAssemblies/ 224 | 225 | # GhostDoc plugin setting file 226 | *.GhostDoc.xml 227 | 228 | # Node.js Tools for Visual Studio 229 | .ntvs_analysis.dat 230 | 231 | # Visual Studio 6 build log 232 | *.plg 233 | 234 | # Visual Studio 6 workspace options file 235 | *.opt 236 | 237 | # Visual Studio LightSwitch build output 238 | **/*.HTMLClient/GeneratedArtifacts 239 | **/*.DesktopClient/GeneratedArtifacts 240 | **/*.DesktopClient/ModelManifest.xml 241 | **/*.Server/GeneratedArtifacts 242 | **/*.Server/ModelManifest.xml 243 | _Pvt_Extensions 244 | 245 | # Paket dependency manager 246 | .paket/paket.exe 247 | paket-files/ 248 | 249 | # FAKE - F# Make 250 | .fake/ 251 | 252 | # JetBrains Rider 253 | .idea/ 254 | *.sln.iml 255 | 256 | # CodeRush 257 | .cr/ 258 | 259 | # Python Tools for Visual Studio (PTVS) 260 | __pycache__/ 261 | *.pyc -------------------------------------------------------------------------------- /.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | $(MsBuildAllProjects);$(MsBuildThisFileFullPath) 5 | 6 | 7 | 8 | 10.4.0 9 | 10 | 11 | $(VersionPrefix).$(BUILD_BUILDID) 12 | 13 | 14 | Copyright © 2020 15 | delaware digital 16 | delaware-digital 17 | http://opensource.org/licenses/MIT 18 | http://github.com/delawarePro/sitecore-data-blaster 19 | 20 | 21 | true 22 | true 23 | 24 | 26 | symbols.nupkg 27 | 28 | 29 | 30 | 31 | 32 | $(Prerelease) 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | all 41 | runtime; build; native; contentfiles; analyzers; buildtransitive 42 | 43 | 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Sitecore DataBlaster is a **low-level API to rapidly load lots of data in a Sitecore database** while still being compatible with other moving parts like caches, indexes and others. Loading a lot of data in Sitecore is typically quite slow. Although Sitecore allows you to perform some performance tweaks, like e.g. BulkUpdateContext or EventDisabler, the central item APIs are not optimized for batch/bulk updates. 2 | 3 | ## What do I use it for? 4 | * **Deserialization** of developer managed content (see below for integration with **Unicorn**) 5 | * Standup fresh Sitecore environments for **Continuous Integration** 6 | * Full and delta **imports** into Sitecore 7 | * **Transfer items** from one db to another when upgrading Sitecore versions 8 | 9 | ## How does it work? 10 | A stream (IEnumerable) of items is bulk copied (SqlBulkCopy) into a temp table after which an elaborate SQL merge script is executed to insert, update and delete items and fields in the database. During the merge, the changes are tracked, so that it can report the exact item changes. These changes are then used to synchronize other parts like e.g. caches and indexes. 11 | 12 | ## Are you crazy? What if Sitecore changes its database schema? 13 | First of all, the database schema hasn't changed significantly in ages. We've been running these scripts from Sitecore 7.1 onwards. Second, database schema changes will have quite some impact on Sitecore as well, so they'll probably be quite careful with this. 14 | 15 | ## Are you sure? Is this proven technology? 16 | We, at [delaware digital](http://digital.delawareconsulting.com), successfully use this approach for a **couple of years** now. Our own serialization tools were built on this library. By using this approach in a specific project, our deserialization time decreased from arround **50+ minutes to less than 1 minute**. 17 | 18 | Next to that, we are strong believers in continuous integration and continous delivery, so **automated testing is crucial** for us. Without this library, automated **integration, web and smoke tests wouldn't really be feasable** on the large projects that we're building and supporting. 19 | 20 | ## Alright, so what breaks when I use this? 21 | The data blaster stores everything in the database in native Sitecore format and we've gone out of our way to make sure everything is **fully supported**, like following components: 22 | * **Bucketing**: auto bucketing items into Sitecore buckets. 23 | * **Item API**: caches are cleared immediately after data blast. 24 | * **Link database**: links between items are detected and updated. 25 | * **History engine**: used to be important for index updates. 26 | * **Publish queue**: if you want to use the incremental publish. 27 | * **Indexes**: optimized index update with auto rebuild and refresh support. 28 | 29 | There's still **one thing not supported yet**, which is updating the event queue. This is typically only important when you have multiple content management nodes. **However** we have an alpha version implementation that will follow soon. 30 | 31 | ## I don't find any automated tests, are you kidding me!? 32 | The automated tests are not located in this repository, because we need a more elaborate setup, but we run over **65 automated tests** on a real Sitecore database on every checkin. 33 | 34 | ## Unicorn you said? 35 | [Unicorn](https://github.com/kamsar/Unicorn) is a cool (de)serialization utility and it's quite optimized for performance. However, it's not performing very well when filling 'empty' Sitecore databases. This is not Unicorn's fault, but the issue of the underlying item API. 36 | 37 | Because filling 'empty' Sitecore databases is typically something we do **very often**, we created, with directions of [kamsar](https://github.com/kamsar), a drop-in integration for Unicorn. Which, in our tests, is faster than the default implementation in all our cases. 38 | 39 | How to get started? 40 | * Install nuget package [Unicorn.DataBlaster](https://www.nuget.org/packages/Unicorn.DataBlaster/) 41 | * Add a configuration file: App_Config/Unicorn/Unicorn.DataBlaster.config 42 | ```xml 43 | 44 | 45 | 46 | 47 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | true 63 | 64 | true 65 | 68 | false 69 | 72 | false 73 | 74 | 75 | 76 | 77 | ``` 78 | 79 | ## How do I use it programmatically? 80 | I thought you'd never ask. Let's do a quick intro of the core classes first. 81 | * [BulkLoader](https://github.com/delawarePro/sitecore-data-blaster/blob/master/src/Sitecore.DataBlaster/Load/BulkLoader.cs): core of the bulk load process. 82 | * [BulkLoadContext](https://github.com/delawarePro/sitecore-data-blaster/blob/master/src/Sitecore.DataBlaster/Load/BulkLoadContext.cs): context object that supports load options and tracking. 83 | * [BulkLoadItem](https://github.com/delawarePro/sitecore-data-blaster/blob/master/src/Sitecore.DataBlaster/Load/BulkLoadItem.cs): Item representation with its fields and load behavior. 84 | 85 | You can clone/fork this repository or you could use a NuGet package: [Sitecore.DataBlaster](https://www.nuget.org/packages/Sitecore.DataBlaster/) 86 | 87 | ### Let's create a simple item. 88 | ```cs 89 | // Get standard Sitecore refrences. 90 | var masterDb = Factory.GetDatabase("master"); 91 | var contentItem = masterDb.GetItem("/sitecore/content"); 92 | var folderTemplate = TemplateManager.GetTemplate(TemplateIDs.Folder, masterDb); 93 | 94 | // Create a new folder as child of the content item with the data blaster. 95 | var bulkLoader = new BulkLoader(); 96 | var context = BulkLoader.NewBulkLoadContext(masterDb.Name); 97 | bulkLoader.LoadItems(context, new[] 98 | { 99 | new BulkLoadItem(BulkLoadAction.Update, folderTemplate, contentItem, "New Folder") 100 | }); 101 | ``` 102 | ### BulkLoadAction 103 | One of the most important parts is choosing the right bulk load action per item: 104 | ```cs 105 | public enum BulkLoadAction 106 | { 107 | /// 108 | /// Adds items and adds missing fields, but doesn't update any fields. 109 | /// 110 | AddOnly = 0, 111 | 112 | /// 113 | /// Only adds items that don't exist yet, does NOT add missing fields to existing items. 114 | /// 115 | AddItemOnly = 6, 116 | 117 | /// 118 | /// Adds items, missing fields to existing items and updates/overwrites fields for which the data is different. 119 | /// 120 | Update = 1, 121 | 122 | /// 123 | /// Adds and updates fields for existing items only. 124 | /// 125 | UpdateExistingItem = 2, 126 | 127 | /// 128 | /// Reverts items to the provided state, removing redundant fields as well. 129 | /// Does NOT remove children that are not provided in the dataset. 130 | /// 131 | Revert = 3, 132 | 133 | /// 134 | /// Reverts items to the provided state, removing redundant fields as well. 135 | /// Removes descendants that are not provided in the dataset. 136 | /// 137 | RevertTree = 4 138 | } 139 | ``` 140 | 141 | ### Other stuff 142 | A lot of options and combinations are available. You can e.g. use the item path to lookup an id of item in the database. This can particulary be useful when importing data for which you don't know the item id in Sitecore. 143 | 144 | Another feature worth mentioning is 'FieldRules' on the bulk load context, which allows you to exclude specific Sitecore fields from the bulk load process. 145 | 146 | If you need more information, please feel free to post an issue. 147 | 148 | ### Debugging 149 | A lot of the logic is implemented in SQL scripts, which are not easy to debug. For this purpose, there's a 'StageDataWithoutProcessing' flag on the bulk load context. In that case, all data will be staged in a table called 'tmp_BulkItemsAndFields'. After that you can execute the SQL scripts one by one, as long as you replace '#' with 'tmp_'. 150 | 151 | ## Questions, suggestions and bugs? 152 | Feel free to post an issue or a PR ;) -------------------------------------------------------------------------------- /Sitecore.DataBlaster.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.8.34330.188 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sitecore.DataBlaster", "src\Sitecore.DataBlaster\Sitecore.DataBlaster.csproj", "{D5B8428D-84FD-48BC-B722-7F9ECA73A43A}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Unicorn.DataBlaster", "src\Unicorn.DataBlaster\Unicorn.DataBlaster.csproj", "{2B27B2B1-BBCA-4F0D-A604-68FF0713DC21}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{878491BE-8601-40EC-85A7-BAC24BF53687}" 11 | ProjectSection(SolutionItems) = preProject 12 | .gitattributes = .gitattributes 13 | .gitignore = .gitignore 14 | Directory.Build.props = Directory.Build.props 15 | license.txt = license.txt 16 | README.md = README.md 17 | EndProjectSection 18 | EndProject 19 | Global 20 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 21 | Debug|Any CPU = Debug|Any CPU 22 | Release|Any CPU = Release|Any CPU 23 | EndGlobalSection 24 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 25 | {D5B8428D-84FD-48BC-B722-7F9ECA73A43A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 26 | {D5B8428D-84FD-48BC-B722-7F9ECA73A43A}.Debug|Any CPU.Build.0 = Debug|Any CPU 27 | {D5B8428D-84FD-48BC-B722-7F9ECA73A43A}.Release|Any CPU.ActiveCfg = Release|Any CPU 28 | {D5B8428D-84FD-48BC-B722-7F9ECA73A43A}.Release|Any CPU.Build.0 = Release|Any CPU 29 | {2B27B2B1-BBCA-4F0D-A604-68FF0713DC21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 30 | {2B27B2B1-BBCA-4F0D-A604-68FF0713DC21}.Debug|Any CPU.Build.0 = Debug|Any CPU 31 | {2B27B2B1-BBCA-4F0D-A604-68FF0713DC21}.Release|Any CPU.ActiveCfg = Release|Any CPU 32 | {2B27B2B1-BBCA-4F0D-A604-68FF0713DC21}.Release|Any CPU.Build.0 = Release|Any CPU 33 | EndGlobalSection 34 | GlobalSection(SolutionProperties) = preSolution 35 | HideSolutionNode = FALSE 36 | EndGlobalSection 37 | GlobalSection(ExtensibilityGlobals) = postSolution 38 | SolutionGuid = {BAC6357F-BD59-49EF-950F-1E8DF1400C05} 39 | EndGlobalSection 40 | EndGlobal 41 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 delaware 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/BulkField.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Sitecore.DataBlaster 5 | { 6 | public abstract class BulkField 7 | { 8 | public BulkItem Item { get; private set; } 9 | 10 | /// 11 | /// Field id. 12 | /// 13 | public Guid Id { get; private set; } 14 | 15 | /// 16 | /// Fieldname, only used for diagnostics. 17 | /// 18 | public string Name { get; set; } 19 | 20 | /// 21 | /// Field value. 22 | /// 23 | public string Value { get; set; } 24 | 25 | public Func Blob { get; set; } 26 | 27 | /// 28 | /// Field maybe a blob field although it may not have a blob stream. 29 | /// Can occur in case of same blob for different fields. 30 | /// 31 | public bool IsBlob { get; private set; } 32 | 33 | /// 34 | /// When set, this field is only processed when item is created in database. 35 | /// 36 | public bool DependsOnCreate { get; set; } 37 | 38 | /// 39 | /// When set, this field is only processed when item is updated in database. 40 | /// 41 | public bool DependsOnUpdate { get; set; } 42 | 43 | protected BulkField(BulkItem item, Guid id, string value, Func blob = null, bool isBlob = false, 44 | string name = null) 45 | { 46 | if (id == Guid.Empty) 47 | throw new ArgumentException("Id of field should not be an empty Guid.", nameof(id)); 48 | if (blob != null && !isBlob) 49 | throw new ArgumentException("You cannot provide a blob for a non-blob field."); 50 | 51 | this.Item = item; 52 | this.Id = id; 53 | this.Name = name; 54 | this.Value = value; 55 | this.Blob = blob; 56 | this.IsBlob = isBlob; 57 | } 58 | 59 | public override string ToString() 60 | { 61 | return $"{Name ?? Id.ToString()}={Value}"; 62 | } 63 | 64 | internal abstract BulkField CopyTo(BulkItem targetItem); 65 | } 66 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/BulkItem.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using Sitecore.Data.Managers; 6 | using Sitecore.Data.Templates; 7 | 8 | namespace Sitecore.DataBlaster 9 | { 10 | public class BulkItem 11 | { 12 | private string _itemPath; 13 | private readonly Dictionary _fields = new Dictionary(); 14 | 15 | /// 16 | /// Item id. 17 | /// 18 | public Guid Id { get; private set; } 19 | 20 | /// 21 | /// Item name 22 | /// 23 | public string Name { get; private set; } 24 | 25 | public Guid TemplateId { get; private set; } 26 | public Guid MasterId { get; private set; } 27 | public Guid ParentId { get; set; } 28 | 29 | /// 30 | /// Sitecore path of the item. 31 | /// 32 | /// Could be null. 33 | public string ItemPath 34 | { 35 | get { return _itemPath; } 36 | set 37 | { 38 | if (value == null) throw new ArgumentNullException(nameof(value)); 39 | _itemPath = value; 40 | 41 | this.Name = _itemPath.Split('/').Last(); 42 | } 43 | } 44 | 45 | 46 | public IEnumerable Fields => _fields.Values; 47 | public int FieldCount => _fields.Count; 48 | 49 | public BulkItem(Guid id, Guid templateId, Guid masterId, Guid parentId, string itemPath) 50 | { 51 | if (id == Guid.Empty) 52 | throw new ArgumentException("Id of item should not be an empty Guid.", nameof(id)); 53 | if (string.IsNullOrWhiteSpace(itemPath)) throw new ArgumentNullException(nameof(itemPath)); 54 | 55 | this.Id = id; 56 | this.TemplateId = templateId; 57 | this.MasterId = masterId; 58 | this.ParentId = parentId; 59 | this.ItemPath = itemPath; 60 | } 61 | 62 | protected BulkItem(BulkItem toCopy) 63 | { 64 | if (toCopy == null) throw new ArgumentNullException(nameof(toCopy)); 65 | 66 | this.Id = toCopy.Id; 67 | this.TemplateId = toCopy.TemplateId; 68 | this.MasterId = toCopy.MasterId; 69 | this.ParentId = toCopy.ParentId; 70 | this.ItemPath = toCopy.ItemPath; 71 | 72 | _fields = toCopy._fields.ToDictionary(x => x.Key, x => x.Value.CopyTo(this)); 73 | } 74 | 75 | private void AddField(BulkFieldKey key, string value, Func blob = null, bool isBlob = false, 76 | string name = null, 77 | Action postProcessor = null) 78 | { 79 | if (value == null && !isBlob) return; 80 | 81 | BulkField field = null; 82 | if (key.Language == null && !key.Version.HasValue) 83 | { 84 | field = new SharedBulkField(this, key.FieldId, value, blob, isBlob, name); 85 | _fields.Add(key, field); 86 | } 87 | else if (key.Language != null && !key.Version.HasValue) 88 | { 89 | field = new UnversionedBulkField(this, key.FieldId, key.Language, value, blob, isBlob, name); 90 | _fields.Add(key, field); 91 | } 92 | else if (key.Language == null && key.Version.HasValue) 93 | { 94 | throw new ArgumentException("You cannot add a language specific field without a version."); 95 | } 96 | else 97 | { 98 | field = new VersionedBulkField(this, key.FieldId, key.Language, key.Version.Value, value, blob, isBlob, 99 | name); 100 | _fields.Add(key, field); 101 | } 102 | 103 | postProcessor?.Invoke(field); 104 | } 105 | 106 | /// 107 | /// Tries to add the field, returns false if the field with name and version already exists. 108 | /// 109 | public bool TryAddField(Guid id, string value, Func blob = null, bool isBlob = false, 110 | string language = null, int? version = null, string name = null, 111 | Action postProcessor = null) 112 | { 113 | if (string.IsNullOrWhiteSpace(language)) language = null; 114 | 115 | var key = new BulkFieldKey(id, language, version); 116 | if (_fields.ContainsKey(key)) return false; 117 | 118 | AddField(key, value, blob, isBlob, name, postProcessor); 119 | return true; 120 | } 121 | 122 | public BulkItem AddField(Guid id, string value, Func blob = null, bool isBlob = false, 123 | string language = null, int? version = null, string name = null, 124 | Action postProcessor = null) 125 | { 126 | if (value == null && !isBlob) return this; 127 | if (string.IsNullOrWhiteSpace(language)) language = null; 128 | AddField(new BulkFieldKey(id, language, version), value, blob, isBlob, name, postProcessor); 129 | return this; 130 | } 131 | 132 | public BulkItem AddField(TemplateField field, string value, Func blob = null, bool isBlob = false, 133 | string language = null, int? version = null, 134 | Action postProcessor = null) 135 | { 136 | return AddField(field.ID.Guid, value, blob, isBlob, language, version, field.Name, postProcessor); 137 | } 138 | 139 | public BulkItem AddSharedField(Guid id, string value, 140 | Func blob = null, bool isBlob = false, string name = null, 141 | Action postProcessor = null) 142 | { 143 | return AddField(id, value, blob, isBlob, null, null, name, postProcessor); 144 | } 145 | 146 | public BulkItem AddUnversionedField(Guid id, string language, string value, 147 | Func blob = null, bool isBlob = false, string name = null, 148 | Action postProcessor = null) 149 | { 150 | return AddField(id, value, blob, isBlob, language, null, name, postProcessor); 151 | } 152 | 153 | public BulkItem AddVersionedField(Guid id, string language, int version, string value, 154 | Func blob = null, bool isBlob = false, string name = null, 155 | Action postProcessor = null) 156 | { 157 | return AddField(id, value, blob, isBlob, language, version, name, postProcessor); 158 | } 159 | 160 | public BulkField GetField(Guid id, string language, int? version) 161 | { 162 | BulkField field; 163 | return _fields.TryGetValue(new BulkFieldKey(id, language, version), out field) ? field : null; 164 | } 165 | 166 | /// 167 | /// Statistics fields are necessary for correct working of Sitecore versions. 168 | /// If not correctly configured, publish might e.g not work. 169 | /// 170 | /// Default language will be added when no language version is present. 171 | /// Language for which a version must be present. 172 | /// Whether to only ensure created and updated fields. 173 | /// Forces modification date to always be set, not only when data is changed. 174 | public void EnsureLanguageVersions(string defaultLanguage = null, IEnumerable mandatoryLanguages = null, 175 | bool timestampsOnly = false, bool forceUpdate = false) 176 | { 177 | var user = Sitecore.Context.User.Name; 178 | var now = DateUtil.IsoNow; 179 | 180 | var versionsByLanguage = new Dictionary>(StringComparer.OrdinalIgnoreCase); 181 | 182 | // Detect versions by language from fields. 183 | foreach (var field in Fields.OfType()) 184 | { 185 | var versioned = field as VersionedBulkField; 186 | var version = versioned?.Version ?? 1; 187 | 188 | HashSet versions; 189 | if (versionsByLanguage.TryGetValue(field.Language, out versions)) 190 | versions.Add(version); 191 | else 192 | versionsByLanguage[field.Language] = new HashSet 193 | { 194 | version 195 | }; 196 | } 197 | 198 | // Ensure mandatory languages. 199 | foreach (var language in mandatoryLanguages ?? Enumerable.Empty()) 200 | { 201 | HashSet versions; 202 | if (!versionsByLanguage.TryGetValue(language, out versions)) 203 | versionsByLanguage[language] = new HashSet 204 | { 205 | 1 206 | }; 207 | } 208 | 209 | // Add default version when no version is present. 210 | if (versionsByLanguage.Count == 0) 211 | versionsByLanguage[defaultLanguage ?? LanguageManager.DefaultLanguage.Name] = new HashSet 212 | { 213 | 1 214 | }; 215 | 216 | foreach (var languageVersion in versionsByLanguage 217 | .SelectMany(pair => pair.Value.Select(x => new 218 | { 219 | Language = pair.Key, 220 | Version = x 221 | }))) 222 | { 223 | TryAddField(FieldIDs.Created.Guid, now, 224 | language: languageVersion.Language, version: languageVersion.Version, 225 | name: "__Created", postProcessor: x => x.DependsOnCreate = true); 226 | 227 | TryAddField(FieldIDs.Updated.Guid, now, 228 | language: languageVersion.Language, version: languageVersion.Version, 229 | name: "__Updated", postProcessor: x => 230 | { 231 | if (!forceUpdate) 232 | x.DependsOnCreate = x.DependsOnUpdate = true; 233 | }); 234 | 235 | if (!timestampsOnly) 236 | { 237 | TryAddField(FieldIDs.CreatedBy.Guid, user, 238 | language: languageVersion.Language, version: languageVersion.Version, 239 | name: "__Created by", postProcessor: x => x.DependsOnCreate = true); 240 | 241 | TryAddField(FieldIDs.UpdatedBy.Guid, user, 242 | language: languageVersion.Language, version: languageVersion.Version, 243 | name: "__Updated by", postProcessor: x => x.DependsOnCreate = x.DependsOnUpdate = true); 244 | 245 | TryAddField(FieldIDs.Revision.Guid, Guid.NewGuid().ToString("D"), 246 | language: languageVersion.Language, version: languageVersion.Version, 247 | name: "__Revision", postProcessor: x => x.DependsOnCreate = x.DependsOnUpdate = true); 248 | } 249 | } 250 | } 251 | 252 | public string[] GetLanguages() 253 | { 254 | return Fields.OfType().Select(x => x.Language).Distinct().ToArray(); 255 | } 256 | 257 | public string GetParentPath() 258 | { 259 | if (string.IsNullOrWhiteSpace(ItemPath)) return null; 260 | var idx = ItemPath.LastIndexOf("/", StringComparison.OrdinalIgnoreCase); 261 | if (idx < 0 || ItemPath.Length == idx + 1) return null; 262 | return ItemPath.Substring(0, idx); 263 | } 264 | 265 | private class BulkFieldKey 266 | { 267 | public Guid FieldId { get; private set; } 268 | public string Language { get; private set; } 269 | public int? Version { get; private set; } 270 | 271 | public BulkFieldKey(Guid fieldId, string language, int? version) 272 | { 273 | FieldId = fieldId; 274 | Language = language; 275 | Version = version; 276 | } 277 | 278 | public override string ToString() 279 | { 280 | return $"{FieldId} ({Language}#{Version})"; 281 | } 282 | 283 | private bool Equals(BulkFieldKey other) 284 | { 285 | return FieldId.Equals(other.FieldId) && 286 | string.Equals(Language, other.Language, StringComparison.OrdinalIgnoreCase) && 287 | Version == other.Version; 288 | } 289 | 290 | public override bool Equals(object obj) 291 | { 292 | if (ReferenceEquals(null, obj)) return false; 293 | if (ReferenceEquals(this, obj)) return true; 294 | if (obj.GetType() != this.GetType()) return false; 295 | return Equals((BulkFieldKey)obj); 296 | } 297 | 298 | public override int GetHashCode() 299 | { 300 | unchecked 301 | { 302 | return (FieldId.GetHashCode() * 397) ^ 303 | (Language == null ? 0 : StringComparer.OrdinalIgnoreCase.GetHashCode(Language)) ^ 304 | Version.GetHashCode(); 305 | } 306 | } 307 | } 308 | } 309 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/ItemReference.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Sitecore.DataBlaster 4 | { 5 | /// 6 | /// Reference to an item by id and path. 7 | /// 8 | public class ItemReference 9 | { 10 | public Guid ItemId { get; private set; } 11 | public string ItemPath { get; private set; } 12 | 13 | public ItemReference(Guid itemId, string itemPath) 14 | { 15 | if (String.IsNullOrWhiteSpace(itemPath)) throw new ArgumentNullException(nameof(itemPath)); 16 | 17 | ItemId = itemId; 18 | ItemPath = itemPath; 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/BulkLoadAction.cs: -------------------------------------------------------------------------------- 1 | namespace Sitecore.DataBlaster.Load 2 | { 3 | public enum BulkLoadAction 4 | { 5 | /// 6 | /// Adds items and adds missing fields, but doesn't update any fields. 7 | /// 8 | AddOnly = 0, 9 | 10 | /// 11 | /// Only adds items that don't exist yet, does NOT add missing fields to existing items. 12 | /// 13 | AddItemOnly = 6, // Keep original numbering for backwards compatibility. 14 | 15 | /// 16 | /// Adds items, missing fields to existing items and updates/overwrites fields for which the data is different. 17 | /// 18 | Update = 1, 19 | 20 | /// 21 | /// Adds and updates fields for existing items only. 22 | /// 23 | UpdateExistingItem = 2, 24 | 25 | /// 26 | /// Reverts items to the provided state, removing redundant fields as well. 27 | /// Does NOT remove children that are not provided in the dataset. 28 | /// 29 | Revert = 3, 30 | 31 | /// 32 | /// Reverts items to the provided state, removing redundant fields as well. 33 | /// Removes descendants that are not provided in the dataset. 34 | /// 35 | RevertTree = 4 36 | 37 | //Delete (todo) 38 | } 39 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/BulkLoadContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using log4net; 5 | using Sitecore.Buckets.Util; 6 | using Sitecore.ContentSearch; 7 | using Sitecore.Data.Items; 8 | using Sitecore.Diagnostics; 9 | 10 | namespace Sitecore.DataBlaster.Load 11 | { 12 | /// 13 | /// Should be created a fresh for every bulk import action. 14 | /// 15 | public class BulkLoadContext 16 | { 17 | private ILog _log; 18 | 19 | public ILog Log 20 | { 21 | get { return _log; } 22 | set 23 | { 24 | if (value == null) throw new ArgumentNullException(nameof(value)); 25 | _log = value; 26 | } 27 | } 28 | 29 | public string FailureMessage { get; private set; } 30 | 31 | public string Database { get; private set; } 32 | 33 | /// 34 | /// Stages data to temp tables, but don't merge it with existing data. 35 | /// Useful for debugging. 36 | /// 37 | public bool StageDataWithoutProcessing { get; set; } 38 | 39 | /// 40 | /// Whether to lookup item ids in database by item name or use the item id provided in the value of the field. 41 | /// 42 | public bool LookupItemIds { get; set; } 43 | 44 | /// 45 | /// Performance optimization when loading items into a specific repository (supports bucketing). 46 | /// 47 | public ItemReference Destination { get; private set; } 48 | 49 | /// 50 | /// Whether to lookup blob ids in database or use the blob id provided in the value of the field. 51 | /// 52 | public bool LookupBlobIds { get; set; } 53 | 54 | /// 55 | /// Whether bulk load allows template changes. 56 | /// Typically used during serialization. 57 | /// When set, provided bulk items should contain ALL fields and not partial data. 58 | /// 59 | public bool AllowTemplateChanges { get; set; } 60 | 61 | /// 62 | /// In the initial version if Sitecore 9.3, they did not put all field records in the right fields table (versioned/unversioned/shared). 63 | /// E.g. the Display Name is configured as unversioned field on the template, but the actual data is stored in the versioned table. 64 | /// We need this extra flag to be able to disable the cleanup operation during bulk load, else the Display Name values will be removed, resulting in empty context menu's. 65 | /// This flag is typically only relevant during deserialization actions. 66 | /// 67 | public bool AllowCleanupOfFields { get; set; } = true; 68 | 69 | /// 70 | /// Forces updates in Sitecore database, so that all loaded items will have an item change. 71 | /// All modification dates will be reset. 72 | /// 73 | public bool ForceUpdates { get; set; } 74 | 75 | /// 76 | /// Additional processing rules for fiels which will affect all items with those specified fields. 77 | /// 78 | public IList FieldRules { get; set; } 79 | 80 | /// 81 | /// Will ensure bucket folder structure for items that are directly added to a parent that is a bucket. 82 | /// Be aware, this needs to do additional database reads while processing the item stream. 83 | /// 84 | public bool BucketIfNeeded { get; set; } 85 | 86 | /// 87 | /// Resolves the paths for items in buckets. 88 | /// 89 | public IDynamicBucketFolderPath BucketFolderPath { get; set; } 90 | 91 | /// 92 | /// Whether to remove updated items from Sitecore caches. Enabled by default. 93 | /// This setting is not impacted by the value of . 94 | /// 95 | public bool RemoveItemsFromCaches { get; set; } 96 | 97 | /// 98 | /// Offers an alternative strategy to remove items from Sitecore caches, by clearing them completely. 99 | /// This setting is not impacted by the value of . 100 | /// 101 | /// When both the imported data set and the Sitecore caches are quite large, there is a performance impact in scanning the caches for entries that must be deleted. 102 | /// In this case it could prove more useful to just clear the caches, instead of spending time to scan them. The performance impact is then in repopulation, though. 103 | /// For settings that have an impact on cache removal performance, see . 104 | /// 105 | public bool ClearCaches { get; set; } 106 | 107 | /// 108 | /// Whether to update the history engine of Sitecore. This engine is e.g. used for index syncs. 109 | /// 110 | public bool? UpdateHistory { get; set; } 111 | 112 | /// 113 | /// Whether to update the publish queue of Sitecore. This queue is used for incremental publishing. 114 | /// 115 | public bool? UpdatePublishQueue { get; set; } 116 | 117 | /// 118 | /// Whether to update the link database. 119 | /// 120 | public bool? UpdateLinkDatabase { get; set; } 121 | 122 | /// 123 | /// Whether to update the indexes of Sitecore. Enabled by default. 124 | /// 125 | public bool UpdateIndexes { get; set; } 126 | 127 | private IList _allIndexes; 128 | private IList _indexesToUpdate; 129 | 130 | /// 131 | /// Which indexes to update, will be detected from database by default. 132 | /// 133 | public IList IndexesToUpdate 134 | { 135 | get 136 | { 137 | if (_indexesToUpdate != null) 138 | return _indexesToUpdate; 139 | 140 | // No specific indexes provided, fallback to all indexes for the current Database 141 | if (_allIndexes != null) 142 | return _allIndexes; 143 | 144 | _allIndexes = ContentSearchManager.Indexes 145 | .Where(idx => idx.Crawlers 146 | .OfType() 147 | .Any(c => Database.Equals(c.Database, StringComparison.OrdinalIgnoreCase))) 148 | .ToList(); 149 | return _allIndexes; 150 | } 151 | set { _indexesToUpdate = value; } 152 | } 153 | 154 | /// 155 | /// Threshold percentage to refresh destination in index instead of updating it one by one. 156 | /// 157 | public int? IndexRefreshThresholdPercentage { get; set; } 158 | 159 | /// 160 | /// Threshold percentage to rebuild index instead of updating it one by one. 161 | /// 162 | public int? IndexRebuildThresholdPercentage { get; set; } 163 | 164 | /// 165 | /// Data is staged in database but no changes are made yet. 166 | /// 167 | public Action OnDataStaged { get; set; } 168 | 169 | /// 170 | /// Data is loaded in database. 171 | /// 172 | public Action OnDataLoaded { get; set; } 173 | 174 | /// 175 | /// Data is indexed. 176 | /// 177 | public Action> OnDataIndexed { get; set; } 178 | 179 | public LinkedList ItemChanges { get; } = new LinkedList(); 180 | 181 | protected internal BulkLoadContext(string database) 182 | { 183 | if (string.IsNullOrEmpty(database)) throw new ArgumentNullException(nameof(database)); 184 | 185 | Database = database; 186 | RemoveItemsFromCaches = true; 187 | UpdateIndexes = true; 188 | 189 | Log = LoggerFactory.GetLogger(typeof(BulkLoader)); 190 | } 191 | 192 | public void LookupItemsIn(Guid itemId, string itemPath) 193 | { 194 | LookupItemIds = true; 195 | Destination = new ItemReference(itemId, itemPath); 196 | } 197 | 198 | public void LookupItemsIn(Item item) 199 | { 200 | if (item == null) throw new ArgumentNullException(nameof(item)); 201 | LookupItemsIn(item.ID.Guid, item.Paths.Path); 202 | } 203 | 204 | public bool ShouldUpdateIndex(ISearchIndex searchIndex, ISearchIndexSummary searchIndexSummary) 205 | { 206 | // Always update when index has been explicitly set as to update 207 | if (_indexesToUpdate != null && _indexesToUpdate.Contains(searchIndex)) 208 | return true; 209 | 210 | // Only update when index is not empty: updating an empty index would trigger a rebuild. 211 | return searchIndexSummary.NumberOfDocuments > 0; 212 | } 213 | 214 | #region Stage results and feedback 215 | 216 | private readonly Dictionary _stageResults = new Dictionary(); 217 | 218 | public bool AnyStageFailed => _stageResults.Any(x => x.Value.HasFlag(StageResult.Failed)); 219 | 220 | protected virtual void AddStageResult(Stage stage, StageResult result) 221 | { 222 | StageResult r; 223 | r = _stageResults.TryGetValue(stage, out r) 224 | ? r | result 225 | : result; 226 | _stageResults[stage] = r; 227 | } 228 | 229 | public virtual void StageSucceeded(Stage stage) 230 | { 231 | AddStageResult(stage, StageResult.Succeeded); 232 | } 233 | 234 | public virtual void StageFailed(Stage stage, Exception ex, string message) 235 | { 236 | AddStageResult(stage, StageResult.Failed); 237 | 238 | if (ex == null) 239 | Log.Fatal(message); 240 | else 241 | Log.Fatal(message + 242 | $"\nException type: {ex.GetType().Name}\nException message: {ex.Message}\nStack trace: {ex.StackTrace}"); 243 | 244 | FailureMessage = message; 245 | } 246 | 247 | public virtual void StageFailed(Stage stage, string message) 248 | { 249 | StageFailed(stage, null, message); 250 | } 251 | 252 | public virtual void SkipItemWarning(string message) 253 | { 254 | Log.Warn(message + " Skipping item."); 255 | } 256 | 257 | public virtual void SkipItemDebug(string message) 258 | { 259 | Log.Debug(message + " Skipping item."); 260 | } 261 | 262 | #endregion 263 | 264 | #region Tracked item data 265 | 266 | /// 267 | /// Tracks path and template info of the bulk item within the context, 268 | /// so that we can check whether bucketing is still needed, or do lookups by path. 269 | /// Doesn't keep a reference to the item, so that we're not too memory intensive. 270 | /// 271 | /// Item to attach. 272 | public virtual void TrackPathAndTemplateInfo(BulkLoadItem item) 273 | { 274 | // Cache template id per item. 275 | var templateCache = GetTemplateCache(); 276 | templateCache[item.Id] = item.TemplateId; 277 | 278 | // Cache path. 279 | IDictionary pathCache = null; 280 | if (!string.IsNullOrWhiteSpace(item.ItemPath)) 281 | { 282 | pathCache = GetPathCache(); 283 | pathCache[item.ItemPath] = item.Id; 284 | } 285 | 286 | // Cache lookup path. 287 | if (!string.IsNullOrWhiteSpace(item.ItemLookupPath)) 288 | { 289 | pathCache = pathCache ?? GetPathCache(); 290 | pathCache[item.ItemLookupPath] = item.Id; 291 | } 292 | } 293 | 294 | private IDictionary GetPathCache() 295 | { 296 | return GetOrAddState("Transform.PathCache", 297 | () => new Dictionary(StringComparer.OrdinalIgnoreCase)); 298 | } 299 | 300 | private IDictionary GetTemplateCache() 301 | { 302 | return GetOrAddState("Import.TemplateCache", 303 | () => new Dictionary()); 304 | } 305 | 306 | public virtual Guid? GetProcessedPath(string itemPath) 307 | { 308 | if (string.IsNullOrWhiteSpace(itemPath)) return null; 309 | 310 | var cache = GetPathCache(); 311 | Guid id; 312 | return cache.TryGetValue(itemPath, out id) ? id : (Guid?)null; 313 | } 314 | 315 | public virtual Guid? GetProcessedItemTemplateId(Guid itemId) 316 | { 317 | var cache = GetTemplateCache(); 318 | Guid id; 319 | return cache.TryGetValue(itemId, out id) ? id : (Guid?)null; 320 | } 321 | 322 | #endregion 323 | 324 | #region Additional state 325 | 326 | private readonly Dictionary _state = 327 | new Dictionary(StringComparer.OrdinalIgnoreCase); 328 | 329 | /// 330 | /// Gets state from the context. 331 | /// 332 | /// Type of the state. 333 | /// Key for the state. 334 | /// Default value when state is not present. 335 | /// Retrieved state or default value. 336 | /// Not thread safe. 337 | public T GetState(string key, T defaultValue = default(T)) 338 | { 339 | object state; 340 | if (!_state.TryGetValue(key, out state)) 341 | { 342 | return defaultValue; 343 | } 344 | return (T)state; 345 | } 346 | 347 | /// 348 | /// Gets or adds new state to the context. 349 | /// 350 | /// Type of the state. 351 | /// Key for the state. 352 | /// Factory to create new state. 353 | /// Retrieved or newly added state. 354 | /// Not thread safe. 355 | public T GetOrAddState(string key, Func stateFactory) 356 | { 357 | object state; 358 | if (!_state.TryGetValue(key, out state)) 359 | { 360 | state = stateFactory(); 361 | _state[key] = state; 362 | } 363 | return (T)state; 364 | } 365 | 366 | #endregion 367 | } 368 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/BulkLoadItem.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Sitecore.Buckets.Util; 3 | using Sitecore.Data.Items; 4 | using Sitecore.Data.Templates; 5 | 6 | namespace Sitecore.DataBlaster.Load 7 | { 8 | public class BulkLoadItem : BulkItem 9 | { 10 | private string _itemLookupPath; 11 | 12 | /// 13 | /// Load action to perform on this item. 14 | /// 15 | public BulkLoadAction LoadAction { get; set; } 16 | 17 | /// 18 | /// Item path to use for lookups. 19 | /// Can contain /* as wildcard for single item path part. 20 | /// Can contain /** as greedy wildcard for multiple item path parts. 21 | /// Can contain /.. to navigate to parent. 22 | /// E.g.: /sitecore/content/sites/SiteName/Products/**/Aricle U12345/.. 23 | /// 24 | /// To support wildcards, will need the SqlClr data blaster module. 25 | public string ItemLookupPath 26 | { 27 | get { return _itemLookupPath; } 28 | set 29 | { 30 | if (value != null) 31 | { 32 | if (value.Contains("*/*")) 33 | throw new ArgumentException("Path expression cannot contain consecutive wildcards."); 34 | if (value.Contains("*/..")) 35 | throw new ArgumentException( 36 | "Path expression cannot contain parent navigation immediately after wildcard."); 37 | if (value.EndsWith("/*") || value.EndsWith("/**")) 38 | throw new ArgumentException("Path expression cannot end with wildcard."); 39 | } 40 | _itemLookupPath = value; 41 | } 42 | } 43 | 44 | /// 45 | /// Whether to deduplicate this item based on its path, after lookupexpressions have been processed. 46 | /// 47 | public bool Deduplicate { get; set; } 48 | 49 | /// 50 | /// Id of item which must be created for this item to be created as well. 51 | /// When other item is only updated, this entire item will be skipped. 52 | /// 53 | public Guid? DependsOnItemCreation { get; set; } 54 | 55 | /// 56 | /// Whether or not this item has already been bucketed. 57 | /// 58 | public bool Bucketed { get; set; } 59 | 60 | /// 61 | /// Templatename, only used for diagnostics. 62 | /// 63 | public string TemplateName { get; set; } 64 | 65 | /// 66 | /// Diagnostic source information for this item. 67 | /// 68 | public object SourceInfo { get; set; } 69 | 70 | public BulkLoadItem(BulkLoadAction loadAction, Guid id, Guid templateId, Guid masterId, Guid parentId, 71 | string itemPath, string templateName = null, object sourceInfo = null) 72 | : base(id, templateId, masterId, parentId, itemPath) 73 | { 74 | if (templateId == Guid.Empty && loadAction != BulkLoadAction.UpdateExistingItem) 75 | throw new ArgumentException("Template id of item should not be an empty Guid.", nameof(id)); 76 | 77 | this.LoadAction = loadAction; 78 | this.TemplateName = templateName; 79 | this.SourceInfo = sourceInfo; 80 | 81 | // When creating bucket folder items, we don't want those items to be bucketed again. 82 | if (BucketConfigurationSettings.BucketTemplateId.Guid.Equals(templateId)) 83 | Bucketed = true; 84 | } 85 | 86 | public BulkLoadItem(BulkLoadAction loadAction, BulkItem item) 87 | : base(item) 88 | { 89 | this.LoadAction = loadAction; 90 | 91 | // When creating bucket folder items, we don't want those items to be bucketed again. 92 | if (BucketConfigurationSettings.BucketTemplateId.Guid.Equals(TemplateId)) 93 | Bucketed = true; 94 | } 95 | 96 | public BulkLoadItem(BulkLoadAction loadAction, Guid templateId, string itemPath, 97 | string templateName = null, object sourceInfo = null) 98 | : this(loadAction, Guid.NewGuid(), templateId, Guid.Empty, Guid.NewGuid(), 99 | itemPath, templateName, sourceInfo) 100 | { 101 | } 102 | 103 | public BulkLoadItem(BulkLoadAction loadAction, Guid templateId, string itemPath, Guid id, 104 | string templateName = null, object sourceInfo = null) 105 | : this(loadAction, id, templateId, Guid.Empty, Guid.NewGuid(), 106 | itemPath, templateName, sourceInfo) 107 | { 108 | } 109 | 110 | public BulkLoadItem(BulkLoadAction loadAction, Template template, Item parent, string name) 111 | : this(loadAction, Guid.NewGuid(), template.ID.Guid, Guid.Empty, parent.ID.Guid, 112 | parent.Paths.Path + "/" + name, template.Name) 113 | { 114 | } 115 | 116 | /// 117 | /// Method to help chaining. 118 | /// 119 | /// Action to perform. 120 | /// This object. 121 | public BulkLoadItem Do(Action action) 122 | { 123 | if (action == null) throw new ArgumentNullException(nameof(action)); 124 | action(this); 125 | return this; 126 | } 127 | } 128 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/FieldRule.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Sitecore.DataBlaster.Load 4 | { 5 | public class FieldRule 6 | { 7 | public Guid ItemId { get; } 8 | public bool SkipOnCreate { get; set; } 9 | public bool SkipOnUpdate { get; set; } 10 | public bool SkipOnDelete { get; set; } 11 | 12 | public FieldRule(Guid itemId, bool skipOnCreate = false, bool skipOnUpdate = false, bool skipOnDelete = false) 13 | { 14 | ItemId = itemId; 15 | SkipOnCreate = skipOnCreate; 16 | SkipOnUpdate = skipOnUpdate; 17 | SkipOnDelete = skipOnDelete; 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/ItemChange.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data; 3 | using System.Linq; 4 | using Sitecore.DataBlaster.Read; 5 | 6 | namespace Sitecore.DataBlaster.Load 7 | { 8 | public class ItemChange 9 | { 10 | private readonly ItemVersionHeader _itemVersionHeader; 11 | public string ItemPath { get; private set; } 12 | public int ItemPathLevel { get; private set; } 13 | public Guid ItemId { get; private set; } 14 | public Guid OriginalItemId { get; private set; } 15 | public Guid TemplateId { get; private set; } 16 | public Guid ParentId { get; private set; } 17 | public Guid OriginalParentId { get; private set; } 18 | public string Language { get; private set; } 19 | public int? Version { get; private set; } 20 | public bool Created { get; private set; } 21 | public bool Saved { get; private set; } 22 | public bool Moved { get; private set; } 23 | public bool Deleted { get; private set; } 24 | public object SourceInfo { get; private set; } 25 | 26 | public ItemChange(IDataRecord record) 27 | { 28 | ItemPath = record.IsDBNull(0) ? null : record.GetString(0); 29 | ItemPathLevel = ItemPath == null ? 0 : ItemPath.Count(c => c == '/'); 30 | ItemId = record.GetGuid(1); 31 | OriginalItemId = record.IsDBNull(2) ? ItemId : record.GetGuid(2); 32 | TemplateId = record.GetGuid(3); 33 | ParentId = record.GetGuid(4); 34 | OriginalParentId = record.GetGuid(5); 35 | Language = record.IsDBNull(6) ? null : record.GetString(6); 36 | Version = record.IsDBNull(7) ? (int?)null : record.GetInt32(7); 37 | Created = record.IsDBNull(8) ? false : record.GetBoolean(8); 38 | Saved = record.IsDBNull(9) ? false : record.GetBoolean(9); 39 | Moved = record.IsDBNull(10) ? false : record.GetBoolean(10); 40 | Deleted = record.IsDBNull(11) ? false : record.GetBoolean(11); 41 | SourceInfo = record.IsDBNull(12) ? null : record.GetValue(12); 42 | } 43 | 44 | public ItemChange(ItemVersionHeader itemVersionHeader, 45 | bool created = false, bool saved = false, bool moved = false, bool deleted = false) 46 | { 47 | _itemVersionHeader = itemVersionHeader; 48 | ItemPath = itemVersionHeader.ItemPath; 49 | ItemPathLevel = itemVersionHeader.ItemPath.Count(c => c == '/'); 50 | ItemId = itemVersionHeader.Id; 51 | OriginalItemId = itemVersionHeader.Id; 52 | TemplateId = itemVersionHeader.TemplateId; 53 | ParentId = itemVersionHeader.ParentId; 54 | OriginalParentId = itemVersionHeader.ParentId; 55 | Language = itemVersionHeader.Language; 56 | Version = itemVersionHeader.Version; 57 | Created = created; 58 | Saved = saved; 59 | Moved = moved; 60 | Deleted = deleted; 61 | SourceInfo = null; 62 | } 63 | } 64 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Links/BulkItemLink.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Sitecore.Data; 3 | using Sitecore.Links; 4 | 5 | namespace Sitecore.DataBlaster.Load.Links 6 | { 7 | public class BulkItemLink : ItemLink 8 | { 9 | public BulkLoadAction ItemAction { get; set; } 10 | 11 | public BulkItemLink(string sourceDatabase, ID sourceItemID, ID sourceFieldID, 12 | string targetDatabase, ID targetItemID, string targetPath) 13 | : base(sourceDatabase, sourceItemID, sourceFieldID, targetDatabase, targetItemID, targetPath) 14 | { 15 | } 16 | 17 | public BulkItemLink Map(Guid originalId, Guid newId) 18 | { 19 | if (SourceItemID.Guid == originalId) 20 | return new BulkItemLink(SourceDatabaseName, new ID(newId), SourceFieldID, 21 | TargetDatabaseName, TargetItemID, TargetPath); 22 | 23 | if (TargetItemID.Guid == originalId) 24 | return new BulkItemLink(SourceDatabaseName, SourceItemID, SourceFieldID, 25 | TargetDatabaseName, new ID(newId), TargetPath); 26 | 27 | return this; 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Links/BulkItemLinkParser.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | using Sitecore.Data; 6 | 7 | namespace Sitecore.DataBlaster.Load.Links 8 | { 9 | /// 10 | /// Parses out links in content of bulk items. 11 | /// 12 | public class BulkItemLinkParser 13 | { 14 | protected Lazy IdRegex = new Lazy(() => 15 | new Regex(@"\{[0-9A-Z]{8}-[0-9A-Z]{4}-[0-9A-Z]{4}-[0-9A-Z]{4}-[0-9A-Z]{12}\}", 16 | RegexOptions.Compiled | RegexOptions.IgnoreCase)); 17 | 18 | protected Lazy LinkRegex = new Lazy(() => 19 | new Regex(@"(?<=~/link\.aspx\?_id=)[0-9A-Z]{32}(?=&)", 20 | RegexOptions.Compiled | RegexOptions.IgnoreCase)); 21 | 22 | protected Lazy> IgnoredFields = new Lazy>(() => 23 | new HashSet(new[] 24 | { 25 | "__Revision" 26 | }, StringComparer.OrdinalIgnoreCase)); 27 | 28 | public virtual IEnumerable ExtractLinks(IEnumerable bulkItems, 29 | BulkLoadContext context, 30 | LinkedList links) 31 | { 32 | foreach (var item in bulkItems) 33 | { 34 | ExtractLinks(context, item, links); 35 | yield return item; 36 | } 37 | } 38 | 39 | protected virtual void ExtractLinks(BulkLoadContext context, BulkLoadItem item, LinkedList links) 40 | { 41 | foreach (var field in item.Fields) 42 | { 43 | ExtractLinks(context, item, field, links); 44 | } 45 | } 46 | 47 | protected virtual void ExtractLinks(BulkLoadContext context, BulkLoadItem item, BulkField field, 48 | LinkedList links) 49 | { 50 | if (string.IsNullOrWhiteSpace(field.Value)) return; 51 | if (IgnoredFields.Value.Contains(field.Name)) return; 52 | 53 | var ids = IdRegex.Value.Matches(field.Value).Cast() 54 | .Select(x => 55 | { 56 | ID id; 57 | return ID.TryParse(x.Value, out id) ? id : (ID)null; 58 | }) 59 | .Concat(LinkRegex.Value.Matches(field.Value).Cast().Select(x => 60 | { 61 | Guid guid; 62 | return Guid.TryParse(x.Value, out guid) ? new ID(guid) : (ID)null; 63 | })) 64 | .Where(x => x != (ID)null); 65 | 66 | foreach (var link in ids 67 | .Select(x => new BulkItemLink( 68 | context.Database, new ID(item.Id), new ID(field.Id), 69 | context.Database, x, x.ToString()))) 70 | { 71 | link.ItemAction = item.LoadAction; 72 | links.AddLast(link); 73 | } 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Paths/AncestorGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Sitecore.Data.Templates; 5 | using Sitecore.DataBlaster.Util; 6 | 7 | namespace Sitecore.DataBlaster.Load.Paths 8 | { 9 | /// 10 | /// Allows generating ancestor in a stream of bulk items. 11 | /// 12 | public class AncestorGenerator 13 | { 14 | private readonly GuidUtility _guidUtility; 15 | 16 | public AncestorGenerator(GuidUtility guidUtility = null) 17 | { 18 | _guidUtility = guidUtility ?? new GuidUtility(); 19 | } 20 | 21 | /// 22 | /// Returns ancestor items for every needed level between the item and the root. 23 | /// 24 | /// Item to generate ancestors for. 25 | /// Root that marks the ancestor that should already exist or be present in the stream. 26 | /// Template for the ancestors to generate. 27 | /// Context to generate the ancestors in. 28 | /// Stream of generated ancestors, migh be empty if ancestors were already created, 29 | /// or it might be partial if shared ancestors were already created for another item in the same context. 30 | public virtual IEnumerable EnsureAncestorBulkItems(BulkLoadItem item, ItemReference root, 31 | Template ancestorTemplate, BulkLoadContext context) 32 | { 33 | return EnsureAncestorBulkItems(item, root, ancestorTemplate, item.Id, context); 34 | } 35 | 36 | protected virtual IEnumerable EnsureAncestorBulkItems(BulkLoadItem item, 37 | ItemReference root, Template ancestorTemplate, Guid dependsOnItemCreation, 38 | BulkLoadContext context) 39 | { 40 | if (item == null) throw new ArgumentNullException(nameof(item)); 41 | if (root == null) throw new ArgumentNullException(nameof(root)); 42 | if (ancestorTemplate == null) throw new ArgumentNullException(nameof(ancestorTemplate)); 43 | if (context == null) throw new ArgumentNullException(nameof(context)); 44 | if (!item.ItemPath.StartsWith(root.ItemPath)) 45 | throw new ArgumentException("Bulk item should be a descendant of the root."); 46 | 47 | // Detect all the ancestors to generate. 48 | var ancestorNames = item.ItemPath.Substring(root.ItemPath.Length) 49 | .Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); 50 | ancestorNames = ancestorNames.Take(ancestorNames.Length - 1).ToArray(); // Don't include item. 51 | 52 | // Generate all ancestors. 53 | var parent = root; 54 | foreach (var name in ancestorNames) 55 | { 56 | var itemPath = $"{parent.ItemPath}/{name}"; 57 | 58 | // Maybe we have already generated this path in a previous call to EnsureAncestors within the same context. 59 | var itemId = context.GetProcessedPath(itemPath); 60 | if (itemId.HasValue) 61 | { 62 | // Continue with next. 63 | parent = new ItemReference(itemId.Value, itemPath); 64 | continue; 65 | } 66 | 67 | // Generate stable guid for this ancestor item within its parent. 68 | itemId = _guidUtility.Create(parent.ItemId, name); 69 | 70 | // In case of forced updates, also update the child item. This will result in an ItemChange which makes sure the item gets re-published. 71 | var childLoadAction = context.ForceUpdates ? BulkLoadAction.Update : BulkLoadAction.AddOnly; 72 | 73 | // Create new bulk item. 74 | var child = new BulkLoadItem(childLoadAction, itemId.Value, ancestorTemplate.ID.Guid, 75 | Guid.Empty, parent.ItemId, itemPath, 76 | templateName: ancestorTemplate.Name, sourceInfo: item.SourceInfo) 77 | { 78 | // Only create ancestor when child is created, skip ancestor creation when child is updated. 79 | DependsOnItemCreation = dependsOnItemCreation 80 | }; 81 | 82 | // Attach asap to context, because import profiles might eagerly bucket. 83 | context.TrackPathAndTemplateInfo(child); 84 | 85 | yield return child; 86 | 87 | parent = new ItemReference(child.Id, child.ItemPath); 88 | } 89 | 90 | // Reset parent reference for initial item. 91 | item.ParentId = parent.ItemId; 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ChangeCacheClearer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using Sitecore.Configuration; 6 | using Sitecore.Data; 7 | using Sitecore.DataBlaster.Load.Sql; 8 | using Sitecore.DataBlaster.Util; 9 | 10 | namespace Sitecore.DataBlaster.Load.Processors 11 | { 12 | public class ChangeCacheClearer : IChangeProcessor 13 | { 14 | private readonly CacheUtil _cachUtil; 15 | 16 | public ChangeCacheClearer(CacheUtil cachehUtil = null) 17 | { 18 | _cachUtil = cachehUtil ?? new CacheUtil(); 19 | } 20 | 21 | public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection changes) 22 | { 23 | if (!loadContext.RemoveItemsFromCaches && !loadContext.ClearCaches) return; 24 | 25 | var stopwatch = Stopwatch.StartNew(); 26 | 27 | // Remove items from database cache. 28 | // We don't do this within the transaction so that items will be re-read from the committed data. 29 | var db = Factory.GetDatabase(loadContext.Database, true); 30 | if (loadContext.ClearCaches) 31 | { 32 | _cachUtil.ClearCaches(db); 33 | loadContext.Log.Info($"Caches cleared: {(int)stopwatch.Elapsed.TotalSeconds}s"); 34 | } 35 | else 36 | { 37 | _cachUtil.RemoveItemsFromCachesInBulk(db, GetCacheClearEntries(loadContext.ItemChanges)); 38 | loadContext.Log.Info($"Items removed from cache: {(int)stopwatch.Elapsed.TotalSeconds}s"); 39 | } 40 | } 41 | 42 | protected virtual IEnumerable> GetCacheClearEntries(IEnumerable itemChanges) 43 | { 44 | if (itemChanges == null) yield break; 45 | 46 | // Since we don't include the language in the cache clear entries, 47 | // we should de-duplicate the ItemChanges by language first to avoid duplicate cache clear entries. 48 | // Result should be one cache-clear entry per Item, not per ItemChange. 49 | var seenKeys = new HashSet(); 50 | var filteredItemChanges = 51 | itemChanges.Where(x => seenKeys.Add($"{x.ItemId}{x.ParentId}{x.OriginalParentId}{x.ItemPath}")); 52 | 53 | foreach (var itemChange in filteredItemChanges) 54 | { 55 | yield return new Tuple(ID.Parse(itemChange.ItemId), ID.Parse(itemChange.ParentId), 56 | itemChange.ItemPath); 57 | 58 | // Support moved items. 59 | if (itemChange.ParentId != itemChange.OriginalParentId) 60 | yield return new Tuple(ID.Parse(itemChange.ItemId), 61 | ID.Parse(itemChange.OriginalParentId), itemChange.ItemPath); 62 | } 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ChangeIndexer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using Sitecore.Abstractions; 6 | using Sitecore.Configuration; 7 | using Sitecore.ContentSearch; 8 | using Sitecore.ContentSearch.Maintenance; 9 | using Sitecore.Data; 10 | using Sitecore.Data.Managers; 11 | using Sitecore.DataBlaster.Load.Sql; 12 | using Sitecore.DataBlaster.Util; 13 | using Sitecore.Events; 14 | using Sitecore.Globalization; 15 | 16 | namespace Sitecore.DataBlaster.Load.Processors 17 | { 18 | public class ChangeIndexer : IChangeProcessor 19 | { 20 | private static readonly Guid BucketFolderTemplate = Guid.Parse(Sitecore.Buckets.Util.Constants.BucketFolder); 21 | 22 | public virtual void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, 23 | ICollection changes) 24 | { 25 | if (!loadContext.UpdateIndexes) return; 26 | if (loadContext.IndexesToUpdate == null || loadContext.IndexesToUpdate.Count == 0) return; 27 | 28 | // We don't index bucket folders. 29 | changes = changes?.Where(x => x.TemplateId != BucketFolderTemplate)?.ToList(); 30 | if (changes == null || changes.Count == 0) return; 31 | 32 | var stopwatch = Stopwatch.StartNew(); 33 | 34 | UpdateIndexes(loadContext, changes); 35 | loadContext.Log.Info($"Updated content search indexes: {(int)stopwatch.Elapsed.TotalSeconds}s"); 36 | 37 | loadContext.OnDataIndexed?.Invoke(loadContext, changes); 38 | Event.RaiseEvent("bulkloader:dataindexed", loadContext); 39 | } 40 | 41 | protected virtual void UpdateIndexes(BulkLoadContext context, ICollection itemChanges) 42 | { 43 | var db = Factory.GetDatabase(context.Database, true); 44 | 45 | foreach (var index in context.IndexesToUpdate) 46 | { 47 | UpdateIndex(context, itemChanges, db, index); 48 | } 49 | } 50 | 51 | protected virtual void UpdateIndex(BulkLoadContext context, ICollection itemChanges, 52 | Database database, ISearchIndex index) 53 | { 54 | BaseJob job = null; 55 | 56 | var indexSummary = index.RequestSummary(); 57 | if (indexSummary == null) 58 | { 59 | context.Log.Warn($"Skipping updating index '{index.Name}' because we could not get its summary."); 60 | return; 61 | } 62 | if (!context.ShouldUpdateIndex(index, indexSummary)) 63 | { 64 | context.Log.Warn($"Skipping updating index '{index.Name}' because it's empty."); 65 | return; 66 | } 67 | 68 | var touchedPercentage = 69 | (uint)Math.Ceiling((double)itemChanges.Count / Math.Max(1, indexSummary.NumberOfDocuments) * 100); 70 | if (context.IndexRebuildThresholdPercentage.HasValue 71 | && touchedPercentage > context.IndexRebuildThresholdPercentage.Value) 72 | { 73 | context.Log.Info($"Rebuilding index '{index.Name}' because {touchedPercentage}% is changed."); 74 | job = IndexCustodian.FullRebuild(index); 75 | } 76 | else if (context.Destination != null 77 | && !itemChanges.Any(ic => ic.Deleted) // Refresh doesn't do deletes. 78 | && context.IndexRefreshThresholdPercentage.HasValue 79 | && touchedPercentage > context.IndexRefreshThresholdPercentage.Value) 80 | { 81 | context.Log.Info( 82 | $"Refreshing index '{index.Name}' from '{context.Destination.ItemPath}' because {touchedPercentage}% is changed."); 83 | job = IndexCustodian.Refresh(index, 84 | new SitecoreIndexableItem(database.GetItem(new ID(context.Destination.ItemId)))); 85 | } 86 | else 87 | { 88 | var sitecoreIds = GetItemsToIndex(itemChanges, database); 89 | context.Log.Info($"Updating index '{index.Name}' with {sitecoreIds.Count} items."); 90 | job = IndexCustodian.IncrementalUpdate(index, sitecoreIds); 91 | } 92 | job.Wait(); 93 | } 94 | 95 | protected virtual IList GetItemsToIndex(ICollection itemChanges, Database db) 96 | { 97 | var identifiers = new List(itemChanges.Count); 98 | 99 | // The SitecoreItemCrawler has a bad habit of updating *all* languages of an item 100 | // when asked to index a single language. However, it will not take care de-indexing 101 | // languages for which the item does not exist (any more). For those, we need to send 102 | // the exact identifier for the version that needs to be removed from the index. 103 | // See: Crawler.Update() 104 | // SitecoreItemCrawler.DoUpdate() 105 | 106 | // So, the strategy here is to separate index 'update' from 'remove' requests. 107 | // For deletes, *all* item changes need to be registered. 108 | // for upserts, it is sufficient to supply one existing entry. 109 | 110 | identifiers.AddRange(itemChanges 111 | .Where(ic => ic.Deleted) 112 | .Select(ic => 113 | { 114 | var language = !string.IsNullOrEmpty(ic.Language) 115 | ? Language.Parse(ic.Language) 116 | : LanguageManager.DefaultLanguage; 117 | var version = ic.Version.HasValue 118 | ? Sitecore.Data.Version.Parse(ic.Version.Value) 119 | : Sitecore.Data.Version.First; 120 | return new SitecoreItemUniqueId(new ItemUri(new ID(ic.ItemId), language, version, db)); 121 | })); 122 | 123 | identifiers.AddRange(itemChanges 124 | .Where(ic => !ic.Deleted) 125 | .GroupBy(x => x.ItemId) 126 | .Select(g => 127 | { 128 | // Find an entry whose language and version is not empty/unknown (= look for versioned field changes). 129 | var ic = g.FirstOrDefault(x => !string.IsNullOrEmpty(x.Language) && x.Version.HasValue) 130 | ?? g.FirstOrDefault(x => !string.IsNullOrEmpty(x.Language)); 131 | 132 | var language = ic != null ? Language.Parse(ic.Language) : LanguageManager.DefaultLanguage; 133 | var version = ic != null && ic.Version.HasValue 134 | ? Sitecore.Data.Version.Parse(ic.Version.Value) 135 | : Sitecore.Data.Version.First; // Take first (1), not latest (0). 136 | 137 | return new SitecoreItemUniqueId(new ItemUri(new ID(g.Key), language, version, db)); 138 | })); 139 | 140 | return identifiers; 141 | } 142 | } 143 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ChangeLogger.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using log4net.spi; 4 | using Sitecore.DataBlaster.Load.Sql; 5 | using Sitecore.DataBlaster.Util; 6 | 7 | namespace Sitecore.DataBlaster.Load.Processors 8 | { 9 | public class ChangeLogger : IChangeProcessor 10 | { 11 | public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection changes) 12 | { 13 | loadContext.Log.Info( 14 | $"Item changes in database: created: {loadContext.ItemChanges.Count(ic => ic.Created)}, " + 15 | $"saved: {loadContext.ItemChanges.Count(ic => ic.Saved)}, moved: {loadContext.ItemChanges.Count(ic => ic.Moved)}, " + 16 | $"deleted: {loadContext.ItemChanges.Count(ic => ic.Deleted)}"); 17 | 18 | if (!loadContext.Log.Logger.IsEnabledFor(Level.TRACE)) return; 19 | 20 | foreach (var change in loadContext.ItemChanges) 21 | { 22 | if (change.Created) 23 | loadContext.Log.Trace( 24 | $"Created item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + 25 | $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); 26 | if (change.Saved & !change.Created) 27 | loadContext.Log.Trace( 28 | $"Saved item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + 29 | $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); 30 | if (change.Moved) 31 | loadContext.Log.Trace( 32 | $"Moved item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + 33 | $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); 34 | if (change.Deleted) 35 | loadContext.Log.Trace( 36 | $"Deleted item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + 37 | $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); 38 | } 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/IChangeProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Sitecore.DataBlaster.Load.Sql; 3 | 4 | namespace Sitecore.DataBlaster.Load.Processors 5 | { 6 | public interface IChangeProcessor 7 | { 8 | void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection changes); 9 | } 10 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/IItemProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Sitecore.DataBlaster.Load.Processors 4 | { 5 | public interface IItemProcessor 6 | { 7 | IEnumerable Process(BulkLoadContext context, IEnumerable items); 8 | } 9 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ISyncInTransaction.cs: -------------------------------------------------------------------------------- 1 | using Sitecore.DataBlaster.Load.Sql; 2 | 3 | namespace Sitecore.DataBlaster.Load.Processors 4 | { 5 | public interface ISyncInTransaction 6 | { 7 | void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext); 8 | } 9 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/IValidateStagedData.cs: -------------------------------------------------------------------------------- 1 | using Sitecore.DataBlaster.Load.Sql; 2 | 3 | namespace Sitecore.DataBlaster.Load.Processors 4 | { 5 | public interface IValidateStagedData 6 | { 7 | bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext); 8 | } 9 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ItemBucketer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Sitecore.Buckets.Extensions; 5 | using Sitecore.Buckets.Util; 6 | using Sitecore.Configuration; 7 | using Sitecore.Data; 8 | using Sitecore.Data.Managers; 9 | using Sitecore.DataBlaster.Load.Paths; 10 | 11 | namespace Sitecore.DataBlaster.Load.Processors 12 | { 13 | /// 14 | /// Generates bucket structure for bulk items. 15 | /// Assumes that provided bulk items are supplied as direct children of the bucket item. 16 | /// 17 | public class ItemBucketer : IItemProcessor 18 | { 19 | private static readonly Guid BucketFolderTemplate = Guid.Parse(Sitecore.Buckets.Util.Constants.BucketFolder); 20 | 21 | private readonly AncestorGenerator _ancestorGenerator; 22 | 23 | public ItemBucketer(AncestorGenerator ancestorGenerator = null) 24 | { 25 | _ancestorGenerator = ancestorGenerator ?? new AncestorGenerator(); 26 | } 27 | 28 | public IEnumerable Process(BulkLoadContext context, IEnumerable items) 29 | { 30 | return !context.BucketIfNeeded 31 | ? items 32 | : items.SelectMany(item => Bucket(item, context, true)); 33 | } 34 | 35 | protected virtual IEnumerable Bucket(BulkLoadItem item, BulkLoadContext context, 36 | bool skipIfNotBucket) 37 | { 38 | if (context == null) throw new ArgumentNullException(nameof(context)); 39 | if (item == null) throw new ArgumentNullException(nameof(item)); 40 | 41 | if (item.Bucketed) 42 | { 43 | yield return item; 44 | yield break; 45 | } 46 | 47 | var db = Factory.GetDatabase(context.Database); 48 | var bucketItem = db.GetItem(new ID(item.ParentId)); 49 | if (bucketItem == null && !skipIfNotBucket) 50 | throw new ArgumentException( 51 | $"Unable to bucket item because parent with id '{item.ParentId}' doesn't exist."); 52 | if (bucketItem == null) 53 | { 54 | yield return item; 55 | yield break; 56 | } 57 | 58 | if (!bucketItem.IsABucket()) 59 | { 60 | if (skipIfNotBucket) 61 | { 62 | yield return item; 63 | yield break; 64 | } 65 | throw new InvalidOperationException( 66 | $"Item with path '{bucketItem.Paths.Path}' is not bucket."); 67 | } 68 | 69 | // Get template for ancestors. 70 | var bucketFolderTemplate = TemplateManager.GetTemplate(new ID(BucketFolderTemplate), db); 71 | 72 | // Default to configured bucket folder generation. 73 | if (context.BucketFolderPath == null) 74 | context.BucketFolderPath = new BucketFolderPathResolver(); 75 | 76 | // Try to find out when item was created. 77 | var createdField = item.Fields.FirstOrDefault( 78 | x => x.Id == FieldIDs.Created.Guid && !string.IsNullOrWhiteSpace(x.Value)); 79 | var created = createdField == null ? DateTime.UtcNow : DateUtil.IsoDateToDateTime(createdField.Value); 80 | 81 | var bucketFolderPath = context.BucketFolderPath.GetFolderPath(db, 82 | item.Name.Replace(' ', '0'), // Sitecore's name based bucket folder generation doesn't handle spaces. 83 | new ID(item.TemplateId), new ID(item.Id), bucketItem.ID, created); 84 | item.ItemPath = bucketItem.Paths.Path + "/" + bucketFolderPath + "/" + item.Name; 85 | item.Bucketed = true; 86 | 87 | // Check if bucket folder path depends on creation date. 88 | if (context.GetState("Load.BucketByDate") == null) 89 | { 90 | created = new DateTime(2000, 01, 01, 01, 01, 01); 91 | var testBucketFolderPath = context.BucketFolderPath.GetFolderPath(db, 92 | item.Name, new ID(item.TemplateId), new ID(item.Id), bucketItem.ID, created); 93 | if (!bucketFolderPath.Equals(testBucketFolderPath, StringComparison.OrdinalIgnoreCase)) 94 | { 95 | context.Log.Warn( 96 | "Bucket strategy is based on creation date, this will affect import performance, " + 97 | "but might also not be repeatable."); 98 | context.GetOrAddState("Load.BucketByDate", () => true); 99 | } 100 | } 101 | 102 | // If bucketing depends on date, lookup item by name with wildcard. 103 | var dependsOnDate = context.GetState("Load.BucketsByDate", false); 104 | if (dependsOnDate.GetValueOrDefault(false)) 105 | { 106 | context.LookupItemIds = true; 107 | item.ItemLookupPath = bucketItem.Paths.Path + "/**/" + item.Name; 108 | } 109 | 110 | foreach (var ancestor in _ancestorGenerator.EnsureAncestorBulkItems(item, 111 | new ItemReference(bucketItem.ID.Guid, bucketItem.Paths.Path), bucketFolderTemplate, context)) 112 | { 113 | // Make sure ancestor doesn't get re-bucketed. 114 | ancestor.Bucketed = true; 115 | yield return ancestor; 116 | } 117 | 118 | yield return item; 119 | } 120 | } 121 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ItemLinker.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Data.SqlClient; 4 | using System.Diagnostics; 5 | using System.Diagnostics.CodeAnalysis; 6 | using System.Linq; 7 | using Sitecore.Configuration; 8 | using Sitecore.Data.SqlServer; 9 | using Sitecore.DataBlaster.Load.Links; 10 | using Sitecore.DataBlaster.Load.Sql; 11 | using Sitecore.DataBlaster.Util.Sql; 12 | 13 | namespace Sitecore.DataBlaster.Load.Processors 14 | { 15 | public class ItemLinker : IItemProcessor, IChangeProcessor 16 | { 17 | private readonly BulkItemLinkParser _itemLinkParser; 18 | 19 | public ItemLinker(BulkItemLinkParser itemLinkParser = null) 20 | { 21 | _itemLinkParser = itemLinkParser ?? new BulkItemLinkParser(); 22 | } 23 | 24 | public IEnumerable Process(BulkLoadContext context, IEnumerable items) 25 | { 26 | if (!context.UpdateLinkDatabase.GetValueOrDefault()) return items; 27 | 28 | var links = GetItemLinksFromContext(context); 29 | return _itemLinkParser.ExtractLinks(items, context, links); 30 | } 31 | 32 | public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection changes) 33 | { 34 | if (!loadContext.UpdateLinkDatabase.GetValueOrDefault()) return; 35 | if (changes.Count == 0) return; 36 | 37 | var stopwatch = Stopwatch.StartNew(); 38 | 39 | // Update link database, is in core database, so we can't do this within the transaction. 40 | // Links are detected when reading the bulk item stream, 41 | // so we assume that the same set will be presented again after a crash. 42 | UpdateLinkDatabase(loadContext, sqlContext, GetItemLinksFromContext(loadContext), changes); 43 | 44 | loadContext.Log.Info($"Updated link database: {(int)stopwatch.Elapsed.TotalSeconds}s"); 45 | } 46 | 47 | protected virtual LinkedList GetItemLinksFromContext(BulkLoadContext context) 48 | { 49 | return context.GetOrAddState("Load.ExtractedLinks", () => new LinkedList()); 50 | } 51 | 52 | protected virtual void UpdateLinkDatabase(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, 53 | LinkedList links, ICollection changes) 54 | { 55 | // Index all items that were actually changed in db. 56 | var touchedItemsMap = changes 57 | .GroupBy(x => x.ItemId) 58 | .ToDictionary(x => x.First().OriginalItemId, x => x.First().ItemId); 59 | 60 | // Get links and filter and map them by touched items. 61 | if (links.Count == 0) return; 62 | var linksForTouchedItems = links 63 | .Where(x => touchedItemsMap.ContainsKey(x.SourceItemID.Guid)) 64 | .Select(x => x.Map(x.SourceItemID.Guid, touchedItemsMap[x.SourceItemID.Guid])); 65 | 66 | // Create temp table, bulk load temp table, merge records in link database. 67 | // The link database is probably not the same physical db as the one were loading data to. 68 | var createLinkTempTable = sqlContext.GetEmbeddedSql(loadContext, "Sql.20.CreateLinkTempTable.sql"); 69 | var mergeLinkData = sqlContext.GetEmbeddedSql(loadContext, "Sql.21.MergeLinkTempData.sql"); 70 | 71 | var connectionString = ((SqlServerLinkDatabase)Factory.GetLinkDatabase()).ConnectionString; 72 | using (var conn = new SqlConnection(connectionString)) 73 | { 74 | conn.Open(); 75 | var linkSqlContext = new SqlContext(conn); 76 | 77 | linkSqlContext.ExecuteSql(createLinkTempTable); 78 | 79 | using (var bulkCopy = new SqlBulkCopy(conn)) 80 | { 81 | bulkCopy.BulkCopyTimeout = int.MaxValue; 82 | bulkCopy.EnableStreaming = true; 83 | bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#ItemLinks"); 84 | 85 | try 86 | { 87 | bulkCopy.WriteToServer(new ItemLinksReader(() => linksForTouchedItems.GetEnumerator())); 88 | } 89 | catch (Exception exception) 90 | { 91 | loadContext.StageFailed(Stage.Load, exception, 92 | $"Write to #ItemLinks failed with message: {exception.Message}"); 93 | return; 94 | } 95 | } 96 | linkSqlContext.ExecuteSql(mergeLinkData); 97 | } 98 | } 99 | 100 | private class ItemLinksReader : AbstractEnumeratorReader 101 | { 102 | private readonly object[] _fields; 103 | 104 | public override int FieldCount => 11; 105 | 106 | [SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors", Justification = 107 | "Not an issue in this case.")] 108 | public ItemLinksReader(Func> enumerator) 109 | : base(enumerator) 110 | { 111 | _fields = new object[FieldCount]; 112 | } 113 | 114 | public override bool Read() 115 | { 116 | if (!base.Read()) 117 | { 118 | // Clear the fields. 119 | for (var i = 0; i < _fields.Length; i++) 120 | _fields[i] = null; 121 | return false; 122 | } 123 | 124 | _fields[0] = Current.SourceDatabaseName; 125 | _fields[1] = Current.SourceItemID.Guid; 126 | _fields[2] = (object)Current.SourceItemLanguage?.Name ?? DBNull.Value; 127 | _fields[3] = (object)Current.SourceItemVersion?.Number ?? DBNull.Value; 128 | _fields[4] = Current.SourceFieldID.Guid; 129 | 130 | _fields[5] = Current.TargetDatabaseName; 131 | _fields[6] = Current.TargetItemID.Guid; 132 | _fields[7] = (object)Current.TargetItemLanguage?.Name ?? DBNull.Value; 133 | _fields[8] = (object)Current.TargetItemVersion?.Number ?? DBNull.Value; 134 | _fields[9] = (object)Current.TargetPath ?? DBNull.Value; 135 | 136 | _fields[10] = Current.ItemAction.ToString(); 137 | 138 | return true; 139 | } 140 | 141 | public override object GetValue(int i) 142 | { 143 | return _fields[i]; 144 | } 145 | } 146 | } 147 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ItemValidator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Sitecore.DataBlaster.Load.Sql; 4 | 5 | namespace Sitecore.DataBlaster.Load.Processors 6 | { 7 | public class ItemValidator : IItemProcessor, IValidateStagedData 8 | { 9 | private bool HasItemsWithoutFields { get; set; } 10 | private bool HasItemsWithoutPaths { get; set; } 11 | 12 | public IEnumerable Process(BulkLoadContext context, IEnumerable items) 13 | { 14 | // We don't support items without fields. 15 | items = items.Where(x => 16 | { 17 | if (x.FieldCount != 0) return true; 18 | context.SkipItemWarning( 19 | $"Item with id '{x.Id}', item path '{x.ItemPath}' and source info '{x.SourceInfo}' has no fields."); 20 | HasItemsWithoutFields = true; 21 | return false; 22 | }); 23 | 24 | // Item path must be available when item ids need to be looked up. 25 | items = items.Where(x => 26 | { 27 | if (!context.LookupItemIds) return true; 28 | if (!string.IsNullOrWhiteSpace(x.ItemPath)) return true; 29 | context.SkipItemWarning($"Item with id '{x.Id}', and source info '{x.SourceInfo}' has no item path."); 30 | HasItemsWithoutPaths = true; 31 | return false; 32 | }); 33 | 34 | return items; 35 | } 36 | 37 | public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) 38 | { 39 | if (HasItemsWithoutFields) 40 | { 41 | loadContext.StageFailed(Stage.Load, "Items without fields were found ."); 42 | return false; 43 | } 44 | 45 | if (HasItemsWithoutPaths) 46 | { 47 | loadContext.StageFailed(Stage.Load, "Items were found without paths."); 48 | return false; 49 | } 50 | 51 | return true; 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ItemVersionEnsurer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Sitecore.DataBlaster.Load.Processors 4 | { 5 | public class ItemVersionEnsurer : IItemProcessor 6 | { 7 | public IEnumerable Process(BulkLoadContext context, IEnumerable items) 8 | { 9 | // Statistic fields need to be present to represent the item versions. 10 | foreach (var item in items) 11 | { 12 | item.EnsureLanguageVersions(forceUpdate: context.ForceUpdates); 13 | yield return item; 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/SyncHistoryTable.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using Sitecore.Configuration; 3 | using Sitecore.DataBlaster.Load.Sql; 4 | 5 | namespace Sitecore.DataBlaster.Load.Processors 6 | { 7 | public class SyncHistoryTable : ISyncInTransaction 8 | { 9 | public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) 10 | { 11 | if (!loadContext.UpdateHistory.GetValueOrDefault()) return; 12 | if (loadContext.ItemChanges.Count == 0) return; 13 | 14 | // In Sitecore 9, history engine is disabled by default 15 | if (!HistoryEngineEnabled(loadContext)) 16 | { 17 | loadContext.Log.Info($"Skipped updating history because history engine is not enabled."); 18 | return; 19 | } 20 | 21 | var stopwatch = Stopwatch.StartNew(); 22 | 23 | var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.09.UpdateHistory.sql"); 24 | sqlContext.ExecuteSql(sql, 25 | commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", Sitecore.Context.User.Name)); 26 | 27 | loadContext.Log.Info($"Updated history: {(int)stopwatch.Elapsed.TotalSeconds}s"); 28 | } 29 | 30 | private bool HistoryEngineEnabled(BulkLoadContext context) 31 | { 32 | var db = Factory.GetDatabase(context.Database, true); 33 | return db.Engines.HistoryEngine.Storage != null; 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/SyncPublishQueue.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using Sitecore.DataBlaster.Load.Sql; 3 | 4 | namespace Sitecore.DataBlaster.Load.Processors 5 | { 6 | public class SyncPublishQueue : ISyncInTransaction 7 | { 8 | public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) 9 | { 10 | if (!loadContext.UpdatePublishQueue.GetValueOrDefault()) return; 11 | if (loadContext.ItemChanges.Count == 0) return; 12 | 13 | var stopwatch = Stopwatch.StartNew(); 14 | 15 | var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.10.UpdatePublishQueue.sql"); 16 | sqlContext.ExecuteSql(sql, 17 | commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", Sitecore.Context.User.Name)); 18 | 19 | loadContext.Log.Info($"Updated publish queue: {(int)stopwatch.Elapsed.TotalSeconds}s"); 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ValidateDataIntegrity.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using Sitecore.DataBlaster.Load.Sql; 3 | 4 | namespace Sitecore.DataBlaster.Load.Processors 5 | { 6 | public class ValidateDataIntegrity : IValidateStagedData 7 | { 8 | public bool ValidateLoadStage(BulkLoadContext context, BulkLoadSqlContext sqlContext) 9 | { 10 | if (!CheckTempData(context, sqlContext)) 11 | { 12 | context.StageFailed(Stage.Load, "Found missing templates, parents, items or fields."); 13 | return false; 14 | } 15 | return true; 16 | } 17 | 18 | [SuppressMessage("Microsoft.Security", "CA2100", Justification = "No user parameters")] 19 | private bool CheckTempData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) 20 | { 21 | var check = sqlContext.GetEmbeddedSql(loadContext, "Sql.07.CheckTempData.sql"); 22 | var hasErrors = false; 23 | 24 | using (var cmd = sqlContext.NewSqlCommand(check)) 25 | using (var reader = cmd.ExecuteReader()) 26 | { 27 | while (reader.Read()) 28 | { 29 | if (!reader.GetBoolean(reader.GetOrdinal("HasParent"))) 30 | loadContext.Log.Error( 31 | $"Unable to find parent '{reader["ParentId"]}' for item with id '{reader["Id"]}', " + 32 | $"item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); 33 | if (!reader.GetBoolean(reader.GetOrdinal("HasTemplate"))) 34 | loadContext.Log.Error( 35 | $"Unable to find template '{reader["TemplateName"]}' with id '{reader["TemplateId"]}' " + 36 | $"for item with id '{reader["Id"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); 37 | hasErrors = true; 38 | } 39 | reader.NextResult(); 40 | while (reader.Read()) 41 | { 42 | if (!reader.GetBoolean(reader.GetOrdinal("HasItem"))) 43 | loadContext.Log.Error( 44 | $"Unable to find item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' " + 45 | $"and source info '{reader["SourceInfo"]}' for field '{reader["FieldName"]}' with id '{reader["FieldId"]}'."); 46 | if (!reader.GetBoolean(reader.GetOrdinal("HasField"))) 47 | loadContext.Log.Error( 48 | $"Unable to find field '{reader["FieldName"]}' with id '{reader["FieldId"]}' " + 49 | $"for item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); 50 | hasErrors = true; 51 | } 52 | } 53 | 54 | return !hasErrors; 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Processors/ValidateNoDuplicates.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using Sitecore.DataBlaster.Load.Sql; 3 | 4 | namespace Sitecore.DataBlaster.Load.Processors 5 | { 6 | public class ValidateNoDuplicates : IValidateStagedData 7 | { 8 | public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) 9 | { 10 | if (!CheckDuplicates(loadContext, sqlContext)) 11 | { 12 | loadContext.StageFailed(Stage.Load, "Duplicate items were found in set to load."); 13 | return false; 14 | } 15 | return true; 16 | } 17 | 18 | [SuppressMessage("Microsoft.Security", "CA2100", Justification = "No user parameters")] 19 | private bool CheckDuplicates(BulkLoadContext context, BulkLoadSqlContext sqlContext) 20 | { 21 | var check = sqlContext.GetEmbeddedSql(context, "Sql.05.CheckDuplicates.sql"); 22 | var hasErrors = false; 23 | 24 | using (var cmd = sqlContext.NewSqlCommand(check)) 25 | using (var reader = cmd.ExecuteReader()) 26 | { 27 | while (reader.Read()) 28 | { 29 | context.Log.Error( 30 | $"Duplicate item found with id '{reader["Id"]}', item path '{reader["ItemPath"]}' " + 31 | $"and source info '{reader["SourceInfo"]}'."); 32 | hasErrors = true; 33 | } 34 | } 35 | return !hasErrors; 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/01.CreateTempTable.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE #BulkItemsAndFields 2 | ( 3 | ItemId UNIQUEIDENTIFIER NOT NULL, 4 | ItemName NVARCHAR(256) COLLATE database_default NOT NULL, 5 | TemplateId UNIQUEIDENTIFIER NOT NULL, 6 | TemplateName NVARCHAR(256) COLLATE database_default, 7 | MasterId UNIQUEIDENTIFIER NOT NULL, 8 | ParentId UNIQUEIDENTIFIER NOT NULL, 9 | ItemPath NVARCHAR(MAX) COLLATE database_default, 10 | ItemPathExpression NVARCHAR(MAX) COLLATE database_default, 11 | WhenItemIdCreated UNIQUEIDENTIFIER, -- Only creates this item when the referenced item is created as well. 12 | OriginalItemId UNIQUEIDENTIFIER NOT NULL, -- When LookupItems updates IDs, we still have a reference to the original id in code. 13 | 14 | ItemAction VARCHAR(50) COLLATE database_default NOT NULL, -- 'AddOnly', 'AddItemOnly', 'Update', 'UpdateExistingItem', 'Revert', 'RevertTree'. 15 | SourceInfo SQL_VARIANT, 16 | 17 | FieldId UNIQUEIDENTIFIER NOT NULL, 18 | FieldName NVARCHAR(256) COLLATE database_default, 19 | Language NVARCHAR(50) COLLATE database_default, -- NULL for shared fields 20 | Version INT, -- NULL for unversioned fields 21 | 22 | -- Value for the field, contains the blob id (GUID) in case of a blob, leave empty to lookup blob id. 23 | Value NVARCHAR(MAX) COLLATE database_default, 24 | 25 | -- In case of same blob for different fields, 26 | -- we only store the blob once, check the blob id (value) to find blob data in other record. 27 | Blob VARBINARY(MAX), 28 | IsBlob BIT NOT NULL, 29 | 30 | FieldAction VARCHAR(50) COLLATE database_default NOT NULL, -- 'AddOnly, 'Update' 31 | 32 | -- If fields only needs to be set when item is created/saved. 33 | WhenCreated BIT NOT NULL, 34 | WhenSaved BIT NOT NULL, 35 | 36 | DeduplicateItem BIT NOT NULL, 37 | 38 | IsShared BIT, 39 | IsUnversioned BIT, 40 | 41 | -- Flags will be set later on and will be used for validation. 42 | HasItem BIT, 43 | HasField BIT, 44 | 45 | RowId INT NOT NULL IDENTITY (1, 1) 46 | ) 47 | 48 | CREATE TABLE #FieldRules 49 | ( 50 | FieldId UNIQUEIDENTIFIER NOT NULL, 51 | SkipOnCreate BIT NOT NULL, 52 | SkipOnUpdate BIT NOT NULL, 53 | SkipOnDelete BIT NOT NULL 54 | ) -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/02.LookupBlobs.sql: -------------------------------------------------------------------------------- 1 | -- Lookup missing blob ids. 2 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Looking up blobs...' 3 | UPDATE 4 | bif 5 | SET 6 | -- Value should contain BlobId in case of blobs. 7 | Value = CONVERT(VARCHAR(50), blobId.Value) 8 | FROM 9 | #BulkItemsAndFields bif 10 | 11 | -- Find current BlobId for this field. 12 | OUTER APPLY 13 | ( 14 | SELECT TOP 1 15 | b.BlobId 16 | FROM 17 | ( 18 | SELECT f.Value 19 | FROM SharedFields f 20 | WHERE f.ItemId = bif.ItemId 21 | AND f.FieldId = bif.FieldId 22 | AND f.Value IS NOT NULL AND f.Value != '' -- We can not convert this to GUID. 23 | 24 | UNION ALL 25 | 26 | -- Blobs cannot be unversioned, it's either shared or versioned. 27 | --SELECT f.Value 28 | --FROM UnversionedFields f 29 | --WHERE f.ItemId = bif.ItemId 30 | -- AND f.FieldId = bif.FieldId 31 | -- AND f.Language = bif.Language 32 | -- AND f.Value IS NOT NULL AND f.Value != '' -- We can not convert this to GUID. 33 | 34 | --UNION ALL 35 | 36 | SELECT f.Value 37 | FROM VersionedFields f 38 | WHERE f.ItemId = bif.ItemId 39 | AND f.FieldId = bif.FieldId 40 | AND f.Language = bif.Language 41 | AND f.Version = bif.Version 42 | AND f.Value IS NOT NULL AND f.Value != '' -- We can not convert this to GUID. 43 | ) f 44 | JOIN Blobs b ON 45 | b.BlobId = TRY_CAST(f.Value AS UNIQUEIDENTIFIER) 46 | ) b 47 | 48 | -- Generate new id for new blobs, will be used later on to add new blob. 49 | CROSS APPLY 50 | ( 51 | SELECT ISNULL(b.BlobId, NEWID()) Value 52 | ) blobId 53 | WHERE 54 | bif.isBlob = 1 55 | AND bif.Value IS NULL 56 | -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/03.LookupItems.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Looking up items...' 2 | 3 | DECLARE @destinationPath VARCHAR(900) = '/sitecore/content' 4 | DECLARE @destinationId UNIQUEIDENTIFIER = '0DE95AE4-41AB-4D01-9EB0-67441B7C2450' 5 | DECLARE @delim VARCHAR(10) = '|' 6 | 7 | -- Build item paths of all items in destination. 8 | ;WITH LookupCTE (Id, Path, ParentId) 9 | AS 10 | ( 11 | SELECT i.ID AS Id, @destinationPath AS Path, i.ParentID 12 | FROM Items i 13 | WHERE i.ID = @destinationId 14 | 15 | UNION ALL 16 | 17 | SELECT i.ID, CAST(cte.Path + '/' + i.Name AS VARCHAR(900)), i.ParentID 18 | FROM Items i 19 | JOIN LookupCTE cte ON 20 | cte.Id = i.ParentID 21 | ) 22 | 23 | SELECT * 24 | INTO #Paths 25 | FROM LookupCTE 26 | 27 | CREATE CLUSTERED INDEX IX_Paths_Path ON #Paths(Path) 28 | 29 | 30 | -- Create temp table with all paths mapped to an id including parent paths and ids. 31 | -- Id can either be the already existing id or the new id for the unexisting item. 32 | SELECT * 33 | INTO #PathIdMapping 34 | FROM 35 | ( 36 | SELECT 37 | new.ItemId AS NewItemId, 38 | ISNULL(existing.Id, new.ItemId) AS ItemId, 39 | ISNULL(existing.Path, new.ItemPath) AS ItemPath, 40 | new.ParentId AS NewParentId, 41 | ISNULL(existingParent.Id, parent.Id) AS ParentId 42 | FROM 43 | ( 44 | SELECT DISTINCT ItemId, ItemPath, ParentId 45 | FROM #BulkItemsAndFields 46 | WHERE ItemPath IS NOT NULL 47 | AND ItemPathExpression IS NULL 48 | ) new 49 | LEFT JOIN #Paths existing ON 50 | existing.Path = new.ItemPath 51 | CROSS APPLY 52 | ( 53 | SELECT 54 | CASE WHEN (existing.Id IS NOT NULL) 55 | THEN existing.ParentId 56 | ELSE NULL 57 | END AS Id, 58 | CASE WHEN (existing.Id IS NOT NULL) 59 | THEN LEFT(existing.Path, LEN(existing.Path) - CHARINDEX('/', REVERSE(existing.Path))) 60 | ELSE LEFT(new.ItemPath, LEN(new.ItemPath) - CHARINDEX('/', REVERSE(new.ItemPath))) 61 | END AS Path 62 | ) parent 63 | LEFT JOIN #Paths existingParent ON 64 | existingParent.Path = parent.Path 65 | ) x 66 | 67 | CREATE CLUSTERED INDEX IX_PathIdMapping_Id ON #PathIdMapping (NewItemId) 68 | 69 | 70 | -- Don't mis out on parents that are refering to new items that don't exist in db yet. 71 | UPDATE map 72 | SET map.ParentId = map2.ItemId 73 | FROM #PathIdMapping map 74 | JOIN #PathIdMapping map2 ON 75 | map2.ItemPath = LEFT(map.ItemPath, LEN(map.ItemPath) - CHARINDEX('/', REVERSE(map.ItemPath))) 76 | WHERE map.ParentId IS NULL 77 | 78 | 79 | -- Update item ids, paths and parents. 80 | UPDATE bif 81 | SET bif.ItemId = map.ItemId, 82 | bif.ItemPath = map.ItemPath, 83 | bif.ParentId = map.ParentId 84 | FROM #BulkItemsAndFields bif 85 | JOIN #PathIdMapping map ON 86 | map.NewItemId = bif.ItemId 87 | WHERE bif.ItemId != map.ItemId 88 | OR bif.ItemPath != map.ItemPath 89 | OR bif.ParentId != map.parentId 90 | 91 | 92 | -- Update fields that contain item ids to newly imported items. 93 | -- See e.g. LookupNameColumnMapping 94 | WHILE (1 = 1) -- We can only update a row 1 time in a single update statement. 95 | BEGIN 96 | UPDATE 97 | bif 98 | SET 99 | bif.Value = REPLACE(bif.Value, ref.NewItemStrId, ref.ItemStrId) 100 | FROM 101 | #BulkItemsAndFields bif 102 | CROSS APPLY 103 | ( 104 | SELECT TOP 1 105 | '{' + CONVERT(NVARCHAR(36), map.NewItemId) + '}' AS NewItemStrId, 106 | '{' + CONVERT(NVARCHAR(36), map.ItemId) + '}' AS ItemStrId 107 | FROM 108 | ( 109 | SELECT LTRIM(RTRIM(SUBSTRING(bif.Value, number, 110 | CHARINDEX(@delim, bif.Value + @delim, number) - number))) AS Value 111 | FROM 112 | (SELECT ROW_NUMBER() OVER (ORDER BY name) AS number FROM sys.all_objects) nr 113 | WHERE 114 | number <= LEN(bif.Value) 115 | AND SUBSTRING(@delim + bif.Value, number, LEN(@delim)) = @delim 116 | ) split 117 | CROSS APPLY 118 | ( 119 | SELECT TRY_CAST(split.Value AS UNIQUEIDENTIFIER) refId 120 | ) ref 121 | LEFT JOIN Items i ON 122 | i.ID = ref.refId 123 | LEFT JOIN #PathIdMapping map ON 124 | i.ID IS NULL -- Only join when item doesn't exist. 125 | AND map.NewItemId = ref.refId 126 | WHERE 127 | map.ItemId IS NOT NULL 128 | ) ref 129 | WHERE 130 | LEFT(LTRIM(bif.Value), 1) = '{' -- Perf optimization. 131 | AND bif.Value != REPLACE(bif.Value, ref.NewItemStrId, ref.ItemStrId) 132 | 133 | IF (@@ROWCOUNT = 0) 134 | BREAK 135 | END -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/04.SplitTempTable.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Splitting temp table...' 2 | 3 | SELECT DISTINCT 4 | ItemId AS Id, 5 | ItemName AS Name, 6 | TemplateId, 7 | TemplateName, 8 | MasterId, 9 | ParentId, 10 | ItemPath, 11 | OriginalItemId AS OriginalId, 12 | ItemAction AS Action, 13 | SourceInfo, 14 | CAST(NULL AS BIT) AS HasParent, 15 | CAST(NULL AS BIT) AS HasTemplate 16 | INTO 17 | #SyncItems 18 | FROM 19 | #BulkItemsAndFields 20 | 21 | -- We don't create a seperate table for the fields anymore, 22 | -- because that would basically just be copying data. -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/05.CheckDuplicates.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Finding duplicates...' 2 | 3 | SELECT 4 | si.Id, si.Name, si.ItemPath, si.SourceInfo 5 | FROM 6 | #SyncItems si 7 | WHERE 8 | si.Id IN 9 | ( 10 | -- Find items with duplicate fields. 11 | SELECT DISTINCT dbl.ItemId 12 | FROM #BulkItemsAndFields dbl 13 | GROUP BY dbl.ItemId, dbl.FieldId, dbl.Language, dbl.Version 14 | HAVING COUNT(*) > 1 15 | ) 16 | 17 | UNION ALL 18 | 19 | SELECT 20 | si.Id, si.Name, si.ItemPath, si.SourceInfo 21 | FROM 22 | #SyncItems si 23 | WHERE 24 | si.ItemPath IN 25 | ( 26 | -- Find items with duplicate paths. 27 | SELECT DISTINCT dbl.ItemPath 28 | FROM #SyncItems dbl 29 | GROUP BY dbl.ItemPath 30 | HAVING COUNT(*) > 1 31 | ) 32 | 33 | ORDER BY 34 | si.ItemPath -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/06.CreateIndexes.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Creating indexes...' 2 | 3 | CREATE INDEX IX_BulkItemsAndFields ON #BulkItemsAndFields (ItemId, FieldId, Language, Version) 4 | CREATE INDEX IX_BulkItemsAndFields_IsShared_IsUnversioned ON #BulkItemsAndFields (IsShared, IsUnversioned) 5 | 6 | CREATE UNIQUE CLUSTERED INDEX IX_SyncItems_Id ON #SyncItems (Id) 7 | 8 | CREATE UNIQUE CLUSTERED INDEX IX_FieldRules_FieldId ON #FieldRules (FieldId) -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/07.CheckTempData.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Checking temp data...' 2 | 3 | DECLARE @SharedFieldId UNIQUEIDENTIFIER = '{BE351A73-FCB0-4213-93FA-C302D8AB4F51}' 4 | DECLARE @UnversionedFieldId UNIQUEIDENTIFIER = '{39847666-389D-409B-95BD-F2016F11EED5}' 5 | 6 | 7 | -- Check if all items have corresponding parents and templates. 8 | -- TODO: check masters? 9 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Validating parents and templates...' 10 | UPDATE 11 | si 12 | SET 13 | HasParent = CASE WHEN pi.ID IS NULL AND spi.Id IS NULL THEN 0 ELSE 1 END, 14 | HasTemplate = CASE WHEN ti.ID IS NULL AND tpi.Id IS NULL AND tpi.Action != 'UpdateExistingItem' THEN 0 ELSE 1 END 15 | FROM #SyncItems si 16 | LEFT JOIN Items pi ON pi.ID = si.ParentId 17 | LEFT JOIN #SyncItems spi ON spi.Id = si.ParentId 18 | LEFT JOIN Items ti ON ti.ID = si.TemplateId 19 | LEFT JOIN #SyncItems tpi ON tpi.Id = si.TemplateId 20 | WHERE si.ParentID != '00000000-0000-0000-0000-000000000000' 21 | 22 | SELECT * FROM #SyncItems WHERE HasParent = 0 OR HasTemplate = 0 23 | 24 | 25 | -- Check if all fields have corresponding items and field items. 26 | -- Check if fields are shared or unversioned. 27 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Validating fields...' 28 | UPDATE 29 | sf 30 | SET 31 | HasItem = CASE WHEN i.ID IS NULL AND si.Id IS NULL THEN 0 ELSE 1 END, 32 | HasField = CASE 33 | WHEN fi.ID IS NULL AND sfi.Id IS NULL THEN 0 34 | WHEN sharunver.IsUnversioned = 1 AND sf.Language IS NULL THEN 0 35 | WHEN sharunver.IsShared = 0 AND sharunver.IsUnversioned = 0 AND (sf.Language IS NULL OR sf.Version IS NULL ) THEN 0 36 | ELSE 1 END, 37 | IsShared = sharunver.IsShared, 38 | IsUnversioned = sharunver.IsUnversioned 39 | FROM 40 | #BulkItemsAndFields sf 41 | LEFT JOIN Items i ON i.ID = sf.ItemId 42 | LEFT JOIN #SyncItems si ON si.Id = sf.ItemId 43 | LEFT JOIN Items fi ON fi.ID = sf.FieldId 44 | LEFT JOIN #SyncItems sfi ON sfi.Id = sf.FieldId 45 | 46 | -- Detect whether field is shared or unversioned from provided data or database. 47 | OUTER APPLY 48 | ( 49 | SELECT TOP 1 CASE WHEN ft.Value = '1' THEN 1 ELSE 0 END AS IsShared 50 | FROM Fields ft 51 | WHERE ft.ItemId = fi.ID AND ft.FieldId = @SharedFieldId 52 | ) fiShared 53 | OUTER APPLY 54 | ( 55 | SELECT TOP 1 CASE WHEN sft.Value = '1' THEN 1 ELSE 0 END AS IsShared 56 | FROM #BulkItemsAndFields sft 57 | WHERE sft.ItemId = sfi.ID AND sft.FieldId = @SharedFieldId 58 | ) sfiShared 59 | OUTER APPLY 60 | ( 61 | SELECT TOP 1 CASE WHEN ft.Value = '1' THEN 1 ELSE 0 END AS IsUnversioned 62 | FROM Fields ft 63 | WHERE ft.ItemId = fi.ID AND ft.FieldId = @UnversionedFieldId 64 | ) fiUnversioned 65 | OUTER APPLY 66 | ( 67 | SELECT TOP 1 CASE WHEN sft.Value = '1' THEN 1 ELSE 0 END AS IsUnversioned 68 | FROM #BulkItemsAndFields sft 69 | WHERE sft.ItemId = sfi.ID AND sft.FieldId = @UnversionedFieldId 70 | ) sfiUnversioned 71 | OUTER APPLY 72 | ( 73 | SELECT 74 | COALESCE(sfiShared.IsShared, fiShared.IsShared, 0) AS IsShared, 75 | CASE WHEN COALESCE(sfiShared.IsShared, fiShared.IsShared, 0) = 1 76 | THEN 0 77 | ELSE COALESCE(sfiUnversioned.IsUnversioned, fiUnversioned.IsUnversioned, 0) END AS IsUnversioned 78 | ) sharunver 79 | 80 | 81 | -- Remove old versions for unversioned fields. 82 | -- Sitecore's sync item doesn't really have the notion of unversioned, so we get duplicates. 83 | DELETE d 84 | FROM #BulkItemsAndFields d 85 | JOIN ( 86 | SELECT sf.ItemId, sf.FieldId, sf.Language, MAX(sf.Version) AS MaxVersion 87 | FROM #BulkItemsAndFields sf 88 | WHERE sf.IsUnversioned = 1 89 | GROUP BY sf.ItemId, sf.FieldId, sf.Language 90 | HAVING COUNT(sf.Version) > 1 91 | ) sf ON sf.ItemId = d.ItemId 92 | AND sf.FieldId = d.FieldId 93 | AND sf.Language = d.Language 94 | AND sf.MaxVersion > d.Version 95 | 96 | 97 | SELECT sf.*, si.ItemPath, si.SourceInfo 98 | FROM #BulkItemsAndFields sf 99 | JOIN #SyncItems si ON 100 | si.Id = sf.ItemId 101 | WHERE sf.HasItem = 0 OR sf.HasField = 0 -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/09.UpdateHistory.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Updating history engine...' 2 | 3 | DECLARE @Category VARCHAR(50) = 'Item' 4 | --DECLARE @UserName NVARCHAR(250) = 'default\Anonymous' 5 | DECLARE @TaskStack VARCHAR(MAX) = '[none]' 6 | DECLARE @AdditionalInfo NVARCHAR(MAX) = '' 7 | 8 | -- Missing: 9 | -- master/template change without changed fields (very unlikely) 10 | -- blob change without field change 11 | -- added/removed version 12 | 13 | DECLARE @Actions TABLE 14 | ( 15 | Action VARCHAR(50), 16 | Created BIT, 17 | Saved BIT, 18 | Moved BIT, 19 | Deleted BIT 20 | ) 21 | INSERT INTO @Actions (Action, Created) VALUES ('Created', 1) 22 | INSERT INTO @Actions (Action, Saved) VALUES ('Saved', 1) 23 | INSERT INTO @Actions (Action, Moved) VALUES ('Moved', 1) 24 | INSERT INTO @Actions (Action, Deleted) VALUES ('Deleted', 1) 25 | 26 | 27 | INSERT INTO 28 | History (Id, Category, Action, ItemId, ItemLanguage, ItemVersion, ItemPath, UserName, TaskStack, AdditionalInfo, Created) 29 | SELECT 30 | NEWID(), @Category, a.Action, 31 | ia.ItemId, ISNULL(ia.Language, ''), ISNULL(ia.Version, 0), ISNULL(ia.ItemPath, ''), @UserName, @TaskStack, @AdditionalInfo, ia.Timestamp 32 | FROM 33 | ( 34 | SELECT si.ItemPath, ia.ItemId, 35 | CAST(MAX(CAST(ia.Created AS INT)) AS BIT) AS Created, 36 | CAST(MAX(CAST(ia.Saved AS INT)) AS BIT) AS Saved, 37 | CAST(MAX(CAST(ia.Moved AS INT)) AS BIT) AS Moved, 38 | CAST(MAX(CAST(ia.Deleted AS INT)) AS BIT) AS Deleted, 39 | ia.Language, 40 | ia.Version, 41 | ia.Timestamp 42 | FROM #ItemActions ia 43 | JOIN #SyncItems si ON 44 | si.Id = ia.ItemId 45 | GROUP BY si.ItemPath, ia.ItemId, ia.Language, ia.Version, ia.Timestamp 46 | ) ia 47 | JOIN @Actions a ON 48 | a.Created = ia.Created 49 | OR a.Saved = ia.Saved 50 | OR a.Moved = ia.Moved 51 | OR a.Deleted = ia.Deleted 52 | ORDER BY 53 | -- Parents need to be inserted first. 54 | -- Don't use CLR for now. 55 | --dbo.ItemPathLevel(ItemPath) 56 | (LEN(ItemPath) - LEN(REPLACE(ItemPath, '/', ''))) -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/10.UpdatePublishQueue.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Updating publish queue...' 2 | 3 | --DECLARE @UserName NVARCHAR(250) = 'default\Anonymous' 4 | 5 | -- Missing: 6 | -- master/template change without changed fields (very unlikely) 7 | -- blob change without field change 8 | -- added/removed version 9 | 10 | DECLARE @Actions TABLE 11 | ( 12 | Action VARCHAR(50), 13 | Created BIT, 14 | Saved BIT, 15 | Moved BIT, 16 | Deleted BIT 17 | ) 18 | INSERT INTO @Actions (Action, Created) VALUES ('Created', 1) 19 | INSERT INTO @Actions (Action, Saved) VALUES ('Saved', 1) 20 | INSERT INTO @Actions (Action, Moved) VALUES ('Moved', 1) 21 | INSERT INTO @Actions (Action, Deleted) VALUES ('Deleted', 1) 22 | 23 | 24 | INSERT INTO 25 | PublishQueue (ID, ItemId, Language, Version, Action, Date) 26 | SELECT 27 | NEWID(), ia.ItemId, ISNULL(ia.Language, '*'), ISNULL(ia.Version, 0), a.Action, ia.Timestamp 28 | FROM 29 | ( 30 | SELECT si.ItemPath, ia.ItemId, 31 | CAST(MAX(CAST(ia.Created AS INT)) AS BIT) AS Created, 32 | CAST(MAX(CAST(ia.Saved AS INT)) AS BIT) AS Saved, 33 | CAST(MAX(CAST(ia.Moved AS INT)) AS BIT) AS Moved, 34 | CAST(MAX(CAST(ia.Deleted AS INT)) AS BIT) AS Deleted, 35 | ia.Language, 36 | ia.Version, 37 | ia.Timestamp 38 | FROM #ItemActions ia 39 | JOIN #SyncItems si ON 40 | si.Id = ia.ItemId 41 | GROUP BY si.ItemPath, ia.ItemId, ia.Language, ia.Version, ia.Timestamp 42 | ) ia 43 | JOIN @Actions a ON 44 | a.Created = ia.Created 45 | OR a.Saved = ia.Saved 46 | OR a.Moved = ia.Moved 47 | OR a.Deleted = ia.Deleted 48 | ORDER BY 49 | -- Parents need to be inserted first. 50 | -- Don't use CLR for now. 51 | --dbo.ItemPathLevel(ItemPath) 52 | (LEN(ItemPath) - LEN(REPLACE(ItemPath, '/', ''))) -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/20.CreateLinkTempTable.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE #ItemLinks 2 | ( 3 | SourceDatabase NVARCHAR(150) COLLATE database_default NOT NULL, 4 | SourceItemId UNIQUEIDENTIFIER NOT NULL, 5 | SourceLanguage NVARCHAR(50) COLLATE database_default, 6 | SourceVersion INT, 7 | SourceFieldId UNIQUEIDENTIFIER NOT NULL, 8 | 9 | TargetDatabase NVARCHAR(150) COLLATE database_default NOT NULL, 10 | TargetItemId UNIQUEIDENTIFIER NOT NULL, 11 | TargetLanguage NVARCHAR(50) COLLATE database_default, 12 | TargetVersion INT, 13 | TargetPath NTEXT COLLATE database_default NOT NULL, 14 | 15 | ItemAction VARCHAR(50) COLLATE database_default NOT NULL, -- 'AddOnly', 'Revert'. 16 | ) -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/21.MergeLinkTempData.sql: -------------------------------------------------------------------------------- 1 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Updating link database...' 2 | 3 | MERGE 4 | Links AS target 5 | USING 6 | #ItemLinks AS source 7 | ON 8 | target.SourceDatabase = source.SourceDatabase 9 | AND target.SourceItemID = source.SourceItemId 10 | AND target.SourceLanguage = ISNULL(source.SourceLanguage, '') 11 | AND target.SourceVersion = ISNULL(source.SourceVersion, 0) 12 | AND target.SourceFieldID = source.SourceFieldId 13 | AND target.TargetItemID = source.TargetItemId 14 | AND target.TargetLanguage = ISNULL(source.TargetLanguage, '') 15 | AND target.TargetVersion = ISNULL(source.TargetVersion, 0) 16 | WHEN MATCHED AND (source.ItemAction = 'Update' OR source.ItemAction = 'Revert') THEN 17 | UPDATE SET 18 | target.TargetPath = source.TargetPath 19 | WHEN NOT MATCHED BY TARGET THEN 20 | INSERT (ID, 21 | SourceDatabase, SourceItemID, SourceLanguage, SourceVersion, SourceFieldID, 22 | TargetDatabase, TargetItemID, TargetLanguage, TargetVersion, TargetPath) 23 | VALUES (NEWID(), 24 | source.SourceDatabase, source.SourceItemId, ISNULL(source.SourceLanguage, ''), ISNULL(source.SourceVersion, 0), source.SourceFieldId, 25 | source.TargetDatabase, source.TargetItemId, ISNULL(source.TargetLanguage, ''), ISNULL(source.TargetVersion, 0), source.TargetPath) 26 | ; 27 | 28 | PRINT CONVERT(VARCHAR(12), GETDATE(), 114) + ': Deleting item links...' 29 | DELETE 30 | l 31 | FROM 32 | Links l 33 | CROSS APPLY 34 | ( 35 | SELECT TOP 1 il.SourceItemId 36 | FROM #ItemLinks il 37 | WHERE (il.ItemAction = 'Revert' OR il.ItemAction = 'RevertTree') 38 | AND il.SourceDatabase = l.SourceDatabase 39 | AND il.SourceItemId = l.SourceItemID 40 | ) isItemToRevert 41 | OUTER APPLY 42 | ( 43 | SELECT TOP 1 il.SourceItemId 44 | FROM #ItemLinks il 45 | WHERE il.SourceDatabase = l.SourceDatabase 46 | AND il.SourceItemId = l.SourceItemID 47 | AND ISNULL(il.SourceLanguage, '') = l.SourceLanguage 48 | AND ISNULL(il.SourceVersion, 0) = l.SourceVersion 49 | AND il.SourceFieldId = l.SourceFieldID 50 | ) source 51 | WHERE 52 | source.SourceItemId IS NULL -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Sql/BulkLoadSqlContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Data.SqlClient; 4 | using System.Diagnostics.CodeAnalysis; 5 | using Sitecore.DataBlaster.Util.Sql; 6 | 7 | namespace Sitecore.DataBlaster.Load.Sql 8 | { 9 | public class BulkLoadSqlContext : SqlContext 10 | { 11 | public BulkLoadSqlContext(SqlConnection connection, Type defaultSubject = null) 12 | : base(connection, defaultSubject: defaultSubject) 13 | { 14 | } 15 | 16 | public virtual string GetEmbeddedSql(BulkLoadContext context, string relativePath, Type subject = null) 17 | { 18 | return PostProcessSql(context, GetEmbeddedSql(relativePath, subject)); 19 | } 20 | 21 | public virtual string PostProcessSql(BulkLoadContext context, string sql) 22 | { 23 | if (context.StageDataWithoutProcessing) 24 | return sql.Replace("#", "tmp_"); 25 | 26 | return sql; 27 | } 28 | 29 | [SuppressMessage("Microsoft.Security", "CA2100", Justification = "No user parameters")] 30 | public virtual void DropStageTables() 31 | { 32 | var sql = 33 | "SELECT TABLE_SCHEMA, TABLE_NAME " + 34 | "FROM INFORMATION_SCHEMA.TABLES " + 35 | "WHERE TABLE_NAME LIKE 'tmp_%'"; 36 | 37 | var toDelete = new List(); 38 | using (var cmd = NewSqlCommand(sql)) 39 | using (var reader = cmd.ExecuteReader()) 40 | { 41 | while (reader.Read()) 42 | { 43 | toDelete.Add($"[{reader.GetString(0)}].[{reader.GetString(1)}]"); 44 | } 45 | } 46 | foreach (var table in toDelete) 47 | { 48 | using (var cmd = NewSqlCommand("DROP TABLE " + table)) 49 | { 50 | cmd.ExecuteNonQuery(); 51 | } 52 | } 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/Stage.cs: -------------------------------------------------------------------------------- 1 | namespace Sitecore.DataBlaster.Load 2 | { 3 | public enum Stage 4 | { 5 | Extract, 6 | Transform, 7 | Bucketing, 8 | Load 9 | } 10 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Load/StageResult.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Sitecore.DataBlaster.Load 4 | { 5 | [Flags] 6 | public enum StageResult 7 | { 8 | Unknown = 0, 9 | Succeeded = 1, 10 | Failed = 2, 11 | HasWarnings = 4, 12 | HasErrors = 8 13 | } 14 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Read/BulkReader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.Data.SqlClient; 5 | using System.Linq; 6 | using Sitecore.Data; 7 | using Sitecore.Data.Items; 8 | using Sitecore.DataBlaster.Util.Sql; 9 | 10 | namespace Sitecore.DataBlaster.Read 11 | { 12 | public class BulkReader 13 | { 14 | /// 15 | /// Gets the item header information for all descendants of an item. 16 | /// 17 | /// Sql context to read from. 18 | /// Ancestor item id. 19 | /// Ancestor item path. 20 | /// Filter descendants by template(s). No inheritance supported. 21 | /// Filter descendants by modification timestamp. 22 | /// Stream of item header information. 23 | public virtual IEnumerable GetDescendantHeaders(SqlContext sqlContext, 24 | Guid itemId, string itemPath, Guid[] ofTemplates = null, DateTime? modifiedSince = null) 25 | { 26 | var templateCsv = ofTemplates == null || ofTemplates.Length == 0 27 | ? null 28 | : string.Join(",", ofTemplates.Select(x => $"'{x:D}'")); 29 | 30 | var sql = sqlContext.GetEmbeddedSqlLines("Sql.GetDescendantHeaders.sql", typeof(BulkReader)) 31 | .ExpandParameterLineIf(() => ofTemplates?.Length > 1, "@templateIdsCsv", templateCsv) 32 | .RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length != 1, "@templateId") 33 | .RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length <= 1, "@templateIdsCsv") 34 | .RemoveParameterLineIf(() => modifiedSince == null, "@modifiedSince"); 35 | 36 | using (var reader = sqlContext.ExecuteReader(sql, commandProcessor: cmd => 37 | { 38 | cmd.Parameters.AddWithValue("@rootItemId", itemId); 39 | cmd.Parameters.AddWithValue("@rootItemPath", itemPath); 40 | if (ofTemplates != null && ofTemplates.Length == 1) 41 | cmd.Parameters.AddWithValue("templateId", ofTemplates[0]); 42 | if (modifiedSince.HasValue) 43 | cmd.Parameters.AddWithValue("@modifiedSince", modifiedSince.Value.ToUniversalTime()); 44 | })) 45 | { 46 | while (reader.Read()) 47 | { 48 | yield return new ItemHeader( 49 | reader.GetGuid(0), 50 | reader.GetString(1), 51 | reader.GetString(2), 52 | reader.GetGuid(3), 53 | reader.GetGuid(4), 54 | reader.GetGuid(5), 55 | reader.GetDateTime(6), 56 | reader.GetDateTime(7) 57 | ); 58 | } 59 | } 60 | } 61 | 62 | /// 63 | /// Gets the item header information for all descendants of an item. 64 | /// 65 | /// Item to get descendant headers for. 66 | /// Filter descendants by template(s). No inheritance supported. 67 | /// Filter descendants by modification timestamp. 68 | /// Stream of item header information. 69 | public virtual IEnumerable GetDescendantHeaders(Item item, 70 | Guid[] ofTemplates = null, DateTime? modifiedSince = null) 71 | { 72 | if (item == null) throw new ArgumentNullException(nameof(item)); 73 | 74 | using (var conn = GetSqlConnection(item.Database, open: true)) 75 | { 76 | foreach (var header in GetDescendantHeaders(new SqlContext(conn, typeof(BulkReader)), 77 | item.ID.Guid, item.Paths.Path, ofTemplates, modifiedSince)) 78 | { 79 | yield return header; 80 | } 81 | } 82 | } 83 | 84 | /// 85 | /// Gets the item version header information for all descendants of an item. 86 | /// 87 | /// Sql context to read from. 88 | /// Ancestor item id. 89 | /// Ancestor item path. 90 | /// Filter descendants by template(s). No inheritance supported. 91 | /// Filter descendants by item modification timestamp. 92 | /// Filter descendants by item version modification timestamp. 93 | /// Stream of item version header information. 94 | public virtual IEnumerable GetDescendantVersionHeaders(SqlContext sqlContext, Guid itemId, 95 | string itemPath, 96 | Guid[] ofTemplates = null, DateTime? itemModifiedSince = null, DateTime? itemVersionModifiedSince = null) 97 | { 98 | var templateCsv = ofTemplates == null || ofTemplates.Length == 0 99 | ? null 100 | : string.Join(",", ofTemplates.Select(x => $"'{x:D}'")); 101 | 102 | var sql = sqlContext.GetEmbeddedSqlLines("Sql.GetDescendantVersionHeaders.sql", typeof(BulkReader)) 103 | .ExpandParameterLineIf(() => ofTemplates?.Length > 1, "@templateIdsCsv", templateCsv) 104 | .RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length != 1, "@templateId") 105 | .RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length <= 1, "@templateIdsCsv") 106 | .RemoveParameterLineIf(() => itemModifiedSince == null, "@itemModifiedSince") 107 | .RemoveParameterLineIf(() => itemVersionModifiedSince == null, "@versionModifiedSince"); 108 | 109 | using (var reader = sqlContext.ExecuteReader(sql, commandProcessor: cmd => 110 | { 111 | cmd.Parameters.AddWithValue("@rootItemId", itemId); 112 | cmd.Parameters.AddWithValue("@rootItemPath", itemPath); 113 | if (ofTemplates != null && ofTemplates.Length == 1) 114 | cmd.Parameters.AddWithValue("templateId", ofTemplates[0]); 115 | if (itemModifiedSince.HasValue) 116 | cmd.Parameters.AddWithValue("@itemModifiedSince", itemModifiedSince.Value.ToUniversalTime()); 117 | if (itemVersionModifiedSince.HasValue) 118 | cmd.Parameters.AddWithValue("@versionModifiedSince", 119 | itemVersionModifiedSince.Value.ToUniversalTime()); 120 | })) 121 | { 122 | while (reader.Read()) 123 | { 124 | yield return new ItemVersionHeader( 125 | reader.GetGuid(0), 126 | reader.GetString(1), 127 | reader.GetString(2), 128 | reader.GetGuid(3), 129 | reader.GetGuid(4), 130 | reader.GetGuid(5), 131 | reader.GetString(6), 132 | reader.GetInt32(7), 133 | reader.GetDateTime(8), 134 | reader.GetDateTime(9) 135 | ); 136 | } 137 | } 138 | } 139 | 140 | /// 141 | /// Gets the item version header information for all descendants of an item. 142 | /// 143 | /// Item to get descendant version headers for. 144 | /// Filter descendants by template(s). No inheritance supported. 145 | /// Filter descendants by item modification timestamp. 146 | /// Filter descendants by item version modification timestamp. 147 | /// Stream of item version header information. 148 | public virtual IEnumerable GetDescendantVersionHeaders(Item item, 149 | Guid[] ofTemplates = null, DateTime? itemModifiedSince = null, DateTime? itemVersionModifiedSince = null) 150 | { 151 | if (item == null) throw new ArgumentNullException(nameof(item)); 152 | 153 | using (var conn = GetSqlConnection(item.Database, open: true)) 154 | { 155 | foreach (var header in GetDescendantVersionHeaders(new SqlContext(conn, typeof(BulkReader)), 156 | item.ID.Guid, item.Paths.Path, ofTemplates, itemModifiedSince, itemVersionModifiedSince)) 157 | { 158 | yield return header; 159 | } 160 | } 161 | } 162 | 163 | /// 164 | /// Gets all descendants of an item. 165 | /// 166 | /// Sql context to read from. 167 | /// Ancestor item id. 168 | /// Ancestor item path. 169 | /// Filter descendants by template(s). No inheritance supported. 170 | /// Filter descendants by modification timestamp. 171 | /// Filter out descendants that are set to 'never publish'. 172 | /// Stream of bulk items. 173 | public virtual IEnumerable GetDescendants(SqlContext sqlContext, Guid itemId, string itemPath, 174 | Guid[] ofTemplates = null, DateTime? modifiedSince = null, bool onlyPublishable = false) 175 | { 176 | var templateCsv = ofTemplates == null || ofTemplates.Length == 0 177 | ? null 178 | : string.Join(",", ofTemplates.Select(x => $"'{x:D}'")); 179 | 180 | var sql = sqlContext.GetEmbeddedSqlLines("Sql.GetDescendants.sql", typeof(BulkReader)) 181 | .ExpandParameterLineIf(() => ofTemplates?.Length > 1, "@templateIdsCsv", templateCsv) 182 | .RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length != 1, "@templateId") 183 | .RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length <= 1, "@templateIdsCsv") 184 | .RemoveParameterLineIf(() => modifiedSince == null, "@modifiedSince") 185 | .RemoveParameterLineIf(() => !onlyPublishable, "@neverPublishFieldId") 186 | .RemoveParameterLineIf(() => !onlyPublishable, "@neverPublish"); 187 | 188 | using (var reader = sqlContext.ExecuteReader(sql, commandProcessor: cmd => 189 | { 190 | cmd.Parameters.AddWithValue("@rootItemId", itemId); 191 | cmd.Parameters.AddWithValue("@rootItemPath", itemPath); 192 | if (ofTemplates != null && ofTemplates.Length == 1) 193 | cmd.Parameters.AddWithValue("templateId", ofTemplates[0]); 194 | if (modifiedSince.HasValue) 195 | cmd.Parameters.AddWithValue("@modifiedSince", modifiedSince.Value.ToUniversalTime()); 196 | if (onlyPublishable) 197 | { 198 | cmd.Parameters.AddWithValue("@neverPublishFieldId", Sitecore.FieldIDs.NeverPublish.Guid); 199 | cmd.Parameters.AddWithValue("@neverPublish", "1"); 200 | } 201 | })) 202 | { 203 | BulkItem item = null; 204 | while (reader.Read()) 205 | { 206 | var recordItemId = reader.GetGuid(0); 207 | 208 | if (item != null && item.Id != recordItemId) 209 | { 210 | yield return item; 211 | item = null; 212 | } 213 | 214 | if (item == null) 215 | item = new BulkItem(recordItemId, 216 | reader.GetGuid(3), reader.GetGuid(4), reader.GetGuid(5), reader.GetString(2)); 217 | 218 | var language = reader.IsDBNull(10) ? null : reader.GetString(10); 219 | var version = reader.IsDBNull(11) ? null : (int?)reader.GetInt32(11); 220 | 221 | if (language != null && version != null) 222 | item.AddVersionedField(reader.GetGuid(8), language, version.Value, reader.GetString(9)); 223 | else if (language != null) 224 | item.AddUnversionedField(reader.GetGuid(8), language, reader.GetString(9)); 225 | else 226 | item.AddSharedField(reader.GetGuid(8), reader.GetString(9)); 227 | } 228 | if (item != null) yield return item; 229 | } 230 | } 231 | 232 | /// 233 | /// Gets all descendants of an item. 234 | /// 235 | /// Item to get descendants for. 236 | /// Filter descendants by template(s). No inheritance supported. 237 | /// Filter descendants by modification timestamp. 238 | /// Filter out descendants that are set to 'never publish'. 239 | /// Stream of bulk items. 240 | public virtual IEnumerable GetDescendants(Item item, 241 | Guid[] ofTemplates = null, DateTime? modifiedSince = null, bool onlyPublishable = false) 242 | { 243 | if (item == null) throw new ArgumentNullException(nameof(item)); 244 | 245 | using (var conn = GetSqlConnection(item.Database, open: true)) 246 | { 247 | foreach (var descendant in GetDescendants(new SqlContext(conn, typeof(BulkReader)), 248 | item.ID.Guid, item.Paths.Path, ofTemplates, modifiedSince, onlyPublishable)) 249 | { 250 | yield return descendant; 251 | } 252 | } 253 | } 254 | 255 | public virtual SqlConnection GetSqlConnection(Database database, bool open = false) 256 | { 257 | var conn = new SqlConnection(ConfigurationManager.ConnectionStrings[database.ConnectionStringName] 258 | .ConnectionString); 259 | if (open) conn.Open(); 260 | return conn; 261 | } 262 | } 263 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Read/ItemHeader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Sitecore.DataBlaster.Read 4 | { 5 | /// 6 | /// Describes an item without a specific version. 7 | /// 8 | public class ItemHeader 9 | { 10 | public Guid Id { get; set; } 11 | public string Name { get; set; } 12 | public string ItemPath { get; set; } 13 | public Guid TemplateId { get; set; } 14 | public Guid MasterId { get; set; } 15 | public Guid ParentId { get; set; } 16 | public DateTime Created { get; set; } 17 | public DateTime Updated { get; set; } 18 | 19 | public ItemHeader(Guid id, string name, string itemPath, 20 | Guid templateId, Guid masterId, Guid parentId, 21 | DateTime created, DateTime updated) 22 | { 23 | if (string.IsNullOrEmpty(name)) throw new ArgumentNullException(nameof(name)); 24 | 25 | this.Id = id; 26 | this.Name = name; 27 | this.ItemPath = itemPath; 28 | this.TemplateId = templateId; 29 | this.MasterId = masterId; 30 | this.ParentId = parentId; 31 | this.Created = created; 32 | this.Updated = updated; 33 | } 34 | 35 | #region Equality members 36 | 37 | protected bool Equals(ItemHeader other) 38 | { 39 | return Id.Equals(other.Id); 40 | } 41 | 42 | public override bool Equals(object obj) 43 | { 44 | if (ReferenceEquals(null, obj)) return false; 45 | if (ReferenceEquals(this, obj)) return true; 46 | var other = obj as ItemHeader; 47 | return other != null && Equals(other); 48 | } 49 | 50 | public override int GetHashCode() 51 | { 52 | return Id.GetHashCode(); 53 | } 54 | 55 | #endregion 56 | } 57 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Read/ItemVersionHeader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Sitecore.DataBlaster.Read 4 | { 5 | /// 6 | /// Describes a specific version of an item. 7 | /// 8 | public class ItemVersionHeader : ItemHeader 9 | { 10 | public string Language { get; set; } 11 | public int Version { get; set; } 12 | 13 | public ItemVersionHeader(Guid id, string name, string itemPath, 14 | Guid templateId, Guid masterId, Guid parentId, 15 | string language, int version, 16 | DateTime created, DateTime updated) 17 | : base(id, name, itemPath, templateId, masterId, parentId, created, updated) 18 | { 19 | if (string.IsNullOrEmpty(language)) throw new ArgumentNullException(nameof(language)); 20 | 21 | this.Language = language; 22 | this.Version = version; 23 | } 24 | 25 | #region Equality members 26 | 27 | protected bool Equals(ItemVersionHeader other) 28 | { 29 | return base.Equals(other) && 30 | string.Equals(Language, other.Language, StringComparison.InvariantCultureIgnoreCase) && 31 | Version == other.Version; 32 | } 33 | 34 | public override bool Equals(object obj) 35 | { 36 | if (ReferenceEquals(null, obj)) return false; 37 | if (ReferenceEquals(this, obj)) return true; 38 | var other = obj as ItemVersionHeader; 39 | return other != null && Equals(other); 40 | } 41 | 42 | public override int GetHashCode() 43 | { 44 | unchecked 45 | { 46 | int hashCode = base.GetHashCode(); 47 | hashCode = (hashCode * 397) ^ StringComparer.InvariantCultureIgnoreCase.GetHashCode(Language); 48 | hashCode = (hashCode * 397) ^ Version; 49 | return hashCode; 50 | } 51 | } 52 | 53 | #endregion 54 | } 55 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Read/Sql/GetDescendantHeaders.sql: -------------------------------------------------------------------------------- 1 | DECLARE @rootItemId UNIQUEIDENTIFIER = '11111111-1111-1111-1111-111111111111' 2 | DECLARE @rootItemPath VARCHAR(MAX) = '/sitecore' 3 | 4 | DECLARE @templateId UNIQUEIDENTIFIER = '455A3E98-A627-4B40-8035-E683A0331AC7' -- Template fields 5 | DECLARE @modifiedSince DATETIME = '2010-01-01' 6 | 7 | -- BEGIN: Lines before this line will be ignored when reading this embedded sql. 8 | 9 | ;WITH LookupCTE (Id, Path, ParentId) 10 | AS 11 | ( 12 | SELECT i.ID AS Id, CAST(@rootItemPath AS VARCHAR(MAX)) AS Path, i.ParentID AS ParentId 13 | FROM Items i 14 | WHERE i.ID = @rootItemId 15 | 16 | UNION ALL 17 | 18 | SELECT i.ID, CAST(cte.Path + '/' + i.Name AS VARCHAR(MAX)), i.ParentID 19 | FROM Items i 20 | JOIN LookupCTE cte ON 21 | cte.Id = i.ParentID 22 | ) 23 | 24 | SELECT 25 | cte.Id, 26 | i.Name, 27 | cte.Path AS ItemPath, 28 | i.TemplateID AS TemplateId, 29 | i.MasterId AS MasterId, 30 | cte.ParentId, 31 | i.Created, 32 | i.Updated 33 | FROM 34 | LookupCTE cte 35 | JOIN Items i ON 36 | i.ID = cte.Id 37 | WHERE 38 | 1 = 1 -- Next lines need to be able to be removed. 39 | AND i.TemplateID = @templateId 40 | AND i.TemplateID IN (@templateIdsCsv) 41 | AND i.Updated >= @modifiedSince -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Read/Sql/GetDescendantVersionHeaders.sql: -------------------------------------------------------------------------------- 1 | DECLARE @rootItemId UNIQUEIDENTIFIER = '11111111-1111-1111-1111-111111111111' 2 | DECLARE @rootItemPath VARCHAR(MAX) = '/sitecore' 3 | 4 | DECLARE @templateId UNIQUEIDENTIFIER = '455A3E98-A627-4B40-8035-E683A0331AC7' -- Template fields 5 | DECLARE @itemModifiedSince DATETIME = '2010-01-01' 6 | DECLARE @versionModifiedSince DATETIME = '2010-01-01' 7 | 8 | -- BEGIN: Lines before this line will be ignored when reading this embedded sql. 9 | 10 | ;WITH LookupCTE (Id, Path, ParentId) 11 | AS 12 | ( 13 | SELECT i.ID AS Id, CAST(@rootItemPath AS VARCHAR(MAX)) AS Path, i.ParentID AS ParentId 14 | FROM Items i 15 | WHERE i.ID = @rootItemId 16 | 17 | UNION ALL 18 | 19 | SELECT i.ID, CAST(cte.Path + '/' + i.Name AS VARCHAR(MAX)), i.ParentID 20 | FROM Items i 21 | JOIN LookupCTE cte ON 22 | cte.Id = i.ParentID 23 | ) 24 | 25 | SELECT 26 | cte.Id, 27 | i.Name, 28 | cte.Path AS ItemPath, 29 | i.TemplateID AS TemplateId, 30 | i.MasterId AS MasterId, 31 | cte.ParentId, 32 | vf.Language, 33 | vf.Version, 34 | i.Created, 35 | i.Updated 36 | FROM 37 | LookupCTE cte 38 | JOIN Items i ON 39 | i.ID = cte.Id 40 | CROSS APPLY 41 | ( 42 | SELECT DISTINCT vf.Language, vf.Version 43 | FROM dbo.VersionedFields vf 44 | WHERE vf.ItemId = i.ID 45 | AND vf.Updated >= @versionModifiedSince 46 | ) vf 47 | WHERE 48 | 1 = 1 -- Next lines need to be able to be removed. 49 | AND i.TemplateID = @templateId 50 | AND i.TemplateID IN (@templateIdsCsv) 51 | AND i.Updated >= @itemModifiedSince -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Read/Sql/GetDescendants.sql: -------------------------------------------------------------------------------- 1 | DECLARE @rootItemId UNIQUEIDENTIFIER = '11111111-1111-1111-1111-111111111111' 2 | DECLARE @rootItemPath VARCHAR(MAX) = '/sitecore' 3 | 4 | DECLARE @templateId UNIQUEIDENTIFIER = '455A3E98-A627-4B40-8035-E683A0331AC7' -- Template fields 5 | DECLARE @modifiedSince DATETIME = '2010-01-01' 6 | DECLARE @neverPublishFieldId UNIQUEIDENTIFIER = '9135200A-5626-4DD8-AB9D-D665B8C11748' 7 | DECLARE @neverPublish NVARCHAR(MAX) = '1' 8 | 9 | -- BEGIN: Lines before this line will be ignored when reading this embedded sql. 10 | 11 | DECLARE @NonPublishableItems TABLE (Id UNIQUEIDENTIFIER) 12 | INSERT INTO @NonPublishableItems (Id) 13 | SELECT DISTINCT np.ItemId 14 | FROM Descendants d 15 | JOIN SharedFields np ON np.ItemId = d.Descendant 16 | WHERE d.Ancestor = @rootItemId 17 | AND np.FieldId = @neverPublishFieldId 18 | AND np.Value = '1' 19 | 20 | ;WITH LookupCTE (Id, Path, ParentId) 21 | AS 22 | ( 23 | SELECT i.ID AS Id, CAST(@rootItemPath AS VARCHAR(MAX)) AS Path, i.ParentID AS ParentId 24 | FROM Items i 25 | WHERE i.ID = @rootItemId 26 | 27 | UNION ALL 28 | 29 | SELECT i.ID, CAST(cte.Path + '/' + i.Name AS VARCHAR(MAX)), i.ParentID 30 | FROM Items i 31 | JOIN LookupCTE cte ON 32 | cte.Id = i.ParentID 33 | WHERE @neverPublish = '1' AND i.ID NOT IN (SELECT Id FROM @NonPublishableItems) 34 | ) 35 | 36 | SELECT 37 | cte.Id AS ItemId, 38 | i.Name AS ItemName, 39 | cte.Path AS ItemPath, 40 | i.TemplateID AS TemplateId, 41 | i.MasterId AS MasterId, 42 | cte.ParentId, 43 | i.Created AS ItemCreated, 44 | i.Updated AS ItemUpdated, 45 | f.FieldId, 46 | f.Value AS FieldValue, 47 | f.Language, 48 | f.Version 49 | FROM 50 | LookupCTE cte 51 | JOIN Items i ON 52 | i.ID = cte.Id 53 | CROSS APPLY 54 | ( 55 | SELECT sf.FieldId, sf.Value AS Value, NULL AS Language, NULL AS Version 56 | FROM dbo.SharedFields sf 57 | WHERE sf.ItemId = i.ID 58 | AND sf.Updated >= @modifiedSince 59 | 60 | UNION ALL 61 | 62 | SELECT uf.FieldId, uf.Value AS Value, uf.Language AS Language, NULL AS Version 63 | FROM dbo.UnversionedFields uf 64 | WHERE uf.ItemId = i.ID 65 | AND uf.Updated >= @modifiedSince 66 | 67 | UNION ALL 68 | 69 | SELECT vf.FieldId, vf.Value AS Value, vf.Language AS Language, vf.Version 70 | FROM dbo.VersionedFields vf 71 | WHERE vf.ItemId = i.ID 72 | AND vf.Updated >= @modifiedSince 73 | ) f 74 | LEFT JOIN dbo.SharedFields np ON np.ItemId = i.ID AND np.FieldId = @neverPublishFieldId 75 | WHERE 76 | 1 = 1 -- Next lines need to be able to be removed. 77 | AND i.TemplateID = @templateId 78 | AND i.TemplateID IN (@templateIdsCsv) 79 | AND i.Updated >= @modifiedSince 80 | AND (np.Id IS NULL OR np.Value != @neverPublish) -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/SharedBulkField.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Sitecore.DataBlaster 5 | { 6 | /// 7 | /// Field which has the same value for all languages and all versions. 8 | /// 9 | public class SharedBulkField : BulkField 10 | { 11 | internal SharedBulkField(BulkItem item, Guid id, string value, 12 | Func blob = null, bool isBlob = false, string name = null) 13 | : base(item, id, value, blob, isBlob, name) 14 | { 15 | } 16 | 17 | internal override BulkField CopyTo(BulkItem targetItem) 18 | => new SharedBulkField(targetItem, Id, Value, Blob, IsBlob, Name); 19 | } 20 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Sitecore.DataBlaster.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | net48 4 | True 5 | false 6 | Sitecore.DataBlaster 7 | Sitecore.DataBlaster 8 | High throughput bulk database access for Sitecore. 9 | sitecore items 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/UnversionedBulkField.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Sitecore.DataBlaster 5 | { 6 | /// 7 | /// Field which has specific values per language but shares the values across versions. 8 | /// 9 | public class UnversionedBulkField : BulkField 10 | { 11 | public string Language { get; private set; } 12 | 13 | internal UnversionedBulkField(BulkItem item, Guid id, string language, string value, 14 | Func blob = null, bool isBlob = false, string name = null) 15 | : base(item, id, value, blob, isBlob, name) 16 | { 17 | if (language == null) throw new ArgumentNullException("language"); 18 | 19 | this.Language = language; 20 | } 21 | 22 | internal override BulkField CopyTo(BulkItem targetItem) 23 | => new UnversionedBulkField(targetItem, Id, Language, Value, Blob, IsBlob, Name); 24 | } 25 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/CacheUtil.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Reflection; 6 | using System.Threading.Tasks; 7 | using Sitecore.Caching; 8 | using Sitecore.Configuration; 9 | using Sitecore.Data; 10 | using Sitecore.Data.DataProviders.Sql; 11 | using Sitecore.Data.Items; 12 | 13 | namespace Sitecore.DataBlaster.Util 14 | { 15 | public class CacheUtil 16 | { 17 | public virtual void RemoveItemFromCaches(Database database, ID itemId, ID parentId, string itemPath = null) 18 | { 19 | if (database == null) throw new ArgumentNullException(nameof(database)); 20 | if (itemId == (ID)null) throw new ArgumentNullException(nameof(itemId)); 21 | if (parentId == (ID)null) throw new ArgumentNullException(nameof(parentId)); 22 | 23 | // Parent needs to be cleared because it holds a reference to its children. 24 | var parentPath = itemPath?.Substring(0, itemPath.LastIndexOf("/", StringComparison.Ordinal)); 25 | RemoveItemFromCaches(database, parentId, parentPath); 26 | 27 | RemoveItemFromCaches(database, itemId, itemPath); 28 | } 29 | 30 | public virtual void RemoveFromCaches(Item item) 31 | { 32 | if (item == null) throw new ArgumentNullException(nameof(item)); 33 | 34 | RemoveItemFromCaches(item.Database, item.ID, item.ParentID, item.Paths.Path); 35 | } 36 | 37 | public virtual void RemoveItemsFromCachesInBulk(Database database, 38 | IEnumerable> itemIdParentIdAndItemPaths) 39 | { 40 | if (database == null) throw new ArgumentNullException(nameof(database)); 41 | if (itemIdParentIdAndItemPaths == null) throw new ArgumentNullException(nameof(itemIdParentIdAndItemPaths)); 42 | 43 | // In previous versions of this method, we did some crazy custom cache clearing to optimize performance. 44 | // Since 8.2 Sitecore now consistently supports cache key indexing, 45 | // so if you have performance issues with partial cache clears, please set enable following settings: 46 | // 47 | // 48 | // 49 | var maxDegreeOfParallelism = Environment.ProcessorCount > 1 ? Environment.ProcessorCount * 0.666 : 1; 50 | Parallel.ForEach(itemIdParentIdAndItemPaths, 51 | new ParallelOptions 52 | { 53 | MaxDegreeOfParallelism = (int)maxDegreeOfParallelism 54 | }, 55 | x => { RemoveItemFromCaches(database, x.Item1, x.Item2, x.Item3); }); 56 | } 57 | 58 | protected virtual void RemoveItemFromCaches(Database database, ID itemId, string itemPath = null) 59 | { 60 | if (database == null) throw new ArgumentNullException(nameof(database)); 61 | if (itemId == (ID)null) throw new ArgumentNullException(nameof(itemId)); 62 | 63 | var dbCaches = database.Caches; 64 | 65 | // To clear the item paths cache we need an actual item, so we get on from the caches to avoid IO. 66 | var item = GetCachedItems(dbCaches, itemId).FirstOrDefault(); 67 | 68 | dbCaches.DataCache.RemoveItemInformation(itemId); 69 | dbCaches.ItemCache.RemoveItem(itemId); // Will also remove the items from the filter cache per site. 70 | dbCaches.StandardValuesCache.RemoveKeysContaining(itemId.ToString()); 71 | if (!string.IsNullOrWhiteSpace(itemPath)) 72 | dbCaches.PathCache.RemoveKeysContaining(itemPath.ToLower()); 73 | if (item != null) 74 | dbCaches.ItemPathsCache.InvalidateCache(item); 75 | 76 | var prefetch = CacheManager.FindCacheByName("SqlDataProvider - Prefetch data(" + database.Name + ")"); 77 | prefetch?.Remove(itemId); 78 | } 79 | 80 | protected virtual IEnumerable GetCachedItems(DatabaseCaches caches, ID itemId) 81 | { 82 | if (caches == null) throw new ArgumentNullException(nameof(caches)); 83 | if (itemId == (ID)null) throw new ArgumentNullException(nameof(itemId)); 84 | 85 | var info = caches.DataCache.GetItemInformation(itemId); 86 | var languages = info?.GetLanguages(); 87 | if (languages == null) yield break; 88 | 89 | foreach (var language in languages) 90 | { 91 | var versions = info.GetVersions(language); 92 | if (versions == null) continue; 93 | 94 | foreach (var version in versions) 95 | { 96 | var item = caches.ItemCache.GetItem(itemId, language, version); 97 | if (item != null) yield return item; 98 | } 99 | } 100 | } 101 | 102 | public virtual void ClearLanguageCache(Database database) 103 | { 104 | if (database == null) throw new ArgumentNullException(nameof(database)); 105 | 106 | // Clear protected property 'Languages' on the SqlDataProvider for the database. 107 | var field = typeof(SqlDataProvider).GetProperty("Languages", 108 | BindingFlags.Instance | BindingFlags.NonPublic); 109 | Debug.Assert(field != null, "Protected field not found in Sitecore, did you change Sitecore version?"); 110 | foreach (var provider in database.GetDataProviders().OfType()) 111 | { 112 | field.SetValue(provider, null); 113 | } 114 | 115 | // Remove the database from the language cache. 116 | var cache = CacheManager.GetNamedInstance("LanguageProvider - Languages", Settings.Caching.SmallCacheSize, 117 | true); 118 | cache.Remove(database.Name); 119 | } 120 | 121 | public virtual void ClearCaches(Database database) 122 | { 123 | if (database == null) throw new ArgumentNullException(nameof(database)); 124 | 125 | var current = Context.Database; 126 | try 127 | { 128 | // http://www.theinsidecorner.com/en/Developers/Caching/CacheStates/ClearAllCaches 129 | // http://stackoverflow.com/questions/3713031/sitecore-clear-cache-programatically 130 | 131 | Context.Database = database; 132 | Context.Database.Engines.TemplateEngine.Reset(); 133 | Context.ClientData.RemoveAll(); 134 | CacheManager.ClearAllCaches(); 135 | } 136 | finally 137 | { 138 | if (current != null) Context.Database = current; 139 | } 140 | } 141 | } 142 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/Chain.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | 5 | namespace Sitecore.DataBlaster.Util 6 | { 7 | /// 8 | /// Simple generic implementation of a chain of processors. 9 | /// 10 | /// 11 | public class Chain : ICollection 12 | { 13 | private readonly IList _processors = new List(); 14 | 15 | public TIo Execute(TIo result, Func processorFunction, bool breakOnDefault = false) 16 | { 17 | foreach (var processor in _processors) 18 | { 19 | result = processorFunction(processor, result); 20 | 21 | if (breakOnDefault && Equals(result, default(TIo))) 22 | return result; 23 | } 24 | return result; 25 | } 26 | 27 | public TResult Execute(Func processorFunction, bool breakOnDefault = false) 28 | { 29 | return Execute(default(TResult), (p, r) => processorFunction(p), breakOnDefault: breakOnDefault); 30 | } 31 | 32 | public void Execute(Action processorFunction) 33 | { 34 | foreach (var processor in _processors) 35 | { 36 | processorFunction(processor); 37 | } 38 | } 39 | 40 | #region Modifiers 41 | 42 | public Chain Add() 43 | where TProcessor : T, new() 44 | { 45 | _processors.Add(new TProcessor()); 46 | return this; 47 | } 48 | 49 | public void InsertBefore(T newProcessor, bool matchExactType = true) 50 | where TReference : T 51 | { 52 | var index = IndexOf(typeof(TReference), matchExactType); 53 | if (index == null) 54 | throw new ArgumentException($"Unable to find a proccessor of type '{typeof(TReference).Name}'."); 55 | 56 | _processors.Insert(index.Value, newProcessor); 57 | } 58 | 59 | public void InsertBefore(bool matchExactType = true) 60 | where TReference : T 61 | where TNew : T, new() 62 | { 63 | InsertBefore(new TNew(), matchExactType: matchExactType); 64 | } 65 | 66 | public void InsertAfter(T newProcessor, bool matchExactType = true) 67 | where TReference : T 68 | { 69 | var index = IndexOf(typeof(TReference), matchExactType); 70 | if (index == null) 71 | throw new ArgumentException($"Unable to find a proccessor of type '{typeof(TReference).Name}'."); 72 | 73 | if (index.Value == Count - 1) 74 | _processors.Add(newProcessor); 75 | else 76 | _processors.Insert(index.Value + 1, newProcessor); 77 | } 78 | 79 | public void InsertAfter(bool matchExactType = true) 80 | where TReference : T 81 | where TNew : T, new() 82 | { 83 | InsertAfter(new TNew(), matchExactType: matchExactType); 84 | } 85 | 86 | public bool Replace(T replacement, bool matchExactType = true) 87 | where TSource : T 88 | { 89 | var index = IndexOf(typeof(TSource), matchExactType); 90 | if (index == null) return false; 91 | 92 | _processors[index.Value] = replacement; 93 | return true; 94 | } 95 | 96 | public bool Replace(bool matchExactType = true) 97 | where TSource : T 98 | where TTarget : T, new() 99 | { 100 | return Replace(new TTarget(), matchExactType: matchExactType); 101 | } 102 | 103 | public void Remove(bool matchExactType = true) 104 | where TProcessor : T 105 | { 106 | int? index; 107 | while ((index = IndexOf(typeof(TProcessor), matchExactType)) != null) 108 | { 109 | _processors.RemoveAt(index.Value); 110 | } 111 | } 112 | 113 | private int? IndexOf(Type processorType, bool matchExactType) 114 | { 115 | for (var i = 0; i < _processors.Count; i++) 116 | { 117 | var processor = _processors[i]; 118 | if (matchExactType && processor.GetType() == processorType || processorType.IsInstanceOfType(processor)) 119 | { 120 | return i; 121 | } 122 | } 123 | return null; 124 | } 125 | 126 | #endregion 127 | 128 | #region ICollection members 129 | 130 | public int Count => _processors.Count; 131 | 132 | public bool IsReadOnly => _processors.IsReadOnly; 133 | 134 | public void Add(T item) 135 | { 136 | _processors.Add(item); 137 | } 138 | 139 | public void Clear() 140 | { 141 | _processors.Clear(); 142 | } 143 | 144 | public bool Contains(T item) 145 | { 146 | if (item == null) throw new ArgumentNullException(nameof(item)); 147 | return _processors.Contains(item); 148 | } 149 | 150 | public void CopyTo(T[] array, int arrayIndex) 151 | { 152 | _processors.CopyTo(array, arrayIndex); 153 | } 154 | 155 | public bool Remove(T item) 156 | { 157 | return _processors.Remove(item); 158 | } 159 | 160 | public IEnumerator GetEnumerator() 161 | { 162 | return _processors.GetEnumerator(); 163 | } 164 | 165 | IEnumerator IEnumerable.GetEnumerator() 166 | { 167 | return ((IEnumerable)_processors).GetEnumerator(); 168 | } 169 | 170 | #endregion 171 | } 172 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/GuidUtility.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delawarePro/sitecore-data-blaster/77a8cf868e828c80e3b17ddb2ed7e1a0f0337e54/src/Sitecore.DataBlaster/Util/GuidUtility.cs -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/ISearchIndexExtensions.cs: -------------------------------------------------------------------------------- 1 | using Sitecore.ContentSearch; 2 | using Sitecore.ContentSearch.Summary; 3 | 4 | namespace Sitecore.DataBlaster.Util 5 | { 6 | public static class ISearchIndexExtensions 7 | { 8 | /// 9 | /// Sitecore moved summary from the interface to the base class. 10 | /// but we only have access to the index from the . 11 | /// 12 | public static ISearchIndexSummary RequestSummary(this ISearchIndex index) 13 | { 14 | return index is IIndexSummarySource summarSource 15 | ? summarSource.GetClient().RequestSummary() 16 | : null; 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/LogExtensions.cs: -------------------------------------------------------------------------------- 1 | using log4net; 2 | using log4net.spi; 3 | 4 | namespace Sitecore.DataBlaster.Util 5 | { 6 | public static class LogExtensions 7 | { 8 | public static void Trace(this ILog log, string message) 9 | { 10 | if (!log.Logger.IsEnabledFor(Level.TRACE)) return; 11 | log.Logger.Log(null, Level.TRACE, message, null); 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/Sql/AbstractEnumeratorReader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.Data.Common; 5 | 6 | namespace Sitecore.DataBlaster.Util.Sql 7 | { 8 | /// 9 | /// Helps mapping an IEnumerator to an IDataReader so it can e.g. be used in bulk copy. 10 | /// 11 | /// Type to enumerate. 12 | public abstract class AbstractEnumeratorReader : DbDataReader 13 | { 14 | private readonly Lazy> _lazyEnumerator; 15 | 16 | private IEnumerator Enumerator => _lazyEnumerator.Value; 17 | 18 | public T Current => Enumerator.Current; 19 | 20 | private bool _closed; 21 | 22 | protected AbstractEnumeratorReader(Func> enumeratorFactory) 23 | { 24 | if (enumeratorFactory == null) 25 | throw new ArgumentNullException(nameof(enumeratorFactory)); 26 | 27 | _lazyEnumerator = new Lazy>(enumeratorFactory); 28 | } 29 | 30 | #region IDataReader Members 31 | 32 | /// 33 | /// Closes the Object. 34 | /// 35 | public override void Close() 36 | { 37 | // Read to end. 38 | while (Read()) 39 | { 40 | } 41 | _closed = true; 42 | } 43 | 44 | /// 45 | /// Gets a value indicating the depth of nesting for the current row. 46 | /// 47 | /// 48 | /// 49 | /// The level of nesting. 50 | /// 51 | public override int Depth => 0; 52 | 53 | /// 54 | /// Gets a value indicating whether the data reader is closed. 55 | /// 56 | /// 57 | /// true if the data reader is closed; otherwise, false. 58 | /// 59 | public override bool IsClosed => _closed; 60 | 61 | /// 62 | /// Advances the data reader to the next result, when reading the results of batch SQL statements. 63 | /// 64 | /// 65 | /// true if there are more rows; otherwise, false. 66 | /// 67 | public override bool NextResult() 68 | { 69 | return Read(); 70 | } 71 | 72 | /// 73 | /// Advances the to the next record. 74 | /// 75 | /// 76 | /// true if there are more rows; otherwise, false. 77 | /// 78 | public override bool Read() 79 | { 80 | return Enumerator.MoveNext(); 81 | } 82 | 83 | /// 84 | /// Gets the number of rows changed, inserted, or deleted by execution of the SQL statement. 85 | /// 86 | /// 87 | /// 88 | /// The number of rows changed, inserted, or deleted; 0 if no rows were affected or the statement failed; and -1 for SELECT statements. 89 | /// 90 | public override int RecordsAffected => 0; 91 | 92 | #endregion 93 | 94 | #region IDataRecord Members 95 | 96 | public override bool GetBoolean(int i) 97 | { 98 | return GetFieldValue(i); 99 | } 100 | 101 | public override byte GetByte(int i) 102 | { 103 | return GetFieldValue(i); 104 | } 105 | 106 | public override long GetBytes(int i, long fieldOffset, byte[] buffer, int bufferoffset, int length) 107 | { 108 | throw new NotSupportedException(); 109 | } 110 | 111 | public override char GetChar(int i) 112 | { 113 | return GetFieldValue(i); 114 | } 115 | 116 | public override long GetChars(int i, long fieldoffset, char[] buffer, int bufferoffset, int length) 117 | { 118 | throw new NotSupportedException(); 119 | } 120 | 121 | public override string GetDataTypeName(int i) 122 | { 123 | throw new NotSupportedException(); 124 | } 125 | 126 | public override DateTime GetDateTime(int i) 127 | { 128 | return GetFieldValue(i); 129 | } 130 | 131 | public override decimal GetDecimal(int i) 132 | { 133 | return GetFieldValue(i); 134 | } 135 | 136 | public override double GetDouble(int i) 137 | { 138 | return GetFieldValue(i); 139 | } 140 | 141 | public override Type GetFieldType(int i) 142 | { 143 | throw new NotSupportedException(); 144 | } 145 | 146 | public override float GetFloat(int i) 147 | { 148 | return GetFieldValue(i); 149 | } 150 | 151 | public override Guid GetGuid(int i) 152 | { 153 | return GetFieldValue(i); 154 | } 155 | 156 | public override short GetInt16(int i) 157 | { 158 | return GetFieldValue(i); 159 | } 160 | 161 | public override int GetInt32(int i) 162 | { 163 | return GetFieldValue(i); 164 | } 165 | 166 | public override long GetInt64(int i) 167 | { 168 | return GetFieldValue(i); 169 | } 170 | 171 | public override string GetName(int i) 172 | { 173 | throw new NotSupportedException(); 174 | } 175 | 176 | public override int GetOrdinal(string name) 177 | { 178 | throw new NotSupportedException(); 179 | } 180 | 181 | public override string GetString(int i) 182 | { 183 | return GetFieldValue(i); 184 | } 185 | 186 | public override int GetValues(object[] values) 187 | { 188 | throw new NotSupportedException(); 189 | } 190 | 191 | public override bool IsDBNull(int i) 192 | { 193 | return DBNull.Value.Equals(GetValue(i)); 194 | } 195 | 196 | public override object this[string name] 197 | { 198 | get { throw new NotSupportedException(); } 199 | } 200 | 201 | public override object this[int i] => GetValue(i); 202 | 203 | #endregion 204 | 205 | #region DbDataReader Members 206 | 207 | public override bool HasRows => true; 208 | 209 | public override IEnumerator GetEnumerator() 210 | { 211 | return new DbEnumerator(this, true); 212 | } 213 | 214 | #endregion 215 | 216 | #region IDisposable Members 217 | 218 | /// 219 | /// Releases unmanaged and - optionally - managed resources 220 | /// 221 | /// true to release both managed and unmanaged resources; false to release only unmanaged resources. 222 | protected override void Dispose(bool disposing) 223 | { 224 | if (disposing) 225 | { 226 | Close(); 227 | Enumerator.Dispose(); 228 | } 229 | } 230 | 231 | #endregion 232 | } 233 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/Sql/SqlContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Data.SqlClient; 4 | using System.Diagnostics.CodeAnalysis; 5 | using System.IO; 6 | using System.Linq; 7 | using System.Text.RegularExpressions; 8 | 9 | namespace Sitecore.DataBlaster.Util.Sql 10 | { 11 | /// 12 | /// Holder class for a connection and a transaction which supports working with embedded sql files. 13 | /// 14 | public class SqlContext 15 | { 16 | private readonly Type _defaultSubject; 17 | 18 | public SqlConnection Connection { get; private set; } 19 | public SqlTransaction Transaction { get; set; } 20 | 21 | public SqlContext(SqlConnection connection, Type defaultSubject = null) 22 | { 23 | _defaultSubject = defaultSubject; 24 | if (connection == null) throw new ArgumentNullException(nameof(connection)); 25 | Connection = connection; 26 | } 27 | 28 | #region Reading embedded SQL files 29 | 30 | protected virtual StreamReader GetEmbeddedSqlReader(string relativePath, Type subject = null) 31 | { 32 | if (subject == null) subject = _defaultSubject ?? this.GetType(); 33 | 34 | var stream = subject.Assembly.GetManifestResourceStream(subject, relativePath); 35 | if (stream == null) 36 | throw new ArgumentException( 37 | $"Could not locate embedded resource '{subject.Namespace}.{relativePath}' in '{subject.Assembly}'."); 38 | 39 | return new StreamReader(stream); 40 | } 41 | 42 | protected virtual string GetEmbeddedSql(string relativePath, Type subject = null) 43 | { 44 | using (var reader = GetEmbeddedSqlReader(relativePath, subject)) 45 | { 46 | return reader.ReadToEnd(); 47 | } 48 | } 49 | 50 | public virtual string GetEmbeddedSql(string relativePath) 51 | { 52 | return GetEmbeddedSql(relativePath, typeof(T)); 53 | } 54 | 55 | #endregion 56 | 57 | #region Manipulation SQL files 58 | 59 | public virtual string ReplaceOneLineSqlStringParameter(string sql, string name, string value) 60 | { 61 | return Regex.Replace(sql, 62 | @"(?<=DECLARE\s+" + name + @"\s+.*?=\s?').*?(?=')", 63 | value); 64 | } 65 | 66 | public virtual string ReplaceOneLineSqlBitParameter(string sql, string name, bool value) 67 | { 68 | return Regex.Replace(sql, 69 | @"(?<=DECLARE\s+" + name + @"\s+.*?=\s)[01]{1}", 70 | value ? "1" : "0"); 71 | } 72 | 73 | #endregion 74 | 75 | #region Handling SQL files as filterable lines 76 | 77 | public IEnumerable GetEmbeddedSqlLines(string relativePath, Type subject = null) 78 | { 79 | using (var reader = GetEmbeddedSqlReader(relativePath, subject)) 80 | { 81 | var begun = false; 82 | 83 | while (reader.Peek() >= 0) 84 | { 85 | var line = reader.ReadLine(); 86 | if (!begun && line.ToUpper().Contains("-- BEGIN")) 87 | { 88 | begun = true; 89 | } 90 | else if (begun) 91 | { 92 | yield return line; 93 | } 94 | } 95 | } 96 | } 97 | 98 | #endregion 99 | 100 | #region Command execution 101 | 102 | [SuppressMessage("Microsoft.Security", "CA2100", Justification = "No user parameters")] 103 | public virtual SqlCommand NewSqlCommand(string sql, int commandTimeout = int.MaxValue) 104 | { 105 | var command = new SqlCommand(sql, Connection) 106 | { 107 | CommandTimeout = int.MaxValue 108 | }; 109 | if (Transaction != null) 110 | command.Transaction = Transaction; 111 | 112 | return command; 113 | } 114 | 115 | public virtual void ExecuteSql(string sql, int commandTimeout = int.MaxValue, 116 | Action commandProcessor = null, bool splitOnGoCommands = false) 117 | { 118 | var commands = splitOnGoCommands 119 | ? Regex.Split(sql, @"^\s*GO\s*$", RegexOptions.Multiline | RegexOptions.IgnoreCase) 120 | : new[] { sql }; 121 | 122 | foreach (var command in commands) 123 | { 124 | if (string.IsNullOrWhiteSpace(command)) continue; 125 | 126 | using (var cmd = NewSqlCommand(command, commandTimeout: commandTimeout)) 127 | { 128 | commandProcessor?.Invoke(cmd); 129 | cmd.ExecuteNonQuery(); 130 | } 131 | } 132 | } 133 | 134 | public SqlDataReader ExecuteReader(string sql, int commandTimeout = int.MaxValue, 135 | Action commandProcessor = null) 136 | { 137 | using (var cmd = NewSqlCommand(sql, commandTimeout: commandTimeout)) 138 | { 139 | commandProcessor?.Invoke(cmd); 140 | return cmd.ExecuteReader(); 141 | } 142 | } 143 | 144 | public SqlDataReader ExecuteReader(IEnumerable sqlLines, int commandTimeout = int.MaxValue, 145 | Action commandProcessor = null) 146 | { 147 | return ExecuteReader(string.Join("\n", sqlLines), commandTimeout, commandProcessor); 148 | } 149 | 150 | public SqlDataReader ExecuteReader(IEnumerable sqlLines, int commandTimeout = int.MaxValue, 151 | Action commandProcessor = null) 152 | { 153 | return ExecuteReader(string.Join("\n", sqlLines.Select(x => x.ToString())), commandTimeout, 154 | commandProcessor); 155 | } 156 | 157 | #endregion 158 | } 159 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/Sql/SqlContextExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | 5 | namespace Sitecore.DataBlaster.Util.Sql 6 | { 7 | public static class SqlLineExtensions 8 | { 9 | public static IEnumerable RemoveParameterLineIf(this IEnumerable sqlLines, 10 | Func predicate, string parameterName) 11 | { 12 | return sqlLines 13 | .Where(line => 14 | !predicate() || line.ToString().IndexOf(parameterName, StringComparison.OrdinalIgnoreCase) < 0); 15 | } 16 | 17 | public static IEnumerable ExpandParameterLineIf(this IEnumerable sqlLines, 18 | Func predicate, 19 | string parameterName, string parameterValue) 20 | { 21 | foreach (var line in sqlLines) 22 | { 23 | if (!predicate()) 24 | { 25 | yield return line; 26 | continue; 27 | } 28 | 29 | var pos = line.ToString().IndexOf(parameterName, StringComparison.OrdinalIgnoreCase); 30 | if (pos < 0) 31 | { 32 | yield return line; 33 | continue; 34 | } 35 | 36 | yield return line.ToString().Substring(0, pos) + parameterValue + 37 | line.ToString().Substring(pos + parameterName.Length); 38 | } 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/Util/Sql/SqlLine.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Sitecore.DataBlaster.Util.Sql 4 | { 5 | public class SqlLine 6 | { 7 | private readonly string _line; 8 | 9 | public SqlLine(string line) 10 | { 11 | if (line == null) throw new ArgumentNullException(nameof(line)); 12 | _line = line; 13 | } 14 | 15 | public override string ToString() 16 | { 17 | return _line; 18 | } 19 | 20 | public static implicit operator SqlLine(string line) 21 | { 22 | return line == null ? null : new SqlLine(line); 23 | } 24 | 25 | public static implicit operator string(SqlLine sqlLine) 26 | { 27 | return sqlLine.ToString(); 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /src/Sitecore.DataBlaster/VersionedBulkField.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Sitecore.DataBlaster 5 | { 6 | /// 7 | /// Field which has specific values per language and version. 8 | /// 9 | public class VersionedBulkField : UnversionedBulkField 10 | { 11 | public int Version { get; private set; } 12 | 13 | internal VersionedBulkField(BulkItem item, Guid id, string language, int version, string value, 14 | Func blob = null, bool isBlob = false, string name = null) 15 | : base(item, id, language, value, blob, isBlob, name) 16 | { 17 | if (version <= 0) throw new ArgumentException("Version should be greater than 0.", "version"); 18 | 19 | Version = version; 20 | } 21 | 22 | internal override BulkField CopyTo(BulkItem targetItem) 23 | => new VersionedBulkField(targetItem, Id, Language, Version, Value, Blob, IsBlob, Name); 24 | } 25 | } -------------------------------------------------------------------------------- /src/Unicorn.DataBlaster/App_Config/Unicorn/Unicorn.DataBlaster.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | true 23 | 24 | 25 | true 26 | 27 | 29 | false 30 | 31 | 33 | false 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/Unicorn.DataBlaster/Logging/SitecoreAndUnicornLog.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using log4net; 3 | using log4net.spi; 4 | 5 | namespace Unicorn.DataBlaster.Logging 6 | { 7 | /// 8 | /// Logs both to standard log4net target and Unicorn target. 9 | /// 10 | public class SitecoreAndUnicornLog : ILog 11 | { 12 | private readonly Unicorn.Logging.ILogger _unicornLogger; 13 | private readonly ILog _sitecoreLog; 14 | 15 | public SitecoreAndUnicornLog(ILog sitecoreLog, Unicorn.Logging.ILogger unicornLogger) 16 | { 17 | if (sitecoreLog == null) throw new ArgumentNullException(nameof(sitecoreLog)); 18 | if (unicornLogger == null) throw new ArgumentNullException(nameof(unicornLogger)); 19 | 20 | _sitecoreLog = sitecoreLog; 21 | _unicornLogger = unicornLogger; 22 | } 23 | 24 | #region Forward to Sitecore and Unicorn log 25 | 26 | public bool IsDebugEnabled => _sitecoreLog.IsDebugEnabled; 27 | 28 | public bool IsInfoEnabled => _sitecoreLog.IsInfoEnabled; 29 | 30 | public bool IsWarnEnabled => _sitecoreLog.IsWarnEnabled; 31 | 32 | public bool IsErrorEnabled => _sitecoreLog.IsErrorEnabled; 33 | 34 | public bool IsFatalEnabled => _sitecoreLog.IsFatalEnabled; 35 | 36 | public ILogger Logger => _sitecoreLog.Logger; 37 | 38 | public void Debug(object message) 39 | { 40 | if (message == null) return; 41 | 42 | _sitecoreLog.Debug(message); 43 | _unicornLogger.Debug(message.ToString()); 44 | } 45 | 46 | public void Debug(object message, Exception t) 47 | { 48 | if (message == null) return; 49 | 50 | _sitecoreLog.Debug(message, t); 51 | _unicornLogger.Debug(message.ToString()); 52 | } 53 | 54 | public void Info(object message) 55 | { 56 | if (message == null) return; 57 | 58 | _sitecoreLog.Info(message); 59 | _unicornLogger.Info(message.ToString()); 60 | } 61 | 62 | public void Info(object message, Exception t) 63 | { 64 | if (message == null) return; 65 | 66 | _sitecoreLog.Info(message, t); 67 | _unicornLogger.Info(message.ToString()); 68 | } 69 | 70 | public void Warn(object message) 71 | { 72 | if (message == null) return; 73 | 74 | _sitecoreLog.Warn(message); 75 | _unicornLogger.Warn(message.ToString()); 76 | } 77 | 78 | public void Warn(object message, Exception t) 79 | { 80 | if (message == null) return; 81 | 82 | _sitecoreLog.Warn(message, t); 83 | _unicornLogger.Warn(message.ToString()); 84 | } 85 | 86 | public void Error(object message) 87 | { 88 | if (message == null) return; 89 | 90 | _sitecoreLog.Error(message); 91 | _unicornLogger.Error(message.ToString()); 92 | } 93 | 94 | public void Error(object message, Exception t) 95 | { 96 | if (message == null) return; 97 | 98 | _sitecoreLog.Error(message, t); 99 | _unicornLogger.Error(message.ToString()); 100 | } 101 | 102 | public void Fatal(object message) 103 | { 104 | if (message == null) return; 105 | 106 | _sitecoreLog.Fatal(message); 107 | _unicornLogger.Error(message.ToString()); 108 | } 109 | 110 | public void Fatal(object message, Exception t) 111 | { 112 | if (message == null) return; 113 | 114 | _sitecoreLog.Fatal(message, t); 115 | _unicornLogger.Error(message.ToString()); 116 | } 117 | 118 | #endregion 119 | } 120 | } -------------------------------------------------------------------------------- /src/Unicorn.DataBlaster/Sync/DataBlasterParameters.cs: -------------------------------------------------------------------------------- 1 | using Sitecore.DataBlaster.Load; 2 | 3 | namespace Unicorn.DataBlaster.Sync 4 | { 5 | /// 6 | /// Parameters that allow tweaking the data blaster integration. 7 | /// 8 | public class DataBlasterParameters 9 | { 10 | /// 11 | /// Explicitly enable the data blaster integration. 12 | /// 13 | public bool EnableDataBlaster { get; set; } 14 | 15 | /// 16 | /// Flag to only stage data in database, used for debugging the data blaster. 17 | /// 18 | public bool StageDataWithoutProcessing { get; set; } 19 | 20 | /// 21 | /// Flag to skip post processing of Unicorn configuration like e.g. users. 22 | /// 23 | public bool SkipUnicornSyncComplete { get; set; } 24 | 25 | /// 26 | /// Flag to skip post processing of Unicorn like e.g. publishing. 27 | /// 28 | public bool SkipUnicornSyncEnd { get; set; } 29 | 30 | /// 31 | /// Force this BulkLoadAction to be used during item extraction instead of the BulkLoadAction derived from the configured evaluator. 32 | /// 33 | public BulkLoadAction? ForceBulkLoadAction { get; set; } 34 | 35 | /// 36 | /// Should be null or a Sitecore item path. 37 | /// When set, only items that are descendants of the configured path will be loaded during sync (deserialization). 38 | /// Useful for e.g. revert tree. 39 | /// 40 | public string AncestorFilter { get; set; } 41 | } 42 | } -------------------------------------------------------------------------------- /src/Unicorn.DataBlaster/Sync/ItemExtractor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Rainbow.Model; 5 | using Rainbow.Storage; 6 | using Sitecore.DataBlaster.Load; 7 | using Unicorn.Configuration; 8 | using Unicorn.Data; 9 | using Unicorn.Evaluators; 10 | using Unicorn.Predicates; 11 | 12 | namespace Unicorn.DataBlaster.Sync 13 | { 14 | /// 15 | /// Extracts items from Unicorn. 16 | /// 17 | public class ItemExtractor 18 | { 19 | private ItemMapper ItemMapper { get; } 20 | 21 | public ItemExtractor(ItemMapper itemMapper = null) 22 | { 23 | ItemMapper = itemMapper ?? new ItemMapper(); 24 | } 25 | 26 | public virtual IEnumerable ExtractBulkItems(BulkLoadContext context, DataBlasterParameters parameters, 27 | IConfiguration[] configurations, string database) 28 | { 29 | var uniqueItems = new HashSet(); 30 | 31 | return GetTreeRoots(configurations, database) 32 | .SelectMany(tr => 33 | { 34 | var action = GetBulkLoadAction(parameters, tr.Item1, tr.Item2); 35 | var dataStore = tr.Item1.Resolve(); 36 | 37 | return dataStore.GetByPath(tr.Item2.Path, database) 38 | .SelectMany(i => GetSelfAndAllDescendants(dataStore, i)) 39 | // For example '/sitecore/layout/Layouts/User Defined' can occur more than once 40 | // because it has children from different configurations. 41 | // Make sure we add the item itself only once. 42 | .Where(item => uniqueItems.Add(item.Id)) 43 | .Where(item => string.IsNullOrEmpty(parameters.AncestorFilter) || item.Path.StartsWith(parameters.AncestorFilter, StringComparison.OrdinalIgnoreCase)) 44 | .Select(y => ItemMapper.ToBulkLoadItem(y, context, action)); 45 | }); 46 | } 47 | 48 | protected virtual IEnumerable GetDatabaseNames(IEnumerable configurations) 49 | { 50 | return configurations 51 | .SelectMany(c => c.Resolve().GetRootPaths().Select(rp => rp.DatabaseName)) 52 | .Distinct(); 53 | } 54 | 55 | protected virtual IEnumerable> GetTreeRoots( 56 | IEnumerable configurations, string db) 57 | { 58 | return configurations 59 | .Select(x => new 60 | { 61 | Configuration = x, 62 | TreeRoots = x.Resolve() 63 | .GetRootPaths() 64 | .Where(rp => rp.DatabaseName.Equals(db, StringComparison.OrdinalIgnoreCase)) 65 | .Select(tr => tr as PresetTreeRoot) 66 | }) 67 | .SelectMany(x => 68 | x.TreeRoots.Select(tr => new Tuple(x.Configuration, tr))); 69 | } 70 | 71 | protected virtual BulkLoadAction GetBulkLoadAction(DataBlasterParameters parameters, IConfiguration configuration, PresetTreeRoot treeRoot) 72 | { 73 | if (parameters.ForceBulkLoadAction.HasValue) 74 | return parameters.ForceBulkLoadAction.Value; 75 | 76 | var evaluator = configuration.Resolve(); 77 | 78 | if (evaluator is SerializedAsMasterEvaluator) 79 | { 80 | // Only revert the tree when there are no exclusions for this tree root. 81 | return treeRoot.Exclusions == null || treeRoot.Exclusions.Count == 0 82 | ? BulkLoadAction.RevertTree 83 | : BulkLoadAction.Revert; 84 | } 85 | 86 | if (evaluator is NewItemOnlyEvaluator) 87 | { 88 | return BulkLoadAction.AddItemOnly; 89 | } 90 | 91 | //if (evaluator is AddOnlyEvaluator) 92 | //{ 93 | // return BulkLoadAction.AddOnly; 94 | //} 95 | 96 | throw new ArgumentException($"Unknown evaluator type: '{evaluator.GetType().Name}'"); 97 | } 98 | 99 | private IEnumerable GetSelfAndAllDescendants(IDataStore targetDataStore, IItemData parentItem) 100 | { 101 | var queue = new Queue(); 102 | queue.Enqueue(parentItem); 103 | while (queue.Count > 0) 104 | { 105 | var item = queue.Dequeue(); 106 | foreach (var child in targetDataStore.GetChildren(item)) 107 | { 108 | queue.Enqueue(child); 109 | } 110 | yield return item; 111 | } 112 | } 113 | } 114 | } -------------------------------------------------------------------------------- /src/Unicorn.DataBlaster/Sync/ItemMapper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using Rainbow.Model; 4 | using Sitecore; 5 | using Sitecore.DataBlaster.Load; 6 | using Convert = System.Convert; 7 | 8 | namespace Unicorn.DataBlaster.Sync 9 | { 10 | /// 11 | /// Maps a Unicorn item to a bulk load item. 12 | /// 13 | public class ItemMapper 14 | { 15 | public virtual BulkLoadItem ToBulkLoadItem(IItemData itemData, BulkLoadContext context, 16 | BulkLoadAction loadAction) 17 | { 18 | if (itemData == null) throw new ArgumentNullException(nameof(itemData)); 19 | if (context == null) throw new ArgumentNullException(nameof(context)); 20 | 21 | var bulkItem = new BulkLoadItem( 22 | loadAction, 23 | itemData.Id, 24 | itemData.TemplateId, 25 | itemData.BranchId, 26 | itemData.ParentId, 27 | itemData.Path, 28 | sourceInfo: itemData.SerializedItemId); 29 | 30 | foreach (var sharedField in itemData.SharedFields) 31 | { 32 | AddSyncField(context, bulkItem, sharedField); 33 | } 34 | 35 | foreach (var languagedFields in itemData.UnversionedFields) 36 | { 37 | foreach (var field in languagedFields.Fields) 38 | { 39 | AddSyncField(context, bulkItem, field, languagedFields.Language.Name); 40 | } 41 | } 42 | 43 | foreach (var versionFields in itemData.Versions) 44 | { 45 | foreach (var field in versionFields.Fields) 46 | { 47 | AddSyncField(context, bulkItem, field, versionFields.Language.Name, versionFields.VersionNumber); 48 | } 49 | 50 | AddStatisticsFieldsWhenMissing(bulkItem, versionFields.Language.Name, versionFields.VersionNumber); 51 | } 52 | 53 | // Serialized items don't contain the original blob id. 54 | context.LookupBlobIds = true; 55 | 56 | return bulkItem; 57 | } 58 | 59 | protected virtual void AddSyncField(BulkLoadContext context, BulkLoadItem bulkItem, IItemFieldValue itemField, 60 | string language = null, int versionNumber = 1) 61 | { 62 | var fieldId = itemField.FieldId; 63 | var fieldValue = itemField.Value; 64 | var fieldName = itemField.NameHint; 65 | var isBlob = itemField.BlobId.HasValue; 66 | 67 | Func blob = null; 68 | if (isBlob) 69 | { 70 | byte[] blobBytes; 71 | try 72 | { 73 | blobBytes = Convert.FromBase64String(fieldValue); 74 | } 75 | catch (Exception ex) 76 | { 77 | blobBytes = new byte[] { }; 78 | context.Log.Error( 79 | $"Unable to read blob from field '{fieldId}' in item with id '{bulkItem.Id}', " + 80 | $"item path '{bulkItem.ParentId}' and source info '{bulkItem.SourceInfo}', defaulting to empty value.", 81 | ex); 82 | } 83 | blob = () => new MemoryStream(blobBytes); 84 | 85 | // Field value needs to be set to the blob id. 86 | fieldValue = itemField.BlobId.Value.ToString("B").ToUpper(); 87 | } 88 | 89 | if (language == null) 90 | { 91 | bulkItem.AddSharedField(fieldId, fieldValue, blob, isBlob, fieldName); 92 | } 93 | else 94 | { 95 | bulkItem.AddVersionedField(fieldId, language, versionNumber, fieldValue, blob, isBlob, fieldName); 96 | } 97 | } 98 | 99 | protected virtual void AddStatisticsFieldsWhenMissing(BulkLoadItem bulkItem, string language, 100 | int versionNumber = 1) 101 | { 102 | var user = Sitecore.Context.User.Name; 103 | 104 | // Make sure revision is updated when item is created or updated so that smart publish works. 105 | // Unicorn doesn't track revisions, so we need to generate one ourselves. 106 | if (bulkItem.GetField(FieldIDs.Revision.Guid, language, versionNumber) == null) 107 | bulkItem.AddVersionedField(FieldIDs.Revision.Guid, language, versionNumber, Guid.NewGuid().ToString("D"), name: "__Revision", 108 | postProcessor: x => x.DependsOnCreate = x.DependsOnUpdate = true); 109 | 110 | if (bulkItem.GetField(FieldIDs.Created.Guid, language, versionNumber) == null) 111 | bulkItem.AddVersionedField(FieldIDs.Created.Guid, language, versionNumber, DateUtil.IsoNow, name: "__Created", 112 | postProcessor: x => x.DependsOnCreate = true); 113 | 114 | if (bulkItem.GetField(FieldIDs.CreatedBy.Guid, language, versionNumber) == null) 115 | bulkItem.AddVersionedField(FieldIDs.CreatedBy.Guid, language, versionNumber, user, name: "__Created by", 116 | postProcessor: x => x.DependsOnCreate = true); 117 | 118 | if (bulkItem.GetField(FieldIDs.Updated.Guid, language, versionNumber) == null) 119 | bulkItem.AddVersionedField(FieldIDs.Updated.Guid, language, versionNumber, DateUtil.IsoNow, name: "__Updated", 120 | postProcessor: x => x.DependsOnCreate = x.DependsOnUpdate = true); 121 | 122 | if (bulkItem.GetField(FieldIDs.UpdatedBy.Guid, language, versionNumber) == null) 123 | bulkItem.AddVersionedField(FieldIDs.UpdatedBy.Guid, language, versionNumber, user, name: "__Updated by", 124 | postProcessor: x => x.DependsOnCreate = x.DependsOnUpdate = true); 125 | } 126 | } 127 | } -------------------------------------------------------------------------------- /src/Unicorn.DataBlaster/Sync/UnicornDataBlaster.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.Linq; 4 | using Sitecore.Configuration; 5 | using Sitecore.DataBlaster.Load; 6 | using Sitecore.DataBlaster.Util; 7 | using Sitecore.Diagnostics; 8 | using Sitecore.Pipelines; 9 | using Unicorn.Configuration; 10 | using Unicorn.DataBlaster.Logging; 11 | using Unicorn.Loader; 12 | using Unicorn.Logging; 13 | using Unicorn.Pipelines.UnicornSyncComplete; 14 | using Unicorn.Pipelines.UnicornSyncEnd; 15 | using Unicorn.Pipelines.UnicornSyncStart; 16 | using Unicorn.Predicates; 17 | using Unicorn.Publishing; 18 | 19 | namespace Unicorn.DataBlaster.Sync 20 | { 21 | public class UnicornDataBlaster : IUnicornSyncStartProcessor 22 | { 23 | public const string DisableDataBlasterSettingName = "Unicorn.DisableDataBlasterByDefault"; 24 | public const string PipelineArgsParametersKey = "DataBlaster.Parameters"; 25 | 26 | private bool? _isUnicornPublishEnabled; 27 | 28 | protected virtual bool IsUnicornPublishEnabled 29 | { 30 | get 31 | { 32 | if (_isUnicornPublishEnabled == null) 33 | { 34 | _isUnicornPublishEnabled = Factory 35 | .GetConfigNode("//sitecore/pipelines/unicornSyncEnd/processor")? 36 | .Attributes?["type"]? 37 | .Value 38 | .StartsWith("Unicorn.Pipelines.UnicornSyncEnd.TriggerAutoPublishSyncedItems") ?? false; 39 | } 40 | 41 | return _isUnicornPublishEnabled.Value; 42 | } 43 | } 44 | 45 | private BulkLoader BulkLoader { get; } 46 | 47 | private ItemExtractor ItemExtractor { get; } 48 | 49 | /// 50 | /// Whether to skip updating the history engine. 51 | /// 52 | /// Skipped by default. 53 | public bool SkipHistoryEngine { get; set; } = true; 54 | 55 | /// 56 | /// Whether to skip updating the publish queue for incremental publishing. 57 | /// 58 | /// Skipped by default. 59 | public bool SkipPublishQueue { get; set; } = true; 60 | 61 | /// 62 | /// Whether to skip updating the link database. 63 | /// 64 | /// If not skipped and at least one config nneds to update the link database, it's updated for all configs. 65 | public bool SkipLinkDatabase { get; set; } = false; 66 | 67 | /// 68 | /// Whether to skip updating the indexes. 69 | /// 70 | /// If not skipped and at least one config nneds to update the indexes, it's updated for all configs. 71 | public bool SkipIndexes { get; set; } = false; 72 | 73 | #region Explicit ctors to support Sitecore.Configuration.DefaultFactory.CreateFromTypeName 74 | 75 | public UnicornDataBlaster() 76 | : this(new BulkLoader(), new ItemExtractor()) 77 | { 78 | } 79 | 80 | public UnicornDataBlaster(BulkLoader bulkLoader) 81 | : this(bulkLoader, new ItemExtractor()) 82 | { 83 | } 84 | 85 | public UnicornDataBlaster(ItemExtractor itemExtractor) 86 | : this(new BulkLoader(), itemExtractor) 87 | { 88 | } 89 | 90 | public UnicornDataBlaster(BulkLoader bulkLoader, ItemExtractor itemExtractor) 91 | { 92 | BulkLoader = bulkLoader; 93 | ItemExtractor = itemExtractor; 94 | } 95 | 96 | #endregion 97 | 98 | public virtual void Process(UnicornSyncStartPipelineArgs args) 99 | { 100 | // Find optional data blaster parameters in custom data of arguments. 101 | args.CustomData.TryGetValue(PipelineArgsParametersKey, out var parms); 102 | var parameters = parms as DataBlasterParameters ?? new DataBlasterParameters(); 103 | 104 | // If not enabled by default, is DataBlaster enabled through parameters? 105 | if (Settings.GetBoolSetting(DisableDataBlasterSettingName, false) && !parameters.EnableDataBlaster) 106 | return; 107 | 108 | try 109 | { 110 | args.Logger.Info( 111 | $"Start Bulk Unicorn Sync for configurations: '{string.Join("', '", args.Configurations.Select(x => x.Name))}'."); 112 | 113 | var watch = Stopwatch.StartNew(); 114 | var startTimestamp = DateTime.Now; 115 | 116 | LoadItems(args.Configurations, parameters, args.Logger); 117 | args.Logger.Info($"Extracted and loaded items ({(int)watch.Elapsed.TotalMilliseconds}ms)"); 118 | 119 | watch.Restart(); 120 | ClearCaches(); 121 | args.Logger.Info($"Caches cleared ({(int)watch.Elapsed.TotalMilliseconds}ms)"); 122 | 123 | ExecuteUnicornSyncComplete(args, parameters, startTimestamp); 124 | ExecuteUnicornSyncEnd(args, parameters); 125 | } 126 | catch (Exception ex) 127 | { 128 | args.Logger.Error(ex); 129 | throw; 130 | } 131 | finally 132 | { 133 | // This will signal that we handled the sync for all configurations, 134 | // and no further handling should be done. 135 | args.SyncIsHandled = true; 136 | } 137 | } 138 | 139 | protected virtual void LoadItems(IConfiguration[] configurations, DataBlasterParameters parameters, 140 | ILogger logger) 141 | { 142 | var databaseNames = configurations 143 | .SelectMany(c => c.Resolve().GetRootPaths().Select(rp => rp.DatabaseName)) 144 | .Distinct(); 145 | 146 | foreach (var databaseName in databaseNames) 147 | { 148 | logger.Info($"Syncing database '{databaseName}'..."); 149 | 150 | var context = CreateBulkLoadContext(BulkLoader, databaseName, configurations, parameters, logger); 151 | var bulkItems = ItemExtractor.ExtractBulkItems(context, parameters, configurations, databaseName); 152 | BulkLoader.LoadItems(context, bulkItems); 153 | 154 | if (context.AnyStageFailed) 155 | throw new Exception( 156 | $"Stage failed during bulkload of database '{databaseName}': {context.FailureMessage}"); 157 | 158 | // Support publishing after sync. 159 | if (IsUnicornPublishEnabled && !databaseName.Equals("core", StringComparison.OrdinalIgnoreCase)) 160 | { 161 | // Sort item changes by path length before sending them to Unicorn publish. 162 | // This way we are sure Parents will always be published before Children. 163 | var sortedItemChanges = context 164 | .ItemChanges 165 | .OrderBy(x => x.ItemPathLevel); 166 | 167 | foreach (var itemChange in sortedItemChanges) 168 | { 169 | ManualPublishQueueHandler.AddItemToPublish(itemChange.ItemId); 170 | } 171 | } 172 | } 173 | } 174 | 175 | protected virtual void ClearCaches() 176 | { 177 | var cacheUtil = new CacheUtil(); 178 | 179 | Factory.GetDatabases() 180 | .ForEach(x => 181 | { 182 | x.Engines.TemplateEngine.Reset(); 183 | cacheUtil.ClearLanguageCache(x); 184 | }); 185 | Sitecore.Caching.CacheManager.ClearAllCaches(); 186 | 187 | // Slow as hell, most people don't use it. 188 | //Translate.ResetCache(); 189 | } 190 | 191 | protected virtual void ExecuteUnicornSyncComplete(UnicornSyncStartPipelineArgs args, 192 | DataBlasterParameters parameters, 193 | DateTime syncStartTimestamp) 194 | { 195 | if (parameters.SkipUnicornSyncComplete) return; 196 | 197 | // Run complete pipelines to support post-processing, e.g. users and roles. 198 | var watch = Stopwatch.StartNew(); 199 | foreach (var config in args.Configurations) 200 | { 201 | CorePipeline.Run("unicornSyncComplete", 202 | new UnicornSyncCompletePipelineArgs(config, syncStartTimestamp)); 203 | } 204 | args.Logger.Info($"Ran sync complete pipelines ({(int)watch.Elapsed.TotalMilliseconds}ms)"); 205 | } 206 | 207 | protected virtual void ExecuteUnicornSyncEnd(UnicornSyncStartPipelineArgs args, 208 | DataBlasterParameters parameters) 209 | { 210 | if (parameters.SkipUnicornSyncEnd) return; 211 | 212 | // When we tell Unicorn that sync is handled, end pipeline is not called anymore. 213 | var watch = Stopwatch.StartNew(); 214 | CorePipeline.Run("unicornSyncEnd", new UnicornSyncEndPipelineArgs(args.Logger, true, args.Configurations)); 215 | args.Logger.Info($"Ran sync end pipeline ({(int)watch.Elapsed.TotalMilliseconds}ms)"); 216 | } 217 | 218 | protected virtual BulkLoadContext CreateBulkLoadContext(BulkLoader bulkLoader, string databaseName, 219 | IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger) 220 | { 221 | var context = bulkLoader.NewBulkLoadContext(databaseName); 222 | 223 | context.Log = new SitecoreAndUnicornLog(LoggerFactory.GetLogger(GetType()), logger); 224 | 225 | context.AllowTemplateChanges = true; 226 | 227 | // In Sitecore 9.3 initial release they did change properties on fields (versioned to unversioned) in the core database but it wasn't done in a clean way. 228 | // There are now more than 3000 field records stored in the versioned table which should have been moved to the unverioned table. 229 | // As we don't do bulk imports, or many template changes during development on the core database, we will disable this by default, and avoid deleting relevant content in the core database. 230 | // Sitecore its default deserialize operation is also not doing cleanup work. 231 | context.AllowCleanupOfFields = !databaseName.Equals("core", StringComparison.InvariantCultureIgnoreCase); 232 | context.StageDataWithoutProcessing = parameters.StageDataWithoutProcessing; 233 | 234 | // Use the shotgun, removing items one by one is too slow for full deserialize. 235 | context.RemoveItemsFromCaches = false; 236 | 237 | context.UpdateHistory = !SkipHistoryEngine; 238 | context.UpdatePublishQueue = !SkipPublishQueue; 239 | context.UpdateLinkDatabase = !SkipLinkDatabase && 240 | configurations.Any(x => x.Resolve().UpdateLinkDatabase); 241 | context.UpdateIndexes = !SkipIndexes && 242 | configurations.Any(x => x.Resolve().UpdateSearchIndex); 243 | 244 | return context; 245 | } 246 | } 247 | } -------------------------------------------------------------------------------- /src/Unicorn.DataBlaster/Unicorn.DataBlaster.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | net48 4 | True 5 | false 6 | Unicorn.DataBlaster 7 | Unicorn.DataBlaster 8 | Using Sitecore.DataBlaster for deserialization. 9 | sitecore serialization 10 | 11 | 13 | content 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | --------------------------------------------------------------------------------