├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── doc-resources ├── Pivot01.png ├── Pivot02.png ├── Sample01.png ├── Sample02.png ├── Sample03.png ├── Sample04.png ├── Sample05.png ├── Sample06.png ├── Sample07.png └── Sample08.png └── src ├── .nuget ├── NuGet.Config ├── NuGet.exe └── NuGet.targets ├── ExperienceExtractor.Client ├── App_Config │ └── Include │ │ └── ExperienceExtractor │ │ └── ExperienceExtractor.config ├── ExperienceExtractor.Client.csproj ├── Properties │ └── AssemblyInfo.cs ├── sitecore │ ├── admin │ │ └── experienceextractor │ │ │ └── Shell.aspx │ └── shell │ │ └── client │ │ └── Applications │ │ └── ExperienceExtractor │ │ ├── Dimensions.cshtml │ │ ├── ExperienceExtractor.css │ │ ├── ExperienceExtractor.js │ │ ├── ExperienceExtractorDimensions.js │ │ ├── ExperienceExtractorFilters.js │ │ ├── ExperienceExtractorStatus.js │ │ ├── ExtractDialog.js │ │ ├── Filters.cshtml │ │ └── Status.cshtml ├── web.Debug.config ├── web.Release.config └── web.config ├── ExperienceExtractor.Components ├── ExperienceExtractor.Components.csproj ├── Mapping │ ├── Sitecore │ │ ├── FactsMapper.cs │ │ ├── ItemDatabaseFieldLookup.cs │ │ ├── RulesFilter.cs │ │ ├── SitecoreFieldLabelProvider.cs │ │ ├── XaFacts.cs │ │ ├── XaFieldMapper.cs │ │ ├── XaHelpers.cs │ │ ├── XaLabelProvider.cs │ │ └── XaTableMapper.cs │ └── Splitting │ │ ├── MvTestFieldMapper.cs │ │ ├── MvTestSplitter.cs │ │ └── PageConditionSplitter.cs ├── Parsing │ ├── Fields │ │ ├── ContactIdFactory.cs │ │ ├── CountFieldFactory.cs │ │ ├── DateFieldFactory.cs │ │ ├── DateTimeFieldFactoryBase.cs │ │ ├── DimensionFactory.cs │ │ ├── FactFieldsFactory.cs │ │ ├── IndexFieldFactory.cs │ │ ├── MvTestFieldsFactory.cs │ │ ├── PartitionFieldFactory.cs │ │ ├── SequenceFieldMapper.cs │ │ ├── SplitMapperFactory.cs │ │ ├── StaticFieldFactory.cs │ │ ├── TimeFieldFactory.cs │ │ ├── TypedFieldFactories.cs │ │ ├── XaFactoryHelpers.cs │ │ ├── XaFactsFieldFactory.cs │ │ └── XaFieldFactory.cs │ ├── Filters │ │ └── RulesFilterFactory.cs │ ├── Helpers │ │ └── Selectors.cs │ ├── JobLoaders │ │ └── DirectJobLoaderFactory.cs │ ├── PostProcessors │ │ ├── AccessPostProcessorFactory.cs │ │ ├── EdmFactory.cs │ │ ├── MsSqlFactory.cs │ │ └── MsSqlJobLoaderFactory.cs │ ├── Splitters │ │ ├── FunnelFactory.cs │ │ └── PageConditionSplitterFactories.cs │ └── Tables │ │ ├── DefaultTableFactory.cs │ │ ├── MatrixFactory.cs │ │ ├── RowRepeater.cs │ │ └── XaTableFactory.cs ├── PostProcessors │ ├── AccessExporter.cs │ ├── PartitionField.cs │ ├── SqlClearOptions.cs │ ├── SqlExporter.cs │ ├── SqlRecordAdapter.cs │ ├── SqlUpdateUtil.cs │ └── SsasExporter.cs ├── Properties │ └── AssemblyInfo.cs ├── Resources │ └── Empty.accdb └── packages.config ├── ExperienceExtractor.MongoDb ├── ExperienceExtractor.MongoDb.csproj ├── Filters │ ├── IMongoNativeFilter.cs │ ├── MongoDateRangeFilter.cs │ ├── MongoIdSampler.cs │ ├── MongoLimitFilter.cs │ └── MongoRandomSampleFilter.cs ├── Helpers.cs ├── MongoDbVisitAggregationContextSource.cs ├── MongoSource.cs ├── Properties │ └── AssemblyInfo.cs └── packages.config ├── ExperienceExtractor.Scheduling ├── ExperienceExtractor.Scheduling.csproj ├── IJobLoader.cs ├── JobUpdateAgent.cs ├── Properties │ └── AssemblyInfo.cs ├── StaticJobLoader.cs ├── UpdateJobWrapper.cs └── packages.config ├── ExperienceExtractor.Tests ├── ExperienceExtractor.Tests.csproj ├── FieldMappers.cs ├── ProcessingScopeTests.cs ├── ProcessingTests.cs ├── Properties │ └── AssemblyInfo.cs ├── RowComparerTests.cs ├── SchemaTests.cs ├── Support │ ├── Helpers.cs │ ├── SimpleDataSource.cs │ ├── StaticFieldMapper.cs │ ├── TestExporter.cs │ ├── TestJobSpecification.cs │ └── TestSets.cs └── packages.config ├── ExperienceExtractor.sln ├── ExperienceExtractor ├── Api │ ├── Http │ │ ├── Configuration │ │ │ └── ExperienceExtractorWebApiConfig.cs │ │ ├── Controllers │ │ │ ├── ExperienceExtractorJobResultsController.cs │ │ │ ├── ExperienceExtractorJobsController.cs │ │ │ └── ExperienceExtractorMetaDataController.cs │ │ └── RequireSitecoreLoginAttribute.cs │ ├── Jobs │ │ ├── ExperienceExtractor.cs │ │ ├── IJobRepository.cs │ │ ├── IJobSpecification.cs │ │ ├── Job.cs │ │ ├── JobExecutionSettings.cs │ │ ├── JobInfo.cs │ │ ├── JobStatus.cs │ │ └── LocalJobRepository.cs │ ├── OData │ │ └── EdmSchemaBuilder.cs │ ├── Parsing │ │ ├── IParseFactory.cs │ │ ├── JobParser.cs │ │ ├── JsonJobParser.cs │ │ ├── JsonParseState.cs │ │ ├── ParseException.cs │ │ ├── ParseFactories.cs │ │ ├── ParseFactoryAttribute.cs │ │ ├── ParseFactoryParameterAttribute.cs │ │ ├── ParseState.cs │ │ └── ReflectionHelpers.cs │ └── Pipelines │ │ ├── ExperienceExtractorApiInitializer.cs │ │ └── XmlConfigHelpers.cs ├── Data │ ├── RowComparer.cs │ ├── Schema │ │ ├── CalculatedField.cs │ │ ├── Field.cs │ │ ├── FieldType.cs │ │ ├── RelationType.cs │ │ ├── SortOrder.cs │ │ ├── TableDataHelpers.cs │ │ ├── TableDataRelation.cs │ │ └── TableDataSchema.cs │ ├── TableData.cs │ ├── TableDataReader.cs │ └── WritableTableData.cs ├── ExperienceExtractor.csproj ├── ExperienceExtractor.licenseheader ├── Export │ ├── BinaryDataPartition.cs │ ├── BinaryTableData.cs │ ├── CsvExporter.cs │ ├── CsvTableData.cs │ ├── DataTableHelpers.cs │ ├── ITableDataBatchWriter.cs │ ├── ITableDataExporter.cs │ ├── ITableDataWriter.cs │ ├── ITablePartitionSource.cs │ ├── MergedTableData.cs │ ├── OutputFormat.cs │ ├── TableDataBatchWriter.cs │ └── TablePartition.cs ├── Mapping │ ├── Dimension.cs │ ├── FieldMapperBase.cs │ ├── FieldMapperSet.cs │ ├── ICalculatedFieldContainer.cs │ ├── IFieldMapper.cs │ ├── IItemFieldLookup.cs │ ├── ILabelProvider.cs │ ├── ITableMapper.cs │ ├── ItemDatabaseFieldLookup.cs │ ├── LabeledFieldMapper.cs │ ├── MappingHelpers.cs │ ├── SimpleFieldMapper.cs │ ├── SimpleTableMapper.cs │ ├── Splitting │ │ ├── ISplitter.cs │ │ ├── MatrixTableMapper.cs │ │ ├── MatrixTypes.cs │ │ ├── SequenceFieldMapper.cs │ │ ├── SequenceType.cs │ │ ├── SimpleSplitter.cs │ │ ├── SplittingFieldMapper.cs │ │ └── SplittingTableMapper.cs │ ├── TableDefinition.cs │ ├── TableMapperBase.cs │ ├── Tables.cs │ └── Time │ │ ├── DateDetailLevel.cs │ │ ├── DateDimension.cs │ │ ├── TimeDetailLevel.cs │ │ └── TimeDimension.cs ├── Processing │ ├── ArrayMap.cs │ ├── DataProcessor.cs │ ├── DataSources │ │ ├── DataSourceBase.cs │ │ ├── IDataFilter.cs │ │ ├── IDataSource.cs │ │ └── TypedDataFilter.cs │ ├── DateTime │ │ ├── DateRangeFilter.cs │ │ ├── ISequenceMapper.cs │ │ └── SequenceTableDataBuilder.cs │ ├── FieldMapperIterator.cs │ ├── Helpers │ │ ├── CollectionHelpers.cs │ │ ├── DisposeActionWrapper.cs │ │ ├── Indexed.cs │ │ └── ProtocolHelper.cs │ ├── IDeferedValue.cs │ ├── ITableDataPostProcessor.cs │ ├── IUpdatingTableDataPostProcessor.cs │ ├── Keys │ │ ├── ByteHelper.cs │ │ ├── Fnv1a32.cs │ │ ├── Fnv1a64.cs │ │ ├── IKeyFactory.cs │ │ ├── KeyFactory.cs │ │ └── KeyFactoryBase.cs │ ├── Labels │ │ ├── LabelLoader.cs │ │ └── LruCache.cs │ ├── ProcessingScope.cs │ ├── TableDataBuilder.cs │ └── TableMap.cs ├── Properties │ └── AssemblyInfo.cs └── packages.config ├── Sitecore Package └── Experience Extractor.package ├── components ├── Amo2Tabular.dll ├── Microsoft.AnalysisServices.DLL └── Readme.txt └── components80 └── Readme.txt /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /doc-resources/Pivot01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Pivot01.png -------------------------------------------------------------------------------- /doc-resources/Pivot02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Pivot02.png -------------------------------------------------------------------------------- /doc-resources/Sample01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample01.png -------------------------------------------------------------------------------- /doc-resources/Sample02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample02.png -------------------------------------------------------------------------------- /doc-resources/Sample03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample03.png -------------------------------------------------------------------------------- /doc-resources/Sample04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample04.png -------------------------------------------------------------------------------- /doc-resources/Sample05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample05.png -------------------------------------------------------------------------------- /doc-resources/Sample06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample06.png -------------------------------------------------------------------------------- /doc-resources/Sample07.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample07.png -------------------------------------------------------------------------------- /doc-resources/Sample08.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/doc-resources/Sample08.png -------------------------------------------------------------------------------- /src/.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/.nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/src/.nuget/NuGet.exe -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Reflection; 14 | using System.Runtime.CompilerServices; 15 | using System.Runtime.InteropServices; 16 | 17 | // General Information about an assembly is controlled through the following 18 | // set of attributes. Change these attribute values to modify the information 19 | // associated with an assembly. 20 | [assembly: AssemblyTitle("Sitecore Experience Extractor, Client Files")] 21 | [assembly: AssemblyDescription("")] 22 | [assembly: AssemblyConfiguration("")] 23 | [assembly: AssemblyCompany("Sitecore Corporation")] 24 | [assembly: AssemblyProduct("Sitecore Experience Extractor")] 25 | [assembly: AssemblyCopyright("Copyright © Sitecore Corporation 2015")] 26 | [assembly: AssemblyTrademark("")] 27 | [assembly: AssemblyCulture("")] 28 | 29 | // Setting ComVisible to false makes the types in this assembly not visible 30 | // to COM components. If you need to access a type in this assembly from 31 | // COM, set the ComVisible attribute to true on that type. 32 | [assembly: ComVisible(false)] 33 | 34 | // The following GUID is for the ID of the typelib if this project is exposed to COM 35 | [assembly: Guid("2a6166bb-6db4-4329-a9d1-cd41534c260f")] 36 | 37 | // Version information for an assembly consists of the following four values: 38 | // 39 | // Major Version 40 | // Minor Version 41 | // Build Number 42 | // Revision 43 | // 44 | // You can specify all the values or you can default the Revision and Build Numbers 45 | // by using the '*' as shown below: 46 | [assembly: AssemblyVersion("0.3.1.*")] 47 | [assembly: AssemblyFileVersion("0.3.1.*")] 48 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/sitecore/shell/client/Applications/ExperienceExtractor/Dimensions.cshtml: -------------------------------------------------------------------------------- 1 | @********************************************************************************************** 2 | Copyright 2015 Sitecore Corporation A/S 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | except in compliance with the License. You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under the 8 | License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | either express or implied. See the License for the specific language governing permissions 10 | and limitations under the License. 11 | ***********************************************************************************************@ 12 | @model RenderingModel 13 | 14 | @using Sitecore.Mvc 15 | @using Sitecore.Mvc.Presentation 16 | @using Sitecore.Web.UI.Controls.Common.UserControls 17 | @using ExperienceExtractor.Api.Http.Configuration 18 | @{ 19 | var rendering = this.Html.Sitecore().Controls().GetUserControl(this.Model.Rendering); 20 | rendering.Requires.Script("client", "ExperienceExtractorDimensions.js"); 21 | 22 | rendering.Class = "sc-experience-extractor sc-experience-extractor-dimensions"; 23 | 24 | } 25 |
26 |
27 |
28 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/sitecore/shell/client/Applications/ExperienceExtractor/ExperienceExtractor.css: -------------------------------------------------------------------------------- 1 | /********************************************************************************************** 2 | Copyright 2015 Sitecore Corporation A/S 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | except in compliance with the License. You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under the 8 | License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | either express or implied. See the License for the specific language governing permissions 10 | and limitations under the License. 11 | ***********************************************************************************************/ 12 | 13 | .sc-experience-extractor .dimension-settings { padding: 10px; z-index: 1000;} 14 | .sc-experience-extractor .dimension-settings-chrome { border: 1px solid #ccc; background-color: white; } 15 | 16 | .sc-experience-extractor .dynatree-separate { margin-top: 10px; display: inline-block; } 17 | 18 | .sc-experience-extractor span.big-label { line-height: 28px; } 19 | 20 | .sc-experience-extractor-filters { padding: 10px;} 21 | 22 | .sc-experience-extractor .sc-Slider .titleBar, .sc-experience-extractor .sc-Slider .tooltip { z-index: 0} 23 | 24 | .sc-experience-extractor .info-box { position: absolute; display: none; } -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/sitecore/shell/client/Applications/ExperienceExtractor/ExperienceExtractor.js: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | define(["sitecore", "jquery"], function (Sitecore, $) { 14 | $("").attr("href", "/sitecore/shell/client/Applications/ExperienceExtractor/ExperienceExtractor.css") 15 | .appendTo("head"); 16 | 17 | var xaDimensionConfig = { 18 | tableDimensions: ["Pages/By Page", "Pages/By Page URL", "Visits/By Goal"], 19 | extractDimensionTables: { 20 | "Geo": ["Visits/By Country", "Visits/By Region", "Visits/By City"] 21 | }, 22 | 23 | sortOrder: { 24 | "Visits/By Country": 0, 25 | "Visits/By Region": 1, 26 | "Visits/By City": 2 27 | } 28 | } 29 | 30 | 31 | return { 32 | job: { 33 | source: {"xdb":{filters:[]}}, 34 | mapper:{}, 35 | postprocessors: ["msaccess"] 36 | }, 37 | xaDimensionConfig: xaDimensionConfig 38 | }; 39 | }); -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/sitecore/shell/client/Applications/ExperienceExtractor/ExperienceExtractorStatus.js: -------------------------------------------------------------------------------- 1 | 2 | define(["sitecore", "jquery", "experienceExtractor"], function (_sc, $, X) { 3 | 4 | _sc.Factories.createBaseComponent({ 5 | name: "ExperienceExtractorStatus", 6 | base: "ControlBase", 7 | selector: ".sc-experience-extractor-status", 8 | attributes: [ 9 | { name: "apiUrl", value: "$el.data:sc-extract-api" } 10 | ], 11 | initialize: function() { 12 | 13 | }, 14 | 15 | getProgressBar: function() { 16 | return this.app[this.model.get("name") + "ProgressBar"].viewModel; 17 | }, 18 | 19 | extendModel: { 20 | extract: function() { 21 | var _this = this; 22 | var model = this.viewModel; 23 | var bar = _this.viewModel.getProgressBar(); 24 | var status = $(".status-text", model.$el); 25 | var progress = 0; 26 | function update() { 27 | progress += 0.1; 28 | bar.percentage(progress*100); 29 | if( progress >= 1 ) { 30 | bar.hide(); 31 | } else { 32 | bar.show(); 33 | setTimeout(update, 100); 34 | } 35 | } 36 | 37 | update(); 38 | } 39 | } 40 | }); 41 | }); -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/sitecore/shell/client/Applications/ExperienceExtractor/ExtractDialog.js: -------------------------------------------------------------------------------- 1 | define(["sitecore", "experienceExtractor"], function (sc, X) { 2 | return sc.Definitions.App.extend({ 3 | initialized: function() { 4 | var _this = this; 5 | _this.on("extract:dialog:button:cancel:clicked", function () { _this.ExtractDialog.hide(); }, _this); 6 | }, 7 | 8 | extract: function() { 9 | this.ExtractDialog.show(); 10 | this.ExperienceExtractorStatus.extract(); 11 | } 12 | }); 13 | }); -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/sitecore/shell/client/Applications/ExperienceExtractor/Status.cshtml: -------------------------------------------------------------------------------- 1 | @model RenderingModel 2 | 3 | @using Sitecore.Mvc 4 | @using Sitecore.Mvc.Presentation 5 | @using Sitecore.Web.UI.Controls.Common.UserControls 6 | @using Sitecore.Web.UI.Controls.Common.Renderings 7 | @using ExperienceExtractor.Api.Http.Configuration 8 | @{ 9 | var rendering = this.Html.Sitecore().Controls().GetUserControl(this.Model.Rendering); 10 | var renderingId = rendering.ControlId; 11 | rendering.Requires.Script("client", "ExperienceExtractorStatus.js"); 12 | 13 | rendering.Attributes["data-sc-extract-api"] = "/" + ExperienceExtractorWebApiConfig.JobApiRoute; 14 | rendering.Class = "sc-experience-extractor sc-experience-extractor-status"; 15 | 16 | } 17 |
18 |
19 | @Html.Sitecore().Controls().Rendering(Html, "{AF6438A4-8295-4925-90B3-2BF97D3515AF}", renderingId + "ProgressBar", "", new {Value=0}) 20 |
21 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/web.Debug.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 17 | 18 | 29 | 30 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/web.Release.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 17 | 18 | 19 | 30 | 31 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Client/web.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Mapping/Sitecore/SitecoreFieldLabelProvider.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Mapping; 14 | using ExperienceExtractor.Processing; 15 | using Sitecore.Globalization; 16 | 17 | namespace ExperienceExtractor.Components.Mapping.Sitecore 18 | { 19 | public class SitecoreFieldLabelProvider : ILabelProvider 20 | { 21 | public IItemFieldLookup Lookup { get; private set; } 22 | public string Path { get; set; } 23 | public Language Language { get; set; } 24 | 25 | public SitecoreFieldLabelProvider(string path, Language language) 26 | { 27 | Path = path; 28 | Language = language; 29 | } 30 | 31 | public void Initialize(DataProcessor processor) 32 | { 33 | Lookup = processor.FieldLookup; 34 | } 35 | 36 | public string GetLabel(object key) 37 | { 38 | return Lookup != null ? Lookup.Lookup(key, Path, Language) : null; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Mapping/Sitecore/XaHelpers.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using Sitecore.Analytics.Aggregation.Data.Model; 16 | using Sitecore.ExperienceAnalytics.Aggregation.Data.Model; 17 | using Sitecore.ExperienceAnalytics.Aggregation.Data.Schema; 18 | using Sitecore.ExperienceAnalytics.Aggregation.Dimensions; 19 | 20 | namespace ExperienceExtractor.Components.Mapping.Sitecore 21 | { 22 | public static class XaHelpers 23 | { 24 | public static IDimension GetDimension(string name) 25 | { 26 | var type = 27 | typeof (DimensionBase).Assembly.GetType("Sitecore.ExperienceAnalytics.Aggregation.Dimensions." + name); 28 | 29 | return Activator.CreateInstance(type, new object[]{Guid.Empty}) as IDimension; 30 | } 31 | 32 | 33 | private static readonly MetricsCalculator _calculator = new MetricsCalculator(); 34 | public static SegmentMetricsValue CalculateMetrics(IVisitAggregationContext context) 35 | { 36 | return _calculator.CalculateMetrics(context); 37 | } 38 | 39 | class MetricsCalculator : DimensionBase 40 | { 41 | public MetricsCalculator() : base(Guid.Empty) 42 | { 43 | } 44 | 45 | public override IEnumerable GetData(IVisitAggregationContext context) 46 | { 47 | throw new NotImplementedException(); 48 | } 49 | 50 | public SegmentMetricsValue CalculateMetrics(IVisitAggregationContext context) 51 | { 52 | return CalculateCommonMetrics(context, 1); 53 | } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Mapping/Splitting/MvTestSplitter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | 15 | namespace ExperienceExtractor.Components.Mapping.Splitting 16 | { 17 | public class MvTestSplitter : PageConditionSplitter 18 | { 19 | public Guid TestId { get; private set; } 20 | 21 | public MvTestSplitter(Guid testId) : 22 | base((s, p) => p.MvTest != null && p.MvTest.Id == testId, true, true, true) 23 | { 24 | TestId = testId; 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/ContactIdFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Api.Parsing; 19 | using ExperienceExtractor.Data.Schema; 20 | using ExperienceExtractor.Mapping; 21 | using Sitecore.Analytics.Aggregation.Data.Model; 22 | 23 | namespace ExperienceExtractor.Components.Parsing.Fields 24 | { 25 | [ParseFactory("contactid", "Contact ID", "Contact ID for unique visitor count")] 26 | public class ContactIdFactory : IParseFactory 27 | { 28 | public IFieldMapper Parse(JobParser parser, ParseState state) 29 | { 30 | return new SimpleFieldMapper(state.AffixName("ContactId"), scope => scope.Current().TryGet(ctx => ctx.Visit.ContactId), typeof(Guid), 31 | valueKind: "ContactId", 32 | calculatedFields: new[] 33 | { 34 | new CalculatedField{Name="Unique visitors", 35 | DaxPattern = string.Format("DISTINCTCOUNT([{0}])", state.AffixName("ContactId")), 36 | ChildDaxPattern = string.Format("CALCULATE(DISTINCTCOUNT(@Parent[{0}]), @TableName)", state.AffixName("ContactId")), 37 | FormatString = CalculatedFieldFormat.Integer} 38 | }); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/CountFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using ExperienceExtractor.Api.Parsing; 15 | using ExperienceExtractor.Data.Schema; 16 | using ExperienceExtractor.Mapping; 17 | 18 | namespace ExperienceExtractor.Components.Parsing.Fields 19 | { 20 | [ParseFactory("count", "Aggregated row count", "The number of rows aggregated or the number of distinct parent rows in the aggregate, i.e. \"count distinct\" with respect to parent scope"), 21 | ParseFactoryParameter("Name", typeof(string), "The name of the field in the target table", defaultValue: "Count"), 22 | ParseFactoryParameter("Scope", typeof(int), "Parent offset. 0 corresponds to current scope, -1 to Parent, -2 to Parent.Parent etc.", "0")] 23 | public class CountFieldFactory : IParseFactory 24 | { 25 | public IFieldMapper Parse(JobParser parser, ParseState state) 26 | { 27 | var name = state.AffixName(state.TryGet("Name", "Count", true)); 28 | 29 | var scopeOffset = Math.Abs(state.TryGet("Scope", 0)); 30 | 31 | return new SimpleFieldMapper(name, ctx => 32 | { 33 | if (scopeOffset == 0) return 1; 34 | var p = ctx; 35 | for (var i = 0; i < scopeOffset && p != null; i++) 36 | { 37 | p = p.Parent; 38 | } 39 | if (p != null) 40 | { 41 | return ctx.OncePerScope(p, 1); 42 | } 43 | 44 | return 0; 45 | }, typeof(int), FieldType.Fact); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/DateFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using ExperienceExtractor.Api.Parsing; 15 | using ExperienceExtractor.Mapping; 16 | using ExperienceExtractor.Mapping.Time; 17 | using ExperienceExtractor.Processing; 18 | 19 | namespace ExperienceExtractor.Components.Parsing.Fields 20 | { 21 | [ParseFactory("date", "Date dimension", "Creates a date dimension table with consecutive dates from the extracted visits. The date table contains information such as year, month, day of week etc. localized a job's language."), 22 | ParseFactoryParameter("Name", typeof(string), "Name of date dimension table if not inlined", defaultValue: "StartDateTime"), 23 | ParseFactoryParameter("UseDateKey", typeof(bool), "Add date columns to main table rather than creating a separate dimension table"), 24 | ParseFactoryParameter("Inline", typeof(bool), "Add date columns to the main table rather than creating a separate dimension table"), 25 | ParseFactoryParameter("Resolution", typeof(string), "Resolution. Can be Year, Quarter, Month or Date"), 26 | ParseFactoryParameter("Key", typeof(string), "Specifices if the date field should be added as a key rather than a dimension in the main table")] 27 | public class DateFieldFactory : DateTimeFieldFactoryBase 28 | { 29 | protected override IFieldMapper Parse(Func selector, string defaultName, 30 | JobParser parser, ParseState state) 31 | { 32 | return new DateDimension( 33 | state.AffixName(state.TryGet("Name", defaultName)), 34 | selector, 35 | useDateForKey: state.TryGet("DateKey", true), 36 | inlineFields: state.TryGet("Inline", false), 37 | cultureInfo: parser.DefaultLanguage.CultureInfo, 38 | key: state.TryGet("Key", false), 39 | detailLevel: 40 | state.TryGet("Resolution", DateDetailLevel.Date, true)); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/DateTimeFieldFactoryBase.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using ExperienceExtractor.Api.Parsing; 15 | using ExperienceExtractor.Mapping; 16 | using ExperienceExtractor.Processing; 17 | using Sitecore.Analytics.Aggregation.Data.Model; 18 | using Sitecore.Analytics.Model; 19 | 20 | namespace ExperienceExtractor.Components.Parsing.Fields 21 | { 22 | [ParseFactoryParameter("Select", typeof(IFieldMapper), "Optional field mapper that selects the date/time field", defaultValue: "Visit's StartDateTime")] 23 | public abstract class DateTimeFieldFactoryBase : IParseFactory 24 | { 25 | public IFieldMapper Parse(JobParser parser, ParseState state) 26 | { 27 | var mapper = state.Select("Select").TryGet(parser.ParseFieldMapper) as SimpleFieldMapper; 28 | 29 | Func selector = 30 | scope => scope.Current().TryGet(v => (DateTime?)v.Visit.StartDateTime); 31 | if (mapper != null) 32 | { 33 | selector = scope => (DateTime?)mapper.Selector(scope); 34 | } 35 | 36 | var defaultName = mapper != null ? mapper.Name : "StartDateTime"; 37 | 38 | return Parse(selector, defaultName, parser, state); 39 | } 40 | 41 | protected abstract IFieldMapper Parse(Func selector, 42 | string defaultName, 43 | JobParser parser, ParseState state); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/FactFieldsFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Api.Parsing; 14 | using ExperienceExtractor.Components.Mapping.Sitecore; 15 | using ExperienceExtractor.Mapping; 16 | 17 | namespace ExperienceExtractor.Components.Parsing.Fields 18 | { 19 | [ParseFactory("facts", "Facts as calculated in Experience Analytics"), 20 | ParseFactoryParameter("Types", typeof(FactTypes), "The facts to include as columns in the table. An array with any of the values Visits, Value, Bounces, Conversions, TimeOnSite, PageViews, Count", "All"), 21 | ParseFactoryParameter("Prefix", typeof(string), "Prefix for that fact column names"), 22 | ParseFactoryParameter("EntireVisit", typeof(bool), "If false, facts are calculated relative to the current page or event. When true, facts are calculated for the entire visit.", defaultValue: "false")] 23 | public class FactFieldsFactory : IParseFactory 24 | { 25 | public IFieldMapper Parse(JobParser parser, ParseState state) 26 | { 27 | var factTypes = XaFactsFieldFactory.ParseFactTypes(state.TryGet("Types", new string[0], true)); 28 | 29 | var prefix = state.TryGet("Prefix"); 30 | 31 | return new FactsMapper(factTypes, state.Prefix(prefix).AffixName, state.TryGet("EntireVisit", false)); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/IndexFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using ExperienceExtractor.Api.Parsing; 15 | using ExperienceExtractor.Data.Schema; 16 | using ExperienceExtractor.Mapping; 17 | 18 | namespace ExperienceExtractor.Components.Parsing.Fields 19 | { 20 | [ParseFactory("index", "Index of current item", "The index of the current item being processed relative to a parent scope"), 21 | ParseFactoryParameter("Name", typeof(string), "Name of the field in the target table", "Index"), 22 | ParseFactoryParameter("Scope", typeof(int), "Parent offset. -1 corresponds to Parent, -2 to Parent.Parent etc.", "-1")] 23 | public class IndexFieldFactory : IParseFactory 24 | { 25 | public IFieldMapper Parse(JobParser parser, ParseState state) 26 | { 27 | var name = state.AffixName(state.TryGet("Name", "Index", true)); 28 | 29 | var scopeOffset = Math.Abs(state.TryGet("Scope", -1, true)); 30 | 31 | return new SimpleFieldMapper(name, ctx => 32 | { 33 | if (scopeOffset == 0) return ctx.GlobalIndex; 34 | 35 | var p = ctx; 36 | for (var i = 0; i < scopeOffset && p != null; i++) 37 | { 38 | p = p.Parent; 39 | } 40 | if (p != null) 41 | { 42 | return ctx.Index(p); 43 | } 44 | 45 | return 0; 46 | }, typeof(int), FieldType.Dimension); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/MvTestFieldsFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using ExperienceExtractor.Api.Parsing; 17 | using ExperienceExtractor.Components.Mapping.Splitting; 18 | using ExperienceExtractor.Mapping; 19 | using ExperienceExtractor.Mapping.Splitting; 20 | using Sitecore.Analytics.Testing; 21 | 22 | namespace ExperienceExtractor.Components.Parsing.Fields 23 | { 24 | 25 | [ParseFactory("mvtest", "Multi variate test fields", "Adds a column with variable number and label for each variable in a MV test, and adds columns split by what happened before and after the test and in total"), 26 | ParseFactoryParameter("TestId", typeof(Guid), "The ID of the MV test in Sitecore"), 27 | ParseFactoryParameter("Fields", typeof(IEnumerable), "Field mappers to split by Before/After/Total.")] 28 | public class MvTestFieldsFactory : IParseFactory 29 | { 30 | public IFieldMapper Parse(JobParser parser, ParseState state) 31 | { 32 | var testIdString = state.Require("TestId", true); 33 | 34 | Guid testId; 35 | if (!Guid.TryParse(testIdString, out testId)) 36 | { 37 | throw state.AttributeError("Invalid test id specified ({0})", testIdString); 38 | } 39 | 40 | 41 | var testSet = TestManager.GetTestSet(testId); 42 | 43 | var testMapper = new MvTestFieldMapper(testSet, state.AffixName(state.TryGet("Name", "MvTest"))); 44 | 45 | if (state.SelectMany("Fields").Any()) 46 | { 47 | var splitter = new SplittingFieldMapper(new MvTestSplitter(testId), 48 | postfix => 49 | state.Postfix(postfix).SelectMany("Fields").Select(parser.ParseFieldMapper).ToArray()); 50 | 51 | return new FieldMapperSet(testMapper.Name, true, new IFieldMapper[] { testMapper, splitter }, state.AffixName); 52 | } 53 | 54 | return testMapper; 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/PartitionFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Api.Parsing; 19 | using ExperienceExtractor.Components.PostProcessors; 20 | using ExperienceExtractor.Data.Schema; 21 | using ExperienceExtractor.Mapping; 22 | using Sitecore.Analytics.Aggregation.Data.Model; 23 | 24 | namespace ExperienceExtractor.Components.Parsing.Fields 25 | { 26 | [ParseFactory("PartitionKey", "Partition field", "Used for incremental updates. The timespan defines a tumbling window, and should be set to a value greater than the max latency time expected for the datastore queried for interactions. (Now() - SaveDateTime).")] 27 | public class PartitionFieldFactory : IParseFactory 28 | { 29 | public IFieldMapper Parse(JobParser parser, ParseState state) 30 | { 31 | var staleTime = TimeSpan.Parse(state.Require("MaxStaleTime", true)); 32 | 33 | return 34 | new SimpleFieldMapper( 35 | s => 36 | SqlUpdateUtil.GetPartitionDate(s.Current().Visit.SaveDateTime.ToUniversalTime(), 37 | staleTime), 38 | new PartitionField 39 | { 40 | Name = "Partition", 41 | FieldType = FieldType.Dimension, 42 | StaleTime = staleTime, 43 | ValueType = typeof (DateTime) 44 | }); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/StaticFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Api.Parsing; 14 | using ExperienceExtractor.Mapping; 15 | using FieldType = ExperienceExtractor.Data.Schema.FieldType; 16 | 17 | namespace ExperienceExtractor.Components.Parsing.Fields 18 | { 19 | [ParseFactory("static", "Static value", "A static value in the table"), 20 | ParseFactoryParameter("Value", typeof(string), "The value"), 21 | ParseFactoryParameter("Name", typeof(string), "The column name")] 22 | public class StaticFieldFactory : IParseFactory 23 | { 24 | public IFieldMapper Parse(JobParser parser, ParseState state) 25 | { 26 | var value = state.Require("Value", true); 27 | 28 | SimpleFieldMapper mapper; 29 | 30 | long longValue; 31 | if (long.TryParse(value, out longValue)) 32 | { 33 | mapper = new SimpleFieldMapper(state.TryGet("Name", value), scope => longValue, typeof (long), FieldType.Dimension); 34 | } 35 | else 36 | { 37 | mapper = new SimpleFieldMapper(state.TryGet("Name", value), scope => value, typeof (string), FieldType.Dimension); 38 | } 39 | 40 | mapper.Hide = state.Require("Hide", false); 41 | 42 | return mapper; 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/TimeFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using ExperienceExtractor.Api.Parsing; 15 | using ExperienceExtractor.Mapping; 16 | using ExperienceExtractor.Mapping.Time; 17 | using ExperienceExtractor.Processing; 18 | 19 | namespace ExperienceExtractor.Components.Parsing.Fields 20 | { 21 | [ParseFactory("time", "Time of day dimension", "Creates a time dimension table with consecutive time of day from 00:00 to 24:00"), 22 | ParseFactoryParameter("Name", typeof(string), "Name of time dimension table if not inlined", defaultValue: "StartDateTimeTime"), 23 | ParseFactoryParameter("Inline", typeof(bool), "Add time columns to main table rather than creating a dimension table"), 24 | ParseFactoryParameter("Resolution", typeof(string), "Resolution. Can be Hour, Quarter or Minute")] 25 | public class TimeFieldFactory : DateTimeFieldFactoryBase 26 | { 27 | protected override IFieldMapper Parse(Func selector, string defaultName, 28 | JobParser parser, ParseState state) 29 | { 30 | return new TimeDimension( 31 | state.AffixName(state.TryGet("Name", defaultName + "Time")), 32 | selector, 33 | inlineFields: state.TryGet("Inline"), 34 | cultureInfo: parser.DefaultLanguage.CultureInfo, 35 | detailLevel: state.TryGet("Resolution", TimeDetailLevel.Hour, true)); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/XaFactsFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Linq; 15 | using ExperienceExtractor.Api.Parsing; 16 | using ExperienceExtractor.Components.Mapping.Sitecore; 17 | using ExperienceExtractor.Mapping; 18 | 19 | namespace ExperienceExtractor.Components.Parsing.Fields 20 | { 21 | [ParseFactory("xafacts", "Facts as calculated in Experience Analytics"), 22 | ParseFactoryParameter("Types", typeof(FactTypes), "The facts to include as columns in the table. An array of any of the values Visits, Value, Bounces, Conversions, TimeOnSite, PageViews, Count", "All")] 23 | public class XaFactsFieldFactory : IParseFactory 24 | { 25 | public static FactTypes ParseFactTypes(string[] factTypes) 26 | { 27 | return factTypes.Length == 0 28 | ? FactTypes.All 29 | : (FactTypes) factTypes.Aggregate(0, (current, s) => current | (int) Enum.Parse(typeof (FactTypes), s, true)); 30 | 31 | } 32 | 33 | public IFieldMapper Parse(JobParser parser, ParseState state) 34 | { 35 | return new XaFacts(state.AffixName, ParseFactTypes(state.TryGet("Types", new string[0], true))); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Fields/XaFieldFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Api.Parsing; 14 | using ExperienceExtractor.Components.Mapping.Sitecore; 15 | using ExperienceExtractor.Mapping; 16 | 17 | namespace ExperienceExtractor.Components.Parsing.Fields 18 | { 19 | [ParseFactory("xa", "Experience Analytics dimension", "Adds key and label field from an Experience Analytics dimension that generates one row per vist"), 20 | ParseFactoryParameter("Dimension", typeof(string), "Dimension's guid or path to dimension item in Sitecore. If a leading slash is omitted in the path it is relative to /sitecore/system/Marketing Control Panel/Experience Analytics/Dimensions/", isMainParameter: true, required: true), 21 | ParseFactoryParameter("KeyName", typeof(string), "The column name for the key value", defaultValue: "[DimensionTypeName]Key"), 22 | ParseFactoryParameter("LabelName", typeof(string), "The column name for the label value", defaultValue: "[DimensionTypeName]Label")] 23 | public class XaFieldFactory : IParseFactory 24 | { 25 | public IFieldMapper Parse(JobParser parser, ParseState state) 26 | { 27 | var dimension = state.ParseDimension(); 28 | 29 | var keyName = state.AffixName(state.TryGet("KeyName", ()=>dimension.GetType().Name + "Key")); 30 | var labelName = state.AffixName(state.TryGet("LabelName", () => dimension.GetType().Name + "Label")); 31 | 32 | 33 | return new LabeledFieldMapper(new XaFieldMapper(dimension, false, keyName), labelName, 34 | XaLabelProvider.FromDimension(dimension, parser.DefaultLanguage), friendlyName: XaFieldMapper.SuggestFriendlyLabelName(labelName)); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/JobLoaders/DirectJobLoaderFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Api.Parsing; 19 | using ExperienceExtractor.Scheduling; 20 | using Newtonsoft.Json.Linq; 21 | 22 | namespace ExperienceExtractor.Components.Parsing.JobLoaders 23 | { 24 | [ParseFactory("direct", "Default job factory", "Parses a JSON job specification")] 25 | public class DirectJobLoaderFactory : IParseFactory 26 | { 27 | public IJobLoader Parse(JobParser parser, ParseState state) 28 | { 29 | var spec = new JsonJobParser((JObject)((JsonParseState) state).Token); 30 | 31 | return new StaticJobLoader(spec); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/PostProcessors/AccessPostProcessorFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Api.Parsing; 14 | using ExperienceExtractor.Components.PostProcessors; 15 | using ExperienceExtractor.Processing; 16 | 17 | namespace ExperienceExtractor.Components.Parsing.PostProcessors 18 | { 19 | [ParseFactory("msaccess", "Microsoft Access", "Compiles the output in a Microsoft Access database. Be sure to install a version of the Microsoft.ACE.OLEDB provider that matches the server's architecture (typically x64)")] 20 | public class AccessPostProcessorFactory : IParseFactory 21 | { 22 | public ITableDataPostProcessor Parse(JobParser parser, ParseState state) 23 | { 24 | return new AccessExporter(); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/PostProcessors/EdmFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Api.OData; 14 | using ExperienceExtractor.Api.Parsing; 15 | using ExperienceExtractor.Components.PostProcessors; 16 | using ExperienceExtractor.Processing; 17 | 18 | namespace ExperienceExtractor.Components.Parsing.PostProcessors 19 | { 20 | [ParseFactory("edm", "EDM/CSDL schema", "Adds EDM/CSDL schema to the output")] 21 | public class EdmFactory : IParseFactory 22 | { 23 | public ITableDataPostProcessor Parse(JobParser parser, ParseState state) 24 | { 25 | return new EdmSchemaBuilder(); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/PostProcessors/MsSqlJobLoaderFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Configuration; 15 | using System.Data.SqlClient; 16 | using System.Spatial; 17 | using ExperienceExtractor.Api.Jobs; 18 | using ExperienceExtractor.Api.Parsing; 19 | using ExperienceExtractor.Scheduling; 20 | using Newtonsoft.Json.Linq; 21 | 22 | namespace ExperienceExtractor.Components.Parsing.PostProcessors 23 | { 24 | [ParseFactory("mssql", "SQL Server job loader")] 25 | public class MsSqlJobLoaderFactory : IParseFactory 26 | { 27 | public IJobLoader Parse(JobParser parser, ParseState state) 28 | { 29 | var connection = state.Require("Database", true); 30 | 31 | if (!connection.Contains("=")) 32 | { 33 | var builder = 34 | new SqlConnectionStringBuilder( 35 | ConfigurationManager.ConnectionStrings["ExperienceExtractor.Sql"].ConnectionString); 36 | builder.InitialCatalog = connection; 37 | connection = builder.ConnectionString; 38 | } 39 | 40 | using (var conn = new SqlConnection(connection)) 41 | { 42 | conn.Open(); 43 | if (DBNull.Value.Equals(new SqlCommand("SELECT OBJECT_ID('Sitecore.JobInfo', 'U')", conn).ExecuteScalar())) 44 | { 45 | throw new ParseException(state, "Database does not contain the job info table. Was it created with Experience Extractor?"); 46 | } 47 | 48 | var json = (string) new SqlCommand("SELECT TOP 1 [Prototype] FROM Sitecore.JobInfo", conn).ExecuteScalar(); 49 | if (string.IsNullOrEmpty(json)) 50 | { 51 | throw new ParseException(state, "Database does not contain a job prototype"); 52 | } 53 | 54 | var job = new JsonJobParser(JObject.Parse(json)); 55 | return new StaticJobLoader(job); 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Parsing/Tables/RowRepeater.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections; 15 | using System.Collections.Generic; 16 | using System.Linq; 17 | using System.Text; 18 | using System.Threading.Tasks; 19 | using ExperienceExtractor.Api.Parsing; 20 | using ExperienceExtractor.Data.Schema; 21 | using ExperienceExtractor.Mapping; 22 | using ExperienceExtractor.Processing; 23 | using Microsoft.AnalysisServices; 24 | 25 | namespace ExperienceExtractor.Components.Parsing.Tables 26 | { 27 | /// 28 | /// Creates a number of random rows for each source record. Useful for performance testing. 29 | /// 30 | [ParseFactory("repeat", "Row randomizer")] 31 | public class RowRepeater : IParseFactory 32 | { 33 | public ITableMapper Parse(JobParser parser, ParseState state) 34 | { 35 | var repeats = state.TryGet("Repeats", 10); 36 | 37 | 38 | var repeatState = new State(); 39 | 40 | var def = parser.ParseTableDefinition(state); 41 | def.FieldMappers = def.FieldMappers.Concat(new[]{ 42 | new SimpleFieldMapper("Repeat", scope => "Value" + repeatState.Level, typeof (string), FieldType.Dimension)}).ToArray(); 43 | 44 | return new SimpleTableMapper(scope => Repeat(scope, repeats, repeatState), def); 45 | } 46 | 47 | static IEnumerable Repeat(ProcessingScope scope, int levels, State repeatState) 48 | { 49 | foreach (var item in ((IEnumerable)scope.CurrentObject)) 50 | { 51 | repeatState.Level = 1; 52 | for (var i = 0; i < levels; i++) 53 | { 54 | yield return item; 55 | ++repeatState.Level; 56 | } 57 | } 58 | } 59 | 60 | class State 61 | { 62 | public int Level { get; set; } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/PostProcessors/PartitionField.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Data.Schema; 19 | 20 | namespace ExperienceExtractor.Components.PostProcessors 21 | { 22 | public class PartitionField : Field 23 | { 24 | public TimeSpan StaleTime { get; set; } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/PostProcessors/SqlClearOptions.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Components.PostProcessors 20 | { 21 | [Flags] 22 | public enum SqlClearOptions 23 | { 24 | None = 0, 25 | Dimensions = 1, 26 | Facts = 2, 27 | All = 3 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Reflection; 14 | using System.Runtime.CompilerServices; 15 | using System.Runtime.InteropServices; 16 | 17 | // General Information about an assembly is controlled through the following 18 | // set of attributes. Change these attribute values to modify the information 19 | // associated with an assembly. 20 | [assembly: AssemblyTitle("Sitecore Experience Extractor, Components")] 21 | [assembly: AssemblyDescription("")] 22 | [assembly: AssemblyConfiguration("")] 23 | [assembly: AssemblyCompany("Sitecore Corporation")] 24 | [assembly: AssemblyProduct("Sitecore Experience Extractor")] 25 | [assembly: AssemblyCopyright("Copyright © Sitecore Corporation 2015")] 26 | [assembly: AssemblyTrademark("")] 27 | [assembly: AssemblyCulture("")] 28 | // Setting ComVisible to false makes the types in this assembly not visible 29 | // to COM components. If you need to access a type in this assembly from 30 | // COM, set the ComVisible attribute to true on that type. 31 | [assembly: ComVisible(false)] 32 | 33 | // The following GUID is for the ID of the typelib if this project is exposed to COM 34 | [assembly: Guid("288b02e9-565c-42a0-891b-a27717584572")] 35 | 36 | // Version information for an assembly consists of the following four values: 37 | // 38 | // Major Version 39 | // Minor Version 40 | // Build Number 41 | // Revision 42 | // 43 | // You can specify all the values or you can default the Build and Revision Numbers 44 | // by using the '*' as shown below: 45 | // [assembly: AssemblyVersion("1.0.*")] 46 | [assembly: AssemblyVersion("0.3.1.*")] 47 | [assembly: AssemblyFileVersion("0.3.1.*")] 48 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/Resources/Empty.accdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/src/ExperienceExtractor.Components/Resources/Empty.accdb -------------------------------------------------------------------------------- /src/ExperienceExtractor.Components/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.MongoDb/Filters/IMongoNativeFilter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Processing.DataSources; 14 | using MongoDB.Driver; 15 | 16 | namespace ExperienceExtractor.MongoDb.Filters 17 | { 18 | public interface IMongoNativeFilter : IDataFilter 19 | { 20 | IMongoQuery MongoQuery { get; } 21 | 22 | void UpdateCursor(MongoCursor cursor); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.MongoDb/Filters/MongoLimitFilter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using MongoDB.Driver; 14 | using Newtonsoft.Json.Linq; 15 | using ExperienceExtractor.Api.Parsing; 16 | using ExperienceExtractor.Processing; 17 | using ExperienceExtractor.Processing.DataSources; 18 | 19 | namespace ExperienceExtractor.MongoDb.Filters 20 | { 21 | 22 | public class MongoLimitFilter : IMongoNativeFilter 23 | { 24 | public int Count { get; set; } 25 | public int Skip { get; set; } 26 | 27 | public MongoLimitFilter(int count, int skip = 0) 28 | { 29 | Count = count; 30 | Skip = skip; 31 | } 32 | 33 | public bool IsStagingFilter { get { return true; } } 34 | 35 | public bool Include(object item) 36 | { 37 | return true; 38 | } 39 | 40 | public IMongoQuery MongoQuery { get; private set; } 41 | public void UpdateCursor(MongoCursor cursor) 42 | { 43 | cursor.SetLimit(Count); 44 | } 45 | 46 | [ParseFactory("limit", "MongoDB Limit filter", "Limits the number of visits extracted from MongoDB"), 47 | ParseFactoryParameter("Count", typeof(int), "The number of visits to extract", isMainParameter: true), 48 | ParseFactoryParameter("Skip", typeof(int), "Skip this number of visits", "0")] 49 | public class Factory : IParseFactory 50 | { 51 | public IDataFilter Parse(JobParser parser, ParseState state) 52 | { 53 | return new MongoLimitFilter(state.Require("Count", true), state.TryGet("Skip", 0)); 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.MongoDb/Filters/MongoRandomSampleFilter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Processing.DataSources; 14 | using MongoDB.Driver; 15 | using Sitecore.Analytics.Aggregation.Data.Model; 16 | 17 | namespace ExperienceExtractor.MongoDb.Filters 18 | { 19 | public class MongoRandomSampleFilter : TypedDataFilter, IMongoNativeFilter, IEstimatedCountFilter 20 | { 21 | public double Start { get; set; } 22 | public double End { get; set; } 23 | 24 | /// 25 | /// If the filter is only used for staging. 26 | /// If true, scheduled updates will include all data (that is, rebuild on first run), if false, scheduled updates will use this sampling 27 | /// 28 | public bool ForStaging { get; set; } 29 | 30 | private MongoIdSampler _sampler = new MongoIdSampler(); 31 | public MongoRandomSampleFilter(double start = 0, double end = 1) 32 | { 33 | Start = start; 34 | End = end; 35 | } 36 | 37 | public bool IsStagingFilter { get { return ForStaging; } } 38 | 39 | protected override bool Include(IVisitAggregationContext item) 40 | { 41 | if (Start > 0 || End < 1) 42 | { 43 | return _sampler.GuidInRange(item.Visit.InteractionId, Start, End); 44 | } 45 | return true; 46 | } 47 | 48 | public IMongoQuery MongoQuery 49 | { 50 | get 51 | { 52 | if (Start > 0 || End < 1) 53 | { 54 | return _sampler.GetIdRange(Start, End); 55 | } 56 | 57 | return null; 58 | } 59 | } 60 | 61 | public void UpdateCursor(MongoCursor cursor) 62 | { 63 | 64 | } 65 | 66 | public long? EstimateCount(long? population) 67 | { 68 | if (!population.HasValue) return null; 69 | 70 | return (long) (population.Value * (End - Start)/1d); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.MongoDb/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Reflection; 14 | using System.Runtime.CompilerServices; 15 | using System.Runtime.InteropServices; 16 | 17 | // General Information about an assembly is controlled through the following 18 | // set of attributes. Change these attribute values to modify the information 19 | // associated with an assembly. 20 | [assembly: AssemblyTitle("Sitecore Experience Extractor, MongoDB components")] 21 | [assembly: AssemblyDescription("")] 22 | [assembly: AssemblyConfiguration("")] 23 | [assembly: AssemblyCompany("Sitecore Corporation")] 24 | [assembly: AssemblyProduct("Sitecore Experience Extractor")] 25 | [assembly: AssemblyCopyright("Copyright © Sitecore Corporation 2015")] 26 | [assembly: AssemblyTrademark("")] 27 | [assembly: AssemblyCulture("")] 28 | 29 | // Setting ComVisible to false makes the types in this assembly not visible 30 | // to COM components. If you need to access a type in this assembly from 31 | // COM, set the ComVisible attribute to true on that type. 32 | [assembly: ComVisible(false)] 33 | 34 | // The following GUID is for the ID of the typelib if this project is exposed to COM 35 | [assembly: Guid("8308ec16-cd1e-47b3-af87-b9bdb7b099fe")] 36 | 37 | // Version information for an assembly consists of the following four values: 38 | // 39 | // Major Version 40 | // Minor Version 41 | // Build Number 42 | // Revision 43 | // 44 | // You can specify all the values or you can default the Build and Revision Numbers 45 | // by using the '*' as shown below: 46 | // [assembly: AssemblyVersion("1.0.*")] 47 | [assembly: AssemblyVersion("0.3.1.*")] 48 | [assembly: AssemblyFileVersion("0.3.1.*")] 49 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.MongoDb/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Scheduling/IJobLoader.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Api.Jobs; 19 | 20 | namespace ExperienceExtractor.Scheduling 21 | { 22 | public interface IJobLoader 23 | { 24 | IJobSpecification Load(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Scheduling/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Reflection; 14 | using System.Runtime.CompilerServices; 15 | using System.Runtime.InteropServices; 16 | 17 | // General Information about an assembly is controlled through the following 18 | // set of attributes. Change these attribute values to modify the information 19 | // associated with an assembly. 20 | [assembly: AssemblyTitle("ExperienceExtractor.Scheduling")] 21 | [assembly: AssemblyDescription("")] 22 | [assembly: AssemblyConfiguration("")] 23 | [assembly: AssemblyCompany("")] 24 | [assembly: AssemblyProduct("ExperienceExtractor.Scheduling")] 25 | [assembly: AssemblyCopyright("Copyright © 2015")] 26 | [assembly: AssemblyTrademark("")] 27 | [assembly: AssemblyCulture("")] 28 | 29 | // Setting ComVisible to false makes the types in this assembly not visible 30 | // to COM components. If you need to access a type in this assembly from 31 | // COM, set the ComVisible attribute to true on that type. 32 | [assembly: ComVisible(false)] 33 | 34 | // The following GUID is for the ID of the typelib if this project is exposed to COM 35 | [assembly: Guid("f7390ea4-5a34-446d-ae4a-f59c7b2deb30")] 36 | 37 | // Version information for an assembly consists of the following four values: 38 | // 39 | // Major Version 40 | // Minor Version 41 | // Build Number 42 | // Revision 43 | // 44 | // You can specify all the values or you can default the Build and Revision Numbers 45 | // by using the '*' as shown below: 46 | // [assembly: AssemblyVersion("1.0.*")] 47 | [assembly: AssemblyVersion("1.0.0.0")] 48 | [assembly: AssemblyFileVersion("1.0.0.0")] 49 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Scheduling/StaticJobLoader.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Api.Jobs; 19 | 20 | namespace ExperienceExtractor.Scheduling 21 | { 22 | public class StaticJobLoader : IJobLoader 23 | { 24 | public IJobSpecification Specification { get; set; } 25 | 26 | public StaticJobLoader(IJobSpecification specification) 27 | { 28 | Specification = specification; 29 | } 30 | 31 | public IJobSpecification Load() 32 | { 33 | return Specification; 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Scheduling/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | using System.Reflection; 13 | using System.Runtime.CompilerServices; 14 | using System.Runtime.InteropServices; 15 | 16 | // General Information about an assembly is controlled through the following 17 | // set of attributes. Change these attribute values to modify the information 18 | // associated with an assembly. 19 | [assembly: AssemblyTitle("ExperienceExtractor.Tests")] 20 | [assembly: AssemblyDescription("")] 21 | [assembly: AssemblyConfiguration("")] 22 | [assembly: AssemblyCompany("")] 23 | [assembly: AssemblyProduct("ExperienceExtractor.Tests")] 24 | [assembly: AssemblyCopyright("Copyright © 2015")] 25 | [assembly: AssemblyTrademark("")] 26 | [assembly: AssemblyCulture("")] 27 | 28 | // Setting ComVisible to false makes the types in this assembly not visible 29 | // to COM components. If you need to access a type in this assembly from 30 | // COM, set the ComVisible attribute to true on that type. 31 | [assembly: ComVisible(false)] 32 | 33 | // The following GUID is for the ID of the typelib if this project is exposed to COM 34 | [assembly: Guid("0090b77b-86b4-482d-9749-dbe90d177014")] 35 | 36 | // Version information for an assembly consists of the following four values: 37 | // 38 | // Major Version 39 | // Minor Version 40 | // Build Number 41 | // Revision 42 | // 43 | // You can specify all the values or you can default the Build and Revision Numbers 44 | // by using the '*' as shown below: 45 | // [assembly: AssemblyVersion("1.0.*")] 46 | [assembly: AssemblyVersion("1.0.0.0")] 47 | [assembly: AssemblyFileVersion("1.0.0.0")] 48 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Tests/Support/Helpers.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | using System; 13 | using System.Collections.Generic; 14 | using System.Linq; 15 | using Sitecore.Analytics.Aggregation.Data.Model; 16 | using ExperienceExtractor.Data; 17 | using ExperienceExtractor.Mapping; 18 | using ExperienceExtractor.Processing; 19 | using ExperienceExtractor.Processing.Helpers; 20 | 21 | namespace ExperienceExtractor.Tests.Support 22 | { 23 | static class Helpers 24 | { 25 | public static Guid ToGuid(this int n) 26 | { 27 | var bytes = new byte[16]; 28 | BitConverter.GetBytes(n).CopyTo(bytes, 0); 29 | return new Guid(bytes); 30 | } 31 | 32 | public static TValue Field(this TableData table, string name, object[] row) 33 | { 34 | var index = table.Schema.Fields.AsIndexed().FirstOrDefault(f => f.Value.Name == name).Index; 35 | return (TValue) row[index]; 36 | } 37 | 38 | public static IEnumerable Fields(this TableData table, string name, IEnumerable rows = null) 39 | { 40 | var index = table.Schema.Fields.AsIndexed().FirstOrDefault(f => f.Value.Name == name).Index; 41 | return (rows ?? table.Rows).Select(row => (TValue) row[index]); 42 | } 43 | 44 | public static IEnumerable Process(this IEnumerable contexts, 45 | Func defintion) 46 | { 47 | return contexts.Process(()=>new SimpleTableMapper(defintion())); 48 | } 49 | 50 | public static IEnumerable Process(this IEnumerable contexts, Func mapper, Action initializer = null) 51 | { 52 | var spec = new TestJobSpecification(contexts, mapper); 53 | 54 | var processor = new DataProcessor(spec.CreateRootMapper()); 55 | if (initializer != null) 56 | { 57 | initializer(processor); 58 | } 59 | processor.Process(spec.CreateDataSource()); 60 | 61 | return processor.Tables; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Tests/Support/SimpleDataSource.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | using System; 13 | using System.Collections; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using Sitecore.Analytics.Aggregation.Data.Model; 19 | using ExperienceExtractor.Processing.DataSources; 20 | 21 | namespace ExperienceExtractor.Tests.Support 22 | { 23 | class SimpleDataSource : DataSourceBase 24 | { 25 | public override long? Count { get { return _src.Length; } } 26 | 27 | private readonly IVisitAggregationContext[] _src; 28 | 29 | public SimpleDataSource(IEnumerable src) 30 | { 31 | _src = src.ToArray(); 32 | } 33 | 34 | public override IEnumerator GetEnumerator() 35 | { 36 | return _src.GetEnumerator(); 37 | } 38 | 39 | 40 | public override void ApplyUpdateFilter(DateTime? lastSaveDate, DateTime? lastSaveDateEnd) 41 | { 42 | throw new NotImplementedException(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Tests/Support/StaticFieldMapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | using System; 13 | using System.Collections.Generic; 14 | using System.Linq; 15 | using System.Text; 16 | using System.Threading.Tasks; 17 | using ExperienceExtractor.Data.Schema; 18 | using ExperienceExtractor.Mapping; 19 | using ExperienceExtractor.Processing; 20 | 21 | namespace ExperienceExtractor.Tests.Support 22 | { 23 | class StaticFieldMapper : IFieldMapper 24 | { 25 | public object Value { get; set; } 26 | 27 | private Field[] _fields; 28 | public StaticFieldMapper(Field field, object value) 29 | { 30 | Value = value; 31 | _fields = new[] {field}; 32 | } 33 | 34 | public IList Fields { get { return _fields; }} 35 | 36 | 37 | public bool SetValues(ProcessingScope scope, IList target) 38 | { 39 | if (Value != null) 40 | { 41 | target[0] = Value; 42 | return true; 43 | } 44 | return false; 45 | } 46 | 47 | public void Initialize(DataProcessor processor) 48 | { 49 | } 50 | 51 | public void InitializeRelatedTables(DataProcessor processor, TableDataBuilder table) 52 | { 53 | } 54 | 55 | public void PostProcessRows(IEnumerable> rows) 56 | { 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Tests/Support/TestExporter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | using System; 13 | using System.Collections.Generic; 14 | using System.Linq; 15 | using System.Text; 16 | using System.Threading.Tasks; 17 | using ExperienceExtractor.Data; 18 | using ExperienceExtractor.Export; 19 | using ExperienceExtractor.Processing; 20 | 21 | namespace ExperienceExtractor.Tests.Support 22 | { 23 | class TestExporter : ITableDataExporter 24 | { 25 | public IEnumerable Tables { get; set; } 26 | 27 | public IEnumerable Export(IEnumerable tables) 28 | { 29 | return (Tables = tables); 30 | } 31 | 32 | public void Dispose() 33 | { 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Tests/Support/TestJobSpecification.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | using System; 13 | using System.Collections.Generic; 14 | using System.Linq; 15 | using System.Text; 16 | using System.Threading.Tasks; 17 | using Sitecore.Analytics.Aggregation.Data.Model; 18 | using ExperienceExtractor.Api.Jobs; 19 | using ExperienceExtractor.Data; 20 | using ExperienceExtractor.Export; 21 | using ExperienceExtractor.Mapping; 22 | using ExperienceExtractor.Processing; 23 | using ExperienceExtractor.Processing.DataSources; 24 | using Sitecore.Globalization; 25 | 26 | namespace ExperienceExtractor.Tests.Support 27 | { 28 | class TestJobSpecification : IJobSpecification 29 | { 30 | private readonly IEnumerable _contexts; 31 | private readonly Func _mapper; 32 | private TestExporter _exporter = new TestExporter(); 33 | 34 | 35 | 36 | public TestJobSpecification(IEnumerable contexts, Func mapper) 37 | { 38 | _contexts = contexts; 39 | _mapper = mapper; 40 | } 41 | 42 | public IEnumerable Tables 43 | { 44 | get { return _exporter.Tables; } 45 | } 46 | 47 | public Language DefaultLanguage { get { return Language.Parse("en"); }} 48 | 49 | public IDataSource CreateDataSource() 50 | { 51 | return new SimpleDataSource(_contexts); 52 | } 53 | 54 | public ITableMapper CreateRootMapper() 55 | { 56 | return _mapper(); 57 | } 58 | 59 | public IEnumerable CreatePostProcessors() 60 | { 61 | yield break; 62 | } 63 | 64 | public ITableDataExporter CreateExporter(string tempPath) 65 | { 66 | return _exporter; 67 | } 68 | 69 | public IList Load(string tempPath) 70 | { 71 | throw new NotImplementedException(); 72 | } 73 | 74 | public string LockKey { get; set; } 75 | 76 | public void Initialize(Job job) 77 | { 78 | 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/ExperienceExtractor.Tests/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Http/Controllers/ExperienceExtractorJobResultsController.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Net; 15 | using System.Net.Http; 16 | using System.Net.Http.Headers; 17 | using System.Web.Http; 18 | using ExperienceExtractor.Api.Jobs; 19 | 20 | namespace ExperienceExtractor.Api.Http.Controllers 21 | { 22 | [RequireSitecoreLogin] 23 | public class ExperienceExtractorJobResultsController : ApiController 24 | { 25 | private readonly IJobRepository _repository; 26 | 27 | public ExperienceExtractorJobResultsController() 28 | : this(ExperienceExtractorApiContainer.JobRepository) 29 | { 30 | 31 | } 32 | 33 | public ExperienceExtractorJobResultsController(IJobRepository repository) 34 | { 35 | _repository = repository; 36 | } 37 | 38 | // GET api/jobs/{id}/result 39 | public HttpResponseMessage Get(Guid id) 40 | { 41 | var stream = _repository.GetZippedResult(id); 42 | if (stream == null) 43 | { 44 | return new HttpResponseMessage(HttpStatusCode.NotFound); 45 | } 46 | 47 | var result = new HttpResponseMessage(HttpStatusCode.OK) {Content = new StreamContent(stream)}; 48 | 49 | result.Content.Headers.ContentType = 50 | new MediaTypeHeaderValue("application/zip"); 51 | 52 | result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") 53 | { 54 | FileName = id.ToString("d") + ".zip" 55 | }; 56 | 57 | return result; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Http/RequireSitecoreLoginAttribute.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using System.Web.Http; 19 | using System.Web.Http.Controllers; 20 | using ExperienceExtractor.Api.Http.Configuration; 21 | using Sitecore; 22 | 23 | namespace ExperienceExtractor.Api.Http 24 | { 25 | [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = true, Inherited = true)] 26 | public class RequireSitecoreLoginAttribute : AuthorizeAttribute 27 | { 28 | protected override bool IsAuthorized(HttpActionContext actionContext) 29 | { 30 | return CheckSecurity(); 31 | } 32 | 33 | public static bool CheckSecurity() 34 | { 35 | if (ExperienceExtractorWebApiConfig.AllowAnonymousAccess) return true; 36 | 37 | var user = Context.User; 38 | return user != null && user.IsAuthenticated && 39 | (user.IsAdministrator || ExperienceExtractorWebApiConfig.AllowedRoles.Any(user.IsInRole) || ExperienceExtractorWebApiConfig.AllowedUsers.Any(userName => userName.Equals(user.Name, StringComparison.InvariantCultureIgnoreCase))); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Jobs/ExperienceExtractor.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using System.Web.Hosting; 15 | using ExperienceExtractor.Api.Parsing; 16 | 17 | namespace ExperienceExtractor.Api.Jobs 18 | { 19 | public class ExperienceExtractorApiContainer 20 | { 21 | public static IJobRepository JobRepository { get; set; } 22 | 23 | public static ParseFactories ParseFactories { get; set; } 24 | 25 | public static Dictionary ItemPaths { get; set; } 26 | 27 | static ExperienceExtractorApiContainer() 28 | { 29 | JobRepository = LocalJobRepository.Instance; 30 | ParseFactories = ExperienceExtractor.Api.Parsing.ParseFactories.Default; 31 | 32 | ItemPaths = new Dictionary(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Jobs/IJobRepository.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.IO; 16 | 17 | namespace ExperienceExtractor.Api.Jobs 18 | { 19 | public interface IJobRepository 20 | { 21 | JobInfo CreateJob(IJobSpecification specification, Action jobEnded = null); 22 | 23 | IEnumerable Get(); 24 | 25 | IEnumerable GetFromLockKey(string lockKey); 26 | 27 | JobInfo Get(Guid id); 28 | 29 | JobInfo Cancel(Guid id); 30 | 31 | Stream GetZippedResult(Guid id); 32 | } 33 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Jobs/IJobSpecification.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using ExperienceExtractor.Data; 15 | using ExperienceExtractor.Export; 16 | using ExperienceExtractor.Mapping; 17 | using ExperienceExtractor.Processing; 18 | using ExperienceExtractor.Processing.DataSources; 19 | using Sitecore.Globalization; 20 | 21 | namespace ExperienceExtractor.Api.Jobs 22 | { 23 | public interface IJobSpecification 24 | { 25 | Language DefaultLanguage { get; } 26 | 27 | IDataSource CreateDataSource(); 28 | 29 | ITableMapper CreateRootMapper(); 30 | 31 | IEnumerable CreatePostProcessors(); 32 | 33 | ITableDataExporter CreateExporter(string tempPath); 34 | 35 | IList Load(string tempPath); 36 | 37 | void Initialize(Job job); 38 | 39 | string LockKey { get; } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Jobs/JobExecutionSettings.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Api.Jobs 20 | { 21 | public class JobExecutionSettings 22 | { 23 | public string TempDirectory { get; set; } 24 | 25 | public int ProcessingThreads { get; set; } 26 | 27 | public int LoadThreads { get; set; } 28 | 29 | public int BatchSize { get; set; } 30 | 31 | public long? SizeLimit { get; set; } 32 | 33 | public int DataSourceBufferSize { get; set; } 34 | 35 | public int FieldCacheSize { get; set; } 36 | 37 | public string DatabaseName { get; set; } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Jobs/JobStatus.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Api.Jobs 14 | { 15 | public enum JobStatus 16 | { 17 | Pending, 18 | Preparing, 19 | Running, 20 | Merging, 21 | PostProcessing, 22 | Completing, 23 | Completed, 24 | Failed, 25 | Canceled 26 | } 27 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Parsing/IParseFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Api.Parsing 14 | { 15 | public interface IParseFactory 16 | { 17 | } 18 | 19 | public interface IParseFactory : IParseFactory 20 | { 21 | //TODO: parser is redundant. State contains that already. 22 | TItem Parse(JobParser parser, ParseState state); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Parsing/ParseException.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | 15 | namespace ExperienceExtractor.Api.Parsing 16 | { 17 | public class ParseException : ApplicationException 18 | { 19 | public ParseState State { get; set; } 20 | 21 | public ParseException(ParseState state, string message) 22 | : base(message) 23 | { 24 | State = state; 25 | } 26 | 27 | public static ParseException MissingAttribute(ParseState state, string name) 28 | { 29 | return new ParseException(state, string.Format("{0} was expected while parsing {1}", name, state.Path)); 30 | } 31 | 32 | public static ParseException AttributeError(ParseState state, string description) 33 | { 34 | return new ParseException(state, description + string.Format(" while parsing {0}", state.Path)); 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Parsing/ParseFactoryAttribute.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | 15 | namespace ExperienceExtractor.Api.Parsing 16 | { 17 | [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] 18 | public class ParseFactoryAttribute : Attribute 19 | { 20 | public string Key { get; set; } 21 | public string Name { get; set; } 22 | public string Description { get; set; } 23 | 24 | public ParseFactoryAttribute(string key, string name, string description = null) 25 | { 26 | Key = key; 27 | Name = name; 28 | Description = description; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Parsing/ParseFactoryParameterAttribute.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Api.Parsing 20 | { 21 | [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] 22 | public class ParseFactoryParameterAttribute : Attribute 23 | { 24 | public string Name { get; set; } 25 | public Type Type { get; set; } 26 | public string Description { get; set; } 27 | public string DefaultValue { get; set; } 28 | public bool IsMainParameter { get; set; } 29 | public bool Required { get; set; } 30 | 31 | public ParseFactoryParameterAttribute(string name, Type type, string description, 32 | string defaultValue = null, bool isMainParameter = false, bool required = false) 33 | { 34 | Name = name; 35 | Type = type; 36 | Description = description; 37 | DefaultValue = defaultValue; 38 | IsMainParameter = isMainParameter; 39 | Required = required; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Parsing/ReflectionHelpers.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Reflection; 17 | using Sitecore.Diagnostics; 18 | 19 | namespace ExperienceExtractor.Api.Parsing 20 | { 21 | static class ReflectionHelpers 22 | { 23 | public static IEnumerable GetAllTypes(this Assembly assembly) 24 | { 25 | return assembly.GetTypes().SelectMany(type => type.ExpandNestedTypes()); 26 | } 27 | 28 | public static IEnumerable ExpandNestedTypes(this Type type) 29 | { 30 | yield return type; 31 | 32 | foreach (var nestedType in type.GetNestedTypes()) 33 | { 34 | foreach (var t in ExpandNestedTypes(nestedType)) 35 | { 36 | yield return t; 37 | } 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Api/Pipelines/XmlConfigHelpers.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using System.Xml; 19 | 20 | namespace ExperienceExtractor.Api.Pipelines 21 | { 22 | static class XmlConfigHelpers 23 | { 24 | public static void SetIfDefined(this string value, Action setter) 25 | { 26 | if (!string.IsNullOrWhiteSpace(value)) 27 | { 28 | setter(value); 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Data/Schema/FieldType.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Data.Schema 14 | { 15 | public enum FieldType 16 | { 17 | Key, 18 | Dimension, 19 | Label, 20 | Fact 21 | } 22 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Data/Schema/RelationType.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Data.Schema 14 | { 15 | /// 16 | /// Relation types for 17 | /// 18 | public enum RelationType 19 | { 20 | /// 21 | /// A dimension is referenced (0..1) 22 | /// 23 | DimensionReference, 24 | 25 | /// 26 | /// A child table references this table (1..*) 27 | /// 28 | Child, 29 | 30 | /// 31 | /// Another table references this table as a dimension (1..*) 32 | /// 33 | Dimension, 34 | 35 | /// 36 | /// Another table references this table as a child in a parent/child relationship (1..1) 37 | /// 38 | Parent 39 | } 40 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Data/Schema/SortOrder.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Data.Schema 14 | { 15 | public enum SortOrder 16 | { 17 | Unspecified, 18 | Ascending, 19 | Descending 20 | } 21 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Data/Schema/TableDataRelation.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using Newtonsoft.Json; 15 | 16 | namespace ExperienceExtractor.Data.Schema 17 | { 18 | /// 19 | /// Defines a relation to another in a . 20 | /// 21 | public class TableDataRelation 22 | { 23 | /// 24 | /// The fields in the table containing this relation that are part of this relation 25 | /// 26 | public IList Fields { get; set; } 27 | 28 | /// 29 | /// The fields in the related table that are part of this relation 30 | /// 31 | public IList RelatedFields { get; set; } 32 | 33 | /// 34 | /// The table 35 | /// 36 | public TableDataSchema RelatedTable { get; set; } 37 | 38 | /// 39 | /// The type of the relation 40 | /// 41 | public RelationType RelationType { get; set; } 42 | } 43 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Data/WritableTableData.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Data.Schema; 19 | using ExperienceExtractor.Export; 20 | 21 | namespace ExperienceExtractor.Data 22 | { 23 | public abstract class WritableTableData : TableData, ITableDataWriter 24 | { 25 | protected WritableTableData(TableDataSchema schema) : base(schema) 26 | { 27 | } 28 | 29 | public abstract void Dispose(); 30 | 31 | public abstract void WriteRows(IEnumerable rows); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Export/BinaryDataPartition.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.IO; 16 | using System.Linq; 17 | using System.Text; 18 | using System.Threading.Tasks; 19 | using ExperienceExtractor.Data.Schema; 20 | 21 | namespace ExperienceExtractor.Export 22 | { 23 | public class BinaryDataPartition : TablePartition 24 | { 25 | public BinaryDataPartition(string directory) 26 | { 27 | Directory = directory; 28 | } 29 | 30 | public string Directory { get; set; } 31 | 32 | public override long Size 33 | { 34 | get 35 | { 36 | var dir = new DirectoryInfo(Directory); 37 | return dir.Exists ? dir.GetFiles().Sum(f => f.Length) : 0; 38 | } 39 | } 40 | 41 | public override ITableDataWriter CreateTableDataWriter(TableDataSchema schema) 42 | { 43 | var data = new BinaryTableData(schema, Path.Combine(Directory, schema.Name + ".bin")); 44 | AddTableData(data); 45 | return data; 46 | } 47 | 48 | public override void Dispose() 49 | { 50 | if (System.IO.Directory.Exists(Directory)) 51 | { 52 | System.IO.Directory.Delete(Directory, true); 53 | } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Export/ITableDataBatchWriter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Data; 19 | using ExperienceExtractor.Mapping; 20 | using ExperienceExtractor.Processing; 21 | 22 | namespace ExperienceExtractor.Export 23 | { 24 | /// 25 | /// Offloads data to a secondary storage while data is processed to preserve memory. 26 | /// 27 | public interface ITableDataBatchWriter : IDisposable 28 | { 29 | /// 30 | /// Writes a batch of tables 31 | /// 32 | /// The tables to write 33 | void WriteBatch(IEnumerable tables); 34 | 35 | /// 36 | /// The merged tables from all the batches written 37 | /// 38 | IEnumerable Tables { get; } 39 | 40 | /// 41 | /// A flag to indicate if prcoessing should end prematurely due to storage limit or other constraints. 42 | /// 43 | bool End { get; } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Export/ITableDataExporter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Data; 19 | using ExperienceExtractor.Mapping; 20 | using ExperienceExtractor.Processing; 21 | 22 | namespace ExperienceExtractor.Export 23 | { 24 | /// 25 | /// Classes implementing this interface persists a list of tables and provides access to the persisted data as tables with the same schemas 26 | /// 27 | public interface ITableDataExporter : IDisposable 28 | { 29 | /// 30 | /// Persists the tables specified and returns tables with the same schemas access accessing the persisted data 31 | /// 32 | /// 33 | /// 34 | IEnumerable Export(IEnumerable tables); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Export/ITableDataWriter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Export 20 | { 21 | public interface ITableDataWriter : IDisposable 22 | { 23 | void WriteRows(IEnumerable rows); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Export/ITablePartitionSource.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Processing; 14 | 15 | namespace ExperienceExtractor.Export 16 | { 17 | /// 18 | /// Classes implementing this interface provides partitions for offloading data from memory while a is processing 19 | /// 20 | public interface ITablePartitionSource 21 | { 22 | /// 23 | /// Returns a new 24 | /// 25 | /// 26 | TablePartition CreatePartition(); 27 | } 28 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Export/OutputFormat.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Export 20 | { 21 | [Flags] 22 | public enum OutputFormat 23 | { 24 | Csv = 0, 25 | BinaryPartitions = 1, 26 | Binary = 3 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Export/TablePartition.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using ExperienceExtractor.Data; 16 | using ExperienceExtractor.Data.Schema; 17 | using ExperienceExtractor.Processing; 18 | 19 | namespace ExperienceExtractor.Export 20 | { 21 | /// 22 | /// A partition of data offloaded to disk during processing in a 23 | /// 24 | public abstract class TablePartition : IDisposable 25 | { 26 | private readonly List _tables = new List(); 27 | 28 | /// 29 | /// The tables in this partition 30 | /// 31 | public IEnumerable Tables { get { return _tables; }} 32 | 33 | /// 34 | /// The byte size of this partition 35 | /// 36 | public abstract long Size { get; } 37 | 38 | /// 39 | /// Adds a table with the specified schema to this partition and returns a for writing data to it 40 | /// 41 | /// The schema to add a table for 42 | /// 43 | public abstract ITableDataWriter CreateTableDataWriter(TableDataSchema schema); 44 | 45 | /// 46 | /// Adds a table to this partition 47 | /// 48 | /// 49 | protected void AddTableData(TableData data) 50 | { 51 | _tables.Add(data); 52 | } 53 | 54 | /// 55 | /// Frees resources used by this partition after it has been merged with others. 56 | /// 57 | public abstract void Dispose(); 58 | } 59 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/FieldMapperBase.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using System.Linq; 15 | using ExperienceExtractor.Data.Schema; 16 | using ExperienceExtractor.Processing; 17 | 18 | namespace ExperienceExtractor.Mapping 19 | { 20 | public abstract class FieldMapperBase : IFieldMapper, ICalculatedFieldContainer 21 | { 22 | private Field[] _fields; 23 | 24 | public IList Fields 25 | { 26 | get { return _fields; } 27 | } 28 | 29 | public virtual IEnumerable CalculatedFields 30 | { 31 | get { yield break; } 32 | } 33 | 34 | protected abstract IEnumerable CreateFields(); 35 | 36 | 37 | public abstract bool SetValues(ProcessingScope scope, IList target); 38 | 39 | 40 | public virtual void Initialize(DataProcessor processor) 41 | { 42 | _fields = CreateFields().ToArray(); 43 | } 44 | 45 | public virtual void InitializeRelatedTables(DataProcessor processor, TableDataBuilder table) 46 | { 47 | 48 | } 49 | 50 | public virtual void PostProcessRows(IEnumerable> rows) 51 | { 52 | 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/FieldMapperSet.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using ExperienceExtractor.Processing; 17 | using ExperienceExtractor.Processing.Keys; 18 | 19 | namespace ExperienceExtractor.Mapping 20 | { 21 | public class FieldMapperSet : Dimension 22 | { 23 | public FieldMapperSet(string tableName, 24 | bool inlineFields, 25 | IEnumerable mappers, 26 | Func referenceNameFormatter = null, 27 | IKeyFactory keyFactory = null, 28 | string fieldNamePrefix = null) 29 | 30 | : base(fieldNamePrefix ?? "", tableName, mappers) 31 | { 32 | InlineFields = inlineFields; 33 | ReferenceNameFormatter = referenceNameFormatter; 34 | HashKeyFactory = keyFactory ?? KeyFactory.Default; 35 | } 36 | 37 | public static IFieldMapper Inline(params IFieldMapper[] mappers) 38 | { 39 | return new FieldMapperSet("", true, mappers) {Key = true}; 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/ICalculatedFieldContainer.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Data.Schema; 19 | 20 | namespace ExperienceExtractor.Mapping 21 | { 22 | public interface ICalculatedFieldContainer 23 | { 24 | IEnumerable CalculatedFields { get; } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/IItemFieldLookup.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using Sitecore.Globalization; 14 | 15 | namespace ExperienceExtractor.Mapping 16 | { 17 | public interface IItemFieldLookup 18 | { 19 | string Lookup(object itemId, string path, Language language = null); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/ILabelProvider.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using ExperienceExtractor.Processing; 15 | 16 | namespace ExperienceExtractor.Mapping 17 | { 18 | public interface ILabelProvider 19 | { 20 | void Initialize(DataProcessor processor); 21 | 22 | string GetLabel(object key); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/ITableMapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Processing; 14 | using ExperienceExtractor.Processing.DataSources; 15 | 16 | namespace ExperienceExtractor.Mapping 17 | { 18 | /// 19 | /// Classes implementing this interface define zero ore more tables in the output of an ETL job, and pushes objects to scope while a is processing. 20 | /// A table mapper can only be used by a single 21 | /// 22 | public interface ITableMapper 23 | { 24 | /// 25 | /// Adds and initializes the tables for this 26 | /// 27 | /// The where this table mapper is used 28 | /// The table currently being initialized by the parent 29 | void Initialize(DataProcessor processor, TableDataBuilder parentTable); 30 | 31 | /// 32 | /// Adds rows to the 's tables from the objects currently in scope, and optionally pushes child objects to scope and passes control on to related . 33 | /// 34 | /// 35 | void Process(ProcessingScope context); 36 | } 37 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/MappingHelpers.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | 16 | namespace ExperienceExtractor.Mapping 17 | { 18 | public static class MappingHelpers 19 | { 20 | public static string Discretize(this TValue x, params TValue[] limits) 21 | { 22 | var comparer = Comparer.Default; 23 | 24 | if (comparer.Compare(x, limits[0]) < 0) return "<" + limits[0]; 25 | for (var i = 1; i < limits.Length; i++) 26 | { 27 | if (comparer.Compare(x, limits[i]) < 0) return limits[i - 1] + "-" + limits[i]; 28 | } 29 | return ">" + limits[limits.Length - 1]; 30 | } 31 | 32 | public static TValue TryGet(this TSource source, Func getter, 33 | TValue defaultValue = default(TValue)) 34 | { 35 | if (source == null) return defaultValue; 36 | 37 | var value = getter(source); 38 | return value != null ? value : defaultValue; 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/SimpleTableMapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections; 15 | using System.Collections.Generic; 16 | using System.Linq; 17 | using ExperienceExtractor.Processing; 18 | using ExperienceExtractor.Processing.Helpers; 19 | 20 | namespace ExperienceExtractor.Mapping 21 | { 22 | public class SimpleTableMapper : TableMapperBase 23 | { 24 | private readonly Func _selector; 25 | 26 | public SimpleTableMapper(string name, Func selector, 27 | IEnumerable fieldMappers, IEnumerable tableMappers = null) : 28 | this(selector, new TableDefinition(name) { FieldMappers = fieldMappers.OrEmpty().ToList(), TableMappers = tableMappers.OrEmpty().ToList() }) 29 | { 30 | } 31 | 32 | public SimpleTableMapper(Func selector, 33 | params TableDefinition[] tables) : 34 | base(tables) 35 | { 36 | _selector = selector ?? (ctx => (IEnumerable)ctx.CurrentObject); 37 | } 38 | 39 | public SimpleTableMapper(params TableDefinition[] tables) : 40 | this(null, tables) 41 | { 42 | } 43 | 44 | protected override IEnumerable SelectRowItems(ProcessingScope context) 45 | { 46 | return _selector(context); 47 | } 48 | 49 | public static SimpleTableMapper Inline(Func parentSelector, Func childSelector, params TableDefinition[] tables) 50 | { 51 | return new SimpleTableMapper(parentSelector, new TableDefinition(null) 52 | { 53 | TableMappers = new[] {new SimpleTableMapper(childSelector, tables)} 54 | }); 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Splitting/ISplitter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using ExperienceExtractor.Processing; 15 | 16 | namespace ExperienceExtractor.Mapping.Splitting 17 | { 18 | /// 19 | /// Provides different splits (views) of an object in scope. 20 | /// 21 | public interface ISplitter 22 | { 23 | /// 24 | /// The names of the splits 25 | /// 26 | string[] Names { get; } 27 | 28 | /// 29 | /// Gets the values for the splits given the current 30 | /// 31 | /// The current scope 32 | /// 33 | IEnumerable GetSplits(ProcessingScope scope); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Splitting/MatrixTableMapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections; 15 | using System.Collections.Generic; 16 | using System.Linq; 17 | using ExperienceExtractor.Processing; 18 | using ExperienceExtractor.Processing.Helpers; 19 | 20 | namespace ExperienceExtractor.Mapping.Splitting 21 | { 22 | public class MatrixTableMapper : TableMapperBase 23 | { 24 | public string Name { get; set; } 25 | public Func Selector { get; set; } 26 | 27 | 28 | public MatrixTableMapper(string name, Func>> selector, Func> fieldMappers, 29 | IEnumerable commonFields = null) 30 | { 31 | Name = name; 32 | Selector = selector; 33 | 34 | var splitter = new SimpleSplitter(new[] { "1", "2" }, o => 35 | { 36 | var co = o as Tuple; 37 | return new[] { co.Item1, co.Item2 }; 38 | }); 39 | 40 | 41 | 42 | TableDefinitions.Add(new TableDefinition(name) 43 | { 44 | FieldMappers = new[] { new SplittingFieldMapper(splitter, fieldMappers) }.Concat(commonFields.OrEmpty()).ToArray() 45 | }); 46 | } 47 | 48 | 49 | 50 | 51 | protected override IEnumerable SelectRowItems(ProcessingScope context) 52 | { 53 | return Selector(context); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Splitting/MatrixTypes.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections; 15 | using System.Collections.Generic; 16 | using System.Linq; 17 | using System.Text; 18 | using System.Threading.Tasks; 19 | using ExperienceExtractor.Processing; 20 | using ExperienceExtractor.Processing.Helpers; 21 | 22 | namespace ExperienceExtractor.Mapping.Splitting 23 | { 24 | 25 | public static class MatrixTypes 26 | { 27 | 28 | public static IEnumerable> CoOcurrence(IEnumerable items) 29 | { 30 | var array = items.ToArray(); 31 | for (var i = 0; i < array.Length; i++) 32 | { 33 | for (var j = 0; j < array.Length; j++) 34 | { 35 | yield return Tuple.Create(array[i], array[j]); 36 | } 37 | } 38 | } 39 | 40 | public static IEnumerable> Links(IEnumerable items) 41 | { 42 | var array = items.ToArray(); 43 | for (var i = 0; i < array.Length - 1; i++) 44 | { 45 | yield return Tuple.Create(array[i], array[i + 1]); 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Splitting/SequenceFieldMapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections; 15 | using System.Collections.Generic; 16 | using System.Linq; 17 | using ExperienceExtractor.Data.Schema; 18 | using ExperienceExtractor.Processing; 19 | 20 | namespace ExperienceExtractor.Mapping.Splitting 21 | { 22 | public class SequenceFieldMapper : FieldMapperBase 23 | { 24 | public string Name { get; set; } 25 | public Func Selector { get; set; } 26 | public SequenceType Type { get; set; } 27 | 28 | public SequenceFieldMapper(string name, Func selector, SequenceType type = SequenceType.Path) 29 | { 30 | Name = name; 31 | Selector = selector; 32 | Type = type; 33 | } 34 | 35 | protected override IEnumerable CreateFields() 36 | { 37 | yield return new Field 38 | { 39 | Name = Name, 40 | FieldType = FieldType.Dimension, 41 | ValueType = typeof(string) 42 | }; 43 | } 44 | 45 | public override bool SetValues(ProcessingScope scope, IList target) 46 | { 47 | var values = Selector(scope); 48 | if (values == null) return false; 49 | 50 | var quotedValues = values.Cast().Where(s => s != null).Select(s => "'" + s.ToString().Replace("'", @"\'") + "'").ToArray(); 51 | 52 | if (Type == SequenceType.Path) 53 | { 54 | target[0] = string.Join(",", quotedValues); 55 | } 56 | else if (Type == SequenceType.Set) 57 | { 58 | target[0] = string.Join(",", new HashSet(quotedValues).OrderBy(s => s)); 59 | } 60 | else 61 | { 62 | target[0] = string.Join(",", quotedValues 63 | .ToLookup(s => s).OrderBy(kv => kv.Key).Select(kv => kv.Key + ": " + kv.Count())); 64 | } 65 | 66 | return true; 67 | } 68 | } 69 | 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Splitting/SequenceType.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Mapping.Splitting 20 | { 21 | public enum SequenceType 22 | { 23 | Path, 24 | Set, 25 | CountSet 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Splitting/SimpleSplitter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using ExperienceExtractor.Processing; 17 | 18 | namespace ExperienceExtractor.Mapping.Splitting 19 | { 20 | public class SimpleSplitter : ISplitter 21 | { 22 | public string[] Names { get; private set; } 23 | public IEnumerable GetSplits(ProcessingScope scope) 24 | { 25 | return Selector(scope.CurrentObject); 26 | } 27 | 28 | public Func> Selector { get; private set; } 29 | 30 | public SimpleSplitter(IEnumerable names, Func> selector) 31 | { 32 | Names = names.ToArray(); 33 | Selector = selector; 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Splitting/SplittingTableMapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Linq; 15 | using ExperienceExtractor.Processing; 16 | 17 | namespace ExperienceExtractor.Mapping.Splitting 18 | { 19 | /// 20 | /// Creates nested tables from a splitter with a table for each split. 21 | /// 22 | public class SplittingTableMapper : ITableMapper 23 | { 24 | public ISplitter Splitter { get; set; } 25 | 26 | public ITableMapper[] TableMappers { get; private set; } 27 | 28 | public SplittingTableMapper(ISplitter splitter, 29 | Func factory) 30 | { 31 | Splitter = splitter; 32 | TableMappers = splitter.Names.Select(factory).ToArray(); 33 | } 34 | 35 | 36 | public void Initialize(DataProcessor processor, TableDataBuilder parentTable) 37 | { 38 | foreach (var mapper in TableMappers) 39 | { 40 | mapper.Initialize(processor, parentTable); 41 | } 42 | } 43 | 44 | public void Process(ProcessingScope context) 45 | { 46 | var i = 0; 47 | foreach (var split in Splitter.GetSplits(context)) 48 | { 49 | using (context.Swap(split)) 50 | { 51 | TableMappers[i++].Process(context); 52 | } 53 | } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/TableDefinition.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using ExperienceExtractor.Data.Schema; 15 | 16 | namespace ExperienceExtractor.Mapping 17 | { 18 | /// 19 | /// Defines a table in the output. 20 | /// 21 | public class TableDefinition 22 | { 23 | /// 24 | /// The name of the table in the output 25 | /// 26 | public string Name { get; set; } 27 | 28 | /// 29 | /// The s that define the fields in this table 30 | /// 31 | public IList FieldMappers { get; set; } 32 | 33 | /// 34 | /// s that define tables related to this table 35 | /// 36 | public IList TableMappers { get; set; } 37 | 38 | 39 | public TableDefinition(string name) 40 | { 41 | Name = name; 42 | FieldMappers = new List(); 43 | TableMappers = new List(); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Time/DateDetailLevel.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Mapping.Time 14 | { 15 | public enum DateDetailLevel 16 | { 17 | Year = 0, 18 | Quarter = 1, 19 | Month = 2, 20 | Date = 3 21 | } 22 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Mapping/Time/TimeDetailLevel.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Mapping.Time 14 | { 15 | public enum TimeDetailLevel 16 | { 17 | Hour = 0, 18 | Quarter = 1, 19 | Minute = 2 20 | } 21 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/DataSources/DataSourceBase.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections; 15 | using System.Collections.Generic; 16 | 17 | namespace ExperienceExtractor.Processing.DataSources 18 | { 19 | public abstract class DataSourceBase : IDataSource 20 | { 21 | public abstract IEnumerator GetEnumerator(); 22 | public abstract long? Count { get; } 23 | public IList Filters { get; set; } 24 | 25 | public event EventHandler ItemLoaded; 26 | 27 | public abstract void ApplyUpdateFilter(System.DateTime? lastSaveDate, System.DateTime? lastSaveDateEnd); 28 | 29 | 30 | protected virtual void OnItemLoaded(int e) 31 | { 32 | EventHandler handler = ItemLoaded; 33 | if (handler != null) handler(this, e); 34 | } 35 | 36 | protected DataSourceBase() 37 | { 38 | Filters = new List(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/DataSources/IDataFilter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Processing.DataSources 14 | { 15 | /// 16 | /// Classes implementing this interface can be used to filter data from a 17 | /// 18 | public interface IDataFilter 19 | { 20 | bool IsStagingFilter { get; } 21 | 22 | /// 23 | /// Returns true if the item matches this filter; false, otherwise 24 | /// 25 | /// The item to evaluate this filter against 26 | /// 27 | bool Include(object item); 28 | } 29 | 30 | /// 31 | /// Classes implementing this interface provides an estimated count of the items after this filter has been applied without querying the underlying data source 32 | /// 33 | public interface IEstimatedCountFilter : IDataFilter 34 | { 35 | /// 36 | /// Estimates the number of source items will be included by this filter 37 | /// 38 | /// The number of source items before this filter is applied 39 | /// 40 | long? EstimateCount(long? population); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/DataSources/IDataSource.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections; 15 | using System.Collections.Generic; 16 | using ExperienceExtractor.Api.Jobs; 17 | 18 | namespace ExperienceExtractor.Processing.DataSources 19 | { 20 | /// 21 | /// Classes implementing this interface provides the data to process in a 22 | /// 23 | public interface IDataSource : IEnumerable 24 | { 25 | /// 26 | /// The estimated number of items returned for processing before client side filters are applied 27 | /// 28 | long? Count { get; } 29 | 30 | /// 31 | /// Filters to limit the items returned for processing 32 | /// 33 | IList Filters { get; set; } 34 | 35 | /// 36 | /// This event is triggered every time an item is returned from the underlying data source before client side filters are applied. 37 | /// The argument to the event handler is the current number of items returned from the underlying data source 38 | /// 39 | event EventHandler ItemLoaded; 40 | 41 | 42 | //TODO: Figure out better name 43 | /// 44 | /// Adjusts filters to only include data >= lastSaveDate. This is used for incremental updates of postprocessor targets. 45 | /// 46 | /// 47 | void ApplyUpdateFilter(System.DateTime? lastSaveDate, System.DateTime? lastSaveDateEnd); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/DataSources/TypedDataFilter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Processing.DataSources 14 | { 15 | /// 16 | /// An that filter objects of the type specified. If the object cannot be cast to this type the filter excludes the item 17 | /// 18 | /// 19 | public abstract class TypedDataFilter : IDataFilter where TItem : class 20 | { 21 | public bool Include(object item) 22 | { 23 | var typedItem = item as TItem; 24 | return typedItem != null && Include(typedItem); 25 | } 26 | 27 | protected abstract bool Include(TItem item); 28 | 29 | 30 | public virtual bool IsStagingFilter { get { return false; } } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/DateTime/DateRangeFilter.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Processing.DataSources; 14 | using Sitecore.Analytics.Aggregation.Data.Model; 15 | 16 | namespace ExperienceExtractor.Processing.DateTime 17 | { 18 | public class DateRangeFilter : TypedDataFilter 19 | { 20 | public System.DateTime? Start { get; set; } 21 | public System.DateTime? End { get; set; } 22 | 23 | public DateRangeFilter(System.DateTime? start = null, System.DateTime? end = null) 24 | { 25 | Start = start; 26 | End = end; 27 | } 28 | 29 | public override bool IsStagingFilter { get { return true; } } 30 | 31 | protected override bool Include(IVisitAggregationContext item) 32 | { 33 | if (item == null) return false; 34 | if (Start.HasValue && item.Visit.StartDateTime < Start) return false; 35 | if (End.HasValue && item.Visit.StartDateTime >= End) return false; 36 | 37 | return true; 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/DateTime/ISequenceMapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using ExperienceExtractor.Mapping; 15 | 16 | namespace ExperienceExtractor.Processing.DateTime 17 | { 18 | public interface ISequenceMapper : IFieldMapper 19 | { 20 | TKey GetKeyFromContext(ProcessingScope context); 21 | 22 | bool SetValues(TKey key, IList values); 23 | 24 | TKey Increment(TKey value); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Helpers/DisposeActionWrapper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | 15 | namespace ExperienceExtractor.Processing.Helpers 16 | { 17 | class DisposeActionWrapper : IDisposable 18 | { 19 | private readonly Action _disposeAction; 20 | 21 | public DisposeActionWrapper(Action disposeAction) 22 | { 23 | _disposeAction = disposeAction; 24 | } 25 | 26 | private bool _disposed; 27 | public void Dispose() 28 | { 29 | if (_disposed) return; 30 | _disposeAction(); 31 | _disposed = true; 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Helpers/Indexed.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Processing.Helpers 14 | { 15 | public struct Indexed 16 | { 17 | public int Index { get; set; } 18 | 19 | public TValue Value { get; set; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Helpers/ProtocolHelper.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | namespace ExperienceExtractor.Processing.Helpers 14 | { 15 | using Api.Http.Configuration; 16 | 17 | public static class ProtocolHelper 18 | { 19 | /// 20 | /// If "forceProtocol" setting is used, this will return the corrected url. 21 | /// 22 | public static string EnforceProtocol(string url) 23 | { 24 | return string.IsNullOrWhiteSpace(ExperienceExtractorWebApiConfig.ForceProtocol) 25 | ? url 26 | : url.Replace("http://", string.Format("{0}://", ExperienceExtractorWebApiConfig.ForceProtocol)); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/IDeferedValue.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Data.Schema; 14 | using ExperienceExtractor.Mapping; 15 | using ExperienceExtractor.Processing.Helpers; 16 | 17 | namespace ExperienceExtractor.Processing 18 | { 19 | /// 20 | /// A value that is set in a row after all the row's fields and dimensions has been set 21 | /// 22 | public interface IDeferedValue 23 | { 24 | object GetValue(TableDataBuilder builder, Indexed field, object[] row); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/ITableDataPostProcessor.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Api.Jobs; 19 | using ExperienceExtractor.Data; 20 | using ExperienceExtractor.Export; 21 | using ExperienceExtractor.Processing.DataSources; 22 | 23 | namespace ExperienceExtractor.Processing 24 | { 25 | /// 26 | /// Performs operations on the table data from a completed processing job 27 | /// 28 | public interface ITableDataPostProcessor 29 | { 30 | 31 | /// 32 | /// A descriptive name for this 33 | /// 34 | string Name { get; } 35 | 36 | /// 37 | /// Processes the rows in the tables specified 38 | /// 39 | /// The temporary directory where the output of a job is collected 40 | /// Tables containing the data from a completed job 41 | /// 42 | void Process(string tempDirectory, IEnumerable tables, IJobSpecification job); 43 | 44 | /// 45 | /// Validates the prerequisites for the post processing (e.g. external connections) before a job starts to give early feedback about errors. 46 | /// 47 | void Validate(IEnumerable tables, IJobSpecification job); 48 | 49 | 50 | bool UpdateDataSource(IEnumerable tables, IDataSource source); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/IUpdatingTableDataPostProcessor.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Processing 20 | { 21 | /// 22 | /// Post processors implementing this interface implements logic to update some external data source. 23 | /// 24 | public interface IUpdatingTableDataPostProcessor : ITableDataPostProcessor 25 | { 26 | void AdjustToUpdate(); 27 | 28 | void AdjustToRebuild(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Keys/Fnv1a32.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | 19 | namespace ExperienceExtractor.Processing.Keys 20 | { 21 | public class Fnv1a32 : KeyFactoryBase 22 | { 23 | public Type Type {get { return typeof (int); }} 24 | public override object CalculateKey(IEnumerable values) 25 | { 26 | unchecked 27 | { 28 | const int fnv32Prime = (int) 16777619U; 29 | var hash = (int)2166136261U; 30 | foreach (var o in values) 31 | { 32 | hash = (hash ^ (o == null ? 0 : o.GetHashCode()))*fnv32Prime; 33 | } 34 | 35 | return hash; 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Keys/Fnv1a64.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using ExperienceExtractor.Mapping; 19 | 20 | namespace ExperienceExtractor.Processing.Keys 21 | { 22 | public class Fnv1a64 : KeyFactoryBase 23 | { 24 | public override object CalculateKey(IEnumerable values) 25 | { 26 | unchecked 27 | { 28 | const long fnv64Prime = 0x100000001b3; 29 | var hash = (long)14695981039346656037; 30 | //foreach (var o in values) 31 | //{ 32 | // hash = (hash ^ (o == null ? 0 : o.GetHashCode())) * fnv64Prime; 33 | //} 34 | foreach (var o in values) 35 | { 36 | foreach (var b in ByteUtil.GetBytes(o)) 37 | { 38 | hash = (hash ^ b) * fnv64Prime; 39 | } 40 | } 41 | 42 | return hash; 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Keys/IKeyFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using ExperienceExtractor.Data.Schema; 15 | 16 | namespace ExperienceExtractor.Processing.Keys 17 | { 18 | /// 19 | /// Provides surrogate keys for tables. 20 | /// 21 | public interface IKeyFactory 22 | { 23 | /// 24 | /// Calculates the key from the values specified. 25 | /// 26 | /// 27 | /// 28 | object CalculateKey(IEnumerable values); 29 | 30 | /// 31 | /// Returns a field for a table schema with a name and value type matching the keys this key factory generates. 32 | /// 33 | /// 34 | /// 35 | Field GetKeyField(TableDataSchema schema); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Keys/KeyFactory.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using ExperienceExtractor.Mapping; 14 | using Sitecore.Shell.Applications.ContentEditor; 15 | using DateTime = System.DateTime; 16 | 17 | namespace ExperienceExtractor.Processing.Keys 18 | { 19 | public static class KeyFactory 20 | { 21 | private static IKeyFactory _default = new Fnv1a64(); 22 | 23 | public static IKeyFactory Default 24 | { 25 | get { return _default; } 26 | set { _default = value; } 27 | } 28 | 29 | 30 | //public static long ToKeyOrder(object orderValue, SortOrder sortOrder = SortOrder.Ascending) 31 | //{ 32 | // if (orderValue is DateTime) 33 | // { 34 | // return ((DateTime) orderValue).Ticks; 35 | // } 36 | 37 | // return orderValue != null ? System.Convert.ToInt64(orderValue) : 0L; 38 | //} 39 | 40 | //public static long GetHash64(params object[] values) 41 | //{ 42 | // return GetHash64((IEnumerable)values); 43 | //} 44 | 45 | ////public static long GetHash64(IEnumerable values) 46 | ////{ 47 | //// unchecked 48 | //// { 49 | //// var hash = 17L; 50 | //// foreach (var o in values) 51 | //// { 52 | //// var h = o == null ? 0 : o.GetHashCode(); 53 | //// hash = (hash << 5) + (h ^ hash); 54 | //// } 55 | //// return hash; 56 | //// } 57 | ////} 58 | 59 | //public static long GetHash64(IEnumerable values) 60 | //{ 61 | // unchecked 62 | // { 63 | // const long fnv64Prime = 0x100000001b3; 64 | // var hash = (long)14695981039346656037; 65 | // foreach (var o in values) 66 | // { 67 | // hash = (hash ^ (o == null ? 0 : o.GetHashCode())) * fnv64Prime; 68 | // } 69 | 70 | // return hash; 71 | // } 72 | //} 73 | } 74 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Keys/KeyFactoryBase.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | using System.Text; 17 | using System.Threading.Tasks; 18 | using Sitecore.Data.Comparers; 19 | using ExperienceExtractor.Data.Schema; 20 | using ExperienceExtractor.Mapping; 21 | 22 | namespace ExperienceExtractor.Processing.Keys 23 | { 24 | public abstract class KeyFactoryBase : IKeyFactory 25 | { 26 | public abstract object CalculateKey(IEnumerable values); 27 | 28 | protected virtual string FormatName(TableDataSchema schema, int? postfix) 29 | { 30 | return schema.Name + "Id" + (postfix.HasValue ? "" + postfix : ""); 31 | } 32 | 33 | public virtual Field GetKeyField(TableDataSchema schema) 34 | { 35 | var name = FormatName(schema, null); 36 | var postfix = 1; 37 | while (schema.Fields.Any(f => f.Name == name)) 38 | { 39 | name = FormatName(schema, postfix++); 40 | } 41 | 42 | return new Field {Name = name, FieldType = FieldType.Key, ValueType = typeof(TKey)}; 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Labels/LabelLoader.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using System.Linq; 15 | using ExperienceExtractor.Mapping; 16 | 17 | namespace ExperienceExtractor.Processing.Labels 18 | { 19 | public class LabelLoader 20 | { 21 | public ILabelProvider LabelProvider { get; private set; } 22 | 23 | private readonly LruCache _cache; 24 | 25 | public LabelLoader(ILabelProvider labelProvider, int cacheSize = 1000) 26 | { 27 | LabelProvider = labelProvider; 28 | _cache = new LruCache(cacheSize); 29 | } 30 | 31 | public void SetLabels(IEnumerable> rows, int keyIndex, int labelIndex) 32 | { 33 | foreach (var row in rows) 34 | { 35 | var key = row[keyIndex]; 36 | if (key != null) 37 | { 38 | if (row[labelIndex] == null) 39 | { 40 | row[labelIndex] = _cache.GetOrAdd(key, LabelProvider.GetLabel); 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/Labels/LruCache.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System; 14 | using System.Collections.Generic; 15 | using System.Linq; 16 | 17 | namespace ExperienceExtractor.Processing.Labels 18 | { 19 | public class LruCache 20 | { 21 | public int MaxSize { get; set; } 22 | 23 | private readonly Dictionary, TValue>> _cache 24 | = new Dictionary, TValue>>(); 25 | 26 | private readonly LinkedList _keyList = new LinkedList(); 27 | 28 | public LruCache(int maxSize) 29 | { 30 | MaxSize = maxSize; 31 | } 32 | 33 | public void Clear() 34 | { 35 | _cache.Clear(); 36 | _keyList.Clear(); 37 | } 38 | 39 | public bool Contains(TKey key) 40 | { 41 | return _cache.ContainsKey(key); 42 | } 43 | 44 | public TValue GetOrAdd(TKey key, Func factory) 45 | { 46 | if (MaxSize <= 0) return factory(key); 47 | 48 | KeyValuePair, TValue> value; 49 | if (!_cache.TryGetValue(key, out value)) 50 | { 51 | if (_cache.Count >= MaxSize) 52 | { 53 | var first = _keyList.First; 54 | _keyList.Remove(first); 55 | _cache.Remove(first.Value); 56 | } 57 | value = new KeyValuePair, TValue>(new LinkedListNode(key), factory(key)); 58 | _cache.Add(key, value); 59 | } 60 | else 61 | { 62 | _keyList.Remove(value.Key); 63 | } 64 | _keyList.AddLast(value.Key); 65 | 66 | return value.Value; 67 | } 68 | 69 | public IEnumerable Keys 70 | { 71 | get { return _keyList; } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/ExperienceExtractor/Processing/TableMap.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Collections.Generic; 14 | using ExperienceExtractor.Data; 15 | 16 | namespace ExperienceExtractor.Processing 17 | { 18 | public class TableMap 19 | { 20 | public List Tables { get; set; } 21 | 22 | public TableMap() 23 | { 24 | Tables = new List(); 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /src/ExperienceExtractor/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //-------------------------------------------------------------------------------------------- 2 | // Copyright 2015 Sitecore Corporation A/S 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 4 | // except in compliance with the License. You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software distributed under the 8 | // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 9 | // either express or implied. See the License for the specific language governing permissions 10 | // and limitations under the License. 11 | // ------------------------------------------------------------------------------------------- 12 | 13 | using System.Reflection; 14 | using System.Runtime.CompilerServices; 15 | using System.Runtime.InteropServices; 16 | 17 | // General Information about an assembly is controlled through the following 18 | // set of attributes. Change these attribute values to modify the information 19 | // associated with an assembly. 20 | [assembly: AssemblyTitle("Sitecore Experience Extractor")] 21 | [assembly: AssemblyDescription("")] 22 | [assembly: AssemblyConfiguration("")] 23 | [assembly: AssemblyCompany("Sitecore Corporation")] 24 | [assembly: AssemblyProduct("Sitecore Experience Extractor")] 25 | [assembly: AssemblyCopyright("Copyright © Sitecore Corporation 2015")] 26 | [assembly: AssemblyTrademark("")] 27 | [assembly: AssemblyCulture("")] 28 | 29 | // Setting ComVisible to false makes the types in this assembly not visible 30 | // to COM components. If you need to access a type in this assembly from 31 | // COM, set the ComVisible attribute to true on that type. 32 | [assembly: ComVisible(false)] 33 | 34 | // The following GUID is for the ID of the typelib if this project is exposed to COM 35 | [assembly: Guid("2e0298f5-f87c-4e5e-bda9-194f8ec522d3")] 36 | 37 | // Version information for an assembly consists of the following four values: 38 | // 39 | // Major Version 40 | // Minor Version 41 | // Build Number 42 | // Revision 43 | // 44 | // You can specify all the values or you can default the Build and Revision Numbers 45 | // by using the '*' as shown below: 46 | // [assembly: AssemblyVersion("1.0.*")] 47 | [assembly: AssemblyVersion("0.3.1.*")] 48 | [assembly: AssemblyFileVersion("0.3.1.*")] 49 | 50 | [assembly:InternalsVisibleTo("ExperienceExtractor.Tests")] -------------------------------------------------------------------------------- /src/ExperienceExtractor/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/components/Amo2Tabular.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/src/components/Amo2Tabular.dll -------------------------------------------------------------------------------- /src/components/Microsoft.AnalysisServices.DLL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sitecore/experience-extractor/2e5ba938ba4cf4f821effeb196e2667f7bd9b706/src/components/Microsoft.AnalysisServices.DLL -------------------------------------------------------------------------------- /src/components/Readme.txt: -------------------------------------------------------------------------------- 1 | Copy the assemblies from Sitecore XP 8.0 (pre update 5) here. -------------------------------------------------------------------------------- /src/components80/Readme.txt: -------------------------------------------------------------------------------- 1 | Copy the assemblies from Sitecore XP 8.0 (pre update 5) here. --------------------------------------------------------------------------------