├── Common ├── Migration │ ├── ErrorHandling │ │ ├── ExceptionResponse.cs │ │ ├── CriticalExceptionResponse.cs │ │ ├── MigrationException.cs │ │ └── MigrationError.cs │ ├── Phase1 │ │ ├── IPhase1Processor.cs │ │ ├── IPhase1PreProcessor.cs │ │ ├── Processors │ │ │ ├── CreateWorkItemsProcessor.cs │ │ │ ├── UpdateWorkItemsProcessor.cs │ │ │ └── BaseWorkItemsProcessor.cs │ │ ├── WitBatchRequestGenerators │ │ │ ├── UpdateWitBatchRequestGenerator.cs │ │ │ └── CreateWitBatchRequestGenerator.cs │ │ └── PreProcessors │ │ │ └── IdentityPreProcessor.cs │ ├── AttachmentLink.cs │ ├── Contexts │ │ ├── IBatchMigrationContext.cs │ │ ├── BatchMigrationContext.cs │ │ ├── IMigrationContext.cs │ │ └── MigrationContext.cs │ ├── Phase2 │ │ ├── IPhase2Processor.cs │ │ └── Processors │ │ │ ├── ClearAllRelationsProcessor.cs │ │ │ ├── TargetPostMoveTagsProcessor.cs │ │ │ ├── RevisionHistoryAttachmentsProcessor.cs │ │ │ ├── RemoteLinksProcessor.cs │ │ │ └── WorkItemLinksProcessor.cs │ ├── Phase3 │ │ ├── IPhase3Processor.cs │ │ └── Processors │ │ │ └── SourcePostMoveTagsProcessor.cs │ └── WorkItemLink.cs ├── Validation │ ├── Target │ │ ├── ITargetValidator.cs │ │ └── ResolveTargetWorkItemIds.cs │ ├── Configuration │ │ ├── IConfigurationValidator.cs │ │ ├── ValidateTargetClient.cs │ │ ├── ValidateSourceClient.cs │ │ ├── ValidateSourceQuery.cs │ │ └── ValidateWorkItemRelationTypes.cs │ ├── WorkItem │ │ ├── ValidateWorkItemRevisions.cs │ │ ├── IWorkItemValidator.cs │ │ └── ValidateClassificationNodes.cs │ ├── ValidationException.cs │ ├── IValidationContext.cs │ └── ValidationContext.cs ├── Config │ ├── IConfigReader.cs │ ├── TargetFieldMap.cs │ ├── ConfigConnection.cs │ ├── EmailNotification.cs │ ├── ConfigReaderJson.cs │ └── ConfigJson.cs ├── RetryExhaustedException.cs ├── RetryPermanentException.cs ├── Extensions │ ├── ArrayExtensions.cs │ ├── StringExtensions.cs │ ├── TaskExtensions.cs │ ├── ConcurrentDictionaryExtensions.cs │ └── DictionaryExtensions.cs ├── FieldNames.cs ├── IProcessor.cs ├── RunOrderAttribute.cs ├── IContext.cs ├── WorkItemMigrationState.cs ├── Common.csproj ├── RevAndPhaseStatus.cs ├── ApiWrappers │ ├── Phase2ApiWrapper.cs │ ├── Phase3ApiWrapper.cs │ └── Phase1ApiWrapper.cs ├── BaseContext.cs ├── WorkItemClientConnection.cs ├── RelationHelpers.cs ├── Constants.cs ├── Emailer.cs ├── ConcurrentSet.cs ├── ValidationHeartbeatLogger.cs ├── MigrationHeartbeatLogger.cs ├── AreaAndIterationPathTree.cs └── RetryHelper.cs ├── Logging ├── ConsoleLoggerProvider.cs ├── LogItemsRecorderProvider.cs ├── LoggingRetryExhaustedException.cs ├── LogDestination.cs ├── Logging.csproj ├── FileLoggerProvider.cs ├── LoggingConstants.cs ├── LogLevelOutputMapping.cs ├── LoggingExtensionMethods.cs ├── LogItemsRecorder.cs ├── LoggingRetryHelper.cs ├── FileLogger.cs ├── LogItem.cs ├── WitBatchRequestLogger.cs ├── ConsoleLogger.cs ├── MigratorLogging.cs └── BulkLogger.cs ├── .github └── workflows │ └── dotnetcore.yml ├── WiMigrator ├── Program.cs ├── WiMigrator.csproj └── sample-configuration.json ├── UnitTests ├── UnitTests.csproj ├── Migration │ ├── Phase2 │ │ └── Processors │ │ │ └── TargetPostMoveTagsProcessorTests.cs │ └── Preprocess │ │ └── PreprocessGitCommitLinksTests.cs ├── Common │ ├── ExtensionMethodsTests.cs │ ├── RevAndPhaseStatusTests.cs │ ├── ArrayExtensionMethodsTests.cs │ ├── ClientHelperTests.cs │ └── BatchApi │ │ └── ApiWrapperHelpersTests.cs ├── Helpers │ └── WorkItemTrackingHelperTests.cs └── Logging │ └── LogLevelOutputMappingTests.cs ├── LICENSE ├── WiMigrator.sln ├── SECURITY.md └── README.md /Common/Migration/ErrorHandling/ExceptionResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Common.Migration 2 | { 3 | public class ExceptionResponse 4 | { 5 | public string Message 6 | { 7 | get; set; 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/IPhase1Processor.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace Common.Migration 4 | { 5 | public interface IPhase1Processor : IProcessor 6 | { 7 | Task Process(IMigrationContext context); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Common/Validation/Target/ITargetValidator.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace Common.Validation 4 | { 5 | public interface ITargetValidator 6 | { 7 | string Name { get; } 8 | 9 | Task Validate(IValidationContext context); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/IPhase1PreProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace Common.Migration 4 | { 5 | public interface IPhase1PreProcessor : IProcessor 6 | { 7 | Task Prepare(IMigrationContext context); 8 | 9 | Task Process(IBatchMigrationContext batchContext); 10 | } 11 | } -------------------------------------------------------------------------------- /Common/Config/IConfigReader.cs: -------------------------------------------------------------------------------- 1 | namespace Common.Config 2 | { 3 | public interface IConfigReader 4 | { 5 | ConfigJson Deserialize(); 6 | 7 | void LoadFromFile(string filePath); 8 | 9 | string GetJsonFromFile(string filePath); 10 | 11 | ConfigJson DeserializeText(string input); 12 | } 13 | } -------------------------------------------------------------------------------- /Common/Migration/ErrorHandling/CriticalExceptionResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Common.Migration 2 | { 3 | public class CriticalExceptionResponse 4 | { 5 | public int Count 6 | { 7 | get; set; 8 | } 9 | 10 | public ExceptionResponse Value 11 | { 12 | get; set; 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Common/RetryExhaustedException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Common 4 | { 5 | class RetryExhaustedException : Exception 6 | { 7 | public RetryExhaustedException() : base() { } 8 | public RetryExhaustedException(string message) : base(message) { } 9 | public RetryExhaustedException(string message, Exception inner) : base(message, inner) { } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Common/RetryPermanentException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Common 4 | { 5 | class RetryPermanentException : Exception 6 | { 7 | public RetryPermanentException() : base() { } 8 | public RetryPermanentException(string message) : base(message) { } 9 | public RetryPermanentException(string message, Exception inner) : base(message, inner) { } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Common/Extensions/ArrayExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Common 4 | { 5 | public static class ArrayExtensions 6 | { 7 | public static T[] SubArray(this T[] array, int index, int length) 8 | { 9 | T[] result = new T[length]; 10 | Array.Copy(array, index, result, 0, length); 11 | return result; 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Common/Validation/Configuration/IConfigurationValidator.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace Common.Validation 4 | { 5 | /// 6 | /// Validates data not related to work items 7 | /// 8 | public interface IConfigurationValidator 9 | { 10 | string Name { get; } 11 | 12 | Task Validate(IValidationContext context); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Logging/ConsoleLoggerProvider.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | 3 | namespace Logging 4 | { 5 | public class ConsoleLoggerProvider : ILoggerProvider 6 | { 7 | public ILogger CreateLogger(string categoryName) 8 | { 9 | return new ConsoleLogger(categoryName); 10 | } 11 | 12 | public void Dispose() 13 | { 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Common/Migration/ErrorHandling/MigrationException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Common.Migration 4 | { 5 | public class MigrationException : Exception 6 | { 7 | public MigrationException() : base() { } 8 | public MigrationException(string message) : base(message) { } 9 | public MigrationException(string message, Exception inner) : base(message, inner) { } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Logging/LogItemsRecorderProvider.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | 3 | namespace Logging 4 | { 5 | public class LogItemsRecorderProvider : ILoggerProvider 6 | { 7 | public ILogger CreateLogger(string categoryName) 8 | { 9 | return new LogItemsRecorder(categoryName); 10 | } 11 | 12 | public void Dispose() 13 | { 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Common/Config/TargetFieldMap.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | 3 | namespace Common.Config 4 | { 5 | public class TargetFieldMap 6 | { 7 | [JsonProperty(Required = Required.DisallowNull)] 8 | public string Value { get; set; } 9 | 10 | [JsonProperty(PropertyName = "field-reference-name", Required = Required.DisallowNull)] 11 | public string FieldReferenceName { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Common/Extensions/StringExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Microsoft.VisualStudio.Services.Common; 3 | 4 | namespace Common 5 | { 6 | public static class StringExtensions 7 | { 8 | public static ISet SplitBySemicolonToHashSet(this string input) 9 | { 10 | string[] parts = input.Split(';'); 11 | return parts.ToHashSet(); 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Logging/LoggingRetryExhaustedException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Logging 4 | { 5 | class LoggingRetryExhaustedException : Exception 6 | { 7 | public LoggingRetryExhaustedException() : base() { } 8 | public LoggingRetryExhaustedException(string message) : base(message) { } 9 | public LoggingRetryExhaustedException(string message, Exception inner) : base(message, inner) { } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Common/FieldNames.cs: -------------------------------------------------------------------------------- 1 | namespace Common 2 | { 3 | public class FieldNames 4 | { 5 | public const string WorkItemType = "System.WorkItemType"; 6 | public const string TeamProject = "System.TeamProject"; 7 | public const string AreaPath = "System.AreaPath"; 8 | public const string IterationPath = "System.IterationPath"; 9 | public const string Watermark = "System.Watermark"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Common/IProcessor.cs: -------------------------------------------------------------------------------- 1 | using Common.Config; 2 | 3 | namespace Common 4 | { 5 | public interface IProcessor 6 | { 7 | /// 8 | /// The name of to use for logging 9 | /// 10 | string Name { get; } 11 | 12 | /// 13 | /// Returns true if this processor should be invoked 14 | /// 15 | bool IsEnabled(ConfigJson config); 16 | } 17 | } -------------------------------------------------------------------------------- /Logging/LogDestination.cs: -------------------------------------------------------------------------------- 1 | namespace Logging 2 | { 3 | /// 4 | /// This is a static class instead of an enum so that we can pass into Log Methods with casting to int. 5 | /// We cannot change parameter of these methods because it is not our code. 6 | /// 7 | public static class LogDestination 8 | { 9 | public const int All = 0; 10 | public const int File = 1; 11 | public const int Console = 2; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /.github/workflows/dotnetcore.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | 13 | - name: Setup .NET Core 14 | uses: actions/setup-dotnet@v1 15 | with: 16 | dotnet-version: 2.2.108 17 | 18 | - name: Build with dotnet 19 | run: dotnet build --configuration Release 20 | 21 | - name: Test with dotnet 22 | run: dotnet test 23 | 24 | -------------------------------------------------------------------------------- /Logging/Logging.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0 5 | portable-net45+win8+wp8+wpa81 6 | 7.1 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /Logging/FileLoggerProvider.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | 3 | namespace Logging 4 | { 5 | public class FileLoggerProvider : ILoggerProvider 6 | { 7 | public FileLoggerProvider() 8 | { 9 | } 10 | 11 | public ILogger CreateLogger(string categoryName) 12 | { 13 | return new FileLogger(categoryName); 14 | } 15 | 16 | public void Flush() 17 | { 18 | FileLogger.Flush(); 19 | } 20 | 21 | public void Dispose() 22 | { 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Logging/LoggingConstants.cs: -------------------------------------------------------------------------------- 1 | namespace Logging 2 | { 3 | public class LoggingConstants 4 | { 5 | // Time interval in seconds for how often we check to see if it is time to write to the log file. 6 | public const int CheckInterval = 1; 7 | // Time interval in seconds for the maximum amount of time we will wait before writing to the log file. 8 | public const int LogInterval = 3; 9 | // Maximum number of items we can have in Queue before writing to the log file. 10 | public const long LogItemsUnloggedLimit = 500; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Common/Validation/WorkItem/ValidateWorkItemRevisions.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 3 | 4 | namespace Common.Validation 5 | { 6 | public class ValidateWorkItemRevisions : IWorkItemValidator 7 | { 8 | public string Name => "Work item revisions"; 9 | 10 | public async Task Prepare(IValidationContext context) 11 | { 12 | } 13 | 14 | public async Task Validate(IValidationContext context, WorkItem workItem) 15 | { 16 | context.SourceWorkItemRevision.TryAdd(workItem.Id.Value, workItem.Rev.Value); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Common/RunOrderAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Common 4 | { 5 | [AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, AllowMultiple = false, Inherited = true)] 6 | public class RunOrderAttribute : Attribute 7 | { 8 | //RunOrderAttribute helps determine the sequence for validation 9 | //Add this attribute to the class that implements validation interface 10 | //if you care about the order it runs. Otherwise, it will be added to the end 11 | public RunOrderAttribute(int order) 12 | { 13 | this.Order = order; 14 | } 15 | 16 | public int Order { get; set; } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Common/Validation/Configuration/ValidateTargetClient.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Logging; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace Common.Validation 6 | { 7 | [RunOrder(2)] 8 | public class ValidateTargetPermissions : IConfigurationValidator 9 | { 10 | private ILogger Logger { get; } = MigratorLogging.CreateLogger(); 11 | 12 | public string Name => "Target account permissions"; 13 | 14 | public async Task Validate(IValidationContext context) 15 | { 16 | await ValidationHelpers.CheckBypassRulesPermission(context.TargetClient, context.Config.TargetConnection.Project); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Common/Migration/AttachmentLink.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 2 | 3 | namespace Common.Migration 4 | { 5 | public class AttachmentLink 6 | { 7 | public AttachmentReference AttachmentReference { get; set; } 8 | public string FileName { get; set; } 9 | public string Comment { get; set; } 10 | public long ResourceSize { get; set; } 11 | 12 | public AttachmentLink(string filename, AttachmentReference aRef, long resourceSize, string comment = null) 13 | { 14 | this.FileName = filename; 15 | this.AttachmentReference = aRef; 16 | this.ResourceSize = resourceSize; 17 | this.Comment = comment; 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /WiMigrator/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.Logging; 3 | using Logging; 4 | 5 | namespace WiMigrator 6 | { 7 | class Program 8 | { 9 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 10 | 11 | static void Main(string[] args) 12 | { 13 | try 14 | { 15 | CommandLine commandLine = new CommandLine(args); 16 | commandLine.Run(); 17 | } 18 | catch (Exception ex) 19 | { 20 | Logger.LogError(0, ex, "Closing application due to an Exception: "); 21 | } 22 | finally 23 | { 24 | MigratorLogging.LogSummary(); 25 | } 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /Common/Migration/ErrorHandling/MigrationError.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Common.Migration 4 | { 5 | //Define where the migration of a workitem can go wrong 6 | [Flags] 7 | public enum FailureReason 8 | { 9 | None = 0, 10 | UnsupportedWorkItemType = 1 << 0, 11 | ProjectDifferentFromSource = 1 << 1, 12 | BadRequest = 1 << 2, 13 | CriticalError = 1 << 3, 14 | UnexpectedError = 1 << 4, 15 | AttachmentUploadError = 1 << 5, 16 | AttachmentDownloadError = 1 << 6, 17 | InlineImageUrlFormatError = 1 << 7, 18 | InlineImageUploadError = 1 << 8, 19 | InlineImageDownloadError = 1 << 9, 20 | DuplicateSourceLinksOnTarget = 1 << 10, 21 | CreateBatchFailureError = 1 << 11, 22 | } 23 | } -------------------------------------------------------------------------------- /Common/Migration/Contexts/IBatchMigrationContext.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 3 | 4 | namespace Common.Migration 5 | { 6 | public interface IBatchMigrationContext 7 | { 8 | int BatchId { get; set; } 9 | IList WorkItemMigrationState { get; set; } 10 | IDictionary SourceInlineImageUrlToTargetInlineImageGuid { get; set; } 11 | IList SourceWorkItems { get; set; } 12 | IDictionary TargetIdToSourceWorkItemMapping { get; set; } 13 | IDictionary SourceToTagsOfWorkItemsSuccessfullyMigrated { get; set; } 14 | IDictionary SourceWorkItemIdToTargetWorkItemIdMapping { get; set; } 15 | } 16 | } -------------------------------------------------------------------------------- /Common/IContext.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Concurrent; 2 | using Common.Config; 3 | 4 | namespace Common 5 | { 6 | public interface IContext 7 | { 8 | ConfigJson Config { get; } 9 | 10 | WorkItemClientConnection SourceClient { get; } 11 | 12 | WorkItemClientConnection TargetClient { get; } 13 | 14 | //Mapping of source work items to their url on the source 15 | ConcurrentDictionary WorkItemIdsUris { get; set; } 16 | 17 | //State of all work items to migrate 18 | ConcurrentBag WorkItemsMigrationState { get; set; } 19 | 20 | //remote relation types, do not need to exist on target since they're 21 | //recreated as hyperlinks 22 | ConcurrentSet RemoteLinkRelationTypes { get; set; } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /UnitTests/UnitTests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0 5 | false 6 | 7.1 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /Common/WorkItemMigrationState.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Common.Migration; 3 | 4 | namespace Common 5 | { 6 | public class WorkItemMigrationState 7 | { 8 | public int SourceId { get; set; } 9 | public int? TargetId { get; set; } 10 | 11 | public FailureReason FailureReason { get; set; } 12 | 13 | public State MigrationState { get; set; } 14 | public RequirementForExisting Requirement { get; set; } 15 | public MigrationCompletionStatus MigrationCompleted { get; set; } 16 | public enum State { Create, Existing, Error } 17 | 18 | [Flags] 19 | public enum RequirementForExisting { None, UpdatePhase1, UpdatePhase2 } 20 | [Flags] 21 | public enum MigrationCompletionStatus { None, Phase1, Phase2, Phase3 } 22 | public RevAndPhaseStatus RevAndPhaseStatus { get; set; } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Logging/LogLevelOutputMapping.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | 3 | namespace Logging 4 | { 5 | public static class LogLevelOutputMapping 6 | { 7 | public static string Get(LogLevel logLevel) 8 | { 9 | switch (logLevel) 10 | { 11 | case LogLevel.Trace: 12 | return "Success"; 13 | case LogLevel.Debug: 14 | return "Debug"; 15 | case LogLevel.Information: 16 | return "Info"; 17 | case LogLevel.Warning: 18 | return "Warning"; 19 | case LogLevel.Error: 20 | return "Error"; 21 | case LogLevel.Critical: 22 | return "Critical"; 23 | default: 24 | return "None"; 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Common/Migration/Phase2/IPhase2Processor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 4 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 5 | 6 | namespace Common.Migration 7 | { 8 | public interface IPhase2Processor : IProcessor 9 | { 10 | /// 11 | /// Preprocess the work item batch 12 | /// 13 | Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems); 14 | 15 | /// 16 | /// Process the work item batch 17 | /// 18 | Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Common/Migration/Phase3/IPhase3Processor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 4 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 5 | 6 | namespace Common.Migration 7 | { 8 | public interface IPhase3Processor : IProcessor 9 | { 10 | /// 11 | /// Preprocess the work item batch 12 | /// 13 | Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems); 14 | 15 | /// 16 | /// Process the work item batch 17 | /// 18 | Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Common/Validation/WorkItem/IWorkItemValidator.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 3 | 4 | namespace Common.Validation 5 | { 6 | /// 7 | /// Validates anything work item related 8 | /// 9 | public interface IWorkItemValidator 10 | { 11 | /// 12 | /// The name of the validator to use for logging 13 | /// 14 | string Name { get; } 15 | 16 | /// 17 | /// Populates the context with any data required for validation 18 | /// 19 | /// 20 | Task Prepare(IValidationContext context); 21 | 22 | /// 23 | /// Validates the work item 24 | /// 25 | Task Validate(IValidationContext context, WorkItem workItem); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Common/Migration/WorkItemLink.cs: -------------------------------------------------------------------------------- 1 | namespace Common.Migration 2 | { 3 | public class WorkItemLink 4 | { 5 | //This is the target id of the link (not the target id of the work item we are migrating) 6 | public int Id {get; set;} 7 | public string ReferenceName {get; set;} 8 | public bool IsDirectional {get; set;} 9 | public bool IsForward {get; set;} 10 | //Index will be used only for updates 11 | public int? Index {get; set;} 12 | public string Comment; 13 | public WorkItemLink(int id, string name, bool isDirectional, bool isForward, string comment, int index) 14 | { 15 | this.Id = id; 16 | this.ReferenceName = name; 17 | this.IsDirectional = isDirectional; 18 | this.IsForward = isForward; 19 | this.Comment = comment; 20 | this.Index = index; 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /Common/Config/ConfigConnection.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | 3 | namespace Common.Config 4 | { 5 | public class ConfigConnection 6 | { 7 | [JsonProperty(Required = Required.Always)] 8 | public string Account { get; set; } 9 | 10 | [JsonProperty(Required = Required.Always)] 11 | public string Project { get; set; } 12 | 13 | [JsonProperty(PropertyName = "access-token", Required = Required.DisallowNull)] 14 | public string AccessToken { get; set; } 15 | 16 | [JsonProperty(PropertyName = "use-integrated-auth", Required = Required.DisallowNull)] 17 | public bool UseIntegratedAuth { get; set; } 18 | 19 | // used by JSON.NET to control weather or not AccessToken gets serialized. 20 | // JSON.NET just knows to apply it to AccessToken because it’s in the method name. 21 | public bool ShouldSerializeAccessToken() 22 | { 23 | return false; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Common/Extensions/TaskExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Threading.Tasks; 5 | using System.Linq; 6 | using System.Threading; 7 | 8 | public static class TaskExtensions 9 | { 10 | public static Task ForEachAsync(this IEnumerable source, int degreeOfParallelism, Func func) 11 | { 12 | var partitionId = 0; 13 | var partitioner = Partitioner.Create(source); 14 | var partitions = partitioner.GetPartitions(degreeOfParallelism); 15 | 16 | var tasks = partitions.Select((partition) => Task.Run(async () => 17 | { 18 | int current; 19 | while (partition.MoveNext()) 20 | { 21 | current = Interlocked.Increment(ref partitionId); 22 | await func(partition.Current, current); 23 | } 24 | })); 25 | 26 | return Task.WhenAll(tasks); 27 | } 28 | } -------------------------------------------------------------------------------- /Logging/LoggingExtensionMethods.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.Logging; 3 | 4 | namespace Logging 5 | { 6 | public static class LoggingExtensionMethods 7 | { 8 | public static void LogSuccess(this ILogger logger, string message, params object[] args) 9 | { 10 | logger.LogTrace(message, args); 11 | } 12 | 13 | public static void LogSuccess(this ILogger logger, int eventId, string message, params object[] args) 14 | { 15 | logger.LogTrace(eventId, message, args); 16 | } 17 | 18 | public static void LogSuccess(this ILogger logger, Exception exception, string message, params object[] args) 19 | { 20 | logger.LogTrace(exception, message, args); 21 | } 22 | 23 | public static void LogSuccess(this ILogger logger, int eventId, Exception exception, string message, params object[] args) 24 | { 25 | logger.LogTrace(eventId, exception, message, args); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Common/Common.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0 5 | portable-net45+win8+wp8+wpa81 6 | 7.1 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /Common/RevAndPhaseStatus.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Microsoft.VisualStudio.Services.Common; 3 | 4 | namespace Common 5 | { 6 | public class RevAndPhaseStatus 7 | { 8 | public int Rev { get; set; } 9 | 10 | public ISet PhaseStatus { get; set; } 11 | 12 | public RevAndPhaseStatus() 13 | { 14 | this.PhaseStatus = new HashSet(); 15 | } 16 | 17 | public RevAndPhaseStatus(string revAndPhaseStatusComment) 18 | { 19 | SetRevAndPhaseStatus(revAndPhaseStatusComment); 20 | } 21 | 22 | public void SetRevAndPhaseStatus(string revAndPhaseStatusComment) 23 | { 24 | string[] parts = revAndPhaseStatusComment.Split(';'); 25 | this.Rev = int.Parse(parts[0]); 26 | string[] phaseStatusStringParts = parts.SubArray(1, parts.Length - 1); 27 | this.PhaseStatus = phaseStatusStringParts.ToHashSet(); 28 | } 29 | 30 | public string GetCommentRepresentation() 31 | { 32 | return $"{this.Rev};{string.Join(";", this.PhaseStatus)}"; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /Common/ApiWrappers/Phase2ApiWrapper.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using Common.Migration; 3 | using Logging; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi; 6 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 7 | 8 | namespace Common.ApiWrappers 9 | { 10 | public class Phase2ApiWrapper : BaseBatchApiWrapper 11 | { 12 | protected override ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | 14 | protected override WorkItemTrackingHttpClient GetWorkItemTrackingHttpClient(IMigrationContext migrationContext) 15 | { 16 | return migrationContext.TargetClient.WorkItemTrackingHttpClient; 17 | } 18 | 19 | protected override void UpdateWorkItemMigrationStatus(IBatchMigrationContext batchContext, int sourceId, WorkItem targetWorkItem) 20 | { 21 | WorkItemMigrationState state = batchContext.WorkItemMigrationState.First(w => w.SourceId == sourceId); 22 | state.MigrationCompleted |= WorkItemMigrationState.MigrationCompletionStatus.Phase2; 23 | } 24 | 25 | protected override void BatchCompleted(IMigrationContext migrationContext, IBatchMigrationContext batchContext) 26 | { 27 | // no-op 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Common/ApiWrappers/Phase3ApiWrapper.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using Common.Migration; 3 | using Logging; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi; 6 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 7 | 8 | namespace Common.ApiWrappers 9 | { 10 | public class Phase3ApiWrapper : BaseBatchApiWrapper 11 | { 12 | protected override ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | 14 | protected override WorkItemTrackingHttpClient GetWorkItemTrackingHttpClient(IMigrationContext migrationContext) 15 | { 16 | return migrationContext.SourceClient.WorkItemTrackingHttpClient; 17 | } 18 | 19 | protected override void UpdateWorkItemMigrationStatus(IBatchMigrationContext batchContext, int sourceId, WorkItem targetWorkItem) 20 | { 21 | WorkItemMigrationState state = batchContext.WorkItemMigrationState.First(w => w.SourceId == sourceId); 22 | state.MigrationCompleted |= WorkItemMigrationState.MigrationCompletionStatus.Phase3; 23 | } 24 | 25 | protected override void BatchCompleted(IMigrationContext migrationContext, IBatchMigrationContext batchContext) 26 | { 27 | // no-op 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /WiMigrator/WiMigrator.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | win10-x64 7 | WiMigrator 8 | 7.1 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | Always 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /UnitTests/Migration/Phase2/Processors/TargetPostMoveTagsProcessorTests.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.VisualStudio.TestTools.UnitTesting; 2 | using Moq; 3 | using Common.Config; 4 | using Common.Migration; 5 | 6 | namespace UnitTests.Migration.Phase2.Processors 7 | { 8 | [TestClass] 9 | public class TargetPostMoveTagsProcessorTests 10 | { 11 | private Mock MigrationContextMock; 12 | 13 | [TestInitialize] 14 | public void Initialize() 15 | { 16 | this.MigrationContextMock = new Mock(); 17 | } 18 | 19 | [TestMethod] 20 | public void GetUpdatedTagsFieldWithPostMove_ReturnsCorrectValue() 21 | { 22 | ConfigJson Config = new ConfigJson(); 23 | Config.TargetPostMoveTag = "sample-post-move-tag"; 24 | this.MigrationContextMock.SetupGet(a => a.Config).Returns(Config); 25 | string tagFieldValue = "originalTag"; 26 | string expected = "originalTag; sample-post-move-tag"; 27 | 28 | TargetPostMoveTagsProcessor targetPostMoveTagsProcessor = new TargetPostMoveTagsProcessor(); 29 | string actual = targetPostMoveTagsProcessor.GetUpdatedTagsFieldWithPostMove(this.MigrationContextMock.Object, tagFieldValue); 30 | 31 | Assert.AreEqual(expected, actual); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Common/Validation/ValidationException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.Services.Common; 3 | using Microsoft.VisualStudio.Services.WebApi; 4 | 5 | namespace Common.Validation 6 | { 7 | public class ValidationException : Exception 8 | { 9 | public ValidationException() : base() { } 10 | public ValidationException(string message) : base(message) { } 11 | public ValidationException(string message, AggregateException innerException) : base(message, UnwrapAggregateException(innerException)) { } 12 | public ValidationException(string message, Exception innerException) : base(message, innerException) { } 13 | public ValidationException(string account, VssUnauthorizedException innerException) : base($"Unable to validate {account}, the scopes \"Work items (read and write) and Identity (read)\" are required.", innerException) { } 14 | public ValidationException(string account, VssServiceResponseException innerException) : base($"Unable to connect to {account}, please verify the account name.", innerException) { } 15 | 16 | private static Exception UnwrapAggregateException(Exception exception) 17 | { 18 | if (exception is AggregateException) 19 | { 20 | return ((AggregateException)exception).InnerException; 21 | } 22 | 23 | return exception; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Logging/LogItemsRecorder.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | using System; 3 | 4 | namespace Logging 5 | { 6 | internal class LogItemsRecorder : ILogger 7 | { 8 | private string categoryName; 9 | 10 | public LogItemsRecorder(string categoryName) 11 | { 12 | this.categoryName = categoryName; 13 | } 14 | 15 | public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 16 | { 17 | if (formatter == null) 18 | { 19 | throw new ArgumentNullException(nameof(formatter)); 20 | } 21 | 22 | if (!IsEnabled(logLevel)) 23 | { 24 | return; 25 | } 26 | 27 | var message = formatter(state, exception); 28 | 29 | if (string.IsNullOrEmpty(message)) 30 | { 31 | return; 32 | } 33 | 34 | LogItem logItem = new LogItem(logLevel, DateTime.Now, message, exception, eventId.Id); 35 | MigratorLogging.logItems.Enqueue(logItem); 36 | } 37 | 38 | public bool IsEnabled(LogLevel logLevel) 39 | { 40 | return true; 41 | } 42 | 43 | public IDisposable BeginScope(TState state) 44 | { 45 | return null; 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /Common/Extensions/ConcurrentDictionaryExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Linq; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | 6 | namespace Common 7 | { 8 | public static class ConcurrentDictionaryExtensions 9 | { 10 | public static bool ContainsKeyIgnoringCase(this ConcurrentDictionary dictionary, string desiredKeyOfAnyCase) 11 | { 12 | return GetKeyIgnoringCase(dictionary, desiredKeyOfAnyCase) != null; 13 | } 14 | 15 | public static string GetKeyIgnoringCase(this ConcurrentDictionary dictionary, string desiredKeyOfAnyCase) 16 | { 17 | return dictionary.FirstOrDefault(a => a.Key.Equals(desiredKeyOfAnyCase, StringComparison.OrdinalIgnoreCase)).Key; 18 | } 19 | 20 | public static bool TryGetValueIgnoringCase(this ConcurrentDictionary dictionary, string desiredKeyOfAnyCase, out WorkItemField value) 21 | { 22 | var key = GetKeyIgnoringCase(dictionary, desiredKeyOfAnyCase); 23 | if (key != null) 24 | { 25 | return dictionary.TryGetValue(key, out value); 26 | } 27 | else 28 | { 29 | value = null; 30 | return false; 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Common/BaseContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using Common.Config; 6 | 7 | namespace Common 8 | { 9 | public abstract class BaseContext : IContext 10 | { 11 | public BaseContext(ConfigJson configJson) 12 | { 13 | this.Config = configJson; 14 | this.SourceClient = ClientHelpers.CreateClient(configJson.SourceConnection); 15 | this.TargetClient = ClientHelpers.CreateClient(configJson.TargetConnection); 16 | } 17 | 18 | /// 19 | /// Constructor for test purposes 20 | /// 21 | public BaseContext() 22 | { 23 | } 24 | 25 | public ConfigJson Config { get; } 26 | 27 | public WorkItemClientConnection SourceClient { get; } 28 | 29 | public WorkItemClientConnection TargetClient { get; } 30 | 31 | public ConcurrentDictionary WorkItemIdsUris { get; set; } 32 | 33 | public ConcurrentBag WorkItemsMigrationState { get; set; } = new ConcurrentBag(); 34 | 35 | public ConcurrentDictionary SourceToTargetIds { get; set; } = new ConcurrentDictionary(); 36 | 37 | public ConcurrentSet RemoteLinkRelationTypes { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Common/Migration/Contexts/BatchMigrationContext.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 3 | 4 | namespace Common.Migration 5 | { 6 | public class BatchMigrationContext : IBatchMigrationContext 7 | { 8 | public int BatchId { get; set; } 9 | public IList WorkItemMigrationState { get; set; } 10 | //Inline image urls in html fields on source mapped to the guids of those on target. Used to replace url on target to point to the attachment file that was actually uploaded to the target. 11 | public IDictionary SourceInlineImageUrlToTargetInlineImageGuid { get; set; } = new Dictionary(); 12 | //List of source workitems that need to be migrated 13 | public IList SourceWorkItems {get; set;} = new List(); 14 | public IDictionary TargetIdToSourceWorkItemMapping { get; set; } = new Dictionary(); 15 | public IDictionary SourceToTagsOfWorkItemsSuccessfullyMigrated { get; set; } = new Dictionary(); 16 | public IDictionary SourceWorkItemIdToTargetWorkItemIdMapping { get; set; } = new Dictionary(); 17 | 18 | public BatchMigrationContext(int batchId, IList workItemMigrationState) 19 | { 20 | this.BatchId = batchId; 21 | this.WorkItemMigrationState = workItemMigrationState; 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /Common/Validation/Configuration/ValidateSourceClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Logging; 4 | using Microsoft.Extensions.Logging; 5 | 6 | namespace Common.Validation 7 | { 8 | [RunOrder(1)] 9 | public class ValidateSourcePermissions : IConfigurationValidator 10 | { 11 | private ILogger Logger { get; } = MigratorLogging.CreateLogger(); 12 | 13 | public string Name => "Source account permissions"; 14 | 15 | public async Task Validate(IValidationContext context) 16 | { 17 | //if optional postmovetag is not specified, then we need only read permissions to the source account 18 | //else if postmovetag is specified in the config file, then we will need write permissions to the source account 19 | if (String.IsNullOrEmpty(context.Config.SourcePostMoveTag)) 20 | { 21 | Logger.LogInformation(LogDestination.File, "Checking read permissions on the source project"); 22 | 23 | await ValidationHelpers.CheckReadPermission(context.SourceClient, context.Config.SourceConnection.Project); 24 | } 25 | else 26 | { 27 | Logger.LogInformation(LogDestination.File, "source-post-move-tag is specified, checking write permissions on the source project"); 28 | await ValidationHelpers.CheckBypassRulesPermission(context.SourceClient, context.Config.SourceConnection.Project); 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/Processors/CreateWorkItemsProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Common.Config; 4 | using Logging; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace Common.Migration 8 | { 9 | [RunOrder(1)] 10 | public class CreateWorkItemsProcessor : BaseWorkItemsProcessor 11 | { 12 | protected override ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | 14 | public override string Name => "Create work items"; 15 | 16 | public override bool IsEnabled(ConfigJson config) 17 | { 18 | return true; 19 | } 20 | 21 | public override void PrepareBatchContext(IBatchMigrationContext batchContext, IList workItemsAndStateToMigrate) 22 | { 23 | } 24 | 25 | public override IList GetWorkItemsAndStateToMigrate(IMigrationContext context) 26 | { 27 | return context.WorkItemsMigrationState.Where(wi => wi.MigrationState == WorkItemMigrationState.State.Create).ToList(); 28 | } 29 | 30 | public override int GetWorkItemsToProcessCount(IBatchMigrationContext batchContext) 31 | { 32 | return batchContext.SourceWorkItems?.Count ?? 0; 33 | } 34 | 35 | public override BaseWitBatchRequestGenerator GetWitBatchRequestGenerator(IMigrationContext context, IBatchMigrationContext batchContext) 36 | { 37 | return new CreateWitBatchRequestGenerator(context, batchContext); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Common/WorkItemClientConnection.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Logging; 3 | using Microsoft.Extensions.Logging; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi; 5 | using Microsoft.VisualStudio.Services.Common; 6 | using Microsoft.VisualStudio.Services.WebApi; 7 | 8 | namespace Common 9 | { 10 | public class WorkItemClientConnection 11 | { 12 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | public WorkItemTrackingHttpClient WorkItemTrackingHttpClient { get; private set; } 14 | protected VssCredentials Credentials { get; private set; } 15 | protected Uri Url { get; private set; } 16 | public VssConnection Connection { get; private set; } 17 | 18 | public WorkItemClientConnection(Uri uri, VssCredentials credentials) 19 | { 20 | Connect(uri, credentials); 21 | } 22 | 23 | private void Connect(Uri uri, VssCredentials credentials) 24 | { 25 | this.Url = uri; 26 | this.Credentials = credentials; 27 | this.Connection = new VssConnection(uri, credentials); 28 | 29 | try 30 | { 31 | this.WorkItemTrackingHttpClient = new WorkItemTrackingHttpClient(uri, credentials, new VssHttpRequestSettings { SendTimeout = TimeSpan.FromMinutes(5) }); 32 | } 33 | catch (Exception e) 34 | { 35 | Logger.LogError(LogDestination.File, e, $"Unable to create the WorkItemTrackingHttpClient for {Url}"); 36 | throw e; 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Common/Migration/Contexts/IMigrationContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | 6 | namespace Common.Migration 7 | { 8 | public interface IMigrationContext : IContext 9 | { 10 | ConcurrentDictionary SourceToTargetIds { get; set; } 11 | 12 | //Mapping of targetId of a work item to attribute id of the hyperlink 13 | ConcurrentDictionary TargetIdToSourceHyperlinkAttributeId { get; set; } 14 | 15 | //relations that do exist on the target 16 | ConcurrentSet ValidatedWorkItemLinkRelationTypes { get; set; } 17 | 18 | ConcurrentDictionary> WorkItemTypes { get; set; } 19 | 20 | ConcurrentDictionary SourceFields { get; set; } 21 | 22 | //Source work item id to Tags Field. Tags Field is null if the work item has no tags 23 | ConcurrentDictionary SourceToTags { get; set; } 24 | 25 | ISet HtmlFieldReferenceNames { get; set; } 26 | 27 | ISet TargetAreaPaths { get; set; } 28 | 29 | ISet TargetIterationPaths { get; set; } 30 | 31 | ISet IdentityFields { get; set; } 32 | 33 | ConcurrentSet ValidatedIdentities { get; set; } 34 | ConcurrentSet InvalidIdentities { get; set; } 35 | 36 | IList UnsupportedFields { get; } 37 | 38 | IList FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Common/Migration/Phase2/Processors/ClearAllRelationsProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using Common.Config; 5 | using Logging; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 8 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 9 | 10 | namespace Common.Migration.Phase2Process 11 | { 12 | [RunOrder(1)] 13 | public class ClearAllRelationsProcessor : IPhase2Processor 14 | { 15 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 16 | 17 | public string Name => Constants.RelationPhaseClearAllRelations; 18 | 19 | public bool IsEnabled(ConfigJson config) 20 | { 21 | return true; 22 | } 23 | 24 | public async Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems) 25 | { 26 | 27 | } 28 | 29 | public async Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem) 30 | { 31 | return GetRemoveAllRelationsOperations(batchContext, targetWorkItem); 32 | } 33 | 34 | public IEnumerable GetRemoveAllRelationsOperations(IBatchMigrationContext batchContext, WorkItem targetWorkItem) 35 | { 36 | return targetWorkItem.Relations?.Select((r, index) => MigrationHelpers.GetRelationRemoveOperation(index)); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Common/Config/EmailNotification.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.ComponentModel; 3 | using Newtonsoft.Json; 4 | 5 | namespace Common.Config 6 | { 7 | public class EmailNotification 8 | { 9 | [JsonProperty(PropertyName = "smtp-server", Required = Required.Always)] 10 | public string SmtpServer { get; set; } 11 | 12 | [JsonProperty(PropertyName = "use-ssl", DefaultValueHandling = DefaultValueHandling.Populate)] 13 | [DefaultValue(false)] 14 | public bool UseSsl { get; set; } 15 | 16 | [JsonProperty(PropertyName = "port", DefaultValueHandling = DefaultValueHandling.Populate)] 17 | [DefaultValue(25)] 18 | public int Port { get; set; } 19 | 20 | [JsonProperty(PropertyName = "from-address", DefaultValueHandling = DefaultValueHandling.Populate)] 21 | [DefaultValue("wimigrator@example.com")] 22 | public string FromAddress { get; set; } 23 | 24 | [JsonProperty(PropertyName = "recipient-addresses", DefaultValueHandling = DefaultValueHandling.Populate)] 25 | public List RecipientAddresses { get; set; } 26 | 27 | [JsonProperty(PropertyName = "user-name", Required = Required.DisallowNull)] 28 | public string UserName { get; set; } 29 | 30 | [JsonProperty(PropertyName = "password", Required = Required.DisallowNull)] 31 | public string Password { get; set; } 32 | 33 | // used by JSON.NET to control weather or not Password gets serialized. 34 | // JSON.NET just knows to apply it to Password because it’s in the method name. 35 | public bool ShouldSerializePassword() 36 | { 37 | return false; 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /UnitTests/Common/ExtensionMethodsTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | using Common; 4 | 5 | namespace UnitTests.Common 6 | { 7 | [TestClass] 8 | public class ExtensionMethodsTests 9 | { 10 | [TestMethod] 11 | public void GetKeyIgnoringCase_DictionaryIsEmptyReturnsNull() 12 | { 13 | IDictionary dictionary = new Dictionary(); 14 | 15 | string desiredKeyOfAnyCase = "desiredKEYofANYcase"; 16 | 17 | string result = dictionary.GetKeyIgnoringCase(desiredKeyOfAnyCase); 18 | 19 | Assert.IsNull(result); 20 | } 21 | 22 | [TestMethod] 23 | public void GetKeyIgnoringCase_DictionaryDoesNotContainReturnsNull() 24 | { 25 | IDictionary dictionary = new Dictionary(); 26 | dictionary.Add("NotWhatWeWant", new object()); 27 | 28 | string desiredKeyOfAnyCase = "desiredKEYofANYcase"; 29 | 30 | string result = dictionary.GetKeyIgnoringCase(desiredKeyOfAnyCase); 31 | 32 | Assert.IsNull(result); 33 | } 34 | 35 | [TestMethod] 36 | public void GetKeyIgnoringCase_DictionaryContainsReturnsCorrectKeyFromDictionary() 37 | { 38 | IDictionary dictionary = new Dictionary(); 39 | dictionary.Add("desiredKeyOfAnyCase", new object()); 40 | 41 | string desiredKeyOfAnyCase = "desiredKEYofANYcase"; 42 | 43 | string expected = "desiredKeyOfAnyCase"; 44 | 45 | string actual = dictionary.GetKeyIgnoringCase(desiredKeyOfAnyCase); 46 | 47 | Assert.AreEqual(expected, actual); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Common/ApiWrappers/Phase1ApiWrapper.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Common.Migration; 4 | using Logging; 5 | using Microsoft.Extensions.Logging; 6 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi; 7 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 8 | using Microsoft.VisualStudio.Services.Common; 9 | 10 | namespace Common.ApiWrappers 11 | { 12 | public class Phase1ApiWrapper : BaseBatchApiWrapper 13 | { 14 | protected override ILogger Logger { get; } = MigratorLogging.CreateLogger(); 15 | 16 | protected override WorkItemTrackingHttpClient GetWorkItemTrackingHttpClient(IMigrationContext migrationContext) 17 | { 18 | return migrationContext.TargetClient.WorkItemTrackingHttpClient; 19 | } 20 | 21 | protected override void UpdateWorkItemMigrationStatus(IBatchMigrationContext batchContext, int sourceId, WorkItem targetWorkItem) 22 | { 23 | WorkItemMigrationState state = batchContext.WorkItemMigrationState.First(w => w.SourceId == sourceId); 24 | state.MigrationCompleted |= WorkItemMigrationState.MigrationCompletionStatus.Phase1; 25 | state.TargetId = targetWorkItem.Id.Value; 26 | } 27 | 28 | protected override void BatchCompleted(IMigrationContext migrationContext, IBatchMigrationContext batchContext) 29 | { 30 | var successfullyMigrated = batchContext.WorkItemMigrationState.Where(m => m.MigrationCompleted == WorkItemMigrationState.MigrationCompletionStatus.Phase1).Select(m => new KeyValuePair(m.SourceId, m.TargetId.Value)); 31 | // add successfully migrated workitems to main cache 32 | if (successfullyMigrated.Any()) 33 | { 34 | migrationContext.SourceToTargetIds.AddRange(successfullyMigrated); 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Logging/LoggingRetryHelper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace Logging 6 | { 7 | public class LoggingRetryHelper 8 | { 9 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 10 | 11 | public static void Retry(Action function, int retryCount, int secsDelay = 1) 12 | { 13 | Exception exception = null; 14 | bool succeeded = true; 15 | for (int i = 0; i < retryCount; i++) 16 | { 17 | try 18 | { 19 | succeeded = true; 20 | function(); 21 | return; 22 | } 23 | catch (Exception ex) 24 | { 25 | exception = ex; 26 | 27 | succeeded = false; 28 | Logger.LogInformation(LogDestination.Console, $"Sleeping for {secsDelay} seconds and retrying again for logging"); 29 | 30 | var task = Task.Delay(secsDelay * 1000); 31 | task.Wait(); 32 | 33 | // add 1 second to delay so that each delay is slightly incrementing in wait time 34 | secsDelay += 1; 35 | } 36 | finally 37 | { 38 | if (succeeded && i >= 1) 39 | { 40 | Logger.LogSuccess(LogDestination.File, $"Logging request succeeded."); 41 | } 42 | } 43 | } 44 | 45 | if (exception is null) 46 | { 47 | throw new LoggingRetryExhaustedException($"Retry count exhausted for logging request."); 48 | } 49 | else 50 | { 51 | throw exception; 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /WiMigrator/sample-configuration.json: -------------------------------------------------------------------------------- 1 | { 2 | "source-connection": { 3 | "account": "https://dev.azure.com/fabrikam", 4 | "project": "fabricam-project", 5 | "access-token": "surf54m2is4yict2sdsdsdsdsdsewrcn4jolgprjmr4rjq", 6 | "use-integrated-auth": "false" 7 | }, 8 | "target-connection": { 9 | "account": "https://dev.azure.com/contoso", 10 | "project": "contoso-project", 11 | "access-token": "urf54m2is4yict2ewrhsdkshdkjsdscn4jolgprjmr4rjq", 12 | "use-integrated-auth": "false" 13 | }, 14 | "query": "Shared Queries/Website/All Tasks", 15 | "heartbeat-frequency-in-seconds": 30, 16 | "query-page-size": 20000, 17 | "parallelism": 1, 18 | "max-attachment-size": 62914560, 19 | "link-parallelism": 1, 20 | "attachment-upload-chunk-size": 1048576, 21 | "skip-existing": true, 22 | "move-history": false, 23 | "move-history-limit": 200, 24 | "move-git-links": false, 25 | "move-attachments": false, 26 | "move-links": true, 27 | "source-post-move-tag": "6F078B6C-2A96-453B-A7C3-EACE6E63BB97", 28 | "target-post-move-tag": "6F078B6C-2A96-453B-A7C3-EACE6E63BB97", 29 | "skip-work-items-with-type-missing-fields": false, 30 | "skip-work-items-with-missing-area-path": false, 31 | "skip-work-items-with-missing-iteration-path": false, 32 | "default-area-path": "contoso-project\\missing area path", 33 | "default-iteration-path": "contoso-project\\missing iteration path", 34 | "clear-identity-display-names": false, 35 | "ensure-identities": false, 36 | "include-web-link": true, 37 | "log-level-for-file": "information", 38 | "field-replacements": { 39 | }, 40 | "send-email-notification": true, 41 | "email-notification": { 42 | "smtp-server": "127.0.0.1", 43 | "use-ssl": false, 44 | "port": "25", 45 | "from-address": "wimigrator@example.com", 46 | "user-name": "un", 47 | "password": "pw", 48 | "recipient-addresses": [ 49 | "test1@test.com", 50 | "test2@test.com" 51 | ] 52 | } 53 | } -------------------------------------------------------------------------------- /Logging/FileLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.Logging; 3 | 4 | namespace Logging 5 | { 6 | public class FileLogger : ILogger 7 | { 8 | private string CategoryName; 9 | 10 | public static BulkLogger bulkLogger = new BulkLogger(); 11 | 12 | public FileLogger(string categoryName) 13 | { 14 | CategoryName = categoryName; 15 | } 16 | 17 | public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 18 | { 19 | int logDestination = eventId.Id; 20 | 21 | if (eventId.Id == LogDestination.All || eventId.Id == LogDestination.File) 22 | { 23 | PerformLogging(logLevel, state, exception, formatter, logDestination); 24 | } 25 | } 26 | 27 | private void PerformLogging(LogLevel logLevel, TState state, Exception exception, Func formatter, int logDestination) 28 | { 29 | if (formatter == null) 30 | { 31 | throw new ArgumentNullException(nameof(formatter)); 32 | } 33 | 34 | if (!IsEnabled(logLevel)) 35 | { 36 | return; 37 | } 38 | 39 | var message = formatter(state, exception); 40 | 41 | if (string.IsNullOrEmpty(message)) 42 | { 43 | return; 44 | } 45 | 46 | LogItem logItem = new LogItem(logLevel, DateTime.Now, message, exception, logDestination); 47 | bulkLogger.WriteToQueue(logItem); 48 | } 49 | 50 | public bool IsEnabled(LogLevel logLevel) 51 | { 52 | int comparison = logLevel.CompareTo(MigratorLogging.configMinimumLogLevel); 53 | return comparison >= 0; 54 | } 55 | 56 | public IDisposable BeginScope(TState state) 57 | { 58 | return null; 59 | } 60 | 61 | public static void Flush() 62 | { 63 | bulkLogger.BulkLoggerLog(); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Common/Validation/IValidationContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | 6 | namespace Common.Validation 7 | { 8 | public interface IValidationContext : IContext 9 | { 10 | //Mapping of targetId of a work item to attribute id of the hyperlink 11 | ConcurrentDictionary TargetIdToSourceHyperlinkAttributeId { get; set; } 12 | 13 | /// 14 | /// Fields to read on the work item for any validation steps that need the full work item, 15 | /// which is populated by the Prepare method by the IWorkITemMetadataValidator 16 | /// 17 | ISet RequestedFields { get; } 18 | 19 | ConcurrentDictionary SourceWorkItemRevision { get; set; } 20 | 21 | ConcurrentDictionary SourceFields { get; set; } 22 | 23 | ConcurrentDictionary TargetFields { get; set; } 24 | 25 | ConcurrentDictionary> SourceTypesAndFields { get; } 26 | 27 | ConcurrentDictionary> TargetTypesAndFields { get; } 28 | 29 | ConcurrentSet ValidatedTypes { get; } 30 | 31 | ConcurrentSet ValidatedFields { get; } 32 | 33 | ISet IdentityFields { get; set; } 34 | 35 | ConcurrentSet SkippedTypes { get; } 36 | 37 | ConcurrentSet SkippedFields { get; } 38 | 39 | ISet TargetAreaPaths { get; set; } 40 | 41 | ConcurrentSet ValidatedAreaPaths { get; } 42 | 43 | ConcurrentSet SkippedAreaPaths { get; } 44 | 45 | ISet TargetIterationPaths { get; set; } 46 | 47 | ConcurrentSet ValidatedIterationPaths { get; } 48 | 49 | ConcurrentSet SkippedIterationPaths { get; } 50 | 51 | ConcurrentSet ValidatedWorkItemLinkRelationTypes { get; set; } 52 | 53 | ConcurrentSet SkippedWorkItems { get; } 54 | 55 | IList FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Logging/LogItem.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | using System; 3 | 4 | namespace Logging 5 | { 6 | public class LogItem 7 | { 8 | public LogLevel LogLevel { get; } 9 | 10 | private DateTime DateTimeStamp { get; } 11 | 12 | public string Message { get; private set; } 13 | 14 | public Exception Exception { get; } 15 | 16 | public int LogDestination { get; } 17 | 18 | public LogItem(LogLevel logLevel, DateTime dateTimeStamp, string message, int logDestination) 19 | { 20 | this.LogLevel = logLevel; 21 | this.DateTimeStamp = dateTimeStamp; 22 | this.Message = message; 23 | this.Exception = null; 24 | this.LogDestination = logDestination; 25 | } 26 | 27 | public LogItem(LogLevel logLevel, DateTime dateTimeStamp, string message, Exception exception, int logDestination) 28 | { 29 | this.LogLevel = logLevel; 30 | this.DateTimeStamp = dateTimeStamp; 31 | this.Message = message; 32 | this.Exception = exception; 33 | this.LogDestination = logDestination; 34 | } 35 | 36 | public string OutputFormat(bool includeExceptionMessage, bool includeLogLevelTimeStamp) 37 | { 38 | if (includeExceptionMessage && this.Exception != null && this.Exception.Message != null) 39 | { 40 | this.Message = $"{this.Message}. {this.Exception.Message}"; 41 | } 42 | if (includeLogLevelTimeStamp) 43 | { 44 | // HH specifies 24-hour time format 45 | string timeStamp = DateTimeStampString(); 46 | string logLevelName = LogLevelName(); 47 | return $"[{logLevelName} @{timeStamp}] {this.Message}"; 48 | } 49 | else 50 | { 51 | return this.Message; 52 | } 53 | } 54 | 55 | public virtual string DateTimeStampString() 56 | { 57 | return this.DateTimeStamp.ToString("HH.mm.ss.fff"); 58 | } 59 | 60 | public virtual string LogLevelName() 61 | { 62 | return LogLevelOutputMapping.Get(this.LogLevel); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/Processors/UpdateWorkItemsProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Common.Config; 4 | using Logging; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace Common.Migration 8 | { 9 | [RunOrder(2)] 10 | public class UpdateWorkItemsProcessor : BaseWorkItemsProcessor 11 | { 12 | protected override ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | 14 | public override string Name => "Update work items"; 15 | 16 | public override bool IsEnabled(ConfigJson config) 17 | { 18 | return !config.SkipExisting; 19 | } 20 | 21 | public override IList GetWorkItemsAndStateToMigrate(IMigrationContext context) 22 | { 23 | return context.WorkItemsMigrationState.Where(w => w.MigrationState == WorkItemMigrationState.State.Existing && w.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase1)).ToList(); 24 | } 25 | 26 | public override void PrepareBatchContext(IBatchMigrationContext batchContext, IList workItemsAndStateToMigrate) 27 | { 28 | foreach (var sourceWorkItem in batchContext.SourceWorkItems) 29 | { 30 | var workItemMigrationState = workItemsAndStateToMigrate.Where(w => w.SourceId == sourceWorkItem.Id.Value).FirstOrDefault(); 31 | if (workItemMigrationState != null && workItemMigrationState.TargetId.HasValue) 32 | { 33 | batchContext.TargetIdToSourceWorkItemMapping.Add(workItemMigrationState.TargetId.Value, sourceWorkItem); 34 | } 35 | else 36 | { 37 | Logger.LogWarning(LogDestination.File, $"Expected source work item {sourceWorkItem.Id} to map to a target work item"); 38 | } 39 | } 40 | } 41 | 42 | public override int GetWorkItemsToProcessCount(IBatchMigrationContext batchContext) 43 | { 44 | return batchContext.TargetIdToSourceWorkItemMapping.Count; 45 | } 46 | 47 | public override BaseWitBatchRequestGenerator GetWitBatchRequestGenerator(IMigrationContext context, IBatchMigrationContext batchContext) 48 | { 49 | return new UpdateWitBatchRequestGenerator(context, batchContext); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /UnitTests/Helpers/WorkItemTrackingHelperTests.cs: -------------------------------------------------------------------------------- 1 | using Common; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace UnitTests.Helpers 5 | { 6 | [TestClass] 7 | public class WorkItemTrackingHelperTests 8 | { 9 | [TestMethod] 10 | public void ParseQueryForPaging_StripsOrderByClause() 11 | { 12 | var query = WorkItemTrackingHelpers.ParseQueryForPaging("SELECT * FROM WorkItems order BY System.Id", null); 13 | Assert.AreEqual("SELECT * FROM WorkItems", query); 14 | } 15 | 16 | [TestMethod] 17 | public void ParseQueryForPaging_NoOrderByClause() 18 | { 19 | var parsedQuery = WorkItemTrackingHelpers.ParseQueryForPaging("SELECT * FROM WorkItems", null); 20 | Assert.AreEqual("SELECT * FROM WorkItems", parsedQuery); 21 | } 22 | 23 | [TestMethod] 24 | public void GetPageableQuery_ValidWhereClause() 25 | { 26 | var query = WorkItemTrackingHelpers.GetPageableQuery("SELECT * FROM WorkItems WHERE System.Id = 1", 1, 1); 27 | Assert.AreEqual("SELECT * FROM WorkItems WHERE (System.Id = 1) AND ((System.Watermark > 1) OR (System.Watermark = 1 AND System.Id > 1)) ORDER BY System.Watermark, System.Id", query); 28 | } 29 | 30 | [TestMethod] 31 | public void GetPageableQuery_NoWhereClause() 32 | { 33 | var query = WorkItemTrackingHelpers.GetPageableQuery("SELECT * FROM WorkItems", 1, 1); 34 | Assert.AreEqual("SELECT * FROM WorkItems WHERE ((System.Watermark > 1) OR (System.Watermark = 1 AND System.Id > 1)) ORDER BY System.Watermark, System.Id", query); 35 | } 36 | 37 | [TestMethod] 38 | public void ParseQueryForPaging_InjectsPostMoveTag() 39 | { 40 | var query = WorkItemTrackingHelpers.ParseQueryForPaging("SELECT * FROM WorkItems order BY System.Id", "Migrated"); 41 | Assert.AreEqual("SELECT * FROM WorkItems WHERE System.Tags NOT CONTAINS 'Migrated'", query); 42 | } 43 | 44 | [TestMethod] 45 | public void ParseQueryForPaging_InjectsPostMoveTagWithWhereClause() 46 | { 47 | var query = WorkItemTrackingHelpers.ParseQueryForPaging("SELECT * FROM WorkItems WHERE System.Id = 1 order BY System.Id", "Migrated"); 48 | Assert.AreEqual("SELECT * FROM WorkItems WHERE (System.Id = 1) AND System.Tags NOT CONTAINS 'Migrated'", query); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /WiMigrator.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26430.6 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WiMigrator", "WiMigrator\WiMigrator.csproj", "{B81E3306-3F10-48C4-9D23-C5775884BC31}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTests", "UnitTests\UnitTests.csproj", "{F11B2357-454C-4630-82F2-4F8145C35E45}" 9 | EndProject 10 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Logging", "Logging\Logging.csproj", "{B92E60AB-DE47-44A1-81B8-E6C89155EC3C}" 11 | EndProject 12 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Common", "Common\Common.csproj", "{3ACD2A9A-09D3-4803-BE37-AD21C0CCE594}" 13 | EndProject 14 | Global 15 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 16 | Debug|Any CPU = Debug|Any CPU 17 | Release|Any CPU = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {B81E3306-3F10-48C4-9D23-C5775884BC31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {B81E3306-3F10-48C4-9D23-C5775884BC31}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {B81E3306-3F10-48C4-9D23-C5775884BC31}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {B81E3306-3F10-48C4-9D23-C5775884BC31}.Release|Any CPU.Build.0 = Release|Any CPU 24 | {F11B2357-454C-4630-82F2-4F8145C35E45}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {F11B2357-454C-4630-82F2-4F8145C35E45}.Debug|Any CPU.Build.0 = Debug|Any CPU 26 | {F11B2357-454C-4630-82F2-4F8145C35E45}.Release|Any CPU.ActiveCfg = Release|Any CPU 27 | {F11B2357-454C-4630-82F2-4F8145C35E45}.Release|Any CPU.Build.0 = Release|Any CPU 28 | {B92E60AB-DE47-44A1-81B8-E6C89155EC3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {B92E60AB-DE47-44A1-81B8-E6C89155EC3C}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {B92E60AB-DE47-44A1-81B8-E6C89155EC3C}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {B92E60AB-DE47-44A1-81B8-E6C89155EC3C}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {3ACD2A9A-09D3-4803-BE37-AD21C0CCE594}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {3ACD2A9A-09D3-4803-BE37-AD21C0CCE594}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {3ACD2A9A-09D3-4803-BE37-AD21C0CCE594}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {3ACD2A9A-09D3-4803-BE37-AD21C0CCE594}.Release|Any CPU.Build.0 = Release|Any CPU 36 | EndGlobalSection 37 | GlobalSection(SolutionProperties) = preSolution 38 | HideSolutionNode = FALSE 39 | EndGlobalSection 40 | EndGlobal 41 | -------------------------------------------------------------------------------- /UnitTests/Common/RevAndPhaseStatusTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | using Common; 4 | 5 | namespace UnitTests.Common 6 | { 7 | [TestClass] 8 | public class RevAndPhaseStatusTests 9 | { 10 | [TestMethod] 11 | public void SetRevAndPhaseStatus_ReturnsCorrectValueForRegularCase() 12 | { 13 | string revAndPhaseStatusComment = "123;attachments;git commit links"; 14 | 15 | ISet expectedPhaseStatus = new HashSet(); 16 | expectedPhaseStatus.Add("attachments"); 17 | expectedPhaseStatus.Add("git commit links"); 18 | 19 | RevAndPhaseStatus expected = new RevAndPhaseStatus(); 20 | expected.Rev = 123; 21 | expected.PhaseStatus = expectedPhaseStatus; 22 | 23 | RevAndPhaseStatus actual = new RevAndPhaseStatus(); 24 | actual.SetRevAndPhaseStatus(revAndPhaseStatusComment); 25 | 26 | Assert.AreEqual(expected.Rev, actual.Rev); 27 | 28 | foreach (string item in expected.PhaseStatus) 29 | { 30 | Assert.IsTrue(actual.PhaseStatus.Contains(item)); 31 | } 32 | } 33 | 34 | [TestMethod] 35 | public void SetRevAndPhaseStatus_ReturnsCorrectValueWhenOnlyRev() 36 | { 37 | string revAndPhaseStatusComment = "123"; 38 | 39 | RevAndPhaseStatus expected = new RevAndPhaseStatus(); 40 | expected.Rev = 123; 41 | expected.PhaseStatus = new HashSet(); 42 | 43 | RevAndPhaseStatus actual = new RevAndPhaseStatus(); 44 | actual.SetRevAndPhaseStatus(revAndPhaseStatusComment); 45 | 46 | Assert.AreEqual(expected.Rev, actual.Rev); 47 | Assert.AreEqual(0, actual.PhaseStatus.Count); 48 | } 49 | 50 | [TestMethod] 51 | public void GetCommentRepresentation_ReturnsCorrectValue() 52 | { 53 | RevAndPhaseStatus revAndPhaseStatus = new RevAndPhaseStatus(); 54 | revAndPhaseStatus.Rev = 123; 55 | 56 | ISet phaseStatus = new HashSet(); 57 | phaseStatus.Add("attachments"); 58 | phaseStatus.Add("git commit links"); 59 | 60 | revAndPhaseStatus.PhaseStatus = phaseStatus; 61 | 62 | string expected = "123;attachments;git commit links"; 63 | string actual = revAndPhaseStatus.GetCommentRepresentation(); 64 | 65 | Assert.AreEqual(expected, actual); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /UnitTests/Logging/LogLevelOutputMappingTests.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | using Logging; 4 | 5 | namespace UnitTests.Logging 6 | { 7 | [TestClass] 8 | public class LogLevelOutputMappingTests 9 | { 10 | [TestMethod] 11 | public void Get_LogLevelTraceReturnsSuccess() 12 | { 13 | string expected = "Success"; 14 | LogLevel logLevel = LogLevel.Trace; 15 | 16 | string actual = LogLevelOutputMapping.Get(logLevel); 17 | Assert.AreEqual(expected, actual); 18 | } 19 | 20 | [TestMethod] 21 | public void Get_LogLevelDebugReturnsDebug() 22 | { 23 | string expected = "Debug"; 24 | LogLevel logLevel = LogLevel.Debug; 25 | 26 | string actual = LogLevelOutputMapping.Get(logLevel); 27 | Assert.AreEqual(expected, actual); 28 | } 29 | 30 | [TestMethod] 31 | public void Get_LogLevelInformationReturnsInfo() 32 | { 33 | string expected = "Info"; 34 | LogLevel logLevel = LogLevel.Information; 35 | 36 | string actual = LogLevelOutputMapping.Get(logLevel); 37 | Assert.AreEqual(expected, actual); 38 | } 39 | 40 | [TestMethod] 41 | public void Get_LogLevelWarningReturnsWarning() 42 | { 43 | string expected = "Warning"; 44 | LogLevel logLevel = LogLevel.Warning; 45 | 46 | string actual = LogLevelOutputMapping.Get(logLevel); 47 | Assert.AreEqual(expected, actual); 48 | } 49 | 50 | [TestMethod] 51 | public void Get_LogLevelErrorReturnsError() 52 | { 53 | string expected = "Error"; 54 | LogLevel logLevel = LogLevel.Error; 55 | 56 | string actual = LogLevelOutputMapping.Get(logLevel); 57 | Assert.AreEqual(expected, actual); 58 | } 59 | 60 | [TestMethod] 61 | public void Get_LogLevelCriticalReturnsCritical() 62 | { 63 | string expected = "Critical"; 64 | LogLevel logLevel = LogLevel.Critical; 65 | 66 | string actual = LogLevelOutputMapping.Get(logLevel); 67 | Assert.AreEqual(expected, actual); 68 | } 69 | 70 | [TestMethod] 71 | public void Get_LogLevelNoneReturnsNone() 72 | { 73 | string expected = "None"; 74 | LogLevel logLevel = LogLevel.None; 75 | 76 | string actual = LogLevelOutputMapping.Get(logLevel); 77 | Assert.AreEqual(expected, actual); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /Common/RelationHelpers.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Text.RegularExpressions; 4 | using Common.Migration; 5 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 6 | 7 | namespace Common 8 | { 9 | public class RelationHelpers 10 | { 11 | public static bool IsRelationHyperlinkToSourceWorkItem(IContext context, WorkItemRelation relation, int sourceId) 12 | { 13 | // only hyperlinks can contain the link to the source work item 14 | if (relation.Rel.Equals(Constants.Hyperlink, StringComparison.OrdinalIgnoreCase)) 15 | { 16 | var hyperlinkToSourceWorkItem = context.WorkItemIdsUris[sourceId]; 17 | 18 | var sourceParts = Regex.Split(hyperlinkToSourceWorkItem, "/_apis/wit/workitems/", RegexOptions.IgnoreCase); 19 | var targetParts = Regex.Split(relation.Url, "/_apis/wit/workitems/", RegexOptions.IgnoreCase); 20 | 21 | if (sourceParts.Length == 2 && targetParts.Length == 2) 22 | { 23 | var sourceIdPart = sourceParts.Last(); 24 | var targetIdPart = targetParts.Last(); 25 | 26 | var sourceAccountPart = sourceParts.First().Split("/", StringSplitOptions.RemoveEmptyEntries); 27 | var targetAccountPart = targetParts.First().Split("/", StringSplitOptions.RemoveEmptyEntries); 28 | 29 | // url of the work item can contain project which we want to ignore since the url we generate does not include project 30 | // and we just need to verify the ids are the same and the account are the same. 31 | if (sourceAccountPart.Length > 1 32 | && targetAccountPart.Length > 1 33 | && string.Equals(sourceIdPart, targetIdPart, StringComparison.OrdinalIgnoreCase) 34 | && string.Equals(sourceAccountPart[1], targetAccountPart[1], StringComparison.OrdinalIgnoreCase)) 35 | { 36 | return true; 37 | } 38 | } 39 | } 40 | 41 | return false; 42 | } 43 | 44 | public static bool IsRemoteLinkType(WorkItemRelationType relationType) 45 | { 46 | return relationType.Attributes.TryGetValueOrDefaultIgnoringCase(Constants.RemoteLinkAttributeKey, out var remote) && remote; 47 | } 48 | 49 | public static bool IsRemoteLinkType(IContext context, string relationReferenceName) 50 | { 51 | return context.RemoteLinkRelationTypes.Contains(relationReferenceName); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Common/Validation/Configuration/ValidateSourceQuery.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Threading.Tasks; 4 | using Logging; 5 | using Microsoft.Extensions.Logging; 6 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 7 | 8 | namespace Common.Validation 9 | { 10 | [RunOrder(3)] 11 | public class ValidateSourceQuery : IConfigurationValidator 12 | { 13 | private ILogger Logger { get; } = MigratorLogging.CreateLogger(); 14 | 15 | public string Name => "Source query"; 16 | 17 | public async Task Validate(IValidationContext context) 18 | { 19 | await VerifyQueryExistsAndIsValid(context); 20 | await RunQuery(context); 21 | } 22 | 23 | private async Task VerifyQueryExistsAndIsValid(IValidationContext context) 24 | { 25 | Logger.LogInformation(LogDestination.File, "Checking if the migration query exists in the source project"); 26 | QueryHierarchyItem query; 27 | try 28 | { 29 | query = await WorkItemTrackingHelpers.GetQueryAsync(context.SourceClient.WorkItemTrackingHttpClient, context.Config.SourceConnection.Project, context.Config.Query); 30 | } 31 | catch (Exception e) 32 | { 33 | throw new ValidationException("Unable to read the migration query", e); 34 | } 35 | 36 | if (query.QueryType != QueryType.Flat) 37 | { 38 | throw new ValidationException("Only flat queries are supported for migration"); 39 | } 40 | } 41 | 42 | private async Task RunQuery(IValidationContext context) 43 | { 44 | Logger.LogInformation(LogDestination.File, "Running the migration query in the source project"); 45 | 46 | try 47 | { 48 | var workItemUris = await WorkItemTrackingHelpers.GetWorkItemIdAndReferenceLinksAsync( 49 | context.SourceClient.WorkItemTrackingHttpClient, 50 | context.Config.SourceConnection.Project, 51 | context.Config.Query, 52 | context.Config.SourcePostMoveTag, 53 | context.Config.QueryPageSize - 1 /* Have to subtract -1 from the page size due to a bug in how query interprets page size */); 54 | 55 | context.WorkItemIdsUris = new ConcurrentDictionary(workItemUris); 56 | } 57 | catch (Exception e) 58 | { 59 | throw new ValidationException("Unable to run the migration query", e); 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Common/Constants.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Common 4 | { 5 | public class Constants 6 | { 7 | public const int BatchSize = 200; 8 | public const int PageSize = 200; 9 | 10 | public const string Hyperlink = "Hyperlink"; 11 | public const string AttachedFile = "AttachedFile"; 12 | public const string WorkItemHistory = "WorkItemHistory"; 13 | public const string RelationAttributeName = "name"; 14 | public const string RelationAttributeResourceSize = "resourceSize"; 15 | public const string RelationArtifactLink = "ArtifactLink"; 16 | 17 | public const string RelationAttributeComment = "comment"; 18 | public const string RelationAttributeId = "id"; 19 | public const string RelationAttributeAuthorizedDate = "authorizedDate"; 20 | public const string RelationAttributeResourceCreatedDate = "resourceCreatedDate"; 21 | public const string RelationAttributeResourceModifiedDate = "resourceModifiedDate"; 22 | public const string RelationAttributeRevisedDate = "revisedDate"; 23 | 24 | public const string RelationAttributeGitCommitNameValue = "Fixed in Commit"; 25 | public const string RelationAttributeGitCommitCommentValue = "(Git Commit Link) Comment: "; 26 | public const string Fields = "fields"; 27 | public const string Relations = "relations"; 28 | public const string UsageAttributeKey = "usage"; 29 | public const string UsageAttributeValue = "workItemLink"; 30 | public const string RemoteLinkAttributeKey = "remote"; 31 | 32 | public const string TagsFieldReferenceName = "System.Tags"; 33 | public const string TeamProjectReferenceName = "System.TeamProject"; 34 | 35 | public const string RelationPhaseClearAllRelations = "clear all relations"; 36 | 37 | public const string RelationPhaseAttachments = "attachments"; 38 | public const string RelationPhaseGitCommitLinks = "git commit links"; 39 | public const string RelationPhaseRevisionHistoryAttachments = "revision history attachments"; 40 | public const string RelationPhaseWorkItemLinks = "work item links"; 41 | public const string RelationPhaseRemoteLinks = "remote work item links"; 42 | public const string RelationPhaseSourcePostMoveTags = "source post move tags"; 43 | public const string RelationPhaseTargetPostMoveTags = "target post move tags"; 44 | 45 | public static readonly ISet RelationPhases = new HashSet(new[] { 46 | RelationPhaseAttachments, 47 | RelationPhaseGitCommitLinks, 48 | RelationPhaseRevisionHistoryAttachments, 49 | RelationPhaseWorkItemLinks, 50 | RelationPhaseRemoteLinks 51 | }); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Common/Extensions/DictionaryExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Common.Config; 5 | 6 | namespace Common 7 | { 8 | public static class DictionaryExtensions 9 | { 10 | public static bool ContainsKeyIgnoringCase(this IDictionary dictionary, string desiredKeyOfAnyCase) 11 | { 12 | return GetKeyIgnoringCase(dictionary, desiredKeyOfAnyCase) != null; 13 | } 14 | 15 | public static string GetKeyIgnoringCase(this IDictionary dictionary, string desiredKeyOfAnyCase) 16 | { 17 | return dictionary.FirstOrDefault(a => a.Key.Equals(desiredKeyOfAnyCase, StringComparison.OrdinalIgnoreCase)).Key; 18 | } 19 | 20 | public static bool TryGetValueIgnoringCase(this IDictionary dictionary, string desiredKeyOfAnyCase, out object value) 21 | { 22 | var key = GetKeyIgnoringCase(dictionary, desiredKeyOfAnyCase); 23 | if (key != null) 24 | { 25 | return dictionary.TryGetValue(key, out value); 26 | } 27 | else 28 | { 29 | value = null; 30 | return false; 31 | } 32 | } 33 | 34 | public static bool TryGetValueOrDefaultIgnoringCase(this IDictionary dictionary, string key, out V value) 35 | { 36 | if (dictionary.TryGetValueIgnoringCase(key, out object objectValue) && objectValue is V) 37 | { 38 | value = (V)objectValue; 39 | return true; 40 | } 41 | else 42 | { 43 | value = default; 44 | return false; 45 | } 46 | } 47 | 48 | public static bool ContainsKeyIgnoringCase(this IDictionary dictionary, string desiredKeyOfAnyCase) 49 | { 50 | return GetKeyIgnoringCase(dictionary, desiredKeyOfAnyCase) != null; 51 | } 52 | 53 | public static string GetKeyIgnoringCase(this IDictionary dictionary, string desiredKeyOfAnyCase) 54 | { 55 | return dictionary.FirstOrDefault(a => a.Key.Equals(desiredKeyOfAnyCase, StringComparison.OrdinalIgnoreCase)).Key; 56 | } 57 | 58 | public static bool TryGetValueIgnoringCase(this IDictionary dictionary, string desiredKeyOfAnyCase, out TargetFieldMap value) 59 | { 60 | var key = GetKeyIgnoringCase(dictionary, desiredKeyOfAnyCase); 61 | if (key != null) 62 | { 63 | return dictionary.TryGetValue(key, out value); 64 | } 65 | else 66 | { 67 | value = null; 68 | return false; 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /UnitTests/Common/ArrayExtensionMethodsTests.cs: -------------------------------------------------------------------------------- 1 | using Common; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace UnitTests.Common 5 | { 6 | [TestClass] 7 | public class ArrayExtensionMethodsTests 8 | { 9 | [TestMethod] 10 | public void SubArray_ReturnsCorrectValueForRegularCase() 11 | { 12 | int[] array = { 0, 1, 2, 3, 4, 5, 6, 7 }; 13 | int index = 3; 14 | int length = 4; 15 | 16 | int[] actual = ArrayExtensions.SubArray(array, index, length); 17 | int[] expected = { 3, 4, 5, 6 }; 18 | 19 | for (int i = 0; i < length; i++) 20 | { 21 | Assert.AreEqual(expected[i], actual[i]); 22 | } 23 | } 24 | 25 | [TestMethod] 26 | public void SubArray_ReturnsCorrectValueForBeginningToMiddle() 27 | { 28 | int[] array = { 0, 1, 2, 3, 4, 5, 6, 7 }; 29 | int index = 0; 30 | int length = 4; 31 | 32 | int[] actual = ArrayExtensions.SubArray(array, index, length); 33 | int[] expected = { 0, 1, 2, 3 }; 34 | 35 | for (int i = 0; i < length; i++) 36 | { 37 | Assert.AreEqual(expected[i], actual[i]); 38 | } 39 | } 40 | 41 | [TestMethod] 42 | public void SubArray_ReturnsCorrectValueForLength1() 43 | { 44 | int[] array = { 0, 1, 2, 3, 4, 5, 6, 7 }; 45 | int index = 3; 46 | int length = 1; 47 | 48 | int[] actual = ArrayExtensions.SubArray(array, index, length); 49 | int[] expected = { 3 }; 50 | 51 | for (int i = 0; i < length; i++) 52 | { 53 | Assert.AreEqual(expected[i], actual[i]); 54 | } 55 | } 56 | 57 | [TestMethod] 58 | public void SubArray_ReturnsCorrectValueForLength0() 59 | { 60 | int[] array = { 0, 1, 2, 3, 4, 5, 6, 7 }; 61 | int index = 3; 62 | int length = 0; 63 | 64 | int[] actual = ArrayExtensions.SubArray(array, index, length); 65 | int[] expected = { }; 66 | 67 | for (int i = 0; i < length; i++) 68 | { 69 | Assert.AreEqual(expected[i], actual[i]); 70 | } 71 | } 72 | 73 | [TestMethod] 74 | public void SubArray_ReturnsCorrectValueForFullLength() 75 | { 76 | int[] array = { 0, 1, 2, 3, 4, 5, 6, 7 }; 77 | int index = 0; 78 | int length = 8; 79 | 80 | int[] actual = ArrayExtensions.SubArray(array, index, length); 81 | int[] expected = { 0, 1, 2, 3, 4, 5, 6, 7 }; 82 | 83 | for (int i = 0; i < length; i++) 84 | { 85 | Assert.AreEqual(expected[i], actual[i]); 86 | } 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /UnitTests/Migration/Preprocess/PreprocessGitCommitLinksTests.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.VisualStudio.TestTools.UnitTesting; 2 | using Common.Migration; 3 | 4 | namespace UnitTests.Migration.Preprocess 5 | { 6 | [TestClass] 7 | public class PreprocessGitCommitLinksTests 8 | { 9 | [TestMethod] 10 | public void ConvertGitCommitLinkToHyperLink_NullTest() 11 | { 12 | string account = "https://dev.azure.com/account/"; 13 | var actualHyperlink = GitCommitLinksProcessor.ConvertGitCommitLinkToHyperLink(1, null, account); 14 | Assert.AreEqual(null, actualHyperlink); 15 | } 16 | 17 | [TestMethod] 18 | public void ConvertGitCommitLinkToHyperLink_EmptyTest() 19 | { 20 | string account = "https://dev.azure.com/account/"; 21 | var actualHyperlink = GitCommitLinksProcessor.ConvertGitCommitLinkToHyperLink(1, "", account); 22 | Assert.AreEqual(null, actualHyperlink); 23 | } 24 | 25 | [TestMethod] 26 | public void ConvertGitCommitLinkToHyperLink_CorrectArtifactLink() 27 | { 28 | var artifactLink = "vstfs:///Git/Commit/b924d696-3eae-4116-8443-9a18392d8544%2ffb240610-b309-4925-8502-65ff76312c40%2fb8b676f5ec7d5b88df15258bec81c8a2ded4a05a"; 29 | string account = "https://dev.azure.com/account/"; 30 | var actualHyperlink = GitCommitLinksProcessor.ConvertGitCommitLinkToHyperLink(1, artifactLink, account); 31 | string expectedHyperlink = "https://dev.azure.com/account/b924d696-3eae-4116-8443-9a18392d8544/_git/fb240610-b309-4925-8502-65ff76312c40/commit/b8b676f5ec7d5b88df15258bec81c8a2ded4a05a"; 32 | Assert.AreEqual(expectedHyperlink, actualHyperlink); 33 | } 34 | 35 | [TestMethod] 36 | public void ConvertGitCommitLinkToHyperLink_BadArtifactLinkWithTwoGuids() 37 | { 38 | string account = "https://dev.azure.com/account/"; 39 | var artifactLink = "vstfs:///Git/Commit/b924d696-3eae-4116-8443-9a18392d8544%2ffb240610-b309-4925-8502-65ff76312c40"; 40 | var actualHyperlink = GitCommitLinksProcessor.ConvertGitCommitLinkToHyperLink(1, artifactLink, account); 41 | Assert.AreEqual(null, actualHyperlink); 42 | } 43 | 44 | [TestMethod] 45 | public void ConvertGitCommitLinkToHyperLink_BadArtifactLink() 46 | { 47 | string account = "https://dev.azure.com/account/"; 48 | //3rd separator is a %2e instead of %2f which stands for / 49 | var artifactLink = "vstfs:///Git/Commit/b924d696-3eae-4116-8443-9a18392d8544%2ffb240610-b309-4925-8502-65ff76312c40%2eb8b676f5ec7d5b88df15258bec81c8a2ded4a05a"; 50 | var actualHyperlink = GitCommitLinksProcessor.ConvertGitCommitLinkToHyperLink(1, artifactLink, account); 51 | Assert.AreEqual(null, actualHyperlink); 52 | } 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Logging/WitBatchRequestLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | 6 | namespace Logging 7 | { 8 | public static class WitBatchRequestLogger 9 | { 10 | public static void Log(IList witBatchRequests, IList witBatchResponses, int batchId) 11 | { 12 | try 13 | { 14 | string filePath = GetFilePathBasedOnTime(batchId); 15 | var canUseWitBatchResponses = witBatchRequests.Count == witBatchResponses?.Count; 16 | 17 | using (var streamWriter = File.AppendText(filePath)) 18 | { 19 | for (int i = 0; i < witBatchRequests.Count; i++) 20 | { 21 | var witBatchRequest = witBatchRequests[i]; 22 | streamWriter.WriteLine($"WIT BATCH REQUEST {i+1}:"); 23 | streamWriter.WriteLine(); 24 | streamWriter.WriteLine($"METHOD: {witBatchRequest.Method}"); 25 | streamWriter.WriteLine($"URI: {witBatchRequest.Uri}"); 26 | streamWriter.WriteLine(); 27 | streamWriter.WriteLine("BODY:"); 28 | streamWriter.WriteLine(witBatchRequest.Body); 29 | streamWriter.WriteLine(); 30 | 31 | if (canUseWitBatchResponses) 32 | { 33 | var witBatchResponse = witBatchResponses[i]; 34 | streamWriter.WriteLine("RESPONSE CODE:"); 35 | streamWriter.WriteLine(witBatchResponse.Code); 36 | streamWriter.WriteLine(); 37 | streamWriter.WriteLine("RESPONSE BODY:"); 38 | streamWriter.WriteLine(witBatchResponse.Body); 39 | streamWriter.WriteLine(); 40 | } 41 | } 42 | } 43 | } 44 | catch 45 | { 46 | // Do nothing because we don't want this to block Program execution. 47 | } 48 | } 49 | 50 | public static string GetFilePathBasedOnTime(int batchId) 51 | { 52 | try 53 | { 54 | string currentDateTime = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss-FFF"); 55 | return $"WitBatchRequestsFromBatch{batchId}WithFailure{currentDateTime}.log"; 56 | } 57 | catch (Exception ex) 58 | { 59 | string defaultFileName = "WitBatchRequestsFromBatchWithFailure_LogFile"; 60 | Console.WriteLine($"Could not give log file a special name due to below Exception. Naming the file: \"{defaultFileName}\" instead.\n{ex}"); 61 | return defaultFileName; 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Common/Migration/Contexts/MigrationContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Collections.ObjectModel; 5 | using Common.Config; 6 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 7 | 8 | namespace Common.Migration 9 | { 10 | public class MigrationContext : BaseContext, IMigrationContext 11 | { 12 | //Mapping of targetId of a work item to attribute id of the hyperlink 13 | public ConcurrentDictionary TargetIdToSourceHyperlinkAttributeId { get; set; } = new ConcurrentDictionary(); 14 | 15 | public ConcurrentSet ValidatedWorkItemLinkRelationTypes { get; set; } 16 | 17 | public ConcurrentDictionary> WorkItemTypes { get; set; } 18 | 19 | public ConcurrentDictionary SourceFields { get; set; } = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); 20 | 21 | public ConcurrentDictionary SourceToTags { get; set; } = new ConcurrentDictionary(); 22 | 23 | public ISet HtmlFieldReferenceNames { get; set; } = new HashSet(StringComparer.OrdinalIgnoreCase); 24 | 25 | public ISet TargetAreaPaths { get; set; } 26 | 27 | public ISet TargetIterationPaths { get; set; } 28 | 29 | public ISet IdentityFields { get; set; } 30 | 31 | public ConcurrentSet ValidatedIdentities { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 32 | 33 | public ConcurrentSet InvalidIdentities { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 34 | 35 | public IList UnsupportedFields => unsupportedFields; 36 | 37 | public IList FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; } 38 | 39 | // List of fields that we do not support in migration because they are related to the board or another reason. 40 | private readonly IList unsupportedFields = new ReadOnlyCollection(new[]{ 41 | "System.BoardColumn", 42 | "System.BoardColumnDone", 43 | "Kanban.Column", 44 | "Kanban.Column.Done", 45 | "System.BoardLane", 46 | "System.AreaId", 47 | "System.IterationId", 48 | "System.IterationLevel1", 49 | "System.IterationLevel2", 50 | "System.IterationLevel3", 51 | "System.IterationLevel4", 52 | "System.IterationLevel5", 53 | "System.IterationLevel6", 54 | "System.IterationLevel7", 55 | "System.AreaLevel1", 56 | "System.AreaLevel2", 57 | "System.AreaLevel3", 58 | "System.AreaLevel4", 59 | "System.AreaLevel5", 60 | "System.AreaLevel6", 61 | "System.AreaLevel7" 62 | }); 63 | 64 | public MigrationContext(ConfigJson configJson) : base(configJson) 65 | { 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /Common/Migration/Phase3/Processors/SourcePostMoveTagsProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using Microsoft.Extensions.Logging; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 6 | using Common.Config; 7 | using Logging; 8 | 9 | namespace Common.Migration.Phase3.Processors 10 | { 11 | public class SourcePostMoveTagsProcessor : IPhase3Processor 12 | { 13 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 14 | 15 | public string Name => Constants.RelationPhaseSourcePostMoveTags; 16 | 17 | public bool IsEnabled(ConfigJson config) 18 | { 19 | return !string.IsNullOrEmpty(config.SourcePostMoveTag); 20 | } 21 | 22 | public async Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems) 23 | { 24 | 25 | } 26 | 27 | public async Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem) 28 | { 29 | IList jsonPatchOperations = new List(); 30 | JsonPatchOperation addPostMoveTagOperation = AddPostMoveTag(migrationContext, sourceWorkItem); 31 | jsonPatchOperations.Add(addPostMoveTagOperation); 32 | return jsonPatchOperations; 33 | } 34 | 35 | // here we modify the in-memory workItem.Fields because that is the easiest way to handle adding Post-Move-Tag 36 | private JsonPatchOperation AddPostMoveTag(IMigrationContext migrationContext, WorkItem sourceWorkItem) 37 | { 38 | string tagKey = sourceWorkItem.Fields.GetKeyIgnoringCase(Constants.TagsFieldReferenceName); 39 | 40 | if (tagKey != null) // Tags Field already exists 41 | { 42 | string existingTagsValue = (string)sourceWorkItem.Fields[tagKey]; 43 | string updatedTagsFieldWithPostMove = GetUpdatedTagsFieldWithPostMove(migrationContext, existingTagsValue); 44 | 45 | KeyValuePair field = new KeyValuePair(tagKey, updatedTagsFieldWithPostMove); 46 | return MigrationHelpers.GetJsonPatchOperationReplaceForField(field); 47 | } 48 | else // Tags Field does not exist, so we add it here 49 | { 50 | KeyValuePair field = new KeyValuePair(Constants.TagsFieldReferenceName, migrationContext.Config.SourcePostMoveTag); 51 | return MigrationHelpers.GetJsonPatchOperationAddForField(field); 52 | } 53 | } 54 | 55 | public string GetUpdatedTagsFieldWithPostMove(IMigrationContext migrationContext, string tagFieldValue) 56 | { 57 | string postMoveTag = migrationContext.Config.SourcePostMoveTag; 58 | return $"{tagFieldValue}; {postMoveTag}"; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /Common/Emailer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net; 3 | using System.Net.Mail; 4 | using Common.Config; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace Logging 8 | { 9 | public class Emailer 10 | { 11 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 12 | 13 | public Emailer() 14 | { 15 | } 16 | 17 | public void SendEmail(ConfigJson configJson, string body) 18 | { 19 | bool sendEmailNotification = configJson.SendEmailNotification; 20 | EmailNotification emailNotification = configJson.EmailNotification; 21 | 22 | // only proceed if sendEmailNotification && emailNotification != null 23 | if (sendEmailNotification && emailNotification == null) 24 | { 25 | throw new Exception("send-email-notification is set to true but there are no email-notification" + 26 | " details specified. Please set send-email-notification to false or specify details for" + 27 | " email-notification in the configuration-file."); 28 | } 29 | else if (!sendEmailNotification || emailNotification == null) 30 | { 31 | return; 32 | } 33 | 34 | SmtpClient client = new SmtpClient(emailNotification.SmtpServer, emailNotification.Port); 35 | client.EnableSsl = emailNotification.UseSsl; 36 | client.UseDefaultCredentials = false; 37 | client.Credentials = new NetworkCredential(emailNotification.UserName, emailNotification.Password); 38 | MailMessage message = new MailMessage(); 39 | 40 | MailAddress from = new MailAddress(emailNotification.FromAddress); 41 | message.From = from; 42 | 43 | if (emailNotification.RecipientAddresses == null || emailNotification.RecipientAddresses.Count == 0) 44 | { 45 | throw new Exception("You must specify one or more recipient-addresses under email-notification."); 46 | } 47 | 48 | foreach (string address in emailNotification.RecipientAddresses) 49 | { 50 | message.To.Add(address); 51 | } 52 | 53 | message.Body = body; 54 | message.BodyEncoding = System.Text.Encoding.UTF8; 55 | message.Subject = $"WiMigrator Summary at {DateTime.Now.ToString()}"; 56 | message.SubjectEncoding = System.Text.Encoding.UTF8; 57 | 58 | try 59 | { 60 | client.Send(message); 61 | } 62 | catch (SmtpException e) 63 | { 64 | Logger.LogError(LogDestination.File, e, "Could not send run summary email because of an issue with the SMTP server." + 65 | " Please ensure that the server works and your configuration file input is correct"); 66 | } 67 | catch (Exception e) 68 | { 69 | Logger.LogError(LogDestination.File, e, "Could not send run summary email because of an Exception"); 70 | } 71 | 72 | message.Dispose(); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Common/Validation/Configuration/ValidateWorkItemRelationTypes.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Logging; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 8 | 9 | namespace Common.Validation 10 | { 11 | public class ValidateWorkItemRelationTypes : IConfigurationValidator 12 | { 13 | private ILogger Logger { get; } = MigratorLogging.CreateLogger(); 14 | 15 | public string Name => "Relation types"; 16 | 17 | public async Task Validate(IValidationContext context) 18 | { 19 | if (context.Config.MoveLinks) 20 | { 21 | await VerifyRelationTypesExistsOnTarget(context); 22 | } 23 | } 24 | 25 | private async Task VerifyRelationTypesExistsOnTarget(IValidationContext context) 26 | { 27 | var sourceRelationTypes = await WorkItemTrackingHelpers.GetRelationTypesAsync(context.SourceClient.WorkItemTrackingHttpClient); 28 | var targetRelationTypes = await WorkItemTrackingHelpers.GetRelationTypesAsync(context.TargetClient.WorkItemTrackingHttpClient); 29 | 30 | foreach (var relationType in sourceRelationTypes) 31 | { 32 | //retrieve relations which are of type workitemlink defined by attribute kvp {"usage", "workitemlink"} 33 | //exclude remote link types because they need to be converted into hyperlinks 34 | if (IsWorkItemLinkType(relationType)) 35 | { 36 | if (RelationHelpers.IsRemoteLinkType(relationType)) 37 | { 38 | context.RemoteLinkRelationTypes.Add(relationType.ReferenceName); 39 | } 40 | else 41 | { 42 | if (TargetHasRelationType(relationType, targetRelationTypes)) 43 | { 44 | context.ValidatedWorkItemLinkRelationTypes.Add(relationType.ReferenceName); 45 | } 46 | else 47 | { 48 | Logger.LogWarning(LogDestination.File, $"Target: Relation type {relationType.ReferenceName} does not exist"); 49 | } 50 | } 51 | } 52 | } 53 | } 54 | 55 | private bool TargetHasRelationType(WorkItemRelationType relation, IList targetRelationTypes) 56 | { 57 | return targetRelationTypes.Where(a => string.Equals(relation.ReferenceName, a.ReferenceName, StringComparison.OrdinalIgnoreCase)).Any(); 58 | } 59 | 60 | private bool IsWorkItemLinkType(WorkItemRelationType relationType) 61 | { 62 | return relationType.Attributes.TryGetValueOrDefaultIgnoringCase(Constants.UsageAttributeKey, out var usage) && 63 | String.Equals(relationType.Attributes[Constants.UsageAttributeKey].ToString(), Constants.UsageAttributeValue, StringComparison.OrdinalIgnoreCase); 64 | } 65 | } 66 | } -------------------------------------------------------------------------------- /Common/Migration/Phase2/Processors/TargetPostMoveTagsProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using Microsoft.Extensions.Logging; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 6 | using Common.Config; 7 | using Logging; 8 | 9 | namespace Common.Migration 10 | { 11 | public class TargetPostMoveTagsProcessor : IPhase2Processor 12 | { 13 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 14 | 15 | public TargetPostMoveTagsProcessor() 16 | { 17 | } 18 | 19 | public string Name => Constants.RelationPhaseTargetPostMoveTags; 20 | 21 | public bool IsEnabled(ConfigJson config) 22 | { 23 | return !string.IsNullOrEmpty(config.TargetPostMoveTag); 24 | } 25 | 26 | public async Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems) 27 | { 28 | 29 | } 30 | 31 | public async Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem) 32 | { 33 | IList jsonPatchOperations = new List(); 34 | JsonPatchOperation addPostMoveTagOperation = AddPostMoveTag(migrationContext, targetWorkItem); 35 | jsonPatchOperations.Add(addPostMoveTagOperation); 36 | return jsonPatchOperations; 37 | } 38 | 39 | // here we modify the in-memory workItem.Fields because that is the easiest way to handle adding Post-Move-Tag 40 | private JsonPatchOperation AddPostMoveTag(IMigrationContext migrationContext, WorkItem targetWorkItem) 41 | { 42 | string tagKey = targetWorkItem.Fields.GetKeyIgnoringCase(Constants.TagsFieldReferenceName); 43 | 44 | if (tagKey != null) // Tags Field already exists 45 | { 46 | string existingTagsValue = (string)targetWorkItem.Fields[tagKey]; 47 | string updatedTagsFieldWithPostMove = GetUpdatedTagsFieldWithPostMove(migrationContext, existingTagsValue); 48 | 49 | KeyValuePair field = new KeyValuePair(tagKey, updatedTagsFieldWithPostMove); 50 | return MigrationHelpers.GetJsonPatchOperationReplaceForField(field); 51 | } 52 | else // Tags Field does not exist, so we add it here 53 | { 54 | KeyValuePair field = new KeyValuePair(Constants.TagsFieldReferenceName, migrationContext.Config.TargetPostMoveTag); 55 | return MigrationHelpers.GetJsonPatchOperationAddForField(field); 56 | } 57 | } 58 | 59 | public string GetUpdatedTagsFieldWithPostMove(IMigrationContext migrationContext, string tagFieldValue) 60 | { 61 | string postMoveTag = migrationContext.Config.TargetPostMoveTag; 62 | return $"{tagFieldValue}; {postMoveTag}"; 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Logging/ConsoleLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.Logging; 3 | 4 | namespace Logging 5 | { 6 | public class ConsoleLogger : ILogger 7 | { 8 | private string CategoryName; 9 | 10 | public ConsoleLogger(string categoryName) 11 | { 12 | CategoryName = categoryName; 13 | } 14 | 15 | public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 16 | { 17 | int logDestination = eventId.Id; 18 | 19 | if (logDestination == LogDestination.All || logDestination == LogDestination.Console) 20 | { 21 | string output = PerformLogging(logLevel, state, exception, formatter, logDestination); 22 | 23 | if (output != null) 24 | { 25 | Print(logLevel, output); 26 | } 27 | } 28 | } 29 | 30 | private string PerformLogging(LogLevel logLevel, TState state, Exception exception, Func formatter, int logDestination) 31 | { 32 | if (formatter == null) 33 | { 34 | throw new ArgumentNullException(nameof(formatter)); 35 | } 36 | 37 | if (!IsEnabled(logLevel)) 38 | { 39 | return null; 40 | } 41 | 42 | var message = formatter(state, exception); 43 | 44 | if (string.IsNullOrEmpty(message)) 45 | { 46 | return null; 47 | } 48 | 49 | LogItem logItem = new LogItem(logLevel, DateTime.Now, message, exception, logDestination); 50 | return logItem.OutputFormat(true, true); 51 | } 52 | 53 | public void Print(LogLevel logLevel, string content) 54 | { 55 | switch (logLevel) 56 | { 57 | case LogLevel.Trace: 58 | Console.ForegroundColor = ConsoleColor.Green; 59 | Console.WriteLine(content); 60 | Console.ResetColor(); 61 | break; 62 | case LogLevel.Warning: 63 | Console.ForegroundColor = ConsoleColor.Yellow; 64 | Console.WriteLine(content); 65 | Console.ResetColor(); 66 | break; 67 | case LogLevel.Error: 68 | Console.ForegroundColor = ConsoleColor.Red; 69 | Console.WriteLine(content); 70 | Console.ResetColor(); 71 | break; 72 | case LogLevel.Critical: 73 | Console.ForegroundColor = ConsoleColor.Red; 74 | Console.WriteLine(content); 75 | Console.ResetColor(); 76 | break; 77 | default: 78 | Console.WriteLine(content); 79 | break; 80 | } 81 | } 82 | 83 | public bool IsEnabled(LogLevel logLevel) 84 | { 85 | return true; 86 | } 87 | 88 | public IDisposable BeginScope(TState state) 89 | { 90 | return null; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Common/Config/ConfigReaderJson.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using Microsoft.Extensions.Logging; 4 | using Newtonsoft.Json; 5 | using Logging; 6 | 7 | namespace Common.Config 8 | { 9 | public class ConfigReaderJson : IConfigReader 10 | { 11 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 12 | 13 | private string FilePath; 14 | private string JsonText; 15 | 16 | public ConfigReaderJson(string filePath) 17 | { 18 | this.FilePath = filePath; 19 | } 20 | 21 | public ConfigJson Deserialize() 22 | { 23 | LoadFromFile(this.FilePath); 24 | return DeserializeText(this.JsonText); 25 | } 26 | 27 | public void LoadFromFile(string filePath) 28 | { 29 | try 30 | { 31 | this.JsonText = GetJsonFromFile(filePath); 32 | } 33 | catch (FileNotFoundException ex) 34 | { 35 | Logger.LogError("Required JSON configuration file was not found. Please ensure that this file is in the correct location."); 36 | throw ex; 37 | } 38 | catch (PathTooLongException ex) 39 | { 40 | Logger.LogError("Required JSON configuration file could not be accessed because the file path is too long. Please store your files for this WiMigrator application in a folder location with a shorter path name."); 41 | throw ex; 42 | } 43 | catch (UnauthorizedAccessException ex) 44 | { 45 | Logger.LogError("Cannot read from the JSON configuration file because you are not authorized to access it. Please try running this application as administrator or moving it to a folder location that does not require special access."); 46 | throw ex; 47 | } 48 | catch (Exception ex) 49 | { 50 | Logger.LogError("Cannot read from the JSON configuration file. Please ensure it is formatted properly."); 51 | throw ex; 52 | } 53 | } 54 | 55 | public string GetJsonFromFile(string filePath) 56 | { 57 | return File.ReadAllText(filePath); 58 | } 59 | 60 | public ConfigJson DeserializeText(string input) 61 | { 62 | ConfigJson result = null; 63 | try 64 | { 65 | result = JsonConvert.DeserializeObject(input); 66 | // If not provided, default to # of processors on the computer 67 | if (result.Parallelism < 1) 68 | { 69 | result.Parallelism = Environment.ProcessorCount; 70 | } 71 | 72 | // If not provided, default to # of processors on the computer 73 | if (result.LinkParallelism < 1) 74 | { 75 | result.LinkParallelism = Environment.ProcessorCount; 76 | } 77 | } 78 | catch (Exception ex) 79 | { 80 | Logger.LogError("Cannot deserialize the JSON text from configuration file. Please ensure it is formatted properly."); 81 | throw ex; 82 | } 83 | 84 | return result; 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/WitBatchRequestGenerators/UpdateWitBatchRequestGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using Microsoft.Extensions.Logging; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 6 | using Newtonsoft.Json; 7 | using Common.ApiWrappers; 8 | using Logging; 9 | 10 | namespace Common.Migration 11 | { 12 | public class UpdateWitBatchRequestGenerator : BaseWitBatchRequestGenerator 13 | { 14 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 15 | 16 | public UpdateWitBatchRequestGenerator() 17 | { 18 | } 19 | 20 | public UpdateWitBatchRequestGenerator(IMigrationContext migrationContext, IBatchMigrationContext batchContext) : base(migrationContext, batchContext) 21 | { 22 | } 23 | 24 | public async override Task Write() 25 | { 26 | var sourceIdToWitBatchRequests = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 27 | foreach (var targetIdToSourceWorkItem in this.batchContext.TargetIdToSourceWorkItemMapping) 28 | { 29 | int targetId = targetIdToSourceWorkItem.Key; 30 | WorkItem sourceWorkItem = targetIdToSourceWorkItem.Value; 31 | 32 | if (WorkItemHasFailureState(sourceWorkItem)) 33 | { 34 | continue; 35 | } 36 | 37 | WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromWorkItem(sourceWorkItem, targetId); 38 | if (witBatchRequest != null) 39 | { 40 | sourceIdToWitBatchRequests.Add((sourceWorkItem.Id.Value, witBatchRequest)); 41 | } 42 | 43 | DecrementIdWithinBatch(sourceWorkItem.Id); 44 | } 45 | 46 | var phase1ApiWrapper = new Phase1ApiWrapper(); 47 | await phase1ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, this.migrationContext, batchContext); 48 | } 49 | 50 | private WitBatchRequest GenerateWitBatchRequestFromWorkItem(WorkItem sourceWorkItem, int targetId) 51 | { 52 | Dictionary headers = new Dictionary(); 53 | headers.Add("Content-Type", "application/json-patch+json"); 54 | 55 | JsonPatchDocument jsonPatchDocument = CreateJsonPatchDocumentFromWorkItemFields(sourceWorkItem); 56 | 57 | string hyperlink = this.migrationContext.WorkItemIdsUris[sourceWorkItem.Id.Value]; 58 | object attributeId = migrationContext.TargetIdToSourceHyperlinkAttributeId[targetId]; 59 | 60 | JsonPatchOperation addHyperlinkWithCommentOperation = MigrationHelpers.GetHyperlinkAddOperation(hyperlink, sourceWorkItem.Rev.ToString(), attributeId); 61 | jsonPatchDocument.Add(addHyperlinkWithCommentOperation); 62 | 63 | string json = JsonConvert.SerializeObject(jsonPatchDocument); 64 | var witBatchRequest = new WitBatchRequest(); 65 | witBatchRequest.Method = "PATCH"; 66 | witBatchRequest.Headers = headers; 67 | witBatchRequest.Uri = $"/_apis/wit/workItems/{targetId}?{this.QueryString}"; 68 | witBatchRequest.Body = json; 69 | 70 | return witBatchRequest; 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | WiMigrator is a command line tool designed with the following goals in mind: 3 | * Migrate work items from one Azure DevOps/TFS project to another 4 | * Real world example of how to use the WIT REST APIs 5 | * Cross platform support 6 | 7 | ![Build Status](https://github.com/microsoft/vsts-work-item-migrator/workflows/Build/badge.svg) 8 | 9 | # Features 10 | * Migrate the latest revision of a work item or set of work items based on the provided query, including: 11 | * Work item links (for work items within the query results set) 12 | * Attachments 13 | * Git commit links (link to the source git commit) 14 | * Work item history (last 200 revisions as an attachment) 15 | * Tagging of the source items that have been migrated 16 | 17 | # Getting Started 18 | ## Requirements 19 | * Source Project on **Azure DevOps** or **TFS 2017 Update 2** or later 20 | * Target Project on **Azure DevOps** or **TFS 2018** or later 21 | * Personal access tokens or NTLM for authentication 22 | * Bypass rules or Project Collection Administrator permissions required on target project 23 | * Process metadata **should** be consistent between the processes 24 | * Limited field mapping support is provided to map fields from the source to target account 25 | * Area/Iteration paths can be defaulted to a specific value when they don't exist on the target 26 | 27 | ## Running 28 | WiMigrator supports the following command line options: 29 | * --validate validates that the metadata between the source and target projects is consistent 30 | * --migrate re-runs validation and then migrates the work items 31 | 32 | Migration runs in two parts: 33 | * Validation 34 | * Configuration settings 35 | * Process metadata is consistent between projects 36 | * Identifies any work items that were previously migrated 37 | * Migration 38 | * Phase 1: Work item fields 39 | * Phase 2: Attachments, links, git commit links, history, target move tag 40 | * Phase 3: Source move tag 41 | 42 | A [sample configuration](WiMigrator/sample-configuration.json) file is provided with [documentation](WiMigrator/migration-configuration.md) of all the settings. 43 | 44 | Execution example: 45 | ``` 46 | dotnet run --validate configuration.json 47 | ``` 48 | 49 | HOWTO [Video](https://www.youtube.com/watch?v=aHbiLYUfomc&feature=youtu.be) 50 | 51 | ## Limitations: 52 | * Artifact links (other than git) are not migrated 53 | * Board fields are not migrated 54 | * Test artifacts (e.g. test results) are not migrated 55 | 56 | # Contributing 57 | 58 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 59 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 60 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 61 | 62 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 63 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 64 | provided by the bot. You will only need to do this once across all repos using our CLA. 65 | 66 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 67 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 68 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 69 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/WitBatchRequestGenerators/CreateWitBatchRequestGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using Microsoft.Extensions.Logging; 4 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 5 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 6 | using Newtonsoft.Json; 7 | using Common.ApiWrappers; 8 | using Logging; 9 | 10 | namespace Common.Migration 11 | { 12 | public class CreateWitBatchRequestGenerator : BaseWitBatchRequestGenerator 13 | { 14 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 15 | 16 | public CreateWitBatchRequestGenerator(IMigrationContext migrationContext, IBatchMigrationContext batchContext) : base(migrationContext, batchContext) 17 | { 18 | } 19 | 20 | public async override Task Write() 21 | { 22 | var sourceIdToWitBatchRequests = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 23 | foreach (var sourceWorkItem in this.batchContext.SourceWorkItems) 24 | { 25 | if (WorkItemHasFailureState(sourceWorkItem)) 26 | { 27 | continue; 28 | } 29 | 30 | WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromWorkItem(sourceWorkItem); 31 | if (witBatchRequest != null) 32 | { 33 | sourceIdToWitBatchRequests.Add((sourceWorkItem.Id.Value, witBatchRequest)); 34 | } 35 | 36 | DecrementIdWithinBatch(sourceWorkItem.Id); 37 | } 38 | 39 | var phase1ApiWrapper = new Phase1ApiWrapper(); 40 | await phase1ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, this.migrationContext, batchContext, verifyOnFailure: true); 41 | } 42 | 43 | private WitBatchRequest GenerateWitBatchRequestFromWorkItem(WorkItem sourceWorkItem) 44 | { 45 | Dictionary headers = new Dictionary(); 46 | headers.Add("Content-Type", "application/json-patch+json"); 47 | 48 | JsonPatchDocument jsonPatchDocument = CreateJsonPatchDocumentFromWorkItemFields(sourceWorkItem); 49 | 50 | JsonPatchOperation insertIdAddOperation = GetInsertBatchIdAddOperation(); 51 | jsonPatchDocument.Add(insertIdAddOperation); 52 | 53 | // add hyperlink to source WorkItem 54 | string sourceWorkItemApiEndpoint = ClientHelpers.GetWorkItemApiEndpoint(this.migrationContext.Config.SourceConnection.Account, sourceWorkItem.Id.Value); 55 | JsonPatchOperation addHyperlinkAddOperation = MigrationHelpers.GetHyperlinkAddOperation(sourceWorkItemApiEndpoint, sourceWorkItem.Rev.ToString()); 56 | jsonPatchDocument.Add(addHyperlinkAddOperation); 57 | 58 | string json = JsonConvert.SerializeObject(jsonPatchDocument); 59 | 60 | string workItemType = jsonPatchDocument.Find(a => a.Path.Contains(FieldNames.WorkItemType)).Value as string; 61 | 62 | var witBatchRequest = new WitBatchRequest(); 63 | witBatchRequest.Method = "PATCH"; 64 | witBatchRequest.Headers = headers; 65 | witBatchRequest.Uri = $"/{this.migrationContext.Config.TargetConnection.Project}/_apis/wit/workItems/${workItemType}?{this.QueryString}"; 66 | witBatchRequest.Body = json; 67 | 68 | return witBatchRequest; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /Common/ConcurrentSet.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Concurrent; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | 7 | namespace Common 8 | { 9 | public class ConcurrentSet : IEnumerable, IReadOnlyCollection, ISet 10 | { 11 | private ConcurrentDictionary dictionary; 12 | 13 | public ConcurrentSet() : this(EqualityComparer.Default) 14 | { 15 | } 16 | 17 | public ConcurrentSet(IEqualityComparer comparer) 18 | { 19 | this.dictionary = new ConcurrentDictionary(comparer); 20 | } 21 | 22 | public ConcurrentSet(IEnumerable source, IEqualityComparer comparer) 23 | { 24 | this.dictionary = new ConcurrentDictionary(source?.Select((s) => new KeyValuePair(s, null)), comparer); 25 | } 26 | 27 | public int Count => this.dictionary.Count; 28 | 29 | public bool IsReadOnly => false; 30 | 31 | public bool Add(T item) 32 | { 33 | return this.dictionary.TryAdd(item, null); 34 | } 35 | 36 | public void Clear() 37 | { 38 | this.dictionary.Clear(); 39 | } 40 | 41 | public bool Contains(T item) 42 | { 43 | return this.dictionary.ContainsKey(item); 44 | } 45 | 46 | public void CopyTo(T[] array, int arrayIndex) 47 | { 48 | throw new NotImplementedException(); 49 | } 50 | 51 | public void ExceptWith(IEnumerable other) 52 | { 53 | throw new NotImplementedException(); 54 | } 55 | 56 | public IEnumerator GetEnumerator() 57 | { 58 | return this.dictionary.Keys.GetEnumerator(); 59 | } 60 | 61 | public void IntersectWith(IEnumerable other) 62 | { 63 | throw new NotImplementedException(); 64 | } 65 | 66 | public bool IsProperSubsetOf(IEnumerable other) 67 | { 68 | throw new NotImplementedException(); 69 | } 70 | 71 | public bool IsProperSupersetOf(IEnumerable other) 72 | { 73 | throw new NotImplementedException(); 74 | } 75 | 76 | public bool IsSubsetOf(IEnumerable other) 77 | { 78 | throw new NotImplementedException(); 79 | } 80 | 81 | public bool IsSupersetOf(IEnumerable other) 82 | { 83 | throw new NotImplementedException(); 84 | } 85 | 86 | public bool Overlaps(IEnumerable other) 87 | { 88 | throw new NotImplementedException(); 89 | } 90 | 91 | public bool Remove(T item) 92 | { 93 | return this.dictionary.TryRemove(item, out _); 94 | } 95 | 96 | public bool SetEquals(IEnumerable other) 97 | { 98 | throw new NotImplementedException(); 99 | } 100 | 101 | public void SymmetricExceptWith(IEnumerable other) 102 | { 103 | throw new NotImplementedException(); 104 | } 105 | 106 | public void UnionWith(IEnumerable other) 107 | { 108 | throw new NotImplementedException(); 109 | } 110 | 111 | void ICollection.Add(T item) 112 | { 113 | if (!this.dictionary.TryAdd(item, null)) 114 | { 115 | throw new Exception(); 116 | } 117 | } 118 | 119 | IEnumerator IEnumerable.GetEnumerator() 120 | { 121 | return this.dictionary.Keys.GetEnumerator(); 122 | } 123 | } 124 | } -------------------------------------------------------------------------------- /UnitTests/Common/ClientHelperTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Concurrent; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | using Common; 6 | using Common.Migration; 7 | 8 | 9 | namespace UnitTests.Common 10 | { 11 | [TestClass] 12 | public class ClientHelperTests 13 | { 14 | [TestMethod] 15 | public void GetSourceWorkItemApiEndpoint_AccountEndingInSlashReturnsCorrectValue() 16 | { 17 | string account = "accountEndingInSlash/"; 18 | int workItemId = 777; 19 | 20 | string expected = "accountEndingInSlash/_apis/wit/workItems/777"; 21 | 22 | string actual = ClientHelpers.GetWorkItemApiEndpoint(account, workItemId); 23 | 24 | Assert.AreEqual(expected, actual); 25 | } 26 | 27 | [TestMethod] 28 | public void GetSourceWorkItemApiEndpoint_AccountWithoutSlashReturnsCorrectValue() 29 | { 30 | string account = "accountWithoutSlash"; 31 | int workItemId = 777; 32 | 33 | string expected = "accountWithoutSlash/_apis/wit/workItems/777"; 34 | 35 | string actual = ClientHelpers.GetWorkItemApiEndpoint(account, workItemId); 36 | 37 | Assert.AreEqual(expected, actual); 38 | } 39 | 40 | [TestMethod] 41 | public void GetWorkItemIdFromApiEndpoint_ReturnsCorrectResultWhenEndpointContainsIdAtEnd() 42 | { 43 | string endpointUri = "https://dev.azure.com/account/_apis/wit/workItems/3543"; 44 | 45 | int expected = 3543; 46 | 47 | int actual = ClientHelpers.GetWorkItemIdFromApiEndpoint(endpointUri); 48 | 49 | Assert.AreEqual(expected, actual); 50 | } 51 | 52 | 53 | [TestMethod] 54 | public void GetWorkItemIdFromApiEndpoint_ReturnsCorrectResultWhenEndpointContainsIdFollowedBySlashAtEnd() 55 | { 56 | string endpointUri = "https://dev.azure.com/account/_apis/wit/workItems/3543/"; 57 | 58 | int expected = 3543; 59 | 60 | int actual = ClientHelpers.GetWorkItemIdFromApiEndpoint(endpointUri); 61 | 62 | Assert.AreEqual(expected, actual); 63 | } 64 | 65 | [TestMethod] 66 | public void GetWorkItemIdFromApiEndpoint_ReturnsCorrectResultWhenEndpointContainsQueryString() 67 | { 68 | string endpointUri = "https://dev.azure.com/account/_apis/wit/workItems/3543?bypassRules=True&suppressNotifications=True&api-version=4.0"; 69 | 70 | int expected = 3543; 71 | 72 | int actual = ClientHelpers.GetWorkItemIdFromApiEndpoint(endpointUri); 73 | 74 | Assert.AreEqual(expected, actual); 75 | } 76 | 77 | [TestMethod] 78 | public void GetNotMigratedWorkItemsFromWorkItemsMigrationState_ReturnsCorrectResult() 79 | { 80 | WorkItemMigrationState notMigratedState = new WorkItemMigrationState(); 81 | notMigratedState.SourceId = 1; 82 | notMigratedState.FailureReason |= FailureReason.UnsupportedWorkItemType; 83 | 84 | WorkItemMigrationState migratedState = new WorkItemMigrationState(); 85 | migratedState.SourceId = 2; 86 | 87 | ConcurrentBag workItemsMigrationState = new ConcurrentBag(); 88 | workItemsMigrationState.Add(notMigratedState); 89 | workItemsMigrationState.Add(migratedState); 90 | 91 | Dictionary result = ClientHelpers.GetNotMigratedWorkItemsFromWorkItemsMigrationState(workItemsMigrationState); 92 | 93 | Assert.AreEqual(1, result.Count); 94 | Assert.AreEqual(1, result.First().Key); 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /Common/ValidationHeartbeatLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using Microsoft.Extensions.Logging; 6 | using Common.Validation; 7 | using Logging; 8 | 9 | namespace Common 10 | { 11 | public class ValidationHeartbeatLogger : IDisposable 12 | { 13 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 14 | 15 | private Timer timer; 16 | private IEnumerable WorkItemMigrationStates; 17 | private IValidationContext ValidationContext; 18 | 19 | public ValidationHeartbeatLogger(IEnumerable workItemMigrationStates, IValidationContext validationContext, int heartbeatFrequencyInSeconds) 20 | { 21 | this.timer = new Timer(Beat, "Some state", TimeSpan.FromSeconds(0), TimeSpan.FromSeconds(heartbeatFrequencyInSeconds)); 22 | this.WorkItemMigrationStates = workItemMigrationStates; 23 | this.ValidationContext = validationContext; 24 | } 25 | 26 | public void Beat() 27 | { 28 | Beat("Some state"); 29 | } 30 | 31 | private void Beat(object state) 32 | { 33 | string line1 = "VALIDATION STATUS:"; 34 | string line2 = $"new work items found: {GetNewWorkItemsFound()}"; 35 | string line3 = $"existing work items found: {GetExistingWorkItemsFound()}"; 36 | string line4 = $"existing work items validated for phase 1: {GetExistingWorkItemsValidatedForPhase1()}"; 37 | string line5 = $"existing work items validated for phase 2: {GetExistingWorkItemsValidatedForPhase2()}"; 38 | 39 | string output = $"{line1}{Environment.NewLine}{line2}{Environment.NewLine}{line3}{Environment.NewLine}{line4}{Environment.NewLine}{line5}"; 40 | 41 | int? workItemsReturnedFromQuery = GetWorkItemsReturnedFromQuery(); 42 | string extraLine; 43 | 44 | if (workItemsReturnedFromQuery != null) 45 | { 46 | extraLine = $"total work items from query to validate: {workItemsReturnedFromQuery}"; 47 | } 48 | else 49 | { 50 | extraLine = $"Waiting for query to retrieve work items to be validated..."; 51 | } 52 | 53 | output = $"{output}{Environment.NewLine}{extraLine}"; 54 | Logger.LogInformation(output); 55 | } 56 | 57 | public void Dispose() 58 | { 59 | this.timer.Dispose(); 60 | } 61 | 62 | private int GetNewWorkItemsFound() 63 | { 64 | return this.WorkItemMigrationStates.Where(w => w.MigrationState == WorkItemMigrationState.State.Create).Count(); 65 | } 66 | 67 | private int GetExistingWorkItemsFound() 68 | { 69 | return this.WorkItemMigrationStates.Where(w => w.MigrationState == WorkItemMigrationState.State.Existing).Count(); 70 | } 71 | 72 | private int GetExistingWorkItemsValidatedForPhase1() 73 | { 74 | return this.WorkItemMigrationStates.Where(w => w.MigrationState == WorkItemMigrationState.State.Existing && w.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase1)).Count(); 75 | } 76 | 77 | private int GetExistingWorkItemsValidatedForPhase2() 78 | { 79 | return this.WorkItemMigrationStates.Where(w => w.MigrationState == WorkItemMigrationState.State.Existing && w.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase2)).Count(); 80 | } 81 | 82 | private int? GetWorkItemsReturnedFromQuery() 83 | { 84 | if (this.ValidationContext.WorkItemIdsUris != null) 85 | { 86 | return this.ValidationContext.WorkItemIdsUris.Count(); 87 | } 88 | else 89 | { 90 | return null; 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /Common/MigrationHeartbeatLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using Microsoft.Extensions.Logging; 6 | using Logging; 7 | 8 | namespace Common 9 | { 10 | public class MigrationHeartbeatLogger : IDisposable 11 | { 12 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | 14 | private Timer timer; 15 | private IEnumerable WorkItemMigrationStates; 16 | 17 | public MigrationHeartbeatLogger(IEnumerable workItemMigrationStates, int heartbeatFrequencyInSeconds) 18 | { 19 | this.timer = new Timer(Beat, "Some state", TimeSpan.FromSeconds(0), TimeSpan.FromSeconds(heartbeatFrequencyInSeconds)); 20 | this.WorkItemMigrationStates = workItemMigrationStates; 21 | } 22 | 23 | public void Beat() 24 | { 25 | Beat("Some state"); 26 | } 27 | 28 | private void Beat(object state) 29 | { 30 | string line1 = "MIGRATION STATUS:"; 31 | string line2 = $"work items that succeeded phase 1 migration: {GetSucceededPhase1WorkItemsCount()}"; 32 | string line3 = $"work items that failed phase 1 migration: {GetFailedPhase1WorkItemsCount()}"; 33 | string line4 = $"work items to be processed in phase 1: {GetPhase1Total()}"; 34 | string line5 = $"work items that succeeded phase 2 migration: {GetSucceededPhase2WorkItemsCount()}"; 35 | string line6 = $"work items that failed phase 2 migration: {GetFailedPhase2WorkItemsCount()}"; 36 | string line7 = $"work items to be processed in phase 2: {GetPhase2Total()}"; 37 | 38 | Logger.LogInformation($"{line1}{Environment.NewLine}{line2}{Environment.NewLine}{line3}{Environment.NewLine}{line4}{Environment.NewLine}{line5}{Environment.NewLine}{line6}{Environment.NewLine}{line7}"); 39 | } 40 | 41 | public void Dispose() 42 | { 43 | this.timer.Dispose(); 44 | } 45 | 46 | private int GetSucceededPhase1WorkItemsCount() 47 | { 48 | return this.WorkItemMigrationStates.Where(w => w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase1) && w.FailureReason == Migration.FailureReason.None).Count(); 49 | } 50 | 51 | private int GetFailedPhase1WorkItemsCount() 52 | { 53 | return this.WorkItemMigrationStates.Where(w => w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase1) && w.FailureReason != Migration.FailureReason.None).Count(); 54 | } 55 | 56 | private int GetSucceededPhase2WorkItemsCount() 57 | { 58 | return this.WorkItemMigrationStates.Where(w => w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase2) && w.FailureReason == Migration.FailureReason.None).Count(); 59 | } 60 | 61 | private int GetFailedPhase2WorkItemsCount() 62 | { 63 | return this.WorkItemMigrationStates.Where(w => w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase2) && w.FailureReason != Migration.FailureReason.None).Count(); 64 | } 65 | 66 | private int GetPhase1Total() 67 | { 68 | int workItemsToCreateCount = this.WorkItemMigrationStates.Where(a => a.MigrationState == WorkItemMigrationState.State.Create).Count(); 69 | int workItemsToUpdate = this.WorkItemMigrationStates.Where(w => w.MigrationState == WorkItemMigrationState.State.Existing && w.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase1)).Count(); 70 | return workItemsToCreateCount + workItemsToUpdate; 71 | } 72 | 73 | private int GetPhase2Total() 74 | { 75 | return this.WorkItemMigrationStates.Where(a => a.MigrationState == WorkItemMigrationState.State.Create || (a.MigrationState == WorkItemMigrationState.State.Existing && a.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase2))).Count(); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Logging/MigratorLogging.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Text; 4 | using Microsoft.Extensions.Logging; 5 | 6 | namespace Logging 7 | { 8 | public class MigratorLogging 9 | { 10 | public static ConcurrentQueue logItems = new ConcurrentQueue(); 11 | public static bool recordLoggingIntoLogItems = true; 12 | public static LogLevel configMinimumLogLevel; 13 | 14 | private static ILoggerFactory LoggerFactory 15 | { 16 | get 17 | { 18 | LoggerFactory loggerFactory = new LoggerFactory(); 19 | 20 | ILoggerProvider consoleLoggerProvider = new ConsoleLoggerProvider(); 21 | loggerFactory.AddProvider(consoleLoggerProvider); 22 | 23 | ILoggerProvider fileLoggerProvider = new FileLoggerProvider(); 24 | loggerFactory.AddProvider(fileLoggerProvider); 25 | 26 | ILoggerProvider logItemsRecorderProvider = new LogItemsRecorderProvider(); 27 | loggerFactory.AddProvider(logItemsRecorderProvider); 28 | 29 | return loggerFactory; 30 | } 31 | } 32 | 33 | public static ILogger CreateLogger() => LoggerFactory.CreateLogger(); 34 | 35 | public static void LogSummary() 36 | { 37 | LoggerFactory loggerFactory = new LoggerFactory(); 38 | 39 | ILoggerProvider consoleLoggerProvider = new ConsoleLoggerProvider(); 40 | loggerFactory.AddProvider(consoleLoggerProvider); 41 | 42 | FileLoggerProvider fileLoggerProvider = new FileLoggerProvider(); 43 | loggerFactory.AddProvider(fileLoggerProvider); 44 | 45 | ILogger summaryLogger = loggerFactory.CreateLogger(); 46 | 47 | int eventId = LogDestination.All; 48 | 49 | summaryLogger.LogInformation(eventId, null, "Summary of errors and warnings:"); 50 | 51 | foreach (var logItem in logItems) 52 | { 53 | LogSummaryItem(logItem, summaryLogger); 54 | } 55 | 56 | fileLoggerProvider.Flush(); 57 | } 58 | 59 | public static void LogSummaryItem(LogItem logItem, ILogger summaryLogger) 60 | { 61 | string output; 62 | 63 | switch (logItem.LogLevel) 64 | { 65 | case LogLevel.Warning: 66 | output = logItem.OutputFormat(true, false); 67 | summaryLogger.LogWarning(logItem.LogDestination, output); 68 | break; 69 | case LogLevel.Error: 70 | output = logItem.OutputFormat(true, false); 71 | summaryLogger.LogError(logItem.LogDestination, output); 72 | break; 73 | case LogLevel.Critical: 74 | output = logItem.OutputFormat(true, false); 75 | summaryLogger.LogCritical(logItem.LogDestination, output); 76 | break; 77 | } 78 | } 79 | 80 | public static string GetLogSummaryText() 81 | { 82 | StringBuilder stringBuilder = new StringBuilder(); 83 | stringBuilder.AppendLine("Summary of errors and warnings:"); 84 | 85 | foreach (LogItem logItem in logItems) 86 | { 87 | string summaryItem = GetSummaryItem(logItem); 88 | 89 | if (summaryItem != null) 90 | { 91 | stringBuilder.AppendLine(summaryItem); 92 | } 93 | } 94 | 95 | return stringBuilder.ToString(); 96 | } 97 | 98 | private static string GetSummaryItem(LogItem logItem) 99 | { 100 | if (logItem.LogLevel == LogLevel.Warning || logItem.LogLevel == LogLevel.Error || logItem.LogLevel == LogLevel.Critical) 101 | { 102 | return logItem.OutputFormat(true, true); 103 | } 104 | else 105 | { 106 | return null; 107 | } 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /UnitTests/Common/BatchApi/ApiWrapperHelpersTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 3 | using Microsoft.VisualStudio.TestTools.UnitTesting; 4 | using Common.ApiWrappers; 5 | 6 | namespace UnitTests.Common.BatchApi 7 | { 8 | [TestClass] 9 | public class ApiWrapperHelpersTests 10 | { 11 | [TestMethod] 12 | public void ResponsesLackExpectedData_WitBatchResponsesIsNullAndBatchIdToWorkItemIdMappingIsPopulatedReturnsTrue() 13 | { 14 | List witBatchResponses = null; 15 | 16 | List<(int SourceId, WitBatchRequest WitBatchRequest)> batchIdToWorkItemIdMapping = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 17 | batchIdToWorkItemIdMapping.Add((0, null)); 18 | 19 | bool result = ApiWrapperHelpers.ResponsesLackExpectedData(witBatchResponses, batchIdToWorkItemIdMapping); 20 | Assert.IsTrue(result); 21 | } 22 | 23 | [TestMethod] 24 | public void ResponsesLackExpectedData_WitBatchResponsesIsEmptyAndBatchIdToWorkItemIdMappingIsPopulatedReturnsTrue() 25 | { 26 | List witBatchResponses = new List(); 27 | 28 | List<(int SourceId, WitBatchRequest WitBatchRequest)> batchIdToWorkItemIdMapping = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 29 | batchIdToWorkItemIdMapping.Add((0, null)); 30 | 31 | bool result = ApiWrapperHelpers.ResponsesLackExpectedData(witBatchResponses, batchIdToWorkItemIdMapping); 32 | Assert.IsTrue(result); 33 | } 34 | 35 | [TestMethod] 36 | public void ResponsesLackExpectedData_WitBatchResponsesIsPopulatedAndBatchIdToWorkItemIdMappingIsPopulatedReturnsFalse() 37 | { 38 | List witBatchResponses = new List(); 39 | witBatchResponses.Add(new WitBatchResponse()); 40 | 41 | List<(int SourceId, WitBatchRequest WitBatchRequest)> batchIdToWorkItemIdMapping = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 42 | batchIdToWorkItemIdMapping.Add((0, null)); 43 | 44 | bool result = ApiWrapperHelpers.ResponsesLackExpectedData(witBatchResponses, batchIdToWorkItemIdMapping); 45 | Assert.IsFalse(result); 46 | } 47 | 48 | [TestMethod] 49 | public void ResponsesLackExpectedData_WitBatchResponsesIsNullAndBatchIdToWorkItemIdMappingIsEmptyReturnsFalse() 50 | { 51 | List witBatchResponses = null; 52 | 53 | List<(int SourceId, WitBatchRequest WitBatchRequest)> batchIdToWorkItemIdMapping = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 54 | 55 | bool result = ApiWrapperHelpers.ResponsesLackExpectedData(witBatchResponses, batchIdToWorkItemIdMapping); 56 | Assert.IsFalse(result); 57 | } 58 | 59 | [TestMethod] 60 | public void ResponsesLackExpectedData_WitBatchResponsesIsEmptyAndBatchIdToWorkItemIdMappingIsEmptyReturnsFalse() 61 | { 62 | List witBatchResponses = new List(); 63 | 64 | List<(int SourceId, WitBatchRequest WitBatchRequest)> batchIdToWorkItemIdMapping = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 65 | 66 | bool result = ApiWrapperHelpers.ResponsesLackExpectedData(witBatchResponses, batchIdToWorkItemIdMapping); 67 | Assert.IsFalse(result); 68 | } 69 | 70 | [TestMethod] 71 | public void ResponsesLackExpectedData_WitBatchResponsesIsPopulatedAndBatchIdToWorkItemIdMappingIsEmptyReturnsFalse() 72 | { 73 | List witBatchResponses = new List(); 74 | witBatchResponses.Add(new WitBatchResponse()); 75 | 76 | List<(int SourceId, WitBatchRequest WitBatchRequest)> batchIdToWorkItemIdMapping = new List<(int SourceId, WitBatchRequest WitBatchRequest)>(); 77 | 78 | bool result = ApiWrapperHelpers.ResponsesLackExpectedData(witBatchResponses, batchIdToWorkItemIdMapping); 79 | Assert.IsFalse(result); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /Common/Migration/Phase2/Processors/RevisionHistoryAttachmentsProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Common.Config; 5 | using Logging; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 8 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 9 | using Newtonsoft.Json; 10 | 11 | namespace Common.Migration 12 | { 13 | public class RevisionHistoryAttachmentsProcessor : IPhase2Processor 14 | { 15 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 16 | 17 | public string Name => Constants.RelationPhaseRevisionHistoryAttachments; 18 | 19 | public bool IsEnabled(ConfigJson config) 20 | { 21 | return config.MoveHistory; 22 | } 23 | 24 | public async Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems) 25 | { 26 | 27 | } 28 | 29 | public async Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem) 30 | { 31 | var jsonPatchOperations = new List(); 32 | var attachments = await UploadAttachmentsToTarget(migrationContext, sourceWorkItem); 33 | foreach (var attachment in attachments) 34 | { 35 | JsonPatchOperation revisionHistoryAttachmentAddOperation = MigrationHelpers.GetRevisionHistoryAttachmentAddOperation(attachment, sourceWorkItem.Id.Value); 36 | jsonPatchOperations.Add(revisionHistoryAttachmentAddOperation); 37 | } 38 | 39 | return jsonPatchOperations; 40 | } 41 | 42 | private async Task> UploadAttachmentsToTarget(IMigrationContext migrationContext, WorkItem sourceWorkItem) 43 | { 44 | var attachmentLinks = new List(); 45 | int updateLimit = migrationContext.Config.MoveHistoryLimit; 46 | int updateCount = 0; 47 | 48 | while (updateCount < updateLimit) 49 | { 50 | var updates = await GetWorkItemUpdates(migrationContext, sourceWorkItem, skip: updateCount); 51 | string attachmentContent = JsonConvert.SerializeObject(updates); 52 | AttachmentReference attachmentReference; 53 | using (MemoryStream stream = new MemoryStream()) 54 | { 55 | var stringBytes = System.Text.Encoding.UTF8.GetBytes(attachmentContent); 56 | await stream.WriteAsync(stringBytes, 0, stringBytes.Length); 57 | stream.Position = 0; 58 | //upload the attachment to the target for each batch of workitem updates 59 | attachmentReference = await WorkItemTrackingHelpers.CreateAttachmentAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, stream); 60 | attachmentLinks.Add( 61 | new AttachmentLink( 62 | $"{Constants.WorkItemHistory}-{sourceWorkItem.Id}-{updateCount}.json", 63 | attachmentReference, 64 | stringBytes.Length, 65 | comment: $"Update range from {updateCount} to {updateCount + updates.Count}")); 66 | } 67 | 68 | updateCount += updates.Count; 69 | 70 | // if we got less than a page size, that means we're on the last 71 | // page and shouldn't try and read another page. 72 | if (updates.Count < Constants.PageSize) 73 | { 74 | break; 75 | } 76 | } 77 | 78 | return attachmentLinks; 79 | } 80 | 81 | private async Task> GetWorkItemUpdates(IMigrationContext migrationContext, WorkItem sourceWorkItem, int skip = 0) 82 | { 83 | return await WorkItemTrackingHelpers.GetWorkItemUpdatesAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, sourceWorkItem.Id.Value, skip); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Common/Validation/WorkItem/ValidateClassificationNodes.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Logging; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 6 | 7 | namespace Common.Validation 8 | { 9 | public class ValidateClassificationNodes : IWorkItemValidator 10 | { 11 | private static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 12 | 13 | public string Name => "Classification nodes"; 14 | 15 | public async Task Prepare(IValidationContext context) 16 | { 17 | context.RequestedFields.Add(FieldNames.AreaPath); 18 | context.RequestedFields.Add(FieldNames.IterationPath); 19 | 20 | Logger.LogInformation("Reading the area and iteration paths from the source and target accounts"); 21 | 22 | try 23 | { 24 | var classificationNodes = await WorkItemTrackingHelpers.GetClassificationNodes(context.TargetClient.WorkItemTrackingHttpClient, context.Config.TargetConnection.Project); 25 | var nodes = new AreaAndIterationPathTree(classificationNodes); 26 | context.TargetAreaPaths = nodes.AreaPathList; 27 | context.TargetIterationPaths = nodes.IterationPathList; 28 | } 29 | catch (Exception e) 30 | { 31 | throw new ValidationException("Unable to read the classification nodes on the source", e); 32 | } 33 | } 34 | 35 | public async Task Validate(IValidationContext context, WorkItem workItem) 36 | { 37 | var areaPath = (string)workItem.Fields[FieldNames.AreaPath]; 38 | var iterationPath = (string)workItem.Fields[FieldNames.IterationPath]; 39 | 40 | if (!AreaAndIterationPathTree.TryReplaceLeadingProjectName(areaPath, context.Config.SourceConnection.Project, context.Config.TargetConnection.Project, out areaPath)) 41 | { 42 | // This is a fatal error because this implies the query is cross project which we do not support, so bail out immediately 43 | throw new ValidationException($"Could not find source project from area path {workItem.Fields[FieldNames.AreaPath]} for work item with id {workItem.Id}"); 44 | } 45 | 46 | if (!AreaAndIterationPathTree.TryReplaceLeadingProjectName(iterationPath, context.Config.SourceConnection.Project, context.Config.TargetConnection.Project, out iterationPath)) 47 | { 48 | // This is a fatal error because this implies the query is cross project which we do not support, so bail out immediately 49 | throw new ValidationException($"Could not find source project from iteration path {workItem.Fields[FieldNames.IterationPath]} for work item with id {workItem.Id}"); 50 | } 51 | 52 | if (!context.ValidatedAreaPaths.Contains(areaPath) && !context.SkippedAreaPaths.Contains(areaPath)) 53 | { 54 | if (!context.TargetAreaPaths.Contains(areaPath)) 55 | { 56 | // only log if we've added this for the first time 57 | if (context.SkippedAreaPaths.Add(areaPath)) 58 | { 59 | Logger.LogWarning(LogDestination.File, $"Area path {areaPath} does not exist on the target for work item with id {workItem.Id}"); 60 | } 61 | } 62 | else 63 | { 64 | context.ValidatedAreaPaths.Add(areaPath); 65 | } 66 | } 67 | 68 | if (!context.ValidatedIterationPaths.Contains(iterationPath) && !context.SkippedIterationPaths.Contains(iterationPath)) 69 | { 70 | if (!context.TargetIterationPaths.Contains(iterationPath)) 71 | { 72 | // only log if we've added this for the first time 73 | if (context.SkippedIterationPaths.Add(iterationPath)) 74 | { 75 | Logger.LogWarning(LogDestination.File, $"Iteration path {iterationPath} does not exist on the target for work item with id {workItem.Id}"); 76 | } 77 | } 78 | else 79 | { 80 | context.ValidatedIterationPaths.Add(iterationPath); 81 | } 82 | } 83 | 84 | // If we're skipping instead of using default values, add the work item to the skipped list 85 | // if it's area or iteration path does not exist on the target. 86 | if ((context.Config.SkipWorkItemsWithMissingIterationPath && context.SkippedIterationPaths.Contains(iterationPath)) || 87 | (context.Config.SkipWorkItemsWithMissingAreaPath && context.SkippedAreaPaths.Contains(areaPath))) 88 | { 89 | context.SkippedWorkItems.Add(workItem.Id.Value); 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Common/Migration/Phase2/Processors/RemoteLinksProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 6 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 7 | using Logging; 8 | using Microsoft.VisualStudio.Services.Common; 9 | using Common.Config; 10 | using System; 11 | 12 | namespace Common.Migration 13 | { 14 | public class RemoteLinksProcessor : IPhase2Processor 15 | { 16 | private static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 17 | 18 | public string Name => Constants.RelationPhaseRemoteLinks; 19 | 20 | public bool IsEnabled(ConfigJson config) 21 | { 22 | return config.MoveLinks; 23 | } 24 | 25 | public async Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems) 26 | { 27 | 28 | } 29 | 30 | public async Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem) 31 | { 32 | IList jsonPatchOperations = new List(); 33 | IEnumerable sourceRemoteLinks = sourceWorkItem.Relations?.Where(r => RelationHelpers.IsRemoteLinkType(migrationContext, r.Rel)); 34 | 35 | if (sourceRemoteLinks != null && sourceRemoteLinks.Any()) 36 | { 37 | foreach (WorkItemRelation sourceRemoteLink in sourceRemoteLinks) 38 | { 39 | string url = ConvertRemoteLinkToHyperlink(sourceRemoteLink.Url); 40 | WorkItemRelation targetRemoteLinkHyperlinkRelation = GetHyperlinkIfExistsOnTarget(targetWorkItem, url); 41 | 42 | if (targetRemoteLinkHyperlinkRelation != null) // is on target 43 | { 44 | JsonPatchOperation remoteLinkHyperlinkAddOperation = MigrationHelpers.GetRelationAddOperation(targetRemoteLinkHyperlinkRelation); 45 | jsonPatchOperations.Add(remoteLinkHyperlinkAddOperation); 46 | } 47 | else // is not on target 48 | { 49 | string comment = string.Empty; 50 | if (sourceRemoteLink.Attributes.ContainsKey(Constants.RelationAttributeComment)) 51 | { 52 | comment = $"{sourceRemoteLink.Attributes[Constants.RelationAttributeComment]}"; 53 | } 54 | 55 | WorkItemRelation newRemoteLinkHyperlinkRelation = new WorkItemRelation(); 56 | newRemoteLinkHyperlinkRelation.Rel = Constants.Hyperlink; 57 | newRemoteLinkHyperlinkRelation.Url = url; 58 | newRemoteLinkHyperlinkRelation.Attributes = new Dictionary(); 59 | newRemoteLinkHyperlinkRelation.Attributes[Constants.RelationAttributeComment] = comment; 60 | 61 | JsonPatchOperation remoteLinkHyperlinkAddOperation = MigrationHelpers.GetRelationAddOperation(newRemoteLinkHyperlinkRelation); 62 | jsonPatchOperations.Add(remoteLinkHyperlinkAddOperation); 63 | } 64 | } 65 | } 66 | 67 | return jsonPatchOperations; 68 | } 69 | 70 | private WorkItemRelation GetHyperlinkIfExistsOnTarget(WorkItem targetWorkItem, string href) 71 | { 72 | if (targetWorkItem.Relations == null) 73 | { 74 | return null; 75 | } 76 | 77 | foreach (WorkItemRelation targetRelation in targetWorkItem.Relations) 78 | { 79 | if (targetRelation.Rel.Equals(Constants.Hyperlink) && targetRelation.Url.Equals(href, StringComparison.OrdinalIgnoreCase)) 80 | { 81 | return targetRelation; 82 | } 83 | } 84 | 85 | return null; 86 | } 87 | 88 | /// 89 | /// Remote links returned refer to the REST reference, and we want the web reference. 90 | /// Format: https://account.visualstudio.com/ad443396-8473-4678-a2ba-0b1cf7cc8837/_apis/wit/workItems/3915636 91 | /// Web: https://account.visualstudio.com/ad443396-8473-4678-a2ba-0b1cf7cc8837/_workitems/edit/3915636 92 | /// 93 | /// 94 | /// 95 | private string ConvertRemoteLinkToHyperlink(string url) 96 | { 97 | return url.Replace("_apis/wit/workitems", "_workitems/edit", StringComparison.OrdinalIgnoreCase); 98 | } 99 | } 100 | } -------------------------------------------------------------------------------- /Common/AreaAndIterationPathTree.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text.RegularExpressions; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 6 | using Logging; 7 | 8 | namespace Common 9 | { 10 | public class AreaAndIterationPathTree 11 | { 12 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | public ISet AreaPathList { get; } = new HashSet(); 14 | public ISet IterationPathList { get; } = new HashSet(); 15 | 16 | public AreaAndIterationPathTree(IList nodeList) 17 | { 18 | if (nodeList != null) 19 | { 20 | foreach (var node in nodeList) 21 | { 22 | if (node.StructureType == TreeNodeStructureType.Area) 23 | { 24 | CreateAreaPathList(node); 25 | } 26 | else if (node.StructureType == TreeNodeStructureType.Iteration) 27 | { 28 | CreateIterationPathList(node); 29 | } 30 | } 31 | } 32 | else 33 | { 34 | //this should never happen 35 | Logger.LogError(LogDestination.All, "Critial error in retrieving Area and Iteration path from target"); 36 | throw new ArgumentNullException("Critial error in retrieving Area and Iteration path from target"); 37 | } 38 | } 39 | 40 | //TODO: find a better place to put this, Get Unit Tests for this again 41 | public static string ReplaceLeadingProjectName(string input, string sourceProject, string targetProject) 42 | { 43 | string replacedInput; 44 | if (!TryReplaceLeadingProjectName(input, sourceProject, targetProject, out replacedInput)) 45 | { 46 | // unexpected, the source/target project should have already been validated 47 | throw new ArgumentException($"Could not find the source project name to replace in the following field value: {input}. Please make sure all team project values match the project name of your source and all area and iteration path values start with the project name of your source."); 48 | } 49 | 50 | return replacedInput; 51 | } 52 | 53 | public static bool TryReplaceLeadingProjectName(string input, string sourceProject, string targetProject, out string replacedInput) 54 | { 55 | replacedInput = null; 56 | // Handles case when System.AreaPath or System.IteraionPath consist of only sourceProject 57 | if (input.Equals(sourceProject, StringComparison.OrdinalIgnoreCase)) 58 | { 59 | replacedInput = targetProject; 60 | } 61 | else if (Regex.IsMatch(input, $@"^{sourceProject}\\", RegexOptions.IgnoreCase)) 62 | { 63 | replacedInput = Regex.Replace(input, $@"^{sourceProject}\\", $"{targetProject}\\", RegexOptions.IgnoreCase); 64 | } 65 | 66 | return replacedInput != null; 67 | } 68 | 69 | private void CreateAreaPathList(WorkItemClassificationNode headnode) 70 | { 71 | //node is the headnode 72 | if (headnode == null) 73 | { 74 | return; 75 | } 76 | //path for the headnode is null 77 | ProcessNode(null, headnode, this.AreaPathList); 78 | } 79 | 80 | private void CreateIterationPathList(WorkItemClassificationNode headnode) 81 | { 82 | //node is the headnode 83 | if (headnode == null) 84 | { 85 | return; 86 | } 87 | //path for the headnode is null 88 | ProcessNode(null, headnode, this.IterationPathList); 89 | } 90 | 91 | private void ProcessNode(string path, WorkItemClassificationNode node, ISet pathList) 92 | { 93 | if (node == null) 94 | { 95 | return; 96 | } 97 | string currentpath; 98 | if (path != null) 99 | { 100 | currentpath = $"{path}\\{node.Name}"; 101 | } 102 | else 103 | { 104 | //very first node will have null path, so it will be just a node name 105 | currentpath = node.Name; 106 | } 107 | 108 | pathList.Add(currentpath); 109 | 110 | if (node.Children != null) 111 | { 112 | foreach (var child in node.Children) 113 | { 114 | ProcessNode(currentpath, child, pathList); 115 | } 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /Common/Validation/Target/ResolveTargetWorkItemIds.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using Logging; 7 | using Microsoft.Extensions.Logging; 8 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi; 9 | using Microsoft.VisualStudio.Services.Common; 10 | 11 | namespace Common.Validation 12 | { 13 | [RunOrder(6)] 14 | public class ResolveTargetWorkItemIds : ITargetValidator 15 | { 16 | private ILogger Logger { get; } = MigratorLogging.CreateLogger(); 17 | 18 | public string Name => "Resolve target work item ids"; 19 | 20 | public async Task Validate(IValidationContext context) 21 | { 22 | await ClassifyWorkItemIds(context); 23 | } 24 | 25 | private async Task ClassifyWorkItemIds(IValidationContext context) 26 | { 27 | // Remove all skipped work items to reduce the load of what we need to query 28 | var workItemIdsUrisToClassify = context.WorkItemIdsUris.Where(wi => !context.SkippedWorkItems.Contains(wi.Key)).ToList(); 29 | var totalNumberOfBatches = ClientHelpers.GetBatchCount(workItemIdsUrisToClassify.Count(), Constants.BatchSize); 30 | 31 | if (workItemIdsUrisToClassify.Any()) 32 | { 33 | var stopwatch = Stopwatch.StartNew(); 34 | Logger.LogInformation(LogDestination.File, "Started querying target account to find previously migrated work items"); 35 | 36 | await workItemIdsUrisToClassify.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async (workItemIdsUris, batchId) => 37 | { 38 | var batchStopwatch = Stopwatch.StartNew(); 39 | Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Started"); 40 | //check if the workitems have already been migrated and add the classified work items to the context 41 | var migrationStates = await FilterWorkItemIds(context, context.TargetClient.WorkItemTrackingHttpClient, workItemIdsUris.ToDictionary(k => k.Key, v => v.Value)); 42 | 43 | if (migrationStates.Any()) 44 | { 45 | foreach (var migrationState in migrationStates) 46 | { 47 | context.WorkItemsMigrationState.Add(migrationState); 48 | } 49 | } 50 | 51 | batchStopwatch.Stop(); 52 | Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Completed in {batchStopwatch.Elapsed.TotalSeconds}s"); 53 | }); 54 | 55 | stopwatch.Stop(); 56 | Logger.LogInformation(LogDestination.File, $"Completed querying target account to find previously migrated work items in {stopwatch.Elapsed.TotalSeconds}s"); 57 | } 58 | } 59 | 60 | private async Task> FilterWorkItemIds(IValidationContext context, WorkItemTrackingHttpClient client, IDictionary workItems) 61 | { 62 | // call GetWorkItemIdsForArtifactUrisAsync for target client to get the mapping of artifacturis and ids 63 | // do a check to see if any of them have already been migrated 64 | var artifactUris = workItems.Select(a => a.Value).ToList(); 65 | var result = await ClientHelpers.QueryArtifactUriToGetIdsFromUris(client, artifactUris); 66 | 67 | IList workItemStateList = new List(); 68 | 69 | //check if any of the workitems have been migrated before 70 | foreach (var workItem in workItems) 71 | { 72 | try 73 | { 74 | if (ClientHelpers.GetMigratedWorkItemId(result, workItem, out int id)) 75 | { 76 | workItemStateList.Add(new WorkItemMigrationState { SourceId = workItem.Key, TargetId = id, MigrationState = WorkItemMigrationState.State.Existing }); 77 | } 78 | else 79 | { 80 | workItemStateList.Add(new WorkItemMigrationState { SourceId = workItem.Key, MigrationState = WorkItemMigrationState.State.Create }); 81 | } 82 | } 83 | catch (Exception e) 84 | { 85 | //edge case where we find more than one workitems in the target for the workitem 86 | Logger.LogError(LogDestination.File, e, e.Message); 87 | //Add this workitem to notmigratedworkitem list 88 | workItemStateList.Add(new WorkItemMigrationState { SourceId = workItem.Key, MigrationState = WorkItemMigrationState.State.Error }); 89 | } 90 | } 91 | 92 | return workItemStateList; 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/Processors/BaseWorkItemsProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Diagnostics; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Common.Config; 6 | using Logging; 7 | using Microsoft.Extensions.Logging; 8 | using Microsoft.VisualStudio.Services.Common; 9 | 10 | namespace Common.Migration 11 | { 12 | public abstract class BaseWorkItemsProcessor : IPhase1Processor 13 | { 14 | protected abstract ILogger Logger { get; } 15 | 16 | public abstract string Name { get; } 17 | 18 | public abstract bool IsEnabled(ConfigJson config); 19 | 20 | public abstract IList GetWorkItemsAndStateToMigrate(IMigrationContext context); 21 | 22 | public abstract void PrepareBatchContext(IBatchMigrationContext batchContext, IList workItemsAndStateToMigrate); 23 | 24 | public abstract int GetWorkItemsToProcessCount(IBatchMigrationContext batchContext); 25 | 26 | public abstract BaseWitBatchRequestGenerator GetWitBatchRequestGenerator(IMigrationContext context, IBatchMigrationContext batchContext); 27 | 28 | public async Task Process(IMigrationContext context) 29 | { 30 | var workItemsAndStateToMigrate = this.GetWorkItemsAndStateToMigrate(context); 31 | var totalNumberOfBatches = ClientHelpers.GetBatchCount(workItemsAndStateToMigrate.Count, Constants.BatchSize); 32 | 33 | if (!workItemsAndStateToMigrate.Any()) 34 | { 35 | Logger.LogInformation(LogDestination.File, $"No work items to process for {this.Name}"); 36 | return; 37 | } 38 | 39 | Logger.LogInformation(LogDestination.All, $"{this.Name} will process {workItemsAndStateToMigrate.Count} work items on the target"); 40 | var preprocessors = ClientHelpers.GetProcessorInstances(context.Config); 41 | foreach (var preprocessor in preprocessors) 42 | { 43 | await preprocessor.Prepare(context); 44 | } 45 | 46 | await workItemsAndStateToMigrate.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async (batchWorkItemsAndState, batchId) => 47 | { 48 | var batchStopwatch = Stopwatch.StartNew(); 49 | Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Starting"); 50 | 51 | IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, batchWorkItemsAndState); 52 | //read the work items 53 | var stepStopwatch = Stopwatch.StartNew(); 54 | 55 | Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Reading source work items"); 56 | await Migrator.ReadSourceWorkItems(context, batchWorkItemsAndState.Select(w => w.SourceId), batchContext); 57 | Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed reading source work items in {stepStopwatch.Elapsed.Seconds}s"); 58 | 59 | this.PrepareBatchContext(batchContext, batchWorkItemsAndState); 60 | 61 | foreach (var preprocessor in preprocessors) 62 | { 63 | stepStopwatch.Restart(); 64 | Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Starting {preprocessor.Name}"); 65 | await preprocessor.Process(batchContext); 66 | Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed {preprocessor.Name} in {stepStopwatch.Elapsed.Seconds}s"); 67 | } 68 | 69 | var workItemsToUpdateCount = this.GetWorkItemsToProcessCount(batchContext); 70 | 71 | Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Number of work items to migrate: {workItemsToUpdateCount}"); 72 | 73 | //migrate the batch of work items 74 | if (workItemsToUpdateCount == 0) 75 | { 76 | batchStopwatch.Stop(); 77 | Logger.LogWarning(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: No work items to migrate"); 78 | } 79 | else 80 | { 81 | stepStopwatch.Restart(); 82 | Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Saving the target work items"); 83 | var witBatchRequestGenerator = this.GetWitBatchRequestGenerator(context, batchContext); 84 | await witBatchRequestGenerator.Write(); 85 | Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed saving the target work items in {stepStopwatch.Elapsed.Seconds}s"); 86 | 87 | batchStopwatch.Stop(); 88 | Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed in {batchStopwatch.Elapsed.TotalSeconds}s"); 89 | } 90 | }); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Common/Validation/ValidationContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Collections.ObjectModel; 5 | using System.IO; 6 | using Common.Config; 7 | using Logging; 8 | using Microsoft.Extensions.Logging; 9 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 10 | using Newtonsoft.Json; 11 | 12 | namespace Common.Validation 13 | { 14 | public class ValidationContext : BaseContext, IValidationContext 15 | { 16 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 17 | 18 | //Mapping of targetId of a work item to attribute id of the hyperlink 19 | public ConcurrentDictionary TargetIdToSourceHyperlinkAttributeId { get; set; } = new ConcurrentDictionary(); 20 | 21 | public ISet RequestedFields { get; } = new HashSet(StringComparer.OrdinalIgnoreCase); 22 | 23 | public ConcurrentDictionary SourceWorkItemRevision { get; set; } = new ConcurrentDictionary(); 24 | 25 | public ConcurrentDictionary SourceFields { get; set; } = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); 26 | 27 | public ConcurrentDictionary TargetFields { get; set; } = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); 28 | 29 | public ConcurrentDictionary> SourceTypesAndFields { get; } = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); 30 | 31 | public ConcurrentDictionary> TargetTypesAndFields { get; } = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); 32 | 33 | public ConcurrentSet ValidatedTypes { get; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 34 | 35 | public ConcurrentSet ValidatedFields { get; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 36 | 37 | public ISet IdentityFields { get; set; } 38 | 39 | public ConcurrentSet SkippedTypes { get; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 40 | 41 | public ConcurrentSet SkippedFields { get; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 42 | 43 | public ISet TargetAreaPaths { get; set; } = new HashSet(StringComparer.OrdinalIgnoreCase); 44 | 45 | public ISet TargetIterationPaths { get; set; } = new HashSet(StringComparer.OrdinalIgnoreCase); 46 | 47 | public ConcurrentSet ValidatedAreaPaths { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 48 | 49 | public ConcurrentSet SkippedAreaPaths { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 50 | 51 | public ConcurrentSet ValidatedIterationPaths { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 52 | 53 | public ConcurrentSet SkippedIterationPaths { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 54 | 55 | public ConcurrentSet ValidatedWorkItemLinkRelationTypes { get; set; } = new ConcurrentSet(StringComparer.OrdinalIgnoreCase); 56 | 57 | public ConcurrentSet SkippedWorkItems { get; } = new ConcurrentSet(); 58 | 59 | public IList FieldsThatRequireSourceProjectToBeReplacedWithTargetProject => fieldsThatRequireSourceProjectToBeReplacedWithTargetProject; 60 | 61 | // This includes area path and iteration path because these fields must have their project name updated to the target project name 62 | private readonly IList fieldsThatRequireSourceProjectToBeReplacedWithTargetProject = new ReadOnlyCollection(new[] { 63 | FieldNames.AreaPath, 64 | FieldNames.IterationPath, 65 | FieldNames.TeamProject 66 | }); 67 | 68 | public ValidationContext(ConfigJson configJson) : base(configJson) 69 | { 70 | MigratorLogging.configMinimumLogLevel = this.Config.LogLevelForFile; 71 | 72 | LogConfigData(); 73 | } 74 | 75 | private void LogConfigData() 76 | { 77 | Logger.LogInformation("Config data:"); 78 | MemoryStream stream = new MemoryStream(); 79 | 80 | using (StreamWriter sw = new StreamWriter(stream)) 81 | { 82 | using (JsonWriter writer = new JsonTextWriter(sw)) 83 | { 84 | writer.Formatting = Formatting.Indented; 85 | JsonSerializer serializer = new JsonSerializer(); 86 | serializer.NullValueHandling = NullValueHandling.Ignore; 87 | serializer.Serialize(writer, this.Config); 88 | 89 | writer.Flush(); 90 | stream.Position = 0; 91 | 92 | StreamReader sr = new StreamReader(stream); 93 | string output = sr.ReadToEnd(); 94 | Logger.LogInformation(output); 95 | } 96 | } 97 | } 98 | 99 | public ValidationContext() : base() 100 | { 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /Logging/BulkLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Diagnostics; 4 | using System.IO; 5 | using System.Text; 6 | using System.Threading; 7 | 8 | namespace Logging 9 | { 10 | public class BulkLogger 11 | { 12 | // we can only let one thread in at a time to log to the file 13 | private static object lockObject = new object(); 14 | private ConcurrentQueue bulkLoggerLogItems; 15 | private Stopwatch stopwatch; 16 | private Timer bulkLoggerCheckTimer; 17 | private string filePath; 18 | 19 | public BulkLogger() 20 | { 21 | this.bulkLoggerLogItems = new ConcurrentQueue(); 22 | this.bulkLoggerCheckTimer = new Timer(BulkLoggerCheck, "Some state", TimeSpan.FromSeconds(LoggingConstants.CheckInterval), TimeSpan.FromSeconds(LoggingConstants.CheckInterval)); 23 | this.stopwatch = Stopwatch.StartNew(); 24 | this.filePath = GetFilePathBasedOnTime(); 25 | Console.WriteLine($"Detailed logging sent to file: {Directory.GetCurrentDirectory()}\\{filePath}"); 26 | } 27 | 28 | public void WriteToQueue(LogItem logItem) 29 | { 30 | if (logItem != null) 31 | { 32 | bulkLoggerLogItems.Enqueue(logItem); 33 | } 34 | } 35 | 36 | private string GetFilePathBasedOnTime() 37 | { 38 | try 39 | { 40 | string currentDateTime = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); 41 | return $"WiMigrator_Migrate_{currentDateTime}.log"; 42 | } 43 | catch (Exception ex) 44 | { 45 | string defaultFileName = "WiMigrator_Migrate_LogFile"; 46 | Console.WriteLine($"Could not give log file a special name due to below Exception. Naming the file: \"{defaultFileName}\" instead.\n{ex}"); 47 | return defaultFileName; 48 | } 49 | } 50 | 51 | private void BulkLoggerCheck(object state) 52 | { 53 | if (LogIntervalHasElapsed() || LogItemsHasReachedLimit()) 54 | { 55 | BulkLoggerLog(); 56 | } 57 | } 58 | 59 | private bool LogIntervalHasElapsed() 60 | { 61 | long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; 62 | return elapsedMilliseconds >= LoggingConstants.LogInterval * 1000; 63 | } 64 | 65 | private bool LogItemsHasReachedLimit() 66 | { 67 | return bulkLoggerLogItems.Count >= LoggingConstants.LogItemsUnloggedLimit; 68 | } 69 | 70 | public void BulkLoggerLog() 71 | { 72 | stopwatch.Restart(); 73 | StringBuilder outputBatchSB = new StringBuilder(); 74 | 75 | if (bulkLoggerLogItems.Count > 0) 76 | { 77 | // We will only dequeue and write the count of items determined at the beginning of iteration. 78 | // Then we will have a predictable end in the case that items are being enqueued during the iteration. 79 | int startingLength = bulkLoggerLogItems.Count; 80 | 81 | for (int i = 0; i < startingLength; i++) 82 | { 83 | if (bulkLoggerLogItems.TryDequeue(out LogItem logItem)) 84 | { 85 | string output = logItem.OutputFormat(false, true); 86 | 87 | if (logItem.Exception != null) 88 | { 89 | output = $"{output}\n{logItem.Exception}"; 90 | } 91 | outputBatchSB.Append(output); 92 | outputBatchSB.AppendLine(); 93 | } 94 | else 95 | { 96 | continue; 97 | } 98 | } 99 | 100 | WriteToFile(outputBatchSB.ToString()); 101 | } 102 | } 103 | 104 | private void WriteToFile(string content) 105 | { 106 | try 107 | { 108 | LoggingRetryHelper.Retry(() => 109 | { 110 | AppendToFile(content); 111 | }, 5); 112 | } 113 | catch (UnauthorizedAccessException ex) 114 | { 115 | Console.WriteLine($"Cannot write to the log file because you are not authorized to access it. Please try running this application as administrator or moving it to a folder location that does not require special access."); 116 | throw ex; 117 | } 118 | catch (PathTooLongException ex) 119 | { 120 | Console.WriteLine($"Cannot write to the log file because the file path is too long. Please store your files for this WiMigrator application in a folder location with a shorter path name."); 121 | throw ex; 122 | } 123 | catch (Exception ex) 124 | { 125 | Console.WriteLine($"Cannot write to the log file: {ex.Message}"); 126 | } 127 | } 128 | 129 | private void AppendToFile(string content) 130 | { 131 | // since we support multithreading, ensure only one thread 132 | // accesses the file at a time. 133 | lock (lockObject) 134 | { 135 | using (var streamWriter = File.AppendText(this.filePath)) 136 | { 137 | streamWriter.Write(content); 138 | } 139 | } 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /Common/Migration/Phase2/Processors/WorkItemLinksProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; 6 | using Microsoft.VisualStudio.Services.WebApi.Patch.Json; 7 | using Logging; 8 | using Microsoft.VisualStudio.Services.Common; 9 | using Common.Config; 10 | 11 | namespace Common.Migration 12 | { 13 | public class WorkItemLinksProcessor : IPhase2Processor 14 | { 15 | private static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 16 | 17 | public string Name => Constants.RelationPhaseWorkItemLinks; 18 | 19 | public bool IsEnabled(ConfigJson config) 20 | { 21 | return config.MoveLinks; 22 | } 23 | 24 | public async Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList sourceWorkItems, IList targetWorkItems) 25 | { 26 | var linkedWorkItemArtifactUrls = new HashSet(); 27 | foreach (WorkItem sourceWorkItem in sourceWorkItems) 28 | { 29 | var relations = GetWorkItemLinkRelations(migrationContext, sourceWorkItem.Relations); 30 | var linkedIds = relations.Select(r => ClientHelpers.GetWorkItemIdFromApiEndpoint(r.Url)); 31 | var uris = linkedIds.Where(id => !migrationContext.SourceToTargetIds.ContainsKey(id)).Select(id => ClientHelpers.GetWorkItemApiEndpoint(migrationContext.Config.SourceConnection.Account, id)); 32 | linkedWorkItemArtifactUrls.AddRange(uris); 33 | } 34 | 35 | await linkedWorkItemArtifactUrls.Batch(Constants.BatchSize).ForEachAsync(migrationContext.Config.Parallelism, async (workItemArtifactUris, batchId) => 36 | { 37 | Logger.LogTrace(LogDestination.File, $"Finding linked work items on target for batch {batchId}"); 38 | var results = await ClientHelpers.QueryArtifactUriToGetIdsFromUris(migrationContext.TargetClient.WorkItemTrackingHttpClient, workItemArtifactUris); 39 | foreach (var result in results.ArtifactUrisQueryResult) 40 | { 41 | if (result.Value != null) 42 | { 43 | if (result.Value.Count() == 1) 44 | { 45 | var sourceId = ClientHelpers.GetWorkItemIdFromApiEndpoint(result.Key); 46 | var targetId = result.Value.First().Id; 47 | 48 | migrationContext.SourceToTargetIds[sourceId] = targetId; 49 | } 50 | } 51 | } 52 | 53 | Logger.LogTrace(LogDestination.File, $"Finished finding linked work items on target for batch {batchId}"); 54 | }); 55 | } 56 | 57 | public async Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem) 58 | { 59 | IList jsonPatchOperations = new List(); 60 | 61 | if (sourceWorkItem.Relations == null) 62 | { 63 | return jsonPatchOperations; 64 | } 65 | 66 | IList sourceWorkItemLinkRelations = GetWorkItemLinkRelations(migrationContext, sourceWorkItem.Relations); 67 | 68 | if (sourceWorkItemLinkRelations.Any()) 69 | { 70 | foreach (WorkItemRelation sourceWorkItemLinkRelation in sourceWorkItemLinkRelations) 71 | { 72 | int linkedSourceId = ClientHelpers.GetWorkItemIdFromApiEndpoint(sourceWorkItemLinkRelation.Url); 73 | int targetWorkItemId = targetWorkItem.Id.Value; 74 | int linkedTargetId; 75 | 76 | if (!migrationContext.SourceToTargetIds.TryGetValue(linkedSourceId, out linkedTargetId)) 77 | { 78 | continue; 79 | } 80 | 81 | string comment = MigrationHelpers.GetCommentFromAttributes(sourceWorkItemLinkRelation); 82 | WorkItemLink newWorkItemLink = new WorkItemLink(linkedTargetId, sourceWorkItemLinkRelation.Rel, false, false, comment, 0); 83 | 84 | JsonPatchOperation workItemLinkAddOperation = MigrationHelpers.GetWorkItemLinkAddOperation(migrationContext, newWorkItemLink); 85 | jsonPatchOperations.Add(workItemLinkAddOperation); 86 | } 87 | } 88 | 89 | return jsonPatchOperations; 90 | } 91 | 92 | private IList GetWorkItemLinkRelations(IMigrationContext migrationContext, IList relations) 93 | { 94 | IList result = new List(); 95 | 96 | if (relations == null) 97 | { 98 | return result; 99 | } 100 | 101 | foreach (WorkItemRelation relation in relations) 102 | { 103 | if (IsRelationWorkItemLink(migrationContext, relation)) 104 | { 105 | result.Add(relation); 106 | } 107 | } 108 | 109 | return result; 110 | } 111 | 112 | private bool IsRelationWorkItemLink(IMigrationContext migrationContext, WorkItemRelation relation) 113 | { 114 | if (migrationContext.ValidatedWorkItemLinkRelationTypes.Contains(relation.Rel)) 115 | { 116 | return true; 117 | } 118 | else 119 | { 120 | return false; 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /Common/Config/ConfigJson.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.ComponentModel; 3 | using Microsoft.Extensions.Logging; 4 | using Newtonsoft.Json; 5 | 6 | namespace Common.Config 7 | { 8 | public class ConfigJson 9 | { 10 | [JsonProperty(Required = Required.Always)] 11 | public string Query { get; set; } 12 | 13 | [JsonProperty(PropertyName = "source-connection", Required = Required.Always)] 14 | public ConfigConnection SourceConnection { get; set; } 15 | 16 | [JsonProperty(PropertyName = "target-connection", Required = Required.Always)] 17 | public ConfigConnection TargetConnection { get; set; } 18 | 19 | [JsonProperty(PropertyName = "parallelism", DefaultValueHandling = DefaultValueHandling.Populate)] 20 | public int Parallelism { get; set; } 21 | 22 | [JsonProperty(PropertyName = "link-parallelism", DefaultValueHandling = DefaultValueHandling.Populate)] 23 | public int LinkParallelism { get; set; } 24 | 25 | [JsonProperty(PropertyName = "heartbeat-frequency-in-seconds", DefaultValueHandling = DefaultValueHandling.Populate)] 26 | [DefaultValue(30)] 27 | public int HeartbeatFrequencyInSeconds { get; set; } 28 | 29 | [JsonProperty(PropertyName = "query-page-size", DefaultValueHandling = DefaultValueHandling.Populate)] 30 | [DefaultValue(20000)] 31 | public int QueryPageSize { get; set; } 32 | 33 | [JsonProperty(PropertyName = "max-attachment-size", DefaultValueHandling = DefaultValueHandling.Populate)] 34 | [DefaultValue(60 * 1024 * 1024)] 35 | public int MaxAttachmentSize { get; set; } 36 | 37 | [JsonProperty(PropertyName = "attachment-upload-chunk-size", DefaultValueHandling = DefaultValueHandling.Populate)] 38 | [DefaultValue(1 * 1024 * 1024)] 39 | public int AttachmentUploadChunkSize { get; set; } 40 | 41 | [JsonProperty(PropertyName = "skip-existing", DefaultValueHandling = DefaultValueHandling.Populate)] 42 | [DefaultValue(true)] 43 | public bool SkipExisting { get; set; } 44 | 45 | [JsonProperty(PropertyName = "move-history", DefaultValueHandling = DefaultValueHandling.Populate)] 46 | [DefaultValue(false)] 47 | public bool MoveHistory { get; set; } 48 | 49 | [JsonProperty(PropertyName = "move-history-limit", DefaultValueHandling = DefaultValueHandling.Populate)] 50 | [DefaultValue(200)] 51 | public int MoveHistoryLimit { get; set; } 52 | 53 | [JsonProperty(PropertyName = "move-git-links", DefaultValueHandling = DefaultValueHandling.Populate)] 54 | [DefaultValue(false)] 55 | public bool MoveGitLinks { get; set; } 56 | 57 | [JsonProperty(PropertyName = "move-attachments", DefaultValueHandling = DefaultValueHandling.Populate)] 58 | [DefaultValue(false)] 59 | public bool MoveAttachments { get; set; } 60 | 61 | [JsonProperty(PropertyName = "move-links", DefaultValueHandling = DefaultValueHandling.Populate)] 62 | [DefaultValue(false)] 63 | public bool MoveLinks { get; set; } 64 | 65 | [JsonProperty(PropertyName = "source-post-move-tag", DefaultValueHandling = DefaultValueHandling.Populate)] 66 | public string SourcePostMoveTag { get; set; } 67 | 68 | [JsonProperty(PropertyName = "target-post-move-tag", DefaultValueHandling = DefaultValueHandling.Populate)] 69 | public string TargetPostMoveTag { get; set; } 70 | 71 | [JsonProperty(PropertyName = "skip-work-items-with-type-missing-fields", DefaultValueHandling = DefaultValueHandling.Populate)] 72 | [DefaultValue(false)] 73 | public bool SkipWorkItemsWithTypeMissingFields { get; set; } 74 | 75 | [JsonProperty(PropertyName = "skip-work-items-with-missing-area-path", DefaultValueHandling = DefaultValueHandling.Populate)] 76 | [DefaultValue(false)] 77 | public bool SkipWorkItemsWithMissingAreaPath { get; set; } 78 | 79 | [JsonProperty(PropertyName = "skip-work-items-with-missing-iteration-path", DefaultValueHandling = DefaultValueHandling.Populate)] 80 | [DefaultValue(false)] 81 | public bool SkipWorkItemsWithMissingIterationPath { get; set; } 82 | 83 | [JsonProperty(PropertyName = "default-area-path", DefaultValueHandling = DefaultValueHandling.Populate)] 84 | public string DefaultAreaPath { get; set; } 85 | 86 | [JsonProperty(PropertyName = "default-iteration-path", DefaultValueHandling = DefaultValueHandling.Populate)] 87 | public string DefaultIterationPath { get; set; } 88 | 89 | [JsonProperty(PropertyName = "clear-identity-display-names", DefaultValueHandling = DefaultValueHandling.Populate)] 90 | [DefaultValue(false)] 91 | public bool ClearIdentityDisplayNames { get; set; } 92 | 93 | [JsonProperty(PropertyName = "ensure-identities", DefaultValueHandling = DefaultValueHandling.Populate)] 94 | [DefaultValue(false)] 95 | public bool EnsureIdentities { get; set; } 96 | 97 | [JsonProperty(PropertyName = "include-web-link", DefaultValueHandling = DefaultValueHandling.Populate)] 98 | [DefaultValue(false)] 99 | public bool IncludeWebLink { get; set; } 100 | 101 | [JsonProperty(PropertyName = "log-level-for-file", DefaultValueHandling = DefaultValueHandling.Populate)] 102 | [DefaultValue(LogLevel.Information)] 103 | public LogLevel LogLevelForFile { get; set; } 104 | 105 | [JsonProperty(PropertyName = "field-replacements", DefaultValueHandling = DefaultValueHandling.Populate)] 106 | public Dictionary FieldReplacements { get; set; } 107 | 108 | [JsonProperty(PropertyName = "send-email-notification", DefaultValueHandling = DefaultValueHandling.Populate)] 109 | [DefaultValue(false)] 110 | public bool SendEmailNotification { get; set; } 111 | 112 | [JsonProperty(PropertyName = "email-notification", Required = Required.DisallowNull)] 113 | public EmailNotification EmailNotification { get; set; } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /Common/RetryHelper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net.Http; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.VisualStudio.Services.Common; 6 | using Logging; 7 | 8 | namespace Common 9 | { 10 | public class RetryHelper 11 | { 12 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 13 | 14 | //If you want to retry then catch the exception and throw. 15 | //If you do not want to retry then do not throw 16 | public static async Task RetryAsync(Func> function, int retryCount, int secsDelay = 1) 17 | { 18 | return await RetryAsync(function, null, retryCount, secsDelay); 19 | } 20 | 21 | public static async Task RetryAsync(Func> function, Func> exceptionHandler, int retryCount, int secsDelay = 1) 22 | { 23 | Guid requestId = Guid.NewGuid(); 24 | Exception exception = null; 25 | bool succeeded = true; 26 | for (int i = 0; i < retryCount; i++) 27 | { 28 | try 29 | { 30 | succeeded = true; 31 | return await function(); 32 | } 33 | catch (Exception ex) 34 | { 35 | exception = TranslateException(requestId, ex); 36 | 37 | if (exceptionHandler != null) 38 | { 39 | try 40 | { 41 | exception = await exceptionHandler(requestId, exception); 42 | } 43 | catch 44 | { 45 | // continue with the original exception handling process 46 | } 47 | } 48 | 49 | if (exception is RetryPermanentException) 50 | { 51 | //exit the for loop as we are not retrying for anything considered permanent 52 | break; 53 | } 54 | 55 | succeeded = false; 56 | Logger.LogTrace(LogDestination.File, $"Sleeping for {secsDelay} seconds and retrying {requestId} again."); 57 | 58 | await Task.Delay(secsDelay * 1000); 59 | 60 | // add 1 second to delay so that each delay is slightly incrementing in wait time 61 | secsDelay += 1; 62 | } 63 | finally 64 | { 65 | if (succeeded && i >= 1) 66 | { 67 | Logger.LogSuccess(LogDestination.File, $"request {requestId} succeeded."); 68 | } 69 | } 70 | } 71 | 72 | if (exception is null) 73 | { 74 | throw new RetryExhaustedException($"Retry count exhausted for {requestId}."); 75 | } 76 | else 77 | { 78 | throw exception; 79 | } 80 | } 81 | 82 | /// 83 | /// Translates the exception to a permanent exception if not retryable 84 | /// 85 | private static Exception TranslateException(Guid requestId, Exception e) 86 | { 87 | var ex = UnwrapIfAggregateException(e); 88 | if (ex is VssServiceException) 89 | { 90 | //Retry in following cases only 91 | //VS402335: QueryTimeoutException 92 | //VS402490: QueryTooManyConcurrentUsers 93 | //VS402491: QueryServerBusy 94 | //TF400733: The request has been canceled: Request was blocked due to exceeding usage of resource 'WorkItemTrackingResource' in namespace 'User.' 95 | if (ex.Message.Contains("VS402335") 96 | || ex.Message.Contains("VS402490") 97 | || ex.Message.Contains("VS402491") 98 | || ex.Message.Contains("TF400733")) 99 | { 100 | Logger.LogWarning(LogDestination.File, ex, $"VssServiceException exception caught for {requestId}:"); 101 | } 102 | else 103 | { 104 | //Specific TF or VS errors. No need to retry DO NOT THROW 105 | return new RetryPermanentException($"Permanent error for {requestId}, not retrying", ex); 106 | } 107 | } 108 | else if (ex is HttpRequestException) 109 | { 110 | // all request exceptions should be considered retryable 111 | Logger.LogWarning(LogDestination.File, ex, $"HttpRequestException exception caught for {requestId}:"); 112 | } 113 | // TF237082: The file you are trying to upload exceeds the supported file upload size 114 | else if (ex.Message.Contains("TF237082")) 115 | { 116 | return new RetryPermanentException($"Permanent error for {requestId}, not retrying", ex); 117 | } 118 | else 119 | { 120 | //Log and throw every other exception for now - example HttpServiceException for connection errors 121 | //Need to retry - in case of connection timeouts or server unreachable etc. 122 | Logger.LogWarning(LogDestination.File, ex, $"Exception caught for {requestId}:"); 123 | } 124 | 125 | return ex; 126 | } 127 | 128 | private static Exception UnwrapIfAggregateException(Exception e) 129 | { 130 | Exception ex; 131 | //Async calls returns AggregateException 132 | //Sync calls returns exception 133 | if (e is AggregateException) 134 | { 135 | ex = e.InnerException; 136 | } 137 | else 138 | { 139 | ex = e; 140 | } 141 | 142 | return ex; 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /Common/Migration/Phase1/PreProcessors/IdentityPreProcessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | using Common.Config; 5 | using Logging; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.TeamFoundation.Framework.Common; 8 | using Microsoft.VisualStudio.Services.Common; 9 | using Microsoft.VisualStudio.Services.Graph.Client; 10 | using Microsoft.VisualStudio.Services.Identity; 11 | using Microsoft.VisualStudio.Services.Identity.Client; 12 | using Microsoft.VisualStudio.Services.Licensing.Client; 13 | using static Microsoft.VisualStudio.Services.Graph.Constants; 14 | 15 | namespace Common.Migration 16 | { 17 | public class IdentityPreProcessor : IPhase1PreProcessor 18 | { 19 | static ILogger Logger { get; } = MigratorLogging.CreateLogger(); 20 | private static string LicensedUsersGroup = SidIdentityHelper.ConstructWellKnownSid(0, 2048); 21 | private static SubjectDescriptor[] Groups = new[] { new SubjectDescriptor(SubjectType.VstsGroup, LicensedUsersGroup) }; 22 | 23 | private IMigrationContext context; 24 | private GraphHttpClient graphClient; 25 | private LicensingHttpClient licensingHttpClient; 26 | private IdentityHttpClient identityHttpClient; 27 | 28 | public string Name => "Identity"; 29 | 30 | public bool IsEnabled(ConfigJson config) 31 | { 32 | return config.EnsureIdentities; 33 | } 34 | 35 | public async Task Prepare(IMigrationContext context) 36 | { 37 | this.context = context; 38 | this.graphClient = context.TargetClient.Connection.GetClient(); 39 | this.licensingHttpClient = context.TargetClient.Connection.GetClient(); 40 | this.identityHttpClient = context.TargetClient.Connection.GetClient(); 41 | } 42 | 43 | public async Task Process(IBatchMigrationContext batchContext) 44 | { 45 | object identityObject = null; 46 | string identityValue = null; 47 | HashSet identitiesToProcess = new HashSet(StringComparer.OrdinalIgnoreCase); 48 | 49 | foreach (var sourceWorkItem in batchContext.SourceWorkItems) 50 | { 51 | foreach (var field in context.IdentityFields) 52 | { 53 | if (sourceWorkItem.Fields.TryGetValueIgnoringCase(field, out identityObject)) 54 | { 55 | identityValue = (string)identityObject; 56 | if (!string.IsNullOrEmpty(identityValue) 57 | && identityValue.Contains("<") && identityValue.Contains(">") && (identityValue.Contains("@"))) 58 | { 59 | // parse out email address from the combo string 60 | identityValue = identityValue.Substring(identityValue.LastIndexOf("<") + 1, identityValue.LastIndexOf(">") - identityValue.LastIndexOf("<") - 1); 61 | 62 | if (!identitiesToProcess.Contains(identityValue) 63 | && !this.context.ValidatedIdentities.Contains(identityValue) 64 | && !this.context.InvalidIdentities.Contains(identityValue)) 65 | { 66 | Logger.LogTrace(LogDestination.File, $"Found identity {identityValue} in batch {batchContext.BatchId} which has not yet been validated for the target account"); 67 | identitiesToProcess.Add(identityValue); 68 | } 69 | } 70 | } 71 | } 72 | } 73 | 74 | Logger.LogTrace(LogDestination.File, $"Adding {identitiesToProcess.Count} identities to the account for batch {batchContext.BatchId}"); 75 | foreach (var identity in identitiesToProcess) 76 | { 77 | try 78 | { 79 | var createUserResult = await RetryHelper.RetryAsync(async () => 80 | { 81 | return await graphClient.CreateUserAsync(new GraphUserPrincipalNameCreationContext() 82 | { 83 | PrincipalName = identity 84 | }); 85 | }, 5); 86 | 87 | // using identity from createUserResult since the identity could be in a mangled format that ReadIdentities does not support 88 | var identities = await RetryHelper.RetryAsync(async () => 89 | { 90 | return await identityHttpClient.ReadIdentitiesAsync(IdentitySearchFilter.MailAddress, createUserResult.MailAddress); 91 | }, 5); 92 | 93 | if (identities.Count == 0) 94 | { 95 | Logger.LogWarning(LogDestination.File, $"Unable to add identity {identity} to the target account for batch {batchContext.BatchId}"); 96 | context.InvalidIdentities.Add(identity); 97 | } 98 | else 99 | { 100 | var assignResult = await RetryHelper.RetryAsync(async () => 101 | { 102 | return await licensingHttpClient.AssignAvailableEntitlementAsync(identities[0].Id, dontNotifyUser: true); 103 | }, 5); 104 | context.ValidatedIdentities.Add(identity); 105 | } 106 | } 107 | catch (Exception ex) 108 | { 109 | Logger.LogWarning(LogDestination.File, ex, $"Unable to add identity {identity} to the target account for batch {batchContext.BatchId}"); 110 | context.InvalidIdentities.Add(identity); 111 | } 112 | } 113 | 114 | Logger.LogTrace(LogDestination.File, $"Completed adding {identitiesToProcess.Count} identities to the account for batch {batchContext.BatchId}"); 115 | } 116 | } 117 | } --------------------------------------------------------------------------------