├── .gitignore ├── FhirLoader.csproj ├── .github └── workflows │ └── release.yaml ├── FhirLoader.sln ├── MetricsCollector.cs ├── README.md ├── SyntheaReferenceResolver.cs └── Program.cs /.gitignore: -------------------------------------------------------------------------------- 1 | obj/ 2 | .vscode/ 3 | .vs/ 4 | *.json 5 | bin/ -------------------------------------------------------------------------------- /FhirLoader.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | release: 10 | runs-on: 'windows-latest' 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | 15 | - uses: actions/setup-dotnet@v1 16 | with: 17 | dotnet-version: '3.1.x' 18 | 19 | - run: dotnet publish --self-contained true --configuration Release --runtime win-x64 -p:PublishSingleFile=true --output .\pub 20 | 21 | - uses: actions/upload-artifact@v2 22 | with: 23 | name: FhirLoader.exe 24 | path: pub/FhirLoader.exe 25 | -------------------------------------------------------------------------------- /FhirLoader.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Version 16 3 | VisualStudioVersion = 16.0.30320.27 4 | MinimumVisualStudioVersion = 10.0.40219.1 5 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FhirLoader", "FhirLoader.csproj", "{FEA23758-131D-4464-BDE9-B351231636ED}" 6 | EndProject 7 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{30EBF985-12C9-41ED-90CE-78FD785E7251}" 8 | ProjectSection(SolutionItems) = preProject 9 | .gitignore = .gitignore 10 | .github\workflows\release.yaml = .github\workflows\release.yaml 11 | EndProjectSection 12 | EndProject 13 | Global 14 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 15 | Debug|Any CPU = Debug|Any CPU 16 | Release|Any CPU = Release|Any CPU 17 | EndGlobalSection 18 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 19 | {FEA23758-131D-4464-BDE9-B351231636ED}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 20 | {FEA23758-131D-4464-BDE9-B351231636ED}.Debug|Any CPU.Build.0 = Debug|Any CPU 21 | {FEA23758-131D-4464-BDE9-B351231636ED}.Release|Any CPU.ActiveCfg = Release|Any CPU 22 | {FEA23758-131D-4464-BDE9-B351231636ED}.Release|Any CPU.Build.0 = Release|Any CPU 23 | EndGlobalSection 24 | GlobalSection(SolutionProperties) = preSolution 25 | HideSolutionNode = FALSE 26 | EndGlobalSection 27 | GlobalSection(ExtensibilityGlobals) = postSolution 28 | SolutionGuid = {9106AB7D-B192-4E47-8FA6-7C17C456DA9B} 29 | EndGlobalSection 30 | EndGlobal 31 | -------------------------------------------------------------------------------- /MetricsCollector.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | 4 | namespace FhirLoader 5 | { 6 | /// 7 | /// Simple class for collecting events. 8 | /// Contains a circular buffer used to bin events. 9 | /// 10 | public class MetricsCollector 11 | { 12 | private readonly object _metricsLock = new object(); 13 | private int _bins = 0; 14 | private int _resolutionMs = 1000; 15 | private int _startBin = 0; 16 | private int _maxBinIndex = 0; 17 | private DateTime? _startTime = null; 18 | private long[] _counts; 19 | 20 | /// 21 | /// Constructor 22 | /// 23 | public MetricsCollector(int bins = 30, int resolutionMs = 1000) 24 | { 25 | _bins = bins; 26 | _counts = new long[bins]; 27 | _resolutionMs = resolutionMs; 28 | } 29 | 30 | /// 31 | /// Register event at specific time 32 | /// 33 | public void Collect(DateTime eventTime) 34 | { 35 | lock (_metricsLock) 36 | { 37 | if (_startTime is null) 38 | { 39 | _startTime = DateTime.Now; 40 | } 41 | 42 | int binIndex = (int)((eventTime - _startTime.Value).TotalMilliseconds / _resolutionMs); 43 | 44 | while (binIndex >= _bins) 45 | { 46 | _counts[_startBin] = 0; 47 | _startBin = (_startBin + 1) % _bins; 48 | _startTime += TimeSpan.FromMilliseconds(_resolutionMs); 49 | binIndex--; 50 | } 51 | 52 | _counts[(binIndex + _startBin) % _bins]++; 53 | 54 | // We keep track of this to make sure that in the warm up, we take the average only of bins used 55 | _maxBinIndex = binIndex; 56 | } 57 | } 58 | 59 | /// 60 | /// Return events per second 61 | /// 62 | public double EventsPerSecond { 63 | get { 64 | lock (_metricsLock) 65 | { 66 | return (double)_counts.Sum() / (_resolutionMs * (_maxBinIndex + 1) / 1000.0); 67 | } 68 | } 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FHIR Server Loading tool 2 | 3 | Simple tool for multi-threaded loading of FHIR data into a FHIR server. The tool will process bundles (like the ones generated by [Synthea](https://github.com/synthetichealth/synthea)). 4 | 5 | Before sending them to the server, the tool will resolve all internal links in the bundle and append all the resources to an ndjson file (one resource per line). If this buffer file already exists, the tool will not create it, so it is also possible to simply specify the buffer file if it is created by another tool. 6 | 7 | ```shell 8 | Usage: 9 | FhirLoader [options] 10 | 11 | Options: 12 | --input-folder inputFolder 13 | --fhir-server-url fhirServerUrl 14 | --authority authority 15 | --client-id clientId 16 | --client-secret clientSecret 17 | --access-token accessToken 18 | --buffer-file-name bufferFileName 19 | --re-create-buffer-if-exists reCreateBufferIfExists 20 | --max-degree-of-parallelism maxDegreeOfParallelism 21 | --refresh-interval refreshInterval 22 | --version Show version information 23 | -?, -h, --help Show help and usage information 24 | ``` 25 | 26 | ## Examples 27 | 28 | ### Authenticating with client id and secret 29 | 30 | ```shell 31 | dotnet run -- --client-secret "XXXX" --client-id "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" --input-folder ..\synthea\output\fhir\ --authority "https://login.microsoftonline.com/{tenant-id}" --fhir-server-url "https://{myfhirserver}.azurehealthcareapis.com" --max-degree-of-parallelism 14 32 | ``` 33 | 34 | ### Authenticating with Azure CLI 35 | 36 | ```pwsh 37 | $fhirServerUrl = "https://{myfhirserver}.azurehealthcareapis.com" 38 | $inputFolder = "..\synthea\output\fhir\" 39 | 40 | # enable healthcare plugin for Azure CLI 41 | az extension add --name healthcareapis 42 | 43 | # authorize user to access FHIR service 44 | az role assignment create ` 45 | --assignee $(az account show --query 'user.name' --output tsv) ` 46 | --scope $(az healthcareapis service list --query "[?properties.authenticationConfiguration.audience=='$fhirServerUrl'].id" --output tsv) ` 47 | --role 'FHIR Data Contributor' 48 | 49 | # run loader 50 | dotnet run -- ` 51 | --access-token $(az account get-access-token --resource $fhirServerUrl --query accessToken --output tsv) ` 52 | --fhir-server-url $fhirServerUrl ` 53 | --input-folder $inputFolder 54 | ``` 55 | -------------------------------------------------------------------------------- /SyntheaReferenceResolver.cs: -------------------------------------------------------------------------------- 1 | 2 | using Newtonsoft.Json.Linq; 3 | using System; 4 | using System.Collections.Generic; 5 | 6 | namespace FhirLoader 7 | { 8 | /// 9 | /// Utility class for resolving Synthea bundle references 10 | /// 11 | public class SyntheaReferenceResolver 12 | { 13 | /// 14 | /// Resolves all UUIDs in Synthea bundle 15 | /// 16 | public static void ConvertUUIDs(JObject bundle) 17 | { 18 | ConvertUUIDs(bundle, CreateUUIDLookUpTable(bundle)); 19 | } 20 | 21 | private static void ConvertUUIDs(JToken tok, Dictionary idLookupTable) 22 | { 23 | switch (tok.Type) 24 | { 25 | case JTokenType.Object: 26 | case JTokenType.Array: 27 | 28 | foreach (var c in tok.Children()) 29 | { 30 | ConvertUUIDs(c, idLookupTable); 31 | } 32 | 33 | return; 34 | case JTokenType.Property: 35 | JProperty prop = (JProperty)tok; 36 | 37 | if (prop.Value.Type == JTokenType.String && 38 | prop.Name == "reference" && 39 | idLookupTable.TryGetValue(prop.Value.ToString(), out var idTypePair)) 40 | { 41 | prop.Value = idTypePair.ResourceType + "/" + idTypePair.Id; 42 | } 43 | else 44 | { 45 | ConvertUUIDs(prop.Value, idLookupTable); 46 | } 47 | 48 | return; 49 | case JTokenType.String: 50 | case JTokenType.Boolean: 51 | case JTokenType.Float: 52 | case JTokenType.Integer: 53 | case JTokenType.Date: 54 | return; 55 | default: 56 | throw new NotSupportedException($"Invalid token type {tok.Type} encountered"); 57 | } 58 | } 59 | 60 | private static Dictionary CreateUUIDLookUpTable(JObject bundle) 61 | { 62 | Dictionary table = new Dictionary(); 63 | JArray entry = (JArray)bundle["entry"]; 64 | 65 | if (entry == null) 66 | { 67 | throw new ArgumentException("Unable to find bundle entries for creating lookup table"); 68 | } 69 | 70 | try 71 | { 72 | foreach (var resourceWrapper in entry) 73 | { 74 | var resource = resourceWrapper["resource"]; 75 | var fullUrl = (string)resourceWrapper["fullUrl"]; 76 | var resourceType = (string)resource["resourceType"]; 77 | var id = (string)resource["id"]; 78 | 79 | table.Add(fullUrl, new IdTypePair { ResourceType = resourceType, Id = id }); 80 | } 81 | } 82 | catch 83 | { 84 | Console.WriteLine("Error parsing resources in bundle"); 85 | throw; 86 | } 87 | 88 | return table; 89 | } 90 | 91 | private class IdTypePair 92 | { 93 | public string Id { get; set; } 94 | 95 | public string ResourceType { get; set; } 96 | } 97 | } 98 | } -------------------------------------------------------------------------------- /Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Net.Http; 4 | using System.Net.Http.Headers; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using System.Threading.Tasks.Dataflow; 9 | using Microsoft.IdentityModel.Clients.ActiveDirectory; 10 | using Newtonsoft.Json; 11 | using Newtonsoft.Json.Linq; 12 | using Polly; 13 | 14 | namespace FhirLoader 15 | { 16 | class Program 17 | { 18 | static void Main( 19 | string inputFolder, 20 | Uri fhirServerUrl, 21 | Uri authority = null, 22 | string clientId = null, 23 | string clientSecret = null, 24 | string accessToken = null, 25 | string bufferFileName = "resources.json", 26 | bool reCreateBufferIfExists = false, 27 | bool forcePost = false, 28 | int maxDegreeOfParallelism = 8, 29 | int refreshInterval = 5) 30 | { 31 | 32 | HttpClient httpClient = new HttpClient(); 33 | MetricsCollector metrics = new MetricsCollector(); 34 | 35 | // Create an ndjson file from the FHIR bundles in folder 36 | if (!(new FileInfo(bufferFileName).Exists) || reCreateBufferIfExists) 37 | { 38 | Console.WriteLine("Creating ndjson buffer file..."); 39 | CreateBufferFile(inputFolder, bufferFileName); 40 | Console.WriteLine("Buffer created."); 41 | } 42 | 43 | bool useAuth = authority != null && clientId != null && clientSecret != null && accessToken == null; 44 | 45 | AuthenticationContext authContext = useAuth ? new AuthenticationContext(authority.AbsoluteUri, new TokenCache()) : null; 46 | ClientCredential clientCredential = useAuth ? new ClientCredential(clientId, clientSecret) : null; 47 | 48 | var randomGenerator = new Random(); 49 | 50 | var actionBlock = new ActionBlock(async resourceString => 51 | { 52 | var resource = JObject.Parse(resourceString); 53 | string resource_type = (string)resource["resourceType"]; 54 | string id = (string)resource["id"]; 55 | 56 | Thread.Sleep(TimeSpan.FromMilliseconds(randomGenerator.Next(50))); 57 | 58 | StringContent content = new StringContent(resourceString, Encoding.UTF8, "application/json"); 59 | var pollyDelays = 60 | new[] 61 | { 62 | TimeSpan.FromMilliseconds(2000 + randomGenerator.Next(50)), 63 | TimeSpan.FromMilliseconds(3000 + randomGenerator.Next(50)), 64 | TimeSpan.FromMilliseconds(5000 + randomGenerator.Next(50)), 65 | TimeSpan.FromMilliseconds(8000 + randomGenerator.Next(50)), 66 | TimeSpan.FromMilliseconds(12000 + randomGenerator.Next(50)), 67 | TimeSpan.FromMilliseconds(16000 + randomGenerator.Next(50)), 68 | }; 69 | 70 | HttpResponseMessage uploadResult = await Policy 71 | .HandleResult(response => !response.IsSuccessStatusCode) 72 | .WaitAndRetryAsync(pollyDelays, (result, timeSpan, retryCount, context) => 73 | { 74 | if (retryCount > 3) 75 | { 76 | Console.WriteLine($"Request failed with {result.Result.StatusCode}. Waiting {timeSpan} before next retry. Retry attempt {retryCount}"); 77 | } 78 | }) 79 | .ExecuteAsync(() => 80 | { 81 | var message = forcePost || string.IsNullOrEmpty(id) 82 | ? new HttpRequestMessage(HttpMethod.Post, new Uri(fhirServerUrl, $"/{resource_type}")) 83 | : new HttpRequestMessage(HttpMethod.Put, new Uri(fhirServerUrl, $"/{resource_type}/{id}")); 84 | 85 | message.Content = content; 86 | 87 | if (useAuth) 88 | { 89 | var authResult = authContext.AcquireTokenAsync(fhirServerUrl.AbsoluteUri.TrimEnd('/'), clientCredential).Result; 90 | message.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authResult.AccessToken); 91 | } 92 | else if (accessToken != null) 93 | { 94 | message.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); 95 | } 96 | 97 | return httpClient.SendAsync(message); 98 | }); 99 | 100 | if (!uploadResult.IsSuccessStatusCode) 101 | { 102 | string resultContent = await uploadResult.Content.ReadAsStringAsync(); 103 | Console.WriteLine(resultContent); 104 | throw new Exception($"Unable to upload to server. Error code {uploadResult.StatusCode}"); 105 | } 106 | 107 | metrics.Collect(DateTime.Now); 108 | }, 109 | new ExecutionDataflowBlockOptions 110 | { 111 | MaxDegreeOfParallelism = maxDegreeOfParallelism 112 | } 113 | ); 114 | 115 | // Start output on timer 116 | var t = new Task( () => { 117 | while (true) 118 | { 119 | Thread.Sleep(1000 * refreshInterval); 120 | Console.WriteLine($"Resources per second: {metrics.EventsPerSecond}"); 121 | } 122 | }); 123 | t.Start(); 124 | 125 | // Read the ndjson file and feed it to the threads 126 | System.IO.StreamReader buffer = new System.IO.StreamReader(bufferFileName); 127 | string line; 128 | while ((line = buffer.ReadLine()) != null) 129 | { 130 | actionBlock.Post(line); 131 | } 132 | 133 | actionBlock.Complete(); 134 | actionBlock.Completion.Wait(); 135 | } 136 | 137 | private static void CreateBufferFile(string inputFolder, string bufferFileName) 138 | { 139 | using (System.IO.StreamWriter outFile = new System.IO.StreamWriter(bufferFileName)) 140 | { 141 | string[] files = Directory.GetFiles(inputFolder, "*.json", SearchOption.TopDirectoryOnly); 142 | 143 | foreach (var file in files) 144 | { 145 | string bundleText = File.ReadAllText(file); 146 | 147 | JObject bundle; 148 | try 149 | { 150 | bundle = JObject.Parse(bundleText); 151 | } 152 | catch (JsonReaderException) 153 | { 154 | Console.WriteLine("Input file is not a valid JSON document"); 155 | throw; 156 | } 157 | 158 | try 159 | { 160 | SyntheaReferenceResolver.ConvertUUIDs(bundle); 161 | } 162 | catch 163 | { 164 | Console.WriteLine("Failed to resolve references in doc"); 165 | throw; 166 | } 167 | 168 | foreach (var r in bundle.SelectTokens("$.entry[*].resource")) 169 | { 170 | outFile.WriteLine(r.ToString(Formatting.None)); 171 | } 172 | } 173 | } 174 | } 175 | } 176 | } 177 | --------------------------------------------------------------------------------