├── .gitignore ├── client └── index.html ├── readme.md ├── server-dotnetcore ├── Comparators │ └── MonthComparator.cs ├── Controllers │ └── CubeController.cs ├── DataLoaders │ ├── DataLoader.cs │ ├── DatabaseConnectionFactory.cs │ ├── IDataLoader.cs │ └── ParserFactory.cs ├── DataStorages │ ├── DataStorage.cs │ ├── DataStorageOptions.cs │ └── IDataStorage.cs ├── DatasourceOptions.cs ├── Extensions │ └── DateTimeExtension.cs ├── IPrepopulatingService.cs ├── JsonConverters │ ├── ColumnTypeJsonConverter.cs │ ├── DataJsonConverter.cs │ ├── MembersResponseJsonConverter.cs │ ├── SelectResponseJsonConverter.cs │ └── ValuesJsonConverter.cs ├── Models │ ├── DataModels │ │ ├── ColumnListDataStructure.cs │ │ ├── DataColumn.cs │ │ ├── DataSliceModel.cs │ │ └── IDataStructure.cs │ ├── FieldsRequest │ │ ├── FieldModel.cs │ │ ├── FieldsRequestModel.cs │ │ ├── SchemaAggregation.cs │ │ ├── SchemaFilter.cs │ │ ├── SchemaFilterElement.cs │ │ └── SchemaModel.cs │ ├── HandshakeRequest │ │ └── HandshakeRequst.cs │ ├── MembersRequest │ │ ├── MembersRequestModel.cs │ │ └── MembersResponseModel.cs │ ├── SelectRequest │ │ ├── AggregationBy.cs │ │ ├── AggregationModel.cs │ │ ├── AggregationRequestModel.cs │ │ ├── FieldFuncValue.cs │ │ ├── FilterModel.cs │ │ ├── HierarchyObject.cs │ │ ├── QueryModel.cs │ │ ├── SelectRequestModel.cs │ │ └── SelectResponceModel.cs │ └── Values │ │ └── ValueModel.cs ├── NetCoreServer.csproj ├── Parsers │ ├── CSVParser.cs │ ├── CSVSerializerOptions.cs │ ├── DatabaseParser.cs │ ├── IParser.cs │ └── JSONParser.cs ├── PrepopulatingCacheService.cs ├── Program.cs ├── Properties │ └── launchSettings.json ├── SettingsConfigurationHelper.cs ├── Startup.cs ├── appsettings.json └── data │ ├── data-types.csv │ ├── data-types.json │ ├── data.csv │ ├── fm-product-sales.csv │ └── fm-product-sales.json ├── server-nodejs ├── api │ └── cube.js ├── data │ ├── data-types.json │ ├── data.json │ └── fm-product-sales.json ├── package.json └── server.js └── tests ├── config.json ├── package-lock.json ├── package.json └── test ├── FieldsRequestSpec.js ├── HandshakeRequestSpec.js ├── MembersRequestSpec.js ├── SelectRequestDrillThroughSpec.js ├── SelectRequestFlatSpec.js └── SelectRequestSpec.js /.gitignore: -------------------------------------------------------------------------------- 1 | /server-nodejs/node_modules/ 2 | /server-nodejs/package-lock.json 3 | /server-dotnetcore/build/ 4 | /server-dotnetcore/[Bb]in/ 5 | /server-dotnetcore/[Oo]bj/ 6 | /server-dotnetcore/project.lock.json 7 | /tests/node_modules 8 | -------------------------------------------------------------------------------- /client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Flexmonster Pivot Table & Charts 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Flexmonster custom data source API server 2 | [![Flexmonster Pivot Table & Charts](https://cdn.flexmonster.com/landing.png)](https://www.flexmonster.com?r=github) 3 | Website: [www.flexmonster.com](https://www.flexmonster.com?r=github) 4 | 5 | ## Flexmonster Pivot Table & Charts 6 | 7 | Flexmonster Pivot is a powerful JavaScript tool for interactive web reporting. It allows you to visualize and analyze data from JSON, CSV, SQL, NoSQL, Elasticsearch, and OLAP data sources quickly and conveniently. Flexmonster is designed to integrate seamlessly with any client-side framework and can be easily embedded into your application. 8 | 9 | This repository holds the source code for sample servers that demonstrate how to implement the [custom data source API](https://www.flexmonster.com/doc/introduction-to-custom-data-source-api?r=github). 10 | 11 | The custom data source API is a communication protocol designed to retrieve aggregated data from a server to Flexmonster Pivot. 12 | The server is responsible for fetching, processing, and aggregating data, which is then passed to Flexmonster in a ready-to-show format. 13 | 14 | Table of contents: 15 | - [Prerequisites](#prerequisites) 16 | - [Installation](#installation) 17 | - [Tests](#tests) 18 | - [Related Flexmonster docs](#related-flexmonster-docs) 19 | 20 | ## Prerequisites 21 | 22 | For the sample Node.js server: 23 | - [Node.js 10 or later](https://nodejs.org/en/) 24 | 25 | For the sample .NET Core server: 26 | - [.NET Core 6.0](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 27 | 28 | ## Installation 29 | 30 | 1. Download a `.zip` archive with the sample project or clone it from GitHub with the following command: 31 | 32 | ```bash 33 | git clone https://github.com/flexmonster/api-data-source.git && cd api-data-source 34 | ``` 35 | 36 | 2. Run the project in one of the following ways: 37 | 38 | - To start the sample Node.js server: 39 | 40 | ``` 41 | cd server-nodejs 42 | npm install 43 | npm start 44 | ``` 45 | 46 | - To start the sample .NET Core server: 47 | 48 | ``` 49 | cd server-dotnetcore 50 | dotnet restore 51 | dotnet run 52 | ``` 53 | 54 | ## Tests 55 | 56 | If needed, you can check a custom data source API server with our [test suite](https://github.com/flexmonster/api-data-source/tree/master/tests). Run the tests with the following commands: 57 | 58 | ``` 59 | cd tests 60 | npm install 61 | npm test 62 | ``` 63 | 64 | Note that these tests will work only if one of the sample servers is running. To learn how your server can be tested, [see our documentation](https://www.flexmonster.com/doc/test-custom-data-source-api-server?r=github). 65 | 66 | ## Related Flexmonster docs 67 | 68 | For details on usage, refer to our documentation: 69 | 70 | - [A quick overview of the sample Node.js server](https://www.flexmonster.com/doc/pivot-table-with-node-js-server?r=github) 71 | - [A quick overview of the sample .NET Core server](https://www.flexmonster.com/doc/pivot-table-with-dot-net-core-server?r=github) 72 | - [Implementing the custom data source API server](https://www.flexmonster.com/doc/implement-custom-data-source-api?r=github) 73 | - [Custom data source API documentation](https://www.flexmonster.com/api/all-requests?r=github) 74 | -------------------------------------------------------------------------------- /server-dotnetcore/Comparators/MonthComparator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics.CodeAnalysis; 4 | using NetCoreServer.Models; 5 | 6 | namespace NetCoreServer.Comparators 7 | { 8 | /// 9 | /// Compare months by its name 10 | /// 11 | public class MonthComparator : Comparer 12 | { 13 | public override int Compare([AllowNull] T x, [AllowNull] T y) 14 | { 15 | 16 | if (x == null || y == null) return -1; 17 | if (Enum.TryParse(x.ToString(), out Month xInEnum)) 18 | { 19 | if (Enum.TryParse(y.ToString(), out Month yInEnum)) 20 | { 21 | return xInEnum.CompareTo(yInEnum); 22 | } 23 | } 24 | if (Enum.TryParse(x.ToString(), out ShortMonth xInShortEnum)) 25 | { 26 | if (Enum.TryParse(y.ToString(), out ShortMonth yInShortEnum)) 27 | { 28 | return xInShortEnum.CompareTo(yInShortEnum); 29 | } 30 | } 31 | return -1; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /server-dotnetcore/Controllers/CubeController.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Mvc; 2 | using Microsoft.Extensions.Caching.Memory; 3 | using NetCoreServer.Comparators; 4 | using NetCoreServer.DataStorages; 5 | using NetCoreServer.Extensions; 6 | using NetCoreServer.JsonConverters; 7 | using NetCoreServer.Models; 8 | using NetCoreServer.Models.DataModels; 9 | using NetCoreServer.Models.Fields; 10 | using NetCoreServer.Models.Handshake; 11 | using NetCoreServer.Models.Members; 12 | using NetCoreServer.Models.Select; 13 | using System; 14 | using System.Collections.Concurrent; 15 | using System.Collections.Generic; 16 | using System.IO; 17 | using System.Linq; 18 | using System.Security.Cryptography; 19 | using System.Text; 20 | using System.Text.Json; 21 | using System.Threading.Tasks; 22 | 23 | namespace NetCoreServer.Controllers 24 | { 25 | [ApiController] 26 | public class CubeController : ControllerBase 27 | { 28 | private const string API_VERSION = "2.9.0"; 29 | private const int MEMBERS_PAGE_SIZE = 50000; 30 | private const int SELECT_PAGE_SIZE = 50000; 31 | 32 | private readonly IMemoryCache _cache; 33 | 34 | private readonly IDataStorage _dataStorage; 35 | 36 | public CubeController(IMemoryCache cache, IDataStorage dataStorage) 37 | { 38 | _cache = cache; 39 | _dataStorage = dataStorage; 40 | } 41 | 42 | /// 43 | /// Handshake requst 44 | /// 45 | /// 46 | /// 47 | [Route("/api/cube/handshake")] 48 | [HttpPost] 49 | public IActionResult Handshake([FromBody]HandshakeRequst request) 50 | { 51 | object response = null; 52 | if (request.Type == RequestType.Handshake) 53 | { 54 | response = new { version = API_VERSION }; 55 | } 56 | return new JsonResult(response); 57 | } 58 | 59 | /// 60 | /// Fields requst 61 | /// 62 | /// requst 63 | /// 64 | [Route("/api/cube/fields")] 65 | [HttpPost] 66 | public async Task PostFields([FromBody]FieldsRequest request) 67 | { 68 | object response = null; 69 | if (request.Index == null) 70 | { 71 | Response.StatusCode = 400; 72 | return new JsonResult("Index property is missing."); 73 | } 74 | if (request.Type == RequestType.Fields) 75 | { 76 | try 77 | { 78 | response = await GetShema(request.Index); 79 | } 80 | catch (Exception e) 81 | { 82 | Console.WriteLine(e.StackTrace); 83 | Response.StatusCode = 500; 84 | return Content(e.Message); 85 | } 86 | } 87 | if (response == null) 88 | { 89 | Response.StatusCode = 400; 90 | return new JsonResult("Incorrect request for this endpoint."); 91 | } 92 | return new JsonResult(response, new JsonSerializerOptions { DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { new ColumnTypeJsonConverter() } }); 93 | } 94 | 95 | /// 96 | /// Members request 97 | /// 98 | /// 99 | /// 100 | [Route("/api/cube/members")] 101 | [HttpPost] 102 | public async Task PostMembers([FromBody]MembersRequest request) 103 | { 104 | if (request.Index == null) 105 | { 106 | Response.StatusCode = 400; 107 | return new JsonResult("Index property is missing."); 108 | } 109 | if (request.Type == RequestType.Members) 110 | { 111 | try 112 | { 113 | var response = await GetMembers(request.Index, request.Page, request.Field); 114 | return Content(response, "application/json"); 115 | } 116 | catch (Exception e) 117 | { 118 | Console.WriteLine(e.StackTrace); 119 | Response.StatusCode = 500; 120 | return new JsonResult(e.Message); 121 | } 122 | } 123 | Response.StatusCode = 400; 124 | return Content("Incorrect request for this endpoint."); 125 | } 126 | 127 | /// 128 | /// Select request 129 | /// 130 | /// requst 131 | /// 132 | [Route("/api/cube/select")] 133 | [HttpPost] 134 | public async Task PostSelect([FromBody]SelectRequest request) 135 | { 136 | string response = null; 137 | 138 | if (request.Index == null) 139 | { 140 | Response.StatusCode = 400; 141 | return new JsonResult("Index property is missing"); 142 | } 143 | if (request.Type == RequestType.Select) 144 | { 145 | try 146 | { 147 | response = await SelectData(request.Index, request.Query, request.Page); 148 | } 149 | catch (Exception e) 150 | { 151 | Console.WriteLine(e.StackTrace); 152 | Response.StatusCode = 500; 153 | return Content("Server error"); 154 | } 155 | } 156 | if (response == null) 157 | { 158 | Response.StatusCode = 400; 159 | return new JsonResult("Incorrect request for this endpoint."); 160 | } 161 | return Content(response); 162 | } 163 | 164 | /// 165 | /// Load schema and create object based on it 166 | /// 167 | /// index 168 | /// 169 | private async Task GetShema(string index) 170 | { 171 | return await _cache.GetOrCreateAsync(index + "schema", 172 | async (cacheEntry) => 173 | { 174 | cacheEntry.SetSize(1); 175 | cacheEntry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(240); 176 | Schema schema = new Schema(); 177 | schema.Sorted = false; 178 | schema.Aggregations.Any = new List { "count", "distinctcount" }; 179 | schema.Aggregations.Date = new List { "count", "distinctcount", "min", "max" }; 180 | schema.Aggregations.Number = new List { "sum", "average", "count", "distinctcount", "min", "max" }; 181 | schema.Filters.Any.Members = true; 182 | schema.Filters.Any.Query = true; 183 | schema.Filters.Any.ValueQuery = true; 184 | var data = await LoadData(index); 185 | var firstElement = data.GetNameAndTypes(); 186 | foreach (var elem in firstElement) 187 | { 188 | FieldModel field = new FieldModel(); 189 | field.UniqueName = elem.Key; 190 | field.Type = elem.Value; 191 | schema.Fields.Add(field); 192 | } 193 | return schema; 194 | }); 195 | } 196 | 197 | /// 198 | /// Load data from file {index}.json 199 | /// 200 | /// index 201 | /// 202 | private async Task LoadData(string index) 203 | { 204 | return await _dataStorage.GetOrAddAsync(index); 205 | } 206 | 207 | private async Task GetMembers(string index, int page, FieldModel field) 208 | { 209 | return (await _cache.GetOrCreateAsync(index + field.UniqueName, 210 | async (cacheEntry) => 211 | { 212 | cacheEntry.SetSize(1); 213 | JsonSerializerOptions options = new JsonSerializerOptions { Converters = { new MembersResponseJsonConverter() } }; 214 | IDataStructure data = await LoadData(index); 215 | var namesAndTypes = data.GetNameAndTypes(); 216 | DataSlice dataSlice = new DataSlice(data); 217 | List members = null; 218 | bool sorted = false; 219 | if (namesAndTypes[field.UniqueName] == ColumnType.stringType) 220 | { 221 | var column = DataSlice.Data.GetColumn(field.UniqueName); 222 | var stringMembers = dataSlice.DataColumnIndexes.Select(index => column[index]).Distinct().ToList(); 223 | if (stringMembers.Count != 0) 224 | { 225 | var first = stringMembers.First(); 226 | if (Enum.TryParse(first.ToString(), out Month m) || Enum.TryParse(first.ToString(), out ShortMonth m1)) 227 | { 228 | stringMembers.Sort(new MonthComparator()); 229 | sorted = true; 230 | } 231 | } 232 | members = stringMembers.ConvertAll(new Converter(str => (object)str)); 233 | } 234 | else 235 | { 236 | var column = DataSlice.Data.GetColumn(field.UniqueName); 237 | members = dataSlice.DataColumnIndexes.Select(index => column[index] as object).Distinct().ToList(); 238 | } 239 | 240 | int pageTotal = (int)Math.Ceiling(members.Count / (double)MEMBERS_PAGE_SIZE); 241 | pageTotal = pageTotal == 0 ? 1 : pageTotal; 242 | string[] responses = new string[pageTotal]; 243 | int currentPage = 0; 244 | while (currentPage < pageTotal) 245 | { 246 | MembersResponse response = new MembersResponse(); 247 | response.Sorted = sorted; 248 | response.Page = currentPage; 249 | response.PageTotal = pageTotal; 250 | int from = currentPage * MEMBERS_PAGE_SIZE; 251 | int size = Math.Min(members.Count, from + MEMBERS_PAGE_SIZE); 252 | for (int i = from; i < size; i++) 253 | { 254 | response.Members.Add(members[i]); 255 | } 256 | responses[currentPage] = JsonSerializer.Serialize(response, options); 257 | currentPage++; 258 | } 259 | return responses; 260 | }))[page]; 261 | } 262 | 263 | /// 264 | /// Gets field's members 265 | /// 266 | /// index 267 | /// page number to load 268 | /// field's name 269 | /// 270 | /// 271 | /// Select data according to query 272 | /// 273 | /// index 274 | /// query 275 | /// page number to load 276 | /// 277 | private async Task SelectData(string index, Query query, int page) 278 | { 279 | var hash = CalculateMD5Hash(index + JsonSerializer.Serialize(query)); 280 | return (await _cache.GetOrCreateAsync(hash, 281 | async (cacheEntry) => 282 | { 283 | cacheEntry.SetSize(1); 284 | var rawData = await LoadData(index); 285 | var nameTypes = rawData.GetNameAndTypes(); 286 | DataSlice data = new DataSlice(rawData); 287 | SelectResponse response = new SelectResponse(); 288 | string[] responses = null; 289 | JsonSerializerOptions options = new JsonSerializerOptions { DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { new SelectResponseJsonConverter() } }; 290 | if (query.Filter != null) 291 | { 292 | data.FilterData(query.Filter); 293 | } 294 | if (query.Aggs != null && query.Aggs.Values != null) 295 | { 296 | response.Aggs = new List(); 297 | query.Aggs.Values.ForEach(aggvalue => aggvalue.Field.Type = nameTypes[aggvalue.Field.UniqueName]); 298 | if (query.Aggs.By != null) 299 | { 300 | if (query.Aggs.By.Rows == null) 301 | { 302 | query.Aggs.By.Rows = new List(); 303 | } 304 | if (query.Aggs.By.Cols == null) 305 | { 306 | query.Aggs.By.Cols = new List(); 307 | } 308 | if (data.DataColumnIndexes.Count() != 0) 309 | { 310 | var tempAggs = new List(); 311 | var concatedFields = query.Aggs.By.Rows.Union(query.Aggs.By.Cols).ToList(); 312 | concatedFields.ForEach(concatedField => concatedField.Type = nameTypes[concatedField.UniqueName]); 313 | data.CalcByFields(concatedFields, query.Aggs.By.Cols, query.Aggs.Values, ref tempAggs); 314 | response.Aggs = tempAggs; 315 | response.Aggs.Add(data.CalcValues(query.Aggs.Values)); 316 | } 317 | else 318 | { 319 | response.Aggs = new List(); 320 | } 321 | } 322 | else 323 | { 324 | response.Aggs.Add(data.CalcValues(query.Aggs.Values)); 325 | } 326 | int pageTotal = (int)Math.Ceiling((double)response.Aggs.Count / SELECT_PAGE_SIZE); 327 | pageTotal = pageTotal == 0 ? 1 : pageTotal; 328 | responses = new string[pageTotal]; 329 | int currentPage = 0; 330 | while (currentPage < pageTotal) 331 | { 332 | SelectResponse partitialResponse = new SelectResponse(); 333 | partitialResponse.Page = currentPage; 334 | partitialResponse.PageTotal = pageTotal; 335 | int from = currentPage * SELECT_PAGE_SIZE; 336 | int size = Math.Min(response.Aggs.Count - from, SELECT_PAGE_SIZE); 337 | partitialResponse.Aggs = response.Aggs.GetRange(from, size); 338 | responses[currentPage] = JsonSerializer.Serialize(partitialResponse, options); 339 | currentPage++; 340 | } 341 | } 342 | if (query.Fields != null) 343 | { 344 | for (int i = 0; i < query.Fields.Count; i++) 345 | { 346 | query.Fields[i].Type = nameTypes[query.Fields[i].UniqueName]; 347 | response.Fields.Add(query.Fields[i]); 348 | } 349 | response.Hits = new List>(); 350 | var limit = query.Limit == 0 ? data.DataColumnIndexes.Count() : Math.Min(query.Limit, data.DataColumnIndexes.Count()); 351 | 352 | for (int i = 0; i < limit; i++) 353 | { 354 | List row = new List(); 355 | query.Fields.ForEach(field => 356 | { 357 | if (field.Type == ColumnType.stringType) 358 | { 359 | row.Add(DataSlice.Data.GetColumn(field.UniqueName)[data.DataColumnIndexes[i]]); 360 | } 361 | else 362 | { 363 | row.Add(DataSlice.Data.GetColumn(field.UniqueName)[data.DataColumnIndexes[i]]); 364 | } 365 | }); 366 | 367 | response.Hits.Add(row); 368 | } 369 | int pageTotal = (int)Math.Ceiling((double)response.Hits.Count / SELECT_PAGE_SIZE); 370 | pageTotal = pageTotal == 0 ? 1 : pageTotal; 371 | responses = new string[pageTotal]; 372 | int currentPage = 0; 373 | while (currentPage < pageTotal) 374 | { 375 | SelectResponse partitialResponse = new SelectResponse(); 376 | partitialResponse.Page = currentPage; 377 | partitialResponse.PageTotal = pageTotal; 378 | partitialResponse.Fields = response.Fields; 379 | partitialResponse.Aggs = response.Aggs; 380 | int from = currentPage * SELECT_PAGE_SIZE; 381 | int size = Math.Min(response.Hits.Count - from, SELECT_PAGE_SIZE); 382 | partitialResponse.Hits = response.Hits.GetRange(from, size); 383 | responses[currentPage] = JsonSerializer.Serialize(partitialResponse, options); 384 | currentPage++; 385 | } 386 | } 387 | return responses; 388 | }))[page]; 389 | } 390 | 391 | 392 | 393 | private string CalculateMD5Hash(string input) 394 | { 395 | MD5 md5 = MD5.Create(); 396 | byte[] inputBytes = Encoding.ASCII.GetBytes(input); 397 | byte[] hash = md5.ComputeHash(inputBytes); 398 | 399 | StringBuilder sb = new StringBuilder(); 400 | for (int i = 0; i < hash.Length; i++) 401 | { 402 | sb.Append(hash[i].ToString("X2")); 403 | } 404 | return sb.ToString(); 405 | } 406 | } 407 | } -------------------------------------------------------------------------------- /server-dotnetcore/DataLoaders/DataLoader.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models; 2 | using NetCoreServer.Models.DataModels; 3 | using NetCoreServer.Parsers; 4 | 5 | namespace NetCoreServer.DataLoaders 6 | { 7 | public class DataLoader : IDataLoader 8 | { 9 | private readonly IParser _parser; 10 | 11 | /// 12 | /// Parse data with given Parser 13 | /// 14 | /// Parser 15 | public DataLoader(IParser parser) 16 | { 17 | _parser = parser; 18 | } 19 | 20 | /// 21 | /// Load data from data source 22 | /// 23 | /// 24 | public IDataStructure Load() 25 | { 26 | var data = new ColumnListDataStructure(); 27 | foreach (var dataBlock in _parser.Parse()) 28 | { 29 | if (data.GetColumnNames().Count == 0) 30 | { 31 | foreach (var column in dataBlock.Keys) 32 | { 33 | data.AddColumn(column, _parser.DataTypes[column]); 34 | } 35 | } 36 | data.AddBlock(dataBlock); 37 | } 38 | return data; 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /server-dotnetcore/DataLoaders/DatabaseConnectionFactory.cs: -------------------------------------------------------------------------------- 1 | using MySql.Data.MySqlClient; 2 | using Npgsql; 3 | using Oracle.ManagedDataAccess.Client; 4 | using System.Data; 5 | using System.Data.SqlClient; 6 | 7 | namespace NetCoreServer.DataLoaders 8 | { 9 | public class DatabaseConnectionFactory 10 | { 11 | /// 12 | /// Get connection based on property in settings 13 | /// 14 | /// Connection string 15 | /// Type of database. Set in property. Can be "mysql", "mssql", "postgresql","oracle" 16 | /// Connection to database 17 | public IDbConnection GetDbConnection(string connectionString, string type) 18 | { 19 | switch (type) 20 | { 21 | case "mysql": 22 | { 23 | return new MySqlConnection(connectionString); 24 | } 25 | case "mssql": 26 | { 27 | return new SqlConnection(connectionString); 28 | } 29 | case "postgresql": 30 | { 31 | return new NpgsqlConnection(connectionString); 32 | } 33 | case "oracle": 34 | { 35 | return new OracleConnection(connectionString); 36 | } 37 | default: return null; 38 | } 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /server-dotnetcore/DataLoaders/IDataLoader.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.DataModels; 2 | 3 | namespace NetCoreServer.DataLoaders 4 | { 5 | public interface IDataLoader 6 | { 7 | IDataStructure Load(); 8 | } 9 | } -------------------------------------------------------------------------------- /server-dotnetcore/DataLoaders/ParserFactory.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.JsonConverters; 2 | using NetCoreServer.Parsers; 3 | using System; 4 | using System.Data; 5 | using System.Text.Json; 6 | 7 | namespace NetCoreServer.DataLoaders 8 | { 9 | 10 | public class ParserFactory : IDisposable 11 | { 12 | private readonly IndexOptions _options; 13 | private readonly DatabaseConnectionFactory _databaseConnectionFactory; 14 | private IDbConnection _dbConnection; 15 | private bool _disposed = false; 16 | 17 | /// 18 | /// Factory to create specific parser 19 | /// 20 | /// Configuration with datasource and other mandatory parameters 21 | /// Index 22 | public ParserFactory(IndexOptions options) 23 | { 24 | _options = options; 25 | _databaseConnectionFactory = new DatabaseConnectionFactory(); 26 | } 27 | 28 | /// 29 | /// Create parser based on Data source type 30 | /// 31 | /// Data source type from settings. Can be "json","csv" or "database" 32 | /// Created Parser 33 | public IParser CreateParser(string index) 34 | { 35 | switch (_options.Type) 36 | { 37 | case "json": 38 | { 39 | var path = (_options as JsonIndexOptions)?.Path; 40 | JsonSerializerOptions serializerOptions = new JsonSerializerOptions 41 | { 42 | AllowTrailingCommas = true, 43 | Converters = { new DataJsonConverter() } 44 | }; 45 | return new JSONParser(path, serializerOptions); 46 | } 47 | case "csv": 48 | { 49 | var csvOptions = _options as CsvIndexOptions; 50 | var path = csvOptions?.Path; 51 | var delimeter = csvOptions?.Delimiter; 52 | CSVSerializerOptions serializerOptions = new CSVSerializerOptions(); 53 | if (delimeter.HasValue) 54 | { 55 | serializerOptions.FieldSeparator = delimeter.Value; 56 | } 57 | return new CSVParser(path, serializerOptions); 58 | } 59 | case "database": 60 | { 61 | var options = _options as DatabaseIndexOptions; 62 | _dbConnection = _databaseConnectionFactory.GetDbConnection(options.ConnectionString, options.DatabaseType); 63 | _dbConnection.Open(); 64 | var dbCommand = _dbConnection.CreateCommand(); 65 | dbCommand.CommandText = options?.Query; 66 | dbCommand.Prepare(); 67 | return new DatabaseParser(dbCommand.ExecuteReader()); 68 | } 69 | default: 70 | { 71 | return null; 72 | } 73 | } 74 | } 75 | 76 | public void Dispose() 77 | { 78 | Dispose(true); 79 | GC.SuppressFinalize(this); 80 | } 81 | 82 | private void Dispose(bool disposing) 83 | { 84 | if (_disposed) 85 | { 86 | return; 87 | } 88 | 89 | if (disposing) 90 | { 91 | //dispose connection because used in outside function, but owner is this class 92 | _dbConnection?.Dispose(); 93 | _disposed = true; 94 | } 95 | } 96 | } 97 | } -------------------------------------------------------------------------------- /server-dotnetcore/DataStorages/DataStorage.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Caching.Memory; 2 | using Microsoft.Extensions.Options; 3 | using Microsoft.Extensions.Primitives; 4 | using NetCoreServer.DataLoaders; 5 | using NetCoreServer.Models.DataModels; 6 | using System; 7 | using System.Collections.Concurrent; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace NetCoreServer.DataStorages 12 | { 13 | public class DataStorage : IDataStorage 14 | { 15 | private readonly IMemoryCache _memoryCache; 16 | private readonly DatasourceOptions _datasourceOptions; 17 | private readonly DataStorageOptions _dataStorageOptions; 18 | private readonly ConcurrentDictionary _locks = new ConcurrentDictionary(); 19 | 20 | public DataStorage(IOptions dataSourceOptions, IOptions dataStorageOptions) 21 | { 22 | _memoryCache = new MemoryCache(new MemoryCacheOptions()); 23 | _datasourceOptions = dataSourceOptions.Value; 24 | _dataStorageOptions = dataStorageOptions.Value; 25 | } 26 | /// 27 | /// Get data from cache or add it if key doesn't present 28 | /// 29 | /// key 30 | /// Loaded data 31 | public async Task GetOrAddAsync(string cacheKey) 32 | { 33 | if (!_memoryCache.TryGetValue(cacheKey, out IDataStructure dataStructure)) 34 | { 35 | SemaphoreSlim certLock = _locks.GetOrAdd(cacheKey, k => new SemaphoreSlim(1, 1)); 36 | await certLock.WaitAsync(); 37 | 38 | try 39 | { 40 | if (!_memoryCache.TryGetValue(cacheKey, out dataStructure)) 41 | { 42 | using (var parserFactory = new ParserFactory(_datasourceOptions.Indexes[cacheKey])) 43 | { 44 | var parser = parserFactory.CreateParser(cacheKey); 45 | var dataLoader = new DataLoader(parser); 46 | dataStructure = dataLoader.Load(); 47 | } 48 | if (_dataStorageOptions.DataRefreshTime != 0) 49 | { 50 | _memoryCache.Set(cacheKey, dataStructure, GetMemoryCacheEntryOptions(_dataStorageOptions.DataRefreshTime)); 51 | } 52 | else 53 | { 54 | _memoryCache.Set(cacheKey, dataStructure); 55 | } 56 | } 57 | } 58 | finally 59 | { 60 | certLock.Release(); 61 | } 62 | } 63 | return dataStructure; 64 | } 65 | 66 | /// 67 | /// Set options for cache entry including PostEvictionCallback 68 | /// 69 | /// Time when cache entry will expire 70 | /// 71 | private MemoryCacheEntryOptions GetMemoryCacheEntryOptions(int expireInMinutes = 60) 72 | { 73 | var expirationTime = DateTime.Now.AddMinutes(expireInMinutes); 74 | var expirationToken = new CancellationChangeToken( 75 | new CancellationTokenSource(TimeSpan.FromMinutes(expireInMinutes + .01)).Token); 76 | 77 | var memoryCacheEntryOptions = new MemoryCacheEntryOptions(); 78 | memoryCacheEntryOptions.SetAbsoluteExpiration(expirationTime); 79 | memoryCacheEntryOptions.AddExpirationToken(expirationToken); 80 | 81 | memoryCacheEntryOptions.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration() 82 | { 83 | EvictionCallback = (key, value, reason, state) => 84 | { 85 | if (reason == EvictionReason.TokenExpired || reason == EvictionReason.Expired) 86 | { 87 | IDataStructure dataStructure = null; 88 | using (var parserFactory = new ParserFactory(_datasourceOptions.Indexes[key as string])) 89 | { 90 | var parser = parserFactory.CreateParser(key as string); 91 | var dataLoader = new DataLoader(parser); 92 | dataStructure = dataLoader.Load(); 93 | } 94 | if (dataStructure != null) 95 | { 96 | _memoryCache.Set(key, dataStructure, GetMemoryCacheEntryOptions(expireInMinutes)); 97 | } 98 | else 99 | { 100 | _memoryCache.Set(key, value, GetMemoryCacheEntryOptions(expireInMinutes)); 101 | } 102 | } 103 | } 104 | }); 105 | 106 | return memoryCacheEntryOptions; 107 | } 108 | } 109 | } -------------------------------------------------------------------------------- /server-dotnetcore/DataStorages/DataStorageOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace NetCoreServer.DataStorages 7 | { 8 | public class DataStorageOptions 9 | { 10 | public int DataRefreshTime { get; set; } 11 | } 12 | } -------------------------------------------------------------------------------- /server-dotnetcore/DataStorages/IDataStorage.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.DataModels; 2 | using System.Threading.Tasks; 3 | 4 | namespace NetCoreServer.DataStorages 5 | { 6 | public interface IDataStorage 7 | { 8 | public Task GetOrAddAsync(string cacheKey); 9 | } 10 | } -------------------------------------------------------------------------------- /server-dotnetcore/DatasourceOptions.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.DataLoaders; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer 5 | { 6 | public class DatasourceOptions 7 | { 8 | public Dictionary Indexes { get; set; } 9 | } 10 | public class IndexOptions 11 | { 12 | public string Type { get; set; } 13 | public IndexOptions() { } 14 | public IndexOptions(string type) 15 | { 16 | Type = type; 17 | } 18 | } 19 | public class JsonIndexOptions : IndexOptions 20 | { 21 | public string Path { get; set; } 22 | public JsonIndexOptions() { } 23 | public JsonIndexOptions(string path) 24 | { 25 | Path = path; 26 | } 27 | 28 | } 29 | public class CsvIndexOptions : IndexOptions 30 | { 31 | public string Path { get; set; } 32 | public char? Delimiter { get; set; } 33 | public CsvIndexOptions() { } 34 | 35 | public CsvIndexOptions(string path) 36 | { 37 | Path = path; 38 | } 39 | } 40 | public class DatabaseIndexOptions : IndexOptions 41 | { 42 | public string Query { get; set; } 43 | public string DatabaseType { get; set; } 44 | public string ConnectionString { get; set; } 45 | 46 | public DatabaseIndexOptions() { } 47 | public DatabaseIndexOptions(string databaseType, string connectionString, string query) 48 | { 49 | ConnectionString = connectionString; 50 | DatabaseType = databaseType; 51 | Query = query; 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /server-dotnetcore/Extensions/DateTimeExtension.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace NetCoreServer.Extensions 7 | { 8 | public static class DateTimeExtensions 9 | { 10 | /// 11 | /// Convert date to Unix timestamp 12 | /// 13 | /// 14 | /// 15 | public static double ToUnixTimestamp(this DateTime date) 16 | { 17 | DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0); 18 | TimeSpan diff = date.Kind == DateTimeKind.Local ? date.ToUniversalTime() - origin : date - origin; 19 | return Math.Floor(diff.TotalMilliseconds); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /server-dotnetcore/IPrepopulatingService.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace NetCoreServer 4 | { 5 | public interface IPrepopulatingService 6 | { 7 | public Task Prepopulate(); 8 | } 9 | } -------------------------------------------------------------------------------- /server-dotnetcore/JsonConverters/ColumnTypeJsonConverter.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.DataModels; 2 | using System; 3 | using System.Text.Json; 4 | using System.Text.Json.Serialization; 5 | 6 | namespace NetCoreServer.JsonConverters 7 | { 8 | /// 9 | /// Converter from JSON to ColumnType 10 | /// 11 | public class ColumnTypeJsonConverter : JsonConverter 12 | { 13 | 14 | public override ColumnType Read(ref Utf8JsonReader reader, 15 | Type typeToConvert, 16 | JsonSerializerOptions options) 17 | { 18 | var type = JsonSerializer.Deserialize(ref reader, options); 19 | if (type == "number") 20 | { 21 | return ColumnType.doubleType; 22 | } 23 | else if (type == "date") 24 | { 25 | return ColumnType.dateType; 26 | } 27 | return ColumnType.stringType; 28 | } 29 | 30 | public override void Write(Utf8JsonWriter writer, 31 | ColumnType value, 32 | JsonSerializerOptions options) 33 | { 34 | if (value == ColumnType.stringType) 35 | writer.WriteStringValue("string"); 36 | else if (value == ColumnType.doubleType) 37 | writer.WriteStringValue("number"); 38 | else if (value == ColumnType.dateType) 39 | writer.WriteStringValue("date"); 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /server-dotnetcore/JsonConverters/DataJsonConverter.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Extensions; 2 | using NetCoreServer.Models.DataModels; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Text.Json; 7 | using System.Text.Json.Serialization; 8 | using System.Threading.Tasks; 9 | 10 | namespace NetCoreServer.JsonConverters 11 | { 12 | /// 13 | /// Converter from JSON to ColumnDataStructure 14 | /// 15 | public class DataJsonConverter : JsonConverter> 16 | { 17 | private Dictionary _columnTypes; 18 | 19 | public Dictionary GetTypes() 20 | { 21 | return _columnTypes; 22 | } 23 | 24 | public override Dictionary Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) 25 | { 26 | reader.Read(); 27 | Dictionary columnList = new Dictionary(); 28 | if (_columnTypes == null) 29 | { 30 | _columnTypes = new Dictionary(); 31 | reader.Read(); 32 | while (reader.TokenType != JsonTokenType.EndObject) 33 | { 34 | var prop = reader.GetString(); 35 | reader.Read(); 36 | if (reader.TokenType == JsonTokenType.String) 37 | { 38 | var value = reader.GetString(); 39 | reader.Read(); 40 | if (DateTime.TryParse(value, out DateTime dateValue)) 41 | { 42 | columnList.Add(prop, new List()); 43 | columnList[prop].Add(dateValue.ToUnixTimestamp()); 44 | _columnTypes.Add(prop, ColumnType.dateType); 45 | } 46 | else 47 | { 48 | columnList.Add(prop, new List()); 49 | columnList[prop].Add(value); 50 | _columnTypes.Add(prop, ColumnType.stringType); 51 | } 52 | } 53 | else if (reader.TokenType == JsonTokenType.Number) 54 | { 55 | var value = reader.GetDouble(); 56 | reader.Read(); 57 | columnList.Add(prop, new List()); 58 | columnList[prop].Add(value); 59 | _columnTypes.Add(prop, ColumnType.doubleType); 60 | } 61 | } 62 | reader.Read(); 63 | } 64 | else 65 | { 66 | foreach (var column in _columnTypes) 67 | { 68 | if (column.Value == ColumnType.stringType) 69 | { 70 | columnList.Add(column.Key, new List()); 71 | } 72 | else 73 | { 74 | columnList.Add(column.Key, new List()); 75 | } 76 | } 77 | } 78 | while (reader.TokenType != JsonTokenType.EndArray) 79 | { 80 | reader.Read(); 81 | int propertyCount = 0; 82 | while (reader.TokenType != JsonTokenType.EndObject) 83 | { 84 | var prop = reader.GetString(); 85 | reader.Read(); 86 | if (_columnTypes.ContainsKey(prop)) 87 | { 88 | if (_columnTypes[prop] == ColumnType.stringType) 89 | { 90 | if (reader.TokenType == JsonTokenType.Number) 91 | { 92 | var value = reader.GetDouble(); 93 | reader.Read(); 94 | columnList[prop].Add(value.ToString()); 95 | } 96 | else 97 | { 98 | var value = reader.GetString(); 99 | reader.Read(); 100 | if (value != "") 101 | { 102 | columnList[prop].Add(value); 103 | } 104 | else 105 | { 106 | columnList[prop].Add((string)null); 107 | } 108 | } 109 | } 110 | else if (_columnTypes[prop] == ColumnType.doubleType) 111 | { 112 | if (reader.TokenType == JsonTokenType.Number) 113 | { 114 | var value = reader.GetDouble(); 115 | reader.Read(); 116 | columnList[prop].Add(value); 117 | } 118 | else 119 | { 120 | var value = reader.GetString(); 121 | reader.Read(); 122 | columnList[prop].Add((double?)null); 123 | } 124 | } 125 | else if (_columnTypes[prop] == ColumnType.dateType) 126 | { 127 | var value = reader.GetString(); 128 | reader.Read(); 129 | if (DateTime.TryParse(value, out DateTime dateValue)) 130 | { 131 | columnList[prop].Add(dateValue.ToUnixTimestamp()); 132 | } 133 | else 134 | { 135 | columnList[prop].Add((double?)null); 136 | } 137 | } 138 | propertyCount++; 139 | } 140 | else 141 | { 142 | reader.Read(); 143 | } 144 | } 145 | if (propertyCount < _columnTypes.Count) 146 | { 147 | var currentCount = columnList.Max(column => column.Value.Count); 148 | foreach (var column in columnList) 149 | { 150 | if (column.Value.Count < currentCount) 151 | { 152 | if (_columnTypes[column.Key] == ColumnType.stringType) 153 | { 154 | column.Value.Add((string)null); 155 | } 156 | else 157 | { 158 | column.Value.Add((double?)null); 159 | } 160 | } 161 | } 162 | } 163 | reader.Read(); 164 | } 165 | reader.Read(); 166 | return columnList; 167 | } 168 | 169 | public override void Write(Utf8JsonWriter writer, Dictionary value, JsonSerializerOptions options) 170 | { 171 | throw new NotImplementedException(); 172 | } 173 | } 174 | } -------------------------------------------------------------------------------- /server-dotnetcore/JsonConverters/MembersResponseJsonConverter.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Extensions; 2 | using NetCoreServer.Models.Members; 3 | using System; 4 | using System.Text.Json; 5 | using System.Text.Json.Serialization; 6 | 7 | namespace NetCoreServer.JsonConverters 8 | { 9 | /// 10 | /// Convertor from MembersResponse object to JSON 11 | /// 12 | public class MembersResponseJsonConverter : JsonConverter 13 | { 14 | public override MembersResponse Read(ref Utf8JsonReader reader, 15 | Type typeToConvert, 16 | JsonSerializerOptions options) 17 | { 18 | var response = new MembersResponse(); 19 | response = JsonSerializer.Deserialize(ref reader, options); 20 | return response; 21 | } 22 | 23 | public override void Write(Utf8JsonWriter writer, 24 | MembersResponse value, 25 | JsonSerializerOptions options) 26 | { 27 | writer.WriteStartObject(); 28 | writer.WritePropertyName("members"); 29 | writer.WriteStartArray(); 30 | foreach (var member in value.Members) 31 | { 32 | writer.WriteStartObject(); 33 | writer.WritePropertyName("value"); 34 | if (member != null) 35 | { 36 | JsonSerializer.Serialize(writer, member, options); 37 | } 38 | else 39 | { 40 | writer.WriteStringValue(""); 41 | } 42 | writer.WriteEndObject(); 43 | } 44 | writer.WriteEndArray(); 45 | writer.WriteBoolean("sorted", value.Sorted); 46 | writer.WriteNumber("page", value.Page); 47 | writer.WriteNumber("pageTotal", value.PageTotal); 48 | 49 | writer.WriteEndObject(); 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /server-dotnetcore/JsonConverters/SelectResponseJsonConverter.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Select; 2 | using System; 3 | using System.Text.Json; 4 | using System.Text.Json.Serialization; 5 | 6 | namespace NetCoreServer.JsonConverters 7 | { 8 | /// 9 | /// Convertor from SelectResponse object to JSON 10 | /// 11 | public class SelectResponseJsonConverter : JsonConverter 12 | { 13 | public override SelectResponse Read(ref Utf8JsonReader reader, 14 | Type typeToConvert, 15 | JsonSerializerOptions options) 16 | { 17 | var response = new SelectResponse(); 18 | response = JsonSerializer.Deserialize(ref reader, options); 19 | return response; 20 | } 21 | 22 | public override void Write(Utf8JsonWriter writer, 23 | SelectResponse value, 24 | JsonSerializerOptions options) 25 | { 26 | writer.WriteStartObject(); 27 | if (value.Fields != null) 28 | { 29 | writer.WritePropertyName("fields"); 30 | 31 | writer.WriteStartArray(); 32 | foreach (var field in value.Fields) 33 | { 34 | writer.WriteStartObject(); 35 | writer.WriteString("uniqueName", field.UniqueName); 36 | writer.WriteEndObject(); 37 | } 38 | writer.WriteEndArray(); 39 | } 40 | if (value.Hits != null) 41 | { 42 | writer.WritePropertyName("hits"); 43 | writer.WriteStartArray(); 44 | for (int i = 0; i < value.Hits.Count; i++) 45 | { 46 | writer.WriteStartArray(); 47 | for (int j = 0; j < value.Hits[i].Count; j++) 48 | { 49 | if (value.Hits[i][j] != null) 50 | { 51 | JsonSerializer.Serialize(writer, value.Hits[i][j], options); 52 | } 53 | else 54 | { 55 | writer.WriteStringValue(""); 56 | } 57 | } 58 | writer.WriteEndArray(); 59 | } 60 | writer.WriteEndArray(); 61 | } 62 | if (value.Aggs != null) 63 | { 64 | writer.WritePropertyName("aggs"); 65 | writer.WriteStartArray(); 66 | foreach (var member in value.Aggs) 67 | { 68 | writer.WriteStartObject(); 69 | writer.WritePropertyName("values"); 70 | writer.WriteStartObject(); 71 | foreach (var memberval in member.Values) 72 | { 73 | writer.WritePropertyName(memberval.Key); 74 | writer.WriteStartObject(); 75 | foreach (var val in memberval.Value) 76 | { 77 | writer.WriteNumber(val.Key, val.Value); 78 | } 79 | writer.WriteEndObject(); 80 | } 81 | writer.WriteEndObject(); 82 | if (member.Keys != null) 83 | { 84 | writer.WritePropertyName("keys"); 85 | writer.WriteStartObject(); 86 | foreach (var memberkey in member.Keys) 87 | { 88 | writer.WritePropertyName(memberkey.Key); 89 | JsonSerializer.Serialize(writer, memberkey.Value, options); 90 | } 91 | writer.WriteEndObject(); 92 | } 93 | writer.WriteEndObject(); 94 | } 95 | writer.WriteEndArray(); 96 | } 97 | writer.WriteNumber("page", value.Page); 98 | writer.WriteNumber("pageTotal", value.PageTotal); 99 | 100 | writer.WriteEndObject(); 101 | } 102 | } 103 | } -------------------------------------------------------------------------------- /server-dotnetcore/JsonConverters/ValuesJsonConverter.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text.Json; 5 | using System.Text.Json.Serialization; 6 | 7 | namespace NetCoreServer.JsonConverters 8 | { 9 | /// 10 | /// Convertor from ValueObject object to JSON depending on it's type 11 | /// 12 | public class ValuesJsonConverter : JsonConverter 13 | { 14 | public override ValueObject Read(ref Utf8JsonReader reader, 15 | Type typeToConvert, 16 | JsonSerializerOptions options) 17 | { 18 | if (reader.TokenType == JsonTokenType.String) 19 | { 20 | var value = reader.GetString(); 21 | return new ValueObject(value); 22 | } 23 | else if (reader.TokenType == JsonTokenType.Number) 24 | { 25 | var intvalue = reader.GetDouble(); 26 | return new ValueObject(intvalue); 27 | } 28 | reader.Read(); 29 | if (reader.TokenType == JsonTokenType.String) 30 | { 31 | var list = new List(); 32 | list.Add(reader.GetString()); 33 | reader.Read(); 34 | list.Add(reader.GetString()); 35 | reader.Read(); 36 | return new ValueObject(list); 37 | } 38 | var numberList = new List(); 39 | numberList.Add(reader.GetDouble()); 40 | reader.Read(); 41 | numberList.Add(reader.GetDouble()); 42 | reader.Read(); 43 | return new ValueObject(numberList); 44 | } 45 | 46 | public override void Write(Utf8JsonWriter writer, 47 | ValueObject value, 48 | JsonSerializerOptions options) 49 | { 50 | JsonSerializer.Serialize(writer, value); 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/DataModels/ColumnListDataStructure.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.DataModels; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | 6 | namespace NetCoreServer.Models 7 | { 8 | public enum Month 9 | { 10 | January, 11 | February, 12 | March, 13 | April, 14 | May, 15 | June, 16 | July, 17 | August, 18 | September, 19 | October, 20 | November, 21 | December 22 | } 23 | 24 | public enum ShortMonth 25 | { 26 | Jan, 27 | Feb, 28 | Mar, 29 | Apr, 30 | May, 31 | Jun, 32 | Jul, 33 | Aug, 34 | Sep, 35 | Oct, 36 | Nov, 37 | Dec 38 | } 39 | 40 | public class ColumnListDataStructure : IDataStructure 41 | { 42 | public Dictionary DataValuesByColumn { get; set; } 43 | 44 | public ColumnListDataStructure() 45 | { 46 | DataValuesByColumn = new Dictionary(); 47 | } 48 | 49 | /// 50 | /// Get column by name 51 | /// 52 | /// Type of column 53 | /// Column name 54 | /// 55 | public DataColumn GetColumn(string columnName) 56 | { 57 | return DataValuesByColumn[columnName]; 58 | } 59 | 60 | /// 61 | /// Get all columns names 62 | /// 63 | /// List of names 64 | public List GetColumnNames() 65 | { 66 | return DataValuesByColumn.Keys.ToList(); 67 | } 68 | 69 | /// 70 | /// Get all columns names and types 71 | /// 72 | /// Dictionary where key - name, value - type 73 | public Dictionary GetNameAndTypes() 74 | { 75 | Dictionary row = new Dictionary(); 76 | foreach (var column in DataValuesByColumn) 77 | { 78 | row.Add(column.Key, column.Value.ColumnType); 79 | } 80 | return row; 81 | } 82 | /// 83 | /// Add column to DataStructure 84 | /// 85 | /// Column name 86 | /// Column type 87 | public void AddColumn(string columnName, ColumnType type) 88 | { 89 | if (ColumnType.stringType == type) 90 | { 91 | DataColumn column = new DataColumn(type); 92 | DataValuesByColumn.Add(columnName, column); 93 | } 94 | else 95 | { 96 | DataColumn column = new DataColumn(type); 97 | DataValuesByColumn.Add(columnName, column); 98 | } 99 | } 100 | 101 | /// 102 | /// Add block of values 103 | /// 104 | /// 105 | public void AddBlock(Dictionary dataBlock) 106 | { 107 | foreach (var column in dataBlock) 108 | { 109 | DataValuesByColumn[column.Key].AddBlock(column.Value); 110 | } 111 | } 112 | } 113 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/DataModels/DataColumn.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCoreServer.Models.DataModels 4 | { 5 | public enum ColumnType 6 | { 7 | stringType = 0, 8 | doubleType, 9 | dateType 10 | } 11 | 12 | public class DataColumn 13 | { 14 | public ColumnType ColumnType { get; set; } 15 | public List Values { get; } 16 | 17 | public T this[int index] 18 | { 19 | get 20 | { 21 | return Values[index]; 22 | } 23 | } 24 | 25 | public DataColumn(ColumnType type) 26 | { 27 | ColumnType = type; 28 | Values = new List(); 29 | } 30 | 31 | public void Add(T value) 32 | { 33 | Values.Add(value); 34 | } 35 | 36 | public void AddBlock(List column) 37 | { 38 | Values.AddRange(column); 39 | } 40 | 41 | public int Count() 42 | { 43 | return Values.Count; 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/DataModels/DataSliceModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.DataModels; 2 | using NetCoreServer.Models.Fields; 3 | using NetCoreServer.Models.Select; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | 8 | namespace NetCoreServer.Models 9 | { 10 | public class DataSlice 11 | { 12 | private int _indexesCount; 13 | public int[] DataColumnIndexes { get; set; } 14 | 15 | public static IDataStructure Data { get; set; } 16 | private static Dictionary _columnNameTypes; 17 | 18 | public DataSlice(IDataStructure data) 19 | { 20 | Data = data; 21 | _columnNameTypes = Data.GetNameAndTypes(); 22 | var firstNameAndType = _columnNameTypes.First(); 23 | int dataVauesCount = 0; 24 | if (firstNameAndType.Value == ColumnType.stringType) 25 | { 26 | dataVauesCount = Data.GetColumn(firstNameAndType.Key).Count(); 27 | } 28 | else 29 | { 30 | dataVauesCount = Data.GetColumn(firstNameAndType.Key).Count(); 31 | } 32 | DataColumnIndexes = new int[dataVauesCount]; 33 | _indexesCount = 0; 34 | for (int i = 0; i < dataVauesCount; i++) 35 | { 36 | DataColumnIndexes[i] = i; 37 | _indexesCount++; 38 | } 39 | } 40 | 41 | public DataSlice(int[] indexes, int count) 42 | { 43 | DataColumnIndexes = indexes; 44 | _indexesCount = count; 45 | } 46 | 47 | /// 48 | /// Filters data 49 | /// 50 | /// filters to apply 51 | public void FilterData(List filters) 52 | { 53 | if (filters.Count == 0) 54 | { 55 | return; 56 | } 57 | foreach (var filter in filters) 58 | { 59 | filter.Field.Type = _columnNameTypes[filter.Field.UniqueName]; 60 | if (filter.Value == null) 61 | { 62 | ParallelQuery indexes = DataColumnIndexes.AsParallel(); 63 | if (filter.Include != null) 64 | { 65 | if (filter.Field.Type == ColumnType.stringType) 66 | { 67 | CheckIncludeFilter(Data.GetColumn(filter.Field.UniqueName), filter.Include, ref indexes); 68 | } 69 | else 70 | { 71 | CheckIncludeFilter(Data.GetColumn(filter.Field.UniqueName), filter.Include, ref indexes); 72 | } 73 | } 74 | else if (filter.Exclude != null) 75 | { 76 | if (filter.Field.Type == ColumnType.stringType) 77 | { 78 | CheckExcludeFilter(Data.GetColumn(filter.Field.UniqueName), filter.Exclude, ref indexes); 79 | } 80 | else 81 | { 82 | CheckExcludeFilter(Data.GetColumn(filter.Field.UniqueName), filter.Exclude, ref indexes); 83 | } 84 | } 85 | else if (filter.Query != null) 86 | { 87 | if (filter.Field.Type == ColumnType.doubleType) 88 | { 89 | var column = Data.GetColumn(filter.Field.UniqueName); 90 | indexes = indexes.Where(index => CheckNumberFilterQuery(column[index], filter.Query)); 91 | } 92 | else if (filter.Field.Type == ColumnType.stringType) 93 | { 94 | var column = Data.GetColumn(filter.Field.UniqueName); 95 | indexes = indexes.Where(index => CheckStringFilterQuery(column[index], filter.Query)); 96 | } 97 | else if (filter.Field.Type == ColumnType.dateType) 98 | { 99 | var column = Data.GetColumn(filter.Field.UniqueName); 100 | indexes = indexes.Where(index => CheckDateFilterQuery(column[index], filter.Query)); 101 | } 102 | } 103 | DataColumnIndexes = indexes.ToArray(); 104 | } 105 | 106 | if (filter.Value != null) 107 | { 108 | filter.Value.Field.Type = _columnNameTypes[filter.Value.Field.UniqueName]; 109 | var calculatedTotalsAggregation = new List(); 110 | CalcByFields(new List { filter.Field }, null, new List { filter.Value }, ref calculatedTotalsAggregation); 111 | var calculatedTotals = new Dictionary(); 112 | foreach (var agg in calculatedTotalsAggregation) 113 | { 114 | calculatedTotals.Add(agg.Keys[filter.Field.UniqueName], agg.Values[filter.Value.Field.UniqueName][filter.Value.Func]); 115 | } 116 | CheckValueFilterQuery(ref calculatedTotals, filter); 117 | if (filter.Field.Type == ColumnType.stringType) 118 | { 119 | var column = Data.GetColumn(filter.Field.UniqueName); 120 | DataColumnIndexes = DataColumnIndexes.Where(index => calculatedTotals.ContainsKey(column[index])).ToArray(); 121 | } 122 | else 123 | { 124 | var column = Data.GetColumn(filter.Field.UniqueName); 125 | DataColumnIndexes = DataColumnIndexes.Where(index => calculatedTotals.ContainsKey(column[index])).ToArray(); 126 | } 127 | } 128 | } 129 | } 130 | 131 | /// 132 | /// Check if calculated data meets filter's requirement 133 | /// 134 | /// Calculated data with given agregation 135 | /// 136 | private void CheckValueFilterQuery(ref Dictionary calculatedTotals, Filter filter) 137 | { 138 | if (filter.Query.ContainsKey("top")) 139 | { 140 | calculatedTotals = calculatedTotals.OrderByDescending(elem => elem.Value) 141 | .Take((int)filter.Query["top"].NumberValue).ToDictionary(elem => elem.Key, elem => elem.Value); 142 | } 143 | else if (filter.Query.ContainsKey("bottom")) 144 | { 145 | calculatedTotals = calculatedTotals.OrderBy(elem => elem.Value) 146 | .Take((int)filter.Query["bottom"].NumberValue).ToDictionary(elem => elem.Key, elem => elem.Value); 147 | } 148 | else if (filter.Query.ContainsKey("equal")) 149 | { 150 | calculatedTotals = calculatedTotals.Where(elem => elem.Value == filter.Query["equal"].NumberValue) 151 | .ToDictionary(elem => elem.Key, elem => elem.Value); 152 | } 153 | else if (filter.Query.ContainsKey("not_equal")) 154 | { 155 | calculatedTotals = calculatedTotals.Where(elem => elem.Value != filter.Query["not_equal"].NumberValue) 156 | .ToDictionary(elem => elem.Key, elem => elem.Value); 157 | } 158 | else if (filter.Query.ContainsKey("greater")) 159 | { 160 | calculatedTotals = calculatedTotals.Where(elem => elem.Value > filter.Query["greater"].NumberValue) 161 | .ToDictionary(elem => elem.Key, elem => elem.Value); 162 | } 163 | else if (filter.Query.ContainsKey("greater_equal")) 164 | { 165 | calculatedTotals = calculatedTotals.Where(elem => elem.Value >= filter.Query["greater_equal"].NumberValue) 166 | .ToDictionary(elem => elem.Key, elem => elem.Value); 167 | } 168 | else if (filter.Query.ContainsKey("less")) 169 | { 170 | calculatedTotals = calculatedTotals.Where(elem => elem.Value < filter.Query["less"].NumberValue) 171 | .ToDictionary(elem => elem.Key, elem => elem.Value); 172 | } 173 | else if (filter.Query.ContainsKey("less_equal")) 174 | { 175 | calculatedTotals = calculatedTotals.Where(elem => elem.Value <= filter.Query["less_equal"].NumberValue) 176 | .ToDictionary(elem => elem.Key, elem => elem.Value); 177 | } 178 | else if (filter.Query.ContainsKey("between")) 179 | { 180 | var from = filter.Query["between"].ListNumberValue[0]; 181 | var to = filter.Query["between"].ListNumberValue[1]; 182 | calculatedTotals = calculatedTotals.Where(elem => elem.Value >= from && elem.Value <= to) 183 | .ToDictionary(elem => elem.Key, elem => elem.Value); 184 | } 185 | else if (filter.Query.ContainsKey("not_between")) 186 | { 187 | var from = filter.Query["not_between"].ListNumberValue[0]; 188 | var to = filter.Query["not_between"].ListNumberValue[1]; 189 | calculatedTotals = calculatedTotals.Where(elem => elem.Value < from || elem.Value > to) 190 | .ToDictionary(elem => elem.Key, elem => elem.Value); 191 | } 192 | } 193 | 194 | /// 195 | /// Checks whether the data item present in include filter 196 | /// 197 | /// data column 198 | /// filter to apply 199 | /// index of data items present in slice 200 | private void CheckIncludeFilter(DataColumn dataColumn, List filter, ref ParallelQuery indexes) 201 | { 202 | if (dataColumn.ColumnType == ColumnType.doubleType || dataColumn.ColumnType == ColumnType.dateType) 203 | { 204 | var numberFilter = filter.Select(value => value?.Member?.NumberValue); 205 | indexes = indexes.Where(index => numberFilter.Contains(dataColumn[index] as double?)); 206 | } 207 | else if (dataColumn.ColumnType == ColumnType.stringType) 208 | { 209 | var stringFilter = filter.Select(value => value?.Member?.StringValue); 210 | indexes = indexes.Where(index => stringFilter.Contains(dataColumn[index] as string)); 211 | } 212 | } 213 | /// 214 | /// Checks whether the data item present in exclude filter 215 | /// 216 | /// data column 217 | /// filter to apply 218 | /// index of data items present in slice 219 | private void CheckExcludeFilter(DataColumn dataColumn, List filter, ref ParallelQuery indexes) 220 | { 221 | if (dataColumn.ColumnType == ColumnType.doubleType || dataColumn.ColumnType == ColumnType.dateType) 222 | { 223 | var numberFilter = filter.Select(value => value?.Member?.NumberValue); 224 | indexes = indexes.Where(index => !numberFilter.Contains(dataColumn[index] as double?)); 225 | } 226 | else if (dataColumn.ColumnType == ColumnType.stringType) 227 | { 228 | var stringFilter = filter.Select(value => value?.Member?.StringValue == "" ? null : value?.Member?.StringValue); 229 | indexes = indexes.Where(index => !stringFilter.Contains(dataColumn[index] as string)); 230 | } 231 | } 232 | 233 | /// 234 | /// Checks whether the DateTime meets the query condition 235 | /// 236 | /// DateTime data value 237 | /// Query object 238 | private bool CheckDateFilterQuery(double? dateElementValue, Dictionary query) 239 | { 240 | if (query.ContainsKey("equal")) 241 | { 242 | var date = query["equal"].NumberValue; 243 | return date.Equals(dateElementValue); 244 | } 245 | if (query.ContainsKey("not_equal")) 246 | { 247 | var date = query["not_equal"].NumberValue; 248 | return !date.Equals(dateElementValue); 249 | } 250 | if (query.ContainsKey("after")) 251 | { 252 | var date = query["after"].NumberValue; 253 | return dateElementValue?.CompareTo(date) > 0; 254 | } 255 | if (query.ContainsKey("after_equal")) 256 | { 257 | var date = query["after_equal"].NumberValue; 258 | return dateElementValue?.CompareTo(date) >= 0; 259 | } 260 | if (query.ContainsKey("before")) 261 | { 262 | var date = query["before"].NumberValue; 263 | return dateElementValue?.CompareTo(date) < 0; 264 | } 265 | if (query.ContainsKey("before_equal")) 266 | { 267 | var date = query["before_equal"].NumberValue; 268 | return dateElementValue?.CompareTo(date) <= 0; 269 | } 270 | if (query.ContainsKey("between")) 271 | { 272 | var v1 = query["between"].ListNumberValue[0]; 273 | var v2 = query["between"].ListNumberValue[1]; 274 | return dateElementValue >= v1 && dateElementValue <= v2; 275 | } 276 | if (query.ContainsKey("not_between")) 277 | { 278 | var v1 = query["not_between"].ListNumberValue[0]; 279 | var v2 = query["not_between"].ListNumberValue[1]; 280 | return dateElementValue < v1 || dateElementValue > v2; 281 | } 282 | return false; 283 | } 284 | 285 | /// 286 | /// Checks whether the number value meets the query condition 287 | /// 288 | /// Number value(store as double) 289 | /// Query object 290 | private bool CheckNumberFilterQuery(double? numberElementValue, Dictionary query) 291 | { 292 | if (query.ContainsKey("equal")) 293 | { 294 | return numberElementValue == query["equal"].NumberValue; 295 | } 296 | if (query.ContainsKey("not_equal")) 297 | { 298 | return numberElementValue != query["not_equal"].NumberValue; 299 | } 300 | if (query.ContainsKey("greater")) 301 | { 302 | return numberElementValue > query["greater"].NumberValue; 303 | } 304 | if (query.ContainsKey("greater_equal")) 305 | { 306 | return numberElementValue >= query["greater_equal"].NumberValue; 307 | } 308 | if (query.ContainsKey("less")) 309 | { 310 | return numberElementValue < query["less"].NumberValue; 311 | } 312 | if (query.ContainsKey("less_equal")) 313 | { 314 | return numberElementValue <= query["less_equal"].NumberValue; 315 | } 316 | if (query.ContainsKey("between")) 317 | { 318 | var v1 = query["between"].ListNumberValue[0]; 319 | var v2 = query["between"].ListNumberValue[1]; 320 | return numberElementValue >= v1 && numberElementValue <= v2; 321 | } 322 | if (query.ContainsKey("not_between")) 323 | { 324 | var v1 = query["not_between"].ListNumberValue[0]; 325 | var v2 = query["not_between"].ListNumberValue[1]; 326 | return numberElementValue < v1 || numberElementValue > v2; 327 | } 328 | return false; 329 | } 330 | 331 | /// 332 | /// Checks whether the number value meets the query condition 333 | /// 334 | /// String value 335 | /// Query object 336 | private bool CheckStringFilterQuery(string stringElementValue, Dictionary query) 337 | { 338 | var dataElementValueToLower = stringElementValue.ToLower(); 339 | if (query.ContainsKey("equal")) 340 | { 341 | return dataElementValueToLower == query["equal"].StringValue.ToLower(); 342 | } 343 | if (query.ContainsKey("not_equal")) 344 | { 345 | return dataElementValueToLower != query["not_equal"].StringValue.ToLower(); 346 | } 347 | if (query.ContainsKey("begin")) 348 | { 349 | return dataElementValueToLower.StartsWith(query["begin"].StringValue.ToLower()); 350 | } 351 | if (query.ContainsKey("not_begin")) 352 | { 353 | return !dataElementValueToLower.StartsWith(query["not_begin"].StringValue.ToLower()); 354 | } 355 | if (query.ContainsKey("end")) 356 | { 357 | return dataElementValueToLower.EndsWith(query["end"].StringValue.ToLower()); 358 | } 359 | if (query.ContainsKey("not_end")) 360 | { 361 | return !dataElementValueToLower.EndsWith(query["not_end"].StringValue.ToLower()); 362 | } 363 | if (query.ContainsKey("contain")) 364 | { 365 | return dataElementValueToLower.Contains(query["contain"].StringValue.ToLower()); 366 | } 367 | if (query.ContainsKey("not_contain")) 368 | { 369 | return !dataElementValueToLower.Contains(query["not_contain"].StringValue.ToLower()); 370 | } 371 | if (query.ContainsKey("greater")) 372 | { 373 | return dataElementValueToLower.CompareTo(query["greater"].StringValue.ToLower()) > 0; 374 | } 375 | if (query.ContainsKey("greater_equal")) 376 | { 377 | return dataElementValueToLower.CompareTo(query["greater_equal"].StringValue.ToLower()) >= 0; 378 | } 379 | if (query.ContainsKey("less")) 380 | { 381 | return dataElementValueToLower.CompareTo(query["less"].StringValue.ToLower()) < 0; 382 | } 383 | if (query.ContainsKey("less_equal")) 384 | { 385 | return dataElementValueToLower.CompareTo(query["less_equal"].StringValue.ToLower()) <= 0; 386 | } 387 | if (query.ContainsKey("between")) 388 | { 389 | var v1 = query["between"].ListStringValue[0].ToLower(); 390 | var v2 = query["between"].ListStringValue[1].ToLower(); 391 | return dataElementValueToLower.CompareTo(v1) >= 0 && dataElementValueToLower.CompareTo(v2) <= 0; 392 | } 393 | if (query.ContainsKey("not_between")) 394 | { 395 | var v1 = query["not_between"].ListStringValue[0].ToLower(); 396 | var v2 = query["not_between"].ListStringValue[1].ToLower(); 397 | return dataElementValueToLower.CompareTo(v1) < 0 || dataElementValueToLower.CompareTo(v2) > 0; 398 | } 399 | return false; 400 | } 401 | 402 | /// 403 | /// Groups data by fields and calculates data values. Works recursively 404 | /// 405 | /// All fields to group by 406 | /// Fields in columns to group by 407 | /// Values to calcluate 408 | /// Output response 409 | /// Key-value pairs that describes specific tuple 410 | public void CalcByFields(List fields, List fieldsInColumns, List values, 411 | ref List response, Dictionary keys = null) 412 | { 413 | if (fields.Count < 1) 414 | { 415 | return; 416 | } 417 | var field = fields[0]; 418 | var fieldsSkipped = fields.Skip(1).ToList(); 419 | var groupsByField = GroupBy(field.UniqueName, field.Type); 420 | foreach (var group in groupsByField) 421 | { 422 | var subdata = new DataSlice(group.Value.ToArray(), _indexesCount); 423 | var item = subdata.CalcValues(values); 424 | if (item.Values.Count != 0) 425 | { 426 | item.Keys = keys != null ? new Dictionary(keys) : new Dictionary(); 427 | item.Keys.Add(field.UniqueName, group.Key); 428 | response.Add(item); 429 | subdata.CalcByFields(fieldsSkipped, fieldsInColumns, values, ref response, item.Keys); 430 | } 431 | } 432 | 433 | if ((fieldsInColumns != null) && fieldsInColumns.Count > 0 && fields.Count > fieldsInColumns.Count) 434 | { 435 | var colField = fieldsInColumns[0]; 436 | var colsSkipped = fieldsInColumns.Skip(1).ToList(); 437 | var colGroupsByField = GroupBy(colField.UniqueName, colField.Type); 438 | foreach (var group in colGroupsByField) 439 | { 440 | var subdata = new DataSlice(group.Value.ToArray(), _indexesCount); 441 | var item = subdata.CalcValues(values); 442 | if (item.Values.Count != 0) 443 | { 444 | item.Keys = keys != null ? new Dictionary(keys) : new Dictionary(); 445 | item.Keys.Add(colField.UniqueName, group.Key); 446 | response.Add(item); 447 | subdata.CalcByFields(colsSkipped, null, values, ref response, item.Keys); 448 | } 449 | } 450 | } 451 | } 452 | 453 | /// 454 | /// Groups data by 455 | /// 456 | /// Field's name 457 | /// Field's type 458 | /// Collection of groups 459 | private Dictionary> GroupBy(string fieldName, ColumnType type) 460 | { 461 | Dictionary> groups = new Dictionary>(); 462 | if (type == ColumnType.stringType) 463 | { 464 | var column = Data.GetColumn(fieldName); 465 | foreach (var index in DataColumnIndexes) 466 | { 467 | var value = column[index] == null ? "" : column[index]; 468 | if (!groups.ContainsKey(value)) 469 | { 470 | groups.Add(value, new List()); 471 | } 472 | groups[value].Add(index); 473 | } 474 | } 475 | else if (type == ColumnType.doubleType || type == ColumnType.dateType) 476 | { 477 | var column = Data.GetColumn(fieldName); 478 | foreach (var index in DataColumnIndexes) 479 | { 480 | object value = null; 481 | if (column[index] != null) 482 | { 483 | value = column[index]; 484 | } 485 | else 486 | { 487 | value = ""; 488 | } 489 | if (!groups.ContainsKey(value)) 490 | { 491 | groups.Add(value, new List()); 492 | } 493 | groups[value].Add(index); 494 | } 495 | } 496 | return groups; 497 | } 498 | 499 | /// 500 | /// Calculates aggregated values 501 | /// 502 | /// Values and its aggregation functions to calculate 503 | /// All calculated aggregations by given field 504 | public Aggregation CalcValues(List values) 505 | { 506 | Aggregation response = new Aggregation 507 | { 508 | Values = new Dictionary>() 509 | }; 510 | values.ForEach( 511 | valrequest => 512 | { 513 | var calcValue = CalcValue(valrequest.Field, valrequest.Func); 514 | if (!double.IsNaN(calcValue)) 515 | { 516 | if (!response.Values.ContainsKey(valrequest.Field.UniqueName)) 517 | { 518 | response.Values.Add(valrequest.Field.UniqueName, new Dictionary()); 519 | } 520 | if (!response.Values[valrequest.Field.UniqueName].ContainsKey(valrequest.Func)) 521 | { 522 | response.Values[valrequest.Field.UniqueName].Add(valrequest.Func, calcValue); 523 | } 524 | } 525 | }); 526 | return response; 527 | } 528 | 529 | /// 530 | /// Calculates aggregated value for specific field 531 | /// 532 | /// field 533 | /// aggregation name 534 | /// Calculated aggregation 535 | private double CalcValue(FieldModel field, string func) 536 | { 537 | if (field.Type == ColumnType.stringType) 538 | { 539 | var validDataColumnIndexes = DataColumnIndexes.Where(index => Data.GetColumn(field.UniqueName)[index] != null).DefaultIfEmpty(-1).ToArray(); 540 | if (validDataColumnIndexes[0] == -1) 541 | { 542 | return 0; 543 | } 544 | if (func == "count") 545 | { 546 | return validDataColumnIndexes.Count(); 547 | } 548 | if (func == "distinctcount") 549 | { 550 | return validDataColumnIndexes.Select(index => Data.GetColumn(field.UniqueName)[index]).Distinct().ToList().Count; 551 | } 552 | } 553 | if (field.Type == ColumnType.doubleType || field.Type == ColumnType.dateType) 554 | { 555 | var validDataColumnIndexes = DataColumnIndexes.Where(index => Data.GetColumn(field.UniqueName)[index].HasValue).DefaultIfEmpty(-1).ToArray(); 556 | var column = Data.GetColumn(field.UniqueName); 557 | if (validDataColumnIndexes[0] == -1) 558 | { 559 | return double.NaN; 560 | } 561 | if (func == "count") 562 | { 563 | return validDataColumnIndexes.Count(); 564 | } 565 | if (func == "distinctcount") 566 | { 567 | return validDataColumnIndexes.Select(index => column[index]).Distinct().ToList().Count; 568 | } 569 | if (func == "sum" || func == "none") 570 | { 571 | return validDataColumnIndexes.Sum(index => column[index].Value); 572 | } 573 | if (func == "average") 574 | { 575 | return validDataColumnIndexes.Average(index => column[index].Value); 576 | } 577 | if (func == "min") 578 | { 579 | return validDataColumnIndexes.Min(index => column[index].Value); 580 | } 581 | if (func == "max") 582 | { 583 | return validDataColumnIndexes.Max(index => column[index].Value); 584 | } 585 | } 586 | return 0; 587 | } 588 | } 589 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/DataModels/IDataStructure.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCoreServer.Models.DataModels 4 | { 5 | public interface IDataStructure 6 | { 7 | void AddColumn(string column, ColumnType type); 8 | 9 | DataColumn GetColumn(string columnName); 10 | 11 | List GetColumnNames(); 12 | 13 | public void AddBlock(Dictionary dataBlock); 14 | 15 | Dictionary GetNameAndTypes(); 16 | } 17 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/FieldsRequest/FieldModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.DataModels; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models.Fields 5 | { 6 | public class FieldModel 7 | { 8 | public string UniqueName { get; set; } 9 | public ColumnType Type { get; set; } 10 | public string Caption { get; set; } 11 | public string Folder { get; set; } 12 | public List Aggregations { get; set; } 13 | public SchemaFilterElement Filter { get; set; } 14 | 15 | public FieldModel() 16 | { 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/FieldsRequest/FieldsRequestModel.cs: -------------------------------------------------------------------------------- 1 | namespace NetCoreServer.Models.Fields 2 | { 3 | public enum RequestType 4 | { 5 | Fields, 6 | Members, 7 | Select, 8 | Handshake 9 | } 10 | 11 | public class FieldsRequest 12 | { 13 | public FieldsRequest() 14 | { 15 | } 16 | 17 | public FieldsRequest(string index, RequestType type) 18 | { 19 | Index = index; 20 | type = Type; 21 | } 22 | 23 | public string Index { get; set; } 24 | 25 | public RequestType Type { get; set; } 26 | } 27 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/FieldsRequest/SchemaAggregation.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCoreServer.Models.Fields 4 | { 5 | public class SchemaAggregation 6 | { 7 | public List Any { get; set; } 8 | 9 | public List Date { get; set; } 10 | 11 | public List Number { get; set; } 12 | 13 | public List String { get; set; } 14 | 15 | public SchemaAggregation() 16 | { 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/FieldsRequest/SchemaFilter.cs: -------------------------------------------------------------------------------- 1 | namespace NetCoreServer.Models.Fields 2 | { 3 | public class SchemaFilter 4 | { 5 | public SchemaFilterElement Any { get; set; } 6 | 7 | public SchemaFilter() 8 | { 9 | Any = new SchemaFilterElement(); 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/FieldsRequest/SchemaFilterElement.cs: -------------------------------------------------------------------------------- 1 | namespace NetCoreServer.Models.Fields 2 | { 3 | public class SchemaFilterElement 4 | { 5 | public bool Members { get; set; } 6 | public bool Query { get; set; } 7 | public bool ValueQuery { get; set; } 8 | 9 | public SchemaFilterElement() 10 | { 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/FieldsRequest/SchemaModel.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCoreServer.Models.Fields 4 | { 5 | public class Schema 6 | { 7 | public List Fields { get; set; } 8 | 9 | public SchemaAggregation Aggregations { get; set; } 10 | 11 | public SchemaFilter Filters { get; set; } 12 | 13 | public bool Sorted { get; set; } 14 | 15 | public Schema() 16 | { 17 | Aggregations = new SchemaAggregation(); 18 | Filters = new SchemaFilter(); 19 | Fields = new List(); 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/HandshakeRequest/HandshakeRequst.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | 3 | namespace NetCoreServer.Models.Handshake 4 | { 5 | public class HandshakeRequst 6 | { 7 | public string Version { get; set; } 8 | public RequestType Type { get; set; } 9 | } 10 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/MembersRequest/MembersRequestModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | using NetCoreServer.Models.Select; 3 | using System.Collections.Generic; 4 | 5 | namespace NetCoreServer.Models.Members 6 | { 7 | public class MembersRequest 8 | { 9 | public MembersRequest() 10 | { 11 | } 12 | 13 | public MembersRequest(string index, RequestType type, FieldModel field, int page) 14 | { 15 | Index = index; 16 | Type = type; 17 | Field = field; 18 | Page = page; 19 | } 20 | 21 | public string Index { get; set; } 22 | 23 | public RequestType Type { get; set; } 24 | 25 | public FieldModel Field { get; set; } 26 | 27 | public int Page { get; set; } 28 | } 29 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/MembersRequest/MembersResponseModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models.Members 5 | { 6 | public class MembersResponse 7 | { 8 | public MembersResponse() 9 | { 10 | Members = new List(); 11 | } 12 | 13 | public List Members { get; set; } 14 | 15 | public bool Sorted { get; set; } 16 | 17 | public int Page { get; set; } 18 | 19 | public int PageTotal { get; set; } 20 | } 21 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/AggregationBy.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models.Select 5 | { 6 | public class AggregationBy 7 | { 8 | public AggregationBy() 9 | { 10 | Rows = new List(); 11 | Cols = new List(); 12 | } 13 | 14 | public List Rows { get; set; } 15 | 16 | public List Cols { get; set; } 17 | } 18 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/AggregationModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models.Select 5 | { 6 | public class Aggregation 7 | { 8 | public Aggregation() 9 | { 10 | } 11 | 12 | public Dictionary> Values { get; set; } 13 | 14 | public Dictionary Keys { get; set; } 15 | } 16 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/AggregationRequestModel.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCoreServer.Models.Select 4 | { 5 | public class AggregationRequest 6 | { 7 | public AggregationRequest() 8 | { 9 | } 10 | 11 | public AggregationBy By { get; set; } 12 | 13 | public List Values { get; set; } 14 | } 15 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/FieldFuncValue.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | 3 | namespace NetCoreServer.Models.Select 4 | { 5 | public class FieldFuncValue 6 | { 7 | public FieldFuncValue() 8 | { 9 | } 10 | 11 | public FieldModel Field { get; set; } 12 | 13 | public string Func { get; set; } 14 | } 15 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/FilterModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models.Select 5 | { 6 | public class Filter 7 | { 8 | public FieldModel Field { get; set; } 9 | 10 | public string FieldType { get; set; } 11 | 12 | public List Include { get; set; } 13 | 14 | public List Exclude { get; set; } 15 | 16 | public Dictionary Query { get; set; } 17 | 18 | public FieldFuncValue Value { get; set; } 19 | 20 | public Filter() 21 | { 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/HierarchyObject.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace NetCoreServer.Models.Select 7 | { 8 | public class HierarchyObject 9 | { 10 | public ValueObject Member { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/QueryModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models.Select 5 | { 6 | public class Query 7 | { 8 | public Query() 9 | { 10 | } 11 | 12 | public List Fields { get; set; } 13 | 14 | public AggregationRequest Aggs { get; set; } 15 | 16 | public List Filter { get; set; } 17 | 18 | public int Limit { get; set; } 19 | } 20 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/SelectRequestModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | 3 | namespace NetCoreServer.Models.Select 4 | { 5 | public class SelectRequest 6 | { 7 | public SelectRequest() 8 | { 9 | } 10 | 11 | public SelectRequest(string index, RequestType type, Query query, int page) 12 | { 13 | Index = index; 14 | Type = type; 15 | Query = query; 16 | Page = page; 17 | } 18 | 19 | public string Index { get; set; } 20 | 21 | public RequestType Type { get; set; } 22 | 23 | public Query Query { get; set; } 24 | 25 | public int Page { get; set; } 26 | } 27 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/SelectRequest/SelectResponceModel.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.Fields; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models.Select 5 | { 6 | public class SelectResponse 7 | { 8 | public SelectResponse() 9 | { 10 | Fields = new List(); 11 | } 12 | 13 | public List Fields { get; set; } 14 | 15 | public List> Hits { get; set; } 16 | 17 | public List Aggs { get; set; } 18 | 19 | public int Page { get; set; } 20 | 21 | public int PageTotal { get; set; } 22 | } 23 | } -------------------------------------------------------------------------------- /server-dotnetcore/Models/Values/ValueModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Models 5 | { 6 | public class ValueObject 7 | { 8 | public ValueObject() 9 | { 10 | } 11 | 12 | public ValueObject(List value) 13 | { 14 | ListStringValue = value; 15 | } 16 | 17 | public ValueObject(List value) 18 | { 19 | ListNumberValue = value; 20 | } 21 | 22 | public ValueObject(string value) 23 | { 24 | StringValue = value; 25 | } 26 | 27 | public ValueObject(double value) 28 | { 29 | NumberValue = value; 30 | } 31 | 32 | public List ListStringValue { get; set; } 33 | 34 | public List ListNumberValue { get; set; } 35 | 36 | public string StringValue { get; set; } 37 | 38 | public double? NumberValue { get; set; } 39 | } 40 | } -------------------------------------------------------------------------------- /server-dotnetcore/NetCoreServer.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | Flexmonster 6 | 2.8.0 7 | Sample server that implement Flexmonster custom data source API. 8 | 9 | 10 | 11 | 3 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /server-dotnetcore/Parsers/CSVParser.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Extensions; 2 | using NetCoreServer.Models; 3 | using NetCoreServer.Models.DataModels; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.Linq; 8 | 9 | namespace NetCoreServer.Parsers 10 | { 11 | public class CSVParser : IParser 12 | { 13 | private readonly CSVSerializerOptions _serializerOptions; 14 | private readonly string _fullFilePath; 15 | private const ushort CHUNK_SIZE = ushort.MaxValue; 16 | 17 | private Dictionary dataBlock; 18 | private int index; 19 | private Dictionary _columnsNames; 20 | private Dictionary _dataTypes; 21 | 22 | public Dictionary DataTypes 23 | { 24 | get 25 | { 26 | return _dataTypes; 27 | } 28 | } 29 | 30 | public CSVParser(string path, CSVSerializerOptions serializerOptions) 31 | { 32 | _fullFilePath = path; 33 | _columnsNames = new Dictionary(); 34 | _dataTypes = new Dictionary(); 35 | _serializerOptions = serializerOptions; 36 | } 37 | 38 | public IEnumerable> Parse() 39 | { 40 | using (StreamReader reader = File.OpenText(_fullFilePath)) 41 | { 42 | string line = ""; 43 | index = 0; 44 | string headerLine = reader.ReadLine(); 45 | List readingChunk = new List(CHUNK_SIZE); 46 | 47 | // first lines are required to detect data types 48 | for (int i = 0; i < 1; i++) 49 | { 50 | line = reader.ReadLine(); 51 | if (line != null) readingChunk.Add(line); 52 | index++; 53 | } 54 | // parse headers 55 | ParseHeader(headerLine, readingChunk); 56 | 57 | while ((line = reader.ReadLine()) != null) 58 | { 59 | line = line.Trim(); 60 | if (line.Length > 0) 61 | { 62 | readingChunk.Add(line); 63 | index++; 64 | if (index % CHUNK_SIZE == 0) 65 | { 66 | ParseBlock(readingChunk); 67 | readingChunk = new List(); 68 | yield return dataBlock; 69 | 70 | var dataBlockColumns = dataBlock.Keys.ToList(); 71 | int i = 0; 72 | foreach (var columnName in dataBlockColumns) 73 | { 74 | if (_dataTypes[columnName] == ColumnType.stringType) 75 | { 76 | dataBlock[columnName] = new List(); 77 | } 78 | else 79 | { 80 | dataBlock[columnName] = new List(); 81 | } 82 | i++; 83 | } 84 | } 85 | } 86 | } 87 | 88 | ParseBlock(readingChunk); 89 | readingChunk = new List(); 90 | yield return dataBlock; 91 | } 92 | } 93 | 94 | private void ParseHeader(string header, List firstLines) 95 | { 96 | var columnNames = header.Split(_serializerOptions.FieldSeparator); 97 | string[][] lines = new string[firstLines.Count][]; 98 | int i = 0; 99 | firstLines.ForEach(line => 100 | { 101 | lines[i] = line.Split(_serializerOptions.FieldSeparator); 102 | i++; 103 | }); 104 | dataBlock = new Dictionary(); 105 | i = 0; 106 | foreach (var columnName in columnNames) 107 | { 108 | List column = new List(firstLines.Count); 109 | for (int j = 0; j < firstLines.Count; j++) 110 | { 111 | column.Add(lines[j][i]); 112 | } 113 | _dataTypes.Add(columnName, DetectType(column)); 114 | _columnsNames.Add(i, columnName); 115 | if (_dataTypes[columnName] == ColumnType.stringType) 116 | { 117 | dataBlock.Add(columnName, new List()); 118 | } 119 | else 120 | { 121 | dataBlock.Add(columnName, new List()); 122 | } 123 | i++; 124 | } 125 | } 126 | 127 | private ColumnType DetectType(List column) 128 | { 129 | int dateCount = 0; 130 | int numberCount = 0; 131 | int stringCount = 0; 132 | int emptyCount = 0; 133 | foreach (var value in column) 134 | { 135 | if (DateTime.TryParse(value, out _)) 136 | { 137 | dateCount++; 138 | } 139 | else if (Double.TryParse(value, out _)) 140 | { 141 | numberCount++; 142 | } 143 | else if (value != "") 144 | { 145 | stringCount++; 146 | } 147 | else 148 | { 149 | emptyCount++; 150 | } 151 | } 152 | if (dateCount >= stringCount) 153 | { 154 | if (dateCount >= numberCount) 155 | { 156 | return ColumnType.dateType; 157 | } 158 | } 159 | if (stringCount > 0) 160 | { 161 | return ColumnType.stringType; 162 | } 163 | if (numberCount > 0) 164 | { 165 | return ColumnType.doubleType; 166 | } 167 | return ColumnType.stringType; 168 | } 169 | 170 | private void ParseBlock(List lines) 171 | { 172 | lines.ForEach(line => 173 | { 174 | ParseLine(line); 175 | }); 176 | } 177 | 178 | private void ParseLine(string line) 179 | { 180 | bool isQuote = false; 181 | char char_; 182 | string value = ""; 183 | int length = line.Length; 184 | int currentWord = 0; 185 | for (int i = 0; i < length; i++) 186 | { 187 | char_ = line[i]; 188 | if (char_ == _serializerOptions.FieldEnclosureToken) 189 | { 190 | isQuote = !isQuote; 191 | } 192 | else 193 | if (char_ == _serializerOptions.FieldSeparator && !isQuote) 194 | { 195 | if (_dataTypes[_columnsNames[currentWord]] == ColumnType.doubleType) 196 | { 197 | if (double.TryParse(value, out double convertedValue)) 198 | dataBlock[_columnsNames[currentWord]].Add(convertedValue); 199 | else 200 | dataBlock[_columnsNames[currentWord]].Add((double?)null); 201 | } 202 | else if (_dataTypes[_columnsNames[currentWord]] == ColumnType.dateType) 203 | { 204 | if (DateTime.TryParse(value, out DateTime convertedValue)) 205 | dataBlock[_columnsNames[currentWord]].Add(convertedValue.ToUnixTimestamp()); 206 | else 207 | dataBlock[_columnsNames[currentWord]].Add((double?)null); 208 | } 209 | else 210 | { 211 | if (value != "") 212 | dataBlock[_columnsNames[currentWord]].Add(value); 213 | else 214 | dataBlock[_columnsNames[currentWord]].Add((string)null); 215 | } 216 | currentWord++; 217 | value = ""; 218 | } 219 | else 220 | { 221 | value += char_; 222 | } 223 | } 224 | if (_dataTypes[_columnsNames[currentWord]] == ColumnType.doubleType) 225 | { 226 | if (double.TryParse(value, out double convertedValue)) 227 | dataBlock[_columnsNames[currentWord]].Add(convertedValue); 228 | else 229 | dataBlock[_columnsNames[currentWord]].Add((double?)null); 230 | } 231 | else if (_dataTypes[_columnsNames[currentWord]] == ColumnType.dateType) 232 | { 233 | if (DateTime.TryParse(value, out DateTime convertedValue)) 234 | dataBlock[_columnsNames[currentWord]].Add(convertedValue.ToUnixTimestamp()); 235 | else 236 | dataBlock[_columnsNames[currentWord]].Add((double?)null); 237 | } 238 | else 239 | { 240 | if (value != "") 241 | dataBlock[_columnsNames[currentWord]].Add(value); 242 | else 243 | dataBlock[_columnsNames[currentWord]].Add((string)null); 244 | } 245 | if (currentWord + 1 < dataBlock.Count) 246 | { 247 | for (int i = currentWord + 1; i < dataBlock.Count; i++) 248 | { 249 | if (_dataTypes[_columnsNames[i]] == ColumnType.doubleType) 250 | { 251 | dataBlock[_columnsNames[i]].Add((double?)null); 252 | } 253 | else 254 | { 255 | dataBlock[_columnsNames[i]].Add((string)null); 256 | } 257 | } 258 | } 259 | } 260 | } 261 | } -------------------------------------------------------------------------------- /server-dotnetcore/Parsers/CSVSerializerOptions.cs: -------------------------------------------------------------------------------- 1 | namespace NetCoreServer.Parsers 2 | { 3 | public class CSVSerializerOptions 4 | { 5 | private char _fieldEnclosureToken; 6 | 7 | public char FieldEnclosureToken 8 | { 9 | get 10 | { 11 | return this._fieldEnclosureToken; 12 | } 13 | set 14 | { 15 | this._fieldEnclosureToken = value; 16 | } 17 | } 18 | 19 | private char _fieldSeparator; 20 | 21 | public char FieldSeparator 22 | { 23 | get 24 | { 25 | return this._fieldSeparator; 26 | } 27 | set 28 | { 29 | this._fieldSeparator = value; 30 | } 31 | } 32 | 33 | public CSVSerializerOptions(char fieldEnclosureToken = '"', char fieldSeparator = ',') 34 | { 35 | _fieldEnclosureToken = fieldEnclosureToken; 36 | _fieldSeparator = fieldSeparator; 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /server-dotnetcore/Parsers/DatabaseParser.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Extensions; 2 | using NetCoreServer.Models; 3 | using NetCoreServer.Models.DataModels; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Data; 7 | 8 | namespace NetCoreServer.Parsers 9 | { 10 | public class DatabaseParser : IParser 11 | { 12 | private const ushort CHUNK_SIZE = ushort.MaxValue; 13 | private readonly IDataReader _dataReader; 14 | 15 | private Dictionary _columnNames; 16 | private Dictionary> _columnAddActionMap; 17 | 18 | private Dictionary _dataBlock; 19 | 20 | public DatabaseParser(IDataReader dataReader) 21 | { 22 | _dataReader = dataReader; 23 | _columnNames = new Dictionary(); 24 | _dataTypes = new Dictionary(); 25 | _columnAddActionMap = new Dictionary>(); 26 | } 27 | 28 | private Dictionary _dataTypes; 29 | 30 | public Dictionary DataTypes 31 | { 32 | get 33 | { 34 | return _dataTypes; 35 | } 36 | } 37 | 38 | public IEnumerable> Parse() 39 | { 40 | _dataBlock = new Dictionary(); 41 | object[][] readingChunk = new object[CHUNK_SIZE][]; 42 | int chunckPosition = 0; 43 | for (int i = 0; i < _dataReader.FieldCount; i++) 44 | { 45 | var columnName = _dataReader.GetName(i); 46 | var columnType = _dataReader.GetFieldType(i); 47 | _columnAddActionMap.Add(i, DetectType(columnType, out ColumnType dataColumnType)); 48 | _columnNames.Add(i, columnName); 49 | _dataTypes.Add(columnName, dataColumnType); 50 | if (dataColumnType == ColumnType.stringType) 51 | { 52 | _dataBlock.Add(columnName, new List()); 53 | } 54 | else 55 | { 56 | _dataBlock.Add(columnName, new List()); 57 | } 58 | } 59 | while (_dataReader.Read()) 60 | { 61 | object[] values = new object[_dataReader.FieldCount]; 62 | _dataReader.GetValues(values); 63 | 64 | readingChunk[chunckPosition] = values; 65 | chunckPosition++; 66 | if (chunckPosition == CHUNK_SIZE) 67 | { 68 | ParseBlock(readingChunk); 69 | readingChunk = new object[CHUNK_SIZE][]; 70 | chunckPosition = 0; 71 | yield return _dataBlock; 72 | foreach (var columnType in _dataTypes) 73 | { 74 | if (columnType.Value == ColumnType.stringType) 75 | { 76 | _dataBlock[columnType.Key] = new List(); 77 | } 78 | else 79 | { 80 | _dataBlock[columnType.Key] = new List(); 81 | } 82 | } 83 | } 84 | } 85 | _dataReader.Close(); 86 | _dataReader.Dispose(); 87 | ParseBlock(readingChunk); 88 | 89 | yield return _dataBlock; 90 | } 91 | 92 | private Action DetectType(Type columnType, out ColumnType dataColumnType) 93 | { 94 | switch (Type.GetTypeCode(columnType)) 95 | { 96 | case TypeCode.DateTime: 97 | { 98 | dataColumnType = ColumnType.dateType; 99 | return (value, dataColumn) => 100 | { 101 | if (value is DBNull) 102 | dataColumn.Add((double?)null); 103 | else 104 | dataColumn.Add(Convert.ToDateTime(value).ToUnixTimestamp()); 105 | }; 106 | } 107 | case TypeCode.Byte: 108 | case TypeCode.SByte: 109 | case TypeCode.UInt16: 110 | case TypeCode.UInt32: 111 | case TypeCode.UInt64: 112 | case TypeCode.Int16: 113 | case TypeCode.Int32: 114 | case TypeCode.Int64: 115 | case TypeCode.Decimal: 116 | case TypeCode.Double: 117 | case TypeCode.Single: 118 | { 119 | dataColumnType = ColumnType.doubleType; 120 | return (value, dataColumn) => 121 | { 122 | if (value is DBNull) 123 | dataColumn.Add((double?)null); 124 | else 125 | dataColumn.Add(Convert.ToDouble(value)); 126 | }; 127 | } 128 | default: 129 | { 130 | dataColumnType = ColumnType.stringType; 131 | return (value, dataColumn) => 132 | { 133 | dataColumn.Add(Convert.ToString(value)); 134 | }; 135 | } 136 | } 137 | } 138 | 139 | private void ParseBlock(object[][] rows) 140 | { 141 | var enumerator = rows.GetEnumerator(); 142 | while (enumerator.MoveNext()) 143 | { 144 | if (enumerator.Current != null) 145 | ParseRow(enumerator.Current as object[]); 146 | } 147 | } 148 | 149 | private void ParseRow(object[] values) 150 | { 151 | for (int i = 0; i < values.Length; i++) 152 | { 153 | _columnAddActionMap[i].Invoke(values[i], _dataBlock[_columnNames[i]]); 154 | } 155 | } 156 | } 157 | } -------------------------------------------------------------------------------- /server-dotnetcore/Parsers/IParser.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.Models.DataModels; 2 | using System.Collections.Generic; 3 | 4 | namespace NetCoreServer.Parsers 5 | { 6 | public interface IParser 7 | { 8 | /// 9 | /// Parse block of data 10 | /// 11 | IEnumerable> Parse(); 12 | 13 | Dictionary DataTypes { get; } 14 | } 15 | } -------------------------------------------------------------------------------- /server-dotnetcore/Parsers/JSONParser.cs: -------------------------------------------------------------------------------- 1 | using NetCoreServer.JsonConverters; 2 | using NetCoreServer.Models.DataModels; 3 | using System; 4 | using System.Buffers; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.Linq; 8 | using System.Text.Json; 9 | 10 | namespace NetCoreServer.Parsers 11 | { 12 | public class JSONParser : IParser 13 | { 14 | private const int CHUNK_SIZE = int.MaxValue / 4; 15 | private readonly string _fullFilePath; 16 | private readonly JsonSerializerOptions _serializerOptions; 17 | private byte[] _buffer = new byte[CHUNK_SIZE]; 18 | private Dictionary _dataBlock = new Dictionary(); 19 | private Lazy> _dataTypes; 20 | private int _offset = 0; 21 | private int inBuffer = 0; 22 | public JSONParser(string path, JsonSerializerOptions serializerOptions) 23 | { 24 | _fullFilePath = path; 25 | _serializerOptions = serializerOptions; 26 | _dataTypes = new Lazy>(() => (_serializerOptions.Converters[0] as DataJsonConverter).GetTypes()); 27 | } 28 | public Dictionary DataTypes 29 | { 30 | get 31 | { 32 | return _dataTypes.Value; 33 | } 34 | } 35 | 36 | public IEnumerable> Parse() 37 | { 38 | using (FileStream fs = new FileStream(_fullFilePath, FileMode.Open, FileAccess.Read)) 39 | { 40 | bool canRead = true; 41 | bool isFirst = true; 42 | while (canRead) 43 | { 44 | byte[] chunk = new byte[CHUNK_SIZE]; 45 | if (isFirst) 46 | { 47 | _offset = 0; 48 | } 49 | else 50 | { 51 | _offset = 1; 52 | chunk[0] = 91; 53 | while (inBuffer != 0) 54 | { 55 | inBuffer--; 56 | chunk[_offset] = _buffer[inBuffer]; 57 | _offset++; 58 | } 59 | } 60 | var count = fs.Read(chunk, _offset, CHUNK_SIZE - _offset - 1); 61 | isFirst = false; 62 | if (count < CHUNK_SIZE - _offset - 1) canRead = false; 63 | for (int i = count + _offset - 1; i > 0; i--) 64 | { 65 | if (chunk[i] != 125) 66 | { 67 | _buffer[inBuffer] = chunk[i]; 68 | inBuffer++; 69 | } 70 | else 71 | { 72 | chunk[i + 1] = 93; 73 | break; 74 | } 75 | } 76 | if (_offset > 0) 77 | { 78 | for (int i = 1; i < CHUNK_SIZE; i++) 79 | { 80 | if (chunk[i] != 123) 81 | { 82 | chunk[i] = 32; 83 | } 84 | else 85 | { 86 | break; 87 | } 88 | } 89 | } 90 | ReadOnlySequence bytesSpan = new ReadOnlySequence(chunk); 91 | Utf8JsonReader jsonReader = new Utf8JsonReader(bytesSpan); 92 | Dictionary dataRows = JsonSerializer.Deserialize>(ref jsonReader, _serializerOptions); 93 | yield return dataRows; 94 | } 95 | } 96 | yield return _dataBlock; 97 | } 98 | } 99 | } -------------------------------------------------------------------------------- /server-dotnetcore/PrepopulatingCacheService.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Options; 2 | using NetCoreServer.DataStorages; 3 | using System.Threading.Tasks; 4 | 5 | namespace NetCoreServer 6 | { 7 | public class PrepopulatingCacheService : IPrepopulatingService 8 | { 9 | private IDataStorage _dataStorage; 10 | private DatasourceOptions _options; 11 | 12 | /// 13 | /// Prepopulate cache with data from DatasourceOptions 14 | /// 15 | /// Storage where data will be loaded 16 | /// Data source options 17 | public PrepopulatingCacheService(IDataStorage dataStorage, IOptions options) 18 | { 19 | _dataStorage = dataStorage; 20 | _options = options.Value; 21 | } 22 | 23 | public async Task Prepopulate() 24 | { 25 | foreach (var index in _options.Indexes.Keys) 26 | { 27 | await _dataStorage.GetOrAddAsync(index); 28 | } 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /server-dotnetcore/Program.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Collections.Generic; 3 | using Microsoft.AspNetCore.Hosting; 4 | using Microsoft.Extensions.Hosting; 5 | using Microsoft.Extensions.Configuration; 6 | using System.Runtime.InteropServices; 7 | using System.Reflection; 8 | using Microsoft.Extensions.Logging.EventLog; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace NetCoreServer 12 | { 13 | public class Program 14 | { 15 | private static readonly Dictionary DEFAULTS = new Dictionary 16 | { 17 | { "settings", "appsettings.json" }, 18 | { "port", "3400" } 19 | }; 20 | private static readonly Dictionary ARGS_MAPPING = new Dictionary 21 | { 22 | { "-s", "settings" }, 23 | { "-p", "port" } 24 | }; 25 | public static void Main(string[] args) 26 | { 27 | 28 | CreateHostBuilder(args).Build().Run(); 29 | } 30 | public static IHostBuilder CreateHostBuilder(string[] args) 31 | { 32 | // loads settings from defauls and command line to find the path to the configuration file 33 | var configBuilder = new ConfigurationBuilder() 34 | .SetBasePath(Directory.GetCurrentDirectory()) 35 | .AddInMemoryCollection(DEFAULTS) 36 | .AddCommandLine(args, Program.ARGS_MAPPING); 37 | 38 | var configuration = configBuilder.Build(); 39 | string defaultConfigFilename = configuration.GetValue("settings"); 40 | 41 | // loads all settings, now including the configuration file 42 | configBuilder.Sources.Clear(); 43 | configBuilder.AddInMemoryCollection(DEFAULTS) 44 | .AddJsonFile(defaultConfigFilename) 45 | .AddCommandLine(args, Program.ARGS_MAPPING); 46 | configuration = configBuilder.Build(); 47 | var hostBuilder = new HostBuilder(); 48 | 49 | hostBuilder.UseContentRoot(Directory.GetCurrentDirectory()); 50 | hostBuilder.ConfigureHostConfiguration(config => 51 | { 52 | config.AddEnvironmentVariables(prefix: "DOTNET_"); 53 | if (args != null) 54 | { 55 | config.AddCommandLine(args); 56 | } 57 | }); 58 | 59 | hostBuilder.ConfigureAppConfiguration((hostingContext, configBuilder) => 60 | { 61 | IHostEnvironment env = hostingContext.HostingEnvironment; 62 | bool reloadOnChange = hostingContext.Configuration.GetValue("hostBuilder:reloadConfigOnChange", defaultValue: true); 63 | configBuilder.AddConfiguration(configuration); 64 | configBuilder.AddEnvironmentVariables(); 65 | if (args != null) 66 | { 67 | configBuilder.AddCommandLine(args); 68 | } 69 | }) 70 | .ConfigureWebHostDefaults(webBuilder => 71 | { 72 | webBuilder.UseStartup() 73 | .UseUrls("http://0.0.0.0:" + configuration.GetValue("port")); 74 | }) 75 | .ConfigureLogging((hostingContext, logging) => 76 | { 77 | bool isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); 78 | 79 | if (isWindows) 80 | { 81 | logging.AddFilter(level => level >= LogLevel.Warning); 82 | } 83 | 84 | logging.AddConfiguration(hostingContext.Configuration.GetSection("Logging")); 85 | logging.AddConsole(); 86 | logging.AddDebug(); 87 | logging.AddEventSourceLogger(); 88 | 89 | if (isWindows) 90 | { 91 | logging.AddEventLog(); 92 | } 93 | }) 94 | .UseDefaultServiceProvider((context, options) => 95 | { 96 | bool isDevelopment = context.HostingEnvironment.IsDevelopment(); 97 | options.ValidateScopes = isDevelopment; 98 | options.ValidateOnBuild = isDevelopment; 99 | }); 100 | return hostBuilder; 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /server-dotnetcore/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/launchsettings.json", 3 | "iisSettings": { 4 | "windowsAuthentication": false, 5 | "anonymousAuthentication": true, 6 | "iisExpress": { 7 | "applicationUrl": "http://localhost:13985", 8 | "sslPort": 44362 9 | } 10 | }, 11 | "profiles": { 12 | "IIS Express": { 13 | "commandName": "IISExpress", 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "NetCoreServer": { 19 | "commandName": "Project", 20 | "applicationUrl": "http://localhost:3400", 21 | "environmentVariables": { 22 | "ASPNETCORE_ENVIRONMENT": "Development" 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /server-dotnetcore/SettingsConfigurationHelper.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Configuration; 2 | using Microsoft.Extensions.DependencyInjection; 3 | using NetCoreServer.DataStorages; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Threading.Tasks; 8 | 9 | namespace NetCoreServer 10 | { 11 | public static class SettingsConfigurationHelper 12 | { 13 | /// 14 | /// Extension method for configuration options from IConfiguration 15 | /// 16 | /// 17 | /// 18 | public static void ConfigureFlexmonsterOptions(this IServiceCollection services, IConfiguration configuration) 19 | { 20 | services.Configure((options) => 21 | { 22 | options.Indexes = new Dictionary(); 23 | foreach (var datasourceConfig in configuration.GetSection("DataSources").GetChildren()) 24 | { 25 | if (datasourceConfig.GetSection("Type").Get() == "json") 26 | { 27 | foreach (var indexConfig in datasourceConfig.GetSection("Indexes").GetChildren()) 28 | { 29 | var jsonConfig = indexConfig.Get(); 30 | jsonConfig.Type = "json"; 31 | options.Indexes.Add(indexConfig.Key, jsonConfig); 32 | } 33 | } 34 | else if (datasourceConfig.GetSection("Type").Get() == "csv") 35 | { 36 | foreach (var indexConfig in datasourceConfig.GetSection("Indexes").GetChildren()) 37 | { 38 | var csvConfig = indexConfig.Get(); 39 | csvConfig.Type = "csv"; 40 | options.Indexes.Add(indexConfig.Key, csvConfig); 41 | } 42 | } 43 | else if (datasourceConfig.GetSection("Type").Get() == "database") 44 | { 45 | foreach (var indexConfig in datasourceConfig.GetSection("Indexes").GetChildren()) 46 | { 47 | var dbConfig = indexConfig.Get(); 48 | dbConfig.ConnectionString = datasourceConfig.GetSection("ConnectionString").Get(); 49 | dbConfig.DatabaseType = datasourceConfig.GetSection("DatabaseType").Get(); 50 | dbConfig.Type = "database"; 51 | options.Indexes.Add(indexConfig.Key, dbConfig); 52 | } 53 | } 54 | else 55 | { 56 | var indexOption = new IndexOptions() { Type = datasourceConfig.GetSection("Type").Get() }; 57 | foreach (var indexConfig in datasourceConfig.GetSection("Indexes").GetChildren()) 58 | { 59 | options.Indexes.Add(indexConfig.Key, indexOption); 60 | } 61 | } 62 | } 63 | }); 64 | services.Configure(configuration.GetSection("DataStorageOptions")); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /server-dotnetcore/Startup.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Builder; 2 | using Microsoft.AspNetCore.Hosting; 3 | using Microsoft.Extensions.Configuration; 4 | using Microsoft.Extensions.DependencyInjection; 5 | using Microsoft.Extensions.Hosting; 6 | using NetCoreServer.DataStorages; 7 | using NetCoreServer.JsonConverters; 8 | using System.Text.Json.Serialization; 9 | 10 | namespace NetCoreServer 11 | { 12 | public class Startup 13 | { 14 | public Startup(IConfiguration configuration) 15 | { 16 | Configuration = configuration; 17 | } 18 | 19 | private readonly string MyAllowSpecificOrigins = "_myAllowSpecificOrigins"; 20 | 21 | public IConfiguration Configuration { get; } 22 | 23 | public void ConfigureServices(IServiceCollection services) 24 | { 25 | services.AddCors(options => 26 | { 27 | options.AddPolicy(MyAllowSpecificOrigins, 28 | builder => 29 | { 30 | builder.AllowAnyOrigin().AllowAnyHeader().AllowAnyMethod(); 31 | }); 32 | }); 33 | services.ConfigureFlexmonsterOptions(Configuration); 34 | services.AddSingleton(); 35 | services.AddScoped(); 36 | services.AddControllers().AddJsonOptions(options => 37 | { 38 | options.JsonSerializerOptions.Converters.Add(new ValuesJsonConverter()); 39 | options.JsonSerializerOptions.Converters.Add(new ColumnTypeJsonConverter()); 40 | options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter()); 41 | }); 42 | services.AddMemoryCache((options) => 43 | { 44 | options.SizeLimit = 100; 45 | }); 46 | } 47 | 48 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IPrepopulatingService prepopulatingService) 49 | { 50 | if (env.IsDevelopment()) 51 | { 52 | app.UseDeveloperExceptionPage(); 53 | } 54 | 55 | app.UseRouting(); 56 | 57 | app.UseCors(MyAllowSpecificOrigins); 58 | 59 | app.UseAuthorization(); 60 | 61 | app.UseEndpoints(endpoints => 62 | { 63 | endpoints.MapControllers(); 64 | }); 65 | prepopulatingService.Prepopulate(); 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /server-dotnetcore/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "DataSources": [ 3 | { 4 | "Type": "json", 5 | "Indexes": { 6 | "data-types": { 7 | "Path": "./data/data-types.json" 8 | } 9 | } 10 | }, 11 | { 12 | "Type": "csv", 13 | "Indexes": { 14 | "fm-product-sales": { 15 | "Path": "./data/fm-product-sales.csv" 16 | }, 17 | "data": { 18 | "Path": "./data/data.csv" 19 | } 20 | } 21 | } 22 | ], 23 | "DataStorageOptions": { 24 | "DataRefreshTime": "60" 25 | } 26 | } -------------------------------------------------------------------------------- /server-dotnetcore/data/data-types.csv: -------------------------------------------------------------------------------- 1 | color,country,city,price,quantity,date,date_year,date_month 2 | red,Canada,Toronto,10,2,2019-07-30T00:00:00.000Z,2019,July 3 | red,Canada,Toronto,10,2,2019-08-01T00:00:00.000Z,2019,August 4 | blue,Canada,Toronto,18,3,2019-08-05T00:00:00.000Z,2019,August 5 | blue,Ukraine,Kyiv,20,4,2019-04-05T00:00:00.000Z,2019,April 6 | -------------------------------------------------------------------------------- /server-dotnetcore/data/data-types.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "color": "red", 4 | "country": "Canada", 5 | "city": "Toronto", 6 | "price": 10, 7 | "quantity": 2, 8 | "date": "2019-07-30T00:00:00.000Z", 9 | "date_year": "2019", 10 | "date_month": "July" 11 | }, 12 | { 13 | "color": "red", 14 | "country": "Canada", 15 | "city": "Toronto", 16 | "price": 10, 17 | "quantity": 2, 18 | "date": "2019-08-01T00:00:00.000Z", 19 | "date_year": "2019", 20 | "date_month": "August" 21 | }, 22 | { 23 | "color": "blue", 24 | "country": "Canada", 25 | "city": "Toronto", 26 | "price": 18, 27 | "quantity": 3, 28 | "date": "2019-08-05T00:00:00.000Z", 29 | "date_year": "2019", 30 | "date_month": "August" 31 | }, 32 | { 33 | "color": "blue", 34 | "country": "Ukraine", 35 | "city": "Kyiv", 36 | "price": 20, 37 | "quantity": 4, 38 | "date": "2019-04-05T00:00:00.000Z", 39 | "date_year": "2019", 40 | "date_month": "April" 41 | } 42 | ] -------------------------------------------------------------------------------- /server-nodejs/api/cube.js: -------------------------------------------------------------------------------- 1 | const cube = require('express').Router(); 2 | const fs = require('fs').promises; 3 | const _ = require('lodash'); 4 | const dataFolder = process.argv[3] || './data'; 5 | 6 | /** 7 | * API endpoints 8 | */ 9 | 10 | const API_VERSION = "2.9.0"; 11 | 12 | cube.post("/handshake", async (req, res) => { 13 | try { 14 | res.json({ version: API_VERSION }); 15 | } catch (err) { 16 | handleError(err, res); 17 | } 18 | }); 19 | 20 | cube.post("/fields", async (req, res) => { 21 | try { 22 | const result = await getFields(req.body.index); 23 | res.json(result); 24 | } catch (err) { 25 | handleError(err, res); 26 | } 27 | }); 28 | 29 | cube.post("/members", async (req, res) => { 30 | try { 31 | const result = await getMembers(req.body.index, req.body.field, req.body.page); 32 | res.json(result); 33 | } catch (err) { 34 | handleError(err, res); 35 | } 36 | }); 37 | 38 | cube.post("/select", async (req, res) => { 39 | try { 40 | const result = await getSelectResult(req.body.index, req.body.query, req.body.page); 41 | res.json(result); 42 | } catch (err) { 43 | handleError(err, res); 44 | } 45 | }); 46 | 47 | // throw an error on other endpoints 48 | cube.post("*", async (req, res) => { 49 | handleError(`Request type '${req.body.type}' is not implemented.`, res); 50 | }); 51 | 52 | /** 53 | * ============== 54 | * FIELDS REQUEST 55 | * ============== 56 | */ 57 | 58 | /** 59 | * Composes the index schema based on the data file from the data folder. 60 | * @param {string} index index name 61 | */ 62 | async function getFields(index) { 63 | if (!fieldsCache[index]) { 64 | const output = { 65 | "fields": [], 66 | "aggregations": { 67 | "any": ["count", "distinctcount"], 68 | "number": ["sum", "count", "distinctcount", "average", "min", "max"], 69 | "date": ["count", "distinctcount", "min", "max"], 70 | }, 71 | "filters": { 72 | "any": { 73 | "members": true, 74 | "query": true 75 | } 76 | } 77 | }; 78 | try { 79 | const fileContent = await fs.readFile(`${dataFolder}/${index}.json`); 80 | const data = JSON.parse(fileContent); 81 | const dataRow = data[0]; 82 | if (dataRow) { 83 | for (const fieldName in dataRow) { 84 | const value = dataRow[fieldName]; 85 | const type = resolveDataType(value); 86 | output.fields.push({ 87 | "uniqueName": fieldName, 88 | "caption": fieldName, 89 | "type": type, 90 | }) 91 | } 92 | } 93 | } catch (err) { 94 | const message = (index == null) 95 | ? "Index property is missing." 96 | : `Index with name "${index}" was not found.`; 97 | throw URIError(message); 98 | } 99 | fieldsCache[index] = output; 100 | } 101 | return fieldsCache[index]; 102 | } 103 | const fieldsCache = {}; 104 | 105 | function resolveDataType(value) { 106 | if (typeof value == "number") { 107 | return "number"; 108 | } 109 | if (typeof value == "string" && value.length >= 10 /* minimal ISO date */ && !isNaN(Date.parse(value))) { 110 | return "date"; 111 | } 112 | return "string"; 113 | } 114 | 115 | /** 116 | * Gets field type by unique name from the index schema. 117 | * @param {object} fields the index schema 118 | * @param {string} fieldName field unique name 119 | */ 120 | function getFieldType(fields, fieldName) { 121 | for (let i = 0; i < fields.fields.length; i++) { 122 | if (fieldName == fields.fields[i].uniqueName) { 123 | return fields.fields[i].type; 124 | } 125 | } 126 | return undefined; 127 | } 128 | 129 | /** 130 | * =============== 131 | * MEMBERS REQUEST 132 | * =============== 133 | */ 134 | 135 | const MEMBERS_PAGE_SIZE = 5000; 136 | 137 | /** 138 | * Gets field members. 139 | * @param {string} index index name 140 | * @param {string} field field object 141 | * @param {number} page page number to load 142 | */ 143 | async function getMembers(index, field, page) { 144 | const data = await getData(index); 145 | const fields = await getFields(index); 146 | const fieldName = field.uniqueName; 147 | const fieldType = getFieldType(fields, fieldName); 148 | const output = { 149 | members: [] 150 | }; 151 | const members = _.uniq(_.map(data, fieldName)); 152 | if (checkMonths(members)) { // custom sort for months 153 | members.sort(monthCompare); 154 | output.sorted = true; 155 | } 156 | page = isNaN(page) ? 0 : page; 157 | const pageTotal = Math.ceil(members.length / MEMBERS_PAGE_SIZE); 158 | if (pageTotal > 1) { 159 | output.page = page; 160 | output.pageTotal = Math.ceil(members.length / MEMBERS_PAGE_SIZE); 161 | } 162 | const from = page * MEMBERS_PAGE_SIZE; 163 | const size = Math.min(members.length, from + MEMBERS_PAGE_SIZE); 164 | for (let i = from; i < size; i++) { 165 | output.members.push(createMember(members[i], fieldType)); 166 | } 167 | return output; 168 | } 169 | 170 | function createMember(value, fieldType) { 171 | if (value || value === 0 || value === false || value === NaN) { 172 | return { 173 | value: value 174 | } 175 | } 176 | else { 177 | return { 178 | value: "" 179 | } 180 | } 181 | } 182 | 183 | const monthOrder = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; 184 | const shortMonthOrder = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; 185 | 186 | /** 187 | * Checks whether the members are months. 188 | * @param {Array} members array of a field's members 189 | */ 190 | function checkMonths(members) { 191 | if (typeof members === 'undefined' || members === null || typeof members.length === 'undefined' || members.length === null) return false; 192 | if (monthOrder.indexOf(members[0]) != -1) return true; 193 | if (shortMonthOrder.indexOf(members[0]) != -1) return true; 194 | return false; 195 | } 196 | 197 | function monthCompare(a, b) { 198 | if (monthOrder.indexOf(a) > -1 || monthOrder.indexOf(b) > -1) { 199 | return monthOrder.indexOf(a) - monthOrder.indexOf(b); 200 | } 201 | return shortMonthOrder.indexOf(a) - shortMonthOrder.indexOf(b); 202 | } 203 | 204 | /** 205 | * ============== 206 | * SELECT REQUEST 207 | * ============== 208 | */ 209 | 210 | const SELECT_PAGE_SIZE = 5000; 211 | 212 | /** 213 | * Gets select data. 214 | * @param {string} index index name 215 | * @param {object} query query object 216 | * @param {number} page page number 217 | */ 218 | async function getSelectResult(index, query, page) { 219 | const output = {}; 220 | let data = await getData(index); 221 | if (query.filter) { 222 | const fields = await getFields(index); 223 | for (const filter of query.filter) { 224 | filter["field"].type = getFieldType(fields, filter["field"].uniqueName); 225 | } 226 | data = filterData(data, query.filter); 227 | } 228 | if (query.aggs && query.aggs.values) { 229 | output.aggs = []; 230 | if (query.aggs.by) { 231 | const rows = query.aggs.by.rows || []; 232 | const cols = query.aggs.by.cols || []; 233 | if (data.length != 0) { 234 | calcByFields(data, rows.concat(cols), cols, query.aggs.values, output.aggs); // data 235 | output.aggs.push(calcValues(data, query.aggs.values)); // grand totals 236 | } 237 | } else { // only grand totals 238 | output.aggs.push(calcValues(data, query.aggs.values)); 239 | } 240 | page = isNaN(page) ? 0 : page; 241 | const pageTotal = Math.ceil(output.aggs.length / SELECT_PAGE_SIZE); 242 | if (pageTotal > 1) { 243 | output.page = page; 244 | output.pageTotal = Math.ceil(output.aggs.length / SELECT_PAGE_SIZE); 245 | const from = page * SELECT_PAGE_SIZE; 246 | const size = Math.min(output.aggs.length, from + SELECT_PAGE_SIZE); 247 | output.aggs = output.aggs.slice(from, size); 248 | } 249 | } 250 | if (query.fields) { 251 | output.fields = query.fields.map(function (f) { 252 | return { 253 | "uniqueName": f.uniqueName 254 | }; 255 | }); 256 | output.hits = []; 257 | const limit = isNaN(query.limit) ? data.length : Math.min(query.limit, data.length); 258 | for (let i = 0; i < limit; i++) { 259 | const row = query.fields.map(f => { 260 | const value = data[i][f.uniqueName] 261 | if (value === undefined || value === null) { 262 | return ""; 263 | } else { 264 | return value; 265 | } 266 | }); 267 | output.hits.push(row) 268 | } 269 | page = isNaN(page) ? 0 : page; 270 | const pageTotal = Math.ceil(output.hits.length / SELECT_PAGE_SIZE); 271 | if (pageTotal > 1) { 272 | output.page = page; 273 | output.pageTotal = Math.ceil(output.hits.length / SELECT_PAGE_SIZE); 274 | const from = page * SELECT_PAGE_SIZE; 275 | const size = Math.min(output.hits.length, from + SELECT_PAGE_SIZE); 276 | output.hits = output.hits.slice(from, size); 277 | } 278 | } 279 | return output; 280 | } 281 | 282 | /** 283 | * Filters data. 284 | * @param {Array} data input data 285 | * @param {Array} filters filters to apply 286 | */ 287 | function filterData(data, filters) { 288 | if (filters.length == 0) { 289 | return data; 290 | } 291 | return _.filter(data, d => { 292 | for (const filter of filters) { 293 | const check = checkFilter(d, filter); 294 | if (!check) { 295 | return false; 296 | } 297 | } 298 | return true; 299 | }); 300 | } 301 | 302 | /** 303 | * Checks whether the data item meets the filter query. 304 | * @param {object} item data item 305 | * @param {object} filter filter object 306 | */ 307 | function checkFilter(item, filter) { 308 | let check = true; 309 | const fieldName = filter["field"].uniqueName; 310 | const fieldType = filter["field"].type; 311 | let value = item[fieldName]; 312 | if (value === undefined || value === null) { 313 | value = ""; 314 | } 315 | if (filter["include"]) { 316 | check = false; 317 | for (let i = 0; i < filter["include"].length; i++) { 318 | if (value === filter["include"][i].member) { 319 | check = true; 320 | break; 321 | } 322 | } 323 | } else if (filter["exclude"]) { 324 | for (let i = 0; i < filter["exclude"].length; i++) { 325 | if (value === filter["exclude"][i].member) { 326 | check = false; 327 | break; 328 | } 329 | } 330 | } else if (filter["query"]) { 331 | const query = filter["query"]; 332 | if (fieldType == "date") { 333 | check = checkDateFilterQuery(value, query); 334 | } else if (fieldType == "number") { 335 | check = checkNumberFilterQuery(value, query); 336 | } else { 337 | check = checkStringFilterQuery(value, query); 338 | } 339 | } 340 | return check; 341 | } 342 | 343 | const MS_DAY = 24 * 60 * 60 * 1000; 344 | 345 | /** 346 | * Checks whether the timestamp meets the query condition. 347 | * @param {number} value Unix timestamp 348 | * @param {object} query query object 349 | */ 350 | function checkDateFilterQuery(value, query) { 351 | if (query["equal"] !== undefined) { 352 | var d = query["equal"]; 353 | return value - d >= 0 && value - d < MS_DAY; 354 | } 355 | if (query["not_equal"] !== undefined) { 356 | var d = query["not_equal"]; 357 | return value < d || value >= d + MS_DAY; 358 | } 359 | if (query["after"] !== undefined) { 360 | var d = query["after"]; 361 | return value >= d + MS_DAY; 362 | } 363 | if (query["after_equal"] !== undefined) { 364 | var d = query["after_equal"]; 365 | return value >= d; 366 | } 367 | if (query["before"] !== undefined) { 368 | var d = query["before"]; 369 | return value < d; 370 | } 371 | if (query["before_equal"] !== undefined) { 372 | var d = query["before_equal"]; 373 | return value < d + MS_DAY; 374 | } 375 | if (query["between"] !== undefined) { 376 | var d1 = query["between"][0]; 377 | var d2 = query["between"][1]; 378 | return value >= d1 && value <= d2; 379 | } 380 | if (query["not_between"] !== undefined) { 381 | var d1 = query["not_between"][0]; 382 | var d2 = query["not_between"][1]; 383 | return value < d1 || value > d2; 384 | } 385 | return false; 386 | } 387 | 388 | /** 389 | * Checks whether the string value meets the query condition. 390 | * @param {string} value string value 391 | * @param {object} query query object 392 | */ 393 | function checkStringFilterQuery(value, query) { 394 | value = String(value).toLowerCase(); 395 | if (query["equal"] !== undefined) { 396 | return value == String(query["equal"]).toLowerCase(); 397 | } 398 | if (query["not_equal"] !== undefined) { 399 | return value != String(query["not_equal"]).toLowerCase(); 400 | } 401 | if (query["begin"] !== undefined) { 402 | return value.indexOf(String(query["begin"]).toLowerCase()) == 0; 403 | } 404 | if (query["not_begin"] !== undefined) { 405 | return value.indexOf(String(query["not_begin"]).toLowerCase()) != 0; 406 | } 407 | if (query["end"] !== undefined) { 408 | const idx = value.lastIndexOf(String(query["end"]).toLowerCase()) 409 | return idx >= 0 && idx + query["end"].length == value.length; 410 | } 411 | if (query["not_end"] !== undefined) { 412 | const idx = value.lastIndexOf(String(query["not_end"]).toLowerCase()) 413 | return idx < 0 || idx + query["not_end"].length != value.length; 414 | } 415 | if (query["contain"] !== undefined) { 416 | const idx = value.indexOf(String(query["contain"]).toLowerCase()) 417 | return idx >= 0; 418 | } 419 | if (query["not_contain"] !== undefined) { 420 | const idx = value.indexOf(String(query["not_contain"]).toLowerCase()) 421 | return idx < 0; 422 | } 423 | if (query["greater"] !== undefined) { 424 | return value > String(query["greater"]).toLowerCase(); 425 | } 426 | if (query["greater_equal"] !== undefined) { 427 | return value >= String(query["greater_equal"]).toLowerCase(); 428 | } 429 | if (query["less"] !== undefined) { 430 | return value < String(query["less"]).toLowerCase(); 431 | } 432 | if (query["less_equal"] !== undefined) { 433 | return value <= String(query["less_equal"]).toLowerCase(); 434 | } 435 | if (query["between"] !== undefined) { 436 | var v1 = String(query["between"][0]).toLowerCase(); 437 | var v2 = String(query["between"][1]).toLowerCase(); 438 | return value >= v1 && value <= v2; 439 | } 440 | if (query["not_between"] !== undefined) { 441 | var v1 = String(query["not_between"][0]).toLowerCase(); 442 | var v2 = String(query["not_between"][1]).toLowerCase() 443 | return value < v1 || value > v2; 444 | } 445 | return false; 446 | } 447 | 448 | /** 449 | * Checks whether the numeric value meets the query condition. 450 | * @param {number} value numeric value 451 | * @param {object} query query object 452 | */ 453 | function checkNumberFilterQuery(value, query) { 454 | if (query["equal"] !== undefined) { 455 | return value === query["equal"]; 456 | } 457 | if (query["not_equal"] !== undefined) { 458 | return value !== query["not_equal"]; 459 | } 460 | if (value === "") { 461 | return false; 462 | } 463 | if (query["greater"] !== undefined) { 464 | return value > query["greater"]; 465 | } 466 | if (query["greater_equal"] !== undefined) { 467 | return value >= query["greater_equal"]; 468 | } 469 | if (query["less"] !== undefined) { 470 | return value < query["less"]; 471 | } 472 | if (query["less_equal"] !== undefined) { 473 | return value <= query["less_equal"]; 474 | } 475 | if (query["between"] !== undefined) { 476 | return value >= query["between"][0] && value <= query["between"][1]; 477 | } 478 | if (query["not_between"] !== undefined) { 479 | return value < query["not_between"][0] || value > query["not_between"][1]; 480 | } 481 | return false; 482 | } 483 | 484 | const groupBy = (array, keyName) => 485 | array.reduce((objectsByKeyValue, obj) => { 486 | let valueObject = obj[keyName]; 487 | if (valueObject === undefined || valueObject === null) { 488 | valueObject = ""; 489 | } 490 | objectsByKeyValue[valueObject] = objectsByKeyValue[valueObject] || {}; 491 | objectsByKeyValue[valueObject].key = valueObject; 492 | objectsByKeyValue[valueObject].values = objectsByKeyValue[valueObject].values || []; 493 | objectsByKeyValue[valueObject].values.push(obj); 494 | return objectsByKeyValue; 495 | }, {}); 496 | 497 | /** 498 | * Groups data by fields and calculates numberic data. Works recursively. 499 | * @param {object[]} data input data 500 | * @param {string[]} fields all fields to group by 501 | * @param {string[]} cols fields in columns to group by 502 | * @param {object[]} values values to calcluate 503 | * @param {object[]} output output response 504 | * @param {object} keys key-value pairs that describes specific tuple 505 | */ 506 | function calcByFields(data, fields, cols, values, output, keys) { 507 | if (fields.length < 1) { 508 | return; 509 | } 510 | const fieldName = fields[0].uniqueName; 511 | const subfields = fields.slice(1); 512 | const groups = groupBy(data, fieldName); 513 | for (const index in groups) { 514 | const key = groups[index].key; 515 | const subdata = groups[index].values; 516 | const item = calcValues(subdata, values); 517 | item.keys = keys ? _.clone(keys) : {}; 518 | item.keys[fieldName] = key; 519 | output.push(item); 520 | calcByFields(subdata, subfields, cols, values, output, item.keys); 521 | } 522 | // column totals 523 | if (cols && cols.length > 0 && fields.length > cols.length) { 524 | const colFieldName = cols[0].uniqueName; 525 | const subCols = cols.slice(1); 526 | const colGroups = groupBy(data, colFieldName); 527 | for (const index in colGroups) { 528 | const key = colGroups[index].key; 529 | const subdata = colGroups[index].values; 530 | const item = calcValues(subdata, values); 531 | item.keys = keys ? _.clone(keys) : {}; 532 | item.keys[colFieldName] = key; 533 | output.push(item); 534 | calcByFields(subdata, subCols, null, values, output, item.keys); 535 | } 536 | } 537 | } 538 | 539 | /** 540 | * Calculates aggregated values. 541 | * @param {object[]} data input data 542 | * @param {object} values values to calculate 543 | */ 544 | function calcValues(data, values) { 545 | const output = { 546 | values: {} 547 | }; 548 | for (const value of values) { 549 | const fieldName = value.field.uniqueName; 550 | if (!output.values[fieldName]) { 551 | output.values[fieldName] = {}; 552 | } 553 | output.values[fieldName][value.func] = calcValue(data, fieldName, value.func); 554 | } 555 | return output; 556 | } 557 | 558 | /** 559 | * Calculates aggregated value for the specific field. 560 | * @param {object} data input data 561 | * @param {string} fieldName field's name 562 | * @param {string} func aggregation name 563 | */ 564 | function calcValue(data, fieldName, func) { 565 | if (func == "sum" || func == "none") { 566 | return _.sumBy(data, fieldName); 567 | } 568 | if (func == "count") { 569 | return _.filter(data, (value) => { 570 | return typeof value[fieldName] == "number" || typeof value[fieldName] == "string"; 571 | }).length; 572 | } 573 | if (func == "distinctcount") { 574 | let notEmptyData = _.filter(data, (value) => { 575 | return typeof value[fieldName] == "number" || typeof value[fieldName] == "string"; 576 | }); 577 | return _.uniqBy(notEmptyData, fieldName).length; 578 | } 579 | if (func == "average") { 580 | return calcAverage(data, fieldName); 581 | } 582 | if (func == "min") { 583 | return _.minBy(data, fieldName)[fieldName]; 584 | } 585 | if (func == "max") { 586 | return _.maxBy(data, fieldName)[fieldName]; 587 | } 588 | return NaN; 589 | } 590 | 591 | /** 592 | * Calculates average value for the specific field. 593 | * @param {object} data input data 594 | * @param {string} fieldName field's name 595 | */ 596 | function calcAverage(data, fieldName) { 597 | let count = 0; 598 | let sum = 0; 599 | for (let i = 0; i < data.length; i++) { 600 | const value = data[i][fieldName]; 601 | if (isNaN(value) || typeof value != "number") { 602 | continue; 603 | } 604 | sum += value; 605 | count++; 606 | } 607 | return sum / count; 608 | } 609 | 610 | /** 611 | * Gets index raw data. Reads the `.json` file from the data folder. 612 | * @param {string} index index name 613 | */ 614 | async function getData(index) { 615 | if (!dataCache[index]) { 616 | const dataRaw = await fs.readFile(`${dataFolder}/${index}.json`); 617 | const data = JSON.parse(dataRaw); 618 | parseDates(data); 619 | dataCache[index] = data; 620 | } 621 | return dataCache[index]; 622 | } 623 | const dataCache = {}; 624 | 625 | function parseDates(data) { 626 | const dateFields = []; 627 | const dataRow = data[0]; 628 | if (dataRow) { 629 | for (const fieldName in dataRow) { 630 | if (resolveDataType(dataRow[fieldName]) == "date") { 631 | dateFields.push(fieldName); 632 | } 633 | } 634 | } 635 | if (dateFields.length > 0) { 636 | for (let i = 0; i < data.length; i++) { 637 | for (const fieldName of dateFields) { 638 | data[i][fieldName] = Date.parse(data[i][fieldName]); 639 | } 640 | } 641 | } 642 | } 643 | 644 | function handleError(err, res, status) { 645 | if (!res) { 646 | throw "The second parameter is required"; 647 | } 648 | if (err instanceof URIError) { 649 | status = 400; 650 | } 651 | console.error(err); 652 | status = status || 500; 653 | var message = "Unknown server error."; 654 | if (typeof err == "string") { 655 | message = err; 656 | } else if (err.message) { 657 | message = err.message; 658 | } 659 | res.status(status).json({ 660 | message 661 | }); 662 | } 663 | 664 | module.exports = cube; -------------------------------------------------------------------------------- /server-nodejs/data/data-types.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "color": "red", 4 | "country": "Canada", 5 | "city": "Toronto", 6 | "price": 10, 7 | "quantity": 2, 8 | "date": "2019-07-30T00:00:00.000Z", 9 | "date_year": "2019", 10 | "date_month": "July" 11 | }, 12 | { 13 | "color": "red", 14 | "country": "Canada", 15 | "city": "Toronto", 16 | "price": 10, 17 | "quantity": 2, 18 | "date": "2019-08-01T00:00:00.000Z", 19 | "date_year": "2019", 20 | "date_month": "August" 21 | }, 22 | { 23 | "color": "blue", 24 | "country": "Canada", 25 | "city": "Toronto", 26 | "price": 18, 27 | "quantity": 3, 28 | "date": "2019-08-05T00:00:00.000Z", 29 | "date_year": "2019", 30 | "date_month": "August" 31 | }, 32 | { 33 | "color": "blue", 34 | "country": "Ukraine", 35 | "city": "Kyiv", 36 | "price": 20, 37 | "quantity": 4, 38 | "date": "2019-04-05T00:00:00.000Z", 39 | "date_year": "2019", 40 | "date_month": "April" 41 | } 42 | ] -------------------------------------------------------------------------------- /server-nodejs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flexmonster-api-server", 3 | "version": "2.9.0", 4 | "description": "Sample server that implements the Flexmonster custom data source API", 5 | "main": "server.js", 6 | "scripts": { 7 | "start": "node server.js" 8 | }, 9 | "keywords": [ 10 | "flexmonster" 11 | ], 12 | "author": "Flexmonster", 13 | "license": "ISC", 14 | "dependencies": { 15 | "body-parser": "^1.19.0", 16 | "cors": "^2.8.5", 17 | "express": "^4.17.1", 18 | "lodash": "^4.17.15", 19 | "moment": "^2.24.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /server-nodejs/server.js: -------------------------------------------------------------------------------- 1 | const express = require("express"); 2 | const cors = require('cors'); 3 | const bodyParser = require('body-parser'); 4 | 5 | const app = express(); 6 | app.use(cors()); 7 | app.use(bodyParser.json()); 8 | 9 | app.use('/api/cube', require('./api/cube.js')); 10 | 11 | const port = process.argv[2] || 3400; 12 | app.listen(port, () => { 13 | console.log(`Server is running on http://localhost:${port}`); 14 | }); -------------------------------------------------------------------------------- /tests/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "http://127.0.0.1:3400/api/cube", 3 | "emptyValue": "", 4 | "valueFilters": false 5 | } 6 | -------------------------------------------------------------------------------- /tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "testforcustomapi", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "author": "", 7 | "license": "ISC", 8 | "scripts": { 9 | "test": "mocha test/*Spec.js", 10 | "debug": "mocha --inspect-brk test/*Spec.js" 11 | }, 12 | "dependencies": { 13 | "axios": "^0.20.0", 14 | "chai": "^4.2.0", 15 | "mocha": "^8.1.3" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/test/FieldsRequestSpec.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios'); 2 | const expect = require('chai').expect; 3 | var config = require('../config.json'); 4 | 5 | const url = config.url + "/fields"; 6 | 7 | const requestBody = { "type": "fields", "index": "data" } 8 | 9 | describe('Fields request', function () { 10 | let promise; 11 | 12 | before(function () { 13 | promise = axios.post(url, requestBody); 14 | }); 15 | 16 | it('should return 200 status code', function (done) { 17 | promise.then(function (response) { 18 | expect(response.status).to.equal(200); 19 | done(); 20 | }) 21 | .catch(function (error) { 22 | done(error); 23 | }); 24 | }); 25 | 26 | it('response type check', function (done) { 27 | promise.then(function (response) { 28 | response.data.fields.forEach((item) => { 29 | expect(item).to.have.property("uniqueName").that.is.a("string"); 30 | expect(item).to.have.property("type").that.is.oneOf(["string", "number", "date"]); 31 | if (item.caption) { 32 | expect(item).to.have.property("caption").that.is.a("string"); 33 | } 34 | if (item.folder) { 35 | expect(item).to.have.property("folder").that.is.a("string"); 36 | } 37 | if (item.interval) { 38 | expect(item.type).to.be("date"); 39 | expect(item).to.have.property("interval").that.is.a("string"); 40 | } 41 | if (item.aggregations) { 42 | expect(item).to.have.property("aggregations").that.is.a("array"); 43 | expect(["sum", "count", "distinctcount", "average", "median", "product", 44 | "min", "max", "stdevp", "stdevs", "none"]).to.include.members(item.aggregations); 45 | } 46 | if (item.filters !== undefined) { 47 | expect(typeof item.filters).to.be.oneOf(["boolean", "object"]); 48 | expect(item.filters).not.to.be.a("array"); 49 | if (typeof item.filters === "object") { 50 | if (item.filters.members !== undefined) { 51 | expect(item.filters.members).to.be.a("boolean"); 52 | } 53 | if (item.filters.query !== undefined) { 54 | expect(item.filters.query).to.satisfy(isBooleanOrArray); 55 | if (Array.isArray(item.filters.query)) { 56 | if (item.type === "string") { 57 | expect(["equal", "not_equal", "begin", "not_begin", "end", "not_end", "contain", "not_contain", 58 | "greater", "greater_equal", "less", "less_equal", "between", "not_between"]).to.include.members(item.filters.query); 59 | } else if (item.type === "number") { 60 | expect(["equal", "not_equal", "greater", "greater_equal", "less", "less_equal", "between", 61 | "not_between"]).to.include.members(item.filters.query); 62 | } else if (item.type === "date") { 63 | expect(["equal", "not_equal", "before", "before_equal", "after", "after_equal", "between", 64 | "not_between", "last", "current", "next"]).to.include.members(item.filters.query); 65 | } 66 | } 67 | } 68 | if (item.filters.valueQuery !== undefined) { 69 | expect(item.filters.valueQuery).to.satisfy(isBooleanOrArray); 70 | if (Array.isArray(item.filters.valueQuery)) { 71 | expect(["top", "bottom", "equal", "not_equal", "greater", "greater_equal", "less", 72 | "less_equal", "between", "not_between"]).to.include.members(item.filters.valueQuery); 73 | } 74 | } 75 | } 76 | } 77 | }); 78 | 79 | if (response.data.aggregations) { 80 | expect(typeof response.data.aggregations).to.equal("object"); 81 | if (Array.isArray(response.data.aggregations)) { 82 | expect(["sum", "count", "distinctcount", "average", "median", "product", 83 | "min", "max", "stdevp", "stdevs", "none"]).to.include.members(response.data.aggregations); 84 | } else if (typeof response.data.aggregations === "object") { 85 | if (response.data.aggregations.any) { 86 | expect(["sum", "count", "distinctcount", "average", "median", "product", 87 | "min", "max", "stdevp", "stdevs", "none"]).to.include.members(response.data.aggregations.any); 88 | } 89 | if (response.data.aggregations.date) { 90 | expect(["sum", "count", "distinctcount", "average", "median", "product", 91 | "min", "max", "stdevp", "stdevs", "none"]).to.include.members(response.data.aggregations.date); 92 | } 93 | if (response.data.aggregations.number) { 94 | expect(["sum", "count", "distinctcount", "average", "median", "product", 95 | "min", "max", "stdevp", "stdevs", "none"]).to.include.members(response.data.aggregations.number); 96 | } 97 | if (response.data.aggregations.string) { 98 | expect(["sum", "count", "distinctcount", "average", "median", "product", 99 | "min", "max", "stdevp", "stdevs", "none"]).to.include.members(response.data.aggregations.string); 100 | } 101 | } 102 | } 103 | 104 | if (response.data.filters !== undefined) { 105 | expect(typeof response.data.filters).to.be.oneOf(["boolean", "object"]); 106 | expect(response.data.filters).not.to.be.a("array"); 107 | if (typeof response.data.filters === "object") { 108 | if (response.data.filters.any) { 109 | expect(typeof response.data.filters.any).to.be.oneOf(["boolean", "object"]); 110 | expect(response.data.filters.any).not.to.be.a("array"); 111 | if (typeof response.data.filters.any === "object") { 112 | if (response.data.filters.any.members !== undefined) { 113 | expect(response.data.filters.any.members).to.be.a("boolean"); 114 | } 115 | if (response.data.filters.any.query !== undefined) { 116 | expect(response.data.filters.any.query).to.satisfy(isBooleanOrArray); 117 | if (Array.isArray(response.data.filters.any.query)) { 118 | expect(["equal", "not_equal", "between", "not_between"]).to.include.members(response.data.filters.any.query); 119 | } 120 | } 121 | if (response.data.filters.any.valueQuery !== undefined) { 122 | expect(response.data.filters.any.valueQuery).to.satisfy(isBooleanOrArray); 123 | if (Array.isArray(response.data.filters.any.valueQuery)) { 124 | expect(["top", "bottom", "equal", "not_equal", "greater", "greater_equal", "less", 125 | "less_equal", "between", "not_between"]).to.include.members(iresponse.data.filters.any.valueQuery); 126 | } 127 | } 128 | } 129 | } 130 | if (response.data.filters.date) { 131 | expect(typeof response.data.filters.date).to.be.oneOf(["boolean", "object"]); 132 | expect(response.data.filters.date).not.to.be.a("array"); 133 | if (typeof response.data.filters.date === "object") { 134 | if (response.data.filters.date.members !== undefined) { 135 | expect(response.data.filters.date.members).to.be.a("boolean"); 136 | } 137 | if (response.data.filters.date.query !== undefined) { 138 | expect(response.data.filters.date.query).to.satisfy(isBooleanOrArray); 139 | if (Array.isArray(response.data.filters.date.query)) { 140 | expect(["equal", "not_equal", "before", "before_equal", "after", "after_equal", "between", 141 | "not_between", "last", "current", "next"]).to.include.members(response.data.filters.date.query); 142 | } 143 | } 144 | if (response.data.filters.date.valueQuery !== undefined) { 145 | expect(response.data.filters.date.valueQuery).to.satisfy(isBooleanOrArray); 146 | if (Array.isArray(response.data.filters.date.valueQuery)) { 147 | expect(["top", "bottom", "equal", "not_equal", "greater", "greater_equal", "less", 148 | "less_equal", "between", "not_between"]).to.include.members(response.data.filters.date.valueQuery); 149 | } 150 | } 151 | } 152 | } 153 | if (response.data.filters.number) { 154 | expect(typeof response.data.filters.number).to.be.oneOf(["boolean", "object"]); 155 | expect(response.data.filters.number).not.to.be.a("array"); 156 | if (typeof response.data.filters.number === "object") { 157 | if (response.data.filters.number.members !== undefined) { 158 | expect(response.data.filters.number.members).to.be.a("boolean"); 159 | } 160 | if (response.data.filters.number.query !== undefined) { 161 | expect(response.data.filters.number.query).to.satisfy(isBooleanOrArray); 162 | if (Array.isArray(response.data.filters.number.query)) { 163 | expect(["equal", "not_equal", "greater", "greater_equal", 164 | "less", "less_equal", "between", "not_between"]).to.include.members(response.data.filters.number.query); 165 | } 166 | } 167 | if (response.data.filters.number.valueQuery !== undefined) { 168 | expect(response.data.filters.number.valueQuery).to.satisfy(isBooleanOrArray); 169 | if (Array.isArray(response.data.filters.number.valueQuery)) { 170 | expect(["top", "bottom", "equal", "not_equal", "greater", "greater_equal", "less", 171 | "less_equal", "between", "not_between"]).to.include.members(response.data.filters.number.valueQuery); 172 | } 173 | } 174 | } 175 | } 176 | if (response.data.filters.string) { 177 | expect(typeof response.data.filters.string).to.be.oneOf(["boolean", "object"]); 178 | expect(response.data.filters.string).not.to.be.a("array"); 179 | if (typeof response.data.filters.string === "object") { 180 | if (response.data.filters.string.members !== undefined) { 181 | expect(response.data.filters.string.members).to.be.a("boolean"); 182 | } 183 | if (response.data.filters.string.query !== undefined) { 184 | expect(response.data.filters.string.query).to.satisfy(isBooleanOrArray); 185 | if (Array.isArray(response.data.filters.string.query)) { 186 | expect(["equal", "not_equal", "begin", "not_begin", "end", "not_end", "contain", 187 | "not_contain", "greater", "greater_equal", "less", "less_equal", "between", "not_between"]).to.include.members(response.data.filters.string.query); 188 | } 189 | } 190 | if (response.data.filters.string.valueQuery !== undefined) { 191 | expect(response.data.filters.string.valueQuery).to.satisfy(isBooleanOrArray); 192 | if (Array.isArray(response.data.filters.string.valueQuery)) { 193 | expect(["top", "bottom", "equal", "not_equal", "greater", "greater_equal", "less", 194 | "less_equal", "between", "not_between"]).to.include.members(response.data.filters.string.valueQuery); 195 | } 196 | } 197 | } 198 | } 199 | } 200 | } 201 | if (response.data.sorted !== undefined) { 202 | expect(response.data).to.have.property("sorted").that.is.a('boolean'); 203 | } 204 | done(); 205 | }) 206 | .catch(function (error) { 207 | done(error); 208 | }); 209 | }); 210 | 211 | 212 | function isBooleanOrArray(query) { 213 | if (Array.isArray(query) || (typeof query === "boolean")) { 214 | return true; 215 | } 216 | return false; 217 | } 218 | 219 | const uniqueNames = ["Category", "Size", "Color", "Destination", "Business Type", "Country", "Order Date", "Price", 220 | "Quantity", "Discount"]; 221 | it('check uniqueName property values', function (done) { 222 | promise.then(function (response) { 223 | response.data.fields.forEach((item) => { 224 | expect(uniqueNames).to.include(item.uniqueName); 225 | }); 226 | done(); 227 | }) 228 | .catch(function (error) { 229 | done(error); 230 | }); 231 | }); 232 | 233 | it('check uniqueName property values(if sorted)', function (done) { 234 | promise.then(function (response) { 235 | if (response.data.sorted) { 236 | expect(response.data.sorted).to.equal(true); 237 | expect(response.data.fields[0]).to.equal("Business Type"); 238 | expect(response.data.fields[1]).to.equal("Category"); 239 | expect(response.data.fields[2]).to.equal("Color"); 240 | expect(response.data.fields[3]).to.equal("Country"); 241 | expect(response.data.fields[4]).to.equal("Destination"); 242 | expect(response.data.fields[5]).to.equal("Discount"); 243 | expect(response.data.fields[6]).to.equal("Order Date"); 244 | expect(response.data.fields[7]).to.equal("Price"); 245 | expect(response.data.fields[8]).to.equal("Quantity"); 246 | expect(response.data.fields[9]).to.equal("Size"); 247 | } 248 | done(); 249 | }) 250 | .catch(function (error) { 251 | done(error); 252 | }); 253 | }); 254 | 255 | 256 | const uniqueNameTypeMap = { 257 | "Category": "string", 258 | "Color": "string", 259 | "Size": "string", 260 | "Destination": "string", 261 | "Business Type": "string", 262 | "Country": "string", 263 | "Order Date": "date", 264 | "Price": "number", 265 | "Quantity": "number", 266 | "Discount": "number" 267 | } 268 | 269 | it('check type property values', function (done) { 270 | promise.then(function (response) { 271 | response.data.fields.forEach((item) => { 272 | expect(item.type).to.equal(uniqueNameTypeMap[item.uniqueName]); 273 | }); 274 | done(); 275 | }) 276 | .catch(function (error) { 277 | done(error); 278 | }); 279 | }); 280 | 281 | }); -------------------------------------------------------------------------------- /tests/test/HandshakeRequestSpec.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios'); 2 | const expect = require('chai').expect; 3 | var config = require('../config.json'); 4 | 5 | const url = config.url + "/handshake"; 6 | 7 | const requestBody = { "type": "handshake", "version": "2.9.0" } 8 | 9 | describe('Handshake request', function () { 10 | let promise; 11 | 12 | before(function () { 13 | promise = axios.post(url, requestBody); 14 | }); 15 | 16 | it('should return 200 status code', function (done) { 17 | promise.then(function (response) { 18 | expect(response.status).to.equal(200); 19 | done(); 20 | }) 21 | .catch(function(error){ 22 | done(error); 23 | }); 24 | }); 25 | 26 | it('should return version in x.x.x format', function (done) { 27 | promise.then(function (response) { 28 | expect(response.data.version).to.match(/^\d+\.\d+\.\d+$/); 29 | done(); 30 | }) 31 | .catch(function(error){ 32 | done(error); 33 | }); 34 | }); 35 | }); -------------------------------------------------------------------------------- /tests/test/MembersRequestSpec.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios'); 2 | const expect = require('chai').expect; 3 | var config = require('../config.json'); 4 | 5 | const url = config.url + "/members"; 6 | 7 | //note - disable interval property if set for date testing 8 | 9 | describe('Members request', function () { 10 | 11 | function typeChecking(response, memberType) { 12 | expect(response.data.members).to.be.a("array"); 13 | response.data.members.forEach((item) => { 14 | if (item.value && item.value !== 0) { 15 | expect(item.value).to.be.a(memberType); 16 | } 17 | if (item.id !== undefined) { 18 | expect(item.id).to.be.a("string"); 19 | } 20 | }); 21 | if (response.data.sorted !== undefined) { 22 | expect(response.data.sorted).to.be.a("boolean"); 23 | } 24 | if (response.data.page !== undefined) { 25 | expect(response.data.page).to.be.a("number"); 26 | } 27 | if (response.data.pageTotal !== undefined) { 28 | expect(response.data.pageTotal).to.be.a("number"); 29 | } 30 | } 31 | 32 | describe('String member', function () { 33 | const requestBody = { "type": "members", "index": "data", "field": { "uniqueName": "Category" } } 34 | let promise; 35 | 36 | before(function () { 37 | promise = axios.post(url, requestBody); 38 | }); 39 | 40 | it('should return 200 status code', function (done) { 41 | promise.then(function (response) { 42 | expect(response.status).to.equal(200); 43 | done(); 44 | }) 45 | .catch(function (error) { 46 | done(error); 47 | }); 48 | }); 49 | 50 | it('response type check', function (done) { 51 | promise.then(function (response) { 52 | typeChecking(response, "string"); 53 | done(); 54 | }) 55 | .catch(function (error) { 56 | done(error); 57 | }); 58 | }); 59 | 60 | 61 | const category = ["Accessories", "Bikes", "Clothing", "Components", "Cars"]; 62 | 63 | it('should return all members', function (done) { 64 | promise.then(function (response) { 65 | expect(response.data.members).to.be.lengthOf(5); 66 | response.data.members.forEach((item) => { 67 | expect(category).to.include(item.value); 68 | }); 69 | if (response.data.page !== undefined) { 70 | expect(response.data.page).to.be.equal(0); 71 | } 72 | if (response.data.pageTotal !== undefined) { 73 | expect(response.data.pageTotal).to.be.equal(1); 74 | } 75 | done(); 76 | }) 77 | .catch(function (error) { 78 | done(error); 79 | }); 80 | }); 81 | 82 | it('should return all members(sorted)', function (done) { 83 | promise.then(function (response) { 84 | if (response.data.sorted) { 85 | expect(response.data.members).to.be.lengthOf(5); 86 | expect(response.data.members[0]).to.equal("Accessories"); 87 | expect(response.data.members[1]).to.equal("Bikes"); 88 | expect(response.data.members[2]).to.equal("Cars"); 89 | expect(response.data.members[3]).to.equal("Clothing"); 90 | expect(response.data.members[4]).to.equal("Components"); 91 | } 92 | done(); 93 | }) 94 | .catch(function (error) { 95 | done(error); 96 | }); 97 | }); 98 | }); 99 | 100 | describe('Number member + nulls', function () { 101 | const requestBody = { "type": "members", "index": "data", "field": { "uniqueName": "Discount" } } 102 | let promise; 103 | 104 | before(function () { 105 | promise = axios.post(url, requestBody); 106 | }); 107 | 108 | it('response type check', function (done) { 109 | promise.then(function (response) { 110 | 111 | typeChecking(response, "number"); 112 | done(); 113 | }) 114 | .catch(function (error) { 115 | done(error); 116 | }); 117 | }); 118 | 119 | 120 | it('should return all members', function (done) { 121 | promise.then(function (response) { 122 | expect(response.data.members).to.be.lengthOf(102); 123 | expect(response.data.members).to.satisfy((members) => { 124 | return members.some( 125 | (item) => { 126 | return (item.value === config.emptyValue) ? true : false; 127 | }); 128 | }); 129 | expect(response.data.members).to.deep.include({ "value": 26 }); 130 | expect(response.data.members).to.deep.include({ "value": 0 }); 131 | expect(response.data.members).to.deep.include({ "value": 100 }); 132 | if (response.data.page !== undefined) { 133 | expect(response.data.page).to.be.equal(0); 134 | } 135 | if (response.data.pageTotal !== undefined) { 136 | expect(response.data.pageTotal).to.be.equal(1); 137 | } 138 | done(); 139 | }) 140 | .catch(function (error) { 141 | done(error); 142 | }); 143 | }); 144 | }); 145 | 146 | describe('Data member', function () { 147 | const requestBody = { "type": "members", "index": "data", "field": { "uniqueName": "Order Date" } } 148 | let promise; 149 | 150 | before(function () { 151 | promise = axios.post(url, requestBody); 152 | }); 153 | 154 | it('response type check', function (done) { 155 | promise.then(function (response) { 156 | 157 | typeChecking(response, "number"); 158 | done(); 159 | }) 160 | .catch(function (error) { 161 | done(error); 162 | }); 163 | }); 164 | 165 | 166 | it('should return all members', function (done) { 167 | promise.then(function (response) { 168 | expect(response.data.members).to.be.lengthOf(623); 169 | expect(response.data.members).to.deep.include({ "value": 1526947200000 }); 170 | expect(response.data.members).to.deep.include({ "value": 1560902400000 }); 171 | expect(response.data.members).to.deep.include({ "value": 1565568000000 }); 172 | if (response.data.page !== undefined) { 173 | expect(response.data.page).to.be.equal(0); 174 | } 175 | if (response.data.pageTotal !== undefined) { 176 | expect(response.data.pageTotal).to.be.equal(1); 177 | } 178 | done(); 179 | }) 180 | .catch(function (error) { 181 | done(error); 182 | }); 183 | }); 184 | }); 185 | }); -------------------------------------------------------------------------------- /tests/test/SelectRequestFlatSpec.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios'); 2 | const expect = require('chai').expect; 3 | var config = require('../config.json'); 4 | 5 | const url = config.url + "/select"; 6 | 7 | describe('Select request for flat table', function () { 8 | 9 | it('Flat request', function (done) { 10 | 11 | const requestBody = { "type": "select", "querytype": "select", "index": "data", "query": { "aggs": { "values": [{ "func": "sum", "field": { "uniqueName": "Price" } }] }, "fields": [{ "uniqueName": "Country" }, { "uniqueName": "Order Date" }, { "uniqueName": "Color" }, { "uniqueName": "Price" }] }, "page": 0 } 12 | 13 | axios.post(url, requestBody).then(function (response) { 14 | expect(response.status).to.equal(200); 15 | expect(response.data.fields).to.deep.include.members([{ "uniqueName": "Country" }, { "uniqueName": "Order Date" }, { "uniqueName": "Color" }, { "uniqueName": "Price" }]); 16 | expect(response.data.hits).to.have.lengthOf(999); 17 | 18 | if (response.data.page !== undefined) { 19 | expect(response.data.page).to.be.a("number"); 20 | } 21 | if (response.data.pageTotal !== undefined) { 22 | expect(response.data.pageTotal).to.be.a("number"); 23 | } 24 | done(); 25 | }) 26 | .catch(function (error) { 27 | done(error); 28 | }); 29 | }); 30 | 31 | it('Flat request with filters - 1', function (done) { 32 | 33 | const requestBody = { "type": "select", "querytype": "select", "index": "data", "query": { "aggs": { "values": [{ "func": "sum", "field": { "uniqueName": "Price" } }] }, "filter": [{ "field": { "uniqueName": "Country" }, "include": [{ "member": "Australia" }, { "member": "Canada" }] }, { "field": { "uniqueName": "Order Date" }, "query": { "between": [1530403200000, 1562025599999] } }, { "field": { "uniqueName": "Color" }, "query": { "contain": "l" } }], "fields": [{ "uniqueName": "Country" }, { "uniqueName": "Order Date" }, { "uniqueName": "Color" }, { "uniqueName": "Price" }] }, "page": 0 } 34 | axios.post(url, requestBody).then(function (response) { 35 | expect(response.status).to.equal(200); 36 | expect(response.data.fields).to.deep.include.members([{ "uniqueName": "Country" }, { "uniqueName": "Order Date" }, { "uniqueName": "Color" }, { "uniqueName": "Price" }]); 37 | expect(response.data.hits).to.have.lengthOf(33); 38 | expect(response.data.hits).to.satisfy((members) => { 39 | return members.every( 40 | (item) => { 41 | return ((item[0] === "Canada") || (item[0] === "Australia")) && (item[1] >= 1530403200000) && (item[1] <= 1562025599999) && ((item[2] === "blue") || (item[2] === "yellow") || (item[2] === "purple")) ? true : false; 42 | }); 43 | }); 44 | done(); 45 | }) 46 | .catch(function (error) { 47 | done(error); 48 | }); 49 | }); 50 | 51 | it('Flat request with filters - 2', function (done) { 52 | 53 | const requestBody = { "type": "select", "querytype": "select", "index": "data", "query": { "aggs": { "values": [{ "func": "sum", "field": { "uniqueName": "Discount" } }, { "func": "sum", "field": { "uniqueName": "Price" } }] }, "filter": [{ "field": { "uniqueName": "Size" }, "query": { "begin": "1" } }, { "field": { "uniqueName": "Discount" }, "include": [{ "member": config.emptyValue }] }, { "field": { "uniqueName": "Price" }, "include": [{ "member": 1241 }] }], "fields": [{ "uniqueName": "Order Date" }, { "uniqueName": "Size" }, { "uniqueName": "Discount" }, { "uniqueName": "Price" }] }, "page": 0 } 54 | axios.post(url, requestBody).then(function (response) { 55 | expect(response.status).to.equal(200); 56 | expect(response.data.fields).to.deep.include.members([{ "uniqueName": "Order Date" }, { "uniqueName": "Size" }, { "uniqueName": "Discount" }, { "uniqueName": "Price" }]); 57 | expect(response.data.hits).to.have.lengthOf(2); 58 | expect(response.data.aggs[0].values).to.have.deep.property("Price", { "sum": 2482 }); 59 | expect(response.data.hits).to.deep.include([1562371200000, "152 oz", config.emptyValue, 1241]); 60 | expect(response.data.hits).to.deep.include([1516838400000, "107 oz", config.emptyValue, 1241]); 61 | done(); 62 | }) 63 | .catch(function (error) { 64 | done(error); 65 | }); 66 | }); 67 | 68 | it('Flat request with filters - 3', function (done) { 69 | 70 | const requestBody = { "type": "select", "querytype": "select", "index": "data", "query": { "aggs": { "values": [{ "func": "sum", "field": { "uniqueName": "Discount" } }, { "func": "sum", "field": { "uniqueName": "Price" } }] }, "filter": [{ "field": { "uniqueName": "Order Date" }, "include": [{ "member": 1514851200000 }, { "member": 1514937600000 }, { "member": 1515024000000 }, { "member": 1515110400000 }] }, { "field": { "uniqueName": "Size" }, "query": { "between": ["1", "2"] } }], "fields": [{ "uniqueName": "Order Date" }, { "uniqueName": "Size" }, { "uniqueName": "Discount" }, { "uniqueName": "Price" }] }, "page": 0 } 71 | axios.post(url, requestBody).then(function (response) { 72 | expect(response.status).to.equal(200); 73 | expect(response.data.fields).to.deep.include.members([{ "uniqueName": "Order Date" }, { "uniqueName": "Size" }, { "uniqueName": "Discount" }, { "uniqueName": "Price" }]); 74 | expect(response.data.hits).to.have.lengthOf(2); 75 | expect(response.data.aggs[0].values).to.have.deep.property("Price", { "sum": 59705 }); 76 | expect(response.data.aggs[0].values).to.have.deep.property("Discount", { "sum": 12 }); 77 | expect(response.data.hits).to.deep.include([1514851200000, "168 oz", config.emptyValue, 250]); 78 | expect(response.data.hits).to.deep.include([1514937600000, "172 oz", 12, 59455]); 79 | done(); 80 | }) 81 | .catch(function (error) { 82 | done(error); 83 | }); 84 | }); 85 | 86 | describe('Select request for flat table with hierarchy', function () { 87 | 88 | it('Flat request with hierarchy - 1', function (done) { 89 | 90 | const requestBody = { "type": "select", "querytype": "select", "index": "data", "query": { "aggs": { "values": [{ "func": "sum", "field": { "uniqueName": "Price" } }] }, "fields": [{ "uniqueName": "Country" }, { "uniqueName": "Color" }, { "uniqueName": "Business Type" }, { "uniqueName": "Price" }] }, "page": 0 } 91 | 92 | axios.post(url, requestBody).then(function (response) { 93 | expect(response.status).to.equal(200); 94 | expect(response.data.fields).to.deep.include.members([{ "uniqueName": "Country" }, { "uniqueName": "Color" }, { "uniqueName": "Business Type" }, { "uniqueName": "Price" }]); 95 | expect(response.data.hits).to.have.lengthOf(999); 96 | expect(response.data.aggs[0].values).to.have.deep.property("Price", { "sum": 6221870 }); 97 | done(); 98 | }) 99 | .catch(function (error) { 100 | done(error); 101 | }); 102 | }); 103 | 104 | }); 105 | }); 106 | --------------------------------------------------------------------------------