├── .github └── FUNDING.yml ├── .gitignore ├── Config.json ├── Gateway Monitor - FromDisk.pbit ├── Gateway Monitor - FromLake.pbit ├── Images ├── Architecture.png ├── AzurePortal_StorageConnStr.png ├── AzurePortal_StorageHierarchicalNamespace.png ├── ConfigFile.png ├── ConfigFile_PathProperty.png ├── PBI_Counters.png ├── PBI_GatewayProfile.png ├── PBI_LogPage.png ├── PBI_MashupLogs.png ├── PBI_MashupProfiles.png ├── PBI_QueriesPage.png ├── PBI_QueryConcurrency.png ├── PBI_RequestsPage.png ├── PBI_TemplateParams.png ├── PBI_TemplateParamsDisk.png ├── Setup_GatewayExportLogs.png ├── Setup_GatewayProperties.png └── Setup_ScheduleTask.png ├── LICENSE ├── PBIP ├── Gateway Monitor.Dataset │ ├── .pbi │ │ └── editorSettings.json │ ├── definition.pbidataset │ ├── diagramLayout.json │ ├── item.config.json │ ├── item.metadata.json │ └── model.bim ├── Gateway Monitor.Report │ ├── StaticResources │ │ ├── RegisteredResources │ │ │ ├── Logs_Background87715584692928.png │ │ │ ├── Mashup_profiles_Background8901081387888303.png │ │ │ ├── Requests_Background8022619583865385.png │ │ │ └── gateway-monitoring-solution-ba7306591542541692.png │ │ └── SharedResources │ │ │ └── BaseThemes │ │ │ └── CY22SU03.json │ ├── datasetDiagramLayout.json │ ├── definition.pbir │ ├── item.config.json │ ├── item.metadata.json │ └── report.json └── Gateway Monitor.pbip ├── README.md ├── Run.cmd ├── Run.ps1 ├── Tools - Get Dataset Refresh History.ps1 ├── UploadGatewayLogs.ps1 └── Utils.psm1 /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: RuiRomano 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pbix 2 | /devicecodetoken.bin 3 | 4 | pbitoken.bin 5 | 6 | pbitoken*.bin 7 | 8 | /Data* 9 | 10 | GraphAppSecret.txt 11 | 12 | ServicePrincipal.json 13 | 14 | /backup 15 | 16 | /Share 17 | 18 | /Config - *.json 19 | 20 | *.pbix 21 | /debug.log 22 | 23 | */data/* 24 | 25 | /Publish 26 | 27 | Test -*.ps1 28 | 29 | local.settings* 30 | 31 | *.docx 32 | *.lnk 33 | 34 | /Dataflow 35 | 36 | /DatasetStats 37 | GatewayInfo.json 38 | GatewayProperties.txt 39 | Logs.lnk 40 | Run-Local.ps1 41 | Diagrams.pptx 42 | ~$Diagrams.pptx 43 | Tools - GetGateways.ps1 44 | Tools - ListGateways.ps1 45 | state.json 46 | Run-Local.cmd 47 | 48 | On-premises data gateway.lnk 49 | 50 | /OtherGatewayLogs/* 51 | gateway-monitoring-solution-backgrounds-1.png 52 | .gitignore 53 | gateway-monitoring-solution-backgrounds-1.pptx 54 | Gateway Monitor - Parameters.txt 55 | 56 | **/.pbi/localSettings.json 57 | **/.pbi/cache.abf -------------------------------------------------------------------------------- /Config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Config.json -------------------------------------------------------------------------------- /Gateway Monitor - FromDisk.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Gateway Monitor - FromDisk.pbit -------------------------------------------------------------------------------- /Gateway Monitor - FromLake.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Gateway Monitor - FromLake.pbit -------------------------------------------------------------------------------- /Images/Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/Architecture.png -------------------------------------------------------------------------------- /Images/AzurePortal_StorageConnStr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/AzurePortal_StorageConnStr.png -------------------------------------------------------------------------------- /Images/AzurePortal_StorageHierarchicalNamespace.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/AzurePortal_StorageHierarchicalNamespace.png -------------------------------------------------------------------------------- /Images/ConfigFile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/ConfigFile.png -------------------------------------------------------------------------------- /Images/ConfigFile_PathProperty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/ConfigFile_PathProperty.png -------------------------------------------------------------------------------- /Images/PBI_Counters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_Counters.png -------------------------------------------------------------------------------- /Images/PBI_GatewayProfile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_GatewayProfile.png -------------------------------------------------------------------------------- /Images/PBI_LogPage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_LogPage.png -------------------------------------------------------------------------------- /Images/PBI_MashupLogs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_MashupLogs.png -------------------------------------------------------------------------------- /Images/PBI_MashupProfiles.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_MashupProfiles.png -------------------------------------------------------------------------------- /Images/PBI_QueriesPage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_QueriesPage.png -------------------------------------------------------------------------------- /Images/PBI_QueryConcurrency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_QueryConcurrency.png -------------------------------------------------------------------------------- /Images/PBI_RequestsPage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_RequestsPage.png -------------------------------------------------------------------------------- /Images/PBI_TemplateParams.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_TemplateParams.png -------------------------------------------------------------------------------- /Images/PBI_TemplateParamsDisk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/PBI_TemplateParamsDisk.png -------------------------------------------------------------------------------- /Images/Setup_GatewayExportLogs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/Setup_GatewayExportLogs.png -------------------------------------------------------------------------------- /Images/Setup_GatewayProperties.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/Setup_GatewayProperties.png -------------------------------------------------------------------------------- /Images/Setup_ScheduleTask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/Images/Setup_ScheduleTask.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Rui Romano 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Dataset/.pbi/editorSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "showHiddenFields": true, 4 | "parallelQueryLoading": true, 5 | "relationshipImportEnabled": true 6 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Dataset/definition.pbidataset: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "settings": { 4 | "qnaEnabled": true 5 | } 6 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Dataset/diagramLayout.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.1.0", 3 | "diagrams": [ 4 | { 5 | "ordinal": 0, 6 | "scrollPosition": { 7 | "x": 0, 8 | "y": 57.333332061767578 9 | }, 10 | "nodes": [ 11 | { 12 | "location": { 13 | "x": 2089.0625, 14 | "y": 68.75 15 | }, 16 | "nodeIndex": "Gateways", 17 | "size": { 18 | "height": 475.76478471564224, 19 | "width": 234 20 | }, 21 | "zIndex": 13 22 | }, 23 | { 24 | "location": { 25 | "x": 690.156512809311, 26 | "y": 519.051840964009 27 | }, 28 | "nodeIndex": "Calendar", 29 | "size": { 30 | "height": 300, 31 | "width": 234 32 | }, 33 | "zIndex": 9 34 | }, 35 | { 36 | "location": { 37 | "x": 795.91117574293071, 38 | "y": 949.61631660819069 39 | }, 40 | "nodeIndex": "Time", 41 | "size": { 42 | "height": 300, 43 | "width": 234 44 | }, 45 | "zIndex": 10 46 | }, 47 | { 48 | "location": { 49 | "x": 1189.28125, 50 | "y": 0 51 | }, 52 | "nodeIndex": "Mashup Container Profile", 53 | "size": { 54 | "height": 300, 55 | "width": 234 56 | }, 57 | "zIndex": 8 58 | }, 59 | { 60 | "location": { 61 | "x": 1705.5539727712071, 62 | "y": 923.54118611840465 63 | }, 64 | "nodeIndex": "Mashup Logs", 65 | "size": { 66 | "height": 407.53836833918081, 67 | "width": 221.5 68 | }, 69 | "zIndex": 11 70 | }, 71 | { 72 | "location": { 73 | "x": 2032.3325764192141, 74 | "y": 546.01781659388655 75 | }, 76 | "nodeIndex": "Param - Counters Measure", 77 | "size": { 78 | "height": 152, 79 | "width": 234 80 | }, 81 | "zIndex": 2 82 | }, 83 | { 84 | "location": { 85 | "x": 1991.0628820960692, 86 | "y": 334.130305676856 87 | }, 88 | "nodeIndex": "Param - Counters Axis", 89 | "size": { 90 | "height": 152, 91 | "width": 234 92 | }, 93 | "zIndex": 3 94 | }, 95 | { 96 | "location": { 97 | "x": 1095.6215743302585, 98 | "y": 412.38623653198272 99 | }, 100 | "nodeIndex": "Logs", 101 | "size": { 102 | "height": 300, 103 | "width": 234 104 | }, 105 | "zIndex": 4 106 | }, 107 | { 108 | "location": { 109 | "x": 1213.5278858092022, 110 | "y": 882.47168033683238 111 | }, 112 | "nodeIndex": "Queries", 113 | "size": { 114 | "height": 414.37987150491404, 115 | "width": 234 116 | }, 117 | "zIndex": 12 118 | }, 119 | { 120 | "location": { 121 | "x": 267.66874999999982, 122 | "y": 826.5625 123 | }, 124 | "nodeIndex": "System Counters", 125 | "size": { 126 | "height": 694.4375, 127 | "width": 334 128 | }, 129 | "zIndex": 5 130 | }, 131 | { 132 | "location": { 133 | "x": 1574.0634899172844, 134 | "y": 493.96703716512934 135 | }, 136 | "nodeIndex": "Requests", 137 | "size": { 138 | "height": 256.25, 139 | "width": 234 140 | }, 141 | "zIndex": 6 142 | }, 143 | { 144 | "location": { 145 | "x": 967.43472032742193, 146 | "y": 1454.1439290586634 147 | }, 148 | "nodeIndex": "Queries - Datasources", 149 | "size": { 150 | "height": 152, 151 | "width": 234 152 | }, 153 | "zIndex": 7 154 | }, 155 | { 156 | "location": { 157 | "x": 1547.8257189305423, 158 | "y": 72.699070871767319 159 | }, 160 | "nodeIndex": "LogsArtifactTrace", 161 | "size": { 162 | "height": 272, 163 | "width": 234 164 | }, 165 | "zIndex": 1 166 | } 167 | ], 168 | "name": "All tables", 169 | "zoomValue": 73.201290315793159, 170 | "pinKeyFieldsToTop": false, 171 | "showExtraHeaderInfo": false, 172 | "hideKeyFieldsWhenCollapsed": false, 173 | "tablesLocked": false 174 | } 175 | ], 176 | "selectedDiagram": "All tables", 177 | "defaultDiagram": "All tables" 178 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Dataset/item.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "logicalId": "3d89307e-9213-4f97-a5ec-de9e4827ec39" 4 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Dataset/item.metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "dataset", 3 | "displayName": "Gateway Monitor" 4 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/Logs_Background87715584692928.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/Logs_Background87715584692928.png -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/Mashup_profiles_Background8901081387888303.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/Mashup_profiles_Background8901081387888303.png -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/Requests_Background8022619583865385.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/Requests_Background8022619583865385.png -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/gateway-monitoring-solution-ba7306591542541692.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiRomano/pbigtwmonitor/9e31cfe70edddcf805ddb8fad4b535e24e32e438/PBIP/Gateway Monitor.Report/StaticResources/RegisteredResources/gateway-monitoring-solution-ba7306591542541692.png -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/StaticResources/SharedResources/BaseThemes/CY22SU03.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CY22SU03", 3 | "dataColors": [ 4 | "#118DFF", 5 | "#12239E", 6 | "#E66C37", 7 | "#6B007B", 8 | "#E044A7", 9 | "#744EC2", 10 | "#D9B300", 11 | "#D64550", 12 | "#197278", 13 | "#1AAB40", 14 | "#15C6F4", 15 | "#4092FF", 16 | "#FFA058", 17 | "#BE5DC9", 18 | "#F472D0", 19 | "#B5A1FF", 20 | "#C4A200", 21 | "#FF8080", 22 | "#00DBBC", 23 | "#5BD667", 24 | "#0091D5", 25 | "#4668C5", 26 | "#FF6300", 27 | "#99008A", 28 | "#EC008C", 29 | "#533285", 30 | "#99700A", 31 | "#FF4141", 32 | "#1F9A85", 33 | "#25891C", 34 | "#0057A2", 35 | "#002050", 36 | "#C94F0F", 37 | "#450F54", 38 | "#B60064", 39 | "#34124F", 40 | "#6A5A29", 41 | "#1AAB40", 42 | "#BA141A", 43 | "#0C3D37", 44 | "#0B511F" 45 | ], 46 | "foreground": "#252423", 47 | "foregroundNeutralSecondary": "#605E5C", 48 | "foregroundNeutralTertiary": "#B3B0AD", 49 | "background": "#FFFFFF", 50 | "backgroundLight": "#F3F2F1", 51 | "backgroundNeutral": "#C8C6C4", 52 | "tableAccent": "#118DFF", 53 | "good": "#1AAB40", 54 | "neutral": "#D9B300", 55 | "bad": "#D64554", 56 | "maximum": "#118DFF", 57 | "center": "#D9B300", 58 | "minimum": "#DEEFFF", 59 | "null": "#FF7F48", 60 | "hyperlink": "#0078d4", 61 | "visitedHyperlink": "#0078d4", 62 | "textClasses": { 63 | "callout": { 64 | "fontSize": 45, 65 | "fontFace": "DIN", 66 | "color": "#252423" 67 | }, 68 | "title": { 69 | "fontSize": 12, 70 | "fontFace": "DIN", 71 | "color": "#252423" 72 | }, 73 | "header": { 74 | "fontSize": 12, 75 | "fontFace": "Segoe UI Semibold", 76 | "color": "#252423" 77 | }, 78 | "label": { 79 | "fontSize": 10, 80 | "fontFace": "Segoe UI", 81 | "color": "#252423" 82 | } 83 | }, 84 | "visualStyles": { 85 | "*": { 86 | "*": { 87 | "*": [ 88 | { 89 | "wordWrap": true 90 | } 91 | ], 92 | "line": [ 93 | { 94 | "transparency": 0 95 | } 96 | ], 97 | "outline": [ 98 | { 99 | "transparency": 0 100 | } 101 | ], 102 | "plotArea": [ 103 | { 104 | "transparency": 0 105 | } 106 | ], 107 | "categoryAxis": [ 108 | { 109 | "showAxisTitle": true, 110 | "gridlineStyle": "dotted" 111 | } 112 | ], 113 | "valueAxis": [ 114 | { 115 | "showAxisTitle": true, 116 | "gridlineStyle": "dotted" 117 | } 118 | ], 119 | "title": [ 120 | { 121 | "titleWrap": true 122 | } 123 | ], 124 | "lineStyles": [ 125 | { 126 | "strokeWidth": 3 127 | } 128 | ], 129 | "wordWrap": [ 130 | { 131 | "show": true 132 | } 133 | ], 134 | "background": [ 135 | { 136 | "show": true, 137 | "transparency": 0 138 | } 139 | ], 140 | "outspacePane": [ 141 | { 142 | "backgroundColor": { 143 | "solid": { 144 | "color": "#ffffff" 145 | } 146 | }, 147 | "foregroundColor": { 148 | "solid": { 149 | "color": "#252423" 150 | } 151 | }, 152 | "transparency": 0, 153 | "border": true, 154 | "borderColor": { 155 | "solid": { 156 | "color": "#B3B0AD" 157 | } 158 | } 159 | } 160 | ], 161 | "filterCard": [ 162 | { 163 | "$id": "Applied", 164 | "transparency": 0, 165 | "foregroundColor": { 166 | "solid": { 167 | "color": "#252423" 168 | } 169 | }, 170 | "border": true 171 | }, 172 | { 173 | "$id": "Available", 174 | "transparency": 0, 175 | "foregroundColor": { 176 | "solid": { 177 | "color": "#252423" 178 | } 179 | }, 180 | "border": true 181 | } 182 | ] 183 | } 184 | }, 185 | "scatterChart": { 186 | "*": { 187 | "bubbles": [ 188 | { 189 | "bubbleSize": -10 190 | } 191 | ], 192 | "general": [ 193 | { 194 | "responsive": true 195 | } 196 | ], 197 | "fillPoint": [ 198 | { 199 | "show": true 200 | } 201 | ], 202 | "legend": [ 203 | { 204 | "showGradientLegend": true 205 | } 206 | ] 207 | } 208 | }, 209 | "lineChart": { 210 | "*": { 211 | "general": [ 212 | { 213 | "responsive": true 214 | } 215 | ], 216 | "smallMultiplesLayout": [ 217 | { 218 | "backgroundTransparency": 0, 219 | "gridLineType": "inner" 220 | } 221 | ] 222 | } 223 | }, 224 | "map": { 225 | "*": { 226 | "bubbles": [ 227 | { 228 | "bubbleSize": -10 229 | } 230 | ] 231 | } 232 | }, 233 | "pieChart": { 234 | "*": { 235 | "legend": [ 236 | { 237 | "show": true, 238 | "position": "RightCenter" 239 | } 240 | ], 241 | "labels": [ 242 | { 243 | "labelStyle": "Data value, percent of total" 244 | } 245 | ] 246 | } 247 | }, 248 | "donutChart": { 249 | "*": { 250 | "legend": [ 251 | { 252 | "show": true, 253 | "position": "RightCenter" 254 | } 255 | ], 256 | "labels": [ 257 | { 258 | "labelStyle": "Data value, percent of total" 259 | } 260 | ] 261 | } 262 | }, 263 | "pivotTable": { 264 | "*": { 265 | "*": [ 266 | { 267 | "showExpandCollapseButtons": true 268 | } 269 | ] 270 | } 271 | }, 272 | "multiRowCard": { 273 | "*": { 274 | "card": [ 275 | { 276 | "outlineWeight": 2, 277 | "barShow": true, 278 | "barWeight": 2 279 | } 280 | ] 281 | } 282 | }, 283 | "kpi": { 284 | "*": { 285 | "trendline": [ 286 | { 287 | "transparency": 20 288 | } 289 | ] 290 | } 291 | }, 292 | "slicer": { 293 | "*": { 294 | "general": [ 295 | { 296 | "responsive": true 297 | } 298 | ], 299 | "date": [ 300 | { 301 | "hideDatePickerButton": false 302 | } 303 | ], 304 | "items": [ 305 | { 306 | "fontColor": { 307 | "solid": { 308 | "color": "#252423" 309 | } 310 | }, 311 | "padding": 4, 312 | "accessibilityContrastProperties": true 313 | } 314 | ] 315 | } 316 | }, 317 | "waterfallChart": { 318 | "*": { 319 | "general": [ 320 | { 321 | "responsive": true 322 | } 323 | ] 324 | } 325 | }, 326 | "columnChart": { 327 | "*": { 328 | "general": [ 329 | { 330 | "responsive": true 331 | } 332 | ], 333 | "legend": [ 334 | { 335 | "showGradientLegend": true 336 | } 337 | ], 338 | "smallMultiplesLayout": [ 339 | { 340 | "backgroundTransparency": 0, 341 | "gridLineType": "inner" 342 | } 343 | ] 344 | } 345 | }, 346 | "clusteredColumnChart": { 347 | "*": { 348 | "general": [ 349 | { 350 | "responsive": true 351 | } 352 | ], 353 | "legend": [ 354 | { 355 | "showGradientLegend": true 356 | } 357 | ], 358 | "smallMultiplesLayout": [ 359 | { 360 | "backgroundTransparency": 0, 361 | "gridLineType": "inner" 362 | } 363 | ] 364 | } 365 | }, 366 | "hundredPercentStackedColumnChart": { 367 | "*": { 368 | "general": [ 369 | { 370 | "responsive": true 371 | } 372 | ], 373 | "legend": [ 374 | { 375 | "showGradientLegend": true 376 | } 377 | ], 378 | "smallMultiplesLayout": [ 379 | { 380 | "backgroundTransparency": 0, 381 | "gridLineType": "inner" 382 | } 383 | ] 384 | } 385 | }, 386 | "barChart": { 387 | "*": { 388 | "general": [ 389 | { 390 | "responsive": true 391 | } 392 | ], 393 | "legend": [ 394 | { 395 | "showGradientLegend": true 396 | } 397 | ], 398 | "smallMultiplesLayout": [ 399 | { 400 | "backgroundTransparency": 0, 401 | "gridLineType": "inner" 402 | } 403 | ] 404 | } 405 | }, 406 | "clusteredBarChart": { 407 | "*": { 408 | "general": [ 409 | { 410 | "responsive": true 411 | } 412 | ], 413 | "legend": [ 414 | { 415 | "showGradientLegend": true 416 | } 417 | ], 418 | "smallMultiplesLayout": [ 419 | { 420 | "backgroundTransparency": 0, 421 | "gridLineType": "inner" 422 | } 423 | ] 424 | } 425 | }, 426 | "hundredPercentStackedBarChart": { 427 | "*": { 428 | "general": [ 429 | { 430 | "responsive": true 431 | } 432 | ], 433 | "legend": [ 434 | { 435 | "showGradientLegend": true 436 | } 437 | ], 438 | "smallMultiplesLayout": [ 439 | { 440 | "backgroundTransparency": 0, 441 | "gridLineType": "inner" 442 | } 443 | ] 444 | } 445 | }, 446 | "areaChart": { 447 | "*": { 448 | "general": [ 449 | { 450 | "responsive": true 451 | } 452 | ], 453 | "smallMultiplesLayout": [ 454 | { 455 | "backgroundTransparency": 0, 456 | "gridLineType": "inner" 457 | } 458 | ] 459 | } 460 | }, 461 | "stackedAreaChart": { 462 | "*": { 463 | "general": [ 464 | { 465 | "responsive": true 466 | } 467 | ], 468 | "smallMultiplesLayout": [ 469 | { 470 | "backgroundTransparency": 0, 471 | "gridLineType": "inner" 472 | } 473 | ] 474 | } 475 | }, 476 | "lineClusteredColumnComboChart": { 477 | "*": { 478 | "general": [ 479 | { 480 | "responsive": true 481 | } 482 | ], 483 | "smallMultiplesLayout": [ 484 | { 485 | "backgroundTransparency": 0, 486 | "gridLineType": "inner" 487 | } 488 | ] 489 | } 490 | }, 491 | "lineStackedColumnComboChart": { 492 | "*": { 493 | "general": [ 494 | { 495 | "responsive": true 496 | } 497 | ], 498 | "smallMultiplesLayout": [ 499 | { 500 | "backgroundTransparency": 0, 501 | "gridLineType": "inner" 502 | } 503 | ] 504 | } 505 | }, 506 | "ribbonChart": { 507 | "*": { 508 | "general": [ 509 | { 510 | "responsive": true 511 | } 512 | ] 513 | } 514 | }, 515 | "group": { 516 | "*": { 517 | "background": [ 518 | { 519 | "show": false 520 | } 521 | ] 522 | } 523 | }, 524 | "basicShape": { 525 | "*": { 526 | "background": [ 527 | { 528 | "show": false 529 | } 530 | ], 531 | "general": [ 532 | { 533 | "keepLayerOrder": true 534 | } 535 | ], 536 | "visualHeader": [ 537 | { 538 | "show": false 539 | } 540 | ] 541 | } 542 | }, 543 | "shape": { 544 | "*": { 545 | "background": [ 546 | { 547 | "show": false 548 | } 549 | ], 550 | "general": [ 551 | { 552 | "keepLayerOrder": true 553 | } 554 | ], 555 | "visualHeader": [ 556 | { 557 | "show": false 558 | } 559 | ] 560 | } 561 | }, 562 | "image": { 563 | "*": { 564 | "background": [ 565 | { 566 | "show": false 567 | } 568 | ], 569 | "general": [ 570 | { 571 | "keepLayerOrder": true 572 | } 573 | ], 574 | "visualHeader": [ 575 | { 576 | "show": false 577 | } 578 | ], 579 | "lockAspect": [ 580 | { 581 | "show": true 582 | } 583 | ] 584 | } 585 | }, 586 | "actionButton": { 587 | "*": { 588 | "background": [ 589 | { 590 | "show": false 591 | } 592 | ], 593 | "visualHeader": [ 594 | { 595 | "show": false 596 | } 597 | ] 598 | } 599 | }, 600 | "pageNavigator": { 601 | "*": { 602 | "background": [ 603 | { 604 | "show": false 605 | } 606 | ], 607 | "visualHeader": [ 608 | { 609 | "show": false 610 | } 611 | ] 612 | } 613 | }, 614 | "bookmarkNavigator": { 615 | "*": { 616 | "background": [ 617 | { 618 | "show": false 619 | } 620 | ], 621 | "visualHeader": [ 622 | { 623 | "show": false 624 | } 625 | ] 626 | } 627 | }, 628 | "textbox": { 629 | "*": { 630 | "general": [ 631 | { 632 | "keepLayerOrder": true 633 | } 634 | ], 635 | "visualHeader": [ 636 | { 637 | "show": false 638 | } 639 | ] 640 | } 641 | }, 642 | "page": { 643 | "*": { 644 | "outspace": [ 645 | { 646 | "color": { 647 | "solid": { 648 | "color": "#FFFFFF" 649 | } 650 | } 651 | } 652 | ], 653 | "background": [ 654 | { 655 | "transparency": 100 656 | } 657 | ] 658 | } 659 | } 660 | } 661 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/datasetDiagramLayout.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.1.0", 3 | "diagrams": [ 4 | { 5 | "ordinal": 0, 6 | "scrollPosition": { 7 | "x": 0, 8 | "y": 57.333332061767578 9 | }, 10 | "nodes": [ 11 | { 12 | "location": { 13 | "x": 2089.0625, 14 | "y": 68.75 15 | }, 16 | "nodeIndex": "Gateways", 17 | "size": { 18 | "height": 475.76478471564224, 19 | "width": 234 20 | }, 21 | "zIndex": 13 22 | }, 23 | { 24 | "location": { 25 | "x": 690.156512809311, 26 | "y": 519.051840964009 27 | }, 28 | "nodeIndex": "Calendar", 29 | "size": { 30 | "height": 300, 31 | "width": 234 32 | }, 33 | "zIndex": 9 34 | }, 35 | { 36 | "location": { 37 | "x": 795.91117574293071, 38 | "y": 949.61631660819069 39 | }, 40 | "nodeIndex": "Time", 41 | "size": { 42 | "height": 300, 43 | "width": 234 44 | }, 45 | "zIndex": 10 46 | }, 47 | { 48 | "location": { 49 | "x": 1189.28125, 50 | "y": 0 51 | }, 52 | "nodeIndex": "Mashup Container Profile", 53 | "size": { 54 | "height": 300, 55 | "width": 234 56 | }, 57 | "zIndex": 8 58 | }, 59 | { 60 | "location": { 61 | "x": 1705.5539727712071, 62 | "y": 923.54118611840465 63 | }, 64 | "nodeIndex": "Mashup Logs", 65 | "size": { 66 | "height": 407.53836833918081, 67 | "width": 221.5 68 | }, 69 | "zIndex": 11 70 | }, 71 | { 72 | "location": { 73 | "x": 2032.3325764192141, 74 | "y": 546.01781659388655 75 | }, 76 | "nodeIndex": "Param - Counters Measure", 77 | "size": { 78 | "height": 152, 79 | "width": 234 80 | }, 81 | "zIndex": 2 82 | }, 83 | { 84 | "location": { 85 | "x": 1991.0628820960692, 86 | "y": 334.130305676856 87 | }, 88 | "nodeIndex": "Param - Counters Axis", 89 | "size": { 90 | "height": 152, 91 | "width": 234 92 | }, 93 | "zIndex": 3 94 | }, 95 | { 96 | "location": { 97 | "x": 1095.6215743302585, 98 | "y": 412.38623653198272 99 | }, 100 | "nodeIndex": "Logs", 101 | "size": { 102 | "height": 300, 103 | "width": 234 104 | }, 105 | "zIndex": 4 106 | }, 107 | { 108 | "location": { 109 | "x": 1213.5278858092022, 110 | "y": 882.47168033683238 111 | }, 112 | "nodeIndex": "Queries", 113 | "size": { 114 | "height": 414.37987150491404, 115 | "width": 234 116 | }, 117 | "zIndex": 12 118 | }, 119 | { 120 | "location": { 121 | "x": 267.66874999999982, 122 | "y": 826.5625 123 | }, 124 | "nodeIndex": "System Counters", 125 | "size": { 126 | "height": 694.4375, 127 | "width": 334 128 | }, 129 | "zIndex": 5 130 | }, 131 | { 132 | "location": { 133 | "x": 1574.0634899172844, 134 | "y": 493.96703716512934 135 | }, 136 | "nodeIndex": "Requests", 137 | "size": { 138 | "height": 256.25, 139 | "width": 234 140 | }, 141 | "zIndex": 6 142 | }, 143 | { 144 | "location": { 145 | "x": 967.43472032742193, 146 | "y": 1454.1439290586634 147 | }, 148 | "nodeIndex": "Queries - Datasources", 149 | "size": { 150 | "height": 152, 151 | "width": 234 152 | }, 153 | "zIndex": 7 154 | }, 155 | { 156 | "location": { 157 | "x": 1547.8257189305423, 158 | "y": 72.699070871767319 159 | }, 160 | "nodeIndex": "LogsArtifactTrace", 161 | "size": { 162 | "height": 272, 163 | "width": 234 164 | }, 165 | "zIndex": 1 166 | } 167 | ], 168 | "name": "All tables", 169 | "zoomValue": 73.201290315793159, 170 | "pinKeyFieldsToTop": false, 171 | "showExtraHeaderInfo": false, 172 | "hideKeyFieldsWhenCollapsed": false, 173 | "tablesLocked": false 174 | } 175 | ], 176 | "selectedDiagram": "All tables", 177 | "defaultDiagram": "All tables" 178 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/definition.pbir: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "datasetReference": { 4 | "byPath": { 5 | "path": "../Gateway Monitor.Dataset" 6 | }, 7 | "byConnection": null 8 | } 9 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/item.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "logicalId": "d4e10d31-15ab-4d30-a04a-99c2067d5908" 4 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.Report/item.metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "report", 3 | "displayName": "Gateway Monitor" 4 | } -------------------------------------------------------------------------------- /PBIP/Gateway Monitor.pbip: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "artifacts": [ 4 | { 5 | "report": { 6 | "path": "Gateway Monitor.Report" 7 | } 8 | } 9 | ], 10 | "settings": { 11 | "enableAutoRecovery": true 12 | } 13 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This project aim to help organizations with multiple gateway clusters centralize all their gateway logs and reports into a central storage (ADLS Gen 2) and allow easy and quick exploration of those logs either by: 2 | 3 | - Easily access all the gateway logs without having to remote access to the gateway server 4 | - Explore the logs with a Power BI Report 5 | - Explore the logs using a SPARK Engine like Azure Synapse Analytics 6 | 7 | ![image](./Images/Architecture.png) 8 | 9 | **To quickly analyze the logs of gateways you dont need to run/deploy the powershell scripts** its possible to use the [disk Power BI template](./Gateway%20Monitor%20-%20FromDisk.pbit) and directly point to the exported gateway logs. The powershell & central storage is only required if you want to keep an fully automated gateway monitoring solution with history log data retention. 10 | 11 | Blog Post: https://www.linkedin.com/pulse/power-bi-gateway-monitoring-troubleshooting-solution-rui-romano/ 12 | 13 | # Setup 14 | 15 | ## Requirements 16 | 17 | - [Azure Data Lake Storage Account (ADLS Gen 2)](https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account) with [Hierarchical Namespace](https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account#enable-the-hierarchical-namespace) enabled 18 | - [PowerShell 7](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.2) on the Gateway Server, with the following modules installed: [Az.Accounts](https://www.powershellgallery.com/packages/Az.Accounts), [Az.Storage](https://www.powershellgallery.com/packages/Az.Storage) 19 | 20 | ### Azure Data Lake Storage Account (ADLS Gen 2) 21 | 22 | Using your Azure Subscription create a new Azure Data Lake Storage resource, follow the steps of following link: 23 | 24 | https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account 25 | 26 | Enable the [Hierarchical Namespace](https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account#enable-the-hierarchical-namespace) when creating the storage account. 27 | 28 | ![image](./Images/AzurePortal_StorageHierarchicalNamespace.png) 29 | 30 | ### PowerShell Modules 31 | 32 | On the gateway server ensure [PowerShell 7](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.2) is installed and install the following required modules: 33 | - [Az.Accounts](https://www.powershellgallery.com/packages/Az.Accounts) 34 | - [Az.Storage](https://www.powershellgallery.com/packages/Az.Storage) 35 | 36 | To install the modules above, open a PowerShell 7 prompt and run the following commands: 37 | 38 | ```powershell 39 | Install-Module Az.Accounts -MinimumVersion "2.8.0" -verbose 40 | 41 | Install-Module Az.Storage -MinimumVersion "4.6.0" -verbose 42 | 43 | Install-Module MicrosoftPowerBIMgmt -MinimumVersion "1.2.1077" -verbose 44 | ``` 45 | 46 | ## Deploy scripts to Gateway Server 47 | 48 | On each Gateway Server you should clone/copy this repo powershell scripts into a local folder (ex: c:\PBIGTWMonitor) 49 | 50 | ### Change Config.Json 51 | 52 | ![image](./Images/ConfigFile.png) 53 | 54 | Open the [Configuration file](./Config.json) and configure the following settings: 55 | 56 | - StorageAccountConnStr 57 | 58 | Open the ADLS Gen 2 storage account, go to "Access Keys" tab and copy the "Connection String" field: 59 | 60 | ![image](./Images/AzurePortal_StorageConnStr.png) 61 | 62 | - GatewayLogsPath 63 | 64 | Location of the Gateway logs & reports files. 65 | 66 | Confirm if the 'GatewayLogsPath' point to the correct path of the gateway logs - [more info](https://docs.microsoft.com/en-us/data-integration/gateway/service-gateway-log-files) 67 | 68 | The script automatically discovers the GatewayId and Number of Cores of the gateway and stores this information on the /metadata/gatewayproperties.json file, but its possible to override these values on the 'GatewayLogsPath' property of the configuration file: 69 | 70 | ![image](./Images/ConfigFile_PathProperty.png) 71 | 72 | - OutputPath 73 | 74 | Temporary location of the gateway logs before copying to blob storage 75 | 76 | - StorageAccountContainerName 77 | 78 | Name of the container in the storage account 79 | 80 | - StorageAccountContainerRootPath 81 | 82 | Root path on the storage container to where the log files will be written to 83 | 84 | ### Schedule Task 85 | 86 | Configure a Windows Schedule Task to run the script [Run.ps1](./Run.ps1) every hour/day 87 | 88 | ![image](./Images/Setup_ScheduleTask.png) 89 | 90 | # Power BI Template 91 | 92 | There are two Power BI templates available: 93 | 94 | - [Lake Template](./Gateway%20Monitor%20-%20FromLake.pbit) - When gateway log data resides in the ADLS Gen 2 lake 95 | - [Disk Template](./Gateway%20Monitor%20-%20FromDisk.pbit) - When gateway log data reside in disk, useful to analyze the logs directly from the gateway server or [gateway log export](https://learn.microsoft.com/en-us/data-integration/gateway/service-gateway-tshoot#collect-logs-from-the-on-premises-data-gateway-app). You dont need to deploy the powershell scripts to use this template, only need to have access to gateway logs. 96 | 97 | 98 | ## Template Parameters 99 | 100 | After opening the Power BI Template file (.pbit) the following parameter window will popup: 101 | 102 | ![image](./Images/PBI_TemplateParams.png) 103 | ![image](./Images/PBI_TemplateParamsDisk.png) 104 | 105 | | Parameter | Description 106 | | ----------- | -------- 107 | | DataLocation | URL Path to the root folder including the container name (default: '/pbigatewaymonitor/RAW'), using the Distributed File System (DFS) endpoint of the storage account, ex: https://storage.dfs.core.windows.net/pbigatewaymonitor/raw; Or the path to the disk location with logs if the disk template is used 108 | | NumberDays | Filter to the log files to be fetched, if '10' Power BI will read only the latest 10 days of logs/queries/counters Default: null (all days) 109 | | SinceDate | Only read log/query files since the specified date. This parameter overrides 'NumberDays' Default: null (all days) 110 | | MaxLogTextLength | Max size of text column of logs. Default: 1000 111 | | LogFilters | Comma separated file names of log files to be fetched. Default: "gatewayerrors,gatewayinfo" If 'None' log files will be excluded 112 | | GatewayFilters | Comma separated gateway id's. Default: All Gateways 113 | 114 | ## Logs Page 115 | 116 | ![image](./Images/PBI_LogPage.png) 117 | 118 | ## Queries Page 119 | 120 | ![image](./Images/PBI_QueriesPage.png) 121 | 122 | ## Gateway Profile 123 | ![image](./Images/PBI_GatewayProfile.png) 124 | 125 | ## Counters Page 126 | 127 | ![image](./Images/PBI_Counters.png) 128 | 129 | ## Requests Page 130 | 131 | ![image](./Images/PBI_RequestsPage.png) 132 | 133 | ## Mashups Profiles Page 134 | 135 | ![image](./Images/PBI_MashupProfiles.png) 136 | 137 | ## Mashups Logs Page 138 | 139 | ![image](./Images/PBI_MashupLogs.png) 140 | 141 | ## Theme 142 | Theme Background Images here: https://alluringbi.com/gallery/ -------------------------------------------------------------------------------- /Run.cmd: -------------------------------------------------------------------------------- 1 | pwsh .\run.ps1 -------------------------------------------------------------------------------- /Run.ps1: -------------------------------------------------------------------------------- 1 | #requires -Version 7 -Modules Az.Accounts, Az.Storage 2 | 3 | param( 4 | [string]$configFilePath = ".\Config.json" 5 | , 6 | [array]$scriptsToRun = @( 7 | ".\UploadGatewayLogs.ps1" 8 | ) 9 | ) 10 | 11 | $ErrorActionPreference = "Stop" 12 | 13 | $currentPath = (Split-Path $MyInvocation.MyCommand.Definition -Parent) 14 | 15 | Set-Location $currentPath 16 | 17 | Import-Module "$currentPath\Utils.psm1" -Force 18 | 19 | Write-Host "Current Path: $currentPath" 20 | 21 | Write-Host "Config Path: $configFilePath" 22 | 23 | if (Test-Path $configFilePath) { 24 | 25 | $config = Get-Content $configFilePath | ConvertFrom-Json 26 | 27 | # Default Values 28 | 29 | if (!$config.OutputPath) { 30 | $config | Add-Member -NotePropertyName "OutputPath" -NotePropertyValue ".\\Data" -Force 31 | } 32 | } 33 | else { 34 | throw "Cannot find config file '$configFilePath'" 35 | } 36 | 37 | try { 38 | 39 | foreach ($scriptToRun in $scriptsToRun) 40 | { 41 | try { 42 | Write-Host "Running '$scriptToRun'" 43 | 44 | & $scriptToRun -config $config 45 | } 46 | catch { 47 | Write-Error "Error on '$scriptToRun' - $($_.Exception.ToString())" -ErrorAction Continue 48 | } 49 | } 50 | } 51 | catch { 52 | 53 | $ex = $_.Exception 54 | 55 | throw 56 | } 57 | -------------------------------------------------------------------------------- /Tools - Get Dataset Refresh History.ps1: -------------------------------------------------------------------------------- 1 | #Requires -Modules @{ ModuleName="MicrosoftPowerBIMgmt"; ModuleVersion="1.2.1026" } 2 | 3 | param( 4 | $datasets = @( 5 | @{workspaceId = "ff9f6b54-83e8-4aa5-901b-a7675e001c77";datasetId = "4aee5203-4d36-4b2e-87b4-904a7bd38016"} 6 | , 7 | @{workspaceId = "7331d174-e08f-4802-acba-898b8cecbc75";datasetId = "ecb5768c-3057-433a-91c0-c56bece634ae"} 8 | ) 9 | ) 10 | 11 | $ErrorActionPreference = "Stop" 12 | 13 | $currentPath = (Split-Path $MyInvocation.MyCommand.Definition -Parent) 14 | 15 | # Get running refreshes, only 1 operation is allowed "Only one refresh operation at a time is accepted for a dataset. If there's a current running refresh operation and another is submitted" 16 | 17 | Connect-PowerBIServiceAccount 18 | 19 | foreach ($dataset in $datasets) 20 | { 21 | Write-Host "Workspace: $($dataset.workspaceId); Dataset: $($dataset.datasetId)" 22 | 23 | Write-Host "Gateway Info" 24 | 25 | $bindedGateways = Invoke-PowerBIRestMethod -url "groups/$($dataset.workspaceId)/datasets/$($dataset.datasetId)/Default.GetBoundGatewayDatasources" -method Get | ConvertFrom-Json | select -ExpandProperty value 26 | 27 | $bindedGateways | Format-Table 28 | 29 | Write-Host "Refresh History" 30 | 31 | # https://docs.microsoft.com/en-us/rest/api/power-bi/datasets/get-refresh-history-in-group 32 | 33 | $refreshes = Invoke-PowerBIRestMethod -url "groups/$($dataset.workspaceId)/datasets/$($dataset.datasetId)/refreshes?`$top=5" -method Get | ConvertFrom-Json | select -ExpandProperty value 34 | 35 | $refreshes | Format-Table 36 | } -------------------------------------------------------------------------------- /UploadGatewayLogs.ps1: -------------------------------------------------------------------------------- 1 | param( 2 | [psobject]$config 3 | , 4 | [string]$stateFilePath 5 | ) 6 | 7 | function ProcessLogFiles ($logFiles, $storagePath, $executionDate, $tempPath) 8 | { 9 | Write-Host "Files modified since last run: $($logFiles.Count)" 10 | 11 | foreach ($logFile in $logFiles) { 12 | 13 | $storagePathTemp = $storagePath 14 | 15 | # Try to parse the date out of the file name 16 | 17 | if ($executionDate) 18 | { 19 | $dateRegExMatches = ($logFile.Name | Select-String -pattern ".+(\d{8}).\d+").Matches 20 | 21 | if ($dateRegExMatches) 22 | { 23 | $fileDateStr = $dateRegExMatches.Groups[1].Value 24 | 25 | $fileDate = [datetime]::ParseExact($fileDateStr, "yyyyMMdd", [Globalization.CultureInfo]::InvariantCulture) 26 | } 27 | else 28 | { 29 | $fileDate = $executionDate 30 | } 31 | 32 | if ($fileDate) 33 | { 34 | $storagePathTemp = ("$storagePath/{0:yyyy}/{0:MM}/{0:dd}" -f $fileDate) 35 | } 36 | } 37 | 38 | if ($tempPath) 39 | { 40 | # Local copy the file, because it could be blocked by Gateway 41 | 42 | Write-Host "Copying file: '$($logFile.FullName)' to '$tempPath'" 43 | 44 | $fileOutputPath = "$tempPath\$($storagePathTemp.Replace("/", "\"))\$($logFile.Name)" 45 | 46 | New-Item -ItemType Directory -Path (Split-Path $fileOutputPath -Parent) -ErrorAction SilentlyContinue | Out-Null 47 | 48 | Copy-Item -Path $logFile.FullName -Destination $fileOutputPath -Force 49 | } 50 | else{ 51 | $fileOutputPath = $logFile.FullName 52 | } 53 | 54 | if ($config.StorageAccountConnStr) 55 | { 56 | Write-Host "Sync '$fileOutputPath' to BlobStorage" 57 | 58 | # Send to Storage 59 | 60 | Add-FileToBlobStorage -storageRootPath $storagePathTemp -filePath $fileOutputPath -storageAccountConnStr $config.StorageAccountConnStr -storageContainerName $config.StorageAccountContainerName 61 | 62 | # If storage account is configured and upload is done the file is deleted from the local file system 63 | 64 | if ($tempPath -and $fileOutputPath) 65 | { 66 | Write-Host "Deleting local file copy: '$fileOutputPath'" 67 | 68 | # Remove the local copy 69 | 70 | Remove-Item $fileOutputPath -Force 71 | } 72 | } 73 | } 74 | } 75 | 76 | 77 | try { 78 | Write-Host "Upload - GatewayLogs Start" 79 | 80 | $stopwatch = [System.Diagnostics.Stopwatch]::new() 81 | $stopwatch.Start() 82 | 83 | $runDate = [datetime]::UtcNow 84 | 85 | $lastRunDate = $null 86 | 87 | $localOutputPath = $config.OutputPath 88 | 89 | # Ensure folders 90 | @($localOutputPath) |% { 91 | New-Item -ItemType Directory -Path $_ -ErrorAction SilentlyContinue | Out-Null 92 | } 93 | 94 | if (!$stateFilePath) { 95 | $stateFilePath = ".\state.json" 96 | } 97 | 98 | if (Test-Path $stateFilePath) { 99 | $state = Get-Content $stateFilePath | ConvertFrom-Json 100 | } 101 | else { 102 | $state = New-Object psobject 103 | } 104 | 105 | if ($state.GatewayLogs.LastRun) { 106 | if (!($state.GatewayLogs.LastRun -is [datetime])) { 107 | $state.GatewayLogs.LastRun = [datetime]::Parse($state.GatewayLogs.LastRun).ToUniversalTime() 108 | } 109 | $lastRunDate = $state.GatewayLogs.LastRun 110 | } 111 | else { 112 | $state | Add-Member -NotePropertyName "GatewayLogs" -NotePropertyValue @{"LastRun" = $null } -Force 113 | } 114 | 115 | Write-Host "LastRun: '$lastRunDate'" 116 | 117 | if (!(Test-Path $config.GatewayLogsPath)) 118 | { 119 | throw "Cannot find gateway logs path '$($config.GatewayLogsPath)' - https://docs.microsoft.com/en-us/data-integration/gateway/service-gateway-log-files" 120 | } 121 | 122 | foreach ($obj in $config.GatewayLogsPath) { 123 | 124 | $gatewayProperties = @{ 125 | GatewayObjectId = $null 126 | } 127 | 128 | $gatewayClusters = $null 129 | 130 | # Manual metadata 131 | 132 | if ($obj -is [PSCustomObject]) 133 | { 134 | $path = $obj.Path 135 | 136 | $gatewayProperties.GatewayObjectId = $obj.GatewayId 137 | $gatewayProperties.GatewayName = $obj.GatewayName 138 | $gatewayProperties.GatewayCluster = $obj.GatewayCluster 139 | $gatewayProperties.Server = $obj.Server 140 | $gatewayProperties.Version = $obj.Version 141 | $gatewayProperties.NumberOfCores = $obj.NumberOfCores 142 | } 143 | else { 144 | $path = $obj 145 | 146 | if (Test-Path "$path\GatewayProperties.txt") 147 | { 148 | $gatewayProperties = Get-Content "$path\GatewayProperties.txt" | ConvertFrom-Json 149 | } 150 | 151 | if (Test-Path "$path\GatewayClusters.txt") 152 | { 153 | $gatewayClusters = Get-Content "$path\GatewayClusters.txt" | ConvertFrom-Json 154 | } 155 | } 156 | 157 | # If GatewayObjectId is not specified try to find it in the logs 158 | 159 | if (!$gatewayProperties.GatewayObjectId) 160 | { 161 | $reportFile = Get-ChildItem -path $path -Recurse |? {$_.Name -ilike "*Report_*.log"} | Sort-Object Length | Select -first 1 162 | 163 | if (!$reportFile) 164 | { 165 | throw "Cannot find any report ('*Report_*.log') file on '$path' to infer the GatewayId. Please ensure there is at least one report. If its a newly installed Gateway you may need to run a refresh and wait a couple of minutes." 166 | } 167 | 168 | $gatewayIdFromCSV = Get-Content -path $reportFile.FullName -First 2 | ConvertFrom-Csv | Select -ExpandProperty GatewayObjectId 169 | 170 | $gatewayProperties.GatewayObjectId = $gatewayIdFromCSV 171 | 172 | } 173 | 174 | # Try to get the gateway core count 175 | 176 | if (!$gatewayProperties.NumberOfCores -and !$gatewayProperties.CpuRelatedInfo) 177 | { 178 | try { 179 | $serverCPU = (Get-ComputerInfo -Property CsProcessors).CsProcessors 180 | 181 | $gatewayProperties.NumberOfCores = $serverCPU.NumberOfLogicalProcessors 182 | } 183 | catch { 184 | Write-Warning "Error getting the server core count" 185 | } 186 | } 187 | 188 | $gatewayId = $gatewayProperties.GatewayObjectId 189 | 190 | if (!$gatewayId) 191 | { 192 | throw "Gateway Id is not defined." 193 | } 194 | 195 | $outputPathLogs = ("$($config.StorageAccountContainerRootPath)/{0:gatewayid}/logs" -f $gatewayId) 196 | 197 | $outputPathMetadata = ("$($config.StorageAccountContainerRootPath)/{0:gatewayid}/metadata" -f $gatewayId) 198 | 199 | $outputPathReports = ("$($config.StorageAccountContainerRootPath)/{0:gatewayid}/reports" -f $gatewayId) 200 | 201 | # GatewayProperties json file 202 | 203 | $gatewayMetadataFilePath = "$localOutputPath\$($config.StorageAccountContainerRootPath)\$gatewayId\metadata\GatewayProperties.json" 204 | 205 | New-Item -ItemType Directory -Path (Split-Path $gatewayMetadataFilePath -Parent) -ErrorAction SilentlyContinue | Out-Null 206 | 207 | ConvertTo-Json $gatewayProperties -Depth 5 | Out-File $gatewayMetadataFilePath -force -Encoding utf8 208 | 209 | ProcessLogFiles -logFiles (Get-ChildItem $gatewayMetadataFilePath) -storagePath $outputPathMetadata 210 | 211 | # GatewayClusters json file 212 | 213 | if ($gatewayClusters) 214 | { 215 | $gatewayClustersFilePath = "$localOutputPath\$($config.StorageAccountContainerRootPath)\$gatewayId\metadata\GatewayClusters.json" 216 | 217 | New-Item -ItemType Directory -Path (Split-Path $gatewayClustersFilePath -Parent) -ErrorAction SilentlyContinue | Out-Null 218 | 219 | ConvertTo-Json $gatewayClusters -Depth 5 | Out-File $gatewayClustersFilePath -force -Encoding utf8 220 | 221 | ProcessLogFiles -logFiles (Get-ChildItem $gatewayClustersFilePath) -storagePath $outputPathMetadata 222 | } 223 | 224 | # Gateway Logs 225 | 226 | $logFiles = @(Get-ChildItem -File -Path "$path\*.log" -ErrorAction SilentlyContinue) 227 | 228 | Write-Host "Gateway log count: $($logFiles.Count)" 229 | 230 | $logFiles = @($logFiles | ? { $_.LastWriteTimeUtc -gt $lastRunDate -or $lastRunDate -eq $null }) 231 | 232 | ProcessLogFiles -logFiles $logFiles -storagePath $outputPathLogs -executionDate $runDate -tempPath $localOutputPath 233 | 234 | # Gateway Reports 235 | 236 | $logFiles = @(Get-ChildItem -File -Path "$path\*Report_*.log" -Recurse -ErrorAction SilentlyContinue) 237 | 238 | Write-Host "Gateway Report log count: $($logFiles.Count)" 239 | 240 | $logFiles = @($logFiles | ? { $_.LastWriteTimeUtc -gt $lastRunDate -or $lastRunDate -eq $null }) 241 | 242 | ProcessLogFiles -logFiles $logFiles -storagePath $outputPathReports -tempPath $localOutputPath 243 | 244 | # Gateway Config Properties 245 | 246 | $logFiles = @(Get-ChildItem -File -Path "$path\*ConfigurationProperties.json" -ErrorAction SilentlyContinue) 247 | 248 | Write-Host "Gateway Config file count: $($logFiles.Count)" 249 | 250 | $logFiles = @($logFiles | ? { $_.LastWriteTimeUtc -gt $lastRunDate -or $lastRunDate -eq $null }) 251 | 252 | ProcessLogFiles -logFiles $logFiles -storagePath $outputPathMetadata -tempPath $localOutputPath 253 | } 254 | 255 | # Save state 256 | 257 | $state.GatewayLogs.LastRun = $runDate.ToString("o") 258 | 259 | New-Item -Path (Split-Path $stateFilePath -Parent) -ItemType Directory -Force -ErrorAction SilentlyContinue | Out-Null 260 | 261 | ConvertTo-Json $state | Out-File $stateFilePath -force -Encoding utf8 262 | 263 | } 264 | finally { 265 | $stopwatch.Stop() 266 | 267 | Write-Host "Ellapsed: $($stopwatch.Elapsed.TotalSeconds)s" 268 | } -------------------------------------------------------------------------------- /Utils.psm1: -------------------------------------------------------------------------------- 1 | function Add-FolderToBlobStorage { 2 | [cmdletbinding()] 3 | param 4 | ( 5 | [string] 6 | $storageAccountName, 7 | [string] 8 | $storageAccountKey, 9 | [string] 10 | $storageAccountConnStr, 11 | [string] 12 | $storageContainerName, 13 | [string] 14 | $storageRootPath, 15 | [string] 16 | $folderPath, 17 | [string] 18 | $rootFolderPath, 19 | [bool] 20 | $ensureContainer = $true 21 | ) 22 | 23 | if ($storageAccountConnStr) { 24 | $ctx = New-AzStorageContext -ConnectionString $storageAccountConnStr 25 | } 26 | else { 27 | $ctx = New-AzStorageContext -StorageAccountName $storageAccountName -StorageAccountKey $storageAccountKey 28 | } 29 | 30 | if ($ensureContainer) { 31 | New-AzStorageContainer -Context $ctx -Name $storageContainerName -Permission Off -ErrorAction SilentlyContinue | Out-Null 32 | } 33 | 34 | $files = @(Get-ChildItem -Path $folderPath -Filter *.* -Recurse -File) 35 | 36 | Write-Host "Adding folder '$folderPath' (files: $($files.Count)) to blobstorage '$storageAccountName/$storageContainerName/$storageRootPath'" 37 | 38 | if (!$rootFolderPath) 39 | { 40 | $rootFolderPath = $folderPath 41 | } 42 | 43 | foreach ($file in $files) { 44 | $filePath = $file.FullName 45 | 46 | Add-FileToBlobStorageInternal -ctx $ctx -filePath $filePath -storageRootPath $storageRootPath -rootFolderPath $rootFolderPath 47 | } 48 | } 49 | 50 | function Add-FileToBlobStorage { 51 | [cmdletbinding()] 52 | param 53 | ( 54 | [string] 55 | $storageAccountName, 56 | [string] 57 | $storageAccountKey, 58 | [string] 59 | $storageAccountConnStr, 60 | [string] 61 | $storageContainerName, 62 | [string] 63 | $storageRootPath, 64 | [string] 65 | $filePath, 66 | [string] 67 | $rootFolderPath, 68 | [bool] 69 | $ensureContainer = $true 70 | ) 71 | 72 | if ($storageAccountConnStr) { 73 | $ctx = New-AzStorageContext -ConnectionString $storageAccountConnStr 74 | } 75 | else { 76 | $ctx = New-AzStorageContext -StorageAccountName $storageAccountName -StorageAccountKey $storageAccountKey 77 | } 78 | 79 | if ($ensureContainer) { 80 | New-AzStorageContainer -Context $ctx -Name $storageContainerName -Permission Off -ErrorAction SilentlyContinue | Out-Null 81 | } 82 | 83 | Add-FileToBlobStorageInternal -ctx $ctx -filePath $filePath -storageRootPath $storageRootPath -rootFolderPath $rootFolderPath 84 | 85 | } 86 | 87 | function Add-FileToBlobStorageInternal { 88 | param 89 | ( 90 | $ctx, 91 | [string] 92 | $storageRootPath, 93 | [string] 94 | $filePath, 95 | [string] 96 | $rootFolderPath 97 | ) 98 | 99 | if (Test-Path $filePath) { 100 | Write-Host "Adding file '$filePath' files to blobstorage '$storageAccountName/$storageContainerName/$storageRootPath'" 101 | 102 | $filePath = Resolve-Path $filePath 103 | 104 | $filePath = $filePath.ToLower() 105 | 106 | $fileName = (Split-Path $filePath -Leaf) 107 | 108 | if ($rootFolderPath) { 109 | $rootFolderPath = Resolve-Path $rootFolderPath 110 | $rootFolderPath = $rootFolderPath.ToLower() 111 | 112 | $parentFolder = (Split-Path $filePath -Parent) 113 | $relativeFolder = $parentFolder.Replace($rootFolderPath, "").Replace("\", "/").TrimStart("/").Trim(); 114 | } 115 | 116 | if (!([string]::IsNullOrEmpty($relativeFolder))) { 117 | $blobName = "$storageRootPath/$relativeFolder/$fileName" 118 | } 119 | else { 120 | $blobName = "$storageRootPath/$fileName" 121 | } 122 | 123 | Set-AzStorageBlobContent -File $filePath -Container $storageContainerName -Blob $blobName -Context $ctx -Force | Out-Null 124 | } 125 | else { 126 | Write-Host "File '$filePath' dont exist" 127 | } 128 | } 129 | 130 | function Get-ArrayInBatches 131 | { 132 | [cmdletbinding()] 133 | param 134 | ( 135 | [array]$array 136 | , 137 | [int]$batchCount 138 | , 139 | [ScriptBlock]$script 140 | , 141 | [string]$label = "Get-ArrayInBatches" 142 | ) 143 | 144 | $skip = 0 145 | 146 | do 147 | { 148 | $batchItems = @($array | Select -First $batchCount -Skip $skip) 149 | 150 | if ($batchItems) 151 | { 152 | Write-Host "[$label] Batch: $($skip + $batchCount) / $($array.Count)" 153 | 154 | Invoke-Command -ScriptBlock $script -ArgumentList @(,$batchItems) 155 | 156 | $skip += $batchCount 157 | } 158 | 159 | } 160 | while($batchItems.Count -ne 0 -and $batchItems.Count -ge $batchCount) 161 | } 162 | --------------------------------------------------------------------------------