├── .github └── workflows │ └── action.yml ├── LICENSE ├── NOTICE ├── README.md ├── default ├── app.conf ├── data │ └── ui │ │ ├── nav │ │ └── default.xml │ │ └── views │ │ ├── ClusterMasterJobs.xml │ │ ├── data_model_rebuild_monitor.xml │ │ ├── data_model_status.xml │ │ ├── detect_excessive_search_use.xml │ │ ├── heavy_forwarder_analysis.xml │ │ ├── heavyforwarders_max_data_queue_sizes_by_name.xml │ │ ├── heavyforwarders_max_data_queue_sizes_by_name_v8.xml │ │ ├── hec_performance.xml │ │ ├── indexer_data_spread.xml │ │ ├── indexer_max_data_queue_sizes_by_name.xml │ │ ├── indexer_max_data_queue_sizes_by_name_v8.xml │ │ ├── issues_per_sourcetype.xml │ │ ├── knowledge_objects_by_app.xml │ │ ├── knowledge_objects_by_app_drilldown.xml │ │ ├── lookup_audit.xml │ │ ├── lookups_in_use_finder.xml │ │ ├── rolled_buckets_by_index.xml │ │ ├── search_head_scheduledsearches_distribution.xml │ │ ├── smartstore_stats.xml │ │ ├── splunk_forwarder_data_balance_tuning.xml │ │ ├── splunk_forwarder_output_tuning.xml │ │ ├── splunk_introspection_io_stats.xml │ │ ├── troubleshooting_indexer_cpu.xml │ │ ├── troubleshooting_indexer_cpu_drilldown.xml │ │ ├── troubleshooting_resource_usage_per_user.xml │ │ └── troubleshooting_resource_usage_per_user_drilldown.xml ├── macros.conf ├── props.conf ├── savedsearches.conf └── transforms.conf ├── lookups ├── splunkadmins_datamodels.csv ├── splunkadmins_eventtypes.csv ├── splunkadmins_hec_reply_code_lookup.csv ├── splunkadmins_indexes_per_role.csv ├── splunkadmins_indexlist.csv ├── splunkadmins_indexlist_by_cluster.csv ├── splunkadmins_lookupfile_owners.csv ├── splunkadmins_macros.csv ├── splunkadmins_rmd5_to_savedsearchname.csv ├── splunkadmins_tags.csv └── splunkadmins_userlist_indexinfo.csv ├── metadata └── default.meta └── static ├── appIcon.png ├── appIconAlt.png ├── appIconAlt_2x.png ├── appIcon_2x.png ├── appLogo.png └── appLogo_2x.png /.github/workflows/action.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to generate build 2 | name: "Generate build, run app inspect and update splunklib" 3 | 4 | on: push 5 | 6 | jobs: 7 | pre-release: 8 | name: "Run on push - Add Utilities & App Inspect" 9 | runs-on: "ubuntu-latest" 10 | 11 | steps: 12 | - uses: VatsalJagani/splunk-app-action@v4 13 | with: 14 | my_github_token: ${{ secrets.MY_GITHUB_TOKEN }} 15 | splunkbase_username: ${{ secrets.SPLUNKBASE_USERNAME }} 16 | splunkbase_password: ${{ secrets.SPLUNKBASE_PASSWORD }} 17 | to_make_permission_changes: true 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Gareth Anderson 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /default/app.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Splunk app configuration file 3 | # 4 | 5 | [install] 6 | is_configured = 0 7 | 8 | [ui] 9 | is_visible = 1 10 | label = SplunkAdmins 11 | # allow 9.1 and above to use themes 12 | supported_themes = light,dark 13 | 14 | [launcher] 15 | author = Gareth Anderson 16 | description = Alerts and dashboards as described in the Splunk 2017 conf presentation How did you get so big? 17 | version = 4.0.4 18 | 19 | [package] 20 | id = SplunkAdmins 21 | check_for_updates = true 22 | 23 | -------------------------------------------------------------------------------- /default/data/ui/views/ClusterMasterJobs.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 6 | 7 | -15m 8 | now 9 | 10 | 11 | 12 | 13 | 2m 14 | 15 |
16 | 17 | 18 | Job Count 19 | 20 | 21 | index=_internal `splunkadmins_clustermaster_oshost` sourcetype=splunkd `splunkadmins_splunkd_source` *CMRepJob running job | timechart span=$span$ count by job 22 | $time.earliest$ 23 | $time.latest$ 24 | 1 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | Fixup Jobs 64 | 65 | 66 | index=_internal `splunkadmins_metrics_source` sourcetype=splunkd name=cmmaster_service `splunkadmins_clustermaster_oshost` group=subtask_counts 67 | | timechart max(to_fix_gen), max(to_fix_rep_factor), max(to_fix_search_factor) span=$span$ 68 | $time.earliest$ 69 | $time.latest$ 70 | 1 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 |
108 | -------------------------------------------------------------------------------- /default/data/ui/views/data_model_rebuild_monitor.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Originally based on the work on URL https://conf.splunk.com/files/2017/slides/running-enterprise-security-at-capacity-tuning-es-with-data-model-acceleration.pdf modified to work without the macros and corrected the datamodel sizing (and misc tweaks). 4 |
5 | 6 | 7 | 8 | | rest /services/admin/summarization by_tstats=t splunk_server=local count=0 9 | | eval datamodel=replace('summary.id',"DM_".'eai:acl.app'."_","") 10 | | fields datamodel 11 | | sort 100 + datamodel 12 | 13 | datamodel 14 | datamodel 15 | 16 | 17 | 18 | acceleration.earliest_time 19 | acceleration.earliest_time 20 | 21 | | rest /services/configs/conf-datamodels| search title=$dm$ | fields acceleration.earliest_time 22 | 0 23 | 24 | 25 | true 26 | 27 |
28 | 29 | 30 | 31 |

$dm$ data modelconfig

32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | | rest /services/configs/conf-datamodels 40 | | search title=$dm$ 41 | | fields acceleration.earliest_time 42 | @d 43 | now 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | | rest /services/configs/conf-datamodels 62 | | search title=$dm$ 63 | | fields acceleration.backfill_time 64 | @d 65 | now 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | | rest /services/admin/summarization by_tstats=t splunk_server=local count=0 84 | | eval datamodel=replace('summary.id',"DM_".'eai:acl.app'."_","") 85 | | fields summary.complete, datamodel 86 | | rename summary.complete AS complete 87 | | search datamodel=$dm$ 88 | | eval complete(%)=round(complete*100,1)."%" 89 | | fields complete(%) 90 | 0.000 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | | rest /services/configs/conf-datamodels 110 | | search title=$dm$ 111 | | fields acceleration.max_concurrent 112 | @d 113 | now 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | | rest /services/configs/conf-datamodels 132 | | search title=$dm$ 133 | | fields acceleration.max_time 134 | @d 135 | now 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | ```The authors original attempt of | `datamodel("Splunk_Audit", "Datamodel_Acceleration | `drop_dm_object_name("Datamodel_Acceleration")` Just did not appear to show accurate numbers when compared to the filesystem of the indexers 154 | The previous attempt at this number via | rest "/services/admin/introspection--disk-objects--summaries?count=-1" ... worked fine *unless* there were multiple search head GUID's in the introspection data in which case it seems to return 1 set only (resulting in highly inaccurate numbers in some cases) 155 | Now querying the introspection data instead as that provides consistently accurate numbers``` 156 | index=_introspection `indexerhosts` component=summaries "data.name"=*$dm$ 157 | | stats latest(data.total_size) AS size by data.search_head_guid, data.related_indexes_count, data.related_indexes, host 158 | | stats sum(size) AS size 159 | @d 160 | now 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 |

$dm$ data model acceleration state

184 | 185 | 186 | 187 | 188 | 189 | $dm$ event counts - Monitor lag and backfill 190 | 191 | Backfill view over the last 2 hours 192 | 193 | | tstats prestats=t summariesonly=t allow_old_summaries=t count from datamodel=$dm$ by _time span=10s 194 | | timechart count span=10s 195 | -2h 196 | now 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | Backfill view over time range of DM acceleration (and -1w) 226 | 227 | |tstats prestats=t allow_old_summaries=t summariesonly=t count from datamodel=$dm$ by _time span=4h| timechart count span=4h 228 | $earliest_token$-1w 229 | now 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | $dm$ recent acceleration jobs 260 | 261 | index=_internal source=*scheduler.log _ACCELERATE_DM_*$dm$_ACCELERATE_ | eval scheduled=strftime(scheduled_time,"%c") 262 | | stats values(scheduled) as scheduled, values(scheduled_time) as scheduled_time, list(status) as statuses, values(run_time) as run_time by savedsearch_name sid | sort - scheduled_time | 263 | eval done=if(isnull(run_time),"running","done") 264 | | eval run_time=tostring(if(isnull(run_time),now()-scheduled_time,run_time),"duration") | fields - scheduled_time savedsearch_name sid 265 | @d 266 | now 267 | 268 | 269 | 270 | 271 | 272 | 273 |
274 | 275 | 276 |
277 | -------------------------------------------------------------------------------- /default/data/ui/views/data_model_status.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Originally based on the work on URL https://conf.splunk.com/files/2017/slides/running-enterprise-security-at-capacity-tuning-es-with-data-model-acceleration.pdf modified to work without the macros (and misc tweaks) 4 |
5 | 6 | 7 | 8 | -4h@m 9 | now 10 | 11 | 12 |
13 | 14 | 15 | 16 | Skipped searches ($timepicker1.earliest$ to $timepicker1.latest$) 17 | 18 | index=_internal `searchheadhosts` sourcetype=scheduler status="skipped" 19 | | eval type=if(match(savedsearch_name,"^_ACCELERATE_"),"DM","non-DM") 20 | | eval reason = if(isnull(reason) OR reason == "", "none", reason) 21 | | eval combo=type . " - " . reason 22 | | timechart span=5m count by combo 23 | $timepicker1.earliest$ 24 | $timepicker1.latest$ 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | Deferred & Skipped searches ($timepicker1.earliest$ to $timepicker1.latest$) 54 | 55 | index=_internal `searchheadhosts` sourcetype=scheduler status=continued OR status=skipped 56 | | eval type=if(match(savedsearch_name,"^_ACCELERATE_"),"DM","non-DM") 57 | | eval status=replace(status,"continued","deferred") 58 | | eval combo=type . "-" . status 59 | | timechart span=5m count by combo 60 | $timepicker1.earliest$ 61 | $timepicker1.latest$ 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | Top Accelerations by Run Duration (on this search head / cluster) 93 | 94 | | rest /services/admin/summarization by_tstats=t splunk_server=local count=0 95 | | eval datamodel=replace('summary.id',(("DM_" . 'eai:acl.app') . "_"),"") 96 | | join max=1 overwrite=1 type=left usetime=0 datamodel 97 | [| rest /services/data/models splunk_server=local count=0 98 | | table title acceleration.cron_schedule eai:digest 99 | | rename title as datamodel 100 | | rename "acceleration.cron_schedule" as cron] 101 | | table datamodel eai:acl.app summary.access_time summary.is_inprogress summary.size summary.latest_time summary.complete summary.buckets_size summary.buckets cron summary.last_error summary.time_range summary.id summary.mod_time eai:digest summary.earliest_time summary.last_sid summary.access_count 102 | | rename "eai:digest" as digest, "summary.earliest_time" as earliest, "summary.id" as summary_id, "summary.latest_time" as latest, "summary.time_range" as retention 103 | | rename "eai:acl.app" as app, "summary.access_count" as access_count, "summary.access_time" as access_time, "summary.buckets" as buckets, "summary.buckets_size" as buckets_size, "summary.complete" as complete, "summary.is_inprogress" as is_inprogress, "summary.last_error" as last_error, "summary.last_sid" as last_sid, "summary.mod_time" as mod_time, "summary.size" as size, "summary.*" as "*", "eai:acl.*" as "*" 104 | | sort datamodel 105 | | rename access_count as "Datamodel_Acceleration.access_count", access_time as "Datamodel_Acceleration.access_time", app as "Datamodel_Acceleration.app", buckets as "Datamodel_Acceleration.buckets", buckets_size as "Datamodel_Acceleration.buckets_size", complete as "Datamodel_Acceleration.complete", cron as "Datamodel_Acceleration.cron", datamodel as "Datamodel_Acceleration.datamodel", digest as "Datamodel_Acceleration.digest", earliest as "Datamodel_Acceleration.earliest", is_inprogress as "Datamodel_Acceleration.is_inprogress", last_error as "Datamodel_Acceleration.last_error", last_sid as "Datamodel_Acceleration.last_sid", latest as "Datamodel_Acceleration.latest", mod_time as "Datamodel_Acceleration.mod_time", retention as "Datamodel_Acceleration.retention", size as "Datamodel_Acceleration.size", summary_id as "Datamodel_Acceleration.summary_id" 106 | | rename "Datamodel_Acceleration.access_count" as access_count, "Datamodel_Acceleration.access_time" as access_time, "Datamodel_Acceleration.app" as app, "Datamodel_Acceleration.buckets" as buckets, "Datamodel_Acceleration.buckets_size" as buckets_size, "Datamodel_Acceleration.complete" as complete, "Datamodel_Acceleration.cron" as cron, "Datamodel_Acceleration.datamodel" as datamodel, "Datamodel_Acceleration.digest" as digest, "Datamodel_Acceleration.earliest" as earliest, "Datamodel_Acceleration.is_inprogress" as is_inprogress, "Datamodel_Acceleration.last_error" as last_error, "Datamodel_Acceleration.last_sid" as last_sid, "Datamodel_Acceleration.latest" as latest, "Datamodel_Acceleration.mod_time" as mod_time, "Datamodel_Acceleration.retention" as retention, "Datamodel_Acceleration.size" as size, "Datamodel_Acceleration.summary_id" as summary_id, "Datamodel_Acceleration.*" as "*" 107 | | join max=1 overwrite=1 type=outer usetime=0 last_sid 108 | [| rest splunk_server=* count=0 /services/search/jobs reportSearch=summarize* 109 | | rename sid as last_sid 110 | | fields last_sid,runDuration] 111 | | eval "size(MB)"=round((size / 1048576),1) 112 | | eval "retention(days)"=if((retention == 0),"unlimited",(retention / 86400)) 113 | | eval "complete(%)"=round((complete * 100),1) 114 | | eval "runDuration(s)"=round(runDuration,1) 115 | | sort 18 - runDuration 116 | | table datamodel,runDuration 117 | | eval concurrent_threshold=300 118 | | eval deferred_threshold=600 119 | | eval skipped_threshold=900 120 | 0.000 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | All skipped scheduled searches ($timepicker1.earliest$ to $timepicker1.latest$) 156 | 157 | index=_internal `searchheadhosts` sourcetype=scheduler status="skipped" 158 | | table _time status savedsearch_name 159 | | sort - _time 160 | $timepicker1.earliest$ 161 | $timepicker1.latest$ 162 | 163 | 164 | 165 |
166 | 167 | 168 |
169 | -------------------------------------------------------------------------------- /default/data/ui/views/detect_excessive_search_use.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Detect repeated search use for the same search query by a particular user during a period of time 4 |
5 | 6 | 7 | 8 | -4h@m 9 | now 10 | 11 | 12 | 13 | 14 | 10m 15 | 16 |
17 | 18 | 19 | Searches occurring more often than expected in the audit logs 20 | 21 | Click any line for drilldown per-username 22 | 23 | index=_audit info=granted "search='" NOT "savedsearch_name=\"Threat - Correlation Searches - Lookup Gen\"" NOT "savedsearch_name=\"Bucket Copy Trigger\"" NOT "search='| copybuckets" NOT "search='search index=_telemetry sourcetype=splunk_telemetry | spath" NOT "savedsearch_name=\"_ACCELERATE_*" 24 | | rex ", search='(?P<search>[\S+\s+]+?)', " 25 | | regex search!="\|\s+(rest|inputlookup|makeresults|tstats count AS \"Count of [^\"]+\"\s+ from sid=)" 26 | | rex "apiEndTime='[^,]+, savedsearch_name=\"(?P<savedsearch_name>[^\"]+)" 27 | | eval apiEndTime=strptime(apiEndTime, "'%a %B %d %H:%M:%S %Y'"), apiStartTime=strptime(apiStartTime, "'%a %B %d %H:%M:%S %Y'") 28 | | eval timePeriod=apiEndTime-apiStartTime 29 | | bin _time span=$span$ 30 | | stats count, values(host) AS hostList, values(savedsearch_name) AS savedSearchName, values(ttl) AS ttl by search, user, _time, timePeriod 31 | | eval frequency = ceil((10*60)/timePeriod) 32 | | fillnull frequency 33 | | where count>4 AND count>frequency 34 | | eval timePeriod=tostring(timePeriod,"duration") 35 | | stats sum(count) AS count, max(count) AS "maxCountPerSpan", values(user) AS userList, values(hostList) AS hostList, values(savedSearchName) AS savedSearchName, values(ttl) AS ttl, earliest(_time) AS firstSeen, latest(_time) AS mostRecent, values(timePeriod) AS timePeriods by search 36 | | eval firstSeen=strftime(firstSeen, "%+"), mostRecent=strftime(mostRecent, "%+") 37 | | eval search=substr(search,0,60) 38 | | sort - count 39 | $time.earliest$ 40 | $time.latest$ 41 | 1 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | $row.userList$ 53 | 54 |
55 | 56 | 57 | 58 | 59 | Results from access logs for $username$ 60 | 61 | Note: cluster command in use, introspection data may better list all dashboards in use 62 | 63 | index=_internal (sourcetype=splunkd_access (method="GET" AND "/services/search/jobs/export") OR method="POST") OR (sourcetype=splunkd_ui_access method=POST "/report?" OR "/search?" OR "/search/jobs" OR "/servicesNS/*/*/search/jobs" OR "/saved/searches" NOT "/search/parser HTTP" NOT "/user-prefs/data/user-prefs/") OR (sourcetype=splunkd_ui_access method=GET "/app/" NOT "/search HTTP" NOT "/dashboards HTTP" NOT "/alerts HTTP" NOT "/reports HTTP") user IN ($username$) 64 | | cluster t=0.95 showcount=true 65 | | rex field=uri "/servicesNS/[^/]+/(?P<app>[^/]+)" 66 | | rex field=uri "/[^/]+/app/(?P<app>[^/]+)/(?P<dashboard_name>[^/\?]+)" 67 | | sort - cluster_count 68 | | table cluster_count, app, uri_path, user, dashboard_name, clientip, sourcetype 69 | $time.earliest$ 70 | $time.latest$ 71 | 72 | 73 | 74 | 75 |
76 | 77 | 78 | 79 | 80 | Introspection data for this $username$ 81 | 82 | Click for drilldown 83 | 84 | index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* data.search_props.user IN ($username$) 85 | | eval mem_used = 'data.mem_used' 86 | | eval app = 'data.search_props.app' 87 | | eval elapsed = 'data.elapsed' 88 | | eval label = 'data.search_props.label' 89 | | eval type = 'data.search_props.type' 90 | | eval mode = 'data.search_props.mode' 91 | | eval user = 'data.search_props.user' 92 | | eval cpuperc = 'data.pct_cpu' 93 | | eval search_head = 'data.search_props.search_head' 94 | | eval read_mb = 'data.read_mb' 95 | | eval provenance='data.search_props.provenance' 96 | | eval label=coalesce(label, provenance) 97 | | eval sid='data.search_props.sid' 98 | | rex field=sid "^remote_[^_]+_(?P<sid>.*)" 99 | | eval sid = "'" . sid . "'" 100 | | fillnull search_head value="*" 101 | | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as searchStartTime, sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU, max(read_mb) AS read_mb, values(sid) AS sids by type, mode, app, user, label, host, search_head, data.pid 102 | | bin searchStartTime span=1m 103 | | stats dc(sids) AS count, sum(totalCPU) AS totalCPU, sum(mem_used) AS totalMemUsed, max(runtime) AS maxRunTime, avg(runtime) AS avgRuntime, avg(avgCPU) AS avgCPUPerIndexer, sum(read_mb) AS totalReadMB, values(sids) AS sids by searchStartTime, type, mode, app, user, search_head, label 104 | | eval maxduration = tostring(maxRunTime, "duration"), averageduration = tostring(avgRuntime, "duration") 105 | | eval Started = strftime(searchStartTime,"%+") 106 | | table Started, count, user, app, label, averageduration, maxduration, search_head, sids, mode, type 107 | $time.earliest$ 108 | $time.latest$ 109 | 110 | 111 | 112 | 113 | ["Started","count","user","app","label","averageduration","maxduration","mode","type"] 114 | 115 | /app/SplunkAdmins/troubleshooting_resource_usage_per_user_drilldown?form.username=$username$&form.sid=$row.sids$&form.app=$row.app$&form.host=*&form.label=*&form.time.earliest=$time.earliest$&form.time.latest=$time.latest$ 116 | 117 |
118 | 119 | 120 |
121 | -------------------------------------------------------------------------------- /default/data/ui/views/heavyforwarders_max_data_queue_sizes_by_name.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 6 | 7 | -4h@m 8 | now 9 | 10 | 11 | 12 | 13 | 1m 14 | 15 | 16 | 17 | `heavyforwarderhosts` 18 | 19 |
20 | 21 | 22 | Parsing Queue Fill Size 23 | 24 | 25 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=parsingqueue) 26 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 27 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 28 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 29 | | eval fill_perc=round((curr/max)*100,2) 30 | | eval combined = host . "_pipe_" . ingest_pipe 31 | | timechart limit=20 useother=false span=$span$ max(fill_perc) by combined 32 | $time.earliest$ 33 | $time.latest$ 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | Aggregation Queue Fill Size 50 | 51 | 52 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=aggqueue) 53 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 54 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 55 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 56 | | eval fill_perc=round((curr/max)*100,2) 57 | | eval combined = host . "_pipe_" . ingest_pipe 58 | | timechart limit=20 useother=false span=$span$ Max(fill_perc) by combined 59 | $time.earliest$ 60 | $time.latest$ 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | Typing Queue Fill Size 73 | 74 | 75 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=typingqueue) 76 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 77 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 78 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 79 | | eval fill_perc=round((curr/max)*100,2) 80 | | eval combined = host . "_pipe_" . ingest_pipe 81 | | timechart limit=20 useother=false span=$span$ Max(fill_perc) by combined 82 | $time.earliest$ 83 | $time.latest$ 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | Index Queue Size 96 | 97 | 98 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=indexqueue) 99 | | eval name=case(name=="aggqueue","2 - Aggregation Queue", 100 | name=="indexqueue", "4 - Indexing Queue", 101 | name=="parsingqueue", "1 - Parsing Queue", 102 | name=="typingqueue", "3 - Typing Queue") 103 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 104 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 105 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 106 | | eval fill_perc=round((curr/max)*100,2) 107 | | eval combined = host . "_pipe_" . ingest_pipe 108 | | timechart limit=20 useother=false span=$span$ Max(fill_perc) by combined 109 | $time.earliest$ 110 | $time.latest$ 111 | 1 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | TCPOut Queue Sizes 144 | 145 | 146 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=tcpout_*) 147 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 148 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 149 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 150 | | eval fill_perc=round((curr/max)*100,2) 151 | | eval combined = host . "_pipe_" . ingest_pipe 152 | | timechart limit=20 useother=false span=$span$ max(fill_perc) by combined 153 | $time.earliest$ 154 | $time.latest$ 155 | 1 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | Blocked Forwarder Queues 189 | 190 | 191 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue max_size_kb>0 | stats count(eval(isnotnull(blocked))) AS blockedCount, count by name, host, _time | eval percBlocked=(100/count)*blockedCount | eval hostQueue = host . "_" . name | where percBlocked>0 | timechart limit=50 useOther=false span=$span$ avg(percBlocked) by hostQueue 192 | $time.earliest$ 193 | $time.latest$ 194 | 1 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | TcpOut KB per second per forwarder 227 | 228 | 229 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=thruput name=cooked_output OR name=uncooked_output 230 | | timechart useother=false span=$span$ limit=20 per_second(kb) by host 231 | 232 | $time.earliest$ 233 | $time.latest$ 234 | 1 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | Forced closures on restart 266 | 267 | A potential indicator of data loss 268 | 269 | | tstats count where index=_internal sourcetype=splunkd $hosts$ `splunkadmins_splunkd_source` TERM("Forcing") groupby _time, host span=1s | timechart sum(count) by host 270 | 271 | $time.earliest$ 272 | $time.latest$ 273 | 1 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | Forwarders that have stopped listening on all ports 305 | 306 | 307 | index=_internal $hosts$ sourcetype=splunkd `splunkadmins_splunkd_source` TERM(WARN) TERM(Stopping) 308 | | timechart count by host span=1m limit=99 309 | -24h@h 310 | now 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 |
319 | -------------------------------------------------------------------------------- /default/data/ui/views/heavyforwarders_max_data_queue_sizes_by_name_v8.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 6 | 7 | -4h@m 8 | now 9 | 10 | 11 | 12 | 13 | 1m 14 | 15 | 16 | 17 | `heavyforwarderhosts` 18 | 19 |
20 | 21 | 22 | Parsing Queue Fill Size 23 | 24 | 25 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=parsingqueue) 26 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 27 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 28 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 29 | | eval fill_perc=round((curr/max)*100,2) 30 | | eval combined = host . "_pipe_" . ingest_pipe 31 | | timechart limit=20 useother=false span=$span$ max(fill_perc) by combined 32 | $time.earliest$ 33 | $time.latest$ 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | Aggregation Queue Fill Size 50 | 51 | 52 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=aggqueue) 53 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 54 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 55 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 56 | | eval fill_perc=round((curr/max)*100,2) 57 | | eval combined = host . "_pipe_" . ingest_pipe 58 | | timechart limit=20 useother=false span=$span$ Max(fill_perc) by combined 59 | $time.earliest$ 60 | $time.latest$ 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | Typing Queue Fill Size 73 | 74 | 75 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=typingqueue) 76 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 77 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 78 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 79 | | eval fill_perc=round((curr/max)*100,2) 80 | | eval combined = host . "_pipe_" . ingest_pipe 81 | | timechart limit=20 useother=false span=$span$ Max(fill_perc) by combined 82 | $time.earliest$ 83 | $time.latest$ 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | Index Queue Size 96 | 97 | 98 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=indexqueue) 99 | | eval name=case(name=="aggqueue","2 - Aggregation Queue", 100 | name=="indexqueue", "4 - Indexing Queue", 101 | name=="parsingqueue", "1 - Parsing Queue", 102 | name=="typingqueue", "3 - Typing Queue") 103 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 104 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 105 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 106 | | eval fill_perc=round((curr/max)*100,2) 107 | | eval combined = host . "_pipe_" . ingest_pipe 108 | | timechart limit=20 useother=false span=$span$ Max(fill_perc) by combined 109 | $time.earliest$ 110 | $time.latest$ 111 | 1 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | TCPOut Queue Sizes 144 | 145 | 146 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=tcpout_*) 147 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 148 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 149 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 150 | | eval fill_perc=round((curr/max)*100,2) 151 | | eval combined = host . "_pipe_" . ingest_pipe 152 | | timechart limit=20 useother=false span=$span$ max(fill_perc) by combined 153 | $time.earliest$ 154 | $time.latest$ 155 | 1 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | Blocked Forwarder Queues 189 | 190 | 191 | index=_internal $hosts$ `splunkadmins_metrics_source` sourcetype=splunkd group=queue max_size_kb>0 | stats count(eval(isnotnull(blocked))) AS blockedCount, count by name, host, _time | eval percBlocked=(100/count)*blockedCount | eval hostQueue = host . "_" . name | where percBlocked>0 | timechart limit=50 useOther=false span=$span$ avg(percBlocked) by hostQueue 192 | $time.earliest$ 193 | $time.latest$ 194 | 1 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | TcpOut KB per second per forwarder 227 | 228 | 229 | | tstats prestats=true sum(PREFIX(kb=)) where index=_internal $hosts$ TERM(group=thruput) TERM(name=cooked_output) OR TERM(name=uncooked_output) sourcetype=splunkd `splunkadmins_metrics_source` groupby host, _time span=1s 230 | | timechart aligntime=latest useother=false span=$span$ limit=20 per_second(kb=) by host 231 | 232 | $time.earliest$ 233 | $time.latest$ 234 | 1 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | Forced closures on restart 266 | 267 | A potential indicator of data loss 268 | 269 | | tstats prestats=true count where index=_internal sourcetype=splunkd $hosts$ `splunkadmins_splunkd_source` TERM("Forcing") groupby _time, host span=1s | timechart count by host 270 | 271 | $time.earliest$ 272 | $time.latest$ 273 | 1 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | Forwarders that have stopped listening on all ports 305 | 306 | 307 | index=_internal $hosts$ sourcetype=splunkd `splunkadmins_splunkd_source` TERM(WARN) TERM(Stopping) 308 | | timechart count by host span=1m limit=99 309 | -24h@h 310 | now 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 |
319 | -------------------------------------------------------------------------------- /default/data/ui/views/hec_performance.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Based on the original version from https://github.com/camrunr/hec_perf_report/blob/master/hec_perf_report.xml 4 | 5 | index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) `splunkadmins_hec_metrics_source` http_event_collector_token 6 | | bucket _time span=$dd_span$ 7 | | stats sum(data.num_of_events) as Events sum(data.total_bytes_received) as Bytes by _time data.token_name 8 | $timepicker.earliest$ 9 | $timepicker.latest$ 10 | 1 11 | $refreshinterval$ 12 | 13 | 14 | index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) `splunkadmins_hec_metrics_source` http_event_collector_token 15 | | bucket _time span=$dd_span$ 16 | | stats sum(data.num_of_events) as Events sum(data.total_bytes_received) as Bytes by _time host 17 | | eval host=replace(host,"\..*","") 18 | $timepicker.earliest$ 19 | $timepicker.latest$ 20 | 1 21 | $refreshinterval$ 22 | 23 |
24 | 25 | 26 | 27 | -4h@m 28 | now 29 | 30 | 31 | 32 | 33 | 1 minute 34 | 5 minutes 35 | 30 minutes 36 | 1 hour 37 | 1 day 38 | 1min 39 | 40 | 41 | 42 | 15 43 | 44 | 45 | 46 | 300 47 | 48 |
49 | 50 | 51 | Events/sec by host 52 | 53 | 54 | timechart limit=$hostcount$ span=$dd_span$ per_second(Events) as Events/sec by host 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | Bytes/sec by host 92 | 93 | 94 | timechart limit=$hostcount$ span=$dd_span$ per_second(Bytes) as Bytes/sec by host 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | Events/sec by input/group 108 | 109 | 110 | timechart span=$dd_span$ per_second(Events) as Events/sec by data.token_name 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | Bytes/sec by input/group 148 | 149 | 150 | timechart span=$dd_span$ per_second(Bytes) as Bytes/sec by data.token_name 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | HEC Batching Efficiency 164 | 165 | 166 | $refreshinterval$ 167 | index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) `splunkadmins_hec_metrics_source` http_event_collector_token 168 | | eval EpR='data.num_of_events'/'data.num_of_requests' 169 | | bucket _time span=5m 170 | | stats sum(data.num_of_events) as events avg(EpR) as events_per_POST sum(data.num_of_requests) as reqs sum(data.total_bytes_received) as Bytes by _time data.token_name 171 | | eval reqs_per_sec=reqs/300, bytes_per_post=Bytes/reqs 172 | | rename data.token_name as token_name 173 | | stats sum(eval(Bytes/1024/1024)) as MBytes sum(events) as Events p50(events_per_POST) as events_per_post p50(bytes_per_post) as bytes_per_post p90(reqs_per_sec) as posts_per_sec by token_name 174 | | eval MBytes = round(MBytes, 2), events_per_post=round(events_per_post,2), bytes_per_post=round(bytes_per_post,2), posts_per_sec=round(posts_per_sec,2) 175 | | sort - posts_per_sec 176 | $timepicker.earliest$ 177 | $timepicker.latest$ 178 | 179 | 180 | 181 | 182 | 183 | [#DC4E41,#DC4E41,#F8BE34,#53A051] 184 | 0,5,10 185 | 186 | 187 | [#53A051,#F8BE34,#DC4E41] 188 | 10,50 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 |
198 | 199 | 200 | 201 | 202 | If useACK is in use num_of_requests_waiting_ack is high then this can be an issue (HEC tokens with useACK will stop allowing data through) 203 | 204 | 205 | $refreshinterval$ 206 | index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) data.series=http_event_collector data.num_of_requests_waiting_ack=* sourcetype=http_event_collector_metrics 207 | | timechart minspan=2m max(data.num_of_requests_waiting_ack) AS num_of_requests_waiting_ack 208 | $timepicker.earliest$ 209 | $timepicker.latest$ 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 |
218 | -------------------------------------------------------------------------------- /default/data/ui/views/indexer_data_spread.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Indexer Data Spread 4 |
5 | 6 | 7 | 8 | -24h@h 9 | now 10 | 11 | 12 | 13 | 14 | 15 | -24h@h 16 | now 17 | 18 | 19 |
20 | 21 | 22 | Spread of data across the indexers 23 | 24 | 25 | | tstats count WHERE index="*" by splunk_server _time span=10m | timechart span=10m sum(count) by splunk_server 26 | $thetime.earliest$ 27 | $thetime.latest$ 28 | 1 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | Indexed data in KB per second per indexer 59 | 60 | 61 | (index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=thruput name=index_thruput) | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* | timechart minspan=30s per_second(kb) by host 62 | $time_tok.earliest$ 63 | $time_tok.latest$ 64 | 1 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | Forwarders and Throughput (from monitoring console) 95 | 96 | 97 | index=_internal sourcetype=splunkd group=tcpin_connections (connectionType=cooked OR connectionType=cookedSSL) fwdType=* guid=* `indexerhosts` | timechart minspan=30s dc(guid) as forwarder_count, per_second(kb) as tcp_KBps | rename forwarder_count as "Forwarder Count", tcp_KBps as "Throughput (KB/s)" 98 | $time_tok.earliest$ 99 | $time_tok.latest$ 100 | 1 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | Incoming TCP Queues 132 | 133 | 134 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=queue name=splunktcpin OR name=tcpin_cooked_pqueue 135 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 136 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 137 | | eval fill_perc=round((curr/max)*100,2) 138 | | timechart minspan=30s Median(fill_perc) AS "fill_percentage" by host 139 | $time_tok.earliest$ 140 | $time_tok.latest$ 141 | 1 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 |
171 | -------------------------------------------------------------------------------- /default/data/ui/views/indexer_max_data_queue_sizes_by_name.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 6 | 7 | -4h@m 8 | now 9 | 10 | 11 |
12 | 13 | 14 | Parsing Queue Fill Size 15 | 16 | 17 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=parsingqueue) 18 | | eval name=case(name=="aggqueue","2 - Aggregation Queue", 19 | name=="indexqueue", "4 - Indexing Queue", 20 | name=="parsingqueue", "1 - Parsing Queue", 21 | name=="typingqueue", "3 - Typing Queue") 22 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 23 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 24 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 25 | | eval fill_perc=round((curr/max)*100,2) 26 | | eval combined = host . "_pipe_" . ingest_pipe 27 | | timechart limit=14 useother=false span=1m Max(fill_perc) by combined 28 | $time.earliest$ 29 | $time.latest$ 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | Aggregation Queue Fill Size 46 | 47 | 48 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=aggqueue) 49 | | eval name=case(name=="aggqueue","2 - Aggregation Queue", 50 | name=="indexqueue", "4 - Indexing Queue", 51 | name=="parsingqueue", "1 - Parsing Queue", 52 | name=="typingqueue", "3 - Typing Queue") 53 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 54 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 55 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 56 | | eval fill_perc=round((curr/max)*100,2) 57 | | eval combined = host . "_pipe_" . ingest_pipe 58 | | timechart limit=14 useother=false span=1m Max(fill_perc) by combined 59 | $time.earliest$ 60 | $time.latest$ 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | Typing Queue Fill Size 73 | 74 | 75 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=typingqueue) 76 | | eval name=case(name=="aggqueue","2 - Aggregation Queue", 77 | name=="indexqueue", "4 - Indexing Queue", 78 | name=="parsingqueue", "1 - Parsing Queue", 79 | name=="typingqueue", "3 - Typing Queue") 80 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 81 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 82 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 83 | | eval fill_perc=round((curr/max)*100,2) 84 | | eval combined = host . "_pipe_" . ingest_pipe 85 | | timechart limit=14 useother=false span=1m Max(fill_perc) by combined 86 | $time.earliest$ 87 | $time.latest$ 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | Indexing Queue Fill Size 100 | 101 | 102 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=indexqueue) 103 | | eval name=case(name=="aggqueue","2 - Aggregation Queue", 104 | name=="indexqueue", "4 - Indexing Queue", 105 | name=="parsingqueue", "1 - Parsing Queue", 106 | name=="typingqueue", "3 - Typing Queue") 107 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 108 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 109 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 110 | | eval fill_perc=round((curr/max)*100,2) 111 | | eval combined = host . "_pipe_" . ingest_pipe 112 | | timechart limit=14 useother=false span=1m Max(fill_perc) by combined 113 | $time.earliest$ 114 | $time.latest$ 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | Shows any replication queue issues that may slowdown/prevent the queues from clearing at the indexer level 127 | 128 | The replication queue appears to directly relate to the indexing queue, any blockage of the indexing queue will then block the replication queue and temporarily slow data ingestion. The replication queue appears to be extremely sensitive to the other indexers indexing queue so it can be a useful measure of an issue... 129 | 130 | index=_internal `indexerhosts` "replication queue for " "full" OR "has room now" sourcetype=splunkd `splunkadmins_splunkd_source` 131 | | rename peer AS guid 132 | | join guid 133 | [| rest /services/search/distributed/peers 134 | | table guid peerName] 135 | | transaction bid guid endswith="has room now" keeporphans=true keepevicted=true 136 | | timechart span=$span$ count, max(duration) AS duration by peerName 137 | -60m@m 138 | now 139 | 1 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | Blocked Indexing Queues 176 | 177 | 178 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=queue | stats count(eval(isnotnull(blocked))) AS blockedCount, count by name, host, _time | eval percBlocked=(100/count)*blockedCount | eval hostQueue = host . "_" . name | timechart useOther=false span=10m avg(percBlocked) by hostQueue 179 | $time.earliest$ 180 | $time.latest$ 181 | 1 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | TCPIn Queue Sizes (Max) 214 | 215 | 216 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=queue (name=splunktcpin OR name=tcpin_cooked_pqueue) 217 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 218 | | eval max=if(isnotnull(max_size_kb),max_size_kb,max_size) 219 | | eval curr=if(isnotnull(current_size_kb),current_size_kb,current_size) 220 | | eval fill_perc=round((curr/max)*100,2) 221 | | eval combined = host . "_pipe_" . ingest_pipe 222 | | timechart limit=14 useother=false span=1m max(fill_perc) by combined 223 | $time.earliest$ 224 | $time.latest$ 225 | 1 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | Thruput Per Indexer 259 | 260 | 261 | index=_internal `indexerhosts` `splunkadmins_metrics_source` sourcetype=splunkd group=thruput name=index_thruput 262 | | eval ingest_pipe = if(isnotnull(ingest_pipe), ingest_pipe, "none") | search ingest_pipe=* 263 | | eval combined = host . "_pipe_" . ingest_pipe 264 | | timechart useother=false span=1m limit=14 per_second(kb) by host 265 | 266 | $time.earliest$ 267 | $time.latest$ 268 | 1 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | Forced closures on restart 300 | 301 | A potential indicator of data loss 302 | 303 | | tstats count where index=_internal sourcetype=splunkd `indexerhosts` `splunkadmins_splunkd_source` TERM("Forcing") groupby _time, host span=1s | timechart sum(count) by host 304 | 305 | $time.earliest$ 306 | $time.latest$ 307 | 1 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 |
337 | -------------------------------------------------------------------------------- /default/data/ui/views/knowledge_objects_by_app.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | List of knowledge objects per app 4 |
5 | 6 | 7 | app 8 | app 9 | 10 | | rest /services/apps/local search="disabled=0" count=0 f=title splunk_server=local 11 | | rename title as app 12 | | table app 13 | -24h@h 14 | now 15 | 16 | 17 | 18 | 19 | all 20 | datamodel 21 | calcfields 22 | macros 23 | type 24 | type 25 | 26 | | rest "/servicesNS/-/$app$/directory" count=0 splunk_server=local 27 | | search eai:acl.app=$app$ 28 | | rename eai:type AS type 29 | | search type!="macros" ```macros only appears in really new versions of Splunk via the directory endpoint, so assume it doesn't exist in this query``` 30 | | stats count by type 31 | | fields - count 32 | -24h@h 33 | now 34 | 35 | all 36 | * 37 | 38 |
39 | 40 | 41 | Knowledge object summary 42 | 43 | 44 | | rest "/servicesNS/-/$app$/directory" count=0 splunk_server=local 45 | | search eai:acl.app=$app$ 46 | | eval updatedEpoch=strptime(updated,"%Y-%m-%dT%H:%M:%S%:z") 47 | | rename eai:type AS type, eai:acl.app AS app, eai:location AS location 48 | | append [ rest splunk_server=local /servicesNS/-/$app$/datamodel/model count=0 f=updated f=eai:appName | rename eai:appName AS app | eval type="datamodel" ] 49 | | append [ | rest splunk_server=local /servicesNS/-/$app$/data/props/calcfields count=0 | eval type="calcfields" | rename eai:acl.app AS app] 50 | | append [ | rest splunk_server=local /servicesNS/-/$app$/configs/conf-macros count=0 | rename eai:appName AS app | eval type="macros"] 51 | | fillnull location value="N/A" 52 | | search app=$app$ 53 | | stats count by type, app, location 54 | -4h@m 55 | now 56 | 1 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 |
67 | 68 | 69 | 70 | 71 | Knowledge Objects by app semi-detailed 72 | 73 | Click any row for the drilldown... 74 | 75 | | rest "/servicesNS/-/$app$/directory" count=0 splunk_server=local 76 | | search eai:acl.app=$app$ 77 | | eval updatedEpoch=strptime(updated,"%Y-%m-%dT%H:%M:%S%:z") 78 | | rename eai:type AS type, eai:acl.app AS app, eai:location AS location 79 | | append [ rest splunk_server=local /servicesNS/-/$app$/datamodel/model count=0 f=updated f=eai:appName | rename eai:appName AS app | eval type="datamodel" ] 80 | | append [ | rest splunk_server=local /servicesNS/-/$app$/data/props/calcfields count=0 | eval type="calcfields" | rename eai:acl.app AS app] 81 | | append [ | rest splunk_server=local /servicesNS/-/$app$/configs/conf-macros count=0 | rename eai:appName AS app | eval type="macros"] 82 | | fillnull location value="N/A" 83 | | search app=$app$, type=$type$ 84 | | stats values(title) AS names, values(updated) AS updated by eai:acl.owner, eai:acl.sharing, type 85 | | rename eai:acl.sharing AS sharing, eai:acl.owner AS owner 86 | -4h@m 87 | now 88 | 1 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | /app/SplunkAdmins/knowledge_objects_by_app_drilldown?form.app=$app$&form.type=$row.type$&form.sharing=$row.sharing$&form.owner=$row.owner$ 99 | 100 |
101 | 102 | 103 |
104 | -------------------------------------------------------------------------------- /default/data/ui/views/knowledge_objects_by_app_drilldown.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | List of knowledge objects per app by user/sharing level 4 |
5 | 6 | 7 | app 8 | app 9 | 10 | | rest /services/apps/local search="disabled=0" count=0 f=title splunk_server=local 11 | | rename title as app 12 | | table app 13 | -24h@h 14 | now 15 | 16 | 17 | 18 | 19 | all 20 | datamodel 21 | calcfields 22 | macros 23 | type 24 | type 25 | 26 | | rest "/servicesNS/-/$app$/directory" count=0 splunk_server=local 27 | | search eai:acl.app=$app$ 28 | | rename eai:type AS type 29 | | stats count by type 30 | | fields - count 31 | -24h@h 32 | now 33 | 34 | all 35 | * 36 | 37 | 38 | 39 | * 40 | 41 | 42 | 43 | All 44 | app 45 | user (private) 46 | global 47 | * 48 | * 49 | 50 | 51 | 52 | * 53 | 54 | 55 | 56 | Yes 57 | No 58 | * 59 | 60 |
61 | 62 | 63 | Knowledge object summary 64 | 65 | 66 | | rest "/servicesNS/-/$app$/directory" count=0 splunk_server=local 67 | | search eai:acl.app=$app$ 68 | | eval updatedEpoch=strptime(updated,"%Y-%m-%dT%H:%M:%S%:z") 69 | | rename eai:type AS type, eai:acl.app AS app, eai:location AS location 70 | | append [ rest splunk_server=local /servicesNS/-/$app$/datamodel/model count=0 f=updated f=eai:appName | rename eai:appName AS app | eval type="datamodel" ] 71 | | append [ | rest splunk_server=local /servicesNS/-/$app$/data/props/calcfields count=0 | eval type="calcfields" | rename eai:acl.app AS app] 72 | | append [ | rest splunk_server=local /servicesNS/-/$app$/configs/conf-macros count=0 | rename eai:appName AS app | eval type="macros"] 73 | | fillnull disabled 74 | | search app=$app$ type=$type$ title=$name$ eai:acl.sharing=$sharing$ disabled=$disabled$ eai:acl.owner=$owner$ 75 | | fillnull location value="N/A" 76 | | rename title AS name, eai:acl.owner AS owner, eai:acl.sharing AS sharing 77 | | eval disabled=case(disabled==0,"false",disabled==1,"true",1==1,"Unknown") 78 | | table name, description, disabled, owner, sharing, type, updated 79 | -4h@m 80 | now 81 | 1 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 |
92 | 93 | 94 |
95 | -------------------------------------------------------------------------------- /default/data/ui/views/lookup_audit.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Dashboard for displaying lookup table files on a Search Head. Created to easily identify large tables which might disrupt Splunk uptime. Created by Discovered Intelligence -- https://discoveredintelligence.ca, modifications by Gareth Anderson 4 | 5 | | rest /servicesNS/nobody/$appselection_rest$/data/lookup-table-files splunk_server=local 6 | | rename eai:acl.app as appname 7 | | regex appname=^$appselection$$ 8 | | dedup appname 9 | | map maxsearches=5000 search=" | rest /servicesNS/-/$appselection_rest$/admin/file-explorer/$splunk_dir|u$%2Fapps%2F$$appname$$%2Flookups splunk_server=local | eval appname=\"$$appname$$\"" 10 | 11 |
12 | 13 | 14 | Show All Lookups 15 | Exclude Blacklisted Lookups 16 | Show Only Blacklisted Lookups 17 | 18 | 19 | 20 | 29 | 30 | 31 | 32 | 33 | 42 | 43 | 44 | 45 | * 46 | 47 | 48 | NonBlackList 49 | NonBlackList 50 | 51 | 52 | 53 | All 54 | appname 55 | appname 56 | 57 | | rest /servicesNS/-/-/data/lookup-table-files splunk_server=local 58 | | where like(title,"%csv") 59 | | rename eai:acl.app as appname 60 | | dedup appname 61 | | sort appname 62 | -15m 63 | now 64 | 65 | 66 | 67 | - 68 | 69 | 70 | $value$ 71 | 72 | 73 | 74 | 75 | 76 | All 77 | Yes 78 | No 79 | * 80 | * 81 | 82 | 83 | 84 | /opt/splunk/etc 85 | 86 |
87 | 88 | 89 | Lookup Files by App 90 | 91 | 92 | | rex field=title "[\\\\/]apps[\\\\/](?P<App>.+)[\\\\/]lookups" 93 | | sort - lastModifiedTime 94 | | eval "Last Modified" = strftime(lastModifiedTime,"%b %d, %Y %H:%M"), fileSize_MB=round((fileSize/1024),3) 95 | | fillnull value=0.000 fileSize_MB 96 | | fields App name fileSize_MB "Last Modified" title 97 | | rex field=title "(?<title>apps.*)$" 98 | | search $blacklist$ 99 | | join type=left name 100 | [| rest /servicesNS/nobody/$appselection_rest$/data/lookup-table-files splunk_server=local 101 | | rename title AS name 102 | | fields + name author] 103 | | eval private_lookup="No" 104 | | append 105 | [| rest /servicesNS/-/$appselection_rest$/data/lookup-table-files splunk_server=local 106 | | regex eai:data="[\\\\/]users[\\\\/]$appselection$[\\\\/][^\\\\/]+[\\\\/]lookups[/\\\\]" 107 | | rename eai:acl.app as appname, eai:userName AS user 108 | | search appname=* 109 | | dedup appname 110 | | map maxsearches=5000 search=" | rest /servicesNS/-/$appselection_rest$/admin/file-explorer/$splunk_dir|u$%2Fusers%2F$$user$$%2F$$appname$$%2Flookups splunk_server=local" 111 | | rex field=title "[\\\\/]users[\\\\/]$appselection$[\\\\/](?<App>.+)[\\\\/]lookups[\\\\/]" 112 | | sort - lastModifiedTime 113 | | eval "Last Modified" = strftime(lastModifiedTime,"%b %d, %Y %H:%M"), fileSize_MB=round((fileSize/1024),3) 114 | | fillnull value=0.000 fileSize_MB 115 | | fields App name fileSize_MB "Last Modified" title 116 | | rex field=title "(?<title>users.*)$" 117 | | search $blacklist$ 118 | | join type=left name 119 | [| rest /servicesNS/-/$appselection_rest$/data/lookup-table-files splunk_server=local 120 | | regex eai:data="$splunk_dir$[\\\\/]users[\\\\/]$appselection$[\\\\/]" 121 | | rename title AS name 122 | | fields + name author] 123 | | eval private_lookup="Yes" 124 | ] 125 | | rename title AS path 126 | | search private_lookup="$priv_lookup$" 127 | | sort - fileSize_MB 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 |
138 | 139 | 140 | 141 | 142 | Lookup Subdirectories by App 143 | 144 | Note: blacklist does not work for this panel and the last modified is directory modification date. If the author is blank then no matching lookup definition of type geo was found. Finally, as per the open ideas, the sub-directories under the lookups directory are never reaped by Splunk as of 8.0.3, it is upto the administrator to remove them as required. Also note they are not blacklisted from the knowledge bundle to the search peers, and finally they are created when the geom command is used so can be different per-search head! 145 | 146 | | eval last_modified = strftime(lastModifiedTime,"%b %d, %Y %H:%M") 147 | | search hasSubNodes=1 148 | | map maxsearches=5000 search=" | rest /servicesNS/-/$appselection_rest$/admin/file-explorer/$splunk_dir|u$%2Fapps%2F$$appname$$%2Flookups%2F$$name$$ splunk_server=local | eval last_modified=\"$$last_modified$$\"" 149 | | rex field=title "(?P<path>[^/\\\\]+[/\\\\](?P<App>[^/\\\\]+)[/\\\\][^/\\\\]+[/\\\\](?P<dirname>[^/\\\\]+))[/\\\\][^/\\\\]+$" 150 | | stats sum(fileSize) AS fileSize, values(last_modified) AS "Last Modified" by dirname, App, path 151 | | append 152 | [| rest /servicesNS/-/$appselection_rest$/data/lookup-table-files splunk_server=local 153 | | regex eai:data="$splunk_dir$[\\\\/]users[/\\\\][^/\\\\]+[/\\\\]$appselection$[\\\\/]" 154 | | rename eai:acl.app as appname, eai:userName AS user 155 | | search appname=* 156 | | dedup appname 157 | | map maxsearches=5000 search=" | rest /servicesNS/-/$appselection_rest$/admin/file-explorer/$splunk_dir|u$%2Fusers%2F$$user$$%2F$$appname$$%2Flookups splunk_server=local | eval appname=\"$$appname$$\", user=\"$$user$$\"" 158 | | search NOT ignoreme="true" 159 | | search hasSubNodes=1 160 | | eval last_modified = strftime(lastModifiedTime,"%b %d, %Y %H:%M") 161 | | fillnull last_modified 162 | | map maxsearches=5000 search=" | rest /servicesNS/-/$appselection_rest$/admin/file-explorer/$splunk_dir|u$%2Fusers%2F$$user$$%2F$$appname$$%2Flookups%2F$$name$$ splunk_server=local | eval last_modified=\"$$last_modified$$\"" 163 | | rex field=title "(?P<path>([^/\\\\]+[/\\\\]){2}(?P<App>[^/\\\\]+)[/\\\\][^/\\\\]+[/\\\\](?P<dirname>[^/\\\\]+))[/\\\\][^/\\\\]+$" 164 | | stats sum(fileSize) AS fileSize, values(last_modified) AS "Last Modified" by dirname, App, path ] 165 | | eval fileSize_MB=round((fileSize/1024),3) 166 | | table App, dirname, fileSize_MB, "Last Modified" path 167 | | join type=left dirname 168 | [| rest /servicesNS/-/$appselection_rest$/data/transforms/lookups splunk_server=local search="type=geo" f=title 169 | | fields + dirname author] 170 | | sort - fileSize_MB 171 | 172 | 173 | 174 | 175 |
176 | 177 | 178 |
179 | -------------------------------------------------------------------------------- /default/data/ui/views/lookups_in_use_finder.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Attempt to detect if a lookup file in question is in use within Splunk 4 |
5 | 6 | 7 | 8 | 9 | 10 | All 11 | - 12 | - 13 | app 14 | app 15 | 16 | | rest /services/apps/local search="disabled=0" count=0 f=title splunk_server=local 17 | | rename title as app 18 | | table app 19 | -15m 20 | now 21 | 22 | 23 | 24 | 25 | 26 | -60m@m 27 | now 28 | 29 | 30 |
31 | 32 | 33 | Dashboard or Scheduled Search lookups 34 | 35 | 36 | | makeresults 37 | | eval filename="$lookup_name$", lookupDefName=null() 38 | | fields - _time 39 | | append 40 | [| rest splunk_server=local "/servicesNS/-/$app$/data/transforms/lookups" f=eai:* f=filename f=title f=updated 41 | | search filename="$lookup_name$" 42 | | fields title 43 | | rename title AS lookupDefName ] 44 | | tail 1 45 | | fillnull lookupDefName value="youwontfindthisone" 46 | | appendpipe 47 | [ | map 48 | [| rest /servicesNS/-/$app$/data/ui/views splunk_server=local f=eai:* f=label f=title 49 | | fields eai:acl.app, label, title, updated, eai:acl.owner, eai:data 50 | | regex eai:data="(input|output)?lookup\s+($lookup_name$|$$lookupDefName$$)" 51 | | eval type="dashboard" 52 | | fields - eai:data ] ] 53 | | appendpipe [ | map 54 | [| rest /servicesNS/-/$app$/saved/searches splunk_server=local f=eai:* f=title f=search f=updated 55 | | fields eai:acl.owner, title, search, updated, eai:acl.app 56 | | regex search="(input|output)?lookup\s+($lookup_name$|$$lookupDefName$$)" 57 | | eval type="report" 58 | | fields - search ]] 59 | | where isnotnull('eai:acl.app') 60 | | eval searchedApp="$app$" 61 | | where 'eai:acl.app'==searchedApp OR "$app$"=="-" 62 | | fields - filename, lookupDefName 63 | | rename eai:acl.app AS app, eai:acl.owner AS owner 64 | -5m 65 | now 66 | 1 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 |
77 | 78 | 79 | 80 | 81 | Audit Logs Check (note no app context available) 82 | 83 | 84 | | makeresults 85 | | eval filename="$lookup_name$", lookupDefName=null() 86 | | fields - _time 87 | | append 88 | [| rest splunk_server=local "/servicesNS/-/$app$/data/transforms/lookups" f=eai:* f=filename f=title f=updated 89 | | search filename="$lookup_name$" 90 | | fields title 91 | | rename title AS lookupDefName ] 92 | | tail 1 93 | | fillnull lookupDefName value="youwontfindthisone" 94 | | appendpipe 95 | [ map 96 | [ search index=_audit "info=granted" "search='search " $lookup_name$ search_id!="'ta_*" 97 | | rex ", search='(?P<search>[\S+\s+]+?)', " 98 | | regex search="(input|output)?lookup\s+($lookup_name$|$$lookupDefName$$)" 99 | | fields user, search, search_id, savedsearch_name] ] 100 | | where isnotnull(user) 101 | | table user, search, search_id, savedsearch_name 102 | $time.earliest$ 103 | $time.latest$ 104 | 1 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 |
115 | 116 | 117 |
118 | -------------------------------------------------------------------------------- /default/data/ui/views/rolled_buckets_by_index.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | A very simple dashboard to determine which index is rolling the largest number of buckets and therefore may require some level of tuning 4 |
5 | 6 | 7 | 8 | -3d 9 | @d 10 | 11 | 12 | 13 | 14 | 3 15 | 7 16 | 14 17 | 30 18 | 60 19 | 7 20 | - 21 | d 22 | 23 |
24 | 25 | 26 | Number of buckets rolled from hot to warm 27 | 28 | Buckets rolled per day per index, top 15 indexes 29 | 30 | index=_internal "HotBucketRoller" sourcetype=splunkd `splunkadmins_splunkd_source` `indexerhosts` "finished moving" 31 | | bin _time span=24h 32 | | chart limit=15 useother=false count by _time, idx 33 | $time.earliest$ 34 | $time.latest$ 35 | 1 36 | 37 | 38 | 39 | 40 | $click.name2$ 41 | 42 | 43 | 44 | 45 | 46 | 47 | Buckets with largest timespan 48 | 49 | Buckets sorted by longest average time period (often indicates a timestamp parsing issue as large time periods trigger the buckets to roll early) 50 | 51 | | dbinspect index=* 52 | | eval timePeriod=(endEpoch-startEpoch)/60/60/24 53 | | stats avg(timePeriod) AS avgTimePeriod, max(timePeriod) AS maxTimePeriod by index 54 | | where avgTimePeriod>5 55 | | sort - avgTimePeriod 56 | $time.earliest$ 57 | $time.latest$ 58 | 1 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | $click.value$ 70 | 71 |
72 | 73 | 74 | 75 | 76 | License usage for index $indexname$ 77 | 78 | Click on an index above for this drilldown to show which the license usage by a particular index 79 | 80 | index=_internal `licensemasterhost` `splunkadmins_license_usage_source` idx=$indexname$ 81 | | bin _time span=24h 82 | | stats sum(b) AS totalB by idx, _time 83 | | eval totalB=totalB/1024/1024/1024 84 | | chart avg(totalB) AS totalGB by _time, idx 85 | $time.earliest$ 86 | $time.latest$ 87 | 1 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | Bucket Info From DBInspect 127 | 128 | Show the length of time for the average bucket from this particular index 129 | 130 | | dbinspect index=$indexname$ 131 | | eval timePeriod=(endEpoch-startEpoch)/60/60/24 132 | | stats avg(timePeriod) AS avgTimePeriod, min(timePeriod) AS minTimePeriod, max(timePeriod) AS maxTimePeriod, max(sizeOnDiskMB) AS maxSizeMB, avg(sizeOnDiskMB) AS avgSizeMB by index 133 | | append 134 | [| rest `splunkindexerhostsvalue` /services/data/indexes 135 | | search title=$indexname$ 136 | | head 1 137 | | table maxDataSize ] 138 | $days$ 139 | $time.latest$ 140 | 1 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 |
151 | 152 | 153 | 154 | 155 | Sourcetype info for $indexname$ 156 | 157 | Click on any sourcetype to drilldown to the historic data in the past week for that sourcetype... 158 | 159 | | tstats count where index=$indexname$ groupby sourcetype 160 | | sort - count 161 | $days$ 162 | $time.latest$ 163 | 1 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | $click.value2$ 174 | 175 |
176 | 177 | 178 | 179 | 180 | Historic data for index $indexname$ indexed in the past $days$ 181 | 182 | Find data indexed in the past $days$ days that is at least 30 days old for sourcetype $sourcetype$ in index $indexname$ 183 | 184 | index=$indexname$ sourcetype=$sourcetype$ _index_earliest=-7d earliest=-300d latest=-30d 185 | | eval indextime=strftime(_indextime, "%+") 186 | $days$ 187 | now 188 | 1 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | Future based data 207 | 208 | Future based data for sourcetype $sourcetype$ in index $indexname$ indexed in the past $days$ days 209 | 210 | index=$indexname$ sourcetype=$sourcetype$ earliest=+5m latest=+5y _index_earliest=$days$ 211 | | eval indextime=strftime(_indextime, "%+") 212 | -5m 213 | now 214 | 1 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 |
231 | -------------------------------------------------------------------------------- /default/data/ui/views/search_head_scheduledsearches_distribution.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Number of scheduler searches per search head 4 |
5 | 6 | 7 | 8 | -24h@h 9 | now 10 | 11 | 12 |
13 | 14 | 15 | Searches per search head 16 | 17 | 18 | index=_internal `searchheadhosts` sourcetype=scheduler status=delegated_remote_completion | timechart count by member_label 19 | $time.earliest$ 20 | $time.latest$ 21 | 1 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | Scheduled searches starting later than 100 seconds after the scheduled time (mostly harmless as the now time period relates to the original scheduled time) 52 | 53 | 54 | None 55 | _ACCELERATE 56 | __NOEXCLUSION__ 57 | __NOEXCLUSION__ 58 | 59 | 60 | 61 | * 62 | * 63 | 64 | 65 | 66 | noexclusion 67 | noexclusion 68 | 69 | 70 | 71 | index=_internal `searchheadhosts` sourcetype=scheduler app=* scheduled_time=* savedsearch_name!=$exclude$ user=$userequals$ user!=$usernotequalto$ | eval time=strftime(_time,"%Y-%m-%d %H:%M:%S") | eval delay_in_start = (dispatch_time - scheduled_time) | where delay_in_start>100 | eval scheduled_time=strftime(scheduled_time,"%Y-%m-%d %H:%M:%S") | eval dispatch_time=strftime(dispatch_time,"%Y-%m-%d %H:%M:%S") | rename time AS endTime | table host,savedsearch_name,delay_in_start, scheduled_time, dispatch_time, endTime, run_time, status, user, app | sort -delay_in_start | dedup host,savedsearch_name,delay_in_start 72 | $time.earliest$ 73 | $time.latest$ 74 | 1 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 |
84 | 85 | 86 |
87 | -------------------------------------------------------------------------------- /default/data/ui/views/smartstore_stats.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Also refer to https://github.com/camrunr/s2_traffic_report/blob/master/s2_traffic_report.xml for an alternative view of SmartStore downloads/uploads. To determine which searches are causing cache misses refer to the SearchHeadLevel - SmartStore cache misses reports in this app. Note that the cache misses combined will require the search to complete while the indexing tier version can catch an in-progress search 4 |
5 | 6 | 7 | 8 | -60m@m 9 | now 10 | 11 | 12 | 13 | 14 | All 15 | download 16 | upload 17 | * 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | All Indexers 26 | `indexerhosts` 27 | 28 |
29 | 30 | 31 | Also refer to 32 | SmartStore S2S Traffic report for an alternative dashboard view or SearchHeadLevel - SmartStore cache misses combined or SmartStore cache misses - remote_searches to find the searches that are triggering the cache misses 33 | 34 | 35 | 36 | 37 | Upload/download latency 38 | 39 | 40 | index=_internal $host$ TERM(status=succeeded) OR TERM(status=failed) sourcetype=splunkd `splunkadmins_splunkd_source` TERM(action=$action$) 41 | | rangemap field=kb under_300=0-307200 300_700=307201-716800 700_1000=716801-1024000 default=over1000 42 | | eval combined = action . "_" . range 43 | | timechart avg(elapsed_ms) AS avg_elapsed_ms, max(elapsed_ms) AS max_elapsed_ms by combined 44 | $time.earliest$ 45 | $time.latest$ 46 | 1 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | Upload/download thruput 86 | 87 | 88 | index=_internal sourcetype=splunkd `splunkadmins_splunkd_source` $host$ TERM(status=succeeded) OR TERM(status=failed) TERM(action=$action$) 89 | | timechart sum(eval(kb/1024)) AS MB by action 90 | $time.earliest$ 91 | $time.latest$ 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | CacheManager Queued download count 102 | 103 | 104 | ```Relates to [cachemanager] max_concurrent_downloads in server.conf. Thanks to Splunk support for the original version of this search``` index=_internal $host$ `splunkadmins_metrics_source` TERM(group=cachemgr_download) sourcetype=splunkd queued 105 | | timechart partial=f limit=50 avg(queued) AS avg_queued by host 106 | | eval ceiling=20 107 | $time.earliest$ 108 | $time.latest$ 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | CacheManager hits/misses 119 | 120 | 121 | 122 | index=_internal $host$ `splunkadmins_metrics_source` sourcetype=splunkd group=cachemgr_bucket TERM(cache_hit=*) OR TERM(cache_miss=*) 123 | | timechart sum(cache_hit) as Hits sum(cache_miss) as Misses 124 | 125 | $time.earliest$ 126 | $time.latest$ 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | Excessive cachemanager downloads 137 | 138 | 139 | ```Thanks to Splunk support for the original version of this search, similar version available in the monitoring console...``` index=_internal $host$ `splunkadmins_splunkd_source` sourcetype=splunkd CacheManager TERM(action=download) TERM(status=succeeded) TERM(download_set=*) 140 | | rex field=cache_id ">*\|(?<index_name>.*)~.*~.*\|" 141 | | eval identifier=(cache_id + host) 142 | | stats count by identifier, index_name 143 | | stats count(eval(count>1)) as duplicate_downloads, sum(count) as all_downloads 144 | count(eval(count>8)) as excessive_duplicate_downloads by index_name 145 | | eval duplicate_percent=if(all_downloads=0,0,round((duplicate_downloads/all_downloads)*100,2)) 146 | | fields index_name, duplicate_percent all_downloads duplicate_downloads excessive_duplicate_downloads 147 | | rename custom_index as Index, duplicate_percent as "Repeat Download %", all_downloads as "All Downloads", duplicate_downloads as "Repeated" 148 | $time.earliest$ 149 | $time.latest$ 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | CacheManager downloads by age/index 160 | 161 | 162 | ```Thanks to Splunk support for the original version of this search``` index=_audit $host$ TERM(action=remote_bucket_download) TERM(info=completed) 163 | | eval gbps=kb/1024/1024 164 | | eval age=round((now()-earliest_time)/60/60/24) 165 | | bucket span=30 age 166 | | rex field=cache_id "^[^\|]+\|(?P<index_name>[^~]+)~[^~]+~[^~]+" 167 | | eval age_index = age. " - ".index_name 168 | |timechart span=60s sum(gbps) by age_index limit=10 useother=f usenull=f 169 | $time.earliest$ 170 | $time.latest$ 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 |
179 | -------------------------------------------------------------------------------- /default/data/ui/views/splunk_forwarder_data_balance_tuning.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Attempt to measure data balance between HF's, original version by Brett Adam's, similar to splunk_forwarder_output_tuning 4 |
5 | 6 | 7 | 8 | -60m@m 9 | now 10 | 11 | 12 | 13 | 14 | `heavyforwarderhosts` 15 | 16 | 17 | 18 | output_name 19 | output_name 20 | 21 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` TERM(group=tcpout_connections) 22 | | rex field=name "(?P<output_name>[^:]+)" 23 | | stats count by output_name 24 | | fields output_name 25 | -60m@m 26 | now 27 | 28 | 29 |
30 | 31 | 32 | Scatter Line Chart of sum by destination IP 33 | 34 | 35 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` component=Metrics TERM(group=tcpout_connections) name=$output_group$* 36 | | timechart span=1m sum(kb) by destIp limit=50 37 | | fillnull value=0 38 | | untable _time server kb 39 | | eval t=_time-now() 40 | | table t kb 41 | $time.earliest$ 42 | $time.latest$ 43 | 1 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | Total KB by destination IP 61 | 62 | 63 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` component=Metrics TERM(group=tcpout_connections) name=$output_group$* 64 | | timechart span=1m sum(kb) by destIp limit=100 65 | | fillnull value=0 66 | $time.earliest$ 67 | $time.latest$ 68 | 1 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | Standard Deviation 110 | 111 | 112 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` component=Metrics TERM(group=tcpout_connections) name=$output_group$* 113 | | timechart span=1m sum(kb) by destIp limit=50 114 | | fillnull value=0 115 | | untable _time destIp kb 116 | | stats avg(kb) as avg stdev(kb) as stdev by _time 117 | | eval devperc = stdev/avg*100 118 | | table _time devperc 119 | $time.earliest$ 120 | $time.latest$ 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | Data Sum 132 | 133 | 134 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` component=Metrics TERM(group=tcpout_connections) kb>0 name=$output_group$* 135 | | bin span=1m _time 136 | | stats sum(kb) as kb by destIp _time 137 | | sort _time 138 | | streamstats sum(kb) as sumkb by destIp 139 | | timechart span=1m max(sumkb) by destIp useother=false limit=50 140 | $time.earliest$ 141 | $time.latest$ 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 |
151 | -------------------------------------------------------------------------------- /default/data/ui/views/splunk_forwarder_output_tuning.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Splunk forwarder to indexer output tuning 4 |
5 | 6 | 7 | 8 | -60m@m 9 | now 10 | 11 | 12 | 13 | 14 | `heavyforwarderhosts` 15 | 16 | 17 | 18 | output_name 19 | output_name 20 | 21 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` TERM(group=tcpout_connections) 22 | | rex field=name "(?P<output_name>[^:]+)" 23 | | stats count by output_name 24 | | fields output_name 25 | -60m@m 26 | now 27 | 28 | 29 | 30 | 31 | Yes 32 | No 33 | 34 |
35 | 36 | 37 | Data output per-second 38 | 39 | 40 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` TERM(group=tcpout_connections) name=$output_group$* 41 | | rex field=name "(?<output_name>[^:]+)" 42 | | search output_name=$output_group$ 43 | | fillnull ingest_pipe 44 | | eval combined = output_name . "_" . ingest_pipe 45 | | bin _time span=1m 46 | | stats sum(kb) AS totalkb by combined, host, _time 47 | | eval totalkb=totalkb/60 48 | | eval combined = $split_by$ . combined 49 | | timechart limit=99 avg(totalkb) AS avgkb, perc95(totalkb) AS perc95kb, min(totalkb) AS minkb by combined 50 | $time.earliest$ 51 | $time.latest$ 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | Destination count 62 | 63 | 64 | index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` group=tcpout_connections name=$output_group$* 65 | | rex field=name "(?<output_name>[^:]+)" 66 | | search output_name=$output_group$ 67 | | bin _time span=5m 68 | | stats dc(destIp) AS destination_count by output_name, host, _time 69 | | stats min(destination_count) AS min_destination_count, avg(destination_count) AS avg_destination_count by output_name 70 | -24h@h 71 | now 72 | 73 | 74 | 75 |
76 | 77 | 78 | 79 | 80 | Data output std deviation 81 | 82 | 83 | ```Credit to Brett Adams``` index=_internal $host$ sourcetype=splunkd `splunkadmins_metrics_source` component=Metrics TERM(group=tcpout_connections) name=$output_group$* 84 | | rex field=name "(?P<destination>[^:]+)" 85 | | search destination=$output_group$* 86 | | timechart span=1m sum(kb) by destIp limit=50 87 | | fillnull value=0 88 | | untable _time destIp kb 89 | | stats avg(kb) as avg stdev(kb) as stdev by _time 90 | | eval dev_perc = stdev/avg*100 91 | | table _time dev_perc 92 | $time.earliest$ 93 | $time.latest$ 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | Dashboard info 104 | 105 | 106 |

Purpose of destination count table? metrics.log only records the tcpout data *if* the connection is open at the time the metrics.log writes, so the count is to sanity-check that the numbers of connections matches the number of forwarders on the backend (this will happen with the below outputs.conf settings combined with regular data flow)

107 |
108 |

Asynchronous load balancing (docs.splunk.com)

109 |

Splunk Asynchronous Forwarding (Lightning-fast data ingestor)

110 |

Purpose of the data output per-second timechart? The current goal is to get close to switching indexers every second for an output group (per-pipeline), note that this will result in more open connections to indexers so only really works if this is deployed to a moderate number of intermediate forwarders (HF's or similar). Note that you want to do this with autoLBVolume, if you lower autoLBFrequency to a very short time period you may result in un-even data balance due to switching frequently when forwarding smaller volumes of data. In my testing so far it would appear that aiming above the average kb/s for the autoLBVolume appears to work well, going too low doesn't work well in my testing so far

111 |

Please read the linked article for information on these settings, note that when using async forwarding the open file descriptor usage is higher than without async forwarding as the connections are held open by forwarders. So this works great on an intermediate forwarding tier, this may not work so well with a very large number of forwarders

112 |

Also note that the maxQueueSize should not be below 10MB (10MB minimium size)

113 |

If you are using an AWS NLB, you may wish to refer to this newer post Asynchronous forwarding with NLB

114 |

Finally while this also works on UF's, there are some reasons why you may want to consider HF's if you are running an intermediate tier, answers post Wrongly merged Events/permanently blocked tcpout queue with Intermediate Universal Forwarder

115 |

Finally you may want to refer to Slow indexer/receiver detection capability

116 |

What config is used to achieve the above?

117 |

outputs.conf file based on 1MB/s 118 |

maxQueueSize = 10MB 119 |

120 |

121 | #autoLBVolume is set below 1/5 of the maxQueueSize due to changes post 7.3.6 which will hopefully be documented in the very near future, minimum 10MB queue 122 |

123 |

124 | autoLBVolume = 1024000 125 |

126 |

127 | autoLBFrequency = 10 128 |

129 |

130 | connectionTTL = 300 131 |

132 | 133 | 134 | 135 | 136 |
137 | -------------------------------------------------------------------------------- /default/data/ui/views/splunk_introspection_io_stats.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 6 | 7 | -24h@h 8 | now 9 | 10 | 11 | 12 | 13 | `indexerhosts` 14 | 15 | 16 | 17 | 1m 18 | 19 |
20 | 21 | 22 | data.avg_total_ms (average wait time) 23 | 24 | perc95 total io service time per host (sum of all disks avg_total_ms) 25 | 26 | index=_introspection sourcetype=splunk_resource_usage component=IOStats $hosts$ data.device=nvme* 27 | | eval avg_total_ms = 'data.avg_total_ms', comment="You may wish to change sum(avg_total_ms) for perc95 or similar depending on your setup..." 28 | | bin _time span=$span$ 29 | | stats sum(avg_total_ms) AS avg_total_ms by host, _time 30 | | timechart span=$span$ partial=f limit=99 perc95(avg_total_ms) AS avg_total_ms by host 31 | $time.earliest$ 32 | $time.latest$ 33 | 1 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | data.read_ps/data.write_ps 73 | 74 | perc95 reads/writes per second (IOPS) 75 | 76 | index=_introspection sourcetype=splunk_resource_usage component=IOStats $hosts$ data.device=nvme* 77 | | eval reads_ps = 'data.reads_ps', writes_ps = 'data.writes_ps' 78 | | bin _time span=$span$ 79 | | stats sum(reads_ps) AS reads_ps, sum(writes_ps) AS writes_ps by host, _time 80 | | timechart span=$span$ partial=f limit=99 perc95(reads_ps) AS reads_ps, perc95(writes_ps) AS writes_ps by host 81 | $time.earliest$ 82 | $time.latest$ 83 | 1 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | data.read_kb_ps/data.write_kb_ps 123 | 124 | perc95 read KB/write KB per second 125 | 126 | index=_introspection sourcetype=splunk_resource_usage component=IOStats $hosts$ data.device=nvme* 127 | | eval reads_kb_ps = 'data.reads_kb_ps', writes_kb_ps = 'data.writes_kb_ps' 128 | | bin _time span=$span$ 129 | | stats sum(reads_kb_ps) AS reads_kb_ps, sum(writes_kb_ps) AS writes_kb_ps by host, _time 130 | | timechart span=$span$ partial=f limit=99 perc95(reads_kb_ps) AS reads_kb_ps, perc95(writes_kb_ps) AS writes_kb_ps by host 131 | $time.earliest$ 132 | $time.latest$ 133 | 1 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 |
171 | -------------------------------------------------------------------------------- /default/data/ui/views/troubleshooting_indexer_cpu_drilldown.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 6 | 7 | -4h@m 8 | now 9 | 10 | 11 | 12 | 13 | avgCPU, memory 14 | totalCPU, memory 15 | duration, totalCPU 16 | duration, avgCPU 17 | totalAVGCPU, totalMemUsed 18 | totalAVGCPU, totalMemUsed 19 | 20 | 21 | 22 | data.search_props.user= 23 | 24 | 25 | 26 | 27 | 28 | true 29 | $value$ 30 | 31 | 32 | 33 | 34 | 35 | * 36 | 37 | 38 | 39 | Yes 40 | 41 | 42 |
43 | 44 | 45 | Usage Drilldown Per PID 46 | 47 | 48 | index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* $user$ data.search_props.app=$app$ 49 | | eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label' 50 | | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu' 51 | | eval read_mb = 'data.read_mb' 52 | | eval sid='data.search_props.sid' 53 | | eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance) 54 | | eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') 55 | | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPUPerMinute by type, mode, app, user, label, host, data.pid, sid 56 | | stats sum(avgCPUPerMinute) AS totalAVGCPUPerMinute, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label, host, sid, data.pid 57 | | eval totalMemUsed = round(totalMemUsed, 2) 58 | | eval Started=strftime(Started,"%+") 59 | | eval duration = tostring(totalRuntime, "duration") 60 | | eval avgCPU = round(totalAVGCPUPerMinute) 61 | | eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas") 62 | | sort - totalRuntime, totalCPU 63 | | fields - totalRuntime, totalAVGCPUPerMinute, sid 64 | $time_tok.earliest$ 65 | $time_tok.latest$ 66 | 1 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 |
76 | 77 | 78 | 79 | 80 | 81 | Usage Drilldown Per Search Label 82 | 83 | index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* $user$ data.search_props.app=$app$ 84 | | eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label' 85 | | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu' 86 | | eval read_mb = 'data.read_mb' 87 | | eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance) 88 | | eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') 89 | | bin _time span=1m 90 | | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPU by type, mode, app, user, label, data.pid, host 91 | | stats sum(avgCPU) AS totalAVGCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label 92 | | eval totalMemUsed = round(totalMemUsed, 2) 93 | | eval Started=strftime(Started,"%+") 94 | | eval duration = tostring(totalRuntime, "duration") 95 | | eval avgCPU = round(totalAVGCPU) 96 | | eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas") 97 | | sort - $sort$ 98 | | fields - totalRuntime, totalAVGCPU 99 | $time_tok.earliest$ 100 | $time_tok.latest$ 101 | 102 | 103 |
104 | 105 | 106 | 107 | 108 | 109 | Recently Used URL By User 110 | 111 | index=_internal sourcetype=splunkd_ui_access user=$uservalue$ `searchheadhosts` | top referer 112 | $time_tok.earliest$ 113 | $time_tok.latest$ 114 | 115 |
116 | 117 | 118 |
119 | -------------------------------------------------------------------------------- /default/data/ui/views/troubleshooting_resource_usage_per_user.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | This dashboard attempts to assist with finding which queries are using excessive amounts of CPU, memory, disk IOPS at the indexing tier and the queries behind them 4 |
5 | 6 | 7 | 8 | -4h@m 9 | now 10 | 11 | 12 | 13 | 14 | Yes 15 | No 16 | "data.search_props.user"!=admin "data.search_props.user"!=splunk-system-user 17 | "data.search_props.user"!=admin "data.search_props.user"!=splunk-system-user 18 | 19 | 20 | 21 | totalCPU 22 | avgCPUPerIndexer 23 | totalduration 24 | averageduration 25 | totalMemUsed 26 | totalReadMB 27 | count 28 | totalCPU 29 | 30 | 31 | 32 | 60m 33 | 34 | 35 | 36 | 37 | "" 38 | 39 |
40 | 41 | 42 | Resource Usage Per User 43 | 44 | count is the number of searches triggered during that time period (dashboards may have multiple searches), introspection is measured in 10 second blocks (so sometimes no stats are available) 45 | 46 | index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* $exclusion$ 47 | | eval mem_used = 'data.mem_used' 48 | | eval app = 'data.search_props.app' 49 | | eval elapsed = 'data.elapsed' 50 | | eval label = 'data.search_props.label' 51 | | eval type = 'data.search_props.type' 52 | | eval mode = 'data.search_props.mode' 53 | | eval user = 'data.search_props.user' 54 | | eval cpuperc = 'data.pct_cpu' 55 | | eval search_head = 'data.search_props.search_head' 56 | | eval read_mb = 'data.read_mb' 57 | | eval provenance='data.search_props.provenance' 58 | | eval label=coalesce(label, provenance) 59 | | eval sid='data.search_props.sid' 60 | | search $filter$ 61 | | rex field=sid "^remote_[^_]+_(?P<sid>.*)" 62 | | eval sid = "'" . sid . "'" 63 | | fillnull search_head value="*" 64 | | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as searchStartTime, sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU, max(read_mb) AS read_mb, values(sid) AS sids by type, mode, app, user, label, host, search_head, data.pid 65 | | bin searchStartTime span=$timespan$ 66 | | stats dc(sids) AS count, sum(totalCPU) AS totalCPU, sum(mem_used) AS totalMemUsed, max(runtime) AS maxRunTime, avg(runtime) AS avgRuntime, avg(avgCPU) AS avgCPUPerIndexer, sum(read_mb) AS totalReadMB, values(sids) AS sids by searchStartTime, type, mode, app, user, search_head, label 67 | | eval maxduration = tostring(maxRunTime, "duration"), averageduration = tostring(avgRuntime, "duration") 68 | | eval Started = strftime(searchStartTime,"%+") 69 | | eval avgCPUPerIndexer = round(avgCPUPerIndexer) 70 | | sort - $sort$ 71 | | eval totalCPU=tostring(totalCPU,"commas"), avgCPUAcrossAllIndexers=tostring(avgCPUAcrossAllIndexers,"commas"), totalReadMB=tostring(totalReadMB, "commas"), totalMemUsed=tostring(totalMemUsed, "commas") 72 | | table Started, count, user, app, label, averageduration, maxduration, totalCPU, avgCPUPerIndexer, totalReadMB, totalMemUsed, search_head, sids, mode, type 73 | $time.earliest$ 74 | $time.latest$ 75 | 1 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | ["Started","count","user","app","label","averageduration","maxduration","totalCPU","avgCPUPerIndexer","totalReadMB","totalMemUsed","mode","type"] 85 | 86 | /app/SplunkAdmins/troubleshooting_resource_usage_per_user_drilldown?form.sid=$row.sids$&form.host=$row.search_head$&form.app=$row.app$&form.label=$row.label$&form.time.earliest=$time.earliest$&form.time.latest=$time.latest$ 87 | 88 |
89 | 90 | 91 |
92 | -------------------------------------------------------------------------------- /default/data/ui/views/troubleshooting_resource_usage_per_user_drilldown.xml: -------------------------------------------------------------------------------- 1 |
2 | 3 | Drilldown for Troubleshooting Resource Usage Per User (Splunk 6.6+ only due to the use of the IN keyword) 4 |
5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -15m 21 | now 22 | 23 | 24 |
25 | 26 | 27 | Query information from audit logs 28 | 29 | 30 | index=_audit host=$host$ "info=granted" OR "info=completed" OR "info=canceled" search_id IN ($sid$) 31 | | rex ", search='(?P<search>[\S+\s+]+?)', " 32 | | stats min(_time) AS time, max(_time) AS max_timestamp, values(user) AS user, values(total_run_time) AS total_run_time, values(result_count) AS result_count, values(search) AS search, values(host) AS host, values(search_et) AS startTime, values(search_lt) AS endTime, values(info) AS info, values(savedsearch_name) AS savedsearch_name by search_id 33 | | eval app="$app$", label="$label$" 34 | | eval endTime=if((info=="completed" OR info=="canceled") AND endTime=="N/A",max_timestamp,endTime) 35 | | eval period=tostring(round(endTime-startTime), "duration") 36 | | eval startTime=strftime(startTime, "%Y-%m-%d %H:%M:%S"), endTime=strftime(endTime, "%Y-%m-%d %H:%M:%S") 37 | | fillnull value="All Time" startTime endTime period 38 | | table time, app, user, total_run_time, result_count, period, search, label, host, startTime, endTime, info, savedsearch_name, search_id 39 | | sort - time 40 | $time.earliest$ 41 | $time.latest$ 42 | 1 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 |
53 | 54 | 55 |
56 | -------------------------------------------------------------------------------- /default/props.conf: -------------------------------------------------------------------------------- 1 | #Splunk does not index the search.log files from the dispatch directory by default 2 | #so create a stanza to take only the parts we care about... 3 | #Example lines to look for include 4 | #05-24-2018 08:31:03.881 ERROR SearchResultTransaction - Got status 502 from https://x.x.x.x:8089/services/streams/search?sh_sid=1527150641.164891_315974D3-2FA6-4A16-839A-A95A0376BA14 5 | #05-24-2018 08:31:03.881 ERROR SearchResultTransaction - HTTP error status message from https://x.x.x.x:8089/services/streams/search?sh_sid=1527150641.164891_315974D3-2FA6-4A16-839A-A95A0376BA14: Error connecting: Connect Timeout 6 | #05-24-2018 08:31:03.881 ERROR DispatchThread - sid:1527150641.164891_315974D3-2FA6-4A16-839A-A95A0376BA14 Unknown error for peer indexername. Search Results might be incomplete. If this occurs frequently, please check on the peer. 7 | #05-28-2018 00:52:17.245 INFO DispatchThread - sid:1527468707.34320_315974D3-DFFC-48EC-86C8-33BD6744EE4F Search auto-finalized after time limit (30 seconds) reached. 8 | #however a better alternative may be [search] 9 | #log_search_messages = true 10 | #In the limits.conf file and then use the search_messages.log file... 11 | [splunk:searchlog] 12 | TIME_PREFIX = ^ 13 | TIME_FORMAT = %m-%d-%Y %H:%M:%S.%3N 14 | SHOULD_LINEMERGE = false 15 | TRANSFORMS-set = setNull,setError,setAutoFinalize 16 | 17 | #Example inputs.conf if you want to use the above in Linux 18 | #[monitor:///opt/splunk/var/run/splunk/dispatch/*/search.log] 19 | #sourcetype = splunk:searchlog 20 | #index = _internal 21 | 22 | #Splunk records failures from search heads to indexer for corrupt buckets in the info.csv log only on the search head level 23 | #the search.log on the indexer peers *will* record this so if your ingesting the search.log from the peers you probably don't need this one... 24 | #The info.csv does show you what the end user will see in terms of errors such as this... 25 | #Examples include: 26 | #,,,,,,,,,,,,,,,,,ERROR,"[hostname] Failed to read size=1 event(s) from rawdata in bucket='_internal~43~E21ADB4E-02B7-4877-8A42-A15CE7F422BD' path='.../db_1515304396_1515080916_.... Rawdata may be corrupt, see search.log. Results may be incomplete!","{}",,,,,,, 27 | #Note that a better alternative may be [search] 28 | #log_search_messages = true 29 | #In the limits.conf file and then use the search_messages.log file... 30 | [splunk:search:info] 31 | SHOULD_LINEMERGE = false 32 | DATETIME_CONFIG = NONE 33 | TRANSFORMS-set = setNull,setWARNorERROR,setAutoFinalize 34 | 35 | #Example inputs.conf if you want to use the above in Linux 36 | #[monitor:///opt/splunk/var/run/splunk/dispatch/*/info.csv] 37 | #sourcetype = splunk:search:info 38 | #index = _internal 39 | #crcSalt = 40 | -------------------------------------------------------------------------------- /default/transforms.conf: -------------------------------------------------------------------------------- 1 | [setNull] 2 | REGEX = . 3 | DEST_KEY = queue 4 | FORMAT = nullQueue 5 | 6 | [setError] 7 | REGEX = ^[01]\d-[0-3]\d-20\d\d \d{2}:\d{2}:\d{2}.\d{3}\s+ERROR\s+ 8 | DEST_KEY = queue 9 | FORMAT = indexQueue 10 | 11 | [setAutoFinalize] 12 | REGEX = Search auto-finalized after 13 | DEST_KEY = queue 14 | FORMAT = indexQueue 15 | 16 | #Only include warning or error entries 17 | [setWARNorERROR] 18 | REGEX = ,(?:ERROR|WARN), 19 | DEST_KEY = queue 20 | FORMAT = indexQueue 21 | 22 | [splunkadmins_macros] 23 | #This config failed below with ERROR KVStoreLookup - KV Store output failed with err: The provided query was invalid. (Document may not contain '$' or '.' in keys.) message: 24 | #Switching back to csv files for now 25 | #collection = splunkadmins_macros 26 | #external_type = kvstore 27 | #fields_list = definition, eai:acl.app, title 28 | batch_index_query = 0 29 | case_sensitive_match = 1 30 | collection = 31 | external_type = 32 | fields_list = 33 | filename = splunkadmins_macros.csv 34 | 35 | [splunkadmins_userlist_indexinfo] 36 | collection = splunkadmins_userlist_indexinfo 37 | #external_type = kvstore 38 | #fields_list = srchIndexesAllowed, srchIndexesDefault, user 39 | filename = splunkadmins_userlist_indexinfo.csv 40 | 41 | [splunkadmins_indexlist] 42 | batch_index_query = 0 43 | case_sensitive_match = 1 44 | filename = splunkadmins_indexlist.csv 45 | 46 | [splunkadmins_indexes_per_role] 47 | batch_index_query = 0 48 | case_sensitive_match = 1 49 | filename = splunkadmins_indexes_per_role.csv 50 | 51 | [splunkadmins_datamodels] 52 | batch_index_query = 0 53 | case_sensitive_match = 0 54 | filename = splunkadmins_datamodels.csv 55 | 56 | [splunkadmins_tags] 57 | batch_index_query = 0 58 | case_sensitive_match = 0 59 | filename = splunkadmins_tags.csv 60 | 61 | [splunkadmins_eventtypes] 62 | batch_index_query = 0 63 | case_sensitive_match = 0 64 | filename = splunkadmins_eventtypes.csv 65 | 66 | [splunkadmins_rmd5_to_savedsearchname] 67 | batch_index_query = 0 68 | case_sensitive_match = 0 69 | filename = splunkadmins_rmd5_to_savedsearchname.csv 70 | 71 | [splunkadmins_indexlist_by_cluster] 72 | batch_index_query = 0 73 | case_sensitive_match = 1 74 | filename = splunkadmins_indexlist_by_cluster.csv 75 | 76 | #Note that the lookup splunkadmins_hec_reply_code_lookup is based on https://github.com/redvelociraptor/gettingsmarter/blob/main/dashboards/hec_reply_codes.csv (previously https://docs.splunk.com/Documentation/Splunk/latest/Data/TroubleshootHTTPEventCollector) and this may change over time 77 | [splunkadmins_hec_reply_code_lookup] 78 | batch_index_query = 0 79 | case_sensitive_match = 1 80 | filename = splunkadmins_hec_reply_code_lookup.csv 81 | 82 | [splunkadmins_lookupfile_owners] 83 | batch_index_query = 0 84 | case_sensitive_match = 1 85 | filename = splunkadmins_lookupfile_owners.csv 86 | -------------------------------------------------------------------------------- /lookups/splunkadmins_datamodels.csv: -------------------------------------------------------------------------------- 1 | datamodel,sharing,app,definition,splunk_server 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_eventtypes.csv: -------------------------------------------------------------------------------- 1 | eventtype,definition,app,sharing,"splunk_server" 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_hec_reply_code_lookup.csv: -------------------------------------------------------------------------------- 1 | status_code,http_status_code_id,http_status_code_text,status_message,reason,action 2 | 0,200,OK,Success,, 3 | 1,403,Forbidden,Token disabled,Client is sending using a disabled token,Splunk Admin needs to enable the token or have client use new token. 4 | 2,401,Unauthorized,Token is required,Client is sending without a token,Splunk Admin needs to find what client is trying to send without a token. 5 | 3,401,Unauthorized,Invalid authorization,Client is sending with an incorrect Authorization Header,"Splunk Admin needs to work with client/user to ensure Authorization Header is correct, most common cause is the word Splunk is missing before the token." 6 | 4,403,Forbidden,Invalid token,Client is sending with a token the receiver(s) don't know of,Splunk Admin needs to work with client/user to ensure they are using a valid token. 7 | 5,400,Bad Request,No data,Client is sending without any data,Splunk Admin needs to work with client/user to ensure sending side is configured to send data properly. If there is a token and a channel ID with no payload this is more than likely AWS Firehose second connection to ensure it can send data to Splunk. It's testing the event endpoint and it's expecting to get a 400 reply code. 8 | 6,400,Bad Request,Invalid data format,Client is sending with data in an invalid format,"Splunk Admin needs to work with client/user to ensure sending side is using a proper format, the raw source should be looked at and the log entry for parsing_err will point to what to look for. In Splunk versions newer then 8.1.2103, as a last resort debug can be used." 9 | 7,400,Bad Request,Incorrect index,Client is trying to send to an index not in the tokens allow list,Splunk Admin needs to work with client/user to ensure the sending side is trying to send to indexes listed in the tokens allow list. Correction can be on the client sending side or adding the index to the token in Splunk. 10 | 8,500,Internal Error,Internal server error,Receiver had an issue client should retry to send,Client should automatically try to resend the data. If the issue happens too often then a support case should be filed so that the issue can be investigated deeper. 11 | 9,503,Service Unavailable,Server is busy,Receiver had an issue receiving client should retry to send,"Client should automatically try to resend data, occasional Server Is Busy messages are expected. If the message happens too often a support case should be filed and investigated further." 12 | 10,400,Bad Request,Data channel is missing,Client is trying to send to a token that has useACK enabled channel id is needed,Splunk Admin needs to work with client/user to ensure they are using the correct token and the sending side is configured properly. 13 | 11,400,Bad Request,Invalid data channel,Client is trying to send with an improperly formatted data channel id,Splunk Admin needs to work with the client/user to ensure they send using a properly formatted data channel id. 14 | 12,400,Bad Request,Event field is required,Client is trying to send without an event field,Splunk Admin needs to work with the client/user to ensure they are sending in a proper format. An event field is not being sent. 15 | 13,400,Bad Request,Event field cannot be blank,Client is trying to send with an empty event field,Splunk Admin needs to work with the client/user to ensure they are sending in a proper format. The event field is empty. 16 | 14,400,Bad Request,ACK is disabled,Client is trying to use useACK on a token that it is not enabled on,Splunk Admin needs to work with the client/user to ensure they are using the correct token for their data in the proper format. 17 | 15,400,Bad Request,Error in handling indexed fields,Client is trying to send where index fields are incorrect,Splunk Admin needs to work with the client/user to ensure they are using index fields correctly for HEC. 18 | 16,400,Bad Request,Query string authorization is not enabled,Client is trying to send with query string authorization where it is not enabled,Splunk Admin needs to open a Support case to enable query string authorization to the token. Understand the security risk of Query string authorization. The HEC token can be logged in plain text as part of the url. 19 | 17,200,OK,HEC is healthy,, 20 | 18,503,Service Unavailable,"HEC is unhealthy, queues are full",Receiver Queues are full, 21 | 19,503,Service Unavailable,"HEC is unhealthy, ack service unavailable",, 22 | 20,503,Service Unavailable,"HEC is unhealthy, queues are full, ack service unavailable",, 23 | 21,400,Bad Request,Invalid token,, 24 | 22,400,Bad Request,Token disabled,, 25 | -------------------------------------------------------------------------------- /lookups/splunkadmins_indexes_per_role.csv: -------------------------------------------------------------------------------- 1 | roles,"splunk_server",srchIndexesAllowed,srchIndexesDefault 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_indexlist.csv: -------------------------------------------------------------------------------- 1 | index 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_indexlist_by_cluster.csv: -------------------------------------------------------------------------------- 1 | indexer_cluster,index 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_lookupfile_owners.csv: -------------------------------------------------------------------------------- 1 | lookup_file,app,owner,owner_from_definitio,sharing,lookup_name 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_macros.csv: -------------------------------------------------------------------------------- 1 | title,app,"splunk_server",definition,sharing 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_rmd5_to_savedsearchname.csv: -------------------------------------------------------------------------------- 1 | RMDvalue,savedsearch_name 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_tags.csv: -------------------------------------------------------------------------------- 1 | tag,definition,app,sharing,"splunk_server" 2 | -------------------------------------------------------------------------------- /lookups/splunkadmins_userlist_indexinfo.csv: -------------------------------------------------------------------------------- 1 | srchIndexesAllowed,srchIndexesDefault,user 2 | -------------------------------------------------------------------------------- /metadata/default.meta: -------------------------------------------------------------------------------- 1 | # Application-level permissions 2 | [] 3 | access = read : [ admin, sc_admin ], write : [ admin, sc_admin ] 4 | 5 | [eventtypes] 6 | export = none 7 | 8 | [props] 9 | export = none 10 | 11 | [transforms] 12 | export = none 13 | 14 | [lookups] 15 | export = none 16 | 17 | [tags] 18 | export = none 19 | 20 | [viewstates] 21 | access = read : [ * ], write : [ * ] 22 | export = none 23 | 24 | -------------------------------------------------------------------------------- /static/appIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gjanders/SplunkAdmins/da8b07f6cb89218435926935ff0e1de72f7ea4ee/static/appIcon.png -------------------------------------------------------------------------------- /static/appIconAlt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gjanders/SplunkAdmins/da8b07f6cb89218435926935ff0e1de72f7ea4ee/static/appIconAlt.png -------------------------------------------------------------------------------- /static/appIconAlt_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gjanders/SplunkAdmins/da8b07f6cb89218435926935ff0e1de72f7ea4ee/static/appIconAlt_2x.png -------------------------------------------------------------------------------- /static/appIcon_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gjanders/SplunkAdmins/da8b07f6cb89218435926935ff0e1de72f7ea4ee/static/appIcon_2x.png -------------------------------------------------------------------------------- /static/appLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gjanders/SplunkAdmins/da8b07f6cb89218435926935ff0e1de72f7ea4ee/static/appLogo.png -------------------------------------------------------------------------------- /static/appLogo_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gjanders/SplunkAdmins/da8b07f6cb89218435926935ff0e1de72f7ea4ee/static/appLogo_2x.png --------------------------------------------------------------------------------