├── LICENSE ├── README.md └── aws-logsearch.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tony Meehan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aws-logsearch 2 | 3 | Search AWS CloudWatch logs all at once on the command line. This uses the aws [sdk-for-go](https://docs.aws.amazon.com/sdk-for-go/api/service/cloudwatchlogs/#pkg-overview). See [Configuring Credentials](https://github.com/aws/aws-sdk-go/#configuring-credentials) to configure your AWS secrets. 4 | 5 | # Installation 6 | 7 | ``` 8 | go get -u github.com/endgameinc/aws-logsearch 9 | ``` 10 | 11 | # Usage 12 | 13 | ``` 14 | # ~/go/bin/aws-logsearch 15 | 16 | Usage: ~/go/bin/aws-logsearch [options] 17 | 18 | Options: 19 | -p, --pattern search pattern 20 | -g, --group search specific log group 21 | -l, --list list log groups 22 | -s, --starttime start time, e.g. 2018-11-05 14:45:03 23 | -e, --endtime end time, e.g. 2018-11-06 00:03:43 24 | -c, --count print first count log matches only 25 | -r, --region aws region (default us-east-1) 26 | ``` 27 | 28 | To search a log group that matches the substring "group1" for "login" and only print the first 5 results, run the following command: 29 | 30 | ``` 31 | # ~/go/bin/aws-logsearch -p "login" -g "group001" -c 5 32 | Found Log Group: /group001/logs 33 | [/group001/logs] [2018-10-08 13:02:47.357 +0000 UTC] [endgame.log] Oct 8 13:02:47 ip-172-16-100-174 app[47690]: INFO 200 POST /api/auth/login (127.0.0.1) 45.02ms 34 | [/group001/logs] [2018-10-08 14.35.07.357 +0000 UTC] [endgame.log] Oct 8 14:35:07 ip-172-16-100-174 app[47683]: INFO 200 POST /api/auth/login (127.0.0.1) 43.23ms 35 | [/group001/logs] [2018-10-08 21:28:00.357 +0000 UTC] [endgame.log] Oct 8 21:28:00 ip-172-16-100-174 app[47690]: INFO 200 POST /api/auth/login (127.0.0.1) 43.90ms 36 | [/group001/logs] [2018-10-08 21:38:11.357 +0000 UTC] [endgame.log] Oct 8 21:38:11 ip-172-16-100-174 app[47683]: INFO 200 POST /api/auth/login (127.0.0.1) 43.46ms 37 | [/group001/logs] [2018-10-08 21:41:05.357 +0000 UTC] [endgame.log] Oct 8 21:41:05 ip-172-16-100-174 app[47683]: WARNING 401 POST /api/auth/login (127.0.0.1) 36.47ms 38 | ``` 39 | 40 | To search all log groups for "login" on or after November 5th, 2018 after midnight (UTC) and print only the first result, run the following command: 41 | 42 | ``` 43 | # ~/go/bin/aws-logsearch -p "login" -s "2018-11-05 00:00:00" -c 1 44 | [/group001/logs] [2018-11-08 16:11:09.357 +0000 UTC] [nginx-access.log] 127.0.0.1 - - [08/Nov/2018:16:10:08 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 1250 "-" "python-requests/2.19.1" "-" - 0.314 38 45 | [/group002/logs] [2018-11-05 00:20:36.862 +0000 UTC] [endgamelog] Nov 5 00:20:36 ip-172-16-100-252 app[27954]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 44.07ms 46 | [/group003/logs] [2018-11-05 00:13:57.877 +0000 UTC] [endgamelog] Nov 5 00:13:57 ip-172-16-100-178 app[10717]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 44.71ms 47 | [/group004/logs] [2018-11-05 00:00:25.85 +0000 UTC] [endgame.log] Nov 5 00:00:25 ip-172-16-100-97 app[20750]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 44.67ms 48 | [/group005/logs] [2018-11-05 00:13:57.838 +0000 UTC] [endgame.log] Nov 5 00:13:57 ip-172-16-100-97 app[12580]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 44.79ms 49 | [/group006/logs] [2018-11-05 00:01:02.318 +0000 UTC] [endgame.log] Nov 5 00:01:02 ip-172-16-100-176 app[24760]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 45.19ms 50 | [/group007/logs] [2018-11-05 00:00:40.661 +0000 UTC] [endgame.log] Nov 5 00:00:40 ip-172-31-86-221.ec2.internal app[114811]: INFO 200 POST /api/auth/login (127.0.0.1) 104.82ms 51 | [/group009/logs] [2018-11-26 22:30:32.405 +0000 UTC] [nginx-access.log] 127.0.0.1 - - [26/Nov/2018:22:29:18 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 1250 "-" "python-requests/2.19.1" "-" - 0.311 29 52 | [/group010/logs] [2018-11-21 17:59:50.867 +0000 UTC] [nginx-access.log] 127.0.0.1 - - [21/Nov/2018:17:58:35 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 1250 "-" "python-requests/2.19.1" "-" - 0.329 38 53 | [/group011/logs] [2018-11-27 19:36:00.494 +0000 UTC] [nginx-access.log] 127.0.0.1 - - [27/Nov/2018:19:34:59 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 1250 "-" "python-requests/2.19.1" "-" - 0.315 37 54 | [/group012/logs] [2018-11-05 00:01:12.046 +0000 UTC] [endgame.log] Nov 5 00:01:11 ip-172-16-100-206 app[2088]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 44.45ms 55 | [/group013/logs] [2018-11-26 16:22:35.52 +0000 UTC] [nginx-access.log] 127.0.0.1 - - [26/Nov/2018:16:21:14 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 1250 "-" "python-requests/2.19.1" "-" - 0.365 66 56 | [/group014/logs] [2018-11-05 00:00:07.077 +0000 UTC] [endgame.log] Nov 5 00:00:06 ip-172-16-100-174 app[69249]: INFO 200 POST /api/auth/login (127.0.0.1) 43.06ms 57 | [/group015/logs] [2018-11-21 18:50:43.437 +0000 UTC] [nginx-access.log] 127.0.0.1 - - [21/Nov/2018:18:49:43 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 1250 "-" "python-requests/2.19.1" "-" - 0.329 41 58 | [/group016/logs] [2018-11-16 18:15:49.196 +0000 UTC] [nginx-access.log] 127.0.0.1 - - [16/Nov/2018:18:14:46 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 1250 "-" "python-requests/2.19.1" "-" - 0.327 43 59 | [/group017/logs] [2018-11-16 18:37:06.362 +0000 UTC] [endgame.log] Nov 16 18:36:05 ip-172-16-100-226 app[9145]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 313.09ms 60 | [/group019/logs] [2018-11-05 15:43:20.041 +0000 UTC] [endgame.log] Nov 5 15:43:19 ip-172-16-100-104 app[32409]: INFO 200 POST /api/auth/login (127.0.0.1) 46.22ms 61 | [/group020/logs] [2018-11-07 23:20:36.402 +0000 UTC] [endgame.log] Nov 7 23:19:34 ip-172-16-100-120 app[8214]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 325.40ms 62 | [/group021/logs] [2018-11-05 12:26:30.92 +0000 UTC] [endgame.log] Nov 5 12:26:30 ip-172-16-100-36 app[50664]: INFO 200 POST /api/auth/login (127.0.0.1) 44.26ms 63 | [/group022/logs] [2018-11-06 15:19:36.358 +0000 UTC] [endgame.log] Nov 6 15:19:36 ip-172-16-100-6 app[4007]: WARNING 401 POST /api/auth/login (127.0.0.1) 334.18ms 64 | [/group023/logs] [2018-11-05 14:33:31.492 +0000 UTC] [endgame.log] Nov 5 14:33:31 ip-172-16-100-72 app[25229]: INFO 200 POST /api/auth/login (127.0.0.1) 44.24ms 65 | [/group024/logs] [2018-11-05 14:27:00.391 +0000 UTC] [endgame.log] Nov 5 14:27:00 ip-172-16-100-133 app[8211]: INFO 200 POST /api/auth/login (127.0.0.1) 44.88ms 66 | [/group025/logs] [2018-11-05 05:00:31.032 +0000 UTC] [endgame.log] Nov 5 05:00:31 ip-172-16-100-128 app[13572]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 44.65ms 67 | [/group026/logs] [2018-11-05 12:29:15.487 +0000 UTC] [endgame.log] Nov 5 12:29:15 ip-172-16-100-147 app[66209]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 43.62ms 68 | [/group027/logs] [2018-11-05 12:29:13.295 +0000 UTC] [endgame.log] Nov 5 12:29:13 ip-172-16-100-142 app[23130]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 43.83ms 69 | [/group028/logs] [2018-11-05 12:29:14.6 +0000 UTC] [endgame.log] Nov 5 12:29:14 ip-172-16-100-228 app[4950]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 44.10ms 70 | [/group029/logs] [2018-11-05 22:45:48.284 +0000 UTC] [endgame.log] Nov 5 22:44:42 ip-172-16-100-101 app[8325]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 312.97ms 71 | [/group032/logs] [2018-11-05 13:07:40.835 +0000 UTC] [endgame.log] Nov 5 13:07:40 ip-172-16-100-186 app[77543]: INFO 200 POST /api/auth/login (127.0.0.1) 44.67ms 72 | [/group033/logs] [2018-11-16 00:45:39.145 +0000 UTC] [endgame.log] Nov 16 00:44:25 ip-172-16-100-32 app[42013]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 347.84ms 73 | [/group034/logs] [2018-11-05 04:39:27.531 +0000 UTC] [endgame.log] Nov 5 04:39:27 ip-172-16-100-131 app[12860]: INFO 200 POST /api/auth/login (127.0.0.1) 44.26ms 74 | [/group035/logs] [2018-11-05 12:29:16.771 +0000 UTC] [endgame.log] Nov 5 12:29:16 ip-172-16-100-31 app[8206]: INFO 200 POST /api/v1/auth/login (127.0.0.1) 43.96ms 75 | [/group036/logs] [2018-11-05 05:00:24.778 +0000 UTC] [endgame.log] Nov 5 05:00:24 ip-172-16-100-79 app[28815]: INFO 200 POST /api/v1/auth/login/ (127.0.0.1) 43.23ms 76 | [/group037/logs] [2018-11-05 15:20:51.171 +0000 UTC] [endgame.log] Nov 5 15:20:51 ip-172-16-100-201 app[22039]: INFO 200 POST /api/auth/login (127.0.0.1) 44.45ms 77 | ``` 78 | -------------------------------------------------------------------------------- /aws-logsearch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/session" 13 | "github.com/aws/aws-sdk-go/service/cloudwatchlogs" 14 | ) 15 | 16 | var usageStr = ` 17 | Usage: %s [options] 18 | 19 | Options: 20 | -p, --pattern search pattern 21 | -g, --group search specific log group 22 | -l, --list list log groups 23 | -s, --starttime start time, e.g. 2018-11-05 14:45:03 24 | -e, --endtime end time, e.g. 2018-11-06 00:03:43 25 | -c, --count print first count log matches only 26 | -r, --region aws region (default us-east-1) 27 | ` 28 | 29 | func usage() { 30 | fmt.Printf("%s\n", fmt.Sprintf(usageStr, os.Args[0])) 31 | os.Exit(0) 32 | } 33 | 34 | func CaseInsensitiveContains(s, substr string) bool { 35 | s, substr = strings.ToUpper(s), strings.ToUpper(substr) 36 | return strings.Contains(s, substr) 37 | } 38 | 39 | func printLogGroups(lgs *cloudwatchlogs.DescribeLogGroupsOutput) { 40 | fmt.Println(lgs) 41 | } 42 | 43 | func searchLogGroup( 44 | logs *cloudwatchlogs.CloudWatchLogs, 45 | name string, 46 | pattern string, 47 | token string, 48 | startTime int64, 49 | endTime int64, 50 | count int) { 51 | 52 | if count == 0 { 53 | return 54 | } 55 | 56 | filter := &cloudwatchlogs.FilterLogEventsInput{} 57 | filter.SetFilterPattern(pattern) 58 | filter.SetLogGroupName(name) 59 | 60 | if startTime != 0 { 61 | filter.SetStartTime(startTime) 62 | } 63 | 64 | if endTime != 0 { 65 | filter.SetEndTime(endTime) 66 | } 67 | 68 | if token != "" { 69 | filter.SetNextToken(token) 70 | } 71 | 72 | resp, err := logs.FilterLogEvents(filter) 73 | if err != nil { 74 | fmt.Println(err) 75 | return 76 | } 77 | 78 | eventCount := 0 79 | 80 | location, err := time.LoadLocation("UTC") 81 | if err != nil { 82 | fmt.Println(err) 83 | return 84 | } 85 | 86 | for _, m := range resp.Events { 87 | if eventCount == count { 88 | return 89 | } 90 | 91 | msg := *m.Message 92 | timestamp := *m.Timestamp 93 | filename := *m.LogStreamName 94 | 95 | t := time.Unix(0, (timestamp * int64(time.Millisecond))).In(location) 96 | fmt.Printf("[%s] [%s] [%s] %s\n", name, t, filename, msg) 97 | 98 | eventCount += 1 99 | } 100 | 101 | if resp.NextToken != nil && *resp.NextToken != "" { 102 | searchLogGroup(logs, name, pattern, *resp.NextToken, startTime, endTime, (count - eventCount)) 103 | } 104 | } 105 | 106 | func searchLogGroups( 107 | logs *cloudwatchlogs.CloudWatchLogs, 108 | lgs *cloudwatchlogs.DescribeLogGroupsOutput, 109 | pattern string, 110 | startTime int64, 111 | endTime int64, 112 | count int) { 113 | 114 | var wg sync.WaitGroup 115 | 116 | for _, v := range lgs.LogGroups { 117 | name := *v.LogGroupName 118 | 119 | wg.Add(1) 120 | go func() { 121 | defer wg.Done() 122 | searchLogGroup(logs, name, pattern, "", startTime, endTime, count) 123 | }() 124 | } 125 | 126 | wg.Wait() 127 | } 128 | 129 | func matchLogGroupName(lgs *cloudwatchlogs.DescribeLogGroupsOutput, group string) string { 130 | for _, v := range lgs.LogGroups { 131 | name := *v.LogGroupName 132 | if CaseInsensitiveContains(name, group) { 133 | return name 134 | } 135 | } 136 | return "" 137 | } 138 | 139 | func parseTimestampToMillis(timestamp string) (int64, error) { 140 | layout := "2006-01-02 15:04:05" 141 | t, err := time.Parse(layout, timestamp) 142 | if err != nil { 143 | return 0, err 144 | } 145 | nanos := t.UnixNano() 146 | return (nanos / int64(time.Millisecond)), nil 147 | } 148 | 149 | func main() { 150 | 151 | desc := "pattern" 152 | defaultValue := "" 153 | pattern := defaultValue 154 | flag.StringVar(&pattern, "p", defaultValue, desc) 155 | flag.StringVar(&pattern, "pattern", defaultValue, desc) 156 | 157 | desc = "list" 158 | list := false 159 | flag.BoolVar(&list, "l", false, desc) 160 | flag.BoolVar(&list, "list", false, desc) 161 | 162 | desc = "log group" 163 | group := "" 164 | flag.StringVar(&group, "g", defaultValue, desc) 165 | flag.StringVar(&group, "group", defaultValue, desc) 166 | 167 | desc = "start time, e.g. 2018-11-05 14:45" 168 | startTimeString := "" 169 | flag.StringVar(&startTimeString, "s", defaultValue, desc) 170 | flag.StringVar(&startTimeString, "startime", defaultValue, desc) 171 | 172 | desc = "end time, e.g. 2018-11-06 00:03" 173 | endTimeString := "" 174 | flag.StringVar(&endTimeString, "e", defaultValue, desc) 175 | flag.StringVar(&endTimeString, "endtime", defaultValue, desc) 176 | 177 | desc = "print first count log matches only" 178 | count := -1 179 | flag.IntVar(&count, "c", count, desc) 180 | flag.IntVar(&count, "count", count, desc) 181 | 182 | desc = "region (default us-east-1)" 183 | region := "us-east-1" 184 | flag.StringVar(®ion, "r", region, desc) 185 | flag.StringVar(®ion, "region", region, desc) 186 | 187 | flag.Usage = usage 188 | flag.Parse() 189 | 190 | sess := session.Must(session.NewSession(&aws.Config{ 191 | Region: aws.String(region), 192 | })) 193 | 194 | logs := cloudwatchlogs.New(sess) 195 | 196 | lgs, err := logs.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{}) 197 | if err != nil { 198 | fmt.Println(err) 199 | os.Exit(1) 200 | } 201 | 202 | if list == true { 203 | printLogGroups(lgs) 204 | os.Exit(0) 205 | } 206 | 207 | startTime := int64(0) 208 | endTime := int64(0) 209 | 210 | if startTimeString != "" { 211 | startTime, err = parseTimestampToMillis(startTimeString) 212 | if err != nil { 213 | fmt.Printf("error parsing start timestamp: %s\n", err) 214 | return 215 | } 216 | } 217 | if endTimeString != "" { 218 | endTime, err = parseTimestampToMillis(endTimeString) 219 | if err != nil { 220 | fmt.Printf("error parsing end timestamp: %s\n", err) 221 | return 222 | } 223 | } 224 | 225 | if group != "" { 226 | matchedGroupName := matchLogGroupName(lgs, group) 227 | if matchedGroupName != "" { 228 | fmt.Printf("Found Log Group: %s\n", matchedGroupName) 229 | searchLogGroup( 230 | logs, 231 | matchedGroupName, 232 | pattern, 233 | "", 234 | startTime, 235 | endTime, 236 | count) 237 | } 238 | os.Exit(0) 239 | } 240 | 241 | if pattern == "" { 242 | usage() 243 | } else { 244 | searchLogGroups(logs, lgs, pattern, startTime, endTime, count) 245 | } 246 | } 247 | --------------------------------------------------------------------------------