├── .editorconfig ├── .gitignore ├── AWSscripts ├── SQS3script.py └── requirements.txt ├── CODEOWNERS ├── Linux Script ├── README.md └── configure-linux.sh ├── Mac Script ├── README.md └── configure-mac.sh ├── Modular Scripts ├── Apache2 │ ├── README.md │ └── configure-apache.sh ├── File Monitoring │ ├── README.md │ └── configure-file-monitoring.sh ├── Nginx │ ├── README.md │ └── configure-nginx.sh ├── S3Logs Monitoring │ ├── README.md │ └── configure-s3-file-monitoring.sh └── Tomcat │ ├── README.md │ └── configure-tomcat.sh ├── README.md ├── configure-syslog.py ├── publish.sh └── tests.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs 2 | # editorconfig.org 3 | 4 | root = true 5 | 6 | [*.sh] 7 | indent_style = space 8 | indent_size = 2 9 | 10 | [*.py] 11 | indent_style = space 12 | indent_size = 4 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | .idea 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | 38 | loggly_setup.log 39 | env_details.txt 40 | 41 | environment 42 | -------------------------------------------------------------------------------- /AWSscripts/SQS3script.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import sys 4 | 5 | import boto3 6 | import botocore 7 | 8 | 9 | class AWS: 10 | """Encapsulates AWS session with resources (S3 bucket, SQS queue, IAM user).""" 11 | QUEUE_BUCKET_POLICY_JSON = """ 12 | { 13 | "Effect": "Allow", 14 | "Principal": { 15 | "AWS": "*" 16 | }, 17 | "Action": "SQS:SendMessage", 18 | "Resource": "%(queue_arn)s", 19 | "Condition": { 20 | "ArnLike": { 21 | "aws:SourceArn": "arn:aws:s3:::%(bucket_name)s" 22 | } 23 | } 24 | } 25 | """ 26 | 27 | QUEUE_ACCOUNT_POLICY_JSON = """ 28 | { 29 | "Effect": "Allow", 30 | "Principal": { 31 | "AWS": "arn:aws:iam::%(account_number)s:root" 32 | }, 33 | "Action": [ 34 | "sqs:ReceiveMessage", 35 | "sqs:GetQueueUrl", 36 | "sqs:GetQueueAttributes", 37 | "sqs:DeleteMessage" 38 | ], 39 | "Resource": "%(queue_arn)s" 40 | } 41 | """ 42 | 43 | QUEUE_POLICY_WHOLE_JSON = """ 44 | { 45 | "Version": "2012-10-17", 46 | "Statement": [%s, %s] 47 | } 48 | """ % (QUEUE_BUCKET_POLICY_JSON, QUEUE_ACCOUNT_POLICY_JSON) 49 | 50 | QUEUE_CONFIGURATIONS_JSON = """ 51 | { 52 | "QueueConfigurations": [ 53 | { 54 | "Id": "LogglyS3Notification", 55 | "Events": ["s3:ObjectCreated:*"], 56 | "QueueArn": "%(queue_arn)s" 57 | } 58 | ] 59 | } 60 | """ 61 | 62 | USER_POLICY_BUCKET_JSON = """ 63 | { 64 | "Effect": "Allow", 65 | "Action":[ 66 | "s3:ListBucket", 67 | "s3:GetObject", 68 | "s3:GetBucketLocation" 69 | ], 70 | "Resource": [ 71 | "arn:aws:s3:::%(bucket_name)s/*", 72 | "arn:aws:s3:::%(bucket_name)s" 73 | ] 74 | } 75 | """ 76 | 77 | USER_POLICY_QUEUE_JSON = """ 78 | { 79 | "Effect": "Allow", 80 | "Action": [ 81 | "sqs:ReceiveMessage", 82 | "sqs:GetQueueUrl", 83 | "sqs:GetQueueAttributes", 84 | "sqs:DeleteMessage" 85 | ], 86 | "Resource": [ 87 | "%(queue_arn)s" 88 | ] 89 | } 90 | """ 91 | 92 | USER_POLICY_WHOLE_JSON = """{ 93 | "Version": "2012-10-17", 94 | "Statement": [%s, %s] 95 | } 96 | """ % (USER_POLICY_BUCKET_JSON, USER_POLICY_QUEUE_JSON) 97 | 98 | class PolicyDocument: 99 | """Represents the JSON AWS policy.""" 100 | 101 | def __init__(self, policy_document): 102 | self._policy_document = policy_document 103 | 104 | def get_policy(self): 105 | return self._policy_document 106 | 107 | def add_statement_or_resource(self, action, resource, statement_to_add): 108 | statement, index = self.get_statement(action) 109 | if not statement: 110 | # No statement contains required action, create a new statement. 111 | self.add_statement(statement_to_add) 112 | else: 113 | # Required action is contained in a statement, check resource. 114 | if not self.get_statement(action, resource)[0]: 115 | # Required resource is missing, add it to the statement with the required action. 116 | self.add_resource_to_statement(resource, index) 117 | 118 | def add_statement(self, statement, index=None): 119 | if index is not None: 120 | self._policy_document['Statement'][index] = statement 121 | else: 122 | self._policy_document['Statement'].append(statement) 123 | 124 | def get_statement(self, action=None, resource=None, effect='Allow'): 125 | """Returns a statement with matching action, resource and effect. For the resource and action it is 126 | sufficient if it is contained in the list. Statement index is also returned.""" 127 | 128 | statements = self._policy_document['Statement'] 129 | for s, i in zip(statements, range(0, len(statements))): 130 | if s['Effect'] != effect: 131 | continue 132 | if action: 133 | s['Action'] = self._make_lowercase(s['Action']) 134 | action = self._make_lowercase(action) 135 | if not self._compare_fields(s['Action'], action): 136 | continue 137 | if resource: 138 | if not self._compare_fields(s['Resource'], resource): 139 | continue 140 | return s, i 141 | return None, None 142 | 143 | def add_resource_to_statement(self, resource, statement_index): 144 | statement = self._policy_document['Statement'][statement_index] 145 | try: 146 | if isinstance(statement['Resource'], list): 147 | statement['Resource'].append(resource) 148 | else: 149 | statement['Resource'] = [statement['Resource'], resource] 150 | except KeyError: 151 | statement['Resource'] = resource 152 | self.add_statement(statement, statement_index) 153 | 154 | def _compare_fields(self, statement_field, given_field): 155 | if isinstance(statement_field, list): 156 | if isinstance(given_field, list): 157 | if set(statement_field) != set(given_field): 158 | return False 159 | elif given_field not in statement_field: 160 | return False 161 | elif statement_field != given_field: 162 | return False 163 | return True 164 | 165 | def _make_lowercase(self, field): 166 | if isinstance(field, list): 167 | for i, e in enumerate(field): 168 | field[i] = e.lower() 169 | else: 170 | field = field.lower() 171 | return field 172 | 173 | def __init__(self, session, bucket, queue, user, account_id): 174 | self._session = session 175 | self._bucket = bucket 176 | self._queue = queue 177 | self._user = user 178 | self._account_id = account_id 179 | 180 | def set_queue_policy(self): 181 | try: 182 | queue_policy = json.loads(self._queue.attributes['Policy']) 183 | except KeyError: 184 | print('Queue policy not found, creating it') 185 | params_dict = {'queue_arn': self._queue.attributes['QueueArn'], 186 | 'bucket_name': self._bucket.name, 187 | 'account_number': self._account_id} 188 | self._queue.set_attributes(Attributes={'Policy': self.QUEUE_POLICY_WHOLE_JSON % params_dict}) 189 | return 190 | pd = self.PolicyDocument(queue_policy) 191 | self._add_bucket_to_queue_policy(pd) 192 | self._add_account_access_to_queue_policy(pd) 193 | 194 | def set_bucket_notification(self): 195 | self._bucket.Notification().put( 196 | NotificationConfiguration=json.loads( 197 | self.QUEUE_CONFIGURATIONS_JSON % {'queue_arn': self._queue.attributes['QueueArn']})) 198 | 199 | def set_user_policy(self): 200 | policy_name = 'LogglyUserPolicy' 201 | try: 202 | policy = self._user.Policy(policy_name) 203 | policy.policy_document # This raises exception on non existent policy. 204 | except botocore.exceptions.ClientError as e: 205 | if e.response['Error']['Code'] == 'NoSuchEntity': 206 | print('Policy {} not found, creating it'.format(policy_name)) 207 | params_dict = {'queue_arn': self._queue.attributes['QueueArn'], 'bucket_name': self._bucket.name} 208 | self._user.create_policy( 209 | PolicyName=policy_name, PolicyDocument=self.USER_POLICY_WHOLE_JSON % params_dict) 210 | return 211 | else: 212 | raise e 213 | pd = self.PolicyDocument(policy.policy_document) 214 | self._add_bucket_to_user_policy(pd) 215 | self._add_queue_to_user_policy(pd) 216 | policy.put(PolicyDocument=json.dumps(pd.get_policy())) 217 | 218 | def _add_bucket_to_queue_policy(self, policy_document): 219 | statement, index = policy_document.get_statement( 220 | action='sqs:sendmessage', resource=self._queue.attributes['QueueArn']) 221 | if not statement: 222 | # Create a whole new statement. 223 | params_dict = {'queue_arn': self._queue.attributes['QueueArn'], 'bucket_name': self._bucket.name} 224 | policy_document.add_statement(json.loads(self.QUEUE_BUCKET_POLICY_JSON % params_dict)) 225 | else: 226 | try: 227 | bucket_arn = statement['Condition']['ArnLike']['aws:SourceArn'] 228 | except KeyError: 229 | # A statement with 'sqs:sendmessage' action already exists without ARN condition, no need to add the 230 | # bucket ARN. 231 | return 232 | if isinstance(bucket_arn, list): 233 | bucket_arn = ",".join(bucket_arn) 234 | if self._bucket.name in bucket_arn: 235 | print("Given bucket already exists in queue's policy") 236 | return 237 | else: 238 | # A statement with 'sqs:sendmessage' action and some ARN condition already exists, 239 | # just append the bucket ARN to its condition. 240 | bucket_arn += ',arn:aws:s3:::' + self._bucket.name 241 | statement['Condition']['ArnLike']['aws:SourceArn'] = bucket_arn.split(',') 242 | policy_document.add_statement(statement, index) 243 | self._queue.set_attributes(Attributes={'Policy': json.dumps(policy_document.get_policy())}) 244 | 245 | def _add_account_access_to_queue_policy(self, policy_document): 246 | statement, _ = policy_document.get_statement( 247 | action='sqs:*', resource=self._queue.attributes['QueueArn']) 248 | if not statement: 249 | self._queue.add_permission(Label='AccountAccess', AWSAccountIds=[self._account_id], Actions=['*']) 250 | 251 | def _add_bucket_to_user_policy(self, policy_document): 252 | policy_document.add_statement_or_resource(["s3:ListBucket", "s3:GetObject", "s3:GetBucketLocation"], 253 | 'arn:aws:s3:::' + self._bucket.name, 254 | self.USER_POLICY_BUCKET_JSON % {'bucket_name': self._bucket.name}) 255 | policy_document.add_statement_or_resource(["s3:ListBucket", "s3:GetObject", "s3:GetBucketLocation"], 256 | 'arn:aws:s3:::' + self._bucket.name + '/*', 257 | self.USER_POLICY_BUCKET_JSON % {'bucket_name': self._bucket.name}) 258 | 259 | def _add_queue_to_user_policy(self, policy_document): 260 | queue_arn = self._queue.attributes['QueueArn'] 261 | policy_document.add_statement_or_resource( 262 | 'sqs:*', queue_arn, self.USER_POLICY_QUEUE_JSON % {'queue_arn': queue_arn}) 263 | 264 | 265 | def get_args(): 266 | parser = argparse.ArgumentParser() 267 | parser.add_argument("--acnumber", dest="acnumber", help="account number") 268 | parser.add_argument("--s3bucket", dest="s3bucket", help="s3 bucket name") 269 | parser.add_argument("--user", dest="user", default='loggly-s3-user', help="user") 270 | parser.add_argument("--admin", dest="admin", default="default", help="admin user name") 271 | parser.add_argument("--sqsname", dest="sqsname", default='loggly-s3-queue', help="sqsname") 272 | 273 | args = parser.parse_args() 274 | 275 | if not args.s3bucket: 276 | parser.error("S3 bucket name not provided") 277 | 278 | if not args.acnumber: 279 | parser.error("Account number not provided") 280 | 281 | if not args.acnumber.isdigit(): 282 | parser.error("Please check your account number, it should only contain digits, no other characters.") 283 | 284 | return args 285 | 286 | 287 | def get_bucket(session, bucket_name): 288 | bucket = session.resource('s3').Bucket(bucket_name) 289 | if bucket.creation_date is None: 290 | region = boto3.session.Session().region_name 291 | print_warn('S3 bucket {} does not exist, please create it and run the script again. Also, make sure the S3 bucket and the SQS queue are in the same region. Current session region: {}'.format(bucket_name, region)) 292 | sys.exit(1) 293 | return bucket 294 | 295 | 296 | def get_queue(session, queue_name): 297 | sqs = session.resource('sqs') 298 | try: 299 | queue = sqs.get_queue_by_name(QueueName=queue_name) 300 | print_warn('Queue {} already exists'.format(queue_name)) 301 | except botocore.exceptions.ClientError as e: 302 | if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue': 303 | print_warn('Queue {} does not exist, creating it.'.format(queue_name)) 304 | queue = sqs.create_queue(QueueName=queue_name) 305 | print_warn('Queue url: {}'.format(queue.url)) 306 | else: 307 | raise e 308 | print_warn('Queue name\n{}'.format(queue_name)) 309 | return queue 310 | 311 | 312 | def get_user(session, user_name): 313 | iam = session.resource('iam') 314 | try: 315 | user = iam.User(user_name) 316 | user.arn # This raises exception on non existent user. 317 | print_warn('IAM user {} already exists'.format(user_name)) 318 | print_warn('Please provide the access key and secret key for the IAM user {} in the form fields'.format(user_name)) 319 | except botocore.exceptions.ClientError as e: 320 | if e.response['Error']['Code'] == 'NoSuchEntity': 321 | print_warn("IAM user {} does not exist, creating it".format(user_name)) 322 | user = iam.create_user(UserName=user_name) 323 | access_key_pair = user.create_access_key_pair() 324 | print_warn("Access key for Loggly") 325 | print_warn(access_key_pair.access_key_id) 326 | print_warn("Secret key for Loggly") 327 | print_warn(access_key_pair.secret_access_key) 328 | print_warn('Please save the above credentials') 329 | else: 330 | raise e 331 | return user 332 | 333 | def print_warn(input): 334 | WARN = '\033[91m' 335 | ENDW = '\033[0m' 336 | print(WARN+str(input)+ENDW) 337 | 338 | def main(): 339 | args = get_args() 340 | try: 341 | session = boto3.Session(profile_name=args.admin) 342 | bucket = get_bucket(session, args.s3bucket) 343 | queue = get_queue(session, args.sqsname) 344 | user = get_user(session, args.user) 345 | aws = AWS(session, bucket, queue, user, args.acnumber) 346 | aws.set_queue_policy() 347 | aws.set_bucket_notification() 348 | aws.set_user_policy() 349 | except Exception as e: 350 | print_warn(e) 351 | return 1 352 | return 0 353 | 354 | 355 | if __name__ == '__main__': 356 | sys.exit(main()) 357 | -------------------------------------------------------------------------------- /AWSscripts/requirements.txt: -------------------------------------------------------------------------------- 1 | boto==2.49.0 2 | boto3==1.9.209 3 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default owners 2 | * @loggly/lg1-devs 3 | -------------------------------------------------------------------------------- /Linux Script/README.md: -------------------------------------------------------------------------------- 1 | # Linux Script 2 | 3 | ## Configure 4 | 5 | Configure your Linux system to send syslogs to Loggly using the following command 6 | 7 | ```bash 8 | sudo bash configure-linux.sh -a SUBDOMAIN -u USERNAME 9 | ``` 10 | 11 | You can also pass your *customer token* as `-t TOKEN`. If it's omitted, the token will be loaded automatically. 12 | 13 | ## Stop 14 | 15 | Stop sending your Linux System logs to Loggly 16 | 17 | ```bash 18 | sudo bash configure-linux.sh -a SUBDOMAIN -r 19 | ``` 20 | -------------------------------------------------------------------------------- /Linux Script/configure-linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #trapping Control + C 4 | #these statements must be the first statements in the script to trap the CTRL C event 5 | 6 | trap ctrl_c INT 7 | 8 | function ctrl_c() { 9 | logMsgToConfigSysLog "INFO" "INFO: Aborting the script." 10 | exit 1 11 | } 12 | 13 | ########## Variable Declarations - Start ########## 14 | 15 | #name of the current script. This will get overwritten by the child script which calls this 16 | SCRIPT_NAME=configure-linux.sh 17 | #version of the current script. This will get overwritten by the child script which calls this 18 | SCRIPT_VERSION=1.21 19 | 20 | #application tag. This will get overwritten by the child script which calls this 21 | APP_TAG= 22 | 23 | #directory location for syslog 24 | RSYSLOG_ETCDIR_CONF=/etc/rsyslog.d 25 | #name and location of loggly syslog file 26 | LOGGLY_RSYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/22-loggly.conf 27 | #name and location of loggly syslog backup file 28 | LOGGLY_RSYSLOG_CONFFILE_BACKUP=$LOGGLY_RSYSLOG_CONFFILE.loggly.bk 29 | 30 | #syslog directory 31 | RSYSLOG_DIR=/var/spool/rsyslog 32 | #rsyslog service name 33 | RSYSLOG_SERVICE=rsyslog 34 | #syslog-ng 35 | SYSLOG_NG_SERVICE=syslog-ng 36 | #rsyslogd 37 | RSYSLOGD=rsyslogd 38 | #minimum version of rsyslog to enable logging to loggly 39 | MIN_RSYSLOG_VERSION=5.8.0 40 | #this variable will hold the users syslog version 41 | RSYSLOG_VERSION= 42 | 43 | #this variable will hold the existing syslog port of 22-loggly.conf 44 | EXISTING_SYSLOG_PORT= 45 | 46 | #this variable will hold the host name 47 | HOST_NAME= 48 | #this variable will hold the name of the linux distribution 49 | LINUX_DIST= 50 | #this variable will hold the path to CA bundle 51 | CA_FILE_PATH="/etc/ssl/certs/ca-certificates.crt" 52 | 53 | #host name for logs-01.loggly.com 54 | LOGS_01_HOST=logs-01.loggly.com 55 | LOGS_01_URL=https://$LOGS_01_HOST 56 | #this variable will contain loggly account url in the format https://$LOGGLY_ACCOUNT.loggly.com 57 | LOGGLY_ACCOUNT_URL= 58 | #loggly.com URL 59 | LOGGLY_COM_URL=https://www.loggly.com 60 | 61 | ######Inputs provided by user###### 62 | #this variable will hold the loggly account name provided by user. 63 | #this is a mandatory input 64 | LOGGLY_ACCOUNT= 65 | #this variable will hold the loggly authentication token provided by user. 66 | #this is a mandatory input 67 | LOGGLY_AUTH_TOKEN= 68 | #this variable will identify if the user has selected to rollback settings 69 | LOGGLY_ROLLBACK= 70 | #this variable will hold the user name provided by user 71 | #this is a mandatory input 72 | LOGGLY_USERNAME= 73 | #this variable will hold the password provided by user 74 | #this is a mandatory input 75 | LOGGLY_PASSWORD= 76 | 77 | #if this variable is set to true then suppress all prompts 78 | SUPPRESS_PROMPT="false" 79 | 80 | #variables used in 22-loggly.conf file 81 | LOGGLY_SYSLOG_PORT=6514 82 | LOGGLY_DISTRIBUTION_ID="41058" 83 | 84 | #Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this 85 | #on how to configure the child application 86 | MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-tls-configuration/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" 87 | 88 | #this variable is set if the script is invoked via some other calling script 89 | IS_INVOKED= 90 | 91 | #this variable will hold if the check env function for linux is invoked 92 | LINUX_ENV_VALIDATED="false" 93 | 94 | #this variable will inform if verification needs to be performed 95 | LINUX_DO_VERIFICATION="true" 96 | 97 | #this variable will enable sending logs over TLS 98 | LOGGLY_TLS_SENDING="true" 99 | 100 | #Setting FORCE_SECURE to false 101 | FORCE_SECURE="false" 102 | 103 | #Setting LOGGLY_REMOVE to false 104 | LOGGLY_REMOVE="false" 105 | 106 | #Setting INSECURE mode to false initially 107 | INSECURE_MODE="false" 108 | 109 | #Setting invalid subdomain value 110 | INVALID_SUBDOMAIN=*".loggly.com"* 111 | 112 | ########## Variable Declarations - End ########## 113 | 114 | #check if the Linux environment is compatible with Loggly. 115 | #Also set few variables after the check. 116 | checkLinuxLogglyCompatibility() { 117 | #check if the user has root permission to run this script 118 | checkIfUserHasRootPrivileges 119 | 120 | #check if the OS is supported by the script. If no, then exit 121 | checkIfSupportedOS 122 | 123 | #check if required dependencies to run the script are not installed. If yes, ask user to install them manually and run the script again. 124 | checkIfRequiredDependenciesAreNotInstalled 125 | 126 | #check if package-manager is present on the machine 127 | checkIfPackageManagerIsPresent 128 | 129 | #set the basic variables needed by this script 130 | setLinuxVariables 131 | 132 | #set TLS package 133 | setTlsPackage 134 | 135 | #check if the Loggly servers are accessible. If no, ask user to check network connectivity & exit 136 | checkIfLogglyServersAccessible 137 | 138 | #check if user credentials are valid. If no, then exit 139 | checkIfValidUserNamePassword 140 | 141 | #get authentication token if not provided 142 | getAuthToken 143 | 144 | #check if authentication token is valid. If no, then exit. 145 | checkIfValidAuthToken 146 | 147 | #checking if syslog-ng is configured as a service 148 | checkifSyslogNgConfiguredAsService 149 | 150 | #check if systemd is present in machine. 151 | checkIfSystemdConfigured 152 | 153 | #check if rsyslog is configured as service. If no, then exit 154 | checkIfRsyslogConfiguredAsService 155 | 156 | #check if multiple rsyslog are present in the system. If yes, then exit 157 | checkIfMultipleRsyslogConfigured 158 | 159 | #check for the minimum version of rsyslog i.e 5.8.0. If no, then exit 160 | checkIfMinVersionOfRsyslog 161 | 162 | #check if selinux service is enforced. if yes, ask the user to manually disable and exit the script 163 | checkIfSelinuxServiceEnforced 164 | 165 | #update rsyslog.conf and adds $MaxMessageSize in it 166 | modifyMaxMessageSize 167 | 168 | 169 | LINUX_ENV_VALIDATED="true" 170 | } 171 | 172 | # executing the script for loggly to install and configure rsyslog. 173 | installLogglyConf() { 174 | #log message indicating starting of Loggly configuration 175 | logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." 176 | 177 | if [ "$LINUX_ENV_VALIDATED" = "false" ]; then 178 | checkLinuxLogglyCompatibility 179 | fi 180 | 181 | #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu 182 | createRsyslogDir 183 | 184 | #if all the above check passes, write the 22-loggly.conf file 185 | checkAuthTokenAndWriteContents 186 | 187 | if [ "$LINUX_DO_VERIFICATION" = "true" ]; then 188 | #check if the logs are going to loggly fro linux system now 189 | checkIfLogsMadeToLoggly 190 | fi 191 | 192 | if [ "$IS_INVOKED" = "" ]; then 193 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." 194 | fi 195 | 196 | } 197 | 198 | #remove loggly configuration from Linux system 199 | removeLogglyConf() { 200 | #log message indicating starting of Loggly configuration 201 | logMsgToConfigSysLog "INFO" "INFO: Initiating uninstall Loggly for Linux." 202 | 203 | #check if the user has root permission to run this script 204 | checkIfUserHasRootPrivileges 205 | 206 | #check if the OS is supported by the script. If no, then exit 207 | checkIfSupportedOS 208 | 209 | #set the basic variables needed by this script 210 | setLinuxVariables 211 | 212 | #remove systemd-rsyslog configuration 213 | revertSystemdChanges 214 | 215 | #remove 22-loggly.conf file 216 | remove22LogglyConfFile 217 | 218 | #restart rsyslog service 219 | restartRsyslog 220 | 221 | #log success message 222 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Uninstalled Loggly configuration from Linux system." 223 | } 224 | 225 | #checks if user has root privileges 226 | checkIfUserHasRootPrivileges() { 227 | #This script needs to be run as root 228 | if [[ $EUID -ne 0 ]]; then 229 | logMsgToConfigSysLog "ERROR" "ERROR: This script must be run as root." 230 | exit 1 231 | fi 232 | } 233 | 234 | #check if package-manager is present on the machine 235 | checkIfPackageManagerIsPresent() { 236 | if [ -x "$(command -v apt-get)" ]; then 237 | PKG_MGR="apt-get" 238 | else 239 | if [ -x "$(command -v yum)" ]; then 240 | PKG_MGR="yum" 241 | fi 242 | fi 243 | } 244 | 245 | #set tls package 246 | setTlsPackage() { 247 | TLS_PACKAGE="rsyslog-gnutls" 248 | TLS_DRIVER="gtls" 249 | 250 | if [ "$LINUX_DIST_IN_LOWER_CASE" == "amazon" ]; then 251 | LINUX_DIST_VERSION=$(cat /etc/system-release | cut -f 4 -d " ") 252 | if [ "$LINUX_DIST_VERSION" -ge 2023 ]; then 253 | TLS_PACKAGE="rsyslog-openssl" 254 | TLS_DRIVER="ossl" 255 | fi 256 | fi 257 | } 258 | 259 | #check if required dependencies to run the script are not installed, If yes then ask user to install them manually and run the script again 260 | checkIfRequiredDependenciesAreNotInstalled() { 261 | if ! [ -x "$(command -v curl)" ]; then 262 | logMsgToConfigSysLog "ERROR" "ERROR: 'Curl' executable could not be found on your machine, since it is a dependent package to run this script, please install it manually and then run the script again." 263 | exit 1 264 | elif ! [ -x "$(command -v ping)" ]; then 265 | logMsgToConfigSysLog "ERROR" "ERROR: 'Ping' executable could not be found on your machine, since it is a dependent package to run this script, please install it manually and then run the script again." 266 | exit 1 267 | fi 268 | } 269 | 270 | #check if supported operating system 271 | checkIfSupportedOS() { 272 | getOs 273 | 274 | LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") 275 | 276 | case "$LINUX_DIST_IN_LOWER_CASE" in 277 | *"ubuntu"*) 278 | echo "INFO: Operating system is Ubuntu." 279 | ;; 280 | *"red"*) 281 | echo "INFO: Operating system is Red Hat." 282 | ;; 283 | *"centos"*) 284 | echo "INFO: Operating system is CentOS." 285 | ;; 286 | *"debian"*) 287 | echo "INFO: Operating system is Debian." 288 | ;; 289 | *"amazon"*) 290 | echo "INFO: Operating system is Amazon AMI." 291 | ;; 292 | *"darwin"*) 293 | #if the OS is mac then exit 294 | logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs/send-mac-logs-to-loggly/" 295 | exit 1 296 | ;; 297 | *) 298 | logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." 299 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 300 | while true; do 301 | read -p "Would you like to continue anyway? (yes/no)" yn 302 | case $yn in 303 | [Yy]*) 304 | break 305 | ;; 306 | [Nn]*) 307 | exit 1 308 | ;; 309 | *) echo "Please answer yes or no." ;; 310 | esac 311 | done 312 | fi 313 | ;; 314 | esac 315 | } 316 | 317 | getOs() { 318 | # Determine OS platform 319 | UNAME=$(uname | tr "[:upper:]" "[:lower:]") 320 | # If Linux, try to determine specific distribution 321 | if [ "$UNAME" == "linux" ]; then 322 | # If available, use LSB to identify distribution 323 | if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then 324 | LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) 325 | # If system-release is available, then try to identify the name 326 | elif [ -f /etc/system-release ]; then 327 | LINUX_DIST=$(cat /etc/system-release | cut -f 1 -d " ") 328 | # Otherwise, use release info file 329 | else 330 | LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) 331 | fi 332 | fi 333 | 334 | # For everything else (or if above failed), just use generic identifier 335 | if [ "$LINUX_DIST" == "" ]; then 336 | LINUX_DIST=$(uname) 337 | fi 338 | } 339 | 340 | #sets linux variables which will be used across various functions 341 | setLinuxVariables() { 342 | #set host name 343 | HOST_NAME=$(hostname) 344 | 345 | #set loggly account url 346 | LOGGLY_ACCOUNT_URL=https://$LOGGLY_ACCOUNT.loggly.com 347 | } 348 | 349 | #checks if all the various endpoints used for configuring loggly are accessible 350 | checkIfLogglyServersAccessible() { 351 | echo "INFO: Checking if $LOGS_01_HOST can be pinged." 352 | if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then 353 | echo "INFO: $LOGS_01_HOST can be pinged." 354 | else 355 | logMsgToConfigSysLog "WARNING" "WARNING: $LOGS_01_HOST cannot be pinged. Please check your network and firewall settings." 356 | fi 357 | 358 | echo "INFO: Checking if $LOGS_01_HOST is reachable." 359 | (/dev/null 2>&1 360 | if [ $? -eq 0 ]; then 361 | echo "INFO: $LOGS_01_HOST is reachable." 362 | else 363 | logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings." 364 | exit 1 365 | fi 366 | 367 | echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." 368 | if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then 369 | echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." 370 | else 371 | logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings." 372 | exit 1 373 | fi 374 | 375 | echo "INFO: Checking if '$LOGGLY_ACCOUNT' subdomain is valid." 376 | if [[ $LOGGLY_ACCOUNT != $INVALID_SUBDOMAIN ]]; then 377 | if curl --head -s --request GET $LOGGLY_ACCOUNT_URL/login | grep -q "accounts/invalid" ; then 378 | logMsgToConfigSysLog "ERROR" "ERROR: This is not a recognized subdomain. Please ask the account owner for the subdomain they signed up with." 379 | exit 1 380 | else 381 | echo "INFO: $LOGGLY_ACCOUNT_URL is valid and reachable." 382 | fi 383 | else 384 | logMsgToConfigSysLog "ERROR" "ERROR: This is not a recognized subdomain. Please ask the account owner for the subdomain they signed up with. Please note that your subdomain is just the first string in your loggly account URL not the entire account name." 385 | exit 1 386 | fi 387 | } 388 | 389 | #check if user name and password is valid 390 | checkIfValidUserNamePassword() { 391 | echo "INFO: Checking if provided username and password is correct." 392 | if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then 393 | logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" 394 | logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. Your username is visible at the top right of the Loggly console. You can reset your password at http://.loggly.com/login." 395 | exit 1 396 | else 397 | logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." 398 | fi 399 | } 400 | 401 | getAuthToken() { 402 | if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then 403 | logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." 404 | #get authentication token if user has not provided one 405 | tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -A1 "\"tokens\":" | grep -v "tokens") 406 | 407 | #get the string from index 0 to last occurence of " 408 | tokenstr=${tokenstr%\"*} 409 | 410 | #get the string from first occurence of " to the end 411 | tokenstr=${tokenstr#*\"} 412 | 413 | LOGGLY_AUTH_TOKEN=$tokenstr 414 | 415 | logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" 416 | fi 417 | } 418 | 419 | #check if authentication token is valid 420 | checkIfValidAuthToken() { 421 | echo "INFO: Checking if provided auth token is correct." 422 | if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then 423 | logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." 424 | else 425 | logMsgToConfigSysLog "ERROR" "ERROR: Invalid account name or authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." 426 | exit 1 427 | fi 428 | } 429 | 430 | #check if rsyslog is configured as service. If it is configured as service and not started, start the service 431 | checkIfRsyslogConfiguredAsService() { 432 | if [ -f /etc/init.d/$RSYSLOG_SERVICE ]; then 433 | logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." 434 | elif [ -f /usr/lib/systemd/system/$RSYSLOG_SERVICE.service ]; then 435 | logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." 436 | else 437 | logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." 438 | exit 1 439 | fi 440 | 441 | #checking if syslog-ng is running as a service 442 | checkifSyslogNgConfiguredAsService 443 | 444 | if [ $(ps -A | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then 445 | logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." 446 | service $RSYSLOG_SERVICE start 447 | fi 448 | } 449 | 450 | checkifSyslogNgConfiguredAsService() { 451 | if [ $(ps -A | grep "$SYSLOG_NG_SERVICE" | wc -l) -gt 0 ]; then 452 | logMsgToConfigSysLog "ERROR" "ERROR: This script does not currently support syslog-ng. Please follow the instructions on this page https://www.loggly.com/docs/syslog-ng-manual-configuration" 453 | exit 1 454 | fi 455 | } 456 | 457 | #check if systemd is present in machine. 458 | checkIfSystemdConfigured() { 459 | FILE="/etc/systemd/journald.conf" 460 | if [ -f "$FILE" ]; then 461 | logMsgToConfigSysLog "INFO" "INFO: Systemd is present. Configuring logs from Systemd to rsyslog." 462 | cp /etc/systemd/journald.conf /etc/systemd/journald.conf.loggly.bk 463 | sed -i 's/.*ForwardToSyslog.*/ForwardToSyslog=Yes/g' /etc/systemd/journald.conf 464 | logMsgToConfigSysLog "INFO" "INFO: Restarting Systemd-journald" 465 | systemctl restart systemd-journald 466 | fi 467 | } 468 | 469 | #check if multiple versions of rsyslog is configured 470 | checkIfMultipleRsyslogConfigured() { 471 | if [ $(ps -A | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then 472 | logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." 473 | exit 1 474 | fi 475 | } 476 | 477 | #check if minimum version of rsyslog required to configure loggly is met 478 | checkIfMinVersionOfRsyslog() { 479 | RSYSLOG_VERSION=$($RSYSLOGD -version | grep "$RSYSLOGD") 480 | RSYSLOG_VERSION=${RSYSLOG_VERSION#* } 481 | RSYSLOG_VERSION=${RSYSLOG_VERSION%,*} 482 | RSYSLOG_VERSION=$RSYSLOG_VERSION | tr -d " " 483 | if [ $(compareVersions "$RSYSLOG_VERSION" "$MIN_RSYSLOG_VERSION" 3) -lt 0 ]; then 484 | logMsgToConfigSysLog "ERROR" "ERROR: Minimum rsyslog version required to run this script is 5.8.0. Please upgrade your rsyslog version or follow the manual instructions." 485 | exit 1 486 | fi 487 | } 488 | 489 | #check if SeLinux service is enforced 490 | checkIfSelinuxServiceEnforced() { 491 | isSelinuxInstalled=$(getenforce -ds 2>/dev/null) 492 | if [ $? -ne 0 ]; then 493 | logMsgToConfigSysLog "INFO" "INFO: selinux status is not enforced." 494 | elif [ $(getenforce | grep "Enforcing" | wc -l) -gt 0 ]; then 495 | logMsgToConfigSysLog "ERROR" "ERROR: selinux status is 'Enforcing'. Please manually restart the rsyslog daemon or turn off selinux by running 'setenforce 0' and then rerun the script." 496 | exit 1 497 | fi 498 | } 499 | 500 | #update rsyslog.conf and adds $MaxMessageSize in it 501 | modifyMaxMessageSize() { 502 | if grep -q '$MaxMessageSize' "/etc/rsyslog.conf"; then 503 | sed -i 's/.*$MaxMessageSize.*/$MaxMessageSize 64k/g' /etc/rsyslog.conf 504 | else 505 | sed -i '1 a $MaxMessageSize 64k' /etc/rsyslog.conf 506 | fi 507 | logMsgToConfigSysLog "INFO" "INFO: Modified \$MaxMessageSize to 64k in rsyslog.conf" 508 | } 509 | 510 | #check if authentication token is valid and then write contents to 22-loggly.conf file to /etc/rsyslog.d directory 511 | checkAuthTokenAndWriteContents() { 512 | if [ "$LOGGLY_AUTH_TOKEN" != "" ]; then 513 | writeContents $LOGGLY_ACCOUNT $LOGGLY_AUTH_TOKEN $LOGGLY_DISTRIBUTION_ID $LOGS_01_HOST $LOGGLY_SYSLOG_PORT 514 | restartRsyslog 515 | else 516 | logMsgToConfigSysLog "ERROR" "ERROR: Loggly auth token is required to configure rsyslog. Please pass -t while running script." 517 | exit 1 518 | fi 519 | } 520 | 521 | setPathToCABundle () { 522 | case "$LINUX_DIST_IN_LOWER_CASE" in 523 | *"debian"* | *"ubuntu"*) 524 | CA_FILE_PATH="/etc/ssl/certs/ca-certificates.crt" 525 | ;; 526 | *"red"* | *"centos"* | *"amazon"*) 527 | CA_FILE_PATH="/etc/ssl/certs/ca-bundle.crt" 528 | ;; 529 | *) 530 | logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly. Verify path to the file with root CA certificates (usually stored in OS trust store) in '$RSYSLOG_ETCDIR_CONF' -> '\$DefaultNetstreamDriverCAFile' and restart rsyslog service or re-run script with '--insecure' attribute. Default path to CA file is '$CA_FILE_PATH'." 531 | ;; 532 | esac 533 | } 534 | 535 | confString() { 536 | setPathToCABundle 537 | RSYSLOG_VERSION_TMP=$(echo $RSYSLOG_VERSION | cut -d "." -f1) 538 | inputStr_TLS_RSYS_7=" 539 | # ------------------------------------------------------- 540 | # Syslog Logging Directives for Loggly ($LOGGLY_ACCOUNT.loggly.com) 541 | # ------------------------------------------------------- 542 | ########################################################## 543 | ### RsyslogTemplate for Loggly ### 544 | ########################################################## 545 | 546 | \$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID tag=\\\"RsyslogTLS\\\"] %msg%\n\" 547 | 548 | # Setup disk assisted queues 549 | \$WorkDirectory /var/spool/rsyslog # where to place spool files 550 | \$ActionQueueFileName fwdRule1 # unique name prefix for spool files 551 | \$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) 552 | \$ActionQueueSaveOnShutdown on # save messages to disk on shutdown 553 | \$ActionQueueType LinkedList # run asynchronously 554 | \$ActionResumeRetryCount -1 # infinite retries if host is down 555 | 556 | #RsyslogGnuTLS 557 | \$DefaultNetstreamDriverCAFile $CA_FILE_PATH 558 | \$ActionSendStreamDriver $TLS_DRIVER 559 | \$ActionSendStreamDriverMode 1 560 | \$ActionSendStreamDriverAuthMode x509/name 561 | \$ActionSendStreamDriverPermittedPeer *.loggly.com 562 | 563 | *.* @@$LOGS_01_HOST:$LOGGLY_SYSLOG_PORT;LogglyFormat 564 | #################END CONFIG FILE######################### 565 | " 566 | inputStr_TLS_RSYS_8=" 567 | # ------------------------------------------------------- 568 | # Syslog Logging Directives for Loggly ($LOGGLY_ACCOUNT.loggly.com) 569 | # ------------------------------------------------------- 570 | # Setup disk assisted queues 571 | \$WorkDirectory /var/spool/rsyslog # where to place spool files 572 | \$ActionQueueFileName fwdRule1 # unique name prefix for spool files 573 | \$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) 574 | \$ActionQueueSaveOnShutdown on # save messages to disk on shutdown 575 | \$ActionQueueType LinkedList # run asynchronously 576 | \$ActionResumeRetryCount -1 # infinite retries if host is down 577 | 578 | #RsyslogGnuTLS 579 | \$DefaultNetstreamDriverCAFile $CA_FILE_PATH 580 | 581 | 582 | template(name=\"LogglyFormat\" type=\"string\" 583 | string=\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID tag=\\\"RsyslogTLS\\\"] %msg%\n\" 584 | ) 585 | 586 | # Send messages to Loggly over TCP using the template. 587 | action(type=\"omfwd\" protocol=\"tcp\" target=\"$LOGS_01_HOST\" port=\"$LOGGLY_SYSLOG_PORT\" template=\"LogglyFormat\" StreamDriver=\"$TLS_DRIVER\" StreamDriverMode=\"1\" StreamDriverAuthMode=\"x509/name\" StreamDriverPermittedPeers=\"*.loggly.com\") 588 | " 589 | 590 | inputStr_NO_TLS=" 591 | # ------------------------------------------------------- 592 | # Syslog Logging Directives for Loggly ($LOGGLY_ACCOUNT.loggly.com) 593 | # ------------------------------------------------------- 594 | # Define the template used for sending logs to Loggly. Do not change this format. 595 | \$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID tag=\\\"Rsyslog\\\"] %msg%\n\" 596 | 597 | \$WorkDirectory /var/spool/rsyslog # where to place spool files 598 | \$ActionQueueFileName fwdRule1 # unique name prefix for spool files 599 | \$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) 600 | \$ActionQueueSaveOnShutdown on # save messages to disk on shutdown 601 | \$ActionQueueType LinkedList # run asynchronously 602 | \$ActionResumeRetryCount -1 # infinite retries if host is down 603 | 604 | # Send messages to Loggly over TCP using the template. 605 | *.* @@$LOGS_01_HOST:$LOGGLY_SYSLOG_PORT;LogglyFormat 606 | # ------------------------------------------------------- 607 | " 608 | if [ "$RSYSLOG_VERSION_TMP" -le "7" ]; then 609 | inputStrTls=$inputStr_TLS_RSYS_7 610 | elif [ "$RSYSLOG_VERSION_TMP" -ge "8" ]; then 611 | inputStrTls=$inputStr_TLS_RSYS_8 612 | fi 613 | inputStr=$inputStr_NO_TLS 614 | } 615 | 616 | #install the certificate and check if gnutls package is installed 617 | installTLSDependencies() { 618 | 619 | if [ $LOGGLY_TLS_SENDING == "true" ]; then 620 | if [ "$SUPPRESS_PROMPT" == "true" ]; then 621 | /bin/bash -c "sudo $PKG_MGR install -y $TLS_PACKAGE" 622 | else 623 | /bin/bash -c "sudo $PKG_MGR install $TLS_PACKAGE" 624 | fi 625 | if [ "$PKG_MGR" == "yum" ]; then 626 | if [ $(rpm -qa | grep -c "$TLS_PACKAGE") -eq 0 ]; then 627 | DEPENDENCIES_INSTALLED="false" 628 | if [ "$FORCE_SECURE" == "true" ]; then 629 | logMsgToConfigSysLog "WARN" "WARN: The $TLS_PACKAGE package could not be download automatically because your package manager could not be found. Please install it and restart the rsyslog service to send logs to Loggly." 630 | fi 631 | fi 632 | elif [ "$PKG_MGR" == "apt-get" ]; then 633 | if [ $(dpkg-query -W -f='${Status}' $TLS_PACKAGE 2>/dev/null | grep -c "ok installed") -eq 0 ]; then 634 | DEPENDENCIES_INSTALLED="false" 635 | if [ "$FORCE_SECURE" == "true" ]; then 636 | logMsgToConfigSysLog "WARN" "WARN: The $TLS_PACKAGE package could not be download automatically because your package manager could not be found. Please install it and restart the rsyslog service to send logs to Loggly." 637 | fi 638 | fi 639 | else 640 | DEPENDENCIES_INSTALLED="false" 641 | fi 642 | inputStr=$inputStrTls 643 | fi 644 | } 645 | 646 | #prompt users if they want to switch to insecure mode on gnutls-package download failure 647 | switchToInsecureModeIfTLSNotFound() { 648 | if [ "$FORCE_SECURE" == "false" ]; then 649 | if [ "$DEPENDENCIES_INSTALLED" == "false" ]; then 650 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 651 | logMsgToConfigSysLog "WARN" "WARN: The $TLS_PACKAGE package could not download automatically either because of your package manager could not be found or due to some other reason." 652 | while true; do 653 | read -p "Do you wish to continue with insecure mode? (yes/no)" yn 654 | case $yn in 655 | [Yy]*) 656 | logMsgToConfigSysLog "INFO" "INFO: Going to overwrite the conf file: $LOGGLY_RSYSLOG_CONFFILE with insecure configuration" 657 | LOGGLY_SYSLOG_PORT=514 658 | break 659 | ;; 660 | [Nn]*) 661 | logMsgToConfigSysLog "INFO" "INFO: Since the $TLS_PACKAGE package could not be installed automatically, please install it yourself and then re-run the script using the --force-secure flag. This option will force the secure TLS configuration instead of falling back on insecure mode. It is useful for Linux distributions where this script cannot automatically detect the dependency using yum or apt-get." 662 | exit 1 663 | ;; 664 | *) echo "Please answer yes or no." ;; 665 | esac 666 | done 667 | else 668 | logMsgToConfigSysLog "WARN" "WARN: The $TLS_PACKAGE package could not download automatically either because of your package manager could not be found or due to some other reason, continuing with insecure mode." 669 | LOGGLY_SYSLOG_PORT=514 670 | 671 | fi 672 | confString 673 | fi 674 | fi 675 | } 676 | 677 | #write the contents to 22-loggly.conf file 678 | writeContents() { 679 | confString 680 | checkScriptRunningMode 681 | installTLSDependencies 682 | switchToInsecureModeIfTLSNotFound 683 | WRITE_SCRIPT_CONTENTS="false" 684 | 685 | if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then 686 | logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." 687 | 688 | STR_SIZE=${#inputStr} 689 | SIZE_FILE=$(stat -c%s "$LOGGLY_RSYSLOG_CONFFILE") 690 | 691 | #actual file size and variable size with same contents always differ in size with one byte 692 | STR_SIZE=$((STR_SIZE + 1)) 693 | 694 | if [ "$STR_SIZE" -ne "$SIZE_FILE" ]; then 695 | 696 | logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file /etc/rsyslog.d/22-loggly.conf content has changed." 697 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 698 | while true; do 699 | read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn 700 | case $yn in 701 | [Yy]*) 702 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP" 703 | mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP 704 | WRITE_SCRIPT_CONTENTS="true" 705 | break 706 | ;; 707 | [Nn]*) 708 | LINUX_DO_VERIFICATION="false" 709 | logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." 710 | break 711 | ;; 712 | *) echo "Please answer yes or no." ;; 713 | esac 714 | done 715 | else 716 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP" 717 | mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP 718 | WRITE_SCRIPT_CONTENTS="true" 719 | fi 720 | else 721 | LINUX_DO_VERIFICATION="false" 722 | fi 723 | else 724 | WRITE_SCRIPT_CONTENTS="true" 725 | fi 726 | 727 | if [ "$WRITE_SCRIPT_CONTENTS" == "true" ]; then 728 | 729 | cat <>$LOGGLY_RSYSLOG_CONFFILE 730 | $inputStr 731 | EOIPFW 732 | 733 | fi 734 | 735 | } 736 | 737 | #create /var/spool/rsyslog directory if not already present. Modify the permission of this directory for Ubuntu 738 | createRsyslogDir() { 739 | if [ -d "$RSYSLOG_DIR" ]; then 740 | logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_DIR already exist, so not creating directory." 741 | if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then 742 | logMsgToConfigSysLog "INFO" "INFO: Changing the permission on the rsyslog in /var/spool" 743 | chown -R syslog:adm $RSYSLOG_DIR 744 | fi 745 | else 746 | logMsgToConfigSysLog "INFO" "INFO: Creating directory $SYSLOGDIR" 747 | mkdir -v $RSYSLOG_DIR 748 | if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then 749 | chown -R syslog:adm $RSYSLOG_DIR 750 | fi 751 | fi 752 | } 753 | 754 | #check if the logs made it to Loggly 755 | checkIfLogsMadeToLoggly() { 756 | logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly." 757 | uuid=$(cat /proc/sys/kernel/random/uuid) 758 | 759 | queryParam="syslog.appName%3ALOGGLYVERIFY%20$uuid" 760 | logger -t "LOGGLYVERIFY" "LOGGLYVERIFY-Test message for verification with UUID $uuid" 761 | 762 | counter=1 763 | maxCounter=10 764 | finalCount=0 765 | 766 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 767 | logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" 768 | 769 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." 770 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 771 | searchAndFetch finalCount "$queryUrl" 772 | let counter=$counter+1 773 | 774 | while [ "$finalCount" -eq 0 ]; do 775 | echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." 776 | sleep 30 777 | echo "INFO: Done waiting. Verifying again." 778 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 779 | searchAndFetch finalCount "$queryUrl" 780 | let counter=$counter+1 781 | if [ "$counter" -gt "$maxCounter" ]; then 782 | logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." 783 | exit 1 784 | fi 785 | done 786 | 787 | if [ "$finalCount" -eq 1 ]; then 788 | if [ "$IS_INVOKED" = "" ]; then 789 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." 790 | exit 0 791 | else 792 | logMsgToConfigSysLog "INFO" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." 793 | fi 794 | fi 795 | 796 | } 797 | 798 | #delete 22-loggly.conf file 799 | remove22LogglyConfFile() { 800 | if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then 801 | rm -rf "$LOGGLY_RSYSLOG_CONFFILE" 802 | fi 803 | } 804 | 805 | revertSystemdChanges() { 806 | FILE="/etc/systemd/journald.conf.loggly.bk" 807 | if [ -f "$FILE" ]; then 808 | cp /etc/systemd/journald.conf.loggly.bk /etc/systemd/journald.conf 809 | rm /etc/systemd/journald.conf.loggly.bk 810 | logMsgToConfigSysLog "INFO" "INFO: Reverted Systemd-rsyslog configuration" 811 | systemctl restart systemd-journald 812 | fi 813 | } 814 | 815 | #compares two version numbers, used for comparing versions of various softwares 816 | compareVersions() { 817 | typeset IFS='.' 818 | typeset -a v1=($1) 819 | typeset -a v2=($2) 820 | typeset n diff 821 | 822 | for ((n = 0; n < $3; n += 1)); do 823 | diff=$((v1[n] - v2[n])) 824 | if [ $diff -ne 0 ]; then 825 | [ $diff -le 0 ] && echo '-1' || echo '1' 826 | return 827 | fi 828 | done 829 | echo '0' 830 | } 831 | 832 | #restart rsyslog 833 | restartRsyslog() { 834 | logMsgToConfigSysLog "INFO" "INFO: Restarting the $RSYSLOG_SERVICE service." 835 | service $RSYSLOG_SERVICE restart 836 | if [ $? -ne 0 ]; then 837 | logMsgToConfigSysLog "WARNING" "WARNING: $RSYSLOG_SERVICE did not restart gracefully. Please restart $RSYSLOG_SERVICE manually." 838 | fi 839 | } 840 | 841 | #logs message to config syslog 842 | logMsgToConfigSysLog() { 843 | #$1 variable will be SUCCESS or ERROR or INFO or WARNING 844 | #$2 variable will be the message 845 | cslStatus=$1 846 | cslMessage=$2 847 | echo "$cslMessage" 848 | currentTime=$(date) 849 | 850 | #for Linux system, we need to use -d switch to decode base64 whereas 851 | #for Mac system, we need to use -D switch to decode 852 | varUname=$(uname) 853 | if [[ $varUname == 'Linux' ]]; then 854 | enabler=$(echo -n MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) 855 | elif [[ $varUname == 'Darwin' ]]; then 856 | enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) 857 | fi 858 | 859 | if [ $? -ne 0 ]; then 860 | echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." 861 | exit 1 862 | fi 863 | 864 | sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" 865 | 866 | #if it is an error, then log message "Script Failed" to config syslog and exit the script 867 | if [[ $cslStatus == "ERROR" ]]; then 868 | sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" 869 | if [ "$varUname" != "Darwin" ]; then 870 | echo $MANUAL_CONFIG_INSTRUCTION 871 | fi 872 | exit 1 873 | fi 874 | 875 | #if it is a success, then log message "Script Succeeded" to config syslog and exit the script 876 | if [[ $cslStatus == "SUCCESS" ]]; then 877 | sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" 878 | exit 0 879 | fi 880 | } 881 | 882 | #payload construction to send log to config syslog 883 | sendPayloadToConfigSysLog() { 884 | if [ "$APP_TAG" = "" ]; then 885 | var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\",\"rsyslog-version\":\"$RSYSLOG_VERSION\",\"insecure-mode\":\"$INSECURE_MODE\",\"suppress-enabled\":\"$SUPPRESS_PROMPT\",\"force-secure-enabled\":\"$FORCE_SECURE\",\"loggly-removed\":\"$LOGGLY_REMOVE\"}" 886 | else 887 | var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\",\"rsyslog-version\":\"$RSYSLOG_VERSION\",\"insecure-mode\":\"$INSECURE_MODE\",\"suppress-enabled\":\"$SUPPRESS_PROMPT\",\"force-secure-enabled\":\"$FORCE_SECURE\",\"loggly-removed\":\"$LOGGLY_REMOVE\"}" 888 | fi 889 | curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 >/dev/null 2>&1 890 | } 891 | 892 | #$1 return the count of records in loggly, $2 is the query param to search in loggly 893 | searchAndFetch() { 894 | url=$2 895 | 896 | result=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $url) 897 | 898 | if [ -z "$result" ]; then 899 | logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." 900 | exit 1 901 | fi 902 | # remove newline characters 903 | result=${result//$'\n'/} 904 | 905 | id=$(echo "$result" | sed -E 's/.*"id"\s*:\s*"([^"]+)".*/\1/g') 906 | 907 | url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" 908 | 909 | # retrieve the data 910 | result=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $url) 911 | count=$(echo "$result" | grep total_events | awk '{print $2}') 912 | count="${count%\,}" 913 | 914 | # If no result is fetched, return 0 915 | isNumberRegex='^[0-9]+$' 916 | if ! [[ $count =~ $isNumberRegex ]] ; then 917 | count=0 918 | fi 919 | eval $1="'$count'" 920 | if [ "$count" -gt 0 ]; then 921 | timestamp=$(echo "$result" | grep timestamp) 922 | fi 923 | } 924 | 925 | #get password in the form of asterisk 926 | getPassword() { 927 | unset LOGGLY_PASSWORD 928 | prompt="Please enter Loggly Password:" 929 | while IFS= read -p "$prompt" -r -s -n 1 char; do 930 | if [[ $char == $'\0' ]]; then 931 | break 932 | fi 933 | prompt='*' 934 | LOGGLY_PASSWORD+="$char" 935 | done 936 | echo 937 | } 938 | 939 | #function to switch system logging to insecure mode if user runs the modular script in insecure mode 940 | switchSystemLoggingToInsecure() { 941 | if [ -f $LOGGLY_RSYSLOG_CONFFILE ]; then 942 | EXISTING_SYSLOG_PORT=$(grep -Eow 6514 $LOGGLY_RSYSLOG_CONFFILE) 943 | if [[ $EXISTING_SYSLOG_PORT == 6514 ]]; then 944 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 945 | while true; do 946 | read -p "You are running the script using insecure mode, but your system logs are using secure mode. The script only supports a single mode for both, so would you like to switch your system logs to insecure mode? (yes/no)" yn 947 | case $yn in 948 | [Yy]*) 949 | logMsgToConfigSysLog "INFO" "INFO: Going to overwrite the conf file: $LOGGLY_RSYSLOG_CONFFILE with insecure configuration" 950 | LOGGLY_TLS_SENDING="false" 951 | LOGGLY_SYSLOG_PORT=514 952 | break 953 | ;; 954 | [Nn]*) 955 | logMsgToConfigSysLog "INFO" "INFO: Please re-run the script in secure mode if you want to setup secure logging" 956 | exit 1 957 | ;; 958 | *) echo "Please answer yes or no." ;; 959 | esac 960 | done 961 | else 962 | logMsgToConfigSysLog "WARN" "WARNING: You are running the script using insecure mode, but your system logs are using secure mode. The script only supports a single mode for both, so we are switching the system logs to insecure mode as well." 963 | LOGGLY_TLS_SENDING="false" 964 | LOGGLY_SYSLOG_PORT=514 965 | fi 966 | fi 967 | fi 968 | } 969 | 970 | #function to switch system logging to secure mode if user runs the modular script in secure mode 971 | switchSystemLoggingToSecure() { 972 | if [ -f $LOGGLY_RSYSLOG_CONFFILE ]; then 973 | EXISTING_SYSLOG_PORT=$(grep -Eow 514 $LOGGLY_RSYSLOG_CONFFILE) 974 | if [[ $EXISTING_SYSLOG_PORT == 514 ]]; then 975 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 976 | while true; do 977 | read -p "You are running the script using secure mode, but your system logs are using insecure mode. The script only supports a single mode for both, so would you like to switch your system logs to secure mode? (yes/no)" yn 978 | case $yn in 979 | [Yy]*) 980 | logMsgToConfigSysLog "INFO" "INFO: Going to overwrite the conf file: $LOGGLY_RSYSLOG_CONFFILE with secure configuration" 981 | LOGGLY_TLS_SENDING="true" 982 | LOGGLY_SYSLOG_PORT=6514 983 | break 984 | ;; 985 | [Nn]*) 986 | logMsgToConfigSysLog "INFO" "INFO: Please re-run the script in insecure mode if you want to setup insecure logging" 987 | exit 1 988 | ;; 989 | *) echo "Please answer yes or no." ;; 990 | esac 991 | done 992 | else 993 | logMsgToConfigSysLog "WARN" "WARNING: You are running the script using secure mode, but your system logs are using insecure mode. The script only supports a single mode for both, so we are switching the system logs to secure mode as well." 994 | LOGGLY_TLS_SENDING="true" 995 | LOGGLY_SYSLOG_PORT=6514 996 | fi 997 | fi 998 | fi 999 | } 1000 | 1001 | #check whether the user is running the script in secure or insecure mode and then switch system logging accordingly. 1002 | checkScriptRunningMode() { 1003 | if [ "$FORCE_SECURE" == "false" ]; then 1004 | if [[ $LOGGLY_SYSLOG_PORT == 514 ]]; then 1005 | switchSystemLoggingToInsecure 1006 | else 1007 | switchSystemLoggingToSecure 1008 | fi 1009 | fi 1010 | } 1011 | 1012 | #display usage syntax 1013 | usage() { 1014 | cat <&1 | grep "Empty reply from server" | wc -l) == 1 ]; then 229 | echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." 230 | else 231 | logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings." 232 | exit 1 233 | fi 234 | 235 | echo "INFO: Checking if '$LOGGLY_ACCOUNT' subdomain is valid." 236 | if [[ $LOGGLY_ACCOUNT != $INVALID_SUBDOMAIN ]]; then 237 | if curl --head -s --request GET $LOGGLY_ACCOUNT_URL/login | grep -q "accounts/invalid" ; then 238 | logMsgToConfigSysLog "ERROR" "ERROR: This is not a recognized subdomain. Please ask the account owner for the subdomain they signed up with." 239 | exit 1 240 | else 241 | echo "INFO: $LOGGLY_ACCOUNT_URL is valid and reachable." 242 | fi 243 | else 244 | logMsgToConfigSysLog "ERROR" "ERROR: This is not a recognized subdomain. Please ask the account owner for the subdomain they signed up with. Please note that your subdomain is just the first string in your loggly account URL not the entire account name." 245 | exit 1 246 | fi 247 | } 248 | 249 | #check if user name and password is valid 250 | checkIfValidUserNamePassword() { 251 | echo "INFO: Checking if provided username and password is correct." 252 | if [ $(curl -s -u "$LOGGLY_USERNAME:$LOGGLY_PASSWORD" $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then 253 | logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" 254 | logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. Your username is visible at the top right of the Loggly console before the @ symbol. You can reset your password at http://.loggly.com/login." 255 | exit 1 256 | else 257 | logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." 258 | fi 259 | } 260 | 261 | #gets the authentication token from the Loggly server 262 | getAuthToken() { 263 | if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then 264 | logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." 265 | #get authentication token if user has not provided one 266 | tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") 267 | 268 | #get the string from index 0 to first occurence of , 269 | tokenstr=${tokenstr%%,*} 270 | 271 | #get the string from index 0 to last occurence of " 272 | tokenstr=${tokenstr%\"*} 273 | 274 | #get the string from first occurence of " to the end 275 | tokenstr=${tokenstr#*\"} 276 | 277 | LOGGLY_AUTH_TOKEN=$tokenstr 278 | 279 | logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" 280 | fi 281 | } 282 | 283 | #check if authentication token is valid 284 | checkIfValidAuthToken() { 285 | echo "INFO: Checking if provided auth token is correct." 286 | if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then 287 | logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." 288 | else 289 | logMsgToConfigSysLog "ERROR" "ERROR: Invalid authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." 290 | exit 1 291 | fi 292 | } 293 | 294 | #this functions check if the min required version is installed in the system 295 | checkIfMinRubyVersionInstalled() { 296 | RUBY_VERSION=$(sudo $RUBY --version | grep "$RUBY") 297 | RUBY_VERSION=${RUBY_VERSION%p*} 298 | RUBY_VERSION=${RUBY_VERSION#* } 299 | RUBY_VERSION=$RUBY_VERSION | tr -d " " 300 | if [ $(compareVersions $RUBY_VERSION $MIN_RUBY_VERSION 3) -lt 0 ]; then 301 | logMsgToConfigSysLog "ERROR" "ERROR: Min ruby version required is 1.9.3." 302 | exit 1 303 | fi 304 | } 305 | 306 | checkIfXCodeCommandlineToolsInstalled() { 307 | logMsgToConfigSysLog "INFO" "INFO: Checking if Xcode command line tools are installed." 308 | 309 | if [ $(xcode-select -p 2>/dev/null | wc -l) == 0 ]; then 310 | logMsgToConfigSysLog "ERROR" "ERROR: $MANUAL_XCODE_INSTALL_INSTRUCTION" 311 | exit 1 312 | else 313 | logMsgToConfigSysLog "INFO" "INFO: Xcode command line tools are installed in your system." 314 | fi 315 | } 316 | 317 | #this functions checks if the Fluentd gem is installed in the system 318 | checkIfFluentdInstalled() { 319 | if [ $(sudo fluentd --setup $LOGGLY_HOME/fluent 2>/dev/null | grep ".loggly/fluent/fluent.conf" | wc -l) == 1 ]; then 320 | logMsgToConfigSysLog "INFO" "INFO: Fluentd is already installed. Not installing." 321 | else 322 | logMsgToConfigSysLog "INFO" "INFO: Fluentd is not installed. Installing Fluentd. This may take a while." 323 | installFluentd 324 | fi 325 | } 326 | 327 | #this function installs the Fluentd in the system 328 | installFluentd() { 329 | #install fluentd gem http://docs.fluentd.org/articles/install-by-gem 330 | sudo gem install fluentd --no-document -n/usr/local/bin 331 | 332 | if [[ ! -d "$LOGGLY_HOME" ]]; then 333 | mkdir $LOGGLY_HOME 334 | fi 335 | 336 | #to check fluentd installed successfully 337 | if [ $(sudo fluentd --setup $LOGGLY_HOME/fluent 2>/dev/null | grep ".loggly/fluent/fluent.conf" | wc -l) == 1 ]; then 338 | logMsgToConfigSysLog "INFO" "INFO: Fluentd installed Successfully" 339 | else 340 | logMsgToConfigSysLog "ERROR" "ERROR: Unable to install fluentd" 341 | exit 1 342 | fi 343 | } 344 | 345 | #this function installs Loggly fluentd plugin 346 | installLogglyFluentdPlugin() { 347 | logMsgToConfigSysLog "INFO" "INFO: Installing Loggly plugin for Fluentd" 348 | sudo gem install fluent-plugin-loggly 349 | logMsgToConfigSysLog "INFO" "INFO: Loggly fluentd plugin installed successfully." 350 | } 351 | 352 | #function to write the contents of fluentd config file 353 | writeLogglyConfFile() { 354 | 355 | FLUENTD_CONF="$HOME/.loggly/fluentd-loggly.conf" 356 | 357 | if [ -f "$FLUENTD_CONF" ]; then 358 | echo "INFO: Conf file already exists. Creating Backup $FLUENTD_CONF $FLUENTD_CONF.bk" 359 | sudo mv $FLUENTD_CONF $FLUENTD_CONF.bk 360 | fi 361 | 362 | logMsgToConfigSysLog "INFO" "INFO: Creating file $FLUENTD_CONF" 363 | 364 | sudo touch $FLUENTD_CONF 365 | 366 | inputStr=" 367 | 368 | type tail 369 | format none 370 | path /var/log/system.log 371 | tag system_logs 372 | 373 | 374 | type loggly 375 | loggly_url http://logs-01.loggly.com/inputs/$LOGGLY_AUTH_TOKEN/tag/Mac 376 | " 377 | 378 | sudo cat <>$FLUENTD_CONF 379 | $inputStr 380 | EOIPFW 381 | 382 | } 383 | 384 | #delete 22-loggly.conf file 385 | removeLogglyConfFile() { 386 | if [ -f "$HOME/.loggly/fluentd-loggly.conf" ]; then 387 | logMsgToConfigSysLog "INFO" "INFO: Deleting file fluentd-loggly.conf" 388 | sudo rm -rf "$HOME/.loggly/fluentd-loggly.conf" 389 | 390 | logMsgToConfigSysLog "INFO" "INFO: Removing Fluentd service" 391 | sudo launchctl unload -F /Library/LaunchDaemons/com.loggly.loggly_fluentd.plist >/dev/null 2>&1 392 | sudo rm -rf /Library/LaunchDaemons/com.loggly.loggly_fluentd.plist 393 | else 394 | logMsgToConfigSysLog "ERROR" "ERROR: There is no conf file to delete" 395 | exit 1 396 | fi 397 | } 398 | 399 | #this function creates a fluentd daemon to send logs to Loggly 400 | configureFluentdAsService() { 401 | logMsgToConfigSysLog "INFO" "INFO: Creating daemon for Loggly conf file." 402 | 403 | #this sets the fluentd installation location 404 | FLUENTD_LOCATION=$(which fluentd) 405 | 406 | PROP_FILE="/Library/LaunchDaemons/com.loggly.loggly_fluentd.plist" 407 | 408 | #if loggly fluentd is already running as a service then unload it 409 | if [ $(sudo launchctl list | grep 'com.loggly.loggly_fluentd' | wc -l) == 1 ]; then 410 | sudo launchctl unload -F $PROP_FILE >/dev/null 2>&1 411 | 412 | #if there was some error while unloading, just remove it 413 | sudo launchctl remove com.loggly.loggly_fluentd >/dev/null 2>&1 414 | fi 415 | 416 | #if plist file is already there then delete it 417 | if [ -f "$PROP_FILE" ]; then 418 | sudo rm -f $PROP_FILE 419 | fi 420 | 421 | sudo touch $PROP_FILE 422 | sudo chmod +x $PROP_FILE 423 | 424 | propStr=" 425 | 426 | 427 | 428 | Label 429 | com.loggly.loggly_fluentd 430 | ProgramArguments 431 | 432 | $FLUENTD_LOCATION 433 | -c 434 | $HOME/.loggly/fluentd-loggly.conf 435 | 436 | RunAtLoad 437 | 438 | StandardErrorPath 439 | /tmp/loggly_fluentd.err 440 | StandardOutPath 441 | /tmp/loggly_fluentd.out 442 | 443 | " 444 | 445 | sudo cat <>$PROP_FILE 446 | $propStr 447 | EOIPFW 448 | 449 | } 450 | 451 | #starts Fluentd Service 452 | startFluentdService() { 453 | logMsgToConfigSysLog "INFO" "INFO: Starting Fluentd as a service" 454 | sudo launchctl load -F $PROP_FILE 455 | logMsgToConfigSysLog "INFO" "INFO: Fluentd started successfully" 456 | } 457 | 458 | #check if the logs made it to Loggly 459 | checkIfLogsMadeToLoggly() { 460 | logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly. Waiting for 30 secs." 461 | 462 | #sleeping for 30 secs so that fluentd service can start doing its work properly 463 | sleep 30 464 | uuid=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | fold -w 32 | head -n 1) 465 | 466 | queryParam="tag%3AMac%20$uuid" 467 | logger -t "Mac" "Mac-Test message for verification with UUID $uuid" 468 | 469 | counter=1 470 | maxCounter=10 471 | finalCount=0 472 | 473 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 474 | logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" 475 | 476 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." 477 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 478 | searchAndFetch finalCount "$queryUrl" 479 | let counter=$counter+1 480 | 481 | while [[ "$finalCount" -eq 0 ]]; do 482 | echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." 483 | sleep 30 484 | echo "INFO: Done waiting. Verifying again." 485 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 486 | searchAndFetch finalCount "$queryUrl" 487 | let counter=$counter+1 488 | if [ "$counter" -gt "$maxCounter" ]; then 489 | logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." 490 | exit 1 491 | fi 492 | done 493 | 494 | if [[ "$finalCount" -eq 1 ]]; then 495 | if [ "$IS_INVOKED" = "" ]; then 496 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Mac system logs to Loggly." 497 | exit 0 498 | else 499 | logMsgToConfigSysLog "INFO" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Mac system logs to Loggly." 500 | fi 501 | fi 502 | 503 | } 504 | 505 | compareVersions() { 506 | typeset IFS='.' 507 | typeset -a v1=($1) 508 | typeset -a v2=($2) 509 | typeset n diff 510 | 511 | for ((n = 0; n < $3; n += 1)); do 512 | diff=$((v1[n] - v2[n])) 513 | if [ $diff -ne 0 ]; then 514 | [ $diff -le 0 ] && echo '-1' || echo '1' 515 | return 516 | fi 517 | done 518 | echo '0' 519 | } 520 | 521 | #logs message to config syslog 522 | logMsgToConfigSysLog() { 523 | #$1 variable will be SUCCESS or ERROR or INFO or WARNING 524 | #$2 variable will be the message 525 | cslStatus=$1 526 | cslMessage=$2 527 | echo "$cslMessage" 528 | currentTime=$(date) 529 | 530 | #for Linux system, we need to use -d switch to decode base64 whereas 531 | #for Mac system, we need to use -D switch to decode 532 | varUname=$(uname) 533 | if [[ $varUname == 'Linux' ]]; then 534 | enabler=$(echo -n MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 --decode) 535 | elif [[ $varUname == 'Darwin' ]]; then 536 | enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 --decode) 537 | fi 538 | 539 | if [ $? -ne 0 ]; then 540 | echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." 541 | exit 1 542 | fi 543 | 544 | sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" 545 | 546 | #if it is an error, then log message "Script Failed" to config syslog and exit the script 547 | if [[ $cslStatus == "ERROR" ]]; then 548 | sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" 549 | echo $MANUAL_CONFIG_INSTRUCTION 550 | exit 1 551 | fi 552 | 553 | #if it is a success, then log message "Script Succeeded" to config syslog and exit the script 554 | if [[ $cslStatus == "SUCCESS" ]]; then 555 | sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" 556 | exit 0 557 | fi 558 | } 559 | 560 | #payload construction to send log to config syslog 561 | sendPayloadToConfigSysLog() { 562 | if [ "$APP_TAG" = "" ]; then 563 | var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"Mac-distribution\":\"$MAC_DIST\", \"messages\":\"$2\",\"ruby-version\":\"$RUBY_VERSION\"}" 564 | else 565 | var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"Mac-distribution\":\"$MAC_DIST\", $APP_TAG, \"messages\":\"$2\",\"ruby-version\":\"$RUBY_VERSION\"}" 566 | fi 567 | curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 >/dev/null 2>&1 568 | } 569 | 570 | #$1 return the count of records in loggly, $2 is the query param to search in loggly 571 | searchAndFetch() { 572 | url=$2 573 | 574 | result=$(curl -s -u "$LOGGLY_USERNAME":"$LOGGLY_PASSWORD" "$url") 575 | 576 | if [ -z "$result" ]; then 577 | logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." 578 | exit 1 579 | fi 580 | id=$(echo "$result" | grep -v "{" | grep id | awk '{print $2}') 581 | # strip last double quote from id 582 | id="${id%\"}" 583 | # strip first double quote from id 584 | id="${id#\"}" 585 | url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" 586 | 587 | # retrieve the data 588 | result=$(curl -s -u "$LOGGLY_USERNAME":"$LOGGLY_PASSWORD" "$url") 589 | count=$(echo "$result" | grep total_events | awk '{print $2}') 590 | count="${count%\,}" 591 | eval $1="'$count'" 592 | if [[ "$count" -gt 0 ]]; then 593 | timestamp=$(echo "$result" | grep timestamp) 594 | fi 595 | } 596 | 597 | #get password in the form of asterisk 598 | getPassword() { 599 | unset LOGGLY_PASSWORD 600 | prompt="Please enter Loggly Password:" 601 | while IFS= read -p "$prompt" -r -s -n 1 char; do 602 | if [[ $char == $'\0' ]]; then 603 | break 604 | fi 605 | prompt='*' 606 | LOGGLY_PASSWORD+="$char" 607 | done 608 | echo 609 | } 610 | 611 | #display usage syntax 612 | usage() { 613 | cat </dev/null; then 117 | logMsgToConfigSysLog "INFO" "INFO: Apache is present as a service." 118 | else 119 | logMsgToConfigSysLog "ERROR" "ERROR: Apache is not configured as a service" 120 | exit 1 121 | fi 122 | 123 | #get the version of apache installed 124 | getApacheVersion 125 | 126 | #check if apache is supported 127 | checkIfSupportedApacheVersion 128 | 129 | #set all the required apache variables by this script 130 | setApacheVariables 131 | } 132 | 133 | #Get the apache service name on various linux flavors 134 | getApacheServiceName() { 135 | #checking if the Linux is yum based or apt-get based 136 | YUM_BASED=$(command -v yum) 137 | APT_GET_BASED=$(command -v apt-get) 138 | 139 | if [ "$YUM_BASED" != "" ]; then 140 | SERVICE="httpd" 141 | APACHE_ACCESS_LOG_FILE="access_log" 142 | APACHE_ERROR_LOG_FILE="error_log" 143 | 144 | elif [ "$APT_GET_BASED" != "" ]; then 145 | SERVICE="apache2" 146 | APACHE_ACCESS_LOG_FILE="access.log" 147 | APACHE_ERROR_LOG_FILE="error.log" 148 | fi 149 | } 150 | 151 | #sets apache variables which will be used across various functions 152 | setApacheVariables() { 153 | LOGGLY_APACHE_LOG_HOME=/var/log/$SERVICE 154 | } 155 | 156 | #gets the version of apache installed on the unix box 157 | getApacheVersion() { 158 | APACHE_VERSION=$($SERVICE -v | grep "Server version: Apache") 159 | APACHE_VERSION=${APACHE_VERSION#*/} 160 | APACHE_VERSION=${APACHE_VERSION% *} 161 | APACHE_VERSION=$APACHE_VERSION | tr -d ' ' 162 | APP_TAG="\"apache-version\":\"$APACHE_VERSION\"" 163 | logMsgToConfigSysLog "INFO" "INFO: Apache version: $APACHE_VERSION" 164 | } 165 | 166 | #checks if the apache version is supported by this script, currently the script 167 | #only supports apache2 168 | checkIfSupportedApacheVersion() { 169 | apacheMajorVersion=${APACHE_VERSION%%.*} 170 | if [[ ($apacheMajorVersion -ne 2) ]]; then 171 | logMsgToConfigSysLog "ERROR" "ERROR: This script only supports Apache version 2." 172 | exit 1 173 | fi 174 | } 175 | 176 | checkLogFileSize() { 177 | accessFileSize=$(wc -c "$1" | cut -f 1 -d ' ') 178 | errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') 179 | fileSize=$((accessFileSize + errorFileSize)) 180 | if [ $fileSize -ge 102400000 ]; then 181 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 182 | while true; do 183 | read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn 184 | case $yn in 185 | [Yy]*) 186 | logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration." 187 | break 188 | ;; 189 | [Nn]*) 190 | logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Discontinuing with Apache Loggly configuration." 191 | exit 1 192 | break 193 | ;; 194 | *) echo "Please answer yes or no." ;; 195 | esac 196 | done 197 | else 198 | logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." 199 | logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration." 200 | fi 201 | elif [ $fileSize -eq 0 ]; then 202 | logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." 203 | exit 1 204 | fi 205 | } 206 | 207 | write21ApacheConfFile() { 208 | #Create apache syslog config file if it doesn't exist 209 | echo "INFO: Checking if apache sysconf file $APACHE_SYSLOG_CONFFILE exist." 210 | if [ -f "$APACHE_SYSLOG_CONFFILE" ]; then 211 | 212 | logMsgToConfigSysLog "WARN" "WARN: Apache syslog file $APACHE_SYSLOG_CONFFILE already exist." 213 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 214 | while true; do 215 | read -p "Do you wish to override $APACHE_SYSLOG_CONFFILE? (yes/no)" yn 216 | case $yn in 217 | [Yy]*) 218 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP" 219 | sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP 220 | write21ApacheFileContents 221 | break 222 | ;; 223 | [Nn]*) 224 | restartRsyslog 225 | break ;; 226 | *) echo "Please answer yes or no." ;; 227 | esac 228 | done 229 | else 230 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP" 231 | sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP 232 | write21ApacheFileContents 233 | fi 234 | else 235 | write21ApacheFileContents 236 | fi 237 | } 238 | 239 | addTagsInConfiguration() { 240 | #split tags by comman(,) 241 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 242 | for i in "${array[@]}"; do 243 | TAG="$TAG tag=\\\"$i\\\" " 244 | done 245 | } 246 | #function to write the contents of apache syslog config file 247 | write21ApacheFileContents() { 248 | logMsgToConfigSysLog "INFO" "INFO: Creating file $APACHE_SYSLOG_CONFFILE" 249 | sudo touch $APACHE_SYSLOG_CONFFILE 250 | sudo chmod o+w $APACHE_SYSLOG_CONFFILE 251 | commonContent=" 252 | \$ModLoad imfile 253 | \$InputFilePollInterval 10 254 | \$WorkDirectory $RSYSLOG_DIR 255 | " 256 | 257 | if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then 258 | commonContent+="\$PrivDropToGroup adm 259 | " 260 | fi 261 | 262 | imfileStr=$commonContent" 263 | 264 | \$ActionSendStreamDriver gtls 265 | \$ActionSendStreamDriverMode 1 266 | \$ActionSendStreamDriverAuthMode x509/name 267 | \$ActionSendStreamDriverPermittedPeer *.loggly.com 268 | 269 | #RsyslogGnuTLS 270 | \$DefaultNetstreamDriverCAFile $CA_FILE_PATH 271 | 272 | # Apache access file: 273 | \$InputFileName $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE 274 | \$InputFileTag apache-access: 275 | \$InputFileStateFile stat-apache-access 276 | \$InputFileSeverity info 277 | \$InputFilePersistStateInterval 20000 278 | \$InputRunFileMonitor 279 | 280 | #Apache Error file: 281 | \$InputFileName $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE 282 | \$InputFileTag apache-error: 283 | \$InputFileStateFile stat-apache-error 284 | \$InputFileSeverity error 285 | \$InputFilePersistStateInterval 20000 286 | \$InputRunFileMonitor 287 | 288 | #Add a tag for apache events 289 | \$template LogglyFormatApache,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" 290 | 291 | if \$programname == 'apache-access' then @@logs-01.loggly.com:6514;LogglyFormatApache 292 | if \$programname == 'apache-access' then ~ 293 | if \$programname == 'apache-error' then @@logs-01.loggly.com:6514;LogglyFormatApache 294 | if \$programname == 'apache-error' then ~ 295 | " 296 | imfileStrNonTls=$commonContent" 297 | 298 | # Apache access file: 299 | \$InputFileName $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE 300 | \$InputFileTag apache-access: 301 | \$InputFileStateFile stat-apache-access 302 | \$InputFileSeverity info 303 | \$InputFilePersistStateInterval 20000 304 | \$InputRunFileMonitor 305 | 306 | #Apache Error file: 307 | \$InputFileName $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE 308 | \$InputFileTag apache-error: 309 | \$InputFileStateFile stat-apache-error 310 | \$InputFileSeverity error 311 | \$InputFilePersistStateInterval 20000 312 | \$InputRunFileMonitor 313 | 314 | #Add a tag for apache events 315 | \$template LogglyFormatApache,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" 316 | 317 | if \$programname == 'apache-access' then @@logs-01.loggly.com:514;LogglyFormatApache 318 | if \$programname == 'apache-access' then ~ 319 | if \$programname == 'apache-error' then @@logs-01.loggly.com:514;LogglyFormatApache 320 | if \$programname == 'apache-error' then ~ 321 | " 322 | 323 | if [ $TLS_SENDING == "false" ]; then 324 | imfileStr=$imfileStrNonTls 325 | fi 326 | 327 | #change the apache-21 file to variable from above and also take the directory of the apache log file. 328 | sudo cat <>$APACHE_SYSLOG_CONFFILE 329 | $imfileStr 330 | EOIPFW 331 | 332 | restartRsyslog 333 | } 334 | 335 | #checks if the apache logs made to loggly 336 | checkIfApacheLogsMadeToLoggly() { 337 | counter=1 338 | maxCounter=10 339 | 340 | apacheInitialLogCount=0 341 | apacheLatestLogCount=0 342 | 343 | TAGS= 344 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 345 | for i in "${array[@]}"; do 346 | if [ "$TAGS" == "" ]; then 347 | TAGS="tag%3A$i" 348 | else 349 | TAGS="$TAGS%20tag%3A$i" 350 | fi 351 | done 352 | 353 | read -p "In order to check if Apache logs are successfully sent to Loggly, apache needs to produce logs. Do you want to create a log record by accessing server? (yes/no)" yn 354 | case $yn in 355 | [Yy]*) 356 | #access a page on apache server in order to create a log record in access log file 357 | WEB_ADDRESS=127.0.0.1 358 | while true; do 359 | read -p "Default check is performed on a localhost address 127.0.0.1 . Are you running an apache server with a different address? (yes/no)" yesno 360 | case $yesno in 361 | [Yy]*) 362 | logMsgToConfigSysLog "INFO" "INFO: Using specific apache server address." 363 | read -p "Please enter Apache server address " -r WEB_ADDRESS 364 | ;; 365 | [Nn]*) 366 | logMsgToConfigSysLog "INFO" "INFO: Using default apache server address." 367 | ;; 368 | *) echo "Please answer yes or no." ;; 369 | esac 370 | 371 | RESPONSE_CODE=$(curl --max-time 10 -s -o /dev/null -i -w "%{http_code}" $WEB_ADDRESS) 372 | #curl return 000 response on reaching max-time 373 | if [ ${RESPONSE_CODE} != "000" ]; then 374 | logMsgToConfigSysLog "INFO" "INFO: Curl on server $WEB_ADDRESS has succeeded" 375 | break; 376 | else 377 | logMsgToConfigSysLog "INFO" "INFO: Curl on server $WEB_ADDRESS has failed." 378 | fi 379 | done 380 | ;; 381 | [Nn]*) 382 | logMsgToConfigSysLog "INFO" "INFO: Creating a sample log record has been skipped." 383 | ;; 384 | *) echo "Please answer yes or no." ;; 385 | esac 386 | 387 | queryParam="$TAGS&from=-15m&until=now&size=1" 388 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 389 | logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" 390 | 391 | logMsgToConfigSysLog "INFO" "INFO: Getting initial apache log count." 392 | #get the initial count of apache logs for past 15 minutes 393 | searchAndFetch apacheInitialLogCount "$queryUrl" 394 | 395 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the apache logs made it to Loggly." 396 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 397 | #get the final count of apache logs for past 15 minutes 398 | searchAndFetch apacheLatestLogCount "$queryUrl" 399 | let counter=$counter+1 400 | 401 | while [ "$apacheLatestLogCount" -le "$apacheInitialLogCount" ]; do 402 | echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." 403 | sleep 30 404 | echo "INFO: Done waiting. Verifying again." 405 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 406 | searchAndFetch apacheLatestLogCount "$queryUrl" 407 | let counter=$counter+1 408 | if [ "$counter" -gt "$maxCounter" ]; then 409 | logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check network and firewall settings and retry." 410 | exit 1 411 | fi 412 | done 413 | 414 | if [ "$apacheLatestLogCount" -gt "$apacheInitialLogCount" ]; then 415 | logMsgToConfigSysLog "INFO" "INFO: Apache logs successfully transferred to Loggly! You are now sending Apache logs to Loggly." 416 | checkIfLogsAreParsedInLoggly 417 | fi 418 | } 419 | #verifying if the logs are being parsed or not 420 | checkIfLogsAreParsedInLoggly() { 421 | apacheInitialLogCount=0 422 | TAG_PARSER= 423 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 424 | 425 | for i in "${array[@]}"; do 426 | TAG_PARSER="$TAG_PARSER%20tag%3A$i " 427 | done 428 | queryParam="logtype%3Aapache$TAG_PARSER&from=-15m&until=now&size=1" 429 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 430 | searchAndFetch apacheInitialLogCount "$queryUrl" 431 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the Apache logs are parsed in Loggly." 432 | if [ "$apacheInitialLogCount" -gt 0 ]; then 433 | logMsgToConfigSysLog "INFO" "INFO: Apache logs successfully parsed in Loggly!" 434 | else 435 | logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" 436 | fi 437 | } 438 | 439 | #remove 21apache.conf file 440 | remove21ApacheConfFile() { 441 | echo "INFO: Deleting the loggly apache syslog conf file." 442 | if [ -f "$APACHE_SYSLOG_CONFFILE" ]; then 443 | sudo rm -rf "$APACHE_SYSLOG_CONFFILE" 444 | fi 445 | echo "INFO: Removed all the modified files." 446 | restartRsyslog 447 | } 448 | 449 | #display usage syntax 450 | usage() { 451 | cat <.conf file 94 | write21ConfFileContents 95 | 96 | fi 97 | 98 | #restart rsyslog 99 | restartRsyslog 100 | 101 | #verify if the file logs made it to loggly 102 | checkIfFileLogsMadeToLoggly 103 | 104 | if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then 105 | #log success message 106 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." 107 | fi 108 | } 109 | 110 | #executing script to remove loggly configuration for File 111 | removeLogglyConfForFile() { 112 | logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." 113 | 114 | #check if the user has root permission to run this script 115 | checkIfUserHasRootPrivileges 116 | 117 | #check if the OS is supported by the script. If no, then exit 118 | checkIfSupportedOS 119 | 120 | #construct variables using filename and filealias 121 | constructFileVariables 122 | 123 | #checks if the conf file exists. if not, then exit. 124 | checkIfConfFileExist 125 | 126 | #remove 21.conf file 127 | remove21ConfFile 128 | 129 | #restart rsyslog 130 | restartRsyslog 131 | 132 | removeStatFile 133 | 134 | #log success message 135 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Rollback completed." 136 | } 137 | 138 | checkIfFileLocationContainSpaces() { 139 | case "$LOGGLY_FILE_TO_MONITOR" in 140 | *\ *) 141 | logMsgToConfigSysLog "ERROR" "ERROR: File location cannot contain spaces." 142 | exit 1 143 | ;; 144 | *) ;; 145 | esac 146 | } 147 | 148 | constructFileVariables() { 149 | #conf file name 150 | FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$FILE_ALIAS.conf" 151 | 152 | #conf file backup name 153 | FILE_SYSLOG_CONFFILE_BACKUP="$RSYSLOG_ETCDIR_CONF/$FILE_ALIAS.loggly.bk" 154 | 155 | #application tag 156 | APP_TAG="\"file-alias\":\"$LOGGLY_FILE_TO_MONITOR_ALIAS\"" 157 | } 158 | 159 | #configures the directory files for file monitoring 160 | configureDirectoryFileMonitoring() { 161 | addTagsInConfiguration 162 | TOTAL_FILES_IN_DIR=$(ls -1 ${LOGGLY_FILE_TO_MONITOR} | wc -l) 163 | logMsgToConfigSysLog "INFO" "INFO: There are $TOTAL_FILES_IN_DIR files in directory. Configuring each file for monitoring present in this directory." 164 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 165 | while true; do 166 | read -p "There are $TOTAL_FILES_IN_DIR files present in this directory. Would you like to configure all the files (yes/no)?" yn 167 | case $yn in 168 | [Yy]*) 169 | installLogglyConf 170 | for file in $(find $LOGGLY_FILE_TO_MONITOR -name '*'); do 171 | configureFilesPresentInDirectory $file $FILE_ALIAS 172 | done 173 | break 174 | ;; 175 | [Nn]*) 176 | exit 1 177 | break 178 | ;; 179 | *) echo "Please answer yes or no." ;; 180 | esac 181 | done 182 | while true; do 183 | read -p "Would you like install a Cron job to sync the files currently in your Directory every 10 minutes? (yes/no)" yn 184 | case $yn in 185 | [Yy]*) 186 | doCronInstallation 187 | break 188 | ;; 189 | [Nn]*) 190 | logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." 191 | break 192 | ;; 193 | *) echo "Please answer yes or no." ;; 194 | esac 195 | done 196 | else 197 | installLogglyConf 198 | for file in $(find $LOGGLY_FILE_TO_MONITOR -name '*'); do 199 | configureFilesPresentInDirectory $file $FILE_ALIAS 200 | done 201 | if [[ ! -f "/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ]]; then 202 | doCronInstallation 203 | fi 204 | fi 205 | } 206 | 207 | #actually configures a file present in the directory for monitoring 208 | configureFilesPresentInDirectory() { 209 | FILE_TO_MONITOR=$1 210 | fileNameWithExt=${1##*/} 211 | uniqueFileName=$(echo "$fileNameWithExt" | tr . _) 212 | var=$(file $FILE_TO_MONITOR) 213 | 214 | #checking if it is a text file otherwise ignore it 215 | #it may be possible that the "text" may contain some uppercase letters like "Text" 216 | var=$(echo $var | tr "[:upper:]" "[:lower:]") 217 | if [[ $var == *text* ]]; then 218 | LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$2 219 | if [ -f ${FILE_TO_MONITOR} ]; then 220 | constructFileVariables 221 | checkFileReadPermission 222 | checkLogFileSize $FILE_TO_MONITOR 223 | STATE_FILE_ALIAS=$(echo -n "$uniqueFileName" | md5sum | tr -d ' ')$FILE_ALIAS 224 | write21ConfFileContents 225 | fi 226 | fi 227 | } 228 | 229 | checkIfWildcardExist() { 230 | TOTAL_FILES_IN_DIR=$(ls -1 ${LOGGLY_FILE_TO_MONITOR} 2>/dev/null | wc -l) 231 | WILDCARDS=('*' '.' '?' '|' ']' '[') 232 | for WILDCARD in "${WILDCARDS[@]}"; do 233 | if [[ $LOGGLY_FILE_TO_MONITOR == *"${WILDCARD}"* && $TOTAL_FILES_IN_DIR -gt 0 ]]; then 234 | IS_WILDCARD="true" 235 | return 0 236 | else 237 | return 1 238 | fi 239 | done 240 | } 241 | 242 | #checks if the file to be monitored exist 243 | checkIfFileExist() { 244 | if [ -f "$LOGGLY_FILE_TO_MONITOR" ]; then 245 | logMsgToConfigSysLog "INFO" "INFO: File $LOGGLY_FILE_TO_MONITOR exists." 246 | else 247 | logMsgToConfigSysLog "ERROR" "ERROR: File $LOGGLY_FILE_TO_MONITOR does not exist. Kindly recheck." 248 | exit 1 249 | fi 250 | } 251 | 252 | #deletes the state file for the current alias, if exists 253 | deleteStateFile() { 254 | restartRsyslog 255 | sudo rm -f $RSYSLOG_DIR/stat-$FILE_ALIAS 256 | restartRsyslog 257 | } 258 | 259 | #check if the file alias is already taken 260 | checkIfFileAliasExist() { 261 | if [ -f "$FILE_SYSLOG_CONFFILE" ]; then 262 | logMsgToConfigSysLog "WARN" "WARN: This file alias is already taken. You must choose a unique file alias for each file." 263 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 264 | while true; do 265 | read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn 266 | case $yn in 267 | [Yy]*) 268 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP" 269 | sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP 270 | deleteStateFile 271 | break 272 | ;; 273 | [Nn]*) 274 | logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" 275 | exit 1 276 | break 277 | ;; 278 | *) echo "Please answer yes or no." ;; 279 | esac 280 | done 281 | else 282 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP" 283 | sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP 284 | deleteStateFile 285 | fi 286 | fi 287 | } 288 | 289 | #check the size of the log file. If the size is greater than 100MB give a warning to the user. If the file size is 0 290 | #then exit 291 | checkLogFileSize() { 292 | monitorFileSize=$(wc -c "$1" | cut -f 1 -d ' ') 293 | if [ $monitorFileSize -ge 102400000 ]; then 294 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 295 | while true; do 296 | read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn 297 | case $yn in 298 | [Yy]*) 299 | logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration." 300 | break 301 | ;; 302 | [Nn]*) 303 | logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." 304 | exit 1 305 | break 306 | ;; 307 | *) echo "Please answer yes or no." ;; 308 | esac 309 | done 310 | else 311 | logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." 312 | logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration." 313 | fi 314 | elif [ $monitorFileSize -eq 0 ]; then 315 | logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR so there won't be any data sent to Loggly. You can generate some logs by writing to this file." 316 | exit 1 317 | else 318 | logMsgToConfigSysLog "INFO" "INFO: File size of $FILE_TO_MONITOR is $monitorFileSize bytes." 319 | fi 320 | } 321 | 322 | #checks the input file has proper read permissions 323 | checkFileReadPermission() { 324 | LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") 325 | #no need to check read permissions with RedHat and CentOS as they also work with ---------- (000)permissions 326 | case "$LINUX_DIST_IN_LOWER_CASE" in 327 | *"redhat"*) ;; 328 | 329 | *"centos"*) ;; 330 | 331 | *) 332 | FILE_PERMISSIONS=$(ls -ld $LOGGLY_FILE_TO_MONITOR) 333 | #checking if the file has read permission for others 334 | PERMISSION_READ_OTHERS=${FILE_PERMISSIONS:7:1} 335 | if [[ $PERMISSION_READ_OTHERS != r ]]; then 336 | logMsgToConfigSysLog "WARN" "WARN: $LOGGLY_FILE_TO_MONITOR does not have proper read permissions. Verification step may fail." 337 | fi 338 | ;; 339 | esac 340 | } 341 | 342 | addTagsInConfiguration() { 343 | #split tags by comman(,) 344 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 345 | for i in "${array[@]}"; do 346 | TAG="$TAG tag=\\\"$i\\\" " 347 | done 348 | } 349 | 350 | doCronInstallation() { 351 | if [[ ! -d "/root/.loggly" ]]; then 352 | mkdir /root/.loggly 353 | fi 354 | CRON_SCRIPT="/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" 355 | logMsgToConfigSysLog "INFO" "INFO: Creating cron script $CRON_SCRIPT" 356 | 357 | sudo touch $CRON_SCRIPT 358 | sudo chmod +x $CRON_SCRIPT 359 | 360 | cronScriptStr="#!/bin/bash 361 | curl -s -o configure-file-monitoring.sh https://www.loggly.com/install/configure-file-monitoring.sh 362 | sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE.bk 363 | sudo rm -f $FILE_SYSLOG_CONFFILE 364 | sudo bash configure-file-monitoring.sh -a $LOGGLY_ACCOUNT -u $LOGGLY_USERNAME -p $LOGGLY_PASSWORD -f $LOGGLY_FILE_TO_MONITOR -l $FILE_ALIAS -tag $LOGGLY_FILE_TAG -s 365 | " 366 | #write to cron script file 367 | 368 | sudo cat <>$CRON_SCRIPT 369 | $cronScriptStr 370 | EOIPFW 371 | 372 | CRON_JOB_TO_MONITOR_FILES="*/10 * * * * sudo bash $CRON_SCRIPT" 373 | CRON_FILE="/tmp/File_Monitor_Cron" 374 | 375 | EXISTING_CRONS=$(sudo crontab -l 2>&1) 376 | case $EXISTING_CRONS in 377 | no*) ;; 378 | 379 | *) 380 | echo "$EXISTING_CRONS" >>$CRON_FILE 381 | ;; 382 | esac 383 | 384 | echo "$CRON_JOB_TO_MONITOR_FILES" >>$CRON_FILE 385 | sudo crontab $CRON_FILE 386 | sudo rm -fr $CRON_FILE 387 | } 388 | 389 | #function to write the contents of syslog config file 390 | write21ConfFileContents() { 391 | logMsgToConfigSysLog "INFO" "INFO: Creating file $FILE_SYSLOG_CONFFILE" 392 | sudo touch $FILE_SYSLOG_CONFFILE 393 | sudo chmod o+w $FILE_SYSLOG_CONFFILE 394 | 395 | rsyslog_version=($(rsyslogd -v)) 396 | r_ver=${rsyslog_version[1]:0:1} 397 | if [ $r_ver -le 7 ]; then 398 | imfileStr=" 399 | \$ModLoad imfile 400 | \$InputFilePollInterval 10 401 | \$WorkDirectory $RSYSLOG_DIR 402 | \$ActionSendStreamDriver gtls 403 | \$ActionSendStreamDriverMode 1 404 | \$ActionSendStreamDriverAuthMode x509/name 405 | \$ActionSendStreamDriverPermittedPeer *.loggly.com 406 | 407 | #RsyslogGnuTLS 408 | \$DefaultNetstreamDriverCAFile $CA_FILE_PATH 409 | 410 | # File access file: 411 | \$InputFileName $FILE_TO_MONITOR 412 | \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS 413 | \$InputFileStateFile stat-$STATE_FILE_ALIAS 414 | \$InputFileSeverity info 415 | \$InputFilePersistStateInterval 20000 416 | \$InputRunFileMonitor 417 | 418 | #Add a tag for file events 419 | template (name=\"$CONF_FILE_FORMAT_NAME\" type=\"string\" string=\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\") 420 | 421 | if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then action(type=\"omfwd\" protocol=\"tcp\" target=\"logs-01.loggly.com\" port=\"6514\" template=\"$CONF_FILE_FORMAT_NAME\") 422 | if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then stop 423 | " 424 | imfileStrNonTls=" 425 | \$ModLoad imfile 426 | \$InputFilePollInterval 10 427 | \$WorkDirectory $RSYSLOG_DIR 428 | 429 | # File access file: 430 | \$InputFileName $FILE_TO_MONITOR 431 | \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS 432 | \$InputFileStateFile stat-$STATE_FILE_ALIAS 433 | \$InputFileSeverity info 434 | \$InputFilePersistStateInterval 20000 435 | \$InputRunFileMonitor 436 | 437 | #Add a tag for file events 438 | template (name=\"$CONF_FILE_FORMAT_NAME\" type=\"string\" string=\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\") 439 | 440 | if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then action(type=\"omfwd\" protocol=\"tcp\" target=\"logs-01.loggly.com\" port=\"514\" template=\"$CONF_FILE_FORMAT_NAME\") 441 | if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then stop 442 | " 443 | else 444 | imfileStr=" 445 | module(load=\"imfile\") 446 | 447 | #RsyslogGnuTLS 448 | \$DefaultNetstreamDriverCAFile $CA_FILE_PATH 449 | 450 | # Input for FILE1 451 | input(type=\"imfile\" tag=\"$LOGGLY_FILE_TO_MONITOR_ALIAS\" ruleset=\"filelog\" file=\"$FILE_TO_MONITOR\") #wildcard is allowed at file level only 452 | 453 | # Add a tag for file events 454 | template(name=\"$CONF_FILE_FORMAT_NAME\" type=\"string\" string=\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\") 455 | 456 | ruleset(name=\"filelog\"){ 457 | action(type=\"omfwd\" protocol=\"tcp\" target=\"logs-01.loggly.com\" port=\"6514\" template=\"$CONF_FILE_FORMAT_NAME\" StreamDriver=\"gtls\" StreamDriverMode=\"1\" StreamDriverAuthMode=\"x509/name\" StreamDriverPermittedPeers=\"*.loggly.com\") 458 | } 459 | " 460 | imfileStrNonTls=" 461 | 462 | module(load=\"imfile\") 463 | 464 | # Input for FILE1 465 | input(type=\"imfile\" tag=\"$LOGGLY_FILE_TO_MONITOR_ALIAS\" ruleset=\"filelog\" file=\"$FILE_TO_MONITOR\") #wildcard is allowed at file level only 466 | 467 | # Add a tag for file events 468 | template(name=\"$CONF_FILE_FORMAT_NAME\" type=\"string\" string=\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\") 469 | 470 | ruleset(name=\"filelog\"){ 471 | action(type=\"omfwd\" protocol=\"tcp\" target=\"logs-01.loggly.com\" port=\"514\" template=\"$CONF_FILE_FORMAT_NAME\") stop 472 | } 473 | " 474 | fi 475 | 476 | if [ $FILE_TLS_SENDING == "false" ]; then 477 | imfileStr=$imfileStrNonTls 478 | fi 479 | 480 | #write to 21-.conf file 481 | sudo cat <>$FILE_SYSLOG_CONFFILE 482 | $imfileStr 483 | EOIPFW 484 | 485 | } 486 | 487 | #checks if the apache logs made to loggly 488 | checkIfFileLogsMadeToLoggly() { 489 | counter=1 490 | maxCounter=10 491 | 492 | fileInitialLogCount=0 493 | fileLatestLogCount=0 494 | queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS&from=-15m&until=now&size=1" 495 | 496 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 497 | logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" 498 | 499 | logMsgToConfigSysLog "INFO" "INFO: Getting initial log count." 500 | #get the initial count of file logs for past 15 minutes 501 | searchAndFetch fileInitialLogCount "$queryUrl" 502 | 503 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the logs made it to Loggly." 504 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 505 | #get the final count of file logs for past 15 minutes 506 | searchAndFetch fileLatestLogCount "$queryUrl" 507 | let counter=$counter+1 508 | 509 | while [ "$fileLatestLogCount" -le "$fileInitialLogCount" ]; do 510 | echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." 511 | sleep 30 512 | echo "INFO: Done waiting. Verifying again." 513 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 514 | searchAndFetch fileLatestLogCount "$queryUrl" 515 | let counter=$counter+1 516 | if [ "$counter" -gt "$maxCounter" ]; then 517 | logMsgToConfigSysLog "ERROR" "ERROR: File logs did not make to Loggly in time. Please check network and firewall settings and retry." 518 | exit 1 519 | fi 520 | done 521 | 522 | if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then 523 | logMsgToConfigSysLog "INFO" "INFO: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." 524 | checkIfLogsAreParsedInLoggly 525 | fi 526 | } 527 | 528 | #verifying if the logs are being parsed or not 529 | checkIfLogsAreParsedInLoggly() { 530 | fileInitialLogCount=0 531 | TAG_PARSER= 532 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 533 | for i in "${array[@]}"; do 534 | TAG_PARSER="$TAG_PARSER%20tag%3A$i " 535 | done 536 | 537 | queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS$TAG_PARSER&from=-15m&until=now&size=1" 538 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 539 | searchAndFetch fileInitialLogCount "$queryUrl" 540 | if [ "$fileInitialLogCount" -gt 0 ]; then 541 | logMsgToConfigSysLog "INFO" "INFO: File logs successfully parsed in Loggly!" 542 | else 543 | logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" 544 | fi 545 | } 546 | 547 | #checks if the conf file exist. Name of conf file is constructed using the file alias name provided 548 | checkIfConfFileExist() { 549 | if [[ ! -f "$FILE_SYSLOG_CONFFILE" ]]; then 550 | if [ $(sudo crontab -l 2>/dev/null | grep "file-monitoring-cron-$FILE_ALIAS.sh" | wc -l) -eq 1 ]; then 551 | logMsgToConfigSysLog "ERROR" "ERROR: Cron is running to refresh configuration. Please try again after sometime." 552 | exit 1 553 | else 554 | logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." 555 | exit 1 556 | fi 557 | fi 558 | } 559 | 560 | #remove 21.conf file 561 | remove21ConfFile() { 562 | echo "INFO: Deleting the loggly syslog conf file $FILE_SYSLOG_CONFFILE." 563 | if [ -f "$FILE_SYSLOG_CONFFILE" ]; then 564 | sudo rm -rf "$FILE_SYSLOG_CONFFILE" 565 | deleteFileFromCrontab 566 | if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then 567 | echo "INFO: Removed all the modified files." 568 | fi 569 | else 570 | logMsgToConfigSysLog "WARN" "WARN: $FILE_SYSLOG_CONFFILE file was not found." 571 | fi 572 | } 573 | 574 | deleteFileFromCrontab() { 575 | if [ -f "/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ]; then 576 | 577 | logMsgToConfigSysLog "INFO" "INFO: Deleting sync Cron." 578 | 579 | #delete cron 580 | sudo crontab -l | grep -v "file-monitoring-cron-$FILE_ALIAS.sh" | crontab - 581 | 582 | #delete cron script 583 | sudo rm -f /root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh 584 | 585 | fi 586 | 587 | } 588 | 589 | removeStatFile() { 590 | sudo rm -f $RSYSLOG_DIR/stat-*$FILE_ALIAS 591 | } 592 | 593 | #display usage syntax 594 | usage() { 595 | cat </dev/null; then 115 | logMsgToConfigSysLog "INFO" "INFO: Nginx is present as a service." 116 | else 117 | logMsgToConfigSysLog "ERROR" "ERROR: Nginx is not configured as a service" 118 | exit 1 119 | fi 120 | 121 | #get the version of nginx installed 122 | getNginxVersion 123 | 124 | #set all the required nginx variables by this script 125 | setNginxVariables 126 | } 127 | 128 | #sets nginx variables which will be used across various functions 129 | setNginxVariables() { 130 | LOGGLY_NGINX_LOG_HOME=/var/log/$SERVICE 131 | } 132 | 133 | #gets the version of nginx installed on the unix box 134 | getNginxVersion() { 135 | NGINX_VERSION=$(nginx -v 2>&1) 136 | NGINX_VERSION=${NGINX_VERSION#*/} 137 | APP_TAG="\"nginx-version\":\"$NGINX_VERSION\"" 138 | logMsgToConfigSysLog "INFO" "INFO: nginx version: $NGINX_VERSION" 139 | } 140 | 141 | checkLogFileSize() { 142 | accessFileSize=$(wc -c "$1" | cut -f 1 -d ' ') 143 | errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') 144 | fileSize=$((accessFileSize + errorFileSize)) 145 | if [ $fileSize -ge 102400000 ]; then 146 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 147 | while true; do 148 | read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn 149 | case $yn in 150 | [Yy]*) 151 | logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration." 152 | break 153 | ;; 154 | [Nn]*) 155 | logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Discontinuing with nginx Loggly configuration." 156 | exit 1 157 | break 158 | ;; 159 | *) echo "Please answer yes or no." ;; 160 | esac 161 | done 162 | else 163 | logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." 164 | logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration." 165 | fi 166 | elif [ $fileSize -eq 0 ]; then 167 | logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from nginx there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." 168 | exit 1 169 | fi 170 | } 171 | 172 | write21NginxConfFile() { 173 | #Create nginx syslog config file if it doesn't exist 174 | echo "INFO: Checking if nginx sysconf file $NGINX_SYSLOG_CONFFILE exist." 175 | if [ -f "$NGINX_SYSLOG_CONFFILE" ]; then 176 | logMsgToConfigSysLog "WARN" "WARN: nginx syslog file $NGINX_SYSLOG_CONFFILE already exist." 177 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 178 | while true; do 179 | read -p "Do you wish to override $NGINX_SYSLOG_CONFFILE? (yes/no)" yn 180 | case $yn in 181 | [Yy]*) 182 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP" 183 | sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP 184 | write21NginxFileContents 185 | break 186 | ;; 187 | [Nn]*) break ;; 188 | *) echo "Please answer yes or no." ;; 189 | esac 190 | done 191 | else 192 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP" 193 | sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP 194 | write21NginxFileContents 195 | fi 196 | else 197 | write21NginxFileContents 198 | fi 199 | } 200 | 201 | addTagsInConfiguration() { 202 | #split tags by comman(,) 203 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 204 | for i in "${array[@]}"; do 205 | TAG="$TAG tag=\\\"$i\\\" " 206 | done 207 | } 208 | #function to write the contents of nginx syslog config file 209 | write21NginxFileContents() { 210 | logMsgToConfigSysLog "INFO" "INFO: Creating file $NGINX_SYSLOG_CONFFILE" 211 | sudo touch $NGINX_SYSLOG_CONFFILE 212 | sudo chmod o+w $NGINX_SYSLOG_CONFFILE 213 | 214 | commonContent=" 215 | \$ModLoad imfile 216 | \$InputFilePollInterval 10 217 | \$WorkDirectory $RSYSLOG_DIR 218 | " 219 | if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then 220 | commonContent+="\$PrivDropToGroup adm 221 | " 222 | fi 223 | 224 | imfileStr+=$commonContent" 225 | 226 | \$ActionSendStreamDriver gtls 227 | \$ActionSendStreamDriverMode 1 228 | \$ActionSendStreamDriverAuthMode x509/name 229 | \$ActionSendStreamDriverPermittedPeer *.loggly.com 230 | 231 | #RsyslogGnuTLS 232 | \$DefaultNetstreamDriverCAFile $CA_FILE_PATH 233 | 234 | # nginx access file: 235 | \$InputFileName $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE 236 | \$InputFileTag nginx-access: 237 | \$InputFileStateFile stat-nginx-access 238 | \$InputFileSeverity info 239 | \$InputFilePersistStateInterval 20000 240 | \$InputRunFileMonitor 241 | 242 | #nginx Error file: 243 | \$InputFileName $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE 244 | \$InputFileTag nginx-error: 245 | \$InputFileStateFile stat-nginx-error 246 | \$InputFileSeverity error 247 | \$InputFilePersistStateInterval 20000 248 | \$InputRunFileMonitor 249 | 250 | #Add a tag for nginx events 251 | \$template LogglyFormatNginx,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" 252 | 253 | if \$programname == 'nginx-access' then @@logs-01.loggly.com:6514;LogglyFormatNginx 254 | if \$programname == 'nginx-access' then ~ 255 | if \$programname == 'nginx-error' then @@logs-01.loggly.com:6514;LogglyFormatNginx 256 | if \$programname == 'nginx-error' then ~ 257 | " 258 | 259 | imfileStrNonTls=$commonContent" 260 | # nginx access file: 261 | \$InputFileName $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE 262 | \$InputFileTag nginx-access: 263 | \$InputFileStateFile stat-nginx-access 264 | \$InputFileSeverity info 265 | \$InputFilePersistStateInterval 20000 266 | \$InputRunFileMonitor 267 | 268 | #nginx Error file: 269 | \$InputFileName $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE 270 | \$InputFileTag nginx-error: 271 | \$InputFileStateFile stat-nginx-error 272 | \$InputFileSeverity error 273 | \$InputFilePersistStateInterval 20000 274 | \$InputRunFileMonitor 275 | 276 | #Add a tag for nginx events 277 | \$template LogglyFormatNginx,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" 278 | 279 | if \$programname == 'nginx-access' then @@logs-01.loggly.com:514;LogglyFormatNginx 280 | if \$programname == 'nginx-access' then ~ 281 | if \$programname == 'nginx-error' then @@logs-01.loggly.com:514;LogglyFormatNginx 282 | if \$programname == 'nginx-error' then ~ 283 | " 284 | 285 | if [ $TLS_SENDING == "false" ]; then 286 | imfileStr=$imfileStrNonTls 287 | fi 288 | 289 | #change the nginx-21 file to variable from above and also take the directory of the nginx log file. 290 | sudo cat <>$NGINX_SYSLOG_CONFFILE 291 | $imfileStr 292 | EOIPFW 293 | 294 | restartRsyslog 295 | } 296 | 297 | #checks if the nginx logs made to loggly 298 | checkIfNginxLogsMadeToLoggly() { 299 | counter=1 300 | maxCounter=10 301 | 302 | nginxInitialLogCount=0 303 | nginxLatestLogCount=0 304 | 305 | TAGS= 306 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 307 | for i in "${array[@]}"; do 308 | if [ "$TAGS" == "" ]; then 309 | TAGS="tag%3A$i" 310 | else 311 | TAGS="$TAGS%20tag%3A$i" 312 | fi 313 | done 314 | 315 | queryParam="$TAGS&from=-15m&until=now&size=1" 316 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 317 | logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" 318 | 319 | logMsgToConfigSysLog "INFO" "INFO: Getting initial nginx log count." 320 | #get the initial count of nginx logs for past 15 minutes 321 | searchAndFetch nginxInitialLogCount "$queryUrl" 322 | 323 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the nginx logs made it to Loggly." 324 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 325 | #get the final count of nginx logs for past 15 minutes 326 | searchAndFetch nginxLatestLogCount "$queryUrl" 327 | let counter=$counter+1 328 | 329 | while [ "$nginxLatestLogCount" -le "$nginxInitialLogCount" ]; do 330 | echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." 331 | sleep 30 332 | echo "INFO: Done waiting. Verifying again." 333 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 334 | searchAndFetch nginxLatestLogCount "$queryUrl" 335 | let counter=$counter+1 336 | if [ "$counter" -gt "$maxCounter" ]; then 337 | logMsgToConfigSysLog "ERROR" "ERROR: Nginx logs did not make to Loggly in time. Please check network and firewall settings and retry." 338 | exit 1 339 | fi 340 | done 341 | 342 | if [ "$nginxLatestLogCount" -gt "$nginxInitialLogCount" ]; then 343 | logMsgToConfigSysLog "INFO" "INFO: Nginx logs successfully transferred to Loggly! You are now sending Nginx logs to Loggly." 344 | checkIfLogsAreParsedInLoggly 345 | fi 346 | } 347 | 348 | #verifying if the logs are being parsed or not 349 | checkIfLogsAreParsedInLoggly() { 350 | nginxInitialLogCount=0 351 | TAG_PARSER= 352 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 353 | for i in "${array[@]}"; do 354 | TAG_PARSER="$TAG_PARSER%20tag%3A$i " 355 | done 356 | queryParam="logtype%3Anginx$TAG_PARSER&from=-15m&until=now&size=1" 357 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 358 | searchAndFetch nginxInitialLogCount "$queryUrl" 359 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the Nginx logs are parsed in Loggly." 360 | if [ "$nginxInitialLogCount" -gt 0 ]; then 361 | logMsgToConfigSysLog "INFO" "INFO: Nginx logs successfully parsed in Loggly!" 362 | else 363 | logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" 364 | fi 365 | } 366 | 367 | #remove 21nginx.conf file 368 | remove21NginxConfFile() { 369 | echo "INFO: Deleting the loggly nginx syslog conf file." 370 | if [ -f "$NGINX_SYSLOG_CONFFILE" ]; then 371 | sudo rm -rf "$NGINX_SYSLOG_CONFFILE" 372 | fi 373 | echo "INFO: Removed all the modified files." 374 | restartRsyslog 375 | } 376 | 377 | #display usage syntax 378 | usage() { 379 | cat </dev/null; then 115 | logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_S3_ALIAS is already taken. Please try with another one." 116 | exit 1 117 | fi 118 | } 119 | 120 | #check if s3cmd utility is installed and configured 121 | checkIfS3cmdInstalledAndConfigured() { 122 | if hash s3cmd 2>/dev/null; then 123 | checkIfS3cmdConfigured 124 | else 125 | logMsgToConfigSysLog "INFO" "INFO: s3cmd is not present on your system. Setting it up on your system" 126 | downloadS3cmd 127 | configureS3cmd 128 | fi 129 | } 130 | 131 | #check if s3cmd utility is configured 132 | checkIfS3cmdConfigured() { 133 | var=$(sudo s3cmd ls 2>/dev/null) 134 | if [ "$var" != "" ]; then 135 | if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then 136 | logMsgToConfigSysLog "INFO" "INFO: s3cmd is already configured on your system" 137 | else 138 | logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" 139 | fi 140 | else 141 | if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then 142 | logMsgToConfigSysLog "INFO" "INFO: s3cmd is not configured on your system. Trying to configure." 143 | configureS3cmd 144 | else 145 | logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" 146 | exit 1 147 | fi 148 | fi 149 | } 150 | 151 | #download and install s3cmd 152 | downloadS3cmd() { 153 | #checking if the Linux is yum based or apt-get based 154 | YUM_BASED=$(command -v yum) 155 | APT_GET_BASED=$(command -v apt-get) 156 | 157 | if [ "$YUM_BASED" != "" ]; then 158 | sudo yum install s3cmd || { 159 | logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST. Please ensure you have EPEL installed." 160 | exit 1 161 | } 162 | elif [ "$APT_GET_BASED" != "" ]; then 163 | sudo apt-get install s3cmd || { 164 | logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." 165 | exit 1 166 | } 167 | else 168 | logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." 169 | exit 1 170 | fi 171 | } 172 | 173 | #configure s3cmd 174 | configureS3cmd() { 175 | sudo s3cmd --configure 176 | IS_S3CMD_CONFIGURED_BY_SCRIPT="true" 177 | #check if s3cmd configured successfully now 178 | checkIfS3cmdConfigured 179 | } 180 | 181 | #check if s3bucket is valid 182 | checkIfValidS3Bucket() { 183 | #check if valid Bucket name 184 | if [[ $LOGGLY_S3_BUCKET_NAME != s3://* ]]; then 185 | logMsgToConfigSysLog "Error" "Error: Invalid s3 Bucket name. Bucket name should start with 's3://'" 186 | exit 1 187 | fi 188 | 189 | if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then 190 | logMsgToConfigSysLog "INFO" "INFO: Check if valid S3 Bucket name." 191 | BUCKET_INFO=$(sudo s3cmd ls -r $LOGGLY_S3_BUCKET_NAME 2>&1) 192 | case $BUCKET_INFO in 193 | ERROR*) 194 | #logging actual error message returned by s3cmd 195 | logMsgToConfigSysLog "ERROR" "$BUCKET_INFO" 196 | exit 1 197 | ;; 198 | "") 199 | logMsgToConfigSysLog "ERROR" "ERROR: No files found in the S3 Bucket $LOGGLY_S3_BUCKET_NAME." 200 | exit 1 201 | ;; 202 | *) 203 | logMsgToConfigSysLog "INFO" "INFO: '$LOGGLY_S3_BUCKET_NAME' is a valid Bucket and accessible." 204 | ;; 205 | esac 206 | fi 207 | } 208 | 209 | createTempDir() { 210 | TEMP_DIR=/tmp/s3monitoring/$LOGGLY_S3_ALIAS 211 | if [ -d "$TEMP_DIR" ]; then 212 | if [ "$(ls -A $TEMP_DIR)" ]; then 213 | logMsgToConfigSysLog "WARN" "WARN: There are some files/folders already present in $TEMP_DIR. If you continue, the files currently inside the $TEMP_DIR will also be configured to send logs to loggly." 214 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 215 | while true; do 216 | read -p "Would you like to continue now anyway? (yes/no)" yn 217 | case $yn in 218 | [Yy]*) 219 | break 220 | ;; 221 | [Nn]*) 222 | logMsgToConfigSysLog "INFO" "INFO: Discontinuing with s3 file monitoring configuration." 223 | exit 1 224 | break 225 | ;; 226 | *) echo "Please answer yes or no." ;; 227 | esac 228 | done 229 | fi 230 | fi 231 | else 232 | if [ -d "/tmp/s3monitoring" ]; then 233 | mkdir /tmp/s3monitoring/$LOGGLY_S3_ALIAS 234 | else 235 | mkdir /tmp/s3monitoring 236 | mkdir /tmp/s3monitoring/$LOGGLY_S3_ALIAS 237 | fi 238 | fi 239 | } 240 | 241 | downloadS3Bucket() { 242 | if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then 243 | #Files are downloaded in nested directory 244 | cd $TEMP_DIR 245 | echo "Downloading files, may take some time..." 246 | sudo s3cmd get -r -f $LOGGLY_S3_BUCKET_NAME >/dev/null 2>&1 247 | if [ $? -ne 0 ]; then 248 | logMsgToConfigSysLog "ERROR" "ERROR: Error downloading files recursively from $LOGGLY_S3_BUCKET_NAME" 249 | exit 1 250 | fi 251 | fi 252 | } 253 | 254 | invokeS3FileMonitoring() { 255 | dir=/tmp/s3monitoring/$LOGGLY_S3_ALIAS 256 | LOGGLY_FILE_TO_MONITOR=$dir 257 | 258 | configureDirectoryFileMonitoring 259 | 260 | IS_ANY_FILE_CONFIGURED="true" 261 | 262 | if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then 263 | restartRsyslog 264 | fi 265 | } 266 | 267 | installCronToSyncS3BucketPeriodically() { 268 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 269 | while true; do 270 | read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn 271 | case $yn in 272 | [Yy]*) 273 | doS3CronInstallation 274 | break 275 | ;; 276 | [Nn]*) 277 | logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." 278 | break 279 | ;; 280 | *) echo "Please answer yes or no." ;; 281 | esac 282 | done 283 | else 284 | doS3CronInstallation 285 | fi 286 | } 287 | 288 | doS3CronInstallation() { 289 | #copying .s3cfg file to /root so that it can be used by crontab for sync 290 | if ! sudo test -f "/root/.s3cfg"; then 291 | sudo cp $HOME/.s3cfg /root 292 | fi 293 | 294 | CRON_FILE="/tmp/s3monitoring/cron_$LOGGLY_S3_ALIAS" 295 | CRON_SYNC_PATH="/tmp/s3monitoring/$LOGGLY_S3_ALIAS" 296 | 297 | #checking if the provided s3 path if of directory or file 298 | IS_DIR="true" 299 | BUCKET_URL_LAST_VALUE=$(echo ${LOGGLY_S3_BUCKET_NAME##*/}) 300 | 301 | if [ "$BUCKET_URL_LAST_VALUE" != "" ]; then 302 | for fle in $(find $CRON_SYNC_PATH -name $BUCKET_URL_LAST_VALUE); do 303 | if [ -f $fle ]; then 304 | IS_DIR="false" 305 | break 306 | fi 307 | done 308 | fi 309 | 310 | #adding file name to the sync folder as the bucket path is 311 | #provided upto a file 312 | if [ "$IS_DIR" == "false" ]; then 313 | CRON_SYNC_PATH="$CRON_SYNC_PATH/$BUCKET_URL_LAST_VALUE" 314 | fi 315 | 316 | logMsgToConfigSysLog "INFO" "INFO: Creating a Cron job to sync $LOGGLY_S3_BUCKET_NAME files to $CRON_SYNC_PATH in every five minutes." 317 | 318 | #setting up cron job 319 | CRON_JOB_TO_SYNC_S3_BUCKET="*/5 * * * * s3cmd sync $LOGGLY_S3_BUCKET_NAME --preserve $CRON_SYNC_PATH" 320 | 321 | EXISTING_CRONS=$(sudo crontab -l 2>&1) 322 | case $EXISTING_CRONS in 323 | no*) ;; 324 | 325 | *) 326 | echo "$EXISTING_CRONS" >>$CRON_FILE 327 | ;; 328 | esac 329 | 330 | echo "$CRON_JOB_TO_SYNC_S3_BUCKET" >>$CRON_FILE 331 | sudo crontab $CRON_FILE 332 | sudo rm -fr $CRON_FILE 333 | } 334 | 335 | deleteTempDir() { 336 | if [ -d "$TEMP_DIR" ]; then 337 | sudo rm -fr $TEMP_DIR 338 | fi 339 | } 340 | 341 | checkIfS3LogsMadeToLoggly() { 342 | counter=1 343 | maxCounter=10 344 | 345 | fileInitialLogCount=0 346 | fileLatestLogCount=0 347 | queryParam="syslog.appName%3A%2A$LOGGLY_S3_ALIAS&from=-5m&until=now&size=1" 348 | 349 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 350 | logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" 351 | 352 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the logs made it to Loggly." 353 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 354 | #get the final count of file logs for past 5 minutes 355 | searchAndFetch fileLatestLogCount "$queryUrl" 356 | let counter=$counter+1 357 | 358 | while [ "$fileLatestLogCount" -le "$fileInitialLogCount" ]; do 359 | echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." 360 | sleep 30 361 | echo "INFO: Done waiting. Verifying again." 362 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 363 | searchAndFetch fileLatestLogCount "$queryUrl" 364 | let counter=$counter+1 365 | if [ "$counter" -gt "$maxCounter" ]; then 366 | logMsgToConfigSysLog "ERROR" "ERROR: S3 logs did not make to Loggly in time. Please check network and firewall settings and retry." 367 | exit 1 368 | fi 369 | done 370 | 371 | if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then 372 | if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then 373 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_S3_BUCKET_NAME logs to Loggly." 374 | 375 | fi 376 | fi 377 | } 378 | 379 | checkIfS3AliasExist() { 380 | if ! ls $RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf &>/dev/null; then 381 | #logMsgToConfigSysLog "INFO" "INFO: $LOGGLY_S3_ALIAS found." 382 | #else 383 | logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_S3_ALIAS does not exist. Please provide the correct s3 alias." 384 | exit 1 385 | fi 386 | } 387 | 388 | removeS3FileMonitoring() { 389 | FILES=$RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf 390 | for f in $FILES; do 391 | aliasName=${f##*/} 392 | aliasName=${aliasName%.*} 393 | aliasName=${aliasName#21-filemonitoring-} 394 | 395 | LOGGLY_FILE_TO_MONITOR_ALIAS=$aliasName 396 | constructFileVariables 397 | remove21ConfFile 398 | done 399 | echo "INFO: Removed all the modified files." 400 | restartRsyslog 401 | } 402 | 403 | deleteS3CronFromCrontab() { 404 | logMsgToConfigSysLog "INFO" "INFO: Deleting sync Cron." 405 | sudo crontab -l | grep -v "/tmp/s3monitoring/$LOGGLY_S3_ALIAS" | crontab - 406 | } 407 | 408 | #display usage syntax 409 | usage() { 410 | cat </dev/null; then 285 | logMsgToConfigSysLog "INFO" "INFO: Tomcat is not configured as a service." 286 | if [ ! -f "$LOGGLY_CATALINA_HOME/bin/startup.sh" ]; then 287 | logMsgToConfigSysLog "WARN" "WARN: Unable to find bin/startup.sh file within $LOGGLY_CATALINA_HOME." 288 | eval $1="false" 289 | fi 290 | fi 291 | fi 292 | } 293 | 294 | #sets tomcat variables which will be used across various functions 295 | setTomcatVariables() { 296 | #set value for catalina conf home path, logging.properties path and 297 | #logging.properties.loggly.bk path 298 | LOGGLY_CATALINA_CONF_HOME=$LOGGLY_CATALINA_HOME/conf 299 | LOGGLY_CATALINA_PROPFILE=$LOGGLY_CATALINA_CONF_HOME/logging.properties 300 | LOGGLY_CATALINA_BACKUP_PROPFILE=$LOGGLY_CATALINA_PROPFILE.loggly.bk 301 | 302 | LOGGLY_CATALINA_LOG_HOME=/var/log/$SERVICE 303 | 304 | #if tomcat is not installed as service, then tomcat logs will be created at would be $CATALINA_HOME/log 305 | if [ ! -f "$LOGGLY_CATALINA_LOG_HOME" ]; then 306 | LOGGLY_CATALINA_LOG_HOME=$LOGGLY_CATALINA_HOME/logs 307 | fi 308 | 309 | #default path for catalina.jar 310 | CATALINA_JAR_PATH=$LOGGLY_CATALINA_HOME/lib/catalina.jar 311 | } 312 | 313 | #get the version of tomcat 314 | getTomcatVersion() { 315 | #check if the identified CATALINA_HOME has the catalina.jar 316 | if [ ! -f "$CATALINA_JAR_PATH" ]; then 317 | #if not, search it throughout the system. If we find no entries or more than 318 | #1 entry, then we cannot determine the version of the tomcat 319 | logMsgToConfigSysLog "INFO" "INFO: Could not find catalina.jar in $LOGGLY_CATALINA_HOME/lib. Searching at other locations, this may take some time." 320 | if [ $(sudo find / -name catalina.jar | grep $SERVICE | wc -l) = 1 ]; then 321 | CATALINA_JAR_PATH=$(sudo find / -name catalina.jar | grep $SERVICE) 322 | logMsgToConfigSysLog "INFO" "INFO: Found catalina.jar at $CATALINA_JAR_PATH" 323 | else 324 | logMsgToConfigSysLog "WARNING" "WARNING: Unable to determine the correct version of tomcat 6. Assuming its >= to 6.0.33." 325 | TOMCAT_VERSION=6.0.33.0 326 | fi 327 | fi 328 | 329 | #get the tomcat version number 330 | if [ -f "$CATALINA_JAR_PATH" ]; then 331 | TOMCAT_VERSION=$(sudo java -cp $CATALINA_JAR_PATH org.apache.catalina.util.ServerInfo | grep "Server number") 332 | TOMCAT_VERSION=${TOMCAT_VERSION#*: } 333 | TOMCAT_VERSION=$TOMCAT_VERSION | tr -d ' ' 334 | APP_TAG="\"tomcat-version\":\"$TOMCAT_VERSION\"" 335 | fi 336 | } 337 | 338 | #checks if the tomcat version is supported by this script, currently the script 339 | #only supports tomcat 6 and tomcat 7 340 | checkIfSupportedTomcatVersion() { 341 | tomcatMajorVersion=${TOMCAT_VERSION%%.*} 342 | if [[ ($tomcatMajorVersion -ne 6) && ($tomcatMajorVersion -ne 7) && ($tomcatMajorVersion -ne 8) ]]; then 343 | logMsgToConfigSysLog "ERROR" "ERROR: This script only supports Tomcat version 6, 7 or 8." 344 | exit 1 345 | fi 346 | } 347 | 348 | #checks if the tomcat is already configured with log4j. If yes, then exit 349 | checkIfTomcatConfiguredWithLog4J() { 350 | echo "INFO: Checking if tomcat is configured with log4j logger." 351 | #default path for log4j files 352 | LOG4J_FILE_PATH=$LOGGLY_CATALINA_HOME/lib/log4j* 353 | #check if the log4j files are present, if yes, then exit 354 | if ls $LOG4J_FILE_PATH >/dev/null 2>&1; then 355 | logMsgToConfigSysLog "ERROR" "ERROR: Script does not support log4j logger. Please see $LOGGLY_COM_URL/docs/java-log4j" 356 | exit 1 357 | else 358 | #if not found in the default path, check in the path where catalina.jar is found 359 | libDirName=$(dirname ${CATALINA_JAR_PATH}) 360 | LOG4J_FILE_PATH=$libDirName/log4j* 361 | if ls $LOG4J_FILE_PATH >/dev/null 2>&1; then 362 | logMsgToConfigSysLog "ERROR" "ERROR: Script does not support log4j logger. Please see $LOGGLY_COM_URL/docs/java-log4j" 363 | exit 1 364 | fi 365 | fi 366 | logMsgToConfigSysLog "INFO" "INFO: Tomcat seems not to be configured with log4j logger." 367 | } 368 | 369 | canTomcatBeRestarted() { 370 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 371 | while true; do 372 | read -p "Tomcat needs to be restarted during configuration. Do you wish to continue? (yes/no)" yn 373 | case $yn in 374 | [Yy]*) 375 | break 376 | ;; 377 | [Nn]*) 378 | logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." 379 | exit 1 380 | break 381 | ;; 382 | *) echo "Please answer yes or no." ;; 383 | esac 384 | done 385 | else 386 | logMsgToConfigSysLog "WARN" "WARN:Tomcat needs to be restarted during configuration." 387 | fi 388 | } 389 | #backup the logging.properties file in the CATALINA_HOME folder 390 | backupLoggingPropertiesFile() { 391 | logMsgToConfigSysLog "INFO" "INFO: Tomcat logging properties file: $LOGGLY_CATALINA_PROPFILE" 392 | # backup the logging properties file just in case it need to reverted. 393 | echo "INFO: Going to back up the properties file: $LOGGLY_CATALINA_PROPFILE to $LOGGLY_CATALINA_BACKUP_PROPFILE" 394 | if [ ! -f $LOGGLY_CATALINA_PROPFILE ]; then 395 | logMsgToConfigSysLog "ERROR" "ERROR: logging.properties file not found!. Looked at location $LOGGLY_CATALINA_PROPFILE" 396 | exit 1 397 | else 398 | # dont take a backup of logging properties file if it is already there 399 | if [ ! -f $LOGGLY_CATALINA_BACKUP_PROPFILE ]; then 400 | sudo cp -f $LOGGLY_CATALINA_PROPFILE $LOGGLY_CATALINA_BACKUP_PROPFILE 401 | fi 402 | fi 403 | 404 | } 405 | 406 | #update logging.properties file to enable log rotation. If the version of tomcat 407 | #is less than 6.0.33, then log rotation cannot be enabled 408 | updateLoggingPropertiesFile() { 409 | #check if tomcat version is less than 6.0.33.0, if yes, throw a warning 410 | if [ $(compareVersions $TOMCAT_VERSION $MIN_TOMCAT_VERSION 4) -lt 0 ]; then 411 | logMsgToConfigSysLog "WARNING" "WARNING: Tomcat version is less than 6.0.33. Log rotation cannot be disabled for version <6.0.33; only catalina.out log will be monitored." 412 | fi 413 | 414 | #Log rotation is not supported on version below 6.0.33.0, logging.properties should not be modified 415 | #in such case. If version is above 6.0.33.0, then do the following 416 | if [ $(compareVersions $TOMCAT_VERSION $MIN_TOMCAT_VERSION 4) -ge 0 ]; then 417 | #removing the end . from logging.properties variable 1catalina.org.apache.juli.FileHandler.prefix = catalina. 418 | if grep -Fq "prefix = catalina." $LOGGLY_CATALINA_PROPFILE; then 419 | sudo sed -i "s/prefix = catalina./prefix = catalina/g" $LOGGLY_CATALINA_PROPFILE 420 | fi 421 | if grep -Fq "prefix = localhost." $LOGGLY_CATALINA_PROPFILE; then 422 | sudo sed -i "s/prefix = localhost./prefix = localhost/g" $LOGGLY_CATALINA_PROPFILE 423 | fi 424 | if grep -Fq "prefix = manager." $LOGGLY_CATALINA_PROPFILE; then 425 | sudo sed -i "s/prefix = manager./prefix = manager/g" $LOGGLY_CATALINA_PROPFILE 426 | fi 427 | if grep -Fq "prefix = host-manager." $LOGGLY_CATALINA_PROPFILE; then 428 | sudo sed -i "s/prefix = host-manager./prefix = host-manager/g" $LOGGLY_CATALINA_PROPFILE 429 | fi 430 | 431 | #Check if the rotatable property is present in logging.properties 432 | if grep -Fq "rotatable" $LOGGLY_CATALINA_PROPFILE; then 433 | #If present, set all the values to false 434 | sed -i -e 's/rotatable = true/rotatable = false/g' $LOGGLY_CATALINA_PROPFILE 435 | fi 436 | 437 | if [ $(fgrep "rotatable = false" "$LOGGLY_CATALINA_PROPFILE" | wc -l) -lt 4 ]; then 438 | #If rotatable property present or not, add the following lines to disable rotation in any case 439 | sudo cat <>$LOGGLY_CATALINA_PROPFILE 440 | 1catalina.org.apache.juli.FileHandler.rotatable = false 441 | 2localhost.org.apache.juli.FileHandler.rotatable = false 442 | 3manager.org.apache.juli.FileHandler.rotatable = false 443 | 4host-manager.org.apache.juli.FileHandler.rotatable = false 444 | EOIPFW 445 | fi 446 | fi 447 | } 448 | 449 | #add renameOnRotate to true in the Valve element to stop access logs 450 | #log rotation 451 | updateServerXML() { 452 | 453 | if ! grep -q 'renameOnRotate="true"' "$LOGGLY_CATALINA_HOME/conf/server.xml"; then 454 | 455 | #Creating backup of server.xml to server.xml.bk 456 | logMsgToConfigSysLog "INFO" "INFO: Creating backup of server.xml to server.xml.bk" 457 | sudo cp $LOGGLY_CATALINA_HOME/conf/server.xml $LOGGLY_CATALINA_HOME/conf/server.xml.bk 458 | if grep -q '"localhost_access_log."' "$LOGGLY_CATALINA_HOME/conf/server.xml"; then 459 | sed -i 's/"localhost_access_log."/"localhost_access_log"/g' $LOGGLY_CATALINA_HOME/conf/server.xml 460 | fi 461 | sed -i 's/"localhost_access_log"/"localhost_access_log"\ renameOnRotate="true"/g' $LOGGLY_CATALINA_HOME/conf/server.xml 462 | logMsgToConfigSysLog "INFO" "INFO: Disabled log rotation for localhost_access_log file in server.xml" 463 | fi 464 | } 465 | addTagsInConfiguration() { 466 | #split tags by comman(,) 467 | IFS=, read -a array <<<"$LOGGLY_FILE_TAG" 468 | for i in "${array[@]}"; do 469 | TAG="$TAG tag=\\\"$i\\\" " 470 | done 471 | } 472 | 473 | write21TomcatConfFile() { 474 | #Create tomcat syslog config file if it doesn't exist 475 | echo "INFO: Checking if tomcat sysconf file $TOMCAT_SYSLOG_CONFFILE exist." 476 | if [ -f "$TOMCAT_SYSLOG_CONFFILE" ]; then 477 | logMsgToConfigSysLog "WARN" "WARN: Tomcat syslog file $TOMCAT_SYSLOG_CONFFILE already exist." 478 | if [ "$SUPPRESS_PROMPT" == "false" ]; then 479 | while true; do 480 | read -p "Do you wish to override $TOMCAT_SYSLOG_CONFFILE? (yes/no)" yn 481 | case $yn in 482 | [Yy]*) 483 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP" 484 | sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP 485 | write21TomcatFileContents 486 | break 487 | ;; 488 | [Nn]*) break ;; 489 | *) echo "Please answer yes or no." ;; 490 | esac 491 | done 492 | else 493 | logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP" 494 | sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP 495 | write21TomcatFileContents 496 | fi 497 | else 498 | write21TomcatFileContents 499 | fi 500 | } 501 | 502 | #function to write the contents of tomcat syslog config file 503 | write21TomcatFileContents() { 504 | logMsgToConfigSysLog "INFO" "INFO: Creating file $TOMCAT_SYSLOG_CONFFILE" 505 | sudo touch $TOMCAT_SYSLOG_CONFFILE 506 | sudo chmod o+w $TOMCAT_SYSLOG_CONFFILE 507 | 508 | commonContent=" 509 | \$ModLoad imfile 510 | \$WorkDirectory $RSYSLOG_DIR 511 | " 512 | if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then 513 | commonContent+="\$PrivDropToGroup adm 514 | " 515 | fi 516 | 517 | imfileStr=$commonContent" 518 | 519 | \$ActionSendStreamDriver gtls 520 | \$ActionSendStreamDriverMode 1 521 | \$ActionSendStreamDriverAuthMode x509/name 522 | \$ActionSendStreamDriverPermittedPeer *.loggly.com 523 | 524 | #RsyslogGnuTLS 525 | \$DefaultNetstreamDriverCAFile $CA_FILE_PATH 526 | 527 | #parameterized token here....... 528 | #Add a tag for tomcat events 529 | \$template LogglyFormatTomcat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" 530 | 531 | # catalina.out 532 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/catalina.out 533 | \$InputFileTag catalina-out 534 | \$InputFileStateFile stat-catalina-out 535 | \$InputFileSeverity info 536 | \$InputFilePersistStateInterval 20000 537 | \$InputRunFileMonitor 538 | if \$programname == 'catalina-out' then @@logs-01.loggly.com:6514;LogglyFormatTomcat 539 | if \$programname == 'catalina-out' then ~ 540 | 541 | # initd.log 542 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/initd.log 543 | \$InputFileTag initd 544 | \$InputFileStateFile stat-initd 545 | \$InputFileSeverity info 546 | \$InputFilePersistStateInterval 20000 547 | \$InputRunFileMonitor 548 | if \$programname == 'initd' then @@logs-01.loggly.com:6514;LogglyFormatTomcat 549 | if \$programname == 'initd' then ~ 550 | " 551 | 552 | #if log rotation is enabled i.e. tomcat version is greater than or equal to 553 | #6.0.33.0, then add the following lines to tomcat syslog conf file 554 | if [ $(compareVersions $TOMCAT_VERSION $MIN_TOMCAT_VERSION 4) -ge 0 ]; then 555 | imfileStr+=$commonContent" 556 | # catalina.log 557 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/catalina.log 558 | \$InputFileTag catalina-log 559 | \$InputFileStateFile stat-catalina-log 560 | \$InputFileSeverity info 561 | \$InputFilePersistStateInterval 20000 562 | \$InputRunFileMonitor 563 | if \$programname == 'catalina-log' then @@logs-01.loggly.com:6514;LogglyFormatTomcat 564 | if \$programname == 'catalina-log' then ~ 565 | 566 | # host-manager.log 567 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/host-manager.log 568 | \$InputFileTag host-manager 569 | \$InputFileStateFile stat-host-manager 570 | \$InputFileSeverity info 571 | \$InputFilePersistStateInterval 20000 572 | \$InputRunFileMonitor 573 | if \$programname == 'host-manager' then @@logs-01.loggly.com:6514;LogglyFormatTomcat 574 | if \$programname == 'host-manager' then ~ 575 | 576 | # localhost.log 577 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/localhost.log 578 | \$InputFileTag localhost-log 579 | \$InputFileStateFile stat-localhost-log 580 | \$InputFileSeverity info 581 | \$InputFilePersistStateInterval 20000 582 | \$InputRunFileMonitor 583 | if \$programname == 'localhost-log' then @@logs-01.loggly.com:6514;LogglyFormatTomcat 584 | if \$programname == 'localhost-log' then ~ 585 | 586 | # manager.log 587 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/manager.log 588 | \$InputFileTag manager 589 | \$InputFileStateFile stat-manager 590 | \$InputFileSeverity info 591 | \$InputFilePersistStateInterval 20000 592 | \$InputRunFileMonitor 593 | if \$programname == 'manager' then @@logs-01.loggly.com:6514;LogglyFormatTomcat 594 | if \$programname == 'manager' then ~ 595 | 596 | # localhost_access_log.txt 597 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/localhost_access_log.txt 598 | \$InputFileTag tomcat-access 599 | \$InputFileStateFile stat-tomcat-access 600 | \$InputFileSeverity info 601 | \$InputFilePersistStateInterval 20000 602 | \$InputRunFileMonitor 603 | if \$programname == 'tomcat-access' then @@logs-01.loggly.com:6514;LogglyFormatTomcat 604 | if \$programname == 'tomcat-access' then ~ 605 | " 606 | fi 607 | 608 | imfileStrNonTls=$commonContent" 609 | 610 | #parameterized token here....... 611 | #Add a tag for tomcat events 612 | \$template LogglyFormatTomcat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" 613 | 614 | # catalina.out 615 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/catalina.out 616 | \$InputFileTag catalina-out 617 | \$InputFileStateFile stat-catalina-out 618 | \$InputFileSeverity info 619 | \$InputFilePersistStateInterval 20000 620 | \$InputRunFileMonitor 621 | if \$programname == 'catalina-out' then @@logs-01.loggly.com:514;LogglyFormatTomcat 622 | if \$programname == 'catalina-out' then ~ 623 | 624 | # initd.log 625 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/initd.log 626 | \$InputFileTag initd 627 | \$InputFileStateFile stat-initd 628 | \$InputFileSeverity info 629 | \$InputFilePersistStateInterval 20000 630 | \$InputRunFileMonitor 631 | if \$programname == 'initd' then @@logs-01.loggly.com:514;LogglyFormatTomcat 632 | if \$programname == 'initd' then ~ 633 | " 634 | 635 | #if log rotation is enabled i.e. tomcat version is greater than or equal to 636 | #6.0.33.0, then add the following lines to tomcat syslog conf file 637 | if [ $(compareVersions $TOMCAT_VERSION $MIN_TOMCAT_VERSION 4) -ge 0 ]; then 638 | imfileStrNonTls+=$commonContent" 639 | # catalina.log 640 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/catalina.log 641 | \$InputFileTag catalina-log 642 | \$InputFileStateFile stat-catalina-log 643 | \$InputFileSeverity info 644 | \$InputFilePersistStateInterval 20000 645 | \$InputRunFileMonitor 646 | if \$programname == 'catalina-log' then @@logs-01.loggly.com:514;LogglyFormatTomcat 647 | if \$programname == 'catalina-log' then ~ 648 | 649 | # host-manager.log 650 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/host-manager.log 651 | \$InputFileTag host-manager 652 | \$InputFileStateFile stat-host-manager 653 | \$InputFileSeverity info 654 | \$InputFilePersistStateInterval 20000 655 | \$InputRunFileMonitor 656 | if \$programname == 'host-manager' then @@logs-01.loggly.com:514;LogglyFormatTomcat 657 | if \$programname == 'host-manager' then ~ 658 | 659 | # localhost.log 660 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/localhost.log 661 | \$InputFileTag localhost-log 662 | \$InputFileStateFile stat-localhost-log 663 | \$InputFileSeverity info 664 | \$InputFilePersistStateInterval 20000 665 | \$InputRunFileMonitor 666 | if \$programname == 'localhost-log' then @@logs-01.loggly.com:514;LogglyFormatTomcat 667 | if \$programname == 'localhost-log' then ~ 668 | 669 | # manager.log 670 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/manager.log 671 | \$InputFileTag manager 672 | \$InputFileStateFile stat-manager 673 | \$InputFileSeverity info 674 | \$InputFilePersistStateInterval 20000 675 | \$InputRunFileMonitor 676 | if \$programname == 'manager' then @@logs-01.loggly.com:514;LogglyFormatTomcat 677 | if \$programname == 'manager' then ~ 678 | 679 | # localhost_access_log.txt 680 | \$InputFileName $LOGGLY_CATALINA_LOG_HOME/localhost_access_log.txt 681 | \$InputFileTag tomcat-access 682 | \$InputFileStateFile stat-tomcat-access 683 | \$InputFileSeverity info 684 | \$InputFilePersistStateInterval 20000 685 | \$InputRunFileMonitor 686 | if \$programname == 'tomcat-access' then @@logs-01.loggly.com:514;LogglyFormatTomcat 687 | if \$programname == 'tomcat-access' then ~ 688 | " 689 | fi 690 | 691 | if [ $TLS_SENDING == "false" ]; 692 | then 693 | imfileStr=$imfileStrNonTls 694 | fi 695 | 696 | #change the tomcat-21 file to variable from above and also take the directory of the tomcat log file. 697 | sudo cat <>$TOMCAT_SYSLOG_CONFFILE 698 | $imfileStr 699 | EOIPFW 700 | 701 | #restart the syslog service. 702 | restartRsyslog 703 | } 704 | 705 | #checks if the tomcat logs made to loggly 706 | checkIfTomcatLogsMadeToLoggly() { 707 | counter=1 708 | maxCounter=10 709 | 710 | tomcatInitialLogCount=0 711 | tomcatLatestLogCount=0 712 | queryParam="tag%3Atomcat&from=-15m&until=now&size=1" 713 | 714 | queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" 715 | logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" 716 | 717 | logMsgToConfigSysLog "INFO" "INFO: Getting initial tomcat log count." 718 | #get the initial count of tomcat logs for past 15 minutes 719 | searchAndFetch tomcatInitialLogCount "$queryUrl" 720 | 721 | logMsgToConfigSysLog "INFO" "INFO: Tomcat needs to be restarted to complete the configuration and verification." 722 | # restart the tomcat service. 723 | restartTomcat 724 | 725 | logMsgToConfigSysLog "INFO" "INFO: Verifying if the tomcat logs made it to Loggly." 726 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 727 | #get the final count of tomcat logs for past 15 minutes 728 | searchAndFetch tomcatLatestLogCount "$queryUrl" 729 | let counter=$counter+1 730 | 731 | while [ "$tomcatLatestLogCount" -le "$tomcatInitialLogCount" ]; do 732 | echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." 733 | sleep 30 734 | echo "INFO: Done waiting. Verifying again." 735 | logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." 736 | searchAndFetch tomcatLatestLogCount "$queryUrl" 737 | let counter=$counter+1 738 | if [ "$counter" -gt "$maxCounter" ]; then 739 | logMsgToConfigSysLog "ERROR" "ERROR: Tomcat logs did not make to Loggly in time. Please check network and firewall settings and retry." 740 | exit 1 741 | fi 742 | done 743 | 744 | if [ "$tomcatLatestLogCount" -gt "$tomcatInitialLogCount" ]; then 745 | logMsgToConfigSysLog "SUCCESS" "SUCCESS: Tomcat logs successfully transferred to Loggly! You are now sending Tomcat logs to Loggly." 746 | exit 0 747 | fi 748 | } 749 | 750 | #restore original loggly properties file from backup 751 | restoreLogglyPropertiesFile() { 752 | echo "INFO: Reverting the logging.properties file." 753 | if [ -f "$LOGGLY_CATALINA_BACKUP_PROPFILE" ]; then 754 | sudo rm -fr $LOGGLY_CATALINA_PROPFILE 755 | sudo cp -f $LOGGLY_CATALINA_BACKUP_PROPFILE $LOGGLY_CATALINA_PROPFILE 756 | sudo rm -fr $LOGGLY_CATALINA_BACKUP_PROPFILE 757 | fi 758 | 759 | logMsgToConfigSysLog "INFO" "INFO: Tomcat needs to be restarted to rollback the configuration." 760 | restartTomcat 761 | } 762 | 763 | restoreServerXML() { 764 | if [ -f "$LOGGLY_CATALINA_HOME/conf/server.xml.bk" ]; then 765 | logMsgToConfigSysLog "INFO" "INFO: Restoring server.xml file from backup" 766 | sudo rm -rf $LOGGLY_CATALINA_HOME/conf/server.xml 767 | sudo cp $LOGGLY_CATALINA_HOME/conf/server.xml.bk $LOGGLY_CATALINA_HOME/conf/server.xml 768 | sudo rm -rf $LOGGLY_CATALINA_HOME/conf/server.xml.bk 769 | fi 770 | } 771 | 772 | #remove 21tomcat.conf file 773 | remove21TomcatConfFile() { 774 | echo "INFO: Deleting the loggly tomcat syslog conf file." 775 | if [ -f "$TOMCAT_SYSLOG_CONFFILE" ]; then 776 | sudo rm -rf "$TOMCAT_SYSLOG_CONFFILE" 777 | fi 778 | 779 | #restart rsyslog 780 | restartRsyslog 781 | } 782 | 783 | #restart tomcat 784 | restartTomcat() { 785 | #sudo service tomcat restart or home/bin/start.sh 786 | if [ $(ps -ef | grep -v grep | grep "$SERVICE" | wc -l) -gt 0 ]; then 787 | logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running." 788 | if [ -f /etc/init.d/$SERVICE ]; then 789 | logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running as service." 790 | logMsgToConfigSysLog "INFO" "INFO: Restarting the tomcat service." 791 | sudo service $SERVICE restart 792 | if [ $? -ne 0 ]; then 793 | logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not restart gracefully. Log rotation may not be disabled. Please restart tomcat manually." 794 | fi 795 | else 796 | logMsgToConfigSysLog "INFO" "INFO: $SERVICE is not running as service." 797 | logMsgToConfigSysLog "INFO" "INFO: Shutting down tomcat." 798 | sudo $LOGGLY_CATALINA_HOME/bin/shutdown.sh 799 | if [ $? -ne 0 ]; then 800 | logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not shut down gracefully." 801 | else 802 | logMsgToConfigSysLog "INFO" "INFO: Done shutting down tomcat." 803 | fi 804 | 805 | logMsgToConfigSysLog "INFO" "INFO: Starting up tomcat." 806 | sudo $LOGGLY_CATALINA_HOME/bin/startup.sh 807 | if [ $? -ne 0 ]; then 808 | logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not start up down gracefully." 809 | else 810 | logMsgToConfigSysLog "INFO" "INFO: Tomcat is up and running." 811 | fi 812 | fi 813 | fi 814 | } 815 | 816 | #display usage syntax 817 | usage() { 818 | cat <Linux Configuration Script 13 | 2. Apache Installation Script 14 | 3. File Monitoring Configuration Script 15 | 4. Tomcat Configuration Script 16 | 5. S3 Bucket Configuration Script 17 | 6. Nginx Script 18 | 7. Mac Script 19 | -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | # Publishes the scripts to Loggly's S3 bucket where they are publically hosted. 2 | # For Loggly's internal use only. Requires keys to publish. 3 | 4 | copy_to_aws() { 5 | aws s3 cp "$1" s3://loggly-install/install/ --acl public-read 6 | } 7 | 8 | declare -a files=("Linux Script/configure-linux.sh" 9 | "Modular Scripts/File Monitoring/configure-file-monitoring.sh" 10 | "Modular Scripts/Apache2/configure-apache.sh" 11 | "Modular Scripts/Nginx/configure-nginx.sh" 12 | "Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh" 13 | "Modular Scripts/Tomcat/configure-tomcat.sh" 14 | "Mac Script/configure-mac.sh" 15 | "AWSscripts/SQS3script.py") 16 | 17 | for file in "${files[@]}";do 18 | copy_to_aws "$file" 19 | done 20 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | import unittest, platform 2 | 3 | confsys = __import__("configure-syslog") 4 | 5 | # "mock" out the LOGGER object 6 | class Amorphous(object): 7 | def __getattr__(self, name): 8 | return lambda x: x 9 | 10 | confsys.LOGGER = Amorphous() 11 | 12 | class TestConfigureSyslog(unittest.TestCase): 13 | 14 | def test_get_syslog_version(self): 15 | 16 | r = confsys.get_syslog_version() 17 | t = r[0] 18 | 19 | self.assertTrue(len(r) > 0) 20 | self.assertTrue(isinstance(t, tuple) ) 21 | 22 | self.assertTrue(t[0] in ['rsyslog', 'syslog-ng'] ) 23 | 24 | # versions should only be of xx.yy form; no more, no less 25 | self.assertEquals(2, len(t[1].split('.')) ) 26 | 27 | def skip_test_new_old_equality(self): 28 | 29 | new_get = confsys.new_get_syslog_version() 30 | 31 | distro_name, version, version_id = platform.linux_distribution() 32 | distro_id = confsys.get_os_id(distro_name) 33 | old_get = confsys.get_syslog_version(distro_id) 34 | 35 | self.assertEquals( new_get, old_get) 36 | 37 | if __name__ == "__main__": 38 | unittest.main() 39 | --------------------------------------------------------------------------------