├── aws-event-triggering ├── example_file.txt ├── s3-lambda-function │ ├── requirements.txt │ └── s3-lambda-function.py └── s3-notification-triggers.sh ├── README.md └── github-api └── list-users.sh /aws-event-triggering/example_file.txt: -------------------------------------------------------------------------------- 1 | my name is abhishek 2 | -------------------------------------------------------------------------------- /aws-event-triggering/s3-lambda-function/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3==1.17.95 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS S3 Event Triggering 2 | 3 | You can also watch the entire implementation using the below YouTube video 4 | 5 | https://youtu.be/qLZ-s7wvK7w 6 | 7 | **NOTE**: REPLACE YOUR AWS ACCOUNT ID IN THE LAMBDA FUNCTION CODE. 8 | 9 | AWS S3 Event triggering is a very popular project used by top companies in the Industry. 10 | 11 | Here are some examples of top companies that use S3 event triggering: 12 | 13 | **Netflix**: Netflix use S3 event triggering to automatically process video files uploaded to Amazon S3, enabling seamless content ingestion and processing. 14 | 15 | **Airbnb**: This lodging and homestays aggregator use S3 event triggering to automatically process and analyze data stored in Amazon S3, such as guest reviews and booking information. 16 | 17 | **Expedia**: They use S3 event triggering to automatically process and analyze data stored in Amazon S3, such as travel bookings, user profiles, and pricing information, to power their personalized travel recommendations and search features. 18 | 19 | 20 | ![Screenshot 2023-04-14 at 7 06 46 PM](https://user-images.githubusercontent.com/43399466/232058778-a7299e9b-9892-471c-a05d-14d773b5b333.png) 21 | -------------------------------------------------------------------------------- /aws-event-triggering/s3-lambda-function/s3-lambda-function.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | 4 | def lambda_handler(event, context): 5 | # Extract relevant information from the S3 event trigger 6 | bucket_name = event['Records'][0]['s3']['bucket']['name'] 7 | object_key = event['Records'][0]['s3']['object']['key'] 8 | 9 | # Perform desired operations with the uploaded file 10 | print(f"File '{object_key}' was uploaded to bucket '{bucket_name}'") 11 | 12 | # Example: Send a notification via SNS 13 | sns_client = boto3.client('sns') 14 | topic_arn = 'arn:aws:sns:us-east-1::s3-lambda-sns' 15 | sns_client.publish( 16 | TopicArn=topic_arn, 17 | Subject='S3 Object Created', 18 | Message=f"File '{object_key}' was uploaded to bucket '{bucket_name}'" 19 | ) 20 | 21 | # Example: Trigger another Lambda function 22 | # lambda_client = boto3.client('lambda') 23 | # target_function_name = 'my-another-lambda-function' 24 | # lambda_client.invoke( 25 | # FunctionName=target_function_name, 26 | # InvocationType='Event', 27 | # Payload=json.dumps({'bucket_name': bucket_name, 'object_key': object_key}) 28 | # ) 29 | 30 | return { 31 | 'statusCode': 200, 32 | 'body': json.dumps('Lambda function executed successfully') 33 | } 34 | 35 | -------------------------------------------------------------------------------- /github-api/list-users.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # GitHub API URL 4 | API_URL="https://api.github.com" 5 | 6 | # GitHub username and personal access token 7 | USERNAME=$username 8 | TOKEN=$token 9 | 10 | # User and Repository information 11 | REPO_OWNER=$1 12 | REPO_NAME=$2 13 | 14 | # Function to make a GET request to the GitHub API 15 | function github_api_get { 16 | local endpoint="$1" 17 | local url="${API_URL}/${endpoint}" 18 | 19 | # Send a GET request to the GitHub API with authentication 20 | curl -s -u "${USERNAME}:${TOKEN}" "$url" 21 | } 22 | 23 | # Function to list users with read access to the repository 24 | function list_users_with_read_access { 25 | local endpoint="repos/${REPO_OWNER}/${REPO_NAME}/collaborators" 26 | 27 | # Fetch the list of collaborators on the repository 28 | collaborators="$(github_api_get "$endpoint" | jq -r '.[] | select(.permissions.pull == true) | .login')" 29 | 30 | # Display the list of collaborators with read access 31 | if [[ -z "$collaborators" ]]; then 32 | echo "No users with read access found for ${REPO_OWNER}/${REPO_NAME}." 33 | else 34 | echo "Users with read access to ${REPO_OWNER}/${REPO_NAME}:" 35 | echo "$collaborators" 36 | fi 37 | } 38 | 39 | # Main script 40 | 41 | echo "Listing users with read access to ${REPO_OWNER}/${REPO_NAME}..." 42 | list_users_with_read_access 43 | -------------------------------------------------------------------------------- /aws-event-triggering/s3-notification-triggers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | # Store the AWS account ID in a variable 6 | aws_account_id=$(aws sts get-caller-identity --query 'Account' --output text) 7 | 8 | # Print the AWS account ID from the variable 9 | echo "AWS Account ID: $aws_account_id" 10 | 11 | # Set AWS region and bucket name 12 | aws_region="us-east-1" 13 | bucket_name="abhishek-ultimate-bucket" 14 | lambda_func_name="s3-lambda-function" 15 | role_name="s3-lambda-sns" 16 | email_address="zyz@gmail.com" 17 | 18 | # Create IAM Role for the project 19 | role_response=$(aws iam create-role --role-name s3-lambda-sns --assume-role-policy-document '{ 20 | "Version": "2012-10-17", 21 | "Statement": [{ 22 | "Action": "sts:AssumeRole", 23 | "Effect": "Allow", 24 | "Principal": { 25 | "Service": [ 26 | "lambda.amazonaws.com", 27 | "s3.amazonaws.com", 28 | "sns.amazonaws.com" 29 | ] 30 | } 31 | }] 32 | }') 33 | 34 | # Extract the role ARN from the JSON response and store it in a variable 35 | role_arn=$(echo "$role_response" | jq -r '.Role.Arn') 36 | 37 | # Print the role ARN 38 | echo "Role ARN: $role_arn" 39 | 40 | # Attach Permissions to the Role 41 | aws iam attach-role-policy --role-name $role_name --policy-arn arn:aws:iam::aws:policy/AWSLambda_FullAccess 42 | aws iam attach-role-policy --role-name $role_name --policy-arn arn:aws:iam::aws:policy/AmazonSNSFullAccess 43 | 44 | # Create the S3 bucket and capture the output in a variable 45 | bucket_output=$(aws s3api create-bucket --bucket "$bucket_name" --region "$aws_region") 46 | 47 | # Print the output from the variable 48 | echo "Bucket creation output: $bucket_output" 49 | 50 | # Upload a file to the bucket 51 | aws s3 cp ./example_file.txt s3://"$bucket_name"/example_file.txt 52 | 53 | # Create a Zip file to upload Lambda Function 54 | zip -r s3-lambda-function.zip ./s3-lambda-function 55 | 56 | sleep 5 57 | # Create a Lambda function 58 | aws lambda create-function \ 59 | --region "$aws_region" \ 60 | --function-name $lambda_func_name \ 61 | --runtime "python3.8" \ 62 | --handler "s3-lambda-function/s3-lambda-function.lambda_handler" \ 63 | --memory-size 128 \ 64 | --timeout 30 \ 65 | --role "arn:aws:iam::$aws_account_id:role/$role_name" \ 66 | --zip-file "fileb://./s3-lambda-function.zip" 67 | 68 | # Add Permissions to S3 Bucket to invoke Lambda 69 | aws lambda add-permission \ 70 | --function-name "$lambda_func_name" \ 71 | --statement-id "s3-lambda-sns" \ 72 | --action "lambda:InvokeFunction" \ 73 | --principal s3.amazonaws.com \ 74 | --source-arn "arn:aws:s3:::$bucket_name" 75 | 76 | # Create an S3 event trigger for the Lambda function 77 | LambdaFunctionArn="arn:aws:lambda:us-east-1:$aws_account_id:function:s3-lambda-function" 78 | aws s3api put-bucket-notification-configuration \ 79 | --region "$aws_region" \ 80 | --bucket "$bucket_name" \ 81 | --notification-configuration '{ 82 | "LambdaFunctionConfigurations": [{ 83 | "LambdaFunctionArn": "'"$LambdaFunctionArn"'", 84 | "Events": ["s3:ObjectCreated:*"] 85 | }] 86 | }' 87 | 88 | # Create an SNS topic and save the topic ARN to a variable 89 | topic_arn=$(aws sns create-topic --name s3-lambda-sns --output json | jq -r '.TopicArn') 90 | 91 | # Print the TopicArn 92 | echo "SNS Topic ARN: $topic_arn" 93 | 94 | # Trigger SNS Topic using Lambda Function 95 | 96 | 97 | # Add SNS publish permission to the Lambda Function 98 | aws sns subscribe \ 99 | --topic-arn "$topic_arn" \ 100 | --protocol email \ 101 | --notification-endpoint "$email_address" 102 | 103 | # Publish SNS 104 | aws sns publish \ 105 | --topic-arn "$topic_arn" \ 106 | --subject "A new object created in s3 bucket" \ 107 | --message "Hello from Abhishek.Veeramalla YouTube channel, Learn DevOps Zero to Hero for Free" 108 | 109 | 110 | --------------------------------------------------------------------------------