├── AWS Gateway ├── AWS Gateway-1 │ └── readme.md ├── AWS Gateway-2- Troubleshooting │ └── readme.md └── readme.md ├── AWS Projects └── vehicle-number-identifier │ ├── README.md │ ├── package-lock.json │ ├── package.json │ ├── public │ ├── favicon.ico │ ├── index.html │ ├── logo192.png │ ├── logo512.png │ ├── manifest.json │ └── robots.txt │ ├── server │ ├── lambda_funtion.py │ ├── publisher.py │ ├── requrements.txt │ └── sample_responses │ │ ├── audi1.json │ │ └── audi2.json │ └── src │ ├── App.css │ ├── App.js │ ├── App.test.js │ ├── MultipleImageUpload.js │ ├── SampleImages │ ├── audi1.png │ ├── audi2.png │ ├── audi3.png │ ├── audi4.png │ └── benz11.png │ ├── index.css │ ├── index.js │ ├── logo.svg │ ├── reportWebVitals.js │ └── setupTests.js ├── AWS S3 └── AWS S3- slides.pdf ├── AWS-Gateway-Lambda-Authorizer ├── Authorizer response.md └── Readme.md ├── AWS-Standarad-vs-FIFO ├── Readme.md └── pythonCode │ ├── index.py │ └── requrement.txt ├── AWSEventBridge └── Readme.md ├── AWSTextract ├── ReadingAllText.py ├── Readme.md ├── lambda_function.py └── requrement.txt ├── AWS_IAM ├── Presentation slides- What is IAM.pdf ├── Readme.md ├── abac_policy.json ├── cross-account-policy.json ├── cross-account-role.json ├── identity-based-policy.json ├── permission-boundary.json ├── s3-resource-based-policy.json └── trust-policy.json ├── AWS_SSM_ParameterStore ├── Readme.md └── lambda_handler.py ├── DynamoDB-Python-Demo ├── index.py ├── readme.md └── requrements.txt ├── ECR_Auto_Scanning_Python └── index.py ├── README.md ├── SumoLogic_DotNetCore_End-To_End ├── .gitignore ├── Controllers │ └── MyController.cs ├── DotNetCoreSumoLogic.csproj ├── Program.cs ├── Properties │ └── launchSettings.json ├── Startup.cs ├── WeatherForecast.cs ├── appsettings.Development.json └── appsettings.json ├── SumoLogic_Serilog ├── .gitignore ├── Controllers │ └── WeatherForecastController.cs ├── DotnetSerilog.csproj ├── Program.cs ├── Properties │ └── launchSettings.json ├── Readme.md ├── WeatherForecast.cs ├── appsettings.Development.json └── appsettings.json ├── SumoLogic_Slack ├── DotNetCoreSumoLogic │ ├── .gitignore │ ├── Controllers │ │ └── MyController.cs │ ├── DotNetCoreSumoLogic.csproj │ ├── Program.cs │ ├── Properties │ │ └── launchSettings.json │ ├── Startup.cs │ ├── WeatherForecast.cs │ ├── appsettings.Development.json │ └── appsettings.json └── Readme.md ├── TextractAsync ├── README.md ├── lambda │ ├── document_analyzer.py │ ├── lambda_function.py │ ├── processor │ │ └── document_processor.py │ ├── requrement.txt │ └── responses │ │ └── test_block.json └── package-deploy.sh └── test.js /AWS Gateway/AWS Gateway-1/readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Demo Video: https://www.youtube.com/watch?v=c3J5uvdfSfE&list=PLruLATXv4pNz2RPn5X6iMmqyvv69NprJS&index=1 3 | 4 | ## Lambda: test-hellow-lambda ( GET USER end point) 5 | 6 | ```python 7 | import json 8 | 9 | def lambda_handler(event, context): 10 | 11 | print(f'event: {event}'); 12 | 13 | # this users object can be fetched from database etc. 14 | users = [ 15 | {"id": 1, "name": "john dove"}, 16 | {"id": 2, "name": "michel wats"} 17 | ] 18 | 19 | return { 20 | 'statusCode': 200, 21 | 'body': json.dumps(users) 22 | } 23 | 24 | ``` 25 | 26 | ## Lambda: demo-post-users ( POST USER end point) 27 | 28 | ```python 29 | import json 30 | 31 | def lambda_handler(event, context): 32 | 33 | print(f'event: {event}') 34 | print(f'Posting user details') 35 | # save the information database 36 | 37 | return { 38 | 'statusCode': 200, 39 | 'body': json.dumps('User added') 40 | } 41 | 42 | ``` 43 | -------------------------------------------------------------------------------- /AWS Gateway/AWS Gateway-2- Troubleshooting/readme.md: -------------------------------------------------------------------------------- 1 | ## Lambda: test-hellow-lambda ( GET USER end point) 2 | 3 | ```python 4 | import json 5 | 6 | def lambda_handler(event, context): 7 | 8 | print(f'event: {event}'); 9 | 10 | # this users object can be fetched from database etc. 11 | users = [ 12 | {"id": 1, "name": "john dove"}, 13 | {"id": 2, "name": "michel wats"} 14 | ] 15 | 16 | return { 17 | 'statusCode': 200, 18 | 'body': users 19 | } 20 | 21 | ``` 22 | 23 | ## Lambda: demo-post-users ( POST USER end point) 24 | 25 | ```python 26 | import json 27 | 28 | def lambda_handler(event, context): 29 | 30 | print(f'event: {event}') 31 | print(f'Posting user details') 32 | # save the information database 33 | 34 | user_name = event['name'] 35 | 36 | 37 | return { 38 | 'statusCode': 200, 39 | 'body': { "message": f'{user_name} added'} 40 | } 41 | 42 | 43 | ``` -------------------------------------------------------------------------------- /AWS Gateway/readme.md: -------------------------------------------------------------------------------- 1 | ## AWS API Gateway Full Course 2 | This video series is about full course of AWS API Gateway. This covers basic concept to advance concept with the demos 3 | Covers securing API gateway, troubleshooting, scaling 4 | Once you went through these video, you should have good understanding of API Gateway. 5 | 6 | Demo video: https://www.youtube.com/watch?v=c3J5uvdfSfE&list=PLruLATXv4pNz2RPn5X6iMmqyvv69NprJS 7 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/README.md: -------------------------------------------------------------------------------- 1 | # VEHICLE NUMBER PLATE IDENTIFIER 2 | This project is a sample AWS project for identify vehichle registation number by providing image. This project use below AWS services. 3 | - API Gateway (SOCKET API) 4 | - Lambda 5 | - Rekognition 6 | - S3 7 | 8 | This app containes two projects which are front end and back end. 9 | - Front-End => React 10 | - Back-End => Python 11 | 12 | ## Application 13 | ![image](https://github.com/CodeSam621/Demo/assets/101965114/d594039c-cb71-467b-a8f4-7a5ebb355725) 14 | 15 | ## STEP BY STEP instructions 16 | Please check this video for step by ste guidance https://youtu.be/EZrXBnca4KY 17 | 18 | ## Backend code ( Lambda code) 19 | You can find the source code for the lambda(`demo-vehicle-sendVehicleInfo`) [here](https://github.com/CodeSam621/Demo/tree/main/AWS%20Projects/vehicle-number-identifier/server) 20 | ## Front-End 21 | 22 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 23 | 24 | ### Available Scripts 25 | 26 | In the project directory, you can run: 27 | 28 | ### `npm start` 29 | 30 | Runs the app in the development mode.\ 31 | Open [http://localhost:3000](http://localhost:3000) to view it in your browser. 32 | 33 | ## Back-End 34 | 35 | Use below 2 python files to create lambda function 36 | 37 | ## Sample images 38 | You can find sample image files [here](https://github.com/CodeSam621/Demo/tree/main/AWS%20Projects/vehicle-number-identifier/src/SampleImages) 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vehicle-number-identifier", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@aws-sdk/client-s3": "^3.535.0", 7 | "@aws-sdk/lib-storage": "^3.535.0", 8 | "@testing-library/jest-dom": "^5.17.0", 9 | "@testing-library/react": "^13.4.0", 10 | "@testing-library/user-event": "^13.5.0", 11 | "react": "^18.2.0", 12 | "react-dom": "^18.2.0", 13 | "react-scripts": "5.0.1", 14 | "web-vitals": "^2.1.4" 15 | }, 16 | "scripts": { 17 | "start": "react-scripts start", 18 | "build": "react-scripts build", 19 | "test": "react-scripts test", 20 | "eject": "react-scripts eject" 21 | }, 22 | "eslintConfig": { 23 | "extends": [ 24 | "react-app", 25 | "react-app/jest" 26 | ] 27 | }, 28 | "browserslist": { 29 | "production": [ 30 | ">0.2%", 31 | "not dead", 32 | "not op_mini all" 33 | ], 34 | "development": [ 35 | "last 1 chrome version", 36 | "last 1 firefox version", 37 | "last 1 safari version" 38 | ] 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/public/favicon.ico -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/public/logo192.png -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/public/logo512.png -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/server/lambda_funtion.py: -------------------------------------------------------------------------------- 1 | import boto3, json 2 | from publisher import publish 3 | 4 | s3_client = boto3.client('s3', "ap-southeast-2") 5 | 6 | def lambda_handler(event, context): 7 | 8 | try: 9 | print(f'event: {event}') 10 | message_body = json.loads(event["body"]) 11 | message = message_body['message'] 12 | bucket = message["bucket"] 13 | key = message["key"] 14 | connection_id = event["requestContext"].get("connectionId") 15 | 16 | print(f'bucket: {bucket}, key: {key}, connection_id: {connection_id} ') 17 | 18 | number_plate = extract_number_plate( bucket, key) 19 | 20 | if(connection_id != None): 21 | publish(connection_id, number_plate) 22 | 23 | return { 24 | 'statusCode': 200, 25 | 'body': json.dumps({"success": True, "message": number_plate}) 26 | } 27 | except Exception as error: 28 | print(f'Error occurrred. {error}') 29 | return { 30 | 'statusCode': 500, 31 | 'body': json.dumps({"success": False, "message": "Failed"}) 32 | } 33 | 34 | def extract_number_plate(bucket, key): 35 | list = get_detected_text_list(bucket, key) 36 | print('List', list) 37 | if(list != None): 38 | return list[0] 39 | return "Unable to find number" 40 | 41 | 42 | def get_detected_text_list( bucket, key): 43 | response = {} 44 | client = boto3.client('rekognition', "us-east-1") 45 | response = client.detect_text(Image= { 46 | 'S3Object': { 47 | 'Bucket': bucket, 48 | 'Name': key 49 | } 50 | }) 51 | print(f'Response: {response}') 52 | 53 | list_of_detected_object = response["TextDetections"] 54 | 55 | list_of_detected_text = [] 56 | if(list_of_detected_object != None): 57 | for item in list_of_detected_object: 58 | validated_text = validate_detected_text(item["DetectedText"]) 59 | if(validated_text != None): 60 | list_of_detected_text.append(validated_text) 61 | 62 | return list_of_detected_text 63 | 64 | 65 | ########### 66 | ## Custom logic to validate the number plate. 67 | ########### 68 | def validate_detected_text(text): 69 | ## validate length 70 | text = text.strip() 71 | if(len(text) != 8): 72 | return None 73 | return text 74 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/server/publisher.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | from botocore.config import Config 4 | 5 | callbackUrl = "https://xxxxxx.execute-api..amazonaws.com/xxxxxxxx" # please replace with your socket API end point 6 | 7 | def publish(connection_id, data): 8 | config = Config( 9 | region_name = 'us-east-1', 10 | signature_version = 'v4' 11 | ) 12 | 13 | try: 14 | client = boto3.client("apigatewaymanagementapi", endpoint_url=callbackUrl, config=config ) 15 | client.post_to_connection(Data = json.dumps({ "success": True, "message": data}), ConnectionId = connection_id) 16 | 17 | except Exception as error: 18 | raise Exception(error) 19 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/server/requrements.txt: -------------------------------------------------------------------------------- 1 | boto3 -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/server/sample_responses/audi1.json: -------------------------------------------------------------------------------- 1 | { 2 | "TextDetections": [ 3 | { 4 | "DetectedText": "17", 5 | "Type": "LINE", 6 | "Id": 0, 7 | "Confidence": 99.45081329345703, 8 | "Geometry": { 9 | "BoundingBox": { 10 | "Width": 0.018278080970048904, 11 | "Height": 0.01953125, 12 | "Left": 0.07823018729686737, 13 | "Top": 0.0498046875 14 | }, 15 | "Polygon": [ 16 | { "X": 0.07823018729686737, "Y": 0.0498046875 }, 17 | { "X": 0.09650827199220657, "Y": 0.0498046875 }, 18 | { "X": 0.09650827199220657, "Y": 0.0693359375 }, 19 | { "X": 0.07823018729686737, "Y": 0.0693359375 } 20 | ] 21 | } 22 | }, 23 | { 24 | "DetectedText": "DE", 25 | "Type": "LINE", 26 | "Id": 1, 27 | "Confidence": 63.244564056396484, 28 | "Geometry": { 29 | "BoundingBox": { 30 | "Width": 0.013914615847170353, 31 | "Height": 0.030600864440202713, 32 | "Left": 0.11286245286464691, 33 | "Top": 0.1185239851474762 34 | }, 35 | "Polygon": [ 36 | { "X": 0.11416646093130112, "Y": 0.1491248458623886 }, 37 | { "X": 0.11286245286464691, "Y": 0.11951480060815811 }, 38 | { "X": 0.12547306716442108, "Y": 0.1185239851474762 }, 39 | { "X": 0.1267770677804947, "Y": 0.1481340378522873 } 40 | ] 41 | } 42 | }, 43 | { 44 | "DetectedText": "VXI6 JYJ", 45 | "Type": "LINE", 46 | "Id": 2, 47 | "Confidence": 93.56248474121094, 48 | "Geometry": { 49 | "BoundingBox": { 50 | "Width": 0.08049864321947098, 51 | "Height": 0.07460111379623413, 52 | "Left": 0.1649831235408783, 53 | "Top": 0.5359601974487305 54 | }, 55 | "Polygon": [ 56 | { "X": 0.17564396560192108, "Y": 0.5359601974487305 }, 57 | { "X": 0.24548177421092987, "Y": 0.5811991691589355 }, 58 | { "X": 0.23482094705104828, "Y": 0.6105613112449646 }, 59 | { "X": 0.1649831235408783, "Y": 0.5653222799301147 } 60 | ] 61 | } 62 | }, 63 | { 64 | "DetectedText": "View gallery", 65 | "Type": "LINE", 66 | "Id": 3, 67 | "Confidence": 99.69119262695312, 68 | "Geometry": { 69 | "BoundingBox": { 70 | "Width": 0.12209758162498474, 71 | "Height": 0.0283203125, 72 | "Left": 0.8444473743438721, 73 | "Top": 0.9228515625 74 | }, 75 | "Polygon": [ 76 | { "X": 0.8444473743438721, "Y": 0.9228515625 }, 77 | { "X": 0.9665449261665344, "Y": 0.9228515625 }, 78 | { "X": 0.9665449261665344, "Y": 0.951171875 }, 79 | { "X": 0.8444473743438721, "Y": 0.951171875 } 80 | ] 81 | } 82 | }, 83 | { 84 | "DetectedText": "17", 85 | "Type": "WORD", 86 | "Id": 4, 87 | "ParentId": 0, 88 | "Confidence": 99.45081329345703, 89 | "Geometry": { 90 | "BoundingBox": { 91 | "Width": 0.018278080970048904, 92 | "Height": 0.01953125, 93 | "Left": 0.07823018729686737, 94 | "Top": 0.0498046875 95 | }, 96 | "Polygon": [ 97 | { "X": 0.07823018729686737, "Y": 0.0498046875 }, 98 | { "X": 0.09650827199220657, "Y": 0.0498046875 }, 99 | { "X": 0.09650827199220657, "Y": 0.0693359375 }, 100 | { "X": 0.07823018729686737, "Y": 0.0693359375 } 101 | ] 102 | } 103 | }, 104 | { 105 | "DetectedText": "DE", 106 | "Type": "WORD", 107 | "Id": 5, 108 | "ParentId": 1, 109 | "Confidence": 63.244564056396484, 110 | "Geometry": { 111 | "BoundingBox": { 112 | "Width": 0.013914615847170353, 113 | "Height": 0.030600864440202713, 114 | "Left": 0.11286245286464691, 115 | "Top": 0.1185239851474762 116 | }, 117 | "Polygon": [ 118 | { "X": 0.11416646093130112, "Y": 0.1491248458623886 }, 119 | { "X": 0.11286245286464691, "Y": 0.11951480060815811 }, 120 | { "X": 0.12547306716442108, "Y": 0.1185239851474762 }, 121 | { "X": 0.1267770677804947, "Y": 0.1481340378522873 } 122 | ] 123 | } 124 | }, 125 | { 126 | "DetectedText": "VXI6", 127 | "Type": "WORD", 128 | "Id": 6, 129 | "ParentId": 2, 130 | "Confidence": 89.02195739746094, 131 | "Geometry": { 132 | "BoundingBox": { 133 | "Width": 0.04657042399048805, 134 | "Height": 0.0518157035112381, 135 | "Left": 0.16595123708248138, 136 | "Top": 0.5364179611206055 137 | }, 138 | "Polygon": [ 139 | { "X": 0.17635075747966766, "Y": 0.5364179611206055 }, 140 | { "X": 0.21252165734767914, "Y": 0.5619957447052002 }, 141 | { "X": 0.20212212204933167, "Y": 0.5882337093353271 }, 142 | { "X": 0.16595123708248138, "Y": 0.5626559257507324 } 143 | ] 144 | } 145 | }, 146 | { 147 | "DetectedText": "JYJ", 148 | "Type": "WORD", 149 | "Id": 7, 150 | "ParentId": 2, 151 | "Confidence": 98.1030044555664, 152 | "Geometry": { 153 | "BoundingBox": { 154 | "Width": 0.044805727899074554, 155 | "Height": 0.049596212804317474, 156 | "Left": 0.20007368922233582, 157 | "Top": 0.5596328377723694 158 | }, 159 | "Polygon": [ 160 | { "X": 0.2096484750509262, "Y": 0.5596328377723694 }, 161 | { "X": 0.24487942457199097, "Y": 0.5808089971542358 }, 162 | { "X": 0.23530462384223938, "Y": 0.6092290878295898 }, 163 | { "X": 0.20007368922233582, "Y": 0.5880529880523682 } 164 | ] 165 | } 166 | }, 167 | { 168 | "DetectedText": "View", 169 | "Type": "WORD", 170 | "Id": 8, 171 | "ParentId": 3, 172 | "Confidence": 99.60192108154297, 173 | "Geometry": { 174 | "BoundingBox": { 175 | "Width": 0.046791888773441315, 176 | "Height": 0.0205078125, 177 | "Left": 0.8444473743438721, 178 | "Top": 0.923828125 179 | }, 180 | "Polygon": [ 181 | { "X": 0.8444473743438721, "Y": 0.923828125 }, 182 | { "X": 0.8912392258644104, "Y": 0.923828125 }, 183 | { "X": 0.8912392258644104, "Y": 0.9443359375 }, 184 | { "X": 0.8444473743438721, "Y": 0.9443359375 } 185 | ] 186 | } 187 | }, 188 | { 189 | "DetectedText": "gallery", 190 | "Type": "WORD", 191 | "Id": 9, 192 | "ParentId": 3, 193 | "Confidence": 99.78046417236328, 194 | "Geometry": { 195 | "BoundingBox": { 196 | "Width": 0.07018782943487167, 197 | "Height": 0.0283203125, 198 | "Left": 0.8963571190834045, 199 | "Top": 0.9228515625 200 | }, 201 | "Polygon": [ 202 | { "X": 0.8963571190834045, "Y": 0.9228515625 }, 203 | { "X": 0.9665449261665344, "Y": 0.9228515625 }, 204 | { "X": 0.9665449261665344, "Y": 0.951171875 }, 205 | { "X": 0.8963571190834045, "Y": 0.951171875 } 206 | ] 207 | } 208 | } 209 | ], 210 | "TextModelVersion": "3.0", 211 | "ResponseMetadata": { 212 | "RequestId": "cc38fdee-23cb-48ce-b7bc-b9381af11132", 213 | "HTTPStatusCode": 200, 214 | "HTTPHeaders": { 215 | "x-amzn-requestid": "cc38fdee-23cb-48ce-b7bc-b9381af11132", 216 | "content-type": "application/x-amz-json-1.1", 217 | "content-length": "4144", 218 | "date": "Sat, 16 Mar 2024 12:39:04 GMT" 219 | }, 220 | "RetryAttempts": 0 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/server/sample_responses/audi2.json: -------------------------------------------------------------------------------- 1 | { 2 | "TextDetections": [ 3 | { 4 | "DetectedText": "10", 5 | "Type": "LINE", 6 | "Id": 0, 7 | "Confidence": 99.59686279296875, 8 | "Geometry": { 9 | "BoundingBox": { 10 | "Width": 0.020471451804041862, 11 | "Height": 0.021484375, 12 | "Left": 0.07749906182289124, 13 | "Top": 0.048828125 14 | }, 15 | "Polygon": [ 16 | { "X": 0.07749906182289124, "Y": 0.048828125 }, 17 | { "X": 0.09797051548957825, "Y": 0.048828125 }, 18 | { "X": 0.09797051548957825, "Y": 0.0703125 }, 19 | { "X": 0.07749906182289124, "Y": 0.0703125 } 20 | ] 21 | } 22 | }, 23 | { 24 | "DetectedText": "OU63 RWO", 25 | "Type": "LINE", 26 | "Id": 1, 27 | "Confidence": 93.72862243652344, 28 | "Geometry": { 29 | "BoundingBox": { 30 | "Width": 0.1385752260684967, 31 | "Height": 0.10234533250331879, 32 | "Left": 0.15920910239219666, 33 | "Top": 0.533059298992157 34 | }, 35 | "Polygon": [ 36 | { "X": 0.17063485085964203, "Y": 0.533059298992157 }, 37 | { "X": 0.29778432846069336, "Y": 0.5894030332565308 }, 38 | { "X": 0.2863585650920868, "Y": 0.635404646396637 }, 39 | { "X": 0.15920910239219666, "Y": 0.579060971736908 } 40 | ] 41 | } 42 | }, 43 | { 44 | "DetectedText": "View gallery", 45 | "Type": "LINE", 46 | "Id": 2, 47 | "Confidence": 99.63088989257812, 48 | "Geometry": { 49 | "BoundingBox": { 50 | "Width": 0.12282870709896088, 51 | "Height": 0.0283203125, 52 | "Left": 0.8437162637710571, 53 | "Top": 0.9228515625 54 | }, 55 | "Polygon": [ 56 | { "X": 0.8437162637710571, "Y": 0.9228515625 }, 57 | { "X": 0.9665449261665344, "Y": 0.9228515625 }, 58 | { "X": 0.9665449261665344, "Y": 0.951171875 }, 59 | { "X": 0.8437162637710571, "Y": 0.951171875 } 60 | ] 61 | } 62 | }, 63 | { 64 | "DetectedText": "10", 65 | "Type": "WORD", 66 | "Id": 3, 67 | "ParentId": 0, 68 | "Confidence": 99.59686279296875, 69 | "Geometry": { 70 | "BoundingBox": { 71 | "Width": 0.020471451804041862, 72 | "Height": 0.021484375, 73 | "Left": 0.07749906182289124, 74 | "Top": 0.048828125 75 | }, 76 | "Polygon": [ 77 | { "X": 0.07749906182289124, "Y": 0.048828125 }, 78 | { "X": 0.09797051548957825, "Y": 0.048828125 }, 79 | { "X": 0.09797051548957825, "Y": 0.0703125 }, 80 | { "X": 0.07749906182289124, "Y": 0.0703125 } 81 | ] 82 | } 83 | }, 84 | { 85 | "DetectedText": "OU63", 86 | "Type": "WORD", 87 | "Id": 4, 88 | "ParentId": 1, 89 | "Confidence": 89.15182495117188, 90 | "Geometry": { 91 | "BoundingBox": { 92 | "Width": 0.07713472098112106, 93 | "Height": 0.07152236253023148, 94 | "Left": 0.16086320579051971, 95 | "Top": 0.5334733128547668 96 | }, 97 | "Polygon": [ 98 | { "X": 0.17156927287578583, "Y": 0.5334733128547668 }, 99 | { "X": 0.23799793422222137, "Y": 0.5660675764083862 }, 100 | { "X": 0.22729185223579407, "Y": 0.6049956679344177 }, 101 | { "X": 0.16086320579051971, "Y": 0.5724013447761536 } 102 | ] 103 | } 104 | }, 105 | { 106 | "DetectedText": "RWO", 107 | "Type": "WORD", 108 | "Id": 5, 109 | "ParentId": 1, 110 | "Confidence": 98.305419921875, 111 | "Geometry": { 112 | "BoundingBox": { 113 | "Width": 0.07461166381835938, 114 | "Height": 0.06655184179544449, 115 | "Left": 0.22157889604568481, 116 | "Top": 0.5644218325614929 117 | }, 118 | "Polygon": [ 119 | { "X": 0.2303103655576706, "Y": 0.5644218325614929 }, 120 | { "X": 0.2961905598640442, "Y": 0.5886967182159424 }, 121 | { "X": 0.2874591052532196, "Y": 0.6309736967086792 }, 122 | { "X": 0.22157889604568481, "Y": 0.6066988110542297 } 123 | ] 124 | } 125 | }, 126 | { 127 | "DetectedText": "View", 128 | "Type": "WORD", 129 | "Id": 6, 130 | "ParentId": 2, 131 | "Confidence": 99.49078369140625, 132 | "Geometry": { 133 | "BoundingBox": { 134 | "Width": 0.04752301052212715, 135 | "Height": 0.0205078125, 136 | "Left": 0.8437162637710571, 137 | "Top": 0.923828125 138 | }, 139 | "Polygon": [ 140 | { "X": 0.8437162637710571, "Y": 0.923828125 }, 141 | { "X": 0.8912392258644104, "Y": 0.923828125 }, 142 | { "X": 0.8912392258644104, "Y": 0.9443359375 }, 143 | { "X": 0.8437162637710571, "Y": 0.9443359375 } 144 | ] 145 | } 146 | }, 147 | { 148 | "DetectedText": "gallery", 149 | "Type": "WORD", 150 | "Id": 7, 151 | "ParentId": 2, 152 | "Confidence": 99.77100372314453, 153 | "Geometry": { 154 | "BoundingBox": { 155 | "Width": 0.07018782943487167, 156 | "Height": 0.0283203125, 157 | "Left": 0.8963571190834045, 158 | "Top": 0.9228515625 159 | }, 160 | "Polygon": [ 161 | { "X": 0.8963571190834045, "Y": 0.9228515625 }, 162 | { "X": 0.9665449261665344, "Y": 0.9228515625 }, 163 | { "X": 0.9665449261665344, "Y": 0.951171875 }, 164 | { "X": 0.8963571190834045, "Y": 0.951171875 } 165 | ] 166 | } 167 | } 168 | ], 169 | "TextModelVersion": "3.0", 170 | "ResponseMetadata": { 171 | "RequestId": "261f6931-d570-4718-a47b-cca4d575521d", 172 | "HTTPStatusCode": 200, 173 | "HTTPHeaders": { 174 | "x-amzn-requestid": "261f6931-d570-4718-a47b-cca4d575521d", 175 | "content-type": "application/x-amz-json-1.1", 176 | "content-length": "3260", 177 | "date": "Sun, 17 Mar 2024 01:15:59 GMT" 178 | }, 179 | "RetryAttempts": 0 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | label#numberPlate { 11 | font-size: 25px; 12 | color: rgb(19, 157, 125); 13 | font-style:oblique; 14 | } 15 | h3 { 16 | font-size: 50px; 17 | color: rgb(143, 112, 197); 18 | } 19 | label#message { 20 | font-size: 25px; 21 | color: rgb(157, 19, 93); 22 | font-style:oblique; 23 | } 24 | 25 | #web-socket-status { 26 | position: absolute; 27 | top: 0px; 28 | right: 0px; 29 | } 30 | 31 | 32 | input { 33 | font-size: 25px; 34 | color: rgb(0, 255, 170); 35 | background-color: rgb(78, 74, 74); 36 | } 37 | 38 | input { 39 | font-size: 25px; 40 | color: rgb(0, 255, 170); 41 | background-color: rgb(78, 74, 74); 42 | } 43 | 44 | @media (prefers-reduced-motion: no-preference) { 45 | .App-logo { 46 | animation: App-logo-spin infinite 20s linear; 47 | } 48 | } 49 | 50 | .App-header { 51 | background-color: #282c34; 52 | min-height: 100vh; 53 | display: flex; 54 | flex-direction: column; 55 | align-items: center; 56 | justify-content: center; 57 | font-size: calc(10px + 2vmin); 58 | color: white; 59 | } 60 | 61 | .App-link { 62 | color: #61dafb; 63 | } 64 | 65 | @keyframes App-logo-spin { 66 | from { 67 | transform: rotate(0deg); 68 | } 69 | to { 70 | transform: rotate(360deg); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/App.js: -------------------------------------------------------------------------------- 1 | import { useState, useEffect } from "react"; 2 | import { S3Client } from "@aws-sdk/client-s3"; 3 | import { Upload } from "@aws-sdk/lib-storage"; 4 | import { MdOnlinePrediction } from "react-icons/md"; 5 | import './App.css'; 6 | 7 | const bucketName = "vehicle-image-bucket" 8 | const creds = { 9 | accessKeyId: "", 10 | secretAccessKey: "", 11 | }; 12 | 13 | const client = new S3Client({ 14 | region: "ap-southeast-2", 15 | signatureVersion: 'v4', 16 | credentials: creds 17 | }); 18 | 19 | function App() { 20 | const [image, setImage] = useState(null); 21 | const [message, setMessage] = useState(""); 22 | const [ws, setWs] = useState(undefined); 23 | const [numberPlate, setNumberPlate] = useState(""); 24 | const [socketStatusColour, setSocketStatusColour]= useState("grey") 25 | 26 | useEffect(() => { 27 | if(!ws){ 28 | console.log('........Connecting to server...........') 29 | const webSocket = new WebSocket("wss://nb1t7abol1.execute-api.us-east-1.amazonaws.com/dev/"); 30 | setWs(webSocket) 31 | } else{ 32 | setSocketStatusColour("grey") 33 | } 34 | },[]); 35 | 36 | 37 | if(ws) { 38 | ws.onopen = (event) => { 39 | console.log('Connection established', event) 40 | setSocketStatusColour("green") 41 | }; 42 | ws.onmessage = function (event) { 43 | console.log(`event`, event) 44 | try { 45 | const data = JSON.parse(event.data); 46 | console.log('Number:', data.message) 47 | setMessage("") 48 | setNumberPlate(`Number plate: ${data.message}`); 49 | 50 | } catch (err) { 51 | console.log(err); 52 | } 53 | }; 54 | } 55 | 56 | const processImage = async (event) => { 57 | setNumberPlate(`Number plate: .....`); 58 | setImage(event.target.files[0]) 59 | 60 | const uploadResult = await uploadFileToS3(event.target.files[0]) 61 | if(uploadResult){ 62 | const imageName = event.target.files[0].name; 63 | await sendMessage(imageName); 64 | } 65 | } 66 | 67 | const uploadFileToS3 = async(image) => { 68 | setMessage("Uploading......") 69 | const target = { Bucket: bucketName, Key: image.name, Body: image }; 70 | try { 71 | const parallelUploads3 = new Upload({ 72 | client: client, 73 | // tags: [{ Key: "connection_id", Value: "123"}], // optional tags 74 | queueSize: 4, // optional concurrency configuration 75 | leavePartsOnError: false, // optional manually handle dropped parts 76 | params: target, 77 | }); 78 | 79 | parallelUploads3.on("httpUploadProgress", (progress) => { 80 | console.log("progress:", progress); 81 | }); 82 | 83 | var result = await parallelUploads3.done(); 84 | 85 | setMessage("Upload is done") 86 | 87 | // console.log('Result:', result); 88 | return true; 89 | 90 | } catch (e) { 91 | console.log(e); 92 | setMessage("Upload is failed"); 93 | } 94 | return false; 95 | } 96 | 97 | const sendMessage = async (imageName) => { 98 | try { 99 | if(ws) { 100 | setMessage(`Sending image ${imageName} info....`) 101 | 102 | const result = await ws.send( JSON.stringify( { 103 | "action": "sendVehicleInfo", 104 | "message": {"bucket": bucketName, "key": imageName} 105 | })); 106 | 107 | console.log('result', result) 108 | setMessage("Processing image....") 109 | } 110 | } 111 | catch(err){ 112 | console.log("error", err) 113 | } 114 | 115 | } 116 | 117 | return ( 118 |
119 |
120 |

Vehicle Number Plate Recognition System

121 |
122 | { 123 | image=== null?
: 124 | } 125 | 126 |

127 | 128 |
129 | 130 | 131 |
132 |
133 | ); 134 | } 135 | 136 | export default App; 137 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import App from './App'; 3 | 4 | test('renders learn react link', () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/MultipleImageUpload.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | const dummy = "https://static.thenounproject.com/png/625182-200.png"; 3 | class MultipleImageUpload extends React.Component { 4 | state = { current: null, old: [] }; 5 | uploadImage = e => { 6 | const { current, old } = this.state; 7 | 8 | this.setState( 9 | { 10 | current: URL.createObjectURL(e.target.files[0]) 11 | }, 12 | () => { 13 | this.setState({ old: [...old, current] }); 14 | } 15 | ); 16 | 17 | this.setState({ current: URL.createObjectURL(e.target.files[0]) }); 18 | }; 19 | render() { 20 | console.log(this.state); 21 | const { old, current } = this.state; 22 | return ( 23 |
24 | {/*
25 |

Current

26 | custom-pic 31 |
*/} 32 |
33 | {/* {old 34 | ? old.map((o, i) => ( 35 | preview 41 | )) 42 | : ""} */} 43 | 44 | 45 |
46 | ); 47 | } 48 | } 49 | export default MultipleImageUpload; 50 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/SampleImages/audi1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/src/SampleImages/audi1.png -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/SampleImages/audi2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/src/SampleImages/audi2.png -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/SampleImages/audi3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/src/SampleImages/audi3.png -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/SampleImages/audi4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/src/SampleImages/audi4.png -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/SampleImages/benz11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS Projects/vehicle-number-identifier/src/SampleImages/benz11.png -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import './index.css'; 4 | import App from './App'; 5 | import reportWebVitals from './reportWebVitals'; 6 | 7 | const root = ReactDOM.createRoot(document.getElementById('root')); 8 | root.render( 9 | 10 | 11 | 12 | ); 13 | 14 | // If you want to start measuring performance in your app, pass a function 15 | // to log results (for example: reportWebVitals(console.log)) 16 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 17 | reportWebVitals(); 18 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = onPerfEntry => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /AWS Projects/vehicle-number-identifier/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /AWS S3/AWS S3- slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS S3/AWS S3- slides.pdf -------------------------------------------------------------------------------- /AWS-Gateway-Lambda-Authorizer/Authorizer response.md: -------------------------------------------------------------------------------- 1 | 2 | ## Authorizer response 3 | ```json 4 | { 5 | "principalId": "any-api-principle-id", 6 | "policyDocument": 7 | { 8 | "Version": "2012-10-17", 9 | "Statement": 10 | [ 11 | { 12 | "Action": "execute-api:Invoke", 13 | "Resource": ["arn:aws:execute-api:YOUR_AWS_REGION:YOURACCOUNTNUMBER:YOUR_API_IDENTIFIER/STAGEING_ENV/VERB/RESOURCE"], 14 | "Effect": "Allow" // Allow OR Deny 15 | } 16 | ] 17 | } 18 | } 19 | ``` 20 | 21 | ## Resource format 22 | "Resource": ["arn:aws:execute-api:YOUR_AWS_REGION:YOURACCOUNTNUMBER:YOUR_API_IDENTIFIER/STAGEING_ENV/VERB/RESOURCE"] 23 | 24 | 25 | ## Authorizer Lambda code 26 | ``` 27 | export const handler = async(event) => { 28 | console.log('event', event) 29 | 30 | const token = event['authorizationToken'] 31 | 32 | console.log('token', token) 33 | 34 | 35 | let permission = "Deny"; 36 | if(token === "my-secret-token") { 37 | permission = "Allow" 38 | } 39 | const authResponse = { 40 | "principalId": "abc123", 41 | "policyDocument": 42 | { 43 | "Version": "2012-10-17", 44 | "Statement": 45 | [ 46 | { 47 | "Action": "execute-api:Invoke", 48 | "Resource": ["arn:aws:execute-api:YOUR_AWS_REGION:YOURACCOUNTNUMBER:YOUR_API_IDENTIFIER/STAGEING_ENV/VERB/RESOURCE"], 49 | "Effect": `${permission}` 50 | } 51 | ] 52 | } 53 | 54 | } 55 | return authResponse; 56 | }; 57 | ``` 58 | 59 | 60 | -------------------------------------------------------------------------------- /AWS-Gateway-Lambda-Authorizer/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Authorizer response 3 | ```json 4 | { 5 | "principalId": "any-api-principle-id", 6 | "policyDocument": 7 | { 8 | "Version": "2012-10-17", 9 | "Statement": 10 | [ 11 | { 12 | "Action": "execute-api:Invoke", 13 | "Resource": ["arn:aws:execute-api:YOUR_AWS_REGION:YOURACCOUNTNUMBER:YOUR_API_IDENTIFIER/STAGEING_ENV/VERB/RESOURCE"], 14 | "Effect": "Allow" // Allow OR Deny 15 | } 16 | ] 17 | } 18 | } 19 | ``` 20 | 21 | ## Resource format 22 | "Resource": ["arn:aws:execute-api:YOUR_AWS_REGION:YOURACCOUNTNUMBER:YOUR_API_IDENTIFIER/STAGEING_ENV/VERB/RESOURCE"] 23 | 24 | 25 | ## Authorizer Lambda code 26 | ``` 27 | export const handler = async(event) => { 28 | console.log('event', event) 29 | 30 | const token = event['authorizationToken'] 31 | 32 | console.log('token', token) 33 | 34 | 35 | let permission = "Deny"; 36 | if(token === "my-secret-token") { 37 | permission = "Allow" 38 | } 39 | const authResponse = { 40 | "principalId": "abc123", 41 | "policyDocument": 42 | { 43 | "Version": "2012-10-17", 44 | "Statement": 45 | [ 46 | { 47 | "Action": "execute-api:Invoke", 48 | "Resource": ["arn:aws:execute-api:YOUR_AWS_REGION:YOURACCOUNTNUMBER:YOUR_API_IDENTIFIER/STAGEING_ENV/VERB/RESOURCE"], 49 | "Effect": `${permission}` 50 | } 51 | ] 52 | } 53 | 54 | } 55 | return authResponse; 56 | }; 57 | ``` 58 | 59 | 60 | -------------------------------------------------------------------------------- /AWS-Standarad-vs-FIFO/Readme.md: -------------------------------------------------------------------------------- 1 | ## SQS quotas 2 | - https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html 3 | 4 | ## Long polling and short polling 5 | - https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-short-and-long-polling.html 6 | 7 | 8 | ## Lambda code 9 | 10 | ``` 11 | import json 12 | 13 | def lambda_handler(event, context): 14 | 15 | for item in event["Records"]: 16 | body = item["body"] 17 | print(f'message: ', body) 18 | 19 | # throw error to simulate the an error in the Lambda 20 | myvalue = float(body) 21 | return { 22 | 'statusCode': 200, 23 | 'body': json.dumps('Hello from Lambda!') 24 | } 25 | ``` -------------------------------------------------------------------------------- /AWS-Standarad-vs-FIFO/pythonCode/index.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import uuid 3 | 4 | # Create SQS client 5 | sqs = boto3.client('sqs') 6 | 7 | # update your queue Url as needed 8 | queue_url = 'https://sqs.us-east-1.amazonaws.com/xxx/demo-queue-standard' 9 | #queue_url = 'https://sqs.us-east-1.amazonaws.com/xxx/demo-queue-fifo.fifo' 10 | 11 | # Send message to SQS queue 12 | response = sqs.send_message( 13 | QueueUrl= queue_url, 14 | # un-comment below lines for FIFO queue 15 | #MessageGroupId= "G3", 16 | #MessageDeduplicationId= str(uuid.uuid4()), 17 | MessageAttributes= { 18 | 'Title': { 19 | 'DataType': 'String', 20 | 'StringValue': 'Mr' 21 | } 22 | }, 23 | MessageBody=(f'700') 24 | ) 25 | 26 | print(response['MessageId']) -------------------------------------------------------------------------------- /AWS-Standarad-vs-FIFO/pythonCode/requrement.txt: -------------------------------------------------------------------------------- 1 | boto3 -------------------------------------------------------------------------------- /AWSEventBridge/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Rule pattern 3 | ``` 4 | { 5 | "detail": { 6 | "type": ["toy"] 7 | } 8 | } 9 | 10 | ``` 11 | ## Sample payload which can be used in Rule evaluation page 12 | 13 | ``` 14 | { 15 | "id": "1b8e2e75-b771-e964-f0e6-fbca6a21dad8", 16 | "detail-type": "test type", 17 | "source": "test", 18 | "account": "sasa", 19 | "resources": [""], 20 | "time": "2022-10-02T12:47:58Z", 21 | "region": "ss", 22 | "detail": { 23 | "type": "toy" 24 | } 25 | } 26 | 27 | ``` 28 | 29 | ## payload event to test in 'send event' page in AWS console 30 | 31 | ``` 32 | { 33 | "name": "lego", 34 | "price": "10.50", 35 | "type": "toy" 36 | } 37 | ``` 38 | 39 | ## Custome payload which can be used to test for Lambda 40 | // payload 41 | ``` 42 | { 43 | "name": "lego", 44 | "price": "10.50", 45 | "type": "toy" 46 | } 47 | ``` 48 | 49 | ## Lambda code to generate events 50 | 51 | ``` 52 | import json 53 | import boto3 54 | import datetime 55 | 56 | event_client = boto3.client('events') 57 | 58 | def lambda_handler(event, context): 59 | 60 | # change the 'EventBusName' as per your account and event bus 61 | response = event_client.put_events( 62 | Entries= [ 63 | { 64 | 'Time': datetime.datetime.now(), 65 | 'Source': 'Lambda Publish', 66 | 'Resources': [], 67 | 'DetailType': 'Demo 1', 68 | 'Detail': json.dumps(event), 69 | 'EventBusName': 'arn:aws:events:us-east-1:xxxxxxx:event-bus/test-bus1', 70 | 'TraceHeader': 'testdemo' 71 | }, 72 | ]) 73 | 74 | ``` 75 | 76 | ## Shopify resource URL 77 | https://shopify.dev/apps/webhooks/configuration/eventbridge 78 | 79 | ## Shopify sample event 80 | ``` 81 | { 82 | "version": "0", 83 | "id": "1b8e2e75-b771-e964-f0e6-fbca6a21dad8", 84 | "detail-type": ["shopifyWebhook"], 85 | "source": "aws.partner/shopify.com/shop-event-bus", 86 | "account": "123456789", 87 | "resources": [""], 88 | "time": "2022-10-02T12:47:58Z", 89 | "region": "ca-central-1", 90 | "detail": { 91 | "payload": { 92 | "product" : { 93 | "id": 1, 94 | "title": "Columbia Las Hermosas", 95 | "body_html": "", 96 | "vendor": "Reunion Island", 97 | "product_type": "Coffee", 98 | "created_at": "2022-10-07T14:55:00-05:00", 99 | "handle": "columbia-las-hermosas", 100 | ... 101 | } 102 | }, 103 | "metadata": { 104 | "X-Shopify-Topic": "products/update", 105 | "X-Shopify-API-Version": "2022-10", 106 | "X-Shopify-Hmac-SHA256": "rncNozYG6CCjmFJjEgUWowYJ60X+oUtsqul1sTwJMpU=", 107 | "X-Shopify-Shop-Domain": "{shop}.myshopify.com", 108 | "X-Shopify-Order-Id": 1 109 | } 110 | } 111 | } 112 | 113 | ``` 114 | 115 | ## Shopify rule pattern 116 | 117 | ``` 118 | { 119 | "detail-type": ["shopifyWebhook"], 120 | "detail": { 121 | "metadata": { 122 | "X-Shopify-Topic": [{ 123 | "prefix": "products" 124 | }] 125 | } 126 | } 127 | } 128 | 129 | ``` -------------------------------------------------------------------------------- /AWSTextract/ReadingAllText.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | from urllib.parse import unquote_plus 4 | 5 | def lambda_handler(event, context): 6 | textract = boto3.client("textract") 7 | print(f'Event: {event}') 8 | 9 | if event: 10 | file_obj = event["Records"][0] 11 | bucketname = str(file_obj["s3"]["bucket"]["name"]) 12 | filename = unquote_plus(str(file_obj["s3"]["object"]["key"])) 13 | 14 | print(f"Bucket: {bucketname} ::: Key: {filename}") 15 | 16 | response = textract.detect_document_text( 17 | Document= { 18 | "S3Object": { 19 | "Bucket": bucketname, 20 | "Name": filename, 21 | } 22 | } 23 | ) 24 | print(json.dumps(response)) 25 | 26 | raw_text = extract_text(response, extract_by="LINE") 27 | print(raw_text) 28 | 29 | return { 30 | "statusCode": 200, 31 | "body": json.dumps("Document processed successfully!"), 32 | } 33 | 34 | return {"statusCode": 500, "body": "Event is null."} 35 | 36 | def extract_text(response, extract_by="LINE"): 37 | line_text = [] 38 | for block in response["Blocks"]: 39 | if block["BlockType"] == extract_by: 40 | line_text.append(block["Text"]) 41 | return line_text -------------------------------------------------------------------------------- /AWSTextract/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWSTextract/Readme.md -------------------------------------------------------------------------------- /AWSTextract/lambda_function.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from collections import defaultdict 3 | from urllib.parse import unquote_plus 4 | 5 | def get_kv_map(bucket, key): 6 | # process using image bytes 7 | client = boto3.client('textract') 8 | response = client.analyze_document(Document={'S3Object': {'Bucket': bucket, "Name": key}}, FeatureTypes=['FORMS']) 9 | 10 | # Get the text blocks 11 | blocks = response['Blocks'] 12 | print(f'BLOCKS: {blocks}') 13 | 14 | # get key and value maps 15 | key_map = {} 16 | value_map = {} 17 | block_map = {} 18 | for block in blocks: 19 | block_id = block['Id'] 20 | block_map[block_id] = block 21 | if block['BlockType'] == "KEY_VALUE_SET": 22 | if 'KEY' in block['EntityTypes']: 23 | key_map[block_id] = block 24 | else: 25 | value_map[block_id] = block 26 | 27 | return key_map, value_map, block_map 28 | 29 | 30 | def get_kv_relationship(key_map, value_map, block_map): 31 | kvs = defaultdict(list) 32 | for block_id, key_block in key_map.items(): 33 | value_block = find_value_block(key_block, value_map) 34 | key = get_text(key_block, block_map) 35 | val = get_text(value_block, block_map) 36 | kvs[key].append(val) 37 | return kvs 38 | 39 | 40 | def find_value_block(key_block, value_map): 41 | for relationship in key_block['Relationships']: 42 | if relationship['Type'] == 'VALUE': 43 | for value_id in relationship['Ids']: 44 | value_block = value_map[value_id] 45 | return value_block 46 | 47 | 48 | def get_text(result, blocks_map): 49 | text = '' 50 | if 'Relationships' in result: 51 | for relationship in result['Relationships']: 52 | if relationship['Type'] == 'CHILD': 53 | for child_id in relationship['Ids']: 54 | word = blocks_map[child_id] 55 | if word['BlockType'] == 'WORD': 56 | text += word['Text'] + ' ' 57 | if word['BlockType'] == 'SELECTION_ELEMENT': 58 | if word['SelectionStatus'] == 'SELECTED': 59 | text += 'X' 60 | 61 | return text 62 | 63 | def lambda_handler(event, context): 64 | file_obj = event["Records"][0] 65 | bucket = unquote_plus(str(file_obj["s3"]["bucket"]["name"])) 66 | file_name = unquote_plus(str(file_obj["s3"]["object"]["key"])) 67 | #file_name ='filled_form.png' 68 | print(f'Bucket: {bucket}, file: {file_name}') 69 | key_map, value_map, block_map = get_kv_map( bucket, file_name) 70 | 71 | # Get Key Value relationship 72 | kvs = get_kv_relationship(key_map, value_map, block_map) 73 | print("\n\n== FOUND KEY : VALUE pairs ===\n") 74 | 75 | for key, value in kvs.items(): 76 | print(key, ":", value) 77 | -------------------------------------------------------------------------------- /AWSTextract/requrement.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | -------------------------------------------------------------------------------- /AWS_IAM/Presentation slides- What is IAM.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeSam621/Demo/0de2c00ec8d09ceddc070f7aa35cf8d6d88ca740/AWS_IAM/Presentation slides- What is IAM.pdf -------------------------------------------------------------------------------- /AWS_IAM/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Full video >>> : https://www.youtube.com/watch?v=ExjW3HCFG1U 3 | ## Connect to EC2 ssh using certificate 4 | - `ssh -i ec2-user@` 5 | - eg: `ssh -i ./demo-ec2-pair.pem ec2-user@ec2-13-33-204-155.ap-southeast-2.compute.amazonaws.com` 6 | - `chmod 400 ./demo-ec2-pair.pem` 7 | - https://chmodcommand.com/chmod-400/ 8 | 9 | ## Policy evaluation logic 10 | - https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html 11 | 12 | ## Cross account access 13 | - `aws sts get-caller-identity` 14 | - `aws sts assume-role --profile john --role-arn "arn:aws:iam:::role/cross-account-role" --role-session-name AWSCLI-Session --external-id lovetocode-id` 15 | - 16 | `export AWS_ACCESS_KEY_ID=REPLACE_WITH_YOUR_ACCESS_ID` 17 | `export AWS_SECRET_ACCESS_KEY=REPLACE_WITH_YOUR_SECRET_ACCESS_KEY` 18 | `export AWS_SESSION_TOKEN=REPLACE_WITH_YOUR_SESSION_TOKEN` 19 | 20 | - Remove env variables 21 | `unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN` 22 | -------------------------------------------------------------------------------- /AWS_IAM/abac_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor1", 6 | "Effect": "Allow", 7 | "Action": "s3:*", 8 | "Resource": "arn:aws:s3:::*", 9 | "Condition": { 10 | "StringEquals": { 11 | "aws:PrincipalTag/group": "admin" 12 | } 13 | } 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /AWS_IAM/cross-account-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "sts:AssumeRole" 8 | ], 9 | "Resource": "*" 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /AWS_IAM/cross-account-role.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "AWS": "arn:aws:iam::YOUR_ACCOUNT_B_ID:root" 8 | }, 9 | "Action": "sts:AssumeRole", 10 | "Condition": { 11 | "StringEquals": { 12 | "sts:ExternalId": "lovetocode-id" 13 | } 14 | } 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /AWS_IAM/identity-based-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "s3:Get*", 8 | "s3:List*", 9 | "s3-object-lambda:Get*", 10 | "s3-object-lambda:List*" 11 | ], 12 | "Resource": "*" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /AWS_IAM/permission-boundary.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "IAMAccess", 6 | "Effect": "Allow", 7 | "Action": "iam:*", 8 | "Resource": "*" 9 | }, 10 | { 11 | "Sid": "DenyCreatingUserWithoutPermisionBoundary", 12 | "Effect": "Deny", 13 | "Action": [ 14 | "iam:CreateUser", 15 | "iam:CreateRole" 16 | ], 17 | "Resource": [ 18 | "arn:aws:iam::YOUR_ACCOUNT_ID:user/*", 19 | "arn:aws:iam::YOUR_ACCOUNT_ID:role/*" 20 | ], 21 | "Condition": { 22 | "StringNotEquals": { 23 | "iam:PermissionsBoundary": "arn:aws:iam::YOUR_ACCOUNT_ID:policy/permission-boundary" 24 | } 25 | } 26 | }, 27 | { 28 | "Sid": "DenyDeletingPolicy", 29 | "Effect": "Deny", 30 | "Action": [ 31 | "iam:DeletePolicy", 32 | "iam:DeletePolicyVersion", 33 | "iam:CreatePolicyVersion", 34 | "iam:SetDefaultPolicyVersion" 35 | ], 36 | "Resource": [ 37 | "arn:aws:iam::YOUR_ACCOUNT_ID:policy/permission-boundary" 38 | ] 39 | }, 40 | { 41 | "Sid": "DenyDeletingPermBoundaryFromAnyUserOrRole", 42 | "Effect": "Deny", 43 | "Action": [ 44 | "iam:DeleteUserPermissionsBoundary", 45 | "iam:DeleteRolePermissionsBoundary" 46 | ], 47 | "Resource": [ 48 | "arn:aws:iam::YOUR_ACCOUNT_ID:user/*", 49 | "arn:aws:iam::YOUR_ACCOUNT_ID:role/*" 50 | ], 51 | "Condition": { 52 | "StringEquals": { 53 | "iam:PermissionsBoundary": "arn:aws:iam::YOUR_ACCOUNT_ID:policy/permission-boundary" 54 | } 55 | } 56 | }, 57 | { 58 | "Sid": "DenyUpdatingPermissionBoundary", 59 | "Effect": "Deny", 60 | "Action": [ 61 | "iam:PutUserPermissionsBoundary", 62 | "iam:PutRolePermissionsBoundary" 63 | ], 64 | "Resource": [ 65 | "arn:aws:iam::YOUR_ACCOUNT_ID:user/*", 66 | "arn:aws:iam::YOUR_ACCOUNT_ID:role/*" 67 | ], 68 | "Condition": { 69 | "StringNotEquals": { 70 | "iam:PermissionsBoundary": "arn:aws:iam::YOUR_ACCOUNT_ID:policy/permission-boundary" 71 | } 72 | } 73 | } 74 | ] 75 | } -------------------------------------------------------------------------------- /AWS_IAM/s3-resource-based-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "Statement1", 6 | "Effect": "Deny", 7 | "Principal": { 8 | "AWS": "arn:aws:iam::YOUR_ACCOUNT_ID:user/YOUR_ACCOUNT_USER_NAME" 9 | }, 10 | "Action": "S3:*", 11 | "Resource": [ 12 | "arn:aws:s3:::YOUR_S3_BUCKET_NAME", 13 | "arn:aws:s3:::YOUR_S3_BUCKET_NAME/*" 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /AWS_IAM/trust-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "Service": "ec2.amazonaws.com" 8 | }, 9 | "Action": "sts:AssumeRole" 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /AWS_SSM_ParameterStore/Readme.md: -------------------------------------------------------------------------------- 1 | ``` 2 | 3 | import boto3 4 | import os 5 | 6 | client = boto3.client('ssm') 7 | 8 | def lambda_handler(event, context): 9 | parameter = client.get_parameter(Name= os.environ['ENV_DB_CONNECTION_STRING'], WithDecryption=True) 10 | print(f'parameter: {parameter}') 11 | 12 | print(f" Password: { parameter ['Parameter']['Value']}"); 13 | ``` 14 | -------------------------------------------------------------------------------- /AWS_SSM_ParameterStore/lambda_handler.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import os 3 | 4 | client = boto3.client('ssm') 5 | 6 | def lambda_handler(event, context): 7 | parameter = client.get_parameter(Name= os.environ['ENV_DB_CONNECTION_STRING'], WithDecryption=True) 8 | print(f'parameter: {parameter}') 9 | 10 | print(f" Password: { parameter ['Parameter']['Value']}"); -------------------------------------------------------------------------------- /DynamoDB-Python-Demo/index.py: -------------------------------------------------------------------------------- 1 | from boto3 import resource 2 | from boto3.dynamodb.conditions import Attr, Key 3 | from datetime import datetime 4 | 5 | demo_table = resource('dynamodb').Table('demo-dynamo-python') 6 | 7 | ############################# insert record ############################# 8 | 9 | def insert(): 10 | print(f'demo_insert') 11 | response = demo_table.put_item( 12 | Item={ 13 | 'customer_id': 'cus-05', # parition key 14 | 'order_id' : 'ord-5', # sort key 15 | 'status': 'pending', 16 | 'created_date' : datetime.now().isoformat() 17 | } 18 | ) 19 | print(f'Insert response: {response}') 20 | 21 | # insert() 22 | 23 | ############################# Scan by attributes ############################# 24 | 25 | def select_scan(): 26 | print(f'demo_select_scan') 27 | filter_expression = Attr('status').eq('pending') 28 | 29 | item_list = [] 30 | dynamo_response = {'LastEvaluatedKey': False} 31 | while 'LastEvaluatedKey' in dynamo_response: 32 | if dynamo_response['LastEvaluatedKey']: 33 | dynamo_response = demo_table.scan( 34 | FilterExpression=filter_expression, 35 | ExclusiveStartKey=dynamo_response['LastEvaluatedKey'] 36 | ) 37 | print(f'response-if: {dynamo_response}') 38 | else: 39 | dynamo_response = demo_table.scan( 40 | FilterExpression=filter_expression, 41 | ) 42 | print(f'response-else: {dynamo_response}') 43 | 44 | for i in dynamo_response['Items']: 45 | item_list.append(i) 46 | 47 | print(f'Number of input tasks to process: {len(item_list)}') 48 | for item in item_list: 49 | print(f'Item: {item}') 50 | 51 | 52 | # select_scan() 53 | 54 | ############################# Query by parition key ############################# 55 | 56 | def query_by_partition_key(customer_value): 57 | print(f'demo_select_query') 58 | 59 | response = {} 60 | filtering_exp = Key('customer_id').eq(customer_value) 61 | response = demo_table.query( 62 | KeyConditionExpression=filtering_exp) 63 | item_list = response["Items"] 64 | for item in item_list: 65 | print(f'Item: {item}') 66 | 67 | 68 | # query_by_partition_key('cus-01') 69 | 70 | ############################# Query by parition key and sort by ASC,DESC ############################# 71 | 72 | def query_by_partition_key_order(customer_value): 73 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_query_by_partition_key_order <<<<<<<<<<<<<<<<<<<<<<') 74 | response = {} 75 | filtering_exp = Key('customer_id').eq(customer_value) 76 | response = demo_table.query( 77 | KeyConditionExpression=filtering_exp, 78 | ScanIndexForward=False) 79 | 80 | item_list = response["Items"] 81 | for item in item_list: 82 | print(f'Item: {item}') 83 | 84 | # query_by_partition_key_order('cus-01') 85 | 86 | ############################# Query by Global index and Local index ############################# 87 | 88 | def query_by_index_key(status_value): 89 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_query_index_key <<<<<<<<<<<<<<<<<<<<<<') 90 | 91 | filtering_exp = Key('status').eq(status_value) 92 | response = demo_table.query( 93 | IndexName="status-index", 94 | KeyConditionExpression=filtering_exp, 95 | ScanIndexForward=False) 96 | 97 | for item in response["Items"]: 98 | print(f'Item: {item}') 99 | 100 | 101 | # query_by_index_key('completed') 102 | 103 | ############################# Query by parition and sort key ############################# 104 | 105 | def query_by_partition_key_and_sort_key(customer_value, order_value): 106 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_query_by_partition_key_and_sort_key <<<<<<<<<<<<<<<<<<<<<<') 107 | 108 | response = {} 109 | filtering_exp = Key('customer_id').eq(customer_value) 110 | filtering_exp2 = Key('order_id').eq(order_value) 111 | response = demo_table.query( 112 | KeyConditionExpression=filtering_exp & filtering_exp2) 113 | 114 | for item in response["Items"]: 115 | print(f'Item: {item}') 116 | 117 | # query_by_partition_key_and_sort_key('cus-01', 'ord-1') 118 | 119 | 120 | ############################# Update record - start ############################# 121 | 122 | def update(customer_value, status_value): 123 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_update <<<<<<<<<<<<<<<<<<<<<<') 124 | response = demo_table.update_item( 125 | Key = { 126 | 'customer_id': customer_value, 127 | }, 128 | UpdateExpression = 'set status=:r, updated_date=:d', 129 | ExpressionAttributeValues={ 130 | ':r': status_value, 131 | ':d': datetime.now().isoformat() 132 | }, 133 | ReturnValues="UPDATED_NEW" 134 | ) 135 | 136 | 137 | def update_with_expression_name(customer_value, status_value): 138 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_update_with_expression_name <<<<<<<<<<<<<<<<<<<<<<') 139 | response = demo_table.update_item( 140 | Key = { 141 | 'customer_id': customer_value, 142 | 'order_id': 'ord-3' 143 | }, 144 | UpdateExpression = 'set #status=:r, updated_date=:d', 145 | ExpressionAttributeValues={ 146 | ':r': status_value, 147 | ':d': datetime.now().isoformat() 148 | }, 149 | ExpressionAttributeNames={ 150 | '#status': 'status' 151 | }, 152 | ReturnValues="UPDATED_NEW" 153 | ) 154 | print(f'Reponse: {response}') 155 | 156 | # update_with_expression_name('cus-02', 'completed') 157 | 158 | ############################# Update record - end ############################# 159 | 160 | 161 | ############################# Batch delete ############################# 162 | def batch_delete_transaction_records(items_to_delete): 163 | print(f'Deleting transactions') 164 | response = {} 165 | with demo_table.batch_writer() as batch: 166 | for item in items_to_delete: 167 | response = batch.delete_item(Key={ 168 | "customer_id": item["id"], # Change key and value names 169 | "order_id": item["order_id"] 170 | }) 171 | # items = [{"id": "cus-04", "order_id": "ord-4" }, {"id": "cus-05", "order_id": "ord-4" }] 172 | # batch_delete_transaction_records(items) -------------------------------------------------------------------------------- /DynamoDB-Python-Demo/readme.md: -------------------------------------------------------------------------------- 1 | ``` python 2 | from boto3 import resource 3 | from boto3.dynamodb.conditions import Attr, Key 4 | from datetime import datetime 5 | 6 | demo_table = resource('dynamodb').Table('demo-dynamo-python') 7 | 8 | ############################# insert record ############################# 9 | 10 | def insert(): 11 | print(f'demo_insert') 12 | response = demo_table.put_item( 13 | Item={ 14 | 'customer_id': 'cus-05', # parition key 15 | 'order_id' : 'ord-5', # sort key 16 | 'status': 'pending', 17 | 'created_date' : datetime.now().isoformat() 18 | } 19 | ) 20 | print(f'Insert response: {response}') 21 | 22 | # insert() 23 | 24 | ############################# Scan by attributes ############################# 25 | 26 | def select_scan(): 27 | print(f'demo_select_scan') 28 | filter_expression = Attr('status').eq('pending') 29 | 30 | item_list = [] 31 | dynamo_response = {'LastEvaluatedKey': False} 32 | while 'LastEvaluatedKey' in dynamo_response: 33 | if dynamo_response['LastEvaluatedKey']: 34 | dynamo_response = demo_table.scan( 35 | FilterExpression=filter_expression, 36 | ExclusiveStartKey=dynamo_response['LastEvaluatedKey'] 37 | ) 38 | print(f'response-if: {dynamo_response}') 39 | else: 40 | dynamo_response = demo_table.scan( 41 | FilterExpression=filter_expression, 42 | ) 43 | print(f'response-else: {dynamo_response}') 44 | 45 | for i in dynamo_response['Items']: 46 | item_list.append(i) 47 | 48 | print(f'Number of input tasks to process: {len(item_list)}') 49 | for item in item_list: 50 | print(f'Item: {item}') 51 | 52 | 53 | # select_scan() 54 | 55 | ############################# Query by parition key ############################# 56 | 57 | def query_by_partition_key(customer_value): 58 | print(f'demo_select_query') 59 | 60 | response = {} 61 | filtering_exp = Key('customer_id').eq(customer_value) 62 | response = demo_table.query( 63 | KeyConditionExpression=filtering_exp) 64 | item_list = response["Items"] 65 | for item in item_list: 66 | print(f'Item: {item}') 67 | 68 | 69 | # query_by_partition_key('cus-01') 70 | 71 | ############################# Query by parition key and sort by ASC,DESC ############################# 72 | 73 | def query_by_partition_key_order(customer_value): 74 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_query_by_partition_key_order <<<<<<<<<<<<<<<<<<<<<<') 75 | response = {} 76 | filtering_exp = Key('customer_id').eq(customer_value) 77 | response = demo_table.query( 78 | KeyConditionExpression=filtering_exp, 79 | ScanIndexForward=False) 80 | 81 | item_list = response["Items"] 82 | for item in item_list: 83 | print(f'Item: {item}') 84 | 85 | # query_by_partition_key_order('cus-01') 86 | 87 | ############################# Query by Global index and Local index ############################# 88 | 89 | def query_by_index_key(status_value): 90 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_query_index_key <<<<<<<<<<<<<<<<<<<<<<') 91 | 92 | filtering_exp = Key('status').eq(status_value) 93 | response = demo_table.query( 94 | IndexName="status-index", 95 | KeyConditionExpression=filtering_exp, 96 | ScanIndexForward=False) 97 | 98 | for item in response["Items"]: 99 | print(f'Item: {item}') 100 | 101 | 102 | # query_by_index_key('completed') 103 | 104 | ############################# Query by parition and sort key ############################# 105 | 106 | def query_by_partition_key_and_sort_key(customer_value, order_value): 107 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_query_by_partition_key_and_sort_key <<<<<<<<<<<<<<<<<<<<<<') 108 | 109 | response = {} 110 | filtering_exp = Key('customer_id').eq(customer_value) 111 | filtering_exp2 = Key('order_id').eq(order_value) 112 | response = demo_table.query( 113 | KeyConditionExpression=filtering_exp & filtering_exp2) 114 | 115 | for item in response["Items"]: 116 | print(f'Item: {item}') 117 | 118 | # query_by_partition_key_and_sort_key('cus-01', 'ord-1') 119 | 120 | 121 | ############################# Update record - start ############################# 122 | 123 | def update(customer_value, status_value): 124 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_update <<<<<<<<<<<<<<<<<<<<<<') 125 | response = demo_table.update_item( 126 | Key = { 127 | 'customer_id': customer_value, 128 | }, 129 | UpdateExpression = 'set status=:r, updated_date=:d', 130 | ExpressionAttributeValues={ 131 | ':r': status_value, 132 | ':d': datetime.now().isoformat() 133 | }, 134 | ReturnValues="UPDATED_NEW" 135 | ) 136 | 137 | 138 | def update_with_expression_name(customer_value, status_value): 139 | print(f'\n\t\t\t>>>>>>>>>>>>>>>>> demo_update_with_expression_name <<<<<<<<<<<<<<<<<<<<<<') 140 | response = demo_table.update_item( 141 | Key = { 142 | 'customer_id': customer_value, 143 | 'order_id': 'ord-3' 144 | }, 145 | UpdateExpression = 'set #status=:r, updated_date=:d', 146 | ExpressionAttributeValues={ 147 | ':r': status_value, 148 | ':d': datetime.now().isoformat() 149 | }, 150 | ExpressionAttributeNames={ 151 | '#status': 'status' 152 | }, 153 | ReturnValues="UPDATED_NEW" 154 | ) 155 | print(f'Reponse: {response}') 156 | 157 | # update_with_expression_name('cus-02', 'completed') 158 | 159 | ############################# Update record - end ############################# 160 | 161 | 162 | ############################# Batch delete ############################# 163 | def batch_delete_transaction_records(items_to_delete): 164 | print(f'Deleting transactions') 165 | response = {} 166 | with demo_table.batch_writer() as batch: 167 | for item in items_to_delete: 168 | response = batch.delete_item(Key={ 169 | "customer_id": item["id"], # Change key and value names 170 | "order_id": item["order_id"] 171 | }) 172 | # items = [{"id": "cus-04", "order_id": "ord-4" }, {"id": "cus-05", "order_id": "ord-4" }] 173 | # batch_delete_transaction_records(items) 174 | ``` 175 | -------------------------------------------------------------------------------- /DynamoDB-Python-Demo/requrements.txt: -------------------------------------------------------------------------------- 1 | boto3 -------------------------------------------------------------------------------- /ECR_Auto_Scanning_Python/index.py: -------------------------------------------------------------------------------- 1 | from json import loads, dumps 2 | import json 3 | import subprocess 4 | import csv 5 | 6 | class Vulnurability: 7 | repo_name: str = "" 8 | pack_name: str = "" 9 | pack_version: str = "" 10 | image_id: str = "" 11 | uri: str = "" 12 | 13 | repos = [] 14 | list_critical = [] 15 | findings = [] 16 | 17 | def get_all_repos(): 18 | get_repos = subprocess.Popen("aws ecr describe-repositories", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) 19 | json_repo_result = json.loads(get_repos.stdout.read()) 20 | repositories = json_repo_result["repositories"] 21 | print(f'Number of repos: {len(repositories)}') 22 | return repositories 23 | 24 | def get_latest_img_digest(repo_name): 25 | get_latest_image_cmd = f"aws ecr describe-images --repository-name {repo_name} --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageDigest'" 26 | get_latest_image = subprocess.Popen(get_latest_image_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) 27 | image_Digest = get_latest_image.stdout.read().decode('utf-8') 28 | #print(f'image_Digest: {image_Digest}') 29 | return image_Digest 30 | 31 | def start_scanning(repo_name, image_Digest): 32 | start_scan_cmd = f"aws ecr start-image-scan --repository-name {repo_name} --image-id imageDigest={image_Digest}" 33 | start_scan = subprocess.Popen(start_scan_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) 34 | scan_result = start_scan.stdout.read().decode('utf-8') 35 | # print(f'scan_result: {scan_result}') 36 | return scan_result 37 | 38 | def wait_scan_results(repo_name, image_Digest): 39 | wait_scan_cmd = f"aws ecr wait image-scan-complete --repository-name {repo_name} --image-id imageDigest={image_Digest}" 40 | wait_scan = subprocess.Popen(wait_scan_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) 41 | wait_scan.stdout.read().decode('utf-8') 42 | 43 | def get_scan_result(repo_name, image_Digest): 44 | cmd_finding = f"aws ecr describe-image-scan-findings --repository-name {repo_name} --image-id imageDigest={image_Digest}" 45 | get_findings = subprocess.Popen(cmd_finding, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) 46 | result = json.loads(get_findings.stdout.read()) 47 | return result 48 | 49 | def finding_vulnarablity(repo_name, json_finding_result): 50 | print_to_json( json_finding_result['imageScanFindings']['findings']) 51 | for result in json_finding_result['imageScanFindings']['findings']: 52 | if(result['severity'] == "CRITICAL"): 53 | item = Vulnurability() 54 | item.repo_name = repo_name 55 | item.uri = result['uri'] 56 | list_critical.append(item) 57 | attributes = result['attributes'] 58 | 59 | for att in attributes: 60 | if(att["key"] == "package_name"): 61 | item.pack_name = att["value"] 62 | if(att["key"] == "package_version"): 63 | item.pack_version = att["value"] 64 | 65 | def write_to_csv(list_critical): 66 | if(len(list_critical) > 0): 67 | csv_columns = ["Repo","package_name","version","uri"] 68 | with open("repo_with_CRITICAL_issues.csv", 'w', newline='', encoding='utf8') as f: 69 | writer = csv.writer(f) 70 | writer.writerow(csv_columns) 71 | for ob in list_critical: 72 | row = [ob.repo_name, ob.pack_name,ob.pack_version,ob.uri] 73 | writer.writerow(row) 74 | 75 | def print_to_json(data): 76 | with open('data.json', 'w') as f: 77 | json.dump(data, f) 78 | 79 | def main(): 80 | # get all the repos 81 | repositories = get_all_repos() 82 | 83 | for repo in repositories: 84 | repo_name = repo["repositoryName"] 85 | print(f" --> repo_name: {repo_name}") 86 | print (f'------------- Starting scanning : {repo_name}') 87 | 88 | # get latest image for the repo 89 | image_Digest = get_latest_img_digest(repo_name) 90 | 91 | print(f'------------- Started the scanning: {repo_name}') 92 | start_scanning(repo_name, image_Digest) 93 | 94 | # waiting scan result 95 | print(f'------------- Waiting to finish the scanning: {repo_name}') 96 | wait_scan_results(repo_name, image_Digest) 97 | 98 | # get scan results 99 | json_finding_result = get_scan_result(repo_name, image_Digest) 100 | 101 | status = json_finding_result['imageScanStatus']['status'] 102 | print(f'Status of scan: {status}') 103 | if(status == "FAILED"): 104 | print(f'>>>>>>>>>>>>>>>>>>>Failed to scan the repo: {repo_name}') 105 | continue 106 | 107 | # finding vulnarabilities 108 | finding_vulnarablity(repo_name, json_finding_result) 109 | 110 | # write all critical issues to csv file 111 | write_to_csv(list_critical) 112 | 113 | print(f'Repos with issues: {len(list_critical)}') 114 | print(f'=====================================================') 115 | 116 | if __name__ == "__main__": 117 | main() 118 | 119 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Demo -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/.gitignore: -------------------------------------------------------------------------------- 1 | # From .NET Core 3.0 you can use the command: `dotnet new gitignore` to generate a customizable .gitignore file 2 | 3 | # custom 4 | *logs 5 | 6 | *.swp 7 | *.*~ 8 | project.lock.json 9 | .DS_Store 10 | *.pyc 11 | 12 | # Visual Studio Code 13 | .vscode 14 | 15 | # User-specific files 16 | *.suo 17 | *.user 18 | *.userosscache 19 | *.sln.docstates 20 | 21 | # Build results 22 | [Dd]ebug/ 23 | [Dd]ebugPublic/ 24 | [Rr]elease/ 25 | [Rr]eleases/ 26 | x64/ 27 | x86/ 28 | build/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | msbuild.log 33 | msbuild.err 34 | msbuild.wrn 35 | 36 | # Visual Studio 2015 37 | .vs/ 38 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/Controllers/MyController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Threading; 6 | using Microsoft.AspNetCore.Mvc; 7 | using Microsoft.Extensions.Logging; 8 | 9 | namespace DotNetCoreSumoLogic.Controllers 10 | { 11 | [ApiController] 12 | [Route("[controller]")] 13 | public class MyController : ControllerBase 14 | { 15 | private static readonly string[] Summaries = new[] 16 | { 17 | "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" 18 | }; 19 | 20 | private readonly ILogger _logger; 21 | 22 | public MyController(ILogger logger) 23 | { 24 | _logger = logger; 25 | } 26 | 27 | [HttpGet] 28 | public IEnumerable Get() 29 | { 30 | _logger.LogInformation("Logging mycontroller index route"); 31 | var rng = new Random(); 32 | return Enumerable.Range(1, 5).Select(index => new WeatherForecast 33 | { 34 | Date = DateTime.Now.AddDays(index), 35 | TemperatureC = rng.Next(-20, 55), 36 | Summary = Summaries[rng.Next(Summaries.Length)] 37 | }) 38 | .ToArray(); 39 | } 40 | 41 | [HttpGet] 42 | [Route("country")] 43 | public IActionResult Country() 44 | { 45 | // Simulate a random delay. So we can get different response time. 46 | Stopwatch watch = new Stopwatch(); 47 | watch.Start(); 48 | Thread.Sleep(new Random().Next(1000, 10000)); 49 | watch.Stop(); 50 | 51 | _logger.LogInformation($"country endpoint took: {watch.ElapsedMilliseconds} to process."); 52 | 53 | return Ok("Australia"); 54 | } 55 | 56 | [HttpGet] 57 | [Route("city")] 58 | public IActionResult City() 59 | { 60 | var cities = new[] { "Melbourne", "Sydney", "Canberra", "Brisbane" }; 61 | 62 | var randomNumber = new Random().Next(0, 4); 63 | var randomCity = cities[randomNumber]; 64 | 65 | _logger.LogInformation($"City endpoint returns {randomCity}."); 66 | return Ok(randomCity); 67 | } 68 | 69 | [HttpGet] 70 | [Route("cityWithDelay")] 71 | public IActionResult CityWithDelay() 72 | { 73 | Stopwatch watch = new Stopwatch(); 74 | watch.Start(); 75 | Thread.Sleep(new Random().Next(1000, 10000)); 76 | 77 | var cities = new[] { "Melbourne", "Sydney", "Canberra", "Brisbane" }; 78 | var randomNumber = new Random().Next(0, 4); 79 | var randomCity = cities[randomNumber]; 80 | 81 | watch.Stop(); 82 | _logger.LogInformation($"Delay City endpoint returns {randomCity} took: {watch.ElapsedMilliseconds}"); 83 | return Ok(randomCity); 84 | } 85 | 86 | [HttpGet] 87 | [Route("random-error")] 88 | public IActionResult RandomError() 89 | { 90 | var randomNumber = new Random().Next(1, 10); 91 | try 92 | { 93 | switch (randomNumber) 94 | { 95 | case 1: 96 | throw new Exception($"my custom error message: {Guid.NewGuid()}"); 97 | case 2: 98 | var ob = new Person(); 99 | // want to create an exception 100 | var street = ob.Address.Street; 101 | break; 102 | case 3: 103 | var listPerson = new List { new Person() { }, new Person() { } }; 104 | // want to create an exception 105 | var person = listPerson.SingleOrDefault(); 106 | break; 107 | case 4: 108 | case 5: 109 | case 6: 110 | case 7: 111 | case 8: 112 | case 9: 113 | throw new Exception("NOT IMPLEMENTED ERROR"); 114 | } 115 | } 116 | catch (Exception ex) 117 | { 118 | _logger.LogError($"Error occurred. Exception: {ex}"); 119 | } 120 | 121 | return Ok(); 122 | } 123 | } 124 | 125 | public class Person 126 | { 127 | public Address Address { get; set; } 128 | } 129 | 130 | public class Address 131 | { 132 | public string Street { get; set; } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/DotNetCoreSumoLogic.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net6.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/Program.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Hosting; 2 | using Microsoft.Extensions.Hosting; 3 | 4 | namespace DotNetCoreSumoLogic 5 | { 6 | public class Program 7 | { 8 | public static void Main(string[] args) 9 | { 10 | CreateHostBuilder(args).Build().Run(); 11 | } 12 | 13 | public static IHostBuilder CreateHostBuilder(string[] args) => 14 | Host.CreateDefaultBuilder(args) 15 | .ConfigureWebHostDefaults(webBuilder => 16 | { 17 | webBuilder.UseStartup(); 18 | }); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/launchsettings.json", 3 | "iisSettings": { 4 | "windowsAuthentication": false, 5 | "anonymousAuthentication": true, 6 | "iisExpress": { 7 | "applicationUrl": "http://localhost:9264", 8 | "sslPort": 44373 9 | } 10 | }, 11 | "profiles": { 12 | "IIS Express": { 13 | "commandName": "IISExpress", 14 | "launchBrowser": true, 15 | "launchUrl": "swagger", 16 | "environmentVariables": { 17 | "ASPNETCORE_ENVIRONMENT": "Development" 18 | } 19 | }, 20 | "DotNetCoreSumoLogic": { 21 | "commandName": "Project", 22 | "dotnetRunMessages": "true", 23 | "launchBrowser": true, 24 | "launchUrl": "swagger", 25 | "applicationUrl": "https://localhost:5001;http://localhost:5000", 26 | "environmentVariables": { 27 | "ASPNETCORE_ENVIRONMENT": "Development" 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/Startup.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Builder; 2 | using Microsoft.AspNetCore.Hosting; 3 | using Microsoft.Extensions.Configuration; 4 | using Microsoft.Extensions.DependencyInjection; 5 | using Microsoft.Extensions.Hosting; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.OpenApi.Models; 8 | using SumoLogic.Logging.AspNetCore; 9 | 10 | namespace DotNetCoreSumoLogic 11 | { 12 | public class Startup 13 | { 14 | public Startup(IConfiguration configuration) 15 | { 16 | Configuration = configuration; 17 | } 18 | 19 | public IConfiguration Configuration { get; } 20 | 21 | // This method gets called by the runtime. Use this method to add services to the container. 22 | public void ConfigureServices(IServiceCollection services) 23 | { 24 | 25 | services.AddControllers(); 26 | services.AddSwaggerGen(c => 27 | { 28 | c.SwaggerDoc("v1", new OpenApiInfo { Title = "DotNetCoreSumoLogic", Version = "v1" }); 29 | }); 30 | } 31 | 32 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 33 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory) 34 | { 35 | loggerFactory.AddSumoLogic( 36 | new LoggerOptions 37 | { 38 | Uri = "PLEASE USE YOUR OWN COLLECTOR END POINT HERE", 39 | IsBuffered = false 40 | }); 41 | 42 | if (env.IsDevelopment()) 43 | { 44 | app.UseDeveloperExceptionPage(); 45 | app.UseSwagger(); 46 | app.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", "DotNetCoreSumoLogic v1")); 47 | } 48 | app.UseHttpsRedirection(); 49 | 50 | app.UseRouting(); 51 | 52 | app.UseAuthorization(); 53 | 54 | app.UseEndpoints(endpoints => 55 | { 56 | endpoints.MapControllers(); 57 | }); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/WeatherForecast.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetCoreSumoLogic 4 | { 5 | public class WeatherForecast 6 | { 7 | public DateTime Date { get; set; } 8 | 9 | public int TemperatureC { get; set; } 10 | 11 | public int TemperatureF => 32 + (int)(TemperatureC / 0.5556); 12 | 13 | public string Summary { get; set; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /SumoLogic_DotNetCore_End-To_End/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | }, 9 | "AllowedHosts": "*" 10 | } 11 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/.gitignore: -------------------------------------------------------------------------------- 1 | # From .NET Core 3.0 you can use the command: `dotnet new gitignore` to generate a customizable .gitignore file 2 | 3 | # custom 4 | *logs 5 | 6 | *.swp 7 | *.*~ 8 | project.lock.json 9 | .DS_Store 10 | *.pyc 11 | 12 | # Visual Studio Code 13 | .vscode 14 | 15 | # User-specific files 16 | *.suo 17 | *.user 18 | *.userosscache 19 | *.sln.docstates 20 | 21 | # Build results 22 | [Dd]ebug/ 23 | [Dd]ebugPublic/ 24 | [Rr]elease/ 25 | [Rr]eleases/ 26 | x64/ 27 | x86/ 28 | build/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | msbuild.log 33 | msbuild.err 34 | msbuild.wrn 35 | 36 | # Visual Studio 2015 37 | .vs/ 38 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/Controllers/WeatherForecastController.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Mvc; 2 | 3 | namespace DotnetSerilog.Controllers; 4 | 5 | [ApiController] 6 | [Route("[controller]")] 7 | public class WeatherForecastController : ControllerBase 8 | { 9 | private static readonly string[] Summaries = new[] 10 | { 11 | "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" 12 | }; 13 | 14 | private readonly ILogger _logger; 15 | 16 | public WeatherForecastController(ILogger logger) 17 | { 18 | _logger = logger; 19 | } 20 | 21 | [HttpGet(Name = "GetWeatherForecast")] 22 | public IEnumerable Get() 23 | { 24 | _logger.LogInformation("Logging mycontroller index route"); 25 | 26 | return Enumerable.Range(1, 5).Select(index => new WeatherForecast 27 | { 28 | Date = DateTime.Now.AddDays(index), 29 | TemperatureC = Random.Shared.Next(-20, 55), 30 | Summary = Summaries[Random.Shared.Next(Summaries.Length)] 31 | }) 32 | .ToArray(); 33 | } 34 | [HttpGet] 35 | [Route("parse-as-json")] 36 | public IActionResult ParseAsJson() 37 | { 38 | _logger.LogInformation("Calling route 'parse-as-json'"); 39 | var randomBoolean = new Random().Next(1, 10)/2 == 1? true: false; 40 | var randomString = randomBoolean? "C#": "JavaScript"; 41 | 42 | var message = new Message{ 43 | Id= Guid.NewGuid(), 44 | Type = randomBoolean? "HTTP": "HTTPS", 45 | Data = new SupportedType []{ 46 | new SupportedType{ VerbType= "POST", Message = "Test message: " + randomString}, 47 | new SupportedType{ VerbType= "GET", Message = "Test message: " + randomString}, 48 | new SupportedType{ VerbType= "DELETE", Message = "Test message: " + randomString}, 49 | } 50 | }; 51 | 52 | _logger.LogInformation("{@message}",message); 53 | return Ok(message); 54 | } 55 | } 56 | 57 | 58 | public class Message 59 | { 60 | public Guid Id { get; set; } 61 | public string Type { get; set; } 62 | public SupportedType[] Data { get; set; } 63 | } 64 | 65 | public class SupportedType 66 | { 67 | public string VerbType { get; set; } 68 | public string Message { get; set; } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/DotnetSerilog.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net6.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.Configuration; 3 | using Serilog.Formatting.Compact; 4 | using Serilog; 5 | using SumoLogic.Logging.Serilog.Extensions; 6 | 7 | var builder = WebApplication.CreateBuilder(args); 8 | 9 | // Add services to the container. 10 | 11 | var logger = new LoggerConfiguration() 12 | .ReadFrom.Configuration(builder.Configuration) 13 | .WriteTo.Console() 14 | .WriteTo.BufferedSumoLogic( 15 | new Uri("YOUR SUMOLOGIC COLLETOR URL"), 16 | sourceName: "ExampleNameSerilogBufferedSink", 17 | formatter: new CompactJsonFormatter()) 18 | .Enrich.FromLogContext() 19 | .CreateLogger(); 20 | 21 | builder.Logging.AddSerilog(logger); 22 | 23 | builder.Services.AddControllers(); 24 | // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle 25 | builder.Services.AddEndpointsApiExplorer(); 26 | builder.Services.AddSwaggerGen(); 27 | 28 | 29 | 30 | var app = builder.Build(); 31 | 32 | // Configure the HTTP request pipeline. 33 | if (app.Environment.IsDevelopment()) 34 | { 35 | app.UseSwagger(); 36 | app.UseSwaggerUI(); 37 | } 38 | app.UseHttpsRedirection(); 39 | 40 | app.UseAuthorization(); 41 | 42 | app.MapControllers(); 43 | 44 | app.Run(); 45 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/launchsettings.json", 3 | "iisSettings": { 4 | "windowsAuthentication": false, 5 | "anonymousAuthentication": true, 6 | "iisExpress": { 7 | "applicationUrl": "http://localhost:43584", 8 | "sslPort": 44308 9 | } 10 | }, 11 | "profiles": { 12 | "DotnetSerilog": { 13 | "commandName": "Project", 14 | "dotnetRunMessages": true, 15 | "launchBrowser": true, 16 | "launchUrl": "swagger", 17 | "applicationUrl": "https://localhost:5001;http://localhost:5235", 18 | "environmentVariables": { 19 | "ASPNETCORE_ENVIRONMENT": "Development" 20 | } 21 | }, 22 | "IIS Express": { 23 | "commandName": "IISExpress", 24 | "launchBrowser": true, 25 | "launchUrl": "swagger", 26 | "environmentVariables": { 27 | "ASPNETCORE_ENVIRONMENT": "Development" 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/Readme.md: -------------------------------------------------------------------------------- 1 | ## DotNet Web API integrate with Serilog and Sumologic 2 | 3 | ## Step by Step guide 4 | You can find the step by step guidance in this video: TBD 5 | 6 | ## Introduction 7 | This project is a step by step guide to setup a `dotnet 6` peoject with `Serilog` logs and `Sumologic`. 8 | After setting up the project, the logs should be available in Sumologic application and optionally you can see in logs file in local folder. The path to the log file in the folder will be config in the `appsettings.json` file. Tested the below packages with dotnet 6 and should be work latest dotnet version like `dotnet 7, 8`, etc 9 | 10 | ## Install packages using `dotnet` cli 11 | dotnet add package Serilog --version=3.1.1 12 | dotnet add package Serilog.AspNetCore --version=8.0.0 13 | dotnet add package Serilog.Formatting.Compact --version=2.0.0 14 | dotnet add package Serilog.Settings.Configuration --version=8.0.0 15 | dotnet add package Serilog.Sinks.SumoLogic --version=2.4.0 16 | dotnet add package SumoLogic.Logging.Serilog --version=1.0.1.7 17 | dotnet add package Serilog --version=3.1.1 18 | 19 | ## Nuget packages in project file (.csproj) should be as below after install above packages 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | ## appsetting.json file 31 | 32 | ``` json 33 | { 34 | "Logging": { 35 | "LogLevel": { 36 | "Default": "Debug", 37 | "Microsoft.AspNetCore": "Warning" 38 | } 39 | }, 40 | "AllowedHosts": "*", 41 | "Serilog": { 42 | "Using": [ "Serilog.Sinks.File" ], 43 | "MinimumLevel": { 44 | "Default": "Information" 45 | }, 46 | "WriteTo": [ 47 | { 48 | "Name": "File", 49 | "Args": { 50 | "path": "./logs/webapi-.log", 51 | "rollingInterval": "Day", 52 | "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId} {Level:u3} {Username} {Message:lj}{Exception}{NewLine}" 53 | } 54 | } 55 | 56 | ] 57 | } 58 | } 59 | 60 | ``` 61 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/WeatherForecast.cs: -------------------------------------------------------------------------------- 1 | namespace DotnetSerilog; 2 | 3 | public class WeatherForecast 4 | { 5 | public DateTime Date { get; set; } 6 | 7 | public int TemperatureC { get; set; } 8 | 9 | public int TemperatureF => 32 + (int)(TemperatureC / 0.5556); 10 | 11 | public string? Summary { get; set; } 12 | } 13 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /SumoLogic_Serilog/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | }, 8 | "AllowedHosts": "*", 9 | "Serilog": { 10 | "Using": [ "Serilog.Sinks.File" ], 11 | "MinimumLevel": { 12 | "Default": "Information" 13 | }, 14 | "WriteTo": [ 15 | { 16 | "Name": "File", 17 | "Args": { 18 | "path": "./logs/webapi-.log", 19 | "rollingInterval": "Day", 20 | "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId} {Level:u3} {Username} {Message:lj}{Exception}{NewLine}" 21 | } 22 | } 23 | 24 | ] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/.gitignore: -------------------------------------------------------------------------------- 1 | # From .NET Core 3.0 you can use the command: `dotnet new gitignore` to generate a customizable .gitignore file 2 | 3 | # custom 4 | *logs 5 | 6 | *.swp 7 | *.*~ 8 | project.lock.json 9 | .DS_Store 10 | *.pyc 11 | 12 | # Visual Studio Code 13 | .vscode 14 | 15 | # User-specific files 16 | *.suo 17 | *.user 18 | *.userosscache 19 | *.sln.docstates 20 | 21 | # Build results 22 | [Dd]ebug/ 23 | [Dd]ebugPublic/ 24 | [Rr]elease/ 25 | [Rr]eleases/ 26 | x64/ 27 | x86/ 28 | build/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | msbuild.log 33 | msbuild.err 34 | msbuild.wrn 35 | 36 | # Visual Studio 2015 37 | .vs/ 38 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/Controllers/MyController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Threading; 6 | using Microsoft.AspNetCore.Mvc; 7 | using Microsoft.Extensions.Logging; 8 | 9 | namespace DotNetCoreSumoLogic.Controllers 10 | { 11 | [ApiController] 12 | [Route("[controller]")] 13 | public class MyController : ControllerBase 14 | { 15 | private static readonly string[] Summaries = new[] 16 | { 17 | "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" 18 | }; 19 | 20 | private readonly ILogger _logger; 21 | 22 | public MyController(ILogger logger) 23 | { 24 | _logger = logger; 25 | } 26 | 27 | [HttpGet] 28 | public IEnumerable Get() 29 | { 30 | _logger.LogInformation("Logging mycontroller index route"); 31 | var rng = new Random(); 32 | return Enumerable.Range(1, 5).Select(index => new WeatherForecast 33 | { 34 | Date = DateTime.Now.AddDays(index), 35 | TemperatureC = rng.Next(-20, 55), 36 | Summary = Summaries[rng.Next(Summaries.Length)] 37 | }) 38 | .ToArray(); 39 | } 40 | 41 | [HttpGet] 42 | [Route("country")] 43 | public IActionResult Country() 44 | { 45 | // Simulate a random delay. So we can get different response time. 46 | Stopwatch watch = new Stopwatch(); 47 | watch.Start(); 48 | Thread.Sleep(new Random().Next(1000, 10000)); 49 | watch.Stop(); 50 | 51 | _logger.LogInformation($"country endpoint took: {watch.ElapsedMilliseconds} to process."); 52 | 53 | return Ok("Australia"); 54 | } 55 | 56 | [HttpGet] 57 | [Route("city")] 58 | public IActionResult City() 59 | { 60 | var cities = new[] { "Melbourne", "Sydney", "Canberra", "Brisbane" }; 61 | 62 | var randomNumber = new Random().Next(0, 4); 63 | var randomCity = cities[randomNumber]; 64 | 65 | _logger.LogInformation($"City endpoint returns {randomCity}."); 66 | return Ok(randomCity); 67 | } 68 | 69 | [HttpGet] 70 | [Route("cityWithDelay")] 71 | public IActionResult CityWithDelay() 72 | { 73 | Stopwatch watch = new Stopwatch(); 74 | watch.Start(); 75 | Thread.Sleep(new Random().Next(1000, 10000)); 76 | 77 | var cities = new[] { "Melbourne", "Sydney", "Canberra", "Brisbane" }; 78 | var randomNumber = new Random().Next(0, 4); 79 | var randomCity = cities[randomNumber]; 80 | 81 | watch.Stop(); 82 | _logger.LogInformation($"Delay City endpoint returns {randomCity} took: {watch.ElapsedMilliseconds}"); 83 | return Ok(randomCity); 84 | } 85 | 86 | [HttpGet] 87 | [Route("random-error")] 88 | public IActionResult RandomError() 89 | { 90 | var randomNumber = new Random().Next(1, 10); 91 | try 92 | { 93 | switch (randomNumber) 94 | { 95 | case 1: 96 | throw new Exception($"my custom error message: {Guid.NewGuid()}"); 97 | case 2: 98 | var ob = new Person(); 99 | // want to create an exception 100 | var street = ob.Address.Street; 101 | break; 102 | case 3: 103 | var listPerson = new List { new Person() { }, new Person() { } }; 104 | // want to create an exception 105 | var person = listPerson.SingleOrDefault(); 106 | break; 107 | case 4: 108 | case 5: 109 | case 6: 110 | case 7: 111 | case 8: 112 | case 9: 113 | throw new Exception("NOT IMPLEMENTED ERROR"); 114 | } 115 | } 116 | catch (Exception ex) 117 | { 118 | _logger.LogError($"Error occurred. Exception: {ex}"); 119 | } 120 | 121 | return Ok(); 122 | } 123 | } 124 | 125 | public class Person 126 | { 127 | public Address Address { get; set; } 128 | } 129 | 130 | public class Address 131 | { 132 | public string Street { get; set; } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/DotNetCoreSumoLogic.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net6.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/Program.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Hosting; 2 | using Microsoft.Extensions.Hosting; 3 | 4 | namespace DotNetCoreSumoLogic 5 | { 6 | public class Program 7 | { 8 | public static void Main(string[] args) 9 | { 10 | CreateHostBuilder(args).Build().Run(); 11 | } 12 | 13 | public static IHostBuilder CreateHostBuilder(string[] args) => 14 | Host.CreateDefaultBuilder(args) 15 | .ConfigureWebHostDefaults(webBuilder => 16 | { 17 | webBuilder.UseStartup(); 18 | }); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/launchsettings.json", 3 | "iisSettings": { 4 | "windowsAuthentication": false, 5 | "anonymousAuthentication": true, 6 | "iisExpress": { 7 | "applicationUrl": "http://localhost:9264", 8 | "sslPort": 44373 9 | } 10 | }, 11 | "profiles": { 12 | "IIS Express": { 13 | "commandName": "IISExpress", 14 | "launchBrowser": true, 15 | "launchUrl": "swagger", 16 | "environmentVariables": { 17 | "ASPNETCORE_ENVIRONMENT": "Development" 18 | } 19 | }, 20 | "DotNetCoreSumoLogic": { 21 | "commandName": "Project", 22 | "dotnetRunMessages": "true", 23 | "launchBrowser": true, 24 | "launchUrl": "swagger", 25 | "applicationUrl": "https://localhost:5001;http://localhost:5000", 26 | "environmentVariables": { 27 | "ASPNETCORE_ENVIRONMENT": "Development" 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/Startup.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Builder; 2 | using Microsoft.AspNetCore.Hosting; 3 | using Microsoft.Extensions.Configuration; 4 | using Microsoft.Extensions.DependencyInjection; 5 | using Microsoft.Extensions.Hosting; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.OpenApi.Models; 8 | using SumoLogic.Logging.AspNetCore; 9 | 10 | namespace DotNetCoreSumoLogic 11 | { 12 | public class Startup 13 | { 14 | public Startup(IConfiguration configuration) 15 | { 16 | Configuration = configuration; 17 | } 18 | 19 | public IConfiguration Configuration { get; } 20 | 21 | // This method gets called by the runtime. Use this method to add services to the container. 22 | public void ConfigureServices(IServiceCollection services) 23 | { 24 | 25 | services.AddControllers(); 26 | services.AddSwaggerGen(c => 27 | { 28 | c.SwaggerDoc("v1", new OpenApiInfo { Title = "DotNetCoreSumoLogic", Version = "v1" }); 29 | }); 30 | } 31 | 32 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 33 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory) 34 | { 35 | loggerFactory.AddSumoLogic( 36 | new LoggerOptions 37 | { 38 | Uri = "PLEASE USE YOUR OWN COLLECTOR END POINT HERE", 39 | IsBuffered = false 40 | }); 41 | 42 | if (env.IsDevelopment()) 43 | { 44 | app.UseDeveloperExceptionPage(); 45 | app.UseSwagger(); 46 | app.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", "DotNetCoreSumoLogic v1")); 47 | } 48 | app.UseHttpsRedirection(); 49 | 50 | app.UseRouting(); 51 | 52 | app.UseAuthorization(); 53 | 54 | app.UseEndpoints(endpoints => 55 | { 56 | endpoints.MapControllers(); 57 | }); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/WeatherForecast.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetCoreSumoLogic 4 | { 5 | public class WeatherForecast 6 | { 7 | public DateTime Date { get; set; } 8 | 9 | public int TemperatureC { get; set; } 10 | 11 | public int TemperatureF => 32 + (int)(TemperatureC / 0.5556); 12 | 13 | public string Summary { get; set; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /SumoLogic_Slack/DotNetCoreSumoLogic/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | }, 9 | "AllowedHosts": "*" 10 | } 11 | -------------------------------------------------------------------------------- /SumoLogic_Slack/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Query to test the Sumologic connection 3 | 4 | ``` 5 | { 6 | "attachments": [ 7 | { 8 | "pretext": "{{TriggerType}} Alert: {{AlertName}}", 9 | "fields": [ 10 | { 11 | "title": "Alert URL", 12 | "value": "{{AlertResponseURL}}" 13 | }, 14 | { 15 | "title": "Description", 16 | "value": "{{Description}}" 17 | }, 18 | { 19 | "title": "Trigger Time", 20 | "value": "{{TriggerTime}}" 21 | }, 22 | { 23 | "title": "Time Range", 24 | "value": "{{TriggerTimeRange}}" 25 | }, 26 | { 27 | "title": "Trigger Condition", 28 | "value": "{{TriggerCondition}}" 29 | }, 30 | { 31 | "title": "Trigger Value", 32 | "value": "{{TriggerValue}}" 33 | }, 34 | { 35 | "title": "Query", 36 | "value": "<{{QueryURL}} | {{Query}}>" 37 | } 38 | ], 39 | "mrkdwn_in": ["text", "pretext"], 40 | "color": "#29A1E6" 41 | } 42 | ] 43 | } 44 | ``` 45 | 46 | 47 | # Sumologic Query 48 | 49 | ``` 50 | // change the _sourceCategory and filter (eg: "Error") as per your logs 51 | _sourceCategory="dev/test-app" 52 | AND "Error" 53 | | formatDate(_receiptTime, "yyyy-MM-dd") as date 54 | | count date 55 | 56 | ``` 57 | -------------------------------------------------------------------------------- /TextractAsync/README.md: -------------------------------------------------------------------------------- 1 | ## Textract AWS tutorial. 2 | This video shows how to extract FORMS, TABLES from pdf using ` start-document-analysis` method. 3 | 4 | - AWS Textract Async: https://youtu.be/BNnFfTZsmjc 5 | - AWS Textract Sync: https://www.youtube.com/watch?v=xlUSJEsFQdk&t=669s 6 | 7 | - Source: https://github.com/CodeSam621/Demo/tree/main/TextractAsync 8 | 9 | ## AWS doc: 10 | - https://awscli.amazonaws.com/v2/documentation/api/latest/reference/textract/start-document-analysis.html 11 | - https://awscli.amazonaws.com/v2/documentation/api/latest/reference/textract/analyze-document.html 12 | -------------------------------------------------------------------------------- /TextractAsync/lambda/document_analyzer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from time import sleep 4 | import logging 5 | from boto3 import client 6 | 7 | logger = logging.getLogger() 8 | logging.basicConfig() 9 | logger.setLevel(logging.INFO) 10 | 11 | textract_client = client('textract') 12 | 13 | WAIT_TIME_FOR_RESULTS = os.environ.get("WAIT_TIME_FOR_RESULTS", 2) 14 | 15 | def analyze_document(bucket, key): 16 | 17 | blocks = trigger_analysis(bucket, key) 18 | 19 | # for debuging 20 | text_file = open(f'./responses/test_block.json', "w") 21 | text_file.write(json.dumps(blocks)) 22 | text_file.close() 23 | 24 | key_map = {} 25 | value_map = {} 26 | block_map = {} 27 | table_blocks = [] 28 | for block in blocks: 29 | block_id = block['Id'] 30 | block_map[block_id] = block 31 | if block['BlockType'] == "KEY_VALUE_SET": 32 | if 'KEY' in block['EntityTypes']: 33 | key_map[block_id] = block 34 | else: 35 | value_map[block_id] = block 36 | elif block['BlockType'] == "TABLE": 37 | table_blocks.append(block) 38 | 39 | return key_map, value_map, block_map, table_blocks 40 | 41 | def trigger_analysis(bucket, key): 42 | response = textract_client.start_document_analysis( 43 | DocumentLocation={ 44 | 'S3Object': { 45 | 'Bucket': bucket, 46 | 'Name': key 47 | } 48 | }, 49 | FeatureTypes=["FORMS", "TABLES"] 50 | ) 51 | job_id = response['JobId'] 52 | logger.info(f"\t\t\t\t -----------Job_id: {job_id} --------------\n\n") 53 | 54 | block_list = [] 55 | get_analysis_result(job_id, next_token = None, block_list = block_list) 56 | return block_list 57 | 58 | def get_analysis_result(job_id, next_token, block_list): 59 | if(next_token == None): 60 | result = textract_client.get_document_analysis(JobId=job_id) 61 | else: 62 | result = textract_client.get_document_analysis(JobId=job_id, NextToken=next_token) 63 | 64 | blocks = result.get("Blocks") 65 | 66 | if(blocks != None): 67 | # logger.info(f'Analysis blocks: {blocks}') 68 | block_list.extend(blocks) 69 | if(result.get("NextToken") != None): 70 | get_analysis_result(job_id, result["NextToken"], block_list ) 71 | 72 | else: 73 | logger.info(f'>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Waiting to finish the analysis >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') 74 | logger.info(f'Analysis result: {result}') 75 | sleep(int(WAIT_TIME_FOR_RESULTS)) 76 | get_analysis_result(job_id, None, block_list) -------------------------------------------------------------------------------- /TextractAsync/lambda/lambda_function.py: -------------------------------------------------------------------------------- 1 | import json 2 | from boto3 import resource 3 | import logging 4 | from processor.document_processor import process_forms_data, process_table_data 5 | from document_analyzer import analyze_document 6 | 7 | logger = logging.getLogger() 8 | logging.basicConfig() 9 | logger.setLevel(logging.INFO) 10 | 11 | def lambda_handler(event, context): 12 | logger.info(f' \n\n\t ---------------------------------- STARTED ----------------------------------\n\n') 13 | logger.info(f'event: { event }') 14 | try: 15 | if(event.get("bucket") == None or event.get("file_key") == None): 16 | return {'status_code': 400, 'message': 'Both bucket and file_key should included in the request'} 17 | 18 | bucket = event["bucket"] 19 | key = event["file_key"] 20 | 21 | key_map, value_map, block_map, table_blocks = analyze_document(bucket, key) 22 | 23 | # process key-value 24 | key_value_maps = process_forms_data(key_map, value_map, block_map) 25 | for key in key_value_maps: 26 | logger.info(f'\t\tForm key: {key}, value: {key_value_maps[key]}') 27 | 28 | # process tables 29 | table_data = process_table_data(block_map, table_blocks) 30 | logger.info(f'\n\n\nTables: {table_data}') 31 | 32 | except Exception as exp: 33 | logger.info(f'Failed extraction. Exception: {str(exp)}') 34 | return {'status_code': 500, 'message': 'Failed extraction'} 35 | 36 | 37 | 38 | # tigger lambda locally 39 | event = {} 40 | event["file_key"] = "sample3.pdf" 41 | event["bucket"] = "sam-textract-test-bucket-new" 42 | lambda_handler(event, None) 43 | -------------------------------------------------------------------------------- /TextractAsync/lambda/processor/document_processor.py: -------------------------------------------------------------------------------- 1 | 2 | from collections import defaultdict 3 | import logging 4 | 5 | logger = logging.getLogger() 6 | logging.basicConfig() 7 | logger.setLevel(logging.INFO) 8 | 9 | def process_forms_data(key_map, value_map, block_map): 10 | kvs = defaultdict(list) 11 | for block_id, key_block in key_map.items(): 12 | value_block = get_value_block(key_block, value_map) 13 | key, confident_of_key = get_text_form(key_block, block_map) 14 | value, confident_of_value = get_text_form(value_block, block_map) 15 | if(value != None): 16 | value = value.strip() 17 | data = {'value': value, 'confident_of_key': confident_of_key, 18 | 'confident_of_value': confident_of_value} 19 | kvs[key].append(data) 20 | return kvs 21 | 22 | 23 | def get_value_block(key_block, value_map): 24 | for relationship in key_block['Relationships']: 25 | if relationship['Type'] == 'VALUE': 26 | for value_id in relationship['Ids']: 27 | value_block = value_map[value_id] 28 | return value_block 29 | 30 | 31 | def get_text_form(result, blocks_map): 32 | text = '' 33 | confident_level = 0 34 | number_of_words = 0 35 | if 'Relationships' in result: 36 | for relationship in result['Relationships']: 37 | if relationship['Type'] == 'CHILD': 38 | for child_id in relationship['Ids']: 39 | word = blocks_map[child_id] 40 | if word['BlockType'] == 'WORD': 41 | text += word['Text'] + ' ' 42 | confident_level += float(word['Confidence']) 43 | number_of_words += 1 44 | if word['BlockType'] == 'SELECTION_ELEMENT': 45 | if word['SelectionStatus'] == 'SELECTED': 46 | text += 'X' 47 | confident_level += float(word['Confidence']) 48 | number_of_words += 1 49 | confident_level = confident_level if confident_level == 0 else (confident_level / number_of_words) 50 | return text, confident_level 51 | 52 | 53 | def get_text_table(result, blocks_map): 54 | text = '' 55 | confident_level = 0 56 | number_of_words = 0 57 | if 'Relationships' in result: 58 | for relationship in result['Relationships']: 59 | if relationship['Type'] == 'CHILD': 60 | for child_id in relationship['Ids']: 61 | word = blocks_map[child_id] 62 | if word['BlockType'] == 'WORD': 63 | text += word['Text'] + ' ' 64 | confident_level += float(word['Confidence']) 65 | number_of_words += 1 66 | if word['BlockType'] == 'SELECTION_ELEMENT': 67 | if word['SelectionStatus'] == 'SELECTED': 68 | text += 'X' 69 | confident_level += float(word['Confidence']) 70 | number_of_words += 1 71 | 72 | confident_level = confident_level if confident_level == 0 else (confident_level / number_of_words) 73 | return { "value": text, "confident" : confident_level } 74 | 75 | # extracting tables data 76 | def process_table_data(blocks_map, table_blocks): 77 | logger.info('\n\nProcessing table data') 78 | table_data = [] 79 | for index, table in enumerate(table_blocks): 80 | logger.info(f'======= TABLE BLOCK ->>: {index} =======') 81 | data = generate_table(table, blocks_map, index + 1) 82 | table_data.append(data) 83 | # logger.info(f'Table text: {data}') 84 | return table_data 85 | 86 | 87 | def generate_table(table_result, blocks_map, table_index): 88 | rows = get_rows_columns_map(table_result, blocks_map) 89 | 90 | table_data = [] 91 | for row_index, cols in rows.items(): 92 | data = [] 93 | for col_index, text in cols.items(): 94 | data.append(text) 95 | table_data.append(data) 96 | return table_data 97 | 98 | 99 | def get_rows_columns_map(table_result, blocks_map): 100 | rows = {} 101 | for relationship in table_result['Relationships']: 102 | if relationship['Type'] == 'CHILD': 103 | for child_id in relationship['Ids']: 104 | cell = blocks_map[child_id] 105 | if cell['BlockType'] == 'CELL': 106 | row_index = cell['RowIndex'] 107 | col_index = cell['ColumnIndex'] 108 | if row_index not in rows: 109 | rows[row_index] = {} 110 | 111 | rows[row_index][col_index] = get_text_table( 112 | cell, blocks_map) 113 | return rows 114 | -------------------------------------------------------------------------------- /TextractAsync/lambda/requrement.txt: -------------------------------------------------------------------------------- 1 | boto3 -------------------------------------------------------------------------------- /TextractAsync/package-deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | # echo $(pwd) 4 | 5 | LAMBDA_NAME="my-lambda-name" 6 | DEPLOY_FILE_PATH=$(pwd)/deploy/$LAMBDA_NAME.zip 7 | 8 | rm -f -r ./deploy/* 9 | cd ./lambda 10 | zip -r9 ../deploy/$LAMBDA_NAME.zip * -x \*.pyc responses/* results/* play_ground/* play_ground/temp_data/* 11 | 12 | echo "Path: $DEPLOY_FILE_PATH" 13 | 14 | echo "Updating lambda with env variable" 15 | aws lambda update-function-configuration --function-name $LAMBDA_NAME --environment Variables="{ENABLE_DEBUG_LOGS='True'}" 16 | echo "The Lambda updated with env variables" 17 | sleep 3 18 | 19 | echo "Updating '$LAMBDA_NAME' with the new package" 20 | aws lambda update-function-code --function-name $LAMBDA_NAME --zip-file fileb://$DEPLOY_FILE_PATH 21 | echo "The Lambda '$LAMBDA_NAME' updated succesfully......" -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | test 2 | --------------------------------------------------------------------------------