├── ELK-stack-setup.md ├── README.md ├── SaltStack ├── install-nginx.sls └── install-saltstack-ubuntu.md ├── angular ├── Dockerfile-angular-nginx.md ├── angular-build-new-env.txt ├── angular-command.md ├── angular-error-fix.md ├── angular-okta-integration.md ├── commands.md └── install-nodejs-ubuntu.md ├── ansible ├── ansible-install-in-ec2-RHEL.md ├── ansible-install-ubuntu.md ├── ansible-playbook-deploy-to-tomcat ├── ansible_playbook_configure_tomcat_amazon_linux.yml └── ansible_playbook_configure_tomcat_ubuntu.yml ├── apache └── install.md ├── aws ├── best-practices │ └── aws-startup-policy.json ├── cloudformation │ ├── cf-ec2-kubectl.yml │ ├── cf-ecs-fargate.yml │ ├── cf-eks-fargate.yml │ ├── cf_base-network-vpc.yml │ ├── cf_network_alb_vpc.yml │ ├── cf_network_wpsite.json │ ├── cf_vpc_peer_vpc1-vpc3-us-east-1.json │ ├── cf_vpc_peer_vpc2-canada.json │ ├── cfn_aws_workspace_network.json │ ├── cfn_networking__bastion_vpc.json │ ├── cfn_vpc_subnets.json │ ├── draft_cf_jenkins_network.yml │ └── readme.txt ├── cloudwatch │ └── amazon-cloudwatch-agent.json ├── codebuild │ ├── buildspec-eks-deployment-v01.yaml │ ├── buildspec-springbootdemo.yml │ └── codeartifact-token-codebuild.md ├── codedeploy │ ├── appspec_ecs_deploy.yml │ └── appspec_lambda_deploy.yml ├── codepipeline │ └── buildspec-angular-deploy.yml ├── data-pipeline │ ├── aws-data-pipeline-issues.txt │ └── sample_data_for_data_pipeline_demo_DynamoDB_format.json ├── docs │ ├── User is not authorized.md │ └── create-lambda-layer-for-python.md ├── ec2 │ ├── add-a-new-user-to-ssh-ec2.md │ ├── aws-configure.md │ ├── awscli-install.md │ ├── ec2-cloud-init-log.md │ ├── readme.md │ ├── shell-scripts │ │ ├── readme.md │ │ ├── userdata-clamav-rhel-docker.sh │ │ ├── userdata-clamav-ubuntu-docker.sh │ │ ├── userdata-clamav-ubuntu.sh │ │ ├── userdata-kubectl-vm.sh │ │ ├── userdata-nfs-mount-efs-RHEL.sh │ │ ├── userdata-win-v01.ps1 │ │ └── userdata-win-v02.ps1 │ ├── upload-file-ec2-s3.md │ └── userdata-nfs-mount-efs-RHEL.sh ├── efs │ ├── README.md │ └── nfs-mount-efs-ec2.md ├── eks │ ├── eks-bastian-host-ec2.md │ ├── eks-provide-access-to-iam-roles.md │ └── readme.md ├── iam │ └── policies │ │ ├── aws-policy-glue-crawler.json │ │ ├── e2esa-eks-demo.json │ │ ├── e2esa-sts-assume-role-policy.json │ │ ├── eks-permission-full.json │ │ ├── s3-allow-SSL-requestsonly.json │ │ ├── s3-policy-rds-snapshot-backup.json │ │ └── s3-restrict-by-user-role.md ├── install-aws-cli.md ├── lambda │ ├── CRUD-lambda-dynamodb-apigw-v3.js │ ├── CRUD-nodejs-apigw-lambda-dynamodb-v2.js │ ├── CRUD-nodejs-apigw-lambda-dynamodb.js │ ├── emi-calc.py │ ├── lambda- verify-face-rekognition-python.py │ ├── lambda-autogenerate-json-upload-to-s3.py │ ├── lambda-eventbridge-trigger-lambda-tocreate-sqs.py │ ├── lambda-file-upload-s3-sqs.py │ ├── lambda-generate-twitter-message-stream.py │ ├── lambda-nodejs-dynamodb.txt │ ├── lambda-nodejs-file-upload-to-efs.py │ ├── lambda-nodejs-upload-file-to-efs-v02.js │ ├── lambda-nodejs-upload-file-to-efs.js │ ├── lambda-presigned-url-file-upload-s3-metadata.py │ ├── lambda-python-file-archive.py │ ├── lambda-python-upload-file-v02.py │ ├── lambda-python-uploadfile-api.py │ ├── lambda-receive-message-from-sqs.py │ ├── lambda-send-message-to-sqs.py │ ├── lambda-send-sqs-message-on-s3-upload-v02.py │ ├── lambda-send-sqs-message-on-s3-upload.py │ ├── lambda-test-input.json │ ├── lambda-upload-file-to-s3.py │ ├── lambda_pandas.js │ ├── read-from-dynamodb.js │ ├── s3-object-tagging.py │ ├── write-to-dynamodb_v1.js │ └── write-to-dynamodb_v2.js ├── networking │ ├── cfn_networking_vpc.json │ └── cfn_vpc_subnets.json └── ssm │ ├── aws-configure-headless.md │ └── secrets-manager.md ├── azure ├── ado-pipeline.md └── policy │ ├── azure-custom-tags-allow-or-deny-policy.json │ ├── azure-custom-tags-check-append.json │ └── azure-custom-tags-list-allow-deny-policy.json ├── clamav ├── clamav-errors.md ├── install-clamav.md └── readme.md ├── create-maven-project-cmd.md ├── create-user.md ├── datascience ├── jupyter-notebook │ ├── List, Tuples, Sets in Python - Python Tutorial Part 1 for beginners.ipynb │ ├── Markdown in Jupyter Notebook.ipynb │ ├── Practice Pandas.ipynb │ ├── Sales_Analysis.ipynb │ ├── Sales_Trend_Analysis.ipynb │ └── seaborn practice.ipynb └── readme.md ├── django ├── create-django-project.md └── install-django.md ├── docker ├── Dockerfile │ ├── Dockerfile-tomEE808 │ └── tomEE-Dockerfile ├── config-file.cnf ├── docker save and load.md ├── docker-compose │ ├── docker-compose-jenkins.yml │ ├── docker-compose-keycloak-mysql-phpmyadmin.yml │ ├── docker-compose-keycloak-mysql.yml │ ├── docker-compose-phpmyadmin.yml │ ├── docker-compose-sonar.yml │ ├── docker-compose-tomEE.yml │ └── docker-compose-wordpress.yml ├── install-docker-aws-ec2.md ├── install-docker-compose.md ├── install-docker-ubuntu.md ├── install-mysql-and-phpmyadmin-on-docker.md ├── install-sonar-docker.md ├── install_jenkins_docker.md └── readme.md ├── docs └── aws │ └── Installing or updating the latest version of the AWS CLI.md ├── git ├── create-Git-PAT-Token.md ├── git-commands.md └── git-common-issues.md ├── gitaction └── pipelines │ └── deploy-terraform-github-aws-01.yml ├── hashicorp-vault ├── config.hcl ├── generating-dynamic-secrets-for-aws.md ├── install-vault.md ├── policy │ └── vault-policy-01.hcl ├── readme.md ├── vault-auth-github.md ├── vault-auth-methods.md ├── vault-aws.md ├── vault-commands.md ├── vault-encrypt-data.md ├── vault-lease.md ├── vault-policy.md ├── vault-production-steps.md ├── vault-secret-engine-kv.md ├── vault-secrets.md └── vault-token.md ├── java └── java-install.md ├── jenkins ├── Jenkinsfile-cicd-docker-ECR ├── Jenkinsfile.md ├── install-jenkins-docker.sh ├── install-tomcat-ec2-linux.md └── install-tomcat-in-ubuntu.md ├── jmeter ├── Dockerfile_01 ├── Dockerfile_02 ├── install-jmeter-docker-standalone.md ├── install-jmeter-ec2-distributed-load-test.md ├── jmeter-common-issues.md └── jmeter-html-report.md ├── kibana └── commands.txt ├── kubernetes ├── alias-and-shortcuts-for-kubernetes.txt ├── cluster-role-binding │ ├── eks-console-full-access.yaml │ └── project-a-ns.yaml ├── cluster-role │ └── clusterrole-01.yaml ├── commands │ ├── k8s-commands-config.md │ └── k8s-handy-commands.md ├── configure-kubeconfig.md ├── create-namespace.yaml ├── deployment │ ├── deployment-def1.yml │ ├── deployment-nginx-lb.yaml │ ├── deployment-nginx.yaml │ ├── deployment-with-service-account.yaml │ ├── myapp-deployment-jfrog.yaml │ ├── nginx-deployment.yaml │ ├── nginx-ecr-deployment-v02.yaml │ └── nginx-ecr-deployment.yaml ├── install-kubeadm-ubuntu-using-calico.txt ├── install-kubeadm-ubuntu-using-flannel-plugin.txt ├── pod │ ├── pod-def1.yml │ └── pod-multi-containers-def.yaml ├── readme.md ├── replicaset │ └── replicaset-def1.yml ├── role-binding │ └── rolebinding-01.yaml ├── role │ └── development-role-01.yaml ├── rolebinding │ └── eks-console-full-access.yaml ├── service-account │ └── serviceaccount-01.yaml ├── service │ ├── service-nlb-internal.yaml │ ├── service-wd-lb.yml │ ├── service-wd-nlb.yaml │ └── service-wo-lb.yml └── target-group-binding.yaml ├── markdown └── icons.md ├── mysql └── install-mysql-ubuntu.txt ├── nexus-install-in-ubuntu.md ├── nodejs └── aws-sqs-message-processor │ ├── consumer.js │ ├── download.js │ ├── index.js │ ├── package.json │ ├── producer.js │ └── readme.md ├── packer ├── aws-ami-v1.pkr.hcl ├── aws-ami-v2.json ├── aws-ami-v3.json ├── aws-ami-v4.pkr.hcl ├── aws-ami-v5.json ├── aws-ami-v6.json ├── aws-ami-v7.json ├── provision.sh └── readme.md ├── pipelines └── azure │ ├── angular-nginx-dkr-ecr-adopipeline.yml │ ├── docker-build-n-push-to-ecr-azurepipeline.yml │ ├── terraform-adopipeline.yml │ ├── terraform-dkr-adopipeline.yml │ └── terraform-scan-iac-adopipeline.yml ├── python ├── anaconda-commands.md ├── analyze-csv-pandas-lambda.py ├── create-conda-environment.md ├── cuda-install.md ├── import-pandas-aws-lambda.py ├── install-geopandas.md ├── install-plotly.md ├── morse-code-decoder.py ├── mysql-connector-python.md ├── python-call-rest-api.py ├── python-commands.md └── troubleshoot-python.md ├── terraform ├── readme.md └── run-terraform-offline.md ├── tomcat └── install-tomcat.md ├── wordpress ├── htaccess.txt └── wordpress-amazon-lightsail-s3.txt └── wsl └── install-wsl.md /ELK-stack-setup.md: -------------------------------------------------------------------------------- 1 | 2 | ## Install Java 3 | 4 | sudo apt-get update 5 | 6 | sudo apt-get install openjdk-8-jdk 7 | 8 | 9 | 10 | ## 1. Download and install public signing key 11 | 12 | wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - 13 | 14 | ## 2. Install apt-transport-https package 15 | 16 | sudo apt-get install apt-transport-https -y 17 | 18 | ## 3. Save directory definitions 19 | 20 | echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list 21 | 22 | 23 | ## 4. Update and Install elasticsearch 24 | 25 | sudo apt-get update && sudo apt-get install elasticsearch && sudo apt-get install logstash && sudo apt-get install kibana 26 | 27 | ## 5. configure elasticsearch 28 | 29 | sudo su 30 | nano /etc/elasticsearch/elasticsearch.yml 31 | 32 | change cluster name 33 | cluster.name: awstg-elk 34 | 35 | give the cluster a descriptive name 36 | node.name: aws-node 37 | 38 | change network binding 39 | network.host: 0.0.0.0 40 | 41 | setup discovery.type as single node 42 | discovery.type: single-node 43 | 44 | ## 6. Start Elasticsearch service 45 | 46 | sudo systemctl start elasticsearch 47 | 48 | ## 7. validate Elasticsearch cluster health 49 | 50 | curl -XGET http://localhost:9200/_cluster/health?pretty 51 | 52 | ## 8. configure kibana 53 | 54 | nano /etc/kibana/kibana.yml 55 | 56 | uncomment server.port 57 | server.port: 5601 58 | 59 | change server.host 60 | server.host: "0.0.0.0" 61 | 62 | change server.name 63 | server.name: "awstg-kibana" 64 | 65 | uncomment elasticsearch.host 66 | elasticsearch.hosts: ["http://localhost:9200"] 67 | 68 | ## 9. start Kibana service 69 | 70 | systemctl start kibana 71 | 72 | ## 10. enable elasticsearch and kibana 73 | 74 | systemctl enable elasticsearch 75 | systemctl enable kibana 76 | -------------------------------------------------------------------------------- /SaltStack/install-nginx.sls: -------------------------------------------------------------------------------- 1 | nginx: 2 | pkg: 3 | - installed 4 | service: 5 | - running 6 | - reload: true 7 | - enable: true 8 | -------------------------------------------------------------------------------- /SaltStack/install-saltstack-ubuntu.md: -------------------------------------------------------------------------------- 1 | Please [refer here](https://docs.saltproject.io/salt/install-guide/en/latest/topics/install-by-operating-system/ubuntu.html) 2 | 3 | ``` 4 | mkdir /etc/apt/keyrings 5 | ``` 6 | 7 | ### Import the Salt Project repository key based on the Ubuntu version 8 | ## Install Salt on Ubuntu 22.04 (Jammy) 9 | ``` 10 | sudo curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/ubuntu/22.04/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg 11 | echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/22.04/amd64/latest jammy main" | sudo tee /etc/apt/sources.list.d/salt.list 12 | ``` 13 | 14 | ## Install Salt on Ubuntu 20.04 (Focal) 15 | ``` 16 | sudo curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg 17 | echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/latest focal main" | sudo tee /etc/apt/sources.list.d/salt.list 18 | ``` 19 | 20 | 21 | 22 | ### Update VM 23 | ``` 24 | sudo apt-get update 25 | ``` 26 | ### Install the salt-minion, salt-master, or other Salt components: 27 | ``` 28 | sudo apt-get install salt-master 29 | sudo apt-get install salt-minion 30 | sudo apt-get install salt-ssh 31 | sudo apt-get install salt-syndic 32 | sudo apt-get install salt-cloud 33 | sudo apt-get install salt-api 34 | ``` 35 | 36 | ### Enable and start the services for salt-minion, salt-master, or other Salt components: 37 | ``` 38 | sudo systemctl enable salt-master && sudo systemctl start salt-master 39 | sudo systemctl enable salt-minion && sudo systemctl start salt-minion 40 | sudo systemctl enable salt-syndic && sudo systemctl start salt-syndic 41 | sudo systemctl enable salt-api && sudo systemctl start salt-api 42 | ``` 43 | 44 | ### Verify installation 45 | ``` 46 | systemctl status salt-master 47 | ``` 48 | -------------------------------------------------------------------------------- /angular/Dockerfile-angular-nginx.md: -------------------------------------------------------------------------------- 1 | ``` 2 | FROM nginx:latest 3 | 4 | WORKDIR /usr/share/nginx/html 5 | 6 | RUN rm -rf ./* 7 | 8 | COPY /dist/datahub . 9 | 10 | ENTRYPOINT ["nginx","-g","daemon off;"] 11 | 12 | EXPOSE 80 13 | 14 | # sudo docker run -d -p 80:80 obliqueo:latest 15 | # curl http://localhost:80 16 | 17 | ``` 18 | -------------------------------------------------------------------------------- /angular/angular-build-new-env.txt: -------------------------------------------------------------------------------- 1 | Production mode: 2 | You can build your application in production mode by running command: 3 | ng build --prod 4 | Build angular project 5 | 6 | ``` 7 | ng build [project] --configuration production 8 | ``` 9 | 10 | When application is built for production mode then environments/environment.ts file gets replaced with environments/environment.prod.ts file. Hence if you are referring to settings from environment.ts file in your code, you don’t have to put any if condition or hard code production URL. 11 | 12 | For each target environment, create a new file under folder environments. like: environment.stg.ts , environment.qa.ts 13 | 14 | update env file with below code snipet and update according to your env settings. 15 | export const environment = { 16 | stg: true, 17 | environmentName: 'Stage', 18 | apiUrl: 'Stage url' 19 | }; 20 | 21 | Use environment settings inside components 22 | Your feature components/services can now use normal environment.ts file for referring settings like API URL etc. 23 | 24 | update your compont ts accordingly. 25 | 26 | import { environment } from 'src/environments/environment'; 27 | 28 | @Component({ 29 | selector: 'app-root', 30 | templateUrl: './app.component.html', 31 | styleUrls: ['./app.component.css'] 32 | }) 33 | export class AppComponent { 34 | title = 'multiple-env-demo'; 35 | environmentName = ''; 36 | environmentUrl = 'Debug api'; 37 | 38 | constructor() { 39 | this.environmentName = environment.environmentName; 40 | this.environmentUrl = environment.apiUrl; 41 | } 42 | } 43 | 44 | 45 | Update angular.json configuration. add a new section under projects/architect/build section. 46 | "build": { 47 | "configurations": { 48 | "production": { 49 | "fileReplacements": [ 50 | { 51 | "replace": "src/environments/environment.ts", 52 | "with": "src/environments/environment.prod.ts" 53 | } 54 | ], 55 | }, 56 | "qa": { 57 | "fileReplacements": [ 58 | { 59 | "replace": "src/environments/environment.ts", 60 | "with": "src/environments/environment.qa.ts" 61 | } 62 | ] 63 | }, 64 | "staging": { 65 | "fileReplacements": [ 66 | { 67 | "replace": "src/environments/environment.ts", 68 | "with": "src/environments/environment.staging.ts" 69 | } 70 | ] 71 | } 72 | } 73 | } 74 | 75 | Build application for custom environment. Now build your application for targeted environment by specific --configuration parameter to ng build command. 76 | ng build --configuration stg 77 | and dist folder shall contains build output with stg configuration. 78 | -------------------------------------------------------------------------------- /angular/angular-command.md: -------------------------------------------------------------------------------- 1 | ng --version 2 | 3 | npm uninstall -g @angular/cli 4 | 5 | npm cache clean --force 6 | 7 | Delete the C:\Users\\AppData\Roaming\npm\node_modules\@angular and and C:\Users\\AppData\Roaming\npm-cachefolder. if it exists after uninstalling 8 | 9 | npm install -g @angular/cli 10 | 11 | ng version 12 | 13 | npm -v 14 | 15 | node -v 16 | 17 | 18 | Build angular project 19 | 20 | ``` 21 | ng build [project] --configuration production 22 | ``` 23 | -------------------------------------------------------------------------------- /angular/angular-error-fix.md: -------------------------------------------------------------------------------- 1 | ## Angular Error and Fix 2 | 3 | Error: An unhandled exception occurred: Cannot find module '@angular-devkit/build-angular/package.json' 4 | Fix: Run this , 5 | ``` 6 | npm install --save-dev @angular-devkit/build-angular 7 | ``` 8 | 9 | Error: 'ng' is not recognized as an internal or external command 10 | Fix: 11 | ``` 12 | npm install -g @angular/cli 13 | ``` 14 | 15 | If error still exists, Go to environment variable > click PATH valiable > add below to values un der PATH then restart terminal 16 | ``` 17 | %USERPROFILE%\AppData\Roaming\npm 18 | %USERPROFILE%\AppData\Roaming\npm\node_modules\angular-cli\bin 19 | ``` 20 | 21 | Error: Cannot find module '@angular/compiler-cli' 22 | ``` 23 | rm -r node_modules 24 | npm cache clean --force 25 | install npm 26 | ``` 27 | 28 | ### set proxy for git config global 29 | ``` 30 | git config --global http.proxy "http://abc.com:8080" 31 | ``` 32 | 33 | ## set ssl false 34 | ``` 35 | npm config set strict-ssl false -g 36 | ``` 37 | 38 | ## Run npm install with proxy 39 | ``` 40 | npm install -g angular/cli --proxy "http://abc.com:8080" 41 | ``` 42 | -------------------------------------------------------------------------------- /angular/commands.md: -------------------------------------------------------------------------------- 1 | *******Angular Notes ******* 2 | // start angular app 3 | npm start // or you can do ng serve 4 | 5 | // run below command if you get ".ps1 cannot be loaded because the execution of scripts is disabled on this system" 6 | powershell Set-ExecutionPolicy RemoteSigned 7 | powershell Set-ExecutionPolicy Restricted 8 | powershell Get-ExecutionPolicy -List 9 | 10 | The @angular/material package provides the components of the Material Design, @angular/cdk is a component development kit that is needed for the Material components to work and hammerjs is a library that provides smooth animations for the component. @angular/flex-layout provides a flexible and responsive grid. It is independent of the Material components but is often used together with it. 11 | 12 | ng add @angular/material 13 | npm i @angular/flex-layout 14 | npm i hammerjs 15 | 16 | Angular-cli from css to scss 17 | 18 | //https://stackoverflow.com/questions/40726081/angular-cli-from-css-to-scss/45255290 19 | 20 | Change the default style extension to scss 21 | Manually change in .angular-cli.json (Angular 5.x and older) or angular.json (Angular 6+) or run: 22 | 23 | ng config defaults.styleExt=scss 24 | if you get an error: Value cannot be found. use the command: 25 | 26 | ng config schematics.@schematics/angular:component.styleext scss 27 | 28 | Experimental support for decorators is a feature that is subject to change in a future release. Set the 'experimentalDecorators' option in your 'tsconfig' or 'jsconfig' to remove this warning.ts(1219) 29 | Soluntion: in VS Code 30 | File -> Preferences -> Settings 31 | find experimentalDecorators and enable experimentalDecorators 32 | 33 | // mat icons https://klarsys.github.io/angular-material-icons/ 34 | 35 | Error: Angular JIT compilation failed: '@angular/compiler' not loaded! 36 | - JIT compilation is discouraged for production use-cases! Consider AOT mode instead. 37 | Solution: 38 | Turn off the AOT by changing "aot": true to "aot: false in angular.json file. I would not recommend this as this improves the performance of the Angular app and improves the catching of error codes in development mode. 39 | 40 | ## for Angular Firebase installation 41 | 42 | npm install -g firebase-tools 43 | npm install firebase @angular/fire 44 | firebase login --reauth 45 | 46 | Build Production mode: 47 | You can build your application in production mode by running command: 48 | ng build --prod 49 | 50 | When application is built for production mode then environments/environment.ts file gets replaced with environments/environment.prod.ts file. Hence if you are referring to settings from environment.ts file in your code, you don’t have to put any if condition or hard code production URL. 51 | 52 | -------------------------------------------------------------------------------- /angular/install-nodejs-ubuntu.md: -------------------------------------------------------------------------------- 1 | 2 | ``` 3 | cd ~ 4 | curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh 5 | 6 | nano /tmp/nodesource_setup.sh 7 | 8 | sudo bash /tmp/nodesource_setup.sh 9 | 10 | sudo apt install nodejs 11 | 12 | node -v 13 | ``` 14 | -------------------------------------------------------------------------------- /ansible/ansible-install-ubuntu.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Install Ansible in Ubuntu 4 | 5 | ### Update and Upgrade the server. After upgrade reboot 6 | 7 | sudo apt-get upgrade -y 8 | 9 | sudo apt-get update 10 | 11 | ### Add repository for ansible 12 | 13 | sudo apt-add-repository ppa:ansible/ansible 14 | 15 | ### Update packages once more 16 | 17 | sudo apt-get update 18 | 19 | ### Install ansible in Ubuntu 20 | 21 | sudo apt-get install ansible -y 22 | 23 | ### Check pytho installation 24 | 25 | python --version 26 | 27 | ### If python is not installed, install it by executing below command 28 | 29 | sudo apt-get install python -y 30 | 31 | -------------------------------------------------------------------------------- /ansible/ansible-playbook-deploy-to-tomcat: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Deploy web app in tomcat 3 | hosts: webservers 4 | become: yes 5 | tasks: 6 | - name: copy war into tomcat 7 | copy: 8 | src: /opt/temp/target/webapptest.war 9 | dest: /usr/local/tomcat/webapps 10 | -------------------------------------------------------------------------------- /ansible/ansible_playbook_configure_tomcat_amazon_linux.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install and configure tomcat 3 | hosts: 3.89.155.217 4 | gather_facts: false 5 | vars: 6 | java_v: java-1.8.0-openjdk 7 | set_java: jre-1.8.0-openjdk.x86_64 8 | tomcat_port: 8181 9 | tomcat_version: 8.5.55 10 | tomcat_url: "https://mirror.csclub.uwaterloo.ca/apache/tomcat/tomcat-{{tomcat_version.split('.')[0]}}/v{{tomcat_version}}/bin/apache-tomcat-{{tomcat_version}}.tar.gz" 11 | become: yes 12 | tasks: 13 | - name: Updating repos 14 | yum: 15 | name: "*" 16 | state: latest 17 | - name: Installing Openjdk 18 | yum: 19 | name: "{{java_v}}" 20 | state: present 21 | - name: Setting default Java 22 | alternatives: 23 | name: java 24 | link: /usr/bin/java 25 | path: /usr/lib/jvm/{{set_java}}/bin/java 26 | - name: Downloading tomcat 27 | get_url: 28 | url: "{{tomcat_url}}" 29 | dest: /usr/local 30 | - name: Extracting downloaded tar file 31 | unarchive: 32 | src: "/usr/local/apache-tomcat-{{tomcat_version}}.tar.gz" 33 | dest: /usr/local 34 | remote_src: yes 35 | - name: Renaming tomcat home 36 | command: mv /usr/local/apache-tomcat-{{tomcat_version}} /usr/local/tomcat 37 | - name: replace tomcat port in server config 38 | template: 39 | src: server.xml.j2 40 | dest: /usr/local/tomcat/conf/server.xml 41 | - name: Updating users and role 42 | template: 43 | src: tomcat-users.xml 44 | dest: /usr/local/tomcat/conf/tomcat-users.xml 45 | - name: Updating context xml restriction 46 | template: 47 | src: context-manager.xml 48 | dest: /usr/local/tomcat/webapps/manager/META-INF/context.xml 49 | - name: Updating restriction for host manager context 50 | template: 51 | src: context-hostmanager.xml 52 | dest: /usr/local/tomcat/webapps/host-manager/META-INF/context.xml 53 | - name: Starting tomcat 54 | shell: nohup /usr/local/tomcat/bin/startup.sh & 55 | -------------------------------------------------------------------------------- /ansible/ansible_playbook_configure_tomcat_ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure in Remote Hosts 3 | hosts: remotehosts 4 | gather_facts: true 5 | become: yes 6 | vars: 7 | tomcat_port: 8282 8 | tasks: 9 | - name: Update repos 10 | apt: 11 | name: "*" 12 | state: latest 13 | - name: Installing OpenJDK 14 | apt: 15 | name: openjdk-8-jdk 16 | - name: Downloading tomcat 17 | get_url: 18 | url: "http://apache.mirror.iweb.ca/tomcat/tomcat-8/v8.5.56/bin/apache-tomcat-8.5.56.tar.gz" 19 | dest: /usr/local 20 | - name: Extracting tomcat 21 | unarchive: 22 | src: "/usr/local/apache-tomcat-8.5.56.tar.gz" 23 | dest: /usr/local 24 | remote_src: yes 25 | - name: Renaming tomcat home 26 | command: mv /usr/local/apache-tomcat-8.5.56 /usr/local/tomcat 27 | - name: Changing port 28 | template: 29 | src: server.xml.j2 30 | dest: /usr/local/tomcat/conf/server.xml 31 | - name: Updating users and role 32 | template: 33 | src: tomcat-users.xml 34 | dest: /usr/local/tomcat/conf/tomcat-users.xml 35 | - name: Updating context xml restriction 36 | template: 37 | src: context-manager.xml 38 | dest: /usr/local/tomcat/webapps/manager/META-INF/context.xml 39 | - name: Updating restriction for host manager context 40 | template: 41 | src: context-hostmanager.xml 42 | dest: /usr/local/tomcat/webapps/host-manager/META-INF/context.xml 43 | - name: Starting tomcat 44 | shell: nohup /usr/local/tomcat/bin/startup.sh & 45 | -------------------------------------------------------------------------------- /apache/install.md: -------------------------------------------------------------------------------- 1 | 2 | ### Execute below commands in an Ubuntu VM to install and setup Apache. 3 | 4 | ``` 5 | sudo apt update 6 | sudo apt install apache2 7 | sudo ufw app list 8 | sudo ufw allow 'Apache' 9 | sudo ufw status 10 | sudo systemctl status apache2 11 | hostname -I 12 | ``` 13 | 14 | Test 15 | 16 | ``` 17 | http://your_server_ip 18 | `` 19 | -------------------------------------------------------------------------------- /aws/best-practices/aws-startup-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "IAM_Statement01", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "iam:CreateRole", 9 | "iam:GetRole", 10 | "iam:DeleteRole", 11 | "iam:DeleteRolePolicy", 12 | "iam:TagRole", 13 | "iam:CreatePolicy", 14 | "iam:DeletePolicy", 15 | "iam:GetPolicy", 16 | "iam:GetRolePolicy", 17 | "iam:PutRolePolicy", 18 | "iam:TagPolicy", 19 | "iam:ListRolePolicies", 20 | "iam:GetPolicyVersion", 21 | "iam:AttachRolePolicy", 22 | "iam:ListAttachedRolePolicies", 23 | "iam:DetachRolePolicy", 24 | "iam:ListPolicyVersions", 25 | "iam:ListInstanceProfilesForRole", 26 | "iam:CreateInstanceProfile", 27 | "iam:GetInstanceProfile", 28 | "iam:RemoveRoleFromInstanceProfile", 29 | "iam:DeleteInstanceProfile", 30 | "iam:AddRoleToInstanceProfile" 31 | ], 32 | "Resource": [ 33 | "arn:aws:iam:::role/myOrg_Role_*", 34 | "arn:aws:iam:::policy/myOrg_Policy_*", 35 | "arn:aws:iam:::instance-profile/myOrg_*" 36 | ] 37 | }, 38 | { 39 | "Sid": "IAM_Statement02", 40 | "Effect": "Allow", 41 | "Action": [ 42 | "iam:PassRole" 43 | ], 44 | "Resource": [ 45 | "arn:aws:iam:::role/myOrg_Role_*", 46 | "arn:aws:iam:::policy/myOrg_Policy_*" 47 | ] 48 | }, 49 | { 50 | "Sid": "Resources_Statement01", 51 | "Effect": "Allow", 52 | "Action": [ 53 | "s3:*", 54 | "ec2:*", 55 | "kms:*", 56 | "rds:*", 57 | "sns:*", 58 | "sqs:*", 59 | "eks:*", 60 | "ecs:*", 61 | "logs:*", 62 | "glue:*", 63 | "events:*", 64 | "lambda:*", 65 | "athena:*", 66 | "states:*", 67 | "dynamodb:*", 68 | "codebuild:*", 69 | "apigateway:*", 70 | "elasticache:*", 71 | "comprehend:*", 72 | "codepipeline:*", 73 | "datapipeline:*", 74 | "lakeformation:*", 75 | "emr-serverless:*", 76 | "elasticmapreduce:*", 77 | "elasticloadbalancing:*" 78 | ], 79 | "Resource": [ 80 | "*" 81 | ] 82 | } 83 | ] 84 | } 85 | -------------------------------------------------------------------------------- /aws/cloudformation/readme.txt: -------------------------------------------------------------------------------- 1 | you can download the cloudformation templates from here 2 | -------------------------------------------------------------------------------- /aws/codebuild/buildspec-eks-deployment-v01.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 0.2 3 | phases: 4 | install: 5 | commands: 6 | - echo Logging in to Amazon ECR... 7 | - $(aws ecr get-login --no-include-email --region $AWS_REGION) 8 | - curl -sS -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/aws-iam-authenticator 9 | - curl -sS -o kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.20.4/2021-04-12/bin/linux/amd64/kubectl 10 | - chmod +x ./kubectl ./aws-iam-authenticator 11 | - export PATH=$PWD/:$PATH 12 | pre_build: 13 | commands: 14 | - TAG="$(date +%Y-%m-%d.%H.%M.%S).$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" 15 | - sed -i 's@CONTAINER_IMAGE@'"$REPOSITORY:$TAG"'@' Kubernetes/eks-deployment.yml 16 | - $(aws ecr get-login --no-include-email) 17 | - export KUBECONFIG=$HOME/.kube/config 18 | build: 19 | commands: 20 | - docker build --tag $REPOSITORY:$TAG . 21 | 22 | post_build: 23 | commands: 24 | - docker push $REPOSITORY:$TAG 25 | - CREDENTIALS=$(aws sts assume-role --role-arn $EKS_KUBECTL_ROLE_ARN --role-session-name codebuild-kubectl --duration-seconds 900) 26 | - export AWS_ACCESS_KEY_ID="$(echo ${CREDENTIALS} | jq -r '.Credentials.AccessKeyId')" 27 | - export AWS_SECRET_ACCESS_KEY="$(echo ${CREDENTIALS} | jq -r '.Credentials.SecretAccessKey')" 28 | - export AWS_SESSION_TOKEN="$(echo ${CREDENTIALS} | jq -r '.Credentials.SessionToken')" 29 | - export AWS_EXPIRATION=$(echo ${CREDENTIALS} | jq -r '.Credentials.Expiration') 30 | - aws eks update-kubeconfig --name $EKS_CLUSTER_NAME 31 | - kubectl apply -f Kubernetes/eks-deployment.yml 32 | - printf '[{"name":"eks-deployment","imageUri":"%s"}]' $REPOSITORY:$TAG > build.json 33 | artifacts: 34 | files: build.json 35 | -------------------------------------------------------------------------------- /aws/codebuild/buildspec-springbootdemo.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | install: 5 | commands: 6 | - apt-get update -y 7 | - apt-get install -y openjdk-8-jdk 8 | pre_build: 9 | commands: 10 | - echo pre build started on 'date' 11 | - mvn clean install 12 | - echo Logging in to Amazon ECR... 13 | - aws --version 14 | - $(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email) 15 | - IMAGE_REPO_NAME= 16 | - REPOSITORY_URI=.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_REPO_NAME 17 | - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) 18 | - IMAGE_TAG=build-$(echo $CODEBUILD_BUILD_ID | awk -F":" '{print $2}') 19 | build: 20 | commands: 21 | - echo Build started 'date' 22 | - docker build -t $REPOSITORY_URI:latest . 23 | - docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$IMAGE_TAG 24 | post_build: 25 | commands: 26 | - echo Build completed on 'date' 27 | - echo Pushing the Docker image... 28 | - docker push $REPOSITORY_URI:$IMAGE_TAG 29 | - docker push $REPOSITORY_URI:latest 30 | - printf $REPOSITORY_URI:$IMAGE_TAG > imagedefinitions.json 31 | - cat imagedefinitions.json 32 | artifacts: 33 | files: 34 | - imagedefinitions.json 35 | - 'target/springbootdemo-0.0.1-SNAPSHOT.jar' 36 | -------------------------------------------------------------------------------- /aws/codebuild/codeartifact-token-codebuild.md: -------------------------------------------------------------------------------- 1 | 2 | ## Pass an auth token using an environment variable refer https://docs.aws.amazon.com/codeartifact/latest/ug/tokens-authentication.html 3 | ### macOS or Linux: 4 | 5 | export CODEARTIFACT_TOKEN=`aws codeartifact get-authorization-token --domain my-domain --domain-owner domain-owner-id --query authorizationToken --output text` 6 | 7 | ### Windows (using default command shell): 8 | 9 | for /f %i in ('aws codeartifact get-authorization-token --domain my-domain --domain-owner domain-owner-id --query authorizationToken --output text') do set CODEARTIFACT_TOKEN=%i 10 | 11 | ### Windows PowerShell: 12 | 13 | $env:CODEARTIFACT_TOKEN = aws codeartifact get-authorization-token --domain my-domain --domain-owner domain-owner-id --query authorizationToken --output text 14 | 15 | -------------------------------------------------------------------------------- /aws/codedeploy/appspec_ecs_deploy.yml: -------------------------------------------------------------------------------- 1 | # rename the file to appspec.yml 2 | # It is for ECS deployment 3 | 4 | version: 0.0 5 | Resources: 6 | - TargetService: 7 | Type: AWS::ECS::Service 8 | Properties: 9 | TaskDefinition: "arn:aws:ecs:us-east-1::task-definition/bg-task-def1:1" 10 | LoadBalancerInfo: 11 | ContainerName: "bg-cont" 12 | ContainerPort: 8181 13 | 14 | 15 | ## notes 16 | # TaskDefinition: "arn:aws:ecs:us-east-1::task-definition/:1" - you will get this ARN from ECS cluster > task definition > json 17 | # check the blue gree deployment section in course https://www.udemy.com/course/aws-cicd-automation-complete-reference-for-prod-ready-system/ 18 | # LoadBalancerInfo: 19 | # ContainerName: "container name" 20 | # ContainerPort: 21 | -------------------------------------------------------------------------------- /aws/codedeploy/appspec_lambda_deploy.yml: -------------------------------------------------------------------------------- 1 | # rename this file to appspec.yml 2 | # it is for Lambda deploy through CodeDeploy 3 | 4 | version: 0.0 5 | Resources: 6 | - myLambdaFunctionName: 7 | Type: AWS::Lambda::Function 8 | Properties: 9 | Name: "myLambdaFunctionName" 10 | Alias: "VLatest" 11 | CurrentVersion: "4" 12 | TargetVersion: "5" 13 | -------------------------------------------------------------------------------- /aws/codepipeline/buildspec-angular-deploy.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | CACHE_CONTROL: "86400" 6 | S3_BUCKET: "{{s3_bucket_url}}" 7 | BUILD_FOLDER: "dist" 8 | phases: 9 | install: 10 | runtime-versions: 11 | nodejs: 10 12 | commands: 13 | - echo Installing source NPM dependencies... 14 | - npm install 15 | - npm install -g @angular/cli 16 | build: 17 | commands: 18 | - echo Build started 19 | - ng build 20 | artifacts: 21 | files: 22 | - '**/*' 23 | base-directory: 'dist*' 24 | discard-paths: yes 25 | -------------------------------------------------------------------------------- /aws/docs/User is not authorized.md: -------------------------------------------------------------------------------- 1 | ## Issue: 2 | ### While trying to provision any resources from AWS CLI and having **MFA enabled** for the IAM user, it shows an encrypted message with error like "User: xxx is not authorized to perform: xxx with an explicit deny in an indentity-based policy " 3 | 4 | So first decode the message to read it. 5 | 6 | Ensure you have 'aws configure' done with valid access credentials. 7 | 8 | ### Step 1: 9 | Decode the error message by below sts command 10 | 11 | ``` 12 | aws sts decode-authorization-message --encoded-message 13 | ``` 14 | 15 | It will show detail decoded message. 16 | 17 | 18 | ### Step 2: 19 | In Decoded message if you find BlockMostAccessUnlessSignedInWithMFA, it means you have your MFA enabled and while using CLI commands you have to use your MFA to generate sessio token. 20 | 21 | To generate session token run below command 22 | 23 | ``` 24 | aws sts get-session-token --serial-number --token-code 25 | 26 | aws sts get-session-token --serial-number arn:aws:iam:::mfa/ --token-code 000000 27 | ``` 28 | 29 | It will give an output with credentials AccessKeyId, SecretAccessKey, SessionToken 30 | 31 | ### Step 3: 32 | If you are using windows, browse to you user directory > .aws then open the 'credentials' in notepad. 33 | replace the existing values of AccessKeyId, SecretAccessKey with Step 2 output. and add one more line as below 34 | 35 | ``` 36 | aws_session_token= 37 | ``` 38 | 39 | save the file. and executing your aws command in cli 40 | 41 | Remember that Step 2 token is valid for a finite time only. Once it expires you have to generate it same way. 42 | 43 | 44 | ****After the token expires when regenerating the token *********** 45 | 46 | Step A: Reset the AccessKeyId, SecretAccessKey values with your original values (the actual values from your IAM record) 47 | Step B: start from Step 1 above. 48 | 49 | Note: If you miss Step A aws sts get-session-token .. command will show error saying invalid credentials , That's why you have to add oroginal credentials back. 50 | -------------------------------------------------------------------------------- /aws/docs/create-lambda-layer-for-python.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | https://pypi.org/project/pillow/ 4 | 5 | follow https://www.youtube.com/watch?v=lrEAu75zhNI 6 | -------------------------------------------------------------------------------- /aws/ec2/add-a-new-user-to-ssh-ec2.md: -------------------------------------------------------------------------------- 1 | This is to add multiple users to SSH an EC2 2 | The primary user (who has the ssh key pair ) loginto the instance and run below commands to add another user for ssh access to that EC2. 3 | The new user need to share private-key of his/her key 4 | 5 | ``` 6 | sudo bash 7 | useradd -m --group sudo 8 | mkdir -p /home//.ssh 9 | echo ssh-rsa >> /home//.ssh/authorized_keys 10 | chmod 700 /home//.ssh 11 | chmod 600 /home//.ssh/authorized_keys 12 | chown : -R /home//.ssh 13 | passwd 14 | ``` 15 | 16 | Note: get from the other EC2 where the user has access. browse to below location to get the keys 17 | ``` 18 | /home//.ssh/authorized_keys 19 | ``` 20 | -------------------------------------------------------------------------------- /aws/ec2/aws-configure.md: -------------------------------------------------------------------------------- 1 | ``` 2 | export AWS_ACCESS_KEY_ID = #### 3 | export AWS_SECRET_ACCESS_KEY = #### 4 | export AWS_SESSION_TOKEN = #### 5 | 6 | unset AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN 7 | 8 | aws sts get-caller-identity 9 | 10 | aws sts assume-role --role-arn arn:aws:iam::1234567890/role/eks-cluster-role --role-session-name AWSCLI-Session 11 | 12 | ``` 13 | -------------------------------------------------------------------------------- /aws/ec2/awscli-install.md: -------------------------------------------------------------------------------- 1 | Zip a directory in ubuntu EC2 instance`` 2 | 3 | ``` 4 | sudo apt install zip -y 5 | 6 | zip -r temp.zip existing_folder 7 | 8 | ``` 9 | Create an IAM role with S3 write access or admin access 10 | 11 | Map the IAM role to an EC2 instance 12 | Install CLI 13 | 14 | ``` 15 | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" 16 | 17 | unzip awscliv2.zip 18 | 19 | sudo ./aws/install 20 | 21 | ``` 22 | -------------------------------------------------------------------------------- /aws/ec2/ec2-cloud-init-log.md: -------------------------------------------------------------------------------- 1 | 2 | cloud-init log location in EC2 instance 3 | ``` 4 | var/log/cloud-init.log 5 | ``` 6 | -------------------------------------------------------------------------------- /aws/ec2/readme.md: -------------------------------------------------------------------------------- 1 | 2 | cloud-init log location in EC2 instance 3 | ``` 4 | var/log/cloud-init.log 5 | ``` 6 | -------------------------------------------------------------------------------- /aws/ec2/shell-scripts/readme.md: -------------------------------------------------------------------------------- 1 | List of shell scripts for EC2 metadata 2 | -------------------------------------------------------------------------------- /aws/ec2/shell-scripts/userdata-clamav-rhel-docker.sh: -------------------------------------------------------------------------------- 1 | # Docker community support "We currently only provide packages for RHEL on s390x (IBM Z). Other architectures are not yet supported for RHEL, but you may be able to install the CentOS packages on RHEL. Refer to the Install Docker Engine on CentOS page for details." 2 | # https://docs.docker.com/engine/install/centos/ 3 | 4 | #!/bin/bash -xe 5 | echo 'Starting Update' 6 | sudo yum install -y yum-utils 7 | sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo 8 | echo 'Create test file' 9 | sudo mkdir /opt/scandir 10 | sudo chmod -R 777 /opt/scandir 11 | echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' > /opt/scandir/bad.txt 12 | echo 'I am good file' > /opt/scandir/good.txt 13 | echo 'Install Docker Engine' 14 | sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin 15 | sudo systemctl start docker 16 | sudo docker run hello-world 17 | echo 'Install clamav image' 18 | sudo docker pull clamav/clamav:1.0.2 19 | sudo groupadd clamav 20 | echo 'Adding clamav user in group' 21 | sudo useradd -g clamav -s /bin/false -c "Clam Antivirus" clamav 22 | sudo chown -R clamav:clamav /var/lib/clamav/ 23 | echo 'Running clamav' 24 | #sudo docker run -it --rm --name "clam_container" --mount source=clam_db, target=/var/lib/clamav --env 'CLAMAV_NO_FRESHCLAMD=true' clamav/clamav:stable 25 | #sudo docker run -it --rm --name "clam_container_01" clamav/clamav:1.0.2_base 26 | #sudo docker volume create clam_db 27 | #sudo docker run -it --rm --name "clam_container_01" --mount source=clam_db,target=/var/lib/clamav clamav/clamav:1.0.2_base 28 | sudo docker run --rm \ 29 | --name "clamav_container_01" \ 30 | --mount source=clam_db,target=/var/lib/clamav \ 31 | --mount type=bind,source=/opt/scandir,target=/scandir \ 32 | --env 'CLAMAV_NO_FRESHCLAMD=false' \ 33 | --env 'FRESHCLAM_CHECKS=24' \ 34 | clamav/clamav:1.0.2_base 35 | sudo docker run --rm \ 36 | --name "clamav_container_02" \ 37 | --mount source=clam_db,target=/var/lib/clamav \ 38 | --mount type=bind,source=/opt/scandir,target=/scandir \ 39 | --env 'CLAMAV_NO_FRESHCLAMD=true' \ 40 | --env 'FRESHCLAM_CHECKS=24' \ 41 | clamav/clamav:1.0.2_base \ 42 | clamscan /scandir -------------------------------------------------------------------------------- /aws/ec2/shell-scripts/userdata-clamav-ubuntu-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo apt-get update 3 | sudo apt-get install ca-certificates curl gnupg 4 | 5 | 6 | sudo install -m 0755 -d /etc/apt/keyrings 7 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg 8 | sudo chmod a+r /etc/apt/keyrings/docker.gpg 9 | 10 | echo \ 11 | "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ 12 | "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ 13 | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 14 | 15 | sudo apt-get update 16 | 17 | sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin 18 | 19 | sudo docker pull clamav/clamav:stable 20 | sudo docker run -it --rm --name "clam_container" --mount source=clam_db, target=/var/lib/clamav --env 'CLAMAV_NO_FRESHCLAMD=true' clamav/clamav:stable -------------------------------------------------------------------------------- /aws/ec2/shell-scripts/userdata-clamav-ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | echo 'Starting Update' 3 | sudo apt-get update -y 4 | echo 'Starting clamav install' 5 | sudo apt-get install -y clamav clamav-base clamav-daemon clamav-freshclam 6 | sudo chmod -R 777 /var/log/clamav 7 | echo 'Check status' 8 | sudo systemctl status clamav-freshclam.service 9 | which freshclam 10 | sudo mkdir /var/run/clamav 11 | echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' > /home/ubuntu/scandir/bad.txt 12 | echo 'I am good file' > /home/ubuntu/scandir/good.txt 13 | sudo chmod -R 777 /var/run/clamav 14 | sudo clamd 15 | sudo clamdscan --fdpass --move=/home/ubuntu/quarantined /home/ubuntu/scandir -------------------------------------------------------------------------------- /aws/ec2/shell-scripts/userdata-kubectl-vm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | echo 'Starting Update' 3 | sudo apt-get update -y 4 | sudo apt-get install unzip 5 | echo 'Installing AWSCLI2' 6 | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" 7 | unzip awscliv2.zip 8 | sudo ./aws/install 9 | aws --version 10 | echo 'Installing kubectl' 11 | curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.4/2023-08-16/bin/linux/amd64/kubectl 12 | chmod +x ./kubectl 13 | mkdir -p $HOME/bin && cp ./kubectl $HOME/bin/kubectl && export PATH=$HOME/bin:$PATH 14 | kubectl version --short --client 15 | # for ARM systems, set ARCH to: `arm64`, `armv6` or `armv7` 16 | ARCH=amd64 17 | PLATFORM=$(uname -s)_$ARCH 18 | curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz" 19 | # (Optional) Verify checksum 20 | curl -sL "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_checksums.txt" | grep $PLATFORM | sha256sum --check 21 | tar -xzf eksctl_$PLATFORM.tar.gz -C /tmp && rm eksctl_$PLATFORM.tar.gz 22 | sudo mv /tmp/eksctl /usr/local/bin 23 | eksctl version 24 | -------------------------------------------------------------------------------- /aws/ec2/shell-scripts/userdata-nfs-mount-efs-RHEL.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | echo 'Starting Update' 3 | sudo yum install -y update yum-utils nfs-utils 4 | echo 'Create efs mount' 5 | sudo mkdir efs 6 | sudo chmod -R 777 efs 7 | sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport fs-050d6dbc42c12259e.efs.us-east-1.amazonaws.com:/ efs 8 | df -h -------------------------------------------------------------------------------- /aws/ec2/upload-file-ec2-s3.md: -------------------------------------------------------------------------------- 1 | ### considering a scenario where you want to zip a directory in your EC2 and then transfer to S3 bucket`` 2 | 3 | ### Zip a directory in ubuntu EC2 instance`` 4 | 5 | ``` 6 | apt install zip 7 | 8 | zip -r temp.zip existing_folder 9 | ``` 10 | 11 | Create an IAM role with S3 write access or admin access 12 | 13 | ### Map the IAM role to an EC2 instance`` 14 | 15 | ### Install CLI 16 | 17 | ``` 18 | $ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" 19 | 20 | unzip awscliv2.zip 21 | 22 | sudo ./aws/install 23 | ``` 24 | 25 | ### Run the AWS s3 cp command to copy the single file to the S3 bucket 26 | 27 | ``` 28 | aws s3 cp s3:// 29 | aws s3 cp /home/ec2-user/copy.zip s3:// 30 | ``` 31 | 32 | ### Run the AWS s3 cp command to recursive file copy to the S3 bucket 33 | 34 | ``` 35 | aws s3 cp s3:// --recursive 36 | aws s3 cp /home/ec2-user/mydir s3:// --recursive 37 | ``` 38 | 39 | ### sync files to S3 bucket 40 | sync will only upload the new or modified files. As it named sync 41 | 42 | ``` 43 | aws s3 sync s3:// 44 | aws s3 sync /home/ec2-user/mydir s3:// 45 | ``` 46 | 47 | 48 | ### if upload fails showing "Warning: Skipping file /home/ec2-user/copy.zip" 49 | 50 | ``` 51 | check file permission 52 | ls -l /home/ec2-user/copy.zip 53 | 54 | you may have -rw-------- on this file . So change the file permission 55 | 56 | chmod 644 /home/ec2-user/copy.zip 57 | 58 | Now the permission should show -rw-r--r-- 59 | 60 | Now run the copy command again 61 | aws s3 cp /home/ec2-user/copy.zip s3:// 62 | ``` 63 | ### Copy from S3 to EC2 64 | ``` 65 | aws s3 cp s3:///copy.zip . (dont miss the '.' It will copy to your corrent directry. Or provide a directory path like /home/ec2-user/) 66 | ``` 67 | ### you can also use s3 presigned url (incase your s3 bucket is private) to download 68 | 1. select the object in s3 bucket > Click Action > Create Pre-signed url by adding time validity 69 | 2. go to your EC2. run below command to download the object to ec2. NOTE: add your pre-signed url into "" 70 | 71 | ``` 72 | wget "https://......" 73 | 74 | If wget is not installed then install wget first. sudo apt-get install wget / sudo yum install wget 75 | ``` 76 | 77 | ### If you want to a diff AWS account suppose Account A to account B. everything remains same. just make sure you have your account B's credential configuied in account A 78 | go to home/ec2-user/.aws and update the credential of account B in 'credential' file of account A 79 | 80 | ### Upload from local system to ec2 81 | ``` 82 | scp -i ~/Downloads/file.pem local_image_file user@ec2_elastic_ip:/home/user/ 83 | ``` 84 | -------------------------------------------------------------------------------- /aws/ec2/userdata-nfs-mount-efs-RHEL.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | echo 'Starting Update' 3 | sudo yum install -y update yum-utils nfs-utils 4 | echo 'Create efs mount' 5 | sudo mkdir efs 6 | sudo chmod -R 777 efs 7 | sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport .efs..amazonaws.com:/ efs 8 | # e.g sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport fs-######.efs.us-east-1.amazonaws.com:/ efs 9 | df -h 10 | -------------------------------------------------------------------------------- /aws/efs/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Create EFS 3 | - Simply create an EFS filesystem and get the file-system id. e.g, fs-######## 4 | - Select the EFS filesystem and go to it's 'Network' tab. select the 'Security Group' 5 | - The Security group must have type NSF for port 2049 open for either 0.0.0.0/0 or any specific source which connects the file-system (e,g the EC2s which creates mount with the this EFS) 6 | 7 | 8 | # Mount EFS with RHEL instance via NFS 9 | - Provision an EC2 with RHEL OS 10 | - Refer userdata for the instalnce [here](https://github.com/e2eSolutionArchitect/scripts/blob/main/aws/ec2/userdata-nfs-mount-efs-RHEL.sh) 11 | - The userdata script is supposed to make a directory for mount point 'efs' 12 | - The command to mount efs with directory 'efs' in ec2 is as below 13 | ``` 14 | e.g sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport fs-######.efs.us-east-1.amazonaws.com:/ efs 15 | ``` 16 | 17 | # Important: 18 | - Make sure to configure security group to have access between EFS ( type NFS - port 2049) and EC2 19 | -------------------------------------------------------------------------------- /aws/efs/nfs-mount-efs-ec2.md: -------------------------------------------------------------------------------- 1 | 2 | # Create EFS 3 | - Simply create an EFS filesystem and get the file-system id. e.g, fs-######## 4 | - Select the EFS filesystem and go to it's 'Network' tab. select the 'Security Group' 5 | - The Security group must have type NSF for port 2049 open for either 0.0.0.0/0 or any specific source which connects the file-system (e,g the EC2s which creates mount with the this EFS) 6 | 7 | 8 | # Mount EFS with RHEL instance via NFS 9 | - Provision an EC2 with RHEL OS 10 | - Refer userdata for the instalnce [here](https://github.com/e2eSolutionArchitect/scripts/blob/main/aws/ec2/userdata-nfs-mount-efs-RHEL.sh) 11 | - The userdata script is supposed to make a directory for mount point 'efs' 12 | - The command to mount efs with directory 'efs' in ec2 is as below 13 | ``` 14 | e.g sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport fs-######.efs.us-east-1.amazonaws.com:/ efs 15 | ``` 16 | 17 | # Important: 18 | - Make sure to configure security group to have access between EFS ( type NFS - port 2049) and EC2 19 | -------------------------------------------------------------------------------- /aws/eks/eks-bastian-host-ec2.md: -------------------------------------------------------------------------------- 1 | configure a bastian host for managing EKS [Click here](https://github.com/e2eSolutionArchitect/terraform/blob/main/providers/aws/examples/e2esa-aws-eks-v2/init.sh) 2 | -------------------------------------------------------------------------------- /aws/eks/eks-provide-access-to-iam-roles.md: -------------------------------------------------------------------------------- 1 | 2 | ``` 3 | aws sts get-caller-identity 4 | 5 | aws sts assume-role --role-arn arn:aws:iam::1234567890/role/eks-cluster-role --role-session-name AWSCLI-Session 6 | 7 | export AWS_ACCESS_KEY_ID = #### 8 | export AWS_SECRET_ACCESS_KEY = #### 9 | export AWS_SESSION_TOKEN = #### 10 | 11 | unset AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN 12 | ``` 13 | -------------------------------------------------------------------------------- /aws/eks/readme.md: -------------------------------------------------------------------------------- 1 | [Click here](https://github.com/e2eSolutionArchitect/scripts/tree/main/kubernetes) for Kubernetes commands and scripts 2 | -------------------------------------------------------------------------------- /aws/iam/policies/e2esa-eks-demo.json: -------------------------------------------------------------------------------- 1 | ## Trust Relationship entry. It is required to get the role access to sts:AssumedRole 2 | { 3 | "Version": "2012-10-17", 4 | "Statement": [ 5 | { 6 | "Effect": "Allow", 7 | "Principal": { 8 | "Service": "eks.amazonaws.com", 9 | "AWS": "arn:aws:iam::11111111:user/myuser" 10 | }, 11 | "Action": "sts:AssumeRole" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /aws/iam/policies/e2esa-sts-assume-role-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": "eks:*", 8 | "Resource": "*" 9 | }, 10 | { 11 | "Sid": "VisualEditor1", 12 | "Effect": "Allow", 13 | "Action": "sts:AssumeRole", 14 | "Resource": "arn:aws:iam::1111111111:role/myrole" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /aws/iam/policies/eks-permission-full.json: -------------------------------------------------------------------------------- 1 | { 2 | "Id": "EKSFullPermission", 3 | "Version": "2012-10-17", 4 | "Statement": [ 5 | { 6 | "Sid": "EKS", 7 | "Effect": "Allow", 8 | "Action": [ 9 | "cloudformation:ListStacks", 10 | "eks:*" 11 | ], 12 | "Resource": "*" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /aws/iam/policies/s3-allow-SSL-requestsonly.json: -------------------------------------------------------------------------------- 1 | # To allow SSL requests only to s3 2 | { 3 | "Id": "S3AllowSSLRequestsOnly", 4 | "Version": "2012-10-17", 5 | "Statement": [ 6 | { 7 | "Sid": "AllowSSLRequestsOnly", 8 | "Action": "s3:*", 9 | "Effect": "Deny", 10 | "Resource": [ 11 | "arn:aws:s3:::DOC-EXAMPLE-BUCKET", 12 | "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*" 13 | ], 14 | "Condition": { 15 | "Bool": { 16 | "aws:SecureTransport": "false" 17 | } 18 | }, 19 | "Principal": "*" 20 | } 21 | ] 22 | } 23 | 24 | -------------------------------------------------------------------------------- /aws/iam/policies/s3-policy-rds-snapshot-backup.json: -------------------------------------------------------------------------------- 1 | # IAM policy used for taking RDS snapshot backup to S3 2 | 3 | { 4 | "Id": "S3AllowRDSSnapshotExport", 5 | "Version": "2012-10-17", 6 | "Statement": [ 7 | { 8 | "Sid": "ExportPolicy", 9 | "Effect": "Allow", 10 | "Action": [ 11 | "s3:PutObject*", 12 | "s3:ListBucket", 13 | "s3:GetObject*", 14 | "s3:DeleteObject*", 15 | "s3:GetBucketLocation" 16 | ], 17 | "Resource": [ 18 | "arn:aws:s3:::DOC-EXAMPLE-BUCKET", 19 | "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*" 20 | ] 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /aws/iam/policies/s3-restrict-by-user-role.md: -------------------------------------------------------------------------------- 1 | referance : https://aws.amazon.com/blogs/security/how-to-restrict-amazon-s3-bucket-access-to-a-specific-iam-role/ 2 | ``` 3 | aws iam get-role --role-name 4 | ``` 5 | 6 | copy the ROLE ID, suppose AROAEXAMPLEID or 111111111111. 7 | 8 | add below policy in s3 bucket policy of bucket name "MyExampleBucket" 9 | 10 | Run the command: aws iam get-user -–user-name USER-NAME 11 | 12 | In the output, look for the userId string, which will begin with AIDAEXAMPLEID for userid and AROAEXAMPLEID for Roleid. 13 | 14 | dont be confused by "aws:userId" you can add userId and RoleID with aws:userId. userid will be a string like "AIDA*" or roleid it will be "AROA*" 15 | 16 | ``` 17 | { 18 | "Version": "2012-10-17", 19 | "Statement": [ 20 | { 21 | "Effect": "Deny", 22 | "Principal": "*", 23 | "Action": "s3:*", 24 | "Resource": [ 25 | "arn:aws:s3:::MyExampleBucket", 26 | "arn:aws:s3:::MyExampleBucket/*" 27 | ], 28 | "Condition": { 29 | "StringNotLike": { 30 | "aws:userId": [ 31 | "AROAEXAMPLEID:*", 32 | "111111111111" 33 | ] 34 | } 35 | } 36 | } 37 | ] 38 | } 39 | ``` 40 | -------------------------------------------------------------------------------- /aws/install-aws-cli.md: -------------------------------------------------------------------------------- 1 | ## Install CLI [refer here](https://github.com/e2eSolutionArchitect/KEDB/blob/main/aws/install-aws-cli.md) 2 | -------------------------------------------------------------------------------- /aws/lambda/emi-calc.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | print('Loading function .... Lambda emi calculator using API gateway') 4 | 5 | def lambda_handler(event, context): 6 | #1. Parse out query string params 7 | principal = event['queryStringParameters']['p'] 8 | rate = event['queryStringParameters']['r'] 9 | time = event['queryStringParameters']['t'] 10 | 11 | principal = int(principal) 12 | rate = float(rate) 13 | time = int(time) 14 | 15 | emi = emi_calculator(principal, rate, time); 16 | 17 | #2. Construct the body of the response object 18 | transactionResponse = {} 19 | transactionResponse['p'] = principal 20 | transactionResponse['r'] = rate 21 | transactionResponse['t'] = time 22 | transactionResponse['emi'] = emi 23 | 24 | #3. Construct http response object 25 | responseObject = {} 26 | responseObject['statusCode'] = 200 27 | responseObject['headers'] = {} 28 | responseObject['headers']['Content-Type'] = 'application/json' 29 | responseObject['body'] = json.dumps(transactionResponse) 30 | 31 | #4. Return the response object 32 | return responseObject 33 | 34 | def emi_calculator(p, r, t): 35 | r = r / (12 * 100) # one month interest 36 | t = t * 12 # one month period 37 | emi = (p * r * pow(1 + r, t)) / (pow(1 + r, t) - 1) 38 | return emi 39 | -------------------------------------------------------------------------------- /aws/lambda/lambda- verify-face-rekognition-python.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | from botocore.exceptions import ClientError 4 | 5 | rekognition = boto3.client('rekognition') 6 | dynamodb = boto3.client('dynamodb') 7 | 8 | bucket='e2esa-mybucket-image' 9 | collectionId='faceprintClnId' 10 | fileName='test.jpg' 11 | threshold = 80 12 | maxFaces=2 13 | 14 | def lambda_handler(event, context): 15 | found = False 16 | try: 17 | response=rekognition.search_faces_by_image(CollectionId=collectionId, 18 | Image={'S3Object':{'Bucket':bucket,'Name':fileName}}, 19 | FaceMatchThreshold=threshold, 20 | MaxFaces=maxFaces) 21 | for match in response['FaceMatches']: 22 | print(match['Face']['FaceId'],match['Face']['Confidence']) 23 | print ('FaceId:' + match['Face']['FaceId']) 24 | print ('Similarity: ' + "{:.2f}".format(match['Similarity']) + "%") 25 | face = dynamodb.get_item(TableName = 'face_recognition', Key= {'RekognitionId':{'S':match['Face']['FaceId']}}) 26 | print(json.dumps(face)) 27 | if 'Item' in face: 28 | print ("Found Person: ", face['Item']['Metadata']['S']) 29 | found = True 30 | 31 | if not found: 32 | print("Person can not be recognized") 33 | 34 | except ClientError as e: 35 | print(e) 36 | return { 37 | 'statusCode': e.args[0], 38 | 'body': json.dumps({'ClientError': str(e)}) 39 | } 40 | except Exception as e: 41 | print(e) 42 | return { 43 | 'statusCode': e.args[0], 44 | 'body': json.dumps({'error': str(e)}) 45 | } 46 | 47 | return { 48 | 'body': found 49 | } 50 | -------------------------------------------------------------------------------- /aws/lambda/lambda-autogenerate-json-upload-to-s3.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | from datetime import date 4 | import random 5 | import calendar 6 | import time 7 | from datetime import datetime 8 | 9 | def lambda_handler(event, context): 10 | usr_id = random.randint(10000000, 100000000) 11 | 12 | current_GMT = time.gmtime() 13 | time_stamp = calendar.timegm(current_GMT) 14 | print("Current timestamp:", time_stamp) 15 | 16 | data_dict = { 17 | "id": usr_id, 18 | "name": "Twitter Dev", 19 | "screen_name": "TwitterDev", 20 | "location": "Internet", 21 | "url": "https:\/\/dev.twitter.com\/", 22 | "description": "Your official source for Twitter Platform news, updates & events. Need technical help?" 23 | } 24 | 25 | 26 | # Convert Dictionary to JSON String 27 | data_string = json.dumps(data_dict, indent=2, default=str) 28 | 29 | 30 | # Upload JSON String to an S3 Object 31 | s3_resource = boto3.resource('s3') 32 | 33 | s3_bucket = s3_resource.Bucket(name='ucal-datalake') 34 | 35 | s3_bucket.put_object( 36 | Key='raw-zone/streaming-data/message-'+str(time_stamp)+'.json', 37 | Body=data_string 38 | ) 39 | 40 | return { 41 | 'statusCode': 200, 42 | 'body': data_string 43 | } 44 | -------------------------------------------------------------------------------- /aws/lambda/lambda-eventbridge-trigger-lambda-tocreate-sqs.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | from botocore.exceptions import ClientError 4 | from datetime import datetime 5 | 6 | def lambda_handler(event, context): 7 | now = datetime.now() 8 | current_time = now.strftime("%d %B %Y, %H:%M:%S %p") 9 | print(event) 10 | try: 11 | event_time = event['time'] 12 | print(event_time) 13 | print(event['detail']) 14 | source_bucket = event['detail']['bucket']['name'] 15 | print(source_bucket) 16 | file_name = event['detail']['object']['key'] 17 | print(file_name) 18 | size = event['detail']['object']['size'] 19 | print(size) 20 | source_ip_address = event['detail']['source-ip-address'] 21 | message_body ={ 22 | 'fileName': file_name, 23 | 'fileSize': size, 24 | 'sourceBucket': source_bucket, 25 | 'sourceIPAddress': source_ip_address, 26 | 'uploadTime': event_time, 27 | 'timeStamp': current_time 28 | } 29 | 30 | # Set up SQS client 31 | sqs = boto3.client('sqs') 32 | queue_url = 'https://sqs.us-east-1.amazonaws.com/306442480424/clamav-waiting-queue' 33 | 34 | # Send message to SQS queue 35 | response = sqs.send_message( 36 | QueueUrl=queue_url, 37 | MessageAttributes={ 38 | 'Service': { 39 | 'DataType': 'String', 40 | 'StringValue': 'ClamAV scanning' 41 | }, 42 | 'Author': { 43 | 'DataType': 'String', 44 | 'StringValue': 'John Doe' 45 | }, 46 | 'CreatedOn': { 47 | 'DataType': 'String', 48 | 'StringValue': current_time 49 | } 50 | }, 51 | MessageBody=json.dumps(message_body, indent = 4) 52 | ) 53 | 54 | print(f"MessageId: {response['MessageId']}") 55 | 56 | except ClientError as e: 57 | response = { 58 | 'statusCode': e.response['ResponseMetadata']['HTTPStatusCode'], 59 | 'body': json.dumps('Error uploading file: ' + e.response['Error']['Message']) 60 | } 61 | return { 62 | 'statusCode': 200, 63 | 'MessageId': response['MessageId'], 64 | 'body': 'Message has been created in SQS queue' 65 | } 66 | -------------------------------------------------------------------------------- /aws/lambda/lambda-file-upload-s3-sqs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from botocore.exceptions import ClientError 5 | import base64 6 | from datetime import datetime 7 | 8 | s3 = boto3.client('s3') 9 | 10 | def lambda_handler(event, context): 11 | try: 12 | file_content = event['body-json'] 13 | content_decoded=base64.b64decode(file_content) 14 | 15 | filename = event['params']['header']['filename'] 16 | print(f"filename: {filename}") 17 | bucket_name = 'e2esa-demo' 18 | file_key = 'upload/' + filename 19 | 20 | # Upload the file to S3 21 | s3.put_object(Bucket=bucket_name, Key=file_key, Body=content_decoded) 22 | # send message to SQS 23 | send_msg = send_sqs_message(filename) 24 | if send_msg.statusCode==200: 25 | print(f"sqs message status: {send_msg.statusCode}") 26 | response = { 27 | 'statusCode': 200, 28 | 'body': json.dumps('File uploaded successfully') 29 | } 30 | 31 | except ClientError as e: 32 | response = { 33 | 'statusCode': e.response['ResponseMetadata']['HTTPStatusCode'], 34 | 'body': json.dumps('Error uploading file: ' + e.response['Error']['Message']) 35 | } 36 | 37 | return response 38 | 39 | def send_sqs_message(data): 40 | try: 41 | # Set up SQS client 42 | sqs = boto3.client('sqs') 43 | queue_url = 'https://sqs.us-east-1.amazonaws.com/306442480424/scanning-queue' 44 | now = datetime.now() 45 | current_time = now.strftime("%H:%M:%S %p") 46 | 47 | # Message to be sent to SQS queue 48 | message_body = "File "+data+" has been sent for scanning at "+current_time 49 | 50 | # Send message to SQS queue 51 | response = sqs.send_message( 52 | QueueUrl=queue_url, 53 | MessageAttributes={ 54 | 'Service': { 55 | 'DataType': 'String', 56 | 'StringValue': 'ClamAV scanning' 57 | }, 58 | 'Author': { 59 | 'DataType': 'String', 60 | 'StringValue': 'John Doe' 61 | }, 62 | 'CreatedOn': { 63 | 'DataType': 'String', 64 | 'StringValue': current_time 65 | } 66 | }, 67 | MessageBody=message_body 68 | ) 69 | 70 | print(f"MessageId: {response['MessageId']}") 71 | except ClientError as e: 72 | response = { 73 | 'statusCode': e.response['ResponseMetadata']['HTTPStatusCode'], 74 | 'body': json.dumps('Error uploading file: ' + e.response['Error']['Message']) 75 | } 76 | 77 | return response -------------------------------------------------------------------------------- /aws/lambda/lambda-generate-twitter-message-stream.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | from datetime import date 4 | import random 5 | import calendar 6 | import time 7 | from datetime import datetime 8 | 9 | def lambda_handler(event, context): 10 | 11 | usr_id = random.randint(10000000, 100000000) 12 | id_str = random.randint(9999999999, 99999999999) 13 | 14 | current_GMT = time.gmtime() 15 | time_stamp = calendar.timegm(current_GMT) 16 | print("Current timestamp:", time_stamp) 17 | 18 | country=["US", "Canada", "Mexico", "India", "Japan" , "Dubai" , "China", "Kenya", "Australia" , "Srilanka"] 19 | hash_tags=create_hashtags(event) 20 | 21 | data_dict = { 22 | "created_at": datetime.fromtimestamp(time_stamp), 23 | "id_str": id_str, 24 | "text": "1\/ Today we\u2019re sharing our vision for the future of the Twitter API platform!\nhttps:\/\/t.co\/XweGngmxlP", 25 | "user": [ 26 | { 27 | "id": usr_id, 28 | "name": "Twitter Dev", 29 | "screen_name": "TwitterDev", 30 | "location": "Internet", 31 | "url": "https:\/\/dev.twitter.com\/", 32 | "description": "Your official source for Twitter Platform news, updates & events. Need technical help? Visit https:\/\/twittercommunity.com\/ \u2328\ufe0f #TapIntoTwitter" 33 | }], 34 | "place": [country[random.randint(0, 9)]], 35 | "entities": { 36 | "hashtags": hash_tags, 37 | "urls": [ 38 | { 39 | "url": "https:\/\/t.co\/XweGngmxlP", 40 | "unwound": { 41 | "url": "https:\/\/cards.twitter.com\/cards\/18ce53wgo4h\/3xo1c", 42 | "title": "Building the Future of the Twitter API Platform" 43 | } 44 | } 45 | ], 46 | "user_mentions": [ 47 | ] 48 | } 49 | } 50 | 51 | 52 | # Convert Dictionary to JSON String 53 | data_string = json.dumps(data_dict, indent=2, default=str) 54 | 55 | 56 | # Upload JSON String to an S3 Object 57 | s3_resource = boto3.resource('s3') 58 | 59 | s3_bucket = s3_resource.Bucket(name='ucal-datalake') 60 | 61 | s3_bucket.put_object( 62 | Key='raw-zone/streaming-data/message-'+str(time_stamp)+'.json', 63 | Body=data_string 64 | ) 65 | 66 | return { 67 | 'statusCode': 200, 68 | 'body': data_string 69 | } 70 | 71 | 72 | def create_hashtags(event): 73 | hashtags_list=["Ucal", "Data608", "Calgary", "Canada", "Alberta" ] 74 | return [ 75 | hashtags_list[random.randint(1,4)] , hashtags_list[random.randint(1,4)] 76 | ] 77 | -------------------------------------------------------------------------------- /aws/lambda/lambda-nodejs-dynamodb.txt: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | const crypto = require("crypto"); 3 | 4 | // Generate unique id with no external dependencies 5 | const generateUUID = () => crypto.randomBytes(16).toString("hex"); 6 | 7 | // Initialising the DynamoDB SDK 8 | const documentClient = new AWS.DynamoDB.DocumentClient(); 9 | 10 | exports.handler = async event => { 11 | const { empNo } = JSON.parse(event.body); 12 | const { empName } = JSON.parse(event.body); 13 | const { empEmail } = JSON.parse(event.body); 14 | 15 | const params = { 16 | TableName: "Employee", // The name of your DynamoDB table 17 | Item: { // Creating an Item with a unique id and with the passed title 18 | empid: generateUUID(), 19 | empNo: empNo, 20 | empName: empName, 21 | empEmail: empEmail 22 | } 23 | }; 24 | try { 25 | const data = await documentClient.put(params).promise(); 26 | const response = { 27 | statusCode: 200 28 | }; 29 | return response; // Returning a 200 if the item has been inserted 30 | } catch (e) { 31 | return { 32 | statusCode: 500, 33 | body: JSON.stringify(e) 34 | }; 35 | } 36 | }; 37 | 38 | 39 | // Test JSON format while testing in Lambda 40 | 41 | { 42 | "body": "{\"empNo\": \"110011\",\"empName\": \"New Employee\",\"empEmail\": \"som@awstechguide.com\"}" 43 | } 44 | 45 | // Test JSON format while testing through a client like POSTMAN 46 | 47 | { 48 | "empNo": "220011", 49 | "empName": "New Employee", 50 | "empEmail": "som@awstechguide.com" 51 | } 52 | -------------------------------------------------------------------------------- /aws/lambda/lambda-nodejs-file-upload-to-efs.py: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs'; 2 | 3 | 4 | export const handler = async (event) => { 5 | try { 6 | let filename = event['params']['header']['filename'] 7 | const fileContent = event['body-json']; 8 | const filePath = '/mnt/upload/'+filename; 9 | 10 | fs.writeFileSync(filePath, fileContent); 11 | 12 | const response = { 13 | statusCode: 200, 14 | body: JSON.stringify('File uploaded successfully') 15 | }; 16 | 17 | return response; 18 | 19 | } catch (error) { 20 | console.error('Error uploading file:', error); 21 | const errorResponse = { 22 | statusCode: 500, 23 | body: JSON.stringify('Error uploading file') 24 | }; 25 | 26 | return errorResponse; 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /aws/lambda/lambda-nodejs-upload-file-to-efs-v02.js: -------------------------------------------------------------------------------- 1 | // NodeJS 2 | import * as fs from 'fs'; 3 | 4 | export const handler = async (event) => { 5 | try { 6 | 7 | const fileContent = JSON.parse(event.body).content; 8 | const filePath = '/mnt/upload/myfile.txt'; 9 | 10 | fs.writeFileSync(filePath, fileContent); 11 | 12 | const response = { 13 | statusCode: 200, 14 | body: JSON.stringify('File uploaded successfully') 15 | }; 16 | 17 | return response; 18 | 19 | } catch (error) { 20 | console.error('Error uploading file:', error); 21 | const errorResponse = { 22 | statusCode: 500, 23 | body: JSON.stringify('Error uploading file') 24 | }; 25 | 26 | return errorResponse; 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /aws/lambda/lambda-nodejs-upload-file-to-efs.js: -------------------------------------------------------------------------------- 1 | // NodeJs script 2 | // Pre-requisite: The lambda should be attached to VPC and the File System should be attached also. 3 | // File system should have accesspoint /mnt/upload/ created already 4 | // API gateway should have mapping templated defined for content type 'multipart/form-data' - When there are no templates defined (recommended) 5 | 6 | import * as fs from 'fs'; 7 | 8 | 9 | export const handler = async (event) => { 10 | try { 11 | let filename = event['params']['header']['filename'] 12 | const fileContent = event['body-json']; 13 | const filePath = '/mnt/upload/'+filename; 14 | 15 | fs.writeFileSync(filePath, fileContent); 16 | 17 | const response = { 18 | statusCode: 200, 19 | body: JSON.stringify('File uploaded successfully') 20 | }; 21 | 22 | return response; 23 | 24 | } catch (error) { 25 | console.error('Error uploading file:', error); 26 | const errorResponse = { 27 | statusCode: 500, 28 | body: JSON.stringify('Error uploading file') 29 | }; 30 | 31 | return errorResponse; 32 | } 33 | }; 34 | 35 | -------------------------------------------------------------------------------- /aws/lambda/lambda-python-file-archive.py: -------------------------------------------------------------------------------- 1 | ## it is a lambda function in python to archive a file when it is dropped in an s3 bucket and read the file content. It is manly to read a small text file 2 | import boto3 3 | import urllib 4 | 5 | def lambda_handler(event, context): 6 | s3_client =boto3.client('s3') 7 | 8 | s3_archive_bucket="arc_bucket" 9 | 10 | bucket_name = event['Records'][0]['s3']['bucket']['name'] 11 | key=event['Records'][0]['s3']['object']['key'] 12 | key=urllib.parse.unquote_plus(key,encoding='utf-8') 13 | message=key + ' file uploaded ' + ' to bucket ' + bucket_name 14 | print(message) 15 | 16 | response=s3_client.get_object(Bucket='s3fileupload',Key=key) 17 | contents= response["Body"].read().decode() 18 | print("file : \n",contents) 19 | 20 | s3_upload_archive =s3_client.put_object(Bucket=s3_archive_bucket, Key=key) 21 | -------------------------------------------------------------------------------- /aws/lambda/lambda-python-upload-file-v02.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from botocore.exceptions import ClientError 5 | import base64 6 | 7 | s3 = boto3.client('s3') 8 | 9 | def lambda_handler(event, context): 10 | try: 11 | file_content = event['body-json'] 12 | content_decoded=base64.b64decode(file_content) 13 | 14 | filename = event['params']['header']['filename'] 15 | print(f"filename: {filename}") 16 | bucket_name = 'e2esa-demo' 17 | #file_key = 'upload/' + event['filename'] 18 | file_key = 'upload/' + filename 19 | 20 | # Upload the file to S3 21 | s3.put_object(Bucket=bucket_name, Key=file_key, Body=content_decoded) 22 | 23 | response = { 24 | 'statusCode': 200, 25 | 'body': json.dumps('File uploaded successfully') 26 | } 27 | 28 | except ClientError as e: 29 | response = { 30 | 'statusCode': e.response['ResponseMetadata']['HTTPStatusCode'], 31 | 'body': json.dumps('Error uploading file: ' + e.response['Error']['Message']) 32 | } 33 | 34 | return response 35 | -------------------------------------------------------------------------------- /aws/lambda/lambda-python-uploadfile-api.py: -------------------------------------------------------------------------------- 1 | # it is a lambda function in python. Used for triggering through an API gateway to upload the file into a s3 bucket. 2 | 3 | import json 4 | import boto3 5 | import base64 6 | 7 | def lambda_handler(event, context): 8 | 9 | s3_client =boto3.client('s3') 10 | s3_bucket="s3fileupload" 11 | 12 | file_content=event["content"] 13 | content_decoded=base64.b64decode(file_content) 14 | s3_upload =s3_client.put_object(Bucket=s3_bucket, Key='uploaded-file.csv', Body=content_decoded) 15 | 16 | return { 17 | 'statusCode': 200, 18 | 'body': json.dumps('Hello from Lambda!') 19 | } 20 | -------------------------------------------------------------------------------- /aws/lambda/lambda-receive-message-from-sqs.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | 4 | def lambda_handler(event, context): 5 | # Set up SQS client 6 | sqs = boto3.client('sqs') 7 | queue_url='https://sqs.us-east-1.amazonaws.com/306442480424/scanning-queue' 8 | 9 | # Receive messages from SQS queue 10 | response = sqs.receive_message( 11 | QueueUrl=queue_url, 12 | MaxNumberOfMessages=1, 13 | VisibilityTimeout=60, 14 | WaitTimeSeconds=10 15 | ) 16 | 17 | # Check if messages were received 18 | if 'Messages' in response: 19 | #print(response['Messages']) 20 | for message in response['Messages']: 21 | message_body = message['Body'] 22 | receipt_handle = message['ReceiptHandle'] 23 | message_id= message['MessageId'] 24 | 25 | #print(f"Received message: {message_body}") 26 | print(json.loads(message_body)) 27 | 28 | # Add processing logic here 29 | 30 | # Delete the message from the queue 31 | sqs.delete_message( 32 | QueueUrl=queue_url, 33 | ReceiptHandle=receipt_handle 34 | ) 35 | 36 | print(f"Deleted message:"+message_id) 37 | else: 38 | print("No messages received from the queue.") 39 | 40 | return { 41 | 'statusCode': 200, 42 | 'body': 'Message processing complete for messageId: '+message_id 43 | } 44 | -------------------------------------------------------------------------------- /aws/lambda/lambda-send-message-to-sqs.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from datetime import datetime 3 | import json 4 | 5 | def lambda_handler(event, context): 6 | now = datetime.now() 7 | current_time = now.strftime("%d %B %Y, %H:%M:%S %p") 8 | ts= datetime.timestamp(now) 9 | print(current_time) 10 | print(ts) 11 | 12 | # Set up SQS client 13 | sqs = boto3.client('sqs') 14 | queue_url = 'https://sqs.us-east-1.amazonaws.com/306442480424/scanning-queue' 15 | message_body ={ 16 | 'fileName': 'test.txt', 17 | 'fileKey': 'upload', 18 | 'scanStatus': 'WAITING', # 'WAITING','SCANNING', 'SCANNED', 19 | 'waitingStartTime': current_time, 20 | 'scanningStartTime': current_time, 21 | 'scanningEndTime': current_time 22 | } 23 | 24 | # Message to be sent to SQS queue 25 | #message_body = "Hello, this is a test message sent at current time "+current_time 26 | 27 | # Send message to SQS queue 28 | response = sqs.send_message( 29 | QueueUrl=queue_url, 30 | MessageAttributes={ 31 | 'Service': { 32 | 'DataType': 'String', 33 | 'StringValue': 'ClamAV scanning' 34 | }, 35 | 'Author': { 36 | 'DataType': 'String', 37 | 'StringValue': 'John Doe' 38 | }, 39 | 'CreatedOn': { 40 | 'DataType': 'String', 41 | 'StringValue': current_time 42 | } 43 | }, 44 | MessageBody=json.dumps(message_body, indent = 4) 45 | ) 46 | 47 | print(f"MessageId: {response['MessageId']}") 48 | 49 | return { 50 | 'statusCode': 200, 51 | 'MessageId': response['MessageId'], 52 | 'body': 'Message sent to SQS: '+json.dumps(message_body, indent = 4) 53 | } 54 | -------------------------------------------------------------------------------- /aws/lambda/lambda-send-sqs-message-on-s3-upload-v02.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | from botocore.exceptions import ClientError 4 | from datetime import datetime 5 | 6 | def lambda_handler(event, context): 7 | now = datetime.now() 8 | current_time = now.strftime("%d %B %Y, %H:%M:%S %p") 9 | try: 10 | for record in event['Records']: 11 | event_time = record['eventTime'] 12 | source_bucket = record['s3']['bucket']['name'] 13 | key = record['s3']['object']['key'] 14 | size = record['s3']['object']['size'] 15 | source_ip_address = record['requestParameters']['sourceIPAddress'] 16 | message_body ={ 17 | 'fileName': key, 18 | 'fileSize': size, 19 | 'sourceBucket': source_bucket, 20 | 'sourceIPAddress': source_ip_address, 21 | 'scanStatus': 'WAITING', # 'WAITING','SCANNING', 'SCANNED', 22 | 'uploadEventTime': event_time, 23 | 'waitingStartTime': current_time, 24 | 'scanningStartTime': current_time, 25 | 'scanningEndTime': current_time 26 | } 27 | 28 | # Set up SQS client 29 | sqs = boto3.client('sqs') 30 | queue_url = 'https://sqs.us-east-1.amazonaws.com/306442480424/scanning-queue' 31 | 32 | # Send message to SQS queue 33 | response = sqs.send_message( 34 | QueueUrl=queue_url, 35 | MessageAttributes={ 36 | 'Service': { 37 | 'DataType': 'String', 38 | 'StringValue': 'ClamAV scanning' 39 | }, 40 | 'Author': { 41 | 'DataType': 'String', 42 | 'StringValue': 'John Doe' 43 | }, 44 | 'CreatedOn': { 45 | 'DataType': 'String', 46 | 'StringValue': current_time 47 | } 48 | }, 49 | MessageBody=json.dumps(message_body, indent = 4) 50 | ) 51 | 52 | print(f"MessageId: {response['MessageId']}") 53 | 54 | except ClientError as e: 55 | response = { 56 | 'statusCode': e.response['ResponseMetadata']['HTTPStatusCode'], 57 | 'body': json.dumps('Error uploading file: ' + e.response['Error']['Message']) 58 | } 59 | return { 60 | 'statusCode': 200, 61 | 'MessageId': response['MessageId'], 62 | 'body': 'Message has been created in SQS' 63 | } 64 | -------------------------------------------------------------------------------- /aws/lambda/lambda-send-sqs-message-on-s3-upload.py: -------------------------------------------------------------------------------- 1 | import json 2 | from botocore.exceptions import ClientError 3 | 4 | def lambda_handler(event, context): 5 | try: 6 | print(event) 7 | print(event['Records']) 8 | for record in event['Records']: 9 | event_time = record['eventTime'] 10 | source_bucket = record['s3']['bucket']['name'] 11 | key = record['s3']['object']['key'] 12 | size = record['s3']['object']['size'] 13 | source_ip_address = record['requestParameters']['sourceIPAddress'] 14 | print(event_time) 15 | print(source_bucket) 16 | print(key) 17 | print(size) 18 | print(source_ip_address) 19 | 20 | except ClientError as e: 21 | response = { 22 | 'statusCode': e.response['ResponseMetadata']['HTTPStatusCode'], 23 | 'body': json.dumps('Error uploading file: ' + e.response['Error']['Message']) 24 | } 25 | return { 26 | 'statusCode': 200, 27 | 'body': 'Message has been sent to SQS' 28 | } 29 | -------------------------------------------------------------------------------- /aws/lambda/lambda-test-input.json: -------------------------------------------------------------------------------- 1 | // Test JSON format while testing in Lambda 2 | 3 | { 4 | "body": "{\"empNo\": \"110011\",\"empName\": \"New Employee\",\"empEmail\": \"som@awstechguide.com\"}" 5 | } 6 | 7 | // Test JSON format while testing through a client like POSTMAN 8 | 9 | { 10 | "empNo": "220011", 11 | "empName": "New Employee", 12 | "empEmail": "som@awstechguide.com" 13 | } 14 | -------------------------------------------------------------------------------- /aws/lambda/lambda-upload-file-to-s3.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from botocore.exceptions import ClientError 5 | import base64 6 | 7 | s3 = boto3.client('s3') 8 | 9 | def lambda_handler(event, context): 10 | try: 11 | file_content = event['body-json'] 12 | content_decoded=base64.b64decode(file_content) 13 | 14 | filename = event['params']['header']['filename'] 15 | print(f"filename: {filename}") 16 | bucket_name = 'e2esa-demo' 17 | file_key = 'upload/' + filename 18 | 19 | # Upload the file to S3 20 | s3.put_object(Bucket=bucket_name, Key=file_key, Body=content_decoded) 21 | 22 | response = { 23 | 'statusCode': 200, 24 | 'body': json.dumps('File uploaded successfully') 25 | } 26 | 27 | except ClientError as e: 28 | response = { 29 | 'statusCode': e.response['ResponseMetadata']['HTTPStatusCode'], 30 | 'body': json.dumps('Error uploading file: ' + e.response['Error']['Message']) 31 | } 32 | 33 | return response 34 | -------------------------------------------------------------------------------- /aws/lambda/lambda_pandas.js: -------------------------------------------------------------------------------- 1 | // Example talks about using pandas in Lambda. 2 | // Please refer the steps https://www.youtube.com/watch?v=lrEAu75zhNI 3 | import json 4 | import pandas as pd 5 | 6 | def lambda_handler(event, context): 7 | a = [5,6,7,8,9] 8 | srs = pd.Series(a) 9 | print(srs) 10 | 11 | return { 12 | 'statusCode': 200, 13 | 'body': json.dumps('Hello from Lambda!') 14 | } 15 | -------------------------------------------------------------------------------- /aws/lambda/read-from-dynamodb.js: -------------------------------------------------------------------------------- 1 | 2 | const AWS = require('aws-sdk'); 3 | 4 | const ddb = new AWS.DynamoDB.DocumentClient({region: 'us-east-1'}); 5 | 6 | exports.handler = async (event, context, callback) => { 7 | await readMessage().then(data => { 8 | data.Items.forEach(function(item) { 9 | console.log(item.message) 10 | }); 11 | callback(null, { 12 | // If success return 200, and items 13 | statusCode: 200, 14 | body: data.Items, 15 | headers: { 16 | 'Access-Control-Allow-Origin': '*', 17 | }, 18 | }) 19 | }).catch((err) => { 20 | // If an error occurs write to the console 21 | console.error(err); 22 | }) 23 | }; 24 | 25 | function readMessage() { 26 | const params = { 27 | TableName: 'dth_transactions', 28 | Limit: 10 29 | } 30 | return ddb.scan(params).promise(); 31 | } 32 | -------------------------------------------------------------------------------- /aws/lambda/s3-object-tagging.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | from botocore.exceptions import ClientError 4 | from botocore.config import Config 5 | import logging 6 | 7 | config = Config(signature_version='s3v4') 8 | s3 = boto3.client('s3', config=config) 9 | 10 | def lambda_handler(event, context): 11 | try: 12 | response ="" 13 | bucket = 'e2esa-mybucket-us-east-1' 14 | key='upload/test.jpg' 15 | response = s3.put_object_tagging( 16 | Bucket=bucket, 17 | Key=key, 18 | Tagging={ 19 | "TagSet": [ 20 | {"Key": "requestid", "Value": "oldvalue"},{"Key": "newkey", "Value": "value2"} 21 | ] 22 | } 23 | ) 24 | print(response) 25 | get_tags_response = s3.get_object_tagging( 26 | Bucket=bucket, 27 | Key=key, 28 | ) 29 | print(get_tags_response) 30 | # Add user-defined Metadata to S3 object 31 | s3_client = boto3.resource('s3') 32 | s3_object = s3_client.Object(bucket, key) 33 | s3_object.metadata.update({'myid':'myvalue'}) 34 | s3_object.copy_from(CopySource={'Bucket':bucket, 'Key':key}, Metadata=s3_object.metadata, MetadataDirective='REPLACE') 35 | 36 | except ClientError as e: 37 | logging.error(e) 38 | return { 39 | 'statusCode': e.args[0], 40 | 'body': json.dumps({'ClientError': str(e)}) 41 | } 42 | except Exception as e: 43 | logging.error(e) 44 | return { 45 | 'statusCode': e.args[0], 46 | 'body': json.dumps({'error': str(e)}) 47 | } 48 | 49 | return { 50 | 'body': response 51 | } 52 | -------------------------------------------------------------------------------- /aws/lambda/write-to-dynamodb_v1.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | const crypto = require("crypto"); 3 | 4 | // Generate unique id with no external dependencies 5 | const generateUUID = () => crypto.randomBytes(16).toString("hex"); 6 | 7 | // Initialising the DynamoDB SDK 8 | const documentClient = new AWS.DynamoDB.DocumentClient(); 9 | 10 | exports.handler = async event => { 11 | const { transactionId } = JSON.parse(event.body); 12 | const { transactionDate } = JSON.parse(event.body); 13 | const { transactionAmount } = JSON.parse(event.body); 14 | const { transactionBy } = JSON.parse(event.body); 15 | const { transactionStatus } = JSON.parse(event.body); 16 | 17 | const params = { 18 | TableName: "dth_transactions", // The name of your DynamoDB table 19 | Item: { // Creating an Item with a unique id and with the passed title 20 | transactionId: generateUUID(), 21 | transactionDate: transactionDate, 22 | transactionAmount: transactionAmount, 23 | transactionBy: transactionBy, 24 | transactionStatus: transactionStatus 25 | } 26 | }; 27 | try { 28 | const data = await documentClient.put(params).promise(); 29 | const response = { 30 | statusCode: 200 31 | }; 32 | return response; // Returning a 200 if the item has been inserted 33 | } catch (e) { 34 | return { 35 | statusCode: 500, 36 | body: JSON.stringify(e) 37 | }; 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /aws/lambda/write-to-dynamodb_v2.js: -------------------------------------------------------------------------------- 1 | 2 | const AWS = require('aws-sdk'); 3 | 4 | const ddb = new AWS.DynamoDB.DocumentClient({region: 'us-east-1'}); 5 | 6 | exports.handler = async (event, context, callback) => { 7 | const requestId = context.awsRequestId; 8 | await createMessage(requestId).then(() => { 9 | callback(null, { 10 | statusCode: 201, 11 | body: '', 12 | headers: { 13 | 'Access-Control-Allow-Origin' : '*' 14 | } 15 | }); 16 | }).catch((err) => { 17 | console.error(err) 18 | }) 19 | }; 20 | 21 | function createMessage(requestId) { 22 | const params = { 23 | TableName: 'dih_transactions', 24 | Item: { 25 | 'messageId' : requestId, 26 | 'message' : 'Hello from lambda' 27 | } 28 | } 29 | return ddb.put(params).promise(); 30 | } 31 | -------------------------------------------------------------------------------- /aws/ssm/aws-configure-headless.md: -------------------------------------------------------------------------------- 1 | aws configure set varname value [--profile profile-name] 2 | 3 | https://docs.aws.amazon.com/cli/latest/reference/configure/set.html 4 | 5 | ``` 6 | $ aws configure set aws_access_key_id default_access_key --profile testing 7 | $ aws configure set aws_secret_access_key default_secret_key --profile testing 8 | $ aws configure set default.region us-west-2 --profile testing 9 | $ aws configure set default.ca_bundle /path/to/ca-bundle.pem --profile testing 10 | $ aws configure set region us-west-1 --profile testing 11 | ``` 12 | -------------------------------------------------------------------------------- /aws/ssm/secrets-manager.md: -------------------------------------------------------------------------------- 1 | 2 | # check if instance IAM profile enough to run below 3 | 4 | ``` 5 | aws secretsmanager describe-secrets --secret-id 6 | aws secretsmanager get-secret-value --secret-id --version-stage AWSCURRENT 7 | ``` 8 | https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/setup-credentials.html 9 | 10 | ``` 11 | export AWS_REGION=your_aws_region 12 | export AWS_ACCESS_KEY_ID=your_access_key_id 13 | export AWS_SECRET_ACCESS_KEY=your_secret_access_key 14 | ``` 15 | 16 | 17 | aws configure set varname value [--profile profile-name] 18 | 19 | https://docs.aws.amazon.com/cli/latest/reference/configure/set.html 20 | 21 | ``` 22 | $ aws configure set aws_access_key_id default_access_key --profile testing 23 | $ aws configure set aws_secret_access_key default_secret_key --profile testing 24 | $ aws configure set default.region us-west-2 --profile testing 25 | $ aws configure set default.ca_bundle /path/to/ca-bundle.pem --profile testing 26 | $ aws configure set region us-west-1 --profile testing 27 | ``` 28 | 29 | ``` 30 | aws ssm put-parameter --name mysecret --type SecureString --value 'secret value' --profile 31 | aws ssm get-parameter --name mysecret --profile 32 | aws ssm get-parameter --name mysecret --with-decryption --profile 33 | ``` 34 | https://docs.aws.amazon.com/systems-manager/latest/userguide/integration-ps-secretsmanager.html 35 | 36 | ``` 37 | aws ssm get-parameter \ 38 | --name /aws/reference/secretsmanager/s1-secret:AWSCURRENT \ 39 | --with-decryption 40 | ``` 41 | -------------------------------------------------------------------------------- /azure/ado-pipeline.md: -------------------------------------------------------------------------------- 1 | Please [click here](https://github.com/e2eSolutionArchitect/hashicorp-packer/tree/main/pipeline/azure) for Azure DevOps pipelines. 2 | Sequential and Parallel pipelines. 3 | -------------------------------------------------------------------------------- /azure/policy/azure-custom-tags-allow-or-deny-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "displayName": "Require custom tags as per org policy on resources", 3 | "policyType": "BuiltIn", 4 | "mode": "Indexed", 5 | "description": "Enforces existence of a tag. Does not apply to resource groups.Enforce each resource to be tagged as per the organizational cloud governance policy", 6 | "metadata": { 7 | "version": "1.0.1", 8 | "category": "Tags" 9 | }, 10 | "policyRule": { 11 | "if": { 12 | "allOf": [ 13 | { 14 | "field": "tags[project]", 15 | "exists": "false" 16 | }, 17 | { 18 | "field": "tags[cost_center]", 19 | "exists": "false" 20 | },{ 21 | "field": "tags[env]", 22 | "exists": "false" 23 | } 24 | ] 25 | }, 26 | "then": { 27 | "effect": "deny" 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /azure/policy/azure-custom-tags-check-append.json: -------------------------------------------------------------------------------- 1 | { 2 | "displayName": "Check required custom tags and append tags if missing mandatory tags as per org policy on resources", 3 | "policyType": "BuiltIn", 4 | "mode": "Indexed", 5 | "description": "Enforces existence of a tag. Does not apply to resource groups.Enforce each resource to be tagged as per the organizational cloud governance policy", 6 | "metadata": { 7 | "version": "1.0.1", 8 | "category": "Tags" 9 | }, 10 | "policyRule": { 11 | "if": { 12 | "allOf": [ 13 | { 14 | "field": "tags[project]", 15 | "exists": "false" 16 | }, 17 | { 18 | "field": "tags[cost_center]", 19 | "exists": "false" 20 | },{ 21 | "field": "tags[env]", 22 | "exists": "false" 23 | } 24 | ] 25 | }, 26 | "then": { 27 | "effect": "append", 28 | "details": [ 29 | { 30 | "field": "tags[project]", 31 | "value": "e2esa-demo" 32 | },{ 33 | "field": "tags[cost_center]", 34 | "value": "CC-E2ESAAZ01" 35 | },{ 36 | "field": "tags[env]", 37 | "value": "dev" 38 | } 39 | ] 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /azure/policy/azure-custom-tags-list-allow-deny-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "displayName": "Require custom tags with selected values as per org policy on resources", 3 | "policyType": "BuiltIn", 4 | "mode": "Indexed", 5 | "description": "Enforces existence of a tag. Does not apply to resource groups.Enforce each resource to be tagged as per the organizational cloud governance policy", 6 | "metadata": { 7 | "version": "1.0.1", 8 | "category": "Tags" 9 | }, 10 | "policyRule": { 11 | "if": { 12 | "allOf": [ 13 | { 14 | "field": "tags[project]", 15 | "notIn": ["e2esa-demo1","e2esa-demo2"] 16 | }, 17 | { 18 | "field": "tags[cost_center]", 19 | "notIn": ["CC-E2ESAAZ01","CC-E2ESAAZ01"] 20 | },{ 21 | "field": "tags[env]", 22 | "notIn": ["dev","stg","prod"] 23 | } 24 | ] 25 | }, 26 | "then": { 27 | "effect": "deny" 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /clamav/clamav-errors.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Error: cli_loaddbdir: No supported database files found in /var/lib/clamav ERROR: Can't open file or directory 4 | While running below command 5 | ``` 6 | sudo docker run --rm \ 7 | --name "clamav_container_01" \ 8 | --mount source=clam_db,target=/var/lib/clamav \ 9 | --mount type=bind,source=/opt/scandir,target=/scandir \ 10 | --env 'CLAMAV_NO_FRESHCLAMD=false' \ 11 | --env 'FRESHCLAM_CHECKS=24' \ 12 | clamav/clamav:1.0.2_base \ 13 | clamscan /scandir 14 | ``` 15 | 16 | Fix: first remove clamscan /scandir part and run. Then update the container name "clamav_container_02" and run the command with clamscan /scandir 17 | Step 1 18 | ``` 19 | sudo docker run --rm \ 20 | --name "clamav_container_01" \ 21 | --mount source=clam_db,target=/var/lib/clamav \ 22 | --mount type=bind,source=/opt/scandir,target=/scandir \ 23 | --env 'CLAMAV_NO_FRESHCLAMD=false' \ 24 | --env 'FRESHCLAM_CHECKS=24' \ 25 | clamav/clamav:1.0.2_base 26 | ``` 27 | Step 2 28 | ``` 29 | sudo docker run --rm \ 30 | --name "clamav_container_02" \ 31 | --mount source=clam_db,target=/var/lib/clamav \ 32 | --mount type=bind,source=/opt/scandir,target=/scandir \ 33 | --env 'CLAMAV_NO_FRESHCLAMD=false' \ 34 | --env 'FRESHCLAM_CHECKS=24' \ 35 | clamav/clamav:1.0.2_base \ 36 | clamscan /scandir 37 | ``` 38 | 39 | 40 | ## Could not connect to clamd on LocalSocket /tmp/clamd.sock: No such file or directory 41 | While running clamDscan 42 | ``` 43 | sudo docker run --rm \ 44 | --name "clamav_container_01" \ 45 | --mount source=clam_db,target=/var/lib/clamav \ 46 | --mount type=bind,source=/opt/scandir,target=/scandir \ 47 | --env 'CLAMAV_NO_FRESHCLAMD=false' \ 48 | --env 'FRESHCLAM_CHECKS=24' \ 49 | clamav/clamav:1.0.2_base \ 50 | clamdscan /scandir 51 | ``` 52 | Fix: 53 | 54 | 55 | --------------------------------------- 56 | 57 | ## ERROR: Can't open /var/log/clamav/freshclam.log in append mode (check permissions!) 58 | ``` 59 | sudo chmod -R 777 /var/log/clamav 60 | which freshclam 61 | ``` 62 | 63 | ## ERROR: /var/log/clamav/freshclam.log is locked by another process 64 | ## freshclam run automatically, If you want to stop the daemon and run it manually: 65 | ``` 66 | sudo systemctl stop clamav-freshclam.service 67 | sudo freshclam 68 | ``` 69 | 70 | ## Error: Could not connect to clamd on LocalSocket /var/run/clamav/clamd.ctl: No such file or directory LibClamAV Error: File tree walk aborted. 71 | If clamd didn't run properly it will generate below error. clamd creates clamd.ctl file in /var/run/clamav/ 72 | 73 | Fix: 74 | ``` 75 | mkdir /var/run/clamav 76 | sudo chmod -R 777 /var/run/clamav 77 | sudo clamd 78 | ``` 79 | -------------------------------------------------------------------------------- /clamav/install-clamav.md: -------------------------------------------------------------------------------- 1 | 2 | # Install in Ubuntu 3 | 4 | ``` 5 | sudo apt-get install -y clamav clamav-base clamav-daemon clamav-freshclam clamav-testfiles 6 | ``` 7 | 8 | # ERROR: Can't open /var/log/clamav/freshclam.log in append mode (check permissions!) 9 | ``` 10 | sudo chmod -R 777 /var/log/clamav 11 | which freshclam 12 | ``` 13 | 14 | # ERROR: /var/log/clamav/freshclam.log is locked by another process 15 | # freshclam run automatically, If you want to stop the daemon and run it manually: 16 | ``` 17 | sudo systemctl stop clamav-freshclam.service 18 | sudo freshclam 19 | ``` 20 | 21 | crontab -e 22 | @hourly /usr/bin/freshclam --quiet 23 | 24 | # Configure clamd 25 | cd /etc/clamav 26 | vi clamd.conf 27 | vi freshclam.conf 28 | 29 | # Start clamav service 30 | 31 | ``` 32 | sudo systemctl start clamav-freshclam.service 33 | sudo systemctl status clamav-freshclam.service 34 | ``` 35 | # Install htop to check momory consumption 36 | ``` 37 | sudo apt-get install htop -y 38 | htop 39 | ``` 40 | 41 | ## Run clamd. check LocalSocket path in clamd.conf 42 | ``` 43 | mkdir /var/run/clamav 44 | sudo chmod -R 777 /var/run/clamav 45 | sudo clamd 46 | ``` 47 | 48 | ## If clamd didn't run properly it will generate below error. clamd creates clamd.ctl file in /var/run/clamav/ 49 | ## Error: Could not connect to clamd on LocalSocket /var/run/clamav/clamd.ctl: No such file or directory LibClamAV Error: File tree walk aborted. 50 | 51 | # Run scan 52 | 53 | ``` 54 | clamdscan --fdpass 55 | clamdscan --fdpass /home/ubuntu/scantest 56 | clamdscan --fdpass --move=/home/ubuntu/quarantined /home/ubuntu/scandir 57 | ``` 58 | 59 | 60 | # Generate example config 61 | ``` 62 | clamconf -g freshclam.conf > freshclam.conf 63 | clamconf -g clamd.conf > clamd.conf 64 | ``` 65 | 66 | 67 | # Enable On Access Scanning 68 | Update clamd.conf with below attributes 69 | ``` 70 | OnAccessMaxFileSize 5M 71 | OnAccessIncludePath /home/ubuntu/scandir/ 72 | OnAccessIncludePath /home/ubuntu/ 73 | OnAccessPrevention yes 74 | OnAccessExcludeUname clamav 75 | OnAccessExtraScanning yes 76 | OnAccessMaxThreads 10 77 | 78 | ``` 79 | ### If getting permission issue to update clamd.conf. change the permission 80 | 81 | ``` 82 | sudo chmod -R 777 /etc/clamav/clamd.conf 83 | ``` 84 | 85 | # Run clamonacc ( OnAccess scan should be enabled in /etc/clamav/clamd.conf first) 86 | ``` 87 | sudo clamonacc 88 | ``` 89 | 90 | # check clamav log file 91 | ``` 92 | tail -f /var/log/clamav/clamav.log 93 | ``` 94 | 95 | # Kernel should be > 3.8 96 | ``` 97 | uname -rm 98 | config-5.19.0-1025-aws 99 | # here it is 5.19.0-1025 100 | 101 | sudo su 102 | cd boot 103 | vi config-5.19.0-1025-aws 104 | 105 | CONFIG_FANOTIFY=y 106 | CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y 107 | ``` 108 | -------------------------------------------------------------------------------- /clamav/readme.md: -------------------------------------------------------------------------------- 1 | Contents referred from ClamAV documentation [here](https://docs.clamav.net) 2 | Docker Hub [here](https://hub.docker.com/r/clamav/clamav) 3 | 4 | check cloud init output log 5 | ``` 6 | tail -f /var/log/cloud-init-output.log 7 | ``` 8 | 9 | Recommended RAM for ClamAV (As of 2020/09/20): Minimum: 3 GiB, Preferred: 4 GiB 10 | 11 | # On RHEL Install 12 | 13 | # On Ubuntu Install 14 | 15 | # On Docker Install - check [here](https://docs.clamav.net/manual/Installing/Docker.html) 16 | 17 | ### All images come in two forms: 18 | 19 | - clamav/clamav:: A release preloaded with signature databases. 20 | 21 | Using this container will save the ClamAV project some bandwidth. Use this if you will keep the image around so that you don't download the entire database set every time you start a new container. Updating with FreshClam from existing databases set does not use much data. 22 | 23 | - clamav/clamav:_base: A release with no signature databases. 24 | 25 | Use this container only if you mount a volume in your container under /var/lib/clamav to persist your signature database databases. This method is the best option because it will reduce data costs for ClamAV and for the Docker registry, but it does require advanced familiarity with Linux and Docker. 26 | 27 | ### clamav/clamav:latest_base and clamav/clamav:latest: These are the same as clamav/clamav:stable_base and clamav/clamav:stable. They exist because many users expect all images to have a "latest". 28 | 29 | ### Persisting the virus database (volume) 30 | The virus database in /var/lib/clamav is by default unique to each container and thus is normally not shared. 31 | 32 | 1. Create a Docker volume using the docker volume command.. Volumes are completely managed by Docker and are the best choice for creating a persistent database volume. 33 | For example, create a "clam_db" volume: 34 | ``` 35 | docker volume create clam_db 36 | ``` 37 | 2. Create a Bind Mount that maps a file system directory to a path within the container. 38 | Run the container with these arguments to mount the a directory from your host environment as a volume in the container. 39 | --mount type=bind,source=/path/to/databases,target=/var/lib/clamav 40 | 41 | ``` 42 | docker run -it --rm \ 43 | --name "clam_container_01" \ 44 | --mount type=bind,source=/path/to/databases,target=/var/lib/clamav \ 45 | clamav/clamav:stable_base 46 | 47 | 48 | ``` 49 | 50 | ## Docker run 51 | 52 | ``` 53 | sudo docker run -it --rm \ 54 | --name "clam_container_01" \ 55 | --mount source=clam_db,target=/var/lib/clamav \ 56 | --mount type=bind,source=/opt/scandir,target=/scandir \ 57 | --env 'CLAMAV_NO_FRESHCLAMD=false' \ 58 | --env 'FRESHCLAM_CHECKS=24' \ 59 | clamav/clamav:1.0.2_base \ 60 | clamscan /scandir 61 | ``` 62 | 63 | '/opt/scandir' - is the file path which exists in VM. it is mapped to docker target '/scandir'. clamscan to scan in the target directory which is '/scandir' indirectly 'opt/scandir' 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /create-maven-project-cmd.md: -------------------------------------------------------------------------------- 1 | ### check maven version 2 | 3 | mvn --version 4 | 5 | ### Run below command in your command prompt 6 | 7 | mvn archetype:generate -DgroupId=com.mycompany.app -DartifactId=my-app -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false 8 | 9 | ### create a parent and child project 10 | 11 | ### parent project 12 | 13 | mvn archetype:generate -DgroupId=com.mycompany.commonconfig -DartifactId=commonconfig-parent 14 | 15 | ### child project 16 | 17 | cd commonconfig-parent 18 | 19 | mvn archetype:generate -DgroupId=com.mycompany.commonconfig -DartifactId=commonconfig-domain 20 | -------------------------------------------------------------------------------- /create-user.md: -------------------------------------------------------------------------------- 1 | ## login to your VM instance 2 | ## Change the user to sudo 3 | 4 | sudo su - 5 | 6 | ## create an user 7 | 8 | useradd -d /home/awstechguide -m awstechguide 9 | 10 | ## create password for this user. a non expiry password 11 | 12 | passwd awstechguide 13 | passwd -x -1 awstechguide 14 | -------------------------------------------------------------------------------- /datascience/readme.md: -------------------------------------------------------------------------------- 1 | # datascience 2 | Data Science stuffs 3 | 4 | 5 | # Check out the bigger repository of script and ready to use code 6 | 7 | https://github.com/awstechguide/ 8 | 9 | # End to End Solution Architect tutorials 10 | 11 | https://www.youtube.com/channel/UC5Juuk7aTvbRmrABMq4onJA/videos 12 | 13 | # Visit e2e Solution Architect 14 | 15 | www.e2eSolutionArchitect.com 16 | 17 | # Coming Soon Data Insights Hub 18 | 19 | https://datainsightshub.com/ 20 | -------------------------------------------------------------------------------- /django/create-django-project.md: -------------------------------------------------------------------------------- 1 | '' make sure you have django installed and VM profile created. For the steps please check 'install-django' in this repo. 2 | 3 | django-admin startproject 4 | django-admin startproject DjangoAPI 5 | 6 | Run server 7 | python manage.py runserver 8 | 9 | Check the local link at port 8000 10 | http://localhost:8000/ 11 | 12 | --Output------------------ 13 | (vmdev) C:\....\e2esa-git-repo\django-angular>django-admin startproject DjangoAPI 14 | 15 | (vmdev) C:\....\e2esa-git-repo\django-angular>cd DjangoAPI 16 | 17 | (vmdev) C:\....\e2esa-git-repo\django-angular\DjangoAPI>code . [Optional: just opening the code in code editor] 18 | 19 | (vmdev) C:\....\e2esa-git-repo\django-angular\DjangoAPI>python manage.py runserver 20 | Watching for file changes with StatReloader 21 | Performing system checks... 22 | 23 | System check identified no issues (0 silenced). 24 | 25 | You have 18 unapplied migration(s). Your project may not work properly until you apply the migrations for app(s): admin, auth, contenttypes, sessions. 26 | Run 'python manage.py migrate' to apply them. 27 | December 25, 2021 - 13:33:01 28 | Django version 4.0, using settings 'DjangoAPI.settings' 29 | Starting development server at http://127.0.0.1:8000/ 30 | Quit the server with CTRL-BREAK. 31 | [25/Dec/2021 13:33:13] "GET / HTTP/1.1" 200 10697 32 | [25/Dec/2021 13:33:13] "GET /static/admin/css/fonts.css HTTP/1.1" 200 423 33 | [25/Dec/2021 13:33:13] "GET /static/admin/fonts/Roboto-Bold-webfont.woff HTTP/1.1" 200 86184 34 | [25/Dec/2021 13:33:13] "GET /static/admin/fonts/Roboto-Regular-webfont.woff HTTP/1.1" 200 85876 35 | [25/Dec/2021 13:33:13] "GET /static/admin/fonts/Roboto-Light-webfont.woff HTTP/1.1" 200 85692 36 | Not Found: /favicon.ico 37 | [25/Dec/2021 13:33:13] "GET /favicon.ico HTTP/1.1" 404 2113 38 | -------------------- 39 | -------------------------------------------------------------------------------- /docker/Dockerfile/Dockerfile-tomEE808: -------------------------------------------------------------------------------- 1 | # need update. not tested 2 | FROM tomee:8.0.8-webprofile 3 | 4 | # As we are using the tomee Docker images, then we dont need to install Java, Tomcat, and other required packages 5 | 6 | LABEL maintainer=”som@awstechguide.com” 7 | 8 | WORKDIR /opt/tomcat/webapps 9 | RUN curl -O -L https://github.com/......../SampleWebApp.war 10 | 11 | EXPOSE 8080 12 | 13 | CMD ["/opt/tomcat/bin/catalina.sh", "run"] 14 | -------------------------------------------------------------------------------- /docker/Dockerfile/tomEE-Dockerfile: -------------------------------------------------------------------------------- 1 | # need update. not tested 2 | FROM centos 3 | 4 | # As we are using the latest Ubuntu/CentOS Docker images, then we need to install Java, Tomcat, and other required packages 5 | 6 | LABEL maintainer=”som@awstechguide.com” 7 | RUN mkdir /opt/tomcat/ 8 | WORKDIR /opt/tomcat 9 | RUN curl -O https://www.apache.org/dyn/closer.cgi/tomee/tomee-8.0.8/apache-tomee-8.0.8-webprofile.tar.gz 10 | RUN tar xvfz apache-tomee*.tar.gz 11 | RUN mv apache-tomee-8.0.8/* /opt/tomcat/. 12 | RUN yum -y install java 13 | RUN java -version 14 | 15 | WORKDIR /opt/tomcat/webapps 16 | RUN curl -O -L https://github.com/......../SampleWebApp.war 17 | 18 | EXPOSE 8080 19 | 20 | CMD ["/opt/tomcat/bin/catalina.sh", "run"] 21 | -------------------------------------------------------------------------------- /docker/config-file.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | innodb_buffer_pool_size=1024M 3 | innodb_read_io_threads=4 4 | innodb_flush_log_at_trx_commit=2 5 | innodb_flush_method=O_DIRECT 6 | 7 | 8 | bind-address=0.0.0.0 9 | -------------------------------------------------------------------------------- /docker/docker save and load.md: -------------------------------------------------------------------------------- 1 | ### export and import docker image. 2 | #### Sometime we need to copy docker image from one VM to another VM. follow below steps for save and load 3 | 4 | ### save/export docker image to tar file 5 | ``` 6 | docker save -o ./dkr-test.tar dkr-image-name:latest 7 | ``` 8 | 9 | copy the tar file to another VM and import there. 'myimage:1.0.0' is new name there 10 | [Click here](https://github.com/e2eSolutionArchitect/scripts/blob/main/aws/ec2/upload-file-ec2-s3.md) for how to copy 11 | 12 | ``` 13 | docker load < ./dkr-test.tar 14 | 15 | after loading check your image list the new image should be there 16 | ``` 17 | NOTE: if you user docker import in this case you will get error. if you use SAVE and user LOAD to import 18 | -------------------------------------------------------------------------------- /docker/docker-compose/docker-compose-jenkins.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | jenkins: 4 | image: jenkins/jenkins:lts 5 | privileged: true 6 | user: root 7 | ports: 8 | - 8081:8080 9 | - 50000:50000 10 | container_name: jenkins 11 | volumes: 12 | - ~/jenkins:/var/jenkins_home 13 | - /var/run/docker.sock:/var/run/docker.sock 14 | - /usr/local/bin/docker:/usr/local/bin/docker 15 | 16 | # create below directory before executing this yml 17 | mkdir ~/jenkins 18 | -------------------------------------------------------------------------------- /docker/docker-compose/docker-compose-keycloak-mysql-phpmyadmin.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | # Database 5 | db: 6 | image: mysql:8.0.23 7 | volumes: 8 | - v_mysql_data:/var/lib/mysql 9 | restart: always 10 | environment: 11 | MYSQL_ROOT_PASSWORD: password 12 | MYSQL_DATABASE: MYDB 13 | MYSQL_USER: testusr 14 | MYSQL_PASSWORD: password 15 | expose: 16 | - '3306' 17 | ports: 18 | - '3306:3306' 19 | networks: 20 | - mysqlnet 21 | # phpmyadmin 22 | phpmyadmin: 23 | depends_on: 24 | - db 25 | image: phpmyadmin/phpmyadmin:5 26 | restart: always 27 | ports: 28 | - '8899:80' 29 | environment: 30 | PMA_HOST: db 31 | MYSQL_ROOT_PASSWORD: password 32 | networks: 33 | - mysqlnet 34 | # Keycloak 35 | keycloak: 36 | depends_on: 37 | - db 38 | image: jboss/keycloak:12.0.2 39 | restart: always 40 | environment: 41 | DB_VENDOR: MYSQL 42 | DB_ADDR: db 43 | DB_DATABASE: MYDB 44 | DB_USER: testusr 45 | DB_PASSWORD: password 46 | #JDBC_PARAMS: "connectTimeout=30000" 47 | KEYCLOAK_USER: admin 48 | KEYCLOAK_PASSWORD: password 49 | expose: 50 | - '8443' 51 | - '8080' 52 | ports: 53 | - '8080:8080' 54 | - '8443:8443' 55 | networks: 56 | - mysqlnet 57 | volumes: 58 | v_mysql_data: 59 | networks: 60 | mysqlnet: 61 | -------------------------------------------------------------------------------- /docker/docker-compose/docker-compose-keycloak-mysql.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | # Database 5 | db: 6 | image: mysql:8.0.23 7 | volumes: 8 | - /var/lib/mysql_data:/var/lib/mysql 9 | - ./config-file.cnf:/etc/mysql/conf.d/config-file.cnf 10 | restart: always 11 | environment: 12 | MYSQL_ROOT_PASSWORD: passowrd 13 | MYSQL_DATABASE: DATABASEDB 14 | MYSQL_USER: testusr 15 | MYSQL_PASSWORD: passowrd 16 | expose: 17 | - 3306 18 | ports: 19 | - 3306:3306 20 | networks: 21 | - mysqlnet 22 | # Keycloak 23 | keycloak: 24 | depends_on: 25 | - db 26 | image: jboss/keycloak:12.0.2 27 | restart: always 28 | environment: 29 | DB_VENDOR: MYSQL 30 | DB_ADDR: db 31 | DB_DATABASE: DATABASEDB 32 | DB_USER: testusr 33 | DB_PASSWORD: password 34 | #JDBC_PARAMS: "useSSL=false" 35 | KEYCLOAK_USER: admin 36 | KEYCLOAK_PASSWORD: password 37 | expose: 38 | - '8443' 39 | - '8080' 40 | ports: 41 | - '8080:8080' 42 | - '8443:8443' 43 | networks: 44 | - mysqlnet 45 | volumes: 46 | v_keycloak_data: 47 | networks: 48 | mysqlnet: 49 | -------------------------------------------------------------------------------- /docker/docker-compose/docker-compose-phpmyadmin.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | # Database 5 | db: 6 | image: mysql:latest 7 | volumes: 8 | - db_data:/var/lib/mysql 9 | - ./config-file.cnf:/etc/mysql/conf.d/config-file.cnf 10 | restart: always 11 | expose: 12 | - '3306' 13 | ports: 14 | - '3306:3306' 15 | environment: 16 | MYSQL_ROOT_PASSWORD: pass@123 17 | MYSQL_DATABASE: testdb 18 | MYSQL_USER: testusr 19 | MYSQL_PASSWORD: pass123 20 | networks: 21 | - dockernet 22 | # phpmyadmin 23 | phpmyadmin: 24 | depends_on: 25 | - db 26 | image: phpmyadmin/phpmyadmin:latest 27 | restart: always 28 | ports: 29 | - '8899:80' 30 | environment: 31 | PMA_HOST: db 32 | MYSQL_ROOT_PASSWORD: pass@123 33 | networks: 34 | - dockernet 35 | networks: 36 | dockernet: 37 | volumes: 38 | db_data: 39 | -------------------------------------------------------------------------------- /docker/docker-compose/docker-compose-sonar.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | sonarqube: 5 | image: sonarqube 6 | ports: 7 | - '9000:9000' 8 | networks: 9 | - sonarnet 10 | environment: 11 | - sonar.jdbc.username=sonar 12 | - sonar.jdbc.password=sonar 13 | - sonar.jdbc.url=jdbc:postgresql://db:5432/sonar 14 | volumes: 15 | - sonarqube_conf:/opt/sonarqube/conf 16 | - sonarqube_data:/opt/sonarqube/data 17 | - sonarqube_extensions:/opt/sonarqube/extensions 18 | ulimits: 19 | nofile: 20 | soft: 65536 21 | hard: 65536 22 | db: 23 | image: postgres 24 | networks: 25 | - sonarnet 26 | environment: 27 | - POSTGRES_USER=sonar 28 | - POSTGRES_PASSWORD=sonar 29 | volumes: 30 | - postgresql:/var/lib/postgresql 31 | # This needs explicit mapping due to https://github.com/docker-library/postgres/blob/4e48e3228a30763913ece952c611e5e9b95c8759/Dockerfile.template#L52 32 | - postgresql_data:/var/lib/postgresql/data 33 | 34 | networks: 35 | sonarnet: 36 | driver: bridge 37 | 38 | volumes: 39 | sonarqube_conf: 40 | sonarqube_data: 41 | sonarqube_extensions: 42 | postgresql: 43 | postgresql_data: 44 | -------------------------------------------------------------------------------- /docker/docker-compose/docker-compose-tomEE.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | tomee: 4 | image: tomee:8.0.8-webprofile 5 | ports: 6 | - 8888:8080 7 | expose: 8 | - 8080 9 | container_name: tomee8 10 | networks: 11 | - net_tomee8 12 | volumes: 13 | - v_tomee8:/home/ubuntu/tomee8 14 | volumes: 15 | v_tomee8: 16 | networks: 17 | net_tomee8: 18 | -------------------------------------------------------------------------------- /docker/docker-compose/docker-compose-wordpress.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | # use atleast t3.small instance type. t2.micro will not be sufficient for this 3 | services: 4 | # Database 5 | db: 6 | image: mysql:8.0.23 7 | volumes: 8 | - v_mysql_data:/var/lib/mysql 9 | restart: always 10 | environment: 11 | MYSQL_ROOT_PASSWORD: password1 12 | MYSQL_DATABASE: MYDB 13 | MYSQL_USER: testuser 14 | MYSQL_PASSWORD: password 15 | networks: 16 | - mysqlnet 17 | # phpmyadmin 18 | phpmyadmin: 19 | depends_on: 20 | - db 21 | image: phpmyadmin/phpmyadmin:5 22 | restart: always 23 | ports: 24 | - '8899:80' 25 | environment: 26 | PMA_HOST: db 27 | MYSQL_ROOT_PASSWORD: password1 28 | networks: 29 | - mysqlnet 30 | # Wordpress 31 | wordpress: 32 | depends_on: 33 | - db 34 | image: wordpress:5.6 35 | ports: 36 | - '8080:80' 37 | restart: always 38 | volumes: 39 | - v_wp_data:/var/www/html 40 | environment: 41 | WORDPRESS_DB_NAME: MYDB 42 | WORDPRESS_DB_HOST: db:3306 43 | WORDPRESS_DB_USER: testuser 44 | WORDPRESS_DB_PASSWORD: password 45 | networks: 46 | - mysqlnet 47 | volumes: 48 | v_mysql_data: 49 | v_wp_data: 50 | networks: 51 | mysqlnet: 52 | -------------------------------------------------------------------------------- /docker/install-docker-aws-ec2.md: -------------------------------------------------------------------------------- 1 | Update the packages on your instance 2 | 3 | ``` 4 | [ec2-user ~]$ sudo yum update -y 5 | ``` 6 | 7 | Install Docker 8 | 9 | ``` 10 | [ec2-user ~]$ sudo yum install docker -y 11 | ``` 12 | 13 | Start the Docker Service 14 | 15 | ``` 16 | [ec2-user ~]$ sudo service docker start 17 | ``` 18 | 19 | Add the ec2-user to the docker group so you can execute Docker commands without using sudo. 20 | 21 | ``` 22 | [ec2-user ~]$ sudo usermod -a -G docker ec2-user 23 | ``` 24 | -------------------------------------------------------------------------------- /docker/install-docker-compose.md: -------------------------------------------------------------------------------- 1 | ### reference : https://docs.docker.com/compose/install/ 2 | 3 | ## Install docker compose 4 | ### Run this command to download the current stable release of Docker Compose: 5 | sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 6 | 7 | ### Apply executable permissions to the binary: 8 | sudo chmod +x /usr/local/bin/docker-compose 9 | 10 | 11 | ### Test the installation 12 | docker-compose --version 13 | -------------------------------------------------------------------------------- /docker/install-sonar-docker.md: -------------------------------------------------------------------------------- 1 | # Important: Please use Ubuntu 18.4 version. Otherwise you may face sonarqube started but the link is not up 2 | 3 | # install docker 4 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 5 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 6 | sudo apt-get update 7 | apt-cache policy docker-ce 8 | sudo apt-get install -y docker-ce 9 | sudo usermod -aG docker ubuntu 10 | sudo systemctl status docker 11 | 12 | #install docker compose 13 | Install docker compose 14 | sudo curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 15 | 16 | Change permission 17 | sudo chmod +x /usr/local/bin/docker-compose 18 | 19 | Check the version 20 | docker-compose --version 21 | 22 | #install sonar 23 | 24 | sudo sysctl -w vm.max_map_count=262144 25 | mkdir sonar 26 | wget https://raw.githubusercontent.com/awstechguide/scripts/master/docker/docker-compose/docker-compose-sonar.yml 27 | sudo docker–compose up 28 | docker ps 29 | -------------------------------------------------------------------------------- /docker/install_jenkins_docker.md: -------------------------------------------------------------------------------- 1 | ## Install and Run jenkins in docker container 2 | 3 | ## Using Ubuntu system 4 | ## Install Docker 5 | ### follow instruction to install docker in ubuntu platform https://docs.docker.com/engine/install/ubuntu/ 6 | 7 | # pull docker image 8 | docker pull jenkins 9 | 10 | # create docker volume 11 | docker volume create jenkins 12 | 13 | # list docker volumes 14 | docker volumn ls 15 | 16 | # run jenkins image and map to docker volume 17 | docker run -p 8080:8080 -p 50000:50000 -v jenkins:/var/jenkins_home jenkins 18 | -------------------------------------------------------------------------------- /docker/readme.md: -------------------------------------------------------------------------------- 1 | ``` 2 | docker run (run a docker image) 3 | docker ps 4 | docker ps -a 5 | 6 | ``` 7 | 8 | ``` 9 | docker image ls (list available docker images) 10 | docker container ls (list running containers) 11 | ``` 12 | 13 | ``` 14 | 15 | docker -v # check version 16 | docker stop 17 | docker ps -a # check stopped containers 18 | docker volume --help # to get docker volume commands 19 | ``` 20 | 21 | ``` 22 | docker image rm -f db2b37ec6181 # delete images forcibly 23 | ``` 24 | 25 | ``` 26 | docker volume create 27 | ``` 28 | 29 | ``` 30 | docker volume inspect // describe a volume and find the mount path 31 | ``` 32 | 33 | ``` 34 | docker volume ls # list volume 35 | ``` 36 | 37 | ``` 38 | docker volume prune // will remove all unused volumes, which are not used by any containers 39 | ``` 40 | 41 | ``` 42 | docker volume rm 43 | ``` 44 | -------------------------------------------------------------------------------- /docs/aws/Installing or updating the latest version of the AWS CLI.md: -------------------------------------------------------------------------------- 1 | Ref https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html 2 | 3 | ``` 4 | sudo curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" 5 | 6 | (optional) sudo yum install unzip 7 | 8 | sudo unzip awscliv2.zip 9 | 10 | sudo ./aws/install 11 | 12 | ''' 13 | -------------------------------------------------------------------------------- /git/create-Git-PAT-Token.md: -------------------------------------------------------------------------------- 1 | #Steps to create Personal Access Token (PAT) 2 | 3 | ## Azure DevOps 4 | 5 | ### User settings (top right corner second last icon) > Personal Access Tokens 6 | 7 | - Create a new token. Important: If you plan to migrate multiple org then select 'All accessible organizations' in the field of 'Organization' 8 | - You will need three scopes. Work Item Read, Code Read, Identity Read. Note: click over 'show all scope' link to get the Identity Read option. 9 | - Create the token 10 | 11 | ## GitHub 12 | - Profile > Settings > Developer Settings > Personal access tokens 13 | -------------------------------------------------------------------------------- /git/git-commands.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ### create new branch from another 4 | 5 | ``` 6 | git checkout -b new-branch old-branch; 7 | git push --set-upstream origin new-branch; 8 | ``` 9 | ### checkout a branch 10 | 11 | ``` 12 | git checkout branchname 13 | ``` 14 | 15 | ### Merge code from feature branch 'dev-branch' to master 16 | 17 | ``` 18 | git add –A 19 | git commit –m "Some commit message" 20 | git checkout master 21 | Switched to branch 'master' 22 | git merge dev-branch (this is to merge changes from dev-branch to master) 23 | git push (push changes in master branch) 24 | ``` 25 | 26 | ### Get the update from a branch that is ahead of your branch. 27 | 28 | ``` 29 | # branch A is a feature branch which is behind of 'main' branch. Here is how to take updates from main TO branch A 30 | 31 | git checkout branch-A 32 | git pull 33 | git merge origin/main 34 | git add . 35 | git commit -m "updated with main branch" 36 | git push 37 | ``` 38 | 39 | 40 | ### git commit history 41 | 42 | ``` 43 | git log --pretty=oneline 44 | ``` 45 | 46 | ### delete branch locally 47 | ``` 48 | git branch -d localBranchName 49 | ``` 50 | ### delete branch remotely 51 | ``` 52 | git push origin --delete remoteBranchName 53 | ``` 54 | ### The command to list all branches in local and remote repositories is: 55 | ``` 56 | git branch -a 57 | ``` 58 | ### If you require only listing the remote branches from Git Bash then use this command: 59 | ``` 60 | git branch -r 61 | ``` 62 | 63 | ### You may also use the show-branch command for seeing the branches and their commits as follows: 64 | ``` 65 | git show-branch 66 | ``` 67 | ### If you want to rename a branch while pointed to any other branch 68 | ``` 69 | git branch -m oldname newname 70 | ``` 71 | ### If you want to rename the current branch 72 | ``` 73 | git branch -m newname 74 | ``` 75 | ### list content 76 | ``` 77 | ls - ltrh 78 | ``` 79 | ### Create tag 80 | 81 | 1. checkout the branch first where you want to create the tag 82 | 2. After that create tag and and push. Follow below commands in sequence 83 | 84 | consider our tag name as 'release-v1.0' 85 | ``` 86 | - git checkout 87 | - git tag . e,g. git tag release-v1.0 88 | - git tag -a release-v1.0 -m "add comments" use it while adding comments with tag 89 | - git push --tags 90 | ``` 91 | 92 | Delete a tag 93 | ``` 94 | git tag -d 95 | ``` 96 | push a tag from local to origin branch 97 | 98 | ``` 99 | git push origin -d release-v1.0 100 | ``` 101 | 102 | -------------------------------------------------------------------------------- /git/git-common-issues.md: -------------------------------------------------------------------------------- 1 | 2 | ## Git common issues 3 | 4 | ### remote: Repository Not found 5 | ``` 6 | git clone https://myusername:mypassword@github.com/path_to/myRepo.git 7 | OR 8 | git clone https://myusername@github.com/path_to/myRepo.git -- it will open a github login popup. you can enter password or use the option to login via browser. 9 | 10 | C:\Som\workspace\terraform>git clone https://github.com/myusername/myrepo.git 11 | Cloning into 'myrepo'... remote: Repository not found. 12 | fatal: repository 'https://github.com/myusername/myrepo.git/' not found 13 | 14 | C:\Som\workspace\terraform>git clone https://myusername@github.com/myusername/myrepo.git 15 | Cloning into 'myrepo'... info: please complete authentication in your browser... remote: Enumerating objects: 18, done. 16 | remote: Counting objects: 100% (18/18), done. remote: Compressing objects: 100% (15/15), done. 17 | remote: Total 18 (delta 4), reused 0 (delta 0), pack-reused 0 Receiving objects: 100% (18/18), 4.99 KiB | 1.66 MiB/s, done. 18 | Resolving deltas: 100% (4/4), done. 19 | 20 | ``` 21 | 22 | ### fatal: unable to access : could not load PEM client certificate, OpenSSL error error:02001002:system library:fopen:No such file or directory, ... 23 | 24 | ### check you git config and remove hhtp.sslcert record from there (if its there) 25 | 26 | ``` 27 | git config -l --show-origin 28 | git config --global --unset http.sslCert 29 | ``` 30 | 31 | ### configure username and email globally 32 | 33 | ``` 34 | git config --global user.name=youname 35 | git config --global user.email=your@email 36 | ``` 37 | ### ignore ssl for git 38 | 39 | ``` 40 | git config --global http.sslverify=false 41 | ``` 42 | 43 | ``` 44 | set no-proxy="abc.com" 45 | ``` 46 | 47 | ## SSL certificate problem: self signed certificate in certificate chain 48 | 49 | ``` 50 | use http.sslVerify=false 51 | instead 52 | git config --global http.sslVerify false # Do NOT do this! 53 | You should never globally disable TLS(/SSL) certificate verification 54 | 55 | Run below command 56 | git -c http.sslVerify=false clone https://github.com/e2eSolutionArchitect/academy.git 57 | ``` 58 | -------------------------------------------------------------------------------- /hashicorp-vault/config.hcl: -------------------------------------------------------------------------------- 1 | # production env config 2 | 3 | storage "raft" { 4 | path = "./vault/data" 5 | node_id = "node1" 6 | } 7 | 8 | listener "tcp" { 9 | address = "0.0.0.0:8200" # 0.0.0.0 is because of installing vault in a remote VM 10 | tls_disable = "true" # for actual production this value should be false 11 | } 12 | 13 | api_addr = "http://127.0.0.1:8200" 14 | cluster_addr = "https://127.0.0.1:8201" 15 | ui = true 16 | -------------------------------------------------------------------------------- /hashicorp-vault/generating-dynamic-secrets-for-aws.md: -------------------------------------------------------------------------------- 1 | 2 | Generating dynamic secrets for AWS 3 | - You need to have ACCESS_KEY, SECRET_KEY as prerequisite 4 | - Enable AWS secret engine 5 | ``` 6 | vault secrets enable -path=aws aws 7 | ``` 8 | - Set root config 9 | ``` 10 | vault write aws/config/root \ 11 | access_key = #### \ 12 | secret_key = #### \ 13 | region = us-east-1 14 | ``` 15 | - Setup role 'my-ec2-role' 16 | ``` 17 | vault write aws/roles/my-ec2-role \ 18 | credential_type=iam_user \ 19 | policy_document=-< = 36 | vault kv put mypath/hello1 key1=value1 37 | ``` 38 | 39 | Note: Once a new 'path' is mentioned the 'path' should be enabled for a secret engine. please check how to enable in the section "Enable AWS secret engine path" below. 40 | 41 | Get key value from path 42 | ``` 43 | vault kv get 44 | vault kv get mypath/hello1 45 | 46 | Read in JSON format 47 | vault kv get -format=json my/path 48 | ``` 49 | 50 | Delete 51 | ``` 52 | vault kv delete 53 | vault kv delete mypath/hello1 54 | ``` 55 | 56 | Get the list of secret engine paths 57 | ``` 58 | vault secrets list 59 | ``` 60 | 61 | Enable AWS secret engine path 62 | ``` 63 | vault secrets enable -path= 64 | vault secrets enable -path=aws aws 65 | ``` 66 | 67 | Disable secret engine path 68 | ``` 69 | vault secrets disable 70 | vault secrets disable mypath/hello1 71 | ``` 72 | 73 | Command to instruct the active node to gracefully stand down 74 | ``` 75 | vault operator step-down 76 | ``` 77 | 78 | - [Generate Dynamic Secrets for AWS](https://github.com/e2eSolutionArchitect/scripts/blob/main/hashicorp-vault/generating-dynamic-secrets-for-aws.md) 79 | - [Policy](https://github.com/e2eSolutionArchitect/scripts/blob/main/hashicorp-vault/vault-policy.md) 80 | 81 | Apply policy to write secrets 82 | ``` 83 | vault kv put -mount=secret creds password="my-super-secret-passwprd123" 84 | ``` 85 | 86 | Notes: 87 | - After initializing Vault or restarting the Vault service, each individual node in the cluster needs to be unsealed. 88 | 89 | Rekey operation using the vault operator rekey command creates new unseal/recovery keys as well as a new master key 90 | ``` 91 | vault operator rekey 92 | ``` 93 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-auth-github.md: -------------------------------------------------------------------------------- 1 | 2 | Create an Authentication method for GitHub 3 | ``` 4 | vault auth list 5 | vault auth enable github 6 | ``` 7 | 8 | Link GitHub Org to Vault 9 | Write Config for Org 10 | ``` 11 | vault write auth/github/config organization= 12 | ``` 13 | 14 | Write config for Teams. web-team, app-team assumed created in github org already 15 | ``` 16 | vault write auth/github/map/teams/my-team values = web-team, app-team 17 | ``` 18 | 19 | Now login using Github method. Use Github Token as password 20 | ``` 21 | vault login -method=github 22 | 23 | ``` 24 | 25 | Create Github Token from Github 26 | - Github > Profile > Developer Settings > Personal Access Token 27 | - Make sure to add 'read-org' permission to the PAT (Personal Access Token) 28 | 29 | 30 | How to remove the authentication method? such as Github 31 | 32 | ``` 33 | vault token revoke -mode path auth/github 34 | # Also disable the auth method 35 | vault auth disable github 36 | ``` 37 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-auth-methods.md: -------------------------------------------------------------------------------- 1 | 2 | 'token' is the default auth method in the vault. 3 | 4 | Auth methods offers the cababilities to use external authentication service in vault. 5 | 6 | Examples of machine auth methods include AppRole, Cloud-based auth methods, tokens, TLS, Kubernetes, and Radius. Examples of human auth methods include Okta, LDAP, GitHub, OIDC, and userpass. 7 | 8 | ``` 9 | vault auth list 10 | vault auth enable 11 | vault auth enable approle 12 | ``` 13 | Check what version the KV store is using 14 | ``` 15 | vault secrets list -detailed 16 | ``` 17 | After enabling auth method we have to assign it to the policy 18 | 19 | ## Associate auth method with policy 20 | 21 | ``` 22 | vault write auth/approle/role/my-role \ 23 | secret_id_ttl=10m \ 24 | token_num_uses=10 \ 25 | token_ttl=20m \ 26 | token_max_ttl=30m \ 27 | secret_id_num_uses=40 \ 28 | token_policies=my-policy 29 | ``` 30 | 'my-policy' is already created policy and 'approle' is the auth method already created 31 | 32 | 33 | ``` 34 | export ROLE_ID="$(vault read -field=role_id auth/approle/role/my-role/role-id)" 35 | export SECRET_ID="$(vault write -f -field=secret_id auth/approle/role/my-role/secret-id)" 36 | ``` 37 | 38 | Write config 39 | ``` 40 | vault write auth/approle/login role_id="$ROLE_ID" secret_id="$SECRET_ID" 41 | 42 | ``` 43 | 44 | Vault ERROR http: server gave HTTP response to HTTPS client 45 | ``` 46 | If you're running Vault in a demo or non-production environment, you can configure Vault to disable TLS. In this case, TLS has been disabled, but the default value for VAULT_ADDR is https://127.0.0.1:8200. Therefore Vault is sending the request over HTTPS, but Vault is responding using HTTP since TLS is disabled. In this case, you should set the VAULT_ADDR environment variable to "http://127.0.0.1:8200". This is true if you're running Vault Dev server as well. 47 | ``` 48 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-aws.md: -------------------------------------------------------------------------------- 1 | - AWS KMS to automatically unseal Vault on private EC2 instances. The subnet where Vault is deployed doesn't have internet access 2 | 3 | Ans: Add a VPC endpoint to enable private connectivity to the KMS service. The other way is to permit outbound access to the Internet simply, but the VPC endpoint is more secure since the traffic never leaves the AWS network 4 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-commands.md: -------------------------------------------------------------------------------- 1 | 2 | - Find the leader node of a cluster 3 | ``` 4 | curl http://127.0.0.1:8200/v1/sys/leader | jq 5 | ``` 6 | 7 | - Command to use easily re-encrypt the original data with the new version of the key. This operation does not reveal the plaintext data. Vault will decrypt the value using the appropriate key in the keyring and then encrypt the resulting plaintext with the newest key in the keyring 8 | ``` 9 | vault write transit/rewrap/ ciphertext= 10 | ``` 11 | 12 | - Delete all versions and metadata for the key permanently. Command would permanently delete the path from Vault 13 | ``` 14 | vault kv metadata delete kv/applications/app01 15 | ``` 16 | 17 | - Permanently delete the current version of the secret. Not all version 18 | ``` 19 | vault kv destroy kv/applications/app01 20 | ``` 21 | 22 | - Soft delete the current version of the secret. 23 | ``` 24 | vault kv delete kv/applications/app01 25 | ``` 26 | 27 | Admin never sees all unseal keys and can not unseal the vault by themselves. each individual user can only decrypt their own unseal key using their PGP keys 28 | ``` 29 | vault operator init -key-shares=3 -key-threshold=2 -pgp-keys="keybase:user1,keybase:user2,keybase:user3" 30 | ``` 31 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-encrypt-data.md: -------------------------------------------------------------------------------- 1 | 2 | - Encrypt creditcard number. keyring is credit card, path is encryption, transit secret engine is mounted at 'encryption' path. The pain text must be encoded to base64, otherwise it will show error. 3 | ``` 4 | 5 | vault write encryption/encrypt/creditcard plaintext=$(base64 <<< "1234 5678 9101 1121") 6 | Key Value 7 | --- ----- 8 | ciphertext vault:v3:cZNHVx+sxdMErXRSuDa1q/pz49fXTn1PScKfhf+PIZPvy8xKfkytpwKcbC0fF2U= 9 | ``` 10 | 11 | 12 | ``` 13 | To rotate a key, use the command 14 | vault write -f transit/keys//rotate 15 | ``` 16 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-lease.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Revoke lease 4 | 5 | ``` 6 | vault lease revoke -prefix -- leases from a secrets engine using the -prefix 7 | vault lease revoke -prefix aws/mycreds 8 | 9 | vault lease revoke aws/mycred/ -- revoke particular lease 10 | 11 | ``` 12 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-policy.md: -------------------------------------------------------------------------------- 1 | 2 | Get policy list 3 | ``` 4 | vault policy list 5 | ``` 6 | 7 | Create policy [vault-policy-01.hcl](https://github.com/e2eSolutionArchitect/scripts/blob/main/hashicorp-vault/vault-policy-01.hcl) 8 | 9 | ``` 10 | vault policy write my-policy vault-policy-01.hcl 11 | ``` 12 | 13 | Check content of a policy 14 | ``` 15 | vault policy read 16 | vault policy read my-policy 17 | ``` 18 | 19 | Delete policy 20 | ``` 21 | vault policy delete 22 | ``` 23 | ## When using policy , it should be attached with a token before it is used 24 | 25 | Attach a token with Policy 26 | 27 | ``` 28 | export VAULT_TOKEN="$(vault token create -field token -policy=my-policy)" 29 | ``` 30 | 31 | Policy to restrict a path access 32 | ``` 33 | path "secret/apps/confidential" { 34 | capabilities =["deny"] 35 | } 36 | 37 | or use + as wildcards instead of calling-out each segments of the path 38 | 39 | path "secret/+/confi*" { 40 | capabilities =["deny"] 41 | } 42 | ``` 43 | A policy to permit new entry for environment=prod at path /secrets/app/top_secrets 44 | 45 | ``` 46 | path "secret/apps/top_secrets" { 47 | capabilities =["create"] 48 | allowed_parameters = { 49 | "environment" = ["dev", "test", "stg", "prod"] 50 | } 51 | } 52 | 53 | or 54 | 55 | path "secret/apps/top_secrets" { 56 | capabilities =["create"] 57 | allowed_parameters = { 58 | "environment" = [] 59 | } 60 | } 61 | 62 | or path "secret/+/top_*" { 63 | capabilities =["create"] 64 | allowed_parameters = { 65 | "*" = [] 66 | } 67 | } 68 | 69 | ``` 70 | 71 | Create policy 72 | ``` 73 | cat user.hcl | vault policy write mypolicy - 74 | 75 | OR 76 | 77 | vault policy write mypolicy - << EOF 78 | path "secret/data/*" { 79 | capabilities = ["create","update"] 80 | } 81 | EOF 82 | 83 | OR 84 | 85 | vault policy write mypolicy policy.hcl 86 | ``` 87 | 88 | 89 | Create policy where user can only access their own private section of KV secret engine 90 | ``` 91 | path "kv/team/{{identity.entity.id}}/*" { 92 | capabilities = ["create", "update", "read", "delete"] 93 | } 94 | path "kv/team/{{identity.entity.id}}" { 95 | capabilities = ["create", "update", "read", "delete"] 96 | } 97 | 98 | ``` 99 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-production-steps.md: -------------------------------------------------------------------------------- 1 | start vault server 2 | ``` 3 | vault server -config = config.hcl # for production/server model 4 | ``` 5 | For the production server please make Vault a service in Linux system 6 | 7 | Set environment variables 8 | ``` 9 | export VAULT_ADDR='http://:8200' 10 | ``` 11 | 12 | Initialize Vault. It will give 5 unseal keys and a root token 13 | ``` 14 | vault operator init 15 | ``` 16 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-secret-engine-kv.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ``` 4 | vault kv put kv/ key=value 5 | vault kv put kv/mypath/mycred username=passwd 6 | vault kv put kv/mypath/mycred @data.txt - data.txt contains the key value pair 7 | ``` 8 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-secrets.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Secrets list in detail 4 | ``` 5 | vault secrets list -detailed 6 | ``` 7 | 8 | enable versioning 9 | ``` 10 | vault kv enable-versioning 11 | ``` 12 | 13 | Delete secret engine 14 | ``` 15 | vault kv metadata delete - permanently deletes the secret with all versions and metadata 16 | vault kv delete - soft delete current version 17 | vault kv destroy - permanently deletes current version of the secret 18 | ``` 19 | 20 | revoke all leases associated with the secret engine mounted at /aws 21 | ``` 22 | vault lease revoke -prefix aws/ 23 | ``` 24 | 25 | Transit secret engine 26 | 27 | ``` 28 | curl \ 29 | --header "X-Vault-Token: hvs.########" \ 30 | --request POST \ 31 | --data @data.json \ 32 | https://prod-vault.abc.com:8200/v1/transit/encrypt/customer-data 33 | ``` 34 | data.json will contain plaintext customer data to be encrypted 35 | 36 | 37 | Force to remove the secret 38 | ``` 39 | vault lease revoke -force -prefix 40 | ``` 41 | 42 | Invalidate a credential 43 | ``` 44 | vault lease revoke aws/creds/.... 45 | ``` 46 | 47 | Steps to encrypt 48 | ``` 49 | vendor secrets enable transit 50 | vault write -f transit/keys/myencrpkey 51 | base64 <<< "my confidential text" 52 | 53 | vault write transit/encrypt/myencrpkey plaintext="dfgdJJDfg#$%#FDgaGFzaGljbZFDgd#$%#$DFGmllZA==" 54 | ``` 55 | 56 | perform API call to read secrets for a particular namespace 'integration' 57 | ``` 58 | curl \ 59 | --header "X-Vault-Token:s.lzrmRe5Y3LMcDRmOttEjWoag" \ 60 | --header "X-Vault-Namespace: integration" \ 61 | --request GET \ 62 | https://vault.example.com:8200/v1/secret/data/my-secret 63 | 64 | ``` 65 | -------------------------------------------------------------------------------- /hashicorp-vault/vault-token.md: -------------------------------------------------------------------------------- 1 | 2 | ``` 3 | vault token create -policy=dev -use-limit=5 4 | ``` 5 | 6 | Regenerate vault token 7 | 8 | ``` 9 | vault token create 10 | ``` 11 | 12 | Login with Root Token. Use your root token as password 13 | ``` 14 | vault login 15 | ``` 16 | 17 | Generate Root Token 18 | 19 | ``` 20 | vault token create - when using a valid root token 21 | vault operator init - when first time initializing vault 22 | 23 | 3rdly when generating a root token using a quorum of recovery keys when using vault auto seal 24 | 25 | ``` 26 | 27 | Revoke root token 28 | ``` 29 | vault token revoke 30 | vault token revoke hvs.hisdf776234kSDFSFhiendsfsdfjh 31 | ``` 32 | 33 | After authenticating, a client is issued a service token which is associated with a policy. That token is used to make all subsequent requests to Vault. 34 | 35 | Login to Vault using GitHub token 36 | ``` 37 | vault login -method=github 38 | ``` 39 | 40 | Token renew [refer](https://developer.hashicorp.com/vault/docs/commands/token/renew#token-renew) 41 | ``` 42 | # Renew a token (this uses the /auth/token/renew endpoint and permission): 43 | vault token renew 44 | vault token renew 96ddf4bc-d217-f3ba-f9bd-017055595017 45 | 46 | vault token renew -increment=30m 47 | vault token renew -increment=30m 96ddf4bc-d217-f3ba-f9bd-017055595017 48 | 49 | ``` 50 | 51 | Use the token to access vault 52 | ``` 53 | vault token create -policy=mypolicy 54 | 55 | Key Value 56 | --- ----- 57 | token hvs.########### 58 | token_accessor ############# 59 | token_duration 24h 60 | token_renewable true 61 | token_policies ["e2esa" "default"] 62 | identity_policies [] 63 | policies ["e2esa" "default"] 64 | 65 | 66 | vault login hvc.######## 67 | or 68 | vault login -method=token hvc.######## 69 | 70 | ``` 71 | 72 | Revoke all leases associated with a role 'myrole' 73 | 74 | ``` 75 | vault lease revoke -prefix database/creds/myrole 76 | ``` 77 | 78 | Vault token look up shows associated policy with vault 79 | ``` 80 | vault token lookup s.DjWW0######## 81 | You can also use -accessor flag if you only know the accessor and not the token. 82 | ``` 83 | 84 | vault operator diagnose is a new command in Vault 1.8 that allows you to troubleshoot a Vault node where the Vault service will not start 85 | vault token capabilities will list the capabilities on a certain path 86 | vault policy list will list the current policies on the Vault node/cluster 87 | -------------------------------------------------------------------------------- /java/java-install.md: -------------------------------------------------------------------------------- 1 | Check existing java version 2 | ``` 3 | java -version 4 | ``` 5 | For installing OpenJDK 11 in Ubuntu 6 | ``` 7 | sudo apt-get update && sudo apt-get upgrade -y 8 | sudo apt-get install openjdk-11-jdk 9 | ``` 10 | Alternatively, if you need JRE only then 11 | 12 | ``` 13 | sudo apt-get install openjdk-11-jre 14 | ``` 15 | Run below commands to install Java 11 on Amazon Linux: 16 | ``` 17 | sudo amazon-linux-extras install java-openjdk11 18 | ``` 19 | Run below commands to install Java 8 on Amazon Linux: 20 | 21 | ``` 22 | sudo yum install java-1.8.0-openjdk 23 | sudo alternatives --config java 24 | ``` 25 | Check which java version acting 26 | ``` 27 | java -version 28 | ``` 29 | if its not the one you installed then run the below command to set the java version. 30 | ``` 31 | sudo sudo update-alternatives --config java 32 | ``` 33 | if you want to uninstall existing java 34 | ``` 35 | sudo yum remove java 36 | ``` 37 | 38 | 39 | -------------------------------------------------------------------------------- /jenkins/Jenkinsfile-cicd-docker-ECR: -------------------------------------------------------------------------------- 1 | pipeline 2 | { 3 | agent any 4 | 5 | environment { 6 | registry =aws_acct_id.dkr.ecr.us-east-2.amazomaws.com/your_ecr_repo 7 | } 8 | 9 | stages { 10 | stage ('Git pull') { 11 | steps { 12 | checkout ([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId:",url: 'https://github.com/awstechguide/testrepo']]]) 13 | } 14 | } 15 | 16 | 17 | stage ('Build Image') { 18 | steps{ 19 | script { 20 | dockerImage =docker.build registry 21 | } 22 | 23 | } 24 | } 25 | 26 | stage ('Upload to ECR') { 27 | steps { 28 | script{ 29 | sh 'aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin .dkr.ecr.us.east-2.amazonaws.com' 30 | sh 'docker push .dkr.ecr.us.east-2.amazonaws.com/:latest' 31 | } 32 | } 33 | 34 | } 35 | 36 | 37 | stage ('Clean container') { 38 | steps { 39 | script{ 40 | sh 'docker ps -f name= -q | xargs --no-run-if-empty docker container stop' 41 | sh 'docker container ls -a -fname= -q | xargs -r docker container rm' 42 | } 43 | } 44 | 45 | } 46 | 47 | 48 | stage ('Docker Run') { 49 | steps { 50 | script{ 51 | sh 'docker run -d -p 8096:5000 --rm --name aws_acct_id.dkr.ecr.us-east-2.amazomaws.com/your_ecr_repo:latest' 52 | } 53 | } 54 | 55 | } 56 | 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /jenkins/Jenkinsfile.md: -------------------------------------------------------------------------------- 1 | `Jenkins declarative pipeline scripts` 2 | 3 | `Below are the pipeline script references with project reference` 4 | 5 | Project Repo: https://github.com/awstechguide/spring-webapp 6 | Jenkinsfile: https://raw.githubusercontent.com/awstechguide/spring-webapp/master/Jenkinsfile 7 | 8 | Jenkins CICD pipeline script with Docker build and Push to AWS ECR : https://raw.githubusercontent.com/awstechguide/scripts/master/jenkins/Jenkinsfile-cicd-docker-ECR 9 | -------------------------------------------------------------------------------- /jenkins/install-jenkins-docker.sh: -------------------------------------------------------------------------------- 1 | # install jenkins 2 | wget -q -O - https://pkg.jenkins.io/debian/jenkins-ci.org.key | sudo apt-key add - 3 | echo deb http://pkg.jenkins.io/debian-stable binary/ | sudo tee /etc/apt/sources.list.d/jenkins.list 4 | sudo apt-get update 5 | sudo apt install openjdk-8-jdk 6 | sudo apt-get install jenkins 7 | sudo systemctl start jenkins 8 | sudo systemctl status jenkins 9 | 10 | # install docker // Ubuntu //https://docs.docker.com/engine/install/ubuntu/ 11 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 12 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 13 | sudo apt-get update 14 | apt-cache policy docker-ce 15 | sudo apt-get install -y docker-ce 16 | sudo usermod -aG docker ubuntu 17 | sudo systemctl status docker 18 | 19 | sudo usermod -a -G docker jenkins 20 | -------------------------------------------------------------------------------- /jenkins/install-tomcat-ec2-linux.md: -------------------------------------------------------------------------------- 1 | ## Check java version in EC2 linux instance 2 | 3 | java -version 4 | 5 | ## Install openjdk 8 6 | 7 | sudo yum install java-1.8.0-openjdk 8 | 9 | ## Remove old version 10 | 11 | sudo yum remove java 12 | 13 | ## Tomcat website http://tomcat.apache.org/ 14 | 15 | ## Copy download link from http://apache.forsale.plus/tomcat/tomcat-8/v8.5.54/bin/apache-tomcat-8.5.54.tar.gz 16 | 17 | ## login to EC2 instance and change user to sudo 18 | 19 | sudo su - 20 | 21 | ## download tar in ec2 22 | 23 | wget 24 | 25 | ## extract 26 | tar -xvzf 27 | 28 | ## Browse to the bin folder under tomcat folder. Two files, namely; startup.sh and shutdown.sh. 29 | 30 | ## start tomcat service 31 | 32 | ./startup.sh 33 | 34 | ## /*Change port number from 8080 to 8090 (if your Jenkins on AWS is also listening to the port 8080) 35 | Browse to conf sub-directory under Tomcat directory and open server.xml file for editing using ‘vi’ command */ 36 | 37 | vi server.xml 38 | 39 | ## Allow port no 8090 under security group in AWS 40 | 41 | 42 | ## Restart the tomcat service (browse to the bin folder) 43 | 44 | ./shutdown.sh 45 | ./startup.sh 46 | 47 | 48 | ### Go to Browser and type- 49 | 50 | http://: defalut is 8080 51 | 52 | ### Configure for manager app access outside of tomcat server: find context file location 53 | 54 | find / -name context.xml 55 | 56 | ## update context.xml by commenting context value eliment only. do the same in both context.xml files. 57 | ### check tomcat manager link 58 | 59 | ## for user creation go to conf/tomcat-users.xml 60 | 61 | ### add role and user in conf/tomcat-users.xml 62 | 63 | 64 | 65 | 66 | # below role and user config is for deploying was from an external system 67 | 68 | 69 | 70 | 71 | restart tomcat 72 | -------------------------------------------------------------------------------- /jenkins/install-tomcat-in-ubuntu.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## update packages 4 | ``` 5 | sudo apt-get update 6 | ``` 7 | ## install openjdk 11 8 | 9 | ``` 10 | sudo apt-get update && sudo apt-get upgrade -y 11 | 12 | sudo apt-get install openjdk-11-jdk 13 | ``` 14 | ### Alternatively, if you need JRE only then 15 | 16 | sudo apt-get install openjdk-11-jre 17 | 18 | ## browser to local directory and download tomcat binary 19 | 20 | ``` 21 | cd /usr/local/ 22 | wget https://mirror.csclub.uwaterloo.ca/apache/tomcat/tomcat-8/v8.5.56/bin/apache-tomcat-8.5.56.tar.gz 23 | tar xvzf apache-tomcat-8.5.56.tar.gz 24 | ``` 25 | 26 | ## rename installation directory 27 | ``` 28 | mv apache-tomcat-8.5.56.tar.gz tomcat 29 | ``` 30 | ## start tomcat 31 | ``` 32 | cd /usr/local/tomcat/bin/ 33 | ./startup.sh 34 | ``` 35 | 36 | ## check process 37 | ``` 38 | ps -ef | grep tomcat 39 | ``` 40 | ## stop tomcat 41 | ``` 42 | ./shutdown.sh 43 | ``` 44 | 45 | ## kill process 46 | ``` 47 | sudo kill -9 48 | ``` 49 | 50 | -------------------------------------------------------------------------------- /jmeter/Dockerfile_01: -------------------------------------------------------------------------------- 1 | # This Dockerfile is to build a custom docker image for jmeter which will be used in AWS ECS container. That's why CMD has been used 2 | # https://openjdk.java.net/install/ 3 | 4 | FROM openjdk:19-jdk-alpine3.15 5 | 6 | # set variables 7 | 8 | ARG JMETER_VERSION="5.4.3" 9 | ENV JMETER_HOME=/opt/apache-jmeter-${JMETER_VERSION} 10 | ENV JMETER_DOWNLOAD_URL=https://archive.apache.org/dift/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.tgz 11 | 12 | ENV TEST_FILE=/opt/apache-jmeter-${JMETER_VERSION}/bin/examples/CSVSample.jmx 13 | ENV TEST_LOG_FILE=/opt/apache-jmeter-${JMETER_VERSION}/bin/examples/test.log 14 | ENV TEST_RESULTS_FILE=/opt/apache-jmeter-${JMETER_VERSION}/bin/examples/test-result.xml 15 | ENV PORT 80 16 | 17 | # COPY examples/test.jmx ${TEST_FILE} 18 | 19 | RUN apk update \ 20 | && apk upgrade \ 21 | && apk add ca-certificates \ 22 | && update-ca-certificates \ 23 | && apk add --update openjdk8-jre tzdata curl unzip bash \ 24 | && apk add --no-cache nss \ 25 | && rm -rf /var/cache/apk/* \ 26 | && mrdir -p /tmp/dependencies \ 27 | && curl -L --silent ${JMETER_DOWNLOAD_URL} > /tmp/dependencies/apache-jmeter-${JMETER_VERSION}.tgz \ 28 | && mkdir -p /opt \ 29 | && tar -xzf /tmp/dependencies/apache-jmeter-${JMETER_VERSION}.tgz -C /opt \ 30 | && rm -rf /tmp/dependencies 31 | 32 | ENV PATH $PATH:$JMETER_HOME/bin 33 | 34 | # main command to run jmeter 35 | 36 | CMD $JMETER_HOME/bin/jmeter -n \ 37 | -t $TEST_FILE \ 38 | -j $TEST_LOG_FILE \ 39 | -l $TEST_RESULTS_FILE \ 40 | -Jmeter.save.saveservice.output_format=xml \ 41 | -Jport=$PORT && \ 42 | echo -e "\n\n==== TEST LOGS ====" && \ 43 | cat $TEST_LOG_FILE && \ 44 | echo -e "\n\n==== TEST RESULTS =====" && \ 45 | echo $TEST_RESULTS_FILE 46 | 47 | -------------------------------------------------------------------------------- /jmeter/Dockerfile_02: -------------------------------------------------------------------------------- 1 | # This Dockerfile is to build a custom docker image for jmeter which will be used in AWS ECS container. That's why CMD has been used 2 | 3 | FROM openjdk:19-jdk-alpine3.15 4 | 5 | # set variables 6 | 7 | ARG JMETER_VERSION="5.4.3" 8 | ENV JMETER_HOME=/opt/apache-jmeter-${JMETER_VERSION} 9 | ENV JMETER_DOWNLOAD_URL=https://archive.apache.org/dift/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.tgz 10 | 11 | ENV TEST_FILE=/opt/apache-jmeter-${JMETER_VERSION}/bin/examples/CSVSample.jmx 12 | ENV TEST_LOG_FILE=/opt/apache-jmeter-${JMETER_VERSION}/bin/examples/test.log 13 | ENV TEST_RESULTS_FILE=/opt/apache-jmeter-${JMETER_VERSION}/bin/examples/test-result.xml 14 | ENV PORT 80 15 | 16 | ENV NUMBER_OF_REQUESTS_PER_CLIENT=10 \ 17 | DELAY_PER_REQUEST=100 \ 18 | NUMBER_OF_CLIENTS=1 \ 19 | RAMP_UP_PERIOD=1 \ 20 | TIMEOUT=2000 21 | 22 | # copy test plan 23 | # COPY examples/test.jmx ${TEST_FILE} 24 | 25 | RUN apk update \ 26 | && apk upgrade \ 27 | && apk add ca-certificates \ 28 | && update-ca-certificates \ 29 | && apk add --update openjdk8-jre tzdata curl unzip bash \ 30 | && apk add --no-cache nss \ 31 | && rm -rf /var/cache/apk/* \ 32 | && mrdir -p /tmp/dependencies \ 33 | && curl -L --silent ${JMETER_DOWNLOAD_URL} > /tmp/dependencies/apache-jmeter-${JMETER_VERSION}.tgz \ 34 | && mkdir -p /opt \ 35 | && tar -xzf /tmp/dependencies/apache-jmeter-${JMETER_VERSION}.tgz -C /opt \ 36 | && rm -rf /tmp/dependencies 37 | 38 | ENV PATH $PATH:$JMETER_HOME/bin 39 | 40 | # main command to run jmeter 41 | 42 | CMD $JMETER_HOME/bin/jmeter -n \ 43 | -t $TEST_FILE \ 44 | -j $TEST_LOG_FILE \ 45 | -l $TEST_RESULTS_FILE \ 46 | -Jmeter.save.saveservice.output_format=xml \ 47 | -Jmeter.save.saveservice.response_data=true \ 48 | -JnumberOfRequestsPerClient=$NUMBER_OF_REQUESTS_PER_CLIENT \ 49 | -JdelayPerRequest=$DELAY_PER_REQUEST \ 50 | -JrumpUpPeriod=$RAMP_UP_PERIOD \ 51 | -Jtimeout=$TIMEOUT \ 52 | -Jport=$PORT && \ 53 | echo -e "\n\n==== TEST LOGS ====" && \ 54 | cat $TEST_LOG_FILE && \ 55 | echo -e "\n\n==== TEST RESULTS =====" && \ 56 | echo $TEST_RESULTS_FILE 57 | 58 | -------------------------------------------------------------------------------- /jmeter/install-jmeter-docker-standalone.md: -------------------------------------------------------------------------------- 1 | Pull docker image 2 | #### Refer: https://hub.docker.com/r/justb4/jmeter 3 | 4 | ``` 5 | docker pull justb4/jmeter 6 | ``` 7 | Check pulled image 8 | 9 | ``` 10 | docker images 11 | ``` 12 | 13 | Tag the image with a handy nick name 14 | 15 | ``` 16 | docker tag justb4/jmeter jmeter 17 | ``` 18 | 19 | Run jmeter on docker 20 | 21 | ``` 22 | docker run jmeter 23 | ``` 24 | you will get X11 display error as we are using cli mode only. there is no GUI. The goal is to run the jmeter in cli mode. so ignore the error for now. 25 | 26 | Now run 27 | ``` 28 | # For Single Node 29 | docker run jmeter -n -t /opt/apache-jmeter-5.4.3/bin/examples/CSVSample.jmx -l results.jtl -Dserver.rmi.ssl.disable=true 30 | 31 | # mount the source to get the jmeter result in your target 32 | docker run --mount type=bind, source="mnt/c/tools/apache-jmeter-5.4.3/bin/", target="/opt/apache-jmeter-5.4.3/bin" jmeter -n -t bin/example.jmx -l bin/results.jtl -Dserver.rmi.ssl.disable=true 33 | 34 | For Distributed load test 35 | docker run -n -t /opt/apache-jmeter-5.4.3/bin/examples/CSVSample.jmx -l results.jtl -R -Dserver.rmi.ssl.disable=true 36 | ``` 37 | 38 | ### Copy the results from stopped containers 39 | 40 | ``` 41 | docker ps -a # to check stopped containers 42 | mkdir jmeter-results # create a dir to store result 43 | cd jmeter-result 44 | docker cp :/opt/apache-jmeter-5.4.3/bin/example . # dont miss the '.'. it will copy the containt of 'example' folder in your local directory 45 | 46 | ``` 47 | -------------------------------------------------------------------------------- /jmeter/jmeter-common-issues.md: -------------------------------------------------------------------------------- 1 | 2 | ## Error in NonGUIDriver Following remote engines could not be configuied: [10.#.#.#,10.#.#.#] 3 | 4 | ### Possible causes: 5 | - Jmeter is not installed properly in worker nodes. check by running 'jmeter --version' 6 | - check cloud-init-output file log 7 | ``` 8 | sudo cat /var/log/cloud-init-output.log 9 | ``` 10 | - worker nodes configuration must be enough to run jmeter. e.g, in AWS EC2 t2.micro cant hold jmeter. recommended to use t2.small onwards 11 | 12 | ## Stuck at configuring remote engine: 10.#.#.# 13 | - check the firewall or security group. it is basically restricting inboud traffic to worker nodes. 14 | -------------------------------------------------------------------------------- /jmeter/jmeter-html-report.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ``` 4 | jmeter -n -t -l e -o 5 | 6 | jmeter -n -t /home/ec2-user/apache-jmeter-5.4.3/example/Sample.jmx -l /home/ec2-user/apache-jmeter-5.4.3/results/result.csv e -o /home/ec2-user/apache-jmeter-5.4.3/reports 7 | 8 | ``` 9 | 10 | Or if you have .jtl file then use below command to generate html report 11 | 12 | ``` 13 | jmeter -g(path of .jtl file) -o(path of output folder where you want to save the results) 14 | 15 | jmeter -g"/home/ec2-user/apache-jmeter-5.4.3/result.jtl" -o"/home/ec2-user/apache-jmeter-5.4.3/" 16 | ``` 17 | -------------------------------------------------------------------------------- /kibana/commands.txt: -------------------------------------------------------------------------------- 1 | `` Kibana default port no 5601. 2 | `` elasticsearch port 9200 3 | 4 | `` install metricbeat 5 | 6 | sudo apt-get install metricbeat 7 | sudo systemctl start metricbeat -- all rich metric like cpu,load, memory, network, processes etc are all being collected into elasticsearch in 10 sec interval 8 | 9 | `` add load testing 10 | 11 | sudo apt install stress 12 | 13 | '' check how many core and available memory you have 14 | 15 | nproc -- to check no of core 16 | 17 | `` check memory 18 | 19 | free -h 20 | 21 | `` create load/stress , max out the cpu for 2 mins or 120 secs 22 | stress -- cpu 1 --timeout 120 23 | 24 | `` now run 5 worker for 3 mins 25 | stress --vm 5 --timeout 180 26 | 27 | 28 | 29 | `` bring logstash server up. browse to logstash bin folder and execute below folder. make sure you have logstask.conf file configured properly 30 | logstash -f logstash.conf 31 | 32 | `` list index in elasticsearch 33 | localhost:9200/_cat/_indices 34 | 35 | `` search in elasticsearch 36 | localhost:9200//_search 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /kubernetes/alias-and-shortcuts-for-kubernetes.txt: -------------------------------------------------------------------------------- 1 | alias kc='f(){ kubectl "$@" ; unset -f f; }; f' 2 | 3 | alias kcg='f(){ kubectl get "$@" --all-namespaces -o wide; unset -f f; }; f' 4 | 5 | alias kce='f(){ kubectl edit "$@"; unset -f f; }; f' 6 | alias kcdl='f(){ kubectl delete "$@"; unset -f f; }; f' 7 | 8 | alias kcgp='f(){ kubectl get pod -o wide; unset -f f; }; f' 9 | alias kcgd='f(){ kubectl get deployment -o wide; unset -f f; }; f' 10 | alias kcgs='f(){ kubectl get service -o wide; unset -f f; }; f' 11 | alias kcgrs='f(){ kubectl get replicaset -o wide; unset -f f; }; f' 12 | 13 | 14 | alias kcd='f(){ kubectl describe "$@" ; unset -f f; }; f' 15 | alias kcdp='f(){ kubectl describe pod "$@" ; unset -f f; }; f' 16 | alias kcdd='f(){ kubectl describe deployment "$@" ; unset -f f; }; f' 17 | alias kcdrs='f(){ kubectl describe replicaset "$@" ; unset -f f; }; f' 18 | alias kcds='f(){ kubectl describe service "$@" ; unset -f f; }; f' 19 | 20 | 21 | kc get ns 22 | kc run nginx-pod --image=nginx:alpine --dry-run=client -o yaml > pod.yaml 23 | kc run redis --image=redis:alpine --labels=tire=db --dry-run=client -o yaml > pod2.yaml 24 | kc expose pod redis --name redis-service --port 6379 --target-port 6379 25 | kc create deployment webapp --image=kodekloud/webapp-color 26 | kc scale deployment webapp --replicas=3 27 | kc run custom-nginx --image=nginx --port 8080 28 | 29 | 30 | kc create ns dev-ns 31 | kc create deployment redis-deploy --namespace=dev-ns --image=redis 32 | kc scale deployment redis-deploy --replicas=2 --namespace=dev-ns 33 | 34 | kc run httpd --image=httpd:alpine 35 | kc run httpd --image=httpd:alpine --port 80 expose --dry-run=client -o yaml 36 | -------------------------------------------------------------------------------- /kubernetes/cluster-role-binding/eks-console-full-access.yaml: -------------------------------------------------------------------------------- 1 | # check updated here https://s3.us-west-2.amazonaws.com/amazon-eks/docs/eks-console-full-access.yaml 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: eks-console-dashboard-full-access-clusterrole 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - nodes 11 | - namespaces 12 | - pods 13 | - configmaps 14 | - endpoints 15 | - events 16 | - limitranges 17 | - persistentvolumeclaims 18 | - podtemplates 19 | - replicationcontrollers 20 | - resourcequotas 21 | - secrets 22 | - serviceaccounts 23 | - services 24 | verbs: 25 | - get 26 | - list 27 | - apiGroups: 28 | - apps 29 | resources: 30 | - deployments 31 | - daemonsets 32 | - statefulsets 33 | - replicasets 34 | verbs: 35 | - get 36 | - list 37 | - apiGroups: 38 | - batch 39 | resources: 40 | - jobs 41 | - cronjobs 42 | verbs: 43 | - get 44 | - list 45 | - apiGroups: 46 | - coordination.k8s.io 47 | resources: 48 | - leases 49 | verbs: 50 | - get 51 | - list 52 | - apiGroups: 53 | - discovery.k8s.io 54 | resources: 55 | - endpointslices 56 | verbs: 57 | - get 58 | - list 59 | - apiGroups: 60 | - events.k8s.io 61 | resources: 62 | - events 63 | verbs: 64 | - get 65 | - list 66 | - apiGroups: 67 | - extensions 68 | resources: 69 | - daemonsets 70 | - deployments 71 | - ingresses 72 | - networkpolicies 73 | - replicasets 74 | verbs: 75 | - get 76 | - list 77 | - apiGroups: 78 | - networking.k8s.io 79 | resources: 80 | - ingresses 81 | - networkpolicies 82 | verbs: 83 | - get 84 | - list 85 | - apiGroups: 86 | - policy 87 | resources: 88 | - poddisruptionbudgets 89 | verbs: 90 | - get 91 | - list 92 | - apiGroups: 93 | - rbac.authorization.k8s.io 94 | resources: 95 | - rolebindings 96 | - roles 97 | verbs: 98 | - get 99 | - list 100 | - apiGroups: 101 | - storage.k8s.io 102 | resources: 103 | - csistoragecapacities 104 | verbs: 105 | - get 106 | - list 107 | --- 108 | apiVersion: rbac.authorization.k8s.io/v1 109 | kind: ClusterRoleBinding 110 | metadata: 111 | name: eks-console-dashboard-full-access-binding 112 | subjects: 113 | - kind: Group 114 | name: eks-console-dashboard-full-access-group 115 | apiGroup: rbac.authorization.k8s.io 116 | roleRef: 117 | kind: ClusterRole 118 | name: eks-console-dashboard-full-access-clusterrole 119 | apiGroup: rbac.authorization.k8s.io 120 | -------------------------------------------------------------------------------- /kubernetes/cluster-role-binding/project-a-ns.yaml: -------------------------------------------------------------------------------- 1 | # Refer: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ 2 | # This RoleBinding will provide all the users in 'project-a' group 'superuser' access scopped to 'project-a' name space ONLY. 3 | # ClusterRoleBinding binds Clusterrole with Service Account 4 | apiVersion: rbac.authorization.k8s.io/v1 5 | # This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. 6 | kind: ClusterRoleBinding 7 | metadata: 8 | name: project-a-ns-full-access 9 | namespace: project-ns-01 10 | subjects: 11 | - kind: Group 12 | name: project-a # Name is case sensitive 13 | apiGroup: rbac.authorization.k8s.io 14 | - kind: ServiceAccount 15 | name: e2esa-service-account-01 16 | namespace: project-ns-01 17 | roleRef: 18 | kind: ClusterRole 19 | name: e2esa-cluster-role-01 20 | apiGroup: rbac.authorization.k8s.io 21 | -------------------------------------------------------------------------------- /kubernetes/cluster-role/clusterrole-01.yaml: -------------------------------------------------------------------------------- 1 | # ClusterRoleBinding binds Clusterrole with Service Account 2 | # ClsuterRole is not NS specific. It is available through out the cluster 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | name: e2esa-cluster-role-01 7 | rules: # Authorization rules 8 | - apiGroups: # 1st API group 9 | - '' # An empty string designates the core API group. 10 | - extensions 11 | resources: 12 | - persistentvolumes 13 | - nodes 14 | - configmaps 15 | - endpoints 16 | - events 17 | - ingresses 18 | - ingresses/status 19 | - services 20 | verbs: 21 | - get 22 | - list 23 | - watch 24 | - create 25 | - update 26 | - path 27 | -------------------------------------------------------------------------------- /kubernetes/commands/k8s-commands-config.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ``` 4 | # Manage configmap output 5 | kubectl get configmap aws-auth -n kube-system -o yaml 6 | kubectl edit configmap aws-auth -n kube-system 7 | 8 | ``` 9 | 10 | ``` 11 | aws eks --region update-kubeconfig --name 12 | aws eks --region us-east-1 update-kubeconfig --name e2esa-demo-eks-cluster 13 | 14 | # Read the config 15 | cat /home/ubuntu/.kube/config 16 | ``` 17 | -------------------------------------------------------------------------------- /kubernetes/create-namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: dev-webapp01 5 | labels: 6 | name: dev-webapp01 7 | -------------------------------------------------------------------------------- /kubernetes/deployment/deployment-def1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: e2esa-webapp01-deployment 5 | namespace: dev-webapp01 6 | labels: 7 | app: e2esa-webapp01 8 | spec: 9 | replicas: 4 10 | template: 11 | metadata: 12 | name: e2esa-webapp01-pod 13 | labels: 14 | tire: front-end 15 | env: dev 16 | app: e2esa-webapp01 17 | spec: 18 | containers: 19 | - name: nginx 20 | image: nginx 21 | selector: 22 | matchLabels: 23 | app: e2esa-webapp01 24 | -------------------------------------------------------------------------------- /kubernetes/deployment/deployment-nginx-lb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: e2esa-webapp01-ns 6 | --- 7 | apiVersion: apps/v1 8 | kind: Deployment 9 | metadata: 10 | namespace: e2esa-webapp01-ns 11 | name: e2esa-webapp01-deployment 12 | spec: 13 | selector: 14 | matchLabels: 15 | app.kubernetes.io/name: e2esa-webapp01 16 | replicas: 2 17 | template: 18 | metadata: 19 | labels: 20 | app.kubernetes.io/name: e2esa-webapp01 21 | spec: 22 | containers: 23 | - image: nginx:latest 24 | imagePullPolicy: Always 25 | name: e2esa-webapp01-img 26 | ports: 27 | - containerPort: 80 28 | --- 29 | apiVersion: v1 30 | kind: Service 31 | metadata: 32 | namespace: e2esa-webapp01-ns 33 | name: e2esa-webapp01-service 34 | spec: 35 | ports: 36 | - port: 80 37 | targetPort: 80 38 | protocol: TCP 39 | type: LoadBalancer 40 | selector: 41 | app.kubernetes.io/name: e2esa-webapp01 42 | -------------------------------------------------------------------------------- /kubernetes/deployment/deployment-nginx.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: e2esa-webapp01-ns 6 | --- 7 | apiVersion: apps/v1 8 | kind: Deployment 9 | metadata: 10 | namespace: e2esa-webapp01-ns 11 | name: e2esa-webapp01-deployment 12 | spec: 13 | selector: 14 | matchLabels: 15 | app.kubernetes.io/name: e2esa-webapp01 16 | replicas: 2 17 | template: 18 | metadata: 19 | labels: 20 | app.kubernetes.io/name: e2esa-webapp01 21 | spec: 22 | containers: 23 | - image: nginx:latest 24 | imagePullPolicy: Always 25 | name: e2esa-webapp01-img 26 | ports: 27 | - containerPort: 80 28 | --- 29 | apiVersion: v1 30 | kind: Service 31 | metadata: 32 | namespace: e2esa-webapp01-ns 33 | name: e2esa-webapp01-service 34 | spec: 35 | ports: 36 | - port: 80 37 | targetPort: 80 38 | protocol: TCP 39 | type: NodePort 40 | selector: 41 | app.kubernetes.io/name: e2esa-webapp01 42 | --- 43 | apiVersion: networking.k8s.io/v1 44 | kind: Ingress 45 | metadata: 46 | namespace: e2esa-webapp01-ns 47 | name: e2esa-webapp01-ingress 48 | annotations: 49 | alb.ingress.kubernetes.io/scheme: internet-facing 50 | alb.ingress.kubernetes.io/target-type: ip 51 | spec: 52 | ingressClassName: alb 53 | rules: 54 | - http: 55 | paths: 56 | - path: / 57 | pathType: Prefix 58 | backend: 59 | service: 60 | name: e2esa-webapp01 61 | port: 62 | number: 80 63 | -------------------------------------------------------------------------------- /kubernetes/deployment/deployment-with-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | namespace: e2esa-webapp01-ns 5 | name: e2esa-webapp01-deployment 6 | spec: 7 | selector: 8 | matchLabels: 9 | app.kubernetes.io/name: e2esa-webapp01 10 | replicas: 2 11 | template: 12 | metadata: 13 | labels: 14 | app.kubernetes.io/name: e2esa-webapp01 15 | spec: 16 | serviceAccountName: e2esa-service-account-01 17 | containers: 18 | - image: nginx:latest 19 | imagePullPolicy: Always 20 | name: e2esa-webapp01-img 21 | ports: 22 | - containerPort: 80 23 | -------------------------------------------------------------------------------- /kubernetes/deployment/myapp-deployment-jfrog.yaml: -------------------------------------------------------------------------------- 1 | # refer: https://jfrog.com/blog/pulling-all-your-kubernetes-cluster-images-from-a-private-artifactory-registry/ 2 | # create the secret first for jfrog authentication 3 | # kubectl create secret docker-registry regcred \ 4 | # --docker-server=my-artifactory.jfrog.io \ 5 | # --docker-username=read-only \ 6 | # --docker-password=my-super-secret-pass \ 7 | # --docker-email=johndoe@example.com \ 8 | # -n my-app-ns 9 | # 10 | # secret/regcred created 11 | 12 | # Patch the default service account to include the imagePullSecrets section. 13 | # By default a service account named default automatically gets created with each namespace and all workloads will automatically use it. 14 | 15 | # kubectl edit serviceaccount default -n my-app-ns 16 | apiVersion: v1 17 | kind: ServiceAccount 18 | imagePullSecrets: 19 | - name: regcred 20 | --- 21 | apiVersion: apps/v1 22 | kind: Deployment 23 | metadata: 24 | name: e2esa-webapp01-deployment 25 | namespace: dev-webapp01 26 | labels: 27 | app: e2esa-webapp01 28 | spec: 29 | replicas: 4 30 | selector: 31 | matchLabels: 32 | app: e2esa-webapp01 33 | template: 34 | metadata: 35 | labels: 36 | app: e2esa-webapp01 37 | spec: 38 | containers: 39 | - name: my-app 40 | image: my-artifactory.jfrog.io/default-docker-virtual/my-app:1.0.1 41 | imagePullPolicy: Always 42 | imagePullSecrets: 43 | - name: regcred 44 | ports: 45 | - containerPort: 80 46 | --- 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | namespace: e2esa-webapp01-ns 51 | name: e2esa-webapp01-service 52 | spec: 53 | ports: 54 | - port: 80 55 | targetPort: 80 56 | protocol: TCP 57 | type: NodePort 58 | selector: 59 | app.kubernetes.io/name: e2esa-webapp01 60 | -------------------------------------------------------------------------------- /kubernetes/deployment/nginx-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: e2esa-webapp01-deployment 5 | namespace: dev-webapp01 6 | labels: 7 | app: e2esa-webapp01 8 | spec: 9 | replicas: 4 10 | selector: 11 | matchLabels: 12 | app: e2esa-webapp01 13 | template: 14 | metadata: 15 | labels: 16 | app: e2esa-webapp01 17 | spec: 18 | containers: 19 | - name: nginx 20 | image: nginx:latest 21 | imagePullPolicy: Always 22 | ports: 23 | - containerPort: 80 24 | -------------------------------------------------------------------------------- /kubernetes/deployment/nginx-ecr-deployment-v02.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: e2esa-webapp01-deployment 5 | namespace: dev-webapp01 6 | labels: 7 | app: e2esa-webapp01 8 | spec: 9 | replicas: 4 10 | strategy: 11 | type: RollingUpdate 12 | maxUnavailable: 2 13 | maxSurge: 2 14 | selector: 15 | matchLabels: 16 | app: e2esa-webapp01 17 | template: 18 | metadata: 19 | labels: 20 | app: e2esa-webapp01 21 | spec: 22 | containers: 23 | - name: nginx 24 | image: public.ecr.aws/nginx/nginx:1.21 25 | securityContext: 26 | priviledged: false 27 | readOnlyRootFileSystem: true 28 | allowPrivilegeEscalation: false 29 | ports: 30 | - containerPort: 80 31 | -------------------------------------------------------------------------------- /kubernetes/deployment/nginx-ecr-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: e2esa-webapp01-deployment 5 | namespace: dev-webapp01 6 | labels: 7 | app: e2esa-webapp01 8 | spec: 9 | replicas: 4 10 | selector: 11 | matchLabels: 12 | app: e2esa-webapp01 13 | template: 14 | metadata: 15 | labels: 16 | app: e2esa-webapp01 17 | spec: 18 | containers: 19 | - name: nginx 20 | image: public.ecr.aws/nginx/nginx:1.21 21 | ports: 22 | - containerPort: 80 23 | -------------------------------------------------------------------------------- /kubernetes/pod/pod-def1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: mypod 5 | labels: 6 | tire: front-end 7 | env: dev 8 | spec: 9 | containers: 10 | - name: nginx 11 | image: nginx 12 | resources: 13 | requests: 14 | memory: "64Mi" 15 | cpu: "250m" 16 | limits: 17 | memory: "128Mi" 18 | cpu: "500m" 19 | -------------------------------------------------------------------------------- /kubernetes/pod/pod-multi-containers-def.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: mypod 5 | labels: 6 | tire: front-end 7 | env: dev 8 | spec: 9 | containers: 10 | - name: nginx 11 | image: nginx:latest 12 | resources: 13 | requests: 14 | memory: "64Mi" 15 | cpu: "250m" 16 | limits: 17 | memory: "128Mi" 18 | cpu: "500m" 19 | - name: log-aggregator 20 | image: log-aggregator:latest 21 | resources: 22 | requests: 23 | memory: "64Mi" 24 | cpu: "250m" 25 | limits: 26 | memory: "128Mi" 27 | cpu: "500m" 28 | -------------------------------------------------------------------------------- /kubernetes/readme.md: -------------------------------------------------------------------------------- 1 | [Click here](https://github.com/e2eSolutionArchitect/scripts/tree/main/aws/eks) for AWS EKS scripts and commands 2 | -------------------------------------------------------------------------------- /kubernetes/replicaset/replicaset-def1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: ReplicaSet 3 | metadata: 4 | name: myreplica 5 | labels: 6 | app: myapp 7 | spec: 8 | replicas: 4 9 | template: 10 | metadata: 11 | name: mypod 12 | labels: 13 | tire: front-end 14 | env: dev 15 | spec: 16 | containers: 17 | - name: nginx 18 | image: nginx 19 | selector: 20 | matchLabels: 21 | tire: front-end 22 | env: dev 23 | -------------------------------------------------------------------------------- /kubernetes/role-binding/rolebinding-01.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: deployment-role-binding-01 5 | namespace: project-ns-01 6 | roleRef: 7 | kind: Role 8 | name: e2esa-development-role-01 # Add the role created for deployment 9 | apiGroup: rbac.authorization.k8s.io 10 | subjects: 11 | - kind: User 12 | name: # add username 13 | apiGroup: rbac.authorization.k8s.io 14 | -------------------------------------------------------------------------------- /kubernetes/role/development-role-01.yaml: -------------------------------------------------------------------------------- 1 | # Role is NS specific. 2 | # Bind Role in RoleBinding. Also map this role to the user configuration in kubernetes ConfigMap 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: Role 5 | metadata: 6 | name: e2esa-development-role-01 7 | namespace: project-ns-01 8 | rules: # Authorization rules 9 | - apiGroups: # 1st API group, Multiple apiGroups can be added 10 | - '' # An empty string designates the core API group. 11 | - extensions 12 | - apps 13 | resources: 14 | - deployments 15 | - replicasets 16 | - pods 17 | verbs: 18 | - create 19 | - get 20 | - list 21 | - update 22 | - path 23 | - watch 24 | -------------------------------------------------------------------------------- /kubernetes/rolebinding/eks-console-full-access.yaml: -------------------------------------------------------------------------------- 1 | # check updated here https://s3.us-west-2.amazonaws.com/amazon-eks/docs/eks-console-full-access.yaml 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: eks-console-dashboard-full-access-clusterrole 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - nodes 11 | - namespaces 12 | - pods 13 | - configmaps 14 | - endpoints 15 | - events 16 | - limitranges 17 | - persistentvolumeclaims 18 | - podtemplates 19 | - replicationcontrollers 20 | - resourcequotas 21 | - secrets 22 | - serviceaccounts 23 | - services 24 | verbs: 25 | - get 26 | - list 27 | - apiGroups: 28 | - apps 29 | resources: 30 | - deployments 31 | - daemonsets 32 | - statefulsets 33 | - replicasets 34 | verbs: 35 | - get 36 | - list 37 | - apiGroups: 38 | - batch 39 | resources: 40 | - jobs 41 | - cronjobs 42 | verbs: 43 | - get 44 | - list 45 | - apiGroups: 46 | - coordination.k8s.io 47 | resources: 48 | - leases 49 | verbs: 50 | - get 51 | - list 52 | - apiGroups: 53 | - discovery.k8s.io 54 | resources: 55 | - endpointslices 56 | verbs: 57 | - get 58 | - list 59 | - apiGroups: 60 | - events.k8s.io 61 | resources: 62 | - events 63 | verbs: 64 | - get 65 | - list 66 | - apiGroups: 67 | - extensions 68 | resources: 69 | - daemonsets 70 | - deployments 71 | - ingresses 72 | - networkpolicies 73 | - replicasets 74 | verbs: 75 | - get 76 | - list 77 | - apiGroups: 78 | - networking.k8s.io 79 | resources: 80 | - ingresses 81 | - networkpolicies 82 | verbs: 83 | - get 84 | - list 85 | - apiGroups: 86 | - policy 87 | resources: 88 | - poddisruptionbudgets 89 | verbs: 90 | - get 91 | - list 92 | - apiGroups: 93 | - rbac.authorization.k8s.io 94 | resources: 95 | - rolebindings 96 | - roles 97 | verbs: 98 | - get 99 | - list 100 | - apiGroups: 101 | - storage.k8s.io 102 | resources: 103 | - csistoragecapacities 104 | verbs: 105 | - get 106 | - list 107 | --- 108 | apiVersion: rbac.authorization.k8s.io/v1 109 | kind: ClusterRoleBinding 110 | metadata: 111 | name: eks-console-dashboard-full-access-binding 112 | subjects: 113 | - kind: Group 114 | name: eks-console-dashboard-full-access-group 115 | apiGroup: rbac.authorization.k8s.io 116 | roleRef: 117 | kind: ClusterRole 118 | name: eks-console-dashboard-full-access-clusterrole 119 | apiGroup: rbac.authorization.k8s.io 120 | -------------------------------------------------------------------------------- /kubernetes/service-account/serviceaccount-01.yaml: -------------------------------------------------------------------------------- 1 | # ClusterRoleBinding binds Clusterrole with Service Account 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: e2esa-service-account-01 7 | name: e2esa-service-account-01 8 | namespace: project-ns-01 9 | -------------------------------------------------------------------------------- /kubernetes/service/service-nlb-internal.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: e2esa-webapp01-service 5 | namespace: dev-webapp01 6 | annotations: 7 | service.beta.kubernetes.io/aws-load-balancer-internal: "true" 8 | service.beta.kubernetes.io/aws-load-balancer-scheme: internal 9 | service.beta.kubernetes.io/aws-load-balancer-type: nlb 10 | labels: 11 | app: e2esa-webapp01 12 | spec: 13 | type: LoadBalancer 14 | ports: 15 | - port: 80 16 | targetPort: 80 17 | nodePort: 30004 18 | selector: 19 | tire: front-end 20 | env: dev 21 | app: e2esa-webapp01 22 | -------------------------------------------------------------------------------- /kubernetes/service/service-wd-lb.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: e2esa-webapp01-service 5 | namespace: dev-webapp01 6 | annotations: 7 | service.beta.kubernetes.io/aws-load-balancer-type: nlb 8 | labels: 9 | app: e2esa-webapp01 10 | spec: 11 | type: LoadBalancer 12 | ports: 13 | - port: 80 14 | targetPort: 80 15 | nodePort: 30004 16 | selector: 17 | tire: front-end 18 | env: dev 19 | app: e2esa-webapp01 20 | -------------------------------------------------------------------------------- /kubernetes/service/service-wd-nlb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: e2esa-webapp01-service 5 | namespace: dev-webapp01 6 | annotations: 7 | service.beta.kubernetes.io/aws-load-balancer-type: external 8 | service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip 9 | service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing 10 | labels: 11 | app: e2esa-webapp01 12 | spec: 13 | type: LoadBalancer 14 | ports: 15 | - port: 80 16 | targetPort: 80 17 | nodePort: 30004 18 | selector: 19 | tire: front-end 20 | env: dev 21 | app: e2esa-webapp01 22 | -------------------------------------------------------------------------------- /kubernetes/service/service-wo-lb.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: e2esa-webapp01-service 5 | annotations: 6 | service.beta.kubernetes.io/aws-load-balancer-type: nlb 7 | labels: 8 | app: e2esa-webapp01 9 | spec: 10 | type: NodePort 11 | ports: 12 | - port: 80 13 | targetPort: 80 14 | nodePort: 30004 15 | type: LoadBalancer 16 | selector: 17 | tire: front-end 18 | env: dev 19 | app: e2esa-webapp01 20 | -------------------------------------------------------------------------------- /kubernetes/target-group-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: elbv2.k8s.aws/v1beta1 2 | kind: TargetGroupBinding 3 | metadata: 4 | name: nginx 5 | spec: 6 | serviceRef: 7 | name: nginx 8 | port: 8080 9 | targetGroupARN: "arn:aws:elasticloadbalancing:eu-south-1:123456789012:targetgroup/tf-20220726090605997700000002/a6527ae0e19830d2" 10 | -------------------------------------------------------------------------------- /markdown/icons.md: -------------------------------------------------------------------------------- 1 | 2 | add the keyword between two : : to get the icon displayed 3 | 4 | | | | | | | | | | 5 | |--- | --- |--- |--- |--- |--- |--- |--- | 6 | | :snowflake:| snowflake | :bulb: |bulb | :chart_with_upwards_trend: |chart_with_upwards_trend | :calendar: |calendar | :clipboard: | clipboard| 7 | | :star: |star | :unlock: | unlock| :lock: | lock| :key: |key | 8 | | :boom: | boom| :high_brightness: | high_brightness| :low_brightness: |low_brightness | :electric_plug: |electric_plug | 9 | | :exclamation: |exclamation | :calling: | calling| :postbox: | postbox| :memo: | memo| 10 | | :grey_question: | grey_question| :moneybag: |moneybag | :dart: |dart | :pencil: |pencil | 11 | | :question: |question | :dollar: |dollar | :e-mail: | e-mail| :envelope: | envelope| 12 | | :grey_exclamation: |grey_exclamation | :trophy: |trophy | :warning: | warning| :: | | 13 | | :+1: |+1 | :gift: |gift | :: | | :: | | 14 | | :thumbsup: |thumbsup | :thumbsdown: |thumbsdown | :ok_hand: | ok_hand| :raising_hand: | raising_hand | 15 | | :bell: | bell| :no_bell: |no_bell | :speaker: | speaker| :mute: |mute | 16 | | :sunny: | sunny| :cloud: |cloud | :cyclone: | cyclone| :floppy_disk: | floppy_disk| 17 | | :computer: |computer | :camera: |camera | :cd: | cd| :tv: | tv| 18 | | :loudspeaker: | loudspeaker| :hourglass: |hourglass | :alarm_clock: | alarm_clock| :watch: |watch | 19 | | :radio: |radio | :loop: | loop| :satellite: |satellite | :mag_right: |mag_right | 20 | | :diamonds: | diamonds| :spades: | spades| :100: |100 | :o: |o | 21 | | :office: |office | :house: | house| :bank: |bank | :bangbang: |bangbang | 22 | | :four: |four | :seven: |seven | :arrow_up: |arrow_up | :arrow_double_up: |arrow_double_up | 23 | | :no_entry_sign: |no_entry_sign | :copyright: |copyright | :registered: |registered | :: | | 24 | | :clock130: |clock130 | :clock11: | clock11| :white_check_mark: | white_check_mark| :: | | 25 | | :red_circle: | red_circle| :heavy_dollar_sign: |heavy_dollar_sign | :large_orange_diamond: |large_orange_diamond | :white_square_button: |white_square_button | 26 | 27 | 28 | ***Reference***: 29 | https://dev.to/nikolab/complete-list-of-github-markdown-emoji-markup-5aia 30 | https://gist.github.com/rxaviers/7360908 31 | -------------------------------------------------------------------------------- /mysql/install-mysql-ubuntu.txt: -------------------------------------------------------------------------------- 1 | 2 | sudo apt install mysql-server 3 | 4 | `` secure the installation by below command and follow the instructions 5 | sudo mysql_secure_installation 6 | -------------------------------------------------------------------------------- /nexus-install-in-ubuntu.md: -------------------------------------------------------------------------------- 1 | 2 | ## update packages 3 | 4 | sudo apt-get update 5 | 6 | ## install openjdk 8 7 | 8 | sudo apt install openjdk-8-jdk 9 | 10 | ## change user to sudo 11 | 12 | sudo su 13 | 14 | ## browse to /opt. where we will install nexus 15 | 16 | ## get nexus binary 17 | 18 | wget https://sonatype-download.global.ssl.fastly.net/nexus/3/nexus-3.24.0-02-unix.tar.gz 19 | 20 | tar -zxvf nexus-3.24.0-02-unix.tar.gz 21 | 22 | mv /opt/nexus-3.24.0-02 /opt/nexus 23 | 24 | ## add user. (best practice to avoid root user. create new user) 25 | 26 | sudo adduser nexus 27 | 28 | ## add priviledge to new user 29 | visudo 30 | 31 | ###add nexus ALL=(ALL) NOPASSWD: ALL 32 | 33 | ## change ownership to new user 34 | sudo chown -R nexus:nexus /opt/nexus 35 | 36 | sudo chown -R nexus:nexus /opt/sonatype-work 37 | 38 | ## update /opt/nexus/bin/nexus.rc file, just uncomment run_as_user 39 | 40 | vi /opt/nexus/bin/nexus.rc 41 | 42 | run_as_user="nexus" 43 | 44 | ## Add nexus as a service at boot time 45 | 46 | sudo ln -s /opt/nexus/bin/nexus /etc/init.d/nexus 47 | 48 | ## start nexus 49 | 50 | /etc/init.d/nexus start 51 | 52 | ## check the services running on ports 53 | 54 | netstat -nlpt 55 | 56 | it should show like below 57 | 58 | Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name 59 | 60 | ### tcp 0 0 0.0.0.0:8081 0.0.0.0:* LISTEN - 61 | tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN - 62 | tcp 0 0 127.0.0.1:36665 0.0.0.0:* LISTEN - 63 | tcp6 0 0 :::22 :::* LISTEN - 64 | 65 | 66 | http://nexuxserver url:8081 (default 8081). Make sure you have 8081 port in your security group (for AWS) / network rule (for Azure) / ingress rule ( for GCP) 67 | 68 | Use below credentials to login 69 | 70 | username : admin 71 | 72 | get initial password from path /opt/sonatype-work/nexus3/admin.password 73 | 74 | run cat /opt/sonatype-work/nexus3/admin.password in your nexus server. 75 | 76 | 77 | # pull artifact from nexus to jenkins and deploy to tomcat 78 | ## in JENKINS build job write below shell script to pull artifact 79 | 80 | wget --user= --password= 81 | 82 | wget --user=admin --password=admin http://35.184.138.42:8081/repository/maven-snapshots/awstechguide/spring-webapp/1.0.0-SNAPSHOT/spring-webapp-1.0.0-20200703.145607-1.WAR 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /nodejs/aws-sqs-message-processor/consumer.js: -------------------------------------------------------------------------------- 1 | const {SQS,ReceiveMessageCommand,DeleteMessageCommand} = require("@aws-sdk/client-sqs"); 2 | require('dotenv').config(); 3 | 4 | const sqsClient = new SQS({ 5 | region: process.env.AWS_REGION, 6 | credentials: { 7 | accessKeyId: process.env.AWS_ACCESS_KEY, 8 | secretAccessKey: process.env.AWS_ACCESS_SECRET_KEY 9 | } 10 | }); 11 | 12 | 13 | const PullMessagesFromQueue = async () => { 14 | try{ 15 | const command = new ReceiveMessageCommand({ 16 | MaxNumberOfMessages: 10, 17 | QueueUrl: process.env.AWS_SQS_QUEUE_URL, 18 | WaitTimeSeconds: 5, 19 | MessageAttributes: ["All"], 20 | VisibilityTimeout: 10, 21 | 22 | }); 23 | const result = await sqsClient.send(command); 24 | //console.log(result.Messages); 25 | 26 | // do some message processing 27 | ProcessMessage(result); 28 | // delete the message after successful processing 29 | const del_result = await DeleteMessageFromQueue(result.Messages[0].ReceiptHandle); 30 | } catch (error) { 31 | console.log(error); 32 | } 33 | }; 34 | 35 | PullMessagesFromQueue(); 36 | 37 | const DeleteMessageFromQueue = async (ReceiptHandle) => { 38 | try{ 39 | const data = await sqsClient.send(new DeleteMessageCommand({ 40 | QueueUrl: process.env.AWS_SQS_QUEUE_URL, 41 | ReceiptHandle: ReceiptHandle, 42 | })) 43 | console.log("deleted successfully......"); 44 | } catch (error) { 45 | console.log(error); 46 | } 47 | }; 48 | 49 | 50 | 51 | const ProcessMessage = async (result) => { 52 | try{ 53 | console.log('calling ProcessMessage'); 54 | console.log(result.Messages); 55 | } catch (error) { 56 | console.log(error); 57 | } 58 | }; -------------------------------------------------------------------------------- /nodejs/aws-sqs-message-processor/download.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | const fs = require('fs'); 3 | require('dotenv').config(); 4 | 5 | // Configure AWS SDK 6 | AWS.config.update({ 7 | accessKeyId: process.env.AWS_ACCESS_KEY, 8 | secretAccessKey: process.env.AWS_ACCESS_SECRET_KEY, 9 | region: process.env.AWS_REGION 10 | }); 11 | 12 | const s3 = new AWS.S3(); 13 | 14 | 15 | 16 | const download = async (filename) => { 17 | var resp =false; 18 | var params = { 19 | Bucket: process.env.AWS_BUCKET_NAME, 20 | Key: filename 21 | }; 22 | console.log("Initiating download....."); 23 | s3.getObject(params, (err, data) => { 24 | if (err) { 25 | console.log(err); 26 | } else { 27 | //console.log(data.Body.toString()); 28 | } 29 | 30 | fs.writeFile(process.env.LOCAL_PATH_TO_SAVE_FILE, data.Body, (writeErr) => { 31 | if (writeErr) { 32 | console.log("File download failed for below reason"); 33 | console.error(writeErr); 34 | }else { 35 | console.log(`File downloaded to ${process.env.LOCAL_PATH_TO_SAVE_FILE}`); 36 | resp = true; 37 | } 38 | }); 39 | }); 40 | return resp; 41 | }; 42 | 43 | module.exports = { download }; -------------------------------------------------------------------------------- /nodejs/aws-sqs-message-processor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqs-message-processor", 3 | "version": "1.0.0", 4 | "description": "npm install", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@aws-sdk/client-sqs": "^3.427.0", 13 | "aws-sdk": "^2.1472.0", 14 | "dotenv": "^16.3.1", 15 | "uuid": "^9.0.1" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /nodejs/aws-sqs-message-processor/producer.js: -------------------------------------------------------------------------------- 1 | const {SQS, SendMessageCommand} = require("@aws-sdk/client-sqs"); 2 | require('dotenv').config(); 3 | const sqsClient = new SQS({ 4 | region: process.env.AWS_REGION, 5 | credentials: { 6 | accessKeyId: process.env.AWS_ACCESS_KEY, 7 | secretAccessKey: process.env.AWS_ACCESS_SECRET_KEY 8 | } 9 | }); 10 | 11 | const SendMessageToQueue = async (body,messageAttributes) => { 12 | try{ 13 | const command = new SendMessageCommand({ 14 | MessageBody: body, 15 | QueueUrl: process.env.AWS_SQS_QUEUE_URL_AWAITING, 16 | MessageAttributes: messageAttributes, 17 | }); 18 | const result = await sqsClient.send(command); 19 | console.log(result); 20 | } catch (error) { 21 | console.log(error); 22 | } 23 | }; 24 | 25 | //SendMessageToQueue("File scan request"); 26 | 27 | module.exports = { SendMessageToQueue }; -------------------------------------------------------------------------------- /nodejs/aws-sqs-message-processor/readme.md: -------------------------------------------------------------------------------- 1 | 2 | ``` 3 | npm install 4 | 5 | npm init 6 | npm i @aws-sdk/client-sqs 7 | npm install --save dotenv 8 | npm install uuid // to generate unique id for message id 9 | npm install aws-sdk 10 | 11 | 12 | node producer.js 13 | node consumer.js 14 | ``` 15 | 16 | -------------------------------------------------------------------------------- /packer/aws-ami-v1.pkr.hcl: -------------------------------------------------------------------------------- 1 | # If you have your default VPC available then use it. 2 | 3 | packer { 4 | required_plugins { 5 | amazon = { 6 | version = ">= 0.0.2" 7 | source = "github.com/hashicorp/amazon" 8 | } 9 | } 10 | } 11 | 12 | source "amazon-ebs" "ubuntu" { 13 | ami_name = "e2esa-aws-ubuntu" 14 | instance_type = "t2.micro" 15 | region = "us-east-1" 16 | source_ami = "ami-0c4f7023847b90238" 17 | ssh_username = "ubuntu" 18 | } 19 | 20 | build { 21 | name = "e2esa-packer" 22 | sources = [ 23 | "source.amazon-ebs.ubuntu" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /packer/aws-ami-v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "builders": [ 3 | { 4 | "type": "amazon-ebs", 5 | "region": "us-east-1", 6 | "source_ami": "ami-09d56f8956ab235b3", 7 | "instance_type": "t2.micro", 8 | "ssh_username": "ubuntu", 9 | "ami_name": "packer-example {{timestamp}}" 10 | } 11 | ], 12 | "provisioners": [ 13 | { 14 | "type": "shell", 15 | "script": "provision.sh" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /packer/aws-ami-v3.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "aws_region": "us-west-2", 4 | "aws_subnet_id": "subnet-xxxxx", 5 | "aws_vpc_id": "vpc-xxxxx", 6 | "aws_ami_name": "CentOS-7-HVM-EBS-{{timestamp}}", 7 | "source_ami": "ami-0c4f7023847b90238" 8 | }, 9 | "builders": [ 10 | { 11 | "vpc_id": "{{user `aws_vpc_id`}}", 12 | "subnet_id": "{{user `aws_subnet_id`}}", 13 | "type": "amazon-ebs", 14 | "ami_virtualization_type": "hvm", 15 | "region": "{{user `aws_region`}}", 16 | "source_ami": "{{user `source_ami`}}", 17 | "ami_name": "{{user `aws_ami_name`}}-{{timestamp}}", 18 | "ami_description": "Latest CentOS AMI with EBS backend on HVM", 19 | "instance_type": "t2.micro", 20 | "ssh_username": "ubuntu", 21 | "account_id": "AccountIDNUMBER", 22 | "source_ami_filter": { 23 | "filters": { 24 | "virtualization-type": "hvm", 25 | "name": "ops_aws_cent_7_*", 26 | "root-device-type": "ebs" 27 | }, 28 | "owners": ["xxxxxxxxxxx"], 29 | "most_recent": true 30 | }, 31 | "tags": {"Environment": "test", "name": "packer.io test"} 32 | } 33 | ], 34 | "provisioners": [ 35 | { 36 | "type": "shell", 37 | "script": "provision.sh" 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /packer/aws-ami-v4.pkr.hcl: -------------------------------------------------------------------------------- 1 | # file: example.pkr.hcl 2 | 3 | # Reference https://www.packer.io/plugins/builders/amazon/ebs 4 | # In order to get these variables to read from the environment, 5 | # set the environment variables to have the same name as the declared 6 | # variables, with the prefix PKR_VAR_. 7 | # You could also hardcode them into the file, but we recommend that. 8 | 9 | data "amazon-ami" "example" { 10 | filters = { 11 | virtualization-type = "hvm" 12 | name = "ubuntu/images/*ubuntu-xenial-16.04-amd64-server-*" 13 | root-device-type = "ebs" 14 | } 15 | owners = ["099720109477"] 16 | most_recent = true 17 | region = "us-east-1" 18 | } 19 | 20 | source "amazon-ebs" "ssm-example" { 21 | ami_name = "packer_AWS {{timestamp}}" 22 | instance_type = "t2.micro" 23 | region = "us-east-1" 24 | source_ami = data.amazon-ami.example.id 25 | ssh_username = "ubuntu" 26 | ssh_interface = "session_manager" 27 | communicator = "ssh" 28 | iam_instance_profile = "myinstanceprofile" 29 | } 30 | 31 | build { 32 | sources = ["source.amazon-ebs.ssm-example"] 33 | 34 | provisioner "shell" { 35 | inline = ["echo Connected via SSM at '${build.User}@${build.Host}:${build.Port}'"] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /packer/aws-ami-v5.json: -------------------------------------------------------------------------------- 1 | # reference https://www.packer.io/plugins/builders/amazon/ebs 2 | { 3 | "builders": [ 4 | { 5 | "type": "amazon-ebs", 6 | "ami_name": "packer-ami-{{timestamp}}", 7 | "instance_type": "t2.micro", 8 | "source_ami_filter": { 9 | "filters": { 10 | "virtualization-type": "hvm", 11 | "name": "ubuntu/images/*ubuntu-xenial-16.04-amd64-server-*", 12 | "root-device-type": "ebs" 13 | }, 14 | "owners": ["099720109477"], 15 | "most_recent": true 16 | }, 17 | "ssh_username": "ubuntu", 18 | "ssh_interface": "session_manager", 19 | "communicator": "ssh", 20 | "iam_instance_profile": "{{user `iam_instance_profile`}}" 21 | } 22 | ], 23 | "provisioners": [ 24 | { 25 | "type": "shell", 26 | "inline": [ 27 | "echo Connected via SSM at '{{build `User`}}@{{build `Host`}}:{{build `Port`}}'" 28 | ] 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /packer/aws-ami-v6.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "aws_region": "us-west-2", 4 | "aws_ami_name": "CentOS-7-HVM-EBS-{{timestamp}}", 5 | "source_ami": "ami-0c4f7023847b90238" 6 | }, 7 | "builders": [ 8 | { 9 | "type": "amazon-ebs", 10 | "region": "eu-west-2", 11 | "source_ami": "{{user `source_ami`}}", 12 | "instance_type": "t2.medium", 13 | "ssh_username": "ec2-user", 14 | "ami_name": ", 15 | "ami_users": ["",""], 16 | "vpc_filter": { 17 | "filters": { 18 | "tag:Name": "My App VPC", 19 | "isDefault": "false" 20 | } 21 | }, 22 | "subnet_filter": { 23 | "filters": { 24 | "state": "available" 25 | }, 26 | "most_free": true, 27 | "random": true 28 | } 29 | } 30 | ], 31 | "provisioners": [ 32 | { 33 | "type": "shell", 34 | "inline": [ 35 | "echo Connected via SSM at '{{build `User`}}@{{build `Host`}}:{{build `Port`}}'" 36 | ] 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /packer/aws-ami-v7.json: -------------------------------------------------------------------------------- 1 | # credit https://medium.com/tide-engineering-team/building-your-gold-amis-using-packer-d3248736b3d8 2 | { 3 | "variables": { 4 | "aws_access_key": "${aws_access_key}", 5 | "aws_secret_key": "${aws_secret_key}", 6 | "vpc_region": "${vpc_region}", 7 | "instance_type": "${instance_type}", 8 | "ssh_username": "${ssh_username}" 9 | }, 10 | "builders": [ 11 | { 12 | "type": "amazon-ebs", 13 | "access_key": "{{user `aws_access_key`}}", 14 | "secret_key": "{{user `aws_secret_key`}}", 15 | "region": "{{user `vpc_region`}}", 16 | "vpc_filter": { 17 | "filters": { 18 | "tag:Name": "AKTestVPC", 19 | "isDefault": "false" 20 | } 21 | }, 22 | "subnet_filter": { 23 | "filters": { 24 | "tag:Name": "TestAKSUbnet" 25 | } 26 | }, 27 | "associate_public_ip_address": true, 28 | "security_group_filter": { 29 | "filters": { 30 | "tag:Name": "AKTestSG" 31 | } 32 | }, 33 | "source_ami_filter": { 34 | "filters": { 35 | "virtualization-type": "hvm", 36 | "name": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*", 37 | "root-device-type": "ebs" 38 | }, 39 | "owners": [ 40 | "679593333241" 41 | ], 42 | "most_recent": true 43 | }, 44 | "instance_type": "{{user `instance_type`}}", 45 | "ssh_username": "{{user `ssh_username`}}", 46 | "ami_name": "gold-ami-ubuntu_18.04_v1-{{isotime | clean_resource_name}}", 47 | } 48 | ], 49 | "provisioners": [ 50 | { 51 | "type": "shell", 52 | "execute_command": "echo 'packer' | {{.Vars}} sudo -S -E bash '{{.Path}}'", 53 | "script": "./scripts/Inspector_Agent_Install.sh" 54 | } 55 | ] 56 | } 57 | -------------------------------------------------------------------------------- /packer/provision.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | sudo apt-get -y update 6 | -------------------------------------------------------------------------------- /packer/readme.md: -------------------------------------------------------------------------------- 1 | It is assumed that for every examples 'aws configure' is done. 2 | Passing aws creds in script/template is not a good practise. 3 | 4 | Reefrence https://www.packer.io/plugins/builders/amazon/ebs 5 | 6 | # Packer main repository [click here](https://github.com/e2eSolutionArchitect/hashicorp-packer/tree/main) 7 | -------------------------------------------------------------------------------- /pipelines/azure/angular-nginx-dkr-ecr-adopipeline.yml: -------------------------------------------------------------------------------- 1 | name: angular-nginx-dkr-ecr-adopipeline 2 | 3 | # This Azure pipeline build a docker image, connects to AWS, push docker image to AWS ECR. 4 | # This pipeline is used to build an Angular app then create docker image and push to ECR 5 | # Make sure you have 'Dockerfile' availabale in same directory 6 | 7 | pool: 8 | name: ADO-pool-01 9 | 10 | parameters: 11 | - name: AWSRegion 12 | default: 'us-east-1' 13 | - name: AWSServiceConnection 14 | default: 'ado-aws-service-con' 15 | - name: AWSAccountId 16 | default: '##########' 17 | - name: AWSECRRepository 18 | default: 'mywebapp' 19 | 20 | stages: 21 | - stage: build 22 | displayName: 'Build Image' 23 | jobs: 24 | - job: Build_Image 25 | displayName: 'Build Image' 26 | steps: 27 | - script: | 28 | npm install 29 | ng build 30 | displayName: 'NPM install and Build' 31 | - task: AWSShellScript@1 32 | displayName: 'Build and Push Docker Image to AWS ECR' 33 | inputs: 34 | awsCredentials: ${{paramaters.AWSServiceConnection}} 35 | regionName: ${{paramaters.AWSRegion}} 36 | scriptType: 'inline' 37 | inlineScript: | 38 | echo 'Connect to ECR' 39 | aws ecr get-login-password --region ${{paramaters.AWSRegion}} | docker login --username AWS --password-stdin ${{paramaters.AWSAccountId}}.dkr.ecr.${{paramaters.AWSRegion}}.amazonaws.com 40 | echo 'build image' 41 | docker build -t ${{paramaters.AWSECRRepository}} . 42 | echo 'tag image' 43 | docker tag ${{paramaters.AWSECRRepository}}:latest ${{paramaters.AWSAccountId}}.dkr.ecr.${{paramaters.AWSRegion}}.amazonaws.com/${{paramaters.AWSECRRepository}}:latest 44 | echo 'push to ECR' 45 | docker push ${{paramaters.AWSAccountId}}.dkr.ecr.${{paramaters.AWSRegion}}.amazonaws.com/${{paramaters.AWSECRRepository}}:latest 46 | echo 'image pushed successfully' 47 | -------------------------------------------------------------------------------- /pipelines/azure/docker-build-n-push-to-ecr-azurepipeline.yml: -------------------------------------------------------------------------------- 1 | name: docker-build-n-push-to-ecr-azurepipeline 2 | 3 | # This Azure pipeline build a docker image, connects to AWS, push docker image to AWS ECR. This is a generic ADO pipeline for build and push any docker image to AWS ECR. 4 | # Make sure you have 'Dockerfile' availabale in same directory 5 | 6 | pool: 7 | name: ADO-pool-01 8 | 9 | parameters: 10 | - name: AWSRegion 11 | default: 'us-east-1' 12 | - name: AWSServiceConnection 13 | default: 'ado-aws-service-con' 14 | - name: AWSAccountId 15 | default: '##########' 16 | - name: AWSECRRepository 17 | default: 'mywebapp' 18 | 19 | stages: 20 | - stage: build 21 | displayName: 'Build Image' 22 | jobs: 23 | - job: Build_Image 24 | displayName: 'Build Image' 25 | steps: 26 | - task: AWSShellScript@1 27 | displayName: 'Build and Push Docker Image to AWS ECR' 28 | inputs: 29 | awsCredentials: ${{paramaters.AWSServiceConnection}} 30 | regionName: ${{paramaters.AWSRegion}} 31 | scriptType: 'inline' 32 | inlineScript: | 33 | echo 'Connect to ECR' 34 | aws ecr get-login-password --region ${{paramaters.AWSRegion}} | docker login --username AWS --password-stdin ${{paramaters.AWSAccountId}}.dkr.ecr.${{paramaters.AWSRegion}}.amazonaws.com 35 | echo 'build image' 36 | docker build -t ${{paramaters.AWSECRRepository}} . 37 | echo 'tag image' 38 | docker tag ${{paramaters.AWSECRRepository}}:latest ${{paramaters.AWSAccountId}}.dkr.ecr.${{paramaters.AWSRegion}}.amazonaws.com/${{paramaters.AWSECRRepository}}:latest 39 | echo 'push to ECR' 40 | docker push ${{paramaters.AWSAccountId}}.dkr.ecr.${{paramaters.AWSRegion}}.amazonaws.com/${{paramaters.AWSECRRepository}}:latest 41 | echo 'image pushed successfully' 42 | -------------------------------------------------------------------------------- /python/anaconda-commands.md: -------------------------------------------------------------------------------- 1 | 2 | Update conda. Please [refer here](https://docs.anaconda.com/anaconda/install/update-version/) for the latest instructions. 3 | 4 | 5 | ``` 6 | update the conda package manager to the latest version. 7 | 8 | conda update conda 9 | 10 | use conda to update Anaconda to the latest version. 11 | 12 | conda update anaconda 13 | 14 | Update Python Packages 15 | 16 | conda update python 17 | 18 | ``` 19 | -------------------------------------------------------------------------------- /python/analyze-csv-pandas-lambda.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | import base64 4 | import pandas as pd 5 | 6 | def lambda_handler(event, context): 7 | 8 | s3_client =boto3.client('s3') 9 | s3_bucket="s3fileupload" 10 | # print the event to find the variables to consider for body or other attributes. 11 | print(json.dumps(event)) 12 | file_content=event["body-json"] 13 | content_decoded=base64.b64decode(file_content) 14 | # through your client POST request if you are sending an attribute for holding filename 'filename' then it will be under 'params' section. 15 | # Note: you have to send a custom attribute to send file name like 'filename'. file name is not available by default. 16 | # secondly note that, API gateway > API > Integration request > Mapping request > select method request passthrough 17 | filename=event["params"]["header"]["filename"] 18 | s3_upload =s3_client.put_object(Bucket=s3_bucket, Key=filename, Body=content_decoded) 19 | 20 | 21 | 22 | #bucket_name = event["Records"][0]["s3"]["bucket"]["name"] 23 | #s3_file_name = event["content"][0]["s3"]["object"]["key"] 24 | #print(s3_file_name) 25 | resp = s3_client.get_object(Bucket=s3_bucket, Key=filename) 26 | 27 | df = pd.read_csv(resp['Body'], sep=',') 28 | #print(df.info()) 29 | 30 | return { 31 | 'statusCode': 200, 32 | 'body': json.dumps('worked') 33 | } 34 | -------------------------------------------------------------------------------- /python/create-conda-environment.md: -------------------------------------------------------------------------------- 1 | #### I assume Anaconda is already installed. 2 | 3 | ## 1. Open Anaconda command prompt 4 | ## 2. Check number of existing environment 5 | run >> conda env list 6 | 7 | output should look like below 8 | (base) C:\Users\...>conda env list 9 | base * C:\ProgramData\Anaconda3 10 | 11 | It means only one env 'base' exist. 12 | 13 | ## 3. check python version 14 | run >> python --version 15 | output will look like 'Python 3.7.6' 16 | 17 | ## 4. Now create a new environment. Example env name 'py37' 18 | run >> conda create --name py37 python=3.7 19 | output will look like below 20 | Collecting package metadata (current_repodata.json): done 21 | Solving environment: done 22 | 23 | ## 5. Activate the environment 24 | run >> conda activate py37 25 | 26 | ## 6. Clone conda environment to another environment in same system 27 | 28 | conda create --name new_env --clone existing_env 29 | -------------------------------------------------------------------------------- /python/cuda-install.md: -------------------------------------------------------------------------------- 1 | 2 | Please refer here for Pytorch CUDE installation - https://pytorch.org/get-started/locally/ 3 | 4 | ``` 5 | pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128 6 | 7 | OR 8 | 9 | pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128 10 | ``` 11 | -------------------------------------------------------------------------------- /python/import-pandas-aws-lambda.py: -------------------------------------------------------------------------------- 1 | # Follow the instruction to add layers in aws lambda to import pandas lib 2 | # Watch this tutorial https://youtu.be/lrEAu75zhNI 3 | # Pundas lib is not available in aws lambda by default. 4 | 5 | import pandas as pd 6 | import json 7 | 8 | def lambda_handler(event, context): 9 | a = [5, 7, 4, 9] 10 | srs = pd.Series(a) 11 | 12 | print(srs) 13 | 14 | return { 15 | 'statusCode': 200, 16 | 'body': json.dumps('imported pandas') 17 | } 18 | -------------------------------------------------------------------------------- /python/install-geopandas.md: -------------------------------------------------------------------------------- 1 | ### Install geopandas in Anaconda distribution. 2 | #### I assume Anaconda is already installed. 3 | 4 | 1. Open Anaconda command prompt 5 | 2. Check number of existing environment 6 | run >> conda env list 7 | 8 | output should look like below 9 | (base) C:\Users\...>conda env list 10 | base * C:\ProgramData\Anaconda3 11 | 12 | It means only one env 'base' exist. 13 | 14 | 3. check python version 15 | run >> python --version 16 | output will look like 'Python 3.7.6' 17 | 18 | 4. Now create a new environment. Example env name 'py37' 19 | run >> conda create --name py37 python=3.7 20 | output will look like below 21 | Collecting package metadata (current_repodata.json): done 22 | Solving environment: done 23 | 24 | 5. Activate the environment 25 | run >> conda activate py37 26 | 27 | 6. run below command to install geopandas 28 | conda install geopandas 29 | Note: It may take few mins to process. So be patient 30 | It will listup all supporting packages. 31 | 32 | Ultimately it will show "The following NEW packages will be INSTALLED:" followed by hundreds of packages. Just enter Y when it asks to proceed or not. 33 | 34 | During installation it may show "solving environment: failed with initial frozen solve. retrying with flexible solve. geopandas". Give it time to complete the process. 35 | -------------------------------------------------------------------------------- /python/install-plotly.md: -------------------------------------------------------------------------------- 1 | ### No module named 'plotly.express' 2 | 3 | Simply running 4 | 5 | conda install plotly 6 | 7 | didnt resolve my plotly.express error. 8 | 9 | If you are using conda just run below command. 10 | 11 | conda install -c plotly plotly=4.8.1 12 | 13 | Note: Run this in other than 'base' environment. 14 | 15 | if you are at base env. change to another environment like 16 | 17 | conda activate 18 | 19 | then run plotly install command. 20 | -------------------------------------------------------------------------------- /python/mysql-connector-python.md: -------------------------------------------------------------------------------- 1 | 2 | pip install mysql-connector-python 3 | pip install Flask-SQLAlchemy 4 | 5 | OR if Anaconda is installed then. install it in a custom conda environment. not in default. 6 | 7 | conda install mysql-connector-python 8 | conda install Flask-SQLAlchemy 9 | -------------------------------------------------------------------------------- /python/python-call-rest-api.py: -------------------------------------------------------------------------------- 1 | # GET call 2 | 3 | -------------------------- 4 | # Option 1 5 | 6 | import requests 7 | api_url = "https://lnvajpvyae.execute-api.us-east-1.amazonaws.com/prod/message" 8 | response = requests.get(api_url) 9 | response.json() 10 | 11 | -------------------------- 12 | 13 | # Option 2 14 | 15 | import requests 16 | import json 17 | import time 18 | 19 | responses = list() 20 | 21 | for i in range(50): 22 | time.sleep(5) # Sleep for 3 seconds 23 | print(i) 24 | api_url = "https://o9mhe0mj74.execute-api.us-east-1.amazonaws.com/prod/message" 25 | responses = requests.get(api_url) 26 | data=json.loads(responses.text) 27 | print(data) 28 | 29 | -------------------------- 30 | -------------------------------------------------------------------------------- /python/python-commands.md: -------------------------------------------------------------------------------- 1 | 2 | ``` 3 | python3 -V 4 | python -version 5 | 6 | python3 7 | python3 demo.py 8 | ``` 9 | 10 | **Create Virtual Python Environment using Windows command prompt** 11 | 12 | ``` 13 | python --version 14 | # OR 15 | python3 -V 16 | 17 | C:\ mkdir pyenv 18 | C:\ cd pyenv 19 | C:\pyenv> python -m venv 20 | C:\pyenv> python -m venv py310 21 | C:\pyenv> .\py310\Scripts\activate 22 | (py310) c:\pyenv> pip install jupyter 23 | (py310) c:\pyenv> jupyter notebook 24 | 25 | # Similarly install any other libs you need. 26 | pip install monai==1.3.0 torch==2.0.1 pydicom==2.4.4 opencv-python==4.8.0.76 numpy==1.23.5 27 | ``` 28 | 29 | ***list python packages*** 30 | 31 | ``` 32 | pip list 33 | ``` 34 | 35 | **Check anaconda env list 36 | 37 | ``` 38 | conda env list 39 | ** activate a conda env 40 | conda activate env_name 41 | ``` 42 | 43 | ** Remove : 44 | ``` 45 | conda remove --name myenv --all 46 | ``` 47 | 48 | ***To create an environment with a specific version of Python*** 49 | ``` 50 | conda create --name myenv python=3.7 51 | ``` 52 | OR 53 | ``` 54 | conda create --name myenv 55 | ``` 56 | 57 | ***Clone*** 58 | ``` 59 | conda create --name new_env --clone myenv 60 | ``` 61 | -------------------------------------------------------------------------------- /python/troubleshoot-python.md: -------------------------------------------------------------------------------- 1 | 2 | ## pyarrow or fastparquet is required for parquet support 3 | ``` 4 | pip install pyarrow 5 | pip install fastparquet 6 | ``` 7 | -------------------------------------------------------------------------------- /terraform/readme.md: -------------------------------------------------------------------------------- 1 | Terraform code for AWS, Azure, GCP [click here](https://github.com/e2eSolutionArchitect/terraform) 2 | HashiCorp Packer code repository [click here](https://github.com/e2eSolutionArchitect/hashicorp-packer) 3 | -------------------------------------------------------------------------------- /terraform/run-terraform-offline.md: -------------------------------------------------------------------------------- 1 | 2 | Step by Step guide https://somspeaks.com/terraform-offline-setup-and-initialization-in-the-windows-system/ 3 | 4 | • Terraform OFFLINE setup and initialization in Windows system (this article) 5 | • Terraform OFFLINE setup and initialization in Unix system on docker (coming soon) 6 | 7 | ## Run Terraform offline 8 | 9 | ## Step 1: (Using PowerShell, if you are using anything else just to below instructions accordingly) 10 | Create a directory in your file system. suppose creating "tf_cache" directory in user home ($HOME) 11 | ``` 12 | mkdir "$HOME/tf_cache" 13 | ``` 14 | 15 | Now simply copy the "registry.terraform.io" folder here. (If you are thinking where will you get it!! While you are conencted to internet. simply run terraform init. it will download .terraform in your working directory. copy "registry.terraform.io" from there and disconnect your internet to make sure below configuration is helping you to run terraform init henceforth without internet connection ) 16 | 17 | ## Step 2: 18 | Create this file inside above directory. 19 | For windows, Create a file "terraform.rc", if other OS then create ".terraformrc". Dont miss the '.' and the file should NOT be like "terraform.rc.txt" 20 | 21 | ## Step 3: 22 | Update the file terraform.rc or .terraformrc with below code block 23 | 24 | ``` 25 | provider_installation { 26 | filesystem_mirror { 27 | path = "path/to/the/new/directory" # "c:/Users//tf_cache" 28 | include = ["registry.terraform.io/hashicorp/*"] 29 | } 30 | direct { 31 | exclude = ["registry.terraform.io/hashicorp/*"] 32 | } 33 | } 34 | 35 | plugin_cache_dir = "path/to/the/new/directory" # "c:/Users//tf_cache" 36 | disable_checkpoint=true 37 | ``` 38 | 39 | ## Step 4: 40 | Setup env variables as follows 41 | 42 | ``` 43 | $env:TF_PLUGIN_CACHE_DIR="$HOME/tf_cache" 44 | $env:TF_CLI_CONFIG_FILE="$HOME/tf_cache/terraform.rc" # terraform.rc for windows, .terraformrc for linux. eg. "c:/Users//tf_cache/terraform.rc" 45 | ``` 46 | 47 | ## Step 5: 48 | create a directory to run your terraform code 49 | simply create a file main.tf 50 | write below code 51 | 52 | ``` 53 | terraform { 54 | required_version ="~>1.1.8" 55 | required_providers { 56 | aws = { 57 | source = "hashicorp/aws" 58 | version = "~> 4.5.0" 59 | } 60 | } 61 | } 62 | ``` 63 | 64 | ## Step 6: 65 | This is GAME TIME !! 66 | Make sure you have your internet disconnected 67 | and run terraform init in your working directory. 68 | If you have followed ablove steps properly, terraform init will pull the package from "tf_cache" to your working directory. It will NOT look for pulling from internet terraform registry. 69 | 70 | A video tutorial will be published shortly here https://www.youtube.com/channel/UC5Juuk7aTvbRmrABMq4onJA/videos 71 | if interested join us at https://e2esolutionarchitect.com/ 72 | 73 | -------------------------------------------------------------------------------- /tomcat/install-tomcat.md: -------------------------------------------------------------------------------- 1 | Install Java first [click here](https://github.com/e2eSolutionArchitect/scripts/blob/main/java/java-install.md) 2 | 3 | Install Tomcat v10.0.20 4 | ``` 5 | mkdir tomcat 6 | cd tomcat 7 | wget https://dlcdn.apache.org/tomcat/tomcat-10/v10.0.20/bin/apache-tomcat-10.0.20.tar.gz 8 | sudo tar xzvf apache-tomcat-10*tar.gz -C /opt/tomcat --strip-components=1 9 | ``` 10 | 11 | Create a user for Tomcat 12 | ``` 13 | sudo useradd -m -d /opt/tomcat -U -s /bin/false tomcat 14 | 15 | # grant ownership the to the new user 16 | sudo chown -R tomcat:tomcat /opt/tomcat/ 17 | sudo chmod -R u+x /opt/tomcat/bin 18 | ``` 19 | Configure Admin user 20 | ``` 21 | sudo nano /opt/tomcat/conf/tomcat-users.xml 22 | ``` 23 | Update passwords 'manager_password' and 'admin_password' 24 | ``` 25 | 26 | 27 | 28 | 29 | 30 | ``` 31 | Remove the restriction for the Manager page, open its config file for editing 32 | ``` 33 | sudo nano /opt/tomcat/webapps/manager/META-INF/context.xml 34 | ``` 35 | Comment out the Valve definition, as shown 36 | ``` 37 | ... 38 | 39 | 41 | 43 | https://docs.bitnami.com/aws/faq/get-started/find-credentials/ 7 | 8 | 9 | # Step: Configure AWS credential in wordpress wp-config.php 10 | -------- 11 | ATTENTION: any misconfiguration or type in this step can break your wordpress website. make sure you keep a back copy of wp-config.php before you start editing as mentioned below. 12 | if anything breaks just replace with your backup copy and it will be fine, 13 | -------- 14 | Add either Code 1 or 2 in wp-config.php before the line which says /* That's all, stop editing! Happy publishing. */ 15 | Refer the video tutorial 16 | 17 | Code 1: 18 | define('AS3CF_AWS_ACCESS_KEY_ID','*****************'); 19 | 20 | define('AS3CF_AWS_SECRET_ACCESS_KEY','*****************'); 21 | 22 | 23 | Code 2 2: 24 | define( 'AS3CF_SETTINGS', serialize( array( 25 | 'provider' => 'aws', 26 | 'access-key-id' => '*****************', 27 | 'secret-access-key' => '**********************************', 28 | ) ) ); 29 | 30 | 31 | Setp : Add S3 bucket policy 32 | 33 | { 34 | "Version": "2012-10-17", 35 | "Id": "PolicyReadS3", 36 | "Statement": [ 37 | { 38 | "Sid": "1", 39 | "Effect": "Allow", 40 | "Principal": "*", 41 | "Action": "s3:GetObject", 42 | "Resource": "arn:aws:s3:::bucketname/*" 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /wsl/install-wsl.md: -------------------------------------------------------------------------------- 1 | # Windows Subsystem for Linux 2 | 3 | Install WSL in Windows PC 4 | ``` 5 | wsl --install -d Ubuntu-22.04 6 | ``` 7 | Reboot your system 8 | 9 | Check WSL version 10 | ``` 11 | wsl --list --verbose 12 | ``` 13 | 14 | Change version 15 | 16 | ``` 17 | wsl --set-default-version 2 18 | ``` 19 | --------------------------------------------------------------------------------