├── Assets ├── crt.sh.png ├── hunter.png ├── havester.png ├── build_with.png ├── knockmail.png ├── phonebook.png ├── wappalyzer.png ├── email_checker.png └── securitytrails.png ├── GitHubPreview.png ├── Script ├── README.md └── SubdomainScript │ ├── README.md │ └── subdomain_enum.sh ├── Email Address ├── Verify Email Address │ └── README.md └── Discover Email Address │ └── README.md ├── Directory Bruteforce Tools └── README.md ├── Identify Web Technology └── README.md ├── Search Engine └── README.md ├── Intrusion Detection System ├── readme.md └── iot-intrusion-detection.ipynb ├── Free Learning Resources ├── Youtube Channel │ └── README.md ├── Cyber Security News │ └── README.md ├── CTF Challenge Resources │ └── README.md └── Cyber Security Courses │ └── README.md ├── LICENSE ├── Sub Domain Enumeration └── README.md ├── Password Attacking Tools └── README.md ├── Network Scanner └── README.md ├── CONTRIBUTING.md ├── Attacks └── README.md ├── README.md └── CODE_OF_CONDUCT.md /Assets/crt.sh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/crt.sh.png -------------------------------------------------------------------------------- /Assets/hunter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/hunter.png -------------------------------------------------------------------------------- /GitHubPreview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/GitHubPreview.png -------------------------------------------------------------------------------- /Assets/havester.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/havester.png -------------------------------------------------------------------------------- /Assets/build_with.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/build_with.png -------------------------------------------------------------------------------- /Assets/knockmail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/knockmail.png -------------------------------------------------------------------------------- /Assets/phonebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/phonebook.png -------------------------------------------------------------------------------- /Assets/wappalyzer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/wappalyzer.png -------------------------------------------------------------------------------- /Assets/email_checker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/email_checker.png -------------------------------------------------------------------------------- /Assets/securitytrails.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Prashant-Bhapkar/Awesome-Security-Resources/HEAD/Assets/securitytrails.png -------------------------------------------------------------------------------- /Script/README.md: -------------------------------------------------------------------------------- 1 | ## Script 2 | 3 | - [Subdomain enumeration Automation](/SubdomainScript/) - Run this scirpt and find your Subdomain 4 | -------------------------------------------------------------------------------- /Email Address/Verify Email Address/README.md: -------------------------------------------------------------------------------- 1 | ## Verify Email Addresses 2 | 3 | - [Email-Checker.net](https://email-checker.net/) 4 | 5 | ![Screenshot](/Assets/email_checker.png) 6 | -------------------------------------------------------------------------------- /Directory Bruteforce Tools/README.md: -------------------------------------------------------------------------------- 1 | ## Directory Bruteforce Tools 2 | 3 | - [Gobuster](https://www.kali.org/tools/gobuster/) 4 | - [DirBuster](https://www.kali.org/tools/dirbuster/) 5 | - [Dirsearch](https://github.com/maurosoria/dirsearch) 6 | -------------------------------------------------------------------------------- /Identify Web Technology/README.md: -------------------------------------------------------------------------------- 1 | ## **Identify Web Technology** 2 | 3 | - [BuildWith.com](https://builtwith.com/) 4 | 5 | ![Screenshot](/Assets/build_with.png) 6 | 7 | - [Wappalyzer (Firefox extension) ](https://addons.mozilla.org/en-US/firefox/addon/wappalyzer/) 8 | 9 | ![Screenshot](/Assets/wappalyzer.png) 10 | 11 | -------------------------------------------------------------------------------- /Search Engine/README.md: -------------------------------------------------------------------------------- 1 | # Search Engine 2 | 3 | - [Dehashed](https://www.dehashed.com/) - View leaked credentials. 4 | 5 | - [SecurityTrails](https://securitytrails.com/) - Extensive DNS data. 6 | 7 | - [Uncover](https://github.com/projectdiscovery/uncover) - uncover is a go wrapper using APIs of well known search engines to quickly discover exposed hosts on the internet. 8 | -------------------------------------------------------------------------------- /Email Address/Discover Email Address/README.md: -------------------------------------------------------------------------------- 1 | ## Discover Email Addresses 2 | 3 | - [Hunter.io](https://hunter.io/) 4 | 5 | ![Screenshot](/Assets/hunter.png) 6 | 7 | - [KnockMail](https://github.com/4w4k3/KnockMail) 8 | 9 | ![Screenshot](/Assets/knockmail.png) 10 | 11 | - [Phonebook.cz](https://phonebook.cz/) 12 | 13 | ![Screenshot](/Assets/phonebook.png) 14 | 15 | - [TheHarvester](https://github.com/laramies/theHarvester) 16 | 17 | ![Screenshot](/Assets/havester.png) 18 | -------------------------------------------------------------------------------- /Intrusion Detection System/readme.md: -------------------------------------------------------------------------------- 1 | In this project, we have trained the model over the N-BaIoT dataset,In order to detect intrusion, scrape the features of network, 116 features in total. Refer the link https://archive.ics.uci.edu/ml/datasets/detection_of_IoT_botnet_attacks_N_BaIoT# to understand the features in the dataset. 2 | Simply store the extracted attributes as a vector,in the corresponding order, and test on the trained model, it will classify the intruded network as malignant and the safe network as benign. -------------------------------------------------------------------------------- /Free Learning Resources/Youtube Channel/README.md: -------------------------------------------------------------------------------- 1 | ## Best youtube Channel resources 2 | 3 | 1. [Spin the hack](https://www.youtube.com/c/SpinTheHack) 4 | 2. [HackerSploit](https://www.youtube.com/c/HackerSploit) 5 | 3. [John Hammond](https://www.youtube.com/c/JohnHammond010) 6 | 4. [NetworkChuck](https://www.youtube.com/c/NetworkChuck) 7 | 5. [intigriti](https://www.youtube.com/c/intigriti) 8 | 6. [Farah Hawa](https://www.youtube.com/c/FarahHawa) 9 | 7. [IppSec](https://www.youtube.com/c/ippsec) 10 | 8. [STÖK](https://www.youtube.com/c/STOKfredrik) 11 | 9. [Nahamsec](https://www.youtube.com/c/Nahamsec) 12 | 10. [Bug Bounty Reports Explained](https://www.youtube.com/c/BugBountyReportsExplained) 13 | -------------------------------------------------------------------------------- /Script/SubdomainScript/README.md: -------------------------------------------------------------------------------- 1 | ## About Script 2 | Automated script for subdomain enumeration 3 | Script uses tools `assetfinder`, `amass`, `subfinder` tools to enumerate subdomains and `httpx` to filter. 4 | 5 | ### Preinstalled tools in your system 6 | - assetfinder 7 | - amass 8 | - subfinder 9 | ## Edit path in subdomain_enum.sh 10 | Enter any path `line 9` of your system to get output 11 | ```bash 12 | path='any/path'$Project 13 | ``` 14 | ## Run script 15 | 16 | ```bash 17 | chmod +x subdomain_enum.sh 18 | ./subdomain_enum.sh 19 | ``` 20 | 21 | ## Script runnig 22 | 23 | ```bash 24 | Enter the name of Domain 25 | [enter domain to enumerate e.g: evil.com] 26 | Enter the Project name 27 | [Enter the name of dir in which output shown] 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Prashant Bhapkar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Free Learning Resources/Cyber Security News/README.md: -------------------------------------------------------------------------------- 1 | # Cyber Security News (Sites, Blogs, Feeds) 2 | 3 | - [CISO Series](https://cisoseries.com/) 4 | 5 | - [The Daily Swig](https://lnkd.in/eF8ixh46) 6 | 7 | - [Threatpost](https://threatpost.com/) 8 | 9 | - [CyberSecurityNews](https://lnkd.in/evjYmhhm) 10 | 11 | - [CyWare](https://lnkd.in/e6bMxSbp) 12 | 13 | - [Schneier](https://www.schneier.com/) 14 | 15 | - [SANS Newsbites](https://lnkd.in/e-ed97KU) 16 | 17 | - [TheHackerNews](https://thehackernews.com/) 18 | 19 | - [SANS ISC](https://isc.sans.edu/) 20 | 21 | - [BleepingComputer](https://lnkd.in/eAQKdrrA) 22 | 23 | - [Malwarebytes](https://lnkd.in/etwyTgpg) 24 | 25 | - [CSO Online](https://www.csoonline.com/) 26 | 27 | - [CyberScoop](https://lnkd.in/eHgYe-wx) 28 | 29 | - [Dark Reading](https://lnkd.in/eeVDQWBv) 30 | 31 | - [Infosecurity](https://lnkd.in/e5ReHVK5) 32 | 33 | - [Krebs on Security](https://lnkd.in/eHbSJK8U) 34 | 35 | - [Naked Security](https://lnkd.in/e_xuezBK) 36 | 37 | - [Talos](https://lnkd.in/e47s62Qp) 38 | 39 | - [The Record](https://therecord.media/) 40 | 41 | - [Sophos News](https://lnkd.in/epp_Rqg8) 42 | 43 | - [The DFIR Report](https://thedfirreport.com/) 44 | -------------------------------------------------------------------------------- /Free Learning Resources/CTF Challenge Resources/README.md: -------------------------------------------------------------------------------- 1 | # CTF Challenges Resources 2 | 3 | - [Ctftime](https://ctftime.org/) - The de facto website for everything CTF related. 4 | - [Defend the Web](https://defendtheweb.net/) - Defend the Web is an interactive security platform where you can learn and challenge your skills. 5 | - [Dvwa](http://www.dvwa.co.uk/) - Damn Vulnerable Web Application is another deliberately insecure web application to practice your skills on. 6 | - [HackTheBox](https://www.hackthebox.eu/) - An online platform to test and advance your skills in penetration testing and cyber security. 7 | - [Hackthissite](https://www.hackthissite.org/) - A site which provides challenges, CTFs, and more to improve your hacking skills. 8 | - [Itsecgames](http://www.itsecgames.com/) - bWAPP or buggy web app is a deliberately insecure web application. 9 | - [Overthewire](http://overthewire.org/wargames/) - Learn and practice security concepts in the form of fun-filled games. 10 | - [PicoCTF](https://picoctf.org/) - Provides you with fun CTF challenges of varying levels of difficulty to practice on. 11 | - [Root-me](https://www.root-me.org/) - Another website which hosts challenges to test your hacking skills. 12 | - [TryHackMe](https://tryhackme.com/) - TryHackMe is a free online platform for learning cyber security, using hands-on exercises and labs. 13 | - [Vulnhub](https://www.vulnhub.com/) - Has a lot of VMs to play with. Some are beginner friendly, some aren't. 14 | -------------------------------------------------------------------------------- /Sub Domain Enumeration/README.md: -------------------------------------------------------------------------------- 1 | # Sub Domain Enumeration 2 | 3 | - [Altdns](https://github.com/infosec-au/altdns) - Altdns is a security tool to discover subdomains by [@infosec-au](https://github.com/infosec-au) 4 | 5 | - [Amass](https://github.com/OWASP/Amass) - We can also find subdomain using Amass by [@OWASP](https://github.com/OWASP/Amass) 6 | 7 | - [AssetFinder](https://github.com/tomnomnom/assetfinder) - Find domains and subdomains potentially related to a given domain by [@tomnomnom](https://github.com/tomnomnom) 8 | 9 | - [CRT.sh](https://crt.sh/) 10 | 11 | ![Screenshot](/Assets/crt.sh.png) 12 | 13 | - [Domain](https://github.com/jhaddix/domain) - It allows to use the two tool one multiple domains within the same session, combine two tools Recon-ng and altdns by [@Jhaddix](https://github.com/jhaddix) 14 | 15 | - [SecurityTrails](https://securitytrails.com/) 16 | 17 | ![Screenshot](/Assets/securitytrails.png) 18 | 19 | - [SubFinder](https://github.com/projectdiscovery/subfinder) - Subfinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. 20 | [@ProjectDiscovery](https://github.com/projectdiscovery) 21 | 22 | - [Sublist3r](https://github.com/aboul3la/Sublist3r) - Sublist3r is a multi-threaded sub-domain enumeration tool for penetration testers by [@aboul3la](https://github.com/aboul3la) 23 | 24 | - [Subdomain enumeration Automation](/Script/SubdomainScript/) - Run this scirpt and find your Subdomain 25 | -------------------------------------------------------------------------------- /Password Attacking Tools/README.md: -------------------------------------------------------------------------------- 1 | # Password Attacking Tools 2 | 3 | - [Hashcat](https://github.com/hashcat/hashcat) - hashcat is the world's fastest and most advanced password recovery utility, supporting five unique modes of attack for over 300 highly-optimized hashing algorithms. hashcat currently supports CPUs, GPUs, and other hardware accelerators on Linux, Windows, and macOS, and has facilities to help enable distributed password cracking. 4 | 5 | - [Hydra](https://github.com/vanhauser-thc/thc-hydra) - Can't even emphasize how strong this tool is, just visit the github repo and look out for yourself 6 | 7 | - [John the Ripper](https://github.com/openwall/john) - John the Ripper is a fast password cracker, currently available for many flavors of Unix, macOS, Windows, DOS, BeOS, and OpenVMS (the latter requires a contributed patch). Its primary purpose is to detect weak Unix passwords. 8 | 9 | - [Ncrack](https://github.com/nmap/ncrack) - Ncrack is a high-speed network authentication cracking tool. It was built to help companies secure their networks by proactively testing all their hosts and networking devices for poor passwords. Security professionals also rely on Ncrack when auditing their clients. 10 | 11 | - [Ophcrack](https://gitlab.com/objectifsecurite/ophcrack) - A windows password cracker based on the faster time-memory trade-off using rainbow tables. This is an evolution of the original ophcrack 1.0 developed at EPFL (http://lasecwww.epfl.ch/~oechslin/projects/ophcrack) Ophrack comes with a Qt Graphical User Interface which uns on Windows, Mac OS X as well as on Unix. 12 | -------------------------------------------------------------------------------- /Script/SubdomainScript/subdomain_enum.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo 'Enter the name of Domain' 3 | read domain 4 | alias echo="echo -e" 5 | echo 'Enter the Project name' 6 | read Project 7 | 8 | 9 | Path='enter path to get output eg: ~/home' 10 | 11 | # if [ !-d $Project ] 12 | # then 13 | mkdir $Path 14 | # fi 15 | 16 | echo "Running subfinder------------" 17 | subfinder -d $domain | tee $Path/subdomain.txt 18 | 19 | echo "Running assetfinder------------" 20 | assetfinder --subs-only $domain | tee -a $Path/subdomain.txt 21 | 22 | echo "Running amass------------------" 23 | 24 | 25 | amass enum --passive -d $domain | tee -a $Path/subdomain.txt 26 | 27 | echo "Searching for the 4 level domain" 28 | 29 | 30 | echo '---------------------------' 31 | 32 | echo "Subdomain collected--------" 33 | 34 | cat $Path/subdomain.txt | wc -l 35 | sort -u $Path/subdomain.txt >> $Path/final.txt 36 | # echo 37 | echo "Removed dublicate-----------" 38 | cat $Path/final.txt | wc -l 39 | echo '-------------------------------' 40 | 41 | echo 42 | 43 | # using httpx 44 | 45 | echo usign httpx 46 | 47 | cat $Path/final.txt | httpx -sc -mc 302,200,403,404 -title -o $Path/filtered.txt 48 | 49 | echo '----------------------------------' 50 | '' 51 | echo Total count `cat $Path/filtered.txt | wc -l` 52 | cat filtered.txt | grep 200 |tee status_200.txt 53 | echo 54 | 55 | 56 | # while domain in filtered.txt 57 | # do 58 | # bash ~/tools/4-ZERO-3/403-bypass.sh -u $domain --exploit >> ./temp/4-ZERO-3use.txt ; cat ./temp/4-ZERO-3use.txt | tee -a 403final.txt ; rm ./temp 59 | # done 60 | 61 | 62 | cd $Path 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /Free Learning Resources/Cyber Security Courses/README.md: -------------------------------------------------------------------------------- 1 | # Free Cyber Security Courses 2 | 3 | - [Certified Information Systems Security Professional at Charles Sturt University](https://itmasters.edu.au/free-short-course-cissp-updated/) 4 | - [Computer Networks at MIT](https://ocw.mit.edu/courses/6-829-computer-networks-fall-2002/) 5 | - [Cryptography I at Stanford](https://www.coursera.org/learn/crypto) 6 | - [CS50's Introduction to Computer Science (Harvard University)](https://www.edx.org/course/introduction-computer-science-harvardx-cs50x) 7 | - [CS50's Introduction to Programming with Python (Harvard University)](https://www.edx.org/course/cs50s-introduction-to-programming-with-python) 8 | - [Cybersecurity Fundamentals (Rochester Institute of Technology)](https://www.edx.org/course/cybersecurity-fundamentals) 9 | - [Cybersecurity Management at Charles Sturt University](https://itmasters.edu.au/free-short-course-cybersecurity-management/) 10 | - [Cybersecurity Risk Management (Rochester Institute of Technology)](https://www.edx.org/course/cybersecurity-risk-management) 11 | - [Hardware Security at University of Maryland](https://www.coursera.org/learn/hardware-security) 12 | - [Internet History, Technology, and Security at University of Michigan](https://online.umich.edu/courses/internet-history-technology-and-security/) 13 | - [Introduction to Cyber Attacks at NYU](https://www.classcentral.com/course/intro-cyber-attacks-9667) 14 | - [Introduction to Cyber security (University of Washington)](https://www.edx.org/course/introduction-to-cybersecurity) 15 | - [Network and Computer Security at MIT](https://ocw.mit.edu/courses/6-857-network-and-computer-security-spring-2014/) 16 | - [Network and Security Foundations at Western Governors University](https://www.edx.org/course/network-security-foundations) 17 | - [Network Security - Advanced Topics (New York University)](https://www.edx.org/course/network-security-advanced-topics) 18 | - [Network Security (Rochester Institute of Technology)](https://www.edx.org/course/network-security-2) 19 | - [Pen Testing at Charles Sturt University](https://itmasters.edu.au/free-short-course-pen-testing/) 20 | - [System Issues in Cloud Computing Specialization at Georgia Tech](https://www.coursera.org/specializations/cloud) 21 | -------------------------------------------------------------------------------- /Network Scanner/README.md: -------------------------------------------------------------------------------- 1 | # Network Scanner 2 | 3 | - [Advanced Ip Scanner](https://www.advanced-ip-scanner.com/) - It is a free and open-source network scanning tool that works in a Windows environment. It can detect and scan any device on a network including wireless devices.[@Video](https://www.youtube.com/watch?v=ahgOYjdDZiE&ab_channel=TheTechnology) 4 | 5 | - [Airbash](https://github.com/tehw0lf/airbash) - Airbash is a POSIX-compliant, fully automated WPA PSK PMKID and handshake capture script aimed at penetration testing. It is compatible with Bash and Android Shell (tested on Kali Linux and Cyanogenmod 10.2) and uses aircrack-ng to scan for clients that are currently connected to access points (AP). 6 | 7 | - [Bettercap](https://github.com/bettercap/bettercap) - bettercap is a powerful, easily extensible and portable framework written in Go which aims to offer to security researchers, red teamers and reverse engineers an easy to use, all-in-one solution with all the features they might possibly need for performing reconnaissance and attacking WiFi networks, Bluetooth Low Energy devices, wireless HID devices and Ethernet networks. 8 | 9 | - [Metasploit Framework](https://github.com/rapid7/metasploit-framework) - The Metasploit Project is a computer security project that provides information about security vulnerabilities and aids in penetration testing and IDS signature development.[@rapid7](https://github.com/rapid7) 10 | 11 | - [Nessus](https://www.tenable.com/downloads/nessus?loginAttempted=true) - Nessus today is available with 70,000+ plug-ins and services/functionalities such as malware detection, web application scanning, and system configuration check, etc. 12 | 13 | - [Nikto](https://www.freecodecamp.org/news/an-introduction-to-web-server-scanning-with-nikto/) - Nikto can perform comprehensive tests against web servers for multiple security threats, including over 6700 potentially dangerous files/programs. 14 | 15 | - [Nmap](https://github.com/nmap/nmap) - Nmap is used to discover hosts and services on a computer network by sending packets and analyzing the responses. [@nmap](https://github.com/nmap/nmap) 16 | 17 | - [OWASP ZAP](https://github.com/zaproxy/zaproxy) - Wireshark is a real-time network protocol analyzer that continuously scans network traffic for vulnerabilities and suspicious activities. [@website](https://www.zaproxy.org/) 18 | 19 | - [Snort3](https://github.com/snort3/snort3) - Snort 3 is the next generation Snort IPS (Intrusion Prevention System. 20 | 21 | - [Unicornscan](https://www.kali.org/tools/unicornscan/) - Unicornscan is an information gathering and port scanning tool like Nmap that provides asynchronous UDP and TCP scanning. It is a light-weight tool. [@Video](https://www.youtube.com/watch?v=X_DdYUeKS-o) 22 | 23 | - [WiFi Pumpkin](https://github.com/P0cL4bs/wifipumpkin3) - wifipumpkin3 is powerful framework for rogue access point attack, written in Python, that allow and offer to security researchers, red teamers and reverse engineers to mount a wireless network to conduct a man-in-the-middle attack. 24 | 25 | - [Wireshark](https://github.com/wireshark/wireshark) - Wireshark is a real-time network protocol analyzer that continuously scans network traffic for vulnerabilities and suspicious activities. [@wireshark](https://github.com/wireshark) 26 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to make your first contribution? 2 | 3 | This documentation aims to simplify and guide the way beginners make their first contribution. If you are looking to make your first contribution, follow the steps below. 4 | 5 | _If you're not comfortable with command line, [here are tutorials using GUI tools.](#tutorials-using-other-tools)_ 6 | 7 | #### If you don't have git on your machine, [install it](https://help.github.com/articles/set-up-git/). 8 | 9 | ## Fork this repository 10 | 11 | Fork this repository by clicking on the fork button on the top of this page. 12 | This will create a copy of this repository in your account. 13 | 14 | ## Clone the repository 15 | 16 | Now clone the forked repository to your machine. Go to your GitHub account, open the forked repository, click on the code button and then click the _copy to clipboard_ icon. 17 | 18 | Open a terminal and run the following git command: 19 | 20 | ``` 21 | git clone "url you just copied" 22 | ``` 23 | 24 | where "url you just copied" (without the quotation marks) is the url to this repository (your fork of this project). See the previous steps to obtain the url. 25 | 26 | For example: 27 | 28 | ``` 29 | git clone https://github.com/this-is-you/Awesome-Security-Resources.git 30 | ``` 31 | 32 | where `this-is-you` is your GitHub username. Here you're copying the contents of the first-contributions repository on GitHub to your computer. 33 | 34 | ## Create a branch 35 | 36 | Change to the repository directory on your computer (if you are not already there): 37 | 38 | ``` 39 | cd Awesome-Security-Resources 40 | ``` 41 | 42 | Now create a branch using the `git checkout` command: 43 | 44 | ``` 45 | git checkout -b your-new-branch-name 46 | ``` 47 | 48 | For example: 49 | 50 | ``` 51 | git checkout -b add-new-file 52 | ``` 53 | 54 | (The name of the branch does not need to have the word _add_ in it, but it's a reasonable thing to include because the purpose of this branch is to add your name to a list.) 55 | 56 | ## Make necessary changes and commit those changes 57 | 58 | Now open add or edit file in a text editor. Add code for any existing algorithm in other language or add some new algorithms. Make sure to update correspond README.md file if needed. Now, save the file. 59 | 60 | - **Note:** 61 | If there are any Screenshot/Image then size should be 1500 * 800 62 | 63 | If you go to the project directory and execute the command `git status`, you'll see there are changes. 64 | 65 | Add those changes to the branch you just created using the `git add` command: 66 | 67 | ``` 68 | git add "name of the file you add or edit" 69 | ``` 70 | 71 | Now commit those changes using the `git commit` command: 72 | 73 | ``` 74 | git commit -m "Add message for the change" 75 | ``` 76 | 77 | ## Push changes to GitHub 78 | 79 | Push your changes using the command `git push`: 80 | 81 | ``` 82 | git push origin 83 | ``` 84 | 85 | replacing `` with the name of the branch you created earlier. 86 | 87 | ## Submit your changes for review 88 | 89 | If you go to your repository on GitHub, you'll see a `Compare & pull request` button. Click on that button. 90 | 91 | Now submit the pull request. 92 | 93 | Soon after reviewing all your changes will be merged into the master branch of this project. You will get a notification email once the changes have been merged. 94 | 95 | ## Where to go from here? 96 | 97 | Congrats! You just completed the standard _fork -> clone -> edit -> pull request_ workflow that you'll encounter often as a contributor! 98 | -------------------------------------------------------------------------------- /Attacks/README.md: -------------------------------------------------------------------------------- 1 | ## Vulnerability Attacks 2 | 3 | - [Clickjacking](https://github.com/D4Vinci/Clickjacking-Tester) - Clickjacking is an attack that fools users into thinking they are clicking on one thing when they are actually clicking on another. 4 | - [Reference CVE]() 5 | - [2017](https://cwe.mitre.org/data/definitions/1021.html) 6 | 7 | - [CORS](https://0xn3va.gitbook.io/cheat-sheets/web-application/cors-misconfiguration) - Cross-origin resource sharing (CORS) is a browser mechanism which enables controlled access to resources located outside of a given domain. It extends and adds flexibility to the same-origin policy (SOP). 8 | [@cheatSheet](https://0xn3va.gitbook.io/cheat-sheets/web-application/cors-misconfiguration) 9 | [@practice](https://portswigger.net/web-security/cors) 10 | - [Reference CVE]() 11 | - [2021](https://security.netapp.com/advisory/ntap-20210318-0002/) 12 | 13 | - [IDOR](https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/Insecure_Direct_Object_Reference_Prevention_Cheat_Sheet.md) - Insecure Direct Object Reference (called IDOR from here) occurs when a application exposes a reference to an internal implementation object. Using this way, it reveals the real identifier and format/pattern used of the element in the storage backend side. 14 | - [Reference CVE]() 15 | - [2022](https://cwe.mitre.org/data/definitions/843.html) 16 | - [2021](https://cwe.mitre.org/data/definitions/639.html) 17 | - [2019](https://hackerone.com/reports/415081) 18 | 19 | - [LFI](https://github.com/OWASP/www-project-web-security-testing-guide/blob/master/v41/4-Web_Application_Security_Testing/07-Input_Validation_Testing/11.1-Testing_for_Local_File_Inclusion.md) -Local file inclusion (also known as LFI) is the process of including files, that are already locally present on the server, through the exploiting of vulnerable inclusion procedures implemented in the application. 20 | - [Reference CVE]() 21 | - [2022](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-41571) 22 | - [2021](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41277) 23 | - [2020](https://hackerone.com/reports/895972) 24 | 25 | - [Session Hijacking](https://github.com/OWASP/www-project-web-security-testing-guide/blob/master/v41/4-Web_Application_Security_Testing/06-Session_Management_Testing/03-Testing_for_Session_Fixation.md) - In these cases, vulnerable web applications authenticate users without first destroying existing sessions associated with said users. This allows attackers to access users' accounts by hijacking their active sessions. 26 | - [Reference CVE]() 27 | - [2021](https://hackerone.com/reports/1201396) 28 | - [2021](https://www.cvedetails.com/cve/CVE-2021-36320/) 29 | 30 | - [SQL Injection](https://github.com/sqlmapproject/sqlmap) - sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. 31 | [@Payload](https://github.com/payloadbox/sql-injection-payload-list) 32 | - [Reference CVE]() 33 | - [2022](https://www.cvedetails.com/vulnerability-list/opsqli-1/sql-injection.html) 34 | 35 | - [Web Cache Poisoning](https://portswigger.net/web-security/web-cache-poisoning) - Cache poisoning is a type of cyber attack in which attackers insert fake information into a domain name system (DNS) cache or web cache for the purpose of harming users. 36 | - [Reference CVE]() 37 | - [2021 - Poisoning your Cache for 1000$ - Approach to Exploitation Walkthrough](https://galnagli.com/Cache_Poisoning/) 38 | 39 | - [XSS](https://github.com/daffainfo/AllAboutBugBounty/blob/master/Cross%20Site%20Scripting.md) - Cross-Site Scripting (XSS) attacks are a type of injection, in which malicious scripts are injected into otherwise benign and trusted web sites.
40 | [@Payloads](https://github.com/payloadbox/xss-payload-list/blob/master/Intruder/xss-payload-list.txt) 41 | [@Practice](https://prompt.ml/0) 42 | - [Reference CVE]() 43 | - [2021](https://www.cvedetails.com/vulnerability-list/year-2021/opxss-1/xss.html) 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](GitHubPreview.png) 2 | 3 | # HACKTOBERFEST-2022 🚀 4 | > Add valuable resources that will help the community. 5 | 6 | # 💡Always Welcome For Contribution!! 7 | Check the contribution [RULES](/CONTRIBUTING.md) 8 | 9 | ## Vulnerability Assesment and Penetration Testing 10 | 11 | - [Directory Bruteforce Tools](/Directory%20Bruteforce_Tools/README.md) 12 | - [Gobuster](/Directory%20Bruteforce_Tools/README.md) 13 | - [DirBuster](/Directory%20Bruteforce_Tools/README.md) 14 | - [Dirsearch](/Directory%20Bruteforce_Tools/README.md) 15 | 16 | - [Email Address](/Email%20Address/) 17 | - [Discover Email Address](/Email%20Address/Discover%20Email%20Address/) 18 | - [Hunter.io](/Email%20Address/Discover%20Email%20Address/README.md) 19 | - [KnockMail](/Email%20Address/Discover%20Email%20Address/README.md) 20 | - [PhoneBook.cz](/Email%20Address/Discover%20Email%20Address/README.md) 21 | - [TheHarvester](/Email%20Address/Discover%20Email%20Address/README.md) 22 | - [Verify Email Address](/Email%20Address/Verify%20Email%20Address/) 23 | - [Email-Checker.net](/Email%20Address/Verify%20Email%20Address/README.md) 24 | 25 | - [Identify Web Technologies](/Identify%20Web%20Technology/) 26 | - [BuiltWith.com](/Identify%20Web%20Technology/README.md) 27 | - [Wappalyzer](/Identify%20Web%20Technology/README.md) 28 | 29 | - [Network Scanner](/Network%20Scanner/) 30 | - [Advanced Ip Scanner](/Network%20Scanner/README.md) 31 | - [Airbash](/Network%20Scanner/README.md) 32 | - [Bettercap](/Network%20Scanner/README.md) 33 | - [Metasploit Framework](/Network%20Scanner/README.md) 34 | - [Nessus](/Network%20Scanner/README.md) 35 | - [Nikto](/Network%20Scanner/README.md) 36 | - [Nmap](/Network%20Scanner/README.md) 37 | - [OWASP ZAP](/Network%20Scanner/README.md) 38 | - [Snort3](/Network%20Scanner/README.md) 39 | - [Unicornscan](/Network%20Scanner/README.md) 40 | - [Wireshark](/Network%20Scanner/README.md) 41 | - [WiFi Pumpkin](/Network%20Scanner/README.md) 42 | 43 | - [Password Attacking Tools](/Password%20Attacking%20Tools/) 44 | - [Hashcat](/Password%20Attacking%20Tools/README.md) 45 | - [Hydra](/Password%20Attacking%20Tools/README.md) 46 | - [John the Ripper](/Password%20Attacking%20Tools/README.md) 47 | - [Ncrack](/Password%20Attacking%20Tools/README.md) 48 | - [Ophcrack](/Password%20Attacking%20Tools/README.md) 49 | 50 | - [Script](/Script/README.md) 51 | - [Automatic Subdomain Enumeration](/Script/SubdomainScript) 52 | 53 | - [Search Engine](/Search%20Engine/) 54 | - [Dehashed](/Search%20Engine/README.md) 55 | - [SecurityTrails](/Search%20Engine/README.md) 56 | - [Uncover](/Search%20Engine/README.md) 57 | 58 | - [Sub Domain Enumeration](/Sub%20Domain%20Enumeration/) 59 | - [AssetFinder](/Sub%20Domain%20Enumeration/README.md) 60 | - [Amass](/Sub%20Domain%20Enumeration/README.md) 61 | - [Attdns](/Sub%20Domain%20Enumeration/README.md) 62 | - [CRT.sh](/Sub%20Domain%20Enumeration/README.md) 63 | - [Domain](/Sub%20Domain%20Enumeration/README.md) 64 | - [SecurityTrails](/Sub%20Domain%20Enumeration/README.md) 65 | - [SubFinder](/Sub%20Domain%20Enumeration/README.md) 66 | - [Sublist3r](/Sub%20Domain%20Enumeration/README.md) 67 | 68 | - [Vulnerability Attacks](/Attacks/README.md) 69 | - [Clickjacking](/Attacks/README.md) 70 | - [CORS](/Attacks/README.md) 71 | - [IDOR](/Attacks/README.md) 72 | - [Local File Inclusion (LFI)](/Attacks/README.md) 73 | - [Session Hijacking](/Attacks/README.md) 74 | - [SQL Injection](/Attacks/README.md) 75 | - [Web Cache Poisoning](/Attacks/README.md) 76 | - [XSS](/Attacks/README.md) 77 | 78 | ## Datasets 79 | 80 | - [Intrusion Detection System](/Intrusion%20Detection%20System/README.md) 81 | 82 | 83 | ## Free Learning Resources 84 | 85 | - [CTF Challenge Resources](/Free%20Learning%20Resources/CTF%20Challenge%20Resources/README.md) 86 | - [Cyber Security Courses](/Free%20Learning%20Resources/Cyber%20Security%20Courses/README.md) 87 | - [Cyber Security News (Sites, Blogs, Feeds)](/Free%20Learning%20Resources/Cyber%20Security%20News/README.md) 88 | - [Youtube Channel](/Free%20Learning%20Resources/Youtube%20Channel/README.md) 89 | 90 | 91 | ## License 92 | 93 | [MIT License](LICENSE) 94 | 95 | 96 | ## Disclaimer 97 | 98 | All the resources are for informational and educational purposes only. We believe that ethical hacking, information security and cyber security should be familiar subjects to anyone using digital information and computers. Don't misuse of the information and we strongly suggest against it. We do not promote, encourage, support or excite any illegal activity or hacking. We cannot be held responsible for any misuse of the given information. 99 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | prashantbhapkar13@gmail.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /Intrusion Detection System/iot-intrusion-detection.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "papermill": { 7 | "duration": 0.012056, 8 | "end_time": "2022-10-05T15:14:29.775324", 9 | "exception": false, 10 | "start_time": "2022-10-05T15:14:29.763268", 11 | "status": "completed" 12 | }, 13 | "tags": [] 14 | }, 15 | "source": [ 16 | "# IoT Intrusion Detection\n", 17 | "\n", 18 | "The N-BaIoT Dataset contains traffic data for 9 IoT devices. The data comprise of both benign traffic and of a variety of malicious attacks. Here we run three deep neural networks to identify cyberattacks on a Provision PT-737E Security Camera." 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": { 25 | "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0", 26 | "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a", 27 | "execution": { 28 | "iopub.execute_input": "2022-10-05T15:14:29.804838Z", 29 | "iopub.status.busy": "2022-10-05T15:14:29.803782Z", 30 | "iopub.status.idle": "2022-10-05T15:14:29.807116Z", 31 | "shell.execute_reply": "2022-10-05T15:14:29.806463Z", 32 | "shell.execute_reply.started": "2022-04-07T04:29:44.510530Z" 33 | }, 34 | "papermill": { 35 | "duration": 0.019869, 36 | "end_time": "2022-10-05T15:14:29.807256", 37 | "exception": false, 38 | "start_time": "2022-10-05T15:14:29.787387", 39 | "status": "completed" 40 | }, 41 | "tags": [] 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "import numpy as np\n", 46 | "import pandas as pd" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 2, 52 | "metadata": { 53 | "execution": { 54 | "iopub.execute_input": "2022-10-05T15:14:29.834764Z", 55 | "iopub.status.busy": "2022-10-05T15:14:29.833995Z", 56 | "iopub.status.idle": "2022-10-05T15:14:55.763681Z", 57 | "shell.execute_reply": "2022-10-05T15:14:55.762731Z", 58 | "shell.execute_reply.started": "2022-04-07T04:29:44.517361Z" 59 | }, 60 | "papermill": { 61 | "duration": 25.94479, 62 | "end_time": "2022-10-05T15:14:55.763868", 63 | "exception": false, 64 | "start_time": "2022-10-05T15:14:29.819078", 65 | "status": "completed" 66 | }, 67 | "tags": [] 68 | }, 69 | "outputs": [], 70 | "source": [ 71 | "benign=pd.read_csv('../input/nbaiot-dataset/5.benign.csv')\n", 72 | "g_c=pd.read_csv('../input/nbaiot-dataset/5.gafgyt.combo.csv')\n", 73 | "g_j=pd.read_csv('../input/nbaiot-dataset/5.gafgyt.junk.csv')\n", 74 | "g_s=pd.read_csv('../input/nbaiot-dataset/5.gafgyt.scan.csv')\n", 75 | "g_t=pd.read_csv('../input/nbaiot-dataset/5.gafgyt.tcp.csv')\n", 76 | "g_u=pd.read_csv('../input/nbaiot-dataset/5.gafgyt.udp.csv')\n", 77 | "m_a=pd.read_csv('../input/nbaiot-dataset/5.mirai.ack.csv')\n", 78 | "m_sc=pd.read_csv('../input/nbaiot-dataset/5.mirai.scan.csv')\n", 79 | "m_sy=pd.read_csv('../input/nbaiot-dataset/5.mirai.syn.csv')\n", 80 | "m_u=pd.read_csv('../input/nbaiot-dataset/5.mirai.udp.csv')\n", 81 | "m_u_p=pd.read_csv('../input/nbaiot-dataset/5.mirai.udpplain.csv')\n", 82 | "\n", 83 | "benign=benign.sample(frac=0.25,replace=False)\n", 84 | "g_c=g_c.sample(frac=0.25,replace=False)\n", 85 | "g_j=g_j.sample(frac=0.5,replace=False)\n", 86 | "g_s=g_s.sample(frac=0.5,replace=False)\n", 87 | "g_t=g_t.sample(frac=0.15,replace=False)\n", 88 | "g_u=g_u.sample(frac=0.15,replace=False)\n", 89 | "m_a=m_a.sample(frac=0.25,replace=False)\n", 90 | "m_sc=m_sc.sample(frac=0.15,replace=False)\n", 91 | "m_sy=m_sy.sample(frac=0.25,replace=False)\n", 92 | "m_u=m_u.sample(frac=0.1,replace=False)\n", 93 | "m_u_p=m_u_p.sample(frac=0.27,replace=False)\n", 94 | "\n", 95 | "benign['type']='benign'\n", 96 | "m_u['type']='mirai_udp'\n", 97 | "g_c['type']='gafgyt_combo'\n", 98 | "g_j['type']='gafgyt_junk'\n", 99 | "g_s['type']='gafgyt_scan'\n", 100 | "g_t['type']='gafgyt_tcp'\n", 101 | "g_u['type']='gafgyt_udp'\n", 102 | "m_a['type']='mirai_ack'\n", 103 | "m_sc['type']='mirai_scan'\n", 104 | "m_sy['type']='mirai_syn'\n", 105 | "m_u_p['type']='mirai_udpplain'\n", 106 | "\n", 107 | "data=pd.concat([benign,m_u,g_c,g_j,g_s,g_t,g_u,m_a,m_sc,m_sy,m_u_p],\n", 108 | " axis=0, sort=False, ignore_index=True)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 3, 114 | "metadata": { 115 | "execution": { 116 | "iopub.execute_input": "2022-10-05T15:14:55.798036Z", 117 | "iopub.status.busy": "2022-10-05T15:14:55.797305Z", 118 | "iopub.status.idle": "2022-10-05T15:14:55.826612Z", 119 | "shell.execute_reply": "2022-10-05T15:14:55.827202Z", 120 | "shell.execute_reply.started": "2022-04-07T04:29:56.897502Z" 121 | }, 122 | "papermill": { 123 | "duration": 0.052311, 124 | "end_time": "2022-10-05T15:14:55.827416", 125 | "exception": false, 126 | "start_time": "2022-10-05T15:14:55.775105", 127 | "status": "completed" 128 | }, 129 | "tags": [] 130 | }, 131 | "outputs": [ 132 | { 133 | "data": { 134 | "text/plain": [ 135 | "type\n", 136 | "benign 15538\n", 137 | "gafgyt_combo 15345\n", 138 | "gafgyt_junk 15449\n", 139 | "gafgyt_scan 14648\n", 140 | "gafgyt_tcp 15676\n", 141 | "gafgyt_udp 15602\n", 142 | "mirai_ack 15138\n", 143 | "mirai_scan 14517\n", 144 | "mirai_syn 16436\n", 145 | "mirai_udp 15625\n", 146 | "mirai_udpplain 15304\n", 147 | "Name: type, dtype: int64" 148 | ] 149 | }, 150 | "execution_count": 3, 151 | "metadata": {}, 152 | "output_type": "execute_result" 153 | } 154 | ], 155 | "source": [ 156 | "#how many instances of each class\n", 157 | "data.groupby('type')['type'].count()" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 4, 163 | "metadata": { 164 | "execution": { 165 | "iopub.execute_input": "2022-10-05T15:14:55.863307Z", 166 | "iopub.status.busy": "2022-10-05T15:14:55.861992Z", 167 | "iopub.status.idle": "2022-10-05T15:14:56.097334Z", 168 | "shell.execute_reply": "2022-10-05T15:14:56.097970Z", 169 | "shell.execute_reply.started": "2022-04-07T04:29:56.929445Z" 170 | }, 171 | "papermill": { 172 | "duration": 0.258914, 173 | "end_time": "2022-10-05T15:14:56.098156", 174 | "exception": false, 175 | "start_time": "2022-10-05T15:14:55.839242", 176 | "status": "completed" 177 | }, 178 | "tags": [] 179 | }, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "text/html": [ 184 | "
\n", 185 | "\n", 198 | "\n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | "
MI_dir_L5_weightMI_dir_L5_meanMI_dir_L5_varianceMI_dir_L3_weightMI_dir_L3_meanMI_dir_L3_varianceMI_dir_L1_weightMI_dir_L1_meanMI_dir_L1_varianceMI_dir_L0.1_weight...HpHp_L0.1_covarianceHpHp_L0.1_pccHpHp_L0.01_weightHpHp_L0.01_meanHpHp_L0.01_stdHpHp_L0.01_magnitudeHpHp_L0.01_radiusHpHp_L0.01_covarianceHpHp_L0.01_pcctype
15102630.64834873.7862842.94621142.92008172.38535820.000020218.49956869.59829142.3159463076.945478...0.00.01.00000074.00.000000e+0074.00.000000e+000.00.0mirai_syn
950811.00000060.0000000.0000001.00000060.0000000.0000001.00000060.0000000.0000001.000000...0.00.01.00000060.00.000000e+0060.00.000000e+000.00.0gafgyt_udp
6270845.17868174.0446141.07812254.39368774.1082122.72061897.15827174.2638528.3366191017.833281...0.00.02.89878474.09.536743e-0774.09.094947e-130.00.0gafgyt_scan
13684521.29500760.0587760.80760733.76128360.1275221.947972145.77083760.1610442.5882161890.731832...0.00.01.00000060.00.000000e+0060.00.000000e+000.00.0mirai_scan
31968170.29983674.0060220.144488281.29587474.0143600.344454835.86054274.0321421.2917075257.659324...0.00.01.00000074.00.000000e+0074.00.000000e+000.00.0gafgyt_combo
\n", 348 | "

5 rows × 116 columns

\n", 349 | "
" 350 | ], 351 | "text/plain": [ 352 | " MI_dir_L5_weight MI_dir_L5_mean MI_dir_L5_variance \\\n", 353 | "151026 30.648348 73.786284 2.946211 \n", 354 | "95081 1.000000 60.000000 0.000000 \n", 355 | "62708 45.178681 74.044614 1.078122 \n", 356 | "136845 21.295007 60.058776 0.807607 \n", 357 | "31968 170.299836 74.006022 0.144488 \n", 358 | "\n", 359 | " MI_dir_L3_weight MI_dir_L3_mean MI_dir_L3_variance \\\n", 360 | "151026 42.920081 72.385358 20.000020 \n", 361 | "95081 1.000000 60.000000 0.000000 \n", 362 | "62708 54.393687 74.108212 2.720618 \n", 363 | "136845 33.761283 60.127522 1.947972 \n", 364 | "31968 281.295874 74.014360 0.344454 \n", 365 | "\n", 366 | " MI_dir_L1_weight MI_dir_L1_mean MI_dir_L1_variance \\\n", 367 | "151026 218.499568 69.598291 42.315946 \n", 368 | "95081 1.000000 60.000000 0.000000 \n", 369 | "62708 97.158271 74.263852 8.336619 \n", 370 | "136845 145.770837 60.161044 2.588216 \n", 371 | "31968 835.860542 74.032142 1.291707 \n", 372 | "\n", 373 | " MI_dir_L0.1_weight ... HpHp_L0.1_covariance HpHp_L0.1_pcc \\\n", 374 | "151026 3076.945478 ... 0.0 0.0 \n", 375 | "95081 1.000000 ... 0.0 0.0 \n", 376 | "62708 1017.833281 ... 0.0 0.0 \n", 377 | "136845 1890.731832 ... 0.0 0.0 \n", 378 | "31968 5257.659324 ... 0.0 0.0 \n", 379 | "\n", 380 | " HpHp_L0.01_weight HpHp_L0.01_mean HpHp_L0.01_std \\\n", 381 | "151026 1.000000 74.0 0.000000e+00 \n", 382 | "95081 1.000000 60.0 0.000000e+00 \n", 383 | "62708 2.898784 74.0 9.536743e-07 \n", 384 | "136845 1.000000 60.0 0.000000e+00 \n", 385 | "31968 1.000000 74.0 0.000000e+00 \n", 386 | "\n", 387 | " HpHp_L0.01_magnitude HpHp_L0.01_radius HpHp_L0.01_covariance \\\n", 388 | "151026 74.0 0.000000e+00 0.0 \n", 389 | "95081 60.0 0.000000e+00 0.0 \n", 390 | "62708 74.0 9.094947e-13 0.0 \n", 391 | "136845 60.0 0.000000e+00 0.0 \n", 392 | "31968 74.0 0.000000e+00 0.0 \n", 393 | "\n", 394 | " HpHp_L0.01_pcc type \n", 395 | "151026 0.0 mirai_syn \n", 396 | "95081 0.0 gafgyt_udp \n", 397 | "62708 0.0 gafgyt_scan \n", 398 | "136845 0.0 mirai_scan \n", 399 | "31968 0.0 gafgyt_combo \n", 400 | "\n", 401 | "[5 rows x 116 columns]" 402 | ] 403 | }, 404 | "execution_count": 4, 405 | "metadata": {}, 406 | "output_type": "execute_result" 407 | } 408 | ], 409 | "source": [ 410 | "#shuffle rows of dataframe \n", 411 | "sampler=np.random.permutation(len(data))\n", 412 | "data=data.take(sampler)\n", 413 | "data.head()" 414 | ] 415 | }, 416 | { 417 | "cell_type": "code", 418 | "execution_count": 5, 419 | "metadata": { 420 | "execution": { 421 | "iopub.execute_input": "2022-10-05T15:14:56.138428Z", 422 | "iopub.status.busy": "2022-10-05T15:14:56.131568Z", 423 | "iopub.status.idle": "2022-10-05T15:14:56.163452Z", 424 | "shell.execute_reply": "2022-10-05T15:14:56.162815Z", 425 | "shell.execute_reply.started": "2022-04-07T04:29:57.086461Z" 426 | }, 427 | "papermill": { 428 | "duration": 0.053214, 429 | "end_time": "2022-10-05T15:14:56.163604", 430 | "exception": false, 431 | "start_time": "2022-10-05T15:14:56.110390", 432 | "status": "completed" 433 | }, 434 | "tags": [] 435 | }, 436 | "outputs": [ 437 | { 438 | "data": { 439 | "text/html": [ 440 | "
\n", 441 | "\n", 454 | "\n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | "
type_benigntype_gafgyt_combotype_gafgyt_junktype_gafgyt_scantype_gafgyt_tcptype_gafgyt_udptype_mirai_acktype_mirai_scantype_mirai_syntype_mirai_udptype_mirai_udpplain
15102600000000100
9508100000100000
6270800010000000
13684500000001000
3196801000000000
\n", 544 | "
" 545 | ], 546 | "text/plain": [ 547 | " type_benign type_gafgyt_combo type_gafgyt_junk type_gafgyt_scan \\\n", 548 | "151026 0 0 0 0 \n", 549 | "95081 0 0 0 0 \n", 550 | "62708 0 0 0 1 \n", 551 | "136845 0 0 0 0 \n", 552 | "31968 0 1 0 0 \n", 553 | "\n", 554 | " type_gafgyt_tcp type_gafgyt_udp type_mirai_ack type_mirai_scan \\\n", 555 | "151026 0 0 0 0 \n", 556 | "95081 0 1 0 0 \n", 557 | "62708 0 0 0 0 \n", 558 | "136845 0 0 0 1 \n", 559 | "31968 0 0 0 0 \n", 560 | "\n", 561 | " type_mirai_syn type_mirai_udp type_mirai_udpplain \n", 562 | "151026 1 0 0 \n", 563 | "95081 0 0 0 \n", 564 | "62708 0 0 0 \n", 565 | "136845 0 0 0 \n", 566 | "31968 0 0 0 " 567 | ] 568 | }, 569 | "execution_count": 5, 570 | "metadata": {}, 571 | "output_type": "execute_result" 572 | } 573 | ], 574 | "source": [ 575 | "#dummy encode labels, store separately\n", 576 | "labels_full=pd.get_dummies(data['type'], prefix='type')\n", 577 | "labels_full.head()" 578 | ] 579 | }, 580 | { 581 | "cell_type": "code", 582 | "execution_count": 6, 583 | "metadata": { 584 | "execution": { 585 | "iopub.execute_input": "2022-10-05T15:14:56.291906Z", 586 | "iopub.status.busy": "2022-10-05T15:14:56.282930Z", 587 | "iopub.status.idle": "2022-10-05T15:14:56.313784Z", 588 | "shell.execute_reply": "2022-10-05T15:14:56.313093Z", 589 | "shell.execute_reply.started": "2022-04-07T04:29:57.120938Z" 590 | }, 591 | "papermill": { 592 | "duration": 0.135417, 593 | "end_time": "2022-10-05T15:14:56.313951", 594 | "exception": false, 595 | "start_time": "2022-10-05T15:14:56.178534", 596 | "status": "completed" 597 | }, 598 | "tags": [] 599 | }, 600 | "outputs": [ 601 | { 602 | "data": { 603 | "text/html": [ 604 | "
\n", 605 | "\n", 618 | "\n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | "
MI_dir_L5_weightMI_dir_L5_meanMI_dir_L5_varianceMI_dir_L3_weightMI_dir_L3_meanMI_dir_L3_varianceMI_dir_L1_weightMI_dir_L1_meanMI_dir_L1_varianceMI_dir_L0.1_weight...HpHp_L0.1_radiusHpHp_L0.1_covarianceHpHp_L0.1_pccHpHp_L0.01_weightHpHp_L0.01_meanHpHp_L0.01_stdHpHp_L0.01_magnitudeHpHp_L0.01_radiusHpHp_L0.01_covarianceHpHp_L0.01_pcc
15102630.64834873.7862842.94621142.92008172.38535820.000020218.49956869.59829142.3159463076.945478...0.000000e+000.00.01.00000074.00.000000e+0074.00.000000e+000.00.0
950811.00000060.0000000.0000001.00000060.0000000.0000001.00000060.0000000.0000001.000000...0.000000e+000.00.01.00000060.00.000000e+0060.00.000000e+000.00.0
6270845.17868174.0446141.07812254.39368774.1082122.72061897.15827174.2638528.3366191017.833281...1.818989e-120.00.02.89878474.09.536743e-0774.09.094947e-130.00.0
13684521.29500760.0587760.80760733.76128360.1275221.947972145.77083760.1610442.5882161890.731832...0.000000e+000.00.01.00000060.00.000000e+0060.00.000000e+000.00.0
31968170.29983674.0060220.144488281.29587474.0143600.344454835.86054274.0321421.2917075257.659324...0.000000e+000.00.01.00000074.00.000000e+0074.00.000000e+000.00.0
\n", 768 | "

5 rows × 115 columns

\n", 769 | "
" 770 | ], 771 | "text/plain": [ 772 | " MI_dir_L5_weight MI_dir_L5_mean MI_dir_L5_variance \\\n", 773 | "151026 30.648348 73.786284 2.946211 \n", 774 | "95081 1.000000 60.000000 0.000000 \n", 775 | "62708 45.178681 74.044614 1.078122 \n", 776 | "136845 21.295007 60.058776 0.807607 \n", 777 | "31968 170.299836 74.006022 0.144488 \n", 778 | "\n", 779 | " MI_dir_L3_weight MI_dir_L3_mean MI_dir_L3_variance \\\n", 780 | "151026 42.920081 72.385358 20.000020 \n", 781 | "95081 1.000000 60.000000 0.000000 \n", 782 | "62708 54.393687 74.108212 2.720618 \n", 783 | "136845 33.761283 60.127522 1.947972 \n", 784 | "31968 281.295874 74.014360 0.344454 \n", 785 | "\n", 786 | " MI_dir_L1_weight MI_dir_L1_mean MI_dir_L1_variance \\\n", 787 | "151026 218.499568 69.598291 42.315946 \n", 788 | "95081 1.000000 60.000000 0.000000 \n", 789 | "62708 97.158271 74.263852 8.336619 \n", 790 | "136845 145.770837 60.161044 2.588216 \n", 791 | "31968 835.860542 74.032142 1.291707 \n", 792 | "\n", 793 | " MI_dir_L0.1_weight ... HpHp_L0.1_radius HpHp_L0.1_covariance \\\n", 794 | "151026 3076.945478 ... 0.000000e+00 0.0 \n", 795 | "95081 1.000000 ... 0.000000e+00 0.0 \n", 796 | "62708 1017.833281 ... 1.818989e-12 0.0 \n", 797 | "136845 1890.731832 ... 0.000000e+00 0.0 \n", 798 | "31968 5257.659324 ... 0.000000e+00 0.0 \n", 799 | "\n", 800 | " HpHp_L0.1_pcc HpHp_L0.01_weight HpHp_L0.01_mean HpHp_L0.01_std \\\n", 801 | "151026 0.0 1.000000 74.0 0.000000e+00 \n", 802 | "95081 0.0 1.000000 60.0 0.000000e+00 \n", 803 | "62708 0.0 2.898784 74.0 9.536743e-07 \n", 804 | "136845 0.0 1.000000 60.0 0.000000e+00 \n", 805 | "31968 0.0 1.000000 74.0 0.000000e+00 \n", 806 | "\n", 807 | " HpHp_L0.01_magnitude HpHp_L0.01_radius HpHp_L0.01_covariance \\\n", 808 | "151026 74.0 0.000000e+00 0.0 \n", 809 | "95081 60.0 0.000000e+00 0.0 \n", 810 | "62708 74.0 9.094947e-13 0.0 \n", 811 | "136845 60.0 0.000000e+00 0.0 \n", 812 | "31968 74.0 0.000000e+00 0.0 \n", 813 | "\n", 814 | " HpHp_L0.01_pcc \n", 815 | "151026 0.0 \n", 816 | "95081 0.0 \n", 817 | "62708 0.0 \n", 818 | "136845 0.0 \n", 819 | "31968 0.0 \n", 820 | "\n", 821 | "[5 rows x 115 columns]" 822 | ] 823 | }, 824 | "execution_count": 6, 825 | "metadata": {}, 826 | "output_type": "execute_result" 827 | } 828 | ], 829 | "source": [ 830 | "#drop labels from training dataset\n", 831 | "data=data.drop(columns='type')\n", 832 | "data.head()" 833 | ] 834 | }, 835 | { 836 | "cell_type": "code", 837 | "execution_count": 7, 838 | "metadata": { 839 | "execution": { 840 | "iopub.execute_input": "2022-10-05T15:14:56.352087Z", 841 | "iopub.status.busy": "2022-10-05T15:14:56.349594Z", 842 | "iopub.status.idle": "2022-10-05T15:14:56.847479Z", 843 | "shell.execute_reply": "2022-10-05T15:14:56.846693Z", 844 | "shell.execute_reply.started": "2022-04-07T04:29:57.196126Z" 845 | }, 846 | "papermill": { 847 | "duration": 0.520198, 848 | "end_time": "2022-10-05T15:14:56.847639", 849 | "exception": false, 850 | "start_time": "2022-10-05T15:14:56.327441", 851 | "status": "completed" 852 | }, 853 | "tags": [] 854 | }, 855 | "outputs": [ 856 | { 857 | "data": { 858 | "text/html": [ 859 | "
\n", 860 | "\n", 873 | "\n", 874 | " \n", 875 | " \n", 876 | " \n", 877 | " \n", 878 | " \n", 879 | " \n", 880 | " \n", 881 | " \n", 882 | " \n", 883 | " \n", 884 | " \n", 885 | " \n", 886 | " \n", 887 | " \n", 888 | " \n", 889 | " \n", 890 | " \n", 891 | " \n", 892 | " \n", 893 | " \n", 894 | " \n", 895 | " \n", 896 | " \n", 897 | " \n", 898 | " \n", 899 | " \n", 900 | " \n", 901 | " \n", 902 | " \n", 903 | " \n", 904 | " \n", 905 | " \n", 906 | " \n", 907 | " \n", 908 | " \n", 909 | " \n", 910 | " \n", 911 | " \n", 912 | " \n", 913 | " \n", 914 | " \n", 915 | " \n", 916 | " \n", 917 | " \n", 918 | " \n", 919 | " \n", 920 | " \n", 921 | " \n", 922 | " \n", 923 | " \n", 924 | " \n", 925 | " \n", 926 | " \n", 927 | " \n", 928 | " \n", 929 | " \n", 930 | " \n", 931 | " \n", 932 | " \n", 933 | " \n", 934 | " \n", 935 | " \n", 936 | " \n", 937 | " \n", 938 | " \n", 939 | " \n", 940 | " \n", 941 | " \n", 942 | " \n", 943 | " \n", 944 | " \n", 945 | " \n", 946 | " \n", 947 | " \n", 948 | " \n", 949 | " \n", 950 | " \n", 951 | " \n", 952 | " \n", 953 | " \n", 954 | " \n", 955 | " \n", 956 | " \n", 957 | " \n", 958 | " \n", 959 | " \n", 960 | " \n", 961 | " \n", 962 | " \n", 963 | " \n", 964 | " \n", 965 | " \n", 966 | " \n", 967 | " \n", 968 | " \n", 969 | " \n", 970 | " \n", 971 | " \n", 972 | " \n", 973 | " \n", 974 | " \n", 975 | " \n", 976 | " \n", 977 | " \n", 978 | " \n", 979 | " \n", 980 | " \n", 981 | " \n", 982 | " \n", 983 | " \n", 984 | " \n", 985 | " \n", 986 | " \n", 987 | " \n", 988 | " \n", 989 | " \n", 990 | " \n", 991 | " \n", 992 | " \n", 993 | " \n", 994 | " \n", 995 | " \n", 996 | " \n", 997 | " \n", 998 | " \n", 999 | " \n", 1000 | " \n", 1001 | " \n", 1002 | " \n", 1003 | " \n", 1004 | " \n", 1005 | " \n", 1006 | " \n", 1007 | " \n", 1008 | " \n", 1009 | " \n", 1010 | " \n", 1011 | " \n", 1012 | " \n", 1013 | " \n", 1014 | " \n", 1015 | " \n", 1016 | " \n", 1017 | " \n", 1018 | " \n", 1019 | " \n", 1020 | " \n", 1021 | " \n", 1022 | "
MI_dir_L5_weightMI_dir_L5_meanMI_dir_L5_varianceMI_dir_L3_weightMI_dir_L3_meanMI_dir_L3_varianceMI_dir_L1_weightMI_dir_L1_meanMI_dir_L1_varianceMI_dir_L0.1_weight...HpHp_L0.1_radiusHpHp_L0.1_covarianceHpHp_L0.1_pccHpHp_L0.01_weightHpHp_L0.01_meanHpHp_L0.01_stdHpHp_L0.01_magnitudeHpHp_L0.01_radiusHpHp_L0.01_covarianceHpHp_L0.01_pcc
151026-0.692430-0.509400-0.601542-0.724490-0.545518-0.623738-0.344585-0.598787-0.6409480.171540...-0.098909-0.097904-0.071396-0.197790-0.383413-0.127628-0.405197-0.09854-0.0952640.0
95081-1.150187-0.615149-0.601668-1.136606-0.645077-0.624537-1.118935-0.679878-0.642534-1.137453...-0.098909-0.097904-0.071396-0.197790-0.463774-0.127628-0.484507-0.09854-0.0952640.0
62708-0.468088-0.507418-0.601622-0.611693-0.531669-0.624429-0.776589-0.559370-0.642222-0.704732...-0.098909-0.097904-0.071396-0.196798-0.383413-0.127628-0.405197-0.09854-0.0952640.0
136845-0.836841-0.614698-0.601634-0.814530-0.644052-0.624460-0.603516-0.678517-0.642437-0.333263...-0.098909-0.097904-0.071396-0.197790-0.463774-0.127628-0.484507-0.09854-0.0952640.0
319681.463724-0.507714-0.6016621.618982-0.532424-0.6245241.853367-0.561328-0.6424861.099560...-0.098909-0.097904-0.071396-0.197790-0.383413-0.127628-0.405197-0.09854-0.0952640.0
\n", 1023 | "

5 rows × 115 columns

\n", 1024 | "
" 1025 | ], 1026 | "text/plain": [ 1027 | " MI_dir_L5_weight MI_dir_L5_mean MI_dir_L5_variance \\\n", 1028 | "151026 -0.692430 -0.509400 -0.601542 \n", 1029 | "95081 -1.150187 -0.615149 -0.601668 \n", 1030 | "62708 -0.468088 -0.507418 -0.601622 \n", 1031 | "136845 -0.836841 -0.614698 -0.601634 \n", 1032 | "31968 1.463724 -0.507714 -0.601662 \n", 1033 | "\n", 1034 | " MI_dir_L3_weight MI_dir_L3_mean MI_dir_L3_variance \\\n", 1035 | "151026 -0.724490 -0.545518 -0.623738 \n", 1036 | "95081 -1.136606 -0.645077 -0.624537 \n", 1037 | "62708 -0.611693 -0.531669 -0.624429 \n", 1038 | "136845 -0.814530 -0.644052 -0.624460 \n", 1039 | "31968 1.618982 -0.532424 -0.624524 \n", 1040 | "\n", 1041 | " MI_dir_L1_weight MI_dir_L1_mean MI_dir_L1_variance \\\n", 1042 | "151026 -0.344585 -0.598787 -0.640948 \n", 1043 | "95081 -1.118935 -0.679878 -0.642534 \n", 1044 | "62708 -0.776589 -0.559370 -0.642222 \n", 1045 | "136845 -0.603516 -0.678517 -0.642437 \n", 1046 | "31968 1.853367 -0.561328 -0.642486 \n", 1047 | "\n", 1048 | " MI_dir_L0.1_weight ... HpHp_L0.1_radius HpHp_L0.1_covariance \\\n", 1049 | "151026 0.171540 ... -0.098909 -0.097904 \n", 1050 | "95081 -1.137453 ... -0.098909 -0.097904 \n", 1051 | "62708 -0.704732 ... -0.098909 -0.097904 \n", 1052 | "136845 -0.333263 ... -0.098909 -0.097904 \n", 1053 | "31968 1.099560 ... -0.098909 -0.097904 \n", 1054 | "\n", 1055 | " HpHp_L0.1_pcc HpHp_L0.01_weight HpHp_L0.01_mean HpHp_L0.01_std \\\n", 1056 | "151026 -0.071396 -0.197790 -0.383413 -0.127628 \n", 1057 | "95081 -0.071396 -0.197790 -0.463774 -0.127628 \n", 1058 | "62708 -0.071396 -0.196798 -0.383413 -0.127628 \n", 1059 | "136845 -0.071396 -0.197790 -0.463774 -0.127628 \n", 1060 | "31968 -0.071396 -0.197790 -0.383413 -0.127628 \n", 1061 | "\n", 1062 | " HpHp_L0.01_magnitude HpHp_L0.01_radius HpHp_L0.01_covariance \\\n", 1063 | "151026 -0.405197 -0.09854 -0.095264 \n", 1064 | "95081 -0.484507 -0.09854 -0.095264 \n", 1065 | "62708 -0.405197 -0.09854 -0.095264 \n", 1066 | "136845 -0.484507 -0.09854 -0.095264 \n", 1067 | "31968 -0.405197 -0.09854 -0.095264 \n", 1068 | "\n", 1069 | " HpHp_L0.01_pcc \n", 1070 | "151026 0.0 \n", 1071 | "95081 0.0 \n", 1072 | "62708 0.0 \n", 1073 | "136845 0.0 \n", 1074 | "31968 0.0 \n", 1075 | "\n", 1076 | "[5 rows x 115 columns]" 1077 | ] 1078 | }, 1079 | "execution_count": 7, 1080 | "metadata": {}, 1081 | "output_type": "execute_result" 1082 | } 1083 | ], 1084 | "source": [ 1085 | "#standardize numerical columns\n", 1086 | "def standardize(df,col):\n", 1087 | " df[col]= (df[col]-df[col].mean())/df[col].std()\n", 1088 | "\n", 1089 | "data_st=data.copy()\n", 1090 | "for i in (data_st.iloc[:,:-1].columns):\n", 1091 | " standardize (data_st,i)\n", 1092 | "\n", 1093 | "data_st.head()" 1094 | ] 1095 | }, 1096 | { 1097 | "cell_type": "code", 1098 | "execution_count": 8, 1099 | "metadata": { 1100 | "execution": { 1101 | "iopub.execute_input": "2022-10-05T15:14:56.880118Z", 1102 | "iopub.status.busy": "2022-10-05T15:14:56.879350Z", 1103 | "iopub.status.idle": "2022-10-05T15:14:56.885543Z", 1104 | "shell.execute_reply": "2022-10-05T15:14:56.886203Z", 1105 | "shell.execute_reply.started": "2022-04-07T04:29:57.584389Z" 1106 | }, 1107 | "papermill": { 1108 | "duration": 0.024672, 1109 | "end_time": "2022-10-05T15:14:56.886411", 1110 | "exception": false, 1111 | "start_time": "2022-10-05T15:14:56.861739", 1112 | "status": "completed" 1113 | }, 1114 | "tags": [] 1115 | }, 1116 | "outputs": [ 1117 | { 1118 | "data": { 1119 | "text/plain": [ 1120 | "array([[-0.69243007, -0.50939981, -0.60154247, ..., -0.09854039,\n", 1121 | " -0.09526363, 0. ],\n", 1122 | " [-1.15018689, -0.61514895, -0.60166822, ..., -0.09854039,\n", 1123 | " -0.09526363, 0. ],\n", 1124 | " [-0.46808843, -0.50741826, -0.6016222 , ..., -0.09854039,\n", 1125 | " -0.09526363, 0. ],\n", 1126 | " ...,\n", 1127 | " [ 1.48184073, 0.89029963, 1.99298099, ..., -0.09854039,\n", 1128 | " -0.09526363, 0. ],\n", 1129 | " [-1.15018689, -0.61514895, -0.60166822, ..., -0.09854039,\n", 1130 | " -0.09526363, 0. ],\n", 1131 | " [ 0.1955008 , 1.45423145, 1.97682988, ..., -0.09854039,\n", 1132 | " -0.09526363, 0. ]])" 1133 | ] 1134 | }, 1135 | "execution_count": 8, 1136 | "metadata": {}, 1137 | "output_type": "execute_result" 1138 | } 1139 | ], 1140 | "source": [ 1141 | "#training data for the neural net\n", 1142 | "train_data_st=data_st.values\n", 1143 | "train_data_st" 1144 | ] 1145 | }, 1146 | { 1147 | "cell_type": "code", 1148 | "execution_count": 9, 1149 | "metadata": { 1150 | "execution": { 1151 | "iopub.execute_input": "2022-10-05T15:14:56.918726Z", 1152 | "iopub.status.busy": "2022-10-05T15:14:56.917973Z", 1153 | "iopub.status.idle": "2022-10-05T15:14:56.928243Z", 1154 | "shell.execute_reply": "2022-10-05T15:14:56.927610Z", 1155 | "shell.execute_reply.started": "2022-04-07T04:29:57.594061Z" 1156 | }, 1157 | "papermill": { 1158 | "duration": 0.027812, 1159 | "end_time": "2022-10-05T15:14:56.928422", 1160 | "exception": false, 1161 | "start_time": "2022-10-05T15:14:56.900610", 1162 | "status": "completed" 1163 | }, 1164 | "tags": [] 1165 | }, 1166 | "outputs": [ 1167 | { 1168 | "data": { 1169 | "text/plain": [ 1170 | "array([[0, 0, 0, ..., 1, 0, 0],\n", 1171 | " [0, 0, 0, ..., 0, 0, 0],\n", 1172 | " [0, 0, 0, ..., 0, 0, 0],\n", 1173 | " ...,\n", 1174 | " [0, 0, 0, ..., 0, 0, 0],\n", 1175 | " [0, 0, 0, ..., 0, 0, 0],\n", 1176 | " [0, 0, 0, ..., 0, 1, 0]], dtype=uint8)" 1177 | ] 1178 | }, 1179 | "execution_count": 9, 1180 | "metadata": {}, 1181 | "output_type": "execute_result" 1182 | } 1183 | ], 1184 | "source": [ 1185 | "#labels for training\n", 1186 | "labels=labels_full.values\n", 1187 | "labels" 1188 | ] 1189 | }, 1190 | { 1191 | "cell_type": "markdown", 1192 | "metadata": { 1193 | "papermill": { 1194 | "duration": 0.013845, 1195 | "end_time": "2022-10-05T15:14:56.956846", 1196 | "exception": false, 1197 | "start_time": "2022-10-05T15:14:56.943001", 1198 | "status": "completed" 1199 | }, 1200 | "tags": [] 1201 | }, 1202 | "source": [ 1203 | "### Keras model" 1204 | ] 1205 | }, 1206 | { 1207 | "cell_type": "code", 1208 | "execution_count": 10, 1209 | "metadata": { 1210 | "execution": { 1211 | "iopub.execute_input": "2022-10-05T15:14:56.998040Z", 1212 | "iopub.status.busy": "2022-10-05T15:14:56.997144Z", 1213 | "iopub.status.idle": "2022-10-05T15:19:04.412842Z", 1214 | "shell.execute_reply": "2022-10-05T15:19:04.411069Z", 1215 | "shell.execute_reply.started": "2022-04-07T04:29:57.604533Z" 1216 | }, 1217 | "papermill": { 1218 | "duration": 247.441846, 1219 | "end_time": "2022-10-05T15:19:04.412992", 1220 | "exception": false, 1221 | "start_time": "2022-10-05T15:14:56.971146", 1222 | "status": "completed" 1223 | }, 1224 | "tags": [] 1225 | }, 1226 | "outputs": [ 1227 | { 1228 | "name": "stdout", 1229 | "output_type": "stream", 1230 | "text": [ 1231 | "Epoch 1/500\n", 1232 | "3968/3968 - 5s - loss: 1.2323 - val_loss: 0.9187\n", 1233 | "Epoch 2/500\n", 1234 | "3968/3968 - 5s - loss: 0.7908 - val_loss: 0.6809\n", 1235 | "Epoch 3/500\n", 1236 | "3968/3968 - 5s - loss: 0.6322 - val_loss: 0.5669\n", 1237 | "Epoch 4/500\n", 1238 | "3968/3968 - 5s - loss: 0.5461 - val_loss: 0.5487\n", 1239 | "Epoch 5/500\n", 1240 | "3968/3968 - 5s - loss: 0.4919 - val_loss: 0.5169\n", 1241 | "Epoch 6/500\n", 1242 | "3968/3968 - 5s - loss: 0.4496 - val_loss: 0.4348\n", 1243 | "Epoch 7/500\n", 1244 | "3968/3968 - 5s - loss: 0.4177 - val_loss: 0.4073\n", 1245 | "Epoch 8/500\n", 1246 | "3968/3968 - 5s - loss: 0.3915 - val_loss: 0.3897\n", 1247 | "Epoch 9/500\n", 1248 | "3968/3968 - 5s - loss: 0.3699 - val_loss: 0.4252\n", 1249 | "Epoch 10/500\n", 1250 | "3968/3968 - 5s - loss: 0.3524 - val_loss: 0.3993\n", 1251 | "Epoch 11/500\n", 1252 | "3968/3968 - 5s - loss: 0.3503 - val_loss: 0.3462\n", 1253 | "Epoch 12/500\n", 1254 | "3968/3968 - 5s - loss: 0.3290 - val_loss: 0.3097\n", 1255 | "Epoch 13/500\n", 1256 | "3968/3968 - 5s - loss: 0.3200 - val_loss: 0.3218\n", 1257 | "Epoch 14/500\n", 1258 | "3968/3968 - 5s - loss: 0.3090 - val_loss: 0.2964\n", 1259 | "Epoch 15/500\n", 1260 | "3968/3968 - 5s - loss: 0.3086 - val_loss: 0.3308\n", 1261 | "Epoch 16/500\n", 1262 | "3968/3968 - 5s - loss: 0.2947 - val_loss: 0.3019\n", 1263 | "Epoch 17/500\n", 1264 | "3968/3968 - 5s - loss: 0.2840 - val_loss: 0.2678\n", 1265 | "Epoch 18/500\n", 1266 | "3968/3968 - 5s - loss: 0.2759 - val_loss: 0.2680\n", 1267 | "Epoch 19/500\n", 1268 | "3968/3968 - 5s - loss: 0.2693 - val_loss: 0.2560\n", 1269 | "Epoch 20/500\n", 1270 | "3968/3968 - 5s - loss: 0.2614 - val_loss: 0.2568\n", 1271 | "Epoch 21/500\n", 1272 | "3968/3968 - 5s - loss: 0.2569 - val_loss: 0.2503\n", 1273 | "Epoch 22/500\n", 1274 | "3968/3968 - 5s - loss: 0.2531 - val_loss: 0.2443\n", 1275 | "Epoch 23/500\n", 1276 | "3968/3968 - 5s - loss: 0.2424 - val_loss: 0.2349\n", 1277 | "Epoch 24/500\n", 1278 | "3968/3968 - 5s - loss: 0.2398 - val_loss: 0.2374\n", 1279 | "Epoch 25/500\n", 1280 | "3968/3968 - 5s - loss: 0.2291 - val_loss: 0.2273\n", 1281 | "Epoch 26/500\n", 1282 | "3968/3968 - 5s - loss: 0.2236 - val_loss: 0.2128\n", 1283 | "Epoch 27/500\n", 1284 | "3968/3968 - 5s - loss: 0.2157 - val_loss: 0.2180\n", 1285 | "Epoch 28/500\n", 1286 | "3968/3968 - 5s - loss: 0.2114 - val_loss: 0.2153\n", 1287 | "Epoch 29/500\n", 1288 | "3968/3968 - 5s - loss: 0.2084 - val_loss: 0.2063\n", 1289 | "Epoch 30/500\n", 1290 | "3968/3968 - 5s - loss: 0.2037 - val_loss: 0.1936\n", 1291 | "Epoch 31/500\n", 1292 | "3968/3968 - 5s - loss: 0.1981 - val_loss: 0.2067\n", 1293 | "Epoch 32/500\n", 1294 | "3968/3968 - 5s - loss: 0.1978 - val_loss: 0.2410\n", 1295 | "Epoch 33/500\n", 1296 | "3968/3968 - 5s - loss: 0.1920 - val_loss: 0.2013\n", 1297 | "Epoch 34/500\n", 1298 | "3968/3968 - 5s - loss: 0.1923 - val_loss: 0.1889\n", 1299 | "Epoch 35/500\n", 1300 | "3968/3968 - 5s - loss: 0.1933 - val_loss: 0.3492\n", 1301 | "Epoch 36/500\n", 1302 | "3968/3968 - 5s - loss: 0.1870 - val_loss: 0.2599\n", 1303 | "Epoch 37/500\n", 1304 | "3968/3968 - 5s - loss: 0.1846 - val_loss: 0.1759\n", 1305 | "Epoch 38/500\n", 1306 | "3968/3968 - 5s - loss: 0.1824 - val_loss: 0.1773\n", 1307 | "Epoch 39/500\n", 1308 | "3968/3968 - 5s - loss: 0.1822 - val_loss: 0.3068\n", 1309 | "Epoch 40/500\n", 1310 | "3968/3968 - 5s - loss: 0.1788 - val_loss: 0.2040\n", 1311 | "Epoch 41/500\n", 1312 | "3968/3968 - 5s - loss: 0.1779 - val_loss: 0.2408\n", 1313 | "Epoch 42/500\n", 1314 | "3968/3968 - 5s - loss: 0.1770 - val_loss: 0.1713\n", 1315 | "Epoch 43/500\n", 1316 | "3968/3968 - 5s - loss: 0.1759 - val_loss: 0.1892\n", 1317 | "Epoch 44/500\n", 1318 | "3968/3968 - 5s - loss: 0.1726 - val_loss: 0.1999\n", 1319 | "Epoch 45/500\n", 1320 | "3968/3968 - 5s - loss: 0.1730 - val_loss: 0.1717\n", 1321 | "Epoch 46/500\n", 1322 | "3968/3968 - 5s - loss: 0.1716 - val_loss: 0.1740\n", 1323 | "Epoch 47/500\n", 1324 | "3968/3968 - 5s - loss: 0.1751 - val_loss: 0.3207\n", 1325 | "Epoch 00047: early stopping\n" 1326 | ] 1327 | }, 1328 | { 1329 | "data": { 1330 | "text/plain": [ 1331 | "" 1332 | ] 1333 | }, 1334 | "execution_count": 10, 1335 | "metadata": {}, 1336 | "output_type": "execute_result" 1337 | } 1338 | ], 1339 | "source": [ 1340 | "#import libraries\n", 1341 | "from sklearn.model_selection import train_test_split\n", 1342 | "from sklearn import metrics\n", 1343 | "from tensorflow.keras.models import Sequential\n", 1344 | "from tensorflow.keras.layers import Dense, Activation\n", 1345 | "from tensorflow.keras.callbacks import EarlyStopping\n", 1346 | "\n", 1347 | "\n", 1348 | "# test/train split 25% test\n", 1349 | "x_train_st, x_test_st, y_train_st, y_test_st = train_test_split(\n", 1350 | " train_data_st, labels, test_size=0.25, random_state=42)\n", 1351 | "\n", 1352 | "# create and fit model\n", 1353 | "model = Sequential()\n", 1354 | "model.add(Dense(10, input_dim=train_data_st.shape[1], activation='relu'))\n", 1355 | "model.add(Dense(40, input_dim=train_data_st.shape[1], activation='relu'))\n", 1356 | "model.add(Dense(10, input_dim=train_data_st.shape[1], activation='relu'))\n", 1357 | "model.add(Dense(1, kernel_initializer='normal'))\n", 1358 | "model.add(Dense(labels.shape[1],activation='softmax'))\n", 1359 | "model.compile(loss='categorical_crossentropy', optimizer='adam')\n", 1360 | "monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, \n", 1361 | " patience=5, verbose=1, mode='auto')\n", 1362 | "model.fit(x_train_st,y_train_st,validation_data=(x_test_st,y_test_st),\n", 1363 | " callbacks=[monitor],verbose=2,epochs=500)" 1364 | ] 1365 | }, 1366 | { 1367 | "cell_type": "code", 1368 | "execution_count": 11, 1369 | "metadata": { 1370 | "execution": { 1371 | "iopub.execute_input": "2022-10-05T15:19:04.504624Z", 1372 | "iopub.status.busy": "2022-10-05T15:19:04.503865Z", 1373 | "iopub.status.idle": "2022-10-05T15:19:05.427865Z", 1374 | "shell.execute_reply": "2022-10-05T15:19:05.426447Z", 1375 | "shell.execute_reply.started": "2022-04-07T04:33:03.563723Z" 1376 | }, 1377 | "papermill": { 1378 | "duration": 0.973144, 1379 | "end_time": "2022-10-05T15:19:05.428062", 1380 | "exception": false, 1381 | "start_time": "2022-10-05T15:19:04.454918", 1382 | "status": "completed" 1383 | }, 1384 | "tags": [] 1385 | }, 1386 | "outputs": [ 1387 | { 1388 | "name": "stdout", 1389 | "output_type": "stream", 1390 | "text": [ 1391 | "accuracy: 0.8896975425330813\n" 1392 | ] 1393 | } 1394 | ], 1395 | "source": [ 1396 | "# metrics\n", 1397 | "pred_st = model.predict(x_test_st)\n", 1398 | "pred_st = np.argmax(pred_st,axis=1)\n", 1399 | "y_eval_st = np.argmax(y_test_st,axis=1)\n", 1400 | "score_st = metrics.accuracy_score(y_eval_st, pred_st)\n", 1401 | "print(\"accuracy: {}\".format(score_st))" 1402 | ] 1403 | }, 1404 | { 1405 | "cell_type": "code", 1406 | "execution_count": 12, 1407 | "metadata": { 1408 | "execution": { 1409 | "iopub.execute_input": "2022-10-05T15:19:05.524988Z", 1410 | "iopub.status.busy": "2022-10-05T15:19:05.524182Z", 1411 | "iopub.status.idle": "2022-10-05T15:22:32.058158Z", 1412 | "shell.execute_reply": "2022-10-05T15:22:32.057477Z", 1413 | "shell.execute_reply.started": "2022-04-07T04:33:04.274592Z" 1414 | }, 1415 | "papermill": { 1416 | "duration": 206.587375, 1417 | "end_time": "2022-10-05T15:22:32.058326", 1418 | "exception": false, 1419 | "start_time": "2022-10-05T15:19:05.470951", 1420 | "status": "completed" 1421 | }, 1422 | "tags": [] 1423 | }, 1424 | "outputs": [ 1425 | { 1426 | "name": "stdout", 1427 | "output_type": "stream", 1428 | "text": [ 1429 | "Epoch 1/100\n", 1430 | "3968/3968 - 6s - loss: 1.1881 - val_loss: 0.8779\n", 1431 | "Epoch 2/100\n", 1432 | "3968/3968 - 6s - loss: 0.7484 - val_loss: 0.6438\n", 1433 | "Epoch 3/100\n", 1434 | "3968/3968 - 6s - loss: 0.5990 - val_loss: 0.5672\n", 1435 | "Epoch 4/100\n", 1436 | "3968/3968 - 5s - loss: 0.5026 - val_loss: 0.4730\n", 1437 | "Epoch 5/100\n", 1438 | "3968/3968 - 6s - loss: 0.4486 - val_loss: 0.3994\n", 1439 | "Epoch 6/100\n", 1440 | "3968/3968 - 5s - loss: 0.3963 - val_loss: 0.3910\n", 1441 | "Epoch 7/100\n", 1442 | "3968/3968 - 6s - loss: 0.3632 - val_loss: 0.3955\n", 1443 | "Epoch 8/100\n", 1444 | "3968/3968 - 6s - loss: 0.3304 - val_loss: 0.3233\n", 1445 | "Epoch 9/100\n", 1446 | "3968/3968 - 6s - loss: 0.2972 - val_loss: 0.2809\n", 1447 | "Epoch 10/100\n", 1448 | "3968/3968 - 6s - loss: 0.2647 - val_loss: 0.2606\n", 1449 | "Epoch 11/100\n", 1450 | "3968/3968 - 6s - loss: 0.2429 - val_loss: 0.2323\n", 1451 | "Epoch 12/100\n", 1452 | "3968/3968 - 5s - loss: 0.2218 - val_loss: 0.2115\n", 1453 | "Epoch 13/100\n", 1454 | "3968/3968 - 6s - loss: 0.2138 - val_loss: 0.2433\n", 1455 | "Epoch 14/100\n", 1456 | "3968/3968 - 5s - loss: 0.2023 - val_loss: 0.1998\n", 1457 | "Epoch 15/100\n", 1458 | "3968/3968 - 6s - loss: 0.2120 - val_loss: 0.2034\n", 1459 | "Epoch 16/100\n", 1460 | "3968/3968 - 6s - loss: 0.1921 - val_loss: 0.2015\n", 1461 | "Epoch 17/100\n", 1462 | "3968/3968 - 6s - loss: 0.1858 - val_loss: 0.1925\n", 1463 | "Epoch 18/100\n", 1464 | "3968/3968 - 6s - loss: 0.1806 - val_loss: 0.1924\n", 1465 | "Epoch 19/100\n", 1466 | "3968/3968 - 5s - loss: 0.1802 - val_loss: 0.1782\n", 1467 | "Epoch 20/100\n", 1468 | "3968/3968 - 5s - loss: 0.1767 - val_loss: 0.1896\n", 1469 | "Epoch 21/100\n", 1470 | "3968/3968 - 5s - loss: 0.1785 - val_loss: 0.1711\n", 1471 | "Epoch 22/100\n", 1472 | "3968/3968 - 6s - loss: 0.1688 - val_loss: 0.1981\n", 1473 | "Epoch 23/100\n", 1474 | "3968/3968 - 6s - loss: 0.1880 - val_loss: 0.1748\n", 1475 | "Epoch 24/100\n", 1476 | "3968/3968 - 5s - loss: 0.1716 - val_loss: 0.1984\n", 1477 | "Epoch 25/100\n", 1478 | "3968/3968 - 6s - loss: 0.1717 - val_loss: 0.1659\n", 1479 | "Epoch 26/100\n", 1480 | "3968/3968 - 5s - loss: 0.1764 - val_loss: 0.1841\n", 1481 | "Epoch 27/100\n", 1482 | "3968/3968 - 6s - loss: 0.1733 - val_loss: 0.1690\n", 1483 | "Epoch 28/100\n", 1484 | "3968/3968 - 5s - loss: 0.1663 - val_loss: 0.1883\n", 1485 | "Epoch 29/100\n", 1486 | "3968/3968 - 6s - loss: 0.1683 - val_loss: 0.1612\n", 1487 | "Epoch 30/100\n", 1488 | "3968/3968 - 6s - loss: 0.1628 - val_loss: 0.1591\n", 1489 | "Epoch 31/100\n", 1490 | "3968/3968 - 6s - loss: 0.1700 - val_loss: 0.1649\n", 1491 | "Epoch 32/100\n", 1492 | "3968/3968 - 5s - loss: 0.1642 - val_loss: 0.1577\n", 1493 | "Epoch 33/100\n", 1494 | "3968/3968 - 6s - loss: 0.1636 - val_loss: 0.1613\n", 1495 | "Epoch 34/100\n", 1496 | "3968/3968 - 6s - loss: 0.1610 - val_loss: 0.1807\n", 1497 | "Epoch 35/100\n", 1498 | "3968/3968 - 5s - loss: 0.1630 - val_loss: 0.1717\n", 1499 | "Epoch 36/100\n", 1500 | "3968/3968 - 5s - loss: 0.1592 - val_loss: 0.1631\n", 1501 | "Epoch 37/100\n", 1502 | "3968/3968 - 6s - loss: 0.1633 - val_loss: 0.1754\n", 1503 | "Epoch 00037: early stopping\n" 1504 | ] 1505 | }, 1506 | { 1507 | "data": { 1508 | "text/plain": [ 1509 | "" 1510 | ] 1511 | }, 1512 | "execution_count": 12, 1513 | "metadata": {}, 1514 | "output_type": "execute_result" 1515 | } 1516 | ], 1517 | "source": [ 1518 | "#second model\n", 1519 | "model2 = Sequential()\n", 1520 | "model2.add(Dense(32, input_dim=train_data_st.shape[1], activation='relu'))\n", 1521 | "model2.add(Dense(72, input_dim=train_data_st.shape[1], activation='relu'))\n", 1522 | "model2.add(Dense(32, input_dim=train_data_st.shape[1], activation='relu'))\n", 1523 | "model2.add(Dense(1, kernel_initializer='normal'))\n", 1524 | "model2.add(Dense(labels.shape[1],activation='softmax'))\n", 1525 | "model2.compile(loss='categorical_crossentropy', optimizer='adam')\n", 1526 | "monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, \n", 1527 | " patience=5, verbose=1, mode='auto')\n", 1528 | "model2.fit(x_train_st,y_train_st,validation_data=(x_test_st,y_test_st),\n", 1529 | " callbacks=[monitor],verbose=2,epochs=100)" 1530 | ] 1531 | }, 1532 | { 1533 | "cell_type": "code", 1534 | "execution_count": 13, 1535 | "metadata": { 1536 | "execution": { 1537 | "iopub.execute_input": "2022-10-05T15:22:32.203016Z", 1538 | "iopub.status.busy": "2022-10-05T15:22:32.202309Z", 1539 | "iopub.status.idle": "2022-10-05T15:22:33.091440Z", 1540 | "shell.execute_reply": "2022-10-05T15:22:33.090675Z", 1541 | "shell.execute_reply.started": "2022-04-07T04:34:57.007747Z" 1542 | }, 1543 | "papermill": { 1544 | "duration": 0.962669, 1545 | "end_time": "2022-10-05T15:22:33.091570", 1546 | "exception": false, 1547 | "start_time": "2022-10-05T15:22:32.128901", 1548 | "status": "completed" 1549 | }, 1550 | "tags": [] 1551 | }, 1552 | "outputs": [ 1553 | { 1554 | "name": "stdout", 1555 | "output_type": "stream", 1556 | "text": [ 1557 | "accuracy: 0.902835538752363\n" 1558 | ] 1559 | } 1560 | ], 1561 | "source": [ 1562 | "# metrics\n", 1563 | "pred_st = model2.predict(x_test_st)\n", 1564 | "pred_st = np.argmax(pred_st,axis=1)\n", 1565 | "y_eval_st = np.argmax(y_test_st,axis=1)\n", 1566 | "score_st = metrics.accuracy_score(y_eval_st, pred_st)\n", 1567 | "print(\"accuracy: {}\".format(score_st))" 1568 | ] 1569 | } 1570 | ], 1571 | "metadata": { 1572 | "kernelspec": { 1573 | "display_name": "Python 3", 1574 | "language": "python", 1575 | "name": "python3" 1576 | }, 1577 | "language_info": { 1578 | "codemirror_mode": { 1579 | "name": "ipython", 1580 | "version": 3 1581 | }, 1582 | "file_extension": ".py", 1583 | "mimetype": "text/x-python", 1584 | "name": "python", 1585 | "nbconvert_exporter": "python", 1586 | "pygments_lexer": "ipython3", 1587 | "version": "3.7.6" 1588 | }, 1589 | "papermill": { 1590 | "duration": 489.114057, 1591 | "end_time": "2022-10-05T15:22:34.183000", 1592 | "environment_variables": {}, 1593 | "exception": null, 1594 | "input_path": "__notebook__.ipynb", 1595 | "output_path": "__notebook__.ipynb", 1596 | "parameters": {}, 1597 | "start_time": "2022-10-05T15:14:25.068943", 1598 | "version": "2.1.0" 1599 | } 1600 | }, 1601 | "nbformat": 4, 1602 | "nbformat_minor": 4 1603 | } 1604 | --------------------------------------------------------------------------------