├── .circleci
└── config.yml
├── .github
└── workflows
│ ├── readme-lint.yaml
│ ├── url-lint.yaml
│ └── yaml-lint.yaml
├── .gitignore
├── .mdlrc
├── Makefile
├── README.md
└── azure-pipelines.yml
/.circleci/config.yml:
--------------------------------------------------------------------------------
1 | # See: https://circleci.com/docs/configuration-reference
2 | version: 2.1
3 | jobs:
4 | test:
5 | docker:
6 | - image: cimg/base:stable
7 | steps:
8 | - checkout
9 | # Replace this with a real test runner invocation
10 | - run:
11 | name: Run tests
12 | command: echo 'replace me with real tests!'
13 | build:
14 | docker:
15 | - image: cimg/base:stable
16 | steps:
17 | - checkout
18 | # Replace this with steps to build a package, or executable
19 | - run:
20 | name: Build an artifact
21 | command: touch example.txt
22 | - store_artifacts:
23 | path: example.txt
24 | deploy:
25 | # This is an example deploy job, not actually used by the workflow
26 | docker:
27 | - image: cimg/base:stable
28 | steps:
29 | # Replace this with steps to deploy to users
30 | - run:
31 | name: deploy
32 | command: '#e.g. ./deploy.sh'
33 | workflows:
34 | myworkflow:
35 | jobs:
36 | - test
37 | # - build:
38 | # requires:
39 | # - test
40 | # - deploy:
41 | # requires:
42 | # - test
43 |
--------------------------------------------------------------------------------
/.github/workflows/readme-lint.yaml:
--------------------------------------------------------------------------------
1 | name: README lint
2 |
3 | on:
4 | - push
5 | - pull_request
6 |
7 | jobs:
8 | lint:
9 | runs-on: ubuntu-latest
10 | steps:
11 | - name: Check out code
12 | uses: actions/checkout@v2
13 | - name: Run mdl
14 | uses: actionshub/markdownlint@main
15 |
--------------------------------------------------------------------------------
/.github/workflows/url-lint.yaml:
--------------------------------------------------------------------------------
1 | name: URL lint
2 |
3 | on:
4 | - push
5 | - pull_request
6 |
7 | jobs:
8 | yaml_lint:
9 | name: YAML lint
10 | uses: HariSekhon/GitHub-Actions/.github/workflows/url_links.yaml@master
11 | with:
12 | url_links_ignored: |
13 | https://api.ipify.org
14 | https://signal.avg.com/hs-fs/hubfs
15 |
--------------------------------------------------------------------------------
/.github/workflows/yaml-lint.yaml:
--------------------------------------------------------------------------------
1 | name: YAML lint
2 |
3 | on:
4 | - push
5 | - pull_request
6 |
7 | jobs:
8 | yaml_lint:
9 | name: YAML lint
10 | uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master
11 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | .DS_Store
3 |
--------------------------------------------------------------------------------
/.mdlrc:
--------------------------------------------------------------------------------
1 | rules "~MD013", "~MD034", "~MD007", "~MD001", "~MD033"
2 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | # this is what executes if no argument is given
2 | lint:
3 | mdl README.md
4 |
5 | readme:
6 | mdl README.md
7 | git add README.md
8 | git commit -m "Updated readme" README.md
9 | git push
10 | make browse
11 |
12 | .PHONY: makefile
13 | makefile:
14 | git add Makefile
15 | git commit -m "Updated makefile" Makefile
16 | git push
17 | open https://github.com/MayaSekhon/DevOps-Tutorial/blob/main/Makefile
18 |
19 | browse:
20 | open https://github.com/MayaSekhon/DevOps-Tutorial
21 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Maya Sekhon - DevOps Zero-to-Hero Tutorial
2 |
3 | [](https://github.com/MayaSekhon/DevOps-Tutorial/actions/workflows/readme-lint.yaml)
4 | [](https://github.com/MayaSekhon/DevOps-Tutorial/actions/workflows/url-lint.yaml)
5 | [](https://github.com/MayaSekhon/DevOps-Tutorial/actions/workflows/yaml-lint.yaml)
6 | [](https://dl.circleci.com/status-badge/redirect/gh/MayaSekhon/DevOps-Zero-to-Hero-Tutorial/tree/main)
7 | [](https://github.com/MayaSekhon/DevOps-Tutorial/commits/main)
8 |
9 | [](https://dev.azure.com/mayavsekhon/Git/_git/DevOps-Tutorial)
10 | [](https://github.com/MayaSekhon/DevOps-Tutorial)
11 | [](https://gitlab.com/MayaSekhon/DevOps-Tutorial)
12 | [](https://bitbucket.org/mayasekhon/devops-tutorial/src/main/)
13 |
14 | ## Index
15 |
16 | - [DevOps Intro](#devops-intro)
17 | - [Linux / Unix Basics](#linux--unix-basics)
18 | - [Unix File System Layout](#unix-file-system-layout)
19 | - [Unix Shells](#unix-shells)
20 | - [Standard Input, Standard Output and Standard Error](#standard-input-standard-output-and-standard-error)
21 | - [Linux / Unix - Basic Commands](#linux--unix---basic-commands)
22 | - [File System](#file-system)
23 | - [String Processing](#string-processing)
24 | - [User Management](#user-management)
25 | - [Process Management](#process-management)
26 | - [Performance & Space Management](#performance--space-management)
27 | - [Networking](#networking)
28 | - [Logging & Tracing](#logging--tracing)
29 | - [Compression](#compression)
30 | - [Networking Basics](#networking-basics)
31 | - [SSH](#ssh)
32 | - [Editor / IDE](#editor--ide)
33 | - [Make / Makefiles](#make--makefiles)
34 | - [Git](#git)
35 | - [Git Commands](#git-commands)
36 | - [Example](#example)
37 | - [GitHub](#github)
38 | - [GitHub alternatives - GitLab, BitBucket, Azure DevOps](#github-alternatives---gitlab-bitbucket-azure-devops)
39 | - [CI/CD - Continuous Integration / Continuous Delivery](#cicd---continuous-integration--continuous-delivery)
40 | - [GitHub Actions](#github-actions)
41 | - [Azure DevOps Pipelines](#azure-devops-pipelines)
42 | - [CircleCI](#circleci)
43 | - [Travis CI](#travis-ci)
44 | - [Jenkins](#jenkins)
45 | - [Virtualization](#virtualization)
46 | - [Popular Virtualization Software](#popular-virtualization-software)
47 | - [Data Formats](#data-formats)
48 |
49 | ## DevOps Intro
50 |
51 | DevOps is short for Development Operations. It's a multi-disciplinary field combining programming and
52 | infrastructure to automate and speed up delivery of software.
53 |
54 | DevOps refers both to the wide variety of tools used, the engineers who operate them and to the methodology of closer collaboration between infrastructure and
55 | software engineering teams for quicker iterations.
56 |
57 | In the real world, DevOps practitioners typically come from a Linux background and favour open-source technologies which
58 | are free, to skip the need for licensing or paperwork such as purchase orders.
59 | Proprietary paid for software tools are used when no free alternatives are good enough. Cloud computing, although proprietary,
60 | is heavily underpinned by open-source technologies such that many of the open-source tools work the same and
61 | the compute-on-demand pay-as-you-go model means lower capex at the expense of higher opex and convenience.
62 |
63 |
64 |
65 | ## Linux / Unix Basics
66 |
67 | Linux is the standard open source operating system, based on Unix design.
68 |
69 | Linux is technically just the operating system kernel, while the many command line tools called Core Utils are typically provided by [GNU](https://www.gnu.org/home.en.html).
70 | Together they form a complete operating system called GNU / Linux which is further wrapped by Linux distributions
71 | which include easy installers and commands to download and install further software (software package management).
72 | The most popular Linux distributions include
73 | [Redhat](https://www.redhat.com/en)
74 | , [Debian](https://www.debian.org/)
75 | and [Ubuntu](https://ubuntu.com/)
76 | or are based on one of those.
77 | There are many more Linux distributions that fill specific niches.
78 |
79 | ### Unix File System Layout
80 |
81 | The [Filesystem Hierarchy Standard](https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard) defines the common layout you'll find across all unix style systems, including Linux.
82 |
83 | Everything is a file on Unix, even devices (found under `/dev`).
84 |
85 | Each open file gets its own file descriptor eg. `/dev/fd/`.
86 |
87 | Binaries & Libraries:
88 |
89 | | Directory | Description |
90 | |-------------------|----------------------------------------------------------------------------------------------------------------------------|
91 | | `/bin` | core binaries |
92 | | `/usr/bin` | more user binaries |
93 | | `/sbin` | system binaries |
94 | | `/usr/sbin` | more system binaries |
95 | | `/usr/local/bin` | 3rd party installed user binaries |
96 | | `/usr/local/sbin` | 3rd party installed system binaries |
97 | | `/lib` | libraries for binaries in `/bin` and `/sbin` |
98 | | `/usr/lib` | libraries for the binaries in `/usr/bin` and `/usr/sbin` |
99 | | `/usr/local/lib` | libraries for the binaries in `/usr/local/bin` and `/usr/local/sbin` |
100 | | `/opt` | another location for installing optional / 3rd party software, often used by major installation programs such as Oracle DB |
101 |
102 | Virtual File Systems:
103 |
104 | | Directory | Description |
105 | |-------------------|----------------------------------------------------------------------------------------------------------------------------|
106 | | `/dev` | device files representing every piece of hardware, disk, device, usb etc. |
107 | | `/proc` | process and kernel info exposed as virtual files |
108 | | `/sys` | system info exposed as virtual files |
109 |
110 | Home Directories:
111 |
112 | | Directory | Description |
113 | |-------------------|----------------------------------------------------------------------------------------------------------------------------|
114 | | `/root` | home directory for the root user |
115 | | `/home` | home directories for each user |
116 | | `/User` | home directories for each user on Mac instead of `/home` |
117 |
118 | System Configurations:
119 |
120 | | Directory | Description |
121 | |-------------------|----------------------------------------------------------------------------------------------------------------------------|
122 | | `/boot` | contains the Linux kernel and `initrd` used to boot the OS |
123 | | `/etc` | configuration files |
124 | | `/usr/local/etc` | 3rd party installed config files |
125 | | `/mnt` | mounted extra filesystems |
126 | | `/tmp` | temporary files (often wiped after shutdown) |
127 | | `/var/tmp` | more temporary runtime files |
128 | | `/var/cache` | temporarily cached files for running software, package manager lists |
129 | | `/var/log` | system log files |
130 |
131 | ### Unix Shells
132 |
133 | The Unix command line is extremely powerful and there are several shells to choose from.
134 | The default shell on Linux is Bash, which is based on the Bourne shell (Bash stands for Bourne again shell).
135 | Other popular shells include ZSH and Fish. Mac has in recent years switched the default shell to `zsh` to avoid GNU GPL open source licensing.
136 |
137 | The `root` account is the superuser admin account which always has UID 0.
138 | There should be no other account with UID 0 otherwise it would also be a root superuser account.
139 |
140 | ### Standard Input, Standard Output and Standard Error
141 |
142 | Standard Input is the data input that is streamed into any command via an interactive prompt, or a pipe (`|`) from another command, or an input redirect (`<`) from a file.
143 | Commands that expect standard input like `cat` or `grep`, if not given file arguments, will hang waiting until they receive some input.
144 |
145 | Standard Output is the main output stream from any command that is printed to the terminal unless you redirect it into a pipe (`|`) to another command's Standard Input, or an output redirect (`>`/ `>>`) to a file.
146 |
147 | Standard Error is the secondary output where only errors or logs are sent. This is also printed to the terminal unless redirected via `2>` / `2>>`
148 |
149 | The standard file descriptors for Standard Input, Standard Output and Standard Error are 0, 1 and 2 respectively and can be used at the command line via their file descriptors at `/dev/fd/0`, `/dev/fd/1`, `dev/fd/2`.
150 |
151 | Most Unix commands often work as standard Unix filter programs where they can accept input from Standard Input or read the contents of a file given as an argument and they output to Standard Output so that they can be chained into any number of subsequent commands separated by pipe symbols.
152 |
153 | This combined with the rich ecosystem of bundled Unix commands is the true power of Unix shells.
154 |
155 | ## Linux / Unix - Basic Commands
156 |
157 | Open a terminal. On Mac open `Terminal`, on Linux open `xterm` or `aterm`. These are applications in which to run
158 | your shell in a window inside your GUI.
159 |
160 | To find a command, it must be in the `$PATH`. You will likely need to extend the path to include custom installation directories like so:
161 |
162 | ```shell
163 | export PATH="$PATH:/path/to/some/directory"
164 | ```
165 |
166 | Common switches are `-h` / `--help`, `-v` / `--verbose`, `-V` / `--version`. The long options with `--` are typically GNU convention
167 |
168 | For more detailed help, type `man `. To search for manual pages run `man -k `.
169 |
170 | | Command | Description |
171 | |-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
172 | | `echo` | prints a given string argument to standard output |
173 | | `clear` | clears the terminal screen, leaving your cursor at the top (`ctrl-l` is a shortcut) |
174 | | `tmux` | terminal multiplexer - runs multiple shells in your terminal window and preserves your shell sessions if your terminal crashes or is accidentally closed |
175 | | `which` | shows the full path to a given command |
176 | | `type` | similar to `which` but finds shell built-in commands |
177 | | `grep` | filters from standard input or a file and only prints to standard output lines that match the given regex filter argument |
178 | | `pbcopy` | copies Standard Input to the GUI clipboard on Mac |
179 | | `pbpaste` | pastes the GUI clipboard to Standard Output on Mac |
180 | | `xclip` | copies Standard Input to the GUI (X) clipboard on Linux |
181 | | `env` | shows environment variables or sets environment variables and runs commands |
182 | | `set` | sets shell options such as `set -e` (usually used in scripts), or without args shows everything defined in the shell such as environment variables, aliases and functions |
183 | | `history` | shows list of commands previously executed in your shell |
184 | | `diff` | compares files line by line, shows the differing lines |
185 | | `date` | shows date and time, sets date and time, or shows the date / time in the format specified by a strftime string |
186 | | `rsync` | transfers / synchronizes files or directories efficiently between two directories by comparing timestamps (or optionally checksums) and only copies the files that are newer than the destination |
187 | | `find` | finds files and directories, optionally perform commands on them, eg. `find . -name README.md` |
188 | | `xargs` | reads standard input and uses it as arguments to the given command eg. `\| xargs ` | |
189 | | `file` | shows the type of a given file eg. `ASCII text` or `POSIX tar archive` |
190 | | `tar` | creates or extracts tarballs (bundle archives of files / directories), usually used for backups eg. `tar cvfz my.tar.gz somedirectory` and `tar xvfz my.tar.gz` |
191 | | `dd` | Data Duplicator - copies raw data from one source to another. Can be used the clone hard drives, partitions or create files of given sizes from special pseudo-devices like `/dev/zero` eg. `dd if=/dev/zero of=/path/to/my/file bs=1024 count=1024` |
192 |
193 | #### File System
194 |
195 | | Command | Description |
196 | |-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
197 | | `pwd` | shows present working directory |
198 | | `cd` | change directory, `cd -` to jump to last directory, `cd` without args to jump to `$HOME` directory |
199 | | `ls -l` | list files and directories |
200 | | `cp` | copies files or directories |
201 | | `mv` | moves files or directories |
202 | | `rm` | deletes files / directories. Common switches are `-r` to recursively delete into directories, and `-f`/`--force` |
203 | | `mkdir` | creates a directory |
204 | | `rmdir` | deletes an empty directory, fails with an error if not empty, in which case you need to use `rm -r` to also delete the directory and its contents (files / subdirectories) |
205 | | `chown` | change ownership of files or directories |
206 | | `chgrp` | change group ownership of files or directories |
207 | | `chmod` | change file octal permissions eg. `chmod 755` |
208 | | `touch` | updates the modified timestamp of a file or creates an empty file if it doesn't exist |
209 | | `cat` | reads the contents of the file or standard input to standard output, your terminal if not redirected or piped to another command |
210 | | `head` | reads the first N lines of a file or standard input |
211 | | `tail` | reads the last N lines of a file or standard input |
212 | | `more` | a paging program that displays one screenfull at a time and allows you to scroll down through longer outputs such as standard output from piped commands or files |
213 | | `less` | a better replacement of `more` that allows you to scroll upwards as well as downwards |
214 | | `tree` | lists contents of directories in a tree-like format |
215 | | `>` | overwrite file |
216 | | `>>` | append file |
217 |
218 | #### String Processing
219 |
220 | | Command | Description |
221 | |----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
222 | | `tr` | replaces characters from standard input |
223 | | `sed` | stream editor - replaces strings or deletes from standard input via regex searches |
224 | | `awk` | text processing language, usually used for quick one-liners, also supports regex matches, can print numbered columns |
225 | | `cut` | cuts out selected portions of each line by bite, character or field eg. 1st and 3rd fields `cut -d ' ' -f 1,3` |
226 | | `column` | aligns input into vertically aligned columns, usually called as `column -t` |
227 | | `vi` | text editor, the classic Unix terminal text editor, doesn't require a GUI, almost universally available on every server. If you need to edit a config file on a server, you will need to use this or another terminal editor program |
228 | | `vim` | `vi` improved - fuller features replacement editor to `vi`. `vi` is more commonly available on minimalist systems such as some servers but `vim` package is available to install |
229 |
230 | #### User Management
231 |
232 | | Command | Description |
233 | |-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
234 | | `id` | shows the current or given user's UID, GID and group memberships |
235 | | `who` / `w` | display who is logged in |
236 | | `sudo` | assumes the permissions of `root` or a given `-u ` for the duration of the following command must be pre-approved in `/etc/sudoers`) conifiguration. Prompts for the current user's password and caches for subsequent `sudo` calls within 5 mins |
237 | | `su` | switch user, defaults to `root` if no user arg is given. Prompts for the target user's password and starts a new shell as that user |
238 | | `useradd` | creates a new user account |
239 | | `userdel` | deletes a user account |
240 | | `gpasswd` | administers the /etc/group and /etc/gshadow |
241 | | `pwgen` | generates a secure, random password of configurable length and complexity eg. `pwgen -1 20` |
242 |
243 | #### Process Management
244 |
245 | | Command | Description |
246 | |-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
247 | | `pgrep` | finds processes by regex |
248 | | `bg` | background run the suspended process from `ctrl-z` |
249 | | `ctrl-c` | kill / cancel the current foreground process in the shell |
250 | | `ctrl-z` | suspend the current foreground process, placing it into the background |
251 | | `jobs` | shows backgrounded jobs, suspended or running |
252 | | `ps` | shows running processes. Commonly called as `ps -ef` or `ps aux` to show all processes on a unix based system |
253 | | `fg` | foreground run the suspended process from `ctrl-z` |
254 | | `wait` | waits for all background processes to finish before returning the shell prompt |
255 | | `kill` | kills a process by PID or sends it a specific signal |
256 | | `killall` | same as above, by name |
257 | | `pkill` | same as above, by regex pattern matching name |
258 | | `nohup` | no hang up - lets a command keep running even if your shell is closed eg. broken ssh connection (this usually results in a HUP signal being sent to the process causing it to exit otherwise) |
259 |
260 | #### Performance & Space Management
261 |
262 | | Command | Description |
263 | |------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
264 | | `df` | disk free - shows disk space for one or all disks eg. `df -h` for human units, `df -h /` for disk space of root disk or `df -h .` for disk space in the current partition where you are (`$PWD`) |
265 | | `du` | disk used - counts the disk space used for given files or directories, eg. `du -h -s $HOME` to see how much space your home directory has taken in human-readable units eg. GB |
266 | | `time` | times how long a command takes |
267 | | `free` | shows total used and free RAM eg. `free -h` for human readable units |
268 | | `top` | shows live process information, usually sorted by CPU or RAM - most useful details are PID, CPU, RAM, USER and COMMAND |
269 | | `lsof` | list open files open files and directories, the processes which currently have them opened, along with the user and PID |
270 | | `vmstat` | virtual memory stats - shows RAM, CPU, disk I/O etc. |
271 | | `dstat` | similar to `vmstat` |
272 | | `lscpu` | shows number of CPUs, cores etc. |
273 | | `nproc` | the number of CPU cores available to the current process (could be less than the hardware if a limit has been applied to your user or process) |
274 |
275 | #### Networking
276 |
277 | | Command | Description |
278 | |-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
279 | | `hostname` | shows the hostname with domain (FQDN), use `-s` for short name without domain |
280 | | `ifconfig` | shows or configures network interfaces, usually used to show your IP address |
281 | | `ip` | similar to `ifconfig`, `ip addr` to show your IP address |
282 | | `ping` | sends an ICMP echo request to a device, expects an ICMP echo reply if online. The most basic networking test of connectivity |
283 | | `traceroute` | print the route packets take to network host, by sending a series of ICMP echo request with increasing TTLs so each router along the path rejects it and exposes its IP |
284 | | `route` | shows or configures the network routing cables eg. `route -n` |
285 | | `netstat` | shows the network connections, connected or listening ports. Commonly called as `netstat -an` or `netstat -lntpu` |
286 | | `host` | performs DNS lookup for a given hostname or FQDN |
287 | | `dig` | similar to `host` but more query options and returns more info |
288 | | `curl` | get a web page URL via HTTP(S) or send data eg. JSON to a web service in an HTTP(S) request |
289 | | `wget` | similar to curl, downloads web pages to local files by default, use `wget -O - ...` to output to stdout to emulate curl's behaviour on minimalist systems that don't have curl installed but have wget bundled inside the busybox shell, such as Alpine Linux |
290 | | `ssh` | Secure Shell - connects to a remote server and opens a shell |
291 | | `scp` | Secure Copy - copies a file to remote server via SSH |
292 | | `ssh-keygen` | generates an SSH public / private key pair for passwordless logins to remote ssh servers (see SSH section further down) |
293 | | `nmap` | Network Map - scans ports on a target server or range of IPs by sending TCP syn packets to check what ports are open, useful to find services on the network |
294 | | `netcat` / `nc` | opens sockets to local or remote systems, useful for testing open ports or text-based services eg. HTTP, SMTP etc. |
295 | | `iptables` | lists or modifies the Linux local IP firewall table eg. `iptable -nL` |
296 |
297 | #### Logging & Tracing
298 |
299 | | Command | Description |
300 | |--------------|----------------------------------------------------------------------------------------------------|
301 | | `dmesg` | shows system kernel logs |
302 | | `journalctl` | opens systemd logs |
303 | | `strace` | traces system calls and signals, eg. file open / read / close, network socket open / send / close |
304 | | `dtruss` | similar to `strace` but for Mac |
305 |
306 | #### Compression
307 |
308 | | Command | Description |
309 | |-------------|--------------------------------------------------------------------------------------------------------------------------------------------|
310 | | `gzip` | compresses files using the gzip compression algorithm, adds the `.gz` file extension |
311 | | `gunzip` | decompresses `.gz` files |
312 | | `bzip2` | compresses files using the bzip2 compression algorithm (more compression but slower) |
313 | | `bunzip2` | decompresses `.bz2` files |
314 | | `zless` | shows compressed or plain text files one screen at a time (pipes gzipped files through `gunzip` before opening in `less`) |
315 | | `bzless` | same as `zless` but for `bzip2` |
316 | | `zip` | creates zip compression archives |
317 | | `unzip` | extracts zip compression archives |
318 | | `md5sum` | generates md5 hash of a file's contents, or validates that a saved md5 checksum hash matches the hash computed for a given file's contents |
319 | | `md5` | same as above, on Mac |
320 | | `shasum` | computes the SHA-1 hash of a file's contents (a hex string that is unique to a given content input) |
321 | | `sha1sum` | same as above, on Mac |
322 | | `sha256sum` | same as above, with longer SHA-256 hash |
323 | | `sha512sum` | same as above, with longer SHA-512 hash |
324 |
325 | ## Networking Basics
326 |
327 | | Command | Description |
328 | |--------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
329 | | IP address | the unique address of the computer / device on a network. Most computers are still using IPv4 addresses in the format `1.2.3.4`, with the eventual intention of migrating to IPv6 |
330 | | Subnet mask / netmask | an address that when AND'd against the IP address, leaves just the network portion of the IP address |
331 | | Network address | the IP address range, used by every device to determine if the remote IP is on the local network or a remote network |
332 | | Switch | device with multiples ports connecting different computers on the local network |
333 | | Router | device which connects 2 or more networks together |
334 | | Gateway | a router that is sent traffic to forward on to other networks |
335 | | Default Gateway | the router that you send all your remote traffic to when you don't have a more specific router to send it to e.g. your home broadband router for all traffic going to the internet |
336 | | DNS - Domain Name System | software that translates host names and domain names into IP addresses for network connections to initiate. Every computer has a DNS client that queries DNS servers on the internet that is used everytime you put a URL into your web browser. Popular DNS server software includes Bind, Microsoft DNS, DNSmasq, djbdns |
337 | | Hostname | the name of your computer on the local network |
338 | | Domain Name | the address suffix used to group websites and email addresses eg. google.com |
339 | | FQDN - Fully Qualified Domain Name | the complete host name and domain address eg. www.google.com |
340 | | URL - Uniform Resource Locator | the full path to a website's webpage eg. https://linkedin.com/in/maya-sekhon |
341 | | TCP - Transmission Control Protocol | connection-based protocol, retransmits lost packets, delivers in sequence, used underneath many other protocols eg. HTTP, SSH |
342 | | UDP - User Datagram Protocol | connectionless protocol, less reliable but lower overhead so faster, doesn't detect packet loss or retransmit or guarantee sequence order. Apps have to manage packet loss and retries themselves. DNS is an example of a UDP-based protocol where the DNS client software retries itself. Other examples include NFS, Video streaming, VoIP (Voice over IP) |
343 | | IP - Internet Protocol | the standard for addressing internet connected devices and routing packets between them |
344 | | IPv4 | the most widely used version of IP with 32-bit addresses usually written in 4 octet format from 0 - 255 eg. 4.2.2.1 or 192.168.1.10. There are ~4.3 billion IPv4 addresses (which are running out) |
345 | | IPv6 | newer format with more IP addresses available due to 128-bit format written in hexadecimal separated by colons Used because IPv4 only has 4 billion IP addresses which are running out |
346 | | Public vs Private IPs | in IPv4, due to limited IPs, three address ranges are reserved for private local network use only. One of these three ranges are used for all home and local office networks. Communicating on the internet requires a public IP
- Private IP address ranges are as follows:
- Class A Private IP Range: 10.0.0.0/8 ie. 10.0.0.0 – 10.255.255.255
- Class B Private IP Range: 172.168.0.0/12 ie. 172.16.0.0 – 172.31.255.255
- Class C Private IP Range: 192.168.0.0/16 ie. 192.168.0.0 – 192.168.255.255 |
347 | | CIDR - Classless Inter-Domain Routing | short way of representing a network combining an IP / netmask bits eg. 10.0.0.0 and 255.0.0.0 as 10.0.0.0/8 where the 8 represents an 8-bit netmask which is the same as 255.0.0.0 |
348 | | NAT - Network Address Translation | local router converts your local devices private IP address to the public IP of the router before sending on to the internet so that servers can reply to the router which forwards the IP packets back to the local device on its private IP |
349 | | Static vs Dynamic IPs | IP addresses can be configured manually or automatically via DHCP |
350 | | DHCP - Dynamic Host Configuration Protocol | client device broadcasts requesting an IP address using its DHCP client software. DHCP server hears the broadcast and replies with an IP address from its pool of configured IP addresses that the client device can use with a lease duration of typically 24 hours. After 24 hours, if the device is still online, it'll request to renew the lease, otherwise the lease will expire and the IP will be returned to the DHCP pool of available addresses |
351 | | MAC address - Media Access Control address | the physical network address on the network card, hardcoded, can be overridden in software |
352 | | OSI model - Open Systems Interconnection | seven-layer reference for responsibilities of each component in networking |
353 | | Firewall | restricts traffic inbound / outbound to protect computers from untrusted networks. Layer 4 firewalls restrict based on IP + Port number combinations. Layer 7 firewalls filter based on application layer protocol knowledge such as HTTP / paths or protocol abuses |
354 | | Web Application Firewall | layer 7 firewall that understands HTTP traffic and blocks common attacks and protocol abuses such as SQL injection |
355 | | NAT Firewall | layer 4 firewall that permits outbound and matching replies only. Maintains a connection table in RAM of source IP:port and destination IP:port combinations so that only matching replies are permitted back into the network to the requesting source computer |
356 | | Port Forwarding | opens a port number on a layer 4 firewall to permit outside traffic to flow into a computer in the internal network eg. forward port 80 to a webserver behind the firewall |
357 | | Load Balancer | accepts traffic on a given port and forwards it to one of several servers, therefore spreading the load of multiple inbound connections between a preconfigured group of servers eg. a web farm. This allows a website to scale to millions of users by allowing many servers to answer HTTP requests. It also allows for high availibility because if any one webserver crashes, it will send traffic to the remaining webservers and not the broken one. It detects if any server in the web farm is broken by using a preconfigured health check, usually a HTTP request with an optional `/path` which it repeats every few seconds to detect if a webserver stops responding properly, in which case it marks it as failed until the webserver starts working properly and the health check passes |
358 |
359 | IPv4 Address Format:
360 |
361 | 
362 |
363 | IPv6 Address Format:
364 |
365 | 
366 |
367 | NAT - Network Address Translation:
368 |
369 | /Public-vs-local-IP-addresses.png?width=2640&name=Public-vs-local-IP-addresses.png)
370 |
371 | Find your public IP address via any of these commands:
372 |
373 | ```shell
374 | curl ifconfig.co
375 | ```
376 |
377 | ```shell
378 | curl ipinfo.io/ip
379 | ```
380 |
381 | ```shell
382 | curl api.ipify.org
383 | ```
384 |
385 | ```shell
386 | curl 'https://api.ipify.org?format=json'
387 | ```
388 |
389 | ## SSH
390 |
391 | SSH stands for Secure Shell. It is the standard for connecting into remote shells on other computers across the network.
392 | The connection is encrypted on port 22 and requires a remote username and password or SSH key.
393 | An SSH server must be running on the remote computer (`sshd`).
394 |
395 | ```shell
396 | ssh maya@somecomputer.domain.com
397 | ```
398 |
399 | ```shell
400 | ssh maya@192.168.1.2
401 | ```
402 |
403 | Create an ssh key
404 |
405 | ```shell
406 | ssh-keygen
407 | ```
408 |
409 | This generates a public and private key pair under `$HOME/.ssh/`,
410 | by default `id_rsa` for the secret private key and `id_rsa.pub` for the public key.
411 |
412 | Copy and paste the public key contents from `$HOME/.ssh/id_rsa.pub` into the `$HOME/.ssh/authorized_keys`
413 | in any computer you want to automatically log into without a password prompt, or any public service like GitHub.
414 |
415 | The public key is safe to send to colleagues via emails etc because you cannot derive the secret private key from it due to one-way asymmetric cryptography, so that they can add you into their servers authorised keys.
416 |
417 | ## Editor / IDE
418 |
419 | Get yourself a good IDE (text editor with fancy features like autocomplete, syntax highlighting, version control etc.)
420 |
421 | There are many to choose from, if you don't already have a favourite one just go with [Intellij IDEA](https://www.jetbrains.com/idea/) community edition (free).
422 |
423 | If you're on Mac and want to be able to open files from the command line using the `idea` command from your shell, you will need to add it to the path:
424 |
425 | ```shell
426 | export PATH="$PATH:/Applications/IntelliJ IDEA CE.app/Contents/MacOS"
427 | ```
428 |
429 | ## Make / Makefiles
430 |
431 | Make is the classic, standard build system that executes the script contents of `Makefile`
432 | in the current directory when you execute the `make` command.
433 |
434 | For example, see the [Makefile](https://github.com/MayaSekhon/DevOps-Tutorial/blob/main/Makefile) in this repo.
435 |
436 | If you're using Intellij, remember to add the [Makefile plugin](https://plugins.jetbrains.com/plugin/9333-makefile-language) for syntax support.
437 |
438 | ## Git
439 |
440 | [Git](https://git-scm.com/) is a distributed version control system which saves every version of your software code and configuration files.
441 |
442 | This allows you to track all changes made over time made by yourself and your colleagues, and handles most merging
443 | of each other's changes as long as they're not on the same lines.
444 |
445 | ### Git Commands
446 |
447 | | Command | Description |
448 | |-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
449 | | `git init` | creates a new Git repository (creates `.git/` directory storing your file changes and metadata) |
450 | | `git clone` | clones a repo locally from an upstream server, such as GitHub |
451 | | `git add` | marks files to be committed |
452 | | `git mv` | moves or renames files (necessary for Git to track the move or rename, don't just use Unix `mv`) |
453 | | `git diff` | shows you uncommitted changes made to files |
454 | | `git commit` | saves selected changed files in a new version hashref (a unique alphanumeric string representing this set of changes) |
455 | | `git push` | pushes your local commits to an upstream shared repository such as GitHub |
456 | | `git pull` | pulls the latest commits from the upstream shared repository |
457 | | `git branch` | show branches / create a new branch |
458 | | `git checkout` | checks out the files at a given ref (branch / tag / hashref) |
459 | | `git merge` | merges a given branch into the current branch |
460 | | `git tag` | creates a tag for the current commit hashref easy to use human name or version number eg. v1.2.3 |
461 | | `git log` | shows the git log eg. `git log --all --graph --decorate` to see multi-branch history merges etc. or `git log -p` to see the patch diffs of each commit in the history |
462 | | `git show` | shows the diff at a specific commit |
463 | | `git config` | configures settings for the local repo or global user settings eg. `git config --global user.name` or `git cnofig --global user.email` |
464 | | `git remote -v` | shows the remote repos and their URLs |
465 |
466 | `.gitignore` - file listing paths to ignore, one per line, can be set globally in your home directory or in the root top level directory of the repository
467 |
468 | #### Example
469 |
470 | ```shell
471 | git clone git@github.com:MayaSekhon/DevOps-Zero-to-Hero-Tutorial
472 |
473 | # edit file
474 | vim README.md
475 |
476 | git diff
477 | git commit -m "updated readme"
478 |
479 | git push
480 | ```
481 |
482 | ### GitHub
483 |
484 | [GitHub](https://github.com/) is a website which stores your Git repositories and has nice management features as well as CI/CD.
485 |
486 | Make sure to enable Two-factor authentication (2FA) in your [security settings](https://github.com/settings/security). Get [Microsoft Authenticator app](https://www.microsoft.com/en-gb/security/mobile-authenticator-app).
487 |
488 | Copy and paste the contents of your `$HOME/.ssh/id_rsa.pub` into your profile [keys settings](https://github.com/settings/keys)
489 | (hint: `pbcopy < $HOME/.ssh/id_rsa.pub` on Mac to copy it straight into your clipboard).
490 |
491 | Use SSH for your git clone / pull / push because you should be using autogenerated complex passwords that are stored in password managers such as [Chrome](https://www.google.com/intl/en_uk/chrome/)
492 | / [Lastpass](https://www.lastpass.com/)
493 | / [1password](https://1password.com/), and have MFA enabled.
494 |
495 | If your organisation uses SSO enforced authentication for corporate controls via Azure Active Directory or similar IdP,
496 | then don't forget to authorize your SSH key for your enterprise GitHub organisation using the `Configure SSO` drop down
497 | to the right of the key.
498 |
499 | ### GitHub alternatives - GitLab, BitBucket, Azure DevOps
500 |
501 | These are all just Git repo hosting websites with CI/CD built in.
502 |
503 | - [Azure DevOps](https://azure.microsoft.com/en-gb/products/devops) - unlimited free CI/CD build minutes
504 | - [GitLab](https://about.gitlab.com/) - similar feature parity to GitHub, but few free CI/CD build minutes (legacy)
505 | - [BitBucket](https://bitbucket.org/product) - less feature rich and few free CI/CD build minutes (legacy)
506 |
507 | GitHub's advantages over these alternatives include:
508 |
509 | - most popular and widely used
510 | - feature rich repository management (rivalled only by GitLab)
511 | - huge ecosystem support and integrations
512 | - GitHub can be used for automatic SSO logins to many other 3rd party developer websites
513 | - better CI/CD, see [GitHub Actions section](#github-actions) further down
514 | - Pull Requests with extensive customization:
515 | - merge control behaviours enforcing peer review approvals
516 | - CI build / lint checks passed enforcement
517 | - CI can update / modify / comment on Pull Requests
518 |
519 | ## CI/CD - Continuous Integration / Continuous Delivery
520 |
521 | Continuous Integration means to automatically run any actions upon changes in the repo related to building artifacts, installing dependencies (eg. software libraries or OS packages), testing, linting, code quality checks etc.
522 |
523 | Continuous Delivery is the next step where the software is delivered eg. deployed to a server (eg. copied and executed to run a new version of a website or software or config).
524 |
525 | CI/CD is done via specialised software that watches your Git repo and automatically runs upon any changes to the files in the repo.
526 |
527 | There are many different CI/CD software tools available to fulfill this function. Some prominent ones include:
528 |
529 | - Cloud Hosted:
530 | - [GitHub Actions](https://github.com/features/actions)
531 | - [CircleCI](https://circleci.com/)
532 | - [GitLab CI/CD](https://docs.gitlab.com/ee/ci/)
533 | - [Azure DevOps Pipelines](https://azure.microsoft.com/en-gb/products/devops/pipelines)
534 | - [BitBucket Pipelines](https://bitbucket.org/product/features/pipelines)
535 | - [Travis CI](https://www.travis-ci.com/)
536 | - [AWS CodeBuild](https://aws.amazon.com/codebuild/)
537 | - [GCP CloudBuild](https://cloud.google.com/build)
538 | - Self Hosted (Install and run on your own server or computer):
539 | - [Jenkins](https://www.jenkins.io/)
540 | - [Concourse](https://concourse-ci.org/)
541 | - [TeamCity](https://www.jetbrains.com/teamcity/)
542 |
543 | and many others. For a more comprehensive list of badges of different CI, see [HariSekhon/CI-CD](https://github.com/HariSekhon/CI-CD) or https://harisekhon.netlify.app/.
544 |
545 | ### GitHub Actions
546 |
547 | CI/CD built into each GitHub repo, requires just dropping in a yaml file into `.github/workflows`.
548 |
549 | - unlimited free build minutes for public projects
550 | - [Marketplace](https://github.com/marketplace?type=actions) of pre-built actions
551 | - widespread support among 3rd party vendors providing ready made actions for their products such as various [SAST](https://en.wikipedia.org/wiki/Static_application_security_testing) tools
552 | - better designed CI/CD than other cloud hosted vendors eg. multi-file workflows, separate repo badges etc.
553 | - community pre-built reusable workflows are ready ro run, such as [HariSekhon/GitHub-Actions](https://github.com/HariSekhon/GitHub-Actions)
554 |
555 | For examples, see [.github/workflows](https://github.com/MayaSekhon/DevOps-Tutorial/tree/main/.github/workflows).
556 |
557 | For a master template, see the [HariSekhon/GitHub-Actions](https://github.com/HariSekhon/GitHub-Actions) repo [main.yaml](https://github.com/HariSekhon/GitHub-Actions/blob/master/main.yaml).
558 |
559 | ### Azure DevOps Pipelines
560 |
561 | Azure DevOps Pipelines is the built-in CI/CD available alongside repos. Simply add `azure-pipelines.yml` to the root of your repo and enable in the website.
562 |
563 | For a simple example, see the local [azure-pipelines.yml](/azure-pipelines.yml), or for a more real-world example,
564 | see DevOps-Bash-tools [azure-pipelines.yml](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/azure-pipelines.yml).
565 |
566 | ### CircleCI
567 |
568 | A well established, polished CI/CD solution with a nice GUI.
569 |
570 | For example, see the local [.circlci/config.yml](https://github.com/MayaSekhon/DevOps-Tutorial/blob/main/.circleci/config.yml)
571 | or the DevOps-Bash-tools [.circleci/config.yml](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.circleci/config.yml)
572 |
573 | ### Travis CI
574 |
575 | The original popular Cloud hosted CI/CD, no longer free for all public projects, considered legacy now.
576 |
577 | Configuration template in [HariSekhon/Templates](https://github.com/HariSekhon/Templates) repo [.travis.yml](https://github.com/HariSekhon/Templates/blob/master/.travis.yml)
578 |
579 | ### Jenkins
580 |
581 | The classic, most powerful and flexible CI/CD.
582 |
583 | Free open-source server software, written in Java. You must install, run, administer and update it yourself.
584 |
585 | Uses a lot of plugins to extend its core functionality.
586 |
587 | Ultra powerful but more difficult to manage because you have to
588 | administer the server yourself, including updating all your plugins, compared to Cloud hosted solutions like the above,
589 | which require no administration.
590 |
591 | Builds use a `Jenkinsfile` written in a DSL language, similar to code with braces and functions. See this master template [Jenkinsfile](https://github.com/HariSekhon/Jenkins/blob/master/Jenkinsfile) example.
592 |
593 | Very powerful and flexible because you can write your own functions in the excellent [Groovy](https://groovy-lang.org/)
594 | programming language. Many such functions can be found in the [HariSekhon/Jenkins](https://github.com/HariSekhon/Jenkins) repo.
595 |
596 | Jenkins can have many agents installed on other servers to run pipelines. Jenkins integrates with the fantastic Kubernetes
597 | platform to dynamically spawn agents in autoscaling Kubernetes clusters as needed. To quickly install Jenkins on Kubernetes
598 | with auto-spawning agents, see the [HariSekhon/Kubernetes-configs](https://github.com/HariSekhon/Kubernetes-configs) repo.
599 |
600 | A single Jenkins server will eventually hit performance and scalability limits in the server itself if coordinating and
601 | scheduling hundreds of pipelines across agents.
602 |
603 | [CloudBees](https://www.cloudbees.com/)
604 | provides commercial software to run and manage multiple Jenkins servers centrally. This is because large enterprises
605 | typically end up with many Jenkins installations for different teams and projects but want centralised control and governance.
606 |
607 | For real-world Jenkins architecture and screenshots see the [HariSekhon/Jenkins](https://github.com/HariSekhon/Jenkins) repo.
608 |
609 | ## Virtualization
610 |
611 | Virtualization allows installing multiple virtual machines on one physical machine, using a special piece of software
612 | called a hypervisor that emulates computer hardware. Each virtual machine (VM) believes it's on its own computer and only sees the
613 | virtual hardware presented by the hypervisor, but not any of the adjacent VMs.
614 | Each VM can have different operating systems and versions installed.
615 |
616 | The hypervisor allocates a fixed amount of computer resources to each VM eg. CPU, RAM, disk space.
617 |
618 | Virtualization is used to utilise large server hardware capacity while maintaining isolation between different applications.
619 | The alternative of trying to install and run many applications on a single bare metal server operating system would be difficult to manage,
620 | with potential clashes of software dependencies, versions and libraries, and insecure because any misbehaving or
621 | hacked application could potentially impact all of the other adjacent applications.
622 | Virtualization solves this by allowing each application to run on its own isolated operating system with resource limits,
623 | to prevent resource starvation by greedy adjacent apps.
624 |
625 | Virtualization is also used on desktop computers to able to run software that would otherwise not be able to run on the base operating system
626 | eg. Windows vs Mac vs Linux applications are specific to each operating system.
627 |
628 | Benefits of virtual machines include:
629 |
630 | - fuller utilisation of hardware capacity
631 | - isolation between apps
632 | - snapshots - can save the computer state at different points in time and can revert to a previous snapshot easily.
633 | This makes riskier changes and upgrades easy to roll back
634 | - migration - virtual machine disks are stored as files on the computer and can be exported, copied, imported to migrate between computers
635 | - appliances - virtual machine disks can be shared as ready-to-run installed computers with all software pre-installed and configured
636 | - failover - enterprise hypervisors can automatically migrate and restart VMs on other servers if the server
637 | running the VM dies for any reason.
638 | The VM disk files are kept on shared network storage, accessible to each server for this.
639 |
640 | ### Popular Virtualization Software
641 |
642 | - Server:
643 | - [VMware vSphere](https://www.vmware.com/uk/products/vsphere.html) - enterprise-grade mature software suite with centralised vCentre and ESXi server hypervisor, migration, failover etc.
644 | - [Microsoft Hyper-V](https://learn.microsoft.com/en-us/virtualization/hyper-v-on-windows/about/) on Windows Server - enterprise features like live VM migration between servers
645 | - Desktop:
646 | - [Virtualbox](https://www.virtualbox.org/) - widely popular and easy to use, but only for x86-based processors eg. Intel or AMD
647 | - [VMware Workstation](https://www.vmware.com/uk/products/workstation-pro.html) / [Fusion](https://www.vmware.com/uk/products/fusion.html) -
648 | desktop virtualization software for Windows / Mac
649 | - [Microsoft Hyper-V](https://learn.microsoft.com/en-us/virtualization/hyper-v-on-windows/about/) on Windows - desktop app hypervisor
650 | - [Qemu](https://www.qemu.org/) - massively versatile but complex and difficult to use, but supports a wider range of processors such as new Apple Silicon (ARM processor)
651 | - [UTM](https://mac.getutm.app/) - easy to use Mac frontend to Qemu for new Apple Silicon processors
652 |
653 | ## Data Formats
654 |
655 | | Format | Description |
656 | |--------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
657 | | [YAML](https://en.wikipedia.org/wiki/YAML) | Yet Another Markup Language - simple way of representing key value pairs, lists, dictionaries. Usually used for config files eg. [readme-lint.yaml](https://github.com/MayaSekhon/DevOps-Tutorial/blob/main/.github/workflows/readme-lint.yaml) |
658 | | [JSON](https://en.wikipedia.org/wiki/JSON) | JavaScript Object Notation - text-based data file written like a dictionary in code with braces, key-value pairs, lists, often used for data interchange between web services eg. `{ "name": "Maya", "hobbies": ["coding", "music"] }` |
659 | | [XML](https://en.wikipedia.org/wiki/XML) | Extensible Markup Language - text-based data file with a start \ and end \ (with a slash) surrounding each field eg. `Maya`, older format used for data interchange in older web services |
660 |
--------------------------------------------------------------------------------
/azure-pipelines.yml:
--------------------------------------------------------------------------------
1 | # Starter pipeline
2 | # Start with a minimal pipeline that you can customize to build and deploy your code.
3 | # Add steps that build, run tests, deploy, and more:
4 | # https://aka.ms/yaml
5 |
6 | trigger:
7 | - main
8 |
9 | pool:
10 | vmImage: ubuntu-latest
11 |
12 | steps:
13 | - script: echo Hello, world!
14 | displayName: 'Run a one-line script'
15 |
16 | - script: |
17 | echo Add other tasks to build, test, and deploy your project.
18 | echo See https://aka.ms/yaml
19 | displayName: 'Run a multi-line script'
20 |
--------------------------------------------------------------------------------