├── .github └── workflows │ ├── python.yml │ └── rust.yml ├── .gitignore ├── LICENSE ├── README.md ├── python ├── graphics.py ├── http.py ├── requirements.txt └── tests │ ├── __init__.py │ └── test_http.py └── rust ├── .gitignore ├── Cargo.toml └── src ├── lib.rs └── main.rs /.github/workflows/python.yml: -------------------------------------------------------------------------------- 1 | name: Python 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | defaults: 13 | run: 14 | working-directory: ./python 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 3.9 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: 3.9 21 | - name: Install requirements 22 | run: pip install -r requirements.txt 23 | - name: Install Black 24 | run: pip install black pytest 25 | - name: Run black --check . 26 | run: black --check . 27 | - name: Test with pytest 28 | run: pytest -v 29 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | defaults: 13 | run: 14 | working-directory: ./rust 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: apt-get install packages 18 | run: sudo apt-get install libgtk-3-dev 19 | - name: Build 20 | run: cargo build --verbose 21 | - name: Format 22 | run: cargo fmt --all -- --check 23 | - name: Lint 24 | run: cargo clippy 25 | - name: Test 26 | run: cargo test -v 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # Byte-compiled / optimized / DLL files 14 | __pycache__/ 15 | *.py[cod] 16 | *$py.class 17 | 18 | # C extensions 19 | *.so 20 | 21 | # Distribution / packaging 22 | .Python 23 | build/ 24 | develop-eggs/ 25 | dist/ 26 | downloads/ 27 | eggs/ 28 | .eggs/ 29 | lib/ 30 | lib64/ 31 | parts/ 32 | sdist/ 33 | var/ 34 | wheels/ 35 | share/python-wheels/ 36 | *.egg-info/ 37 | .installed.cfg 38 | *.egg 39 | MANIFEST 40 | 41 | # PyInstaller 42 | # Usually these files are written by a python script from a template 43 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 44 | *.manifest 45 | *.spec 46 | 47 | # Installer logs 48 | pip-log.txt 49 | pip-delete-this-directory.txt 50 | 51 | # Unit test / coverage reports 52 | htmlcov/ 53 | .tox/ 54 | .nox/ 55 | .coverage 56 | .coverage.* 57 | .cache 58 | nosetests.xml 59 | coverage.xml 60 | *.cover 61 | *.py,cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | cover/ 65 | 66 | # Translations 67 | *.mo 68 | *.pot 69 | 70 | # Django stuff: 71 | *.log 72 | local_settings.py 73 | db.sqlite3 74 | db.sqlite3-journal 75 | 76 | # Flask stuff: 77 | instance/ 78 | .webassets-cache 79 | 80 | # Scrapy stuff: 81 | .scrapy 82 | 83 | # Sphinx documentation 84 | docs/_build/ 85 | 86 | # PyBuilder 87 | .pybuilder/ 88 | target/ 89 | 90 | # Jupyter Notebook 91 | .ipynb_checkpoints 92 | 93 | # IPython 94 | profile_default/ 95 | ipython_config.py 96 | 97 | # pyenv 98 | # For a library or package, you might want to ignore these files since the code is 99 | # intended to run in multiple environments; otherwise, check them in: 100 | # .python-version 101 | 102 | # pipenv 103 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 104 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 105 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 106 | # install all needed dependencies. 107 | #Pipfile.lock 108 | 109 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 110 | __pypackages__/ 111 | 112 | # Celery stuff 113 | celerybeat-schedule 114 | celerybeat.pid 115 | 116 | # SageMath parsed files 117 | *.sage.py 118 | 119 | # Environments 120 | .env 121 | .venv 122 | env/ 123 | venv/ 124 | ENV/ 125 | env.bak/ 126 | venv.bak/ 127 | 128 | # Spyder project settings 129 | .spyderproject 130 | .spyproject 131 | 132 | # Rope project settings 133 | .ropeproject 134 | 135 | # mkdocs documentation 136 | /site 137 | 138 | # mypy 139 | .mypy_cache/ 140 | .dmypy.json 141 | dmypy.json 142 | 143 | # Pyre type checker 144 | .pyre/ 145 | 146 | # pytype static type analyzer 147 | .pytype/ 148 | 149 | # Cython debug symbols 150 | cython_debug/ 151 | 152 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 rust-kr 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web Browser Engineering 2 | 3 | This is a port of [Web Browser Engineering](https://browser.engineering/) series from Python to Rust done by Korean Rust User Group. 4 | 5 | # Table 6 | 7 | | Chapter | Author | 8 | |-----------------------|-----------| 9 | | Downloading Web Pages | @sanxiyn | 10 | | Drawing to the Screen | @corona10 | 11 | 12 | # What's changed 13 | 14 | | Library | Python | Rust | 15 | |---------|-------------------------------------------------------------|-------------------------------------------------| 16 | | TLS | [ssl](https://docs.python.org/3/library/ssl.html) | [rustls](https://github.com/ctz/rustls) | 17 | | GUI | [tkinter](https://docs.python.org/3/library/tkinter.html) | [druid](https://github.com/linebender/druid) | 18 | | gzip | [gzip](https://docs.python.org/3/library/gzip.html) | [flate2](https://github.com/rust-lang/flate2-rs)| 19 | | deflate | [zlib](https://docs.python.org/3/library/zlib.html) | [flate2](https://github.com/rust-lang/flate2-rs)| 20 | | brotli | [brotli](https://github.com/google/brotli) | TBD | 21 | -------------------------------------------------------------------------------- /python/graphics.py: -------------------------------------------------------------------------------- 1 | import tkinter 2 | 3 | import http 4 | 5 | 6 | WIDTH, HEIGHT = 800, 600 7 | HSTEP, VSTEP = 13, 18 8 | SCROLL_STEP = 100 9 | 10 | 11 | class Browser: 12 | def __init__(self): 13 | self.window = tkinter.Tk() 14 | self.scroll = 0 15 | self.min_scroll = 0 16 | self.max_scroll = 0 17 | self.window.title("Browser-engineering") 18 | self.window.bind("", self.scrollup) 19 | self.window.bind("", self.scrolldown) 20 | self.window.bind("", self.mousewheel) 21 | self.canvas = tkinter.Canvas(self.window, width=WIDTH, height=HEIGHT) 22 | self.canvas.pack() 23 | 24 | def load(self, url): 25 | headers, body = http.request(url) 26 | text = http.lex(body) 27 | self.display_list = self.layout(text) 28 | self.render() 29 | 30 | def layout(self, text): 31 | display_list = [] 32 | cursor_x, cursor_y = HSTEP, VSTEP 33 | for c in text: 34 | self.max_scroll = max(self.max_scroll, cursor_y) 35 | display_list.append((cursor_x, cursor_y, c)) 36 | cursor_x += HSTEP 37 | if cursor_x >= WIDTH - HSTEP or c == "\n": 38 | cursor_y += VSTEP 39 | cursor_x = HSTEP 40 | return display_list 41 | 42 | def render(self): 43 | self.canvas.delete("all") 44 | for x, y, c in self.display_list: 45 | if y > self.scroll + HEIGHT: 46 | continue 47 | if y + VSTEP < self.scroll: 48 | continue 49 | self.canvas.create_text(x, y - self.scroll, text=c) 50 | 51 | def scrolldown(self, e): 52 | self.scroll += SCROLL_STEP 53 | self.scroll = min(self.max_scroll, self.scroll) 54 | self.render() 55 | 56 | def scrollup(self, e): 57 | self.scroll -= SCROLL_STEP 58 | self.scroll = max(self.scroll, self.min_scroll) 59 | self.render() 60 | 61 | def mousewheel(self, e): 62 | if e.delta > 0: 63 | self.scroll -= SCROLL_STEP 64 | self.scroll = max(self.scroll, self.min_scroll) 65 | self.render() 66 | elif e.delta < 0: 67 | self.scroll += SCROLL_STEP 68 | self.scroll = min(self.max_scroll, self.scroll) 69 | self.render() 70 | 71 | 72 | if __name__ == "__main__": 73 | import sys 74 | 75 | Browser().load(sys.argv[1]) 76 | tkinter.mainloop() 77 | -------------------------------------------------------------------------------- /python/http.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import socket 3 | import ssl 4 | import sys 5 | import zlib 6 | 7 | try: 8 | import brotli 9 | except ImportError: 10 | brotli = None 11 | 12 | 13 | def request(url): 14 | # 1. Parse scheme 15 | scheme, url = url.split(":", 1) 16 | assert scheme in ["http", "https", "data"], f"Unknown scheme {scheme}" 17 | port = 80 if scheme == "http" else 443 18 | 19 | # Exercise data scheme 20 | if scheme == "data": 21 | content_type, body = url.split(",", 1) 22 | return {"content-type": content_type}, body 23 | 24 | # 2. Parse host 25 | host, path = url.removeprefix("//").split("/", 1) 26 | path = "/" + path 27 | 28 | # 3. Parse port 29 | if ":" in host: 30 | host, port = host.split(":", 1) 31 | port = int(port) 32 | 33 | # 4. Connect 34 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as sock: 35 | if scheme == "https": 36 | ctx = ssl.create_default_context() 37 | with ctx.wrap_socket(sock, server_hostname=host) as ssock: 38 | return _get_headers_and_body(ssock, host, port, path) 39 | return _get_headers_and_body(sock, host, port, path) 40 | 41 | 42 | def _get_headers_and_body(sock, host, port, path): 43 | sock.connect((host, port)) 44 | accept_encoding = ["gzip", "deflate"] 45 | if brotli: 46 | accept_encoding.append("br") 47 | accept_encoding = ",".join(accept_encoding) 48 | 49 | # 5. Send request 50 | sock.send(f"GET {path} HTTP/1.1\r\n".encode()) 51 | sock.send(f"Host: {host}\r\n".encode()) 52 | sock.send(f"Connection: close\r\n".encode()) 53 | sock.send(f"User-Agent: Mozilla/5.0 ({sys.platform})\r\n".encode()) 54 | sock.send(f"Accept-Encoding: {accept_encoding}\r\n".encode()) 55 | sock.send("\r\n".encode()) 56 | 57 | # 6. Receive response 58 | with sock.makefile("rb", newline="\r\n") as response: 59 | # 7. Read status line 60 | line = response.readline().decode() 61 | # 8. Parse status line 62 | version, status, explanation = line.split(" ", 2) 63 | 64 | # 9. Check status 65 | assert status in ("200", "301", "302"), f"{status}: {explanation}" 66 | 67 | # 10. Parse headers 68 | headers = {} 69 | while True: 70 | line = response.readline().decode() 71 | if line == "\r\n": 72 | break 73 | header, value = line.split(":", 1) 74 | headers[header.lower()] = value.strip() 75 | 76 | if "location" in headers: 77 | return request(headers["location"]) 78 | 79 | if "transfer-encoding" in headers: 80 | encoding = headers["transfer-encoding"] 81 | if encoding == "chunked": 82 | body = unchunked(response) 83 | else: 84 | raise RuntimeError(f"Unsupported transfer-encoding: {encoding}") 85 | else: 86 | body = response.read() 87 | 88 | if "content-encoding" in headers: 89 | encoding = headers["content-encoding"] 90 | body = decompress(body, encoding) 91 | 92 | body = body.decode() 93 | # 12. Return 94 | return headers, body 95 | 96 | 97 | def unchunked(response): 98 | ret = b"" 99 | 100 | def get_chunk_size(): 101 | chunk_size = response.readline().rstrip() 102 | return int(chunk_size, 16) 103 | 104 | while True: 105 | chunk_size = get_chunk_size() 106 | if chunk_size == 0: 107 | break 108 | else: 109 | ret += response.read(chunk_size) 110 | response.read(2) 111 | return ret 112 | 113 | 114 | def decompress(data, encoding): 115 | if encoding == "gzip": 116 | return gzip.decompress(data) 117 | elif encoding == "deflate": 118 | return zlib.decompress(data, wbits=-zlib.MAX_WBITS) 119 | elif encoding == "br": 120 | if brotli is None: 121 | raise RuntimeError("please install brotli package: pip install brotli") 122 | return brotli.decompress(data) 123 | elif encoding == "identity": 124 | return data 125 | else: 126 | raise RuntimeError(f"unexpected content-encoding: {encoding}") 127 | 128 | 129 | def lex(body): 130 | # TODO: Will be removed in future course. 131 | def get_body(origin): 132 | import re 133 | 134 | body_re = r"<\s*body.*?>([\s\S]*)<\s*\/body\s?>" 135 | m = re.search(body_re, origin, flags=re.MULTILINE) 136 | if not m: 137 | return origin 138 | return m.group() 139 | 140 | # TODO: This logic will be removed in future course. 141 | body = get_body(body) 142 | text = "" 143 | in_angle = False 144 | for c in body: 145 | if c == "<": 146 | in_angle = True 147 | elif c == ">": 148 | in_angle = False 149 | elif not in_angle: 150 | text += c 151 | return text 152 | -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | brotli==1.0.9 2 | -------------------------------------------------------------------------------- /python/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append("../") 4 | -------------------------------------------------------------------------------- /python/tests/test_http.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from http import request, lex 4 | 5 | 6 | class RequestTest(unittest.TestCase): 7 | def test_http_request(self): 8 | http_sites = ["http://www.google.com/", "http://example.com/"] 9 | for site in http_sites: 10 | headers, body = request(site) 11 | self.assertGreater(len(body), 0) 12 | self.assertIn("content-type", headers) 13 | 14 | def test_https_request(self): 15 | https_sites = [ 16 | "https://www.google.com/", 17 | "https://www.facebook.com/", 18 | "https://example.com/", 19 | ] 20 | for site in https_sites: 21 | headers, body = request(site) 22 | self.assertGreater(len(body), 0) 23 | self.assertIn("content-type", headers) 24 | 25 | def test_data_request(self): 26 | headers, body = request("data:text/html,Hello world") 27 | self.assertEqual(body, "Hello world") 28 | self.assertEqual(headers["content-type"], "text/html") 29 | 30 | def test_lex(self): 31 | origin = " test " 32 | ret = lex(origin) 33 | self.assertEqual(ret, " test ") 34 | 35 | def test_redirect(self): 36 | redirect_sites = [ 37 | "http://www.naver.com/", 38 | "http://browser.engineering/redirect", 39 | ] 40 | for site in redirect_sites: 41 | headers, body = request(site) 42 | self.assertGreater(len(body), 0) 43 | self.assertIn("content-type", headers) 44 | 45 | 46 | if __name__ == "__main__": 47 | unittest.main() 48 | -------------------------------------------------------------------------------- /rust/.gitignore: -------------------------------------------------------------------------------- 1 | Cargo.lock 2 | target 3 | -------------------------------------------------------------------------------- /rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "browser-engineering" 3 | version = "0.1.0" 4 | edition = "2018" 5 | authors = [ 6 | "Seo Sanghyeon ", 7 | "Dong-hee Na ", 8 | "Yi Hyunjoon ", 9 | ] 10 | 11 | [lib] 12 | name = "lib" 13 | path = "src/lib.rs" 14 | 15 | [[bin]] 16 | name = "browser" 17 | path = "src/main.rs" 18 | 19 | [dependencies] 20 | druid = "0.7.0" 21 | flate2 = "1.0" 22 | rustls = "0.19" 23 | webpki = "0.21" 24 | webpki-roots = "0.21" 25 | clap = "2.33" 26 | regex = "1" 27 | -------------------------------------------------------------------------------- /rust/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod http { 2 | use std::collections::HashMap; 3 | use std::env; 4 | use std::fmt; 5 | use std::io::{self, BufRead, BufReader, Read, Write}; 6 | use std::net::TcpStream; 7 | use std::sync::Arc; 8 | 9 | use flate2::bufread::{DeflateDecoder, GzDecoder}; 10 | use regex::bytes::Regex; 11 | use rustls::{ClientConfig, ClientSession, StreamOwned}; 12 | use webpki::DNSNameRef; 13 | 14 | enum Stream { 15 | Tcp(TcpStream), 16 | Tls(StreamOwned), 17 | } 18 | 19 | impl Read for Stream { 20 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 21 | match self { 22 | Self::Tcp(stream) => stream.read(buf), 23 | Self::Tls(stream) => match stream.read(buf) { 24 | Ok(len) => Ok(len), 25 | Err(err) if err.kind() == io::ErrorKind::ConnectionAborted => { 26 | // https://github.com/ctz/rustls/issues/380 27 | Ok(0) 28 | } 29 | Err(err) => Err(err), 30 | }, 31 | } 32 | } 33 | } 34 | 35 | impl Write for Stream { 36 | fn write(&mut self, buf: &[u8]) -> io::Result { 37 | match self { 38 | Self::Tcp(stream) => stream.write(buf), 39 | Self::Tls(stream) => stream.write(buf), 40 | } 41 | } 42 | 43 | fn flush(&mut self) -> io::Result<()> { 44 | match self { 45 | Self::Tcp(stream) => stream.flush(), 46 | Self::Tls(stream) => stream.flush(), 47 | } 48 | } 49 | } 50 | 51 | #[derive(Debug)] 52 | enum ContentEncoding { 53 | Gzip, 54 | Compress, 55 | Deflate, 56 | Identity, 57 | Brotli, 58 | } 59 | 60 | #[derive(Debug)] 61 | struct EncodingError; 62 | 63 | impl std::str::FromStr for ContentEncoding { 64 | type Err = EncodingError; 65 | 66 | fn from_str(s: &str) -> Result { 67 | if "gzip".eq_ignore_ascii_case(s) { 68 | Ok(Self::Gzip) 69 | } else if "compress".eq_ignore_ascii_case(s) { 70 | Ok(Self::Compress) 71 | } else if "deflate".eq_ignore_ascii_case(s) { 72 | Ok(Self::Deflate) 73 | } else if "identity".eq_ignore_ascii_case(s) { 74 | Ok(Self::Identity) 75 | } else if "br".eq_ignore_ascii_case(s) { 76 | Ok(Self::Brotli) 77 | } else { 78 | Err(EncodingError) 79 | } 80 | } 81 | } 82 | 83 | // In Python, string.split(delimiter, 1) 84 | // Replace with str::split_once when it stabilizes 85 | fn split2<'a>(string: &'a str, delimiter: &str) -> Option<(&'a str, &'a str)> { 86 | let mut split = string.splitn(2, delimiter); 87 | Some((split.next()?, split.next()?)) 88 | } 89 | 90 | fn decompressor<'a, R: BufRead + 'a>( 91 | reader: R, 92 | encoding: ContentEncoding, 93 | ) -> Box { 94 | use ContentEncoding::*; 95 | match encoding { 96 | Gzip => Box::new(GzDecoder::new(reader)), 97 | Deflate => Box::new(DeflateDecoder::new(reader)), 98 | Identity => Box::new(reader), 99 | _ => unimplemented!(), 100 | } 101 | } 102 | 103 | #[derive(Debug)] 104 | pub enum RequestError { 105 | Unreachable, 106 | MalformedUrl, 107 | UnknownScheme(String), 108 | ConnectionError, 109 | StatusError(String, String), 110 | MalformedResponse, 111 | UnsupportedEncoding, 112 | } 113 | 114 | impl fmt::Display for RequestError { 115 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 116 | match self { 117 | RequestError::Unreachable => f.write_str("Unreachable"), 118 | RequestError::MalformedUrl => f.write_str("Malformed URL"), 119 | RequestError::UnknownScheme(scheme) => { 120 | write!(f, "Unknown scheme: {}", scheme) 121 | } 122 | RequestError::ConnectionError => f.write_str("Connection error"), 123 | RequestError::StatusError(status, reason) => { 124 | write!(f, "Status error: {} {}", status, reason) 125 | } 126 | RequestError::MalformedResponse => f.write_str("Malformed response"), 127 | RequestError::UnsupportedEncoding => f.write_str("Unsupported encoding"), 128 | } 129 | } 130 | } 131 | 132 | impl std::error::Error for RequestError {} 133 | 134 | pub fn request(url: &str) -> Result<(HashMap, Vec), RequestError> { 135 | // 1. Parse scheme 136 | let (scheme, url) = split2(url, ":").unwrap_or(("https", url)); 137 | let default_port = match scheme { 138 | "http" => 80, 139 | "https" => 443, 140 | "data" => { 141 | // Exercise data scheme 142 | let (content_type, body) = split2(url, ",").ok_or(RequestError::MalformedUrl)?; 143 | let mut headers = HashMap::new(); 144 | headers.insert("content-type".to_owned(), content_type.to_owned()); 145 | return Ok((headers, body.as_bytes().to_vec())); 146 | } 147 | _ => return Err(RequestError::UnknownScheme(scheme.to_string())), 148 | }; 149 | let url = url.strip_prefix("//").unwrap_or(url); 150 | 151 | // 2. Parse host 152 | let (host, path) = split2(url, "/").ok_or(RequestError::MalformedUrl)?; 153 | let path = format!("/{}", path); 154 | 155 | // 3. Parse port 156 | let (host, port) = if host.contains(':') { 157 | let (host, port) = split2(host, ":").ok_or(RequestError::Unreachable)?; 158 | let port = port.parse().or(Err(RequestError::MalformedUrl))?; 159 | (host, port) 160 | } else { 161 | (host, default_port) 162 | }; 163 | 164 | // 4. Connect 165 | let stream = TcpStream::connect((host, port)).or(Err(RequestError::ConnectionError))?; 166 | let mut stream = if scheme != "https" { 167 | Stream::Tcp(stream) 168 | } else { 169 | let mut config = ClientConfig::new(); 170 | config 171 | .root_store 172 | .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); 173 | let host = DNSNameRef::try_from_ascii_str(host).or(Err(RequestError::MalformedUrl))?; 174 | let client = ClientSession::new(&Arc::new(config), host); 175 | let stream = StreamOwned::new(client, stream); 176 | Stream::Tls(stream) 177 | }; 178 | 179 | // 5. Send request 180 | write!( 181 | stream, 182 | "GET {} HTTP/1.1\r\n\ 183 | Host: {}\r\n\ 184 | Connction: close\r\n\ 185 | User-Agent: Mozilla/5.0 ({})\r\n\ 186 | Accept-Encoding: gzip,deflate\r\n\ 187 | \r\n", 188 | path, 189 | host, 190 | env::consts::OS 191 | ) 192 | .or(Err(RequestError::ConnectionError))?; 193 | 194 | // 6. Receive response 195 | let mut reader = BufReader::new(stream); 196 | 197 | // 7. Read status line 198 | let mut line = String::new(); 199 | reader 200 | .read_line(&mut line) 201 | .or(Err(RequestError::MalformedResponse))?; 202 | 203 | // 8. Parse status line 204 | let (_version, status) = split2(&line, " ").ok_or(RequestError::MalformedResponse)?; 205 | let (status, explanation) = split2(status, " ").ok_or(RequestError::MalformedResponse)?; 206 | 207 | // 9. Check status 208 | match status { 209 | "200" | "301" | "302" => (), 210 | _ => { 211 | return Err(RequestError::StatusError( 212 | status.to_string(), 213 | explanation.to_string(), 214 | )) 215 | } 216 | }; 217 | 218 | // 10. Parse headers 219 | let mut headers = HashMap::new(); 220 | loop { 221 | line.clear(); 222 | reader 223 | .read_line(&mut line) 224 | .or(Err(RequestError::MalformedResponse))?; 225 | if line == "\r\n" { 226 | break; 227 | } 228 | let (header, value) = split2(&line, ":").ok_or(RequestError::MalformedResponse)?; 229 | let header = header.to_ascii_lowercase(); 230 | let value = value.trim(); 231 | headers.insert(header, value.to_string()); 232 | } 233 | 234 | if let Some(url) = headers.get("location") { 235 | return request(url); 236 | } 237 | 238 | let content_encoding: ContentEncoding = match headers.get("content-encoding") { 239 | Some(encoding) => encoding 240 | .parse() 241 | .or(Err(RequestError::UnsupportedEncoding))?, 242 | None => ContentEncoding::Identity, 243 | }; 244 | 245 | // 11. Read body 246 | // TODO(corona10): Implement ChunkedReader 247 | let mut unchunked; // for chunked 248 | let mut reader = match headers.get("transfer-encoding") { 249 | Some(encoding) => { 250 | unchunked = Vec::new(); 251 | if "chunked".eq_ignore_ascii_case(encoding) { 252 | loop { 253 | let mut line = String::new(); 254 | reader 255 | .read_line(&mut line) 256 | .or(Err(RequestError::MalformedResponse))?; 257 | let n_bytes = u64::from_str_radix(line.trim_end(), 16).unwrap_or(0); 258 | if n_bytes == 0 { 259 | break; 260 | } 261 | let mut chunk = vec![0u8; n_bytes as usize]; 262 | reader 263 | .read_exact(&mut chunk) 264 | .or(Err(RequestError::MalformedResponse))?; 265 | reader.read_exact(&mut vec![0u8; 2]).unwrap(); 266 | unchunked.write_all(&chunk).unwrap(); 267 | } 268 | } else { 269 | unimplemented!() 270 | } 271 | decompressor(BufReader::new(unchunked.as_slice()), content_encoding) 272 | } 273 | None => decompressor(BufReader::new(reader), content_encoding), 274 | }; 275 | let body = { 276 | let mut body = Vec::new(); 277 | reader 278 | .read_to_end(&mut body) 279 | .or(Err(RequestError::MalformedResponse))?; 280 | body 281 | }; 282 | 283 | // In Rust, connection is closed when stream is dropped 284 | 285 | // 12. Return 286 | Ok((headers, body)) 287 | } 288 | 289 | pub fn lex(body: &[u8]) -> String { 290 | fn get_body(origin: &[u8]) -> &[u8] { 291 | let body_re = Regex::new(r"<\s*body.*?>([\s\S]*)<\s*/body\s?>").unwrap(); 292 | match body_re.find(origin) { 293 | Some(m) => &origin[m.start()..m.end()], 294 | None => origin, 295 | } 296 | } 297 | // 13. Print content 298 | let mut in_angle = false; 299 | let mut out: Vec = Vec::new(); 300 | let body = get_body(body); 301 | for c in body { 302 | match *c { 303 | b'<' => in_angle = true, 304 | b'>' => in_angle = false, 305 | _ => { 306 | if !in_angle { 307 | out.push(*c); 308 | } 309 | } 310 | } 311 | } 312 | String::from_utf8(out).expect("utf-8 website is expected") 313 | } 314 | } 315 | 316 | pub mod display { 317 | use druid::piet::{FontFamily, Text, TextLayoutBuilder}; 318 | use druid::widget::prelude::*; 319 | use druid::Color; 320 | use std::cmp; 321 | 322 | const WIDTH: i32 = 800; 323 | const HEIGHT: i32 = 600; 324 | const HSTEP: i32 = 13; 325 | const VSTEP: i32 = 12; 326 | const SCROLL_STEP: i32 = 100; 327 | 328 | struct Character { 329 | x: i32, 330 | y: i32, 331 | ch: char, 332 | } 333 | 334 | pub struct BrowserWidget { 335 | display_list: Vec, 336 | scroll: i32, 337 | min_scroll: i32, 338 | max_scroll: i32, 339 | } 340 | 341 | impl BrowserWidget { 342 | pub fn new(text: String) -> BrowserWidget { 343 | let mut cursor_x = HSTEP; 344 | let mut cursor_y = VSTEP; 345 | let mut max_scroll = 0; 346 | let mut display_list = Vec::new(); 347 | for c in text.chars() { 348 | max_scroll = cmp::max(max_scroll, cursor_y); 349 | display_list.push(Character { 350 | x: cursor_x, 351 | y: cursor_y, 352 | ch: c, 353 | }); 354 | cursor_x += VSTEP; 355 | if cursor_x >= WIDTH - HSTEP || c == '\n' { 356 | cursor_y += VSTEP; 357 | cursor_x = HSTEP; 358 | } 359 | } 360 | BrowserWidget { 361 | display_list, 362 | scroll: 0, 363 | min_scroll: 0, 364 | max_scroll, 365 | } 366 | } 367 | 368 | pub fn get_height() -> f64 { 369 | HEIGHT as f64 370 | } 371 | 372 | pub fn get_width() -> f64 { 373 | WIDTH as f64 374 | } 375 | } 376 | 377 | impl Widget for BrowserWidget { 378 | fn event(&mut self, ctx: &mut EventCtx, _event: &Event, _data: &mut i32, _env: &Env) { 379 | match _event { 380 | Event::Wheel(e) => { 381 | if e.wheel_delta.y < 0.0 { 382 | self.scroll -= SCROLL_STEP; 383 | self.scroll = cmp::max(self.scroll, self.min_scroll); 384 | } else if e.wheel_delta.y > 0.0 { 385 | self.scroll += SCROLL_STEP; 386 | self.scroll = cmp::min(self.scroll, self.max_scroll); 387 | } 388 | *_data = self.scroll; 389 | ctx.request_update(); 390 | } 391 | _ => {} 392 | } 393 | } 394 | 395 | fn lifecycle( 396 | &mut self, 397 | _ctx: &mut LifeCycleCtx, 398 | _event: &LifeCycle, 399 | _data: &i32, 400 | _env: &Env, 401 | ) { 402 | } 403 | 404 | fn update(&mut self, ctx: &mut UpdateCtx, old_data: &i32, data: &i32, _env: &Env) { 405 | if old_data != data { 406 | ctx.request_paint(); 407 | } 408 | } 409 | 410 | fn layout( 411 | &mut self, 412 | _layout_ctx: &mut LayoutCtx, 413 | bc: &BoxConstraints, 414 | _data: &i32, 415 | _env: &Env, 416 | ) -> Size { 417 | bc.max() 418 | } 419 | 420 | fn paint(&mut self, ctx: &mut PaintCtx, _data: &i32, _env: &Env) { 421 | let size = ctx.size(); 422 | let rect = size.to_rect(); 423 | ctx.fill(rect, &Color::WHITE); 424 | for ch in &self.display_list { 425 | if ch.y > self.scroll + HEIGHT { 426 | continue; 427 | } 428 | 429 | if ch.y + VSTEP < self.scroll { 430 | continue; 431 | } 432 | 433 | let text = ctx.text(); 434 | let layout = text 435 | .new_text_layout(String::from(ch.ch)) 436 | .font(FontFamily::default(), 12.0) 437 | .text_color(Color::BLACK) 438 | .build() 439 | .unwrap(); 440 | ctx.draw_text(&layout, (ch.x as f64, ch.y as f64 - self.scroll as f64)); 441 | } 442 | } 443 | } 444 | } 445 | 446 | #[cfg(test)] 447 | mod tests { 448 | use super::*; 449 | 450 | #[test] 451 | fn test_http_request() -> Result<(), String> { 452 | let http_sites = vec!["http://www.google.com/", "http://example.com/"]; 453 | for site in http_sites { 454 | let (header, body) = http::request(site).unwrap(); 455 | assert_eq!(header.contains_key("content-type"), true); 456 | assert_eq!(body.len() > 0, true); 457 | } 458 | Ok(()) 459 | } 460 | 461 | #[test] 462 | fn test_https_request() -> Result<(), String> { 463 | let https_sites = vec!["https://www.google.com/", "https://example.com/"]; 464 | for site in https_sites { 465 | let (header, body) = http::request(site).unwrap(); 466 | assert_eq!(header.contains_key("content-type"), true); 467 | assert_eq!(body.len() > 0, true); 468 | } 469 | Ok(()) 470 | } 471 | 472 | #[test] 473 | fn test_data_request() -> Result<(), String> { 474 | let (header, body) = http::request("data:text/html,Hello world").unwrap(); 475 | assert_eq!(header.get("content-type").unwrap(), "text/html"); 476 | assert_eq!(std::str::from_utf8(&body).unwrap(), "Hello world"); 477 | Ok(()) 478 | } 479 | 480 | #[test] 481 | fn test_lex() -> Result<(), String> { 482 | let origin = " test "; 483 | assert_eq!(http::lex(origin.as_bytes()), " test "); 484 | Ok(()) 485 | } 486 | 487 | #[test] 488 | fn test_redirect() -> Result<(), String> { 489 | let redirect_sites = vec![ 490 | "http://www.naver.com/", 491 | "http://browser.engineering/redirect", 492 | ]; 493 | for site in redirect_sites { 494 | let (header, body) = http::request(site).unwrap(); 495 | assert!(header.contains_key("content-type")); 496 | assert!(!body.is_empty()); 497 | } 498 | Ok(()) 499 | } 500 | } 501 | -------------------------------------------------------------------------------- /rust/src/main.rs: -------------------------------------------------------------------------------- 1 | use druid::{AppLauncher, LocalizedString, WindowDesc}; 2 | use lib::display::BrowserWidget; 3 | use lib::http::{lex, request}; 4 | 5 | const APP_NAME: &str = "Browser-engineering"; 6 | struct BrowserApplication {} 7 | 8 | impl BrowserApplication { 9 | fn run(&self, url: &str) { 10 | let (_headers, body) = request(&url).unwrap_or_else(|e| panic!("{}", e)); 11 | let text = lex(&body); 12 | let browser_widget = || -> BrowserWidget { BrowserWidget::new(text) }; 13 | let window = WindowDesc::new(browser_widget) 14 | .title(LocalizedString::new(APP_NAME)) 15 | .window_size((BrowserWidget::get_width(), BrowserWidget::get_height())); 16 | AppLauncher::with_window(window) 17 | .use_simple_logger() 18 | .launch(0) 19 | .expect("launch failed"); 20 | } 21 | } 22 | 23 | pub fn main() { 24 | use clap::{App, Arg}; 25 | let matches = App::new(APP_NAME) 26 | .arg(Arg::with_name("url").value_name("URL").takes_value(true)) 27 | .get_matches(); 28 | let url = matches 29 | .value_of("url") 30 | .expect("required argument at the moment"); 31 | 32 | let app = BrowserApplication {}; 33 | app.run(url); 34 | } 35 | --------------------------------------------------------------------------------