├── tidb ├── .gitignore ├── mapreduce │ ├── .gitignore │ ├── go.mod │ ├── go.sum │ ├── urltop10.go │ ├── Makefile │ ├── README.md │ ├── urltop10_example.go │ ├── urltop10_test.go │ └── utils.go ├── mergesort │ ├── go.mod │ ├── Makefile │ ├── mergesort.go │ ├── go.sum │ ├── bench_test.go │ ├── mergesort_test.go │ └── README.md ├── join │ ├── Makefile │ ├── go.mod │ ├── benchmark_test.go │ ├── join.go │ ├── join_test.go │ ├── join_example.go │ └── README.md └── README.md ├── courses ├── rust │ ├── .gitignore │ ├── docs │ │ ├── what-next.md │ │ ├── qq-qr.jpg │ │ ├── qq2-qr.jpg │ │ ├── roadmap.md │ │ ├── prerequisites.md │ │ └── etc │ │ │ └── parallel-diagrams.txt │ ├── projects │ │ ├── project-1 │ │ │ ├── src │ │ │ │ ├── lib.rs │ │ │ │ ├── kv.rs │ │ │ │ └── bin │ │ │ │ │ └── kvs.rs │ │ │ ├── Cargo.toml │ │ │ └── tests │ │ │ │ └── tests.rs │ │ ├── project-2 │ │ │ ├── src │ │ │ │ ├── lib.rs │ │ │ │ ├── error.rs │ │ │ │ └── bin │ │ │ │ │ └── kvs.rs │ │ │ └── Cargo.toml │ │ ├── project-3 │ │ │ ├── src │ │ │ │ ├── lib.rs │ │ │ │ ├── common.rs │ │ │ │ ├── engines │ │ │ │ │ ├── mod.rs │ │ │ │ │ └── sled.rs │ │ │ │ ├── error.rs │ │ │ │ ├── client.rs │ │ │ │ ├── server.rs │ │ │ │ └── bin │ │ │ │ │ ├── kvs-server.rs │ │ │ │ │ └── kvs-client.rs │ │ │ ├── Cargo.toml │ │ │ └── benches │ │ │ │ └── engine_bench.rs │ │ ├── project-4 │ │ │ ├── src │ │ │ │ ├── lib.rs │ │ │ │ ├── thread_pool │ │ │ │ │ ├── naive.rs │ │ │ │ │ ├── rayon.rs │ │ │ │ │ ├── mod.rs │ │ │ │ │ └── shared_queue.rs │ │ │ │ ├── common.rs │ │ │ │ ├── engines │ │ │ │ │ ├── mod.rs │ │ │ │ │ └── sled.rs │ │ │ │ ├── error.rs │ │ │ │ ├── client.rs │ │ │ │ ├── server.rs │ │ │ │ └── bin │ │ │ │ │ ├── kvs-client.rs │ │ │ │ │ └── kvs-server.rs │ │ │ ├── Cargo.toml │ │ │ └── tests │ │ │ │ └── thread_pool.rs │ │ └── project-5 │ │ │ ├── src │ │ │ ├── lib.rs │ │ │ ├── common.rs │ │ │ ├── thread_pool │ │ │ │ ├── naive.rs │ │ │ │ ├── rayon.rs │ │ │ │ ├── mod.rs │ │ │ │ └── shared_queue.rs │ │ │ ├── engines │ │ │ │ ├── mod.rs │ │ │ │ └── sled.rs │ │ │ ├── error.rs │ │ │ ├── server.rs │ │ │ ├── bin │ │ │ │ ├── kvs-client.rs │ │ │ │ └── kvs-server.rs │ │ │ └── client.rs │ │ │ ├── Cargo.toml │ │ │ └── tests │ │ │ └── thread_pool.rs │ └── building-blocks │ │ ├── bb-5.md │ │ ├── bb-1.md │ │ ├── bb-3.md │ │ └── bb-4.md ├── dss │ ├── .gitignore │ ├── Cargo.toml │ ├── raft │ │ ├── src │ │ │ ├── kvraft │ │ │ │ ├── mod.rs │ │ │ │ ├── errors.rs │ │ │ │ ├── client.rs │ │ │ │ └── server.rs │ │ │ ├── proto │ │ │ │ ├── raft.proto │ │ │ │ ├── kvraft.proto │ │ │ │ └── mod.rs │ │ │ ├── lib.rs │ │ │ └── raft │ │ │ │ ├── errors.rs │ │ │ │ └── persister.rs │ │ ├── build.rs │ │ └── Cargo.toml │ ├── percolator │ │ ├── build.rs │ │ ├── src │ │ │ ├── lib.rs │ │ │ ├── service.rs │ │ │ ├── client.rs │ │ │ └── server.rs │ │ ├── Cargo.toml │ │ ├── proto │ │ │ └── msg.proto │ │ └── README.md │ ├── labcodec │ │ ├── build.rs │ │ ├── demonstration │ │ │ └── README.md │ │ ├── Cargo.toml │ │ ├── proto │ │ │ └── fixture.proto │ │ └── src │ │ │ └── lib.rs │ ├── linearizability │ │ ├── Cargo.toml │ │ └── src │ │ │ ├── bitset.rs │ │ │ └── model.rs │ ├── labrpc │ │ ├── Cargo.toml │ │ ├── src │ │ │ ├── error.rs │ │ │ ├── client.rs │ │ │ └── server.rs │ │ ├── examples │ │ │ └── echo.rs │ │ └── benches │ │ │ └── rpc.rs │ ├── Makefile │ └── README.md ├── README.md └── tp102-how-to-use-git-github.md ├── media ├── talent-plan-logo.png ├── execution-example.jpg ├── talent-plan-framework.jpg └── talent-plan-students.png ├── talent-challenge-program ├── PROJECT_IDEA_TEMPLATE.md ├── MENTEE_APPLY_TEMPLATE.md ├── s1.md └── README.md ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── feature-requirement.md │ └── question.md └── pull_request_template.md ├── talent-challenge-program2021 ├── MENTEE_APPLY_TEMPLATE.md ├── PROJECT_IDEA_TEMPLATE.md ├── project-ideas.md ├── schedule.md ├── README.md └── selected-projects.md ├── .circleci └── config.yml ├── talent-plan-1.0 ├── README-CN.md ├── 1.0-lp-tikv.md └── 1.0-lp-tidb.md ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md └── README.md /tidb/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /tidb/mapreduce/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /courses/rust/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | target/ 3 | -------------------------------------------------------------------------------- /tidb/mapreduce/go.mod: -------------------------------------------------------------------------------- 1 | module talent 2 | 3 | go 1.12 4 | -------------------------------------------------------------------------------- /courses/dss/.gitignore: -------------------------------------------------------------------------------- 1 | **/target 2 | **/*.rs.bk 3 | 6.824-golabs-2018 4 | -------------------------------------------------------------------------------- /courses/rust/docs/what-next.md: -------------------------------------------------------------------------------- 1 | # What's next now that you've completed PNA Rust? 2 | 3 | Coming soon! 4 | -------------------------------------------------------------------------------- /media/talent-plan-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/talent-plan/HEAD/media/talent-plan-logo.png -------------------------------------------------------------------------------- /courses/rust/docs/qq-qr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/talent-plan/HEAD/courses/rust/docs/qq-qr.jpg -------------------------------------------------------------------------------- /courses/rust/docs/qq2-qr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/talent-plan/HEAD/courses/rust/docs/qq2-qr.jpg -------------------------------------------------------------------------------- /media/execution-example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/talent-plan/HEAD/media/execution-example.jpg -------------------------------------------------------------------------------- /media/talent-plan-framework.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/talent-plan/HEAD/media/talent-plan-framework.jpg -------------------------------------------------------------------------------- /media/talent-plan-students.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/talent-plan/HEAD/media/talent-plan-students.png -------------------------------------------------------------------------------- /tidb/mapreduce/go.sum: -------------------------------------------------------------------------------- 1 | github.com/pingcap/talent-plan v0.0.0-20190408125936-2f97dda786d6 h1:Kr1alXUfrJVBcLQb9tbrZGpInKkBhGLZsuMKNfesH1I= 2 | -------------------------------------------------------------------------------- /courses/rust/projects/project-1/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | //! A simple key/value store. 3 | 4 | pub use kv::KvStore; 5 | 6 | mod kv; 7 | -------------------------------------------------------------------------------- /courses/dss/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "labcodec", 4 | "labrpc", 5 | "linearizability", 6 | "raft", 7 | "percolator" 8 | ] 9 | -------------------------------------------------------------------------------- /courses/dss/raft/src/kvraft/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod client; 2 | #[cfg(test)] 3 | pub mod config; 4 | pub mod errors; 5 | pub mod server; 6 | #[cfg(test)] 7 | mod tests; 8 | -------------------------------------------------------------------------------- /tidb/mergesort/go.mod: -------------------------------------------------------------------------------- 1 | module pingcap/talentplan/tidb/mergesort 2 | 3 | go 1.12 4 | 5 | require github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 6 | -------------------------------------------------------------------------------- /tidb/join/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | all: test bench 4 | 5 | test: 6 | go test 7 | 8 | bench: 9 | go test -bench Benchmark -run xx -count 5 -benchmem 10 | -------------------------------------------------------------------------------- /tidb/README.md: -------------------------------------------------------------------------------- 1 | # Distributed Systems in Go 2 | 3 | * Week 1: [Merge Sort](./mergesort) 4 | * Week 2: [Map Reduce](./mapreduce) 5 | * Week 4: [Parallel Join](./join) 6 | 7 | -------------------------------------------------------------------------------- /courses/dss/percolator/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | prost_build::compile_protos(&["proto/msg.proto"], &["proto"]).unwrap(); 3 | println!("cargo:rerun-if-changed=proto"); 4 | } 5 | -------------------------------------------------------------------------------- /tidb/mergesort/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | all: test bench 4 | 5 | test: 6 | go test 7 | 8 | bench: 9 | go test -bench Benchmark -run xx -count 5 -benchmem 10 | 11 | -------------------------------------------------------------------------------- /courses/dss/labcodec/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | prost_build::compile_protos(&["proto/fixture.proto"], &["proto"]).unwrap(); 3 | println!("cargo:rerun-if-changed=proto"); 4 | } 5 | -------------------------------------------------------------------------------- /tidb/join/go.mod: -------------------------------------------------------------------------------- 1 | module join 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 7 | github.com/pingcap/tidb v2.1.7+incompatible 8 | ) 9 | -------------------------------------------------------------------------------- /courses/rust/projects/project-2/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | //! A simple key/value store. 3 | 4 | pub use error::{KvsError, Result}; 5 | pub use kv::KvStore; 6 | 7 | mod error; 8 | mod kv; 9 | -------------------------------------------------------------------------------- /talent-challenge-program/PROJECT_IDEA_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ``` 2 | #### TiDB Ecosystem Project Name 3 | ##### Title 4 | - Description: 5 | - Recommended Skills: 6 | - Mentor(s): 7 | - Upstream Issue or RFC (URL): 8 | ``` -------------------------------------------------------------------------------- /tidb/mergesort/mergesort.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // MergeSort performs the merge sort algorithm. 4 | // Please supplement this function to accomplish the home work. 5 | func MergeSort(src []int64) { 6 | } 7 | -------------------------------------------------------------------------------- /courses/dss/linearizability/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "linearizability" 3 | version = "0.1.0" 4 | edition = "2018" 5 | publish = false 6 | 7 | [dev-dependencies] 8 | regex = "1.3" 9 | lazy_static = "1.4" 10 | -------------------------------------------------------------------------------- /courses/dss/labcodec/demonstration/README.md: -------------------------------------------------------------------------------- 1 | # Demonstration 2 | 3 | Expanded version of generated rust files. Files in the folder are for the sake 4 | of understanding how procedural macro works, and they are not compilable. 5 | -------------------------------------------------------------------------------- /tidb/mapreduce/urltop10.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // URLTop10 . 4 | func URLTop10(nWorkers int) RoundsArgs { 5 | // YOUR CODE HERE :) 6 | // And don't forget to document your idea. 7 | panic("YOUR CODE HERE") 8 | return nil 9 | } 10 | -------------------------------------------------------------------------------- /tidb/mergesort/go.sum: -------------------------------------------------------------------------------- 1 | github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= 2 | github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= 3 | -------------------------------------------------------------------------------- /courses/dss/labcodec/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "labcodec" 3 | version = "0.1.0" 4 | build = "build.rs" 5 | edition = "2018" 6 | publish = false 7 | 8 | [dependencies] 9 | prost = "0.6" 10 | 11 | [build-dependencies] 12 | prost-build = "0.6" 13 | -------------------------------------------------------------------------------- /courses/rust/docs/roadmap.md: -------------------------------------------------------------------------------- 1 | # PNA Rust Roadmap 2 | 3 | We've got a lot of ideas about how to extend and expand this course. But the 4 | roadmap isn't written yet. In the meantime, [notes.md] contains a bunch of 5 | unorganized thoughts about the matter. 6 | 7 | [notes.md]: ./etc/notes.md 8 | -------------------------------------------------------------------------------- /tidb/mapreduce/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | all: test_example test_homework cleanup gendata 4 | 5 | test_example: 6 | go test -v -run=TestExampleURLTop 7 | 8 | test_homework: 9 | go test -v -run=TestURLTop 10 | 11 | cleanup: 12 | go test -v -run=TestCleanData 13 | 14 | gendata: 15 | go test -v -run=TestGenData 16 | -------------------------------------------------------------------------------- /courses/rust/projects/project-1/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvs" 3 | version = "0.1.0" 4 | authors = ["Yilin Chen "] 5 | description = "A key-value store" 6 | edition = "2018" 7 | 8 | [dependencies] 9 | clap = "2.32.0" 10 | 11 | [dev-dependencies] 12 | assert_cmd = "0.11.0" 13 | predicates = "1.0.0" 14 | -------------------------------------------------------------------------------- /courses/dss/raft/src/proto/raft.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package raftpb; 4 | 5 | // Example RequestVote RPC arguments structure. 6 | message RequestVoteArgs { 7 | // Your data here (2A, 2B). 8 | } 9 | 10 | // Example RequestVote RPC reply structure. 11 | message RequestVoteReply { 12 | // Your data here (2A). 13 | } 14 | -------------------------------------------------------------------------------- /courses/rust/building-blocks/bb-5.md: -------------------------------------------------------------------------------- 1 | # PNA Rust — Building Blocks 5 2 | 3 | Let's learn some building blocks! 4 | 5 | Put your other projects and concerns aside. Take a breath and relax. Here 6 | are some fun resources for you to explore. 7 | 8 | Read all the readings and perform all the exercises. 9 | 10 | Coming soon! 11 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | //! A simple key/value store. 3 | 4 | pub use client::KvsClient; 5 | pub use engines::{KvStore, KvsEngine, SledKvsEngine}; 6 | pub use error::{KvsError, Result}; 7 | pub use server::KvsServer; 8 | mod client; 9 | mod common; 10 | mod engines; 11 | mod error; 12 | mod server; 13 | -------------------------------------------------------------------------------- /courses/dss/raft/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[allow(unused_imports)] 2 | #[macro_use] 3 | extern crate log; 4 | #[allow(unused_imports)] 5 | #[macro_use] 6 | extern crate prost_derive; 7 | 8 | pub mod kvraft; 9 | mod proto; 10 | pub mod raft; 11 | 12 | /// A place holder for suppressing unused_variables warning. 13 | fn your_code_here(_: T) -> ! { 14 | unimplemented!() 15 | } 16 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | //! A simple key/value store. 3 | 4 | pub use client::KvsClient; 5 | pub use engines::{KvStore, KvsEngine, SledKvsEngine}; 6 | pub use error::{KvsError, Result}; 7 | pub use server::KvsServer; 8 | 9 | mod client; 10 | mod common; 11 | mod engines; 12 | mod error; 13 | mod server; 14 | pub mod thread_pool; 15 | -------------------------------------------------------------------------------- /tidb/join/benchmark_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func BenchmarkJoin(b *testing.B) { 6 | for i := 0; i < b.N; i++ { 7 | Join("./t/r0.tbl", "./t/r2.tbl", []int{0}, []int{1}) 8 | } 9 | } 10 | 11 | func BenchmarkJoinExample(b *testing.B) { 12 | for i := 0; i < b.N; i++ { 13 | JoinExample("./t/r0.tbl", "./t/r2.tbl", []int{0}, []int{1}) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /courses/dss/labcodec/proto/fixture.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package fixture; 4 | 5 | // A simple protobuf message. 6 | message Msg { 7 | enum Type { 8 | UNKNOWN = 0; 9 | PUT = 1; 10 | GET = 2; 11 | DEL = 3; 12 | } 13 | 14 | Type type = 1; 15 | uint64 id = 2; 16 | string name = 3; 17 | repeated bytes paylad = 4; 18 | } 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug Report" 3 | about: something isn't working as expected 4 | labels: type/bug 5 | --- 6 | 7 | ## Bug Report 8 | 9 | Please answer these questions before submitting your issue. Thanks! 10 | 11 | ### 1. What did you do? 12 | 13 | ### 2. What did you expect to see? 14 | 15 | ### 3. What did you see instead? 16 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | //! A simple key/value store. 3 | 4 | #[macro_use] 5 | extern crate log; 6 | 7 | pub use client::KvsClient; 8 | pub use engines::{KvStore, KvsEngine, SledKvsEngine}; 9 | pub use error::{KvsError, Result}; 10 | pub use server::KvsServer; 11 | 12 | mod client; 13 | mod common; 14 | mod engines; 15 | mod error; 16 | mod server; 17 | pub mod thread_pool; 18 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/common.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Debug, Serialize, Deserialize)] 4 | pub enum Request { 5 | Get { key: String }, 6 | Set { key: String, value: String }, 7 | Remove { key: String }, 8 | } 9 | 10 | #[derive(Debug, Serialize, Deserialize)] 11 | pub enum Response { 12 | Get(Option), 13 | Set, 14 | Remove, 15 | Err(String), 16 | } 17 | -------------------------------------------------------------------------------- /courses/dss/percolator/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[allow(unused_imports)] 2 | #[macro_use] 3 | extern crate log; 4 | 5 | // After you finish the implementation, `#[allow(unused)]` should be removed. 6 | #[allow(dead_code, unused)] 7 | mod client; 8 | #[allow(unused)] 9 | mod server; 10 | mod service; 11 | #[cfg(test)] 12 | mod tests; 13 | 14 | // This is related to protobuf as described in `msg.proto`. 15 | mod msg { 16 | include!(concat!(env!("OUT_DIR"), "/msg.rs")); 17 | } 18 | -------------------------------------------------------------------------------- /courses/rust/projects/project-2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvs" 3 | version = "0.1.0" 4 | authors = ["Yilin Chen "] 5 | description = "A key-value store" 6 | edition = "2018" 7 | 8 | [dependencies] 9 | clap = "2.32.0" 10 | failure = "0.1.5" 11 | serde = { version = "1.0.89", features = ["derive"] } 12 | serde_json = "1.0.39" 13 | 14 | [dev-dependencies] 15 | assert_cmd = "0.11.0" 16 | predicates = "1.0.0" 17 | tempfile = "3.0.7" 18 | walkdir = "2.2.7" 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-requirement.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature Request" 3 | about: I have a suggestion! 4 | labels: type/enhancement 5 | --- 6 | 7 | ## Feature Request 8 | 9 | please make a clear and concise description for the following questions. 10 | 11 | ### Is your feature request related to a problem? 12 | 13 | 14 | ### Describe the feature you'd like 15 | 16 | 17 | ### Describe alternatives you've considered 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F914 Question" 3 | about: I have a question! 4 | labels: type/question 5 | --- 6 | 7 | ## General Question 8 | 9 | ## Tips 10 | 11 | Expect for asking questions on Github, you can also discuss it in the channel 12 | **#wg-talent-plan-courses**. of 13 | [tidbcommunity](https://join.slack.com/t/tidbcommunity/shared_invite/enQtNzc0MzI4ODExMDc4LWYwYmIzMjZkYzJiNDUxMmZlN2FiMGJkZjAyMzQ5NGU0NGY0NzI3NTYwMjAyNGQ1N2I2ZjAxNzc1OGUwYWM0NzE) 14 | slack workspace. 15 | -------------------------------------------------------------------------------- /courses/rust/docs/prerequisites.md: -------------------------------------------------------------------------------- 1 | # PNA Rust Prerequisites 2 | 3 | Don't meet all the [prerequisites][pre] to take this course? Here are some ideas 4 | for how to proceed. 5 | 6 | * A crash course on Rust by Michael Snoyman (Oct 2018) - [blog series][snoyman-blog] | [print friendly][snoyman-html] 7 | 8 | [pre]: ../README.md#user-content-prerequisites 9 | [snoyman-blog]: https://www.snoyman.com/blog/2018/10/introducing-rust-crash-course 10 | [snoyman-html]: https://www.snoyman.com/series/rust-crash-course 11 | -------------------------------------------------------------------------------- /courses/dss/labrpc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "labrpc" 3 | version = "0.1.0" 4 | edition = "2018" 5 | publish = false 6 | 7 | [dependencies] 8 | async-trait = "0.1" 9 | futures = { version = "0.3", features = ["thread-pool"] } 10 | futures-timer = "3.0" 11 | log = "0.4" 12 | prost = "0.6" 13 | rand = "0.7" 14 | 15 | labcodec = { path = "../labcodec" } 16 | 17 | [dev-dependencies] 18 | criterion = "0.3" 19 | env_logger = "0.7" 20 | prost-derive = "0.6" 21 | 22 | [[bench]] 23 | name = "rpc" 24 | path = "benches/rpc.rs" 25 | harness = false 26 | -------------------------------------------------------------------------------- /courses/dss/percolator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "percolator" 3 | version = "0.1.0" 4 | authors = ["Ryan Leung "] 5 | edition = "2018" 6 | build = "build.rs" 7 | publish = false 8 | 9 | [dependencies] 10 | async-trait = "0.1" 11 | futures = "0.3" 12 | futures-timer = "3.0" 13 | log = "0.4" 14 | prost = "0.6" 15 | prost-derive = "0.6" 16 | 17 | labrpc = { path = "../labrpc" } 18 | labcodec = { path = "../labcodec" } 19 | 20 | [build-dependencies] 21 | prost-build = "0.6" 22 | 23 | [dev-dependencies] 24 | env_logger = "0.7" 25 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/thread_pool/naive.rs: -------------------------------------------------------------------------------- 1 | use std::thread; 2 | 3 | use super::ThreadPool; 4 | use crate::Result; 5 | 6 | /// It is actually not a thread pool. It spawns a new thread every time 7 | /// the `spawn` method is called. 8 | pub struct NaiveThreadPool; 9 | 10 | impl ThreadPool for NaiveThreadPool { 11 | fn new(_threads: u32) -> Result { 12 | Ok(NaiveThreadPool) 13 | } 14 | 15 | fn spawn(&self, job: F) 16 | where 17 | F: FnOnce() + Send + 'static, 18 | { 19 | thread::spawn(job); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /courses/dss/raft/src/kvraft/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{error, fmt, result}; 2 | 3 | #[derive(Clone, Debug, PartialEq, Eq)] 4 | pub enum Error { 5 | NoLeader, 6 | } 7 | 8 | impl fmt::Display for Error { 9 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 10 | write!(f, "{:?}", self) 11 | } 12 | } 13 | 14 | impl error::Error for Error { 15 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 16 | match *self { 17 | Error::NoLeader => None, 18 | } 19 | } 20 | } 21 | 22 | pub type Result = result::Result; 23 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/thread_pool/naive.rs: -------------------------------------------------------------------------------- 1 | use std::thread; 2 | 3 | use super::ThreadPool; 4 | use crate::Result; 5 | 6 | /// It is actually not a thread pool. It spawns a new thread every time 7 | /// the `spawn` method is called. 8 | #[derive(Clone)] 9 | pub struct NaiveThreadPool; 10 | 11 | impl ThreadPool for NaiveThreadPool { 12 | fn new(_threads: u32) -> Result { 13 | Ok(NaiveThreadPool) 14 | } 15 | 16 | fn spawn(&self, job: F) 17 | where 18 | F: FnOnce() + Send + 'static, 19 | { 20 | thread::spawn(job); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tidb/join/join.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Join accepts a join query of two relations, and returns the sum of 4 | // relation0.col0 in the final result. 5 | // Input arguments: 6 | // f0: file name of the given relation0 7 | // f1: file name of the given relation1 8 | // offset0: offsets of which columns the given relation0 should be joined 9 | // offset1: offsets of which columns the given relation1 should be joined 10 | // Output arguments: 11 | // sum: sum of relation0.col0 in the final result 12 | func Join(f0, f1 string, offset0, offset1 []int) (sum uint64) { 13 | return sum 14 | } 15 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/common.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Debug, Serialize, Deserialize)] 4 | pub enum Request { 5 | Get { key: String }, 6 | Set { key: String, value: String }, 7 | Remove { key: String }, 8 | } 9 | 10 | #[derive(Debug, Serialize, Deserialize)] 11 | pub enum GetResponse { 12 | Ok(Option), 13 | Err(String), 14 | } 15 | 16 | #[derive(Debug, Serialize, Deserialize)] 17 | pub enum SetResponse { 18 | Ok(()), 19 | Err(String), 20 | } 21 | 22 | #[derive(Debug, Serialize, Deserialize)] 23 | pub enum RemoveResponse { 24 | Ok(()), 25 | Err(String), 26 | } 27 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/common.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Debug, Serialize, Deserialize)] 4 | pub enum Request { 5 | Get { key: String }, 6 | Set { key: String, value: String }, 7 | Remove { key: String }, 8 | } 9 | 10 | #[derive(Debug, Serialize, Deserialize)] 11 | pub enum GetResponse { 12 | Ok(Option), 13 | Err(String), 14 | } 15 | 16 | #[derive(Debug, Serialize, Deserialize)] 17 | pub enum SetResponse { 18 | Ok(()), 19 | Err(String), 20 | } 21 | 22 | #[derive(Debug, Serialize, Deserialize)] 23 | pub enum RemoveResponse { 24 | Ok(()), 25 | Err(String), 26 | } 27 | -------------------------------------------------------------------------------- /talent-challenge-program/MENTEE_APPLY_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Mentee Apply Template 2 | 3 | Mail to: talent-plan@tidb.io 4 | 5 | ## Mail Subject: 6 | 7 | TCP Mentee Application - {project name} - {personal name} 8 | 9 | ## Message body 10 | 11 | Please make a self-introduction briefly, including but not limited to: 12 | 13 | * your name 14 | * your nationality 15 | * your education background 16 | * your performance in talent plan courses or what you have contributed to TiDB projects 17 | * the project you are applying for 18 | 19 | 20 | ## Attachments 21 | 22 | Please attach your updated profile and any other materials that demonstrate your abilities. 23 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvs" 3 | version = "0.1.0" 4 | authors = ["Yilin Chen "] 5 | description = "A key-value store" 6 | edition = "2018" 7 | 8 | [dependencies] 9 | clap = "2.33.0" 10 | structopt = "0.2.15" 11 | failure = "0.1.5" 12 | serde = { version = "1.0.89", features = ["derive"] } 13 | serde_json = "1.0.39" 14 | log = "0.4.6" 15 | env_logger = "0.6.1" 16 | sled = "0.34.6" 17 | 18 | [dev-dependencies] 19 | assert_cmd = "0.11" 20 | criterion = "0.3" 21 | predicates = "1.0.0" 22 | rand = "0.6.5" 23 | tempfile = "3.0.7" 24 | walkdir = "2.2.7" 25 | 26 | [[bench]] 27 | name = "engine_bench" 28 | harness = false -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/thread_pool/rayon.rs: -------------------------------------------------------------------------------- 1 | use super::ThreadPool; 2 | use crate::{KvsError, Result}; 3 | 4 | /// Wrapper of rayon::ThreadPool 5 | pub struct RayonThreadPool(rayon::ThreadPool); 6 | 7 | impl ThreadPool for RayonThreadPool { 8 | fn new(threads: u32) -> Result { 9 | let pool = rayon::ThreadPoolBuilder::new() 10 | .num_threads(threads as usize) 11 | .build() 12 | .map_err(|e| KvsError::StringError(format!("{}", e)))?; 13 | Ok(RayonThreadPool(pool)) 14 | } 15 | 16 | fn spawn(&self, job: F) 17 | where 18 | F: FnOnce() + Send + 'static, 19 | { 20 | self.0.spawn(job) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /courses/dss/raft/src/proto/kvraft.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package kvraftpb; 4 | 5 | enum Op { 6 | Unknown = 0; 7 | Put = 1; 8 | Append = 2; 9 | } 10 | 11 | /// Put or Append 12 | message PutAppendRequest { 13 | string key = 1; 14 | string value = 2; 15 | // "Put" or "Append" 16 | Op op = 3; 17 | // You'll have to add definitions here. 18 | } 19 | 20 | message PutAppendReply { 21 | bool wrong_leader = 1; 22 | string err = 2; 23 | } 24 | 25 | message GetRequest { 26 | string key = 1; 27 | // You'll have to add definitions here. 28 | } 29 | 30 | message GetReply { 31 | bool wrong_leader = 1; 32 | string err = 2; 33 | string value = 3; 34 | } 35 | -------------------------------------------------------------------------------- /talent-challenge-program2021/MENTEE_APPLY_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Mentee Application Form Template 2 | 3 | Mail to: talent-plan@tidb.io 4 | 5 | ## Mail Subject: 6 | 7 | TCP Mentee Application - {project name} - {personal name} 8 | 9 | ## Message body 10 | 11 | Please make a self-introduction briefly, including but not limited to: 12 | 13 | * Your name 14 | * Your nationality 15 | * Your education background 16 | * The project you are applying for 17 | * Your performance in talent plan courses or what you have contributed to TiDB projects 18 | * Any other evidences that prove your capabilities 19 | 20 | 21 | ## Attachments 22 | 23 | Please attach your updated profile and any other materials that demonstrate your abilities. 24 | -------------------------------------------------------------------------------- /courses/dss/raft/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let includes = &[std::path::PathBuf::from("src/proto")]; 3 | let mut protos = Vec::new(); 4 | for include in includes { 5 | for file in std::fs::read_dir(include).unwrap() { 6 | let file = file.unwrap(); 7 | if file.file_type().unwrap().is_dir() { 8 | continue; 9 | } 10 | let path = file.path(); 11 | if path.extension().unwrap() == "proto" { 12 | protos.push(path); 13 | } 14 | } 15 | } 16 | prost_build::compile_protos(&protos, includes).unwrap(); 17 | for p in protos { 18 | println!("cargo:rerun-if-changed={}", p.display()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /talent-challenge-program2021/PROJECT_IDEA_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ``` 2 | 3 | ## Description 4 | 5 | what's the problem, how to solve it 6 | 7 | ## Document Collection 8 | 9 | - Proposal doc: ${proposal doc} 10 | - Weekly report: ${weekly report link} 11 | 12 | ## Talent Challenge Program information 13 | 14 | - Mentor of this issue: @github_id 15 | - Recommended skills: ${skills and URL link} 16 | - Estimated Workloads: (XL, man-month) 17 | 18 | ## Milestones and action items 19 | 20 | Milestone 1: ${milestone descption}, Expected finish date: ${date} 21 | 22 | - [ ] action item 1 23 | - [ ] action item 2 24 | 25 | Milestone 2: ${milestone descption}, Expected finish date: ${date} 26 | 27 | - [ ] action item 1 28 | - [ ] action item 2 29 | 30 | ``` 31 | -------------------------------------------------------------------------------- /courses/README.md: -------------------------------------------------------------------------------- 1 | # All public courses 2 | 3 | Here we list all public courses of Talent Plan, including: 4 | 5 | - [TP 101: Introduction to open source software](tp101-intro-to-oss.md) 6 | - [TP 102: How to use Git and GitHub](tp102-how-to-use-git-github.md) 7 | - [TP 103: Build a welcoming community](tp103-open-source-community.md) 8 | - [TP 201: Practical Networked Applications in Rust](rust/README.md) 9 | - [TP 202: Distributed Systems in Rust](dss/README.md) 10 | - [TP 301: TinySQL, a distributed relational database in Go](https://github.com/pingcap-incubator/tinysql) 11 | - [TP 302: TinyKV, a distributed key value database in Go](https://github.com/pingcap-incubator/tinykv) 12 | - TP 401: Deep Dive into TiDB(WIP) 13 | - TP 402: Deep Dive into TiKV(WIP) 14 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/thread_pool/rayon.rs: -------------------------------------------------------------------------------- 1 | use super::ThreadPool; 2 | use crate::{KvsError, Result}; 3 | use std::sync::Arc; 4 | 5 | /// Wrapper of rayon::ThreadPool 6 | #[derive(Clone)] 7 | pub struct RayonThreadPool(Arc); 8 | 9 | impl ThreadPool for RayonThreadPool { 10 | fn new(threads: u32) -> Result { 11 | let pool = rayon::ThreadPoolBuilder::new() 12 | .num_threads(threads as usize) 13 | .build() 14 | .map_err(|e| KvsError::StringError(format!("{}", e)))?; 15 | Ok(RayonThreadPool(Arc::new(pool))) 16 | } 17 | 18 | fn spawn(&self, job: F) 19 | where 20 | F: FnOnce() + Send + 'static, 21 | { 22 | self.0.spawn(job) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /courses/dss/raft/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft" 3 | version = "0.1.0" 4 | authors = [ 5 | "Neil Shen ", 6 | "ShuNing ", 7 | "Connor ", 8 | "庄天翼 ", 9 | "MIT 6.824 <6824-staff@lists.csail.mit.edu>" 10 | ] 11 | edition = "2018" 12 | publish = false 13 | 14 | [dependencies] 15 | async-trait = "0.1" 16 | futures = "0.3" 17 | futures-timer = "3.0" 18 | log = "0.4" 19 | prost = "0.6" 20 | prost-derive = "0.6" 21 | rand = "0.7" 22 | 23 | labcodec = { path = "../labcodec" } 24 | labrpc = { path = "../labrpc" } 25 | linearizability = { path = "../linearizability"} 26 | 27 | [dev-dependencies] 28 | env_logger = "0.7" 29 | 30 | [build-dependencies] 31 | prost-build = "0.6" 32 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | dss: 4 | docker: 5 | - image: circleci/rust:stretch 6 | environment: 7 | RUST_BACKTRACE: "1" 8 | RUSTFLAGS: "-Dwarnings" 9 | working_directory: ~/talent-plan/courses/dss 10 | steps: 11 | - checkout: 12 | path: ~/talent-plan 13 | - run: rustup component add clippy-preview rustfmt-preview 14 | - run: cargo fmt --all -- --check 15 | - run: cargo clippy --all --tests -- -D clippy::all 16 | - run: cargo test -p labcodec -p labrpc 17 | - run: cargo test -p raft -- --test raft::persister 18 | - run: 19 | command: cargo test -p linearizability 20 | no_output_timeout: 30m 21 | 22 | workflows: 23 | version: 2 24 | ci-test: 25 | jobs: 26 | - dss 27 | -------------------------------------------------------------------------------- /courses/dss/percolator/src/service.rs: -------------------------------------------------------------------------------- 1 | use crate::msg::{ 2 | CommitRequest, CommitResponse, GetRequest, GetResponse, PrewriteRequest, PrewriteResponse, 3 | TimestampRequest, TimestampResponse, 4 | }; 5 | 6 | labrpc::service! { 7 | service timestamp { 8 | rpc get_timestamp(TimestampRequest) returns (TimestampResponse); 9 | } 10 | } 11 | 12 | pub use timestamp::{add_service as add_tso_service, Client as TSOClient, Service}; 13 | 14 | labrpc::service! { 15 | service transaction { 16 | rpc get(GetRequest) returns (GetResponse); 17 | rpc prewrite(PrewriteRequest) returns (PrewriteResponse); 18 | rpc commit(CommitRequest) returns (CommitResponse); 19 | } 20 | } 21 | 22 | pub use transaction::{add_service as add_transaction_service, Client as TransactionClient}; 23 | -------------------------------------------------------------------------------- /courses/dss/raft/src/raft/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{error, fmt, result}; 2 | 3 | #[derive(Clone, Debug, PartialEq, Eq)] 4 | pub enum Error { 5 | Encode(labcodec::EncodeError), 6 | Decode(labcodec::DecodeError), 7 | Rpc(labrpc::Error), 8 | NotLeader, 9 | } 10 | 11 | impl fmt::Display for Error { 12 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 13 | write!(f, "{:?}", self) 14 | } 15 | } 16 | 17 | impl error::Error for Error { 18 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 19 | match *self { 20 | Error::Encode(ref e) => Some(e), 21 | Error::Decode(ref e) => Some(e), 22 | Error::Rpc(ref e) => Some(e), 23 | _ => None, 24 | } 25 | } 26 | } 27 | 28 | pub type Result = result::Result; 29 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 8 | 9 | ### What problem does this PR solve? 10 | 11 | - Issue number: close #xxx 12 | - Breif description of the problem: 13 | 14 | ### What is changed and how it works? 15 | 16 | ### Check List 17 | 18 | Tests 19 | 20 | - Unit test 21 | - Integration test 22 | - Manual test (add detailed scripts or steps below) 23 | - No code 24 | 25 | Side effects 26 | 27 | - Possible performance regression 28 | - Increased code complexity 29 | 30 | Related changes 31 | 32 | - Need to update the documentation 33 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvs" 3 | version = "0.1.0" 4 | authors = ["Yilin Chen "] 5 | description = "A key-value store" 6 | edition = "2018" 7 | 8 | [dependencies] 9 | clap = "2.33.0" 10 | structopt = "0.2.15" 11 | failure = "0.1.5" 12 | serde = { version = "1.0.89", features = ["derive"] } 13 | serde_json = "1.0.39" 14 | log = "0.4.6" 15 | env_logger = "0.6.1" 16 | sled = "0.34.6" 17 | crossbeam = "0.7.1" 18 | rayon = "1.0.3" 19 | num_cpus = "1.10.0" 20 | crossbeam-skiplist = { git = "https://github.com/crossbeam-rs/crossbeam.git", branch = "master" } 21 | 22 | [dev-dependencies] 23 | assert_cmd = "0.11" 24 | criterion = "0.3" 25 | crossbeam-utils = "0.6.5" 26 | predicates = "1.0.0" 27 | rand = "0.6.5" 28 | tempfile = "3.0.7" 29 | walkdir = "2.2.7" 30 | panic-control = "0.1.4" 31 | -------------------------------------------------------------------------------- /tidb/mergesort/bench_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | ) 7 | 8 | func BenchmarkMergeSort(b *testing.B) { 9 | numElements := 16 << 20 10 | src := make([]int64, numElements) 11 | original := make([]int64, numElements) 12 | prepare(original) 13 | 14 | b.ResetTimer() 15 | for i := 0; i < b.N; i++ { 16 | b.StopTimer() 17 | copy(src, original) 18 | b.StartTimer() 19 | MergeSort(src) 20 | } 21 | } 22 | 23 | func BenchmarkNormalSort(b *testing.B) { 24 | numElements := 16 << 20 25 | src := make([]int64, numElements) 26 | original := make([]int64, numElements) 27 | prepare(original) 28 | 29 | b.ResetTimer() 30 | for i := 0; i < b.N; i++ { 31 | b.StopTimer() 32 | copy(src, original) 33 | b.StartTimer() 34 | sort.Slice(src, func(i, j int) bool { return src[i] < src[j] }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /courses/dss/Makefile: -------------------------------------------------------------------------------- 1 | export RUSTFLAGS=-Dwarnings 2 | export RUST_TEST_THREADS=1 3 | export RUST_BACKTRACE=1 4 | 5 | LOG_LEVEL ?= raft=info,percolator=info 6 | 7 | check: 8 | cargo fmt --all -- --check 9 | cargo clippy --all --tests -- -D clippy::all 10 | 11 | test: test_others test_2 test_3 12 | 13 | test_2: test_2a test_2b test_2c test_2d 14 | 15 | test_2a: cargo_test_2a 16 | 17 | test_2b: cargo_test_2b 18 | 19 | test_2c: cargo_test_2c 20 | 21 | test_2d: cargo_test_2d 22 | 23 | test_3: test_3a test_3b 24 | 25 | test_3a: cargo_test_3a 26 | 27 | test_3b: cargo_test_3b 28 | 29 | cargo_test_%: check 30 | RUST_LOG=${LOG_LEVEL} cargo test -p raft -- --nocapture --test $* 31 | 32 | test_others: check 33 | RUST_LOG=${LOG_LEVEL} cargo test -p labrpc -p labcodec -- --nocapture 34 | 35 | test_percolator: check 36 | RUST_LOG=${LOG_LEVEL} cargo test -p percolator -- --nocapture 37 | -------------------------------------------------------------------------------- /courses/dss/percolator/proto/msg.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffers are a flexible, efficient, automated mechanism for serializing structured data. 2 | // You can define how you want your data to be structured. 3 | // Details can be found in https://developers.google.com/protocol-buffers/docs/proto3. 4 | 5 | // Once you have defined the message, the `build.rs` will generate the corresponding data structure in `OUT_DIR`. 6 | // You can use the structure by importing the `msg` module. 7 | // Example: 8 | // use crate::msg::CommitRequest; 9 | 10 | syntax = "proto3"; 11 | 12 | package msg; 13 | 14 | message TimestampRequest {} 15 | 16 | message TimestampResponse {} 17 | 18 | message GetRequest {} 19 | 20 | message GetResponse {} 21 | 22 | message PrewriteRequest {} 23 | 24 | message PrewriteResponse {} 25 | 26 | message CommitRequest { 27 | bool is_primary = 1; 28 | } 29 | 30 | message CommitResponse {} 31 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/engines/mod.rs: -------------------------------------------------------------------------------- 1 | pub use self::kvs::KvStore; 2 | pub use self::sled::SledKvsEngine; 3 | use crate::Result; 4 | 5 | mod kvs; 6 | mod sled; 7 | 8 | /// Trait for a key value storage engine. 9 | pub trait KvsEngine: Clone + Send + 'static { 10 | /// Sets the value of a string key to a string. 11 | /// 12 | /// If the key already exists, the previous value will be overwritten. 13 | fn set(&self, key: String, value: String) -> Result<()>; 14 | 15 | /// Gets the string value of a given string key. 16 | /// 17 | /// Returns `None` if the given key does not exist. 18 | fn get(&self, key: String) -> Result>; 19 | 20 | /// Removes a given key. 21 | /// 22 | /// # Errors 23 | /// 24 | /// It returns `KvsError::KeyNotFound` if the given key is not found. 25 | fn remove(&self, key: String) -> Result<()>; 26 | } 27 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvs" 3 | version = "0.1.0" 4 | authors = ["Yilin Chen "] 5 | description = "A key-value store" 6 | edition = "2018" 7 | 8 | [dependencies] 9 | clap = "2.33.0" 10 | structopt = "0.2.15" 11 | failure = "0.1.5" 12 | serde = { version = "1.0.89", features = ["derive"] } 13 | serde_json = "1.0.39" 14 | log = "0.4.6" 15 | env_logger = "0.6.1" 16 | sled = "0.22.1" 17 | crossbeam = "0.7.1" 18 | rayon = "1.0.3" 19 | num_cpus = "1.10.0" 20 | crossbeam-skiplist = { version = "0.0.0", git = "https://github.com/crossbeam-rs/crossbeam.git", rev = "8cc906b" } 21 | tokio = "0.1.21" 22 | tokio-serde-json = "0.2.0" 23 | 24 | [dev-dependencies] 25 | assert_cmd = "0.11" 26 | criterion = "0.2.11" 27 | crossbeam-utils = "0.6.5" 28 | predicates = "1.0.0" 29 | rand = "0.6.5" 30 | tempfile = "3.0.7" 31 | walkdir = "2.2.7" 32 | panic-control = "0.1.4" -------------------------------------------------------------------------------- /courses/dss/labrpc/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::{error, fmt, result}; 2 | 3 | use futures::channel::oneshot::Canceled; 4 | 5 | use labcodec::{DecodeError, EncodeError}; 6 | 7 | #[derive(Clone, Debug, PartialEq, Eq)] 8 | pub enum Error { 9 | Unimplemented(String), 10 | Encode(EncodeError), 11 | Decode(DecodeError), 12 | Recv(Canceled), 13 | Timeout, 14 | Stopped, 15 | Other(String), 16 | } 17 | 18 | impl fmt::Display for Error { 19 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 20 | write!(f, "{:?}", self) 21 | } 22 | } 23 | 24 | impl error::Error for Error { 25 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 26 | match *self { 27 | Error::Encode(ref e) => Some(e), 28 | Error::Decode(ref e) => Some(e), 29 | Error::Recv(ref e) => Some(e), 30 | _ => None, 31 | } 32 | } 33 | } 34 | 35 | pub type Result = result::Result; 36 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/engines/mod.rs: -------------------------------------------------------------------------------- 1 | //! This module provides various key value storage engines. 2 | 3 | use crate::Result; 4 | 5 | /// Trait for a key value storage engine. 6 | pub trait KvsEngine { 7 | /// Sets the value of a string key to a string. 8 | /// 9 | /// If the key already exists, the previous value will be overwritten. 10 | fn set(&mut self, key: String, value: String) -> Result<()>; 11 | 12 | /// Gets the string value of a given string key. 13 | /// 14 | /// Returns `None` if the given key does not exist. 15 | fn get(&mut self, key: String) -> Result>; 16 | 17 | /// Removes a given key. 18 | /// 19 | /// # Errors 20 | /// 21 | /// It returns `KvsError::KeyNotFound` if the given key is not found. 22 | fn remove(&mut self, key: String) -> Result<()>; 23 | } 24 | 25 | mod kvs; 26 | mod sled; 27 | 28 | pub use self::kvs::KvStore; 29 | pub use self::sled::SledKvsEngine; 30 | -------------------------------------------------------------------------------- /tidb/mergesort/mergesort_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | "sort" 6 | "testing" 7 | "time" 8 | 9 | "github.com/pingcap/check" 10 | ) 11 | 12 | var _ = check.Suite(&sortTestSuite{}) 13 | 14 | func TestT(t *testing.T) { 15 | check.TestingT(t) 16 | } 17 | 18 | func prepare(src []int64) { 19 | rand.Seed(time.Now().Unix()) 20 | for i := range src { 21 | src[i] = rand.Int63() 22 | } 23 | } 24 | 25 | type sortTestSuite struct{} 26 | 27 | func (s *sortTestSuite) TestMergeSort(c *check.C) { 28 | lens := []int{1, 3, 5, 7, 11, 13, 17, 19, 23, 29, 1024, 1 << 13, 1 << 17, 1 << 19, 1 << 20} 29 | 30 | for i := range lens { 31 | src := make([]int64, lens[i]) 32 | expect := make([]int64, lens[i]) 33 | prepare(src) 34 | copy(expect, src) 35 | MergeSort(src) 36 | sort.Slice(expect, func(i, j int) bool { return expect[i] < expect[j] }) 37 | for i := 0; i < len(src); i++ { 38 | c.Assert(src[i], check.Equals, expect[i]) 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /courses/dss/raft/src/proto/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod raftpb { 2 | include!(concat!(env!("OUT_DIR"), "/raftpb.rs")); 3 | 4 | labrpc::service! { 5 | service raft { 6 | rpc request_vote(RequestVoteArgs) returns (RequestVoteReply); 7 | 8 | // Your code here if more rpc desired. 9 | // rpc xxx(yyy) returns (zzz) 10 | } 11 | } 12 | pub use self::raft::{ 13 | add_service as add_raft_service, Client as RaftClient, Service as RaftService, 14 | }; 15 | } 16 | 17 | pub mod kvraftpb { 18 | include!(concat!(env!("OUT_DIR"), "/kvraftpb.rs")); 19 | 20 | labrpc::service! { 21 | service kv { 22 | rpc get(GetRequest) returns (GetReply); 23 | rpc put_append(PutAppendRequest) returns (PutAppendReply); 24 | 25 | // Your code here if more rpc desired. 26 | // rpc xxx(yyy) returns (zzz) 27 | } 28 | } 29 | pub use self::kv::{add_service as add_kv_service, Client as KvClient, Service as KvService}; 30 | } 31 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/engines/mod.rs: -------------------------------------------------------------------------------- 1 | pub use self::kvs::KvStore; 2 | pub use self::sled::SledKvsEngine; 3 | use crate::KvsError; 4 | 5 | use tokio::prelude::Future; 6 | 7 | mod kvs; 8 | mod sled; 9 | 10 | /// Trait for a key value storage engine. 11 | pub trait KvsEngine: Clone + Send + 'static { 12 | /// Sets the value of a string key to a string. 13 | /// 14 | /// If the key already exists, the previous value will be overwritten. 15 | fn set(&self, key: String, value: String) -> Box + Send>; 16 | 17 | /// Gets the string value of a given string key. 18 | /// 19 | /// Returns `None` if the given key does not exist. 20 | fn get(&self, key: String) -> Box, Error = KvsError> + Send>; 21 | 22 | /// Removes a given key. 23 | /// 24 | /// # Errors 25 | /// 26 | /// It returns `KvsError::KeyNotFound` if the given key is not found. 27 | fn remove(&self, key: String) -> Box + Send>; 28 | } 29 | -------------------------------------------------------------------------------- /courses/rust/projects/project-2/src/error.rs: -------------------------------------------------------------------------------- 1 | use failure::Fail; 2 | use std::io; 3 | 4 | /// Error type for kvs. 5 | #[derive(Fail, Debug)] 6 | pub enum KvsError { 7 | /// IO error. 8 | #[fail(display = "{}", _0)] 9 | Io(#[cause] io::Error), 10 | /// Serialization or deserialization error. 11 | #[fail(display = "{}", _0)] 12 | Serde(#[cause] serde_json::Error), 13 | /// Removing non-existent key error. 14 | #[fail(display = "Key not found")] 15 | KeyNotFound, 16 | /// Unexpected command type error. 17 | /// It indicated a corrupted log or a program bug. 18 | #[fail(display = "Unexpected command type")] 19 | UnexpectedCommandType, 20 | } 21 | 22 | impl From for KvsError { 23 | fn from(err: io::Error) -> KvsError { 24 | KvsError::Io(err) 25 | } 26 | } 27 | 28 | impl From for KvsError { 29 | fn from(err: serde_json::Error) -> KvsError { 30 | KvsError::Serde(err) 31 | } 32 | } 33 | 34 | /// Result type for kvs. 35 | pub type Result = std::result::Result; 36 | -------------------------------------------------------------------------------- /talent-challenge-program/s1.md: -------------------------------------------------------------------------------- 1 | # Talent Challenge Program Season 1 2 | 3 | _Status: ongoing_ 4 | 5 | ### Timeline 6 | _Note: this timeline is proposed and may be subject to change before the formal program launch announcement_ 7 | 8 | - June 7 - June 19: project applications opened 9 | - projects are invited to submit their proposals to [Project Ideas](project-ideas.md) page; 10 | - June 21: selected projects/slots are announced; 11 | - June 21 - July 5: mentees applications opened, projects are selecting mentees; 12 | - July 8 - mentees are selected by the mentors, coding starts; 13 | - August 5 - 1st evaluation checkpoint; 14 | - Mentors verify the quality of the completed tasks; 15 | - First 50% of the stipend is being paid to the mentee if this stage is passed; 16 | - September 16 - 2nd (final) evaluation checkpoint; 17 | - Coding ends; 18 | - The other 50% of the stipend is being paid to the mentees; 19 | - September 23: results announced! 20 | 21 | ### Selected Projects 22 | 23 | Selected projects are listed on the [Selected Projects](selected-projects.md) page 24 | -------------------------------------------------------------------------------- /talent-challenge-program2021/project-ideas.md: -------------------------------------------------------------------------------- 1 | ## Project ideas 2 | 3 | Project maintainers and mentors, please submit the ideas below section using the template. Project ideas selected in season 1 will be listed in [Selected Projects](selected-projects.md) page 4 | 5 | ### Template 6 | 7 | ``` 8 | ## TiDB Ecosystem Project Name 9 | 10 | ## Title 11 | 12 | ## Description 13 | 14 | what's the problem, how to solve it 15 | 16 | ## Document Collection 17 | 18 | - Proposal doc: ${proposal doc} 19 | - Weekly report: ${weekly report link} 20 | 21 | ## Talent Challenge Program information 22 | 23 | - Mentor of this issue: @github_id 24 | - Recommended skills: ${skills and URL link} 25 | - Estimated Workloads: (XL, man-month) 26 | 27 | ## Milestones and action items 28 | 29 | Milestone 1: ${milestone descption}, Expected finish date: ${date} 30 | 31 | - [ ] action item 1 32 | - [ ] action item 2 33 | 34 | Milestone 2: ${milestone descption}, Expected finish date: ${date} 35 | 36 | - [ ] action item 1 37 | - [ ] action item 2 38 | 39 | ``` 40 | 41 | ### Proposed Project ideas 42 | 43 | 44 | -------------------------------------------------------------------------------- /talent-plan-1.0/README-CN.md: -------------------------------------------------------------------------------- 1 | # Talent Plan Courses 1.0 2 | 3 | Talent Plan 1.0 课程中共有 2 条学习路径供大家选择，分别是： 4 | 5 | * 面向分布式关系型数据库 SQL 层的 [TiDB 方向](1.0-lp-tidb.md) 6 | * 面向分布式 Key-value 数据库存储层的 [TiKV 方向](1.0-lp-tikv.md) 7 | 8 | 正在学习 Talent Plan 1.0 课程的小伙伴可通过上述链接查看相关路径的课程详情。 9 | 10 | > ***特别提醒：*** 11 | > 12 | > Talent Plan 2.0 课程已正式上线，相比 1.0 版本，新增了用 Go 语言全新设计的分布式关系型数据库 TinySQL 课程和分布式 Key-value 数据库 TinyKV 课程，并结合大家的兴趣爱好和知识背景全新规划了 5 条推荐的学习路径，分别是： 13 | > 14 | > * 路径 1：[实现一个 Mini 版本的分布式关系型数据库](https://learn.pingcap.com/learner/talent-plan/implement-a-mini-distributed-relational-database) 15 | > * 路径 2：[实现一个 Mini 版本的分布式 Key-value 数据库](https://learn.pingcap.com/learner/talent-plan/implement-a-mini-distributed-key-value-database) 16 | > * 路径 3：[参与工业级开源分布式关系型数据库 TiDB 的开发实践](https://learn.pingcap.com/learner/talent-plan/become-a-tidb-contributor) 17 | > * 路径 4：[参与工业级开源分布式 Key-value 数据库 TiKV 的开发实践](https://learn.pingcap.com/learner/talent-plan/become-a-tikv-contributor) 18 | > * 路径 5：[Rust 编程原理与实践](>https://learn.pingcap.com/learner/talent-plan/rust-programming) 19 | > 20 | > 小伙伴们也可直接通过 [Talent Plan 官网](https://learn.pingcap.com/learner/talent-plan) 开始 Talent Plan 2.0 课程的学习。 21 | -------------------------------------------------------------------------------- /tidb/mergesort/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | This is the Merge Sort home work for PingCAP Talent Plan Online of week 1. 4 | 5 | There are 16, 000, 000 int64 values stored in an unordered array. Please 6 | supplement the `MergeSort()` function defined in `mergesort.go` to sort this 7 | array. 8 | 9 | Requirements and rating principles: 10 | * (30%) Pass the unit test. 11 | * (20%) Performs better than `sort.Slice()`. 12 | * (40%) Have a document to describe your idea and record the process of performance optimization with `pprof`. 13 | * (10%) Have a good code style. 14 | 15 | NOTE: **go 1.12 is required** 16 | 17 | ## How to use 18 | 19 | Please supplement the `MergeSort()` function defined in `mergesort.go` to accomplish 20 | the home work. 21 | 22 | **NOTE**: 23 | 1. There is a builtin unit test defined in `mergesort_test.go`, however, you still 24 | can write your own unit tests. 25 | 2. There is a builtin benchmark test defined in `bench_test.go`, you should run 26 | this benchmark to ensure that your parallel merge sort is fast enough. 27 | 28 | 29 | How to test: 30 | ``` 31 | make test 32 | ``` 33 | 34 | How to benchmark: 35 | ``` 36 | make bench 37 | ``` 38 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/engines/sled.rs: -------------------------------------------------------------------------------- 1 | use super::KvsEngine; 2 | use crate::{KvsError, Result}; 3 | use sled::{Db, Tree}; 4 | 5 | /// Wrapper of `sled::Db` 6 | #[derive(Clone)] 7 | pub struct SledKvsEngine(Db); 8 | 9 | impl SledKvsEngine { 10 | /// Creates a `SledKvsEngine` from `sled::Db`. 11 | pub fn new(db: Db) -> Self { 12 | SledKvsEngine(db) 13 | } 14 | } 15 | 16 | impl KvsEngine for SledKvsEngine { 17 | fn set(&self, key: String, value: String) -> Result<()> { 18 | let tree: &Tree = &self.0; 19 | tree.insert(key, value.into_bytes()).map(|_| ())?; 20 | tree.flush()?; 21 | Ok(()) 22 | } 23 | 24 | fn get(&self, key: String) -> Result> { 25 | let tree: &Tree = &self.0; 26 | Ok(tree 27 | .get(key)? 28 | .map(|i_vec| AsRef::<[u8]>::as_ref(&i_vec).to_vec()) 29 | .map(String::from_utf8) 30 | .transpose()?) 31 | } 32 | 33 | fn remove(&self, key: String) -> Result<()> { 34 | let tree: &Tree = &self.0; 35 | tree.remove(key)?.ok_or(KvsError::KeyNotFound)?; 36 | tree.flush()?; 37 | Ok(()) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/engines/sled.rs: -------------------------------------------------------------------------------- 1 | use super::KvsEngine; 2 | use crate::{KvsError, Result}; 3 | use sled::{Db, Tree}; 4 | 5 | /// Wrapper of `sled::Db` 6 | #[derive(Clone)] 7 | pub struct SledKvsEngine(Db); 8 | 9 | impl SledKvsEngine { 10 | /// Creates a `SledKvsEngine` from `sled::Db`. 11 | pub fn new(db: Db) -> Self { 12 | SledKvsEngine(db) 13 | } 14 | } 15 | 16 | impl KvsEngine for SledKvsEngine { 17 | fn set(&mut self, key: String, value: String) -> Result<()> { 18 | let tree: &Tree = &self.0; 19 | tree.insert(key, value.into_bytes()).map(|_| ())?; 20 | tree.flush()?; 21 | Ok(()) 22 | } 23 | 24 | fn get(&mut self, key: String) -> Result> { 25 | let tree: &Tree = &self.0; 26 | Ok(tree 27 | .get(key)? 28 | .map(|i_vec| AsRef::<[u8]>::as_ref(&i_vec).to_vec()) 29 | .map(String::from_utf8) 30 | .transpose()?) 31 | } 32 | 33 | fn remove(&mut self, key: String) -> Result<()> { 34 | let tree: &Tree = &self.0; 35 | tree.remove(key)?.ok_or(KvsError::KeyNotFound)?; 36 | tree.flush()?; 37 | Ok(()) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/thread_pool/mod.rs: -------------------------------------------------------------------------------- 1 | //! This module provides various thread pools. All thread pools should implement 2 | //! the `ThreadPool` trait. 3 | 4 | use crate::Result; 5 | 6 | mod naive; 7 | mod rayon; 8 | mod shared_queue; 9 | 10 | pub use self::naive::NaiveThreadPool; 11 | pub use self::rayon::RayonThreadPool; 12 | pub use self::shared_queue::SharedQueueThreadPool; 13 | 14 | /// The trait that all thread pools should implement. 15 | pub trait ThreadPool { 16 | /// Creates a new thread pool, immediately spawning the specified number of 17 | /// threads. 18 | /// 19 | /// Returns an error if any thread fails to spawn. All previously-spawned threads 20 | /// are terminated. 21 | fn new(threads: u32) -> Result 22 | where 23 | Self: Sized; 24 | 25 | /// Spawns a function into the thread pool. 26 | /// 27 | /// Spawning always succeeds, but if the function panics the threadpool continues 28 | /// to operate with the same number of threads — the thread count is not 29 | /// reduced nor is the thread pool destroyed, corrupted or invalidated. 30 | fn spawn(&self, job: F) 31 | where 32 | F: FnOnce() + Send + 'static; 33 | } 34 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/thread_pool/mod.rs: -------------------------------------------------------------------------------- 1 | //! This module provides various thread pools. All thread pools should implement 2 | //! the `ThreadPool` trait. 3 | 4 | use crate::Result; 5 | 6 | mod naive; 7 | mod rayon; 8 | mod shared_queue; 9 | 10 | pub use self::naive::NaiveThreadPool; 11 | pub use self::rayon::RayonThreadPool; 12 | pub use self::shared_queue::SharedQueueThreadPool; 13 | 14 | /// The trait that all thread pools should implement. 15 | pub trait ThreadPool: Clone + Send + 'static { 16 | /// Creates a new thread pool, immediately spawning the specified number of 17 | /// threads. 18 | /// 19 | /// Returns an error if any thread fails to spawn. All previously-spawned threads 20 | /// are terminated. 21 | fn new(threads: u32) -> Result 22 | where 23 | Self: Sized; 24 | 25 | /// Spawns a function into the thread pool. 26 | /// 27 | /// Spawning always succeeds, but if the function panics the threadpool continues 28 | /// to operate with the same number of threads — the thread count is not 29 | /// reduced nor is the thread pool destroyed, corrupted or invalidated. 30 | fn spawn(&self, job: F) 31 | where 32 | F: FnOnce() + Send + 'static; 33 | } 34 | -------------------------------------------------------------------------------- /courses/dss/labrpc/examples/echo.rs: -------------------------------------------------------------------------------- 1 | use futures::executor::block_on; 2 | use prost_derive::Message; 3 | 4 | use labrpc::*; 5 | 6 | /// A Hand-written protobuf messages 7 | #[derive(Clone, PartialEq, Message)] 8 | pub struct Echo { 9 | #[prost(int64, tag = "1")] 10 | pub x: i64, 11 | } 12 | 13 | service! { 14 | service echo { 15 | rpc ping(Echo) returns (Echo); 16 | } 17 | } 18 | use echo::{add_service, Client, Service}; 19 | 20 | #[derive(Clone)] 21 | struct EchoService; 22 | 23 | #[async_trait::async_trait] 24 | impl Service for EchoService { 25 | async fn ping(&self, input: Echo) -> Result { 26 | Ok(input) 27 | } 28 | } 29 | 30 | fn main() { 31 | let rn = Network::new(); 32 | let server_name = "echo_server"; 33 | let mut builder = ServerBuilder::new(server_name.to_owned()); 34 | add_service(EchoService, &mut builder).unwrap(); 35 | let server = builder.build(); 36 | rn.add_server(server); 37 | 38 | let client_name = "client"; 39 | let client = Client::new(rn.create_client(client_name.to_owned())); 40 | rn.enable(client_name, true); 41 | rn.connect(client_name, server_name); 42 | 43 | let reply = block_on(async { client.ping(&Echo { x: 777 }).await.unwrap() }); 44 | assert_eq!(reply, Echo { x: 777 }); 45 | println!("{:?}", reply); 46 | } 47 | -------------------------------------------------------------------------------- /courses/rust/projects/project-1/src/kv.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | /// The `KvStore` stores string key/value pairs. 4 | /// 5 | /// Key/value pairs are stored in a `HashMap` in memory and not persisted to disk. 6 | /// 7 | /// Example: 8 | /// 9 | /// ```rust 10 | /// # use kvs::KvStore; 11 | /// let mut store = KvStore::new(); 12 | /// store.set("key".to_owned(), "value".to_owned()); 13 | /// let val = store.get("key".to_owned()); 14 | /// assert_eq!(val, Some("value".to_owned())); 15 | /// ``` 16 | #[derive(Default)] 17 | pub struct KvStore { 18 | map: HashMap, 19 | } 20 | 21 | impl KvStore { 22 | /// Creates a `KvStore`. 23 | pub fn new() -> KvStore { 24 | KvStore { 25 | map: HashMap::new(), 26 | } 27 | } 28 | 29 | /// Sets the value of a string key to a string. 30 | /// 31 | /// If the key already exists, the previous value will be overwritten. 32 | pub fn set(&mut self, key: String, value: String) { 33 | self.map.insert(key, value); 34 | } 35 | 36 | /// Gets the string value of a given string key. 37 | /// 38 | /// Returns `None` if the given key does not exist. 39 | pub fn get(&self, key: String) -> Option { 40 | self.map.get(&key).cloned() 41 | } 42 | 43 | /// Remove a given key. 44 | pub fn remove(&mut self, key: String) { 45 | self.map.remove(&key); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tidb/join/join_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pingcap/check" 7 | ) 8 | 9 | type joinTestSuite struct{} 10 | 11 | func TestT(t *testing.T) { 12 | check.TestingT(t) 13 | } 14 | 15 | var _ = check.Suite(&joinTestSuite{}) 16 | 17 | func (s *joinTestSuite) TestJoin(c *check.C) { 18 | for _, t := range []struct { 19 | f0 string 20 | f1 string 21 | offsets0 []int 22 | offsets1 []int 23 | sum uint64 24 | }{ 25 | // r0 join r0 on r0.col0 = r0.col1 26 | {"./t/r0.tbl", "./t/r0.tbl", []int{0}, []int{1}, 0x16CBF2D}, 27 | // r0 join r1 on r0.col0 = r1.col0 28 | {"./t/r0.tbl", "./t/r1.tbl", []int{0}, []int{0}, 0xC1D73B}, 29 | // r0 join r1 on r0.col1 = r1.col1 30 | {"./t/r0.tbl", "./t/r1.tbl", []int{1}, []int{1}, 0x33010}, 31 | // r0 join r2 on r0.col0 = r2.col0 32 | {"./t/r0.tbl", "./t/r2.tbl", []int{0}, []int{0}, 0x1F235}, 33 | // r0 join r1 on r0.col0 = r1.col0 and r0.col1 = r1.col1 34 | {"./t/r0.tbl", "./t/r1.tbl", []int{0, 1}, []int{0, 1}, 0}, 35 | // r1 join r2 on r1.col0 = r2.col0 36 | {"./t/r1.tbl", "./t/r2.tbl", []int{0}, []int{0}, 0x18CDA}, 37 | // r2 join r2 on r2.col0 = r2.col0 and r2.col1 = r2.col1 38 | {"./t/r2.tbl", "./t/r2.tbl", []int{0, 1}, []int{0, 1}, 0x5B385}, 39 | } { 40 | c.Assert(Join(t.f0, t.f1, t.offsets0, t.offsets1), check.Equals, t.sum) 41 | //c.Assert(JoinExample(t.f0, t.f1, t.offsets0, t.offsets1), check.Equals, t.sum) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/error.rs: -------------------------------------------------------------------------------- 1 | use failure::Fail; 2 | use std::io; 3 | use std::string::FromUtf8Error; 4 | 5 | /// Error type for kvs 6 | #[derive(Fail, Debug)] 7 | pub enum KvsError { 8 | /// IO error 9 | #[fail(display = "IO error: {}", _0)] 10 | Io(#[cause] io::Error), 11 | /// Serialization or deserialization error 12 | #[fail(display = "serde_json error: {}", _0)] 13 | Serde(#[cause] serde_json::Error), 14 | /// Removing non-existent key error 15 | #[fail(display = "Key not found")] 16 | KeyNotFound, 17 | /// Unexpected command type error. 18 | /// It indicated a corrupted log or a program bug. 19 | #[fail(display = "Unexpected command type")] 20 | UnexpectedCommandType, 21 | /// Key or value is invalid UTF-8 sequence 22 | #[fail(display = "UTF-8 error: {}", _0)] 23 | Utf8(#[cause] FromUtf8Error), 24 | /// Sled error 25 | #[fail(display = "sled error: {}", _0)] 26 | Sled(#[cause] sled::Error), 27 | /// Error with a string message 28 | #[fail(display = "{}", _0)] 29 | StringError(String), 30 | } 31 | 32 | impl From for KvsError { 33 | fn from(err: io::Error) -> KvsError { 34 | KvsError::Io(err) 35 | } 36 | } 37 | 38 | impl From for KvsError { 39 | fn from(err: serde_json::Error) -> KvsError { 40 | KvsError::Serde(err) 41 | } 42 | } 43 | 44 | impl From for KvsError { 45 | fn from(err: FromUtf8Error) -> KvsError { 46 | KvsError::Utf8(err) 47 | } 48 | } 49 | 50 | impl From for KvsError { 51 | fn from(err: sled::Error) -> KvsError { 52 | KvsError::Sled(err) 53 | } 54 | } 55 | 56 | /// Result type for kvs 57 | pub type Result = std::result::Result; 58 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/error.rs: -------------------------------------------------------------------------------- 1 | use failure::Fail; 2 | use std::io; 3 | use std::string::FromUtf8Error; 4 | 5 | /// Error type for kvs 6 | #[derive(Fail, Debug)] 7 | pub enum KvsError { 8 | /// IO error 9 | #[fail(display = "IO error: {}", _0)] 10 | Io(#[cause] io::Error), 11 | /// Serialization or deserialization error 12 | #[fail(display = "serde_json error: {}", _0)] 13 | Serde(#[cause] serde_json::Error), 14 | /// Removing non-existent key error 15 | #[fail(display = "Key not found")] 16 | KeyNotFound, 17 | /// Unexpected command type error. 18 | /// It indicated a corrupted log or a program bug. 19 | #[fail(display = "Unexpected command type")] 20 | UnexpectedCommandType, 21 | /// Key or value is invalid UTF-8 sequence 22 | #[fail(display = "UTF-8 error: {}", _0)] 23 | Utf8(#[cause] FromUtf8Error), 24 | /// Sled error 25 | #[fail(display = "sled error: {}", _0)] 26 | Sled(#[cause] sled::Error), 27 | /// Error with a string message 28 | #[fail(display = "{}", _0)] 29 | StringError(String), 30 | } 31 | 32 | impl From for KvsError { 33 | fn from(err: io::Error) -> KvsError { 34 | KvsError::Io(err) 35 | } 36 | } 37 | 38 | impl From for KvsError { 39 | fn from(err: serde_json::Error) -> KvsError { 40 | KvsError::Serde(err) 41 | } 42 | } 43 | 44 | impl From for KvsError { 45 | fn from(err: FromUtf8Error) -> KvsError { 46 | KvsError::Utf8(err) 47 | } 48 | } 49 | 50 | impl From for KvsError { 51 | fn from(err: sled::Error) -> KvsError { 52 | KvsError::Sled(err) 53 | } 54 | } 55 | 56 | /// Result type for kvs 57 | pub type Result = std::result::Result; 58 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/error.rs: -------------------------------------------------------------------------------- 1 | use failure::Fail; 2 | use std::io; 3 | use std::string::FromUtf8Error; 4 | 5 | /// Error type for kvs 6 | #[derive(Fail, Debug)] 7 | pub enum KvsError { 8 | /// IO error 9 | #[fail(display = "IO error: {}", _0)] 10 | Io(#[cause] io::Error), 11 | /// Serialization or deserialization error 12 | #[fail(display = "serde_json error: {}", _0)] 13 | Serde(#[cause] serde_json::Error), 14 | /// Removing non-existent key error 15 | #[fail(display = "Key not found")] 16 | KeyNotFound, 17 | /// Unexpected command type error. 18 | /// It indicated a corrupted log or a program bug. 19 | #[fail(display = "Unexpected command type")] 20 | UnexpectedCommandType, 21 | /// Key or value is invalid UTF-8 sequence 22 | #[fail(display = "UTF-8 error: {}", _0)] 23 | Utf8(#[cause] FromUtf8Error), 24 | /// Sled error 25 | #[fail(display = "sled error: {}", _0)] 26 | Sled(#[cause] sled::Error), 27 | /// Error with a string message 28 | #[fail(display = "{}", _0)] 29 | StringError(String), 30 | } 31 | 32 | impl From for KvsError { 33 | fn from(err: io::Error) -> KvsError { 34 | KvsError::Io(err) 35 | } 36 | } 37 | 38 | impl From for KvsError { 39 | fn from(err: serde_json::Error) -> KvsError { 40 | KvsError::Serde(err) 41 | } 42 | } 43 | 44 | impl From for KvsError { 45 | fn from(err: FromUtf8Error) -> KvsError { 46 | KvsError::Utf8(err) 47 | } 48 | } 49 | 50 | impl From for KvsError { 51 | fn from(err: sled::Error) -> KvsError { 52 | KvsError::Sled(err) 53 | } 54 | } 55 | 56 | /// Result type for kvs 57 | pub type Result = std::result::Result; 58 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide 2 | 3 | Thanks for taking the time to contribute to Talent Plan! Contributions of any kind are welcome. 4 | 5 | ## Ways to contribute 6 | 7 | We love contributions from the community. Here are just a few of the ways you can contribute: 8 | 9 | - **Report bugs:** if you find a bug while taking courses, you may want to do a quick check of past issues to see if you can find any help there. Otherwise, create a new bug issue in the course repository using the provided template. 10 | - **Answer questions:** check out the open issues to see if you can help answer learner questions. 11 | - **Suggest changes:** this could come in the form of a new learning outcome, updates to the content, updates to the bot responses, or the logic of the course. If you'd like, you can fork the course and suggest the change that way, but most people like to talk about it first in an issue. 12 | - **Tackle an issue:** if you see an issue you would like to resolve, please feel free to fork the course and submit a PR (check out the instructions below). 13 | - **Translate a course:** if you have the ability to translate a course, we'd love to see it. Please fork the course and submit a PR. We'll need a second native speaker to confirm the translation so if you have anyone in mind, please @ mention them in your PR for a review. 14 | 15 | 16 | ## How to Contribute 17 | 18 | 1. Check out which [contributions](#ways-to-contribute) above you are willing to make 19 | 1. Fork this repository. 20 | 2. Commit your changes to your branch 21 | 3. Open a pull request to upstream 22 | 4. Request a review from the reviewers 23 | 24 | Want to konw more about how to use Git & GitHub to make contributions? Click [how to use Git & GitHub](courses/tp102-how-to-use-git-github.md) courses for more help 25 | -------------------------------------------------------------------------------- /courses/dss/raft/src/kvraft/client.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::proto::kvraftpb::*; 4 | 5 | enum Op { 6 | Put(String, String), 7 | Append(String, String), 8 | } 9 | 10 | pub struct Clerk { 11 | pub name: String, 12 | pub servers: Vec, 13 | // You will have to modify this struct. 14 | } 15 | 16 | impl fmt::Debug for Clerk { 17 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 18 | f.debug_struct("Clerk").field("name", &self.name).finish() 19 | } 20 | } 21 | 22 | impl Clerk { 23 | pub fn new(name: String, servers: Vec) -> Clerk { 24 | // You'll have to add code here. 25 | // Clerk { name, servers } 26 | crate::your_code_here((name, servers)) 27 | } 28 | 29 | /// fetch the current value for a key. 30 | /// returns "" if the key does not exist. 31 | /// keeps trying forever in the face of all other errors. 32 | // 33 | // you can send an RPC with code like this: 34 | // if let Some(reply) = self.servers[i].get(args).wait() { /* do something */ } 35 | pub fn get(&self, key: String) -> String { 36 | // You will have to modify this function. 37 | crate::your_code_here(key) 38 | } 39 | 40 | /// shared by Put and Append. 41 | // 42 | // you can send an RPC with code like this: 43 | // let reply = self.servers[i].put_append(args).unwrap(); 44 | fn put_append(&self, op: Op) { 45 | // You will have to modify this function. 46 | crate::your_code_here(op) 47 | } 48 | 49 | pub fn put(&self, key: String, value: String) { 50 | self.put_append(Op::Put(key, value)) 51 | } 52 | 53 | pub fn append(&self, key: String, value: String) { 54 | self.put_append(Op::Append(key, value)) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /courses/dss/linearizability/src/bitset.rs: -------------------------------------------------------------------------------- 1 | use std::num::Wrapping; 2 | 3 | #[derive(Clone)] 4 | pub struct Bitset(Vec); 5 | 6 | impl Bitset { 7 | pub fn new(bits: usize) -> Self { 8 | let extra = if bits % 64 != 0 { 1 } else { 0 }; 9 | Bitset(vec![0; bits / 64 + extra]) 10 | } 11 | 12 | pub fn set(&mut self, pos: usize) { 13 | let (major, minor) = bitset_index(pos); 14 | self.0[major] |= 1 << minor; 15 | } 16 | 17 | pub fn clear(&mut self, pos: usize) { 18 | let (major, minor) = bitset_index(pos); 19 | self.0[major] &= !(1 << minor); 20 | } 21 | 22 | fn popcnt(&self) -> usize { 23 | let mut total = 0; 24 | for b in &self.0 { 25 | let mut v = *b; 26 | v = (v & 0x5555_5555_5555_5555) + ((v & 0xAAAA_AAAA_AAAA_AAAA) >> 1); 27 | v = (v & 0x3333_3333_3333_3333) + ((v & 0xCCCC_CCCC_CCCC_CCCC) >> 2); 28 | v = (v & 0x0F0F_0F0F_0F0F_0F0F) + ((v & 0xF0F0_F0F0_F0F0_F0F0) >> 4); 29 | v = (Wrapping(v) * Wrapping(0x0101_0101_0101_0101)).0; 30 | total += ((v >> 56) & 0xFF) as usize; 31 | } 32 | total 33 | } 34 | 35 | pub fn hash(&self) -> u64 { 36 | let mut hash = self.popcnt() as u64; 37 | for v in &self.0 { 38 | hash ^= v; 39 | } 40 | hash 41 | } 42 | 43 | pub fn equals(&self, b2: &Bitset) -> bool { 44 | let b = &self.0; 45 | let b2 = &b2.0; 46 | if b.len() != b2.len() { 47 | return false; 48 | } 49 | for i in 0..b.len() { 50 | if b[i] != b2[i] { 51 | return false; 52 | } 53 | } 54 | true 55 | } 56 | } 57 | 58 | fn bitset_index(pos: usize) -> (usize, usize) { 59 | (pos / 64, pos % 64) 60 | } 61 | -------------------------------------------------------------------------------- /courses/rust/projects/project-1/src/bin/kvs.rs: -------------------------------------------------------------------------------- 1 | use clap::{App, AppSettings, Arg, SubCommand}; 2 | use std::process::exit; 3 | 4 | fn main() { 5 | let matches = App::new(env!("CARGO_PKG_NAME")) 6 | .version(env!("CARGO_PKG_VERSION")) 7 | .author(env!("CARGO_PKG_AUTHORS")) 8 | .about(env!("CARGO_PKG_DESCRIPTION")) 9 | .setting(AppSettings::DisableHelpSubcommand) 10 | .setting(AppSettings::SubcommandRequiredElseHelp) 11 | .setting(AppSettings::VersionlessSubcommands) 12 | .subcommand( 13 | SubCommand::with_name("set") 14 | .about("Set the value of a string key to a string") 15 | .arg(Arg::with_name("KEY").help("A string key").required(true)) 16 | .arg( 17 | Arg::with_name("VALUE") 18 | .help("The string value of the key") 19 | .required(true), 20 | ), 21 | ) 22 | .subcommand( 23 | SubCommand::with_name("get") 24 | .about("Get the string value of a given string key") 25 | .arg(Arg::with_name("KEY").help("A string key").required(true)), 26 | ) 27 | .subcommand( 28 | SubCommand::with_name("rm") 29 | .about("Remove a given key") 30 | .arg(Arg::with_name("KEY").help("A string key").required(true)), 31 | ) 32 | .get_matches(); 33 | 34 | match matches.subcommand() { 35 | ("set", Some(_matches)) => { 36 | eprintln!("unimplemented"); 37 | exit(1); 38 | } 39 | ("get", Some(_matches)) => { 40 | eprintln!("unimplemented"); 41 | exit(1); 42 | } 43 | ("rm", Some(_matches)) => { 44 | eprintln!("unimplemented"); 45 | exit(1); 46 | } 47 | _ => unreachable!(), 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /courses/dss/percolator/src/client.rs: -------------------------------------------------------------------------------- 1 | use labrpc::*; 2 | 3 | use crate::service::{TSOClient, TransactionClient}; 4 | 5 | // BACKOFF_TIME_MS is the wait time before retrying to send the request. 6 | // It should be exponential growth. e.g. 7 | //| retry time | backoff time | 8 | //|--------------|----------------| 9 | //| 1 | 100 | 10 | //| 2 | 200 | 11 | //| 3 | 400 | 12 | const BACKOFF_TIME_MS: u64 = 100; 13 | // RETRY_TIMES is the maximum number of times a client attempts to send a request. 14 | const RETRY_TIMES: usize = 3; 15 | 16 | /// Client mainly has two purposes: 17 | /// One is getting a monotonically increasing timestamp from TSO (Timestamp Oracle). 18 | /// The other is do the transaction logic. 19 | #[derive(Clone)] 20 | pub struct Client { 21 | // Your definitions here. 22 | } 23 | 24 | impl Client { 25 | /// Creates a new Client. 26 | pub fn new(tso_client: TSOClient, txn_client: TransactionClient) -> Client { 27 | // Your code here. 28 | Client {} 29 | } 30 | 31 | /// Gets a timestamp from a TSO. 32 | pub fn get_timestamp(&self) -> Result { 33 | // Your code here. 34 | unimplemented!() 35 | } 36 | 37 | /// Begins a new transaction. 38 | pub fn begin(&mut self) { 39 | // Your code here. 40 | unimplemented!() 41 | } 42 | 43 | /// Gets the value for a given key. 44 | pub fn get(&self, key: Vec) -> Result> { 45 | // Your code here. 46 | unimplemented!() 47 | } 48 | 49 | /// Sets keys in a buffer until commit time. 50 | pub fn set(&mut self, key: Vec, value: Vec) { 51 | // Your code here. 52 | unimplemented!() 53 | } 54 | 55 | /// Commits a transaction. 56 | pub fn commit(&self) -> Result { 57 | // Your code here. 58 | unimplemented!() 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /talent-challenge-program2021/schedule.md: -------------------------------------------------------------------------------- 1 | # Talent Challenge Program 2021 Schedule 2 | 3 | _Status: ongoing_ 4 | 5 | ### Timeline 6 | _Note: this timeline is proposed and may be subject to change before the formal program launch announcement_ 7 | 8 | #### Season 1 9 | 10 | - June 15 - July 15: mentees applications opened, projects are selecting mentees; 11 | - July 19 - mentees are selected by the mentors, coding starts; 12 | - 1st evaluation checkpoint (mentor decides); 13 | - Mentors verify the quality of the completed tasks; 14 | - First 50% of the stipend is being paid to the mentee if this stage is passed; 15 | - 2nd (final) evaluation checkpoint (mentor decides); 16 | - Coding ends; 17 | - The other 50% of the stipend is being paid to the mentees; 18 | - results announced! 19 | 20 | #### Season 2 21 | 22 | - Sep 01 - Sep 15: mentees applications opened, projects are selecting mentees; 23 | - Sep 20 - mentees are selected by the mentors, coding starts; 24 | - 1st evaluation checkpoint (mentor decides); 25 | - Mentors verify the quality of the completed tasks; 26 | - First 50% of the stipend is being paid to the mentee if this stage is passed; 27 | - 2nd (final) evaluation checkpoint (mentor decides); 28 | - Coding ends; 29 | - The other 50% of the stipend is being paid to the mentees; 30 | - results announced! 31 | 32 | #### Season 3 33 | 34 | - Nov 01 - Nov 15: mentees applications opened, projects are selecting mentees; 35 | - Nov 18 - mentees are selected by the mentors, coding starts; 36 | - 1st evaluation checkpoint (mentor decides); 37 | - Mentors verify the quality of the completed tasks; 38 | - First 50% of the stipend is being paid to the mentee if this stage is passed; 39 | - 2nd (final) evaluation checkpoint (mentor decides); 40 | - Coding ends; 41 | - The other 50% of the stipend is being paid to the mentees; 42 | - results announced! 43 | 44 | ### Selected Projects 45 | 46 | Selected projects are listed on the [Selected Projects](selected-projects.md) page 47 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/tests/thread_pool.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::{AtomicUsize, Ordering}; 2 | use std::sync::Arc; 3 | 4 | use kvs::thread_pool::*; 5 | use kvs::Result; 6 | 7 | use crossbeam_utils::sync::WaitGroup; 8 | 9 | fn spawn_counter(pool: P) -> Result<()> { 10 | const TASK_NUM: usize = 20; 11 | const ADD_COUNT: usize = 1000; 12 | 13 | let wg = WaitGroup::new(); 14 | let counter = Arc::new(AtomicUsize::new(0)); 15 | 16 | for _ in 0..TASK_NUM { 17 | let counter = Arc::clone(&counter); 18 | let wg = wg.clone(); 19 | pool.spawn(move || { 20 | for _ in 0..ADD_COUNT { 21 | counter.fetch_add(1, Ordering::SeqCst); 22 | } 23 | drop(wg); 24 | }) 25 | } 26 | 27 | wg.wait(); 28 | assert_eq!(counter.load(Ordering::SeqCst), TASK_NUM * ADD_COUNT); 29 | Ok(()) 30 | } 31 | 32 | fn spawn_panic_task() -> Result<()> { 33 | const TASK_NUM: usize = 1000; 34 | 35 | let pool = P::new(4)?; 36 | for _ in 0..TASK_NUM { 37 | pool.spawn(move || { 38 | // It suppresses flood of panic messages to the console. 39 | // You may find it useful to comment this out during development. 40 | panic_control::disable_hook_in_current_thread(); 41 | 42 | panic!(); 43 | }) 44 | } 45 | 46 | spawn_counter(pool) 47 | } 48 | 49 | #[test] 50 | fn naive_thread_pool_spawn_counter() -> Result<()> { 51 | let pool = NaiveThreadPool::new(4)?; 52 | spawn_counter(pool) 53 | } 54 | 55 | #[test] 56 | fn shared_queue_thread_pool_spawn_counter() -> Result<()> { 57 | let pool = SharedQueueThreadPool::new(4)?; 58 | spawn_counter(pool) 59 | } 60 | 61 | #[test] 62 | fn rayon_thread_pool_spawn_counter() -> Result<()> { 63 | let pool = RayonThreadPool::new(4)?; 64 | spawn_counter(pool) 65 | } 66 | 67 | #[test] 68 | fn shared_queue_thread_pool_panic_task() -> Result<()> { 69 | spawn_panic_task::() 70 | } 71 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/tests/thread_pool.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::{AtomicUsize, Ordering}; 2 | use std::sync::Arc; 3 | 4 | use kvs::thread_pool::*; 5 | use kvs::Result; 6 | 7 | use crossbeam_utils::sync::WaitGroup; 8 | 9 | fn spawn_counter(pool: P) -> Result<()> { 10 | const TASK_NUM: usize = 20; 11 | const ADD_COUNT: usize = 1000; 12 | 13 | let wg = WaitGroup::new(); 14 | let counter = Arc::new(AtomicUsize::new(0)); 15 | 16 | for _ in 0..TASK_NUM { 17 | let counter = Arc::clone(&counter); 18 | let wg = wg.clone(); 19 | pool.spawn(move || { 20 | for _ in 0..ADD_COUNT { 21 | counter.fetch_add(1, Ordering::SeqCst); 22 | } 23 | drop(wg); 24 | }) 25 | } 26 | 27 | wg.wait(); 28 | assert_eq!(counter.load(Ordering::SeqCst), TASK_NUM * ADD_COUNT); 29 | Ok(()) 30 | } 31 | 32 | fn spawn_panic_task() -> Result<()> { 33 | const TASK_NUM: usize = 1000; 34 | 35 | let pool = P::new(4)?; 36 | for _ in 0..TASK_NUM { 37 | pool.spawn(move || { 38 | // It suppresses flood of panic messages to the console. 39 | // You may find it useful to comment this out during development. 40 | panic_control::disable_hook_in_current_thread(); 41 | 42 | panic!(); 43 | }) 44 | } 45 | 46 | spawn_counter(pool) 47 | } 48 | 49 | #[test] 50 | fn naive_thread_pool_spawn_counter() -> Result<()> { 51 | let pool = NaiveThreadPool::new(4)?; 52 | spawn_counter(pool) 53 | } 54 | 55 | #[test] 56 | fn shared_queue_thread_pool_spawn_counter() -> Result<()> { 57 | let pool = SharedQueueThreadPool::new(4)?; 58 | spawn_counter(pool) 59 | } 60 | 61 | #[test] 62 | fn rayon_thread_pool_spawn_counter() -> Result<()> { 63 | let pool = RayonThreadPool::new(4)?; 64 | spawn_counter(pool) 65 | } 66 | 67 | #[test] 68 | fn shared_queue_thread_pool_panic_task() -> Result<()> { 69 | spawn_panic_task::() 70 | } 71 | -------------------------------------------------------------------------------- /courses/dss/README.md: -------------------------------------------------------------------------------- 1 | # Distributed Systems in Rust 2 | 3 | A training course about the distributed systems in [Rust]. 4 | 5 | Subjects covered include: 6 | 7 | - [Raft consensus algorithm] (including a fault-tolerant key-value storage service 8 | using Raft) 9 | - [Percolator transaction model] 10 | 11 | After completing this course you will have the knowledge to implement a basic 12 | key-value storage service with transaction and fault-tolerant in Rust. 13 | 14 | **Important note: Distributed Systems in Rust is in an alpha state** 15 | It might contain bugs. Any feedback is greatly appreciated. Please [file issues] 16 | if you have any problem. And also You are encouraged to fix problems on your own 17 | and submit pull requests. 18 | 19 | ## The goal of this course 20 | 21 | The goal of this course is to teach the Rust programmers who are interested in 22 | distributed systems to know about how to make the distributed systems reliable 23 | and how to implement the distributed transaction. 24 | 25 | ## Who is this for? 26 | 27 | Distributed Systems in Rust is for experienced _Rust_ programmers, who are 28 | familiar with the Rust language. If you are not, you can first learn our [rust] 29 | lessons. 30 | 31 | ## A PingCAP-specific note 32 | 33 | This course, combined with [Deep Dive TiKV], is intended to be enough to enable 34 | programmers to meaningfully contribute to [TiKV]. It is most specifically 35 | designed to teach those in the Chinese Rust community enough Rust to work on 36 | TiKV. The language used is intended to be simple so that those who read only a 37 | little English can follow. If you find any of the language difficult to 38 | understand please [file issues]. 39 | 40 | ## License 41 | 42 | [CC-BY 4.0](https://opendefinition.org/licenses/cc-by/) 43 | 44 | 45 | [rust]: ../rust/README.md 46 | [file issues]: https://github.com/pingcap/talent-plan/issues/ 47 | [Deep Dive TiKV]: https://tikv.github.io/deep-dive-tikv/overview/introduction.html 48 | [TiKV]: https://github.com/tikv/tikv/ 49 | [Rust]: https://www.rust-lang.org/ 50 | [Raft consensus algorithm]: raft/README.md 51 | [Percolator transaction model]: percolator/README.md 52 | -------------------------------------------------------------------------------- /tidb/join/join_example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "io" 6 | "os" 7 | "strconv" 8 | "unsafe" 9 | 10 | "github.com/pingcap/tidb/util/mvmap" 11 | ) 12 | 13 | // JoinExample performs a simple hash join algorithm. 14 | func JoinExample(f0, f1 string, offset0, offset1 []int) (sum uint64) { 15 | tbl0, tbl1 := readCSVFileIntoTbl(f0), readCSVFileIntoTbl(f1) 16 | hashtable := buildHashTable(tbl0, offset0) 17 | for _, row := range tbl1 { 18 | rowIDs := probe(hashtable, row, offset1) 19 | for _, id := range rowIDs { 20 | v, err := strconv.ParseUint(tbl0[id][0], 10, 64) 21 | if err != nil { 22 | panic("JoinExample panic\n" + err.Error()) 23 | } 24 | sum += v 25 | } 26 | } 27 | return sum 28 | } 29 | 30 | func readCSVFileIntoTbl(f string) (tbl [][]string) { 31 | csvFile, err := os.Open(f) 32 | if err != nil { 33 | panic("ReadFileIntoTbl " + f + " fail\n" + err.Error()) 34 | } 35 | defer csvFile.Close() 36 | 37 | csvReader := csv.NewReader(csvFile) 38 | for { 39 | row, err := csvReader.Read() 40 | if err == io.EOF { 41 | break 42 | } else if err != nil { 43 | panic("ReadFileIntoTbl " + f + " fail\n" + err.Error()) 44 | } 45 | tbl = append(tbl, row) 46 | } 47 | return tbl 48 | } 49 | 50 | func buildHashTable(data [][]string, offset []int) (hashtable *mvmap.MVMap) { 51 | var keyBuffer []byte 52 | valBuffer := make([]byte, 8) 53 | hashtable = mvmap.NewMVMap() 54 | for i, row := range data { 55 | for j, off := range offset { 56 | if j > 0 { 57 | keyBuffer = append(keyBuffer, '_') 58 | } 59 | keyBuffer = append(keyBuffer, []byte(row[off])...) 60 | } 61 | *(*int64)(unsafe.Pointer(&valBuffer[0])) = int64(i) 62 | hashtable.Put(keyBuffer, valBuffer) 63 | keyBuffer = keyBuffer[:0] 64 | } 65 | return 66 | } 67 | 68 | func probe(hashtable *mvmap.MVMap, row []string, offset []int) (rowIDs []int64) { 69 | var keyHash []byte 70 | var vals [][]byte 71 | for i, off := range offset { 72 | if i > 0 { 73 | keyHash = append(keyHash, '_') 74 | } 75 | keyHash = append(keyHash, []byte(row[off])...) 76 | } 77 | vals = hashtable.Get(keyHash, vals) 78 | for _, val := range vals { 79 | rowIDs = append(rowIDs, *(*int64)(unsafe.Pointer(&val[0]))) 80 | } 81 | return rowIDs 82 | } 83 | -------------------------------------------------------------------------------- /courses/rust/docs/etc/parallel-diagrams.txt: -------------------------------------------------------------------------------- 1 | Used in p4. Edited with asciiflow.com. 2 | 3 | 4 | --> read/write reqs over time --> --> read/write reqs over time --> 5 | thread thread 6 | + +--------+--------+--------+--------+ + +--------+ 7 | T1 | | R1 | R2 | W1 | W2 | T1 | | R1 | 8 | + +--------+--------+--------+--------+ | +-----------------+ 9 | T2 | | R2 | 10 | --> read/write reqs over time --> | +-----------------+ 11 | thread T3 | | W1 | 12 | + +--------+ | +-----------------+ 13 | T1 | | R1 | T4 | | W2 | 14 | | +--------+ + +--------+ 15 | T2 | | R2 | 16 | | +-----------------+ 17 | T3 | | W1 | --> read/write reqs over time --> 18 | | +-----------------+ thread 19 | T4 | | W2 | + +--------+ 20 | + +--------+ T1 | | R1 | 21 | | +--------+ 22 | --> read/write reqs over time --> T2 | | R2 | 23 | thread | +--------+ 24 | + +--------+ T3 | | W1 | 25 | T1 | | R1 | | +-----------------+ 26 | | +--------+ T4 | | W2 | 27 | T2 | | R2 | + +--------+ 28 | | +--------+ 29 | T3 | | W1 | 30 | | +--------+ 31 | T4 | | W2 | 32 | + +--------+ 33 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/client.rs: -------------------------------------------------------------------------------- 1 | use crate::common::{GetResponse, RemoveResponse, Request, SetResponse}; 2 | use crate::{KvsError, Result}; 3 | use serde::Deserialize; 4 | use serde_json::de::{Deserializer, IoRead}; 5 | use std::io::{BufReader, BufWriter, Write}; 6 | use std::net::{TcpStream, ToSocketAddrs}; 7 | 8 | /// Key value store client 9 | pub struct KvsClient { 10 | reader: Deserializer>>, 11 | writer: BufWriter, 12 | } 13 | 14 | impl KvsClient { 15 | /// Connect to `addr` to access `KvsServer`. 16 | pub fn connect(addr: A) -> Result { 17 | let tcp_reader = TcpStream::connect(addr)?; 18 | let tcp_writer = tcp_reader.try_clone()?; 19 | Ok(KvsClient { 20 | reader: Deserializer::from_reader(BufReader::new(tcp_reader)), 21 | writer: BufWriter::new(tcp_writer), 22 | }) 23 | } 24 | 25 | /// Get the value of a given key from the server. 26 | pub fn get(&mut self, key: String) -> Result> { 27 | serde_json::to_writer(&mut self.writer, &Request::Get { key })?; 28 | self.writer.flush()?; 29 | let resp = GetResponse::deserialize(&mut self.reader)?; 30 | match resp { 31 | GetResponse::Ok(value) => Ok(value), 32 | GetResponse::Err(msg) => Err(KvsError::StringError(msg)), 33 | } 34 | } 35 | 36 | /// Set the value of a string key in the server. 37 | pub fn set(&mut self, key: String, value: String) -> Result<()> { 38 | serde_json::to_writer(&mut self.writer, &Request::Set { key, value })?; 39 | self.writer.flush()?; 40 | let resp = SetResponse::deserialize(&mut self.reader)?; 41 | match resp { 42 | SetResponse::Ok(_) => Ok(()), 43 | SetResponse::Err(msg) => Err(KvsError::StringError(msg)), 44 | } 45 | } 46 | 47 | /// Remove a string key in the server. 48 | pub fn remove(&mut self, key: String) -> Result<()> { 49 | serde_json::to_writer(&mut self.writer, &Request::Remove { key })?; 50 | self.writer.flush()?; 51 | let resp = RemoveResponse::deserialize(&mut self.reader)?; 52 | match resp { 53 | RemoveResponse::Ok(_) => Ok(()), 54 | RemoveResponse::Err(msg) => Err(KvsError::StringError(msg)), 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/client.rs: -------------------------------------------------------------------------------- 1 | use crate::common::{GetResponse, RemoveResponse, Request, SetResponse}; 2 | use crate::{KvsError, Result}; 3 | use serde::Deserialize; 4 | use serde_json::de::{Deserializer, IoRead}; 5 | use std::io::{BufReader, BufWriter, Write}; 6 | use std::net::{TcpStream, ToSocketAddrs}; 7 | 8 | /// Key value store client 9 | pub struct KvsClient { 10 | reader: Deserializer>>, 11 | writer: BufWriter, 12 | } 13 | 14 | impl KvsClient { 15 | /// Connect to `addr` to access `KvsServer`. 16 | pub fn connect(addr: A) -> Result { 17 | let tcp_reader = TcpStream::connect(addr)?; 18 | let tcp_writer = tcp_reader.try_clone()?; 19 | Ok(KvsClient { 20 | reader: Deserializer::from_reader(BufReader::new(tcp_reader)), 21 | writer: BufWriter::new(tcp_writer), 22 | }) 23 | } 24 | 25 | /// Get the value of a given key from the server. 26 | pub fn get(&mut self, key: String) -> Result> { 27 | serde_json::to_writer(&mut self.writer, &Request::Get { key })?; 28 | self.writer.flush()?; 29 | let resp = GetResponse::deserialize(&mut self.reader)?; 30 | match resp { 31 | GetResponse::Ok(value) => Ok(value), 32 | GetResponse::Err(msg) => Err(KvsError::StringError(msg)), 33 | } 34 | } 35 | 36 | /// Set the value of a string key in the server. 37 | pub fn set(&mut self, key: String, value: String) -> Result<()> { 38 | serde_json::to_writer(&mut self.writer, &Request::Set { key, value })?; 39 | self.writer.flush()?; 40 | let resp = SetResponse::deserialize(&mut self.reader)?; 41 | match resp { 42 | SetResponse::Ok(_) => Ok(()), 43 | SetResponse::Err(msg) => Err(KvsError::StringError(msg)), 44 | } 45 | } 46 | 47 | /// Remove a string key in the server. 48 | pub fn remove(&mut self, key: String) -> Result<()> { 49 | serde_json::to_writer(&mut self.writer, &Request::Remove { key })?; 50 | self.writer.flush()?; 51 | let resp = RemoveResponse::deserialize(&mut self.reader)?; 52 | match resp { 53 | RemoveResponse::Ok(_) => Ok(()), 54 | RemoveResponse::Err(msg) => Err(KvsError::StringError(msg)), 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /talent-plan-1.0/1.0-lp-tikv.md: -------------------------------------------------------------------------------- 1 | # 路径 2：TiKV 方向 2 | 3 | ## 课程目标 4 | * 掌握 Rust 语言 5 | * 掌握分布式一致性算法 Raft 6 | * 掌握分布式存储基本原理 7 | 8 | ## 课程内容 9 | 10 | ### Section 1 熟悉 Rust 语言基础知识 11 | 12 | **学习资料：** 13 | 14 | * [Practical Networked Applications in Rust](https://github.com/pingcap/talent-plan/tree/master/courses/rust) 15 | * 阅读《[Rust 编程之道](https://book.douban.com/subject/30418895/)》或者《[深入浅出 Rust](https://book.douban.com/subject/30312231/)》或者 [Rust 官方教程](https://doc.rust-lang.org/book/) 三选一 16 | 17 | **课程作业：** 18 | 19 | * 完成 [Practical Networked Applications in Rust](https://github.com/pingcap/talent-plan/tree/master/courses/rust) 中 Project 1、2（给该课程提出改进意见并帮助改进，根据意见及改进情况会有额外加分哦~） 20 | 21 | ### Section 2 熟悉 Rust 语言基础知识 22 | 23 | **学习资料：** 24 | 25 | * [Practical Networked Applications in Rust](https://github.com/pingcap/talent-plan/tree/master/courses/rust) 26 | * 阅读《[Rust 编程之道](https://book.douban.com/subject/30418895/)》或者《[深入浅出 Rust](https://book.douban.com/subject/30312231/)》或者 [Rust 官方教程](https://doc.rust-lang.org/book/) 三选一 27 | 28 | **课程作业：** 29 | 30 | * 完成 [Practical Networked Applications in Rust](https://github.com/pingcap/talent-plan/tree/master/courses/rust) 中 Project 3、4（给该课程提出改进意见并帮助改进，根据意见及改进情况会有额外加分哦~） 31 | 32 | ### Section 3 分布式存储基础知识 33 | 34 | **学习资料：** 35 | 36 | * 学习 Raft Paper 和 [6.824 Lab 2: Raft](https://pdos.csail.mit.edu/6.824/labs/lab-raft.html) 线上课程 37 | 38 | **课程作业：** 39 | 40 | * 使用 Rust 语言 完成 6.824 Lab 2 [raft 作业](https://github.com/pingcap/talent-plan/tree/master/dss) 41 | 42 | ### Section 4 分布式系统基础知识 43 | 44 | **学习资料：** 45 | 46 | * 完成 [6.824 Lab 3: Fault-tolerant Key/Value Service](https://pdos.csail.mit.edu/6.824/labs/lab-kvraft.html) 线上课程 47 | 48 | **课程作业：** 49 | 50 | * 使用 Rust 语言 完成 6.824 Lab 3 [kvraft 作业](https://github.com/pingcap/talent-plan/tree/master/dss) 51 | 52 | ### Section 5 阅读 TiKV 源码 53 | 54 | **学习资料：** 55 | 56 | * [TiKV 源码解析](https://pingcap.com/blog-cn/#TiKV-%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90)（持续更新中） 57 | 58 | **课程作业：** 59 | 60 | * 给 TiKV Repo 提 PR，例如修复 [for new contributors issues](https://github.com/tikv/tikv/issues?q=is%3Aissue+is%3Aopen+label%3A%22D%3A+Easy%22+label%3A%22S%3A+HelpWanted%22)。 61 | * 给 PD Repo 提 PR，例如修复 [for new contributors issues]()。 62 | 63 | ## 考核方式 64 | 65 | 根据提交到程序是否能通过测试给出 Pass or Fail 66 | 67 | 1. 代码可以跑过 PingCAP 提供的测试程序 50% 68 | 69 | a. make check 70 | 71 | b. make test 72 | 73 | 2. 代码风格 30% 74 | 3. 文档情况 20% 75 | 76 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/thread_pool/shared_queue.rs: -------------------------------------------------------------------------------- 1 | use std::thread; 2 | 3 | use super::ThreadPool; 4 | use crate::Result; 5 | 6 | use crossbeam::channel::{self, Receiver, Sender}; 7 | 8 | // Note for Rust training course: the thread pool is not implemented using 9 | // `catch_unwind` because it would require the task to be `UnwindSafe`. 10 | 11 | /// A thread pool using a shared queue inside. 12 | /// 13 | /// If a spawned task panics, the old thread will be destroyed and a new one will be 14 | /// created. It fails silently when any failure to create the thread at the OS level 15 | /// is captured after the thread pool is created. So, the thread number in the pool 16 | /// can decrease to zero, then spawning a task to the thread pool will panic. 17 | #[derive(Clone)] 18 | pub struct SharedQueueThreadPool { 19 | tx: Sender>, 20 | } 21 | 22 | impl ThreadPool for SharedQueueThreadPool { 23 | fn new(threads: u32) -> Result { 24 | let (tx, rx) = channel::unbounded::>(); 25 | for _ in 0..threads { 26 | let rx = TaskReceiver(rx.clone()); 27 | thread::Builder::new().spawn(move || run_tasks(rx))?; 28 | } 29 | Ok(SharedQueueThreadPool { tx }) 30 | } 31 | 32 | /// Spawns a function into the thread pool. 33 | /// 34 | /// # Panics 35 | /// 36 | /// Panics if the thread pool has no thread. 37 | fn spawn(&self, job: F) 38 | where 39 | F: FnOnce() + Send + 'static, 40 | { 41 | self.tx 42 | .send(Box::new(job)) 43 | .expect("The thread pool has no thread."); 44 | } 45 | } 46 | 47 | #[derive(Clone)] 48 | struct TaskReceiver(Receiver>); 49 | 50 | impl Drop for TaskReceiver { 51 | fn drop(&mut self) { 52 | if thread::panicking() { 53 | let rx = self.clone(); 54 | if let Err(e) = thread::Builder::new().spawn(move || run_tasks(rx)) { 55 | error!("Failed to spawn a thread: {}", e); 56 | } 57 | } 58 | } 59 | } 60 | 61 | fn run_tasks(rx: TaskReceiver) { 62 | loop { 63 | match rx.0.recv() { 64 | Ok(task) => { 65 | task(); 66 | } 67 | Err(_) => debug!("Thread exits because the thread pool is destroyed."), 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/thread_pool/shared_queue.rs: -------------------------------------------------------------------------------- 1 | use std::thread; 2 | 3 | use super::ThreadPool; 4 | use crate::Result; 5 | 6 | use crossbeam::channel::{self, Receiver, Sender}; 7 | 8 | use log::{debug, error}; 9 | 10 | // Note for Rust training course: the thread pool is not implemented using 11 | // `catch_unwind` because it would require the task to be `UnwindSafe`. 12 | 13 | /// A thread pool using a shared queue inside. 14 | /// 15 | /// If a spawned task panics, the old thread will be destroyed and a new one will be 16 | /// created. It fails silently when any failure to create the thread at the OS level 17 | /// is captured after the thread pool is created. So, the thread number in the pool 18 | /// can decrease to zero, then spawning a task to the thread pool will panic. 19 | pub struct SharedQueueThreadPool { 20 | tx: Sender>, 21 | } 22 | 23 | impl ThreadPool for SharedQueueThreadPool { 24 | fn new(threads: u32) -> Result { 25 | let (tx, rx) = channel::unbounded::>(); 26 | for _ in 0..threads { 27 | let rx = TaskReceiver(rx.clone()); 28 | thread::Builder::new().spawn(move || run_tasks(rx))?; 29 | } 30 | Ok(SharedQueueThreadPool { tx }) 31 | } 32 | 33 | /// Spawns a function into the thread pool. 34 | /// 35 | /// # Panics 36 | /// 37 | /// Panics if the thread pool has no thread. 38 | fn spawn(&self, job: F) 39 | where 40 | F: FnOnce() + Send + 'static, 41 | { 42 | self.tx 43 | .send(Box::new(job)) 44 | .expect("The thread pool has no thread."); 45 | } 46 | } 47 | 48 | #[derive(Clone)] 49 | struct TaskReceiver(Receiver>); 50 | 51 | impl Drop for TaskReceiver { 52 | fn drop(&mut self) { 53 | if thread::panicking() { 54 | let rx = self.clone(); 55 | if let Err(e) = thread::Builder::new().spawn(move || run_tasks(rx)) { 56 | error!("Failed to spawn a thread: {}", e); 57 | } 58 | } 59 | } 60 | } 61 | 62 | fn run_tasks(rx: TaskReceiver) { 63 | loop { 64 | match rx.0.recv() { 65 | Ok(task) => { 66 | task(); 67 | } 68 | Err(_) => debug!("Thread exits because the thread pool is destroyed."), 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /tidb/mapreduce/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | This is the Map-Reduce homework for PingCAP Talent Plan Online of week 2. 4 | 5 | There is a uncompleted Map-Reduce framework, you should complete it and use it to extract the 10 most frequent URLs from data files. 6 | 7 | ## Getting familiar with the source 8 | 9 | The simple Map-Reduce framework is defined in `mapreduce.go`. 10 | 11 | It is uncompleted and you should fill your code below comments `YOUR CODE HERE`. 12 | 13 | The map and reduce function are defined as same as MIT 6.824 lab 1. 14 | ``` 15 | type ReduceF func(key string, values []string) string 16 | type MapF func(filename string, contents string) []KeyValue 17 | ``` 18 | 19 | There is an example in `urltop10_example.go` which is used to extract the 10 most frequent URLs. 20 | 21 | After completing the framework, you can run this example by `make test_example`. 22 | 23 | And then please implement your own `MapF` and `ReduceF` in `urltop10.go` to accomplish this task. 24 | 25 | After filling your code, please use `make test_homework` to test. 26 | 27 | All data files will be generated at runtime, and you can use `make cleanup` to clean all test data. 28 | 29 | Please output URLs by lexicographical order and ensure that your result has the same format as test data so that you can pass all tests. 30 | 31 | Each test cases has **different data distribution** and you should take it into account. 32 | 33 | ## Requirements and rating principles 34 | 35 | * (40%) Performs better than `urltop10_example`. 36 | * (20%) Pass all test cases. 37 | * (30%) Have a document to describe your idea and record the process of performance optimization (both the framework and your own code) with `pprof`. 38 | * (10%) Have a good code style. 39 | 40 | NOTE: **go 1.12 is required** 41 | 42 | ## How to use 43 | 44 | Fill your code below comments `YOUR CODE HERE` in `mapreduce.go` to complete this framework. 45 | 46 | Implement your own `MapF` and `ReduceF` in `urltop10.go` and use `make test_homework` to test it. 47 | 48 | There is a builtin unit test defined in `urltop10_test.go`, however, you still can write your own unit tests. 49 | 50 | How to run example: 51 | ``` 52 | make test_example 53 | ``` 54 | 55 | How to test your implementation: 56 | ``` 57 | make test_homework 58 | ``` 59 | 60 | How to clean up all test data: 61 | ``` 62 | make cleanup 63 | ``` 64 | 65 | How to generate test data again: 66 | ``` 67 | make gendata 68 | ``` 69 | -------------------------------------------------------------------------------- /tidb/mapreduce/urltop10_example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | // ExampleURLTop10 generates RoundsArgs for getting the 10 most frequent URLs. 11 | // There are two rounds in this approach. 12 | // The first round will do url count. 13 | // The second will sort results generated in the first round and 14 | // get the 10 most frequent URLs. 15 | func ExampleURLTop10(nWorkers int) RoundsArgs { 16 | var args RoundsArgs 17 | // round 1: do url count 18 | args = append(args, RoundArgs{ 19 | MapFunc: ExampleURLCountMap, 20 | ReduceFunc: ExampleURLCountReduce, 21 | NReduce: nWorkers, 22 | }) 23 | // round 2: sort and get the 10 most frequent URLs 24 | args = append(args, RoundArgs{ 25 | MapFunc: ExampleURLTop10Map, 26 | ReduceFunc: ExampleURLTop10Reduce, 27 | NReduce: 1, 28 | }) 29 | return args 30 | } 31 | 32 | // ExampleURLCountMap is the map function in the first round 33 | func ExampleURLCountMap(filename string, contents string) []KeyValue { 34 | lines := strings.Split(contents, "\n") 35 | kvs := make([]KeyValue, 0, len(lines)) 36 | for _, l := range lines { 37 | l = strings.TrimSpace(l) 38 | if len(l) == 0 { 39 | continue 40 | } 41 | kvs = append(kvs, KeyValue{Key: l}) 42 | } 43 | return kvs 44 | } 45 | 46 | // ExampleURLCountReduce is the reduce function in the first round 47 | func ExampleURLCountReduce(key string, values []string) string { 48 | return fmt.Sprintf("%s %s\n", key, strconv.Itoa(len(values))) 49 | } 50 | 51 | // ExampleURLTop10Map is the map function in the second round 52 | func ExampleURLTop10Map(filename string, contents string) []KeyValue { 53 | lines := strings.Split(contents, "\n") 54 | kvs := make([]KeyValue, 0, len(lines)) 55 | for _, l := range lines { 56 | kvs = append(kvs, KeyValue{"", l}) 57 | } 58 | return kvs 59 | } 60 | 61 | // ExampleURLTop10Reduce is the reduce function in the second round 62 | func ExampleURLTop10Reduce(key string, values []string) string { 63 | cnts := make(map[string]int, len(values)) 64 | for _, v := range values { 65 | v := strings.TrimSpace(v) 66 | if len(v) == 0 { 67 | continue 68 | } 69 | tmp := strings.Split(v, " ") 70 | n, err := strconv.Atoi(tmp[1]) 71 | if err != nil { 72 | panic(err) 73 | } 74 | cnts[tmp[0]] = n 75 | } 76 | 77 | us, cs := TopN(cnts, 10) 78 | buf := new(bytes.Buffer) 79 | for i := range us { 80 | fmt.Fprintf(buf, "%s: %d\n", us[i], cs[i]) 81 | } 82 | return buf.String() 83 | } 84 | -------------------------------------------------------------------------------- /courses/rust/building-blocks/bb-1.md: -------------------------------------------------------------------------------- 1 | # PNA Rust — Building Blocks 1 2 | 3 | Let's learn some building blocks! 4 | 5 | Put your other projects and concerns aside. Take a breath and relax. Here 6 | are some fun resources for you to explore. 7 | 8 | Read all the readings and perform all the exercises. 9 | 10 | - **[Exercise: Write a Good CLI Program]**. Writing a CLI program in Rust. This 11 | is a good warmup to the CLI program you'll be writing in this course, and the 12 | techniques used by this author may provide an interesting contrast to those we 13 | suggest. Follow along and write the same code. Can you reproduce their 14 | results? 15 | 16 | - **[Reading: The Cargo manifest format]**. A single page in [The Cargo Book], 17 | this will give you an idea of how your project can be customized a bit if you 18 | so choose. This is a page you will come back to repeatedly as a Rust 19 | programmer. 20 | 21 | - **[Reading: Cargo environment variables]**. Also from The Cargo Book, and also 22 | a page that you will see many times in the future. Environment variables are 23 | one way that it communicates with rustc, allowing it to set the various 24 | [`env!`] macros at build time, in both your program source code and build 25 | scripts. It is also a way for scripts and other systems to communicate to 26 | Cargo. 27 | 28 | - **[Reading: Rust API Guidelines: Documentation]**. The Rust project is 29 | opinionated about how Rust source is written. This page is on how to document 30 | Rust projects, but the whole book is worth reading. These are written by 31 | experienced Rust developers, but are in an incomplete state. Note the GitHub 32 | organization it belongs to — [`rust-lang-nursery`]. It contains many 33 | interesting projects. 34 | 35 | 36 | [Reading: Rust API Guidelines: Documentation]: https://rust-lang-nursery.github.io/api-guidelines/documentation.html 37 | [Reading: The Cargo manifest format]: https://doc.rust-lang.org/cargo/reference/manifest.html 38 | [Reading: Cargo environment variables]: https://doc.rust-lang.org/cargo/reference/environment-variables.html 39 | [The Cargo Book]: https://doc.rust-lang.org/cargo/reference/manifest.html 40 | [`env!`]: https://doc.rust-lang.org/std/macro.env.html 41 | [`rust-lang-nursery`]: https://github.com/rust-lang-nursery 42 | [Reading: The rustup documentation]: https://github.com/rust-lang/rustup.rs/blob/master/README.md 43 | [Exercise: Write a Good CLI Program]: https://qiita.com/tigercosmos/items/678f39b1209e60843cc3 44 | 45 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/server.rs: -------------------------------------------------------------------------------- 1 | use crate::common::{Request, Response}; 2 | use crate::{KvsEngine, KvsError, Result}; 3 | use std::net::SocketAddr; 4 | use tokio::codec::{FramedRead, FramedWrite, LengthDelimitedCodec}; 5 | use tokio::net::{TcpListener, TcpStream}; 6 | use tokio::prelude::*; 7 | use tokio_serde_json::{ReadJson, WriteJson}; 8 | 9 | /// The server of a key value store. 10 | pub struct KvsServer { 11 | engine: E, 12 | } 13 | 14 | impl KvsServer { 15 | /// Create a `KvsServer` with a given storage engine. 16 | pub fn new(engine: E) -> Self { 17 | KvsServer { engine } 18 | } 19 | 20 | /// Run the server listening on the given address 21 | pub fn run(self, addr: SocketAddr) -> Result<()> { 22 | let listener = TcpListener::bind(&addr)?; 23 | let server = listener 24 | .incoming() 25 | .map_err(|e| error!("IO error: {}", e)) 26 | .for_each(move |tcp| { 27 | let engine = self.engine.clone(); 28 | serve(engine, tcp).map_err(|e| error!("Error on serving client: {}", e)) 29 | }); 30 | tokio::run(server); 31 | Ok(()) 32 | } 33 | } 34 | 35 | fn serve(engine: E, tcp: TcpStream) -> impl Future { 36 | let (read_half, write_half) = tcp.split(); 37 | let read_json = ReadJson::new(FramedRead::new(read_half, LengthDelimitedCodec::new())); 38 | let resp_stream = read_json 39 | .map_err(KvsError::from) 40 | .and_then( 41 | move |req| -> Box + Send> { 42 | match req { 43 | Request::Get { key } => Box::new(engine.get(key).map(Response::Get)), 44 | Request::Set { key, value } => { 45 | Box::new(engine.set(key, value).map(|_| Response::Set)) 46 | } 47 | Request::Remove { key } => { 48 | Box::new(engine.remove(key).map(|_| Response::Remove)) 49 | } 50 | } 51 | }, 52 | ) 53 | .then(|resp| -> Result { 54 | match resp { 55 | Ok(resp) => Ok(resp), 56 | Err(e) => Ok(Response::Err(format!("{}", e))), 57 | } 58 | }); 59 | let write_json = WriteJson::new(FramedWrite::new(write_half, LengthDelimitedCodec::new())); 60 | write_json 61 | .sink_map_err(KvsError::from) 62 | .send_all(resp_stream) 63 | .map(|_| ()) 64 | } 65 | -------------------------------------------------------------------------------- /courses/dss/labrpc/benches/rpc.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 4 | use futures::executor::block_on; 5 | use prost_derive::Message; 6 | 7 | use labrpc::{service, Network, Result, Server, ServerBuilder}; 8 | 9 | service! { 10 | /// A simple bench-purpose service. 11 | service bench { 12 | /// Doc comments. 13 | rpc handler(BenchArgs) returns (BenchReply); 14 | } 15 | } 16 | use bench::{add_service, Client as BenchClient, Service}; 17 | 18 | // Hand-written protobuf messages. 19 | #[derive(Clone, PartialEq, Message)] 20 | pub struct BenchArgs { 21 | #[prost(int64, tag = "1")] 22 | pub x: i64, 23 | } 24 | 25 | #[derive(Clone, PartialEq, Message)] 26 | pub struct BenchReply { 27 | #[prost(string, tag = "1")] 28 | pub x: String, 29 | } 30 | 31 | #[derive(Default)] 32 | struct BenchInner { 33 | log2: Vec, 34 | } 35 | #[derive(Clone)] 36 | pub struct BenchService { 37 | inner: Arc>, 38 | } 39 | impl BenchService { 40 | fn new() -> BenchService { 41 | BenchService { 42 | inner: Arc::default(), 43 | } 44 | } 45 | } 46 | 47 | #[async_trait::async_trait] 48 | impl Service for BenchService { 49 | async fn handler(&self, args: BenchArgs) -> Result { 50 | self.inner.lock().unwrap().log2.push(args.x); 51 | Ok(BenchReply { 52 | x: format!("handler-{}", args.x), 53 | }) 54 | } 55 | } 56 | 57 | fn bench_suit() -> (Network, Server, BenchService) { 58 | let net = Network::new(); 59 | let server_name = "test_server".to_owned(); 60 | let mut builder = ServerBuilder::new(server_name); 61 | let bench_server = BenchService::new(); 62 | add_service(bench_server.clone(), &mut builder).unwrap(); 63 | let server = builder.build(); 64 | net.add_server(server.clone()); 65 | (net, server, bench_server) 66 | } 67 | 68 | fn bench_rpc(c: &mut Criterion) { 69 | let (net, server, _bench_server) = bench_suit(); 70 | let server_name = server.name(); 71 | let client_name = "client"; 72 | let client = BenchClient::new(net.create_client(client_name.to_owned())); 73 | net.connect(client_name, server_name); 74 | net.enable(client_name, true); 75 | 76 | c.bench_function("rpc", |b| { 77 | b.iter(|| { 78 | black_box(block_on(async { 79 | client.handler(&BenchArgs { x: 111 }).await.unwrap() 80 | })); 81 | }) 82 | // i7-8650U, 13 microseconds per RPC 83 | }); 84 | } 85 | 86 | criterion_group!(benches, bench_rpc); 87 | criterion_main!(benches); 88 | -------------------------------------------------------------------------------- /courses/dss/labcodec/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A thin wrapper of [prost](https://docs.rs/prost/0.6.1/prost/) 2 | 3 | /// A labcodec message. 4 | pub trait Message: prost::Message + Default {} 5 | impl Message for T {} 6 | 7 | /// A message encoding error. 8 | pub type EncodeError = prost::EncodeError; 9 | /// A message decoding error. 10 | pub type DecodeError = prost::DecodeError; 11 | 12 | /// Encodes the message to a `Vec`. 13 | pub fn encode(message: &M, buf: &mut Vec) -> Result<(), EncodeError> { 14 | buf.reserve(message.encoded_len()); 15 | message.encode(buf)?; 16 | Ok(()) 17 | } 18 | 19 | /// Decodes an message from the buffer. 20 | pub fn decode(buf: &[u8]) -> Result { 21 | M::decode(buf) 22 | } 23 | 24 | #[cfg(test)] 25 | mod tests { 26 | mod fixture { 27 | // The generated rust file: 28 | // labs6824/target/debug/build/labcodec-hashhashhashhash/out/fixture.rs 29 | // 30 | // It looks like: 31 | // 32 | // ```no_run 33 | // /// A simple protobuf message. 34 | // #[derive(Clone, PartialEq, Message)] 35 | // pub struct Msg { 36 | // #[prost(enumeration="msg::Type", tag="1")] 37 | // pub type_: i32, 38 | // #[prost(uint64, tag="2")] 39 | // pub id: u64, 40 | // #[prost(string, tag="3")] 41 | // pub name: String, 42 | // #[prost(bytes, repeated, tag="4")] 43 | // pub paylad: ::std::vec::Vec>, 44 | // } 45 | // pub mod msg { 46 | // #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Enumeration)] 47 | // pub enum Type { 48 | // Unknown = 0, 49 | // Put = 1, 50 | // Get = 2, 51 | // Del = 3, 52 | // } 53 | // } 54 | // ``` 55 | include!(concat!(env!("OUT_DIR"), "/fixture.rs")); 56 | } 57 | 58 | use super::{decode, encode}; 59 | 60 | #[test] 61 | fn test_basic_encode_decode() { 62 | let msg = fixture::Msg { 63 | r#type: fixture::msg::Type::Put as _, 64 | id: 42, 65 | name: "the answer".to_owned(), 66 | paylad: vec![vec![7; 3]; 2], 67 | }; 68 | let mut buf = vec![]; 69 | encode(&msg, &mut buf).unwrap(); 70 | let msg1 = decode(&buf).unwrap(); 71 | assert_eq!(msg, msg1); 72 | } 73 | 74 | #[test] 75 | fn test_default() { 76 | let msg = fixture::Msg::default(); 77 | let msg1 = decode(&[]).unwrap(); 78 | assert_eq!(msg, msg1); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /tidb/mapreduce/urltop10_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path" 8 | "runtime" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func testDataScale() ([]DataSize, []int) { 14 | dataSize := []DataSize{1 * MB, 10 * MB, 100 * MB, 500 * MB, 1 * GB} 15 | nMapFiles := []int{5, 10, 20, 40, 60} 16 | return dataSize, nMapFiles 17 | } 18 | 19 | const ( 20 | dataDir = "/tmp/mr_homework" 21 | ) 22 | 23 | func dataPrefix(i int, ds DataSize, nMap int) string { 24 | return path.Join(dataDir, fmt.Sprintf("case%d-%s-%d", i, ds, nMap)) 25 | } 26 | 27 | func TestGenData(t *testing.T) { 28 | gens := AllCaseGenFs() 29 | dataSize, nMapFiles := testDataScale() 30 | for k := range dataSize { 31 | for i, gen := range gens { 32 | fmt.Printf("generate data file for cast%d, dataSize=%v, nMap=%v\n", i, dataSize[k], nMapFiles[k]) 33 | prefix := dataPrefix(i, dataSize[k], nMapFiles[k]) 34 | gen(prefix, int(dataSize[k]), nMapFiles[k]) 35 | } 36 | } 37 | } 38 | 39 | func TestCleanData(t *testing.T) { 40 | if err := os.RemoveAll(dataDir); err != nil { 41 | log.Fatal(err) 42 | } 43 | } 44 | 45 | func TestExampleURLTop(t *testing.T) { 46 | rounds := ExampleURLTop10(GetMRCluster().NWorkers()) 47 | testURLTop(t, rounds) 48 | } 49 | 50 | func TestURLTop(t *testing.T) { 51 | rounds := URLTop10(GetMRCluster().NWorkers()) 52 | testURLTop(t, rounds) 53 | } 54 | 55 | func testURLTop(t *testing.T, rounds RoundsArgs) { 56 | if len(rounds) == 0 { 57 | t.Fatalf("no rounds arguments, please finish your code") 58 | } 59 | mr := GetMRCluster() 60 | 61 | // run all cases 62 | gens := AllCaseGenFs() 63 | dataSize, nMapFiles := testDataScale() 64 | for k := range dataSize { 65 | for i, gen := range gens { 66 | // generate data 67 | prefix := dataPrefix(i, dataSize[k], nMapFiles[k]) 68 | c := gen(prefix, int(dataSize[k]), nMapFiles[k]) 69 | 70 | runtime.GC() 71 | 72 | // run map-reduce rounds 73 | begin := time.Now() 74 | inputFiles := c.MapFiles 75 | for idx, r := range rounds { 76 | jobName := fmt.Sprintf("Case%d-Round%d", i, idx) 77 | ch := mr.Submit(jobName, prefix, r.MapFunc, r.ReduceFunc, inputFiles, r.NReduce) 78 | inputFiles = <-ch 79 | } 80 | cost := time.Since(begin) 81 | 82 | // check result 83 | if len(inputFiles) != 1 { 84 | panic("the length of result file list should be 1") 85 | } 86 | result := inputFiles[0] 87 | 88 | if errMsg, ok := CheckFile(c.ResultFile, result); !ok { 89 | t.Fatalf("Case%d FAIL, dataSize=%v, nMapFiles=%v, cost=%v\n%v\n", i, dataSize[k], nMapFiles[k], cost, errMsg) 90 | } else { 91 | fmt.Printf("Case%d PASS, dataSize=%v, nMapFiles=%v, cost=%v\n", i, dataSize[k], nMapFiles[k], cost) 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /courses/dss/percolator/README.md: -------------------------------------------------------------------------------- 1 | # The Percolator lab 2 | 3 | ## What is Percolator 4 | 5 | Percolator is a system built by Google for incremental processing on a very 6 | large data set. Percolator also provides a distributed transaction protocol with 7 | ACID snapshot-isolation semantics. You can find more details in the paper: 8 | [Large-scale Incremental Processing Using Distributed Transactions and Notifications](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/36726.pdf). 9 | 10 | ## Lab prerequisites 11 | 12 | To start this lab, there are some prerequisites you need to: 13 | 14 | 1. be familiar with Rust (You can also learn something from our Rust training 15 | course) 16 | 17 | 2. know about how protobuf works 18 | 19 | 3. have basic knowledge of how RPC works 20 | 21 | 4. have basic knowledge of what is the distributed transaction 22 | 23 | ## Concepts of the lab 24 | 25 | ### Server 26 | 27 | There are two kinds of servers which provide different services in this lab: the 28 | TSO server and the storage server. 29 | 30 | #### TSO server 31 | 32 | Percolator relies on a service named *timestamp oracle*. The TSO server 33 | implemented by `TimestampOracle` can produce timestamps in a strictly increasing 34 | order. All transactions need to get the unique timestamp to indicate the 35 | execution order. 36 | 37 | #### Storage server 38 | 39 | Percolator is built upon the Bigtable, which presents a multi-dimensional sorted 40 | map to users. In this lab, the storage server implemented by `MemoryStorage`, 41 | which consists of three columns, is used to simulate the Bigtable. These columns 42 | which implemented by `BTreeMap` are similar to the column in the Bigtable. In 43 | particular, the `MemoryStorage` has three columns: `Write`, `Data`, `Lock` to 44 | keep consistent with the Bigtable. 45 | 46 | Besides, the storage also needs to provide the basic operations like `read`, 47 | `write` and `erase` to manipulate the data stored in it. 48 | 49 | ### Client 50 | 51 | The client will `begin` a transaction which contains a set of operations, like 52 | `get` and `set`, and call `commit` to commit a transaction. Also, the client 53 | will call `get_timestamp` to obtain a timestamp. 54 | 55 | More implementation details can be found in the paper. 56 | 57 | ## Writing your own implementation 58 | 59 | There are some comments leaving in this project such as "Your definitions here" 60 | or "Your code here". You need to write the code by yourself according to the paper. 61 | There are not many strict restrictions, and thus you can define as many variables 62 | in both *struct* and *proto* as required to implement the functionality. 63 | 64 | ## Testing your work 65 | 66 | You can directly run the following command in the current directory: 67 | 68 | ```sh 69 | make test_percolator 70 | ``` 71 | -------------------------------------------------------------------------------- /courses/dss/linearizability/src/model.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::PartialEq; 2 | use std::fmt::Debug; 3 | use std::fmt::Display; 4 | use std::marker::Send; 5 | 6 | #[derive(Debug)] 7 | pub enum Value { 8 | Input(I), 9 | Output(O), 10 | None, 11 | } 12 | 13 | impl Value { 14 | pub fn input(&self) -> &I { 15 | if let Value::Input(i) = self { 16 | i 17 | } else { 18 | panic!("Not a input") 19 | } 20 | } 21 | 22 | pub fn output(&self) -> &O { 23 | if let Value::Output(o) = self { 24 | o 25 | } else { 26 | panic!("Not a output") 27 | } 28 | } 29 | } 30 | 31 | #[derive(Debug)] 32 | pub struct Operation { 33 | pub input: I, 34 | pub call: i64, // invocation time 35 | pub output: O, 36 | pub finish: i64, // response time 37 | } 38 | 39 | pub enum EventKind { 40 | CallEvent, 41 | ReturnEvent, 42 | } 43 | 44 | pub struct Event { 45 | pub kind: EventKind, 46 | pub value: T, 47 | pub id: usize, 48 | } 49 | 50 | pub type Operations = Vec>; 51 | pub type Events = Vec>>; 52 | 53 | pub trait Model: Clone + Send + 'static { 54 | type State: Clone + Display + PartialEq; 55 | type Input: Send + Debug + 'static; 56 | type Output: Send + Debug + 'static; 57 | 58 | // Partition functions, such that a history is linearizable if an only 59 | // if each partition is linearizable. If you don't want to implement 60 | // this, you can always use the `NoPartition` functions implemented 61 | // below. 62 | fn partition( 63 | &self, 64 | history: Operations, 65 | ) -> Vec> { 66 | vec![history] 67 | } 68 | 69 | fn partition_event( 70 | &self, 71 | history: Events, 72 | ) -> Vec> { 73 | vec![history] 74 | } 75 | 76 | // Initial state of the system. 77 | fn init(&self) -> Self::State; 78 | 79 | // Step function for the system. Returns whether or not the system 80 | // could take this step with the given inputs and outputs and also 81 | // returns the new state. This should not mutate the existing state. 82 | fn step( 83 | &self, 84 | state: &Self::State, 85 | input: &Self::Input, 86 | output: &Self::Output, 87 | ) -> (bool, Self::State); 88 | 89 | // Equality on states. If you are using a simple data type for states, 90 | // you can use the `ShallowEqual` function implemented below. 91 | fn equal(&self, state1: &Self::State, state2: &Self::State) -> bool { 92 | state1 == state2 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/server.rs: -------------------------------------------------------------------------------- 1 | use crate::common::{GetResponse, RemoveResponse, Request, SetResponse}; 2 | use crate::{KvsEngine, Result}; 3 | use log::{debug, error}; 4 | use serde_json::Deserializer; 5 | use std::io::{BufReader, BufWriter, Write}; 6 | use std::net::{TcpListener, TcpStream, ToSocketAddrs}; 7 | 8 | /// The server of a key value store. 9 | pub struct KvsServer { 10 | engine: E, 11 | } 12 | 13 | impl KvsServer { 14 | /// Create a `KvsServer` with a given storage engine. 15 | pub fn new(engine: E) -> Self { 16 | KvsServer { engine } 17 | } 18 | 19 | /// Run the server listening on the given address 20 | pub fn run(mut self, addr: A) -> Result<()> { 21 | let listener = TcpListener::bind(addr)?; 22 | for stream in listener.incoming() { 23 | match stream { 24 | Ok(stream) => { 25 | if let Err(e) = self.serve(stream) { 26 | error!("Error on serving client: {}", e); 27 | } 28 | } 29 | Err(e) => error!("Connection failed: {}", e), 30 | } 31 | } 32 | Ok(()) 33 | } 34 | 35 | fn serve(&mut self, tcp: TcpStream) -> Result<()> { 36 | let peer_addr = tcp.peer_addr()?; 37 | let reader = BufReader::new(&tcp); 38 | let mut writer = BufWriter::new(&tcp); 39 | let req_reader = Deserializer::from_reader(reader).into_iter::(); 40 | 41 | macro_rules! send_resp { 42 | ($resp:expr) => {{ 43 | let resp = $resp; 44 | serde_json::to_writer(&mut writer, &resp)?; 45 | writer.flush()?; 46 | debug!("Response sent to {}: {:?}", peer_addr, resp); 47 | };}; 48 | } 49 | 50 | for req in req_reader { 51 | let req = req?; 52 | debug!("Receive request from {}: {:?}", peer_addr, req); 53 | match req { 54 | Request::Get { key } => send_resp!(match self.engine.get(key) { 55 | Ok(value) => GetResponse::Ok(value), 56 | Err(e) => GetResponse::Err(format!("{}", e)), 57 | }), 58 | Request::Set { key, value } => send_resp!(match self.engine.set(key, value) { 59 | Ok(_) => SetResponse::Ok(()), 60 | Err(e) => SetResponse::Err(format!("{}", e)), 61 | }), 62 | Request::Remove { key } => send_resp!(match self.engine.remove(key) { 63 | Ok(_) => RemoveResponse::Ok(()), 64 | Err(e) => RemoveResponse::Err(format!("{}", e)), 65 | }), 66 | }; 67 | } 68 | Ok(()) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /courses/rust/projects/project-2/src/bin/kvs.rs: -------------------------------------------------------------------------------- 1 | use clap::{App, AppSettings, Arg, SubCommand}; 2 | use kvs::{KvStore, KvsError, Result}; 3 | use std::env::current_dir; 4 | use std::process::exit; 5 | 6 | fn main() -> Result<()> { 7 | let matches = App::new(env!("CARGO_PKG_NAME")) 8 | .version(env!("CARGO_PKG_VERSION")) 9 | .author(env!("CARGO_PKG_AUTHORS")) 10 | .about(env!("CARGO_PKG_DESCRIPTION")) 11 | .setting(AppSettings::DisableHelpSubcommand) 12 | .setting(AppSettings::SubcommandRequiredElseHelp) 13 | .setting(AppSettings::VersionlessSubcommands) 14 | .subcommand( 15 | SubCommand::with_name("set") 16 | .about("Set the value of a string key to a string") 17 | .arg(Arg::with_name("KEY").help("A string key").required(true)) 18 | .arg( 19 | Arg::with_name("VALUE") 20 | .help("The string value of the key") 21 | .required(true), 22 | ), 23 | ) 24 | .subcommand( 25 | SubCommand::with_name("get") 26 | .about("Get the string value of a given string key") 27 | .arg(Arg::with_name("KEY").help("A string key").required(true)), 28 | ) 29 | .subcommand( 30 | SubCommand::with_name("rm") 31 | .about("Remove a given key") 32 | .arg(Arg::with_name("KEY").help("A string key").required(true)), 33 | ) 34 | .get_matches(); 35 | 36 | match matches.subcommand() { 37 | ("set", Some(matches)) => { 38 | let key = matches.value_of("KEY").unwrap(); 39 | let value = matches.value_of("VALUE").unwrap(); 40 | 41 | let mut store = KvStore::open(current_dir()?)?; 42 | store.set(key.to_string(), value.to_string())?; 43 | } 44 | ("get", Some(matches)) => { 45 | let key = matches.value_of("KEY").unwrap(); 46 | 47 | let mut store = KvStore::open(current_dir()?)?; 48 | if let Some(value) = store.get(key.to_string())? { 49 | println!("{}", value); 50 | } else { 51 | println!("Key not found"); 52 | } 53 | } 54 | ("rm", Some(matches)) => { 55 | let key = matches.value_of("KEY").unwrap(); 56 | 57 | let mut store = KvStore::open(current_dir()?)?; 58 | match store.remove(key.to_string()) { 59 | Ok(()) => {} 60 | Err(KvsError::KeyNotFound) => { 61 | println!("Key not found"); 62 | exit(1); 63 | } 64 | Err(e) => return Err(e), 65 | } 66 | } 67 | _ => unreachable!(), 68 | } 69 | Ok(()) 70 | } 71 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/server.rs: -------------------------------------------------------------------------------- 1 | use crate::common::{GetResponse, RemoveResponse, Request, SetResponse}; 2 | use crate::thread_pool::ThreadPool; 3 | use crate::{KvsEngine, Result}; 4 | use log::{debug, error}; 5 | use serde_json::Deserializer; 6 | use std::io::{BufReader, BufWriter, Write}; 7 | use std::net::{TcpListener, TcpStream, ToSocketAddrs}; 8 | 9 | /// The server of a key value store. 10 | pub struct KvsServer { 11 | engine: E, 12 | pool: P, 13 | } 14 | 15 | impl KvsServer { 16 | /// Create a `KvsServer` with a given storage engine. 17 | pub fn new(engine: E, pool: P) -> Self { 18 | KvsServer { engine, pool } 19 | } 20 | 21 | /// Run the server listening on the given address 22 | pub fn run(self, addr: A) -> Result<()> { 23 | let listener = TcpListener::bind(addr)?; 24 | for stream in listener.incoming() { 25 | let engine = self.engine.clone(); 26 | self.pool.spawn(move || match stream { 27 | Ok(stream) => { 28 | if let Err(e) = serve(engine, stream) { 29 | error!("Error on serving client: {}", e); 30 | } 31 | } 32 | Err(e) => error!("Connection failed: {}", e), 33 | }) 34 | } 35 | Ok(()) 36 | } 37 | } 38 | 39 | fn serve(engine: E, tcp: TcpStream) -> Result<()> { 40 | let peer_addr = tcp.peer_addr()?; 41 | let reader = BufReader::new(&tcp); 42 | let mut writer = BufWriter::new(&tcp); 43 | let req_reader = Deserializer::from_reader(reader).into_iter::(); 44 | 45 | macro_rules! send_resp { 46 | ($resp:expr) => {{ 47 | let resp = $resp; 48 | serde_json::to_writer(&mut writer, &resp)?; 49 | writer.flush()?; 50 | debug!("Response sent to {}: {:?}", peer_addr, resp); 51 | };}; 52 | } 53 | 54 | for req in req_reader { 55 | let req = req?; 56 | debug!("Receive request from {}: {:?}", peer_addr, req); 57 | match req { 58 | Request::Get { key } => send_resp!(match engine.get(key) { 59 | Ok(value) => GetResponse::Ok(value), 60 | Err(e) => GetResponse::Err(format!("{}", e)), 61 | }), 62 | Request::Set { key, value } => send_resp!(match engine.set(key, value) { 63 | Ok(_) => SetResponse::Ok(()), 64 | Err(e) => SetResponse::Err(format!("{}", e)), 65 | }), 66 | Request::Remove { key } => send_resp!(match engine.remove(key) { 67 | Ok(_) => RemoveResponse::Ok(()), 68 | Err(e) => RemoveResponse::Err(format!("{}", e)), 69 | }), 70 | }; 71 | } 72 | Ok(()) 73 | } 74 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/bin/kvs-server.rs: -------------------------------------------------------------------------------- 1 | use clap::arg_enum; 2 | use kvs::*; 3 | use log::LevelFilter; 4 | use log::{error, info, warn}; 5 | use std::env::current_dir; 6 | use std::fs; 7 | use std::net::SocketAddr; 8 | use std::process::exit; 9 | use structopt::StructOpt; 10 | 11 | const DEFAULT_LISTENING_ADDRESS: &str = "127.0.0.1:4000"; 12 | const DEFAULT_ENGINE: Engine = Engine::kvs; 13 | 14 | #[derive(StructOpt, Debug)] 15 | #[structopt(name = "kvs-server")] 16 | struct Opt { 17 | #[structopt( 18 | long, 19 | help = "Sets the listening address", 20 | value_name = "IP:PORT", 21 | raw(default_value = "DEFAULT_LISTENING_ADDRESS"), 22 | parse(try_from_str) 23 | )] 24 | addr: SocketAddr, 25 | #[structopt( 26 | long, 27 | help = "Sets the storage engine", 28 | value_name = "ENGINE-NAME", 29 | raw(possible_values = "&Engine::variants()") 30 | )] 31 | engine: Option, 32 | } 33 | 34 | arg_enum! { 35 | #[allow(non_camel_case_types)] 36 | #[derive(Debug, Copy, Clone, PartialEq, Eq)] 37 | enum Engine { 38 | kvs, 39 | sled 40 | } 41 | } 42 | 43 | fn main() { 44 | env_logger::builder().filter_level(LevelFilter::Info).init(); 45 | let mut opt = Opt::from_args(); 46 | let res = current_engine().and_then(move |curr_engine| { 47 | if opt.engine.is_none() { 48 | opt.engine = curr_engine; 49 | } 50 | if curr_engine.is_some() && opt.engine != curr_engine { 51 | error!("Wrong engine!"); 52 | exit(1); 53 | } 54 | run(opt) 55 | }); 56 | if let Err(e) = res { 57 | error!("{}", e); 58 | exit(1); 59 | } 60 | } 61 | 62 | fn run(opt: Opt) -> Result<()> { 63 | let engine = opt.engine.unwrap_or(DEFAULT_ENGINE); 64 | info!("kvs-server {}", env!("CARGO_PKG_VERSION")); 65 | info!("Storage engine: {}", engine); 66 | info!("Listening on {}", opt.addr); 67 | 68 | // write engine to engine file 69 | fs::write(current_dir()?.join("engine"), format!("{}", engine))?; 70 | 71 | match engine { 72 | Engine::kvs => run_with_engine(KvStore::open(current_dir()?)?, opt.addr), 73 | Engine::sled => run_with_engine(SledKvsEngine::new(sled::open(current_dir()?)?), opt.addr), 74 | } 75 | } 76 | 77 | fn run_with_engine(engine: E, addr: SocketAddr) -> Result<()> { 78 | let server = KvsServer::new(engine); 79 | server.run(addr) 80 | } 81 | 82 | fn current_engine() -> Result> { 83 | let engine = current_dir()?.join("engine"); 84 | if !engine.exists() { 85 | return Ok(None); 86 | } 87 | 88 | match fs::read_to_string(engine)?.parse() { 89 | Ok(engine) => Ok(Some(engine)), 90 | Err(e) => { 91 | warn!("The content of engine file is invalid: {}", e); 92 | Ok(None) 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /courses/rust/building-blocks/bb-3.md: -------------------------------------------------------------------------------- 1 | # PNA Rust — Building Blocks 3 2 | 3 | Let's learn some building blocks! 4 | 5 | Put your other projects and concerns aside. Take a breath and relax. Here 6 | are some fun resources for you to explore. 7 | 8 | Read all the readings and perform all the exercises. 9 | 10 | - **[Reading: `log` crate API][l]**. The original Rust logging crate. Just read 11 | the crate-level documentation (the front page). You may need to click little `[+]` 12 | or `[-]` buttons to make the crate docs visible. This will give you an idea 13 | about how logging works in Rust. 14 | 15 | - **[Reading: `slog` crate API][sl]**. Another popular logging crate, designed 16 | for "structured logging". Again, just read the crate-level docs, to compare to 17 | `log`. You might also want to look at ["Introduction to structured logging 18 | with slog"][sli]. 19 | 20 | - **[Reading: Benefits of Structured Logging vs basic logging][lvsl]**. A 21 | StackOverflow discussion about the differences between traditional, 22 | text-oriented, line logging and structured logging. 23 | 24 | - **[Reading: Redis Protocol specification][rp]**. The protocol spec for 25 | [Redis], an in-memory key-value store. Think about what their design 26 | priorities were. When reading this it also helps to have the Redis [commands] 27 | on hand. 28 | 29 | - **Exercise: Write a Redis ping-pong client and server using `std::io`**. Write 30 | a simple client and server that speaks the Redis protocol, with the client 31 | issuing [PING] commands and the server responding appropriately. Use the 32 | [`std::io`] APIs to read and write bytes directly. Does your client work with 33 | an actual Redis server? 34 | 35 | - **Exercise: Write a Redis ping-pong client and server with serialized 36 | messages**. Same as above, but this time define the protocol with types and 37 | write a [`serde` data format][df] to indirectly read and write messages 38 | through serialization. 39 | 40 | - **[Reading: Statistically Rigorous Java Performance Evaluation][pe]**. 41 | Although it is specifically about Java, and discusses topics relevant to 42 | garbage-collected languages, it is a good example of the kind of thinking 43 | necessary to create effective benchmarks. 44 | 45 | 46 | 47 | 48 | [pe]: https://dri.es/files/oopsla07-georges.pdf 49 | [df]: https://serde.rs/data-format.html 50 | [`std::io`]: https://doc.rust-lang.org/std/io/ 51 | [PING]: https://redis.io/commands/ping 52 | [commands]: https://redis.io/commands 53 | [Redis]: https://redis.io/ 54 | [rp]: https://redis.io/topics/protocol 55 | [l]: https://docs.rs/log/ 56 | [sl]: https://docs.rs/slog/ 57 | [sli]: https://github.com/slog-rs/slog/wiki/Introduction-to-structured-logging-with-slog 58 | [lvsl]: https://softwareengineering.stackexchange.com/questions/312197/benefits-of-structured-logging-vs-basic-logging 59 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/benches/engine_bench.rs: -------------------------------------------------------------------------------- 1 | use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; 2 | use kvs::{KvStore, KvsEngine, SledKvsEngine}; 3 | use rand::prelude::*; 4 | use sled; 5 | use tempfile::TempDir; 6 | 7 | fn set_bench(c: &mut Criterion) { 8 | let mut group = c.benchmark_group("set_bench"); 9 | group.bench_function("kvs", |b| { 10 | b.iter_batched( 11 | || { 12 | let temp_dir = TempDir::new().unwrap(); 13 | (KvStore::open(temp_dir.path()).unwrap(), temp_dir) 14 | }, 15 | |(mut store, _temp_dir)| { 16 | for i in 1..(1 << 12) { 17 | store.set(format!("key{}", i), "value".to_string()).unwrap(); 18 | } 19 | }, 20 | BatchSize::SmallInput, 21 | ) 22 | }); 23 | group.bench_function("sled", |b| { 24 | b.iter_batched( 25 | || { 26 | let temp_dir = TempDir::new().unwrap(); 27 | (SledKvsEngine::new(sled::open(&temp_dir).unwrap()), temp_dir) 28 | }, 29 | |(mut db, _temp_dir)| { 30 | for i in 1..(1 << 12) { 31 | db.set(format!("key{}", i), "value".to_string()).unwrap(); 32 | } 33 | }, 34 | BatchSize::SmallInput, 35 | ) 36 | }); 37 | group.finish(); 38 | } 39 | 40 | fn get_bench(c: &mut Criterion) { 41 | let mut group = c.benchmark_group("get_bench"); 42 | for i in &vec![8, 12, 16, 20] { 43 | group.bench_with_input(format!("kvs_{}", i), i, |b, i| { 44 | let temp_dir = TempDir::new().unwrap(); 45 | let mut store = KvStore::open(temp_dir.path()).unwrap(); 46 | for key_i in 1..(1 << i) { 47 | store 48 | .set(format!("key{}", key_i), "value".to_string()) 49 | .unwrap(); 50 | } 51 | let mut rng = SmallRng::from_seed([0; 16]); 52 | b.iter(|| { 53 | store 54 | .get(format!("key{}", rng.gen_range(1, 1 << i))) 55 | .unwrap(); 56 | }) 57 | }); 58 | } 59 | for i in &vec![8, 12, 16, 20] { 60 | group.bench_with_input(format!("sled_{}", i), i, |b, i| { 61 | let temp_dir = TempDir::new().unwrap(); 62 | let mut db = SledKvsEngine::new(sled::open(&temp_dir).unwrap()); 63 | for key_i in 1..(1 << i) { 64 | db.set(format!("key{}", key_i), "value".to_string()) 65 | .unwrap(); 66 | } 67 | let mut rng = SmallRng::from_seed([0; 16]); 68 | b.iter(|| { 69 | db.get(format!("key{}", rng.gen_range(1, 1 << i))).unwrap(); 70 | }) 71 | }); 72 | } 73 | group.finish(); 74 | } 75 | 76 | criterion_group!(benches, set_bench, get_bench); 77 | criterion_main!(benches); 78 | -------------------------------------------------------------------------------- /courses/dss/labrpc/src/client.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::sync::{Arc, Mutex}; 3 | 4 | use futures::channel::mpsc::UnboundedSender; 5 | use futures::channel::oneshot; 6 | use futures::executor::ThreadPool; 7 | use futures::future::{self, FutureExt}; 8 | 9 | use crate::error::{Error, Result}; 10 | use crate::server::RpcFuture; 11 | 12 | pub struct Rpc { 13 | pub(crate) client_name: String, 14 | pub(crate) fq_name: &'static str, 15 | pub(crate) req: Option>, 16 | pub(crate) resp: Option>>>, 17 | pub(crate) hooks: Arc>>>, 18 | } 19 | 20 | impl Rpc { 21 | pub(crate) fn take_resp_sender(&mut self) -> Option>>> { 22 | self.resp.take() 23 | } 24 | } 25 | 26 | impl fmt::Debug for Rpc { 27 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 28 | f.debug_struct("Rpc") 29 | .field("client_name", &self.client_name) 30 | .field("fq_name", &self.fq_name) 31 | .finish() 32 | } 33 | } 34 | 35 | pub trait RpcHooks: Sync + Send + 'static { 36 | fn before_dispatch(&self, fq_name: &str, req: &[u8]) -> Result<()>; 37 | fn after_dispatch(&self, fq_name: &str, resp: Result>) -> Result>; 38 | } 39 | 40 | #[derive(Clone)] 41 | pub struct Client { 42 | // this end-point's name 43 | pub(crate) name: String, 44 | // copy of Network.sender 45 | pub(crate) sender: UnboundedSender, 46 | pub(crate) hooks: Arc>>>, 47 | 48 | pub worker: ThreadPool, 49 | } 50 | 51 | impl Client { 52 | pub fn call(&self, fq_name: &'static str, req: &Req) -> RpcFuture> 53 | where 54 | Req: labcodec::Message, 55 | Rsp: labcodec::Message + 'static, 56 | { 57 | let mut buf = vec![]; 58 | if let Err(e) = labcodec::encode(req, &mut buf) { 59 | return Box::pin(future::err(Error::Encode(e))); 60 | } 61 | 62 | let (tx, rx) = oneshot::channel(); 63 | let rpc = Rpc { 64 | client_name: self.name.clone(), 65 | fq_name, 66 | req: Some(buf), 67 | resp: Some(tx), 68 | hooks: self.hooks.clone(), 69 | }; 70 | 71 | // Sends requests and waits responses. 72 | if self.sender.unbounded_send(rpc).is_err() { 73 | return Box::pin(future::err(Error::Stopped)); 74 | } 75 | 76 | Box::pin(rx.then(|res| async move { 77 | match res { 78 | Ok(Ok(resp)) => labcodec::decode(&resp).map_err(Error::Decode), 79 | Ok(Err(e)) => Err(e), 80 | Err(e) => Err(Error::Recv(e)), 81 | } 82 | })) 83 | } 84 | 85 | pub fn set_hooks(&self, hooks: Arc) { 86 | *self.hooks.lock().unwrap() = Some(hooks); 87 | } 88 | 89 | pub fn clear_hooks(&self) { 90 | *self.hooks.lock().unwrap() = None; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/bin/kvs-client.rs: -------------------------------------------------------------------------------- 1 | use clap::AppSettings; 2 | use kvs::{KvsClient, Result}; 3 | use std::net::SocketAddr; 4 | use std::process::exit; 5 | use structopt::StructOpt; 6 | 7 | #[derive(StructOpt, Debug)] 8 | #[structopt( 9 | name = "kvs-client", 10 | raw(global_settings = "&[\ 11 | AppSettings::DisableHelpSubcommand,\ 12 | AppSettings::VersionlessSubcommands]") 13 | )] 14 | struct Opt { 15 | #[structopt(subcommand)] 16 | command: Command, 17 | } 18 | 19 | #[derive(StructOpt, Debug)] 20 | enum Command { 21 | #[structopt(name = "get", about = "Get the string value of a given string key")] 22 | Get { 23 | #[structopt(name = "KEY", help = "A string key")] 24 | key: String, 25 | #[structopt( 26 | long, 27 | help = "Sets the server address", 28 | value_name = "IP:PORT", 29 | default_value = "127.0.0.1:4000", 30 | parse(try_from_str) 31 | )] 32 | addr: SocketAddr, 33 | }, 34 | #[structopt(name = "set", about = "Set the value of a string key to a string")] 35 | Set { 36 | #[structopt(name = "KEY", help = "A string key")] 37 | key: String, 38 | #[structopt(name = "VALUE", help = "The string value of the key")] 39 | value: String, 40 | #[structopt( 41 | long, 42 | help = "Sets the server address", 43 | value_name = "IP:PORT", 44 | default_value = "127.0.0.1:4000", 45 | parse(try_from_str) 46 | )] 47 | addr: SocketAddr, 48 | }, 49 | #[structopt(name = "rm", about = "Remove a given string key")] 50 | Remove { 51 | #[structopt(name = "KEY", help = "A string key")] 52 | key: String, 53 | #[structopt( 54 | long, 55 | help = "Sets the server address", 56 | value_name = "IP:PORT", 57 | default_value = "127.0.0.1:4000", 58 | parse(try_from_str) 59 | )] 60 | addr: SocketAddr, 61 | }, 62 | } 63 | 64 | fn main() { 65 | let opt = Opt::from_args(); 66 | if let Err(e) = run(opt) { 67 | eprintln!("{}", e); 68 | exit(1); 69 | } 70 | } 71 | 72 | fn run(opt: Opt) -> Result<()> { 73 | match opt.command { 74 | Command::Get { key, addr } => { 75 | let mut client = KvsClient::connect(addr)?; 76 | if let Some(value) = client.get(key)? { 77 | println!("{}", value); 78 | } else { 79 | println!("Key not found"); 80 | } 81 | } 82 | Command::Set { key, value, addr } => { 83 | let mut client = KvsClient::connect(addr)?; 84 | client.set(key, value)?; 85 | } 86 | Command::Remove { key, addr } => { 87 | let mut client = KvsClient::connect(addr)?; 88 | client.remove(key)?; 89 | } 90 | } 91 | Ok(()) 92 | } 93 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/engines/sled.rs: -------------------------------------------------------------------------------- 1 | use crate::thread_pool::ThreadPool; 2 | use crate::{KvsEngine, KvsError, Result}; 3 | use sled::Db; 4 | use tokio::prelude::*; 5 | use tokio::sync::oneshot; 6 | 7 | /// Wrapper of `sled::Db` 8 | #[derive(Clone)] 9 | pub struct SledKvsEngine { 10 | pool: P, 11 | db: Db, 12 | } 13 | 14 | impl SledKvsEngine

{ 15 | /// Creates a `SledKvsEngine` from `sled::Db`. 16 | /// 17 | /// Operations are run in the given thread pool. `concurrency` specifies the number of 18 | /// threads in the thread pool. 19 | pub fn new(db: Db, concurrency: u32) -> Result { 20 | let pool = P::new(concurrency)?; 21 | Ok(SledKvsEngine { pool, db }) 22 | } 23 | } 24 | 25 | impl KvsEngine for SledKvsEngine

{ 26 | fn set(&self, key: String, value: String) -> Box + Send> { 27 | let db = self.db.clone(); 28 | let (tx, rx) = oneshot::channel(); 29 | self.pool.spawn(move || { 30 | let res = db 31 | .set(key, value.into_bytes()) 32 | .and_then(|_| db.flush()) 33 | .map(|_| ()) 34 | .map_err(KvsError::from); 35 | if tx.send(res).is_err() { 36 | error!("Receiving end is dropped"); 37 | } 38 | }); 39 | Box::new( 40 | rx.map_err(|e| KvsError::StringError(format!("{}", e))) 41 | .flatten(), 42 | ) 43 | } 44 | 45 | fn get(&self, key: String) -> Box, Error = KvsError> + Send> { 46 | let db = self.db.clone(); 47 | let (tx, rx) = oneshot::channel(); 48 | self.pool.spawn(move || { 49 | let res = (move || { 50 | Ok(db 51 | .get(key)? 52 | .map(|i_vec| AsRef::<[u8]>::as_ref(&i_vec).to_vec()) 53 | .map(String::from_utf8) 54 | .transpose()?) 55 | })(); 56 | if tx.send(res).is_err() { 57 | error!("Receiving end is dropped"); 58 | } 59 | }); 60 | Box::new( 61 | rx.map_err(|e| KvsError::StringError(format!("{}", e))) 62 | .flatten(), 63 | ) 64 | } 65 | 66 | fn remove(&self, key: String) -> Box + Send> { 67 | let db = self.db.clone(); 68 | let (tx, rx) = oneshot::channel(); 69 | self.pool.spawn(move || { 70 | let res = (|| { 71 | db.del(key)?.ok_or(KvsError::KeyNotFound)?; 72 | db.flush()?; 73 | Ok(()) 74 | })(); 75 | if tx.send(res).is_err() { 76 | error!("Receiving end is dropped"); 77 | } 78 | }); 79 | Box::new( 80 | rx.map_err(|e| KvsError::StringError(format!("{}", e))) 81 | .flatten(), 82 | ) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /tidb/mapreduce/utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "sort" 10 | "strings" 11 | ) 12 | 13 | // RoundArgs contains arguments used in a map-reduce round. 14 | type RoundArgs struct { 15 | MapFunc MapF 16 | ReduceFunc ReduceF 17 | NReduce int 18 | } 19 | 20 | // RoundsArgs represents arguments used in multiple map-reduce rounds. 21 | type RoundsArgs []RoundArgs 22 | 23 | type urlCount struct { 24 | url string 25 | cnt int 26 | } 27 | 28 | // TopN returns topN urls in the urlCntMap. 29 | func TopN(urlCntMap map[string]int, n int) ([]string, []int) { 30 | ucs := make([]*urlCount, 0, len(urlCntMap)) 31 | for k, v := range urlCntMap { 32 | ucs = append(ucs, &urlCount{k, v}) 33 | } 34 | sort.Slice(ucs, func(i, j int) bool { 35 | if ucs[i].cnt == ucs[j].cnt { 36 | return ucs[i].url < ucs[j].url 37 | } 38 | return ucs[i].cnt > ucs[j].cnt 39 | }) 40 | urls := make([]string, 0, n) 41 | cnts := make([]int, 0, n) 42 | for i, u := range ucs { 43 | if i == n { 44 | break 45 | } 46 | urls = append(urls, u.url) 47 | cnts = append(cnts, u.cnt) 48 | } 49 | return urls, cnts 50 | } 51 | 52 | // CheckFile checks if these two files are same. 53 | func CheckFile(expected, got string) (string, bool) { 54 | c1, err := ioutil.ReadFile(expected) 55 | if err != nil { 56 | panic(err) 57 | } 58 | c2, err := ioutil.ReadFile(got) 59 | if err != nil { 60 | panic(err) 61 | } 62 | s1 := strings.TrimSpace(string(c1)) 63 | s2 := strings.TrimSpace(string(c2)) 64 | if s1 == s2 { 65 | return "", true 66 | } 67 | 68 | errMsg := fmt.Sprintf("expected:\n%s\n, but got:\n%s\n", c1, c2) 69 | return errMsg, false 70 | } 71 | 72 | // CreateFileAndBuf opens or creates a specific file for writing. 73 | func CreateFileAndBuf(fpath string) (*os.File, *bufio.Writer) { 74 | dir := path.Dir(fpath) 75 | os.MkdirAll(dir, 0777) 76 | f, err := os.OpenFile(fpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) 77 | if err != nil { 78 | panic(err) 79 | } 80 | return f, bufio.NewWriterSize(f, 1<<20) 81 | } 82 | 83 | // OpenFileAndBuf opens a specific file for reading. 84 | func OpenFileAndBuf(fpath string) (*os.File, *bufio.Reader) { 85 | f, err := os.OpenFile(fpath, os.O_RDONLY, 0666) 86 | if err != nil { 87 | panic(err) 88 | } 89 | return f, bufio.NewReader(f) 90 | } 91 | 92 | // WriteToBuf write strs to this buffer. 93 | func WriteToBuf(buf *bufio.Writer, strs ...string) { 94 | for _, str := range strs { 95 | if _, err := buf.WriteString(str); err != nil { 96 | panic(err) 97 | } 98 | } 99 | } 100 | 101 | // SafeClose flushes this buffer and closes this file. 102 | func SafeClose(f *os.File, buf *bufio.Writer) { 103 | if buf != nil { 104 | if err := buf.Flush(); err != nil { 105 | panic(err) 106 | } 107 | } 108 | if err := f.Close(); err != nil { 109 | panic(err) 110 | } 111 | } 112 | 113 | // FileOrDirExist tests if this file or dir exist in a simple way. 114 | func FileOrDirExist(p string) bool { 115 | _, err := os.Stat(p) 116 | return err == nil 117 | } 118 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/bin/kvs-client.rs: -------------------------------------------------------------------------------- 1 | use clap::AppSettings; 2 | use kvs::{KvsClient, Result}; 3 | use std::net::SocketAddr; 4 | use std::process::exit; 5 | use structopt::StructOpt; 6 | use tokio::prelude::*; 7 | 8 | #[derive(StructOpt, Debug)] 9 | #[structopt( 10 | name = "kvs-client", 11 | raw(global_settings = "&[\ 12 | AppSettings::DisableHelpSubcommand,\ 13 | AppSettings::VersionlessSubcommands]") 14 | )] 15 | struct Opt { 16 | #[structopt(subcommand)] 17 | command: Command, 18 | } 19 | 20 | #[derive(StructOpt, Debug)] 21 | enum Command { 22 | #[structopt(name = "get", about = "Get the string value of a given string key")] 23 | Get { 24 | #[structopt(name = "KEY", help = "A string key")] 25 | key: String, 26 | #[structopt( 27 | long, 28 | help = "Sets the server address", 29 | value_name = "IP:PORT", 30 | default_value = "127.0.0.1:4000", 31 | parse(try_from_str) 32 | )] 33 | addr: SocketAddr, 34 | }, 35 | #[structopt(name = "set", about = "Set the value of a string key to a string")] 36 | Set { 37 | #[structopt(name = "KEY", help = "A string key")] 38 | key: String, 39 | #[structopt(name = "VALUE", help = "The string value of the key")] 40 | value: String, 41 | #[structopt( 42 | long, 43 | help = "Sets the server address", 44 | value_name = "IP:PORT", 45 | default_value = "127.0.0.1:4000", 46 | parse(try_from_str) 47 | )] 48 | addr: SocketAddr, 49 | }, 50 | #[structopt(name = "rm", about = "Remove a given string key")] 51 | Remove { 52 | #[structopt(name = "KEY", help = "A string key")] 53 | key: String, 54 | #[structopt( 55 | long, 56 | help = "Sets the server address", 57 | value_name = "IP:PORT", 58 | default_value = "127.0.0.1:4000", 59 | parse(try_from_str) 60 | )] 61 | addr: SocketAddr, 62 | }, 63 | } 64 | 65 | fn main() { 66 | let opt = Opt::from_args(); 67 | if let Err(e) = run(opt) { 68 | eprintln!("{}", e); 69 | exit(1); 70 | } 71 | } 72 | 73 | fn run(opt: Opt) -> Result<()> { 74 | match opt.command { 75 | Command::Get { key, addr } => { 76 | let client = KvsClient::connect(addr); 77 | if let (Some(value), _) = client.and_then(move |client| client.get(key)).wait()? { 78 | println!("{}", value); 79 | } else { 80 | println!("Key not found"); 81 | } 82 | } 83 | Command::Set { key, value, addr } => { 84 | let client = KvsClient::connect(addr); 85 | client 86 | .and_then(move |client| client.set(key, value)) 87 | .wait()?; 88 | } 89 | Command::Remove { key, addr } => { 90 | let client = KvsClient::connect(addr); 91 | client.and_then(move |client| client.remove(key)).wait()?; 92 | } 93 | } 94 | Ok(()) 95 | } 96 | -------------------------------------------------------------------------------- /courses/rust/projects/project-3/src/bin/kvs-client.rs: -------------------------------------------------------------------------------- 1 | use clap::AppSettings; 2 | use kvs::{KvsClient, Result}; 3 | use std::net::SocketAddr; 4 | use std::process::exit; 5 | use structopt::StructOpt; 6 | 7 | const DEFAULT_LISTENING_ADDRESS: &str = "127.0.0.1:4000"; 8 | const ADDRESS_FORMAT: &str = "IP:PORT"; 9 | 10 | #[derive(StructOpt, Debug)] 11 | #[structopt( 12 | name = "kvs-client", 13 | raw(global_settings = "&[\ 14 | AppSettings::DisableHelpSubcommand,\ 15 | AppSettings::VersionlessSubcommands]") 16 | )] 17 | struct Opt { 18 | #[structopt(subcommand)] 19 | command: Command, 20 | } 21 | 22 | #[derive(StructOpt, Debug)] 23 | enum Command { 24 | #[structopt(name = "get", about = "Get the string value of a given string key")] 25 | Get { 26 | #[structopt(name = "KEY", help = "A string key")] 27 | key: String, 28 | #[structopt( 29 | long, 30 | help = "Sets the server address", 31 | raw(value_name = "ADDRESS_FORMAT"), 32 | raw(default_value = "DEFAULT_LISTENING_ADDRESS"), 33 | parse(try_from_str) 34 | )] 35 | addr: SocketAddr, 36 | }, 37 | #[structopt(name = "set", about = "Set the value of a string key to a string")] 38 | Set { 39 | #[structopt(name = "KEY", help = "A string key")] 40 | key: String, 41 | #[structopt(name = "VALUE", help = "The string value of the key")] 42 | value: String, 43 | #[structopt( 44 | long, 45 | help = "Sets the server address", 46 | raw(value_name = "ADDRESS_FORMAT"), 47 | raw(default_value = "DEFAULT_LISTENING_ADDRESS"), 48 | parse(try_from_str) 49 | )] 50 | addr: SocketAddr, 51 | }, 52 | #[structopt(name = "rm", about = "Remove a given string key")] 53 | Remove { 54 | #[structopt(name = "KEY", help = "A string key")] 55 | key: String, 56 | #[structopt( 57 | long, 58 | help = "Sets the server address", 59 | raw(value_name = "ADDRESS_FORMAT"), 60 | raw(default_value = "DEFAULT_LISTENING_ADDRESS"), 61 | parse(try_from_str) 62 | )] 63 | addr: SocketAddr, 64 | }, 65 | } 66 | 67 | fn main() { 68 | let opt = Opt::from_args(); 69 | if let Err(e) = run(opt) { 70 | eprintln!("{}", e); 71 | exit(1); 72 | } 73 | } 74 | 75 | fn run(opt: Opt) -> Result<()> { 76 | match opt.command { 77 | Command::Get { key, addr } => { 78 | let mut client = KvsClient::connect(addr)?; 79 | if let Some(value) = client.get(key)? { 80 | println!("{}", value); 81 | } else { 82 | println!("Key not found"); 83 | } 84 | } 85 | Command::Set { key, value, addr } => { 86 | let mut client = KvsClient::connect(addr)?; 87 | client.set(key, value)?; 88 | } 89 | Command::Remove { key, addr } => { 90 | let mut client = KvsClient::connect(addr)?; 91 | client.remove(key)?; 92 | } 93 | } 94 | Ok(()) 95 | } 96 | -------------------------------------------------------------------------------- /talent-plan-1.0/1.0-lp-tidb.md: -------------------------------------------------------------------------------- 1 | # 路径 1：TiDB 方向 2 | 3 | ## 课程目标 4 | * 掌握 Go 语言 5 | * 掌握分布式计算基本原理 6 | * 掌握数据库基础知识 7 | * 掌握优化器和执行引擎基本原理 8 | 9 | ## 课程内容 10 | 11 | ### Section 1 熟悉 Go 语言基础知识 12 | 13 | **学习资料：** 14 | 15 | * 阅读许老师的书 [《Go 语言开发》](https://book.douban.com/subject/11577300/) 16 | * [Go 官方教程](http://tour.golang.org/) 17 | * [Effective Go](https://golang.org/doc/effective_go.html) 18 | * [Profiling Go Programs](https://blog.golang.org/profiling-go-programs) 19 | 20 | **课程作业：** 21 | 22 | * 用 Go 语言实现一个多路归并的 Merge Sort：内存中有一个无序的 int64 数组，数组中有 16M 个整数，要求使用多个 goroutine 对这个无序数组进行排序，使得有元素按照从小到大的顺序排列，并且排序的性能需要比单线程 Quick Sort 快。 23 | * 根据[框架](https://github.com/pingcap/talent-plan/tree/master/tidb/mergesort)完成程序，**要求跑通测试程序，并使用 Go Profile 工具分析性能瓶颈。需要有文档、单元测试、性能瓶颈分析。** 24 | 25 | ### Section 2 分布式计算基础知识 26 | 27 | **学习资料：** 28 | 29 | * 学习并使用 Go 语言完成 [6.824 Lab 1: MapReduce](https://pdos.csail.mit.edu/6.824/labs/lab-1.html) 30 | 31 | **课程作业：** 32 | 33 | * 使用 Go 语言，使用框架代码，用 MapReduce + Shuffle 的方式完成作业：从包含大量 URL 的数据文件中，求出现次数最多的前 10 个 URL 和他们的出现次数，尽量利用 CPU 多核特性和内存资源。 34 | * 根据[作业框架](https://github.com/pingcap/talent-plan/tree/master/tidb/mapreduce)完成作业，详细的作业要求、框架的使用方法和评分规则请看框架的 README 文件 35 | 36 | ### Section 3 数据库基础知识 37 | 38 | **学习资料：** 39 | 40 | * [《数据库系统概念》](https://book.douban.com/subject/1929984/)部分章节阅读和自学，或者 [CMU 15-445](https://15445.courses.cs.cmu.edu/fall2018/schedule.html) 对应部分内容 41 | * 第 3 章：SQL 42 | * 第 11 章：索引与散列 43 | * 第 12 章：查询处理 44 | * 第 13 章：查询优化 45 | * 第 14 章：事务 46 | * 第 15 章：并发控制 47 | * 第 18 章：并行数据库 48 | * 第 19 章：分布式数据库 49 | 50 | **课程作业：** 51 | 52 | * select t1.a, count(*), avg(t1.b) from t1 left outer join t2 on t1.a=t2.a group by t1.a，请给出所有可能的逻辑执行计划（画出 Plan 树），并分析 t1 的数据分布对各种逻辑执行计划执行性能的影响。画 Plan 树时可以利用 [Graphviz 作图](http://www.webgraphviz.com)，例如对于 select * from t where a > 1，一个可能的执行计划可以如下图所示： 53 | 54 | ![Execution-example](../media/execution-example.jpg) 55 | 56 | ### Section 4 SQL 优化器和执行引擎基本原理 57 | 58 | **学习资料：** 59 | 60 | * [CMU 15-721](https://15721.courses.cs.cmu.edu/spring2018/) 数据库课程，自学 + 在线视频教学 61 | * Optimizer Implementation (Part I) 62 | * Optimizer Implementation (Part II) 63 | * Cost Models 64 | * Execution & Scheduling 65 | * Vectorized Execution (Part I) 66 | * Vectorized Execution (Part II) 67 | * Parallel Join Algorithms (Hashing) 68 | * Parallel Join Algorithms (Sorting) 69 | 70 | **课程作业：** 71 | 72 | * 用 Go 语言，实现一个 Join 算法，从两个 CSV 文件中读取数据，在一个或多个等值条件上进行连接，并对最终结果求和。请参考 [README](https://github.com/pingcap/talent-plan/tree/master/tidb/join) ，使用提供的框架完成作业，并跑通测试程序。提交的作业要求代码整洁规范，附有文档，并提供对程序的性能瓶颈分析及优化思路。 73 | 74 | ### Section 5 阅读 TiDB 源码 75 | 76 | **学习资料：** 77 | 78 | * [TiDB 源码阅读 24 章经](https://pingcap.com/blog-cn/#TiDB-%E6%BA%90%E7%A0%81%E9%98%85%E8%AF%BB) 79 | 80 | **课程作业：** 81 | 82 | * 给 TiDB Repo 提 PR，例如修复 [for new contributors issues](https://github.com/pingcap/tidb/issues?utf8=%E2%9C%93&q=is%3Aissue+is%3Aopen+label%3A%22for+new+contributors%22+)。 83 | 84 | ## 考核要点 85 | 86 | 完成各项作业，代码通过邮件提交到 ts-team@pingcap.com，要求使用 Golang 1.12，需要有代码、测试、注释、文档。 87 | 88 | 我们会根据以下规则进行评分，最终得到一个 0~100 分的分数 89 | 90 | 1. 代码可以跑过 PingCAP 提供的测试程序（一些题目有性能要求，希望在规定的时间内跑完）50% 91 | 2. 单元测试覆盖率 30% 92 | 3. [代码风格](https://github.com/pingcap/community/blob/master/CONTRIBUTING.md#code-style) 10% 93 | 4. 文档情况 10% 94 | -------------------------------------------------------------------------------- /talent-challenge-program/README.md: -------------------------------------------------------------------------------- 1 | # Talent Challenge Program 2 | 3 | Talent Challenge program is a platform that offers a structured remote mentoring program for **college students** successfully completing talent plan courses or having contributed to TiDB projects. Open source projects related to TiDB ecosystems will be selected together with professional mentors. Students **successfully completing talent plan courses or having contributed to TiDB projects** can apply for the projects, one at a time, and finish the projects with the help of the mentors in a given time. Mentees gain real-world experience by working closely with mentors. Students who successfully complete the projects will receive **bonus and better job opportunities**. 4 | 5 | ### Current cycle 6 | 7 | Talent Challenge program season 1 is ongoing, check the timeline and projects available on [dedicated page](s1.md). 8 | 9 | ## Communication 10 | 11 | Feel free to contact us in the **#talent-challenge** channel of the [tidbcommunity](https://slack.tidb.io/invite?team=tidb-community&channel=talent-challenge-&ref=github/talent-plan) slack workspace if any questions and suggestions about Talent Challenge program arise. 12 | 13 | ## How to Participate 14 | 15 | You can participate in Talent Challenge program based on your role. Talent Challenge supports the following roles. 16 | 17 | ### Project Maintainers/Mentors 18 | 19 | Project maintainers or mentors are requested to submit the ideas related to **TiDB Ecosystem** under the [Project Ideas](project-ideas.md) page using the [template](PROJECT_IDEA_TEMPLATE.md). 20 | 21 | PMC(Project Management Committee) of TiDB community will select the projects that will participate in the Talent Challenge program round and they will appear on the [Selected Projects](selected-projects.md) page after the selection. 22 | 23 | *Note:* 24 | 25 | - *Projects to be selected must have one mentor at least.* 26 | - *Mentors should be prepared to donate their time to help mentees with the project.* 27 | 28 | ### Mentees 29 | 30 | *Note:* 31 | 32 | - Only **students who have completed talent plan courses or having contributed to TiDB projects** can apply for Talent Challenge program. 33 | - You are eligible to participate as a mentee in a single project per-cycle. 34 | 35 | **Prior to the opening of applications, mentee candidates are encouraged to talk to mentors about the project.** This helps them refine their ideas and deepen their understanding of the project. 36 | 37 | #### Mentee selection process 38 | 39 | 1. Mentee candidates who want to participate in Talent Challenge are expected to join Slack Channel ([#talent-challenge](https://slack.tidb.io/invite?team=tidb-community&channel=talent-challenge-&ref=github/talent-plan)) to have self-introductions. 40 | 2. The mentors would communicate with mentee candidates about the project they would focus on and assign mentee candidates coding tasks. 41 | 3. Mentee candidates need to conduct coding tasks for their desired project under the guidance of mentors and submit their resumes to [talent-plan@tidb.io](maito:talent-plan@tidb.io) using the [template](MENTEE_APPLY_TEMPLATE.md). 42 | 4. The mentors would decide whether to accept mentee candidates in the project based on their scores of talent plan courses, coding assignments, and capabilities. 43 | -------------------------------------------------------------------------------- /courses/rust/projects/project-4/src/bin/kvs-server.rs: -------------------------------------------------------------------------------- 1 | use clap::arg_enum; 2 | use kvs::thread_pool::*; 3 | use kvs::*; 4 | use log::LevelFilter; 5 | use log::{error, info, warn}; 6 | use std::env; 7 | use std::env::current_dir; 8 | use std::fs; 9 | use std::net::SocketAddr; 10 | use std::process::exit; 11 | use structopt::StructOpt; 12 | 13 | const DEFAULT_LISTENING_ADDRESS: &str = "127.0.0.1:4000"; 14 | const DEFAULT_ENGINE: Engine = Engine::kvs; 15 | 16 | #[derive(StructOpt, Debug)] 17 | #[structopt(name = "kvs-server")] 18 | struct Opt { 19 | #[structopt( 20 | long, 21 | help = "Sets the listening address", 22 | value_name = "IP:PORT", 23 | raw(default_value = "DEFAULT_LISTENING_ADDRESS"), 24 | parse(try_from_str) 25 | )] 26 | addr: SocketAddr, 27 | #[structopt( 28 | long, 29 | help = "Sets the storage engine", 30 | value_name = "ENGINE-NAME", 31 | raw(possible_values = "&Engine::variants()") 32 | )] 33 | engine: Option, 34 | } 35 | 36 | arg_enum! { 37 | #[allow(non_camel_case_types)] 38 | #[derive(Debug, Copy, Clone, PartialEq, Eq)] 39 | enum Engine { 40 | kvs, 41 | sled 42 | } 43 | } 44 | 45 | fn main() { 46 | env_logger::builder() 47 | .filter_level(LevelFilter::Info) 48 | .init(); 49 | let mut opt = Opt::from_args(); 50 | let res = current_engine().and_then(move |curr_engine| { 51 | if opt.engine.is_none() { 52 | opt.engine = curr_engine; 53 | } 54 | if curr_engine.is_some() && opt.engine != curr_engine { 55 | error!("Wrong engine!"); 56 | exit(1); 57 | } 58 | run(opt) 59 | }); 60 | if let Err(e) = res { 61 | error!("{}", e); 62 | exit(1); 63 | } 64 | } 65 | 66 | fn run(opt: Opt) -> Result<()> { 67 | let engine = opt.engine.unwrap_or(DEFAULT_ENGINE); 68 | info!("kvs-server {}", env!("CARGO_PKG_VERSION")); 69 | info!("Storage engine: {}", engine); 70 | info!("Listening on {}", opt.addr); 71 | 72 | // write engine to engine file 73 | fs::write(current_dir()?.join("engine"), format!("{}", engine))?; 74 | 75 | let pool = RayonThreadPool::new(num_cpus::get() as u32)?; 76 | 77 | match engine { 78 | Engine::kvs => run_with(KvStore::open(env::current_dir()?)?, pool, opt.addr), 79 | Engine::sled => run_with( 80 | SledKvsEngine::new(sled::open(env::current_dir()?)?), 81 | pool, 82 | opt.addr, 83 | ), 84 | } 85 | } 86 | 87 | pub fn run_with(engine: E, pool: P, addr: SocketAddr) -> Result<()> { 88 | let server = KvsServer::new(engine, pool); 89 | server.run(addr) 90 | } 91 | 92 | fn current_engine() -> Result> { 93 | let engine = current_dir()?.join("engine"); 94 | if !engine.exists() { 95 | return Ok(None); 96 | } 97 | 98 | match fs::read_to_string(engine)?.parse() { 99 | Ok(engine) => Ok(Some(engine)), 100 | Err(e) => { 101 | warn!("The content of engine file is invalid: {}", e); 102 | Ok(None) 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /talent-challenge-program2021/README.md: -------------------------------------------------------------------------------- 1 | # Talent Challenge Program 2021 2 | 3 | Talent Challenge program is a platform that offers a structured remote mentoring program for **college students** who have successfully completed talent plan courses or contributed to TiDB projects. Open source projects related to TiDB ecosystems will be selected together with professional mentors. Students can apply for the projects, one at a time, and finish the projects with the help of the mentors in a given time. Mentees gain real-world experiences by working closely with mentors. Students who successfully complete the projects will receive **bonus and better job opportunities**. 4 | 5 | ### Current cycle 6 | 7 | Talent Challenge program 2021 is ongoing, check the timeline and projects available on [dedicated page](schedule.md). 8 | 9 | ## Communication 10 | 11 | Feel free to contact us in the **#talent-challenge** channel of the [tidbcommunity](https://slack.tidb.io/invite?team=tidb-community&channel=talent-challenge-&ref=github/talent-plan) slack workspace if any questions and suggestions about Talent Challenge program arise. 12 | 13 | ## How to Participate 14 | 15 | You can participate in Talent Challenge program based on your role. Talent Challenge supports the following roles. 16 | 17 | ### Project Maintainers/Mentors 18 | 19 | Project maintainers or mentors are requested to submit the ideas related to **TiDB Ecosystem** under the [Project Ideas](project-ideas.md) page using the [template](PROJECT_IDEA_TEMPLATE.md). 20 | 21 | PMC(Project Management Committee) of TiDB community will select the projects that will participate in the Talent Challenge program round and they will appear on the [Selected Projects](selected-projects.md) page after the selection. 22 | 23 | *Note:* 24 | 25 | - *Projects to be selected must have one mentor at least.* 26 | - *Mentors should be prepared to donate their time to help mentees with the project.* 27 | 28 | ### Mentees 29 | 30 | *Note:* 31 | 32 | - Must be college students at school. 33 | - **Students who have completed talent plan courses or contributed to TiDB projects** are encouraged to apply for the program and they will be considered as priority, or who have proven capabilites on distributed database development. 34 | - You are eligible to participate as a mentee in a single project per-cycle. 35 | 36 | **Prior to the opening of applications, mentee candidates are encouraged to talk to mentors about the project.** This helps them refine their ideas and deepen their understanding of the project. 37 | 38 | #### Mentee selection process 39 | 40 | 1. Mentee candidates who want to participate in Talent Challenge are expected to join Slack Channel ([#talent-challenge](https://slack.tidb.io/invite?team=tidb-community&channel=talent-challenge-&ref=github/talent-plan)) to have self-introductions. 41 | 2. The mentors would communicate with mentee candidates about the project they would focus on and assign mentee candidates coding tasks. 42 | 3. Mentee candidates need to conduct coding tasks for their desired project under the guidance of mentors and submit their resumes to [talent-plan@tidb.io](maito:talent-plan@tidb.io) using the [template](MENTEE_APPLY_TEMPLATE.md). 43 | 4. The mentors would decide whether to accept mentee candidates in the project based on their scores of talent plan courses, coding assignments, and capabilities. 44 | -------------------------------------------------------------------------------- /tidb/join/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | This is the homework for PingCAP Talent Plan Online of week 4. This homework is a simplified version of [ACM SIGMOD Programming Contest 2018](http://sigmod18contest.db.in.tum.de/index.shtml). 4 | 5 | The task is to evaluate batches of join queries on a set of pre-defined relations. Each join query specifies two relations, multiple (equality) join predicates, and one (sum) aggregation. The challenge is to fully utilize the CPU and memory resources and execute the queries as fast as possible. 6 | 7 | NOTE: **go 1.12 is required** 8 | 9 | ## Details 10 | 11 | The simple interface `Join(f0, f1 string, offset0, offset1 []int) (sum uint64)` is defined in `join.go`. Our test harness will feed two relations and two columns' offsets array to the interface every time, and check the correctness of the output result. Explaination to the four input arguments and one output argument of the interface are list as follows: 12 | 13 | - **f0**: File name of the given relation0. 14 | - **f1**: File name of the given relation1. 15 | - **offset0**: Offsets of which columns the given relation0 should be joined. 16 | - **offset1**: Offsets of which columns the given relation1 should be joined. 17 | - **sum** (output argument): Sum of the relation0.col0 in the final join result. 18 | 19 | The (equality) join predicates are specified by the `offset0/1`. The form of the join predicates is like: 20 | ``` go 21 | relation0.cols[offset[0]] = relation1.cols[offset[0]] and relation0.cols[offset[1]] = relation1.cols[offset[1]]... 22 | ``` 23 | 24 | **Example**: `Join("/path/T0", "/path/T1", []int{0, 1}, []int{2, 3})` 25 | 26 | Translated to SQL: 27 | 28 | ``` sql 29 | SELECT SUM(T0.COL0) 30 | FROM T0, T1 31 | ON T0.COL0=T1.COL2 AND T0.COL1=T1.COL3 32 | ``` 33 | 34 | We provide a sample as `join_example.go: JoinExample` which performs a simple hash join algorithm. It uses the relation0 to build the hash table, and probe the hash table for every row in relation1. 35 | 36 | ## Requirements and rating principles 37 | 38 | - (30%) Pass all test cases. 39 | - (20%) Perform better than `join_example.go:JoinExample`. 40 | - (35%) Have a document to describe your idea and record the process of performance optimization with `pprof`. 41 | - (15%) Keep a good code style. 42 | 43 | Note: 44 | 1. For your check sums, you do not have to worry about numeric overflows as long as you are using 64 bit unsigned integers. 45 | 2. More large datasets are provided [here](https://drive.google.com/drive/u/1/folders/10-iJNGKmKXgMmvBYnKt88RTwC0iA1XM-), you can use them to help profile your program. 46 | 3. We'll use the `BenchmarkJoin` and `BenchmarkJoinExample` which can be found in `benchmark_test.go` to evaluate your program. Test data will **NOT** be outside of what we've provided. 47 | 48 | ## How to use 49 | 50 | 1. Please implement your own `Join` in join.go to accomplish this task. 51 | 2. We provide CSV versions (.tbl files) of three relations in directory `t`. You can load them into a DBMS to test your program. 52 | 1. **r0.tbl**: 2 columns * 10,000 records 53 | 2. **r1.tbl**: 4 columns * 5,000 records 54 | 3. **r2.tbl**: 4 columns * 500 records 55 | 3. There is already a built-in unit test `JoinTest` defined in `join_test.go`. You can write your own unit tests, but please make sure `JoinTest` can be passed before. 56 | 4. Use `make test` to run all the unit tests. 57 | 5. Use `make bench` to run all the benchmarks. 58 | -------------------------------------------------------------------------------- /courses/dss/raft/src/kvraft/server.rs: -------------------------------------------------------------------------------- 1 | use futures::channel::mpsc::unbounded; 2 | 3 | use crate::proto::kvraftpb::*; 4 | use crate::raft; 5 | 6 | pub struct KvServer { 7 | pub rf: raft::Node, 8 | me: usize, 9 | // snapshot if log grows this big 10 | maxraftstate: Option, 11 | // Your definitions here. 12 | } 13 | 14 | impl KvServer { 15 | pub fn new( 16 | servers: Vec, 17 | me: usize, 18 | persister: Box, 19 | maxraftstate: Option, 20 | ) -> KvServer { 21 | // You may need initialization code here. 22 | 23 | let (tx, apply_ch) = unbounded(); 24 | let rf = raft::Raft::new(servers, me, persister, tx); 25 | 26 | crate::your_code_here((rf, maxraftstate, apply_ch)) 27 | } 28 | } 29 | 30 | impl KvServer { 31 | /// Only for suppressing deadcode warnings. 32 | #[doc(hidden)] 33 | pub fn __suppress_deadcode(&mut self) { 34 | let _ = &self.me; 35 | let _ = &self.maxraftstate; 36 | } 37 | } 38 | 39 | // Choose concurrency paradigm. 40 | // 41 | // You can either drive the kv server by the rpc framework, 42 | // 43 | // ```rust 44 | // struct Node { server: Arc> } 45 | // ``` 46 | // 47 | // or spawn a new thread runs the kv server and communicate via 48 | // a channel. 49 | // 50 | // ```rust 51 | // struct Node { sender: Sender } 52 | // ``` 53 | #[derive(Clone)] 54 | pub struct Node { 55 | // Your definitions here. 56 | } 57 | 58 | impl Node { 59 | pub fn new(kv: KvServer) -> Node { 60 | // Your code here. 61 | crate::your_code_here(kv); 62 | } 63 | 64 | /// the tester calls kill() when a KVServer instance won't 65 | /// be needed again. you are not required to do anything 66 | /// in kill(), but it might be convenient to (for example) 67 | /// turn off debug output from this instance. 68 | pub fn kill(&self) { 69 | // If you want to free some resources by `raft::Node::kill` method, 70 | // you should call `raft::Node::kill` here also to prevent resource leaking. 71 | // Since the test framework will call kvraft::Node::kill only. 72 | // self.server.kill(); 73 | 74 | // Your code here, if desired. 75 | } 76 | 77 | /// The current term of this peer. 78 | pub fn term(&self) -> u64 { 79 | self.get_state().term() 80 | } 81 | 82 | /// Whether this peer believes it is the leader. 83 | pub fn is_leader(&self) -> bool { 84 | self.get_state().is_leader() 85 | } 86 | 87 | pub fn get_state(&self) -> raft::State { 88 | // Your code here. 89 | raft::State { 90 | ..Default::default() 91 | } 92 | } 93 | } 94 | 95 | #[async_trait::async_trait] 96 | impl KvService for Node { 97 | // CAVEATS: Please avoid locking or sleeping here, it may jam the network. 98 | async fn get(&self, arg: GetRequest) -> labrpc::Result { 99 | // Your code here. 100 | crate::your_code_here(arg) 101 | } 102 | 103 | // CAVEATS: Please avoid locking or sleeping here, it may jam the network. 104 | async fn put_append(&self, arg: PutAppendRequest) -> labrpc::Result { 105 | // Your code here. 106 | crate::your_code_here(arg) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/bin/kvs-server.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate log; 3 | #[macro_use] 4 | extern crate clap; 5 | 6 | use kvs::thread_pool::*; 7 | use kvs::{KvStore, KvsEngine, KvsServer, Result, SledKvsEngine}; 8 | use log::LevelFilter; 9 | use std::env; 10 | use std::env::current_dir; 11 | use std::fs; 12 | use std::net::SocketAddr; 13 | use std::process::exit; 14 | use structopt::StructOpt; 15 | 16 | const DEFAULT_LISTENING_ADDRESS: &str = "127.0.0.1:4000"; 17 | const DEFAULT_ENGINE: Engine = Engine::kvs; 18 | 19 | #[derive(StructOpt, Debug)] 20 | #[structopt(name = "kvs-server")] 21 | struct Opt { 22 | #[structopt( 23 | long, 24 | help = "Sets the listening address", 25 | value_name = "IP:PORT", 26 | raw(default_value = "DEFAULT_LISTENING_ADDRESS"), 27 | parse(try_from_str) 28 | )] 29 | addr: SocketAddr, 30 | #[structopt( 31 | long, 32 | help = "Sets the storage engine", 33 | value_name = "ENGINE-NAME", 34 | raw(possible_values = "&Engine::variants()") 35 | )] 36 | engine: Option, 37 | } 38 | 39 | arg_enum! { 40 | #[allow(non_camel_case_types)] 41 | #[derive(Debug, Copy, Clone, PartialEq, Eq)] 42 | enum Engine { 43 | kvs, 44 | sled 45 | } 46 | } 47 | 48 | fn main() { 49 | env_logger::builder().filter_level(LevelFilter::Info).init(); 50 | let mut opt = Opt::from_args(); 51 | let res = current_engine().and_then(move |curr_engine| { 52 | if opt.engine.is_none() { 53 | opt.engine = curr_engine; 54 | } 55 | if curr_engine.is_some() && opt.engine != curr_engine { 56 | error!("Wrong engine!"); 57 | exit(1); 58 | } 59 | run(opt) 60 | }); 61 | if let Err(e) = res { 62 | error!("{}", e); 63 | exit(1); 64 | } 65 | } 66 | 67 | fn run(opt: Opt) -> Result<()> { 68 | let engine = opt.engine.unwrap_or(DEFAULT_ENGINE); 69 | info!("kvs-server {}", env!("CARGO_PKG_VERSION")); 70 | info!("Storage engine: {}", engine); 71 | info!("Listening on {}", opt.addr); 72 | 73 | // write engine to engine file 74 | fs::write(current_dir()?.join("engine"), format!("{}", engine))?; 75 | 76 | let concurrency = num_cpus::get() as u32; 77 | match engine { 78 | Engine::kvs => run_with( 79 | KvStore::::open(env::current_dir()?, concurrency)?, 80 | opt.addr, 81 | ), 82 | Engine::sled => run_with( 83 | SledKvsEngine::::new( 84 | sled::Db::start_default(env::current_dir()?)?, 85 | concurrency, 86 | )?, 87 | opt.addr, 88 | ), 89 | } 90 | } 91 | 92 | pub fn run_with(engine: E, addr: SocketAddr) -> Result<()> { 93 | let server = KvsServer::new(engine); 94 | server.run(addr) 95 | } 96 | 97 | fn current_engine() -> Result> { 98 | let engine = current_dir()?.join("engine"); 99 | if !engine.exists() { 100 | return Ok(None); 101 | } 102 | 103 | match fs::read_to_string(engine)?.parse() { 104 | Ok(engine) => Ok(Some(engine)), 105 | Err(e) => { 106 | warn!("The content of engine file is invalid: {}", e); 107 | Ok(None) 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /talent-challenge-program2021/selected-projects.md: -------------------------------------------------------------------------------- 1 | ## Selected Projects in Season 1 2 | 3 | Selected Projects in Sprint 1 are listed below. Mentee candidates who want to participate in a certain project please refer to [Mentee Selection Process](README.md#mentees) 4 | 5 | The difficulty of the project is divided into three levels: Very hard, Hard, and Medium, the corresponding pre-tax bonus is RMB 10000 to a very hard project, RMB 8000 to a hard project and RMB 5000 to a medium project respectively. 6 | 7 | ### Template 8 | 9 | ``` 10 | 11 | ## Description 12 | 13 | what's the problem, how to solve it 14 | 15 | ## Document Collection 16 | 17 | - Proposal doc: ${proposal doc} 18 | - Weekly report: ${weekly report link} 19 | 20 | ## Talent Challenge Program information 21 | 22 | - Mentor of this issue: @github_id 23 | - Recommended skills: ${skills and URL link} 24 | - Estimated Workloads: (XL, man-month) 25 | 26 | ## Milestones and action items 27 | 28 | Milestone 1: ${milestone descption}, Expected finish date: ${date} 29 | 30 | - [ ] action item 1 31 | - [ ] action item 2 32 | 33 | Milestone 2: ${milestone descption}, Expected finish date: ${date} 34 | 35 | - [ ] action item 1 36 | - [ ] action item 2 37 | 38 | ``` 39 | 40 | ## List of Selected Projects 41 | 42 | #### SQL 43 | 44 | * [tidb/issues/25149](https://github.com/pingcap/tidb/issues/25149) Optimize CTE performance, [@wjhuang2016](https://github.com/wjhuang2016) 45 | * [tidb/issues/14486](https://github.com/pingcap/tidb/issues/14486) (Occupied)Implement missing JSON functions, [@nullnotnil](https://github.com/nullnotnil) 46 | * [tidb/issues/25152](https://github.com/pingcap/tidb/issues/25152) GBK charset, [@xiongjiwei](https://github.com/xiongjiwei) 47 | * [tidb/issues/25150](https://github.com/pingcap/tidb/issues/25150) (Occupied)Expression index GA, [@wjhuang2016](https://github.com/wjhuang2016) 48 | * [tidb/issues/25204](https://github.com/pingcap/tidb/issues/25204) Local temporary table, [@djshow832](https://github.com/djshow832) 49 | * [tidb/issues/14580](https://github.com/pingcap/tidb/issues/14580) Only full group by check，needed things(relational algebra, graph theory), [@winoros](https://github.com/winoros) 50 | * [tidb/issues/25293](https://github.com/pingcap/tidb/issues/25293) (Occupied)Optimizing hotspot small tables, [@tiancaiamao](https://github.com/tiancaiamao) 51 | * [tidb/issues/13709](https://github.com/pingcap/tidb/issues/13709) (Occupied)Cascades planner,[@chrysanxia](https://github.com/chrysanxia) 52 | * [tidb/issues/18745](https://github.com/pingcap/tidb/issues/18745) (Occupied)Support to export historical statistics,[@qw4990](https://github.com/qw4990) 53 | 54 | #### Storage 55 | 56 | * [tidb/issues/25281](https://github.com/pingcap/tidb/issues/25281) (Occupied)Historical Hotspot Region, [@nolouch](https://github.com/nolouch) 57 | * [pd/issues/4095](https://github.com/tikv/pd/issues/4095)(Occupied)Enhance Split Table Stability and Usability,[@Yisaer](https://github.com/Yisaer) 58 | * [pd/issues/3736](https://github.com/tikv/pd/issues/3736) Build scheduling model with Simulink, [@bufferflies](https://github.com/bufferflies) 59 | * [tikv/issues/10867](https://github.com/tikv/tikv/issues/10867) (Occupied)Support limiting read flow for TiKV IO rate limiter,[@Connor1996](https://github.com/connor1996) 60 | 61 | #### Talent Plan 62 | 63 | * [tidb/issues/25321](https://github.com/pingcap/tidb/issues/25321)（Occupied） TinySQL: design and implement a learning-friendly distributed database [@rebelice](https://github.com/rebelice) 64 | -------------------------------------------------------------------------------- /courses/tp102-how-to-use-git-github.md: -------------------------------------------------------------------------------- 1 | # TP 102: How to Use Git and GitHub 2 | 3 | Git and GitHub are commonly used tools in open source practice. Whether you are considering contribute to an open source project or start your own open source project. This course includes a collection of Git and GitHub related learning materials and will guide you through everything you need to start contributing, from managing notifications to merging pull requests. 4 | 5 | ***Important note:*** ***How to Use Git and GitHub is in an alpha 6 | state.*** Its scope is limited. If you are taking it now you 7 | are brave, but you are also an early tester and your feedback is greatly 8 | appreciated. You are encouraged to reporting bugs, suggest changes, update the contents, etc. **See [Contributing Guide](../CONTRIBUTING.md) for more details.** 9 | 10 | 11 | ## Who this is for 12 | 13 | New GitHub users, users new to Git 14 | 15 | ## What you will learn? 16 | 17 | * what GitHub is 18 | * how to use GitHub 19 | * how to create a branch and a commit 20 | * how to review a pull request 21 | * how to manage merge conflicts 22 | * how to communicate using Markdown 23 | 24 | ### Prerequisites: 25 | 26 | None. This course is a great introduction for your first day on GitHub. 27 | 28 | ## Course details 29 | 30 | ### Chapter 1: Introduction to GitHub 31 | 32 | **Learning Material(s):** 33 | 34 | * [Introduction to GitHub](https://lab.github.com/githubtraining/introduction-to-github), an open source course created by [The GitHub Training Team](https://lab.github.com/githubtraining) 35 | 36 | **Suggested Assignments:** 37 | 38 | * Finish the lab. 39 | 40 | ### Chapter 2: Reviewing pull requests 41 | 42 | **Learning Material(s):** 43 | 44 | * [Reviewing pull requests](https://lab.github.com/githubtraining/reviewing-pull-requests), an open source course created by [The GitHub Training Team](https://lab.github.com/githubtraining) 45 | 46 | **Suggested Assignments:** 47 | 48 | * Finish the lab. 49 | 50 | ### Chapter 3: Managing merge conflicts 51 | 52 | **Learning Material(s):** 53 | 54 | * [Managing merge conflicts](https://lab.github.com/githubtraining/managing-merge-conflicts), an open source course created by [The GitHub Training Team](https://lab.github.com/githubtraining) 55 | 56 | **Suggested Assignments:** 57 | 58 | * Finish the lab. 59 | 60 | ### Chapter 4: Communicating using Markdown 61 | 62 | **Learning Material(s):** 63 | 64 | * [Communicating using Markdown](https://lab.github.com/githubtraining/communicating-using-markdown), an open source course created by [The GitHub Training Team](https://lab.github.com/githubtraining) 65 | 66 | **Suggested Assignments:** 67 | 68 | * Finish the lab. 69 | 70 | ### Chapter 5: Git Handbook 71 | 72 | * [Git Handbook](https://guides.github.com/introduction/git-handbook/), the official learning material provided by [GitHub Guides](https://guides.github.com/) 73 | 74 | **Suggested Assignments:** 75 | 76 | No assignments, the git skill will be tested all the time you participate in an open source project. 77 | 78 | ## Users who took this course also took 79 | 80 | * [TP 101: Introduction to Open Source Software](tp101-intro-to-oss.md) 81 | * [TP 103: Build A Welcoming Open Source Community](tp103-open-source-community.md) 82 | 83 | 84 | ## Acknowledgement 85 | 86 | Special Thanks to: 87 | 88 | * `Producer Wen`, `Qinyao Yang`, and `Jason Zhang`for testing this course and updating the materials. 89 | * [The GitHub Training Team](https://lab.github.com/githubtraining) for creating open source courses above. 90 | -------------------------------------------------------------------------------- /courses/dss/percolator/src/server.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::sync::{Arc, Mutex}; 3 | use std::time::Duration; 4 | 5 | use crate::msg::*; 6 | use crate::service::*; 7 | use crate::*; 8 | 9 | // TTL is used for a lock key. 10 | // If the key's lifetime exceeds this value, it should be cleaned up. 11 | // Otherwise, the operation should back off. 12 | const TTL: u64 = Duration::from_millis(100).as_nanos() as u64; 13 | 14 | #[derive(Clone, Default)] 15 | pub struct TimestampOracle { 16 | // You definitions here if needed. 17 | } 18 | 19 | #[async_trait::async_trait] 20 | impl timestamp::Service for TimestampOracle { 21 | // example get_timestamp RPC handler. 22 | async fn get_timestamp(&self, _: TimestampRequest) -> labrpc::Result { 23 | // Your code here. 24 | unimplemented!() 25 | } 26 | } 27 | 28 | // Key is a tuple (raw key, timestamp). 29 | pub type Key = (Vec, u64); 30 | 31 | #[derive(Clone, PartialEq)] 32 | pub enum Value { 33 | Timestamp(u64), 34 | Vector(Vec), 35 | } 36 | 37 | #[derive(Debug, Clone)] 38 | pub struct Write(Vec, Vec); 39 | 40 | pub enum Column { 41 | Write, 42 | Data, 43 | Lock, 44 | } 45 | 46 | // KvTable is used to simulate Google's Bigtable. 47 | // It provides three columns: Write, Data, and Lock. 48 | #[derive(Clone, Default)] 49 | pub struct KvTable { 50 | write: BTreeMap, 51 | data: BTreeMap, 52 | lock: BTreeMap, 53 | } 54 | 55 | impl KvTable { 56 | // Reads the latest key-value record from a specified column 57 | // in MemoryStorage with a given key and a timestamp range. 58 | #[inline] 59 | fn read( 60 | &self, 61 | key: Vec, 62 | column: Column, 63 | ts_start_inclusive: Option, 64 | ts_end_inclusive: Option, 65 | ) -> Option<(&Key, &Value)> { 66 | // Your code here. 67 | unimplemented!() 68 | } 69 | 70 | // Writes a record to a specified column in MemoryStorage. 71 | #[inline] 72 | fn write(&mut self, key: Vec, column: Column, ts: u64, value: Value) { 73 | // Your code here. 74 | unimplemented!() 75 | } 76 | 77 | #[inline] 78 | // Erases a record from a specified column in MemoryStorage. 79 | fn erase(&mut self, key: Vec, column: Column, commit_ts: u64) { 80 | // Your code here. 81 | unimplemented!() 82 | } 83 | } 84 | 85 | // MemoryStorage is used to wrap a KvTable. 86 | // You may need to get a snapshot from it. 87 | #[derive(Clone, Default)] 88 | pub struct MemoryStorage { 89 | data: Arc>, 90 | } 91 | 92 | #[async_trait::async_trait] 93 | impl transaction::Service for MemoryStorage { 94 | // example get RPC handler. 95 | async fn get(&self, req: GetRequest) -> labrpc::Result { 96 | // Your code here. 97 | unimplemented!() 98 | } 99 | 100 | // example prewrite RPC handler. 101 | async fn prewrite(&self, req: PrewriteRequest) -> labrpc::Result { 102 | // Your code here. 103 | unimplemented!() 104 | } 105 | 106 | // example commit RPC handler. 107 | async fn commit(&self, req: CommitRequest) -> labrpc::Result { 108 | // Your code here. 109 | unimplemented!() 110 | } 111 | } 112 | 113 | impl MemoryStorage { 114 | fn back_off_maybe_clean_up_lock(&self, start_ts: u64, key: Vec) { 115 | // Your code here. 116 | unimplemented!() 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at [ts-team@pingcap.com](mailto:ts-team@pingcap.com). All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /courses/dss/raft/src/raft/persister.rs: -------------------------------------------------------------------------------- 1 | //! Support for Raft and kvraft to save persistent 2 | //! Raft state (log &c) and k/v server snapshots. 3 | //! 4 | //! we will use the original persister.rs to test your code for grading. 5 | //! so, while you can modify this code to help you debug, please 6 | //! test with the original before submitting. 7 | 8 | use std::sync::{Arc, Mutex}; 9 | 10 | pub trait Persister: Send + 'static { 11 | fn raft_state(&self) -> Vec; 12 | fn save_raft_state(&self, state: Vec); 13 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec); 14 | fn snapshot(&self) -> Vec; 15 | } 16 | 17 | impl Persister for Box { 18 | fn raft_state(&self) -> Vec { 19 | (**self).raft_state() 20 | } 21 | fn save_raft_state(&self, state: Vec) { 22 | (**self).save_raft_state(state) 23 | } 24 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec) { 25 | (**self).save_state_and_snapshot(state, snapshot) 26 | } 27 | fn snapshot(&self) -> Vec { 28 | (**self).snapshot() 29 | } 30 | } 31 | 32 | impl Persister for Arc { 33 | fn raft_state(&self) -> Vec { 34 | (**self).raft_state() 35 | } 36 | fn save_raft_state(&self, state: Vec) { 37 | (**self).save_raft_state(state) 38 | } 39 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec) { 40 | (**self).save_state_and_snapshot(state, snapshot) 41 | } 42 | fn snapshot(&self) -> Vec { 43 | (**self).snapshot() 44 | } 45 | } 46 | 47 | #[derive(Default)] 48 | pub struct SimplePersister { 49 | states: Mutex<( 50 | Vec, // raft state 51 | Vec, // snapshot 52 | )>, 53 | } 54 | 55 | impl SimplePersister { 56 | pub fn new() -> SimplePersister { 57 | SimplePersister { 58 | states: Mutex::default(), 59 | } 60 | } 61 | } 62 | 63 | impl Persister for SimplePersister { 64 | fn raft_state(&self) -> Vec { 65 | self.states.lock().unwrap().0.clone() 66 | } 67 | 68 | fn save_raft_state(&self, state: Vec) { 69 | self.states.lock().unwrap().0 = state; 70 | } 71 | 72 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec) { 73 | self.states.lock().unwrap().0 = state; 74 | self.states.lock().unwrap().1 = snapshot; 75 | } 76 | 77 | fn snapshot(&self) -> Vec { 78 | self.states.lock().unwrap().1.clone() 79 | } 80 | } 81 | 82 | #[cfg(test)] 83 | mod tests { 84 | use super::*; 85 | 86 | #[test] 87 | fn test_object_safety() { 88 | let sp = SimplePersister::new(); 89 | sp.save_raft_state(vec![111]); 90 | let obj: Box = Box::new(sp); 91 | assert_eq!(obj.raft_state(), vec![111]); 92 | obj.save_state_and_snapshot(vec![222], vec![123]); 93 | assert_eq!(obj.raft_state(), vec![222]); 94 | assert_eq!(obj.snapshot(), vec![123]); 95 | 96 | let cloneable_obj: Arc = Arc::new(obj); 97 | assert_eq!(cloneable_obj.raft_state(), vec![222]); 98 | assert_eq!(cloneable_obj.snapshot(), vec![123]); 99 | 100 | let cloneable_obj_ = cloneable_obj.clone(); 101 | cloneable_obj.save_raft_state(vec![233]); 102 | assert_eq!(cloneable_obj_.raft_state(), vec![233]); 103 | assert_eq!(cloneable_obj_.snapshot(), vec![123]); 104 | 105 | let sp = SimplePersister::new(); 106 | let obj: Arc = Arc::new(sp); 107 | let _box_obj: Box = Box::new(obj); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /courses/rust/projects/project-5/src/client.rs: -------------------------------------------------------------------------------- 1 | use crate::common::{Request, Response}; 2 | use crate::KvsError; 3 | use std::net::SocketAddr; 4 | use tokio::codec::{FramedRead, FramedWrite, LengthDelimitedCodec}; 5 | use tokio::io::{ReadHalf, WriteHalf}; 6 | use tokio::net::TcpStream; 7 | use tokio::prelude::*; 8 | use tokio_serde_json::{ReadJson, WriteJson}; 9 | 10 | /// Key value store client 11 | pub struct KvsClient { 12 | read_json: ReadJson, LengthDelimitedCodec>, Response>, 13 | write_json: WriteJson, LengthDelimitedCodec>, Request>, 14 | } 15 | 16 | impl KvsClient { 17 | /// Connect to `addr` to access `KvsServer`. 18 | pub fn connect(addr: SocketAddr) -> impl Future { 19 | TcpStream::connect(&addr) 20 | .map(|tcp| { 21 | let (read_half, write_half) = tcp.split(); 22 | let read_json = 23 | ReadJson::new(FramedRead::new(read_half, LengthDelimitedCodec::new())); 24 | let write_json = 25 | WriteJson::new(FramedWrite::new(write_half, LengthDelimitedCodec::new())); 26 | KvsClient { 27 | read_json, 28 | write_json, 29 | } 30 | }) 31 | .map_err(|e| e.into()) 32 | } 33 | 34 | /// Get the value of a given key from the server. 35 | pub fn get(self, key: String) -> impl Future, Self), Error = KvsError> { 36 | self.send_request(Request::Get { key }) 37 | .and_then(move |(resp, client)| match resp { 38 | Some(Response::Get(value)) => Ok((value, client)), 39 | Some(Response::Err(msg)) => Err(KvsError::StringError(msg)), 40 | Some(_) => Err(KvsError::StringError("Invalid response".to_owned())), 41 | None => Err(KvsError::StringError("No response received".to_owned())), 42 | }) 43 | } 44 | 45 | /// Set the value of a string key in the server. 46 | pub fn set(self, key: String, value: String) -> impl Future { 47 | self.send_request(Request::Set { key, value }) 48 | .and_then(move |(resp, client)| match resp { 49 | Some(Response::Set) => Ok(client), 50 | Some(Response::Err(msg)) => Err(KvsError::StringError(msg)), 51 | Some(_) => Err(KvsError::StringError("Invalid response".to_owned())), 52 | None => Err(KvsError::StringError("No response received".to_owned())), 53 | }) 54 | } 55 | 56 | /// Remove a string key in the server. 57 | pub fn remove(self, key: String) -> impl Future { 58 | self.send_request(Request::Remove { key }) 59 | .and_then(move |(resp, client)| match resp { 60 | Some(Response::Remove) => Ok(client), 61 | Some(Response::Err(msg)) => Err(KvsError::StringError(msg)), 62 | Some(_) => Err(KvsError::StringError("Invalid response".to_owned())), 63 | None => Err(KvsError::StringError("No response received".to_owned())), 64 | }) 65 | } 66 | 67 | fn send_request( 68 | self, 69 | req: Request, 70 | ) -> impl Future, Self), Error = KvsError> { 71 | let read_json = self.read_json; 72 | self.write_json 73 | .send(req) 74 | .and_then(move |write_json| { 75 | read_json 76 | .into_future() 77 | .map(move |(resp, read_json)| { 78 | let client = KvsClient { 79 | read_json, 80 | write_json, 81 | }; 82 | (resp, client) 83 | }) 84 | .map_err(|(err, _)| err) 85 | }) 86 | .map_err(|e| e.into()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /courses/dss/labrpc/src/server.rs: -------------------------------------------------------------------------------- 1 | use std::collections::hash_map::{Entry, HashMap}; 2 | use std::fmt; 3 | use std::sync::atomic::{AtomicUsize, Ordering}; 4 | use std::sync::Arc; 5 | 6 | use futures::future::{self, BoxFuture}; 7 | 8 | use crate::error::{Error, Result}; 9 | 10 | static ID_ALLOC: AtomicUsize = AtomicUsize::new(0); 11 | 12 | pub type RpcFuture = BoxFuture<'static, T>; 13 | 14 | pub type Handler = dyn FnOnce(&[u8]) -> RpcFuture>>; 15 | 16 | pub trait HandlerFactory: Sync + Send + 'static { 17 | fn handler(&self, name: &'static str) -> Box; 18 | } 19 | 20 | pub struct ServerBuilder { 21 | name: String, 22 | // Service name -> service methods 23 | pub(crate) services: HashMap<&'static str, Box>, 24 | } 25 | 26 | impl ServerBuilder { 27 | pub fn new(name: String) -> ServerBuilder { 28 | ServerBuilder { 29 | name, 30 | services: HashMap::new(), 31 | } 32 | } 33 | 34 | pub fn add_service( 35 | &mut self, 36 | service_name: &'static str, 37 | factory: Box, 38 | ) -> Result<()> { 39 | match self.services.entry(service_name) { 40 | Entry::Occupied(_) => Err(Error::Other(format!( 41 | "{} has already registered", 42 | service_name 43 | ))), 44 | Entry::Vacant(entry) => { 45 | entry.insert(factory); 46 | Ok(()) 47 | } 48 | } 49 | } 50 | 51 | pub fn build(self) -> Server { 52 | Server { 53 | core: Arc::new(ServerCore { 54 | name: self.name, 55 | services: self.services, 56 | id: ID_ALLOC.fetch_add(1, Ordering::Relaxed), 57 | count: AtomicUsize::new(0), 58 | }), 59 | } 60 | } 61 | } 62 | 63 | pub(crate) struct ServerCore { 64 | pub(crate) name: String, 65 | pub(crate) id: usize, 66 | 67 | pub(crate) services: HashMap<&'static str, Box>, 68 | pub(crate) count: AtomicUsize, 69 | } 70 | 71 | #[derive(Clone)] 72 | pub struct Server { 73 | pub(crate) core: Arc, 74 | } 75 | 76 | impl Server { 77 | pub fn count(&self) -> usize { 78 | self.core.count.load(Ordering::Relaxed) 79 | } 80 | 81 | pub fn name(&self) -> &str { 82 | &self.core.name 83 | } 84 | 85 | pub(crate) fn dispatch(&self, fq_name: &'static str, req: &[u8]) -> RpcFuture>> { 86 | self.core.count.fetch_add(1, Ordering::Relaxed); 87 | let mut names = fq_name.split('.'); 88 | let service_name = match names.next() { 89 | Some(n) => n, 90 | None => { 91 | return Box::pin(future::err(Error::Unimplemented(format!( 92 | "unknown {}", 93 | fq_name 94 | )))); 95 | } 96 | }; 97 | let method_name = match names.next() { 98 | Some(n) => n, 99 | None => { 100 | return Box::pin(future::err(Error::Unimplemented(format!( 101 | "unknown {}", 102 | fq_name 103 | )))); 104 | } 105 | }; 106 | if let Some(factory) = self.core.services.get(service_name) { 107 | let handle = factory.handler(method_name); 108 | handle(req) 109 | } else { 110 | Box::pin(future::err(Error::Unimplemented(format!( 111 | "unknown {}", 112 | fq_name 113 | )))) 114 | } 115 | } 116 | } 117 | 118 | impl fmt::Debug for Server { 119 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 120 | f.debug_struct("Server") 121 | .field("name", &self.core.name) 122 | .field("id", &self.core.id) 123 | .finish() 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Welcome to learn Talent Plan Courses! 2 | 3 | ![Talent Plan Logo](media/talent-plan-logo.png) 4 | 5 | Talent Plan is an open source training program initiated by PingCAP. It aims to create or combine some open source learning materials for people interested in open source, distributed systems, Rust, Golang, and other infrastructure knowledge. As such, it provides a series of courses focused on open source collaboration, rust programming, distributed database and systems. 6 | 7 | > Note: 8 | 9 | > Each course is developed by different teams, so they may vary in their organization and learning outcomes. Please see the individual course documentation for details. 10 | 11 | ## Our Courses 12 | 13 | ### Series 1: Open Source Collaboration 14 | 15 | Open source collaboration includes a series of open-source related learning materials to help enthusiasts gain basic knowledge of what open source software is, the differences between existing open-source software licenses, how to participate in open source projects, and what a welcoming open source community looks like. 16 | 17 | Courses include: 18 | 19 | - [TP 101: Introduction to open source software](courses/tp101-intro-to-oss.md) 20 | - [TP 102: How to use Git and GitHub](courses/tp102-how-to-use-git-github.md) 21 | - [TP 103: Build a welcoming community](courses/tp103-open-source-community.md) 22 | 23 | ### Series 2: Rust Programming 24 | 25 | This series is core to TALENT-PLAN. It builds your understanding of Rust as a programming language and provides opportunities for you to practice with it. 26 | 27 | Courses include: 28 | 29 | - [TP 201: Practical Networked Applications in Rust](courses/rust/README.md). A series of projects that incrementally develop a single Rust project from the ground up into a high-performance, networked, parallel and asynchronous key/value store. Along the way various real-world Rust development subject matter are explored and discussed. 30 | 31 | - [TP 202: Distributed Systems in Rust](courses/dss/README.md). Adapted from the [MIT 6.824](http://nil.csail.mit.edu/6.824/2017/index.html) distributed systems coursework, this course focuses on implementing important distributed algorithms, including the [Raft](https://raft.github.io/) consensus algorithm, and the [Percolator](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/36726.pdf) distributed transaction protocol. 32 | 33 | ### Series 3: Distributed Database 34 | 35 | This series provides information on TinySQL and TinyKV, which are distributed databases in Go. 36 | 37 | Courses include: 38 | 39 | - [TP 301: TinySQL, a distributed relational database in Go](https://github.com/pingcap-incubator/tinysql) 40 | - [TP 302: TinyKV, a distributed key value database in Go](https://github.com/pingcap-incubator/tinykv) 41 | 42 | ### Series 4: Deep Dive into TiDB Ecosystems 43 | 44 | This series provides information on TiDB and TiKV, which are distributed databases developed by PingCAP. 45 | 46 | Courses include: 47 | 48 | - TP 401: Deep Dive into TiDB(WIP) 49 | - TP 402: Deep Dive into TiKV(WIP) 50 | 51 | 52 | See [Courses](courses/README.md) for more details. 53 | 54 | # Contributing to talent plan 55 | 56 | Contributions of any kind are welcome! Check out the [Contributing Guide](CONTRIBUTING.md) in this repository for more information on how you can contribute to Talent Plan. 57 | 58 | We love our community and take great care to ensure it is fun, safe and rewarding. Please review our [Code of Conduct](/CODE_OF_CONDUCT.md) for community expectations and guidelines for reporting concerns. 59 | 60 | 61 | ## We're here to help 62 | 63 | If you have questions about building (or taking) courses, you can ask in the channel **#wg-talent-plan-courses** of the [tidbcommunity](https://tidbcommunity.slack.com/join/shared_invite/enQtNzc0MzI4ODExMDc4LWYwYmIzMjZkYzJiNDUxMmZlN2FiMGJkZjAyMzQ5NGU0NGY0NzI3NTYwMjAyNGQ1N2I2ZjAxNzc1OGUwYWM0NzE#/shared-invite/email) slack workspace. 64 | 65 | ## License 66 | 67 | These courses may be freely used and modified for any purpose, under the terms of each course's individual license. See the courses for details. 68 | -------------------------------------------------------------------------------- /courses/rust/building-blocks/bb-4.md: -------------------------------------------------------------------------------- 1 | # PNA Rust — Building Blocks 4 2 | 3 | Let's learn some building blocks! 4 | 5 | Put your other projects and concerns aside. Take a breath and relax. Here 6 | are some fun resources for you to explore. 7 | 8 | Read all the readings and perform all the exercises. Also watch the video. 9 | 10 | - **[Reading: Fearless Concurrency with Rust][f]**. This is a classic Rust blog 11 | post from [Aaron Turon][at] that clearly explains why concurrency is so easy 12 | in Rust. The title is also the origin of using the word "fearless" to describe 13 | various Rust properties. 14 | 15 | - **[Reading: What is the difference between concurrency and parallelism?][d]**. 16 | This is a 10 second read, but something to keep in mind. The two words are 17 | often used interchangably. We'll mostly use the word "concurrent" since it is 18 | more general than "parallel". Sometimes we'll use the word "parallel" to be 19 | more specific, sometimes because it sounds better… 20 | 21 | - **[Reading: Rust: A unique perspective][ru]**. An explanation of the dangers 22 | of mutable aliasing and how Rust solves the problem. This one is by [Matt 23 | Brubeck][mb], from the [Servo] team. 24 | 25 | - **[Video: Rust Concurrency Explained][ex]**. A more in-depth talk by [Alex 26 | Crichton][ac]. Aaron and Alex wrote many of the concurrent data structures in 27 | the standard library. Alex has given versions of this talk for years, and it 28 | is a pretty great overview of what Rust can do. 29 | 30 | - **[Reading: `std::sync`][ss]**. Once again, the standard library documentation 31 | provides not only good documentation about the library, but about the subject 32 | in general. This provides an overview of most of the concurrent types provided 33 | by the standard library. 34 | 35 | - **[Exercise: Basic multithreading][bmt]**. This is a simple multithreading 36 | exercise from the [rustlings] project. It is small enough that it can be 37 | completed on [play.rust-lang.org]. 38 | 39 | - **Exercise: Write a thread pool**. 40 | 41 | A [thread pool] runs jobs (functions) on a set of reusable threads, which can 42 | be more efficient than spawning a new thread for every job. 43 | 44 | Create a simple thread pool with the following type signature: 45 | 46 | ```rust 47 | impl ThreadPool { 48 | fn new(threads: u32) -> Result; 49 | 50 | fn spawn(&self, job: F) where F: FnOnce() + Send + 'static; 51 | } 52 | ``` 53 | 54 | The `new` function should immediately spawn `threads` number of threads, and 55 | then those threads will wait for jobs to be spawned. When a thread recieves a 56 | job, it runs it to completion, then waits for the next job. 57 | 58 | The [`threadpool`][tp1] crate and Rayon's [`ThreadPool`][tp2] may provide inspiration. 59 | 60 | - **[Reading: Lock-free vs wait-free concurrency][lf]**. It seems like everybody 61 | wants their code to be "lock-free". What's that mean? 62 | 63 | 76 | 77 | [lf]: https://rethinkdb.com/blog/lock-free-vs-wait-free-concurrency/ 78 | [play.rust-lang.org]: https://play.rust-lang.org/ 79 | [tp1]: https://docs.rs/threadpool/1.7.1/threadpool/struct.ThreadPool.html 80 | [tp2]: https://docs.rs/rayon/1.0.3/rayon/struct.ThreadPool.html 81 | [thread pool]: https://softwareengineering.stackexchange.com/questions/173575/what-is-a-thread-pool#173581 82 | [ss]: https://doc.rust-lang.org/std/sync/index.html 83 | [Servo]: https://github.com/servo/servo 84 | [mb]: https://github.com/mbrubeck/ 85 | [ru]: https://limpet.net/mbrubeck/2019/02/07/rust-a-unique-perspective.html 86 | [ac]: https://github.com/alexcrichton/ 87 | [ex]: https://www.youtube.com/watch?v=Dbytx0ivH7Q 88 | [f]: https://blog.rust-lang.org/2015/04/10/Fearless-Concurrency.html 89 | [d]: https://stackoverflow.com/questions/1050222/what-is-the-difference-between-concurrency-and-parallelism#1050257 90 | [at]: https://github.com/aturon 91 | [bmt]: https://github.com/rust-lang/rustlings/blob/master/exercises/threads/threads1.rs 92 | [rustlings]: https://github.com/rust-lang/rustlings/ 93 | -------------------------------------------------------------------------------- /courses/rust/projects/project-1/tests/tests.rs: -------------------------------------------------------------------------------- 1 | use assert_cmd::prelude::*; 2 | use kvs::KvStore; 3 | use predicates::str::contains; 4 | use std::process::Command; 5 | 6 | // `kvs` with no args should exit with a non-zero code. 7 | #[test] 8 | fn cli_no_args() { 9 | Command::cargo_bin("kvs").unwrap().assert().failure(); 10 | } 11 | 12 | // `kvs -V` should print the version 13 | #[test] 14 | fn cli_version() { 15 | Command::cargo_bin("kvs") 16 | .unwrap() 17 | .args(&["-V"]) 18 | .assert() 19 | .stdout(contains(env!("CARGO_PKG_VERSION"))); 20 | } 21 | 22 | // `kvs get ` should print "unimplemented" to stderr and exit with non-zero code 23 | #[test] 24 | fn cli_get() { 25 | Command::cargo_bin("kvs") 26 | .unwrap() 27 | .args(&["get", "key1"]) 28 | .assert() 29 | .failure() 30 | .stderr(contains("unimplemented")); 31 | } 32 | 33 | // `kvs set ` should print "unimplemented" to stderr and exit with non-zero code 34 | #[test] 35 | fn cli_set() { 36 | Command::cargo_bin("kvs") 37 | .unwrap() 38 | .args(&["set", "key1", "value1"]) 39 | .assert() 40 | .failure() 41 | .stderr(contains("unimplemented")); 42 | } 43 | 44 | // `kvs rm ` should print "unimplemented" to stderr and exit with non-zero code 45 | #[test] 46 | fn cli_rm() { 47 | Command::cargo_bin("kvs") 48 | .unwrap() 49 | .args(&["rm", "key1"]) 50 | .assert() 51 | .failure() 52 | .stderr(contains("unimplemented")); 53 | } 54 | 55 | #[test] 56 | fn cli_invalid_get() { 57 | Command::cargo_bin("kvs") 58 | .unwrap() 59 | .args(&["get"]) 60 | .assert() 61 | .failure(); 62 | 63 | Command::cargo_bin("kvs") 64 | .unwrap() 65 | .args(&["get", "extra", "field"]) 66 | .assert() 67 | .failure(); 68 | } 69 | 70 | #[test] 71 | fn cli_invalid_set() { 72 | Command::cargo_bin("kvs") 73 | .unwrap() 74 | .args(&["set"]) 75 | .assert() 76 | .failure(); 77 | 78 | Command::cargo_bin("kvs") 79 | .unwrap() 80 | .args(&["set", "missing_field"]) 81 | .assert() 82 | .failure(); 83 | 84 | Command::cargo_bin("kvs") 85 | .unwrap() 86 | .args(&["set", "extra", "extra", "field"]) 87 | .assert() 88 | .failure(); 89 | } 90 | 91 | #[test] 92 | fn cli_invalid_rm() { 93 | Command::cargo_bin("kvs") 94 | .unwrap() 95 | .args(&["rm"]) 96 | .assert() 97 | .failure(); 98 | 99 | Command::cargo_bin("kvs") 100 | .unwrap() 101 | .args(&["rm", "extra", "field"]) 102 | .assert() 103 | .failure(); 104 | } 105 | 106 | #[test] 107 | fn cli_invalid_subcommand() { 108 | Command::cargo_bin("kvs") 109 | .unwrap() 110 | .args(&["unknown", "subcommand"]) 111 | .assert() 112 | .failure(); 113 | } 114 | 115 | // Should get previously stored value 116 | #[test] 117 | fn get_stored_value() { 118 | let mut store = KvStore::new(); 119 | 120 | store.set("key1".to_owned(), "value1".to_owned()); 121 | store.set("key2".to_owned(), "value2".to_owned()); 122 | 123 | assert_eq!(store.get("key1".to_owned()), Some("value1".to_owned())); 124 | assert_eq!(store.get("key2".to_owned()), Some("value2".to_owned())); 125 | } 126 | 127 | // Should overwrite existent value 128 | #[test] 129 | fn overwrite_value() { 130 | let mut store = KvStore::new(); 131 | 132 | store.set("key1".to_owned(), "value1".to_owned()); 133 | assert_eq!(store.get("key1".to_owned()), Some("value1".to_owned())); 134 | 135 | store.set("key1".to_owned(), "value2".to_owned()); 136 | assert_eq!(store.get("key1".to_owned()), Some("value2".to_owned())); 137 | } 138 | 139 | // Should get `None` when getting a non-existent key 140 | #[test] 141 | fn get_non_existent_value() { 142 | let mut store = KvStore::new(); 143 | 144 | store.set("key1".to_owned(), "value1".to_owned()); 145 | assert_eq!(store.get("key2".to_owned()), None); 146 | } 147 | 148 | #[test] 149 | fn remove_key() { 150 | let mut store = KvStore::new(); 151 | 152 | store.set("key1".to_owned(), "value1".to_owned()); 153 | store.remove("key1".to_owned()); 154 | assert_eq!(store.get("key1".to_owned()), None); 155 | } 156 | --------------------------------------------------------------------------------