├── artifact ├── .gitignore ├── vm-init.sh ├── Vagrantfile ├── vm-setup.sh └── README.md ├── eval ├── data │ └── .gitignore ├── fig │ └── .gitignore ├── plot.sh ├── eval.sh ├── tests.sh ├── nfsdist.bt ├── largefile.plot ├── scale.plot ├── largefile.sh ├── serial.patch ├── global-txn-lock.patch ├── scale.py ├── bench.plot ├── scale.sh ├── latency.sh ├── aggregate-times.py ├── bench.py ├── latency-tcp.sh ├── bench.sh └── loc.py ├── .github ├── mailbot.json └── workflows │ └── build.yml ├── bench ├── stop-go-nfsd.sh ├── run-go-clnt.sh ├── stop-linux.sh ├── start-go-nfsd.sh ├── run-fscq.sh ├── run-linux.sh ├── app-bench.sh ├── run-go-nfsd.sh └── start-linux.sh ├── .editorconfig ├── .gitignore ├── simple ├── 0super.go ├── fh.go ├── recover_example.go ├── mount.go ├── mkfs.go ├── inode.go ├── simple_test.go └── ops.go ├── TODO ├── nfs ├── stats_test.go ├── stats.go ├── mount.go ├── lorder.go ├── nfs_ls.go ├── nfs.go └── nfs_clnt.go ├── Makefile ├── go.mod ├── cmd ├── simple-nfsd │ ├── start.go │ └── main.go ├── lookup │ └── main.go ├── largefile │ └── main.go ├── fs-largefile │ └── main.go ├── smallfile │ └── main.go ├── clnt-null │ └── main.go ├── txn-bench │ └── main.go ├── go-nfsd │ └── main.go ├── fs-smallfile │ └── main.go └── clnt-smallfile │ └── main.go ├── README.md ├── dcache └── dcache.go ├── fh └── nfs_fh.go ├── dir ├── dir_test.go ├── dcache.go └── dir.go ├── LICENSE ├── fstxn ├── fsstate.go ├── commit.go └── fstxn.go ├── util ├── timed_disk │ └── disk.go └── stats │ └── stats.go ├── kvs ├── kvs_test.go └── kvs.go ├── cache └── cache.go ├── super └── super.go ├── shrinker └── shrinker.go ├── inode ├── shrink.go └── inode.go ├── go.sum ├── n.txt └── alloctxn └── alloctxn.go /artifact/.gitignore: -------------------------------------------------------------------------------- 1 | /.vagrant/ 2 | -------------------------------------------------------------------------------- /eval/data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /eval/fig/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /.github/mailbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "commitEmailFormat": "html", 3 | "commitList": "nickolai@csail.mit.edu,kaashoek@mit.edu,tchajed@mit.edu" 4 | } 5 | -------------------------------------------------------------------------------- /bench/stop-go-nfsd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Usage: ./stop-go-nfsd.sh 5 | # 6 | 7 | killall -s SIGINT go-nfsd 8 | sudo umount -f /mnt/nfs 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | 7 | [*.sh] 8 | indent_style = space 9 | indent_size = 4 10 | -------------------------------------------------------------------------------- /eval/plot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ./bench.py data/bench-raw.txt 4 | ./scale.py data/scale-raw.txt 5 | ./bench.plot -o fig/bench.pdf 6 | ./bench.plot -ssd -o fig/bench-ssd.pdf 7 | gnuplot scale.plot 8 | gnuplot largefile.plot 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /Goose 2 | # created by benchmarking scripts 3 | nfs.out 4 | # compiled binaries 5 | fs-smallfile 6 | clnt-smallfile 7 | fs-largefile 8 | go-nfsd 9 | txn-bench 10 | # profiling output 11 | cpu.prof 12 | # per-iteration timing 13 | *-[0-9]*.txt 14 | -------------------------------------------------------------------------------- /eval/eval.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | 4 | # run the entire eval 5 | 6 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 7 | cd "$DIR" 8 | 9 | ./loc.py | tee data/lines-of-code.txt 10 | ./bench.sh -ssd ~/disk.img 11 | ./scale.sh 12 12 | ./plot.sh 13 | -------------------------------------------------------------------------------- /eval/tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | 4 | cd "$GO_NFSD_PATH" 5 | ./bench/run-go-nfsd.sh $LTP_PATH/testcases/kernel/fs/fsstress/fsstress \ 6 | -l 200 -n 100 -p 4 -d /mnt/nfs 7 | ./bench/run-go-nfsd.sh $LTP_PATH/testcases/kernel/fs/fsx-linux/fsx-linux \ 8 | -N 100000 /mnt/nfs/x 9 | -------------------------------------------------------------------------------- /simple/0super.go: -------------------------------------------------------------------------------- 1 | package simple 2 | 3 | import ( 4 | "github.com/mit-pdos/go-journal/addr" 5 | "github.com/mit-pdos/go-journal/common" 6 | ) 7 | 8 | func block2addr(blkno common.Bnum) addr.Addr { 9 | return addr.MkAddr(blkno, 0) 10 | } 11 | 12 | func nInode() common.Inum { 13 | return common.Inum(common.INODEBLK) 14 | } 15 | 16 | func inum2Addr(inum common.Inum) addr.Addr { 17 | return addr.MkAddr(common.LOGSIZE, (uint64(inum) * common.INODESZ * 8)) 18 | } 19 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | Potential improvements: 2 | 3 | - The txn layer could implement more fine-grained locking for CommitWait. 4 | For example, if a transaction is writing full blocks (no inodes or 5 | bits), it's not necessary to acquire the txn commit lock. Even more 6 | aggressively, txn could acquire locks (using a lockmap) for every disk 7 | block that requires an installation read (sorting the disk block addrs 8 | to avoid deadlock). Not clear if that optimization is worth it, though. 9 | -------------------------------------------------------------------------------- /bench/run-go-clnt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Usage: ./run-go-clnt.sh go run ./cmd/clnt-smallfile/main.go 5 | # 6 | 7 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 8 | # root of repo 9 | cd $DIR/.. 10 | 11 | # taskset 0xc go run ./cmd/go-nfsd/ -disk /dev/shm/goose.img & 12 | go run ./cmd/go-nfsd/ -disk /dev/shm/goose.img & 13 | sleep 1 14 | killall -0 go-nfsd # make sure server is running 15 | # taskset 0x3 $1 /mnt/nfs 16 | echo "$@" 17 | "$@" 18 | killall go-nfsd 19 | -------------------------------------------------------------------------------- /nfs/stats_test.go: -------------------------------------------------------------------------------- 1 | package nfs 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mit-pdos/go-nfsd/nfstypes" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestOpNames(t *testing.T) { 11 | assert := assert.New(t) 12 | 13 | // make sure nfsopNames list is sensible 14 | assert.Equal(NUM_NFS_OPS, len(nfsopNames)) 15 | // first operation 16 | assert.Equal("NULL", nfsopNames[nfstypes.NFSPROC3_NULL]) 17 | assert.Equal("FSINFO", nfsopNames[nfstypes.NFSPROC3_FSINFO]) 18 | // the last operation 19 | assert.Equal("COMMIT", nfsopNames[nfstypes.NFSPROC3_COMMIT]) 20 | } 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOPATH := $(shell go env GOPATH) 2 | GOOSE_DIRS := super cache fh fstxn kvs nfstypes simple 3 | 4 | # Things that don't goose yet: 5 | # . 6 | # dcache: map with string keys 7 | # inode: time package 8 | # nfstypes: need to ignore nfs_xdr.go 9 | # dir 10 | 11 | COQ_PKGDIR := Goose/github_com/mit_pdos/go_nfsd 12 | 13 | all: check goose-output 14 | 15 | check: 16 | test -z "$$(gofmt -d .)" 17 | go vet ./... 18 | 19 | goose-output: $(patsubst %,${COQ_PKGDIR}/%.v,$(GOOSE_DIRS)) 20 | 21 | ${COQ_PKGDIR}/%.v: % %/* 22 | $(GOPATH)/bin/goose -out Goose ./$< 23 | 24 | clean: 25 | rm -rf Goose 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mit-pdos/go-nfsd 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/goose-lang/goose v0.7.1 7 | github.com/goose-lang/primitive v0.1.0 8 | github.com/goose-lang/std v0.4.1 9 | github.com/mit-pdos/go-journal v0.5.4 10 | github.com/rodaine/table v1.2.0 11 | github.com/stretchr/testify v1.9.0 12 | github.com/tchajed/marshal v0.6.2 13 | github.com/zeldovich/go-rpcgen v0.1.5 14 | golang.org/x/sys v0.22.0 15 | ) 16 | 17 | require ( 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/pmezard/go-difflib v1.0.0 // indirect 20 | gopkg.in/yaml.v3 v3.0.1 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /cmd/simple-nfsd/start.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/goose-lang/primitive/disk" 7 | 8 | "github.com/mit-pdos/go-journal/util" 9 | "github.com/mit-pdos/go-nfsd/simple" 10 | ) 11 | 12 | func MakeNfs(name string) *simple.Nfs { 13 | sz := uint64(100 * 1024) 14 | 15 | var d disk.Disk 16 | util.DPrintf(1, "MakeNfs: creating file disk at %s", name) 17 | var err error 18 | d, err = disk.NewFileDisk(name, sz) 19 | if err != nil { 20 | panic(fmt.Errorf("could not create file disk: %v", err)) 21 | } 22 | 23 | nfs := simple.MakeNfs(d) 24 | if nfs == nil { 25 | panic("could not initialize nfs") 26 | } 27 | 28 | return nfs 29 | } 30 | -------------------------------------------------------------------------------- /eval/nfsdist.bt: -------------------------------------------------------------------------------- 1 | BEGIN 2 | { 3 | } 4 | 5 | kprobe:nfsd3_proc_getattr, 6 | kprobe:nfsd3_proc_write, 7 | kprobe:nfsd3_proc_lookup, 8 | kprobe:nfsd3_proc_remove, 9 | kprobe:nfsd3_proc_create, 10 | kprobe:nfsd3_proc_null 11 | { 12 | @start[tid] = nsecs; 13 | @name[tid] = func; 14 | } 15 | 16 | kretprobe:nfsd3_proc_getattr, 17 | kretprobe:nfsd3_proc_write, 18 | kretprobe:nfsd3_proc_lookup, 19 | kretprobe:nfsd3_proc_remove, 20 | kretprobe:nfsd3_proc_create, 21 | kretprobe:nfsd3_proc_null 22 | /@start[tid]/ 23 | { 24 | @us[@name[tid]] = avg((nsecs - @start[tid]) / 1000); 25 | @counts[@name[tid]] = count(); 26 | delete(@start[tid]); 27 | delete(@name[tid]); 28 | } 29 | 30 | END 31 | { 32 | clear(@start); 33 | clear(@name); 34 | } 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GoNFS 2 | 3 | [![CI](https://github.com/mit-pdos/go-nfsd/actions/workflows/build.yml/badge.svg)](https://github.com/mit-pdos/go-nfsd/actions/workflows/build.yml) 4 | 5 | An NFSv3 server that uses [GoJournal](https://github.com/mit-pdos/go-journal) to 6 | write operations atomically to disk. Unlike many other NFS servers, GoNFS 7 | implements the file-system abstraction on top of a disk (which can be any file, 8 | including an in-memory file in `/tmp`, an ordinary file in another file system, 9 | or a block device). 10 | 11 | ## GoJournal artifact 12 | 13 | The artifact for the OSDI 2021 GoJournal paper is in this repo at 14 | [artifact](artifact/). See the README there for detailed instructions on 15 | obtaining and using the artifact. 16 | -------------------------------------------------------------------------------- /simple/fh.go: -------------------------------------------------------------------------------- 1 | package simple 2 | 3 | import ( 4 | "github.com/tchajed/marshal" 5 | 6 | "github.com/mit-pdos/go-journal/common" 7 | "github.com/mit-pdos/go-nfsd/nfstypes" 8 | ) 9 | 10 | type Fh struct { 11 | Ino common.Inum 12 | } 13 | 14 | func MakeFh(fh3 nfstypes.Nfs_fh3) Fh { 15 | dec := marshal.NewDec(fh3.Data) 16 | i := dec.GetInt() 17 | return Fh{Ino: common.Inum(i)} 18 | } 19 | 20 | func (fh Fh) MakeFh3() nfstypes.Nfs_fh3 { 21 | enc := marshal.NewEnc(16) 22 | enc.PutInt(uint64(fh.Ino)) 23 | fh3 := nfstypes.Nfs_fh3{Data: enc.Finish()} 24 | return fh3 25 | } 26 | 27 | func MkRootFh3() nfstypes.Nfs_fh3 { 28 | enc := marshal.NewEnc(16) 29 | enc.PutInt(uint64(common.ROOTINUM)) 30 | return nfstypes.Nfs_fh3{Data: enc.Finish()} 31 | } 32 | -------------------------------------------------------------------------------- /artifact/vm-init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | cd 6 | 7 | sudo apt-get -y update 8 | sudo apt-get -y upgrade 9 | sudo apt-get -y install zsh 10 | 11 | if [ ! -e ~/.oh-my-zsh ]; then 12 | wget https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh 13 | sh install.sh --unattended 14 | rm install.sh 15 | fi 16 | sudo chsh -s /usr/bin/zsh "$USER" 17 | sed -i 's/ZSH_THEME=.*/ZSH_THEME="dst"/' ~/.zshrc 18 | rm -f ~/.bashrc 19 | 20 | sudo passwd -d "$USER" 21 | sudo sed -e 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/' -i /etc/ssh/sshd_config 22 | sudo systemctl restart sshd 23 | 24 | if [ ! -e go-nfsd ]; then 25 | git clone https://github.com/mit-pdos/go-nfsd 26 | fi 27 | 28 | git config --global pull.ff only 29 | -------------------------------------------------------------------------------- /bench/stop-linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Usage: ./stop-linux.sh [disk file] 5 | # 6 | 7 | native=false 8 | 9 | while true; do 10 | case "$1" in 11 | -native=true) 12 | shift 13 | native=true 14 | ;; 15 | -native=false) 16 | shift 17 | native=false 18 | ;; 19 | *) 20 | break 21 | ;; 22 | esac 23 | done 24 | 25 | disk_file=/dev/shm/nfs3.img 26 | if [ ! -z "$1" ]; then 27 | disk_file="$1" 28 | fi 29 | 30 | sudo umount -f /mnt/nfs 31 | 32 | if [ "$native" = "false" ]; then 33 | sudo systemctl stop nfs-server.service 34 | sudo umount /srv/nfs/bench 35 | fi 36 | 37 | # do not attempt to remove block devices 38 | if [ -f "$disk_file" ]; then 39 | rm -f "$disk_file" 40 | fi 41 | -------------------------------------------------------------------------------- /simple/recover_example.go: -------------------------------------------------------------------------------- 1 | package simple 2 | 3 | import ( 4 | "github.com/goose-lang/primitive/disk" 5 | 6 | "github.com/mit-pdos/go-journal/common" 7 | "github.com/mit-pdos/go-nfsd/nfstypes" 8 | ) 9 | 10 | func exampleWorker(nfs *Nfs, ino common.Inum) { 11 | fh := Fh{Ino: ino} 12 | buf := make([]byte, 1024) 13 | nfs.NFSPROC3_GETATTR(nfstypes.GETATTR3args{Object: fh.MakeFh3()}) 14 | nfs.NFSPROC3_READ(nfstypes.READ3args{File: fh.MakeFh3(), Offset: 0, Count: 1024}) 15 | nfs.NFSPROC3_WRITE(nfstypes.WRITE3args{File: fh.MakeFh3(), Offset: 0, Count: 1024, Data: buf}) 16 | return 17 | } 18 | 19 | func RecoverExample(d disk.Disk) { 20 | nfs := Recover(d) 21 | go func() { exampleWorker(nfs, 3) }() 22 | go func() { exampleWorker(nfs, 3) }() 23 | go func() { exampleWorker(nfs, 4) }() 24 | } 25 | -------------------------------------------------------------------------------- /dcache/dcache.go: -------------------------------------------------------------------------------- 1 | package dcache 2 | 3 | import ( 4 | "github.com/mit-pdos/go-journal/common" 5 | ) 6 | 7 | type Dentry struct { 8 | Inum common.Inum 9 | Off uint64 10 | } 11 | 12 | type Dcache struct { 13 | cache map[string]Dentry 14 | Lastoff uint64 15 | } 16 | 17 | func MkDcache() *Dcache { 18 | return &Dcache{ 19 | cache: make(map[string]Dentry), 20 | Lastoff: uint64(0), 21 | } 22 | } 23 | 24 | func (dc *Dcache) Add(name string, inum common.Inum, off uint64) { 25 | dc.cache[name] = Dentry{Inum: inum, Off: off} 26 | } 27 | 28 | func (dc *Dcache) Lookup(name string) (Dentry, bool) { 29 | d, ok := dc.cache[name] 30 | return d, ok 31 | } 32 | 33 | func (dc *Dcache) Del(name string) bool { 34 | _, ok := dc.cache[name] 35 | if ok { 36 | delete(dc.cache, name) 37 | } 38 | return ok 39 | } 40 | -------------------------------------------------------------------------------- /eval/largefile.plot: -------------------------------------------------------------------------------- 1 | set terminal pdf dashed noenhanced size 3in,1.5in 2 | set output "fig/largefile.pdf" 3 | 4 | set style data histogram 5 | set style histogram cluster gap 1 6 | set rmargin at screen .95 7 | 8 | set xrange [-1:4] 9 | set yrange [0:*] 10 | set grid y 11 | set ylabel "Throughput (MB/s)" 12 | set ytics scale 0.5,0 nomirror 13 | set xtics scale 0,0 14 | set key top right 15 | set style fill solid 1 border rgb "black" 16 | 17 | set datafile separator "\t" 18 | 19 | plot "data/largefile.data" \ 20 | using "linux-ssd":xtic(1) title "Linux (data=journal)" lc rgb '#b6d7a8' lt 1, \ 21 | '' using "gonfs-ssd-unstable":xtic(1) title "GoNFS (unstable)" lc rgb '#3a81ba' lt 1, \ 22 | '' using "linux-ssd-sync":xtic(1) title "Linux (sync)" lc rgb "#e4ffd9" lt 1, \ 23 | '' using "gonfs-ssd":xtic(1) title "GoNFS (stable)" lc rgb "#8dbbe0" lt 1 24 | -------------------------------------------------------------------------------- /nfs/stats.go: -------------------------------------------------------------------------------- 1 | package nfs 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | "github.com/mit-pdos/go-nfsd/util/stats" 8 | ) 9 | 10 | const NUM_NFS_OPS = 22 11 | 12 | func (nfs *Nfs) recordOp(op uint32, start time.Time) { 13 | nfs.stats[op].Record(start) 14 | } 15 | 16 | var nfsopNames = []string{ 17 | "NULL", 18 | "GETATTR", 19 | "SETATTR", 20 | "LOOKUP", 21 | "ACCESS", 22 | "READLINK", 23 | "READ", 24 | "WRITE", 25 | "CREATE", 26 | "MKDIR", 27 | "SYMLINK", 28 | "MKNOD", 29 | "REMOVE", 30 | "RMDIR", 31 | "RENAME", 32 | "LINK", 33 | "READDIR", 34 | "READDIRPLUS", 35 | "FSSTAT", 36 | "FSINFO", 37 | "PATHCONF", 38 | "COMMIT", 39 | } 40 | 41 | func (nfs *Nfs) WriteOpStats(w io.Writer) { 42 | stats.WriteTable(nfsopNames, nfs.stats[:], w) 43 | } 44 | 45 | func (nfs *Nfs) ResetOpStats() { 46 | for i := range nfs.stats { 47 | nfs.stats[i].Reset() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | jobs: 11 | test: 12 | strategy: 13 | matrix: 14 | go-version: ['stable'] 15 | fail-fast: false 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Install Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version: ${{ matrix.go-version }} 23 | - name: Install Goose 24 | run: | 25 | go install github.com/goose-lang/goose/cmd/goose@latest 26 | - name: Install dependencies 27 | run: | 28 | go get -t ./... 29 | - name: Check style 30 | run: | 31 | gofmt -w . 32 | git diff --exit-code 33 | go vet ./... 34 | - name: Test 35 | run: | 36 | go test -v ./... 37 | make goose-output 38 | -------------------------------------------------------------------------------- /eval/scale.plot: -------------------------------------------------------------------------------- 1 | set terminal pdf dashed noenhanced size 3.5in,2.0in 2 | set output "fig/scale.pdf" 3 | 4 | set auto x 5 | set yrange [0:*] 6 | set xtics 1 7 | set ylabel "files / sec" 8 | set format y '%.0f' 9 | set xlabel "\# clients" 10 | set key top left 11 | 12 | set style data line 13 | 14 | set style line 81 lt 0 # dashed 15 | set style line 81 lt rgb "#808080" # grey 16 | set grid back linestyle 81 17 | 18 | set border 3 back linestyle 80 19 | set style line 1 lt rgb "#00A000" lw 2 pt 6 20 | set style line 2 lt rgb "#5060D0" lw 2 pt 1 21 | set style line 3 lt rgb "#A00000" lw 2 pt 2 22 | set style line 4 lt rgb "#F25900" lw 2 pt 9 23 | 24 | plot \ 25 | "data/gnfs.data" using 1:($2) with linespoints ls 1 title 'GoNFS', \ 26 | "data/linux-nfs.data" using 1:($2) with linespoints ls 2 title 'Linux NFS', \ 27 | "data/serial.data" using 1:($2) with linespoints ls 3 title 'Serial GoNFS', \ 28 | 29 | # "fig/ext3.data" using 1:($2) with linespoints title 'Linux Ext3', \ 30 | -------------------------------------------------------------------------------- /fh/nfs_fh.go: -------------------------------------------------------------------------------- 1 | package fh 2 | 3 | import ( 4 | "github.com/goose-lang/std" 5 | "github.com/tchajed/marshal" 6 | 7 | "github.com/mit-pdos/go-journal/common" 8 | "github.com/mit-pdos/go-nfsd/nfstypes" 9 | ) 10 | 11 | type Fh struct { 12 | Ino common.Inum 13 | Gen uint64 14 | } 15 | 16 | func MakeFh(fh3 nfstypes.Nfs_fh3) Fh { 17 | dec := marshal.NewDec(fh3.Data) 18 | i := dec.GetInt() 19 | g := dec.GetInt() 20 | return Fh{Ino: common.Inum(i), Gen: g} 21 | } 22 | 23 | func (fh Fh) MakeFh3() nfstypes.Nfs_fh3 { 24 | enc := marshal.NewEnc(16) 25 | enc.PutInt(uint64(fh.Ino)) 26 | enc.PutInt(uint64(fh.Gen)) 27 | fh3 := nfstypes.Nfs_fh3{Data: enc.Finish()} 28 | return fh3 29 | } 30 | 31 | func MkRootFh3() nfstypes.Nfs_fh3 { 32 | enc := marshal.NewEnc(16) 33 | enc.PutInt(uint64(common.ROOTINUM)) 34 | enc.PutInt(uint64(1)) 35 | return nfstypes.Nfs_fh3{Data: enc.Finish()} 36 | } 37 | 38 | func Equal(h1 nfstypes.Nfs_fh3, h2 nfstypes.Nfs_fh3) bool { 39 | return std.BytesEqual(h1.Data, h2.Data) 40 | } 41 | -------------------------------------------------------------------------------- /simple/mount.go: -------------------------------------------------------------------------------- 1 | package simple 2 | 3 | import ( 4 | "github.com/mit-pdos/go-nfsd/nfstypes" 5 | 6 | "log" 7 | ) 8 | 9 | func (nfs *Nfs) MOUNTPROC3_NULL() { 10 | } 11 | 12 | func (nfs *Nfs) MOUNTPROC3_MNT(args nfstypes.Dirpath3) nfstypes.Mountres3 { 13 | reply := new(nfstypes.Mountres3) 14 | log.Printf("Mount %v\n", args) 15 | reply.Fhs_status = nfstypes.MNT3_OK 16 | reply.Mountinfo.Fhandle = MkRootFh3().Data 17 | return *reply 18 | } 19 | 20 | func (nfs *Nfs) MOUNTPROC3_UMNT(args nfstypes.Dirpath3) { 21 | log.Printf("Unmount %v\n", args) 22 | } 23 | 24 | func (nfs *Nfs) MOUNTPROC3_UMNTALL() { 25 | log.Printf("Unmountall\n") 26 | } 27 | 28 | func (nfs *Nfs) MOUNTPROC3_DUMP() nfstypes.Mountopt3 { 29 | log.Printf("Dump\n") 30 | return nfstypes.Mountopt3{P: nil} 31 | } 32 | 33 | func (nfs *Nfs) MOUNTPROC3_EXPORT() nfstypes.Exportsopt3 { 34 | res := nfstypes.Exports3{ 35 | Ex_dir: "", 36 | Ex_groups: nil, 37 | Ex_next: nil, 38 | } 39 | res.Ex_dir = "/" 40 | return nfstypes.Exportsopt3{P: &res} 41 | } 42 | -------------------------------------------------------------------------------- /dir/dir_test.go: -------------------------------------------------------------------------------- 1 | package dir 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mit-pdos/go-nfsd/nfstypes" 7 | "github.com/zeldovich/go-rpcgen/xdr" 8 | ) 9 | 10 | func TestFattr3Size(t *testing.T) { 11 | var e nfstypes.Fattr3 12 | bs, err := xdr.EncodeBuf(&e) 13 | if err != nil { 14 | panic(err) 15 | } 16 | if len(bs) != int(fattr3XDRsize) { 17 | t.Fatalf("size of fattr3 is %d != %d", len(bs), fattr3XDRsize) 18 | } 19 | } 20 | 21 | func TestNameBaggage(t *testing.T) { 22 | var e nfstypes.Filename3 23 | bs, err := xdr.EncodeBuf(&e) 24 | if err != nil { 25 | panic(err) 26 | } 27 | if len(bs) != 4 { 28 | t.Fatalf("size of empty filename is %d != 4", len(bs)) 29 | } 30 | } 31 | 32 | func TestEntryPlus3Baggage(t *testing.T) { 33 | var e nfstypes.Entryplus3 34 | e.Name_attributes.Attributes_follow = true 35 | e.Name_handle.Handle_follows = true 36 | bs, err := xdr.EncodeBuf(&e) 37 | if err != nil { 38 | panic(err) 39 | } 40 | if len(bs) > int(entryplus3Baggage) { 41 | t.Fatalf("size of entryplus3 is %d > %d", len(bs), entryplus3Baggage) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /simple/mkfs.go: -------------------------------------------------------------------------------- 1 | package simple 2 | 3 | import ( 4 | "github.com/goose-lang/primitive/disk" 5 | "github.com/mit-pdos/go-journal/jrnl" 6 | 7 | "github.com/mit-pdos/go-journal/lockmap" 8 | "github.com/mit-pdos/go-journal/obj" 9 | ) 10 | 11 | type Nfs struct { 12 | t *obj.Log 13 | l *lockmap.LockMap 14 | } 15 | 16 | func Mkfs(d disk.Disk) *obj.Log { 17 | log := obj.MkLog(d) 18 | op := jrnl.Begin(log) 19 | inodeInit(op) 20 | ok := op.CommitWait(true) 21 | if !ok { 22 | return nil 23 | } 24 | return log 25 | } 26 | 27 | func Recover(d disk.Disk) *Nfs { 28 | log := obj.MkLog(d) // runs recovery 29 | lockmap := lockmap.MkLockMap() 30 | 31 | nfs := &Nfs{ 32 | t: log, 33 | l: lockmap, 34 | } 35 | return nfs 36 | } 37 | 38 | func MakeNfs(d disk.Disk) *Nfs { 39 | log := obj.MkLog(d) // runs recovery 40 | 41 | op := jrnl.Begin(log) 42 | inodeInit(op) 43 | ok := op.CommitWait(true) 44 | if !ok { 45 | return nil 46 | } 47 | 48 | lockmap := lockmap.MkLockMap() 49 | 50 | nfs := &Nfs{ 51 | t: log, 52 | l: lockmap, 53 | } 54 | 55 | return nfs 56 | } 57 | -------------------------------------------------------------------------------- /bench/start-go-nfsd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | # 6 | # Usage: ./start-go-nfsd.sh 7 | # 8 | # default disk is /dev/shm/goose.img but can be overriden by passing -disk again 9 | # 10 | 11 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 12 | # root of repo 13 | cd "$DIR"/.. 14 | 15 | nfs_mount_opts="" 16 | extra_args=() 17 | while [[ "$#" -gt 0 ]]; do 18 | case "$1" in 19 | -nfs-mount-opts) 20 | shift 21 | nfs_mount_opts="$1" 22 | shift 23 | ;; 24 | -*=*) 25 | extra_args+=("$1") 26 | shift 27 | ;; 28 | -*) 29 | extra_args+=("$1" "$2") 30 | shift 31 | shift 32 | ;; 33 | esac 34 | done 35 | 36 | go build ./cmd/go-nfsd 37 | ./go-nfsd -disk /dev/shm/goose.img "${extra_args[@]}" >nfs.out 2>&1 & 38 | sleep 2 39 | killall -0 go-nfsd # make sure server is running 40 | killall -SIGUSR1 go-nfsd # reset stats after recovery 41 | sudo mount -t nfs -o "$nfs_mount_opts" localhost:/ /mnt/nfs 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /nfs/mount.go: -------------------------------------------------------------------------------- 1 | package nfs 2 | 3 | import ( 4 | "github.com/mit-pdos/go-journal/util" 5 | "github.com/mit-pdos/go-nfsd/fh" 6 | "github.com/mit-pdos/go-nfsd/nfstypes" 7 | 8 | "log" 9 | ) 10 | 11 | func (nfs *Nfs) MOUNTPROC3_NULL() { 12 | util.DPrintf(1, "MOUNT Null\n") 13 | } 14 | 15 | func (nfs *Nfs) MOUNTPROC3_MNT(args nfstypes.Dirpath3) nfstypes.Mountres3 { 16 | reply := new(nfstypes.Mountres3) 17 | util.DPrintf(1, "MOUNT Mount %v\n", args) 18 | reply.Fhs_status = nfstypes.MNT3_OK 19 | reply.Mountinfo.Fhandle = fh.MkRootFh3().Data 20 | return *reply 21 | } 22 | 23 | func (nfs *Nfs) MOUNTPROC3_UMNT(args nfstypes.Dirpath3) { 24 | util.DPrintf(1, "MOUNT Unmount %v\n", args) 25 | } 26 | 27 | func (nfs *Nfs) MOUNTPROC3_UMNTALL() { 28 | log.Printf("Unmountall\n") 29 | } 30 | 31 | func (nfs *Nfs) MOUNTPROC3_DUMP() nfstypes.Mountopt3 { 32 | log.Printf("Dump\n") 33 | return nfstypes.Mountopt3{P: nil} 34 | } 35 | 36 | func (nfs *Nfs) MOUNTPROC3_EXPORT() nfstypes.Exportsopt3 { 37 | res := nfstypes.Exports3{ 38 | Ex_dir: "", 39 | Ex_groups: nil, 40 | Ex_next: nil, 41 | } 42 | res.Ex_dir = "/" 43 | return nfstypes.Exportsopt3{P: &res} 44 | } 45 | -------------------------------------------------------------------------------- /eval/largefile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # for experimenting with the many modes of largefile 4 | 5 | set -e 6 | 7 | blue=$(tput setaf 4 || printf "") 8 | reset=$(tput sgr0 || printf "") 9 | 10 | info() { 11 | echo -e "${blue}$1${reset}" 12 | } 13 | 14 | info "go-nfsd" 15 | ./bench/run-go-nfsd.sh -disk "/dev/shm/disk.img" -unstable=true ./fs-largefile 16 | ./bench/run-go-nfsd.sh -disk "/dev/shm/disk.img" -unstable=false ./fs-largefile 17 | ./bench/run-go-nfsd.sh -disk "" -unstable=true ./fs-largefile 18 | ./bench/run-go-nfsd.sh -disk "" -unstable=false ./fs-largefile 19 | 20 | echo 21 | info "Linux (ext4)" 22 | ./bench/run-linux.sh -fs ext4 -disk "/dev/shm/disk.img" -mount-opts "data=journal" ./fs-largefile 23 | ./bench/run-linux.sh -fs ext4 -disk "/dev/shm/disk.img" -mount-opts "data=ordered" ./fs-largefile 24 | ./bench/run-linux.sh -fs ext4 -disk "/dev/shm/disk.img" -mount-opts "sync,data=journal" ./fs-largefile 25 | ./bench/run-linux.sh -fs ext4 -disk "/dev/shm/disk.img" -mount-opts "sync,data=ordered" ./fs-largefile 26 | 27 | echo 28 | info "Linux (btrfs)" 29 | ./bench/run-linux.sh -fs btrfs -disk "/dev/shm/disk.img" -mount-opts "" ./fs-largefile 30 | ./bench/run-linux.sh -fs btrfs -disk "/dev/shm/disk.img" -mount-opts "flushoncommit" ./fs-largefile 31 | -------------------------------------------------------------------------------- /bench/run-fscq.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | disk_file=/dev/shm/disk.img 5 | fscq_path="" 6 | mnt_path="/mnt/nfs" 7 | 8 | while [[ "$#" -gt 0 ]]; do 9 | case "$1" in 10 | -disk) 11 | shift 12 | disk_file="$1" 13 | shift 14 | ;; 15 | -fscq) 16 | shift 17 | fscq_path="$1" 18 | shift 19 | ;; 20 | -mnt) 21 | shift 22 | mnt_path="$1" 23 | shift 24 | ;; 25 | -*) 26 | echo "Unexpected argument $1" 2>&1 27 | exit 1 28 | ;; 29 | *) 30 | break 31 | ;; 32 | esac 33 | done 34 | 35 | if [[ -z "$fscq_path" && -n "$FSCQ_PATH" ]]; then 36 | fscq_path="$FSCQ_PATH" 37 | fi 38 | 39 | if [ ! -d "$fscq_path" ]; then 40 | echo "Please set FSCQ_PATH or pass -fscq" 2>&1 41 | exit 1 42 | fi 43 | 44 | set -u 45 | 46 | function cleanup { 47 | fusermount -u "$mnt_path" 48 | if [ -f "$disk_file" ]; then 49 | rm -f "$disk_file" 50 | fi 51 | } 52 | trap cleanup EXIT 53 | 54 | "$fscq_path"/src/mkfs "$disk_file" 55 | "$fscq_path"/src/fscq "$disk_file" \ 56 | -o big_writes,atomic_o_trunc,use_ino,kernel_cache \ 57 | "$mnt_path" 58 | sleep 1 59 | echo "# fscq -disk $disk_file" 60 | echo "run $*" 1>&2 61 | "$@" 62 | -------------------------------------------------------------------------------- /fstxn/fsstate.go: -------------------------------------------------------------------------------- 1 | package fstxn 2 | 3 | import ( 4 | "github.com/mit-pdos/go-journal/alloc" 5 | "github.com/mit-pdos/go-journal/common" 6 | "github.com/mit-pdos/go-journal/lockmap" 7 | "github.com/mit-pdos/go-journal/obj" 8 | "github.com/mit-pdos/go-nfsd/cache" 9 | "github.com/mit-pdos/go-nfsd/super" 10 | ) 11 | 12 | const ICACHESZ uint64 = 100 13 | 14 | type FsState struct { 15 | Super *super.FsSuper 16 | Txn *obj.Log 17 | Icache *cache.Cache 18 | Lockmap *lockmap.LockMap 19 | Balloc *alloc.Alloc 20 | Ialloc *alloc.Alloc 21 | } 22 | 23 | func readBitmap(super *super.FsSuper, start common.Bnum, len uint64) []byte { 24 | var bitmap []byte 25 | for i := uint64(0); i < len; i++ { 26 | blk := super.Disk.Read(uint64(start) + i) 27 | bitmap = append(bitmap, blk...) 28 | } 29 | return bitmap 30 | } 31 | 32 | func MkFsState(super *super.FsSuper, log *obj.Log) *FsState { 33 | balloc := alloc.MkAlloc(readBitmap(super, super.BitmapBlockStart(), 34 | super.NBlockBitmap)) 35 | ialloc := alloc.MkAlloc(readBitmap(super, super.BitmapInodeStart(), 36 | super.NInodeBitmap)) 37 | icache := cache.MkCache(ICACHESZ) 38 | st := &FsState{ 39 | Super: super, 40 | Txn: log, 41 | Icache: icache, 42 | Lockmap: lockmap.MkLockMap(), 43 | Balloc: balloc, 44 | Ialloc: ialloc, 45 | } 46 | return st 47 | } 48 | -------------------------------------------------------------------------------- /bench/run-linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # 6 | # Usage: ./run-linux.sh go run ./cmd/fs-smallfile/main.go 7 | # 8 | # takes same flags as start-linux.sh but uses /dev/shm/nfs3.img as default disk 9 | # 10 | 11 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 12 | # root of repo 13 | cd "$DIR"/.. 14 | 15 | disk_file="/dev/shm/disk.img" 16 | extra_args=() 17 | native=false 18 | while true; do 19 | case "$1" in 20 | -disk) 21 | shift 22 | disk_file="$1" 23 | shift 24 | ;; 25 | -native) 26 | shift 27 | native=true 28 | ;; 29 | -*) 30 | extra_args+=("$1" "$2") 31 | shift 32 | shift 33 | ;; 34 | # stop gathering start-linux.sh flags as soon as a non-option is reached 35 | # remainder of command line is treated as command to run 36 | *) 37 | break 38 | ;; 39 | esac 40 | done 41 | 42 | ./bench/start-linux.sh -native="$native" -disk "$disk_file" "${extra_args[@]}" || exit 1 43 | 44 | function cleanup { 45 | ./bench/stop-linux.sh -native="$native" "$disk_file" 46 | } 47 | trap cleanup EXIT 48 | 49 | native_text="" 50 | if [ "$native" = "true" ]; then 51 | native_text="-native" 52 | fi 53 | 54 | echo "# Linux "$native_text" -disk $disk_file ${extra_args[*]}" 55 | echo "run $*" 1>&2 56 | "$@" 57 | -------------------------------------------------------------------------------- /fstxn/commit.go: -------------------------------------------------------------------------------- 1 | package fstxn 2 | 3 | // putInodes may free an inode so must be done before commit 4 | func (op *FsTxn) preCommit() { 5 | op.Atxn.PreCommit() 6 | } 7 | 8 | func (op *FsTxn) postCommit() { 9 | op.releaseInodes() 10 | op.Atxn.PostCommit() 11 | } 12 | 13 | func (op *FsTxn) commitWait(wait bool) bool { 14 | op.preCommit() 15 | ok := op.Atxn.Op.CommitWait(wait) 16 | op.postCommit() 17 | return ok 18 | } 19 | 20 | func (op *FsTxn) Commit() bool { 21 | return op.commitWait(true) 22 | } 23 | 24 | // Commit data, but will also commit everything else, since we don't 25 | // support log-by-pass writes. 26 | func (op *FsTxn) CommitData() bool { 27 | return op.Commit() 28 | } 29 | 30 | // Commit transaction, but don't write to stable storage 31 | func (op *FsTxn) CommitUnstable() bool { 32 | return op.commitWait(false) 33 | } 34 | 35 | // Flush log. We don't have to flush data from other file handles, but 36 | // that is only an option if we do log-by-pass writes. 37 | func (op *FsTxn) CommitFh() bool { 38 | op.preCommit() 39 | ok := op.Fs.Txn.Flush() 40 | op.postCommit() 41 | return ok 42 | } 43 | 44 | // An aborted transaction may free an inode, which results in dirty 45 | // buffers that need to be written to log. So, call commit. 46 | func (op *FsTxn) Abort() bool { 47 | op.releaseInodes() 48 | op.Atxn.PostAbort() 49 | return true 50 | } 51 | -------------------------------------------------------------------------------- /util/timed_disk/disk.go: -------------------------------------------------------------------------------- 1 | package timed_disk 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | "github.com/goose-lang/primitive/disk" 8 | "github.com/mit-pdos/go-nfsd/util/stats" 9 | ) 10 | 11 | type Disk struct { 12 | d disk.Disk 13 | ops [3]stats.Op 14 | } 15 | 16 | func New(d disk.Disk) *Disk { 17 | return &Disk{d: d} 18 | } 19 | 20 | const ( 21 | readOp int = iota 22 | writeOp 23 | barrierOp 24 | ) 25 | 26 | var ops = []string{"disk.Read", "disk.Write", "disk.Barrier"} 27 | 28 | // assert that Disk implements disk.Disk 29 | var _ disk.Disk = &Disk{} 30 | 31 | func (d *Disk) ReadTo(a uint64, b disk.Block) { 32 | defer d.ops[readOp].Record(time.Now()) 33 | d.d.ReadTo(a, b) 34 | } 35 | 36 | func (d *Disk) Read(a uint64) disk.Block { 37 | buf := make(disk.Block, disk.BlockSize) 38 | d.ReadTo(a, buf) 39 | return buf 40 | } 41 | 42 | func (d *Disk) Write(a uint64, b disk.Block) { 43 | defer d.ops[writeOp].Record(time.Now()) 44 | d.d.Write(a, b) 45 | } 46 | 47 | func (d *Disk) Barrier() { 48 | defer d.ops[barrierOp].Record(time.Now()) 49 | d.d.Barrier() 50 | } 51 | 52 | func (d *Disk) Size() uint64 { 53 | return d.d.Size() 54 | } 55 | 56 | func (d *Disk) Close() { 57 | d.d.Close() 58 | } 59 | 60 | func (d *Disk) WriteStats(w io.Writer) { 61 | stats.WriteTable(ops, d.ops[:], w) 62 | } 63 | 64 | func (d *Disk) ResetStats() { 65 | for i := range d.ops { 66 | d.ops[i].Reset() 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /bench/app-bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Usage: app-bench.sh xv6-repo path-to-fs 4 | # 5 | # path-to-fs should have the file system being tested mounted. 6 | # 7 | # Clones xv6-repo to path-to-fs/xv6, then compiles the kernel. 8 | # 9 | 10 | set -eu 11 | 12 | if [ $# -ne 2 ]; then 13 | echo "$0 xv6-repo top-dir" 14 | exit 1 15 | fi 16 | xv6_repo="$1" 17 | fs_dir="$2" 18 | 19 | echo "=== app-bench $xv6_repo $fs_dir ===" 1>&2 20 | cd "$fs_dir" 21 | 22 | # run benchmark out of /tmp to avoid overhead of reading source repo 23 | tmp_dir="/tmp" 24 | if [ -d "/dev/shm" ]; then 25 | tmp_dir="/dev/shm" 26 | fi 27 | xv6_tmp="$tmp_dir/xv6" 28 | rm -rf "$xv6_tmp" 29 | cp -r "$xv6_repo" "$xv6_tmp" 30 | # just in case, do one clone for warmup 31 | git clone --quiet "$xv6_tmp" "$tmp_dir/xv6-copy" 32 | rm -rf "$tmp_dir/xv6-copy" 33 | 34 | time_file="/tmp/time" 35 | 36 | #echo "=== git clone ===" 37 | /usr/bin/time -f "clone real %e user %U sys %S" -o "$time_file" git clone --quiet "$xv6_tmp" xv6 38 | clone_time="$(cut -d ' ' -f3 <"$time_file")" 39 | cat "$time_file" 1>&2 40 | 41 | #echo "=== compile xv6 ===" 42 | cd xv6 43 | /usr/bin/time -f "compile real %e user %U sys %S" -o "$time_file" make --quiet kernel 44 | compile_time="$(cut -d ' ' -f3 <"$time_file")" 45 | cat "$time_file" 1>&2 46 | rm -f "$time_file" 47 | 48 | total_time=$(awk "BEGIN{ print $clone_time + $compile_time }") 49 | throughput=$(awk "BEGIN{ print 1.0 / $total_time }") 50 | echo "total real $total_time s" 1>&2 51 | echo "app-bench $throughput app/s" 52 | 53 | rm -rf "$xv6_tmp" 54 | -------------------------------------------------------------------------------- /kvs/kvs_test.go: -------------------------------------------------------------------------------- 1 | package kvs 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "testing" 8 | 9 | "github.com/goose-lang/primitive/disk" 10 | "github.com/mit-pdos/go-journal/common" 11 | ) 12 | 13 | const DISKSZ = 10000 14 | 15 | func mkdataval(b byte, sz uint64) []byte { 16 | data := make([]byte, sz) 17 | for i := range data { 18 | data[i] = b 19 | } 20 | return data 21 | } 22 | 23 | func TestGetAndPuts(t *testing.T) { 24 | fmt.Printf("TestGetAndPuts\n") 25 | 26 | os.Remove(DISKNAME) 27 | d, err := disk.NewFileDisk(DISKNAME, DISKSZ) 28 | if err != nil { 29 | panic(fmt.Errorf("could not create file disk: %v", err)) 30 | } 31 | kvs := MkKVS(d, DISKSZ) 32 | 33 | pairs := []KVPair{} 34 | keys := []uint64{} 35 | vals := [][]byte{} 36 | for i := 0; i < 10; i++ { 37 | keys = append(keys, common.LOGSIZE+uint64(i)) 38 | vals = append(vals, mkdataval(byte(i), disk.BlockSize)) 39 | pairs = append(pairs, KVPair{keys[i], vals[i]}) 40 | } 41 | 42 | ok := kvs.MultiPut(pairs) 43 | if !ok { 44 | log.Fatalf("Puts failed") 45 | } 46 | 47 | for i := 0; i < 10; i++ { 48 | p, ok := kvs.Get(keys[i]) 49 | if !ok { 50 | log.Fatalf("Get failed?") 51 | } 52 | for j := range p.Val { 53 | if p.Val[j] != vals[i][j] { 54 | log.Fatalf("%d: Got %d, expected %d", i, p.Val[j], vals[i][j]) 55 | } 56 | } 57 | } 58 | /*keys = append(keys, 12) 59 | if kvs.Get(keys[10]) != nil { 60 | log.Fatalf("Returned nonpresent key") 61 | }*/ 62 | kvs.Delete() 63 | err = os.Remove(DISKNAME) 64 | if err != nil { 65 | panic(err) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /artifact/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | # The most common configuration options are documented and commented below. 6 | # For a complete reference, please see the online documentation at 7 | # https://docs.vagrantup.com. 8 | 9 | config.vm.box = "bento/ubuntu-21.04" 10 | 11 | config.vm.hostname = "gojournal-vm" 12 | 13 | # support ssh -p 10322 vagrant@localhost to tunnel into guest 14 | config.vm.network "forwarded_port", guest: 22, host: 10322, host_ip: "127.0.0.1" 15 | 16 | # configure the VirtualBox VM 17 | config.vm.provider "virtualbox" do |vb| 18 | vb.name = "gojournal-vm" 19 | 20 | vb.memory = 8192 21 | vb.cpus = 4 22 | 23 | # Use Host I/O cache (speeds up disk performance) 24 | vb.customize ["storagectl", :id, "--name", "SATA Controller", 25 | "--hostiocache", "on"] 26 | # Mark disk as an SSD (enables TRIM support) 27 | vb.customize ["storageattach", :id, "--storagectl", "SATA Controller", 28 | "--port", "0", 29 | "--discard", "on", 30 | "--nonrotational", "on"] 31 | end 32 | 33 | # provision the machine 34 | # 35 | # NOTE: these scripts take 10-15 minutes to run 36 | config.vm.provision "shell", path: "vm-init.sh", privileged: false 37 | config.vm.provision "shell" do |s| 38 | s.path = "vm-setup.sh" 39 | s.privileged = false 40 | # you can also use -no-ocaml or -no-coq to skip those steps 41 | # (those are the slowest; FSCQ's dependencies take a lot of space) 42 | s.args = ["-no-fscq"] 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /eval/serial.patch: -------------------------------------------------------------------------------- 1 | diff --git a/wal/installer.go b/wal/installer.go 2 | index 24bcb55a..8cb9a4a0 100644 3 | --- a/wal/installer.go 4 | +++ b/wal/installer.go 5 | @@ -60,14 +60,14 @@ func (l *Walog) logInstall() (uint64, LogPosition) { 6 | return 0, installEnd 7 | } 8 | 9 | - l.memLock.Unlock() 10 | + //l.memLock.Unlock() 11 | 12 | util.DPrintf(5, "logInstall up to %d\n", installEnd) 13 | installBlocks(l.d, bufs) 14 | l.d.Barrier() 15 | Advance(l.d, installEnd) 16 | 17 | - l.memLock.Lock() 18 | + //l.memLock.Lock() 19 | l.st.cutMemLog(installEnd) 20 | l.condInstall.Broadcast() 21 | 22 | diff --git a/wal/logger.go b/wal/logger.go 23 | index fabc2700..ba11a347 100644 24 | --- a/wal/logger.go 25 | +++ b/wal/logger.go 26 | @@ -34,11 +34,11 @@ func (l *Walog) logAppend(circ *circularAppender) bool { 27 | if len(newbufs) == 0 { 28 | return false 29 | } 30 | - l.memLock.Unlock() 31 | + //l.memLock.Unlock() 32 | 33 | circ.Append(l.d, diskEnd, newbufs) 34 | 35 | - l.memLock.Lock() 36 | + //l.memLock.Lock() 37 | 38 | primitive.Linearize() 39 | 40 | diff --git a/wal/wal.go b/wal/wal.go 41 | index dbc69db0..19c15dce 100644 42 | --- a/wal/wal.go 43 | +++ b/wal/wal.go 44 | @@ -101,7 +101,9 @@ func (l *Walog) ReadInstalled(blkno common.Bnum) disk.Block { 45 | // linearize between the l.memLog.Unlock() and the eventual disk read, due to 46 | // potential concurrent cache or disk writes). 47 | func (l *Walog) Read(blkno common.Bnum) disk.Block { 48 | - blk, ok := l.ReadMem(blkno) 49 | + l.memLock.Lock() 50 | + defer l.memLock.Unlock() 51 | + blk, ok := l.st.readMem(blkno) 52 | if ok { 53 | return blk 54 | } 55 | -------------------------------------------------------------------------------- /cmd/lookup/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "runtime/pprof" 9 | "strconv" 10 | "time" 11 | 12 | go_nfs "github.com/mit-pdos/go-nfsd/nfs" 13 | "github.com/mit-pdos/go-nfsd/nfstypes" 14 | ) 15 | 16 | const BENCHDISKSZ uint64 = 100 * 1000 17 | 18 | var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") 19 | 20 | func main() { 21 | flag.Parse() 22 | if *cpuprofile != "" { 23 | f, err := os.Create(*cpuprofile) 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | pprof.StartCPUProfile(f) 28 | defer pprof.StopCPUProfile() 29 | } 30 | PLookup() 31 | } 32 | 33 | func Lookup(clnt *go_nfs.NfsClient, dirfh nfstypes.Nfs_fh3, name string) { 34 | reply := clnt.LookupOp(dirfh, name) 35 | if reply.Status != nfstypes.NFS3_OK { 36 | panic("Lookup") 37 | } 38 | fh := reply.Resok.Object 39 | attr := clnt.GetattrOp(fh) 40 | if attr.Status != nfstypes.NFS3_OK { 41 | panic("Lookup") 42 | } 43 | } 44 | 45 | func PLookup() { 46 | const N = 1 * time.Second 47 | const NTHREAD = 4 48 | for i := 1; i <= NTHREAD; i++ { 49 | res := go_nfs.Parallel(i, BENCHDISKSZ, 50 | func(clnt *go_nfs.NfsClient, dirfh nfstypes.Nfs_fh3) int { 51 | s := strconv.Itoa(i) 52 | name := "x" + s 53 | clnt.CreateOp(dirfh, name) 54 | start := time.Now() 55 | i := 0 56 | for true { 57 | Lookup(clnt, dirfh, name) 58 | i++ 59 | t := time.Now() 60 | elapsed := t.Sub(start) 61 | if elapsed >= N { 62 | break 63 | } 64 | } 65 | return i 66 | }) 67 | fmt.Printf("Lookup: %d file in %d usec with %d threads\n", 68 | res, N.Nanoseconds()/1e3, i) 69 | 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /bench/run-go-nfsd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Usage: ./run-go-nfsd.sh go run ./cmd/fs-smallfile/main.go 5 | # 6 | 7 | # taskset 0xc go run ./cmd/go-nfsd/ -disk /dev/shm/goose.img & 8 | 9 | # go run ./cmd/go-nfsd/ -disk /dev/shm/goose.img -cpuprofile=nfsd.prof & 10 | 11 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 12 | # root of repo 13 | cd "$DIR"/.. 14 | 15 | disk_file=/dev/shm/goose.img 16 | cpu_list="" 17 | extra_args=() 18 | while true; do 19 | case "$1" in 20 | -disk) 21 | shift 22 | disk_file="$1" 23 | shift 24 | ;; 25 | --cpu-list) 26 | shift 27 | cpu_list="$1" 28 | shift 29 | ;; 30 | # some argument in -foo=value syntax 31 | -*=*) 32 | extra_args+=("$1") 33 | shift 34 | ;; 35 | -*) 36 | extra_args+=("$1" "$2") 37 | shift 38 | shift 39 | ;; 40 | *) 41 | break 42 | ;; 43 | esac 44 | done 45 | 46 | set -eu 47 | 48 | if [ -e "$disk_file" ]; then 49 | dd status=none if=/dev/zero of="$disk_file" bs=4K count=200000 conv=notrunc 50 | sync "$disk_file" 51 | fi 52 | 53 | if [ -z "$cpu_list" ]; then 54 | ./bench/start-go-nfsd.sh -disk "$disk_file" "${extra_args[@]}" || exit 1 55 | else 56 | taskset --cpu-list "$cpu_list" ./bench/start-go-nfsd.sh -disk "$disk_file" "${extra_args[@]}" || exit 1 57 | fi 58 | 59 | function cleanup { 60 | ./bench/stop-go-nfsd.sh 61 | if [ -f "$disk_file" ]; then 62 | rm -f "$disk_file" 63 | fi 64 | } 65 | trap cleanup EXIT 66 | 67 | # taskset 0x3 $1 /mnt/nfs 68 | echo "# go-nfsd -disk $disk_file ${extra_args[*]}" 69 | echo "run $*" 1>&2 70 | "$@" 71 | -------------------------------------------------------------------------------- /cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | 7 | "github.com/mit-pdos/go-journal/util" 8 | ) 9 | 10 | type Cslot struct { 11 | Obj interface{} 12 | } 13 | 14 | type entry struct { 15 | slot Cslot 16 | lru *list.Element 17 | id uint64 18 | } 19 | 20 | type Cache struct { 21 | mu *sync.Mutex 22 | entries map[uint64]*entry 23 | lru *list.List 24 | sz uint64 25 | cnt uint64 26 | } 27 | 28 | func MkCache(sz uint64) *Cache { 29 | entries := make(map[uint64]*entry, sz) 30 | return &Cache{ 31 | mu: new(sync.Mutex), 32 | entries: entries, 33 | lru: list.New(), 34 | cnt: 0, 35 | sz: sz, 36 | } 37 | } 38 | 39 | func (c *Cache) PrintCache() { 40 | for k, v := range c.entries { 41 | util.DPrintf(0, "Entry %v %v\n", k, v) 42 | } 43 | } 44 | 45 | func (c *Cache) evict() { 46 | e := c.lru.Front() 47 | if e == nil { 48 | c.PrintCache() 49 | panic("evict") 50 | } 51 | entry := e.Value.(*entry) 52 | c.lru.Remove(e) 53 | util.DPrintf(5, "evict: %d\n", entry.id) 54 | delete(c.entries, entry.id) 55 | c.cnt = c.cnt - 1 56 | } 57 | 58 | func (c *Cache) LookupSlot(id uint64) *Cslot { 59 | c.mu.Lock() 60 | e := c.entries[id] 61 | if e != nil { 62 | if id != e.id { 63 | panic("LookupSlot") 64 | } 65 | if e.lru != nil { 66 | c.lru.Remove(e.lru) 67 | e.lru = c.lru.PushBack(e) 68 | } 69 | c.mu.Unlock() 70 | return &e.slot 71 | } 72 | if c.cnt >= c.sz { 73 | c.evict() 74 | } 75 | enew := &entry{ 76 | slot: Cslot{Obj: nil}, 77 | lru: nil, 78 | id: id, 79 | } 80 | c.entries[id] = enew 81 | enew.lru = c.lru.PushBack(enew) 82 | c.cnt = c.cnt + 1 83 | c.mu.Unlock() 84 | return &enew.slot 85 | } 86 | -------------------------------------------------------------------------------- /util/stats/stats.go: -------------------------------------------------------------------------------- 1 | // package stats tracks operation latencies 2 | package stats 3 | 4 | import ( 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "sync/atomic" 9 | "time" 10 | 11 | "github.com/rodaine/table" 12 | ) 13 | 14 | type Op struct { 15 | count uint32 16 | nanos uint64 17 | } 18 | 19 | func (op *Op) Record(start time.Time) { 20 | atomic.AddUint32(&op.count, 1) 21 | dur := time.Now().Sub(start) 22 | atomic.AddUint64(&op.nanos, uint64(dur.Nanoseconds())) 23 | } 24 | 25 | func (op *Op) Reset() { 26 | atomic.StoreUint32(&op.count, 0) 27 | atomic.StoreUint64(&op.nanos, 0) 28 | } 29 | 30 | func (op Op) MicrosPerOp() float64 { 31 | return float64(op.nanos) / float64(op.count) / 1e3 32 | } 33 | 34 | func WriteTable(names []string, ops []Op, w io.Writer) { 35 | if len(names) != len(ops) { 36 | panic("mismatched names and ops lists") 37 | } 38 | tbl := table.New("op", "count", "") 39 | loadedOps := make([]Op, len(ops)) 40 | var totalOp Op 41 | for i := range ops { 42 | op := Op{ 43 | count: atomic.LoadUint32(&ops[i].count), 44 | nanos: atomic.LoadUint64(&ops[i].nanos), 45 | } 46 | loadedOps[i] = op 47 | totalOp.count += op.count 48 | totalOp.nanos += op.nanos 49 | } 50 | for i, name := range names { 51 | op := loadedOps[i] 52 | if op.count > 0 { 53 | micros := fmt.Sprintf("%0.1f us/op", op.MicrosPerOp()) 54 | tbl.AddRow(name, op.count, micros) 55 | } 56 | } 57 | totalSeconds := float64(totalOp.nanos) / 1e9 58 | tbl.AddRow("total", totalOp.count, fmt.Sprintf("%0.1f s", totalSeconds)) 59 | tbl.WithWriter(w) 60 | tbl.Print() 61 | } 62 | 63 | func FormatTable(names []string, ops []Op) string { 64 | buf := new(bytes.Buffer) 65 | WriteTable(names, ops, buf) 66 | return buf.String() 67 | } 68 | -------------------------------------------------------------------------------- /kvs/kvs.go: -------------------------------------------------------------------------------- 1 | package kvs 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/goose-lang/primitive/disk" 7 | "github.com/mit-pdos/go-journal/addr" 8 | "github.com/mit-pdos/go-journal/common" 9 | "github.com/mit-pdos/go-journal/jrnl" 10 | "github.com/mit-pdos/go-journal/obj" 11 | "github.com/mit-pdos/go-journal/util" 12 | ) 13 | 14 | // 15 | // KVS using txns to implement multiput / multiget transactions 16 | // Keys == Block addresses 17 | // 18 | 19 | const DISKNAME string = "goose_kvs.img" 20 | 21 | type KVS struct { 22 | sz uint64 23 | log *obj.Log 24 | } 25 | 26 | type KVPair struct { 27 | Key uint64 28 | Val []byte 29 | } 30 | 31 | func MkKVS(d disk.Disk, sz uint64) *KVS { 32 | /*if sz > d.Size() { 33 | panic("kvs larger than disk") 34 | }*/ 35 | // XXX just need to assume that the kvs is less than the disk size? 36 | log := obj.MkLog(d) 37 | kvs := &KVS{ 38 | sz: sz, 39 | log: log, 40 | } 41 | return kvs 42 | } 43 | 44 | func (kvs *KVS) MultiPut(pairs []KVPair) bool { 45 | op := jrnl.Begin(kvs.log) 46 | for _, p := range pairs { 47 | if p.Key >= kvs.sz || p.Key < common.LOGSIZE { 48 | panic(fmt.Errorf("out-of-bounds put at %v", p.Key)) 49 | } 50 | akey := addr.MkAddr(p.Key, 0) 51 | op.OverWrite(akey, common.NBITBLOCK, p.Val) 52 | } 53 | ok := op.CommitWait(true) 54 | return ok 55 | } 56 | 57 | func (kvs *KVS) Get(key uint64) (*KVPair, bool) { 58 | if key > kvs.sz || key < common.LOGSIZE { 59 | panic(fmt.Errorf("out-of-bounds get at %v", key)) 60 | } 61 | op := jrnl.Begin(kvs.log) 62 | akey := addr.MkAddr(key, 0) 63 | data := util.CloneByteSlice(op.ReadBuf(akey, common.NBITBLOCK).Data) 64 | ok := op.CommitWait(true) 65 | return &KVPair{ 66 | Key: key, 67 | Val: data, 68 | }, ok 69 | } 70 | 71 | func (kvs *KVS) Delete() { 72 | kvs.log.Shutdown() 73 | } 74 | -------------------------------------------------------------------------------- /cmd/largefile/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "runtime/pprof" 9 | "time" 10 | 11 | "github.com/goose-lang/primitive/disk" 12 | 13 | "github.com/mit-pdos/go-nfsd/fh" 14 | go_nfs "github.com/mit-pdos/go-nfsd/nfs" 15 | "github.com/mit-pdos/go-nfsd/nfstypes" 16 | ) 17 | 18 | const ( 19 | FILESIZE uint64 = 50 * 1024 * 1024 20 | WSIZE uint64 = disk.BlockSize 21 | MB uint64 = 1024 * 1024 22 | BENCHDISKSZ uint64 = 100 * 1000 23 | ) 24 | 25 | var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") 26 | 27 | func main() { 28 | flag.Parse() 29 | if *cpuprofile != "" { 30 | f, err := os.Create(*cpuprofile) 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | pprof.StartCPUProfile(f) 35 | defer pprof.StopCPUProfile() 36 | } 37 | largeFile() 38 | } 39 | 40 | func mkdata(sz uint64) []byte { 41 | data := make([]byte, sz) 42 | for i := range data { 43 | data[i] = byte(i % 128) 44 | } 45 | return data 46 | } 47 | 48 | func largeFile() { 49 | data := mkdata(WSIZE) 50 | clnt := go_nfs.MkNfsClient(BENCHDISKSZ) 51 | defer clnt.Shutdown() 52 | dir := fh.MkRootFh3() 53 | 54 | start := time.Now() 55 | 56 | name := "largefile" 57 | clnt.CreateOp(dir, name) 58 | reply := clnt.LookupOp(dir, name) 59 | fh := reply.Resok.Object 60 | n := FILESIZE / WSIZE 61 | for j := uint64(0); j < n; j++ { 62 | clnt.WriteOp(fh, j*WSIZE, data, nfstypes.UNSTABLE) 63 | } 64 | clnt.CommitOp(fh, n*WSIZE) 65 | attr := clnt.GetattrOp(fh) 66 | if uint64(attr.Resok.Obj_attributes.Size) != FILESIZE { 67 | panic("large") 68 | } 69 | 70 | t := time.Now() 71 | elapsed := t.Sub(start) 72 | tput := float64(FILESIZE/MB) / elapsed.Seconds() 73 | fmt.Printf("largefile: %v MB throughput %.2f MB/s\n", FILESIZE/MB, tput) 74 | 75 | clnt.RemoveOp(dir, name) 76 | } 77 | -------------------------------------------------------------------------------- /super/super.go: -------------------------------------------------------------------------------- 1 | package super 2 | 3 | import ( 4 | "github.com/goose-lang/primitive/disk" 5 | 6 | "github.com/mit-pdos/go-journal/addr" 7 | "github.com/mit-pdos/go-journal/common" 8 | ) 9 | 10 | type FsSuper struct { 11 | Disk disk.Disk 12 | Size uint64 13 | nLog uint64 // including commit block 14 | NBlockBitmap uint64 15 | NInodeBitmap uint64 16 | nInodeBlk uint64 17 | Maxaddr uint64 18 | } 19 | 20 | func MkFsSuper(d disk.Disk) *FsSuper { 21 | sz := d.Size() 22 | nblockbitmap := (sz / common.NBITBLOCK) + 1 23 | 24 | return &FsSuper{ 25 | Disk: d, 26 | Size: sz, 27 | nLog: common.LOGSIZE, 28 | NBlockBitmap: nblockbitmap, 29 | NInodeBitmap: common.NINODEBITMAP, 30 | nInodeBlk: (common.NINODEBITMAP * common.NBITBLOCK * common.INODESZ) / disk.BlockSize, 31 | Maxaddr: sz} 32 | } 33 | 34 | func (fs *FsSuper) MaxBnum() common.Bnum { 35 | return common.Bnum(fs.Maxaddr) 36 | } 37 | 38 | func (fs *FsSuper) BitmapBlockStart() common.Bnum { 39 | return common.Bnum(fs.nLog) 40 | } 41 | 42 | func (fs *FsSuper) BitmapInodeStart() common.Bnum { 43 | return fs.BitmapBlockStart() + common.Bnum(fs.NBlockBitmap) 44 | } 45 | 46 | func (fs *FsSuper) InodeStart() common.Bnum { 47 | return fs.BitmapInodeStart() + common.Bnum(fs.NInodeBitmap) 48 | } 49 | 50 | func (fs *FsSuper) DataStart() common.Bnum { 51 | return fs.InodeStart() + common.Bnum(fs.nInodeBlk) 52 | } 53 | 54 | func (fs *FsSuper) Block2addr(blkno common.Bnum) addr.Addr { 55 | return addr.MkAddr(blkno, 0) 56 | } 57 | 58 | func (fs *FsSuper) NInode() common.Inum { 59 | return common.Inum(fs.nInodeBlk * common.INODEBLK) 60 | } 61 | 62 | func (fs *FsSuper) Inum2Addr(inum common.Inum) addr.Addr { 63 | return addr.MkAddr(fs.InodeStart()+common.Bnum(uint64(inum)/common.INODEBLK), 64 | (uint64(inum)%common.INODEBLK)*common.INODESZ*8) 65 | } 66 | -------------------------------------------------------------------------------- /cmd/fs-largefile/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path" 8 | "time" 9 | ) 10 | 11 | const ( 12 | MB uint64 = 1024 * 1024 13 | WSIZE = 16 * 4096 14 | ) 15 | 16 | // makefile directly reports throughput in MB/s 17 | func makefile(name string, data []byte, size uint64) float64 { 18 | start := time.Now() 19 | f, err := os.Create(name) 20 | if err != nil { 21 | panic(err) 22 | } 23 | for i := uint64(0); i < size/WSIZE; i++ { 24 | _, err = f.Write(data) 25 | if err != nil { 26 | panic(err) 27 | } 28 | } 29 | err = f.Sync() 30 | if err != nil { 31 | panic(err) 32 | } 33 | err = f.Close() 34 | if err != nil { 35 | panic(err) 36 | } 37 | elapsed := time.Now().Sub(start) 38 | return float64(size) / float64(MB) / elapsed.Seconds() 39 | } 40 | 41 | func mkdata(sz uint64) []byte { 42 | data := make([]byte, sz) 43 | for i := range data { 44 | data[i] = byte(i % 128) 45 | } 46 | return data 47 | } 48 | 49 | func main() { 50 | mnt := flag.String("mnt", "/mnt/nfs", "directory to write files to") 51 | sizeMB := flag.Uint64("file-size", 100, "file size (in MB)") 52 | deleteAfter := flag.Bool("delete", false, "delete files after running benchmark") 53 | warmup := flag.Bool("warmup", true, "run warmup first") 54 | flag.Parse() 55 | 56 | warmupFile := path.Join(*mnt, "large.warmup") 57 | file := path.Join(*mnt, "large") 58 | filesize := *sizeMB * MB 59 | warmupsize := 100 * MB 60 | if filesize < warmupsize { 61 | warmupsize = filesize 62 | } 63 | 64 | data := mkdata(WSIZE) 65 | 66 | if *warmup { 67 | tput := makefile(warmupFile, data, warmupsize) 68 | fmt.Printf("# warmup %d MB throughput %.2f MB/s\n", warmupsize/MB, tput) 69 | } 70 | 71 | tput := makefile(file, data, filesize) 72 | fmt.Printf("fs-largefile: %v MB throughput %.2f MB/s\n", filesize/MB, tput) 73 | 74 | if *deleteAfter { 75 | os.Remove(warmupFile) 76 | os.Remove(file) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /dir/dcache.go: -------------------------------------------------------------------------------- 1 | package dir 2 | 3 | import ( 4 | "github.com/mit-pdos/go-journal/common" 5 | "github.com/mit-pdos/go-nfsd/dcache" 6 | "github.com/mit-pdos/go-nfsd/fstxn" 7 | "github.com/mit-pdos/go-nfsd/inode" 8 | "github.com/mit-pdos/go-nfsd/nfstypes" 9 | ) 10 | 11 | func mkDcache(dip *inode.Inode, op *fstxn.FsTxn) { 12 | dip.Dcache = dcache.MkDcache() 13 | Apply(dip, op, 0, dip.Size, 100000000, 14 | func(ip *inode.Inode, name string, inum common.Inum, off uint64) { 15 | dip.Dcache.Add(name, inum, off) 16 | }) 17 | } 18 | 19 | func LookupName(dip *inode.Inode, op *fstxn.FsTxn, name nfstypes.Filename3) (common.Inum, uint64) { 20 | if dip.Kind != nfstypes.NF3DIR { 21 | return common.NULLINUM, 0 22 | } 23 | var inum = common.NULLINUM 24 | var finalOffset uint64 = 0 25 | if dip.Dcache == nil { 26 | mkDcache(dip, op) 27 | } 28 | dentry, ok := dip.Dcache.Lookup(string(name)) 29 | if ok { 30 | inum = dentry.Inum 31 | finalOffset = dentry.Off 32 | } 33 | return inum, finalOffset 34 | } 35 | 36 | func AddName(dip *inode.Inode, op *fstxn.FsTxn, inum common.Inum, name nfstypes.Filename3) bool { 37 | if dip.Kind != nfstypes.NF3DIR || uint64(len(name)) >= MAXNAMELEN { 38 | return false 39 | } 40 | if dip.Dcache == nil { 41 | mkDcache(dip, op) 42 | } 43 | off, ok := AddNameDir(dip, op, inum, name, dip.Dcache.Lastoff) 44 | if ok { 45 | dip.Dcache.Lastoff = off 46 | dip.Dcache.Add(string(name), inum, off) 47 | } 48 | return ok 49 | } 50 | 51 | func RemName(dip *inode.Inode, op *fstxn.FsTxn, name nfstypes.Filename3) bool { 52 | if dip.Kind != nfstypes.NF3DIR || uint64(len(name)) >= MAXNAMELEN { 53 | return false 54 | } 55 | if dip.Dcache == nil { 56 | mkDcache(dip, op) 57 | } 58 | off, ok := RemNameDir(dip, op, name) 59 | if ok { 60 | dip.Dcache.Lastoff = off 61 | ok := dip.Dcache.Del(string(name)) 62 | if !ok { 63 | panic("RemName") 64 | } 65 | return true 66 | } 67 | return false 68 | } 69 | -------------------------------------------------------------------------------- /nfs/lorder.go: -------------------------------------------------------------------------------- 1 | package nfs 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/mit-pdos/go-journal/common" 7 | "github.com/mit-pdos/go-journal/util" 8 | "github.com/mit-pdos/go-nfsd/dir" 9 | "github.com/mit-pdos/go-nfsd/fh" 10 | "github.com/mit-pdos/go-nfsd/fstxn" 11 | "github.com/mit-pdos/go-nfsd/inode" 12 | "github.com/mit-pdos/go-nfsd/nfstypes" 13 | ) 14 | 15 | // Lock inodes in sorted order, but return the pointers in the same order as in inums 16 | // Caller must revalidate inodes. 17 | func lockInodes(op *fstxn.FsTxn, inums []common.Inum) []*inode.Inode { 18 | util.DPrintf(1, "lock inodes %v\n", inums) 19 | sorted := make([]common.Inum, len(inums)) 20 | copy(sorted, inums) 21 | sort.Slice(sorted, func(i, j int) bool { return inums[i] < inums[j] }) 22 | var inodes = make([]*inode.Inode, len(inums)) 23 | for _, inm := range sorted { 24 | ip := op.GetInodeInum(inm) 25 | if ip == nil { 26 | op.Abort() 27 | return nil 28 | } 29 | // put in same position as in inums 30 | pos := func(inm common.Inum) int { 31 | for i, v := range inums { 32 | if v == inm { 33 | return i 34 | } 35 | } 36 | panic("func") 37 | }(inm) 38 | inodes[pos] = ip 39 | } 40 | return inodes 41 | } 42 | 43 | func twoInums(inum1, inum2 common.Inum) []common.Inum { 44 | inums := make([]common.Inum, 2) 45 | inums[0] = inum1 46 | inums[1] = inum2 47 | return inums 48 | } 49 | 50 | // First lookup inode up for child, then for parent, because parent 51 | // inum > child inum and then revalidate that child is still in parent 52 | // directory. 53 | func lookupOrdered(op *fstxn.FsTxn, name nfstypes.Filename3, parent fh.Fh, inm common.Inum) []*inode.Inode { 54 | util.DPrintf(5, "NFS lookupOrdered child %d parent %v\n", inm, parent) 55 | inodes := lockInodes(op, twoInums(inm, parent.Ino)) 56 | if inodes == nil { 57 | return nil 58 | } 59 | dip := inodes[1] 60 | if dip.Gen != parent.Gen { 61 | op.Abort() 62 | return nil 63 | } 64 | child, _ := dir.LookupName(dip, op, name) 65 | if child == common.NULLINUM || child != inm { 66 | op.Abort() 67 | return nil 68 | } 69 | return inodes 70 | } 71 | -------------------------------------------------------------------------------- /eval/global-txn-lock.patch: -------------------------------------------------------------------------------- 1 | diff --git a/txn/txn.go b/txn/txn.go 2 | index bcdbca4..6c1cb4c 100644 3 | --- a/txn/txn.go 4 | +++ b/txn/txn.go 5 | @@ -6,30 +6,31 @@ 6 | package txn 7 | 8 | import ( 9 | + "sync" 10 | + 11 | "github.com/goose-lang/primitive/disk" 12 | 13 | "github.com/mit-pdos/go-journal/addr" 14 | "github.com/mit-pdos/go-journal/jrnl" 15 | - "github.com/mit-pdos/go-journal/lockmap" 16 | "github.com/mit-pdos/go-journal/obj" 17 | "github.com/mit-pdos/go-journal/util" 18 | ) 19 | 20 | type Log struct { 21 | - log *obj.Log 22 | - locks *lockmap.LockMap 23 | + log *obj.Log 24 | + lock *sync.Mutex 25 | } 26 | 27 | type Txn struct { 28 | buftxn *jrnl.Op 29 | - locks *lockmap.LockMap 30 | - acquired map[uint64]bool 31 | + m *sync.Mutex 32 | + acquired map[uint64]bool // unused 33 | } 34 | 35 | func Init(d disk.Disk) *Log { 36 | twophasePre := &Log{ 37 | - log: obj.MkLog(d), 38 | - locks: lockmap.MkLockMap(), 39 | + log: obj.MkLog(d), 40 | + lock: new(sync.Mutex), 41 | } 42 | return twophasePre 43 | } 44 | @@ -38,10 +39,11 @@ func Init(d disk.Disk) *Log { 45 | func Begin(tsys *Log) *Txn { 46 | trans := &Txn{ 47 | buftxn: jrnl.Begin(tsys.log), 48 | - locks: tsys.locks, 49 | + m: tsys.lock, 50 | acquired: make(map[uint64]bool), 51 | } 52 | util.DPrintf(5, "tp Begin: %v\n", trans) 53 | + trans.m.Lock() 54 | return trans 55 | } 56 | 57 | @@ -50,14 +52,10 @@ func (tsys *Log) Flush() { 58 | } 59 | 60 | func (txn *Txn) acquireNoCheck(addr addr.Addr) { 61 | - flatAddr := addr.Flatid() 62 | - txn.locks.Acquire(flatAddr) 63 | - txn.acquired[flatAddr] = true 64 | } 65 | 66 | func (txn *Txn) isAlreadyAcquired(addr addr.Addr) bool { 67 | - flatAddr := addr.Flatid() 68 | - return txn.acquired[flatAddr] 69 | + return true 70 | } 71 | 72 | func (txn *Txn) Acquire(addr addr.Addr) { 73 | @@ -68,9 +66,7 @@ func (txn *Txn) Acquire(addr addr.Addr) { 74 | } 75 | 76 | func (txn *Txn) ReleaseAll() { 77 | - for flatAddr := range txn.acquired { 78 | - txn.locks.Release(flatAddr) 79 | - } 80 | + txn.m.Unlock() 81 | } 82 | 83 | func (txn *Txn) readBufNoAcquire(addr addr.Addr, sz uint64) []byte { 84 | -------------------------------------------------------------------------------- /nfs/nfs_ls.go: -------------------------------------------------------------------------------- 1 | package nfs 2 | 3 | import ( 4 | "github.com/mit-pdos/go-journal/common" 5 | "github.com/mit-pdos/go-nfsd/dir" 6 | "github.com/mit-pdos/go-nfsd/fh" 7 | "github.com/mit-pdos/go-nfsd/fstxn" 8 | "github.com/mit-pdos/go-nfsd/inode" 9 | "github.com/mit-pdos/go-nfsd/nfstypes" 10 | ) 11 | 12 | func Ls3(dip *inode.Inode, op *fstxn.FsTxn, start nfstypes.Cookie3, dircount, maxcount nfstypes.Count3) nfstypes.Dirlistplus3 { 13 | var lst *nfstypes.Entryplus3 14 | var last *nfstypes.Entryplus3 15 | eof := dir.Apply(dip, op, uint64(start), uint64(dircount), uint64(maxcount), 16 | func(ip *inode.Inode, name string, inum common.Inum, off uint64) { 17 | fattr := ip.MkFattr() 18 | fh := &fh.Fh{Ino: ip.Inum, Gen: ip.Gen} 19 | ph := nfstypes.Post_op_fh3{ 20 | Handle_follows: true, 21 | Handle: fh.MakeFh3(), 22 | } 23 | pa := nfstypes.Post_op_attr{ 24 | Attributes_follow: true, 25 | Attributes: fattr, 26 | } 27 | e := &nfstypes.Entryplus3{ 28 | Fileid: nfstypes.Fileid3(inum), 29 | Name: nfstypes.Filename3(name), 30 | Cookie: nfstypes.Cookie3(off), 31 | Name_attributes: pa, 32 | Name_handle: ph, 33 | Nextentry: nil, 34 | } 35 | if last == nil { 36 | lst = e 37 | last = e 38 | } else { 39 | last.Nextentry = e 40 | last = e 41 | } 42 | }) 43 | dl := nfstypes.Dirlistplus3{Entries: lst, Eof: eof} 44 | return dl 45 | } 46 | 47 | func Readdir3(dip *inode.Inode, op *fstxn.FsTxn, 48 | start nfstypes.Cookie3, count nfstypes.Count3) nfstypes.Dirlist3 { 49 | var lst *nfstypes.Entry3 50 | var last *nfstypes.Entry3 51 | eof := dir.ApplyEnts(dip, op, uint64(start), uint64(count), 52 | func(name string, inum common.Inum, off uint64) { 53 | e := &nfstypes.Entry3{ 54 | Fileid: nfstypes.Fileid3(inum), 55 | Name: nfstypes.Filename3(name), 56 | Cookie: nfstypes.Cookie3(off), 57 | Nextentry: nil, 58 | } 59 | if last == nil { 60 | lst = e 61 | last = e 62 | } else { 63 | last.Nextentry = e 64 | last = e 65 | } 66 | }) 67 | dl := nfstypes.Dirlist3{Entries: lst, Eof: eof} 68 | return dl 69 | } 70 | -------------------------------------------------------------------------------- /cmd/smallfile/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "runtime/pprof" 9 | "strconv" 10 | "time" 11 | 12 | go_nfs "github.com/mit-pdos/go-nfsd/nfs" 13 | "github.com/mit-pdos/go-nfsd/nfstypes" 14 | ) 15 | 16 | const BENCHDISKSZ uint64 = 100 * 1000 17 | 18 | var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") 19 | 20 | func main() { 21 | flag.Parse() 22 | if *cpuprofile != "" { 23 | f, err := os.Create(*cpuprofile) 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | pprof.StartCPUProfile(f) 28 | defer pprof.StopCPUProfile() 29 | } 30 | PSmallFile() 31 | } 32 | 33 | func SmallFile(clnt *go_nfs.NfsClient, dirfh nfstypes.Nfs_fh3, name string, data []byte) { 34 | reply := clnt.LookupOp(dirfh, name) 35 | if reply.Status == nfstypes.NFS3_OK { 36 | panic("SmallFile") 37 | } 38 | clnt.CreateOp(dirfh, name) 39 | reply = clnt.LookupOp(dirfh, name) 40 | if reply.Status != nfstypes.NFS3_OK { 41 | panic("SmallFile") 42 | } 43 | attr := clnt.GetattrOp(reply.Resok.Object) 44 | if attr.Status != nfstypes.NFS3_OK { 45 | panic("SmallFile") 46 | } 47 | clnt.WriteOp(reply.Resok.Object, 0, data, nfstypes.FILE_SYNC) 48 | attr = clnt.GetattrOp(reply.Resok.Object) 49 | if attr.Status != nfstypes.NFS3_OK { 50 | panic("SmallFile") 51 | } 52 | res := clnt.RemoveOp(dirfh, name) 53 | if res.Status != nfstypes.NFS3_OK { 54 | panic("SmallFile") 55 | } 56 | } 57 | 58 | func mkdata(sz uint64) []byte { 59 | data := make([]byte, sz) 60 | for i := range data { 61 | data[i] = byte(i % 128) 62 | } 63 | return data 64 | } 65 | 66 | func PSmallFile() { 67 | const N = 10 * time.Second 68 | const NTHREAD = 20 69 | for i := 1; i <= NTHREAD; i++ { 70 | res := go_nfs.Parallel(i, BENCHDISKSZ, 71 | func(clnt *go_nfs.NfsClient, dirfh nfstypes.Nfs_fh3) int { 72 | data := mkdata(uint64(100)) 73 | start := time.Now() 74 | i := 0 75 | for true { 76 | s := strconv.Itoa(i) 77 | SmallFile(clnt, dirfh, "x"+s, data) 78 | i++ 79 | t := time.Now() 80 | elapsed := t.Sub(start) 81 | if elapsed >= N { 82 | break 83 | } 84 | } 85 | return i 86 | }) 87 | fmt.Printf("smallfile: %v file/s with %d threads\n", 88 | float64(res)/N.Seconds(), i) 89 | 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /cmd/clnt-null/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "math/rand" 7 | "net" 8 | "time" 9 | 10 | "github.com/zeldovich/go-rpcgen/rfc1057" 11 | "github.com/zeldovich/go-rpcgen/rfc1813" 12 | "github.com/zeldovich/go-rpcgen/xdr" 13 | ) 14 | 15 | var N time.Duration 16 | 17 | func pmap_client(host string, prog, vers uint32) *rfc1057.Client { 18 | var cred rfc1057.Opaque_auth 19 | cred.Flavor = rfc1057.AUTH_NONE 20 | 21 | pmapc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", host, rfc1057.PMAP_PORT)) 22 | if err != nil { 23 | panic(err) 24 | } 25 | defer pmapc.Close() 26 | pmap := rfc1057.MakeClient(pmapc, rfc1057.PMAP_PROG, rfc1057.PMAP_VERS) 27 | 28 | arg := rfc1057.Mapping{ 29 | Prog: prog, 30 | Vers: vers, 31 | Prot: rfc1057.IPPROTO_TCP, 32 | } 33 | var res xdr.Uint32 34 | err = pmap.Call(rfc1057.PMAPPROC_GETPORT, cred, cred, &arg, &res) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | svcc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", host, res)) 40 | if err != nil { 41 | panic(err) 42 | } 43 | return rfc1057.MakeClient(svcc, prog, vers) 44 | } 45 | 46 | type nfsclnt struct { 47 | clnt *rfc1057.Client 48 | cred rfc1057.Opaque_auth 49 | verf rfc1057.Opaque_auth 50 | } 51 | 52 | func (c *nfsclnt) null() { 53 | var arg xdr.Void 54 | var res xdr.Void 55 | 56 | err := c.clnt.Call(rfc1813.NFSPROC3_NULL, c.cred, c.verf, &arg, &res) 57 | if err != nil { 58 | panic(err) 59 | } 60 | } 61 | 62 | func client(cred_unix rfc1057.Opaque_auth) (n int, elapsed time.Duration) { 63 | var cred_none rfc1057.Opaque_auth 64 | cred_none.Flavor = rfc1057.AUTH_NONE 65 | nfs := pmap_client("localhost", rfc1813.NFS_PROGRAM, rfc1813.NFS_V3) 66 | clnt := &nfsclnt{clnt: nfs, cred: cred_unix, verf: cred_none} 67 | start := time.Now() 68 | for { 69 | clnt.null() 70 | n++ 71 | elapsed = time.Now().Sub(start) 72 | if elapsed >= N { 73 | return 74 | } 75 | } 76 | } 77 | 78 | func main() { 79 | flag.DurationVar(&N, "benchtime", 10*time.Second, "time to run each iteration for") 80 | flag.Parse() 81 | 82 | var err error 83 | 84 | var unix rfc1057.Auth_unix 85 | var cred_unix rfc1057.Opaque_auth 86 | cred_unix.Flavor = rfc1057.AUTH_UNIX 87 | cred_unix.Body, err = xdr.EncodeBuf(&unix) 88 | if err != nil { 89 | panic(err) 90 | } 91 | 92 | rand.Seed(time.Now().UnixNano()) 93 | 94 | n, elapsed := client(cred_unix) 95 | fmt.Printf("null-bench: NULL takes %.1f us\n", float64(elapsed.Microseconds())/float64(n)) 96 | } 97 | -------------------------------------------------------------------------------- /eval/scale.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import re 5 | from os.path import join 6 | 7 | import argparse 8 | import pandas as pd 9 | 10 | 11 | def parse_raw(lines): 12 | fs = None 13 | data = [] 14 | 15 | def get_bench_data(pattern, line): 16 | m = re.match(pattern, line) 17 | if m: 18 | return { 19 | "fs": fs, 20 | "bench": m.group("bench"), 21 | "val": float(m.group("val")), 22 | } 23 | return None 24 | 25 | for line in lines: 26 | if re.match(r"""^#""", line): 27 | continue 28 | m = re.match(r"""fs=(?P.*)""", line) 29 | if m: 30 | fs = m.group("fs") 31 | continue 32 | m = re.match( 33 | r"""fs-smallfile: (?P\d*) (?P[0-9.]*) file/sec""", 34 | line, 35 | ) 36 | if m: 37 | data.append( 38 | { 39 | "fs": fs, 40 | "clients": int(m.group("clients")), 41 | "throughput": float(m.group("val")), 42 | } 43 | ) 44 | continue 45 | print("ignored line: " + line, end="", file=sys.stderr) 46 | 47 | return pd.DataFrame.from_records(data) 48 | 49 | 50 | def from_tidy(tidy_df): 51 | df = tidy_df.pivot_table(index="clients", columns="fs", values="throughput") 52 | # propagate serial results forward 53 | df.fillna(method="ffill", inplace=True) 54 | return df 55 | 56 | 57 | def main(): 58 | parser = argparse.ArgumentParser() 59 | parser.add_argument( 60 | "-o", 61 | "--output", 62 | default="data", 63 | help="directory to output *.data files to", 64 | ) 65 | parser.add_argument( 66 | "bench", type=argparse.FileType("r"), help="path to scale.sh raw output" 67 | ) 68 | 69 | args = parser.parse_args() 70 | 71 | df = from_tidy(parse_raw(args.bench)) 72 | with open(join(args.output, "gnfs.data"), "w") as f: 73 | print(df["gonfs"].to_csv(sep="\t", header=False), end="", file=f) 74 | with open(join(args.output, "linux-nfs.data"), "w") as f: 75 | print(df["linux"].to_csv(sep="\t", header=False), end="", file=f) 76 | with open(join(args.output, "serial.data"), "w") as f: 77 | print(df["serial-gonfs"].to_csv(sep="\t", header=False), end="", file=f) 78 | 79 | 80 | if __name__ == "__main__": 81 | main() 82 | -------------------------------------------------------------------------------- /eval/bench.plot: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | output=fig/bench.pdf 6 | input=data/bench.data 7 | ssd=false 8 | 9 | usage() { 10 | echo "Usage: $0 [--input INPUT] [--output OUTPUT] [-ssd]" 1>&2 11 | echo "defaults to plotting RAM benchmarks from data/bench.data" 1>&2 12 | } 13 | 14 | while [[ "$#" -gt 0 ]]; do 15 | case "$1" in 16 | -i | --input) 17 | shift 18 | input="$1" 19 | shift 20 | ;; 21 | -o | --output) 22 | shift 23 | output="$1" 24 | shift 25 | ;; 26 | -ssd) 27 | ssd=true 28 | shift 29 | ;; 30 | -h | -help | --help) 31 | usage 32 | exit 0 33 | ;; 34 | *) 35 | echo "unknown option $1" 1>&2 36 | usage 37 | exit 1 38 | ;; 39 | esac 40 | done 41 | 42 | # shellcheck disable=SC2016 43 | if [ "$ssd" = "true" ]; then 44 | line1='column("linux-ssd")/column("linux-ssd")' 45 | line2='column("gonfs-ssd")/column("linux-ssd")' 46 | label=" (SSD)" 47 | label1=$(awk 'NR==2 {printf "%.0f", $4}' "$input") 48 | label2=$(awk 'NR==3 {printf "%.0f", $4}' "$input") 49 | label3=$(awk 'NR==4 {printf "%.3f", $4}' "$input") 50 | else 51 | label="" 52 | line1='column("linux")/column("linux")' 53 | line2='column("gonfs")/column("linux")' 54 | label1=$(awk 'NR==2 {printf "%.0f", $2}' "$input") 55 | label2=$(awk 'NR==3 {printf "%.0f", $2}' "$input") 56 | label3=$(awk 'NR==4 {printf "%0.3f", $2}' "$input") 57 | fi 58 | 59 | gnuplot <<-EOF 60 | set terminal pdf dashed noenhanced size 3.5in,1.5in 61 | set output "${output}" 62 | 63 | set style data histogram 64 | set style histogram cluster gap 1 65 | set rmargin at screen .95 66 | 67 | set xrange [-1:4] 68 | set yrange [0:1.45] 69 | set grid y 70 | set ylabel "Relative througput" 71 | set ytics scale 0.5,0 nomirror 72 | set xtics scale 0,0 73 | set key top right 74 | set style fill solid 1 border rgb "black" 75 | 76 | set label '${label1} file/s' at (0.15 -4./7),1.1 right rotate by 90 offset character 0,-1 77 | set label '${label2} MB/s' at (1.15 -4./7),1.1 right rotate by 90 offset character 0,-1 78 | set label '${label3} app/s' at (2.15 -4./7),1.1 right rotate by 90 offset character 0,-1 79 | 80 | set datafile separator "\t" 81 | 82 | plot "${input}" \ 83 | using (${line1}):xtic(1) title "Linux${label}" lc rgb '#b6d7a8' lt 1, \ 84 | '' using (${line2}):xtic(1) title "GoNFS${label}" lc rgb '#3a81ba' lt 1 85 | EOF 86 | -------------------------------------------------------------------------------- /eval/scale.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | blue=$(tput setaf 4) 6 | red=$(tput setaf 1) 7 | reset=$(tput sgr0) 8 | 9 | info() { 10 | echo -e "${blue}$1${reset}" 1>&2 11 | } 12 | 13 | error() { 14 | echo -e "${red}$1${reset}" 1>&2 15 | } 16 | 17 | if [ ! -d "$GO_NFSD_PATH" ]; then 18 | echo "\$GO_NFSD_PATH is unset" 1>&2 19 | exit 1 20 | fi 21 | 22 | if [ ! -d "$GO_JOURNAL_PATH" ]; then 23 | echo "\$GO_JOURNAL_PATH is unset" 1>&2 24 | exit 1 25 | fi 26 | 27 | help() { 28 | echo "Usage: $0 [-disk ] [threads]" 29 | echo "disk defaults to ~/disk.img (assuming the root file system is on an SSD)" 30 | echo "threads defaults to 10" 31 | } 32 | 33 | output_file="eval/data/scale-raw.txt" 34 | disk_file="$HOME/disk.img" 35 | while true; do 36 | case "$1" in 37 | -disk) 38 | shift 39 | disk_file="$1" 40 | shift 41 | ;; 42 | -o | --output) 43 | shift 44 | output_file="$1" 45 | shift 46 | ;; 47 | -help | --help) 48 | help 49 | exit 0 50 | ;; 51 | -*) 52 | error "unexpected flag $1" 53 | help 54 | exit 1 55 | ;; 56 | *) 57 | break 58 | ;; 59 | esac 60 | done 61 | 62 | threads=12 63 | if [[ $# -gt 0 ]]; then 64 | threads="$1" 65 | fi 66 | 67 | cd "$GO_NFSD_PATH" 68 | 69 | do_eval() { 70 | info "GoNFS smallfile scalability" 71 | echo "fs=gonfs" 72 | ./bench/run-go-nfsd.sh -disk "$disk_file" go run ./cmd/fs-smallfile -threads="$threads" 73 | 74 | echo 1>&2 75 | info "Linux smallfile scalability" 76 | echo "fs=linux" 77 | ./bench/run-linux.sh -disk "$disk_file" go run ./cmd/fs-smallfile -threads="$threads" 78 | 79 | echo 1>&2 80 | info "Serial GoNFS (holding locks)" 81 | 82 | # we change the local checkout of go-journal 83 | pushd "$GO_JOURNAL_PATH" >/dev/null 84 | git apply "$GO_NFSD_PATH/eval/serial.patch" 85 | popd >/dev/null 86 | # ... and then also point go-nfsd to the local version 87 | go mod edit -replace github.com/mit-pdos/go-journal="$GO_JOURNAL_PATH" 88 | 89 | echo "fs=serial-gonfs" 90 | ./bench/run-go-nfsd.sh -disk "$disk_file" go run ./cmd/fs-smallfile -start=1 -threads="$threads" 91 | 92 | go mod edit -dropreplace github.com/mit-pdos/go-journal 93 | pushd "$GO_JOURNAL_PATH" >/dev/null 94 | git restore wal/installer.go wal/logger.go wal/wal.go 95 | popd >/dev/null 96 | } 97 | 98 | if [ "$output_file" = "-" ]; then 99 | do_eval 100 | else 101 | do_eval | tee "$output_file" 102 | fi 103 | -------------------------------------------------------------------------------- /shrinker/shrinker.go: -------------------------------------------------------------------------------- 1 | package shrinker 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/mit-pdos/go-journal/common" 7 | "github.com/mit-pdos/go-journal/util" 8 | "github.com/mit-pdos/go-nfsd/fstxn" 9 | ) 10 | 11 | type ShrinkerSt struct { 12 | mu *sync.Mutex 13 | condShut *sync.Cond 14 | nthread uint32 15 | fsstate *fstxn.FsState 16 | crash bool 17 | } 18 | 19 | func MkShrinkerSt(st *fstxn.FsState) *ShrinkerSt { 20 | mu := new(sync.Mutex) 21 | shrinkst := &ShrinkerSt{ 22 | mu: mu, 23 | condShut: sync.NewCond(mu), 24 | nthread: 0, 25 | fsstate: st, 26 | crash: false, 27 | } 28 | return shrinkst 29 | } 30 | 31 | func (shrinkst *ShrinkerSt) crashed() bool { 32 | shrinkst.mu.Lock() 33 | crashed := shrinkst.crash 34 | shrinkst.mu.Unlock() 35 | return crashed 36 | } 37 | 38 | // If caller changes file size and shrinking is in progress (because 39 | // an earlier call truncated the file), then help/wait with/for 40 | // shrinking. Also, called by shrinker. 41 | func (shrinkst *ShrinkerSt) DoShrink(inum common.Inum) bool { 42 | var more = true 43 | var ok = true 44 | for more { 45 | op := fstxn.Begin(shrinkst.fsstate) 46 | ip := op.GetInodeInumFree(inum) 47 | if ip == nil { 48 | panic("shrink") 49 | } 50 | util.DPrintf(1, "%p: doShrink %v\n", op.Atxn.Id(), ip.Inum) 51 | more = ip.Shrink(op.Atxn) 52 | ok = op.Commit() 53 | if !ok { 54 | break 55 | } 56 | if shrinkst.crashed() { 57 | break 58 | } 59 | } 60 | return ok 61 | } 62 | 63 | func (shrinker *ShrinkerSt) Shutdown() { 64 | shrinker.mu.Lock() 65 | for shrinker.nthread > 0 { 66 | util.DPrintf(1, "Shutdown: shrinker wait %d\n", shrinker.nthread) 67 | shrinker.condShut.Wait() 68 | } 69 | shrinker.mu.Unlock() 70 | } 71 | 72 | func (shrinker *ShrinkerSt) Crash() { 73 | shrinker.mu.Lock() 74 | shrinker.crash = true 75 | for shrinker.nthread > 0 { 76 | util.DPrintf(1, "Crash: wait %d\n", shrinker.nthread) 77 | shrinker.condShut.Wait() 78 | } 79 | shrinker.mu.Unlock() 80 | } 81 | 82 | // for large files, start a separate thread 83 | func (shrinkst *ShrinkerSt) StartShrinker(inum common.Inum) { 84 | util.DPrintf(1, "start shrink thread\n") 85 | shrinkst.mu.Lock() 86 | shrinkst.nthread = shrinkst.nthread + 1 87 | shrinkst.mu.Unlock() 88 | go func() { shrinkst.shrinker(inum) }() 89 | } 90 | 91 | func (shrinkst *ShrinkerSt) shrinker(inum common.Inum) { 92 | ok := shrinkst.DoShrink(inum) 93 | if !ok { 94 | panic("shrink") 95 | } 96 | util.DPrintf(1, "Shrinker: done shrinking # %d\n", inum) 97 | shrinkst.mu.Lock() 98 | shrinkst.nthread = shrinkst.nthread - 1 99 | shrinkst.condShut.Signal() 100 | shrinkst.mu.Unlock() 101 | } 102 | -------------------------------------------------------------------------------- /inode/shrink.go: -------------------------------------------------------------------------------- 1 | package inode 2 | 3 | import ( 4 | "github.com/goose-lang/primitive/disk" 5 | "github.com/mit-pdos/go-journal/jrnl" 6 | 7 | "github.com/mit-pdos/go-journal/common" 8 | "github.com/mit-pdos/go-journal/util" 9 | "github.com/mit-pdos/go-nfsd/alloctxn" 10 | ) 11 | 12 | // 13 | // Freeing of a file. Freeing a large file may require multiple 14 | // transactions to ensure that the indirect blocks modified due to a 15 | // free fit in the write-ahead log. In this case the caller of 16 | // Shrink() is responsible for starting another shrink transaction. 17 | // 18 | 19 | func (ip *Inode) shrinkFits(op *alloctxn.AllocTxn, nblk uint64) bool { 20 | return op.Op.NDirty()+nblk < jrnl.LogBlocks 21 | } 22 | 23 | func (ip *Inode) IsShrinking() bool { 24 | cursz := util.RoundUp(ip.Size, disk.BlockSize) 25 | s := ip.ShrinkSize > cursz 26 | return s 27 | } 28 | 29 | func (ip *Inode) freeIndex(op *alloctxn.AllocTxn, index uint64) { 30 | op.FreeBlock(ip.blks[index]) 31 | ip.blks[index] = 0 32 | } 33 | 34 | // Frees indirect bn. Assumes if bn is cleared, then all blocks > bn 35 | // have been cleared 36 | func (ip *Inode) indshrink(op *alloctxn.AllocTxn, root common.Bnum, level uint64, bn uint64) common.Bnum { 37 | if root == common.NULLBNUM { 38 | return 0 39 | } 40 | if level == 0 { 41 | return root 42 | } 43 | divisor := pow(level - 1) 44 | off := (bn / divisor) 45 | ind := bn % divisor 46 | boff := off * 8 47 | b := op.ReadBlock(root) 48 | nxtroot := b.BnumGet(boff) 49 | op.AssertValidBlock(nxtroot) 50 | if nxtroot != 0 { 51 | freeroot := ip.indshrink(op, nxtroot, level-1, ind) 52 | if freeroot != 0 { 53 | b.BnumPut(boff, 0) 54 | op.FreeBlock(freeroot) 55 | } 56 | } 57 | if off == 0 && ind == 0 { 58 | return root 59 | } else { 60 | return common.NULLBNUM 61 | } 62 | } 63 | 64 | // Frees as many blocks as possible, and returns if more shrinking is necessary. 65 | // 5: inode block, 2xbitmap block, indirect block, double indirect 66 | func (ip *Inode) Shrink(op *alloctxn.AllocTxn) bool { 67 | util.DPrintf(1, "Shrink: from %d to %d\n", ip.ShrinkSize, 68 | util.RoundUp(ip.Size, disk.BlockSize)) 69 | for ip.IsShrinking() && ip.shrinkFits(op, 5) { 70 | ip.ShrinkSize -= 1 71 | if ip.ShrinkSize < NDIRECT { 72 | ip.freeIndex(op, ip.ShrinkSize) 73 | } else { 74 | var off = ip.ShrinkSize - NDIRECT 75 | if off < NBLKBLK { 76 | freeroot := ip.indshrink(op, ip.blks[INDIRECT], 1, off) 77 | if freeroot != 0 { 78 | ip.freeIndex(op, INDIRECT) 79 | } 80 | } else { 81 | off = off - NBLKBLK 82 | freeroot := ip.indshrink(op, ip.blks[DINDIRECT], 2, off) 83 | if freeroot != 0 { 84 | ip.freeIndex(op, DINDIRECT) 85 | } 86 | } 87 | } 88 | } 89 | ip.WriteInode(op) 90 | return ip.IsShrinking() 91 | } 92 | -------------------------------------------------------------------------------- /eval/latency.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # run various performance benchmarks 4 | 5 | set -eu 6 | 7 | blue=$(tput setaf 4) 8 | red=$(tput setaf 1) 9 | reset=$(tput sgr0) 10 | 11 | info() { 12 | echo -e "${blue}$1${reset}" 1>&2 13 | } 14 | error() { 15 | echo -e "${red}$1${reset}" 1>&2 16 | } 17 | 18 | usage() { 19 | echo "Usage: $0 [-ssd ]" 1>&2 20 | echo "SSD benchmarks will be skipped if -ssd is not passed" 21 | } 22 | 23 | ssd_file="" 24 | 25 | while [[ "$#" -gt 0 ]]; do 26 | case "$1" in 27 | -ssd) 28 | shift 29 | ssd_file="$1" 30 | shift 31 | ;; 32 | *) 33 | error "unexpected argument $1" 34 | usage 35 | exit 1 36 | ;; 37 | esac 38 | done 39 | 40 | if [ ! -d "$GO_NFSD_PATH" ]; then 41 | echo "GO_NFSD_PATH is unset" 1>&2 42 | exit 1 43 | fi 44 | 45 | cd "$GO_NFSD_PATH" 46 | 47 | info "GoNFS (smallfile)" 48 | echo "#GoNFS (smallfile)" >eval/data/gonfs-latencies.txt 49 | ./bench/run-go-nfsd.sh -stats true -unstable=false -disk "" go run ./cmd/fs-smallfile -benchtime=20s 50 | cat nfs.out >>eval/data/gonfs-latencies.txt 51 | 52 | info "GoNFS (null)" 53 | echo "#GoNFS (null)" >>eval/data/gonfs-latencies.txt 54 | ./bench/run-go-nfsd.sh -stats true -unstable=false -disk "" go run ./cmd/clnt-null -benchtime=20s >>eval/data/gonfs-latencies.txt 55 | 56 | info "\n\nResults: " 57 | cat eval/data/gonfs-latencies.txt 58 | 59 | echo 1>&2 60 | info "Linux ext4 over NFS" 61 | echo "#Linuxt ext4 over NFS (smallfile)" >eval/data/linux-latencies.txt 62 | sudo bpftrace ./eval/nfsdist.bt >>eval/data/linux-latencies.txt & 63 | ./bench/run-linux.sh go run ./cmd/fs-smallfile -benchtime=20s 64 | sudo killall bpftrace 65 | sleep 1 66 | 67 | echo 1>&2 68 | info "Linux ext4 over NFS (null)" 69 | echo "#Linuxt ext4 over NFS (null)" >>eval/data/linux-latencies.txt 70 | sudo bpftrace ./eval/nfsdist.bt >eval/data/linux-latencies.bpf.txt & 71 | ./bench/run-linux.sh go run ./cmd/clnt-null -benchtime=20s >>eval/data/linux-latencies.txt 72 | sudo killall bpftrace 73 | sleep 1 74 | cat eval/data/linux-latencies.bpf.txt >>eval/data/linux-latencies.txt 75 | 76 | info "\n\nResults: " 77 | cat eval/data/linux-latencies.txt 78 | 79 | if [ -n "$ssd_file" ]; then 80 | echo 1>&2 81 | info "GoNFS (SSD)" 82 | echo "fs=gonfs-ssd" 83 | ./bench/run-go-nfsd.sh -stats true -unstable=false -disk "$ssd_file" go run ./cmd/fs-smallfile -benchtime=20s 84 | cat nfs.out >eval/data/gonfs-disk-latencies.txt 85 | cat eval/data/gonfs-disk-latencies.txt 86 | 87 | echo 1>&2 88 | info "Linux ext4 over NFS (SSD)" 89 | sudo bpftrace ./eval/nfsdist.bt >eval/data/linux-disk-latencies.txt & 90 | ./bench/run-linux.sh -disk "$ssd_file" go run ./cmd/fs-smallfile -benchtime=20s 91 | sudo killall bpftrace 92 | sleep 1 93 | cat eval/data/linux-disk-latencies.txt 94 | fi 95 | -------------------------------------------------------------------------------- /cmd/txn-bench/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/goose-lang/goose/machine/disk" 9 | 10 | "github.com/mit-pdos/go-journal/addr" 11 | "github.com/mit-pdos/go-journal/txn" 12 | ) 13 | 14 | func testSequence(tsys *txn.Log, data []byte, tid uint64) { 15 | txnbuf := txn.Begin(tsys) 16 | for i := uint64(0); i < 16; i++ { 17 | txnbuf.OverWrite(addr.MkAddr(i+513, 8*tid), 8, data) 18 | } 19 | txnbuf.Commit(true) 20 | } 21 | 22 | func mkdata(sz uint64) []byte { 23 | data := make([]byte, sz) 24 | for i := range data { 25 | data[i] = byte(i % 128) 26 | } 27 | return data 28 | } 29 | 30 | func client(tsys *txn.Log, duration time.Duration, tid uint64) int { 31 | data := mkdata(1) 32 | start := time.Now() 33 | i := 0 34 | for { 35 | testSequence(tsys, data, tid) 36 | i++ 37 | t := time.Now() 38 | elapsed := t.Sub(start) 39 | if elapsed >= duration { 40 | break 41 | } 42 | } 43 | return i 44 | } 45 | 46 | func run(tsys *txn.Log, duration time.Duration, nt int) int { 47 | count := make(chan int) 48 | for i := 0; i < nt; i++ { 49 | go func(tid int) { 50 | count <- client(tsys, duration, uint64(tid)) 51 | }(i) 52 | } 53 | n := 0 54 | for i := 0; i < nt; i++ { 55 | n += <-count 56 | } 57 | return n 58 | } 59 | 60 | func zeroDisk(d disk.Disk) { 61 | zeroblock := make([]byte, 4096) 62 | sz := d.Size() 63 | for i := uint64(0); i < sz; i++ { 64 | d.Write(i, zeroblock) 65 | } 66 | d.Barrier() 67 | } 68 | 69 | func main() { 70 | var err error 71 | var duration time.Duration 72 | var nthread int 73 | var diskfile string 74 | var filesizeMegabytes uint64 75 | flag.DurationVar(&duration, "benchtime", 10*time.Second, "time to run each iteration for") 76 | flag.IntVar(&nthread, "threads", 1, "number of threads to run till") 77 | flag.StringVar(&diskfile, "disk", "", "disk image (empty for MemDisk)") 78 | flag.Uint64Var(&filesizeMegabytes, "size", 400, "size of file system (in MB)") 79 | flag.Parse() 80 | if nthread < 1 { 81 | panic("invalid start") 82 | } 83 | 84 | diskBlocks := 1500 + filesizeMegabytes*1024/4 85 | var d disk.Disk 86 | if diskfile == "" { 87 | d = disk.NewMemDisk(diskBlocks) 88 | } else { 89 | d, err = disk.NewFileDisk(diskfile, diskBlocks) 90 | if err != nil { 91 | panic(fmt.Errorf("could not create disk: %w", err)) 92 | } 93 | } 94 | zeroDisk(d) 95 | 96 | tsys := txn.Init(d) 97 | 98 | // warmup (skip if running for very little time, for example when using a 99 | // duration of 0s to run just one iteration) 100 | if duration > 500*time.Millisecond { 101 | run(tsys, 500*time.Millisecond, nthread) 102 | } 103 | 104 | count := run(tsys, duration, nthread) 105 | fmt.Printf("txn-bench: %v %v txn/sec\n", nthread, float64(count)/duration.Seconds()) 106 | } 107 | -------------------------------------------------------------------------------- /eval/aggregate-times.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # aggregate the results of running tshark over an NFS packet capture 4 | # 5 | # gather data with 6 | # tshark -i lo -f tcp -w nfs.pcap 7 | # 8 | # then process with 9 | # tshark -Tfields -e 'nfs.procedure_v3' -e 'rpc.time' -r nfs.pcap '(nfs && rpc.time)' | ./aggregate-times.py 10 | # 11 | # note that running tshark over a trace takes a while 12 | 13 | import re 14 | import sys 15 | import numpy as np 16 | 17 | proc_mapping = { 18 | 0: "NULL", 19 | 1: "GETATTR", 20 | 2: "SETATTR", 21 | 3: "LOOKUP", 22 | 4: "ACCESS", 23 | 6: "READ", 24 | 7: "WRITE", 25 | 8: "CREATE", 26 | 9: "MKDIR", 27 | 10: "SYMLINK", 28 | 12: "REMOVE", 29 | 13: "RMDIR", 30 | 14: "RENAME", 31 | 15: "LINK", 32 | 16: "READDIR", 33 | 17: "READDIRPLUS", 34 | 18: "FSSTAT", 35 | 19: "FSINFO", 36 | 20: "PATHCONF", 37 | 21: "COMMIT", 38 | } 39 | 40 | 41 | def proc_latencies(f): 42 | latencies_s = {} 43 | for line in f: 44 | m = re.match(r"""(?P.*)\t(?P