├── BUILD.bazel ├── LICENSE ├── README.md ├── WORKSPACE ├── errors.go ├── eventfd.go ├── examples ├── cat │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── concurrent-cat │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── cp │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── echo-with-callback │ ├── BUILD.bazel │ ├── README.md │ └── server.go ├── echo │ ├── BUILD.bazel │ ├── README.md │ ├── client.go │ └── server.go ├── hardlink │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── link │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── mkdir │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── mv │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── nvme-id-ctrl │ ├── README.md │ └── main.go ├── rm │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── symlink │ ├── BUILD.bazel │ ├── README.md │ └── main.go └── timeout │ ├── link-with-timeout │ ├── BUILD.bazel │ ├── README.md │ └── main.go │ ├── request-with-timeout │ ├── BUILD.bazel │ ├── README.md │ └── main.go │ └── timer │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── fixed_buffers.go ├── fixed_files.go ├── go.mod ├── go.sum ├── iouring.go ├── iouring_test.go ├── link_request.go ├── mmap.go ├── options.go ├── poller.go ├── prep_request.go ├── probe.go ├── request.go ├── syscall ├── BUILD.bazel ├── io_uring_enter.go ├── io_uring_register.go ├── io_uring_setup.go └── types.go ├── timeout.go ├── types.go ├── user_data.go └── utils.go /BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | load("@bazel_gazelle//:def.bzl", "gazelle") 3 | 4 | # gazelle:prefix github.com/iceber/iouring-go 5 | gazelle(name = "gazelle") 6 | 7 | go_library( 8 | name = "iouring-go", 9 | srcs = [ 10 | "errors.go", 11 | "eventfd.go", 12 | "fixed_buffers.go", 13 | "fixed_files.go", 14 | "iouring.go", 15 | "link_request.go", 16 | "mmap.go", 17 | "options.go", 18 | "poller.go", 19 | "prep_request.go", 20 | "probe.go", 21 | "request.go", 22 | "timeout.go", 23 | "types.go", 24 | "user_data.go", 25 | "utils.go", 26 | ], 27 | importpath = "github.com/iceber/iouring-go", 28 | visibility = ["//visibility:public"], 29 | deps = select({ 30 | "@io_bazel_rules_go//go/platform:android": [ 31 | "//syscall", 32 | "@org_golang_x_sys//unix:go_default_library", 33 | ], 34 | "@io_bazel_rules_go//go/platform:linux": [ 35 | "//syscall", 36 | "@org_golang_x_sys//unix:go_default_library", 37 | ], 38 | "//conditions:default": [], 39 | }), 40 | ) 41 | 42 | go_test( 43 | name = "iouring-go_test", 44 | srcs = ["iouring_test.go"], 45 | embed = [":iouring-go"], 46 | ) 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 IceberGu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What is io_uring 2 | [io_uring](http://kernel.dk/io_uring.pdf) 3 | 4 | [io_uring-wahtsnew](https://kernel.dk/io_uring-whatsnew.pdf) 5 | 6 | [LWN io_uring](https://lwn.net/Kernel/Index/#io_uring) 7 | 8 | [Lord of the io_uring](https://unixism.net/loti/) 9 | 10 | [【译】高性能异步 IO —— io_uring (Effecient IO with io_uring)](http://icebergu.com/archives/linux-iouring) 11 | 12 | [Go 与异步 IO - io_uring 的思考](http://icebergu.com/archives/go-iouring) 13 | 14 | # Features 15 | - [x] register a file set for io_uring instance 16 | - [x] support file IO 17 | - [x] support socket IO 18 | - [x] support IO timeout 19 | - [x] link request 20 | - [x] set timer 21 | - [x] add request extra info, could get it from the result 22 | - [ ] set logger 23 | - [ ] register buffers and IO with buffers 24 | - [ ] support SQPoll 25 | 26 | # OS Requirements 27 | * Linux Kernel >= 5.6 28 | 29 | 30 | # Installation 31 | ``` 32 | go get github.com/iceber/iouring-go 33 | ``` 34 | [doc](https://pkg.go.dev/github.com/iceber/iouring-go) 35 | 36 | # Quickstart 37 | ```golang 38 | package main 39 | 40 | import ( 41 | "fmt" 42 | "os" 43 | 44 | "github.com/iceber/iouring-go" 45 | ) 46 | 47 | var str = "io with iouring" 48 | 49 | func main() { 50 | iour, err := iouring.New(1) 51 | if err != nil { 52 | panic(fmt.Sprintf("new IOURing error: %v", err)) 53 | } 54 | defer iour.Close() 55 | 56 | file, err := os.Create("./tmp.txt") 57 | if err != nil { 58 | panic(err) 59 | } 60 | 61 | ch := make(chan iouring.Result, 1) 62 | 63 | prepRequest := iouring.Write(int(file.Fd()), []byte(str)) 64 | if _, err := iour.SubmitRequest(prepRequest, ch); err != nil { 65 | panic(err) 66 | } 67 | 68 | result := <-ch 69 | i, err := result.ReturnInt() 70 | if err != nil { 71 | fmt.Println("write error: ", err) 72 | return 73 | } 74 | 75 | fmt.Printf("write byte: %d\n", i) 76 | } 77 | ``` 78 | 79 | # Request With Extra Info 80 | ```golang 81 | prepRequest := iouring.Write(int(file.Fd()), []byte(str)).WithInfo(file.Name()) 82 | 83 | request, err := iour.SubmitRequest(prepRequest, nil) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | <- request.Done() 89 | info, ok := request.GetRequestInfo().(string) 90 | ``` 91 | 92 | # Cancel Request 93 | ```golang 94 | prepR := iouring.Timeout(5 * time.Second) 95 | request, err := iour.SubmitRequest(prepR, nil) 96 | if err != nil { 97 | panic(err) 98 | } 99 | 100 | if _, err := request.Cancel(); err != nil{ 101 | fmt.Printf("cancel request error: %v\n", err) 102 | return 103 | } 104 | 105 | <- request.Done() 106 | if err := request.Err(); err != nil{ 107 | if err == iouring.ErrRequestCanceled{ 108 | fmt.Println("request is canceled"0 109 | return 110 | } 111 | fmt.Printf("request error: %v\n", err) 112 | return 113 | } 114 | ``` 115 | 116 | 117 | # Submit multitude request 118 | 119 | ```golang 120 | var offset uint64 121 | buf1 := make([]byte, 1024) 122 | prep1:= iouring.Pread(fd, buf1, offset) 123 | 124 | offset += 1024 125 | buf2 := make([]byte, 1024) 126 | prep2:= iouring.Pread(fd, buf1, offset) 127 | 128 | requests, err := iour.SubmitRequests([]iouring.PrepRequest{prep1,prep2}, nil) 129 | if err != nil{ 130 | panic(err) 131 | } 132 | <- requests.Done() 133 | fmt.Println("requests are completed") 134 | ``` 135 | requests is concurrent execution 136 | 137 | # Link request 138 | ```golang 139 | var offset uint64 140 | buf := make([]byte, 1024) 141 | prep1 := iouring.Pread(fd, buf1, offset) 142 | prep2 := iouring.Write(int(os.Stdout.Fd()), buf) 143 | 144 | iour.SubmitLinkRequests([]iouring.PrepRequest{prep1, prep2}, nil) 145 | ``` 146 | 147 | # Examples 148 | [cat](https://github.com/Iceber/iouring-go/tree/main/examples/cat) 149 | 150 | [concurrent-cat](https://github.com/Iceber/iouring-go/tree/main/examples/concurrent-cat) 151 | 152 | [cp](https://github.com/Iceber/iouring-go/tree/main/examples/cp) 153 | 154 | [request-with-timeout](https://github.com/Iceber/iouring-go/tree/main/examples/timeout/request-with-timeout) 155 | 156 | [link-request](https://github.com/Iceber/iouring-go/tree/main/examples/link) 157 | 158 | [link-with-timeout](https://github.com/Iceber/iouring-go/tree/main/examples/timeout/link-with-timeout) 159 | 160 | [timer](https://github.com/Iceber/iouring-go/tree/main/examples/timeout/timer) 161 | 162 | [echo](https://github.com/Iceber/iouring-go/tree/main/examples/echo) 163 | 164 | [echo-with-callback](https://github.com/Iceber/iouring-go/tree/main/examples/echo-with-callback) 165 | 166 | # TODO 167 | * add tests 168 | * arguments type (eg. int and int32) 169 | * set logger 170 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 2 | 3 | http_archive( 4 | name = "io_bazel_rules_go", 5 | sha256 = "8e968b5fcea1d2d64071872b12737bbb5514524ee5f0a4f54f5920266c261acb", 6 | urls = [ 7 | "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.28.0/rules_go-v0.28.0.zip", 8 | "https://github.com/bazelbuild/rules_go/releases/download/v0.28.0/rules_go-v0.28.0.zip", 9 | ], 10 | ) 11 | 12 | http_archive( 13 | name = "bazel_gazelle", 14 | sha256 = "62ca106be173579c0a167deb23358fdfe71ffa1e4cfdddf5582af26520f1c66f", 15 | urls = [ 16 | "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz", 17 | "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz", 18 | ], 19 | ) 20 | 21 | load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") 22 | load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") 23 | 24 | go_rules_dependencies() 25 | 26 | go_register_toolchains(version = "1.17") 27 | 28 | gazelle_dependencies() 29 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring 4 | 5 | import "errors" 6 | 7 | var ( 8 | ErrIOURingClosed = errors.New("iouring closed") 9 | 10 | ErrRequestCanceled = errors.New("request is canceled") 11 | ErrRequestNotFound = errors.New("request is not found") 12 | ErrRequestCompleted = errors.New("request has already been completed") 13 | ErrRequestNotCompleted = errors.New("request is not completed") 14 | ErrNoRequestCallback = errors.New("no request callback") 15 | 16 | ErrUnregisteredFile = errors.New("file is unregistered") 17 | ) 18 | -------------------------------------------------------------------------------- /eventfd.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring 4 | 5 | import ( 6 | "os" 7 | "unsafe" 8 | 9 | "golang.org/x/sys/unix" 10 | 11 | iouring_syscall "github.com/iceber/iouring-go/syscall" 12 | ) 13 | 14 | func (iour *IOURing) registerEventfd() error { 15 | eventfd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC) 16 | if err != nil { 17 | return os.NewSyscallError("eventfd", err) 18 | } 19 | iour.eventfd = eventfd 20 | 21 | return iouring_syscall.IOURingRegister( 22 | iour.fd, 23 | iouring_syscall.IORING_REGISTER_EVENTFD, 24 | unsafe.Pointer(&iour.eventfd), 1, 25 | ) 26 | } 27 | -------------------------------------------------------------------------------- /examples/cat/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "cat_lib", 5 | srcs = ["main.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/cat", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "cat", 13 | embed = [":cat_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/cat/README.md: -------------------------------------------------------------------------------- 1 | # cat 2 | 3 | ``` 4 | go build . 5 | 6 | ./cat file1 file2 7 | ``` 8 | -------------------------------------------------------------------------------- /examples/cat/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/iceber/iouring-go" 8 | ) 9 | 10 | const blockSize int64 = 32 * 1024 11 | 12 | var buffers [][]byte 13 | 14 | func getBuffers(size int64) [][]byte { 15 | blocks := int(size / blockSize) 16 | if size%blockSize != 0 { 17 | blocks++ 18 | } 19 | 20 | for i := 0; i < blocks-len(buffers); i++ { 21 | buffers = append(buffers, make([]byte, blockSize)) 22 | } 23 | 24 | bs := buffers[:blocks] 25 | if size%blockSize != 0 { 26 | bs[blocks-1] = bs[blocks-1][:size%blockSize] 27 | } 28 | return bs 29 | } 30 | 31 | func readAndPrint(iour *iouring.IOURing, file *os.File) error { 32 | stat, err := file.Stat() 33 | if err != nil { 34 | return err 35 | } 36 | size := stat.Size() 37 | buffers := getBuffers(size) 38 | 39 | request, err := iour.SubmitRequest(iouring.Readv(int(file.Fd()), buffers), nil) 40 | <-request.Done() 41 | if err := request.Err(); err != nil { 42 | return request.Err() 43 | } 44 | 45 | fmt.Println(file.Name(), ":") 46 | for _, buffer := range request.GetRequestBuffers() { 47 | fmt.Printf("%s", buffer) 48 | } 49 | fmt.Println() 50 | return err 51 | } 52 | 53 | func main() { 54 | if len(os.Args) <= 1 { 55 | fmt.Printf("Usage: %s file1 file2 ...\n", os.Args[0]) 56 | } 57 | 58 | iour, err := iouring.New(1) 59 | if err != nil { 60 | panic(err) 61 | } 62 | defer iour.Close() 63 | 64 | for _, filename := range os.Args[1:] { 65 | file, err := os.Open(filename) 66 | if err != nil { 67 | fmt.Printf("open file error: %v\n", err) 68 | return 69 | } 70 | if err := readAndPrint(iour, file); err != nil { 71 | fmt.Printf("cat %s error: %v\n", filename, err) 72 | return 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /examples/concurrent-cat/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "concurrent-cat_lib", 5 | srcs = ["main.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/concurrent-cat", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "concurrent-cat", 13 | embed = [":concurrent-cat_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/concurrent-cat/README.md: -------------------------------------------------------------------------------- 1 | # concurrent-cat 2 | 3 | ``` 4 | go build . 5 | 6 | ./concurrent-cat file1 file2 7 | ``` 8 | -------------------------------------------------------------------------------- /examples/concurrent-cat/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/iceber/iouring-go" 8 | ) 9 | 10 | var blockSize int64 = 32 * 1024 11 | 12 | func main() { 13 | if len(os.Args) <= 1 { 14 | fmt.Printf("Usage: %s file1 file2 ...\n", os.Args[0]) 15 | } 16 | 17 | iour, err := iouring.New(10) 18 | if err != nil { 19 | panic(err) 20 | } 21 | defer iour.Close() 22 | 23 | compCh := make(chan iouring.Result, 1) 24 | 25 | go func() { 26 | for _, filename := range os.Args[1:] { 27 | file, err := os.Open(filename) 28 | if err != nil { 29 | panic(fmt.Sprintf("open %s error: %v", filename, err)) 30 | } 31 | if err := read(iour, file, compCh); err != nil { 32 | panic(fmt.Sprintf("submit read %s request error: %v", filename, err)) 33 | } 34 | } 35 | }() 36 | 37 | files := len(os.Args) - 1 38 | 39 | var prints int 40 | for result := range compCh { 41 | filename := result.GetRequestInfo().(string) 42 | if err := result.Err(); err != nil { 43 | fmt.Printf("read %s error: %v\n", filename, result.Err()) 44 | } 45 | 46 | fmt.Printf("%s: \n", filename) 47 | for _, buffer := range result.GetRequestBuffers() { 48 | fmt.Printf("%s", buffer) 49 | } 50 | fmt.Println() 51 | 52 | prints++ 53 | if prints == files { 54 | break 55 | } 56 | } 57 | 58 | fmt.Println("cat successful") 59 | } 60 | 61 | func read(iour *iouring.IOURing, file *os.File, ch chan iouring.Result) error { 62 | stat, err := file.Stat() 63 | if err != nil { 64 | return err 65 | } 66 | size := stat.Size() 67 | 68 | blocks := int(size / blockSize) 69 | if size%blockSize != 0 { 70 | blocks++ 71 | } 72 | 73 | buffers := make([][]byte, blocks) 74 | for i := 0; i < blocks; i++ { 75 | buffers[i] = make([]byte, blockSize) 76 | } 77 | if size%blockSize != 0 { 78 | buffers[blocks-1] = buffers[blocks-1][:size%blockSize] 79 | } 80 | 81 | prepRequest := iouring.Readv(int(file.Fd()), buffers).WithInfo(file.Name()) 82 | _, err = iour.SubmitRequest(prepRequest, ch) 83 | return err 84 | } 85 | -------------------------------------------------------------------------------- /examples/cp/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "cp_lib", 5 | srcs = ["main.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/cp", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "cp", 13 | embed = [":cp_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/cp/README.md: -------------------------------------------------------------------------------- 1 | # cp 2 | 3 | ``` 4 | go build . 5 | 6 | ./cp src dest 7 | ``` 8 | -------------------------------------------------------------------------------- /examples/cp/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/iceber/iouring-go" 9 | ) 10 | 11 | const entries uint = 64 12 | const blockSize int64 = 32 * 1024 13 | 14 | func main() { 15 | now := time.Now() 16 | 17 | if len(os.Args) != 3 { 18 | fmt.Printf("Usage: %s file1 file2\n", os.Args[0]) 19 | return 20 | } 21 | 22 | iour, err := iouring.New(entries) 23 | if err != nil { 24 | panic(fmt.Sprintf("new IOURing error: %v", err)) 25 | } 26 | defer iour.Close() 27 | 28 | src, err := os.Open(os.Args[1]) 29 | if err != nil { 30 | fmt.Printf("Open src file failed: %v\n", err) 31 | return 32 | } 33 | defer src.Close() 34 | 35 | dest, err := os.Create(os.Args[2]) 36 | if err != nil { 37 | fmt.Printf("create dest file failed: %v\n", err) 38 | return 39 | } 40 | defer dest.Close() 41 | 42 | if err := iour.RegisterFiles([]*os.File{src, dest}); err != nil { 43 | panic(err) 44 | } 45 | 46 | stat, err := src.Stat() 47 | if err != nil { 48 | panic(err) 49 | } 50 | size := stat.Size() 51 | 52 | var reads int 53 | var writes int 54 | var offset uint64 55 | 56 | ch := make(chan iouring.Result, entries) 57 | prepRequests := make([]iouring.PrepRequest, 0, entries) 58 | for size > 0 { 59 | if reads >= int(entries) { 60 | break 61 | } 62 | readSize := size 63 | if readSize > blockSize { 64 | readSize = blockSize 65 | } 66 | 67 | b := make([]byte, readSize) 68 | prepRequest := iouring.Pread(int(src.Fd()), b, offset).WithInfo(offset) 69 | prepRequests = append(prepRequests, prepRequest) 70 | 71 | size -= readSize 72 | offset += uint64(readSize) 73 | reads++ 74 | } 75 | 76 | if _, err := iour.SubmitRequests(prepRequests, ch); err != nil { 77 | panic(err) 78 | } 79 | 80 | for comp := 0; comp < reads+writes; comp++ { 81 | result := <-ch 82 | if err := result.Err(); err != nil { 83 | panic(err) 84 | } 85 | 86 | if result.Opcode() == iouring.OpRead { 87 | b, _ := result.GetRequestBuffer() 88 | offset := result.GetRequestInfo().(uint64) 89 | prep := iouring.Pwrite(int(dest.Fd()), b, offset) 90 | if _, err := iour.SubmitRequest(prep, ch); err != nil { 91 | panic(err) 92 | } 93 | writes++ 94 | continue 95 | } 96 | 97 | if size <= 0 { 98 | continue 99 | } 100 | 101 | readSize := size 102 | if readSize > blockSize { 103 | readSize = blockSize 104 | } 105 | 106 | b, _ := result.GetRequestBuffer() 107 | prepRequest := iouring.Pread(int(src.Fd()), b[:readSize], offset).WithInfo(offset) 108 | if _, err := iour.SubmitRequest(prepRequest, ch); err != nil { 109 | panic(err) 110 | } 111 | size -= readSize 112 | offset += uint64(readSize) 113 | reads++ 114 | } 115 | fmt.Printf("cp successful: %v\n", time.Now().Sub(now)) 116 | } 117 | -------------------------------------------------------------------------------- /examples/echo-with-callback/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "echo-with-callback_lib", 5 | srcs = ["server.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/echo-with-callback", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "echo-with-callback", 13 | embed = [":echo-with-callback_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/echo-with-callback/README.md: -------------------------------------------------------------------------------- 1 | # echo with callback 2 | ``` 3 | go build server.go 4 | 5 | ``` 6 | 7 | ## run server 8 | ``` 9 | ./server 10 | ``` 11 | 12 | ## run client 13 | ``` 14 | go run ../echo/client.go 15 | ``` 16 | -------------------------------------------------------------------------------- /examples/echo-with-callback/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "syscall" 8 | 9 | "github.com/iceber/iouring-go" 10 | ) 11 | 12 | const readSize = 1024 13 | 14 | var ( 15 | iour *iouring.IOURing 16 | resulter chan iouring.Result 17 | ) 18 | 19 | func main() { 20 | if len(os.Args) != 2 { 21 | fmt.Printf("Usage: %s \n", os.Args[0]) 22 | return 23 | } 24 | 25 | var err error 26 | iour, err = iouring.New(1024) 27 | if err != nil { 28 | panic(err) 29 | } 30 | defer iour.Close() 31 | 32 | resulter = make(chan iouring.Result, 10) 33 | 34 | fd := listenSocket(os.Args[1]) 35 | prepAccept := iouring.Accept(fd).WithCallback(accept) 36 | if _, err := iour.SubmitRequest(prepAccept, resulter); err != nil { 37 | panicf("submit accept request error: %v", err) 38 | } 39 | 40 | fmt.Println("echo server running...") 41 | for result := range resulter { 42 | result.Callback() 43 | } 44 | } 45 | 46 | func accept(result iouring.Result) error { 47 | prepAccept := iouring.Accept(result.Fd()).WithCallback(accept) 48 | if _, err := iour.SubmitRequest(prepAccept, resulter); err != nil { 49 | panicf("submit accept request error: %v", err) 50 | } 51 | 52 | if err := result.Err(); err != nil { 53 | panicf("accept error: %v", err) 54 | } 55 | 56 | connFd := result.ReturnValue0().(int) 57 | sockaddr := result.ReturnValue1().(*syscall.SockaddrInet4) 58 | 59 | clientAddr := fmt.Sprintf("%s:%d", net.IPv4(sockaddr.Addr[0], sockaddr.Addr[1], sockaddr.Addr[2], sockaddr.Addr[3]), sockaddr.Port) 60 | fmt.Printf("Client Conn: %s\n", clientAddr) 61 | 62 | buffer := make([]byte, readSize) 63 | prep := iouring.Read(connFd, buffer).WithInfo(clientAddr).WithCallback(read) 64 | if _, err := iour.SubmitRequest(prep, resulter); err != nil { 65 | panicf("submit read request error: %v", err) 66 | } 67 | return nil 68 | } 69 | 70 | func read(result iouring.Result) error { 71 | clientAddr := result.GetRequestInfo().(string) 72 | if err := result.Err(); err != nil { 73 | panicf("[%s] read error: %v", clientAddr, err) 74 | } 75 | 76 | num := result.ReturnValue0().(int) 77 | buf, _ := result.GetRequestBuffer() 78 | content := buf[:num] 79 | 80 | connPrintf(clientAddr, "read byte: %v\ncontent: %s\n", num, content) 81 | 82 | prep := iouring.Write(result.Fd(), content).WithInfo(clientAddr).WithCallback(write) 83 | if _, err := iour.SubmitRequest(prep, resulter); err != nil { 84 | panicf("[%s] submit write request error: %v", clientAddr, err) 85 | } 86 | return nil 87 | } 88 | 89 | func write(result iouring.Result) error { 90 | clientAddr := result.GetRequestInfo().(string) 91 | if err := result.Err(); err != nil { 92 | panicf("[%s] write error: %v", clientAddr, err) 93 | } 94 | connPrintf(clientAddr, "write successful\n") 95 | 96 | prep := iouring.Close(result.Fd()).WithInfo(clientAddr).WithCallback(close) 97 | if _, err := iour.SubmitRequest(prep, resulter); err != nil { 98 | panicf("[%s] submit write request error: %v", clientAddr, err) 99 | } 100 | return nil 101 | } 102 | 103 | func close(result iouring.Result) error { 104 | clientAddr := result.GetRequestInfo().(string) 105 | if err := result.Err(); err != nil { 106 | panicf("[%s] close error: %v", clientAddr, err) 107 | } 108 | connPrintf(clientAddr, "close successful\n") 109 | return nil 110 | } 111 | 112 | func listenSocket(addr string) int { 113 | fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, 0) 114 | if err != nil { 115 | panic(err) 116 | } 117 | 118 | tcpAddr, err := net.ResolveTCPAddr("tcp", addr) 119 | if err != nil { 120 | panic(err) 121 | } 122 | 123 | sockaddr := &syscall.SockaddrInet4{Port: tcpAddr.Port} 124 | copy(sockaddr.Addr[:], tcpAddr.IP.To4()) 125 | if err := syscall.Bind(fd, sockaddr); err != nil { 126 | panic(err) 127 | } 128 | 129 | if err := syscall.Listen(fd, syscall.SOMAXCONN); err != nil { 130 | panic(err) 131 | } 132 | return fd 133 | } 134 | 135 | func panicf(format string, a ...interface{}) { 136 | panic(fmt.Sprintf(format, a...)) 137 | } 138 | 139 | func connPrintf(addr string, format string, a ...interface{}) { 140 | prefix := fmt.Sprintf("[%s]", addr) 141 | fmt.Printf(prefix+format, a...) 142 | } 143 | -------------------------------------------------------------------------------- /examples/echo/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "echo_lib", 5 | srcs = [ 6 | "client.go", 7 | "server.go", 8 | ], 9 | importpath = "github.com/iceber/iouring-go/examples/echo", 10 | visibility = ["//visibility:private"], 11 | deps = ["//:iouring-go"], 12 | ) 13 | 14 | go_binary( 15 | name = "echo", 16 | embed = [":echo_lib"], 17 | visibility = ["//visibility:public"], 18 | ) 19 | -------------------------------------------------------------------------------- /examples/echo/README.md: -------------------------------------------------------------------------------- 1 | # echo 2 | ``` 3 | go build server.go 4 | 5 | go build client.go 6 | ``` 7 | 8 | ## run server 9 | ``` 10 | ./server 11 | ``` 12 | 13 | ## run client 14 | ``` 15 | ./client 16 | ``` 17 | -------------------------------------------------------------------------------- /examples/echo/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | ) 8 | 9 | func main() { 10 | if len(os.Args) != 3 { 11 | fmt.Printf("Usage: %s \n", os.Args[0]) 12 | return 13 | } 14 | serverAddr, msg := os.Args[1], os.Args[2] 15 | 16 | c, err := net.Dial("tcp", serverAddr) 17 | if err != nil { 18 | panic(err) 19 | } 20 | defer c.Close() 21 | 22 | n, err := c.Write([]byte(msg)) 23 | if err != nil { 24 | panic(err) 25 | } 26 | fmt.Println("write byte: ", n) 27 | 28 | buf := make([]byte, 1024) 29 | if _, err = c.Read(buf); err != nil { 30 | panic(err) 31 | } 32 | 33 | fmt.Printf("echo: %s\n", buf) 34 | } 35 | -------------------------------------------------------------------------------- /examples/echo/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "syscall" 8 | 9 | "github.com/iceber/iouring-go" 10 | ) 11 | 12 | const readSize = 1024 13 | 14 | var ( 15 | iour *iouring.IOURing 16 | resulter chan iouring.Result 17 | ) 18 | 19 | func main() { 20 | if len(os.Args) != 2 { 21 | fmt.Printf("Usage: %s \n", os.Args[0]) 22 | return 23 | } 24 | 25 | var err error 26 | iour, err = iouring.New(1024) 27 | if err != nil { 28 | panic(err) 29 | } 30 | defer iour.Close() 31 | 32 | resulter = make(chan iouring.Result, 10) 33 | 34 | fd := listenSocket(os.Args[1]) 35 | if _, err := iour.SubmitRequest(iouring.Accept(fd), resulter); err != nil { 36 | panicf("submit accept request error: %v", err) 37 | } 38 | 39 | fmt.Println("echo server running...") 40 | for { 41 | result := <-resulter 42 | switch result.Opcode() { 43 | case iouring.OpAccept: 44 | if _, err := iour.SubmitRequest(iouring.Accept(fd), resulter); err != nil { 45 | panicf("submit accept request error: %v", err) 46 | } 47 | accept(result) 48 | 49 | case iouring.OpRead: 50 | read(result) 51 | 52 | case iouring.OpWrite: 53 | write(result) 54 | 55 | case iouring.OpClose: 56 | close(result) 57 | } 58 | } 59 | } 60 | 61 | func accept(result iouring.Result) { 62 | if err := result.Err(); err != nil { 63 | panicf("accept error: %v", err) 64 | } 65 | 66 | connFd := result.ReturnValue0().(int) 67 | sockaddr := result.ReturnValue1().(*syscall.SockaddrInet4) 68 | 69 | clientAddr := fmt.Sprintf("%s:%d", net.IPv4(sockaddr.Addr[0], sockaddr.Addr[1], sockaddr.Addr[2], sockaddr.Addr[3]), sockaddr.Port) 70 | fmt.Printf("Client Conn: %s\n", clientAddr) 71 | 72 | buffer := make([]byte, readSize) 73 | prep := iouring.Read(connFd, buffer).WithInfo(clientAddr) 74 | if _, err := iour.SubmitRequest(prep, resulter); err != nil { 75 | panicf("submit read request error: %v", err) 76 | } 77 | } 78 | 79 | func read(result iouring.Result) { 80 | clientAddr := result.GetRequestInfo().(string) 81 | if err := result.Err(); err != nil { 82 | panicf("[%s] read error: %v", clientAddr, err) 83 | } 84 | 85 | num := result.ReturnValue0().(int) 86 | buf, _ := result.GetRequestBuffer() 87 | content := buf[:num] 88 | 89 | connPrintf(clientAddr, "read byte: %v\ncontent: %s\n", num, content) 90 | 91 | prep := iouring.Write(result.Fd(), content).WithInfo(clientAddr) 92 | if _, err := iour.SubmitRequest(prep, resulter); err != nil { 93 | panicf("[%s] submit write request error: %v", clientAddr, err) 94 | } 95 | } 96 | 97 | func write(result iouring.Result) { 98 | clientAddr := result.GetRequestInfo().(string) 99 | if err := result.Err(); err != nil { 100 | panicf("[%s] write error: %v", clientAddr, err) 101 | } 102 | connPrintf(clientAddr, "write successful\n") 103 | 104 | prep := iouring.Close(result.Fd()).WithInfo(clientAddr) 105 | if _, err := iour.SubmitRequest(prep, resulter); err != nil { 106 | panicf("[%s] submit write request error: %v", clientAddr, err) 107 | } 108 | } 109 | 110 | func close(result iouring.Result) { 111 | clientAddr := result.GetRequestInfo().(string) 112 | if err := result.Err(); err != nil { 113 | panicf("[%s] close error: %v", clientAddr, err) 114 | } 115 | connPrintf(clientAddr, "close successful\n") 116 | } 117 | 118 | func listenSocket(addr string) int { 119 | fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, 0) 120 | if err != nil { 121 | panic(err) 122 | } 123 | 124 | tcpAddr, err := net.ResolveTCPAddr("tcp", addr) 125 | if err != nil { 126 | panic(err) 127 | } 128 | 129 | sockaddr := &syscall.SockaddrInet4{Port: tcpAddr.Port} 130 | copy(sockaddr.Addr[:], tcpAddr.IP.To4()) 131 | if err := syscall.Bind(fd, sockaddr); err != nil { 132 | panic(err) 133 | } 134 | 135 | if err := syscall.Listen(fd, syscall.SOMAXCONN); err != nil { 136 | panic(err) 137 | } 138 | return fd 139 | } 140 | 141 | func panicf(format string, a ...interface{}) { 142 | panic(fmt.Sprintf(format, a...)) 143 | } 144 | 145 | func connPrintf(addr string, format string, a ...interface{}) { 146 | prefix := fmt.Sprintf("[%s]", addr) 147 | fmt.Printf(prefix+format, a...) 148 | } 149 | -------------------------------------------------------------------------------- /examples/hardlink/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "hardlink_lib", 5 | srcs = ["main.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/hardlink", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "hardlink", 13 | embed = [":hardlink_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/hardlink/README.md: -------------------------------------------------------------------------------- 1 | # hardlink 2 | 3 | ``` 4 | go build . 5 | 6 | ./hardlink target link 7 | ``` 8 | -------------------------------------------------------------------------------- /examples/hardlink/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/iceber/iouring-go" 9 | ) 10 | 11 | func main() { 12 | if len(os.Args) != 3 { 13 | fmt.Printf("Usage: %s target link\n", os.Args[0]) 14 | return 15 | } 16 | target := os.Args[1] 17 | link := os.Args[2] 18 | 19 | iour, err := iouring.New(1) 20 | if err != nil { 21 | panic(err) 22 | } 23 | defer iour.Close() 24 | 25 | targetDir, err := os.Open(filepath.Dir(target)) 26 | if err != nil { 27 | fmt.Printf("open old directory error: %v\n", err) 28 | return 29 | } 30 | linkDir, err := os.Open(filepath.Dir(link)) 31 | if err != nil { 32 | fmt.Printf("open new directory error: %v\n", err) 33 | return 34 | } 35 | 36 | pr, err := iouring.Linkat( 37 | int(targetDir.Fd()), filepath.Base(target), 38 | int(linkDir.Fd()), filepath.Base(link), 0) 39 | if err != nil { 40 | fmt.Printf("prep request error: %v\n", err) 41 | return 42 | } 43 | request, err := iour.SubmitRequest(pr, nil) 44 | <-request.Done() 45 | if err := request.Err(); err != nil { 46 | return 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /examples/link/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "link_lib", 5 | srcs = ["main.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/link", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "link", 13 | embed = [":link_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/link/README.md: -------------------------------------------------------------------------------- 1 | # link 2 | ``` 3 | go build . 4 | ./link 5 | 6 | str1 str1 str1 str1 7 | str2 str2 str2 str2 str2 8 | write str1 9 | write str2 10 | read fd to buffer 11 | read buffer to stdout 12 | ``` 13 | -------------------------------------------------------------------------------- /examples/link/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/iceber/iouring-go" 8 | ) 9 | 10 | const entries uint = 64 11 | 12 | var ( 13 | str1 = "str1 str1 str1 str1\n" 14 | str2 = "str2 str2 str2 str2 str2\n" 15 | ) 16 | 17 | func main() { 18 | iour, err := iouring.New(entries) 19 | if err != nil { 20 | panic(fmt.Sprintf("new IOURing error: %v", err)) 21 | } 22 | defer iour.Close() 23 | 24 | file, err := os.Create("./tmp") 25 | if err != nil { 26 | panic(err) 27 | } 28 | 29 | prepWrite1 := iouring.Write(int(file.Fd()), []byte(str1)).WithInfo("write str1") 30 | prepWrite2 := iouring.Pwrite(int(file.Fd()), []byte(str2), uint64(len(str1))).WithInfo("write str2") 31 | 32 | buffer := make([]byte, len(str1)+len(str2)) 33 | prepRead1 := iouring.Read(int(file.Fd()), buffer).WithInfo("read fd to buffer") 34 | prepRead2 := iouring.Write(int(os.Stdout.Fd()), buffer).WithInfo("read buffer to stdout") 35 | 36 | ch := make(chan iouring.Result, 4) 37 | _, err = iour.SubmitLinkRequests( 38 | []iouring.PrepRequest{ 39 | prepWrite1, 40 | prepWrite2, 41 | prepRead1, 42 | prepRead2, 43 | }, 44 | ch, 45 | ) 46 | if err != nil { 47 | panic(err) 48 | } 49 | 50 | for i := 0; i < 4; i++ { 51 | result := <-ch 52 | info := result.GetRequestInfo().(string) 53 | fmt.Println(info) 54 | if err := result.Err(); err != nil { 55 | fmt.Printf("error: %v\n", err) 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /examples/mkdir/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "mkdir_lib", 5 | srcs = ["main.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/mkdir", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "mkdir", 13 | embed = [":mkdir_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/mkdir/README.md: -------------------------------------------------------------------------------- 1 | # mkdir 2 | 3 | ``` 4 | go build . 5 | 6 | ./mkdir dir_path 7 | ``` 8 | -------------------------------------------------------------------------------- /examples/mkdir/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/iceber/iouring-go" 9 | ) 10 | 11 | func main() { 12 | if len(os.Args) != 2 { 13 | fmt.Printf("Usage: %s dir_path\n", os.Args[0]) 14 | return 15 | } 16 | path := os.Args[1] 17 | 18 | iour, err := iouring.New(1) 19 | if err != nil { 20 | panic(err) 21 | } 22 | defer iour.Close() 23 | 24 | pathDir, err := os.Open(filepath.Dir(path)) 25 | if err != nil { 26 | fmt.Printf("open directory error: %v\n", err) 27 | return 28 | } 29 | defer pathDir.Close() 30 | 31 | pr, err := iouring.Mkdirat(int(pathDir.Fd()), filepath.Base(path), 0) 32 | if err != nil { 33 | fmt.Printf("prep request error: %v\n", err) 34 | return 35 | } 36 | request, err := iour.SubmitRequest(pr, nil) 37 | <-request.Done() 38 | if err := request.Err(); err != nil { 39 | fmt.Printf("submit iouring request error: %v\n", err) 40 | return 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /examples/mv/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "mv_lib", 5 | srcs = ["main.go"], 6 | importpath = "github.com/iceber/iouring-go/examples/mv", 7 | visibility = ["//visibility:private"], 8 | deps = ["//:iouring-go"], 9 | ) 10 | 11 | go_binary( 12 | name = "mv", 13 | embed = [":mv_lib"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /examples/mv/README.md: -------------------------------------------------------------------------------- 1 | # mv 2 | 3 | ``` 4 | go build . 5 | 6 | ./mv old_path new_path 7 | ``` 8 | -------------------------------------------------------------------------------- /examples/mv/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/iceber/iouring-go" 9 | ) 10 | 11 | func main() { 12 | if len(os.Args) != 3 { 13 | fmt.Printf("Usage: %s old_path new_path\n", os.Args[0]) 14 | return 15 | } 16 | oldPath := os.Args[1] 17 | newPath := os.Args[2] 18 | 19 | iour, err := iouring.New(1) 20 | if err != nil { 21 | panic(err) 22 | } 23 | defer iour.Close() 24 | 25 | oldPathDir, err := os.Open(filepath.Dir(oldPath)) 26 | if err != nil { 27 | fmt.Printf("open old directory error: %v\n", err) 28 | return 29 | } 30 | newPathDir, err := os.Open(filepath.Dir(newPath)) 31 | if err != nil { 32 | fmt.Printf("open new directory error: %v\n", err) 33 | return 34 | } 35 | 36 | pr, err := iouring.Renameat( 37 | int(oldPathDir.Fd()), filepath.Base(oldPath), 38 | int(newPathDir.Fd()), filepath.Base(newPath)) 39 | if err != nil { 40 | fmt.Printf("prep request error: %v\n", err) 41 | return 42 | } 43 | request, err := iour.SubmitRequest(pr, nil) 44 | <-request.Done() 45 | if err := request.Err(); err != nil { 46 | return 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /examples/nvme-id-ctrl/README.md: -------------------------------------------------------------------------------- 1 | # nvme-id-ctrl 2 | 3 | ``` 4 | go build . 5 | 6 | ./nvme-id-ctrl nvme-char-device 7 | ``` 8 | 9 | ## before run 10 | 11 | To run this example, -------------------------------------------------------------------------------- /examples/nvme-id-ctrl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | "unsafe" 8 | 9 | "github.com/iceber/iouring-go" 10 | iouring_syscall "github.com/iceber/iouring-go/syscall" 11 | ) 12 | 13 | const ( 14 | nvmeOpCodeAdminIdentify = uint8(0x06) 15 | 16 | nvmeIdentifyCnsCtrl = uint32(0x01) 17 | 18 | entries uint = 64 19 | 20 | nrBits = 8 21 | typeBits = 8 22 | sizeBits = 14 23 | 24 | nrShift = 0 25 | typeShift = nrShift + nrBits 26 | sizeShift = typeShift + typeBits 27 | dirShift = sizeShift + sizeBits 28 | 29 | iocRead = uint64(2) 30 | iocWrite = uint64(1) 31 | iocWrRd = iocWrite | iocRead 32 | 33 | iocInOut = iocWrRd << dirShift 34 | 35 | iocNVMeType = 'N' << typeShift 36 | 37 | uringCmdAdmin = iocInOut | iocNVMeType | (0x82 << nrShift) | uint64(unsafe.Sizeof(AdminCmd{})<= len(fds) { 146 | goto update 147 | } 148 | } 149 | } 150 | register.fds = append(register.fds, fds[fdi:]...) 151 | 152 | update: 153 | if err := register.fresh(0, len(register.fds)); err != nil { 154 | return err 155 | } 156 | 157 | for i, spares := range updatedSpares { 158 | if spares > 0 { 159 | register.sparseIndexs[i] = spares 160 | continue 161 | } 162 | delete(register.sparseIndexs, i) 163 | } 164 | for i, fd := range register.fds { 165 | register.indexs.Store(fd, i) 166 | } 167 | 168 | return nil 169 | } 170 | 171 | func (register *fileRegister) RegisterFile(fd int32) error { 172 | if fd < 0 { 173 | return nil 174 | } 175 | 176 | if _, ok := register.GetFileIndex(fd); ok { 177 | return nil 178 | } 179 | 180 | register.lock.Lock() 181 | defer register.lock.Unlock() 182 | 183 | if !register.registered { 184 | register.fds = []int32{fd} 185 | return register.register() 186 | } 187 | 188 | var fdi int 189 | var spares int 190 | for fdi, spares = range register.sparseIndexs { 191 | break 192 | } 193 | register.fds[fdi] = fd 194 | 195 | if err := register.fresh(fdi, 1); err != nil { 196 | return err 197 | } 198 | 199 | if spares == 1 { 200 | delete(register.sparseIndexs, fdi) 201 | } else { 202 | register.sparseIndexs[fdi]-- 203 | } 204 | 205 | register.indexs.Store(fd, fdi) 206 | return nil 207 | } 208 | 209 | func (register *fileRegister) UnregisterFile(fd int32) error { 210 | if fd < 0 { 211 | return nil 212 | } 213 | 214 | register.lock.Lock() 215 | defer register.lock.Unlock() 216 | 217 | fdi, ok := register.deleteFile(fd) 218 | if !ok { 219 | return nil 220 | } 221 | 222 | return register.fresh(fdi, 1) 223 | } 224 | 225 | func (register *fileRegister) UnregisterFiles(fds []int32) error { 226 | register.lock.Lock() 227 | defer register.lock.Unlock() 228 | 229 | var unregistered bool 230 | for _, fd := range fds { 231 | if fd < 0 { 232 | continue 233 | } 234 | 235 | _, ok := register.deleteFile(fd) 236 | if !ok { 237 | continue 238 | } 239 | unregistered = true 240 | } 241 | if unregistered { 242 | return nil 243 | } 244 | 245 | return register.fresh(0, len(register.fds)) 246 | } 247 | 248 | func (register *fileRegister) deleteFile(fd int32) (fdi int, ok bool) { 249 | var v interface{} 250 | /* 251 | go version >= 1.15 252 | 253 | v, ok = register.index.LoadAndDelete(fd) 254 | */ 255 | v, ok = register.indexs.Load(fd) 256 | if !ok { 257 | return 258 | } 259 | register.indexs.Delete(fd) 260 | 261 | fdi = v.(int) 262 | register.fds[fdi] = -1 263 | 264 | var updated bool 265 | for i, sparse := range register.sparseIndexs { 266 | if i+sparse == fdi { 267 | register.sparseIndexs[i]++ 268 | updated = true 269 | break 270 | } 271 | } 272 | 273 | if !updated { 274 | register.sparseIndexs[fdi] = 1 275 | } 276 | return 277 | } 278 | 279 | func (register *fileRegister) fresh(offset int, length int) error { 280 | update := iouring_syscall.IOURingFilesUpdate{ 281 | Offset: uint32(offset), 282 | Fds: ®ister.fds[offset], 283 | } 284 | return iouring_syscall.IOURingRegister( 285 | register.iouringFd, 286 | iouring_syscall.IORING_REGISTER_FILES_UPDATE, 287 | unsafe.Pointer(&update), 288 | uint32(len(register.fds)), 289 | ) 290 | } 291 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/iceber/iouring-go 2 | 3 | go 1.15 4 | 5 | require golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d h1:L/IKR6COd7ubZrs2oTnTi73IhgqJ71c9s80WsQnh0Es= 2 | golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 3 | -------------------------------------------------------------------------------- /iouring.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "errors" 8 | "log" 9 | "os" 10 | "runtime" 11 | "sync" 12 | "syscall" 13 | 14 | iouring_syscall "github.com/iceber/iouring-go/syscall" 15 | ) 16 | 17 | // IOURing contains iouring_syscall submission and completion queue. 18 | // It's safe for concurrent use by multiple goroutines. 19 | type IOURing struct { 20 | params *iouring_syscall.IOURingParams 21 | fd int 22 | 23 | eventfd int 24 | cqeSign chan struct{} 25 | 26 | sq *SubmissionQueue 27 | cq *CompletionQueue 28 | 29 | async bool 30 | drain bool 31 | Flags uint32 32 | Features uint32 33 | 34 | submitLock sync.Mutex 35 | 36 | userDataLock sync.RWMutex 37 | userDatas map[uint64]*UserData 38 | 39 | fileRegister FileRegister 40 | 41 | fdclosed bool 42 | closer chan struct{} 43 | closed chan struct{} 44 | } 45 | 46 | // New return a IOURing instance by IOURingOptions 47 | func New(entries uint, opts ...IOURingOption) (*IOURing, error) { 48 | iour := &IOURing{ 49 | params: &iouring_syscall.IOURingParams{}, 50 | userDatas: make(map[uint64]*UserData), 51 | cqeSign: make(chan struct{}, 1), 52 | closer: make(chan struct{}), 53 | closed: make(chan struct{}), 54 | } 55 | 56 | for _, opt := range opts { 57 | opt(iour) 58 | } 59 | 60 | var err error 61 | iour.fd, err = iouring_syscall.IOURingSetup(entries, iour.params) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | if err := mmapIOURing(iour); err != nil { 67 | munmapIOURing(iour) 68 | return nil, err 69 | } 70 | 71 | iour.fileRegister = &fileRegister{ 72 | iouringFd: iour.fd, 73 | sparseIndexs: make(map[int]int), 74 | } 75 | iour.Flags = iour.params.Flags 76 | iour.Features = iour.params.Features 77 | 78 | if err := iour.registerEventfd(); err != nil { 79 | iour.Close() 80 | return nil, err 81 | } 82 | 83 | if err := registerIOURing(iour); err != nil { 84 | iour.Close() 85 | return nil, err 86 | } 87 | 88 | go iour.run() 89 | return iour, nil 90 | } 91 | 92 | // Size iouring submission queue size 93 | func (iour *IOURing) Size() int { 94 | return int(*iour.sq.entries) 95 | } 96 | 97 | // Close IOURing 98 | func (iour *IOURing) Close() error { 99 | iour.submitLock.Lock() 100 | defer iour.submitLock.Unlock() 101 | 102 | select { 103 | case <-iour.closer: 104 | default: 105 | close(iour.closer) 106 | } 107 | 108 | if iour.eventfd > 0 { 109 | if err := removeIOURing(iour); err != nil { 110 | return err 111 | } 112 | syscall.Close(iour.eventfd) 113 | iour.eventfd = -1 114 | } 115 | 116 | <-iour.closed 117 | 118 | if err := munmapIOURing(iour); err != nil { 119 | return err 120 | } 121 | 122 | if !iour.fdclosed { 123 | if err := syscall.Close(iour.fd); err != nil { 124 | return os.NewSyscallError("close", err) 125 | } 126 | iour.fdclosed = true 127 | } 128 | 129 | return nil 130 | } 131 | 132 | // IsClosed IOURing is closed 133 | func (iour *IOURing) IsClosed() (closed bool) { 134 | select { 135 | case <-iour.closer: 136 | closed = true 137 | default: 138 | } 139 | return 140 | } 141 | 142 | func (iour *IOURing) getSQEntry() iouring_syscall.SubmissionQueueEntry { 143 | for { 144 | sqe := iour.sq.getSQEntry() 145 | if sqe != nil { 146 | return sqe 147 | } 148 | runtime.Gosched() 149 | } 150 | } 151 | 152 | func (iour *IOURing) doRequest(sqe iouring_syscall.SubmissionQueueEntry, request PrepRequest, ch chan<- Result) (*UserData, error) { 153 | userData := makeUserData(iour, ch) 154 | 155 | request(sqe, userData) 156 | userData.setOpcode(sqe.Opcode()) 157 | 158 | sqe.SetUserData(userData.id) 159 | 160 | userData.request.fd = int(sqe.Fd()) 161 | if sqe.Fd() >= 0 { 162 | if index, ok := iour.fileRegister.GetFileIndex(int32(sqe.Fd())); ok { 163 | sqe.SetFdIndex(int32(index)) 164 | } else if iour.Flags&iouring_syscall.IORING_SETUP_SQPOLL != 0 && 165 | iour.Features&iouring_syscall.IORING_FEAT_SQPOLL_NONFIXED == 0 { 166 | /* 167 | Before version 5.10 of the Linux kernel, to successful use SQPoll, the application 168 | must register a set of files to be used for IO through iour.RegisterFiles. 169 | Failure to do so will result in submitted IO being errored with EBADF 170 | 171 | The presence of this feature can be detected by the IORING_FEAT_SQPOLL_NONFIXED 172 | In Version 5.10 and later, it is no longer necessary to register files to use SQPoll 173 | */ 174 | 175 | return nil, ErrUnregisteredFile 176 | } 177 | } 178 | 179 | if iour.async { 180 | sqe.SetFlags(iouring_syscall.IOSQE_FLAGS_ASYNC) 181 | } 182 | if iour.drain { 183 | sqe.SetFlags(iouring_syscall.IOSQE_FLAGS_IO_DRAIN) 184 | } 185 | return userData, nil 186 | } 187 | 188 | // SubmitRequest by Request function and io result is notified via channel 189 | // return request id, can be used to cancel a request 190 | func (iour *IOURing) SubmitRequest(request PrepRequest, ch chan<- Result) (Request, error) { 191 | iour.submitLock.Lock() 192 | defer iour.submitLock.Unlock() 193 | 194 | if iour.IsClosed() { 195 | return nil, ErrIOURingClosed 196 | } 197 | 198 | sqe := iour.getSQEntry() 199 | userData, err := iour.doRequest(sqe, request, ch) 200 | if err != nil { 201 | iour.sq.fallback(1) 202 | return nil, err 203 | } 204 | 205 | iour.userDataLock.Lock() 206 | iour.userDatas[userData.id] = userData 207 | iour.userDataLock.Unlock() 208 | 209 | if _, err = iour.submit(); err != nil { 210 | iour.userDataLock.Lock() 211 | delete(iour.userDatas, userData.id) 212 | iour.userDataLock.Unlock() 213 | return nil, err 214 | } 215 | 216 | return userData.request, nil 217 | } 218 | 219 | // SubmitRequests by Request functions and io results are notified via channel 220 | func (iour *IOURing) SubmitRequests(requests []PrepRequest, ch chan<- Result) (RequestSet, error) { 221 | // TODO(iceber): no length limit 222 | if len(requests) > int(*iour.sq.entries) { 223 | return nil, errors.New("too many requests") 224 | } 225 | 226 | iour.submitLock.Lock() 227 | defer iour.submitLock.Unlock() 228 | 229 | if iour.IsClosed() { 230 | return nil, ErrIOURingClosed 231 | } 232 | 233 | var sqeN uint32 234 | userDatas := make([]*UserData, 0, len(requests)) 235 | for _, request := range requests { 236 | sqe := iour.getSQEntry() 237 | sqeN++ 238 | 239 | userData, err := iour.doRequest(sqe, request, ch) 240 | if err != nil { 241 | iour.sq.fallback(sqeN) 242 | return nil, err 243 | } 244 | userDatas = append(userDatas, userData) 245 | } 246 | 247 | // must be located before the lock operation to 248 | // avoid the compiler's adjustment of the code order. 249 | // issue: https://github.com/Iceber/iouring-go/issues/8 250 | rset := newRequestSet(userDatas) 251 | 252 | iour.userDataLock.Lock() 253 | for _, data := range userDatas { 254 | iour.userDatas[data.id] = data 255 | } 256 | iour.userDataLock.Unlock() 257 | 258 | if _, err := iour.submit(); err != nil { 259 | iour.userDataLock.Lock() 260 | for _, data := range userDatas { 261 | delete(iour.userDatas, data.id) 262 | } 263 | iour.userDataLock.Unlock() 264 | 265 | return nil, err 266 | } 267 | 268 | return rset, nil 269 | } 270 | 271 | func (iour *IOURing) needEnter(flags *uint32) bool { 272 | if (iour.Flags & iouring_syscall.IORING_SETUP_SQPOLL) == 0 { 273 | return true 274 | } 275 | 276 | if iour.sq.needWakeup() { 277 | *flags |= iouring_syscall.IORING_SQ_NEED_WAKEUP 278 | return true 279 | } 280 | return false 281 | } 282 | 283 | func (iour *IOURing) submit() (submitted int, err error) { 284 | submitted = iour.sq.flush() 285 | 286 | var flags uint32 287 | if !iour.needEnter(&flags) || submitted == 0 { 288 | return 289 | } 290 | 291 | if (iour.Flags & iouring_syscall.IORING_SETUP_IOPOLL) != 0 { 292 | flags |= iouring_syscall.IORING_ENTER_FLAGS_GETEVENTS 293 | } 294 | 295 | submitted, err = iouring_syscall.IOURingEnter(iour.fd, uint32(submitted), 0, flags, nil) 296 | return 297 | } 298 | 299 | /* 300 | func (iour *IOURing) submitAndWait(waitCount uint32) (submitted int, err error) { 301 | submitted = iour.sq.flush() 302 | 303 | var flags uint32 304 | if !iour.needEnter(&flags) && waitCount == 0 { 305 | return 306 | } 307 | 308 | if waitCount != 0 || (iour.Flags&iouring_syscall.IORING_SETUP_IOPOLL) != 0 { 309 | flags |= iouring_syscall.IORING_ENTER_FLAGS_GETEVENTS 310 | } 311 | 312 | submitted, err = iouring_syscall.IOURingEnter(iour.fd, uint32(submitted), waitCount, flags, nil) 313 | return 314 | } 315 | */ 316 | 317 | func (iour *IOURing) getCQEvent(wait bool) (cqe iouring_syscall.CompletionQueueEvent, err error) { 318 | var tryPeeks int 319 | for { 320 | if cqe = iour.cq.peek(); cqe != nil { 321 | // Copy CQE. 322 | cqe = cqe.Clone() 323 | iour.cq.advance(1) 324 | return 325 | } 326 | 327 | if !wait && !iour.sq.cqOverflow() { 328 | err = syscall.EAGAIN 329 | return 330 | } 331 | 332 | if iour.sq.cqOverflow() { 333 | _, err = iouring_syscall.IOURingEnter(iour.fd, 0, 0, iouring_syscall.IORING_ENTER_FLAGS_GETEVENTS, nil) 334 | if err != nil { 335 | return 336 | } 337 | continue 338 | } 339 | 340 | if tryPeeks++; tryPeeks < 3 { 341 | runtime.Gosched() 342 | continue 343 | } 344 | 345 | select { 346 | case <-iour.cqeSign: 347 | case <-iour.closer: 348 | return nil, ErrIOURingClosed 349 | } 350 | } 351 | } 352 | 353 | func (iour *IOURing) run() { 354 | for { 355 | cqe, err := iour.getCQEvent(true) 356 | if cqe == nil || err != nil { 357 | if err == ErrIOURingClosed { 358 | close(iour.closed) 359 | return 360 | } 361 | log.Println("runComplete error: ", err) 362 | continue 363 | } 364 | 365 | // log.Println("cqe user data", (cqe.UserData)) 366 | 367 | iour.userDataLock.Lock() 368 | userData := iour.userDatas[cqe.UserData()] 369 | if userData == nil { 370 | iour.userDataLock.Unlock() 371 | log.Println("runComplete: notfound user data ", uintptr(cqe.UserData())) 372 | continue 373 | } 374 | delete(iour.userDatas, cqe.UserData()) 375 | iour.userDataLock.Unlock() 376 | 377 | userData.request.complate(cqe) 378 | 379 | // ignore link timeout 380 | if userData.opcode == iouring_syscall.IORING_OP_LINK_TIMEOUT { 381 | continue 382 | } 383 | 384 | if userData.resulter != nil { 385 | userData.resulter <- userData.request 386 | } 387 | } 388 | } 389 | 390 | // Result submit cancel request 391 | func (iour *IOURing) submitCancel(id uint64) (Request, error) { 392 | if iour == nil { 393 | return nil, ErrRequestCompleted 394 | } 395 | 396 | return iour.SubmitRequest(cancelRequest(id), nil) 397 | } 398 | 399 | func cancelRequest(id uint64) PrepRequest { 400 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 401 | userData.request.resolver = cancelResolver 402 | sqe.PrepOperation(iouring_syscall.IORING_OP_ASYNC_CANCEL, -1, id, 0, 0) 403 | } 404 | } 405 | -------------------------------------------------------------------------------- /iouring_test.go: -------------------------------------------------------------------------------- 1 | package iouring 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func testSubmitRequests(t *testing.T, nreqs uint) { 10 | f, err := os.Open("/dev/zero") // For read access. 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | defer f.Close() 15 | iour, err := New(nreqs) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | defer iour.Close() 20 | 21 | fd := int(f.Fd()) 22 | for iter := 0; iter < 2; iter++ { 23 | var offset uint64 24 | preqs := make([]PrepRequest, nreqs) 25 | bufs := make([][]byte, nreqs) 26 | for i := range preqs { 27 | bufs[i] = make([]byte, 2) 28 | preqs[i] = Pread(fd, bufs[i], offset) 29 | offset += 2 30 | } 31 | 32 | requests, err := iour.SubmitRequests(preqs, nil) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | <-requests.Done() 37 | errResults := requests.ErrResults() 38 | if errResults != nil { 39 | t.Fatal(errResults[0].Err()) 40 | } 41 | } 42 | } 43 | 44 | func TestSubmitRequests(t *testing.T) { 45 | for i := uint(0); i < 8; i++ { 46 | nreqs := uint(1 << i) 47 | t.Run(fmt.Sprintf("%d", nreqs), func(t *testing.T) { testSubmitRequests(t, nreqs) }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /link_request.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "errors" 8 | "time" 9 | "unsafe" 10 | 11 | "golang.org/x/sys/unix" 12 | 13 | iouring_syscall "github.com/iceber/iouring-go/syscall" 14 | ) 15 | 16 | func (iour *IOURing) SubmitLinkRequests(requests []PrepRequest, ch chan<- Result) (RequestSet, error) { 17 | return iour.submitLinkRequest(requests, ch, false) 18 | } 19 | 20 | func (iour *IOURing) SubmitHardLinkRequests(requests []PrepRequest, ch chan<- Result) (RequestSet, error) { 21 | return iour.submitLinkRequest(requests, ch, true) 22 | } 23 | 24 | func (iour *IOURing) submitLinkRequest(requests []PrepRequest, ch chan<- Result, hard bool) (RequestSet, error) { 25 | // TODO(iceber): no length limit 26 | if len(requests) > int(*iour.sq.entries) { 27 | return nil, errors.New("too many requests") 28 | } 29 | 30 | flags := iouring_syscall.IOSQE_FLAGS_IO_LINK 31 | if hard { 32 | flags = iouring_syscall.IOSQE_FLAGS_IO_HARDLINK 33 | } 34 | 35 | iour.submitLock.Lock() 36 | defer iour.submitLock.Unlock() 37 | 38 | if iour.IsClosed() { 39 | return nil, ErrIOURingClosed 40 | } 41 | 42 | var sqeN uint32 43 | userDatas := make([]*UserData, 0, len(requests)) 44 | for i := range requests { 45 | sqe := iour.getSQEntry() 46 | sqeN++ 47 | 48 | userData, err := iour.doRequest(sqe, requests[i], ch) 49 | if err != nil { 50 | iour.sq.fallback(sqeN) 51 | return nil, err 52 | } 53 | userDatas = append(userDatas, userData) 54 | 55 | sqe.CleanFlags(iouring_syscall.IOSQE_FLAGS_IO_HARDLINK | iouring_syscall.IOSQE_FLAGS_IO_LINK) 56 | if i < len(requests)-1 { 57 | sqe.SetFlags(flags) 58 | } 59 | } 60 | 61 | // must be located before the lock operation to 62 | // avoid the compiler's adjustment of the code order. 63 | // issue: https://github.com/Iceber/iouring-go/issues/8 64 | rset := newRequestSet(userDatas) 65 | 66 | iour.userDataLock.Lock() 67 | for _, data := range userDatas { 68 | iour.userDatas[data.id] = data 69 | } 70 | iour.userDataLock.Unlock() 71 | 72 | if _, err := iour.submit(); err != nil { 73 | iour.userDataLock.Lock() 74 | for _, data := range userDatas { 75 | delete(iour.userDatas, data.id) 76 | } 77 | iour.userDataLock.Unlock() 78 | 79 | return nil, err 80 | } 81 | 82 | return rset, nil 83 | } 84 | 85 | func linkTimeout(t time.Duration) PrepRequest { 86 | timespec := unix.NsecToTimespec(t.Nanoseconds()) 87 | 88 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 89 | userData.hold(×pec) 90 | userData.request.resolver = timeoutResolver 91 | 92 | sqe.PrepOperation(iouring_syscall.IORING_OP_LINK_TIMEOUT, -1, uint64(uintptr(unsafe.Pointer(×pec))), 1, 0) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /mmap.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "reflect" 10 | "syscall" 11 | "unsafe" 12 | 13 | iouring_syscall "github.com/iceber/iouring-go/syscall" 14 | ) 15 | 16 | const uint32Size = uint32(unsafe.Sizeof(uint32(0))) 17 | 18 | func mmapIOURing(iour *IOURing) (err error) { 19 | defer func() { 20 | if err != nil { 21 | munmapIOURing(iour) 22 | } 23 | }() 24 | iour.sq = new(SubmissionQueue) 25 | iour.cq = new(CompletionQueue) 26 | 27 | if err = mmapSQ(iour); err != nil { 28 | return err 29 | } 30 | 31 | if (iour.params.Features & iouring_syscall.IORING_FEAT_SINGLE_MMAP) != 0 { 32 | iour.cq.ptr = iour.sq.ptr 33 | } 34 | 35 | if err = mmapCQ(iour); err != nil { 36 | return err 37 | } 38 | 39 | if err = mmapSQEs(iour); err != nil { 40 | return err 41 | } 42 | return nil 43 | } 44 | 45 | func mmapSQ(iour *IOURing) (err error) { 46 | sq := iour.sq 47 | params := iour.params 48 | 49 | sq.size = params.SQOffset.Array + params.SQEntries*uint32Size 50 | sq.ptr, err = mmap(iour.fd, sq.size, iouring_syscall.IORING_OFF_SQ_RING) 51 | if err != nil { 52 | return fmt.Errorf("mmap sq ring: %w", err) 53 | } 54 | 55 | sq.head = (*uint32)(unsafe.Pointer(sq.ptr + uintptr(params.SQOffset.Head))) 56 | sq.tail = (*uint32)(unsafe.Pointer(sq.ptr + uintptr(params.SQOffset.Tail))) 57 | sq.mask = (*uint32)(unsafe.Pointer(sq.ptr + uintptr(params.SQOffset.RingMask))) 58 | sq.entries = (*uint32)(unsafe.Pointer(sq.ptr + uintptr(params.SQOffset.RingEntries))) 59 | sq.flags = (*uint32)(unsafe.Pointer(sq.ptr + uintptr(params.SQOffset.Flags))) 60 | sq.dropped = (*uint32)(unsafe.Pointer(sq.ptr + uintptr(params.SQOffset.Dropped))) 61 | 62 | sq.array = *(*[]uint32)(unsafe.Pointer(&reflect.SliceHeader{ 63 | Data: sq.ptr + uintptr(params.SQOffset.Array), 64 | Len: int(params.SQEntries), 65 | Cap: int(params.SQEntries), 66 | })) 67 | 68 | return nil 69 | } 70 | 71 | func mmapCQ(iour *IOURing) (err error) { 72 | params := iour.params 73 | cq := iour.cq 74 | 75 | cqes := makeCompletionQueueRing(params.Flags) 76 | 77 | cq.size = params.CQOffset.Cqes + params.CQEntries*cqes.entrySz() 78 | if cq.ptr == 0 { 79 | cq.ptr, err = mmap(iour.fd, cq.size, iouring_syscall.IORING_OFF_CQ_RING) 80 | if err != nil { 81 | return fmt.Errorf("mmap cq ring: %w", err) 82 | } 83 | } 84 | 85 | cq.head = (*uint32)(unsafe.Pointer(cq.ptr + uintptr(params.CQOffset.Head))) 86 | cq.tail = (*uint32)(unsafe.Pointer(cq.ptr + uintptr(params.CQOffset.Tail))) 87 | cq.mask = (*uint32)(unsafe.Pointer(cq.ptr + uintptr(params.CQOffset.RingMask))) 88 | cq.entries = (*uint32)(unsafe.Pointer(cq.ptr + uintptr(params.CQOffset.RingEntries))) 89 | cq.flags = (*uint32)(unsafe.Pointer(cq.ptr + uintptr(params.CQOffset.Flags))) 90 | cq.overflow = (*uint32)(unsafe.Pointer(cq.ptr + uintptr(params.CQOffset.Overflow))) 91 | 92 | cqes.assignQueue(cq.ptr+uintptr(params.CQOffset.Cqes), int(params.CQEntries)) 93 | cq.cqes = cqes 94 | 95 | return nil 96 | } 97 | 98 | func mmapSQEs(iour *IOURing) error { 99 | params := iour.params 100 | 101 | sqes := makeSubmissionQueueRing(params.Flags) 102 | 103 | ptr, err := mmap(iour.fd, params.SQEntries*sqes.entrySz(), iouring_syscall.IORING_OFF_SQES) 104 | if err != nil { 105 | return fmt.Errorf("mmap sqe array: %w", err) 106 | } 107 | 108 | sqes.assignQueue(ptr, int(params.SQEntries)) 109 | iour.sq.sqes = sqes 110 | 111 | return nil 112 | } 113 | 114 | func munmapIOURing(iour *IOURing) error { 115 | if iour.sq != nil && iour.sq.ptr != 0 { 116 | if iour.sq.sqes.isActive() { 117 | err := munmap(iour.sq.sqes.mappedPtr(), iour.sq.sqes.ringSz()) 118 | if err != nil { 119 | return fmt.Errorf("ummap sqe array: %w", err) 120 | } 121 | iour.sq.sqes = nil 122 | } 123 | 124 | if err := munmap(iour.sq.ptr, iour.sq.size); err != nil { 125 | return fmt.Errorf("munmap sq: %w", err) 126 | } 127 | if iour.sq.ptr == iour.cq.ptr { 128 | iour.cq = nil 129 | } 130 | iour.sq = nil 131 | } 132 | 133 | if iour.cq != nil && iour.cq.ptr != 0 { 134 | if err := munmap(iour.cq.ptr, iour.cq.size); err != nil { 135 | return fmt.Errorf("munmap cq: %w", err) 136 | } 137 | iour.cq = nil 138 | } 139 | 140 | return nil 141 | } 142 | 143 | func mmap(fd int, length uint32, offset uint64) (uintptr, error) { 144 | ptr, _, errno := syscall.Syscall6( 145 | syscall.SYS_MMAP, 146 | 0, 147 | uintptr(length), 148 | syscall.PROT_READ|syscall.PROT_WRITE, 149 | syscall.MAP_SHARED|syscall.MAP_POPULATE, 150 | uintptr(fd), 151 | uintptr(offset), 152 | ) 153 | if errno != 0 { 154 | return 0, os.NewSyscallError("mmap", errno) 155 | } 156 | return uintptr(ptr), nil 157 | } 158 | 159 | func munmap(ptr uintptr, length uint32) error { 160 | _, _, errno := syscall.Syscall( 161 | syscall.SYS_MUNMAP, 162 | ptr, 163 | uintptr(length), 164 | 0, 165 | ) 166 | if errno != 0 { 167 | return os.NewSyscallError("munmap", errno) 168 | } 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "time" 8 | 9 | iouring_syscall "github.com/iceber/iouring-go/syscall" 10 | ) 11 | 12 | type IOURingOption func(*IOURing) 13 | 14 | // WithSQPoll a kernel thread is created to perform submission queue polling 15 | // In Version 5.10 and later, allow using this as non-root, 16 | // if the user has the CAP_SYS_NICE capability 17 | func WithSQPoll() IOURingOption { 18 | return func(iour *IOURing) { 19 | iour.params.Flags |= iouring_syscall.IORING_SETUP_SQPOLL 20 | } 21 | } 22 | 23 | // WithSQPollThreadCPU the poll thread will be bound to the cpu set, only meaningful when WithSQPoll option 24 | func WithSQPollThreadCPU(cpu uint32) IOURingOption { 25 | return func(iour *IOURing) { 26 | iour.params.Flags |= iouring_syscall.IORING_SETUP_SQ_AFF 27 | iour.params.SQThreadCPU = cpu 28 | } 29 | } 30 | 31 | func WithSQPollThreadIdle(idle time.Duration) IOURingOption { 32 | return func(iour *IOURing) { 33 | iour.params.SQThreadIdle = uint32(idle / time.Millisecond) 34 | } 35 | } 36 | 37 | // WithParams use params 38 | func WithParams(params *iouring_syscall.IOURingParams) IOURingOption { 39 | return func(iour *IOURing) { 40 | iour.params = params 41 | } 42 | } 43 | 44 | // WithCQSize create the completion queue with size entries 45 | // size must bue greater than entries 46 | func WithCQSize(size uint32) IOURingOption { 47 | return func(iour *IOURing) { 48 | iour.params.Flags |= iouring_syscall.IORING_SETUP_CQSIZE 49 | iour.params.CQEntries = size 50 | } 51 | } 52 | 53 | // WithAttachWQ new iouring instance being create will share the asynchronous worker thread 54 | // backend of the specified io_uring ring, rather than create a new separate thread pool 55 | func WithAttachWQ(iour *IOURing) IOURingOption { 56 | return func(iour *IOURing) { 57 | iour.params.Flags |= iouring_syscall.IORING_SETUP_ATTACH_WQ 58 | iour.params.WQFd = uint32(iour.fd) 59 | } 60 | } 61 | 62 | func WithAsync() IOURingOption { 63 | return func(iour *IOURing) { 64 | iour.async = true 65 | } 66 | } 67 | 68 | // WithDisableRing the io_uring ring starts in a disabled state 69 | // In this state, restrictions can be registered, but submissions are not allowed 70 | // Available since 5.10 71 | func WithDisableRing() IOURingOption { 72 | return func(iour *IOURing) { 73 | iour.params.Flags |= iouring_syscall.IORING_SETUP_R_DISABLED 74 | } 75 | } 76 | 77 | // WithDrain every SQE will not be started before previously submitted SQEs have completed 78 | func WithDrain() IOURingOption { 79 | return func(iour *IOURing) { 80 | iour.drain = true 81 | } 82 | } 83 | 84 | // WithSQE128 every SQE will have 128B entry size to append IOCTL command 85 | func WithSQE128() IOURingOption { 86 | return func(iour *IOURing) { 87 | iour.params.Flags |= iouring_syscall.IORING_SETUP_SQE128 88 | } 89 | } 90 | 91 | // WithCQE32 every CQE will have 32B entry size to append IOCTL return data 92 | func WithCQE32() IOURingOption { 93 | return func(iour *IOURing) { 94 | iour.params.Flags |= iouring_syscall.IORING_SETUP_CQE32 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /poller.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring 4 | 5 | import ( 6 | "os" 7 | "sync" 8 | 9 | "golang.org/x/sys/unix" 10 | ) 11 | 12 | const initEpollEvents = 1 13 | 14 | type iourPoller struct { 15 | sync.Mutex 16 | 17 | fd int 18 | iours map[int]*IOURing 19 | events []unix.EpollEvent 20 | } 21 | 22 | var ( 23 | poller *iourPoller 24 | initpollerLock sync.Mutex 25 | ) 26 | 27 | func initpoller() error { 28 | // fast path 29 | if poller != nil { 30 | return nil 31 | } 32 | 33 | initpollerLock.Lock() 34 | defer initpollerLock.Unlock() 35 | if poller != nil { 36 | return nil 37 | } 38 | 39 | epfd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC) 40 | if err != nil { 41 | return os.NewSyscallError("epoll_create1", err) 42 | } 43 | 44 | poller = &iourPoller{ 45 | fd: epfd, 46 | iours: make(map[int]*IOURing), 47 | events: make([]unix.EpollEvent, initEpollEvents), 48 | } 49 | 50 | go poller.run() 51 | return nil 52 | } 53 | 54 | func registerIOURing(iour *IOURing) error { 55 | if err := initpoller(); err != nil { 56 | return err 57 | } 58 | 59 | if err := unix.EpollCtl(poller.fd, unix.EPOLL_CTL_ADD, iour.eventfd, 60 | &unix.EpollEvent{Fd: int32(iour.eventfd), Events: unix.EPOLLIN | unix.EPOLLET}, 61 | ); err != nil { 62 | return os.NewSyscallError("epoll_ctl_add", err) 63 | } 64 | 65 | poller.Lock() 66 | poller.iours[iour.eventfd] = iour 67 | poller.Unlock() 68 | return nil 69 | } 70 | 71 | func removeIOURing(iour *IOURing) error { 72 | poller.Lock() 73 | delete(poller.iours, iour.eventfd) 74 | poller.Unlock() 75 | 76 | return os.NewSyscallError("epoll_ctl_del", 77 | unix.EpollCtl(poller.fd, unix.EPOLL_CTL_DEL, iour.eventfd, nil)) 78 | } 79 | 80 | func (poller *iourPoller) run() { 81 | for { 82 | n, err := unix.EpollWait(poller.fd, poller.events, -1) 83 | if err != nil { 84 | continue 85 | } 86 | 87 | for i := 0; i < n; i++ { 88 | fd := int(poller.events[i].Fd) 89 | poller.Lock() 90 | iour, ok := poller.iours[fd] 91 | poller.Unlock() 92 | if !ok { 93 | continue 94 | } 95 | 96 | select { 97 | case iour.cqeSign <- struct{}{}: 98 | default: 99 | } 100 | } 101 | 102 | poller.adjust() 103 | } 104 | } 105 | 106 | func (poller *iourPoller) adjust() { 107 | poller.Lock() 108 | l := len(poller.iours) - len(poller.events) 109 | poller.Unlock() 110 | 111 | if l <= 0 { 112 | return 113 | } 114 | 115 | events := make([]unix.EpollEvent, l*2) 116 | poller.events = append(poller.events, events...) 117 | } 118 | -------------------------------------------------------------------------------- /prep_request.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "errors" 8 | "os" 9 | "syscall" 10 | "unsafe" 11 | 12 | "golang.org/x/sys/unix" 13 | 14 | iouring_syscall "github.com/iceber/iouring-go/syscall" 15 | ) 16 | 17 | type PrepRequest func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) 18 | 19 | // WithInfo request with extra info 20 | func (prepReq PrepRequest) WithInfo(info interface{}) PrepRequest { 21 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 22 | prepReq(sqe, userData) 23 | userData.SetRequestInfo(info) 24 | } 25 | } 26 | 27 | func (prepReq PrepRequest) WithDrain() PrepRequest { 28 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 29 | prepReq(sqe, userData) 30 | sqe.SetFlags(iouring_syscall.IOSQE_FLAGS_IO_DRAIN) 31 | } 32 | } 33 | 34 | func (prepReq PrepRequest) WithCallback(callback RequestCallback) PrepRequest { 35 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 36 | prepReq(sqe, userData) 37 | userData.SetRequestCallback(callback) 38 | } 39 | } 40 | 41 | func (iour *IOURing) Read(file *os.File, b []byte, ch chan<- Result) (Request, error) { 42 | fd := int(file.Fd()) 43 | if fd < 0 { 44 | return nil, errors.New("invalid file") 45 | } 46 | 47 | return iour.SubmitRequest(Read(fd, b), ch) 48 | } 49 | 50 | func (iour *IOURing) Write(file *os.File, b []byte, ch chan<- Result) (Request, error) { 51 | fd := int(file.Fd()) 52 | if fd < 0 { 53 | return nil, errors.New("invalid file") 54 | } 55 | 56 | return iour.SubmitRequest(Write(fd, b), ch) 57 | } 58 | 59 | func (iour *IOURing) Pread(file *os.File, b []byte, offset uint64, ch chan<- Result) (Request, error) { 60 | fd := int(file.Fd()) 61 | if fd < 0 { 62 | return nil, errors.New("invalid file") 63 | } 64 | 65 | return iour.SubmitRequest(Pread(fd, b, offset), ch) 66 | } 67 | 68 | func (iour *IOURing) Pwrite(file *os.File, b []byte, offset uint64, ch chan<- Result) (Request, error) { 69 | fd := int(file.Fd()) 70 | if fd < 0 { 71 | return nil, errors.New("invalid file") 72 | } 73 | 74 | return iour.SubmitRequest(Pwrite(fd, b, offset), ch) 75 | } 76 | 77 | func Nop() PrepRequest { 78 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 79 | sqe.PrepOperation(iouring_syscall.IORING_OP_NOP, -1, 0, 0, 0) 80 | } 81 | } 82 | 83 | func Read(fd int, b []byte) PrepRequest { 84 | var bp unsafe.Pointer 85 | if len(b) > 0 { 86 | bp = unsafe.Pointer(&b[0]) 87 | } else { 88 | bp = unsafe.Pointer(&_zero) 89 | } 90 | 91 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 92 | userData.request.resolver = fdResolver 93 | userData.SetRequestBuffer(b, nil) 94 | 95 | sqe.PrepOperation( 96 | iouring_syscall.IORING_OP_READ, 97 | int32(fd), 98 | uint64(uintptr(bp)), 99 | uint32(len(b)), 100 | 0, 101 | ) 102 | } 103 | } 104 | 105 | func Pread(fd int, b []byte, offset uint64) PrepRequest { 106 | var bp unsafe.Pointer 107 | if len(b) > 0 { 108 | bp = unsafe.Pointer(&b[0]) 109 | } else { 110 | bp = unsafe.Pointer(&_zero) 111 | } 112 | 113 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 114 | userData.request.resolver = fdResolver 115 | userData.SetRequestBuffer(b, nil) 116 | 117 | sqe.PrepOperation( 118 | iouring_syscall.IORING_OP_READ, 119 | int32(fd), 120 | uint64(uintptr(bp)), 121 | uint32(len(b)), 122 | uint64(offset), 123 | ) 124 | } 125 | } 126 | 127 | func Write(fd int, b []byte) PrepRequest { 128 | var bp unsafe.Pointer 129 | if len(b) > 0 { 130 | bp = unsafe.Pointer(&b[0]) 131 | } else { 132 | bp = unsafe.Pointer(&_zero) 133 | } 134 | 135 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 136 | userData.request.resolver = fdResolver 137 | userData.SetRequestBuffer(b, nil) 138 | 139 | sqe.PrepOperation( 140 | iouring_syscall.IORING_OP_WRITE, 141 | int32(fd), 142 | uint64(uintptr(bp)), 143 | uint32(len(b)), 144 | 0, 145 | ) 146 | } 147 | } 148 | 149 | func Pwrite(fd int, b []byte, offset uint64) PrepRequest { 150 | var bp unsafe.Pointer 151 | if len(b) > 0 { 152 | bp = unsafe.Pointer(&b[0]) 153 | } else { 154 | bp = unsafe.Pointer(&_zero) 155 | } 156 | 157 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 158 | userData.request.resolver = fdResolver 159 | userData.SetRequestBuffer(b, nil) 160 | 161 | sqe.PrepOperation( 162 | iouring_syscall.IORING_OP_WRITE, 163 | int32(fd), 164 | uint64(uintptr(bp)), 165 | uint32(len(b)), 166 | uint64(offset), 167 | ) 168 | } 169 | } 170 | 171 | func Readv(fd int, bs [][]byte) PrepRequest { 172 | iovecs := bytes2iovec(bs) 173 | 174 | var bp unsafe.Pointer 175 | if len(iovecs) > 0 { 176 | bp = unsafe.Pointer(&iovecs[0]) 177 | } else { 178 | bp = unsafe.Pointer(&_zero) 179 | } 180 | 181 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 182 | userData.request.resolver = fdResolver 183 | userData.SetRequestBuffers(bs) 184 | 185 | sqe.PrepOperation( 186 | iouring_syscall.IORING_OP_READV, 187 | int32(fd), 188 | uint64(uintptr(bp)), 189 | uint32(len(iovecs)), 190 | 0, 191 | ) 192 | } 193 | } 194 | 195 | func Preadv(fd int, bs [][]byte, offset uint64) PrepRequest { 196 | iovecs := bytes2iovec(bs) 197 | 198 | var bp unsafe.Pointer 199 | if len(iovecs) > 0 { 200 | bp = unsafe.Pointer(&iovecs[0]) 201 | } else { 202 | bp = unsafe.Pointer(&_zero) 203 | } 204 | 205 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 206 | userData.request.resolver = fdResolver 207 | userData.SetRequestBuffers(bs) 208 | 209 | sqe.PrepOperation(iouring_syscall.IORING_OP_READV, 210 | int32(fd), 211 | uint64(uintptr(bp)), 212 | uint32(len(iovecs)), 213 | offset, 214 | ) 215 | } 216 | } 217 | 218 | func Writev(fd int, bs [][]byte) PrepRequest { 219 | iovecs := bytes2iovec(bs) 220 | 221 | var bp unsafe.Pointer 222 | if len(iovecs) > 0 { 223 | bp = unsafe.Pointer(&iovecs[0]) 224 | } else { 225 | bp = unsafe.Pointer(&_zero) 226 | } 227 | 228 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 229 | userData.request.resolver = fdResolver 230 | userData.SetRequestBuffers(bs) 231 | 232 | sqe.PrepOperation( 233 | iouring_syscall.IORING_OP_WRITEV, 234 | int32(fd), 235 | uint64(uintptr(bp)), 236 | uint32(len(iovecs)), 237 | 0, 238 | ) 239 | } 240 | } 241 | 242 | func Pwritev(fd int, bs [][]byte, offset int64) PrepRequest { 243 | iovecs := bytes2iovec(bs) 244 | 245 | var bp unsafe.Pointer 246 | if len(iovecs) > 0 { 247 | bp = unsafe.Pointer(&iovecs[0]) 248 | } else { 249 | bp = unsafe.Pointer(&_zero) 250 | } 251 | 252 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 253 | userData.request.resolver = fdResolver 254 | userData.SetRequestBuffers(bs) 255 | 256 | sqe.PrepOperation( 257 | iouring_syscall.IORING_OP_WRITEV, 258 | int32(fd), 259 | uint64(uintptr(bp)), 260 | uint32(len(iovecs)), 261 | uint64(offset), 262 | ) 263 | } 264 | } 265 | 266 | func Send(sockfd int, b []byte, flags int) PrepRequest { 267 | var bp unsafe.Pointer 268 | if len(b) > 0 { 269 | bp = unsafe.Pointer(&b[0]) 270 | } else { 271 | bp = unsafe.Pointer(&_zero) 272 | } 273 | 274 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 275 | userData.SetRequestBuffer(b, nil) 276 | 277 | sqe.PrepOperation( 278 | iouring_syscall.IORING_OP_SEND, 279 | int32(sockfd), 280 | uint64(uintptr(bp)), 281 | uint32(len(b)), 282 | 0, 283 | ) 284 | sqe.SetOpFlags(uint32(flags)) 285 | } 286 | } 287 | 288 | func Recv(sockfd int, b []byte, flags int) PrepRequest { 289 | var bp unsafe.Pointer 290 | if len(b) > 0 { 291 | bp = unsafe.Pointer(&b[0]) 292 | } else { 293 | bp = unsafe.Pointer(&_zero) 294 | } 295 | 296 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 297 | userData.SetRequestBuffer(b, nil) 298 | 299 | sqe.PrepOperation( 300 | iouring_syscall.IORING_OP_RECV, 301 | int32(sockfd), 302 | uint64(uintptr(bp)), 303 | uint32(len(b)), 304 | 0, 305 | ) 306 | sqe.SetOpFlags(uint32(flags)) 307 | } 308 | } 309 | 310 | func Sendmsg(sockfd int, p, oob []byte, to syscall.Sockaddr, flags int) (PrepRequest, error) { 311 | var ptr unsafe.Pointer 312 | var salen uint32 313 | if to != nil { 314 | var err error 315 | ptr, salen, err = sockaddr(to) 316 | if err != nil { 317 | return nil, err 318 | } 319 | } 320 | 321 | msg := &syscall.Msghdr{} 322 | msg.Name = (*byte)(ptr) 323 | msg.Namelen = uint32(salen) 324 | var iov syscall.Iovec 325 | if len(p) > 0 { 326 | iov.Base = &p[0] 327 | iov.SetLen(len(p)) 328 | } 329 | var dummy byte 330 | if len(oob) > 0 { 331 | if len(p) == 0 { 332 | var sockType int 333 | sockType, err := syscall.GetsockoptInt(sockfd, syscall.SOL_SOCKET, syscall.SO_TYPE) 334 | if err != nil { 335 | return nil, err 336 | } 337 | // send at least one normal byte 338 | if sockType != syscall.SOCK_DGRAM { 339 | iov.Base = &dummy 340 | iov.SetLen(1) 341 | } 342 | } 343 | msg.Control = &oob[0] 344 | msg.SetControllen(len(oob)) 345 | } 346 | msg.Iov = &iov 347 | msg.Iovlen = 1 348 | 349 | resolver := func(req Request) { 350 | result := req.(*request) 351 | result.r0 = int(result.res) 352 | errResolver(result) 353 | if result.err != nil { 354 | return 355 | } 356 | 357 | if len(oob) > 0 && len(p) == 0 { 358 | result.r0 = 0 359 | } 360 | } 361 | 362 | msgptr := unsafe.Pointer(msg) 363 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 364 | userData.hold(msg, to) 365 | userData.request.resolver = resolver 366 | userData.SetRequestBuffer(p, oob) 367 | 368 | sqe.PrepOperation(iouring_syscall.IORING_OP_SENDMSG, int32(sockfd), uint64(uintptr(msgptr)), 1, 0) 369 | sqe.SetOpFlags(uint32(flags)) 370 | }, nil 371 | } 372 | 373 | func Recvmsg(sockfd int, p, oob []byte, to syscall.Sockaddr, flags int) (PrepRequest, error) { 374 | var msg syscall.Msghdr 375 | var rsa syscall.RawSockaddrAny 376 | msg.Name = (*byte)(unsafe.Pointer(&rsa)) 377 | msg.Namelen = uint32(syscall.SizeofSockaddrAny) 378 | var iov syscall.Iovec 379 | if len(p) > 0 { 380 | iov.Base = &p[0] 381 | iov.SetLen(len(p)) 382 | } 383 | var dummy byte 384 | if len(oob) > 0 { 385 | if len(p) == 0 { 386 | var sockType int 387 | sockType, err := syscall.GetsockoptInt(sockfd, syscall.SOL_SOCKET, syscall.SO_TYPE) 388 | if err != nil { 389 | return nil, err 390 | } 391 | // receive at least one normal byte 392 | if sockType != syscall.SOCK_DGRAM { 393 | iov.Base = &dummy 394 | iov.SetLen(1) 395 | } 396 | } 397 | msg.Control = &oob[0] 398 | msg.SetControllen(len(oob)) 399 | } 400 | msg.Iov = &iov 401 | msg.Iovlen = 1 402 | 403 | resolver := func(req Request) { 404 | result := req.(*request) 405 | result.r0 = int(result.res) 406 | errResolver(result) 407 | if result.err != nil { 408 | return 409 | } 410 | 411 | if len(oob) > 0 && len(p) == 0 { 412 | result.r0 = 0 413 | } 414 | } 415 | 416 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 417 | userData.hold(&msg, &rsa) 418 | userData.request.resolver = resolver 419 | userData.SetRequestBuffer(p, oob) 420 | 421 | sqe.PrepOperation( 422 | iouring_syscall.IORING_OP_RECVMSG, 423 | int32(sockfd), 424 | uint64(uintptr(unsafe.Pointer(&msg))), 425 | 1, 426 | 0, 427 | ) 428 | sqe.SetOpFlags(uint32(flags)) 429 | }, nil 430 | } 431 | 432 | func Accept(sockfd int) PrepRequest { 433 | var rsa syscall.RawSockaddrAny 434 | var len uint32 = syscall.SizeofSockaddrAny 435 | 436 | resolver := func(req Request) { 437 | result := req.(*request) 438 | fd := int(result.res) 439 | errResolver(result) 440 | if result.err != nil { 441 | return 442 | } 443 | 444 | result.r0 = fd 445 | result.r1, result.err = anyToSockaddr(&rsa) 446 | if result.err != nil { 447 | syscall.Close(fd) 448 | result.r0 = 0 449 | } 450 | } 451 | 452 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 453 | userData.hold(&len) 454 | userData.request.resolver = resolver 455 | sqe.PrepOperation(iouring_syscall.IORING_OP_ACCEPT, int32(sockfd), uint64(uintptr(unsafe.Pointer(&rsa))), 0, uint64(uintptr(unsafe.Pointer(&len)))) 456 | } 457 | } 458 | 459 | func Accept4(sockfd int, flags int) PrepRequest { 460 | var rsa syscall.RawSockaddrAny 461 | var len uint32 = syscall.SizeofSockaddrAny 462 | 463 | resolver := func(req Request) { 464 | result := req.(*request) 465 | fd := int(result.res) 466 | errResolver(result) 467 | if result.err != nil { 468 | return 469 | } 470 | 471 | if len > syscall.SizeofSockaddrAny { 472 | panic("RawSockaddrAny too small") 473 | } 474 | 475 | result.r0 = fd 476 | result.r1, result.err = anyToSockaddr(&rsa) 477 | if result.err != nil { 478 | syscall.Close(fd) 479 | result.r0 = 0 480 | } 481 | } 482 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 483 | userData.hold(&rsa, &len) 484 | userData.request.resolver = resolver 485 | 486 | sqe.PrepOperation( 487 | iouring_syscall.IORING_OP_ACCEPT, 488 | int32(sockfd), 489 | uint64(uintptr(unsafe.Pointer(&rsa))), 490 | 0, 491 | uint64(uintptr(unsafe.Pointer(&len))), 492 | ) 493 | sqe.SetOpFlags(uint32(flags)) 494 | } 495 | } 496 | 497 | func Connect(sockfd int, sa syscall.Sockaddr) (PrepRequest, error) { 498 | ptr, n, err := sockaddr(sa) 499 | if err != nil { 500 | return nil, err 501 | } 502 | 503 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 504 | userData.hold(sa) 505 | userData.request.resolver = errResolver 506 | 507 | sqe.PrepOperation( 508 | iouring_syscall.IORING_OP_CONNECT, 509 | int32(sockfd), 510 | uint64(uintptr(ptr)), 511 | 0, 512 | uint64(n), 513 | ) 514 | }, nil 515 | } 516 | 517 | func Openat(dirfd int, path string, flags uint32, mode uint32) (PrepRequest, error) { 518 | flags |= syscall.O_LARGEFILE 519 | b, err := syscall.ByteSliceFromString(path) 520 | if err != nil { 521 | return nil, err 522 | } 523 | 524 | bp := unsafe.Pointer(&b[0]) 525 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 526 | userData.hold(&b) 527 | userData.request.resolver = fdResolver 528 | 529 | sqe.PrepOperation( 530 | iouring_syscall.IORING_OP_OPENAT, 531 | int32(dirfd), 532 | uint64(uintptr(bp)), 533 | mode, 534 | 0, 535 | ) 536 | sqe.SetOpFlags(flags) 537 | }, nil 538 | } 539 | 540 | func Openat2(dirfd int, path string, how *unix.OpenHow) (PrepRequest, error) { 541 | b, err := syscall.ByteSliceFromString(path) 542 | if err != nil { 543 | return nil, err 544 | } 545 | 546 | bp := unsafe.Pointer(&b[0]) 547 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 548 | userData.hold(&b) 549 | userData.request.resolver = fdResolver 550 | 551 | sqe.PrepOperation( 552 | iouring_syscall.IORING_OP_OPENAT2, 553 | int32(dirfd), 554 | uint64(uintptr(bp)), 555 | unix.SizeofOpenHow, 556 | uint64(uintptr(unsafe.Pointer(how))), 557 | ) 558 | }, nil 559 | } 560 | 561 | func Statx(dirfd int, path string, flags uint32, mask int, stat *unix.Statx_t) (PrepRequest, error) { 562 | b, err := syscall.ByteSliceFromString(path) 563 | if err != nil { 564 | return nil, err 565 | } 566 | 567 | bp := unsafe.Pointer(&b[0]) 568 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 569 | userData.request.resolver = errResolver 570 | userData.hold(&b, stat) 571 | 572 | sqe.PrepOperation( 573 | iouring_syscall.IORING_OP_STATX, 574 | int32(dirfd), 575 | uint64(uintptr(bp)), 576 | uint32(mask), 577 | uint64(uintptr(unsafe.Pointer(stat))), 578 | ) 579 | sqe.SetOpFlags(flags) 580 | }, nil 581 | } 582 | 583 | func Fsync(fd int) PrepRequest { 584 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 585 | userData.request.resolver = errResolver 586 | sqe.PrepOperation(iouring_syscall.IORING_OP_FSYNC, int32(fd), 0, 0, 0) 587 | } 588 | } 589 | 590 | func Fdatasync(fd int) PrepRequest { 591 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 592 | userData.request.resolver = errResolver 593 | sqe.PrepOperation(iouring_syscall.IORING_OP_FSYNC, int32(fd), 0, 0, 0) 594 | sqe.SetOpFlags(iouring_syscall.IORING_FSYNC_DATASYNC) 595 | } 596 | } 597 | 598 | func Fallocate(fd int, mode uint32, off int64, length int64) PrepRequest { 599 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 600 | userData.request.resolver = errResolver 601 | 602 | sqe.PrepOperation( 603 | iouring_syscall.IORING_OP_FALLOCATE, 604 | int32(fd), 605 | uint64(length), 606 | uint32(mode), 607 | uint64(off), 608 | ) 609 | } 610 | } 611 | 612 | func Close(fd int) PrepRequest { 613 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 614 | userData.request.resolver = errResolver 615 | sqe.PrepOperation(iouring_syscall.IORING_OP_CLOSE, int32(fd), 0, 0, 0) 616 | } 617 | } 618 | 619 | func Madvise(b []byte, advice int) PrepRequest { 620 | var bp unsafe.Pointer 621 | if len(b) > 0 { 622 | bp = unsafe.Pointer(&b[0]) 623 | } else { 624 | bp = unsafe.Pointer(&_zero) 625 | } 626 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 627 | userData.request.resolver = errResolver 628 | userData.SetRequestBuffer(b, nil) 629 | 630 | sqe.PrepOperation( 631 | iouring_syscall.IORING_OP_MADVISE, 632 | -1, 633 | uint64(uintptr(bp)), 634 | uint32(len(b)), 635 | 0, 636 | ) 637 | sqe.SetOpFlags(uint32(advice)) 638 | } 639 | } 640 | 641 | func EpollCtl(epfd int, op int, fd int, event *syscall.EpollEvent) PrepRequest { 642 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 643 | userData.request.resolver = errResolver 644 | 645 | sqe.PrepOperation( 646 | iouring_syscall.IORING_OP_EPOLL_CTL, 647 | int32(epfd), 648 | uint64(uintptr(unsafe.Pointer(event))), 649 | uint32(op), 650 | uint64(fd), 651 | ) 652 | } 653 | } 654 | 655 | func Mkdirat(dirFd int, path string, mode uint32) (PrepRequest, error) { 656 | b, err := syscall.ByteSliceFromString(path) 657 | if err != nil { 658 | return nil, err 659 | } 660 | 661 | bp := unsafe.Pointer(&b[0]) 662 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 663 | userData.hold(&b) 664 | userData.request.resolver = fdResolver 665 | 666 | sqe.PrepOperation( 667 | iouring_syscall.IORING_OP_MKDIRAT, 668 | int32(dirFd), 669 | uint64(uintptr(bp)), 670 | mode, 671 | 0, 672 | ) 673 | }, nil 674 | } 675 | 676 | func Unlinkat(fd int, path string, flags int32) (PrepRequest, error) { 677 | b, err := syscall.ByteSliceFromString(path) 678 | if err != nil { 679 | return nil, err 680 | } 681 | 682 | bp := unsafe.Pointer(&b[0]) 683 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 684 | userData.hold(&b) 685 | userData.request.resolver = fdResolver 686 | 687 | sqe.PrepOperation( 688 | iouring_syscall.IORING_OP_UNLINKAT, 689 | int32(fd), 690 | uint64(uintptr(bp)), 691 | 0, 692 | 0, 693 | ) 694 | sqe.SetOpFlags(uint32(flags)) 695 | }, nil 696 | } 697 | 698 | func Symlinkat(target string, newDirFd int, linkPath string) (PrepRequest, error) { 699 | bTarget, err := syscall.ByteSliceFromString(target) 700 | if err != nil { 701 | return nil, err 702 | } 703 | bLink, err := syscall.ByteSliceFromString(linkPath) 704 | if err != nil { 705 | return nil, err 706 | } 707 | 708 | bpTarget := unsafe.Pointer(&bTarget[0]) 709 | bpLink := unsafe.Pointer(&bLink[0]) 710 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 711 | userData.hold(&bTarget, &bLink) 712 | userData.request.resolver = fdResolver 713 | 714 | sqe.PrepOperation( 715 | iouring_syscall.IORING_OP_SYMLINKAT, 716 | int32(newDirFd), 717 | uint64(uintptr(bpTarget)), 718 | 0, 719 | uint64(uintptr(bpLink)), 720 | ) 721 | }, nil 722 | } 723 | 724 | func Renameat2(oldDirFd int, oldPath string, newDirFd int, newPath string, flags int) (PrepRequest, error) { 725 | bOldPath, err := syscall.ByteSliceFromString(oldPath) 726 | if err != nil { 727 | return nil, err 728 | } 729 | bNewPath, err := syscall.ByteSliceFromString(newPath) 730 | if err != nil { 731 | return nil, err 732 | } 733 | 734 | bpOldPath := unsafe.Pointer(&bOldPath[0]) 735 | bpNewPath := unsafe.Pointer(&bNewPath[0]) 736 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 737 | userData.hold(&bOldPath, &bNewPath) 738 | userData.request.resolver = fdResolver 739 | 740 | sqe.PrepOperation( 741 | iouring_syscall.IORING_OP_RENAMEAT, 742 | int32(oldDirFd), uint64(uintptr(bpOldPath)), 743 | uint32(newDirFd), uint64(uintptr(bpNewPath)), 744 | ) 745 | sqe.SetOpFlags(uint32(flags)) 746 | }, nil 747 | } 748 | 749 | func Renameat(oldDirFd int, oldPath string, newDirFd int, newPath string) (PrepRequest, error) { 750 | return Renameat2(oldDirFd, oldPath, newDirFd, newPath, 0) 751 | } 752 | 753 | func Linkat(targetDirFd int, targetPath string, linkDirFd int, linkPath string, flags int) (PrepRequest, error) { 754 | bTargetPath, err := syscall.ByteSliceFromString(targetPath) 755 | if err != nil { 756 | return nil, err 757 | } 758 | bLinkPath, err := syscall.ByteSliceFromString(linkPath) 759 | if err != nil { 760 | return nil, err 761 | } 762 | 763 | bpTargetPath := unsafe.Pointer(&bTargetPath[0]) 764 | bpLinkPath := unsafe.Pointer(&bLinkPath[0]) 765 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 766 | userData.hold(&bpTargetPath, &bpLinkPath) 767 | userData.request.resolver = fdResolver 768 | 769 | sqe.PrepOperation( 770 | iouring_syscall.IORING_OP_LINKAT, 771 | int32(targetDirFd), 772 | uint64(uintptr(bpTargetPath)), 773 | uint32(linkDirFd), 774 | uint64(uintptr(bpLinkPath)), 775 | ) 776 | sqe.SetOpFlags(uint32(flags)) 777 | }, nil 778 | } 779 | -------------------------------------------------------------------------------- /probe.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring 4 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "errors" 8 | "sync" 9 | "sync/atomic" 10 | "syscall" 11 | 12 | iouring_syscall "github.com/iceber/iouring-go/syscall" 13 | ) 14 | 15 | type ResultResolver func(req Request) 16 | 17 | type RequestCallback func(result Result) error 18 | 19 | type Request interface { 20 | Result 21 | 22 | Cancel() (Request, error) 23 | Done() <-chan struct{} 24 | 25 | GetRes() (int, error) 26 | // Can Only be used in ResultResolver 27 | SetResult(r0, r1 interface{}, err error) error 28 | } 29 | 30 | type Result interface { 31 | Fd() int 32 | Opcode() uint8 33 | GetRequestBuffer() (b0, b1 []byte) 34 | GetRequestBuffers() [][]byte 35 | GetRequestInfo() interface{} 36 | FreeRequestBuffer() 37 | 38 | Err() error 39 | ReturnValue0() interface{} 40 | ReturnValue1() interface{} 41 | ReturnExtra1() uint64 42 | ReturnExtra2() uint64 43 | ReturnFd() (int, error) 44 | ReturnInt() (int, error) 45 | 46 | Callback() error 47 | } 48 | 49 | var _ Request = &request{} 50 | 51 | type request struct { 52 | iour *IOURing 53 | 54 | id uint64 55 | opcode uint8 56 | res int32 57 | 58 | once sync.Once 59 | resolving bool 60 | resolver ResultResolver 61 | 62 | callback RequestCallback 63 | 64 | fd int 65 | b0 []byte 66 | b1 []byte 67 | bs [][]byte 68 | 69 | err error 70 | r0 interface{} 71 | r1 interface{} 72 | ext1 uint64 73 | ext2 uint64 74 | 75 | requestInfo interface{} 76 | 77 | set *requestSet 78 | done chan struct{} 79 | } 80 | 81 | func (req *request) resolve() { 82 | if req.resolver == nil { 83 | return 84 | } 85 | 86 | select { 87 | case <-req.done: 88 | default: 89 | return 90 | } 91 | 92 | req.once.Do(func() { 93 | req.resolving = true 94 | req.resolver(req) 95 | req.resolving = false 96 | 97 | req.resolver = nil 98 | }) 99 | } 100 | 101 | func (req *request) complate(cqe iouring_syscall.CompletionQueueEvent) { 102 | req.res = cqe.Result() 103 | req.ext1 = cqe.Extra1() 104 | req.ext2 = cqe.Extra2() 105 | req.iour = nil 106 | close(req.done) 107 | 108 | if req.set != nil { 109 | req.set.complateOne() 110 | req.set = nil 111 | } 112 | } 113 | 114 | func (req *request) isDone() bool { 115 | select { 116 | case <-req.done: 117 | return true 118 | default: 119 | } 120 | return false 121 | } 122 | 123 | func (req *request) Callback() error { 124 | if !req.isDone() { 125 | return ErrRequestCompleted 126 | } 127 | 128 | if req.callback == nil { 129 | return ErrNoRequestCallback 130 | } 131 | 132 | return req.callback(req) 133 | } 134 | 135 | // Cancel request if request is not completed 136 | func (req *request) Cancel() (Request, error) { 137 | if req.isDone() { 138 | return nil, ErrRequestCompleted 139 | } 140 | 141 | return req.iour.submitCancel(req.id) 142 | } 143 | 144 | func (req *request) Done() <-chan struct{} { 145 | return req.done 146 | } 147 | 148 | func (req *request) Opcode() uint8 { 149 | return req.opcode 150 | } 151 | 152 | func (req *request) Fd() int { 153 | return req.fd 154 | } 155 | 156 | func (req *request) GetRequestBuffer() (b0, b1 []byte) { 157 | return req.b0, req.b1 158 | } 159 | 160 | func (req *request) GetRequestBuffers() [][]byte { 161 | return req.bs 162 | } 163 | 164 | func (req *request) GetRequestInfo() interface{} { 165 | return req.requestInfo 166 | } 167 | 168 | func (req *request) Err() error { 169 | req.resolve() 170 | return req.err 171 | } 172 | 173 | func (req *request) ReturnValue0() interface{} { 174 | req.resolve() 175 | return req.r0 176 | } 177 | 178 | func (req *request) ReturnValue1() interface{} { 179 | req.resolve() 180 | return req.r1 181 | } 182 | 183 | func (req *request) ReturnExtra1() uint64 { 184 | return req.ext1 185 | } 186 | 187 | func (req *request) ReturnExtra2() uint64 { 188 | return req.ext2 189 | } 190 | 191 | func (req *request) ReturnFd() (int, error) { 192 | return req.ReturnInt() 193 | } 194 | 195 | func (req *request) ReturnInt() (int, error) { 196 | req.resolve() 197 | 198 | if req.err != nil { 199 | return -1, req.err 200 | } 201 | 202 | fd, ok := req.r0.(int) 203 | if !ok { 204 | return -1, errors.New("req value is not int") 205 | } 206 | 207 | return fd, nil 208 | } 209 | 210 | func (req *request) FreeRequestBuffer() { 211 | req.b0 = nil 212 | req.b1 = nil 213 | req.bs = nil 214 | } 215 | 216 | func (req *request) GetRes() (int, error) { 217 | if !req.isDone() { 218 | return 0, ErrRequestNotCompleted 219 | } 220 | 221 | return int(req.res), nil 222 | } 223 | 224 | func (req *request) SetResult(r0, r1 interface{}, err error) error { 225 | if !req.isDone() { 226 | return ErrRequestNotCompleted 227 | } 228 | if !req.resolving { 229 | return errors.New("request is not resolving") 230 | } 231 | 232 | req.r0, req.r1, req.err = r0, r1, err 233 | return nil 234 | } 235 | 236 | type RequestSet interface { 237 | Len() int 238 | Done() <-chan struct{} 239 | Requests() []Request 240 | ErrResults() []Result 241 | } 242 | 243 | var _ RequestSet = &requestSet{} 244 | 245 | type requestSet struct { 246 | requests []Request 247 | 248 | complates int32 249 | done chan struct{} 250 | } 251 | 252 | func newRequestSet(userData []*UserData) *requestSet { 253 | set := &requestSet{ 254 | requests: make([]Request, len(userData)), 255 | done: make(chan struct{}), 256 | } 257 | 258 | for i, data := range userData { 259 | set.requests[i] = data.request 260 | data.request.set = set 261 | } 262 | return set 263 | } 264 | 265 | func (set *requestSet) complateOne() { 266 | if atomic.AddInt32(&set.complates, 1) == int32(len(set.requests)) { 267 | close(set.done) 268 | } 269 | } 270 | 271 | func (set *requestSet) Len() int { 272 | return len(set.requests) 273 | } 274 | 275 | func (set *requestSet) Done() <-chan struct{} { 276 | return set.done 277 | } 278 | 279 | func (set *requestSet) Requests() []Request { 280 | return set.requests 281 | } 282 | 283 | func (set *requestSet) ErrResults() (results []Result) { 284 | for _, req := range set.requests { 285 | if req.Err() != nil { 286 | results = append(results, req) 287 | } 288 | } 289 | return 290 | } 291 | 292 | func errResolver(req Request) { 293 | result := req.(*request) 294 | if result.res < 0 { 295 | result.err = syscall.Errno(-result.res) 296 | if result.err == syscall.ECANCELED { 297 | // request is canceled 298 | result.err = ErrRequestCanceled 299 | } 300 | } 301 | } 302 | 303 | func fdResolver(req Request) { 304 | result := req.(*request) 305 | if errResolver(result); result.err != nil { 306 | return 307 | } 308 | result.r0 = int(result.res) 309 | } 310 | 311 | func timeoutResolver(req Request) { 312 | result := req.(*request) 313 | if errResolver(result); result.err != nil { 314 | // if timeout got completed through expiration of the timer 315 | // result.res is -ETIME and result.err is syscall.ETIME 316 | if result.err == syscall.ETIME { 317 | result.err = nil 318 | result.r0 = TimeoutExpiration 319 | } 320 | return 321 | } 322 | 323 | // if timeout got completed through requests completing 324 | // result.res is 0 325 | if result.res == 0 { 326 | result.r0 = CountCompletion 327 | } 328 | } 329 | 330 | func removeTimeoutResolver(req Request) { 331 | result := req.(*request) 332 | if errResolver(result); result.err != nil { 333 | switch result.err { 334 | case syscall.EBUSY: 335 | // timeout request was found bu expiration was already in progress 336 | result.err = ErrRequestCompleted 337 | case syscall.ENOENT: 338 | // timeout request not found 339 | result.err = ErrRequestNotFound 340 | } 341 | return 342 | } 343 | 344 | // timeout request is found and cacelled successfully 345 | // result.res value is 0 346 | } 347 | 348 | func cancelResolver(req Request) { 349 | result := req.(*request) 350 | if errResolver(result); result.err != nil { 351 | switch result.err { 352 | case syscall.ENOENT: 353 | result.err = ErrRequestNotFound 354 | case syscall.EALREADY: 355 | result.err = nil 356 | result.r0 = RequestMaybeCanceled 357 | } 358 | return 359 | } 360 | 361 | if result.res == 0 { 362 | result.r0 = RequestCanceledSuccessfully 363 | } 364 | } 365 | -------------------------------------------------------------------------------- /syscall/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "syscall", 5 | srcs = [ 6 | "io_uring_enter.go", 7 | "io_uring_register.go", 8 | "io_uring_setup.go", 9 | "types.go", 10 | ], 11 | importpath = "github.com/iceber/iouring-go/syscall", 12 | visibility = ["//visibility:public"], 13 | deps = select({ 14 | "@io_bazel_rules_go//go/platform:android": [ 15 | "@org_golang_x_sys//unix:go_default_library", 16 | ], 17 | "@io_bazel_rules_go//go/platform:linux": [ 18 | "@org_golang_x_sys//unix:go_default_library", 19 | ], 20 | "//conditions:default": [], 21 | }), 22 | ) 23 | -------------------------------------------------------------------------------- /syscall/io_uring_enter.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring_syscall 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | "unsafe" 9 | 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | // IOUringEnter flags 14 | const ( 15 | IORING_ENTER_FLAGS_GETEVENTS uint32 = 1 << iota 16 | IORING_ENTER_FLAGS_SQ_WAKEUP 17 | IORING_ENTER_FLAGS_SQ_WAIT 18 | ) 19 | 20 | func IOURingEnter(fd int, toSubmit uint32, minComplete uint32, flags uint32, sigset *unix.Sigset_t) (int, error) { 21 | res, _, errno := syscall.Syscall6( 22 | SYS_IO_URING_ENTER, 23 | uintptr(fd), 24 | uintptr(toSubmit), 25 | uintptr(minComplete), 26 | uintptr(flags), 27 | uintptr(unsafe.Pointer(sigset)), 28 | 0, 29 | ) 30 | if errno != 0 { 31 | return 0, os.NewSyscallError("iouring_enter", errno) 32 | } 33 | if res < 0 { 34 | return 0, os.NewSyscallError("iouring_enter", syscall.Errno(-res)) 35 | } 36 | 37 | return int(res), nil 38 | } 39 | -------------------------------------------------------------------------------- /syscall/io_uring_register.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring_syscall 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | "unsafe" 9 | ) 10 | 11 | const ( 12 | IORING_REGISTER_BUFFERS uint8 = iota 13 | IORING_UNREGISTER_BUFFERS 14 | IORING_REGISTER_FILES 15 | IORING_UNREGISTER_FILES 16 | IORING_REGISTER_EVENTFD 17 | IORING_UNREGISTER_EVENTFD 18 | IORING_REGISTER_FILES_UPDATE 19 | IORING_REGISTER_EVENTFD_ASYNC 20 | IORING_REGISTER_PROBE 21 | IORING_REGISTER_PERSONALITY 22 | IORING_UNREGISTER_PERSONALITY 23 | IORING_REGISTER_RESTRICTIONS 24 | IORING_REGISTER_ENABLE_RINGS 25 | ) 26 | 27 | type IOURingFilesUpdate struct { 28 | Offset uint32 29 | recv uint32 30 | Fds *int32 31 | } 32 | 33 | func IOURingRegister(fd int, opcode uint8, args unsafe.Pointer, nrArgs uint32) error { 34 | for { 35 | _, _, errno := syscall.Syscall6( 36 | SYS_IO_URING_REGISTER, 37 | uintptr(fd), 38 | uintptr(opcode), 39 | uintptr(args), 40 | uintptr(nrArgs), 41 | 0, 42 | 0, 43 | ) 44 | if errno != 0 { 45 | // EINTR may be returned when blocked 46 | if errno == syscall.EINTR { 47 | continue 48 | } 49 | return os.NewSyscallError("iouring_register", errno) 50 | } 51 | return nil 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /syscall/io_uring_setup.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring_syscall 5 | 6 | import ( 7 | "os" 8 | "syscall" 9 | "unsafe" 10 | ) 11 | 12 | const ( 13 | IORING_SETUP_IOPOLL uint32 = 1 << iota 14 | IORING_SETUP_SQPOLL 15 | IORING_SETUP_SQ_AFF 16 | IORING_SETUP_CQSIZE 17 | IORING_SETUP_CLAMP 18 | IORING_SETUP_ATTACH_WQ 19 | IORING_SETUP_R_DISABLED 20 | IORING_SETUP_SUBMIT_ALL 21 | 22 | IORING_SETUP_COOP_TASKRUN 23 | 24 | IORING_SETUP_TASKRUN_FLAG 25 | IORING_SETUP_SQE128 26 | IORING_SETUP_CQE32 27 | ) 28 | 29 | // io_uring features supported by current kernel version 30 | const ( 31 | IORING_FEAT_SINGLE_MMAP uint32 = 1 << iota 32 | IORING_FEAT_NODROP 33 | IORING_FEAT_SUBMIT_STABLE 34 | IORING_FEAT_RW_CUR_POS 35 | IORING_FEAT_CUR_PERSONALITY 36 | IORING_FEAT_FAST_POLL 37 | IORING_FEAT_POLL_32BITS 38 | IORING_FEAT_SQPOLL_NONFIXED 39 | ) 40 | 41 | // IOURingParams the flags, sq_thread_cpu, sq_thread_idle and WQFd fields are used to configure the io_uring instance 42 | type IOURingParams struct { 43 | SQEntries uint32 // specifies the number of submission queue entries allocated 44 | CQEntries uint32 // when IORING_SETUP_CQSIZE flag is specified 45 | Flags uint32 // a bit mast of 0 or more of the IORING_SETUP_* 46 | SQThreadCPU uint32 // when IORING_SETUP_SQPOLL and IORING_SETUP_SQ_AFF flags are specified 47 | SQThreadIdle uint32 // when IORING_SETUP_SQPOLL flag is specified 48 | Features uint32 49 | WQFd uint32 // when IORING_SETUP_ATTACH_WQ flag is specified 50 | Resv [3]uint32 51 | 52 | SQOffset SubmissionQueueRingOffset 53 | CQOffset CompletionQueueRingOffset 54 | } 55 | 56 | // SubmissionQueueRingOffset describes the offsets of various ring buffer fields 57 | type SubmissionQueueRingOffset struct { 58 | Head uint32 59 | Tail uint32 60 | RingMask uint32 61 | RingEntries uint32 62 | Flags uint32 63 | Dropped uint32 64 | Array uint32 65 | Resv1 uint32 66 | Resv2 uint64 67 | } 68 | 69 | // CompletionQueueRingOffset describes the offsets of various ring buffer fields 70 | type CompletionQueueRingOffset struct { 71 | Head uint32 72 | Tail uint32 73 | RingMask uint32 74 | RingEntries uint32 75 | Overflow uint32 76 | Cqes uint32 77 | Flags uint32 78 | Resv1 uint32 79 | Resv2 uint64 80 | } 81 | 82 | func IOURingSetup(entries uint, params *IOURingParams) (int, error) { 83 | res, _, errno := syscall.RawSyscall( 84 | SYS_IO_URING_SETUP, 85 | uintptr(entries), 86 | uintptr(unsafe.Pointer(params)), 87 | 0, 88 | ) 89 | if errno != 0 { 90 | return int(res), os.NewSyscallError("iouring_setup", errno) 91 | } 92 | 93 | return int(res), nil 94 | } 95 | -------------------------------------------------------------------------------- /syscall/types.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring_syscall 5 | 6 | import ( 7 | "fmt" 8 | "reflect" 9 | "unsafe" 10 | ) 11 | 12 | const ( 13 | SYS_IO_URING_SETUP = 425 14 | SYS_IO_URING_ENTER = 426 15 | SYS_IO_URING_REGISTER = 427 16 | ) 17 | 18 | // IORING Offset 19 | const ( 20 | IORING_OFF_SQ_RING uint64 = 0 21 | IORING_OFF_CQ_RING uint64 = 0x8000000 22 | IORING_OFF_SQES uint64 = 0x10000000 23 | ) 24 | 25 | const ( 26 | IORING_OP_NOP uint8 = iota 27 | IORING_OP_READV 28 | IORING_OP_WRITEV 29 | IORING_OP_FSYNC 30 | IORING_OP_READ_FIXED 31 | IORING_OP_WRITE_FIXED 32 | IORING_OP_POLL_ADD 33 | IORING_OP_POLL_REMOVE 34 | IORING_OP_SYNC_FILE_RANGE 35 | IORING_OP_SENDMSG 36 | IORING_OP_RECVMSG 37 | IORING_OP_TIMEOUT 38 | IORING_OP_TIMEOUT_REMOVE 39 | IORING_OP_ACCEPT 40 | IORING_OP_ASYNC_CANCEL 41 | IORING_OP_LINK_TIMEOUT 42 | IORING_OP_CONNECT 43 | IORING_OP_FALLOCATE 44 | IORING_OP_OPENAT 45 | IORING_OP_CLOSE 46 | IORING_OP_FILES_UPDATE 47 | IORING_OP_STATX 48 | IORING_OP_READ 49 | IORING_OP_WRITE 50 | IORING_OP_FADVISE 51 | IORING_OP_MADVISE 52 | IORING_OP_SEND 53 | IORING_OP_RECV 54 | IORING_OP_OPENAT2 55 | IORING_OP_EPOLL_CTL 56 | IORING_OP_SPLICE 57 | IORING_OP_PROVIDE_BUFFERS 58 | IORING_OP_REMOVE_BUFFERS 59 | IORING_OP_TEE 60 | IORING_OP_SHUTDOWN 61 | IORING_OP_RENAMEAT 62 | IORING_OP_UNLINKAT 63 | IORING_OP_MKDIRAT 64 | IORING_OP_SYMLINKAT 65 | IORING_OP_LINKAT 66 | IORING_OP_MSG_RING 67 | IORING_OP_FSETXATTR 68 | IORING_OP_SETXATTR 69 | IORING_OP_FGETXATTR 70 | IORING_OP_GETXATTR 71 | IORING_OP_SOCKET 72 | IORING_OP_URING_CMD 73 | IORING_OP_SEND_ZC 74 | IORING_OP_SENDMSG_ZC 75 | 76 | /* this goes last, obviously */ 77 | IORING_OP_LAST 78 | ) 79 | 80 | const ( 81 | IORING_SQ_NEED_WAKEUP uint32 = 1 << iota 82 | IORING_SQ_CQ_OVERFLOW 83 | ) 84 | 85 | const ( 86 | IOSQE_FLAGS_FIXED_FILE uint8 = 1 << iota 87 | IOSQE_FLAGS_IO_DRAIN 88 | IOSQE_FLAGS_IO_LINK 89 | IOSQE_FLAGS_IO_HARDLINK 90 | IOSQE_FLAGS_ASYNC 91 | IOSQE_FLAGS_BUFFER_SELECT 92 | ) 93 | 94 | const IOSQE_SYNC_DATASYNC uint = 1 95 | const IOSQE_TIMEOUT_ABS uint = 1 96 | const IOSQE_SPLICE_F_FD_IN_FIXED = 1 << 31 97 | 98 | type SubmissionQueueEntry interface { 99 | Opcode() uint8 100 | Reset() 101 | PrepOperation(op uint8, fd int32, addrOrSpliceOffIn uint64, len uint32, offsetOrCmdOp uint64) 102 | Fd() int32 103 | SetFdIndex(index int32) 104 | SetOpFlags(opflags uint32) 105 | SetUserData(userData uint64) 106 | SetFlags(flag uint8) 107 | CleanFlags(flags uint8) 108 | SetIoprio(ioprio uint16) 109 | SetBufIndex(bufIndex uint16) 110 | SetBufGroup(bufGroup uint16) 111 | SetPersonality(personality uint16) 112 | SetSpliceFdIn(fdIn int32) 113 | 114 | CMD(castType interface{}) interface{} 115 | } 116 | 117 | type sqeCore struct { 118 | opcode uint8 119 | flags uint8 120 | ioprio uint16 121 | fd int32 122 | offset uint64 123 | addr uint64 124 | len uint32 125 | opFlags uint32 126 | userdata uint64 127 | 128 | bufIndexOrGroup uint16 129 | personality uint16 130 | spliceFdIn int32 131 | } 132 | 133 | func (sqe *sqeCore) Opcode() uint8 { 134 | return sqe.opcode 135 | } 136 | 137 | func (sqe *sqeCore) PrepOperation(op uint8, fd int32, addrOrSpliceOffIn uint64, len uint32, offsetOrCmdOp uint64) { 138 | sqe.opcode = op 139 | sqe.fd = fd 140 | sqe.addr = addrOrSpliceOffIn 141 | sqe.len = len 142 | sqe.offset = offsetOrCmdOp 143 | } 144 | 145 | func (sqe *sqeCore) Fd() int32 { 146 | return sqe.fd 147 | } 148 | 149 | func (sqe *sqeCore) SetFdIndex(index int32) { 150 | sqe.fd = index 151 | sqe.flags |= IOSQE_FLAGS_FIXED_FILE 152 | } 153 | 154 | func (sqe *sqeCore) SetOpFlags(opflags uint32) { 155 | sqe.opFlags = opflags 156 | } 157 | 158 | func (sqe *sqeCore) SetUserData(userData uint64) { 159 | sqe.userdata = userData 160 | } 161 | 162 | func (sqe *sqeCore) SetFlags(flags uint8) { 163 | sqe.flags |= flags 164 | } 165 | 166 | func (sqe *sqeCore) CleanFlags(flags uint8) { 167 | sqe.flags ^= flags 168 | } 169 | 170 | func (sqe *sqeCore) SetIoprio(ioprio uint16) { 171 | sqe.ioprio = ioprio 172 | } 173 | 174 | func (sqe *sqeCore) SetBufIndex(bufIndex uint16) { 175 | sqe.bufIndexOrGroup = bufIndex 176 | } 177 | 178 | func (sqe *sqeCore) SetBufGroup(bufGroup uint16) { 179 | sqe.bufIndexOrGroup = bufGroup 180 | } 181 | 182 | func (sqe *sqeCore) SetPersonality(personality uint16) { 183 | sqe.personality = personality 184 | } 185 | 186 | func (sqe *sqeCore) SetSpliceFdIn(fdIn int32) { 187 | sqe.spliceFdIn = fdIn 188 | } 189 | 190 | type SubmissionQueueEntry64 struct { 191 | sqeCore 192 | 193 | extra [2]uint64 194 | } 195 | 196 | func (sqe *SubmissionQueueEntry64) Reset() { 197 | *sqe = SubmissionQueueEntry64{} 198 | } 199 | 200 | func (sqe *SubmissionQueueEntry64) CMD(_ interface{}) interface{} { 201 | panic(fmt.Errorf("unsupported interface for CMD command")) 202 | } 203 | 204 | type SubmissionQueueEntry128 struct { 205 | sqeCore 206 | 207 | cmd [80]uint8 208 | } 209 | 210 | func (sqe *SubmissionQueueEntry128) Reset() { 211 | *sqe = SubmissionQueueEntry128{} 212 | } 213 | 214 | func (sqe *SubmissionQueueEntry128) CMD(castType interface{}) interface{} { 215 | return reflect.NewAt(reflect.TypeOf(castType), unsafe.Pointer(&sqe.cmd[0])).Interface() 216 | } 217 | 218 | type CompletionQueueEvent interface { 219 | UserData() uint64 220 | Result() int32 221 | Extra1() uint64 222 | Extra2() uint64 223 | Flags() uint32 224 | Clone() CompletionQueueEvent 225 | } 226 | 227 | type cqeCore struct { 228 | userData uint64 229 | result int32 230 | flags uint32 231 | } 232 | 233 | func (cqe *cqeCore) UserData() uint64 { 234 | return cqe.userData 235 | } 236 | 237 | func (cqe *cqeCore) Result() int32 { 238 | return cqe.result 239 | } 240 | 241 | func (cqe *cqeCore) Flags() uint32 { 242 | return cqe.flags 243 | } 244 | 245 | type CompletionQueueEvent16 struct { 246 | cqeCore 247 | } 248 | 249 | func (cqe *CompletionQueueEvent16) Extra1() uint64 { 250 | return 0 251 | } 252 | 253 | func (cqe *CompletionQueueEvent16) Extra2() uint64 { 254 | return 0 255 | } 256 | 257 | func (cqe *CompletionQueueEvent16) Clone() CompletionQueueEvent { 258 | dest := &CompletionQueueEvent16{} 259 | *dest = *cqe 260 | return dest 261 | } 262 | 263 | type CompletionQueueEvent32 struct { 264 | cqeCore 265 | 266 | extra1 uint64 267 | extra2 uint64 268 | } 269 | 270 | func (cqe *CompletionQueueEvent32) Extra1() uint64 { 271 | return cqe.extra1 272 | } 273 | 274 | func (cqe *CompletionQueueEvent32) Extra2() uint64 { 275 | return cqe.extra2 276 | } 277 | 278 | func (cqe *CompletionQueueEvent32) Clone() CompletionQueueEvent { 279 | dest := &CompletionQueueEvent32{} 280 | *dest = *cqe 281 | return dest 282 | } 283 | 284 | const IORING_FSYNC_DATASYNC uint32 = 1 285 | const IORING_TIMEOUT_ABS uint32 = 1 286 | -------------------------------------------------------------------------------- /timeout.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "time" 8 | "unsafe" 9 | 10 | "golang.org/x/sys/unix" 11 | 12 | iouring_syscall "github.com/iceber/iouring-go/syscall" 13 | ) 14 | 15 | func (prepReq PrepRequest) WithTimeout(timeout time.Duration) []PrepRequest { 16 | linkRequest := func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 17 | prepReq(sqe, userData) 18 | sqe.SetFlags(iouring_syscall.IOSQE_FLAGS_IO_LINK) 19 | } 20 | return []PrepRequest{linkRequest, linkTimeout(timeout)} 21 | } 22 | 23 | func Timeout(t time.Duration) PrepRequest { 24 | timespec := unix.NsecToTimespec(t.Nanoseconds()) 25 | 26 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 27 | userData.hold(×pec) 28 | userData.request.resolver = timeoutResolver 29 | 30 | sqe.PrepOperation(iouring_syscall.IORING_OP_TIMEOUT, -1, uint64(uintptr(unsafe.Pointer(×pec))), 1, 0) 31 | } 32 | } 33 | 34 | func TimeoutWithTime(t time.Time) (PrepRequest, error) { 35 | timespec, err := unix.TimeToTimespec(t) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 41 | userData.hold(×pec) 42 | userData.request.resolver = timeoutResolver 43 | 44 | sqe.PrepOperation(iouring_syscall.IORING_OP_TIMEOUT, -1, uint64(uintptr(unsafe.Pointer(×pec))), 1, 0) 45 | sqe.SetOpFlags(iouring_syscall.IORING_TIMEOUT_ABS) 46 | }, nil 47 | } 48 | 49 | func CountCompletionEvent(n uint64) PrepRequest { 50 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 51 | userData.request.resolver = timeoutResolver 52 | 53 | sqe.PrepOperation(iouring_syscall.IORING_OP_TIMEOUT, -1, 0, 0, n) 54 | } 55 | } 56 | 57 | func RemoveTimeout(id uint64) PrepRequest { 58 | return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 59 | userData.request.resolver = removeTimeoutResolver 60 | 61 | sqe.PrepOperation(iouring_syscall.IORING_OP_TIMEOUT, -1, id, 0, 0) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package iouring 5 | 6 | import ( 7 | "reflect" 8 | "sync/atomic" 9 | "unsafe" 10 | 11 | iouring_syscall "github.com/iceber/iouring-go/syscall" 12 | ) 13 | 14 | // iouring operations 15 | const ( 16 | OpNop uint8 = iota 17 | OpReadv 18 | OpWritev 19 | OpFsync 20 | OpReadFixed 21 | OpWriteFixed 22 | OpPollAdd 23 | OpPollRemove 24 | OpSyncFileRange 25 | OpSendmsg 26 | OpRecvmsg 27 | OpTimeout 28 | OpTimeoutRemove 29 | OpAccept 30 | OpAsyncCancel 31 | OpLinkTimeout 32 | OpConnect 33 | OpFallocate 34 | OpOpenat 35 | OpClose 36 | OpFilesUpdate 37 | OpStatx 38 | OpRead 39 | OpWrite 40 | OpFadvise 41 | OpMadvise 42 | OpSend 43 | OpRecv 44 | OpOpenat2 45 | OpEpollCtl 46 | OpSplice 47 | OpProvideBuffers 48 | OpRemoveBuffers 49 | OpTee 50 | OpShutdown 51 | OpRenameat 52 | OpUnlinkat 53 | OpMkdirat 54 | OpSymlinkat 55 | OpLinkat 56 | ) 57 | 58 | // cancel operation return value 59 | const ( 60 | RequestCanceledSuccessfully = 0 61 | RequestMaybeCanceled = 1 62 | ) 63 | 64 | // timeout operation return value 65 | const ( 66 | TimeoutExpiration = 0 67 | CountCompletion = 1 68 | ) 69 | 70 | var _zero uintptr 71 | 72 | type SubmissionQueueRing interface { 73 | isActive() bool 74 | entrySz() uint32 75 | ringSz() uint32 76 | assignQueue(ptr uintptr, len int) 77 | mappedPtr() uintptr 78 | index(index uint32) iouring_syscall.SubmissionQueueEntry 79 | } 80 | 81 | func makeSubmissionQueueRing(flags uint32) SubmissionQueueRing { 82 | if flags&iouring_syscall.IORING_SETUP_SQE128 == 0 { 83 | return new(SubmissionQueueRing64) 84 | } else { 85 | return new(SubmissionQueueRing128) 86 | } 87 | } 88 | 89 | type SubmissionQueueRing64 struct { 90 | queue []iouring_syscall.SubmissionQueueEntry64 91 | } 92 | 93 | func (ring *SubmissionQueueRing64) isActive() bool { 94 | return ring.queue != nil && len(ring.queue) > 0 95 | } 96 | 97 | func (ring *SubmissionQueueRing64) entrySz() uint32 { 98 | return uint32(unsafe.Sizeof(iouring_syscall.SubmissionQueueEntry64{})) 99 | } 100 | 101 | func (ring *SubmissionQueueRing64) ringSz() uint32 { 102 | return uint32(len(ring.queue)) * ring.entrySz() 103 | } 104 | 105 | func (ring *SubmissionQueueRing64) assignQueue(ptr uintptr, len int) { 106 | ring.queue = *(*[]iouring_syscall.SubmissionQueueEntry64)( 107 | unsafe.Pointer(&reflect.SliceHeader{ 108 | Data: ptr, 109 | Len: len, 110 | Cap: len, 111 | })) 112 | } 113 | 114 | func (ring *SubmissionQueueRing64) mappedPtr() uintptr { 115 | return uintptr(unsafe.Pointer(&ring.queue[0])) 116 | } 117 | 118 | func (ring *SubmissionQueueRing64) index(index uint32) iouring_syscall.SubmissionQueueEntry { 119 | return &ring.queue[index] 120 | } 121 | 122 | type SubmissionQueueRing128 struct { 123 | queue []iouring_syscall.SubmissionQueueEntry128 124 | } 125 | 126 | func (ring *SubmissionQueueRing128) isActive() bool { 127 | return ring.queue != nil && len(ring.queue) > 0 128 | } 129 | 130 | func (ring *SubmissionQueueRing128) entrySz() uint32 { 131 | return uint32(unsafe.Sizeof(iouring_syscall.SubmissionQueueEntry128{})) 132 | } 133 | 134 | func (ring *SubmissionQueueRing128) ringSz() uint32 { 135 | return uint32(len(ring.queue)) * ring.entrySz() 136 | } 137 | 138 | func (ring *SubmissionQueueRing128) assignQueue(ptr uintptr, len int) { 139 | ring.queue = *(*[]iouring_syscall.SubmissionQueueEntry128)( 140 | unsafe.Pointer(&reflect.SliceHeader{ 141 | Data: ptr, 142 | Len: len, 143 | Cap: len, 144 | })) 145 | } 146 | 147 | func (ring *SubmissionQueueRing128) mappedPtr() uintptr { 148 | return uintptr(unsafe.Pointer(&ring.queue[0])) 149 | } 150 | 151 | func (ring *SubmissionQueueRing128) index(index uint32) iouring_syscall.SubmissionQueueEntry { 152 | return &ring.queue[index] 153 | } 154 | 155 | type SubmissionQueue struct { 156 | ptr uintptr 157 | size uint32 158 | 159 | head *uint32 160 | tail *uint32 161 | mask *uint32 162 | entries *uint32 // specifies the number of submission queue ring entries 163 | flags *uint32 // used by the kernel to communicate stat information to the application 164 | dropped *uint32 // incrementd for each invalid submission queue entry encountered in the ring buffer 165 | 166 | array []uint32 167 | sqes SubmissionQueueRing // submission queue ring 168 | 169 | sqeHead uint32 170 | sqeTail uint32 171 | } 172 | 173 | func (queue *SubmissionQueue) getSQEntry() iouring_syscall.SubmissionQueueEntry { 174 | head := atomic.LoadUint32(queue.head) 175 | next := queue.sqeTail + 1 176 | 177 | if (next - head) <= *queue.entries { 178 | sqe := queue.sqes.index(queue.sqeTail & *queue.mask) 179 | queue.sqeTail = next 180 | sqe.Reset() 181 | return sqe 182 | } 183 | return nil 184 | } 185 | 186 | func (queue *SubmissionQueue) fallback(i uint32) { 187 | queue.sqeTail -= i 188 | } 189 | 190 | func (queue *SubmissionQueue) cqOverflow() bool { 191 | return (atomic.LoadUint32(queue.flags) & iouring_syscall.IORING_SQ_CQ_OVERFLOW) != 0 192 | } 193 | 194 | func (queue *SubmissionQueue) needWakeup() bool { 195 | return (atomic.LoadUint32(queue.flags) & iouring_syscall.IORING_SQ_NEED_WAKEUP) != 0 196 | } 197 | 198 | // sync internal status with kernel ring state on the SQ side 199 | // return the number of pending items in the SQ ring, for the shared ring. 200 | func (queue *SubmissionQueue) flush() int { 201 | if queue.sqeHead == queue.sqeTail { 202 | return int(*queue.tail - *queue.head) 203 | } 204 | 205 | tail := *queue.tail 206 | for toSubmit := queue.sqeTail - queue.sqeHead; toSubmit > 0; toSubmit-- { 207 | queue.array[tail&*queue.mask] = queue.sqeHead & *queue.mask 208 | tail++ 209 | queue.sqeHead++ 210 | } 211 | 212 | atomic.StoreUint32(queue.tail, tail) 213 | return int(tail - *queue.head) 214 | } 215 | 216 | type CompletionQueueRing interface { 217 | isActive() bool 218 | entrySz() uint32 219 | ringSz() uint32 220 | assignQueue(ptr uintptr, len int) 221 | mappedPtr() uintptr 222 | index(index uint32) iouring_syscall.CompletionQueueEvent 223 | } 224 | 225 | func makeCompletionQueueRing(flags uint32) CompletionQueueRing { 226 | if flags&iouring_syscall.IORING_SETUP_CQE32 == 0 { 227 | return new(CompletionQueueRing16) 228 | } else { 229 | return new(CompletionQueueRing32) 230 | } 231 | } 232 | 233 | type CompletionQueueRing16 struct { 234 | queue []iouring_syscall.CompletionQueueEvent16 235 | } 236 | 237 | func (ring *CompletionQueueRing16) isActive() bool { 238 | return ring.queue != nil && len(ring.queue) > 0 239 | } 240 | 241 | func (ring *CompletionQueueRing16) entrySz() uint32 { 242 | return uint32(unsafe.Sizeof(iouring_syscall.CompletionQueueEvent16{})) 243 | } 244 | 245 | func (ring *CompletionQueueRing16) ringSz() uint32 { 246 | return uint32(len(ring.queue)) * ring.entrySz() 247 | } 248 | 249 | func (ring *CompletionQueueRing16) assignQueue(ptr uintptr, len int) { 250 | ring.queue = *(*[]iouring_syscall.CompletionQueueEvent16)( 251 | unsafe.Pointer(&reflect.SliceHeader{ 252 | Data: ptr, 253 | Len: len, 254 | Cap: len, 255 | })) 256 | } 257 | 258 | func (ring *CompletionQueueRing16) mappedPtr() uintptr { 259 | return uintptr(unsafe.Pointer(&ring.queue[0])) 260 | } 261 | 262 | func (ring *CompletionQueueRing16) index(index uint32) iouring_syscall.CompletionQueueEvent { 263 | return &ring.queue[index] 264 | } 265 | 266 | type CompletionQueueRing32 struct { 267 | queue []iouring_syscall.CompletionQueueEvent32 268 | } 269 | 270 | func (ring *CompletionQueueRing32) isActive() bool { 271 | return ring.queue != nil && len(ring.queue) > 0 272 | } 273 | 274 | func (ring *CompletionQueueRing32) entrySz() uint32 { 275 | return uint32(unsafe.Sizeof(iouring_syscall.CompletionQueueEvent32{})) 276 | } 277 | 278 | func (ring *CompletionQueueRing32) ringSz() uint32 { 279 | return uint32(len(ring.queue)) * ring.entrySz() 280 | } 281 | 282 | func (ring *CompletionQueueRing32) assignQueue(ptr uintptr, len int) { 283 | ring.queue = *(*[]iouring_syscall.CompletionQueueEvent32)( 284 | unsafe.Pointer(&reflect.SliceHeader{ 285 | Data: ptr, 286 | Len: len, 287 | Cap: len, 288 | })) 289 | } 290 | 291 | func (ring *CompletionQueueRing32) mappedPtr() uintptr { 292 | return uintptr(unsafe.Pointer(&ring.queue[0])) 293 | } 294 | 295 | func (ring *CompletionQueueRing32) index(index uint32) iouring_syscall.CompletionQueueEvent { 296 | return &ring.queue[index] 297 | } 298 | 299 | type CompletionQueue struct { 300 | ptr uintptr 301 | size uint32 302 | 303 | head *uint32 304 | tail *uint32 305 | mask *uint32 306 | entries *uint32 307 | flags *uint32 308 | overflow *uint32 309 | 310 | cqes CompletionQueueRing 311 | } 312 | 313 | func (queue *CompletionQueue) peek() (cqe iouring_syscall.CompletionQueueEvent) { 314 | head := *queue.head 315 | if head != atomic.LoadUint32(queue.tail) { 316 | // if head < atomic.LoadUint32(queue.tail) { 317 | cqe = queue.cqes.index(head & *queue.mask) 318 | } 319 | return 320 | } 321 | 322 | func (queue *CompletionQueue) advance(num uint32) { 323 | if num != 0 { 324 | atomic.AddUint32(queue.head, num) 325 | } 326 | } 327 | -------------------------------------------------------------------------------- /user_data.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring 4 | 5 | import ( 6 | "unsafe" 7 | ) 8 | 9 | type UserData struct { 10 | id uint64 11 | 12 | resulter chan<- Result 13 | opcode uint8 14 | 15 | holds []interface{} 16 | request *request 17 | } 18 | 19 | func (data *UserData) SetResultResolver(resolver ResultResolver) { 20 | data.request.resolver = resolver 21 | } 22 | 23 | func (data *UserData) SetRequestInfo(info interface{}) { 24 | data.request.requestInfo = info 25 | } 26 | 27 | func (data *UserData) SetRequestCallback(callback RequestCallback) { 28 | data.request.callback = callback 29 | } 30 | 31 | func (data *UserData) SetRequestBuffer(b0, b1 []byte) { 32 | data.request.b0, data.request.b1 = b0, b1 33 | } 34 | 35 | func (data *UserData) SetRequestBuffers(bs [][]byte) { 36 | data.request.bs = bs 37 | } 38 | 39 | func (data *UserData) Hold(vars ...interface{}) { 40 | data.holds = append(data.holds, vars) 41 | } 42 | 43 | func (data *UserData) hold(vars ...interface{}) { 44 | data.holds = vars 45 | } 46 | 47 | func (data *UserData) setOpcode(opcode uint8) { 48 | data.opcode = opcode 49 | data.request.opcode = opcode 50 | } 51 | 52 | // TODO(iceber): use sync.Poll 53 | func makeUserData(iour *IOURing, ch chan<- Result) *UserData { 54 | userData := &UserData{ 55 | resulter: ch, 56 | request: &request{iour: iour, done: make(chan struct{})}, 57 | } 58 | 59 | userData.id = uint64(uintptr(unsafe.Pointer(userData))) 60 | userData.request.id = userData.id 61 | return userData 62 | } 63 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package iouring 4 | 5 | import ( 6 | "syscall" 7 | "unsafe" 8 | ) 9 | 10 | var zero uintptr 11 | 12 | func bytes2iovec(bs [][]byte) []syscall.Iovec { 13 | iovecs := make([]syscall.Iovec, len(bs)) 14 | for i, b := range bs { 15 | iovecs[i].SetLen(len(b)) 16 | if len(b) > 0 { 17 | iovecs[i].Base = &b[0] 18 | } else { 19 | iovecs[i].Base = (*byte)(unsafe.Pointer(&zero)) 20 | } 21 | } 22 | return iovecs 23 | } 24 | 25 | //go:linkname sockaddr syscall.Sockaddr.sockaddr 26 | func sockaddr(addr syscall.Sockaddr) (unsafe.Pointer, uint32, error) 27 | 28 | //go:linkname anyToSockaddr syscall.anyToSockaddr 29 | func anyToSockaddr(rsa *syscall.RawSockaddrAny) (syscall.Sockaddr, error) 30 | --------------------------------------------------------------------------------