├── Makefile ├── README.md ├── ceph_fileline.c └── cephfs_readline ├── config ├── ngx_http_cephfs_readline_handler.c ├── ngx_http_cephfs_readline_handler.h ├── ngx_http_cephfs_readline_module.c └── ngx_http_cephfs_readline_module.h /Makefile: -------------------------------------------------------------------------------- 1 | CC = gcc 2 | CEPH_LIB = -L/home/rpmbuild/BUILD/ceph-12.2.8.1/build/lib/ -lrados 3 | CEPH_INCLUDE = -I/home/rpmbuild/BUILD/ceph-12.2.8.1/src/include -I/home/rpmbuild/BUILD/ceph-12.2.8.1/src/ -I/home/rpmbuild/BUILD/ceph-12.2.8.1/build/include/ -I/home/rpmbuild/BUILD/ceph-12.2.8.1/src/client/ 4 | 5 | all: 6 | $(CC) -g ceph_fileline.c $(CEPH_LIB) -o ceph-fileline $(CEPH_INCLUDE) 7 | 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 1.背景说明 2 | 继上次分享的 [Ceph介绍及原理架构分享](https://www.jianshu.com/p/cc3ece850433) 和 [分布式存储Ceph之PG状态详解](https://www.jianshu.com/p/36c2d5682d87) ，这次分享点干货。 3 | 用户需要从cephfs存储系统中检索一个大文件指定关键字的一行信息， 并且对延迟和性能要求比较高。 4 | 5 | # 2. 原始方案 6 | ## 2.1 流程图 7 | 8 | ![image.png](https://upload-images.jianshu.io/upload_images/2099201-152dc1a964a7f60f.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 9 | 10 | ## 2.2 说明 11 | - 假如用户拉取的文件大小是16M, 文件按照4M切分，散落到四个数据片上 12 | - 用户首先请求cephfs拉取文件信息 13 | - cephfs会根据crush算法找计算文件散落到那几个数据片上 14 | - cephfs会拉取文件所属的数据片然后聚合起来 15 | - cephfs文件拉取后返回给用户 16 | - 用户拉取完整个文件，开始做过滤关键字操作 17 | 18 | ## 2.3 实战 19 | ``` 20 | //检索2.7G文件为例 21 | $ ll -lh nginx/logs/access.log.2018102911 22 | -rw-rw-r-- 1 root root 2.7G Oct 29 12:07 nginx/logs/access.log.2018102911 23 | 24 | //grep 模拟花费12s 25 | $ time grep "xxxyyyzzzqqq" nginx/logs/access.log.2018102911 26 | 27 | real 0m12.355s 28 | user 0m0.302s 29 | sys 0m0.823s 30 | ``` 31 | 32 | ## 2.4 优缺点 33 | **优点** 34 | - 简单方便 35 | - 开发成本低 36 | 37 | **缺点** 38 | - 用户端检索延迟大，影响用户体验 39 | - 客户端集群网卡带宽波动较大，带宽有限，每次都需要把大日志文件拉取到客户端 40 | - 对ceph集群负载也有波动影响 41 | 42 | ## 2.5 总结 43 | 用户拉取文件，必须先通过cephfs拉取文件到本地，然后根据关键字检索这行数据。如果用户检索量比较大的时候，并且文件大小都不统一，拉取文件越大网络延迟越高，并且在大文件中过滤关键字效率非常低，严重影响用户的体验。 44 | 45 | # 3. 优化方案 46 | ## 3.1 流程图 47 | ![image.png](https://upload-images.jianshu.io/upload_images/2099201-8830428557d4f17b.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 48 | 49 | ## 3.2 说明 50 | - 用户发起请求输入文件名和key关键字到达索引层 51 | - 索引层根据key找到对应的offset信息，然后传给dss-readline 52 | - dss-readline根据cephfs cursh算法找到对应的object信息和offset信息 53 | - 根据dss-readline用户输入的offset找到对应的object块信息 54 | - dss-readline直接获取需要块的offset 该行的信息 55 | 56 | ## 3.3 实战 57 | ```shell 58 | //查找2.8G文件offset对应的信息 59 | $ ll nginx/logs/access.log.2018110216 -lh 60 | -rw-rw-r-- 1 root root 2.8G Nov 2 17:08 nginx/logs/access.log.2018110216 61 | 62 | //sed的方式模拟，花费12s 63 | $ time sed -n "1024p" nginx/logs/access.log.2018110216 64 | 65 | real 0m12.042s 66 | user 0m1.191s 67 | sys 0m0.929s 68 | 69 | //dss_readfile 自研工具, 输入参数:poolname, filename, offset 可以看出来花费91ms 70 | //usage: dss_readfile 71 | time ./dss_readfile data nginx/logs/access.log.2018110216 1024 72 | 73 | real 0m0.091s 74 | user 0m0.042s 75 | sys 0m0.011s 76 | ``` 77 | 78 | ## 3.4 优缺点 79 | **缺点** 80 | - 需要额外开发成本 81 | 82 | **优点** 83 | - 提升用户体验，从以前检索单个2.8G文件耗时10s左右， 优化后控制在100ms左右 84 | - 客户端网络网卡带宽可用率得到提升 85 | - 减少对ceph集群的冲击影响 86 | 87 | 88 | ## 3.5 总结 89 | **思路：** 90 | 由于文件信息是放到服务端，进行切片存储到数据节点。 91 | 我们能不能只拉取我需要的块信息，不用全量拉取到本地，答案是肯定的。 92 | 93 | - 根据文件信息查找所有的object、offset信息 94 | - 根据offset找到需要检索的object信息 95 | - 找到对应的object，读取该object对应的offset位置的信息（一行数据可能会拆分多个object) 96 | 97 | **优点：** 98 | - 提升用户体验，从以前检索单个2.8G文件耗时10s左右， 优化后控制在100ms左右 99 | - 客户端网络网卡带宽可用率得到提升 100 | - 减少对ceph集群的冲击影响 101 | 102 | # 4. 深入分析 103 | ## 4.1 文件对应object信息 104 | ### 4.1.1 Jewel版本 105 | ```shell 106 | //Ceph Jewel版本里，有个cephfs的工具，可以获取file的location信息 107 | //根据offset查找object信息 108 | $ cephfs /mnt/kernel_log_push.log show_location -l 4194304 109 | WARNING: This tool is deprecated. Use the layout.* xattrs to query and modify layouts. 110 | location.file_offset: 4194304 111 | location.object_offset:0 112 | location.object_no: 1 113 | location.object_size: 4194304 114 | location.object_name: 10002b63282.00000001 115 | location.block_offset: 0 116 | location.block_size: 4194304 117 | location.osd: 67 118 | 119 | //file object map 信息 120 | $ cephfs /mnt/kernel_log_push.log map 121 | WARNING: This tool is deprecated. Use the layout.* xattrs to query and modify layouts. 122 | FILE OFFSET OBJECT OFFSET LENGTH OSD 123 | 0 10002b63282.00000000 0 4194304 61 124 | 4194304 10002b63282.00000001 0 4194304 67 125 | 8388608 10002b63282.00000002 0 4194304 70 126 | 12582912 10002b63282.00000003 0 4194304 68 127 | ``` 128 | ### 4.1.2 源码跟踪 129 | **ceph jewel版本，cephfs代码** 130 | [https://github.com/ceph/ceph/blob/v10.2.9/src/cephfs.cc#L117](https://github.com/ceph/ceph/blob/v10.2.9/src/cephfs.cc#L117) 131 | 132 | ```c/c++ 133 | struct ceph_ioctl_layout layout; 134 | memset(&layout, 0, sizeof(layout)); 135 | //获取layout信息 136 | err = ioctl(fd, CEPH_IOC_GET_LAYOUT, (unsigned long)&layout); 137 | if (err) { 138 | cerr << "Error getting layout: " << cpp_strerror(errno) << endl; 139 | return 1; 140 | } 141 | 142 | printf("%15s %24s %12s %12s %s\n", 143 | "FILE OFFSET", "OBJECT", "OFFSET", "LENGTH", "OSD"); 144 | for (long long off = 0; off < st.st_size; off += layout.stripe_unit) { 145 | struct ceph_ioctl_dataloc location; 146 | location.file_offset = off; 147 | //获取location 信息 148 | err = ioctl(fd, CEPH_IOC_GET_DATALOC, (unsigned long)&location); 149 | if (err) { 150 | cerr << "Error getting location: " << cpp_strerror(errno) << endl; 151 | return 1; 152 | } 153 | printf("%15lld %24s %12lld %12lld %d\n", 154 | off, location.object_name, (long long)location.object_offset, 155 | (long long)location.block_size, (int)location.osd); 156 | } 157 | ``` 158 | **CEPH_IOC_GET_DATALOC代码** 159 | 160 | ```c/c++ 161 | //定义/src/client/ioctl.h 162 | //https://github.com/ceph/ceph/blob/d038e1da7a6c9b31ba4463b8ebedb9908981a55e/src/client/ioctl.h#L46 163 | #define CEPH_IOC_GET_DATALOC _IOWR(CEPH_IOCTL_MAGIC, 3, \ 164 | struct ceph_ioctl_dataloc) 165 | 166 | //fuse 代码跟踪, 发现只支持layout 167 | //https://github.com/ceph/ceph/blob/d038e1da7a6c9b31ba4463b8ebedb9908981a55e/src/client/fuse_ll.cc#L631 168 | static void fuse_ll_ioctl(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg, struct fuse_file_info *fi, 169 | unsigned flags, const void *in_buf, size_t in_bufsz, size_t out_bufsz) 170 | { 171 | CephFuse::Handle *cfuse = fuse_ll_req_prepare(req); 172 | 173 | if (flags & FUSE_IOCTL_COMPAT) { 174 | fuse_reply_err(req, ENOSYS); 175 | return; 176 | } 177 | 178 | switch (static_cast(cmd)) { 179 | case CEPH_IOC_GET_LAYOUT: { 180 | file_layout_t layout; 181 | struct ceph_ioctl_layout l; 182 | Fh *fh = (Fh*)fi->fh; 183 | cfuse->client->ll_file_layout(fh, &layout); 184 | l.stripe_unit = layout.stripe_unit; 185 | l.stripe_count = layout.stripe_count; 186 | l.object_size = layout.object_size; 187 | l.data_pool = layout.pool_id; 188 | fuse_reply_ioctl(req, 0, &l, sizeof(struct ceph_ioctl_layout)); 189 | } 190 | break; 191 | default: 192 | fuse_reply_err(req, EINVAL); 193 | } 194 | } 195 | 196 | //kernel cephfs代码, 支持layout, 支持dataloc 197 | // /usr/src/debug/kernel-3.10.0-693.17.1.el7/linux-3.10.0-693.17.1.el7.x86_64/fs/ceph/ioctl.c 198 | long ceph_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 199 | { 200 | dout("ioctl file %p cmd %u arg %lu\n", file, cmd, arg); 201 | switch (cmd) { 202 | case CEPH_IOC_GET_LAYOUT: 203 | return ceph_ioctl_get_layout(file, (void __user *)arg); 204 | 205 | case CEPH_IOC_SET_LAYOUT: 206 | return ceph_ioctl_set_layout(file, (void __user *)arg); 207 | 208 | case CEPH_IOC_SET_LAYOUT_POLICY: 209 | return ceph_ioctl_set_layout_policy(file, (void __user *)arg); 210 | 211 | case CEPH_IOC_GET_DATALOC: 212 | return ceph_ioctl_get_dataloc(file, (void __user *)arg); 213 | 214 | case CEPH_IOC_LAZYIO: 215 | return ceph_ioctl_lazyio(file); 216 | 217 | case CEPH_IOC_SYNCIO: 218 | return ceph_ioctl_syncio(file); 219 | } 220 | 221 | return -ENOTTY; 222 | } 223 | 224 | static long ceph_ioctl_get_dataloc(struct file *file, void __user *arg) 225 | { 226 | ... 227 | r = ceph_calc_file_object_mapping(&ci->i_layout, dl.file_offset, len, 228 | &dl.object_no, &dl.object_offset, 229 | &olen); 230 | if (r < 0) { 231 | up_read(&osdc->lock); 232 | return -EIO; 233 | } 234 | dl.file_offset -= dl.object_offset; 235 | dl.object_size = ceph_file_layout_object_size(ci->i_layout); 236 | dl.block_size = ceph_file_layout_su(ci->i_layout); 237 | 238 | /* block_offset = object_offset % block_size */ 239 | tmp = dl.object_offset; 240 | dl.block_offset = do_div(tmp, dl.block_size); 241 | 242 | snprintf(dl.object_name, sizeof(dl.object_name), "%llx.%08llx", 243 | ceph_ino(inode), dl.object_no); 244 | 245 | ... 246 | ``` 247 | 248 | ### 4.1.2 Luminous版本 249 | Luminous版本里，没有src/cephfs.cc文件, 发现test_ioctls.c 其实有相关的测试代码。 250 | https://github.com/ceph/ceph/blob/master/src/client/test_ioctls.c 251 | /src/client/test_ioctls.c 252 | 253 | ```c/c++ 254 | int main(int argc, char **argv) 255 | { 256 | ... 257 | fd = open(fn, O_CREAT|O_RDWR, 0644); 258 | if (fd < 0) { 259 | perror("couldn't open file"); 260 | return 1; 261 | } 262 | 263 | /* get layout */ 264 | err = ioctl(fd, CEPH_IOC_GET_LAYOUT, (unsigned long)&l); 265 | if (err < 0) { 266 | perror("ioctl IOC_GET_LAYOUT error"); 267 | return 1; 268 | } 269 | printf("layout:\n stripe_unit %lld\n stripe_count %lld\n object_size %lld\n data_pool %lld\n", 270 | (long long)l.stripe_unit, (long long)l.stripe_count, (long long)l.object_size, (long long)l.data_pool); 271 | 272 | /* dataloc */ 273 | dl.file_offset = atoll(argv[2]); 274 | err = ioctl(fd, CEPH_IOC_GET_DATALOC, (unsigned long)&dl); 275 | if (err < 0) { 276 | perror("ioctl IOC_GET_DATALOC error"); 277 | return 1; 278 | } 279 | 280 | printf("dataloc:\n"); 281 | printf(" file_offset %lld (of object start)\n", (long long)dl.file_offset); 282 | printf(" object '%s'\n object_offset %lld\n object_size %lld object_no %lld\n", 283 | dl.object_name, (long long)dl.object_offset, (long long)dl.object_size, (long long)dl.object_no); 284 | printf(" block_offset %lld\n block_size %lld\n", 285 | (long long)dl.block_offset, (long long)dl.block_size); 286 | ... 287 | ``` 288 | 289 | ### 4.2 总结 290 | - 目前只有kernel版本支持CEPH_IOC_GET_DATALOC 291 | - 根据文件以及offset可以获取对应的object信息。 目前只支持内核kernel版本。 292 | 293 | 294 | ## 4.3 获取这个对象offset对应行的信息 295 | **问题点：** 296 | - 一行数据可能会拆分为两个对象 297 | - 一行数据结尾符是否存在\n 298 | - 一行数据超大等问题 299 | 300 | **解决方案：** 301 | - 用户给的offset属于这一行的开头， 只需要读取当前读取数据是否存在\n。 302 | a. 如果存在\n证明该行，属于完整的行。 303 | b. 否则不存在\n证明该行，被拆分为两个对象，读取当前offset对应的object 信息以及下一个对象的信息，直到遇到\n结束，然后合并两个对象读取的数据为完整的行。 304 | - 超大行或者不存在结尾符\n 自动截取1024字节数。 305 | 306 | ## 4.4 通过librados库，读取object的信息 307 | 308 | ```c/c++ 309 | /* Declare the cluster handle and required arguments. */ 310 | int err; 311 | char cluster_name[] = "ceph"; 312 | char user_name[] = "client.admin"; 313 | uint64_t flags = 0; 314 | rados_t cluster; 315 | rados_ioctx_t io; 316 | rados_completion_t comp; 317 | 318 | 319 | 320 | /* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */ 321 | err = rados_create2(&cluster, cluster_name, user_name, flags); 322 | if (err < 0) { 323 | fprintf(stderr, "error: couldn't create the cluster handle poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 324 | poolname, object_name, offset, strerror(-err)); 325 | return 1; 326 | } 327 | 328 | /* Read a Ceph configuration file to configure the cluster handle. */ 329 | err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf"); 330 | if (err < 0) { 331 | fprintf(stderr, "error: cannot read config file poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 332 | poolname, object_name, offset, strerror(-err)); 333 | return 1; 334 | } 335 | 336 | /* Connect to the cluster */ 337 | err = rados_connect(cluster); 338 | if (err < 0) { 339 | fprintf(stderr, "error: cannot connect to cluster poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 340 | poolname, object_name, offset, strerror(-err)); 341 | return 1; 342 | } 343 | 344 | //create io 345 | err = rados_ioctx_create(cluster, poolname, &io); 346 | if (err < 0) { 347 | fprintf(stderr, "error: cannot open rados pool poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 348 | poolname, object_name, offset, strerror(-err)); 349 | rados_shutdown(cluster); 350 | return 1; 351 | } 352 | 353 | /* 354 | * Read data from the cluster asynchronously. 355 | * First, set up asynchronous I/O completion. 356 | */ 357 | err = rados_aio_create_completion(NULL, NULL, NULL, &comp); 358 | if (err < 0) { 359 | fprintf(stderr, "error: could not create aio completion poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 360 | poolname, object_name, offset, strerror(-err)); 361 | rados_ioctx_destroy(io); 362 | rados_shutdown(cluster); 363 | return 1; 364 | } 365 | 366 | /* Next, read data using rados_aio_read. */ 367 | err = rados_aio_read(io, object_name, comp, line, line_size, offset); 368 | if (err < 0) { 369 | fprintf(stderr, "error: cannot read object poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 370 | poolname, object_name, offset, strerror(-err)); 371 | rados_ioctx_destroy(io); 372 | rados_shutdown(cluster); 373 | return 1; 374 | } 375 | /* Wait for the operation to complete */ 376 | rados_aio_wait_for_complete(comp); 377 | /* Release the asynchronous I/O complete handle to avoid memory leaks. */ 378 | rados_aio_release(comp); 379 | 380 | rados_ioctx_destroy(io); 381 | rados_shutdown(cluster); 382 | ``` 383 | 384 | ## 4.5 项目工具 385 | 386 | **1. 源码地址** 387 | - https://github.com/lidaohang/cephfs_readline 388 | 389 | **2. dss_readfile工具** 390 | - 根据存储池、文件信息、offset获取对应的信息 391 | 392 | ```shell 393 | //usage: dss_readfile 394 | ./dss_readfile data nginx/logs/access.log.2018110216 1024 395 | 396 | ``` 397 | **3. ngx_cephfs_readline** 398 | - 为了提升性能以及用户体验，基于ceph module + librados 开发，充分利用nginx优秀的高并发性能。 399 | 400 | ```json 401 | //接口 402 | 403 | http://127.0.0.1 :8088/v1/dss-cephfs/readfile 404 | 405 | 406 | //请求body 407 | { 408 | "poolname":"data", 409 | "filename":"/mnt/business.log.2018101708", 410 | "offset":1024 411 | } 412 | 413 | //响应body 414 | { 415 | "code":1, 416 | "cost": 50, 417 | "data":"[INFO][2018-10-17T08:59:49.018+0800] xxxxxx" 418 | } 419 | ``` 420 | 421 | **依赖包** 422 | 423 | ``` 424 | #json库 425 | wget http://github.com/lloyd/yajl/tarball/1.0.12 426 | 427 | #ceph包 428 | git clone https://github.com/ceph/ceph.git 429 | 430 | #nginx 431 | wget http://nginx.org/download/nginx-1.14.0.tar.gz 432 | 433 | #ngx_cephfs_readline 模块 434 | git clone https://github.com/lidaohang/cephfs_readline.git 435 | 436 | ``` 437 | 438 | **安装** 439 | ``` 440 | cd nginx-1.14.0 441 | ./configure --add-module=cephfs_readline 442 | make && make install 443 | 444 | ``` 445 | 446 | **使用** 447 | ``` 448 | //配置文件nginx.conf 449 | server { 450 | listen 8012; 451 | server_name 127.0.0.1; 452 | 453 | access_log logs/access.log main; 454 | 455 | location /v1/dss-cephfs/readfile { 456 | cephfs_readline on; 457 | } 458 | ``` 459 | 460 | **效果** 461 | ``` 462 | curl 127.0.0.1:8012/v1/dss-cephfs/readfile -X POST -d '{"poolname":"data","filename":"/mnt//business.log.2018101708","offset":19990595}' 463 | {"code":0,"cost":51,"data":"[INFO][2018-10-17T08:59:49.018+0800] xxx"} 464 | ``` 465 | -------------------------------------------------------------------------------- /ceph_fileline.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include "ioctl.h" 17 | 18 | 19 | #define DSS_OK 0 20 | #define DSS_ERROR -1 21 | 22 | 23 | static const int object_size = 4194304; 24 | 25 | 26 | 27 | static 28 | int get_object_offset(const size_t src_offset) { 29 | size_t object_count; 30 | size_t object_offset;; 31 | 32 | object_count = src_offset / object_size; 33 | return (src_offset - object_count * object_size); 34 | } 35 | 36 | 37 | static 38 | int get_object_name(const char *filename,const size_t offset, char *dest_object_name) { 39 | 40 | int fd, err; 41 | struct ceph_ioctl_layout l; 42 | struct ceph_ioctl_dataloc dl; 43 | 44 | 45 | fd = open(filename, O_RDONLY, 0644); 46 | if (fd < 0) { 47 | printf("error: open() filename=[%s] failed with error=[%s]\n",filename, strerror(fd)); 48 | return DSS_ERROR; 49 | } 50 | 51 | dl.file_offset = offset; 52 | err = ioctl(fd, CEPH_IOC_GET_DATALOC, (unsigned long)&dl); 53 | if (err < 0) { 54 | printf("error: getting location [%s]\n", strerror(err)); 55 | return DSS_ERROR; 56 | } 57 | 58 | if (dl.object_name == NULL || strlen(dl.object_name) <= 0) { 59 | printf("error: dl.object_name is zero file_name=[%s] offset=[%d]", filename, offset); 60 | return DSS_ERROR; 61 | } 62 | 63 | memcpy(dest_object_name, dl.object_name, strlen(dl.object_name) + 1); 64 | 65 | 66 | close(fd); 67 | 68 | return DSS_OK; 69 | } 70 | 71 | static 72 | int get_rados_line(const char *poolname, const char *object_name, const size_t offset, const size_t line_size, char *line) { 73 | 74 | /* Declare the cluster handle and required arguments. */ 75 | int err; 76 | char cluster_name[] = "ceph"; 77 | char user_name[] = "client.admin"; 78 | uint64_t flags = 0; 79 | rados_t cluster; 80 | rados_ioctx_t io; 81 | rados_completion_t comp; 82 | 83 | 84 | /* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */ 85 | err = rados_create2(&cluster, cluster_name, user_name, flags); 86 | if (err < 0) { 87 | fprintf(stderr, "error: couldn't create the cluster handle poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 88 | poolname, object_name, offset, strerror(-err)); 89 | return DSS_ERROR; 90 | } 91 | 92 | /* Read a Ceph configuration file to configure the cluster handle. */ 93 | err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf"); 94 | if (err < 0) { 95 | fprintf(stderr, "error: cannot read config file poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 96 | poolname, object_name, offset, strerror(-err)); 97 | return DSS_ERROR; 98 | } 99 | 100 | /* Connect to the cluster */ 101 | err = rados_connect(cluster); 102 | if (err < 0) { 103 | fprintf(stderr, "error: cannot connect to cluster poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 104 | poolname, object_name, offset, strerror(-err)); 105 | return DSS_ERROR; 106 | } 107 | 108 | //create io 109 | err = rados_ioctx_create(cluster, poolname, &io); 110 | if (err < 0) { 111 | fprintf(stderr, "error: cannot open rados pool poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 112 | poolname, object_name, offset, strerror(-err)); 113 | rados_shutdown(cluster); 114 | return DSS_ERROR; 115 | } 116 | 117 | /* 118 | * Read data from the cluster asynchronously. 119 | * First, set up asynchronous I/O completion. 120 | */ 121 | err = rados_aio_create_completion(NULL, NULL, NULL, &comp); 122 | if (err < 0) { 123 | fprintf(stderr, "error: could not create aio completion poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 124 | poolname, object_name, offset, strerror(-err)); 125 | rados_ioctx_destroy(io); 126 | rados_shutdown(cluster); 127 | return DSS_ERROR; 128 | } 129 | 130 | /* Next, read data using rados_aio_read. */ 131 | err = rados_aio_read(io, object_name, comp, line, line_size, offset); 132 | if (err < 0) { 133 | fprintf(stderr, "error: cannot read object poolname=[%s] object_name=[%s] offset=[%d] error=[%s]\n", 134 | poolname, object_name, offset, strerror(-err)); 135 | rados_ioctx_destroy(io); 136 | rados_shutdown(cluster); 137 | return DSS_ERROR; 138 | } 139 | /* Wait for the operation to complete */ 140 | rados_aio_wait_for_complete(comp); 141 | /* Release the asynchronous I/O complete handle to avoid memory leaks. */ 142 | rados_aio_release(comp); 143 | 144 | rados_ioctx_destroy(io); 145 | rados_shutdown(cluster); 146 | 147 | return DSS_OK; 148 | } 149 | 150 | static 151 | int handler(const char *poolname, const char *filename, size_t offset, size_t line_size, char *line) { 152 | int rc; 153 | size_t dest_offset; 154 | char object_name[20]; 155 | 156 | //get object_name 157 | rc = get_object_name(filename, offset, object_name); 158 | if (rc == DSS_ERROR) { 159 | return DSS_ERROR; 160 | } 161 | 162 | //get object_name offset 163 | dest_offset = get_object_offset(offset); 164 | 165 | rc = get_rados_line(poolname, object_name, dest_offset, line_size, line); 166 | if (rc == DSS_ERROR) { 167 | return DSS_ERROR; 168 | } 169 | 170 | return DSS_OK; 171 | } 172 | 173 | 174 | int main(int argc, char **argv) 175 | { 176 | int rc; 177 | size_t line_size = 1024; 178 | char *filename; 179 | char *poolname; 180 | size_t offset; 181 | char line[line_size], next_object_line[line_size], *p; 182 | 183 | if (argc != 4) { 184 | printf("usage: dss_readfile \n"); 185 | return 1; 186 | } 187 | 188 | poolname = argv[1]; 189 | filename = argv[2]; 190 | offset = atoll(argv[3]); 191 | 192 | rc = handler(poolname, filename, offset, line_size, line); 193 | if (rc == DSS_ERROR) { 194 | return 1; 195 | } 196 | 197 | if (strlen(line) == line_size) { 198 | printf("success: %s", line); 199 | return 0; 200 | } 201 | 202 | if ((p = strchr(line, '\n')) != NULL) { 203 | *p = '\0'; 204 | printf("success: %s", line); 205 | return 0; 206 | } 207 | 208 | //next object_name 209 | offset = offset + 1024; 210 | rc = handler(poolname, filename, offset, line_size, next_object_line); 211 | if (rc == DSS_ERROR) { 212 | return 1; 213 | } 214 | 215 | if ((p = strchr(next_object_line, '\n')) != NULL) { 216 | *p = '\0'; 217 | printf("success: %s%s", line, next_object_line); 218 | return 0; 219 | } 220 | 221 | strncpy(line, next_object_line, line_size - strlen(line)); 222 | 223 | printf("success: %s", line); 224 | return 0; 225 | } 226 | 227 | -------------------------------------------------------------------------------- /cephfs_readline/config: -------------------------------------------------------------------------------- 1 | ngx_addon_name=ngx_http_cephfs_readline_module 2 | 3 | HTTP_MODULES="$HTTP_MODULES ngx_http_cephfs_readline_module" 4 | NGX_ADDON_SRCS="$NGX_ADDON_SRCS 5 | $ngx_addon_dir/ngx_http_cephfs_readline_handler.c 6 | $ngx_addon_dir/ngx_http_cephfs_readline_module.c" 7 | 8 | DIR="/home/rpmbuild/BUILD/" 9 | 10 | YAJL_INCS=" 11 | -I/usr/local/include/yajl/" 12 | 13 | YAJL_LIBS=" 14 | -L/usr/local/lib/yajl/lib/ -lyajl_s" 15 | 16 | CEPH_LIBS=" 17 | -L$DIR/ceph-12.2.8.1/build/lib/ -lrados" 18 | 19 | CEPH_INCS=" 20 | $DIR/ceph-12.2.8.1/src/include/ \ 21 | $DIR/ceph-12.2.8.1/src/ \ 22 | $DIR/ceph-12.2.8.1/build/include/ \ 23 | $DIR/ceph-12.2.8.1/src/client/" 24 | 25 | CORE_INCS="$CORE_INCS $CEPH_INCS $YAJL_INCS $ngx_addon_dir " 26 | CORE_LIBS="$CORE_LIBS $YAJL_LIBS $CEPH_LIBS -ldl -lrt" 27 | -------------------------------------------------------------------------------- /cephfs_readline/ngx_http_cephfs_readline_handler.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Author: lihang 3 | * 4 | * File: ngx_http_cephfs_readline_handler.c 5 | * Create Date: 2018-11-02 6 | * 7 | */ 8 | #include "ngx_http_cephfs_readline_handler.h" 9 | 10 | #include "ngx_http_cephfs_readline_handler.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include "ioctl.h" 30 | 31 | static const size_t line_size = 1048576; 32 | static const size_t object_size = 4194304; 33 | 34 | static ngx_int_t 35 | ngx_http_cephfs_request_parser(ngx_http_request_t *r); 36 | 37 | static ngx_int_t 38 | ngx_http_cephfs_response_body(ngx_http_request_t *r); 39 | 40 | static rados_t cluster = NULL; 41 | 42 | ngx_int_t 43 | ngx_http_cephfs_readline_start(ngx_cycle_t *cycle) 44 | { 45 | int err; 46 | ngx_uint_t flags = 0; 47 | ngx_str_t cluster_name = ngx_string("ceph"); 48 | ngx_str_t ceph_conf = ngx_string("/etc/ceph/ceph.conf"); 49 | ngx_str_t user_name = ngx_string("client.admin"); 50 | 51 | 52 | /* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */ 53 | err = rados_create2(&cluster, (char *)cluster_name.data, (char* ) user_name.data, flags); 54 | if ( err < 0 ) { 55 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 56 | "[cephfs_readline] couldn't create the cluster handle error=[%s]",strerror(-err)); 57 | return NGX_ERROR; 58 | } 59 | 60 | /* Read a Ceph configuration file to configure the cluster handle. */ 61 | err = rados_conf_read_file(cluster, (char *)ceph_conf.data); 62 | if ( err < 0 ) { 63 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 64 | "[cephfs_readline] cannot read config file error=[%s]", strerror(-err)); 65 | return NGX_ERROR; 66 | } 67 | 68 | /* Connect to the cluster */ 69 | err = rados_connect(cluster); 70 | if ( err < 0 ) { 71 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 72 | "[cephfs_readline] cannot connect to cluster error=[%s]", strerror(-err)); 73 | return NGX_ERROR; 74 | } 75 | 76 | return NGX_OK; 77 | } 78 | 79 | void 80 | ngx_http_cephfs_readline_exit(ngx_cycle_t *cycle) 81 | { 82 | if ( cluster ) { 83 | rados_shutdown(cluster); 84 | } 85 | } 86 | 87 | 88 | static ngx_int_t 89 | ngx_http_cephfs_get_object_offset(const size_t offset) 90 | { 91 | size_t object_count; 92 | 93 | object_count = offset / object_size; 94 | return (offset - object_count * object_size); 95 | } 96 | 97 | static ngx_int_t 98 | ngx_http_cephfs_get_object_name(ngx_http_request_t *r) 99 | { 100 | int fd, err; 101 | struct ceph_ioctl_dataloc dl; 102 | ngx_http_cephfs_readline_ctx_t *ctx; 103 | 104 | 105 | ctx = (ngx_http_cephfs_readline_ctx_t*)ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 106 | if ( ctx == NULL ) { 107 | return NGX_ERROR; 108 | } 109 | 110 | if ( ctx->filename.data == NULL || ctx->filename.len == 0 ) { 111 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 112 | "[cephfs_readline] filename is empty"); 113 | return NGX_ERROR; 114 | } 115 | ctx->filename.data[ctx->filename.len] = 0; 116 | 117 | fd = open((char*)ctx->filename.data, O_RDONLY, 0644); 118 | if ( fd < 0 ) { 119 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 120 | "[cephfs_readline] open() filename=[%s] failed error=[%s]", ctx->filename.data, strerror(fd)); 121 | return NGX_ERROR; 122 | } 123 | 124 | if ( ctx->next ) { 125 | dl.file_offset = ctx->next_offset; 126 | }else { 127 | dl.file_offset = ctx->offset; 128 | } 129 | err = ioctl(fd, CEPH_IOC_GET_DATALOC, (unsigned long)&dl); 130 | if ( err < 0 ) { 131 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 132 | "[cephfs_readline] error: getting location [%s]", strerror(err)); 133 | return NGX_ERROR; 134 | } 135 | 136 | if ( ctx->next ) { 137 | ctx->next_object_name.len = strlen(dl.object_name); 138 | ctx->next_object_name.data = (u_char*)ngx_pcalloc(r->pool, ctx->next_object_name.len + 1); 139 | if ( ctx->next_object_name.data == NULL ) { 140 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 141 | "[cephfs_readline] ctx->object_name.data malloc is failed"); 142 | 143 | close(fd); 144 | return NGX_ERROR; 145 | } 146 | ngx_memcpy(ctx->next_object_name.data, dl.object_name, ctx->next_object_name.len); 147 | ctx->next_object_name.data[ctx->next_object_name.len] = 0; 148 | 149 | close(fd); 150 | ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0, 151 | "[cephfs_readline] object_name.len=[%d] object_name.data=[%s]", ctx->next_object_name.len, ctx->next_object_name.data); 152 | 153 | return NGX_OK; 154 | } 155 | 156 | ctx->object_name.len = strlen(dl.object_name); 157 | ctx->object_name.data = (u_char*)ngx_pcalloc(r->pool, ctx->object_name.len + 1); 158 | if ( ctx->object_name.data == NULL ) { 159 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 160 | "[cephfs_readline] ctx->object_name.data malloc is failed"); 161 | 162 | close(fd); 163 | return NGX_ERROR; 164 | } 165 | ngx_memcpy(ctx->object_name.data, dl.object_name, ctx->object_name.len); 166 | ctx->object_name.data[ctx->object_name.len] = 0; 167 | 168 | close(fd); 169 | ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0, 170 | "[cephfs_readline] object_name.len=[%d] object_name.data=[%s]", ctx->object_name.len, ctx->object_name.data); 171 | 172 | return NGX_OK; 173 | } 174 | 175 | static ngx_int_t 176 | ngx_http_cephfs_get_rados_line(ngx_http_request_t *r) 177 | { 178 | int err; 179 | char buffer[line_size]; 180 | ngx_uint_t offset; 181 | rados_ioctx_t io; 182 | rados_completion_t comp; 183 | ngx_http_cephfs_readline_ctx_t *ctx; 184 | 185 | 186 | ctx = (ngx_http_cephfs_readline_ctx_t*)ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 187 | if ( ctx == NULL ) { 188 | return NGX_ERROR; 189 | } 190 | 191 | if ( ctx->poolname.data == NULL || ctx->poolname.len == 0 ) { 192 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 193 | "[cephfs_readline] poolname is empty"); 194 | return NGX_ERROR; 195 | } 196 | 197 | //create io 198 | ctx->poolname.data[ctx->poolname.len] = 0; 199 | 200 | err = rados_ioctx_create(cluster, (char *)ctx->poolname.data, &io); 201 | if ( err < 0 ) { 202 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 203 | "[cephfs_readline] cannot open rados pool poolname=[%s] object_name=[%s] error=[%s]", 204 | ctx->poolname.data, ctx->object_name.data, strerror(-err)); 205 | 206 | return NGX_ERROR; 207 | } 208 | 209 | /* 210 | * Read data from the cluster asynchronously. 211 | * First, set up asynchronous I/O completion. 212 | */ 213 | err = rados_aio_create_completion(NULL, NULL, NULL, &comp); 214 | if ( err < 0 ) { 215 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 216 | "[cephfs_readline] could not create aio completion poolname=[%s] object_name=[%s] error=[%s]", 217 | ctx->poolname.data, ctx->object_name.data, strerror(-err)); 218 | 219 | rados_ioctx_destroy(io); 220 | return NGX_ERROR; 221 | } 222 | 223 | offset = ctx->next ? ctx->next_dest_offset : ctx->dest_offset; 224 | 225 | /* Next, read data using rados_aio_read. */ 226 | err = rados_aio_read(io, (char *)ctx->object_name.data, comp, buffer, line_size, offset); 227 | if (err < 0) { 228 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 229 | "[cephfs_readline] cannot read object poolname=[%s] object_name=[%s] offset=[%d] error=[%s]", 230 | ctx->poolname.data, ctx->object_name.data, ctx->offset, strerror(-err)); 231 | 232 | rados_aio_release(comp); 233 | rados_ioctx_destroy(io); 234 | return NGX_ERROR; 235 | } 236 | /* Wait for the operation to complete */ 237 | rados_aio_wait_for_complete(comp); 238 | err = rados_aio_get_return_value(comp); 239 | if ( err < 0 ) { 240 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 241 | "[cephfs_readline] reados_aio_get_return_value poolname=[%s] object_name=[%s] offset=[%d] error=[%s]", 242 | ctx->poolname.data, ctx->object_name.data, ctx->offset, strerror(-err)); 243 | 244 | rados_aio_release(comp); 245 | rados_ioctx_destroy(io); 246 | return NGX_ERROR; 247 | } 248 | buffer[err] = 0; 249 | 250 | if ( ctx->next ) { 251 | ctx->next_line.len = strlen(buffer); 252 | ctx->next_line.data = ngx_pcalloc(r->pool, ctx->next_line.len); 253 | if ( ctx->next_line.data == NULL ) { 254 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 255 | "[cephfs_readline] ctx->next_line.data malloc is failed"); 256 | /* Release the asynchronous I/O complete handle to avoid memory leaks. */ 257 | rados_aio_release(comp); 258 | rados_ioctx_destroy(io); 259 | return NGX_ERROR; 260 | } 261 | ngx_memcpy(ctx->next_line.data, buffer, ctx->next_line.len); 262 | 263 | ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0, 264 | "[cephfs_readline] next_line.len=[%d] next_line.data=[%s]", ctx->next_line.len, ctx->next_line.data); 265 | rados_aio_release(comp); 266 | rados_ioctx_destroy(io); 267 | return NGX_OK; 268 | } 269 | 270 | ctx->line.len = strlen(buffer); 271 | ctx->line.data = ngx_pcalloc(r->pool, ctx->line.len); 272 | if ( ctx->line.data == NULL ) { 273 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 274 | "[cephfs_readline] ctx->next_line.data malloc is failed"); 275 | /* Release the asynchronous I/O complete handle to avoid memory leaks. */ 276 | rados_aio_release(comp); 277 | rados_ioctx_destroy(io); 278 | return NGX_ERROR; 279 | } 280 | ngx_memcpy(ctx->line.data, buffer, ctx->line.len); 281 | 282 | ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0, 283 | "[cephfs_readline] line.len=[%d] line.data=[%s]", ctx->line.len, ctx->line.data); 284 | 285 | /* Release the asynchronous I/O complete handle to avoid memory leaks. */ 286 | rados_aio_release(comp); 287 | rados_ioctx_destroy(io); 288 | 289 | return NGX_OK; 290 | } 291 | 292 | 293 | static ngx_int_t 294 | ngx_http_cephfs_readline_process(ngx_http_request_t *r) 295 | { 296 | ngx_int_t rc; 297 | ngx_http_cephfs_readline_ctx_t *ctx; 298 | 299 | 300 | ctx = (ngx_http_cephfs_readline_ctx_t*)ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 301 | if ( ctx == NULL ) { 302 | return NGX_ERROR; 303 | } 304 | 305 | //get object_name 306 | rc = ngx_http_cephfs_get_object_name(r); 307 | if ( rc != NGX_OK ) { 308 | return NGX_ERROR; 309 | } 310 | 311 | //get object_name offset 312 | if ( ctx->next ) { 313 | ctx->next_dest_offset = ngx_http_cephfs_get_object_offset(ctx->next_offset); 314 | }else { 315 | ctx->dest_offset = ngx_http_cephfs_get_object_offset(ctx->offset); 316 | } 317 | 318 | //get rados readline 319 | rc = ngx_http_cephfs_get_rados_line(r); 320 | if ( rc != NGX_OK ) { 321 | return NGX_ERROR; 322 | } 323 | 324 | return NGX_OK; 325 | } 326 | 327 | ngx_int_t 328 | ngx_http_cephfs_readline_request(ngx_http_request_t *r) 329 | { 330 | char *p; 331 | ngx_int_t rc; 332 | struct timeval tv; 333 | ngx_http_cephfs_readline_ctx_t *ctx; 334 | 335 | 336 | ctx = (ngx_http_cephfs_readline_ctx_t*)ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 337 | if ( ctx == NULL ) { 338 | ctx->code = NGX_ERROR; 339 | goto done; 340 | } 341 | 342 | rc = ngx_http_cephfs_request_parser(r); 343 | if ( rc != NGX_OK ) { 344 | ctx->code = NGX_ERROR; 345 | goto done; 346 | } 347 | 348 | rc = ngx_http_cephfs_readline_process(r); 349 | if ( rc != NGX_OK ) { 350 | ctx->code = NGX_ERROR; 351 | goto done; 352 | } 353 | 354 | if ((p = strchr((char*)ctx->line.data, '\n')) != NULL) { 355 | *p = '\0'; 356 | ctx->body.data = ctx->line.data; 357 | ctx->body.len = strlen((char*)ctx->body.data); 358 | ctx->code = NGX_OK; 359 | goto done; 360 | } 361 | 362 | if ( ctx->line.len == line_size ) { 363 | ctx->body.data = ctx->line.data; 364 | ctx->body.len = ctx->line.len; 365 | ctx->code = NGX_OK; 366 | goto done; 367 | } 368 | 369 | //next object_name 370 | ctx->next = 1; 371 | ctx->next_offset = ctx->offset + line_size; 372 | rc = ngx_http_cephfs_readline_process(r); 373 | if ( rc != NGX_OK ) { 374 | ctx->code = NGX_ERROR; 375 | goto done; 376 | } 377 | 378 | if ((p = strchr((char*)ctx->next_line.data, '\n')) != NULL) { 379 | *p = '\0'; 380 | 381 | ctx->body.len = ctx->line.len + strlen((char*)ctx->next_line.data); 382 | ctx->body.data = ngx_pcalloc(r->connection->pool, ctx->body.len); 383 | if ( ctx->body.data == NULL ) { 384 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "[cephfs_readline] ctx->body.data malloc is failed"); 385 | ctx->code = NGX_ERROR; 386 | goto done; 387 | } 388 | ngx_sprintf(ctx->body.data, "%s%s", ctx->line.data, ctx->next_line.data); 389 | ctx->code = NGX_OK; 390 | goto done; 391 | } 392 | 393 | ctx->body.len = ctx->line.len + (line_size - ctx->line.len); 394 | ctx->body.data = ngx_pcalloc(r->connection->pool, ctx->body.len); 395 | if ( ctx->body.data == NULL ) { 396 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "[cephfs_readline] ctx->body.data malloc is failed"); 397 | ctx->code = NGX_ERROR; 398 | goto done; 399 | } 400 | strncpy((char*)ctx->body.data, (char*)ctx->line.data, ctx->line.len); 401 | strncpy((char*)ctx->body.data, (char*)ctx->next_line.data, line_size - ctx->line.len); 402 | 403 | done: 404 | ngx_gettimeofday(&tv); 405 | ctx->cost = (tv.tv_sec - r->start_sec) * 1000 406 | + (tv.tv_usec / 1000 - r->start_msec); 407 | 408 | rc = ngx_http_cephfs_response_body(r); 409 | if ( rc != NGX_OK ) { 410 | return NGX_ERROR; 411 | } 412 | 413 | return NGX_OK; 414 | } 415 | 416 | 417 | static ngx_int_t 418 | ngx_http_cephfs_response_body(ngx_http_request_t *r) 419 | { 420 | size_t len; 421 | const unsigned char *buf; 422 | yajl_gen g; 423 | ngx_http_cephfs_readline_ctx_t *ctx; 424 | 425 | 426 | ctx = (ngx_http_cephfs_readline_ctx_t*)ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 427 | if ( ctx == NULL ) { 428 | ctx->code = NGX_ERROR; 429 | return NGX_ERROR; 430 | } 431 | 432 | g = yajl_gen_alloc(NULL); 433 | if ( g == NULL ) { 434 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 435 | "[cephfs_readline] yajl_gen_alloc is null"); 436 | return NGX_ERROR; 437 | } 438 | yajl_gen_config(g, yajl_gen_beautify, 0); 439 | 440 | yajl_gen_map_open(g); 441 | 442 | yajl_gen_string(g, (const unsigned char *) NGX_HTTP_CEPHFS_READLINE_CODE, strlen(NGX_HTTP_CEPHFS_READLINE_CODE)); 443 | yajl_gen_integer(g, ctx->code); 444 | 445 | yajl_gen_string(g, (const unsigned char *) NGX_HTTP_CEPHFS_READLINE_COST, strlen(NGX_HTTP_CEPHFS_READLINE_COST)); 446 | yajl_gen_integer(g, ctx->cost); 447 | 448 | yajl_gen_string(g, (const unsigned char *) NGX_HTTP_CEPHFS_READLINE_DATA, strlen(NGX_HTTP_CEPHFS_READLINE_DATA)); 449 | yajl_gen_string(g, ctx->body.data, ctx->body.len); 450 | 451 | yajl_gen_map_close(g); 452 | 453 | yajl_gen_status status = yajl_gen_get_buf(g, &buf, &len); 454 | if(status != yajl_gen_status_ok) { 455 | yajl_gen_free(g); 456 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 457 | "[cephfs_readline] yajl_gen_get_buf is null"); 458 | return NGX_ERROR; 459 | } 460 | 461 | ctx->response_body.len = len; 462 | ctx->response_body.data = (u_char*)ngx_pcalloc(r->pool, ctx->response_body.len); 463 | if ( ctx->response_body.data == NULL ) { 464 | yajl_gen_free(g); 465 | return NGX_ERROR; 466 | } 467 | ngx_memcpy(ctx->response_body.data, buf, ctx->response_body.len); 468 | yajl_gen_free(g); 469 | 470 | return NGX_OK; 471 | } 472 | 473 | static ngx_int_t 474 | ngx_http_cephfs_request_parser(ngx_http_request_t *r) 475 | { 476 | yajl_val node; 477 | ngx_http_cephfs_readline_ctx_t *ctx; 478 | 479 | 480 | ctx = (ngx_http_cephfs_readline_ctx_t*)ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 481 | if ( ctx == NULL ) { 482 | return NGX_ERROR; 483 | } 484 | ctx->request_body->pos[ctx->request_body->last - ctx->request_body->pos] = 0; 485 | 486 | node = yajl_tree_parse((char*)ctx->request_body->pos, NULL, 0); 487 | if (( node == NULL ) || (!YAJL_IS_OBJECT(node))) { 488 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 489 | "[cephfs_readline] yajl_tree_parse is null"); 490 | return NGX_ERROR; 491 | } 492 | 493 | const yajl_val filename = yajl_tree_get(node , (const char*[]){NGX_HTTP_CEPHFS_READLINE_FILENAME, 0}, yajl_t_string); 494 | const yajl_val poolname = yajl_tree_get(node , (const char*[]){NGX_HTTP_CEPHFS_READLINE_POOLNAME, 0}, yajl_t_string); 495 | const yajl_val offset = yajl_tree_get(node , (const char*[]){NGX_HTTP_CEPHFS_READLINE_OFFSET, 0}, yajl_t_number); 496 | 497 | if ( !filename || !poolname || !offset ) { 498 | yajl_tree_free(node); 499 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 500 | "[cephfs_readline] yajl_tree_parse json fromat illegal"); 501 | return NGX_ERROR; 502 | } 503 | 504 | const char *filename_buf = YAJL_GET_STRING(filename); 505 | if ( filename_buf == NULL ) { 506 | yajl_tree_free(node); 507 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 508 | "[cephfs_readline] yajl_tree_parse json fromat illegal filenme is empty"); 509 | return NGX_ERROR; 510 | } 511 | 512 | ctx->filename.len = strlen(filename_buf); 513 | ctx->filename.data = ngx_pcalloc(r->pool, ctx->filename.len); 514 | if ( ctx->filename.data == NULL ) { 515 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 516 | "[cephfs_readline] ctx->filename malloc is failed"); 517 | return NGX_ERROR; 518 | } 519 | ngx_memcpy(ctx->filename.data, filename_buf, ctx->filename.len); 520 | 521 | 522 | const char *poolname_buf = YAJL_GET_STRING(poolname); 523 | if ( poolname_buf == NULL ) { 524 | yajl_tree_free(node); 525 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 526 | "[cephfs_readline] yajl_tree_parse json fromat illegal poolname is empty"); 527 | return NGX_ERROR; 528 | } 529 | 530 | ctx->poolname.len = strlen(poolname_buf); 531 | ctx->poolname.data = ngx_pcalloc(r->pool, ctx->poolname.len); 532 | if ( ctx->poolname.data == NULL ) { 533 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 534 | "[cephfs_readline] ctx->poolname malloc is failed"); 535 | return NGX_ERROR; 536 | } 537 | ngx_memcpy(ctx->poolname.data, poolname_buf, ctx->poolname.len); 538 | 539 | ctx->offset = YAJL_GET_INTEGER(offset); 540 | 541 | return NGX_OK; 542 | } 543 | 544 | -------------------------------------------------------------------------------- /cephfs_readline/ngx_http_cephfs_readline_handler.h: -------------------------------------------------------------------------------- 1 | #ifndef __NGX_HTTP_CEPHFS_READLINE_HANDLER_H__ 2 | #define __NGX_HTTP_CEPHFS_READLINE_HANDLER_H__ 3 | 4 | #include "ngx_http_cephfs_readline_module.h" 5 | 6 | 7 | #define NGX_HTTP_CEPHFS_READLINE_FILENAME "filename" 8 | #define NGX_HTTP_CEPHFS_READLINE_POOLNAME "poolname" 9 | #define NGX_HTTP_CEPHFS_READLINE_OFFSET "offset" 10 | #define NGX_HTTP_CEPHFS_READLINE_CODE "code" 11 | #define NGX_HTTP_CEPHFS_READLINE_COST "cost" 12 | #define NGX_HTTP_CEPHFS_READLINE_DATA "data" 13 | 14 | 15 | extern ngx_module_t ngx_http_cephfs_readline_module; 16 | 17 | ngx_int_t 18 | ngx_http_cephfs_readline_start(ngx_cycle_t *cycle); 19 | 20 | void 21 | ngx_http_cephfs_readline_exit(ngx_cycle_t *cycle); 22 | 23 | ngx_int_t 24 | ngx_http_cephfs_readline_request(ngx_http_request_t *r); 25 | 26 | 27 | #endif /* __NGX_HTTP_CEPHFS_READLINE_HANDLER_H__ */ 28 | -------------------------------------------------------------------------------- /cephfs_readline/ngx_http_cephfs_readline_module.c: -------------------------------------------------------------------------------- 1 | /* Author: lihanglucien 2 | * 3 | * File: ngx_http_cephfs_readline_module.c 4 | * Create Date: 2018-11-02 5 | * 6 | */ 7 | #include "ngx_http_cephfs_readline_handler.h" 8 | 9 | 10 | static ngx_int_t 11 | ngx_http_cephfs_readline_handler(ngx_http_request_t *r); 12 | 13 | static void 14 | ngx_http_cephfs_readline_request_body_handler(ngx_http_request_t *r); 15 | 16 | static ngx_buf_t * 17 | ngx_http_cephfs_readline_read_body(ngx_http_request_t *r); 18 | 19 | static ngx_buf_t * 20 | ngx_http_cephfs_readline_read_body_from_file(ngx_http_request_t *r); 21 | 22 | static ngx_int_t 23 | ngx_http_cephfs_readline_init(ngx_conf_t *cf); 24 | 25 | static void * 26 | ngx_http_cephfs_readline_create_loc_conf(ngx_conf_t *cf); 27 | 28 | static char* 29 | ngx_http_cephfs_readline_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child); 30 | 31 | static ngx_int_t 32 | ngx_http_cephfs_readline_init_process(ngx_cycle_t *cycle); 33 | 34 | static void 35 | ngx_http_cephfs_readline_exit_process(ngx_cycle_t *cycle); 36 | 37 | 38 | static ngx_command_t ngx_http_cephfs_readline_commands[] = { 39 | 40 | { ngx_string("cephfs_readline"), 41 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, 42 | ngx_conf_set_flag_slot, 43 | NGX_HTTP_LOC_CONF_OFFSET, 44 | offsetof(ngx_http_cephfs_readline_conf_t, enable), 45 | NULL }, 46 | 47 | ngx_null_command 48 | }; 49 | 50 | static ngx_http_module_t ngx_http_cephfs_readline_module_ctx = { 51 | NULL, /* preconfiguration */ 52 | ngx_http_cephfs_readline_init, /* postconfiguration */ 53 | 54 | NULL, /* create main configuration */ 55 | NULL, /* init main configuration */ 56 | 57 | NULL, /* create server configuration */ 58 | NULL, /* merge server configuration */ 59 | 60 | ngx_http_cephfs_readline_create_loc_conf, /* create location configration */ 61 | ngx_http_cephfs_readline_merge_loc_conf /* merge location configration */ 62 | }; 63 | 64 | ngx_module_t ngx_http_cephfs_readline_module = { 65 | NGX_MODULE_V1, 66 | &ngx_http_cephfs_readline_module_ctx, /* module context */ 67 | ngx_http_cephfs_readline_commands, /* module directives */ 68 | NGX_HTTP_MODULE, /* module type */ 69 | NULL, /* init master */ 70 | NULL, /* init module */ 71 | ngx_http_cephfs_readline_init_process, /* init process */ 72 | NULL, /* init thread */ 73 | NULL, /* exit thread */ 74 | ngx_http_cephfs_readline_exit_process, /* exit process */ 75 | NULL, /* exit master */ 76 | NGX_MODULE_V1_PADDING 77 | }; 78 | 79 | 80 | static ngx_int_t 81 | ngx_http_cephfs_readline_handler(ngx_http_request_t *r) 82 | { 83 | ngx_int_t rc; 84 | ngx_http_cephfs_readline_ctx_t *ctx; 85 | ngx_http_cephfs_readline_conf_t *lrcf; 86 | 87 | lrcf = ngx_http_get_module_loc_conf(r, ngx_http_cephfs_readline_module); 88 | if ( lrcf == NULL ) { 89 | return NGX_DECLINED; 90 | } 91 | 92 | if ( lrcf->enable == NGX_CONF_UNSET || lrcf->enable == 0 ) { 93 | return NGX_DECLINED; 94 | } 95 | 96 | ctx = (ngx_http_cephfs_readline_ctx_t*)ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 97 | if ( ctx == NULL ) { 98 | ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_cephfs_readline_ctx_t)); 99 | if ( ctx == NULL ) { 100 | return NGX_ERROR; 101 | } 102 | ngx_http_set_ctx(r, ctx, ngx_http_cephfs_readline_module); 103 | } 104 | 105 | ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0, 106 | "[cephfs_readline] ngx_http_cephfs_readline start"); 107 | 108 | 109 | rc = ngx_http_read_client_request_body(r, ngx_http_cephfs_readline_request_body_handler); 110 | if ( rc >= NGX_HTTP_SPECIAL_RESPONSE ) { 111 | return rc; 112 | } 113 | 114 | 115 | return NGX_DONE; 116 | } 117 | 118 | static void 119 | ngx_http_cephfs_readline_send_response(ngx_http_request_t *r, ngx_int_t code, 120 | ngx_str_t *content) 121 | { 122 | ngx_int_t rc; 123 | ngx_buf_t *b; 124 | ngx_chain_t out; 125 | 126 | 127 | /* set the 'Content-type' header */ 128 | ngx_str_set(&r->headers_out.content_type, "text/json"); 129 | r->headers_out.status = code; 130 | 131 | r->headers_out.content_length_n = content->len; 132 | 133 | rc = ngx_http_send_header(r); 134 | if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { 135 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 136 | "[cephfs_readline] ngx_send_header error rc=[%d]", rc); 137 | ngx_http_finalize_request(r, rc); 138 | return; 139 | } 140 | 141 | if ( content->len == 0 ) { 142 | ngx_http_finalize_request(r, ngx_http_send_special(r, NGX_HTTP_FLUSH)); 143 | return; 144 | } 145 | 146 | 147 | /* allocate a buffer for your response body */ 148 | b = ngx_create_temp_buf(r->pool, content->len); 149 | if ( b == NULL ) { 150 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 151 | "[cephfs_readline] request_body malloc is failed"); 152 | ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); 153 | return; 154 | } 155 | 156 | /* attach this buffer to the buffer chain */ 157 | out.buf = b; 158 | out.next = NULL; 159 | 160 | /* adjust the pointers of the buffer */ 161 | b->pos = content->data; 162 | b->last = content->data + content->len; 163 | b->memory = 1; /* this buffer is in memory */ 164 | b->last_buf = 1; /* this is the last buffer in the buffer chain */ 165 | 166 | /* send the buffer chain of your response */ 167 | ngx_http_finalize_request(r, ngx_http_output_filter(r, &out)); 168 | } 169 | 170 | 171 | static void 172 | ngx_http_cephfs_readline_request_body_handler(ngx_http_request_t *r) 173 | { 174 | ngx_int_t rc; 175 | ngx_int_t code; 176 | ngx_http_cephfs_readline_ctx_t *ctx; 177 | 178 | ctx = ngx_http_get_module_ctx(r, ngx_http_cephfs_readline_module); 179 | if ( ctx == NULL ) { 180 | return; 181 | } 182 | 183 | if (r->method != NGX_HTTP_POST) { 184 | code = NGX_HTTP_NOT_ALLOWED; 185 | goto finish; 186 | } 187 | 188 | if (r->request_body == NULL || r->request_body->bufs == NULL) { 189 | code = NGX_HTTP_NO_CONTENT; 190 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 191 | "[cephfs_readline] request_body no content"); 192 | goto finish; 193 | } 194 | 195 | if ( r->request_body != NULL && r->request_body->bufs != NULL ) { 196 | if ( r->request_body->temp_file ) { 197 | ctx->request_body = ngx_http_cephfs_readline_read_body_from_file(r); 198 | } else { 199 | ctx->request_body = ngx_http_cephfs_readline_read_body(r); 200 | } 201 | } 202 | 203 | if ( ctx->request_body == NULL ) { 204 | code = NGX_HTTP_INTERNAL_SERVER_ERROR; 205 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 206 | "[cephfs_readline] request_body out of memory"); 207 | goto finish; 208 | } 209 | 210 | //request process 211 | rc = ngx_http_cephfs_readline_request(r); 212 | if ( rc != NGX_OK ) { 213 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 214 | "[cephfs_readline] cephfs_readline_request is error"); 215 | } 216 | 217 | code = NGX_HTTP_OK; 218 | 219 | finish: 220 | 221 | ngx_http_cephfs_readline_send_response(r, code, &ctx->response_body); 222 | } 223 | 224 | static ngx_buf_t * 225 | ngx_http_cephfs_readline_read_body(ngx_http_request_t *r) 226 | { 227 | size_t len; 228 | ngx_buf_t *buf, *next, *body; 229 | ngx_chain_t *cl; 230 | 231 | cl = r->request_body->bufs; 232 | buf = cl->buf; 233 | 234 | if (cl->next == NULL) { 235 | return buf; 236 | } else { 237 | next = cl->next->buf; 238 | len = (buf->last - buf->pos) + (next->last - next->pos); 239 | 240 | body = ngx_create_temp_buf(r->pool, len); 241 | if (body == NULL) { 242 | return NULL; 243 | } 244 | body->last = ngx_cpymem(body->last, buf->pos, buf->last - buf->pos); 245 | body->last = ngx_cpymem(body->last, next->pos, next->last - next->pos); 246 | } 247 | 248 | return body; 249 | } 250 | 251 | static ngx_buf_t * 252 | ngx_http_cephfs_readline_read_body_from_file(ngx_http_request_t *r) 253 | { 254 | size_t len; 255 | ssize_t size; 256 | ngx_buf_t *buf, *body; 257 | ngx_chain_t *cl; 258 | 259 | len = 0; 260 | cl = r->request_body->bufs; 261 | 262 | while (cl) { 263 | buf = cl->buf; 264 | if (buf->in_file) { 265 | len += buf->file_last - buf->file_pos; 266 | } else { 267 | len += buf->last - buf->pos; 268 | } 269 | cl = cl->next; 270 | } 271 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 272 | "[cephfs_readline] read post body file size %ui", len); 273 | 274 | body = ngx_create_temp_buf(r->pool, len); 275 | if (body == NULL) { 276 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 277 | "[cephfs_readline] cannot allocate enough space to store the request body, request len [%d]", len); 278 | return NULL; 279 | } 280 | cl = r->request_body->bufs; 281 | 282 | while (cl) { 283 | buf = cl->buf; 284 | if (buf->in_file) { 285 | size = ngx_read_file(buf->file, body->last, 286 | buf->file_last - buf->file_pos, buf->file_pos); 287 | if (size == NGX_ERROR) { 288 | return NULL; 289 | } 290 | body->last += size; 291 | } else { 292 | body->last = ngx_cpymem(body->last, buf->pos, buf->last - buf->pos); 293 | } 294 | cl = cl->next; 295 | } 296 | return body; 297 | } 298 | 299 | static void * 300 | ngx_http_cephfs_readline_create_loc_conf(ngx_conf_t *cf) 301 | { 302 | ngx_http_cephfs_readline_conf_t *conf; 303 | 304 | conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_cephfs_readline_conf_t)); 305 | if (conf == NULL) { 306 | return NULL; 307 | } 308 | 309 | conf->enable = NGX_CONF_UNSET; 310 | 311 | return conf; 312 | } 313 | 314 | static char* 315 | ngx_http_cephfs_readline_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) 316 | { 317 | ngx_http_cephfs_readline_conf_t *prev = parent; 318 | ngx_http_cephfs_readline_conf_t *conf = child; 319 | 320 | ngx_conf_merge_value(conf->enable, prev->enable, 0); 321 | 322 | return NGX_CONF_OK; 323 | } 324 | 325 | static ngx_int_t 326 | ngx_http_cephfs_readline_init(ngx_conf_t *cf) 327 | { 328 | ngx_http_handler_pt *h; 329 | ngx_http_core_main_conf_t *cmcf; 330 | 331 | cmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module); 332 | 333 | h = ngx_array_push(&cmcf->phases[NGX_HTTP_CONTENT_PHASE].handlers); 334 | if (h == NULL) { 335 | return NGX_ERROR; 336 | } 337 | 338 | *h = ngx_http_cephfs_readline_handler; 339 | 340 | return NGX_OK; 341 | } 342 | 343 | static ngx_int_t 344 | ngx_http_cephfs_readline_init_process(ngx_cycle_t *cycle) 345 | { 346 | ngx_log_error(NGX_LOG_DEBUG, cycle->log, 0, "[cephfs_readline] init process"); 347 | 348 | return ngx_http_cephfs_readline_start(cycle); 349 | } 350 | 351 | 352 | static void 353 | ngx_http_cephfs_readline_exit_process(ngx_cycle_t *cycle) 354 | { 355 | ngx_log_error(NGX_LOG_DEBUG, cycle->log, 0, "[cephfs_readline] exit process"); 356 | 357 | ngx_http_cephfs_readline_exit(cycle); 358 | } 359 | 360 | -------------------------------------------------------------------------------- /cephfs_readline/ngx_http_cephfs_readline_module.h: -------------------------------------------------------------------------------- 1 | #ifndef __NGX_HTTP_CEPHFS_READLINE_MODULE_H__ 2 | #define __NGX_HTTP_CEPHFS_READLINE_MODULE_H__ 3 | 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | typedef struct { 11 | ngx_str_t filename; 12 | ngx_str_t poolname; 13 | ngx_int_t offset; 14 | ngx_str_t object_name; 15 | ngx_str_t line; 16 | ngx_int_t dest_offset; 17 | 18 | ngx_int_t next; 19 | ngx_int_t next_offset; 20 | ngx_str_t next_object_name; 21 | ngx_str_t next_line; 22 | ngx_int_t next_dest_offset; 23 | 24 | ngx_str_t body; 25 | ngx_str_t response_body; 26 | ngx_buf_t *request_body; 27 | 28 | ngx_int_t code; 29 | ngx_int_t cost; 30 | } ngx_http_cephfs_readline_ctx_t; 31 | 32 | 33 | typedef struct { 34 | ngx_flag_t enable; 35 | } ngx_http_cephfs_readline_conf_t; 36 | 37 | 38 | #endif /* __NGX_HTTP_CEPHFS_READLINE_MODULE_H__ */ 39 | 40 | --------------------------------------------------------------------------------