7 |
8 | local tracker = require('resty.fastdfs.tracker')
9 | local storage = require('resty.fastdfs.storage')
10 | local tk = tracker:new()
11 | tk:set_timeout(3000)
12 | local ok, err = tk:connect({host='192.168.85.249',port=22122})
13 |
14 | function _dump_res(res)
15 | for i in pairs(res) do
16 | ngx.say(string.format("%s:%s",i, res[i]))
17 | end
18 | ngx.say("")
19 | end
20 |
21 | if not ok then
22 | ngx.say('connect error:' .. err)
23 | ngx.exit(200)
24 | end
25 |
26 | local res, err = tk:query_storage_store()
27 | if not res then
28 | ngx.say("query storage error:" .. err)
29 | ngx.exit(200)
30 | end
31 |
32 | --[[
33 | local res, err = tk:query_storage_update1("group1/M00/00/00/wKhV-VFbkMQEAAAAAAAAAKbO3LA494.txt")
34 | if not res then
35 | ngx.say("query storage error:" .. err)
36 | ngx.exit(200)
37 | end
38 |
39 | local res, err = tk:query_storage_fetch1("group1/M00/00/00/wKhV-VFbkFfR1owfAAAAD6bO3LA348.txt")
40 | if not res then
41 | ngx.say("query storage error:" .. err)
42 | ngx.exit(200)
43 | end
44 |
45 | ]]
46 |
47 | if not res then
48 | ngx.say("query storage error:" .. err)
49 | ngx.exit(200)
50 | end
51 |
52 |
53 |
54 | local st = storage:new()
55 | st:set_timeout(3000)
56 | local ok, err = st:connect(res)
57 | if not ok then
58 | ngx.say("connect storage error:" .. err)
59 | ngx.exit(200)
60 | end
61 |
62 |
63 |
64 | local ok, err = st:delete_file1("group1/M00/00/00/wKhV-VFY71sEAAAAAAAAAKbO3LA277.txt")
65 | if not ok then
66 | ngx.say("Fail:")
67 | else
68 | ngx.say("OK")
69 | end
70 |
71 |
72 |
73 | local res, err = st:upload_by_buff('abcdedfg','txt')
74 | if not res then
75 | ngx.say("upload error:" .. err)
76 | ngx.exit(200)
77 | end
78 |
79 |
80 | local res, err = st:upload_appender_by_buff('abcdedfg','txt')
81 | if not res then
82 | ngx.say("upload error:" .. err)
83 | ngx.exit(200)
84 | end
85 |
86 |
87 | local ok, err = st:append_by_buff1("group1/M00/00/00/wKhV-VFbkMQEAAAAAAAAAKbO3LA494.txt","abcdedfg\n")
88 | if not ok then
89 | ngx.say("Fail:")
90 | else
91 | ngx.say("OK")
92 | end
93 |
94 |
95 | local res, err = tk:query_storage_update1("group1/M00/00/00/wKhV-VFbkFfR1owfAAAAD6bO3LA348.txt")
96 | if not res then
97 | ngx.say("query storate error:" .. err)
98 | ngx.exit(200)
99 | end
100 |
101 |
102 |
103 |
104 |
105 |
106 | Azure Wang (王非)azure1st@gmail.com
113 | 114 |This module is licensed under the BSD license.
121 | 122 |Copyright (C) 2013, by Azure Wang (王非) azure1st@gmail.com.
123 | 124 |All rights reserved.
125 | 126 |Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
127 | 128 |Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
131 |Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
134 |THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
136 | -------------------------------------------------------------------------------- /lib/resty/fastdfs/storage.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2013 Azure Wang 2 | local utils = require('resty.fastdfs.utils') 3 | local strip_string = utils.strip_string 4 | local fix_string = utils.fix_string 5 | local buf2int = utils.buf2int 6 | local int2buf = utils.int2buf 7 | local copy_sock = utils.copy_sock 8 | local read_fdfs_header = utils.read_fdfs_header 9 | local split_fileid = utils.split_fileid 10 | local tcp = ngx.socket.tcp 11 | local string = string 12 | local table = table 13 | local setmetatable = setmetatable 14 | local error = error 15 | 16 | module(...) 17 | 18 | local VERSION = '0.2.0' 19 | 20 | local FDFS_PROTO_PKG_LEN_SIZE = 8 21 | local FDFS_FILE_EXT_NAME_MAX_LEN = 6 22 | local FDFS_FILE_PREFIX_MAX_LEN = 16 23 | local FDFS_PROTO_CMD_QUIT = 82 24 | local STORAGE_PROTO_CMD_UPLOAD_FILE = 11 25 | local STORAGE_PROTO_CMD_DELETE_FILE = 12 26 | -- local STORAGE_PROTO_CMD_SET_METADATA = 13 27 | local STORAGE_PROTO_CMD_DOWNLOAD_FILE = 14 28 | -- local STORAGE_PROTO_CMD_GET_METADATA = 15 29 | local STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE = 21 30 | local STORAGE_PROTO_CMD_QUERY_FILE_INFO = 22 31 | local STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE = 23 32 | local STORAGE_PROTO_CMD_APPEND_FILE = 24 33 | local STORAGE_PROTO_CMD_MODIFY_FILE = 34 34 | local STORAGE_PROTO_CMD_TRUNCATE_FILE = 36 35 | 36 | local mt = { __index = _M } 37 | 38 | function new(self) 39 | local sock, err = tcp() 40 | if not sock then 41 | return nil, err 42 | end 43 | return setmetatable({ sock = sock }, mt) 44 | end 45 | 46 | function connect(self, opts) 47 | local sock = self.sock 48 | if not sock then 49 | return nil, "not initialized" 50 | end 51 | 52 | self.group_name = opts.group_name 53 | self.store_path_index = opts.store_path_index 54 | 55 | local host = opts.host 56 | local port = opts.port or 23000 57 | local ok, err = sock:connect(host, port) 58 | if not ok then 59 | return nil, err 60 | end 61 | return true 62 | end 63 | 64 | function send_request(self, req, data_sock, size) 65 | local sock = self.sock 66 | if not sock then 67 | return nil, "not initialized" 68 | end 69 | local bytes, err = sock:send(req) 70 | if not bytes then 71 | return nil, "storage send request error:" .. err 72 | end 73 | if data_sock and size then 74 | local ok, err = copy_sock(data_sock, sock, size) 75 | if not ok then 76 | return nil, "storate send data by sock error:" .. err 77 | end 78 | end 79 | return true 80 | end 81 | 82 | function read_upload_result(self) 83 | local sock = self.sock 84 | if not sock then 85 | return nil, "not initialized" 86 | end 87 | -- read request header 88 | local hdr, err = read_fdfs_header(sock) 89 | if not hdr then 90 | return nil, "read storage header error:" .. err 91 | end 92 | if hdr.status ~= 0 then 93 | return nil, "read storage status error:" .. hdr.status 94 | end 95 | if hdr.len > 0 and hdr.status == 0 then 96 | local res = {} 97 | local buf = sock:receive(hdr.len) 98 | res.group_name = strip_string(string.sub(buf, 1, 16)) 99 | res.file_name = strip_string(string.sub(buf, 17, hdr.len)) 100 | return res 101 | else 102 | return nil, "upload fail:" .. hdr.status 103 | end 104 | end 105 | 106 | function read_update_result(self, op_name) 107 | local sock = self.sock 108 | if not sock then 109 | return nil, "not initialized" 110 | end 111 | local hdr, err = read_fdfs_header(sock) 112 | if not hdr then 113 | return nil, "read storage header error:" .. err 114 | end 115 | if hdr.status == 0 then 116 | return true 117 | else 118 | return nil, op_name .. " error:" .. hdr.status 119 | end 120 | end 121 | 122 | function read_download_result(self) 123 | local sock = self.sock 124 | if not sock then 125 | return nil, "not initialized" 126 | end 127 | local hdr, err = read_fdfs_header(sock) 128 | if not hdr then 129 | return nil, "read storage header error:" .. err 130 | end 131 | if hdr.status ~= 0 then 132 | return nil, "read storage status error:" .. hdr.status 133 | end 134 | if hdr.len > 0 then 135 | local data, err, partial = sock:receive(hdr.len) 136 | if not data then 137 | return nil, "read file body error:" .. err 138 | end 139 | return data 140 | end 141 | return '' 142 | end 143 | 144 | function read_download_result_cb(self, cb) 145 | local sock = self.sock 146 | if not sock then 147 | return nil, "not initialized" 148 | end 149 | -- read request header 150 | local hdr, err = read_fdfs_header(sock) 151 | if not hdr then 152 | return nil, "read storage header error:" .. err 153 | end 154 | if hdr.status ~= 0 then 155 | return nil, "read storage status error:" .. hdr.status 156 | end 157 | local buff_size = 1024 * 16 158 | local read_size = 0 159 | local remain = hdr.len 160 | local out_buf = {} 161 | while remain > 0 do 162 | if remain > buff_size then 163 | read_size = buff_size 164 | remain = remain - read_size 165 | else 166 | read_size = remain 167 | remain = 0 168 | end 169 | local data, err, partial = sock:receive(read_size) 170 | if not data then 171 | return nil, "read data error:" .. err 172 | end 173 | cb(data) 174 | end 175 | return true 176 | end 177 | 178 | -- build upload method 179 | local function build_upload_request(cmd, size, ext, path_index) 180 | local req = {} 181 | table.insert(req, int2buf(size + 15)) -- length 182 | table.insert(req, string.char(cmd)) -- command 183 | table.insert(req, "\00") -- status 184 | table.insert(req, string.char(path_index)) 185 | table.insert(req, int2buf(size)) 186 | table.insert(req, fix_string(ext, FDFS_FILE_EXT_NAME_MAX_LEN)) 187 | return req 188 | end 189 | 190 | -- upload method 191 | function upload_by_buff(self, buff, ext) 192 | local size = string.len(buff) 193 | -- build request 194 | local req = build_upload_request(STORAGE_PROTO_CMD_UPLOAD_FILE, size, ext, self.store_path_index) 195 | table.insert(req, buff) 196 | -- send 197 | local ok, err = self:send_request(req) 198 | if not ok then 199 | return nil, err 200 | end 201 | return self:read_upload_result() 202 | end 203 | 204 | function upload_by_sock(self, sock, size, ext) 205 | -- build request 206 | local req = build_upload_request(STORAGE_PROTO_CMD_UPLOAD_FILE, size, ext, self.store_path_index) 207 | -- send 208 | local ok, err = self:send_request(req, sock, size) 209 | if not ok then 210 | return nil, err 211 | end 212 | return self:read_upload_result() 213 | end 214 | 215 | -- uoload_appender method 216 | function upload_appender_by_buff(self, buff, ext) 217 | local size = string.len(buff) 218 | -- build request 219 | local req = build_upload_request(STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, size, ext, self.store_path_index) 220 | table.insert(req, buff) 221 | -- send 222 | local ok, err = self:send_request(req) 223 | if not ok then 224 | return nil, err 225 | end 226 | return self:read_upload_result() 227 | end 228 | 229 | function upload_appender_by_sock(self, sock, size, ext) 230 | -- build request 231 | local req = build_upload_request(STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, size, ext, self.store_path_index) 232 | -- send 233 | local ok, err = self:send_request(req, sock, size) 234 | if not ok then 235 | return nil, err 236 | end 237 | return self:read_upload_result() 238 | end 239 | 240 | -- build upload_slave_request method 241 | local function build_upload_slave_request(cmd, file_name, prefix, size , ext) 242 | local req = {} 243 | table.insert(req, int2buf(16 + FDFS_FILE_PREFIX_MAX_LEN + FDFS_FILE_EXT_NAME_MAX_LEN + string.len(file_name) + size)) 244 | table.insert(req, string.char(cmd)) 245 | table.insert(req, "\00") 246 | table.insert(req, int2buf(string.len(file_name))) 247 | table.insert(req, int2buf(size)) 248 | table.insert(req, fix_string(prefix, FDFS_FILE_PREFIX_MAX_LEN)) 249 | table.insert(req, fix_string(ext, FDFS_FILE_EXT_NAME_MAX_LEN)) 250 | table.insert(req, file_name) 251 | return req 252 | end 253 | 254 | -- upload_slave method 255 | function upload_slave_by_buff(self, file_name, prefix, buff, ext) 256 | local size = string.len(buff) 257 | if not ext then 258 | ext = string.match(file_name, "%.(%w+)$") 259 | end 260 | -- build request 261 | local req = build_upload_slave_request(STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, file_name, prefix, size, ext) 262 | table.insert(req, buff) 263 | -- send 264 | local ok, err = self:send_request(req) 265 | if not ok then 266 | return nil, err 267 | end 268 | return self:read_upload_result() 269 | end 270 | 271 | function upload_slave_by_buff1(self, fileid, prefix, buff, ext) 272 | local group_name, file_name, err = split_fileid(fileid) 273 | if not group_name or not file_name then 274 | return nil, "fileid error:" .. err 275 | end 276 | return self:upload_slave_by_buff(file_name, prefix, buff, ext) 277 | end 278 | 279 | function upload_slave_by_sock(self, file_name, prefix, sock, size, ext) 280 | if not ext then 281 | ext = string.match(file_name, "%.(%w+)$") 282 | end 283 | local req = build_upload_slave_request(STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, file_name, prefix, size, ext) 284 | -- send 285 | local ok, err = self:send_request(req, sock, size) 286 | if not ok then 287 | return nil, err 288 | end 289 | return self:read_upload_result() 290 | end 291 | 292 | function upload_slave_by_sock1(self, fileid, prefix, sock, size, ext) 293 | local group_name, file_name, err = split_fileid(fileid) 294 | if not group_name or not file_name then 295 | return nil, "fileid error:" .. err 296 | end 297 | return self:upload_slave_by_sock(file_name, prefix, sock, size, ext) 298 | end 299 | 300 | -- build delete request 301 | local function build_delete_request(cmd, group_name, file_name) 302 | if not group_name then 303 | return nil , "not group_name" 304 | end 305 | if not file_name then 306 | return nil, "not file_name" 307 | end 308 | local req = {} 309 | table.insert(req, int2buf(16 + string.len(file_name))) 310 | table.insert(req, string.char(cmd)) 311 | table.insert(req, "\00") 312 | table.insert(req, fix_string(group_name, 16)) 313 | table.insert(req, file_name) 314 | return req 315 | end 316 | 317 | -- delete method 318 | function delete_file(self, group_name, file_name) 319 | local req, err = build_delete_request(STORAGE_PROTO_CMD_DELETE_FILE, group_name, file_name) 320 | if not req then 321 | return nil, err 322 | end 323 | -- send request 324 | local ok, err = self:send_request(req) 325 | if not ok then 326 | return nil, err 327 | end 328 | return self:read_update_result("delete_file") 329 | end 330 | 331 | function delete_file1(self, fileid) 332 | local group_name, file_name, err = split_fileid(fileid) 333 | if not group_name or not file_name then 334 | return nil, "fileid error:" .. err 335 | end 336 | return self:delete_file(group_name, file_name) 337 | end 338 | 339 | -- build truncate method 340 | local function build_truncate_request(cmd, file_name, remain_bytes) 341 | if not file_name then 342 | return nil, "not file_name" 343 | end 344 | local file_name_len = string.len(file_name) 345 | local req = {} 346 | table.insert(req, int2buf(16 + file_name_len)) 347 | table.insert(req, string.char(cmd)) 348 | table.insert(req, "\00") 349 | table.insert(req, int2buf(file_name_len)) 350 | table.insert(req, int2buf(remain_bytes)) 351 | table.insert(req, file_name) 352 | return req 353 | end 354 | -- truncate method 355 | function truncate_file(self, file_name) 356 | local req, err = build_truncate_request(STORAGE_PROTO_CMD_TRUNCATE_FILE, file_name, 0) 357 | if not req then 358 | return nil, err 359 | end 360 | -- send request 361 | local ok, err = self:send_request(req) 362 | if not ok then 363 | return nil, err 364 | end 365 | return self:read_update_result("truncate_file") 366 | end 367 | 368 | function truncate_file1(self, fileid) 369 | local group_name, file_name, err = split_fileid(fileid) 370 | if not group_name or not file_name then 371 | return nil, "fileid error:" .. err 372 | end 373 | return self:truncate_file(file_name) 374 | end 375 | 376 | -- build download request 377 | local function build_download_request(cmd, group_name, file_name) 378 | if not group_name then 379 | return nil , "not group_name" 380 | end 381 | if not file_name then 382 | return nil, "not file_name" 383 | end 384 | local req = {} 385 | table.insert(req, int2buf(32 + string.len(file_name))) 386 | table.insert(req, string.char(cmd)) 387 | table.insert(req, "\00") 388 | table.insert(req, string.rep("\00", 16)) -- download file offset 389 | table.insert(req, fix_string(group_name, 16)) 390 | table.insert(req, file_name) 391 | return req 392 | end 393 | 394 | -- download method 395 | function download_file_to_buff(self, group_name, file_name) 396 | local req, err = build_download_request(STORAGE_PROTO_CMD_DOWNLOAD_FILE, group_name, file_name) 397 | if not req then 398 | return nil, err 399 | end 400 | -- send 401 | local ok, err = self:send_request(req) 402 | if not ok then 403 | return nil, err 404 | end 405 | return self:read_download_result() 406 | end 407 | 408 | function download_file_to_buff1(self, fileid) 409 | local group_name, file_name, err = split_fileid(fileid) 410 | if not group_name or not file_name then 411 | return nil, "fileid error:" .. err 412 | end 413 | return self:download_file_to_buff(group_name, file_name) 414 | end 415 | 416 | function download_file_to_callback(self,group_name, file_name, cb) 417 | local req, err = build_download_request(STORAGE_PROTO_CMD_DOWNLOAD_FILE, group_name, file_name) 418 | if not req then 419 | return nil, err 420 | end 421 | -- send 422 | local ok, err = self:send_request(req) 423 | if not ok then 424 | return nil, err 425 | end 426 | return self:read_download_result_cb(cb) 427 | end 428 | 429 | function download_file_to_callback1(self, fileid, cb) 430 | local group_name, file_name, err = split_fileid(fileid) 431 | if not group_name or not file_name then 432 | return nil, "fileid error:" .. err 433 | end 434 | return self:download_file_to_callback(group_name, file_name, cb) 435 | end 436 | 437 | -- build append request 438 | local function build_append_request(cmd, group_name, file_name, size) 439 | if not group_name then 440 | return nil , "not group_name" 441 | end 442 | if not file_name then 443 | return nil, "not file_name" 444 | end 445 | local file_name_len = string.len(file_name) 446 | local req = {} 447 | table.insert(req, int2buf(16 + size + file_name_len)) 448 | table.insert(req, string.char(cmd)) 449 | table.insert(req, "\00") 450 | table.insert(req, int2buf(file_name_len)) 451 | table.insert(req, int2buf(size)) 452 | table.insert(req, file_name) 453 | return req 454 | end 455 | 456 | -- append method 457 | function append_by_buff(self, group_name, file_name, buff) 458 | local size = string.len(buff) 459 | local req, err = build_append_request(STORAGE_PROTO_CMD_APPEND_FILE, group_name, file_name, size) 460 | if not req then 461 | return nil, err 462 | end 463 | table.insert(req, buff) 464 | -- send request 465 | local ok, err = self:send_request(req) 466 | if not ok then 467 | return nil, err 468 | end 469 | return self:read_update_result("append_by_buff") 470 | end 471 | 472 | function append_by_buff1(self, fileid, buff) 473 | local group_name, file_name, err = split_fileid(fileid) 474 | if not group_name or not file_name then 475 | return nil, "fileid error:" .. err 476 | end 477 | return self:append_by_buff(group_name, file_name, buff) 478 | end 479 | 480 | function append_by_sock(self, group_name, file_name, sock, size) 481 | local req, err = build_append_request(STORAGE_PROTO_CMD_APPEND_FILE, group_name, file_name, size) 482 | if not req then 483 | return nil, err 484 | end 485 | -- send data 486 | local ok, err = self:send_request(req, sock, size) 487 | if not ok then 488 | return nil, err 489 | end 490 | return self:read_update_result("append_by_sock") 491 | end 492 | 493 | function append_by_sock1(self, fileid, sock, size) 494 | local group_name, file_name, err = split_fileid(fileid) 495 | if not group_name or not file_name then 496 | return nil, "fileid error:" .. err 497 | end 498 | return self:append_by_sock(group_name, file_name, sock, size) 499 | end 500 | 501 | -- build modify request 502 | local function build_modify_request(cmd, file_name, offset, size) 503 | if not file_name then 504 | return nil, "not file_name" 505 | end 506 | local file_name_len = string.len(file_name) 507 | local req = {} 508 | table.insert(req, int2buf(size + file_name_len + 24)) 509 | table.insert(req, string.char(STORAGE_PROTO_CMD_MODIFY_FILE)) 510 | table.insert(req, "\00") 511 | table.insert(req, int2buf(file_name_len)) 512 | table.insert(req, int2buf(offset)) 513 | table.insert(req, int2buf(size)) 514 | table.insert(req, file_name) 515 | return req 516 | end 517 | 518 | -- modify method 519 | function modify_by_buff(self, file_name, buff, offset) 520 | local size = string.len(buff) 521 | local req, err = build_modify_request(STORAGE_PROTO_CMD_MODIFY_FILE, file_name, offset, size) 522 | table.insert(req, buff) 523 | if not req then 524 | return nil, err 525 | end 526 | -- send request 527 | local ok, err = self:send_request(req) 528 | if not ok then 529 | return nil, err 530 | end 531 | return self:read_update_result("modify_by_buff") 532 | end 533 | 534 | function modify_by_buff1(self, fileid, buff, offset) 535 | local group_name, file_name, err = split_fileid(fileid) 536 | if not group_name or not file_name then 537 | return nil, "fileid error:" .. err 538 | end 539 | return self:modify_by_buff(file_name, buff, offset) 540 | end 541 | 542 | function modify_by_sock(self, file_name, sock, size, offset) 543 | local req, err = build_modify_request(STORAGE_PROTO_CMD_MODIFY_FILE, file_name, offset, size) 544 | if not req then 545 | return nil, err 546 | end 547 | -- send data 548 | local ok, err = self:send_request(req, sock, size) 549 | if not ok then 550 | return nil, err 551 | end 552 | return self:read_update_result("modify_by_sock") 553 | end 554 | 555 | function modify_by_sock1(self, fileid, sock, size, offset) 556 | local group_name, file_name, err = split_fileid(fileid) 557 | if not group_name or not file_name then 558 | return nil, "fileid error:" .. err 559 | end 560 | return self:modify_by_sock(file_name, sock, size, offset) 561 | end 562 | -- set variavle method 563 | function set_timeout(self, timeout) 564 | local sock = self.sock 565 | if not sock then 566 | return nil, "not initialized" 567 | end 568 | return sock:settimeout(timeout) 569 | end 570 | 571 | function set_keepalive(self, ...) 572 | local sock = self.sock 573 | if not sock then 574 | return nil, "not initialized" 575 | end 576 | return sock:setkeepalive(...) 577 | end 578 | 579 | local class_mt = { 580 | -- to prevent use of casual module global variables 581 | __newindex = function (table, key, val) 582 | error('attempt to write to undeclared variable "' .. key .. '"') 583 | end 584 | } 585 | 586 | setmetatable(_M, class_mt) 587 | -------------------------------------------------------------------------------- /lib/resty/fastdfs/tracker.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2013 Azure Wang 2 | local utils = require('resty.fastdfs.utils') 3 | local tcp = ngx.socket.tcp 4 | local setmetatable = setmetatable 5 | local error = error 6 | local strip_string = utils.strip_string 7 | local fix_string = utils.fix_string 8 | local buf2int = utils.buf2int 9 | local int2buf = utils.int2buf 10 | local read_int = utils.read_int 11 | local read_fdfs_header = utils.read_fdfs_header 12 | local string = string 13 | local table = table 14 | local date = os.date 15 | 16 | module(...) 17 | 18 | local VERSION = '0.2.1' 19 | 20 | local FDFS_PROTO_PKG_LEN_SIZE = 8 21 | local FDFS_FILE_EXT_NAME_MAX_LEN = 6 22 | local FDFS_PROTO_CMD_QUIT = 82 23 | local TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP = 90 24 | local TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS = 91 25 | local TRACKER_PROTO_CMD_SERVER_LIST_STORAGE = 92 26 | local TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE = 101 27 | local TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE = 102 28 | local TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE = 103 29 | local TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE = 104 30 | local TRACKER_PROTO_CMD_RESP = 100 31 | 32 | local FDFS_STORAGE_STATUS_INIT = 0 33 | local FDFS_STORAGE_STATUS_WAIT_SYNC = 1 34 | local FDFS_STORAGE_STATUS_SYNCING = 2 35 | local FDFS_STORAGE_STATUS_IP_CHANGED = 3 36 | local FDFS_STORAGE_STATUS_DELETED = 4 37 | local FDFS_STORAGE_STATUS_OFFLINE = 5 38 | local FDFS_STORAGE_STATUS_ONLINE = 6 39 | local FDFS_STORAGE_STATUS_ACTIVE = 7 40 | local FDFS_STORAGE_STATUS_RECOVERY = 9 41 | 42 | local storage_status = {} 43 | storage_status[FDFS_STORAGE_STATUS_INIT] = "INIT" 44 | storage_status[FDFS_STORAGE_STATUS_WAIT_SYNC] = "WAIT_SYNC" 45 | storage_status[FDFS_STORAGE_STATUS_SYNCING] = "SYNCING" 46 | storage_status[FDFS_STORAGE_STATUS_IP_CHANGED] = "IP_CHANGED" 47 | storage_status[FDFS_STORAGE_STATUS_DELETED] = "DELETED" 48 | storage_status[FDFS_STORAGE_STATUS_OFFLINE] = "OFFLINE" 49 | storage_status[FDFS_STORAGE_STATUS_ONLINE] = "ONLINE" 50 | storage_status[FDFS_STORAGE_STATUS_ACTIVE] = "ACTIVE" 51 | storage_status[FDFS_STORAGE_STATUS_RECOVERY] = "RECOVERY" 52 | 53 | local mt = { __index = _M } 54 | 55 | local function format_time(t) 56 | if t <= 0 then 57 | return '' 58 | end 59 | return date("%Y-%m-%d %H:%M:%S", t) 60 | end 61 | 62 | function new(self) 63 | local sock, err = tcp() 64 | if not sock then 65 | return nil, err 66 | end 67 | return setmetatable({ sock = sock }, mt) 68 | end 69 | 70 | function connect(self, opts) 71 | local sock = self.sock 72 | if not sock then 73 | return nil, "not initialized" 74 | end 75 | local host = opts.host 76 | local port = opts.port or 22122 77 | local ok, err = sock:connect(host, port) 78 | if not ok then 79 | return nil, err 80 | end 81 | return 1 82 | end 83 | 84 | function query_storage_store(self, group_name) 85 | local sock = self.sock 86 | if not sock then 87 | return nil, "not initialized" 88 | end 89 | local out = {} 90 | if group_name then 91 | -- query upload with group_name 92 | -- package length 93 | table.insert(out, int2buf(16)) 94 | -- cmd 95 | table.insert(out, string.char(TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE)) 96 | -- status 97 | table.insert(out, "\00") 98 | -- group name 99 | table.insert(out, fix_string(group_name, 16)) 100 | else 101 | -- query upload without group_name 102 | -- package length 103 | table.insert(out, string.rep("\00", FDFS_PROTO_PKG_LEN_SIZE)) 104 | -- cmd 105 | table.insert(out, string.char(TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE)) 106 | -- status 107 | table.insert(out, "\00") 108 | end 109 | -- send request 110 | local bytes, err = sock:send(out) 111 | if not bytes then 112 | return nil, "tracker send request error:" .. err 113 | end 114 | -- read request header 115 | local hdr, err = read_fdfs_header(sock) 116 | if not hdr then 117 | return nil, "read tracker header error:" .. err 118 | end 119 | -- read data 120 | if hdr.len > 0 then 121 | local res = {} 122 | local buf = sock:receive(hdr.len) 123 | res.group_name = strip_string(string.sub(buf, 1, 16)) 124 | res.host = strip_string(string.sub(buf, 17, 31)) 125 | res.port = buf2int(string.sub(buf, 32, 39)) 126 | res.store_path_index = string.byte(string.sub(buf, 40, 40)) 127 | return res 128 | else 129 | return nil, "not receive data" 130 | end 131 | end 132 | 133 | function query_storage_update(self, group_name, file_name) 134 | local sock = self.sock 135 | if not sock then 136 | return nil, "not initialized" 137 | end 138 | local out = {} 139 | -- package length 140 | table.insert(out, int2buf(16 + string.len(file_name))) 141 | -- cmd 142 | table.insert(out, string.char(TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE)) 143 | -- status 144 | table.insert(out, "\00") 145 | -- group_name 146 | table.insert(out, fix_string(group_name, 16)) 147 | -- file name 148 | table.insert(out, file_name) 149 | -- send request 150 | local bytes, err = sock:send(out) 151 | if not bytes then 152 | return nil, "tracker send request error:" .. err 153 | end 154 | -- read request header 155 | local hdr, err = read_fdfs_header(sock) 156 | if not hdr then 157 | return nil, "read tracker header error:" .. err 158 | end 159 | -- read data 160 | if hdr.len > 0 then 161 | local res = {} 162 | local buf = sock:receive(hdr.len) 163 | res.group_name = strip_string(string.sub(buf, 1, 16)) 164 | res.host = strip_string(string.sub(buf, 17, 31)) 165 | res.port = buf2int(string.sub(buf, 32, 39)) 166 | return res 167 | else 168 | return nil, "not receive data" 169 | end 170 | end 171 | 172 | function query_storage_fetch(self, group_name, file_name) 173 | local sock = self.sock 174 | if not sock then 175 | return nil, "not initialized" 176 | end 177 | local out = {} 178 | -- package length 179 | table.insert(out, int2buf(16 + string.len(file_name))) 180 | -- cmd 181 | table.insert(out, string.char(TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE)) 182 | -- status 183 | table.insert(out, "\00") 184 | -- group_name 185 | table.insert(out, fix_string(group_name, 16)) 186 | -- file name 187 | table.insert(out, file_name) 188 | -- send request 189 | local bytes, err = sock:send(out) 190 | if not bytes then 191 | return nil, "tracker send request error:" .. err 192 | end 193 | -- read request header 194 | local hdr, err = read_fdfs_header(sock) 195 | if not hdr then 196 | return nil, "read tracker header error:" .. err 197 | end 198 | -- read data 199 | if hdr.len > 0 then 200 | local res = {} 201 | local buf = sock:receive(hdr.len) 202 | res.group_name = strip_string(string.sub(buf, 1, 16)) 203 | res.host = strip_string(string.sub(buf, 17, 31)) 204 | res.port = buf2int(string.sub(buf, 32, 39)) 205 | return res 206 | else 207 | return nil, "error no:" .. hdr.status 208 | end 209 | end 210 | 211 | function query_storage_update1(self, fileid) 212 | local group_name, file_name, err = utils.split_fileid(fileid) 213 | if not group_name or not file_name then 214 | return nil, "fileid error:" .. err 215 | end 216 | return self:query_storage_update(group_name, file_name) 217 | end 218 | 219 | function query_storage_fetch1(self, fileid) 220 | local group_name, file_name, err = utils.split_fileid(fileid) 221 | if not group_name or not file_name then 222 | return nil, "fileid error:" .. err 223 | end 224 | return self:query_storage_fetch(group_name, file_name) 225 | end 226 | 227 | function list_groups(self) 228 | local sock = self.sock 229 | if not sock then 230 | return nil, "not initialized" 231 | end 232 | local out = {} 233 | -- body length 234 | table.insert(out, string.rep("\00", FDFS_PROTO_PKG_LEN_SIZE)) 235 | -- cmd 236 | table.insert(out, string.char(TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS)) 237 | -- status 238 | table.insert(out, "\00") 239 | -- send request 240 | local bytes, err = sock:send(out) 241 | if not bytes then 242 | return nil, "tracker send request error:" .. err 243 | end 244 | -- read request header 245 | local hdr, err = read_fdfs_header(sock) 246 | if not hdr then 247 | return nil, "read tracker header error:" .. err 248 | end 249 | local body_len 250 | if self.v4 then 251 | body_len = 105 252 | else 253 | body_len = 97 254 | end 255 | -- read body 256 | if hdr.len > 0 then 257 | if hdr.len % body_len ~= 0 then 258 | return nil, "response body error" 259 | end 260 | local body, err, part = sock:receive(hdr.len) 261 | if not body then 262 | return nil, "read response body error" 263 | end 264 | local res = {} 265 | res.count = hdr.len / body_len 266 | res.groups = {} 267 | for i=1, res.count do 268 | local group = {} 269 | local pos = body_len * (i-1) + 1 270 | group.group_name = strip_string(string.sub(body, pos, pos + 16)) 271 | pos = pos + 17 272 | if self.v4 then 273 | group.total_mb, pos = read_int(body, pos) 274 | end 275 | group.free_mb, pos = read_int(body, pos) 276 | group.trunk_free_mb, pos = read_int(body, pos) 277 | group.count, pos = read_int(body, pos) 278 | group.storage_port, pos = read_int(body, pos) 279 | group.storage_http_port, pos = read_int(body, pos) 280 | group.active_count, pos = read_int(body, pos) 281 | group.current_write_server, pos = read_int(body, pos) 282 | group.store_path_count, pos = read_int(body, pos) 283 | group.subdir_count_per_path, pos = read_int(body, pos) 284 | group.current_trunk_file_id, pos = read_int(body, pos) 285 | table.insert(res.groups, group) 286 | end 287 | return res 288 | else 289 | return nil, "response body is empty" 290 | end 291 | end 292 | 293 | function list_one_group(self, group_name) 294 | local sock = self.sock 295 | if not sock then 296 | return nil, "not initialized" 297 | end 298 | local out = {} 299 | -- body length 300 | table.insert(out, int2buf(16)) 301 | -- cmd 302 | table.insert(out, string.char(TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP)) 303 | -- status 304 | table.insert(out, "\00") 305 | -- group_name 306 | table.insert(out, fix_string(group_name, 16)) 307 | -- send request 308 | local bytes, err = sock:send(out) 309 | if not bytes then 310 | return nil, "tracker send request error:" .. err 311 | end 312 | -- read request header 313 | local hdr, err = read_fdfs_header(sock) 314 | if not hdr then 315 | return nil, "read tracker header error:" .. err 316 | end 317 | local body_len 318 | if self.v4 then 319 | body_len = 105 320 | else 321 | body_len = 97 322 | end 323 | -- read body 324 | if hdr.len > 0 then 325 | if hdr.len ~= body_len then 326 | return nil, "response body length error" 327 | end 328 | local body, err, part = sock:receive(hdr.len) 329 | if not body then 330 | return nil, "read response body error" 331 | end 332 | local group = {} 333 | local pos = 1 334 | group.group_name = strip_string(string.sub(body, pos, pos + 16)) 335 | pos = pos + 17 336 | if self.v4 then 337 | group.total_mb, pos = read_int(body, pos) 338 | end 339 | group.free_mb, pos = read_int(body, pos) 340 | group.trunk_free_mb, pos = read_int(body, pos) 341 | group.count, pos = read_int(body, pos) 342 | group.storage_port, pos = read_int(body, pos) 343 | group.storage_http_port, pos = read_int(body, pos) 344 | group.active_count, pos = read_int(body, pos) 345 | group.current_write_server, pos = read_int(body, pos) 346 | group.store_path_count, pos = read_int(body, pos) 347 | group.subdir_count_per_path, pos = read_int(body, pos) 348 | group.current_trunk_file_id, pos = read_int(body, pos) 349 | return group 350 | else 351 | return nil, "response body is empty" 352 | end 353 | end 354 | 355 | function list_servers(self, group_name) 356 | local sock = self.sock 357 | if not sock then 358 | return nil, "not initialized" 359 | end 360 | local out = {} 361 | -- body length 362 | table.insert(out, int2buf(16)) 363 | -- cmd 364 | table.insert(out, string.char(TRACKER_PROTO_CMD_SERVER_LIST_STORAGE)) 365 | -- status 366 | table.insert(out, "\00") 367 | -- group_name 368 | table.insert(out, fix_string(group_name, 16)) 369 | -- send request 370 | local bytes, err = sock:send(out) 371 | if not bytes then 372 | return nil, "tracker send request error:" .. err 373 | end 374 | -- read request header 375 | local hdr, err = read_fdfs_header(sock) 376 | if not hdr then 377 | return nil, "read tracker header error:" .. err 378 | end 379 | if hdr.len > 0 then 380 | local body_len 381 | if self.v4 then 382 | body_len = 600 383 | else 384 | body_len = 584 385 | end 386 | if hdr.len % body_len ~= 0 then 387 | return nil, "response body length error" 388 | end 389 | local body, err, part = sock:receive(hdr.len) 390 | if not body then 391 | return nil, "read response body error" 392 | end 393 | local res = {} 394 | res.count = hdr.len / body_len 395 | res.servers = {} 396 | for i=1, res.count do 397 | local server = {} 398 | local pos = body_len * (i-1) + 1 399 | server.status = storage_status[string.byte(body, pos)] or "UNKNOW" 400 | if self.v4 then 401 | server.id = strip_string(string.sub(body, pos+1 , pos+16)) 402 | server.ip_addr = strip_string(string.sub(body, pos+17 , pos+32)) 403 | server.domain_name = strip_string(string.sub(body, pos+33 , pos+160)) 404 | server.src_id = strip_string(string.sub(body, pos+161, pos+176)) 405 | pos = pos + 176 + 1 406 | else 407 | server.ip_addr = strip_string(string.sub(body, pos+1 , pos+16)) 408 | server.domain_name = strip_string(string.sub(body, pos+17 , pos+144)) 409 | server.src_ip_addr = strip_string(string.sub(body, pos+145, pos+160)) 410 | pos = pos + 160 + 1 411 | end 412 | server.version = strip_string(string.sub(body, pos, pos+5)) 413 | pos = pos + 6 414 | local tmp 415 | tmp, pos = read_int(body, pos) 416 | server.join_time = format_time(tmp) 417 | tmp, pos = read_int(body, pos) 418 | server.up_time = format_time(tmp) 419 | server.total_mb, pos = read_int(body, pos) 420 | server.free_mb, pos = read_int(body, pos) 421 | server.upload_priority, pos = read_int(body, pos) 422 | server.store_path_count, pos = read_int(body, pos) 423 | server.subdir_count_per_path, pos = read_int(body, pos) 424 | server.current_write_path, pos = read_int(body, pos) 425 | server.storage_port, pos = read_int(body, pos) 426 | server.storage_http_port, pos = read_int(body, pos) 427 | -- FDFSStorageStatBuff 428 | server.total_upload_count, pos = read_int(body, pos) 429 | server.success_upload_count, pos = read_int(body, pos) 430 | server.total_append_count, pos = read_int(body, pos) 431 | server.success_append_count, pos = read_int(body, pos) 432 | server.total_modify_count, pos = read_int(body, pos) 433 | server.success_modify_count, pos = read_int(body, pos) 434 | server.total_truncate_count, pos = read_int(body, pos) 435 | server.success_truncate_count, pos = read_int(body, pos) 436 | server.total_set_meta_count, pos = read_int(body, pos) 437 | server.success_set_meta_count, pos = read_int(body, pos) 438 | server.total_delete_count, pos = read_int(body, pos) 439 | server.success_delete_count, pos = read_int(body, pos) 440 | server.total_download_count, pos = read_int(body, pos) 441 | server.success_download_count, pos = read_int(body, pos) 442 | server.total_get_meta_count, pos = read_int(body, pos) 443 | server.success_get_meta_count, pos = read_int(body, pos) 444 | server.total_create_link_count, pos = read_int(body, pos) 445 | server.success_create_link_count, pos = read_int(body, pos) 446 | server.total_delete_link_count, pos = read_int(body, pos) 447 | server.success_delete_link_count, pos = read_int(body, pos) 448 | server.total_upload_bytes, pos = read_int(body, pos) 449 | server.success_upload_bytes, pos = read_int(body, pos) 450 | server.total_append_bytes, pos = read_int(body, pos) 451 | server.success_append_bytes, pos = read_int(body, pos) 452 | server.total_modify_bytes, pos = read_int(body, pos) 453 | server.success_modify_bytes, pos = read_int(body, pos) 454 | server.total_download_bytes, pos = read_int(body, pos) 455 | server.success_download_bytes, pos = read_int(body, pos) 456 | server.total_sync_in_bytes, pos = read_int(body, pos) 457 | server.success_sync_in_bytes, pos = read_int(body, pos) 458 | server.total_sync_out_bytes, pos = read_int(body, pos) 459 | server.success_sync_out_bytes, pos = read_int(body, pos) 460 | server.total_file_open_count, pos = read_int(body, pos) 461 | server.success_file_open_count, pos = read_int(body, pos) 462 | server.total_file_read_count, pos = read_int(body, pos) 463 | server.success_file_read_count, pos = read_int(body, pos) 464 | server.total_file_write_count, pos = read_int(body, pos) 465 | server.success_file_write_count, pos = read_int(body, pos) 466 | tmp, pos = read_int(body, pos) 467 | server.last_source_update = format_time(tmp) 468 | tmp, pos = read_int(body, pos) 469 | server.last_sync_update = format_time(tmp) 470 | tmp, pos = read_int(body, pos) 471 | server.last_synced_timestamp = format_time(tmp) 472 | tmp, pos = read_int(body, pos) 473 | server.last_heart_beat_time = format_time(tmp) 474 | server.if_trunk_server = string.byte(body, pos) 475 | table.insert(res.servers, server) 476 | end 477 | return res 478 | else 479 | return nil, "response body is empty" 480 | end 481 | end 482 | 483 | function set_v4(self, flg) 484 | self.v4 = flg 485 | end 486 | 487 | function set_timeout(self, timeout) 488 | local sock = self.sock 489 | if not sock then 490 | return nil, "not initialized" 491 | end 492 | return sock:settimeout(timeout) 493 | end 494 | 495 | function set_keepalive(self, ...) 496 | local sock = self.sock 497 | if not sock then 498 | return nil, "not initialized" 499 | end 500 | return sock:setkeepalive(...) 501 | end 502 | 503 | local class_mt = { 504 | -- to prevent use of casual module global variables 505 | __newindex = function (table, key, val) 506 | error('attempt to write to undeclared variable "' .. key .. '"') 507 | end 508 | } 509 | 510 | setmetatable(_M, class_mt) 511 | 512 | -------------------------------------------------------------------------------- /lib/resty/fastdfs/utils.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) 2013 Azure Wang 2 | local strfind = string.find 3 | local strsub = string.sub 4 | local strbyte = string.byte 5 | local strrep = string.rep 6 | local strchar = string.char 7 | local strrep = string.rep 8 | local band = bit.band 9 | local bor = bit.bor 10 | local rshift = bit.rshift 11 | local lshift = bit.lshift 12 | local strlen = string.len 13 | 14 | module(...) 15 | 16 | local VERSION = '0.1.2' 17 | 18 | function split_fileid(fileid) 19 | local pos = strfind(fileid, '/') 20 | if not pos then 21 | return nil, nil, "fileid not contain /" 22 | else 23 | local group_name = strsub(fileid, 1, pos-1) 24 | local file_name = strsub(fileid, pos + 1) 25 | return group_name, file_name 26 | end 27 | end 28 | 29 | function int2buf(n) 30 | -- only trans 32bit full is 64bit 31 | return strrep("\00", 4) .. strchar(band(rshift(n, 24), 0xff), band(rshift(n, 16), 0xff), band(rshift(n, 8), 0xff), band(n, 0xff)) 32 | end 33 | 34 | function buf2int(buf) 35 | local c1, c2, c3, c4, c5, c6, c7, c8 = strbyte(buf, 1, 8) 36 | local lo = bor(lshift(c5, 24), lshift(c6, 16),lshift(c7, 8), c8) 37 | local hi = bor(lshift(c1, 24), lshift(c2, 16),lshift(c3, 8), c4) 38 | return lo + hi * 4294967296 39 | end 40 | 41 | function fix_string(str, fix_length) 42 | if not str then 43 | return strrep("\00", fix_length) 44 | end 45 | local len = strlen(str) 46 | if len > fix_length then 47 | len = fix_length 48 | end 49 | local fix_str = strsub(str, 1, len) 50 | if len < fix_length then 51 | fix_str = fix_str .. strrep("\00", fix_length - len ) 52 | end 53 | return fix_str 54 | end 55 | 56 | function strip_string(str) 57 | local pos = strfind(str, "\00") 58 | if pos then 59 | return strsub(str, 1, pos - 1) 60 | else 61 | return str 62 | end 63 | end 64 | 65 | function read_int(buf, pos) 66 | return buf2int(strsub(buf, pos, pos + 7)), pos + 8 67 | end 68 | 69 | function read_fdfs_header(sock) 70 | local header = {} 71 | local buf, err = sock:receive(10) 72 | if not buf then 73 | return nil, "read fdfs header error:" .. err 74 | end 75 | header.len = buf2int(strsub(buf, 1, 8)) 76 | header.cmd = strbyte(buf, 9) 77 | header.status = strbyte(buf, 10) 78 | return header 79 | end 80 | 81 | function copy_sock(src, dst, size) 82 | local copy_count = 0 83 | local buff_size = 1024 * 32 84 | while true do 85 | local chunk, _, part = src:receive(buff_size) 86 | if not part then 87 | local bytes, err = dst:send(chunk) 88 | if not bytes then 89 | return nil, "copy sock send data error:" .. err 90 | end 91 | copy_count = copy_count + bytes 92 | else 93 | -- part have data, not read full end 94 | local bytes, err = dst:send(part) 95 | if not bytes then 96 | return nil, "copy sock send data error:" .. err 97 | end 98 | copy_count = copy_count + bytes 99 | break 100 | end 101 | end 102 | if copy_count ~= size then 103 | -- copy not full 104 | return nil, "copy sock not full" 105 | end 106 | return 1 107 | end 108 | --------------------------------------------------------------------------------