├── rebar ├── doc ├── clear_docs ├── overview.edoc ├── gen_docs └── Article1.md ├── rebar.config ├── ebin └── mongodb.app ├── src ├── mongodb_app.erl ├── resource_pool.erl ├── mongo_cursor.erl ├── mongo_query.erl ├── mongo_protocol.erl ├── mongo_connect.erl ├── mvar.erl ├── mongodb_tests.erl ├── mongo_replset.erl └── mongo.erl ├── include └── mongo_protocol.hrl └── README.md /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TonyGen/mongodb-erlang/HEAD/rebar -------------------------------------------------------------------------------- /doc/clear_docs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rm *.html edoc-info erlang.png stylesheet.css 3 | -------------------------------------------------------------------------------- /doc/overview.edoc: -------------------------------------------------------------------------------- 1 | @doc 2 | See ReadMe for overview. 3 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {deps, [ 2 | {bson, ".*", {git, "git://github.com/TonyGen/bson-erlang", "HEAD"}} 3 | ]}. 4 | 5 | {lib_dirs, ["deps"]}. 6 | 7 | {erl_opts, [debug_info, fail_on_warning]}. -------------------------------------------------------------------------------- /doc/gen_docs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Generate html docs from src code comments using edoc 3 | erl -pa ../../bson/ebin/ ../../mongodb/ebin -eval 'edoc:application (mongodb, [{index_columns, 1}, {sort_functions, false}, {preprocess, true}]), init:stop()' -noshell 4 | -------------------------------------------------------------------------------- /ebin/mongodb.app: -------------------------------------------------------------------------------- 1 | {application, mongodb, 2 | [{description, "Client interface to MongoDB, also known as the driver. See www.mongodb.org"}, 3 | {vsn, "0.2.1"}, 4 | {modules, [mongodb_app, mongo, mongo_protocol, mongo_connect, mongo_query, mongo_cursor, mvar, mongodb_tests, mongo_replset, resource_pool]}, 5 | {registered, []}, 6 | {applications, [kernel, stdlib]}, 7 | {mod, {mongodb_app, []}} 8 | ]}. 9 | -------------------------------------------------------------------------------- /src/mongodb_app.erl: -------------------------------------------------------------------------------- 1 | %@doc Init some internal global variables used by mongodb app 2 | -module (mongodb_app). 3 | 4 | -behaviour (application). 5 | -export ([start/2, stop/1]). 6 | 7 | -behaviour (supervisor). 8 | -export ([init/1]). 9 | 10 | -export ([gen_objectid/0, next_requestid/0]). % API 11 | 12 | %% Behaviour callbacks 13 | 14 | start (_, []) -> supervisor:start_link ({local, ?MODULE}, ?MODULE, []). 15 | 16 | stop (_) -> ok. 17 | 18 | %% Supervisor callbacks 19 | 20 | %@doc Create global vars which will be owned by this supervisor (and die with it) 21 | init ([]) -> 22 | ets:new (?MODULE, [named_table, public]), 23 | ets:insert (?MODULE, [ 24 | {oid_counter, 0}, 25 | {oid_machineprocid, oid_machineprocid()}, 26 | {requestid_counter, 0} ]), 27 | {ok, {{one_for_one,3,10}, []}}. 28 | 29 | %% API functions 30 | 31 | -spec next_requestid () -> mongo_protocol:requestid(). % IO 32 | %@doc Fresh request id 33 | next_requestid() -> ets:update_counter (?MODULE, requestid_counter, 1). 34 | 35 | -spec gen_objectid () -> bson:objectid(). % IO 36 | %@doc Fresh object id 37 | gen_objectid() -> 38 | Now = bson:unixtime_to_secs (bson:timenow()), 39 | MPid = ets:lookup_element (?MODULE, oid_machineprocid, 2), 40 | N = ets:update_counter (?MODULE, oid_counter, 1), 41 | bson:objectid (Now, MPid, N). 42 | 43 | -spec oid_machineprocid () -> <<_:40>>. % IO 44 | %@doc Fetch hostname and os pid and compress into a 5 byte id 45 | oid_machineprocid() -> 46 | OSPid = list_to_integer (os:getpid()), 47 | {ok, Hostname} = inet:gethostname(), 48 | <> = erlang:md5 (Hostname), 49 | <>. 50 | -------------------------------------------------------------------------------- /include/mongo_protocol.hrl: -------------------------------------------------------------------------------- 1 | % Wire protocol message types (records) 2 | 3 | -type db() :: atom(). 4 | 5 | -type collection() :: atom(). % without db prefix 6 | 7 | -type cursorid() :: integer(). 8 | 9 | -type selector() :: bson:document(). 10 | 11 | -record (insert, { 12 | collection :: collection(), 13 | documents :: [bson:document()] }). 14 | 15 | -record (update, { 16 | collection :: collection(), 17 | upsert = false :: boolean(), 18 | multiupdate = false :: boolean(), 19 | selector :: selector(), 20 | updater :: bson:document() | modifier() }). 21 | 22 | -type modifier() :: bson:document(). 23 | 24 | -record (delete, { 25 | collection :: collection(), 26 | singleremove = false :: boolean(), 27 | selector :: selector() }). 28 | 29 | -record (killcursor, { 30 | cursorids :: [cursorid()] }). 31 | 32 | -record ('query', { 33 | tailablecursor = false :: boolean(), 34 | slaveok = false :: boolean(), 35 | nocursortimeout = false :: boolean(), 36 | awaitdata = false :: boolean(), 37 | collection :: collection(), 38 | skip = 0 :: skip(), 39 | batchsize = 0 :: batchsize(), 40 | selector :: selector(), 41 | projector = [] :: projector() }). 42 | 43 | -type projector() :: bson:document(). 44 | -type skip() :: integer(). 45 | -type batchsize() :: integer(). % 0 = default batch size. negative closes cursor 46 | 47 | -record (getmore, { 48 | collection :: collection(), 49 | batchsize = 0 :: batchsize(), 50 | cursorid :: cursorid() }). 51 | 52 | -record (reply, { 53 | cursornotfound :: boolean(), 54 | queryerror :: boolean(), 55 | awaitcapable :: boolean(), 56 | cursorid :: cursorid(), 57 | startingfrom :: integer(), 58 | documents :: [bson:document()] }). 59 | -------------------------------------------------------------------------------- /src/resource_pool.erl: -------------------------------------------------------------------------------- 1 | %@doc A set of N resources handed out randomly, and recreated on expiration 2 | -module (resource_pool). 3 | 4 | -export_type ([factory/1, create/1, expire/1, is_expired/1]). 5 | -export_type ([pool/1]). 6 | -export ([new/2, get/1, close/1, is_closed/1]). 7 | 8 | -type maybe(A) :: {A} | {}. 9 | -type err_or(A) :: {ok, A} | {error, any()}. 10 | 11 | -spec trans_error (fun (() -> err_or(A))) -> A. % IO throws any() 12 | %@doc Convert error return to throw 13 | trans_error (Act) -> case Act() of {ok, A} -> A; {error, Reason} -> throw (Reason) end. 14 | 15 | -type factory(A) :: {any(), create(A), expire(A), is_expired(A)}. 16 | % Object for creating, destroying, and checking resources of type A. 17 | % any() is the input to create(A). kept separate so we can identify the factory when printed. 18 | -type create(A) :: fun ((any()) -> err_or(A)). % IO 19 | -type expire(A) :: fun ((A) -> ok). % IO 20 | -type is_expired(A) :: fun ((A) -> boolean()). % IO 21 | 22 | -opaque pool(A) :: {factory(A), mvar:mvar (array:array (maybe(A)))}. 23 | % Pool of N resources of type A, created on demand, recreated on expiration, and handed out randomly 24 | 25 | -spec new (factory(A), integer()) -> pool(A). 26 | %@doc Create empty pool that will create and destroy resources using given factory and allow up to N resources at once 27 | new (Factory, MaxSize) -> {Factory, mvar:new (array:new (MaxSize, [{fixed, false}, {default, {}}]))}. 28 | 29 | -spec get (pool(A)) -> err_or(A). % IO 30 | %@doc Return a random resource from the pool, creating one if necessary. Error if failed to create 31 | get ({{Input,Create,_,IsExpired}, VResources}) -> 32 | New = fun (Array, I) -> Res = trans_error (fun () -> Create (Input) end), {array:set (I, {Res}, Array), Res} end, 33 | Check = fun (Array, I, Res) -> case IsExpired (Res) of true -> New (Array, I); false -> {Array, Res} end end, 34 | try mvar:modify (VResources, fun (Array) -> 35 | R = random:uniform (array:size (Array)) - 1, 36 | case array:get (R, Array) of 37 | {Res} -> Check (Array, R, Res); 38 | {} -> New (Array, R) end end) 39 | of Resource -> {ok, Resource} 40 | catch Reason -> {error, Reason} end. 41 | 42 | -spec close (pool(_)) -> ok. % IO 43 | %@doc Close pool and all its resources 44 | close ({{_,_,Expire,_}, VResources}) -> 45 | mvar:with (VResources, fun (Array) -> 46 | array:map (fun (_I, MRes) -> case MRes of {Res} -> Expire (Res); {} -> ok end end, Array) end), 47 | mvar:terminate (VResources). 48 | 49 | -spec is_closed (pool(_)) -> boolean(). % IO 50 | %@doc Has pool been closed? 51 | is_closed ({_, VResources}) -> mvar:is_terminated (VResources). 52 | -------------------------------------------------------------------------------- /src/mongo_cursor.erl: -------------------------------------------------------------------------------- 1 | %@doc A cursor references pending query results on a server. 2 | % TODO: terminate cursor after idle for 10 minutes. 3 | -module (mongo_cursor). 4 | 5 | -export_type ([maybe/1]). 6 | -export_type ([cursor/0, expired/0]). 7 | 8 | -export ([next/1, rest/1, close/1, is_closed/1]). % API 9 | -export ([cursor/4]). % for mongo_query 10 | 11 | -include ("mongo_protocol.hrl"). 12 | 13 | -type maybe(A) :: {A} | {}. 14 | 15 | -opaque cursor() :: mvar:mvar (state()). 16 | % Thread-safe cursor, ie. access to query results 17 | 18 | -type expired() :: {cursor_expired, cursor()}. 19 | 20 | -type state() :: {env(), batch()}. 21 | -type env() :: {mongo_connect:dbconnection(), collection(), batchsize()}. 22 | -type batch() :: {cursorid(), [bson:document()]}. 23 | 24 | -spec cursor (mongo_connect:dbconnection(), collection(), batchsize(), {cursorid(), [bson:document()]}) -> cursor(). % IO 25 | %@doc Create new cursor from result batch 26 | cursor (DbConn, Collection, BatchSize, Batch) -> 27 | mvar:new ({{DbConn, Collection, BatchSize}, Batch}, fun finalize/1). 28 | 29 | -spec close (cursor()) -> ok. % IO 30 | %@doc Close cursor 31 | close (Cursor) -> mvar:terminate (Cursor). 32 | 33 | -spec is_closed (cursor()) -> boolean(). % IO 34 | %@doc Is cursor closed 35 | is_closed (Cursor) -> mvar:is_terminated (Cursor). 36 | 37 | -spec rest (cursor()) -> [bson:document()]. % IO throws expired() & mongo_connect:failure() 38 | %@doc Return remaining documents in query result 39 | rest (Cursor) -> case next (Cursor) of 40 | {} -> []; 41 | {Doc} -> [Doc | rest (Cursor)] end. 42 | 43 | -spec next (cursor()) -> maybe (bson:document()). % IO throws expired() & mongo_connect:failure() 44 | %@doc Return next document in query result or nothing if finished. 45 | next (Cursor) -> 46 | Next = fun ({Env, Batch}) -> 47 | {Batch1, MDoc} = xnext (Env, Batch), 48 | {{Env, Batch1}, MDoc} end, 49 | try mvar:modify (Cursor, Next) 50 | of {} -> close (Cursor), {}; {Doc} -> {Doc} 51 | catch expired -> close (Cursor), throw ({cursor_expired, Cursor}) end. 52 | 53 | -spec xnext (env(), batch()) -> {batch(), maybe (bson:document())}. % IO throws expired & mongo_connect:failure() 54 | %@doc Get next document in cursor, fetching next batch from server if necessary 55 | xnext (Env = {DbConn, Coll, BatchSize}, {CursorId, Docs}) -> case Docs of 56 | [Doc | Docs1] -> {{CursorId, Docs1}, {Doc}}; 57 | [] -> case CursorId of 58 | 0 -> {{0, []}, {}}; 59 | _ -> 60 | Getmore = #getmore {collection = Coll, batchsize = BatchSize, cursorid = CursorId}, 61 | Reply = mongo_connect:call (DbConn, [], Getmore), 62 | xnext (Env, batch_reply (Reply)) end end. 63 | 64 | -spec batch_reply (mongo_protocol:reply()) -> batch(). % IO throws expired 65 | %@doc Extract next batch of results from reply. Throw expired if cursor not found on server. 66 | batch_reply (#reply { 67 | cursornotfound = CursorNotFound, queryerror = false, awaitcapable = _, 68 | cursorid = CursorId, startingfrom = _, documents = Docs }) -> if 69 | CursorNotFound -> throw (expired); 70 | true -> {CursorId, Docs} end. 71 | 72 | -spec finalize (state()) -> ok. % IO. Result ignored 73 | %@doc Kill cursor on server if not already 74 | finalize ({{DbConn, _, _}, {CursorId, _}}) -> case CursorId of 75 | 0 -> ok; 76 | _ -> mongo_connect:send (DbConn, [#killcursor {cursorids = [CursorId]}]) end. 77 | -------------------------------------------------------------------------------- /src/mongo_query.erl: -------------------------------------------------------------------------------- 1 | %@doc Write, find, and command operations 2 | -module (mongo_query). 3 | 4 | -export_type ([write/0, 'query'/0, command/0]). 5 | -export_type ([getlasterror_request/0, getlasterror_reply/0]). 6 | -export_type ([not_master/0, unauthorized/0]). 7 | 8 | -export ([write/3, write/2, find_one/2, find/2, command/3]). 9 | 10 | -include ("mongo_protocol.hrl"). 11 | 12 | % QIO means does IO and may throw mongo_connect:failure(), not_master(), unauthorized(). 13 | 14 | -type not_master() :: not_master. 15 | % Thrown when attempting to query a slave when the query requires master (slaveok = false). 16 | -type unauthorized() :: unauthorized. 17 | % Thrown when attempting to query a db that requires auth but authentication has not been performed yet. 18 | 19 | -type write() :: #insert{} | #update{} | #delete{}. 20 | 21 | -type getlasterror_request() :: bson:document(). 22 | % Parameters to getlasterror request. See http://www.mongodb.org/display/DOCS/getLastError+Command. 23 | -type getlasterror_reply() :: bson:document(). 24 | % Reply to getlasterror request. See http://www.mongodb.org/display/DOCS/getLastError+Command. 25 | 26 | -spec write (mongo_connect:dbconnection(), write(), getlasterror_request()) -> getlasterror_reply(). % QIO 27 | %@doc Send write and getlasterror request to mongodb over connection and wait for and return getlasterror reply. Bad getlasterror params are ignored. 28 | % Caller is responsible for checking for error in reply; if 'err' field is null then success, otherwise it holds error message string. 29 | write (DbConn, Write, GetlasterrorParams) -> 30 | Query = #'query' {batchsize = -1, collection = '$cmd', 31 | selector = bson:append ({getlasterror, 1}, GetlasterrorParams)}, 32 | Reply = mongo_connect:call (DbConn, [Write], Query), 33 | {0, [Doc | _]} = query_reply (Reply), 34 | Doc. 35 | 36 | -spec write (mongo_connect:dbconnection(), write()) -> ok. % IO 37 | %@doc Send write to mongodb over connection asynchronously. Does not wait for reply hence may silently fail. Doesn't even throw connection failure if connection is closed. 38 | write (DbConn, Write) -> 39 | mongo_connect:send (DbConn, [Write]). 40 | 41 | -type command() :: bson:document(). 42 | 43 | -spec command (mongo_connect:dbconnection(), command(), boolean()) -> bson:document(). % QIO 44 | %@doc Send command to mongodb over connection and wait for reply and return it. Boolean arg indicates slave-ok or not. 'bad_command' error if bad command. 45 | command (DbConn, Command, SlaveOk) -> 46 | Query = #'query' {collection = '$cmd', selector = Command, slaveok = SlaveOk}, 47 | {Doc} = find_one (DbConn, Query), 48 | Ok = bson:at (ok, Doc), 49 | if Ok == true orelse Ok == 1 -> Doc; true -> erlang:error ({bad_command, Doc}) end. % bad_command parsed by mongo:auth 50 | 51 | -type 'query'() :: #'query'{}. 52 | 53 | -type maybe(A) :: {A} | {}. 54 | 55 | -spec find_one (mongo_connect:dbconnection(), 'query'()) -> maybe (bson:document()). % QIO 56 | %@doc Send read request to mongodb over connection and wait for reply. Return first result or nothing if empty. 57 | find_one (DbConn, Query) -> 58 | Query1 = Query #'query' {batchsize = -1}, 59 | Reply = mongo_connect:call (DbConn, [], Query1), 60 | {0, Docs} = query_reply (Reply), % batchsize negative so cursor should be closed (0) 61 | case Docs of [] -> {}; [Doc | _] -> {Doc} end. 62 | 63 | -spec find (mongo_connect:dbconnection(), 'query'()) -> mongo_cursor:cursor(). % QIO 64 | %@doc Send read request to mongodb over connection and wait for reply of first batch. Return a cursor holding this batch and able to fetch next batch on demand. 65 | find (DbConn, Query) -> 66 | Reply = mongo_connect:call (DbConn, [], Query), 67 | mongo_cursor:cursor (DbConn, Query #'query'.collection, Query #'query'.batchsize, query_reply (Reply)). 68 | 69 | -spec query_reply (mongo_protocol:reply()) -> {cursorid(), [bson:document()]}. % QIO 70 | %@doc Extract cursorid and results from reply. 'bad_query' error if query error. 71 | query_reply (#reply { 72 | cursornotfound = false, queryerror = QueryError, awaitcapable = _, 73 | cursorid = Cid, startingfrom = _, documents = Docs }) -> 74 | case QueryError of 75 | false -> {Cid, Docs}; 76 | true -> case bson:at (code, hd (Docs)) of 77 | 13435 -> throw (not_master); 78 | 10057 -> throw (unauthorized); 79 | _ -> erlang:error ({bad_query, hd (Docs)}) end end. 80 | -------------------------------------------------------------------------------- /src/mongo_protocol.erl: -------------------------------------------------------------------------------- 1 | %@doc MongoDB wire protocol 2 | -module(mongo_protocol). 3 | 4 | -export_type ([db/0]). 5 | -export_type ([notice/0, request/0, reply/0]). 6 | -export_type ([message/0]). 7 | -export_type ([requestid/0]). 8 | 9 | -export ([bit/1, bool/1, dbcoll/2, put_message/3, get_reply/1]). 10 | 11 | -include ("mongo_protocol.hrl"). 12 | -include_lib ("bson/include/bson_binary.hrl"). 13 | -import (bson_binary, [put_cstring/1, put_document/1, get_document/1]). 14 | 15 | -type notice() :: #insert{} | #update{} | #delete{} | #killcursor{}. 16 | % A notice is an asynchronous message sent to the server (no reply expected) 17 | 18 | -type request() :: #'query'{} | #getmore{}. 19 | % A request is a syncronous message sent to the server (reply expected) 20 | 21 | -type reply() :: #reply{}. 22 | % A reply to a request 23 | 24 | -type requestid() :: integer(). % message id 25 | 26 | -define (put_header (Opcode), ?put_int32 (RequestId), ?put_int32 (0), ?put_int32 (Opcode)). 27 | % RequestId expected to be in scope at call site 28 | 29 | -define (get_header (Opcode, ResponseTo), ?get_int32 (_RequestId), ?get_int32 (ResponseTo), ?get_int32 (Opcode)). 30 | 31 | -define (ReplyOpcode, 1). 32 | -define (UpdateOpcode, 2001). 33 | -define (InsertOpcode, 2002). 34 | -define (QueryOpcode, 2004). 35 | -define (GetmoreOpcode, 2005). 36 | -define (DeleteOpcode, 2006). 37 | -define (KillcursorOpcode, 2007). 38 | 39 | -spec bit (boolean()) -> 0 | 1. 40 | bit (false) -> 0; 41 | bit (true) -> 1. 42 | 43 | -spec bool (0 | 1) -> boolean(). 44 | bool (0) -> false; 45 | bool (1) -> true. 46 | 47 | -spec dbcoll (db(), collection()) -> bson:utf8(). 48 | %@doc Concat db and collection name with period (.) in between 49 | dbcoll (Db, Coll) -> <<(atom_to_binary (Db, utf8)) /binary, $., (atom_to_binary (Coll, utf8)) /binary>>. 50 | 51 | -type message() :: notice() | request(). 52 | 53 | -spec put_message (db(), message(), requestid()) -> binary(). 54 | put_message (Db, Message, RequestId) -> case Message of 55 | #insert {collection = Coll, documents = Docs} -> << 56 | ?put_header (?InsertOpcode), 57 | ?put_int32 (0), 58 | (put_cstring (dbcoll (Db, Coll))) /binary, 59 | << <<(put_document (Doc)) /binary>> || Doc <- Docs>> /binary >>; 60 | #update {collection = Coll, upsert = U, multiupdate = M, selector = Sel, updater = Up} -> << 61 | ?put_header (?UpdateOpcode), 62 | ?put_int32 (0), 63 | (put_cstring (dbcoll (Db, Coll))) /binary, 64 | ?put_bits32 (0,0,0,0,0,0, bit(M), bit(U)), 65 | (put_document (Sel)) /binary, 66 | (put_document (Up)) /binary >>; 67 | #delete {collection = Coll, singleremove = R, selector = Sel} -> << 68 | ?put_header (?DeleteOpcode), 69 | ?put_int32 (0), 70 | (put_cstring (dbcoll (Db, Coll))) /binary, 71 | ?put_bits32 (0,0,0,0,0,0,0, bit(R)), 72 | (put_document (Sel)) /binary >>; 73 | #killcursor {cursorids = Cids} -> << 74 | ?put_header (?KillcursorOpcode), 75 | ?put_int32 (0), 76 | ?put_int32 (length (Cids)), 77 | << <> || Cid <- Cids>> /binary >>; 78 | #'query' {tailablecursor = TC, slaveok = SOK, nocursortimeout = NCT, awaitdata = AD, 79 | collection = Coll, skip = Skip, batchsize = Batch, selector = Sel, projector = Proj} -> << 80 | ?put_header (?QueryOpcode), 81 | ?put_bits32 (0, 0, bit(AD), bit(NCT), 0, bit(SOK), bit(TC), 0), 82 | (put_cstring (dbcoll (Db, Coll))) /binary, 83 | ?put_int32 (Skip), 84 | ?put_int32 (Batch), 85 | (put_document (Sel)) /binary, 86 | (case Proj of [] -> <<>>; _ -> put_document (Proj) end) /binary >>; 87 | #getmore {collection = Coll, batchsize = Batch, cursorid = Cid} -> << 88 | ?put_header (?GetmoreOpcode), 89 | ?put_int32 (0), 90 | (put_cstring (dbcoll (Db, Coll))) /binary, 91 | ?put_int32 (Batch), 92 | ?put_int64 (Cid) >> 93 | end. 94 | 95 | -spec get_reply (binary()) -> {requestid(), reply(), binary()}. 96 | get_reply (<< 97 | ?get_header (?ReplyOpcode, ResponseTo), 98 | ?get_bits32 (_,_,_,_, AwaitCapable, _, QueryError, CursorNotFound), 99 | ?get_int64 (CursorId), 100 | ?get_int32 (StartingFrom), 101 | ?get_int32 (NumDocs), 102 | Bin /binary >>) -> 103 | {Docs, BinRest} = get_docs (NumDocs, Bin), 104 | Reply = #reply { 105 | cursornotfound = bool (CursorNotFound), queryerror = bool (QueryError), awaitcapable = bool (AwaitCapable), 106 | cursorid = CursorId, startingfrom = StartingFrom, documents = Docs }, 107 | {ResponseTo, Reply, BinRest}. 108 | 109 | get_docs (0, Bin) -> {[], Bin}; 110 | get_docs (NumDocs, Bin) when NumDocs > 0 -> 111 | {Doc, Bin1} = get_document (Bin), 112 | {Docs, Bin2} = get_docs (NumDocs - 1, Bin1), 113 | {[Doc | Docs], Bin2}. 114 | -------------------------------------------------------------------------------- /doc/Article1.md: -------------------------------------------------------------------------------- 1 | # Design of the Erlang MongoDB driver 2 | #### By Tony Hannan, June 2011 3 | 4 | I am a 10gen employee an author of the official [Erlang MongoDB driver](http://github.com/TonyGen/mongodb-erlang). In Nov 2010, I was assigned the task of writing a production-quality Erlang driver. Today, I would say the official driver is production-quality. Below I highlight some design decisions. For detailed documentation with code examples please see the links at the end of this article. 5 | 6 | ### BSON 7 | 8 | At the highest level, the driver is divided into two library applications, [mongodb](http://github.com/TonyGen/mongodb-erlang) and [bson](http://github.com/TonyGen/bson-erlang). Bson is defined independently of MongoDB at [bsonspec.org](http://bsonspec.org). One design decision was how to represent Bson documents in Erlang. Conceptually, a document is a record, but unlike an Erlang record, a Bson document does not have a single type tag. Futhermore, the same MongoDB collection can hold different types of records. So I decided to represent a Bson document as a tuple with labels interleaved with values, as in `{name, Name, address, Address}`. An alternative would have been to represent a document as a list of label-value pairs, but I wanted to reserve lists for Bson arrays. 9 | 10 | A Bson value is one of several types. One of these types is the document type itself, making it recursive. Several value types are not primitive, like objectid and javascript, so I had to create a tagged tuple for each of them. I defined them all to have an odd number of elements to distinguish them from a document which has an even number of elements. Finally, to distinguish between a string and a list of integers, which is indistinguishable in Erlang, I require Bson strings to be binary (UTF-8). Therefore, a plain Erlang string is interpreted as a Bson array of integers, so make sure to always encode your strings, as in `<<"hello">>` or `bson:utf8("hello")`. 11 | 12 | ### Var 13 | 14 | The mongodb driver has a couple of objects like connection and cursor that maintain mutable state. The only way to mutate state in Erlang is to have a process that maintains its own state and updates it when it receives messages from other processes. The Erlang programmer typically creates a process for each mutable object and defines the messages it may receive and the action to take for each message. He usually provides helper functions on top for the clients to call that hide the actual messages being sent and returned. Erlang OTP provides a small framework called *gen_server* to facilitate this process definition but it is still non-trivial. To alleviate this burden I created another abstraction on top of gen_server called *var*. A var is an object (process) that holds a value of some type A that may change. Its basic operation is `modify (var(A), fun ((A) -> {A, B})) -> B`, which applies the function to the current value of the var then changes the var's value to the first item of the result while returning the second item of the result to the client. This is done atomically thanks to the sequential nature of Erlang processes. The function may perform side effects (sending/receiving messages or doing IO), in which case the var acts like a mutex since only one function can execute against the var at a time. In essence, using var or even just gen_server changes the programming paradigm from message passing to protected shared state, which is more like Haskell for example. 15 | 16 | With var it is now very easy to create objects that protect a shared resource or have internal mutable state. A TCP connection to a MongoDB server is one such resource that needs protection. The connection object in mongodb is implemented as a var holding a TCP connection. Every read and write operation gets exclusive access to the TCP connection when sending and receiving its messages to and from the server. This allows multiple user processes to read and write to the same mongodb connection concurrently. 17 | 18 | ### DB Action 19 | 20 | Every read/write operation may throw a DB exception. Furthermore, every read/write operation requires a DB context that includes the connection to use, the database to access, whether slave is ok (read_mode), and whether to confirm writes (write_mode). We group a series of read/write operations that perform a single high-level task into a function called a *DB action*. We then execute the action within a single exception handler and with a single DB context in dynamic scope (using Erlang's process dictionary). `mongo:do (write_mode(), read_mode(), connection(), db(), action(A)) -> {ok, A} | {failure, failure()}` sets up the context, runs the action, and catches and returns any DB failure. Note, it will not catch and return other types of exceptions like programming errors. 21 | 22 | You may notice that a DB action is analogous to a DB transaction for a relational database in that the action aborts when one of its operations fails. However, for scalability reasons, MongoDB does not support ACID across multiple read/write operations, so the operations before the failed operation remain in effect. Your failure handler must be prepared to recover from this intermediate state. If your DB action is conceptually a single high-level task, then it should not be to hard to undo and redo that task even from an intermediate state. 23 | 24 | ### Documentation 25 | 26 | Detailed documentation with examples can be found in the ReadMe's of the two libraries, [mongodb](http://github.com/TonyGen/mongodb-erlang) and [bson](http://github.com/TonyGen/bson-erlang), and in their source code comments and test modules. 27 | -------------------------------------------------------------------------------- /src/mongo_connect.erl: -------------------------------------------------------------------------------- 1 | %@doc Thread-safe TCP connection to a MongoDB server with synchronous call and asynchronous send interface. 2 | -module (mongo_connect). 3 | 4 | -export_type ([host/0, connection/0, dbconnection/0, failure/0]). 5 | 6 | -export ([host_port/1, read_host/1, show_host/1]). 7 | -export ([connect/1, connect/2, conn_host/1, close/1, is_closed/1]). 8 | 9 | -export ([call/3, send/2]). % for mongo_query and mongo_cursor 10 | 11 | -include_lib ("bson/include/bson_binary.hrl"). 12 | 13 | -type host() :: {inet:hostname(), 0..65535} | inet:hostname(). 14 | % Hostname and port. Port defaults to 27017 when missing 15 | 16 | -spec host_port (host()) -> host(). 17 | %@doc Port explicitly filled in with defaut if missing 18 | host_port ({Hostname, Port}) -> {hostname_string (Hostname), Port}; 19 | host_port (Hostname) -> {hostname_string (Hostname), 27017}. 20 | 21 | -spec hostname_string (inet:hostname()) -> string(). 22 | %@doc Convert possible hostname atom to string 23 | hostname_string (Name) when is_atom (Name) -> atom_to_list (Name); 24 | hostname_string (Name) -> Name. 25 | 26 | -spec show_host (host()) -> bson:utf8(). 27 | %@doc UString representation of host, ie. "Hostname:Port" 28 | show_host (Host) -> 29 | {Hostname, Port} = host_port (Host), 30 | bson:utf8 (Hostname ++ ":" ++ integer_to_list (Port)). 31 | 32 | -spec read_host (bson:utf8()) -> host(). 33 | %@doc Interpret ustring as host, ie. "Hostname:Port" -> {Hostname, Port} 34 | read_host (UString) -> case string:tokens (bson:str (UString), ":") of 35 | [Hostname] -> host_port (Hostname); 36 | [Hostname, Port] -> {Hostname, list_to_integer (Port)} end. 37 | 38 | -type reason() :: any(). 39 | 40 | -opaque connection() :: {connection, host(), mvar:mvar (gen_tcp:socket()), timeout()}. 41 | % Thread-safe, TCP connection to a MongoDB server. 42 | % Passive raw binary socket. 43 | % Type not opaque to mongo:connection_mode/2 44 | 45 | -spec connect (host()) -> {ok, connection()} | {error, reason()}. % IO 46 | %@doc Create connection to given MongoDB server or return reason for connection failure. 47 | connect (Host) -> connect (Host, infinity). 48 | 49 | -spec connect (host(), timeout()) -> {ok, connection()} | {error, reason()}. % IO 50 | %@doc Create connection to given MongoDB server or return reason for connection failure. Timeout is used for initial connection and every call. 51 | connect (Host, TimeoutMS) -> try mvar:create (fun () -> tcp_connect (host_port (Host), TimeoutMS) end, fun gen_tcp:close/1) 52 | of VSocket -> {ok, {connection, host_port (Host), VSocket, TimeoutMS}} 53 | catch Reason -> {error, Reason} end. 54 | 55 | -spec conn_host (connection()) -> host(). 56 | %@doc Host this is connected to 57 | conn_host ({connection, Host, _VSocket, _}) -> Host. 58 | 59 | -spec close (connection()) -> ok. % IO 60 | %@doc Close connection. 61 | close ({connection, _Host, VSocket, _}) -> mvar:terminate (VSocket). 62 | 63 | -spec is_closed (connection()) -> boolean(). % IO 64 | %@doc Has connection been closed? 65 | is_closed ({connection, _, VSocket, _}) -> mvar:is_terminated (VSocket). 66 | 67 | -type dbconnection() :: {mongo_protocol:db(), connection()}. 68 | 69 | -type failure() :: {connection_failure, connection(), reason()}. 70 | 71 | -spec call (dbconnection(), [mongo_protocol:notice()], mongo_protocol:request()) -> mongo_protocol:reply(). % IO throws failure() 72 | %@doc Synchronous send and reply. Notices are sent right before request in single block. Exclusive access to connection during entire call. 73 | call ({Db, Conn = {connection, _Host, VSocket, TimeoutMS}}, Notices, Request) -> 74 | {MessagesBin, RequestId} = messages_binary (Db, Notices ++ [Request]), 75 | Call = fun (Socket) -> 76 | tcp_send (Socket, MessagesBin), 77 | <> = tcp_recv (Socket, 4, TimeoutMS), 78 | tcp_recv (Socket, N-4, TimeoutMS) end, 79 | try mvar:with (VSocket, Call) of 80 | ReplyBin -> 81 | {RequestId, Reply, <<>>} = mongo_protocol:get_reply (ReplyBin), 82 | Reply % ^ ResponseTo must match RequestId 83 | catch 84 | throw: Reason -> close (Conn), throw ({connection_failure, Conn, Reason}); 85 | exit: {noproc, _} -> throw ({connection_failure, Conn, closed}) end. 86 | 87 | -spec send (dbconnection(), [mongo_protocol:notice()]) -> ok. % IO throws failure() 88 | %@doc Asynchronous send (no reply). Don't know if send succeeded. Exclusive access to the connection during send. 89 | send ({Db, Conn = {connection, _Host, VSocket, _}}, Notices) -> 90 | {NoticesBin, _} = messages_binary (Db, Notices), 91 | Send = fun (Socket) -> tcp_send (Socket, NoticesBin) end, 92 | try mvar:with (VSocket, Send) 93 | catch 94 | throw: Reason -> close (Conn), throw ({connection_failure, Conn, Reason}); 95 | exit: {noproc, _} -> throw ({connection_failure, Conn, closed}) end. 96 | 97 | -spec messages_binary (mongo_protocol:db(), [mongo_protocol:message()]) -> {binary(), mongo_protocol:requestid()}. 98 | %@doc Binary representation of messages 99 | messages_binary (Db, Messages) -> 100 | Build = fun (Message, {Bin, _}) -> 101 | RequestId = mongodb_app:next_requestid(), 102 | MBin = mongo_protocol:put_message (Db, Message, RequestId), 103 | {<>, RequestId} end, 104 | lists:foldl (Build, {<<>>, 0}, Messages). 105 | 106 | % Util % 107 | 108 | tcp_connect ({Hostname, Port}, TimeoutMS) -> case gen_tcp:connect (Hostname, Port, [binary, {active, false}, {packet, 0}], TimeoutMS) of 109 | {ok, Socket} -> Socket; 110 | {error, Reason} -> throw (Reason) end. 111 | 112 | tcp_send (Socket, Binary) -> case gen_tcp:send (Socket, Binary) of 113 | ok -> ok; 114 | {error, Reason} -> throw (Reason) end. 115 | 116 | tcp_recv (Socket, N, TimeoutMS) -> case gen_tcp:recv (Socket, N, TimeoutMS) of 117 | {ok, Binary} -> Binary; 118 | {error, Reason} -> throw (Reason) end. 119 | -------------------------------------------------------------------------------- /src/mvar.erl: -------------------------------------------------------------------------------- 1 | %@doc A mvar is a process that holds a value (its content) and provides exclusive access to it. When a mvar terminates it executes it given finalize procedure, which is needed for content that needs to clean up when terminating. When a mvar is created it executes its supplied initialize procedure, which creates the initial content from within the mvar process so if the initial content is another linked dependent process (such as a socket) it will terminate when the mvar terminates without the need for a finalizer. A mvar itself dependently links to its parent process (the process that created it) and thus terminates when its parent process terminates. 2 | -module (mvar). 3 | 4 | -export_type ([mvar/1]). 5 | -export ([create/2, new/2, new/1]). 6 | -export ([modify/2, modify_/2, with/2, read/1, write/2]). 7 | -export ([terminate/1, is_terminated/1]). 8 | 9 | -behaviour (gen_server). 10 | -export ([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). 11 | 12 | -type mvar(_) :: pid() | atom(). 13 | % Unregistered or registered process holding a value of given paramterized type 14 | 15 | -type initializer(A) :: fun (() -> A). % IO throws X 16 | % Creates initial value. Any throw will be caught and re-thrown in the creating process. 17 | -type finalizer(A) :: fun ((A) -> ok). % IO 18 | % Closes supplied value. Any exception will not be caught causing an exit signal to be sent to parent (creating) process. 19 | 20 | -spec create (initializer(A), finalizer(A)) -> mvar(A). % IO throws X 21 | %@doc Create new mvar with initial content created from initializer (run within new mvar process so it owns it). Any throw in initializer will be caught and re-thrown in the calling process. Other exceptions will not be caught causing an exit signal to be sent to calling process. When the mvar terminates then given finalizer will be executed against current content. Any exception raised in finalizer (when terminating) will be sent as an exit signal to the parent (calling) process. 22 | create (Initialize, Finalize) -> 23 | Ref = make_ref(), 24 | case gen_server:start_link (?MODULE, {self(), Ref, Initialize, Finalize}, []) of 25 | {ok, Pid} -> Pid; 26 | ignore -> receive {mvar_init_throw, Ref, Thrown} -> throw (Thrown) end end. 27 | 28 | -spec new (A, finalizer(A)) -> mvar(A). % IO 29 | %@doc Same as create/2 except initial value given directly 30 | new (Value, Finalize) -> create (fun () -> Value end, Finalize). 31 | 32 | -spec new (A) -> mvar(A). % IO 33 | %@doc Same as new/2 except no finalizer 34 | new (Value) -> new (Value, fun (_) -> ok end). 35 | 36 | -type modifier(A,B) :: fun ((A) -> {A, B}). % IO throws X 37 | 38 | -spec modify (mvar(A), modifier(A,B)) -> B. % IO throws X 39 | %@doc Atomically modify content and return associated result. Any throw is caught and re-thrown in caller. Errors are not caught and will terminate var and send exit signal to parent. 40 | modify (Var, Modify) -> case gen_server:call (Var, {modify, Modify}, infinity) of 41 | {ok, B} -> B; 42 | {throw, Thrown} -> throw (Thrown) end. 43 | 44 | -spec modify_ (mvar(A), fun ((A) -> A)) -> ok. % IO throws X 45 | %@doc Same as modify but don't return anything 46 | modify_ (Var, Modify) -> modify (Var, fun (A) -> {Modify (A), ok} end). 47 | 48 | -spec with (mvar(A), fun ((A) -> B)) -> B. % IO throws X, fun IO throws X 49 | %@doc Execute Procedure with exclusive access to content but don't modify it. 50 | with (Var, Act) -> modify (Var, fun (A) -> {A, Act (A)} end). 51 | 52 | -spec read (mvar(A)) -> A. % IO 53 | %@doc Return content 54 | read (Var) -> with (Var, fun (A) -> A end). 55 | 56 | -spec write (mvar(A), A) -> A. % IO 57 | %@doc Change content and return previous content 58 | write (Var, Value) -> modify (Var, fun (A) -> {Value, A} end). 59 | 60 | -spec terminate (mvar(_)) -> ok. % IO 61 | %@doc Terminate mvar. Its finalizer will be executed. Future accesses to this mvar will fail, although repeated termination is fine. 62 | terminate (Var) -> catch gen_server:call (Var, stop, infinity), ok. 63 | 64 | -spec is_terminated (mvar(_)) -> boolean(). % IO 65 | %@doc Has mvar been terminated? 66 | is_terminated (Var) -> not is_process_alive (Var). 67 | 68 | % gen_server callbacks % 69 | 70 | -type state(A) :: {A, finalizer(A)}. 71 | 72 | -spec init ({pid(), reference(), initializer(A), finalizer(A)}) -> {ok, state(A)} | ignore. % IO 73 | % Create initial value using initializer and return it in state with finalizer. Catch throws in initializer and report it to caller via direct send and `ignore` result. `create` will pick this up an re-throw it in caller. An error in initializer will cause process to abort and exit signal being sent to caller. 74 | init ({Caller, ThrowRef, Initialize, Finalize}) -> try Initialize() 75 | of A -> {ok, {A, Finalize}} 76 | catch Thrown -> Caller ! {mvar_init_throw, ThrowRef, Thrown}, ignore end. 77 | 78 | -spec handle_call 79 | ({modify, modifier(A,B)}, {pid(), tag()}, state(A)) -> {reply, {ok, B} | {throw, any()}, state(A)}; 80 | (stop, {pid(), tag()}, state(A)) -> {stop, normal, ok, state(A)}. % IO 81 | % Modify content and return associated value. Catch any throws and return them to `modify` to be re-thrown. Errors will abort this mvar process and send exit signal to linked owner. 82 | handle_call ({modify, Modify}, _From, {A, X}) -> try Modify (A) 83 | of {A1, B} -> {reply, {ok, B}, {A1, X}} 84 | catch Thrown -> {reply, {throw, Thrown}, {A, X}} end; 85 | % Terminate mvar 86 | handle_call (stop, _From, State) -> {stop, normal, ok, State}. 87 | 88 | -spec terminate (reason(), state(_)) -> any(). % IO. Result ignored 89 | % Execute finalizer upon termination 90 | terminate (_Reason, {A, Finalize}) -> Finalize (A). 91 | 92 | -type tag() :: any(). % Unique tag 93 | -type reason() :: any(). 94 | 95 | handle_cast (_Request, State) -> {noreply, State}. 96 | 97 | handle_info (Message, State) -> 98 | io:format ("received: ~p~n", [Message]), 99 | {noreply, State}. 100 | 101 | code_change (_OldVsn, State, _Extra) -> {ok, State}. 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is the MongoDB driver for Erlang. [MongoDB](http://www.mongodb.org) is a document-oriented database management system. A driver is a client library that provides an API for connecting to MongoDB servers, performing queries and updates on those servers, and performing administrative tasks like creating indexes and viewing statistics. 2 | 3 | This version of the driver supports connecting to a single server or replica set, and pooling of both types of connections. Both connection types and pools are thread-safe, i.e. multiple processes can use the same connection/pool simultaneously without interfering with each other. 4 | 5 | This driver is implemented as an Erlang application named *mongodb*. It depends on another Erlang library application named [*bson*](http://github.com/TonyGen/bson-erlang), which defines the document type and its standard binary representation. You need both of these. Below we describe the mongodb application; you should also see the bson application to understand the document type. 6 | 7 | ### Installing 8 | 9 | Download and compile each application 10 | 11 | $ git clone git://github.com/TonyGen/bson-erlang.git bson 12 | $ git clone git://github.com/TonyGen/mongodb-erlang.git mongodb 13 | $ cd bson 14 | $ erlc -o ebin -I include src/*.erl 15 | $ cd ../mongodb 16 | $ erlc -o ebin -I include -I .. src/*.erl 17 | $ cd .. 18 | 19 | Then install them in your standard Erlang library location or include them in your path on startup 20 | 21 | $ erl -pa bson/ebin mongodb/ebin 22 | 23 | ### Starting 24 | 25 | The mongodb application needs be started before using (to initialize an internal ets table of counters) 26 | 27 | > application:start (mongodb). 28 | 29 | Although the mongodb application includes several modules, you should only need to use *mongo*, which is the top-level interface for the driver, and the generic *pool* if desired. Likewise, you should only need to use the *bson* module in the bson application. 30 | 31 | ### Connecting 32 | 33 | To connect to a mongodb server listening on `localhost:27017` (or any address & port of your choosing) 34 | 35 | > Host = {localhost, 27017}. 36 | > {ok, Conn} = mongo:connect (Host). 37 | 38 | `27017` is the default port so you could elide it in this case and just supply `localhost` as the argument. `mongo:connect` returns `{error, Reason}` if it failed to connect. 39 | 40 | Remember to close the connection when finished using it via `mongo:disconnect`. 41 | 42 | To connect to a replica set named "rs1" with seed list of members: `localhost:27017` & `localhost:27018` 43 | 44 | > Replset = {<<"rs1">>, [{localhost, 27017}, {localhost, 27018}]}. 45 | > Conn = mongo:rs_connect (Replset). 46 | 47 | `mongo:do` below will connect to the primary or a secondary in the replica set depending on the read-mode supplied. 48 | 49 | ### Querying 50 | 51 | A database operation happens in the context of a connection (single server or replica set), database, read-mode, and write-mode. These four parameters are supplied once at the beginning of a sequence of read/write operations. Furthermore, if one of the operations fails no further operations in the sequence are executed and an error is returned for the entire sequence. 52 | 53 | > mongo:do (safe, master, Conn, test, fun() -> 54 | mongo:delete (foo, {}), 55 | mongo:insert (foo, {x,1, y,2}), 56 | mongo:find (foo, {x,1}) end). 57 | 58 | `safe`, along with `{safe, GetLastErrorParams}` and `unsafe`, are write-modes. Safe mode makes a *getLastError* request after every write in the sequence. If the reply says it failed then the rest of the sequence is aborted and `mongo:do` returns `{failure, {write_failure, Reason}}`, or `{failure, not_master}` when connected to a slave. An example write failure is attempting to insert a duplicate key that is indexed to be unique. Alternatively, unsafe mode issues every write without a confirmation, so if a write fails you won't know about it and remaining operations will be executed. This is unsafe but faster because you there is no round-trip delay. 59 | 60 | `master`, along with `slave_ok`, are read-modes. `master` means every query in the sequence must read fresh data (from a master/primary server). If the connected server is not a master then the first read will fail, the remaining operations will be aborted, and `mongo:do` will return `{failure, not_master}`. `slave_ok` means every query is allowed to read stale data from a slave/secondary (fresh data from a master is fine too). 61 | 62 | `Conn` is the connection or rs_connection we send the operations to. If the connection fails during one of the operations then the remaining operations are aborted and `{failure, {connection_failure, Reason}}` is returned. 63 | 64 | `test` is the name of the database in this example. Collections accessed in the sequence (`foo` in this example) are taken from the database given in this fourth argument. If a collection is missing from the database it will be automatically created upon first access. 65 | 66 | If there are no errors in the sequence of operations then the result of the last operation is returned as the result of the entire `mongo:do` command. It is wrapped in an *ok* tuple as in `{ok, Result}` to distinguish it from an error. 67 | 68 | `mongo:find` returns a *cursor* holding the pending list of results, which are accessed using `mongo:next` to get the next result, and `mongo:rest` to get the remaining results. Either one throws `{cursor_expired, Cursor}` if the cursor was idle for more than 10 minutes. This exception is caught by `mongo:do` and returned as `{failure, {cursor_expired, Cursor}}`. `mongo:rest` also closes the cursor, otherwise you should close the cursor when finished using `mongo:close_cursor`. 69 | 70 | See the [*mongo*](http://github.com/TonyGen/mongodb-erlang/blob/master/src/mongo.erl) module for a description of all operations. A type specification is provided with each operation so you know the expected arguments and results. The spec line also has a comment if it performs a side-effect such as IO and what exceptions it may throw. No comment means it is a pure function. Also, see the [*bson* module](http://github.com/TonyGen/bson-erlang/blob/master/src/bson.erl) in the bson application for details on the document type and its value types. 71 | 72 | ### Administering 73 | 74 | This driver does not provide helper functions for commands. Use `mongo:command` directly and refer to the [MongoDB documentation](http://www.mongodb.org/display/DOCS/Commands) for how to issue raw commands. 75 | 76 | There are functions for complex commands like `mongo:auth`, `mongo:add_user`, and `mongo:create_index`. 77 | 78 | ### Pooling 79 | 80 | A single (replset-)connection is thread-safe, i.e. multiple `mongo:do` actions can access it simultaneously. However, if you want to increase concurrency by using multiple connection simultaneously, you can create a pool of connections using the generic `resource_pool` module with the appropriate factory object supplied by `mongo` module. 81 | 82 | To create a connection pool of max size 10 to a single Host 83 | 84 | > Pool = resource_pool:new (mongo:connect_factory (Host), 10). 85 | 86 | To create a rs-connection pool of max size 10 to a Replset 87 | 88 | > Pool = resource_pool:new (mongo:rs_connect_factory (Replset), 10). 89 | 90 | To get a (replset-)connection from the pool 91 | 92 | > {ok, Conn} = resource_pool:get (Pool). 93 | 94 | `Conn` can then be supplied to `mongo:do`. `resource_pool:get` will return `{error, Reason}` if can't connect. 95 | 96 | Close the pool when done using it and all its connections via `resource_pool:close`. 97 | 98 | ### More Documentation 99 | 100 | [API Docs](http://api.mongodb.org/erlang/mongodb/) - Documentation generated from source code comments 101 | -------------------------------------------------------------------------------- /src/mongodb_tests.erl: -------------------------------------------------------------------------------- 1 | %@doc Unit tests. 2 | % For test to work, a mongodb server must be listening on 127.0.0.1:27017. 3 | % For test_rs to work, a mongodb replica set named "rs1" must be listening on 127.0.0.1:27017 and 127.0.0.1:27018. 4 | -module(mongodb_tests). 5 | 6 | -include_lib("eunit/include/eunit.hrl"). 7 | -include ("mongo_protocol.hrl"). 8 | 9 | -export ([test/0, test_rs/0]). 10 | 11 | test() -> eunit:test ({setup, 12 | fun () -> application:start (mongodb), 13 | io:format (user, "~n** Make sure mongod is running on 127.0.0.1:27017 **~n~n", []) end, 14 | fun (_) -> application:stop (mongodb) end, 15 | [fun var_test/0, 16 | fun var_finalize_test/0, 17 | fun app_test/0, 18 | fun connect_test/0, 19 | fun mongo_test/0, 20 | fun resource_pool_test/0 21 | ]}). 22 | 23 | test_rs() -> eunit:test ({setup, 24 | fun () -> application:start (mongodb), 25 | io:format (user, "~n** Make sure replica set is running on 127.0.0.1:27017 & 27018 **~n~n", []) end, 26 | fun (_) -> application:stop (mongodb) end, 27 | [fun replset_test/0, 28 | fun mongo_rs_test/0 29 | ]}). 30 | 31 | var_test() -> 32 | Var = mvar:new (0), 33 | 0 = mvar:write (Var, 1), 34 | 1 = mvar:read (Var), 35 | foo = mvar:modify (Var, fun (N) -> {N+1, foo} end), 36 | 2 = mvar:read (Var), 37 | foo = (catch mvar:with (Var, fun (_) -> throw (foo) end)), 38 | mvar:terminate (Var), 39 | {exit, {noproc, _}} = try mvar:read (Var) catch C:E -> {C, E} end, 40 | mvar:terminate (Var). % repeat termination is no-op (not failure) 41 | 42 | var_finalize_test() -> 43 | Var0 = mvar:new ({}), 44 | Var = mvar:new (0, fun (N) -> mvar:write (Var0, N) end), 45 | {} = mvar:read (Var0), 46 | 0 = mvar:read (Var), 47 | mvar:terminate (Var), 48 | 0 = mvar:read (Var0), 49 | mvar:terminate (Var0). 50 | 51 | % This test must be run first right after application start (assumes counter table contain initial values) 52 | app_test() -> 53 | 1 = mongodb_app:next_requestid(), 54 | UnixSecs = bson:unixtime_to_secs (bson:timenow()), 55 | {<>} = bson:objectid (UnixSecs, <<1,2,3,4,5>>, 0), 56 | {<>} = mongodb_app:gen_objectid(). % high two timestamp bytes should match 57 | 58 | % Mongod server must be running on 127.0.0.1:27017 59 | connect_test() -> 60 | {error, _} = mongo_connect:connect ({"127.0.0.1", 26555}), 61 | {ok, Conn} = mongo_connect:connect ({"127.0.0.1", 27017}), 62 | DbConn = {test, Conn}, 63 | Res = mongo_query:write (DbConn, #delete {collection = foo, selector = {}}, {}), 64 | {null} = bson:lookup (err, Res), 65 | Doc0 = {'_id', 0, text, <<"hello">>}, 66 | Doc1 = {'_id', 1, text, <<"world">>}, 67 | Res1 = mongo_query:write (DbConn, #insert {collection = foo, documents = [Doc0, Doc1]}, {}), 68 | {null} = bson:lookup (err, Res1), 69 | ok = mongo_query:write (DbConn, #update {collection = foo, selector = {'_id', 1}, updater = {'$set', {text, <<"world!!">>}}}), 70 | Doc1X = bson:update (text, <<"world!!">>, Doc1), 71 | Cursor = mongo_query:find (DbConn, #'query' {collection = foo, selector = {}}), 72 | [Doc0, Doc1X] = mongo_cursor:rest (Cursor), 73 | true = mongo_cursor:is_closed (Cursor), 74 | #reply {cursornotfound = true} = mongo_connect:call (DbConn, [], #getmore {collection = foo, cursorid = 2938725639}), 75 | mongo_connect:close (Conn), 76 | true = mongo_connect:is_closed (Conn). 77 | 78 | % Mongod server must be running on 127.0.0.1:27017 79 | mongo_test() -> 80 | {ok, Conn} = mongo:connect ("127.0.0.1"), 81 | mongo:do (safe, master, Conn, baseball, fun () -> 82 | mongo:delete (team, {}), 83 | Teams0 = [ 84 | {name, <<"Yankees">>, home, {city, <<"New York">>, state, <<"NY">>}, league, <<"American">>}, 85 | {name, <<"Mets">>, home, {city, <<"New York">>, state, <<"NY">>}, league, <<"National">>}, 86 | {name, <<"Phillies">>, home, {city, <<"Philadelphia">>, state, <<"PA">>}, league, <<"National">>}, 87 | {name, <<"Red Sox">>, home, {city, <<"Boston">>, state, <<"MA">>}, league, <<"American">>} ], 88 | Ids = mongo:insert_all (team, Teams0), 89 | 4 = mongo:count (team, {}), 90 | Teams = lists:zipwith (fun (Id, Team) -> bson:append ({'_id', Id}, Team) end, Ids, Teams0), 91 | Teams = mongo:rest (mongo:find (team, {})), 92 | NationalTeams = lists:filter (fun (Team) -> bson:lookup (league, Team) == {<<"National">>} end, Teams), 93 | NationalTeams = mongo:rest (mongo:find (team, {league, <<"National">>})), 94 | TeamNames = lists:map (fun (Team) -> {name, bson:at (name, Team)} end, Teams), 95 | TeamNames = mongo:rest (mongo:find (team, {}, {'_id', 0, name, 1})), 96 | BostonTeam = lists:last (Teams), 97 | {BostonTeam} = mongo:find_one (team, {home, {city, <<"Boston">>, state, <<"MA">>}}), 98 | mongo:delete_one (team, {}), 99 | 3 = mongo:count (team, {}) 100 | end), 101 | mongo:disconnect (Conn). 102 | 103 | % Mongod server must be running on 127.0.0.1:27017 104 | resource_pool_test() -> 105 | Pool = resource_pool:new (mongo:connect_factory ({"127.0.0.1", 27017}), 2), 106 | Do = fun (Conn) -> mongo:do (safe, master, Conn, admin, fun () -> mongo:command ({listDatabases, 1}) end) end, 107 | lists:foreach (fun (_) -> 108 | {ok, Conn} = resource_pool:get (Pool), 109 | {ok, Doc} = Do (Conn), 110 | {_} = bson:lookup (databases, Doc) end, 111 | lists:seq (1,8)), 112 | resource_pool:close (Pool), 113 | true = resource_pool:is_closed (Pool). 114 | 115 | % Replica set named "rs1" must be running on localhost:27017 & 27018 116 | replset_test() -> % TODO: change from connect_test 117 | RS0 = mongo_replset:connect ({<<"rs0">>,[localhost]}), 118 | {error, [{not_member, _, _, _} | _]} = mongo_replset:primary (RS0), 119 | mongo_replset:close (RS0), 120 | RS1 = mongo_replset:connect ({<<"rs1">>,[localhost]}), 121 | {ok, Conn} = mongo_replset:primary (RS1), 122 | DbConn = {test, Conn}, 123 | Res = mongo_query:write (DbConn, #delete {collection = foo, selector = {}}, {}), 124 | {null} = bson:lookup (err, Res), 125 | Doc0 = {'_id', 0, text, <<"hello">>}, 126 | Doc1 = {'_id', 1, text, <<"world">>}, 127 | Res1 = mongo_query:write (DbConn, #insert {collection = foo, documents = [Doc0, Doc1]}, {}), 128 | {null} = bson:lookup (err, Res1), 129 | ok = mongo_query:write (DbConn, #update {collection = foo, selector = {'_id', 1}, updater = {'$set', {text, <<"world!!">>}}}), 130 | Doc1X = bson:update (text, <<"world!!">>, Doc1), 131 | Cursor = mongo_query:find (DbConn, #'query' {collection = foo, selector = {}}), 132 | [Doc0, Doc1X] = mongo_cursor:rest (Cursor), 133 | {ok, Conn2} = mongo_replset:secondary_ok (RS1), 134 | DbConn2 = {test, Conn2}, 135 | Cursor2 = mongo_query:find (DbConn2, #'query' {collection = foo, selector = {}, slaveok = true}), 136 | [Doc0, Doc1X] = mongo_cursor:rest (Cursor2), 137 | mongo_replset:close (RS1), 138 | true = mongo_replset:is_closed (RS1). 139 | 140 | % Replica set named "rs1" must be running on localhost:27017 & 27018 141 | mongo_rs_test() -> 142 | RsConn = mongo:rs_connect ({<<"rs1">>,["127.0.0.1"]}), 143 | {ok, {Teams1, Ids1}} = mongo:do (safe, master, RsConn, baseball, fun () -> 144 | mongo:delete (team, {}), 145 | Teams0 = [ 146 | {name, <<"Yankees">>, home, {city, <<"New York">>, state, <<"NY">>}, league, <<"American">>}, 147 | {name, <<"Mets">>, home, {city, <<"New York">>, state, <<"NY">>}, league, <<"National">>}, 148 | {name, <<"Phillies">>, home, {city, <<"Philadelphia">>, state, <<"PA">>}, league, <<"National">>}, 149 | {name, <<"Red Sox">>, home, {city, <<"Boston">>, state, <<"MA">>}, league, <<"American">>} ], 150 | Ids0 = mongo:insert_all (team, Teams0), 151 | {Teams0, Ids0} 152 | end), 153 | timer:sleep (200), 154 | mongo:do (safe, slave_ok, RsConn, baseball, fun () -> 155 | 4 = mongo:count (team, {}), 156 | Teams = lists:zipwith (fun (Id, Team) -> bson:append ({'_id', Id}, Team) end, Ids1, Teams1), 157 | Teams = mongo:rest (mongo:find (team, {})), 158 | NationalTeams = lists:filter (fun (Team) -> bson:lookup (league, Team) == {<<"National">>} end, Teams), 159 | NationalTeams = mongo:rest (mongo:find (team, {league, <<"National">>})), 160 | TeamNames = lists:map (fun (Team) -> {name, bson:at (name, Team)} end, Teams), 161 | TeamNames = mongo:rest (mongo:find (team, {}, {'_id', 0, name, 1})), 162 | BostonTeam = lists:last (Teams), 163 | {BostonTeam} = mongo:find_one (team, {home, {city, <<"Boston">>, state, <<"MA">>}}) 164 | end), 165 | mongo:rs_disconnect (RsConn). 166 | -------------------------------------------------------------------------------- /src/mongo_replset.erl: -------------------------------------------------------------------------------- 1 | %@doc Get connection to appropriate server in a replica set 2 | -module (mongo_replset). 3 | 4 | -export_type ([replset/0, rs_connection/0]). 5 | -export ([connect/1, connect/2, primary/1, secondary_ok/1, close/1, is_closed/1]). % API 6 | 7 | -type maybe(A) :: {A} | {}. 8 | -type err_or(A) :: {ok, A} | {error, reason()}. 9 | -type reason() :: any(). 10 | 11 | % -spec find (fun ((A) -> boolean()), [A]) -> maybe(A). 12 | % return first element in list that satisfies predicate 13 | % find (Pred, []) -> {}; 14 | % find (Pred, [A | Tail]) -> case Pred (A) of true -> {A}; false -> find (Pred, Tail) end. 15 | 16 | -spec until_success ([A], fun ((A) -> B)) -> B. % EIO, fun EIO 17 | %@doc Apply fun on each element until one doesn't fail. Fail if all fail or list is empty 18 | until_success ([], _Fun) -> throw ([]); 19 | until_success ([A | Tail], Fun) -> try Fun (A) 20 | catch Reason -> try until_success (Tail, Fun) 21 | catch Reasons -> throw ([Reason | Reasons]) end end. 22 | 23 | -spec rotate (integer(), [A]) -> [A]. 24 | %@doc Move first N element of list to back of list 25 | rotate (N, List) -> 26 | {Front, Back} = lists:split (N, List), 27 | Back ++ Front. 28 | 29 | -type host() :: mongo_connect:host(). 30 | -type connection() :: mongo_connect:connection(). 31 | 32 | -type replset() :: {rs_name(), [host()]}. 33 | % Identify replset. Hosts is just seed list, not necessarily all hosts in replica set 34 | -type rs_name() :: bson:utf8(). 35 | 36 | -spec connect (replset()) -> rs_connection(). % IO 37 | %@doc Create new cache of connections to replica set members starting with seed members. No connection attempted until primary or secondary_ok called. 38 | connect (ReplSet) -> connect (ReplSet, infinity). 39 | 40 | -spec connect (replset(), timeout()) -> rs_connection(). % IO 41 | %@doc Create new cache of connections to replica set members starting with seed members. No connection attempted until primary or secondary_ok called. Timeout used for initial connection and every call. 42 | connect ({ReplName, Hosts}, TimeoutMS) -> 43 | Dict = dict:from_list (lists:map (fun (Host) -> {mongo_connect:host_port (Host), {}} end, Hosts)), 44 | {rs_connection, ReplName, mvar:new (Dict), TimeoutMS}. 45 | 46 | -opaque rs_connection() :: {rs_connection, rs_name(), mvar:mvar(connections()), timeout()}. 47 | % Maintains set of connections to some if not all of the replica set members. Opaque except to mongo:connect_mode 48 | % Type not opaque to mongo:connection_mode/2 49 | -type connections() :: dict:dictionary (host(), maybe(connection())). 50 | % All hosts listed in last member_info fetched are keys in dict. Value is {} if no attempt to connect to that host yet 51 | 52 | -spec primary (rs_connection()) -> err_or(connection()). % IO 53 | %@doc Return connection to current primary in replica set 54 | primary (ReplConn) -> try 55 | MemberInfo = fetch_member_info (ReplConn), 56 | primary_conn (2, ReplConn, MemberInfo) 57 | of Conn -> {ok, Conn} 58 | catch Reason -> {error, Reason} end. 59 | 60 | -spec secondary_ok (rs_connection()) -> err_or(connection()). % IO 61 | %@doc Return connection to a current secondary in replica set or primary if none 62 | secondary_ok (ReplConn) -> try 63 | {_Conn, Info} = fetch_member_info (ReplConn), 64 | Hosts = lists:map (fun mongo_connect:read_host/1, bson:at (hosts, Info)), 65 | R = random:uniform (length (Hosts)) - 1, 66 | secondary_ok_conn (ReplConn, rotate (R, Hosts)) 67 | of Conn -> {ok, Conn} 68 | catch Reason -> {error, Reason} end. 69 | 70 | -spec close (rs_connection()) -> ok. % IO 71 | %@doc Close replset connection 72 | close ({rs_connection, _, VConns, _}) -> 73 | CloseConn = fun (_, MCon, _) -> case MCon of {Con} -> mongo_connect:close (Con); {} -> ok end end, 74 | mvar:with (VConns, fun (Dict) -> dict:fold (CloseConn, ok, Dict) end), 75 | mvar:terminate (VConns). 76 | 77 | -spec is_closed (rs_connection()) -> boolean(). % IO 78 | %@doc Has replset connection been closed? 79 | is_closed ({rs_connection, _, VConns, _}) -> mvar:is_terminated (VConns). 80 | 81 | % EIO = IO that may throw error of any type 82 | 83 | -type member_info() :: {connection(), bson:document()}. 84 | % Result of isMaster query on a server connnection. Returned fields are: setName, ismaster, secondary, hosts, [primary]. primary only present when ismaster = false 85 | 86 | -spec primary_conn (integer(), rs_connection(), member_info()) -> connection(). % EIO 87 | %@doc Return connection to primary designated in member_info. Only chase primary pointer N times. 88 | primary_conn (0, _ReplConn, MemberInfo) -> throw ({false_primary, MemberInfo}); 89 | primary_conn (Tries, ReplConn, {Conn, Info}) -> case bson:at (ismaster, Info) of 90 | true -> Conn; 91 | false -> case bson:lookup (primary, Info) of 92 | {HostString} -> 93 | MemberInfo = connect_member (ReplConn, mongo_connect:read_host (HostString)), 94 | primary_conn (Tries - 1, ReplConn, MemberInfo); 95 | {} -> throw ({no_primary, {Conn, Info}}) end end. 96 | 97 | -spec secondary_ok_conn (rs_connection(), [host()]) -> connection(). % EIO 98 | %@doc Return connection to a live secondaries in replica set, or primary if none 99 | secondary_ok_conn (ReplConn, Hosts) -> try 100 | until_success (Hosts, fun (Host) -> 101 | {Conn, Info} = connect_member (ReplConn, Host), 102 | case bson:at (secondary, Info) of true -> Conn; false -> throw (not_secondary) end end) 103 | catch _ -> primary_conn (2, ReplConn, fetch_member_info (ReplConn)) end. 104 | 105 | -spec fetch_member_info (rs_connection()) -> member_info(). % EIO 106 | %@doc Retrieve isMaster info from a current known member in replica set. Update known list of members from fetched info. 107 | fetch_member_info (ReplConn = {rs_connection, _ReplName, VConns, _}) -> 108 | OldHosts_ = dict:fetch_keys (mvar:read (VConns)), 109 | {Conn, Info} = until_success (OldHosts_, fun (Host) -> connect_member (ReplConn, Host) end), 110 | OldHosts = sets:from_list (OldHosts_), 111 | NewHosts = sets:from_list (lists:map (fun mongo_connect:read_host/1, bson:at (hosts, Info))), 112 | RemovedHosts = sets:subtract (OldHosts, NewHosts), 113 | AddedHosts = sets:subtract (NewHosts, OldHosts), 114 | mvar:modify_ (VConns, fun (Dict) -> 115 | Dict1 = sets:fold (fun remove_host/2, Dict, RemovedHosts), 116 | Dict2 = sets:fold (fun add_host/2, Dict1, AddedHosts), 117 | Dict2 end), 118 | case sets:is_element (mongo_connect:conn_host (Conn), RemovedHosts) of 119 | false -> {Conn, Info}; 120 | true -> % Conn connected to member but under wrong name (eg. localhost instead of 127.0.0.1) so it was closed and removed because it did not match a host in isMaster info. Reconnect using correct name. 121 | Hosts = dict:fetch_keys (mvar:read (VConns)), 122 | until_success (Hosts, fun (Host) -> connect_member (ReplConn, Host) end) end. 123 | 124 | add_host (Host, Dict) -> dict:store (Host, {}, Dict). 125 | 126 | remove_host (Host, Dict) -> 127 | MConn = dict:fetch (Host, Dict), 128 | Dict1 = dict:erase (Host, Dict), 129 | case MConn of {Conn} -> mongo_connect:close (Conn); {} -> ok end, 130 | Dict1. 131 | 132 | -spec connect_member (rs_connection(), host()) -> member_info(). % EIO 133 | %@doc Connect to host and verify membership. Cache connection in rs_connection 134 | connect_member ({rs_connection, ReplName, VConns, TimeoutMS}, Host) -> 135 | Conn = get_connection (VConns, Host, TimeoutMS), 136 | Info = try get_member_info (Conn) catch _ -> 137 | mongo_connect:close (Conn), 138 | Conn1 = get_connection (VConns, Host, TimeoutMS), 139 | get_member_info (Conn1) end, 140 | case bson:at (setName, Info) of 141 | ReplName -> {Conn, Info}; 142 | _ -> 143 | mongo_connect:close (Conn), 144 | throw ({not_member, ReplName, Host, Info}) end. 145 | 146 | get_connection (VConns, Host, TimeoutMS) -> mvar:modify (VConns, fun (Dict) -> 147 | case dict:find (Host, Dict) of 148 | {ok, {Conn}} -> case mongo_connect:is_closed (Conn) of 149 | false -> {Dict, Conn}; 150 | true -> new_connection (Dict, Host, TimeoutMS) end; 151 | _ -> new_connection (Dict, Host, TimeoutMS) end end). 152 | 153 | new_connection (Dict, Host, TimeoutMS) -> case mongo_connect:connect (Host, TimeoutMS) of 154 | {ok, Conn} -> {dict:store (Host, {Conn}, Dict), Conn}; 155 | {error, Reason} -> throw ({cant_connect, Reason}) end. 156 | 157 | get_member_info (Conn) -> mongo_query:command ({admin, Conn}, {isMaster, 1}, true). 158 | -------------------------------------------------------------------------------- /src/mongo.erl: -------------------------------------------------------------------------------- 1 | %@doc Top-level client interface to MongoDB 2 | -module (mongo). 3 | 4 | -export_type ([maybe/1]). 5 | 6 | -export_type ([host/0, connection/0]). 7 | -export ([connect/1, connect/2, disconnect/1, connect_factory/1, connect_factory/2]). 8 | -export_type ([replset/0, rs_connection/0]). 9 | -export ([rs_connect/1, rs_connect/2, rs_disconnect/1, rs_connect_factory/1, rs_connect_factory/2]). 10 | 11 | -export_type ([action/1, db/0, write_mode/0, read_mode/0, failure/0]). 12 | -export ([do/5, this_db/0]). 13 | 14 | -export_type ([collection/0, selector/0, projector/0, skip/0, batchsize/0, modifier/0]). 15 | -export ([insert/2, insert_all/2]). 16 | -export ([save/2, replace/3, repsert/3, modify/3]). 17 | -export ([delete/2, delete_one/2]). 18 | -export ([find_one/2, find_one/3, find_one/4]). 19 | -export ([find/2, find/3, find/4, find/5]). 20 | -export ([count/2, count/3]). 21 | 22 | -export_type ([cursor/0]). 23 | -export ([next/1, rest/1, close_cursor/1]). 24 | 25 | -export_type ([command/0]). 26 | -export ([command/1]). 27 | 28 | -export_type ([username/0, password/0]). 29 | -export ([auth/2]). 30 | 31 | -export_type ([permission/0]). 32 | -export ([add_user/3]). 33 | 34 | -export_type ([index_spec/0, key_order/0]). 35 | -export ([create_index/2]). 36 | 37 | -export ([copy_database/3, copy_database/5]). 38 | 39 | -include ("mongo_protocol.hrl"). 40 | 41 | -type reason() :: any(). 42 | 43 | % Server % 44 | 45 | -type host() :: mongo_connect:host(). 46 | % Hostname or ip address with or without port. Port defaults to 27017 when missing. 47 | % Eg. "localhost" or {"localhost", 27017} 48 | -type connection() :: mongo_connect:connection(). 49 | 50 | -spec connect (host()) -> {ok, connection()} | {error, reason()}. % IO 51 | %@doc Connect to given MongoDB server 52 | connect (Host) -> mongo_connect:connect (Host). 53 | 54 | -spec connect (host(), timeout()) -> {ok, connection()} | {error, reason()}. % IO 55 | %@doc Connect to given MongoDB server. Timeout used for initial connection and every query and safe write. 56 | connect (Host, TimeoutMS) -> mongo_connect:connect (Host, TimeoutMS). 57 | 58 | -spec disconnect (connection()) -> ok. % IO 59 | %@doc Close connection to server 60 | disconnect (Conn) -> mongo_connect:close (Conn). 61 | 62 | -spec connect_factory (host()) -> resource_pool:factory(connection()). 63 | %@doc Factory for use with a connection pool. See resource_pool module. 64 | connect_factory (Host) -> connect_factory (Host, infinity). 65 | 66 | -spec connect_factory (host(), timeout()) -> resource_pool:factory(connection()). 67 | %@doc Factory for use with a connection pool. See resource_pool module. 68 | connect_factory (Host, TimeoutMS) -> {Host, fun (H) -> connect (H, TimeoutMS) end, fun disconnect/1, fun mongo_connect:is_closed/1}. 69 | 70 | % Replica Set % 71 | 72 | -type replset() :: mongo_replset:replset(). 73 | -type rs_connection() :: mongo_replset:rs_connection(). 74 | 75 | -spec rs_connect (replset()) -> rs_connection(). % IO 76 | %@doc Create new cache of connections to replica set members starting with seed members. No connection attempted until rs_primary or rs_secondary_ok called. 77 | rs_connect (Replset) -> mongo_replset:connect (Replset). 78 | 79 | -spec rs_connect (replset(), timeout()) -> rs_connection(). % IO 80 | %@doc Create new cache of connections to replica set members starting with seed members. No connection attempted until rs_primary or rs_secondary_ok called. Timeout used for initial connection and every query and safe write. 81 | rs_connect (Replset, TimeoutMS) -> mongo_replset:connect (Replset, TimeoutMS). 82 | 83 | -spec rs_disconnect (rs_connection()) -> ok. % IO 84 | %@doc Close cache of replset connections 85 | rs_disconnect (ReplsetConn) -> mongo_replset:close (ReplsetConn). 86 | 87 | -spec rs_connect_factory (replset()) -> resource_pool:factory(rs_connection()). 88 | %@doc Factory for use with a rs_connection pool. See resource_pool module. 89 | rs_connect_factory (ReplSet) -> rs_connect_factory (ReplSet, infinity). 90 | 91 | -spec rs_connect_factory (replset(), timeout()) -> resource_pool:factory(rs_connection()). 92 | %@doc Factory for use with a rs_connection pool. See resource_pool module. 93 | rs_connect_factory (Replset, TimeoutMS) -> {Replset, fun (RS) -> RC = rs_connect (RS, TimeoutMS), {ok, RC} end, fun rs_disconnect/1, fun mongo_replset:is_closed/1}. 94 | 95 | % Action % 96 | 97 | -type action(A) :: fun (() -> A). 98 | % An Action does IO, reads process dict {mongo_action_context, #context{}}, and throws failure() 99 | 100 | -type failure() :: 101 | mongo_connect:failure() | % thrown by read and safe write 102 | mongo_query:not_master() | % thrown by read and safe write 103 | mongo_query:unauthorized() | % thrown by read and safe write 104 | write_failure() | % thrown by safe write 105 | mongo_cursor:expired(). % thrown by cursor next/rest 106 | 107 | -record (context, { 108 | write_mode :: write_mode(), 109 | read_mode :: read_mode(), 110 | dbconn :: mongo_connect:dbconnection() }). 111 | 112 | -spec do (write_mode(), read_mode(), connection() | rs_connection(), db(), action(A)) -> {ok, A} | {failure, failure()}. % IO 113 | %@doc Execute mongo action under given write_mode, read_mode, connection, and db. Return action result or failure. 114 | do (WriteMode, ReadMode, Connection, Database, Action) -> case connection_mode (ReadMode, Connection) of 115 | {error, Reason} -> {failure, {connection_failure, Reason}}; 116 | {ok, Conn} -> 117 | PrevContext = get (mongo_action_context), 118 | put (mongo_action_context, #context {write_mode = WriteMode, read_mode = ReadMode, dbconn = {Database, Conn}}), 119 | try Action() of 120 | Result -> {ok, Result} 121 | catch 122 | throw: E = {connection_failure, _, _} -> {failure, E}; 123 | throw: E = not_master -> {failure, E}; 124 | throw: E = unauthorized -> {failure, E}; 125 | throw: E = {write_failure, _, _} -> {failure, E}; 126 | throw: E = {cursor_expired, _} -> {failure, E} 127 | after 128 | case PrevContext of undefined -> erase (mongo_action_context); _ -> put (mongo_action_context, PrevContext) end 129 | end end. 130 | 131 | -spec connection_mode (read_mode(), connection() | rs_connection()) -> {ok, connection()} | {error, reason()}. % IO 132 | %@doc For rs_connection return appropriate primary or secondary connection 133 | connection_mode (_, Conn = {connection, _, _, _}) -> {ok, Conn}; 134 | connection_mode (master, RsConn = {rs_connection, _, _, _}) -> mongo_replset:primary (RsConn); 135 | connection_mode (slave_ok, RsConn = {rs_connection, _, _, _}) -> mongo_replset:secondary_ok (RsConn). 136 | 137 | -spec this_db () -> db(). % Action 138 | %@doc Current db in context that we are querying 139 | this_db () -> {Db, _} = (get (mongo_action_context)) #context.dbconn, Db. 140 | 141 | % Write % 142 | 143 | -type write_mode() :: unsafe | safe | {safe, mongo_query:getlasterror_request()}. 144 | % Every write inside an action() will use this write mode. 145 | % unsafe = asynchronous write (no reply) and hence may silently fail; 146 | % safe = synchronous write, wait for reply and fail if connection or write failure; 147 | % {safe, Params} = same as safe but with extra params for getlasterror, see its documentation at http://www.mongodb.org/display/DOCS/getLastError+Command. 148 | 149 | -type write_failure() :: {write_failure, error_code(), bson:utf8()}. 150 | -type error_code() :: integer(). 151 | 152 | -spec write (mongo_query:write()) -> ok. % Action 153 | %@doc Do unsafe unacknowledged fast write or safe acknowledged slower write depending on our context. When safe, throw write_failure if acknowledgment (getlasterror) reports error. 154 | write (Write) -> 155 | Context = get (mongo_action_context), 156 | case Context #context.write_mode of 157 | unsafe -> mongo_query:write (Context #context.dbconn, Write); 158 | SafeMode -> 159 | Params = case SafeMode of safe -> {}; {safe, Param} -> Param end, 160 | Ack = mongo_query:write (Context #context.dbconn, Write, Params), 161 | case bson:lookup (err, Ack) of 162 | {} -> ok; {null} -> ok; 163 | {String} -> case bson:at (code, Ack) of 164 | 10058 -> throw (not_master); 165 | Code -> throw ({write_failure, Code, String}) end end end. 166 | 167 | -spec insert (collection(), bson:document()) -> bson:value(). % Action 168 | %@doc Insert document into collection. Return its '_id' value, which is auto-generated if missing. 169 | insert (Coll, Doc) -> [Value] = insert_all (Coll, [Doc]), Value. 170 | 171 | -spec insert_all (collection(), [bson:document()]) -> [bson:value()]. % Action 172 | %@doc Insert documents into collection. Return their '_id' values, which are auto-generated if missing. 173 | insert_all (Coll, Docs) -> 174 | Docs1 = lists:map (fun assign_id/1, Docs), 175 | write (#insert {collection = Coll, documents = Docs1}), 176 | lists:map (fun (Doc) -> bson:at ('_id', Doc) end, Docs1). 177 | 178 | -spec assign_id (bson:document()) -> bson:document(). % IO 179 | %@doc If doc has no '_id' field then generate a fresh object id for it 180 | assign_id (Doc) -> case bson:lookup ('_id', Doc) of 181 | {_Value} -> Doc; 182 | {} -> bson:append ({'_id', mongodb_app:gen_objectid()}, Doc) end. 183 | 184 | -spec save (collection(), bson:document()) -> ok. % Action 185 | %@doc If document has no '_id' field then insert it, otherwise update it and insert only if missing. 186 | save (Coll, Doc) -> case bson:lookup ('_id', Doc) of 187 | {} -> insert (Coll, Doc), ok; 188 | {Id} -> repsert (Coll, {'_id', Id}, Doc) end. 189 | 190 | -spec replace (collection(), selector(), bson:document()) -> ok. % Action 191 | %@doc Replace first document selected with given document. 192 | replace (Coll, Selector, Doc) -> update (false, false, Coll, Selector, Doc). 193 | 194 | -spec repsert (collection(), selector(), bson:document()) -> ok. % Action 195 | %@doc Replace first document selected with given document, or insert it if selection is empty. 196 | repsert (Coll, Selector, Doc) -> update (true, false, Coll, Selector, Doc). 197 | 198 | -spec modify (collection(), selector(), modifier()) -> ok. % Action 199 | %@doc Update all documents selected using modifier 200 | modify (Coll, Selector, Mod) -> update (false, true, Coll, Selector, Mod). 201 | 202 | -spec update (boolean(), boolean(), collection(), selector(), bson:document()) -> ok. % Action 203 | update (Upsert, MultiUpdate, Coll, Sel, Doc) -> 204 | write (#update {collection = Coll, upsert = Upsert, multiupdate = MultiUpdate, selector = Sel, updater = Doc}). 205 | 206 | -spec delete (collection(), selector()) -> ok. % Action 207 | %@doc Delete selected documents 208 | delete (Coll, Selector) -> 209 | write (#delete {collection = Coll, singleremove = false, selector = Selector}). 210 | 211 | -spec delete_one (collection(), selector()) -> ok. % Action 212 | %@doc Delete first selected document. 213 | delete_one (Coll, Selector) -> 214 | write (#delete {collection = Coll, singleremove = true, selector = Selector}). 215 | 216 | % Read % 217 | 218 | -type read_mode() :: master | slave_ok. 219 | % Every query inside an action() will use this mode. 220 | % master = Server must be master/primary so reads are consistent (read latest writes). 221 | % slave_ok = Server may be slave/secondary so reads may not be consistent (may read stale data). Slaves will eventually get the latest writes, so technically this is called eventually-consistent. 222 | 223 | slave_ok (#context {read_mode = slave_ok}) -> true; 224 | slave_ok (#context {read_mode = master}) -> false. 225 | 226 | -type maybe(A) :: {A} | {}. 227 | 228 | -spec find_one (collection(), selector()) -> maybe (bson:document()). % Action 229 | %@doc Return first selected document, if any 230 | find_one (Coll, Selector) -> find_one (Coll, Selector, []). 231 | 232 | -spec find_one (collection(), selector(), projector()) -> maybe (bson:document()). % Action 233 | %@doc Return projection of first selected document, if any. Empty projection [] means full projection. 234 | find_one (Coll, Selector, Projector) -> find_one (Coll, Selector, Projector, 0). 235 | 236 | -spec find_one (collection(), selector(), projector(), skip()) -> maybe (bson:document()). % Action 237 | %@doc Return projection of Nth selected document, if any. Empty projection [] means full projection. 238 | find_one (Coll, Selector, Projector, Skip) -> 239 | Context = get (mongo_action_context), 240 | Query = #'query' { 241 | collection = Coll, selector = Selector, projector = Projector, 242 | skip = Skip, slaveok = slave_ok (Context) }, 243 | mongo_query:find_one (Context #context.dbconn, Query). 244 | 245 | -spec find (collection(), selector()) -> cursor(). % Action 246 | %@doc Return selected documents. 247 | find (Coll, Selector) -> find (Coll, Selector, []). 248 | 249 | -spec find (collection(), selector(), projector()) -> cursor(). % Action 250 | %@doc Return projection of selected documents. Empty projection [] means full projection. 251 | find (Coll, Selector, Projector) -> find (Coll, Selector, Projector, 0). 252 | 253 | -spec find (collection(), selector(), projector(), skip()) -> cursor(). % Action 254 | %@doc Return projection of selected documents starting from Nth document. Empty projection means full projection. 255 | find (Coll, Selector, Projector, Skip) -> find (Coll, Selector, Projector, Skip, 0). 256 | 257 | -spec find (collection(), selector(), projector(), skip(), batchsize()) -> cursor(). % Action 258 | %@doc Return projection of selected documents starting from Nth document in batches of batchsize. 0 batchsize means default batch size. Negative batch size means one batch only. Empty projection means full projection. 259 | find (Coll, Selector, Projector, Skip, BatchSize) -> 260 | Context = get (mongo_action_context), 261 | Query = #'query' { 262 | collection = Coll, selector = Selector, projector = Projector, 263 | skip = Skip, batchsize = BatchSize, slaveok = slave_ok (Context) }, 264 | mongo_query:find (Context #context.dbconn, Query). 265 | 266 | -type cursor() :: mongo_cursor:cursor(). 267 | 268 | -spec next (cursor()) -> maybe (bson:document()). % IO throws mongo_connect:failure() & mongo_cursor:expired() (this is a subtype of Action) 269 | %@doc Return next document in query result cursor, if any. 270 | next (Cursor) -> mongo_cursor:next (Cursor). 271 | 272 | -spec rest (cursor()) -> [bson:document()]. % IO throws mongo_connect:failure() & mongo_cursor:expired() (this is a subtype of Action) 273 | %@doc Return remaining documents in query result cursor. 274 | rest (Cursor) -> mongo_cursor:rest (Cursor). 275 | 276 | -spec close_cursor (cursor()) -> ok. % IO (IO is a subtype of Action) 277 | %@doc Close cursor 278 | close_cursor (Cursor) -> mongo_cursor:close (Cursor). 279 | 280 | -spec count (collection(), selector()) -> integer(). % Action 281 | %@doc Count selected documents 282 | count (Coll, Selector) -> count (Coll, Selector, 0). 283 | 284 | -spec count (collection(), selector(), integer()) -> integer(). % Action 285 | %@doc Count selected documents up to given max number; 0 means no max. Ie. stops counting when max is reached to save processing time. 286 | count (Coll, Selector, Limit) -> 287 | CollStr = atom_to_binary (Coll, utf8), 288 | Command = if 289 | Limit =< 0 -> {count, CollStr, 'query', Selector}; 290 | true -> {count, CollStr, 'query', Selector, limit, Limit} end, 291 | Doc = command (Command), 292 | trunc (bson:at (n, Doc)). % Server returns count as float 293 | 294 | % Command % 295 | 296 | -type command() :: mongo_query:command(). 297 | 298 | -spec command (command()) -> bson:document(). % Action 299 | %@doc Execute given MongoDB command and return its result. 300 | command (Command) -> 301 | Context = get (mongo_action_context), 302 | mongo_query:command (Context #context.dbconn, Command, slave_ok (Context)). 303 | 304 | % Authentication % 305 | 306 | -type username() :: bson:utf8(). 307 | -type password() :: bson:utf8(). 308 | -type nonce() :: bson:utf8(). 309 | 310 | -spec auth (username(), password()) -> boolean(). % Action 311 | %@doc Authenticate with the database (if server is running in secure mode). Return whether authentication was successful or not. Reauthentication is required for every new pipe. 312 | auth (Username, Password) -> 313 | Nonce = bson:at (nonce, command ({getnonce, 1})), 314 | try command ({authenticate, 1, user, Username, nonce, Nonce, key, pw_key (Nonce, Username, Password)}) 315 | of _ -> true 316 | catch error:{bad_command, _} -> false end. 317 | 318 | -spec pw_key (nonce(), username(), password()) -> bson:utf8(). 319 | pw_key (Nonce, Username, Password) -> bson:utf8 (binary_to_hexstr (crypto:md5 ([Nonce, Username, pw_hash (Username, Password)]))). 320 | 321 | -spec pw_hash (username(), password()) -> bson:utf8(). 322 | pw_hash (Username, Password) -> bson:utf8 (binary_to_hexstr (crypto:md5 ([Username, <<":mongo:">>, Password]))). 323 | 324 | -spec binary_to_hexstr (binary()) -> string(). 325 | binary_to_hexstr (Bin) -> 326 | lists:flatten ([io_lib:format ("~2.16.0b", [X]) || X <- binary_to_list (Bin)]). 327 | 328 | -type permission() :: read_write | read_only. 329 | 330 | -spec add_user (permission(), username(), password()) -> ok. % Action 331 | %@doc Add user with given access rights (permission) 332 | add_user (Permission, Username, Password) -> 333 | User = case find_one (system.users, {user, Username}) of {} -> {user, Username}; {Doc} -> Doc end, 334 | Rec = {readOnly, case Permission of read_only -> true; read_write -> false end, pwd, pw_hash (Username, Password)}, 335 | save (system.users, bson:merge (Rec, User)). 336 | 337 | % Index % 338 | 339 | -type index_spec() :: bson:document(). 340 | % The following fields are required: 341 | % key : key_order() 342 | % The following fields are optional: 343 | % name : bson:utf8() 344 | % unique : boolean() 345 | % dropDups : boolean() 346 | % Additional fields are allowed specific to the index, for example, when creating a Geo index you may also supply 347 | % min & max fields. See http://www.mongodb.org/display/DOCS/Geospatial+Indexing for details. 348 | 349 | -type key_order() :: bson:document(). 350 | % Fields to index on and whether ascending (1) or descending (-1) or Geo (<<"2d">>). Eg. {x,1, y,-1} or {loc, <<"2d">>} 351 | 352 | -spec create_index (collection(), index_spec() | key_order()) -> ok. % Action 353 | %@doc Create index on collection according to given spec. Allow user to just supply key 354 | create_index (Coll, IndexSpec) -> 355 | Db = this_db (), 356 | Index = bson:append ({ns, mongo_protocol:dbcoll (Db, Coll)}, fillout_indexspec (IndexSpec)), 357 | insert ('system.indexes', Index). 358 | 359 | -spec fillout_indexspec (index_spec() | key_order()) -> index_spec(). 360 | % Fill in missing optonal fields with defaults. Allow user to just supply key_order 361 | fillout_indexspec (IndexSpec) -> case bson:lookup (key, IndexSpec) of 362 | {Key} when is_tuple (Key) -> bson:merge (IndexSpec, {key, Key, name, gen_index_name (Key), unique, false, dropDups, false}); 363 | {_} -> {key, IndexSpec, name, gen_index_name (IndexSpec), unique, false, dropDups, false}; % 'key' happens to be a user field 364 | {} -> {key, IndexSpec, name, gen_index_name (IndexSpec), unique, false, dropDups, false} end. 365 | 366 | -spec gen_index_name (key_order()) -> bson:utf8(). 367 | gen_index_name (KeyOrder) -> 368 | AsName = fun (Label, Order, Name) -> << 369 | Name /binary, $_, 370 | (atom_to_binary (Label, utf8)) /binary, $_, 371 | (if 372 | is_integer (Order) -> bson:utf8 (integer_to_list (Order)); 373 | is_atom (Order) -> atom_to_binary (Order, utf8); 374 | is_binary (Order) -> Order; 375 | true -> <<>> end) /binary >> end, 376 | bson:doc_foldl (AsName, <<"i">>, KeyOrder). 377 | 378 | % Admin 379 | 380 | -spec copy_database (db(), host(), db()) -> bson:document(). % Action 381 | % Copy database from given host to the server I am connected to. Must be connected to 'admin' database. 382 | copy_database (FromDb, FromHost, ToDb) -> 383 | command ({copydb, 1, fromhost, mongo_connect:show_host (FromHost), fromdb, atom_to_binary (FromDb, utf8), todb, atom_to_binary (ToDb, utf8)}). 384 | 385 | -spec copy_database (db(), host(), db(), username(), password()) -> bson:document(). % Action 386 | % Copy database from given host, authenticating with given username and password, to the server I am connected to. Must be connected to 'admin' database. 387 | copy_database (FromDb, FromHost, ToDb, Username, Password) -> 388 | Nonce = bson:at (nonce, command ({copydbgetnonce, 1, fromhost, mongo_connect:show_host (FromHost)})), 389 | command ({copydb, 1, fromhost, mongo_connect:show_host (FromHost), fromdb, atom_to_binary (FromDb, utf8), todb, atom_to_binary (ToDb, utf8), username, Username, nonce, Nonce, key, pw_key (Nonce, Username, Password)}). 390 | --------------------------------------------------------------------------------