├── +dj ├── +config │ ├── load.m │ ├── restore.m │ ├── save.m │ ├── saveGlobal.m │ └── saveLocal.m ├── +internal │ ├── AutoPopulate.m │ ├── Declare.m │ ├── ExternalMapping.m │ ├── ExternalTable.m │ ├── GeneralRelvar.m │ ├── Header.m │ ├── Master.m │ ├── Settings.m │ ├── Table.m │ ├── TableAccessor.m │ ├── UserRelation.m │ ├── ask.m │ ├── fromCamelCase.m │ ├── hash.m │ ├── shorthash.m │ └── toCamelCase.m ├── +lib │ ├── DataHash.m │ ├── HMAC.m │ ├── getpass.m │ ├── isString.m │ ├── saveJSONfile.m │ └── startsWith.m ├── +store_plugins │ ├── File.m │ └── S3.m ├── AutoPopulate.m ├── BatchPopulate.m ├── Computed.m ├── Connection.m ├── ERD.m ├── Imported.m ├── Jobs.m ├── Lookup.m ├── Manual.m ├── Part.m ├── Relvar.m ├── Schema.m ├── Table.m ├── close.m ├── config.m ├── conn.m ├── createSchema.m ├── kill.m ├── kill_quick.m ├── new.m ├── reload.m ├── setPassword.m ├── setup.m ├── struct.m └── version.m ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md └── workflows │ ├── development.yaml │ └── publish-docs.yaml ├── .gitignore ├── CHANGELOG.md ├── LICENSE.txt ├── LNX-docker-compose.yaml ├── README.md ├── docs ├── .docker │ ├── Dockerfile │ ├── apk_requirements.txt │ └── pip_requirements.txt ├── .markdownlint.yaml ├── README.md ├── cspell.json ├── docker-compose.yaml ├── mkdocs.yaml └── src │ ├── .overrides │ ├── .icons │ │ └── main │ │ │ └── company-logo.svg │ ├── assets │ │ ├── images │ │ │ └── company-logo-blue.png │ │ └── stylesheets │ │ │ └── extra.css │ └── partials │ │ └── nav.html │ ├── changelog.md │ ├── concepts │ └── existing-pipelines.md │ ├── getting-started │ └── index.md │ ├── images │ └── shapes_pipeline.svg │ ├── query-lang │ ├── common-commands.md │ ├── iteration.md │ └── operators.md │ ├── reproduce │ ├── make-method.md │ └── table-tiers.md │ └── tutorials.md ├── erd.m ├── local-docker-compose.yaml └── tests ├── Main.m ├── Prep.m ├── TestBlob.m ├── TestConfig.m ├── TestConnection.m ├── TestDeclaration.m ├── TestDelete.m ├── TestERD.m ├── TestExternalFile.m ├── TestExternalS3.m ├── TestFetch.m ├── TestPopulate.m ├── TestProjection.m ├── TestRelationalOperand.m ├── TestSchema.m ├── TestTls.m ├── TestUuid.m └── test_schemas ├── +Company ├── Duty.m ├── Employee.m └── Machine.m ├── +External ├── Dimension.m ├── Document.m └── Image.m ├── +Lab ├── ActiveSession.m ├── Rig.m ├── Session.m ├── SessionAnalysis.m └── Subject.m ├── +TestLab ├── Duty.m └── User.m ├── +University ├── A.m ├── Date.m ├── Float.m ├── Integer.m ├── Message.m ├── String.m └── Student.m ├── config.json ├── config_lite.json └── store_config.json /+dj/+config/load.m: -------------------------------------------------------------------------------- 1 | function load(fname) 2 | % LOAD(fname) 3 | % Description: 4 | % Updates the setting from config file in JSON format. 5 | % Inputs: 6 | % fname[optional, default=dj.internal.Settings.LOCALFILE]: (string) Config file path. 7 | % Examples: 8 | % dj.config.load('/path/to/dj_local_conf.json') 9 | % dj.config.load 10 | switch nargin 11 | case 0 12 | dj.internal.Settings.load(); 13 | case 1 14 | dj.internal.Settings.load(fname); 15 | otherwise 16 | error('Exceeded 1 input limit.'); 17 | end 18 | end -------------------------------------------------------------------------------- /+dj/+config/restore.m: -------------------------------------------------------------------------------- 1 | function out = restore() 2 | % RESTORE() 3 | % Description: 4 | % Restores the configuration to initial default. 5 | % Examples: 6 | % dj.config.restore 7 | out = dj.internal.Settings.restore(); 8 | end -------------------------------------------------------------------------------- /+dj/+config/save.m: -------------------------------------------------------------------------------- 1 | function save(fname) 2 | % SAVE(fname) 3 | % Description: 4 | % Saves the settings in JSON format to the given file path. 5 | % Inputs: 6 | % fname[required]: (string) Config file path. 7 | % Examples: 8 | % dj.config.save('/path/to/dj_local_conf.json') 9 | switch nargin 10 | case 1 11 | dj.internal.Settings.save(fname); 12 | otherwise 13 | error('Requires 1 input.'); 14 | end 15 | end -------------------------------------------------------------------------------- /+dj/+config/saveGlobal.m: -------------------------------------------------------------------------------- 1 | function saveGlobal() 2 | % SAVEGLOBAL() 3 | % Description: 4 | % Saves the settings in the global config file. 5 | % Examples: 6 | % dj.config.saveGlobal 7 | dj.internal.Settings.saveGlobal(); 8 | end -------------------------------------------------------------------------------- /+dj/+config/saveLocal.m: -------------------------------------------------------------------------------- 1 | function saveLocal() 2 | % SAVELOCAL() 3 | % Description: 4 | % Saves the settings in the local config file. 5 | % Examples: 6 | % dj.config.saveLocal 7 | dj.internal.Settings.saveLocal(); 8 | end -------------------------------------------------------------------------------- /+dj/+internal/ExternalMapping.m: -------------------------------------------------------------------------------- 1 | % dj.internal.ExternalMapping - The external manager contains all the tables for all external 2 | % stores for a given schema. 3 | % :Example: 4 | % e = dj.internal.ExternalMapping(schema) 5 | % external_table = e.table(store) 6 | classdef ExternalMapping < handle 7 | properties 8 | schema 9 | tables 10 | end 11 | methods 12 | function self = ExternalMapping(schema) 13 | self.schema = schema; 14 | self.tables = struct(); 15 | end 16 | function store_table = table(self, store) 17 | keys = fieldnames(self.tables); 18 | if all(~strcmp(store, keys)) 19 | self.tables.(store) = dj.internal.ExternalTable(... 20 | self.schema.conn, store, self.schema); 21 | end 22 | store_table = self.tables.(store); 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /+dj/+internal/Master.m: -------------------------------------------------------------------------------- 1 | classdef Master < handle 2 | % mix-in class for dj.Relvar classes to make them process parts in a 3 | % master/part relationship. 4 | 5 | methods 6 | function list = getParts(self) 7 | % find classnames that begin with me and are dj.Part 8 | info = meta.class.fromName(class(self)); 9 | classNames = {info.ContainingPackage.ClassList.Name}; 10 | list = cellfun(@feval, classNames(dj.lib.startsWith(classNames, class(self))), 'uni', false); 11 | list = list(cellfun(@(x) isa(x, 'dj.Part'), list)); 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /+dj/+internal/Settings.m: -------------------------------------------------------------------------------- 1 | classdef Settings < matlab.mixin.Copyable 2 | properties (Constant) 3 | LOCALFILE = './dj_local_conf.json' 4 | GLOBALFILE = '~/.datajoint_config.json' 5 | DEFAULTS = struct( ... 6 | 'databaseHost', 'localhost', ... 7 | 'databasePassword', [], ... 8 | 'databaseUser', [], ... 9 | 'databasePort', 3306, ... 10 | 'databaseUse_tls', [], ... 11 | 'databaseReconnect_transaction', false, ... 12 | 'connectionInit_function', [], ... 13 | 'loglevel', 'INFO', ... 14 | 'safemode', true, ... 15 | 'displayLimit', 12, ... how many rows to display when previewing a relation 16 | 'displayDiagram_hierarchy_radius', [2 1], ... levels up and down the hierachy to display in `erd schema.Table` 17 | 'displayDiagram_font_size', 12, ... font size to use in ERD labels 18 | 'displayCount', true, ... optionally display count of records in result 19 | 'queryPopulate_check', true, ... 20 | 'queryPopulate_ancestors', false, ... 21 | 'queryBigint_to_double', false, ... 22 | 'queryIgnore_extra_insert_fields', false, ... when false, throws an error in `insert(self, tuple)` when tuple has extra fields. 23 | 'use_32bit_dims', false ... 24 | ) 25 | end 26 | properties 27 | result 28 | end 29 | methods(Static) 30 | function out = Settings(name, value) 31 | current_state = stateAccess; 32 | out.result = current_state; 33 | if nargin == 1 || nargin == 2 34 | assert(ischar(name), 'DataJoint:Config:InvalidType', ... 35 | 'Setting name must be a string'); 36 | token = regexp(['.', name], '(\W)(\w+)', 'tokens'); 37 | token = vertcat(token{:}).'; 38 | token(1,:) = strrep(strrep(token(1,:), '{', '{}'), '(', '()'); 39 | value_vector = str2double(token(2,:)); 40 | index = ~isnan(value_vector); 41 | token(2, index) = num2cell(num2cell(value_vector(index))); 42 | subscript = substruct(token{:}); 43 | if nargout 44 | try 45 | out.result = subsref(current_state, subscript); 46 | catch ME 47 | switch ME.identifier 48 | case 'MATLAB:nonExistentField' 49 | error('DataJoint:Config:InvalidKey', ... 50 | 'Setting `%s` does not exist', name); 51 | otherwise 52 | rethrow(ME); 53 | end 54 | end 55 | else 56 | out.result = []; 57 | end 58 | end 59 | if nargin == 2 60 | new_state = subsasgn(current_state, subscript, value); 61 | stateAccess('set', new_state); 62 | if strcmp(subscript(1).subs, 'use_32bit_dims') 63 | ternary = @(varargin)varargin{end-varargin{1}}; 64 | setenv('MYM_USE_32BIT_DIMS', ternary(value, 'true', 'false')); 65 | end 66 | end 67 | end 68 | function out = restore() 69 | out = stateAccess('restore'); 70 | end 71 | function save(fname) 72 | c = dj.internal.Settings; 73 | dj.lib.saveJSONfile(c.result, fname); 74 | end 75 | function load(fname) 76 | if ~nargin 77 | fname = dj.internal.Settings.LOCALFILE; 78 | end 79 | raw = fileread(fname); 80 | new_state = fixProps(jsondecode(raw), raw); 81 | stateAccess('load', new_state); 82 | end 83 | function saveLocal() 84 | dj.internal.Settings.save(dj.internal.Settings.LOCALFILE); 85 | end 86 | function saveGlobal() 87 | location = dj.internal.Settings.GLOBALFILE; 88 | if ispc 89 | location = strrep(location, '~', strrep(getenv('USERPROFILE'), '\', '/')); 90 | end 91 | dj.internal.Settings.save(location); 92 | end 93 | end 94 | end 95 | function data = fixProps(data, raw) 96 | newFields = fieldnames(data); 97 | for i=1:length(newFields) 98 | for j=1:length(data.(newFields{i})) 99 | if isstruct(data.(newFields{i})(j)) 100 | if exist('res','var') 101 | res(end + 1) = fixProps(data.(newFields{i})(j), raw); 102 | else 103 | res = fixProps(data.(newFields{i})(j), raw); 104 | end 105 | if j == length(data.(newFields{i})) 106 | data.(newFields{i}) = res; 107 | clear res; 108 | end 109 | end 110 | end 111 | newFields{i} = regexprep(regexp(raw, ... 112 | regexprep(newFields{i},'_','.'), 'match', 'once'), ... 113 | '\.[a-zA-Z0-9]','${upper($0(2))}'); 114 | end 115 | data = cell2struct(struct2cell(data), newFields); 116 | end 117 | function out = stateAccess(operation, new) 118 | function envVarUpdate() 119 | % optional environment variables specifying the connection. 120 | if getenv('DJ_HOST') 121 | STATE.databaseHost = getenv('DJ_HOST'); 122 | end 123 | if getenv('DJ_USER') 124 | STATE.databaseUser = getenv('DJ_USER'); 125 | end 126 | if getenv('DJ_PASS') 127 | STATE.databasePassword = getenv('DJ_PASS'); 128 | end 129 | if getenv('DJ_INIT') 130 | STATE.connectionInit_function = getenv('DJ_INIT'); 131 | end 132 | end 133 | switch nargin 134 | case 0 135 | operation = ''; 136 | case 1 137 | case 2 138 | otherwise 139 | error('Exceeded 2 input limit.') 140 | end 141 | persistent STATE 142 | if (isempty(STATE) && ~strcmpi(operation, 'load')) || strcmpi(operation, 'restore') 143 | % default settings 144 | STATE = orderfields(dj.internal.Settings.DEFAULTS); 145 | if exist(dj.internal.Settings.LOCALFILE, 'file') == 2 146 | dj.internal.Settings.load(dj.internal.Settings.LOCALFILE); 147 | elseif exist(dj.internal.Settings.GLOBALFILE, 'file') == 2 148 | dj.internal.Settings.load(dj.internal.Settings.GLOBALFILE); 149 | end 150 | envVarUpdate(); 151 | end 152 | % return STATE prior to change 153 | out = STATE; 154 | if any(strcmpi(operation, {'set', 'load'})) 155 | if isempty(STATE) 156 | STATE = rmfield(dj.internal.Settings.DEFAULTS, intersect(fieldnames( ... 157 | dj.internal.Settings.DEFAULTS), fieldnames(new))); 158 | names = [fieldnames(STATE); fieldnames(new)]; 159 | STATE = orderfields(... 160 | cell2struct([struct2cell(STATE); struct2cell(new)], names, 1)); 161 | else 162 | % merge with existing STATE 163 | STATE = rmfield(STATE, intersect(fieldnames(STATE), fieldnames(new))); 164 | names = [fieldnames(STATE); fieldnames(new)]; 165 | STATE = orderfields(... 166 | cell2struct([struct2cell(STATE); struct2cell(new)], names, 1)); 167 | end 168 | if strcmpi(operation, 'load') 169 | envVarUpdate(); 170 | end 171 | end 172 | end 173 | -------------------------------------------------------------------------------- /+dj/+internal/TableAccessor.m: -------------------------------------------------------------------------------- 1 | % dj.internal.TableAccessor allows access to tables without requiring the 2 | % creation of classes. 3 | % 4 | % Initialization: 5 | % v = dj.internal.TableAccessor(schema); 6 | % 7 | % MATLAB does not allow the creation of classes without creating 8 | % corresponding classes. 9 | % 10 | % TableAccessor provides a way to access all tables in a schema without 11 | % having to first create the classes. A TableAccessor object is created as 12 | % a property of a schema during the schema's creation. This property is 13 | % named schema.v for 'virtual class generator.' The TableAccessor v itself 14 | % has properties that refer to the tables of the schema. 15 | % 16 | % For example, one can access the Session table using schema.v.Session with 17 | % no need for any Session class to exist in Matlab. Tab completion of table 18 | % names is possible because the table names are added as dynamic properties 19 | % of TableAccessor. 20 | classdef TableAccessor < dynamicprops 21 | 22 | methods 23 | 24 | function self = TableAccessor(schema) 25 | for className = schema.classNames 26 | splitName = strsplit(className{1}, '.'); 27 | name = splitName{2}; 28 | addprop(self, name); 29 | self.(name) = dj.internal.UserRelation(className{1}); 30 | end 31 | end 32 | end 33 | 34 | end 35 | -------------------------------------------------------------------------------- /+dj/+internal/UserRelation.m: -------------------------------------------------------------------------------- 1 | classdef UserRelation < dj.Relvar & dj.internal.Master 2 | methods 3 | function self = UserRelation(varargin) 4 | self@dj.Relvar(varargin{:}) 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /+dj/+internal/ask.m: -------------------------------------------------------------------------------- 1 | function choice = ask(question,choices) 2 | if nargin<=1 3 | choices = {'yes','no'}; 4 | end 5 | choice = ''; 6 | choiceStr = sprintf('/%s',choices{:}); 7 | while ~ismember(choice, lower(choices)) 8 | choice = lower(input([question ' (' choiceStr(2:end) ') > '], 's')); 9 | end -------------------------------------------------------------------------------- /+dj/+internal/fromCamelCase.m: -------------------------------------------------------------------------------- 1 | function str = fromCamelCase(str) 2 | % converts CamelCase to underscore_compound_words. 3 | % 4 | % Examples: 5 | % fromCamelCase('oneTwoThree') --> 'one_two_three' 6 | % fromCamelCase('OneTwoThree') --> 'one_two_three' 7 | % fromCamelCase('one two three') --> !error! white space is not allowed 8 | % fromCamelCase('ABC') --> 'a_b_c' 9 | 10 | assert(isempty(regexp(str, '\s', 'once')), 'white space is not allowed') 11 | assert(~ismember(str(1), '0':'9'), 'string cannot begin with a digit') 12 | 13 | assert(~isempty(regexp(str, '^[a-zA-Z0-9]*$', 'once')), ... 14 | 'fromCamelCase string can only contain alphanumeric characters'); 15 | str = regexprep(str, '([A-Z])', '_${lower($1)}'); 16 | str = str(1+(str(1)=='_'):end); % remove leading underscore 17 | end -------------------------------------------------------------------------------- /+dj/+internal/hash.m: -------------------------------------------------------------------------------- 1 | function s = hash(data, inputType) 2 | if nargin<2 3 | inputType = 'array'; 4 | end 5 | s = strrep(strrep(... 6 | dj.lib.DataHash(data, struct('Input', inputType, 'Method', 'SHA-256', 'Format', 'base64')), ... 7 | '/', '_'), ... 8 | '+', '-'); % make URL safe -------------------------------------------------------------------------------- /+dj/+internal/shorthash.m: -------------------------------------------------------------------------------- 1 | function s = shorthash(varargin) 2 | s = dj.internal.hash(varargin{:}); 3 | s = s(1:8); -------------------------------------------------------------------------------- /+dj/+internal/toCamelCase.m: -------------------------------------------------------------------------------- 1 | function str = toCamelCase(str) 2 | % converts underscore_compound_words to CamelCase 3 | % and double underscores in the middle into dots 4 | % 5 | % Not always exactly invertible 6 | % 7 | % Examples: 8 | % toCamelCase('one') --> 'One' 9 | % toCamelCase('one_two_three') --> 'OneTwoThree' 10 | % toCamelCase('#$one_two,three') --> 'OneTwoThree' 11 | % toCamelCase('One_Two_Three') --> !error! upper case only mixes with alphanumerics 12 | % toCamelCase('5_two_three') --> !error! cannot start with a digit 13 | 14 | assert(isempty(regexp(str, '\s', 'once')), 'white space is not allowed') 15 | assert(~ismember(str(1), '0':'9'), 'string cannot begin with a digit') 16 | assert(isempty(regexp(str, '[A-Z]', 'once')), ... 17 | 'underscore_compound_words must not contain uppercase characters') 18 | str = regexprep(str, '(^|[_\W]+)([a-zA-Z])', '${upper($2)}'); 19 | end -------------------------------------------------------------------------------- /+dj/+lib/HMAC.m: -------------------------------------------------------------------------------- 1 | function hash = HMAC(key,message,method) 2 | % Copyright (c) 2014, Peter Grunnet Wang 3 | % All rights reserved. 4 | 5 | % Redistribution and use in source and binary forms, with or without 6 | % modification, are permitted provided that the following conditions are met: 7 | 8 | % * Redistributions of source code must retain the above copyright notice, this 9 | % list of conditions and the following disclaimer. 10 | 11 | % * Redistributions in binary form must reproduce the above copyright notice, 12 | % this list of conditions and the following disclaimer in the documentation 13 | % and/or other materials provided with the distribution 14 | % * Neither the name of nor the names of its 15 | % contributors may be used to endorse or promote products derived from this 16 | % software without specific prior written permission. 17 | % THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | % AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | % IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | % DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 21 | % FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | % DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | % SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | % CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | % OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | % OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | % 28 | % key: input secret key in char 29 | % message: input message in char 30 | % method: hash method, either: 31 | % 'SHA-1', 'SHA-256', 'SHA-384', 'SHA-512' 32 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 33 | if strcmp(method,'SHA-1') == 1 34 | Blocksize = 64; 35 | elseif strcmp(method,'SHA-256') == 1 36 | Blocksize = 64; 37 | elseif strcmp(method,'SHA-384') == 1 38 | Blocksize = 128; 39 | elseif strcmp(method,'SHA-512') == 1 40 | Blocksize = 128; 41 | else 42 | error('HMAC:Algorithm:Unavailable', ... 43 | 'Algorithm `%s` not available.', method); 44 | end 45 | % if key length > Blocksize calculate Hash and format as binary 46 | if length(key) > Blocksize 47 | Opt.Method = method; 48 | Opt.Format = 'uint8'; 49 | Opt.Input = 'bin'; 50 | 51 | Hash_key = dj.lib.DataHash(uint8(key),Opt) 52 | 53 | for i = length(Hash_key):Blocksize 54 | Hash_key(1,i) = 0; 55 | end 56 | key_bin = uint82bin8(Hash_key); 57 | end 58 | % if key length < Blocksize right pad with zeros and format as binary 59 | if (length(key) > 0) && (length(key) < Blocksize) 60 | key_bin = str2bin8(key); 61 | L = length(key); 62 | for j = L+1:Blocksize 63 | key_bin{1,j} = [0 0 0 0 0 0 0 0]; 64 | end 65 | end 66 | % if key length = 0 right pad with zeros and format as binary 67 | if length(key) ==0 68 | for j = 1:Blocksize 69 | key_bin{1,j} = [0 0 0 0 0 0 0 0]; 70 | end 71 | end 72 | % if key length = Blocksize format key as binary numbers 73 | if length(key) == Blocksize 74 | key_bin = str2bin8(key); 75 | end 76 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 77 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 78 | % format inner and outer padding as binary cell arrays 79 | i_pad = [0 0 1 1 0 1 1 0]; 80 | o_pad = [0 1 0 1 1 1 0 0]; 81 | for i = 1:Blocksize 82 | i_pad_bin{1,i} = i_pad; 83 | o_pad_bin{1,i} = o_pad; 84 | end 85 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 86 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 87 | % calculate key xor ipad and key xor opad 88 | i_pad_key_bin = bit8xor(key_bin,i_pad_bin); 89 | o_pad_key_bin = bit8xor(key_bin,o_pad_bin); 90 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 91 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 92 | % Change format to uint8 93 | i_pad_key_hex = bin82hex(i_pad_key_bin); 94 | i_pad_key_uint8 = hex2uint8(i_pad_key_hex); 95 | o_pad_key_hex = bin82hex(o_pad_key_bin); 96 | o_pad_key_uint8 = hex2uint8(o_pad_key_hex); 97 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 98 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 99 | % concatenate (i_pad_key || message) 100 | concat_i_pad = [i_pad_key_uint8,uint8(message)]; 101 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 102 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 103 | % Calculate Hash of i_pad_key||message 104 | Opt.Method = method; 105 | Opt.Format = 'uint8'; 106 | Opt.Input = 'bin'; 107 | Hash_i_pad_uint8 = dj.lib.DataHash(concat_i_pad,Opt); 108 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 109 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 110 | % concatenate (o_pad || Hash_i_pad) 111 | concat_o_pad = [o_pad_key_uint8,Hash_i_pad_uint8]; 112 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 113 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 114 | % Calculate final Hash in HEX 115 | Opt.Method = method; 116 | Opt.Format = 'HEX'; 117 | Opt.Input = 'bin'; 118 | hash = dj.lib.DataHash(concat_o_pad,Opt); 119 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 120 | end 121 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 122 | % FORMAT HELP FUNCTIONS 123 | function bin = str2bin8(str) 124 | for i = 1:length(str) 125 | temp = dec2bin(str2num(sprintf('%u',str(i))),8); 126 | for j = 1:8 127 | bin{1,i}(1,j) = str2num(temp(j)); 128 | end 129 | end 130 | end 131 | function hex = bin82hex(bin) 132 | hex = cell(1,length(bin)); 133 | for i = 1:length(bin) 134 | string_bin = num2str(bin{1,i}(1,1)) ; 135 | for j = 2:8 136 | temp = num2str(bin{1,i}(1,j)); 137 | string_bin = strcat(string_bin,temp); 138 | end 139 | hex{1,i} = dec2hex(bin2dec(string_bin)); 140 | end 141 | end 142 | function v = bit8xor(a,b) 143 | if length(a) == length(b) 144 | for i = 1:length(a) 145 | v{1,i} = xor(a{1,i},b{1,i}); 146 | end 147 | else 148 | ERROR = sprintf('%s','Input cells must be the same length') 149 | end 150 | end 151 | function out = hex2uint8(hex) 152 | for i = 1:length(hex) 153 | out(i) = hex2dec(hex{1,i}); 154 | end 155 | out = uint8(out); 156 | end 157 | function bin = uint82bin8(in) 158 | bin = cell(1,length(in)); 159 | for i = 1:length(in) 160 | temp = dec2bin(in(1,i),8); 161 | for j = 1:8 162 | bin{1,i}(1,j) = str2num(temp(j)); 163 | end 164 | end 165 | end 166 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -------------------------------------------------------------------------------- /+dj/+lib/getpass.m: -------------------------------------------------------------------------------- 1 | function pass = getpass(prompt) 2 | %GETPASS Open up a dialog box for the user to enter password. Password 3 | %will be hidden as the user enters text. You can pass in an optional 4 | %argument prompt to be used as the dialog box title. Defaults to "Enter 5 | %password". 6 | 7 | if nargin < 1 8 | prompt = 'Enter password'; 9 | end 10 | 11 | screenSize = get(0, 'ScreenSize'); 12 | 13 | % configure the diaglog box 14 | hfig = figure( ... 15 | 'Menubar', 'none', ... 16 | 'Units', 'Pixels', ... 17 | 'NumberTitle', 'off', ... 18 | 'Resize', 'off', ... 19 | 'Name', prompt, ... 20 | 'Position', [(screenSize(3:4)-[300 75])/2 300 75], ... 21 | 'Color', [0.8 0.8 0.8], ... 22 | 'WindowStyle', 'modal'); 23 | 24 | hpass = uicontrol( ... 25 | 'Parent', hfig, ... 26 | 'Style', 'Text', ... 27 | 'Tag', 'password', ... 28 | 'Units', 'Pixels', ... 29 | 'Position', [51 30 198 18], ... 30 | 'FontSize', 15, ... 31 | 'BackGroundColor', [1 1 1]); 32 | 33 | set(hfig,'KeyPressFcn',{@keypress_cb, hpass}, 'CloseRequestFcn','uiresume') 34 | 35 | % wait for password entry 36 | uiwait 37 | pass = get(hpass,'userdata'); 38 | if isempty(pass) 39 | pass = ''; 40 | end 41 | 42 | % remove the figure to prevent passwork leakage 43 | delete(hfig) 44 | 45 | end 46 | 47 | function keypress_cb(hObj, data, hpass) 48 | % Callback function to handle actual key strokes 49 | 50 | pass = get(hpass,'userdata'); 51 | 52 | switch data.Key 53 | case 'backspace' 54 | pass = pass(1:end-1); 55 | case 'return' 56 | uiresume 57 | return 58 | otherwise 59 | % append the typed character 60 | pass = [pass data.Character]; 61 | end 62 | set(hpass, 'userdata', pass) 63 | set(hpass, 'String', char('*' * sign(pass))) 64 | end 65 | -------------------------------------------------------------------------------- /+dj/+lib/isString.m: -------------------------------------------------------------------------------- 1 | function ret = isString(s) 2 | % a MATLAB version-safe function that tells returns true if 3 | % argument is a string or a character array 4 | ret = ischar(s) || exist('isstring', 'builtin') && isstring(s) && isscalar(s); 5 | end -------------------------------------------------------------------------------- /+dj/+lib/saveJSONfile.m: -------------------------------------------------------------------------------- 1 | function saveJSONfile(data, jsonFileName) 2 | % Modified from FileExchange entry: 3 | % https://www.mathworks.com/matlabcentral/fileexchange/... 4 | % 50965-structure-to-json?focused=3876199&tab=function 5 | % saves the values in the structure 'data' to a file in JSON indented format. 6 | % 7 | % Example: 8 | % data.name = 'chair'; 9 | % data.color = 'pink'; 10 | % data.metrics.height = 0.3; 11 | % data.metrics.width = 1.3; 12 | % saveJSONfile(data, 'out.json'); 13 | % 14 | % Output 'out.json': 15 | % { 16 | % "name" : "chair", 17 | % "color" : "pink", 18 | % "metrics" : { 19 | % "height" : 0.3, 20 | % "width" : 1.3 21 | % } 22 | % } 23 | % 24 | fid = fopen(jsonFileName,'w'); 25 | writeElement(fid, data,''); 26 | fprintf(fid,'\n'); 27 | fclose(fid); 28 | end 29 | function writeElement(fid, data,tabs) 30 | namesOfFields = fieldnames(data); 31 | tabs = sprintf('%s\t',tabs); 32 | fprintf(fid,'{\n%s',tabs); 33 | key = true; 34 | for i = 1:length(namesOfFields) - 1 35 | currentField = namesOfFields{i}; 36 | currentElementValue = data.(currentField); 37 | writeSingleElement(fid, currentField,currentElementValue,tabs, key); 38 | fprintf(fid,',\n%s',tabs); 39 | end 40 | currentField = namesOfFields{end}; 41 | currentElementValue = data.(currentField); 42 | writeSingleElement(fid, currentField,currentElementValue,tabs, key); 43 | fprintf(fid,'\n%s}',tabs(1:end-1)); 44 | end 45 | function writeSingleElement(fid, currentField,currentElementValue,tabs, key) 46 | % if this is an array and not a string then iterate on every 47 | % element, if this is a single element write it 48 | currentField = regexprep(currentField,'[a-z0-9][A-Z]','${$0(1)}.${lower($0(2))}'); 49 | if key 50 | fprintf(fid,'"%s" : ' , currentField); 51 | end 52 | if length(currentElementValue) > 1 && ~ischar(currentElementValue) 53 | fprintf(fid,'[\n%s\t',tabs); 54 | for m = 1:length(currentElementValue)-1 55 | if isstruct(currentElementValue(m)) 56 | writeElement(fid, currentElementValue(m),tabs); 57 | else 58 | writeSingleElement(fid, '',currentElementValue(m),tabs, false) 59 | end 60 | fprintf(fid,',\n%s\t',tabs); 61 | end 62 | if isstruct(currentElementValue(end)) 63 | writeElement(fid, currentElementValue(end),tabs); 64 | else 65 | writeSingleElement(fid, '',currentElementValue(end),tabs, false) 66 | end 67 | fprintf(fid,'\n%s]',tabs); 68 | elseif isstruct(currentElementValue) 69 | writeElement(fid, currentElementValue,tabs); 70 | elseif isempty(currentElementValue) 71 | fprintf(fid,'null'); 72 | elseif isnumeric(currentElementValue) 73 | fprintf(fid,'%g' ,currentElementValue); 74 | elseif islogical(currentElementValue) 75 | if currentElementValue 76 | fprintf(fid,'true'); 77 | else 78 | fprintf(fid,'false'); 79 | end 80 | else %ischar or something else ... 81 | fprintf(fid,'"%s"',currentElementValue); 82 | end 83 | end -------------------------------------------------------------------------------- /+dj/+lib/startsWith.m: -------------------------------------------------------------------------------- 1 | function ret = startsWith(s, pattern) 2 | % a MATLAB version-safe function that checks whether the string s 3 | % starts with the given pattern 4 | 5 | ret = strncmp(s, pattern, length(pattern)); 6 | 7 | end 8 | 9 | -------------------------------------------------------------------------------- /+dj/+store_plugins/File.m: -------------------------------------------------------------------------------- 1 | % dj.store_plugins.File - an external storage class for local file stores. 2 | classdef File 3 | properties (Hidden, Constant) 4 | % mode = -1(reject), 0(optional), 1(require) 5 | validation_config = struct( ... 6 | 'datajoint_type', struct( ... 7 | 'mode', @(datajoint_type) 1, ... 8 | 'type_check', @(self) ischar(self) && any(strcmpi(... 9 | self, {'blob', 'filepath'}))... 10 | ), ... 11 | 'protocol', struct( ... 12 | 'mode', @(datajoint_type) 1, ... 13 | 'type_check', @(self) ischar(self) ... 14 | ), ... 15 | 'location', struct( ... 16 | 'mode', @(datajoint_type) 1, ... 17 | 'type_check', @(self) ischar(self) ... 18 | ), ... 19 | 'subfolding', struct( ... 20 | 'mode', @(datajoint_type) -1 + any(strcmpi(datajoint_type, {'blob'})), ... 21 | 'type_check', @(self) all(floor(self) == self), ... 22 | 'default', [2; 2] ... 23 | ) ... 24 | ) 25 | backward_validation_config = struct( ... 26 | 'protocol', struct( ... 27 | 'mode', @(unused) 1, ... 28 | 'type_check', @(self) ischar(self) ... 29 | ), ... 30 | 'location', struct( ... 31 | 'mode', @(unused) 1, ... 32 | 'type_check', @(self) ischar(self) ... 33 | ), ... 34 | 'subfolding', struct( ... 35 | 'mode', @(unused) 0, ... 36 | 'type_check', @(self) all(floor(self) == self), ... 37 | 'default', [2; 2] ... 38 | ) ... 39 | ) 40 | end 41 | properties 42 | protocol 43 | datajoint_type 44 | location 45 | type_config 46 | end 47 | methods (Static) 48 | function remove_object(external_filepath) 49 | % delete an object from the store 50 | delete(external_filepath); 51 | end 52 | function upload_buffer(buffer, external_filepath) 53 | % put blob 54 | [~,start_idx,~] = regexp(external_filepath, '/', 'match', 'start', 'end'); 55 | if ~isfolder(external_filepath(1:(start_idx(end)-1))) 56 | mkdir(external_filepath(1:(start_idx(end)-1))); 57 | end 58 | fileID = fopen(external_filepath, 'w'); 59 | fwrite(fileID, buffer); 60 | fclose(fileID); 61 | end 62 | function result = download_buffer(external_filepath) 63 | % get blob 64 | fileID = fopen(external_filepath, 'r'); 65 | result = fread(fileID); 66 | fclose(fileID); 67 | end 68 | function result = exists(external_filepath) 69 | % get blob metadata 70 | result = exist(external_filepath, 'file') == 2 || ... 71 | exist(external_filepath, 'dir') == 7; 72 | end 73 | end 74 | methods 75 | function self = File(config) 76 | % initialize store 77 | self.protocol = config.protocol; 78 | self.location = strrep(config.location, '\', '/'); 79 | 80 | self.type_config = struct(); 81 | if dj.internal.ExternalTable.BACKWARD_SUPPORT_DJPY012 && ~any(strcmp(... 82 | 'datajoint_type', fieldnames(config))) 83 | self.type_config.subfolding = config.subfolding; 84 | else 85 | self.datajoint_type = config.datajoint_type; 86 | if strcmpi(self.datajoint_type, 'blob') 87 | self.type_config.subfolding = config.subfolding; 88 | end 89 | end 90 | if ~self.exists(self.location) 91 | error('DataJoint:FilePlugin:DirectoryInaccessible', ... 92 | 'Directory `%s` not accessible.', self.location); 93 | end 94 | end 95 | function external_filepath = make_external_filepath(self, relative_filepath) 96 | % resolve the complete external path based on the relative path 97 | external_filepath = [self.location '/' relative_filepath]; 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /+dj/AutoPopulate.m: -------------------------------------------------------------------------------- 1 | % dj.AutoPopulate is deprecated and will be removed in future versions 2 | 3 | classdef AutoPopulate < dj.internal.AutoPopulate 4 | 5 | methods 6 | function self = AutoPopulate 7 | warning('DataJoint:deprecate', ... 8 | 'dj.AutoPopulate is deprecated and will be removed in future versions. Replace with dj.Imported and dj.Computed (subclasses of dj.Relvar).') 9 | end 10 | end 11 | end -------------------------------------------------------------------------------- /+dj/BatchPopulate.m: -------------------------------------------------------------------------------- 1 | % dj.BatchPopulate is deprecated and will be removed in future versions 2 | 3 | % dj.BatchPopulate is an abstract mixin class that allows a dj.Relvar object 4 | % to automatically populate its table by pushing jobs to a cluster via the 5 | % Parallel Computing Toolbox 6 | % 7 | 8 | 9 | classdef BatchPopulate < dj.AutoPopulate 10 | 11 | methods 12 | 13 | function self = BatchPopulate() 14 | warning('DataJoint:deprecate', ... 15 | 'dj.BatchPopulate is no longer supported and will be removed in future versions') 16 | end 17 | 18 | function varargout = batch_populate(self, varargin) 19 | % dj.BatchPopulate/batch_populate works identically to dj.AutoPopulate/parpopulate 20 | % except that it spools jobs to the cluster. It creates one job per key, 21 | % so using batch_populate only makes sense for long-running calculations. 22 | % 23 | % The job reservation table .Jobs required by parpopulate is also 24 | % used by batch_populate. 25 | % See also dj.AutoPopulate/parpopulate 26 | 27 | self.schema.conn.cancelTransaction % rollback any unfinished transaction 28 | self.useReservations = true; 29 | self.populateSanityChecks 30 | self.executionEngine = @(key, fun, args) ... 31 | batchExecutionEngine(self, key, fun, args); 32 | 33 | [varargout{1:nargout}] = self.populate_(varargin{:}); 34 | end 35 | end 36 | 37 | 38 | methods(Access = protected) 39 | function user_data = get_job_user_data(self, key) %#ok 40 | % Override this function in order to customize the UserData argument 41 | % for job creation. This can be used to supply the io_resource field 42 | % for example. 43 | user_data = struct(); 44 | end 45 | 46 | function batchExecutionEngine(self, key, fun, args) 47 | % getDefaultScheduler returns an instance of the job scheduler we 48 | % use in the lab. 49 | % For general use, replace with 50 | % sge = parcluster; 51 | % or 52 | % sge = parcluster('profile_name') 53 | % for some appropriate profile that has been created. 54 | 55 | sge = getDefaultScheduler(); 56 | 57 | % setPath() returns a cell array with all aditional MATLAB 58 | % paths on the shared file system that should be passed on to 59 | % the worker 60 | pathDeps = setPath(); 61 | j = createJob(sge, batch_path_keyword(), pathDeps, ... 62 | 'UserData', self.get_job_user_data(key)); 63 | createTask(j, fun, 0, args); 64 | submit(j); 65 | end 66 | end 67 | end 68 | 69 | 70 | 71 | 72 | function path_kw = batch_path_keyword() 73 | % Returns the keyword that is used to pass the user path 74 | % to a cluster job. This is due to ridiculous API 75 | % changes in the PCT by Mathworks. 76 | 77 | if verLessThan('matlab', '7.12') 78 | path_kw = 'PathDependencies'; 79 | else 80 | path_kw = 'AdditionalPaths'; 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /+dj/Computed.m: -------------------------------------------------------------------------------- 1 | classdef Computed < dj.internal.AutoPopulate 2 | % defines a computed table 3 | properties(Constant) 4 | tierRegexp = sprintf('(?%s%s)', ... 5 | dj.Schema.tierPrefixes{strcmp(dj.Schema.allowedTiers, 'computed')}, ... 6 | dj.Schema.baseRegexp) 7 | end 8 | methods 9 | function self = Computed(varargin) 10 | self@dj.internal.AutoPopulate(varargin{:}) 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /+dj/Connection.m: -------------------------------------------------------------------------------- 1 | classdef Connection < handle 2 | 3 | properties(SetAccess = private) 4 | host 5 | user 6 | initQuery % initializing function or query executed for each new session 7 | use_tls 8 | inTransaction = false 9 | connId % connection handle 10 | serverId % database connection id 11 | packages % maps database names to package names 12 | schemas % registered schema objects 13 | 14 | % dependency lookups by table name 15 | foreignKeys % maps table names to their referenced table names (primary foreign key) 16 | end 17 | 18 | properties(Access = private) 19 | password 20 | end 21 | 22 | properties(Dependent) 23 | isConnected 24 | end 25 | 26 | methods 27 | 28 | function self=Connection(host, username, password, initQuery, use_tls) 29 | % specify the connection to the database. 30 | % initQuery is the SQL query to be executed at the start 31 | % of each new session. 32 | dj.setup('prompt', dj.config('safemode')); 33 | self.host = host; 34 | self.user = username; 35 | self.password = password; 36 | if nargin>=4 37 | self.initQuery = initQuery; 38 | end 39 | if nargin>=5 40 | self.use_tls = use_tls; 41 | end 42 | self.foreignKeys = struct([]); 43 | self.packages = containers.Map; 44 | self.schemas = containers.Map; 45 | end 46 | 47 | 48 | function register(self, schema) 49 | self.schemas(schema.dbname) = schema; 50 | end 51 | 52 | function addPackage(self, dbname, package) 53 | self.packages(dbname) = package; 54 | end 55 | 56 | 57 | function loadDependencies(self, schema) 58 | % load dependencies from SHOW CREATE TABLE 59 | pat = cat(2,... 60 | 'FOREIGN KEY\s+\((?[`\w, ]+)\)\s+',... % attrs1 61 | 'REFERENCES\s+(?[^\s]+)\s+',... % referenced table name 62 | '\((?[`\w, ]+)\)'); 63 | 64 | for tabName = schema.headers.keys 65 | fk = self.query(sprintf('SHOW CREATE TABLE `%s`.`%s`', schema.dbname, ... 66 | tabName{1})); 67 | fk = strtrim(regexp(fk.('Create Table'){1},'\n','split')'); 68 | fk = regexp(fk, pat, 'names'); 69 | fk = [fk{:}]; 70 | from = sprintf('`%s`.`%s`', schema.dbname, tabName{1}); 71 | 72 | for s=fk 73 | s.from = from; 74 | s.ref = s.ref; 75 | s.attrs = regexp(s.attrs, '\w+', 'match'); 76 | s.ref_attrs = regexp(s.ref_attrs, '\w+', 'match'); 77 | s.primary = all(ismember(s.attrs, schema.headers(tabName{1}).primaryKey)); 78 | s.multi = ~all(ismember(schema.headers(tabName{1}).primaryKey, s.attrs)); 79 | if isempty(regexp(s.ref,'`\.`','once')) 80 | s.ref = sprintf('`%s`.%s',schema.dbname,s.ref); 81 | end 82 | s.aliased = ~isequal(s.attrs, s.ref_attrs); 83 | self.foreignKeys = [self.foreignKeys, s]; 84 | end 85 | end 86 | end 87 | 88 | 89 | function [names, isprimary] = parents(self, child, primary) 90 | if isempty(self.foreignKeys) 91 | names = {}; 92 | isprimary = []; 93 | else 94 | ix = strcmp(child, {self.foreignKeys.from}); 95 | if nargin>2 96 | ix = ix & primary == [self.foreignKeys.primary]; 97 | end 98 | names = {self.foreignKeys(ix).ref}; 99 | if nargout > 1 100 | isprimary = [self.foreignKeys(ix).primary]; 101 | end 102 | end 103 | end 104 | 105 | 106 | function [names, isprimary] = children(self, parent, primary) 107 | if isempty(self.foreignKeys) 108 | names = {}; 109 | isprimary = []; 110 | else 111 | ix = strcmp(parent, {self.foreignKeys.ref}); 112 | if nargin>2 113 | ix = ix & primary == [self.foreignKeys.primary]; 114 | end 115 | names = {self.foreignKeys(ix).from}; 116 | if nargout > 1 117 | isprimary = [self.foreignKeys(ix).primary]; 118 | end 119 | end 120 | end 121 | 122 | 123 | function className = tableToClass(self, fullTableName, strict) 124 | % convert '`dbname`.`table_name`' to 'package.ClassName' 125 | % If strict (false by default), throw error if the dbname is not found. 126 | % If not strict and the name is not found, then className=tableName 127 | 128 | strict = nargin>=3 && strict; 129 | s = regexp(fullTableName, '^`(?.+)`.`(?[#~\w\d]+)`$','names'); 130 | className = fullTableName; 131 | if ~isempty(s) && self.packages.isKey(s.dbname) 132 | className = sprintf('%s.%s',self.packages(s.dbname),dj.internal.toCamelCase(... 133 | s.tablename)); 134 | elseif strict 135 | error('Unknown package for "%s". Activate its schema first.', fullTableName) 136 | end 137 | end 138 | 139 | 140 | function reload(self) 141 | % reload all schemas 142 | self.clearDependencies 143 | for s=self.packages.values 144 | reload(feval([s{1} '.getSchema'])) 145 | end 146 | end 147 | 148 | 149 | function ret = get.isConnected(self) 150 | ret = ~isempty(self.connId) && 0==mym(self.connId, 'status'); 151 | 152 | if ~ret && self.inTransaction 153 | if dj.config('databaseReconnect_transaction') 154 | warning 'Reconnected after server disconnected during a transaction' 155 | else 156 | error 'Server disconnected during a transaction' 157 | end 158 | end 159 | end 160 | 161 | 162 | function ret = query(self, queryStr, varargin) 163 | % dj.Connection/query - query(connection, queryStr, varargin) issue an 164 | % SQL query and return the result if any. 165 | % The same connection is re-used by all DataJoint objects. 166 | if ~self.isConnected 167 | self.connId=mym(-1, 'open', self.host, self.user, self.password, self.use_tls); 168 | 169 | if ~isempty(self.initQuery) 170 | mym(self.connId, self.initQuery); 171 | end 172 | 173 | tmp = mym(self.connId, 'SELECT CONNECTION_ID() as id'); 174 | self.serverId = uint64(tmp.id); 175 | 176 | end 177 | 178 | v = varargin; 179 | if dj.config('queryBigint_to_double') 180 | v{end+1} = 'bigint_to_double'; 181 | end 182 | if nargout>0 183 | ret=mym(self.connId, queryStr, v{:}); 184 | else 185 | mym(self.connId, queryStr, v{:}); 186 | end 187 | end 188 | 189 | 190 | function startTransaction(self) 191 | self.query('START TRANSACTION WITH CONSISTENT SNAPSHOT') 192 | self.inTransaction = true; 193 | end 194 | 195 | 196 | function commitTransaction(self) 197 | assert(self.inTransaction, 'No transaction to commit') 198 | self.query('COMMIT') 199 | self.inTransaction = false; 200 | end 201 | 202 | 203 | function cancelTransaction(self) 204 | self.inTransaction = false; 205 | self.query('ROLLBACK') 206 | end 207 | 208 | 209 | function close(self) 210 | if self.isConnected 211 | fprintf('closing DataJoint connection #%d\n', self.connId) 212 | mym(self.connId, 'close') 213 | end 214 | self.inTransaction = false; 215 | end 216 | 217 | 218 | function delete(self) 219 | self.clearDependencies 220 | self.close 221 | end 222 | 223 | 224 | function clearDependencies(self, schema) 225 | if nargin<2 226 | % remove all if the schema is not specified 227 | self.foreignKeys = struct([]); 228 | elseif ~isempty(self.foreignKeys) 229 | % remove references from the given schema 230 | % self.referenced.remove 231 | tableNames = cellfun(@(s) ... 232 | sprintf('`%s`.`%s`', schema.dbname, s), ... 233 | schema.tableNames.values, 'uni', false); 234 | self.foreignKeys(ismember({self.foreignKeys.from}, tableNames)) = []; 235 | end 236 | end 237 | 238 | end 239 | end 240 | -------------------------------------------------------------------------------- /+dj/ERD.m: -------------------------------------------------------------------------------- 1 | classdef ERD < handle 2 | % Entity relationship diagram (ERD) is a directed graph formed between 3 | % nodes (tables in the database) connected with foreign key dependencies. 4 | 5 | properties(SetAccess=protected) 6 | conn % a dj.Connection object contianing dependencies 7 | nodes % list of nodes 8 | graph % the digraph object 9 | end 10 | 11 | 12 | methods 13 | 14 | function self = ERD(obj) 15 | % initialize ERD node list. obj may be a schema object or a 16 | % single Relvar object or another ERD. 17 | if nargin==0 18 | % default constructor 19 | self.conn = []; 20 | self.graph = []; 21 | self.nodes = {}; 22 | else 23 | switch true 24 | case isa(obj, 'dj.internal.Table') 25 | self.nodes = {obj.fullTableName}; 26 | self.conn = obj.schema.conn; 27 | case isa(obj, 'dj.Schema') 28 | self.nodes = cellfun(@(s) sprintf('`%s`.`%s`', obj.dbname, s), ... 29 | obj.tableNames.values, 'uni', false); 30 | self.conn = obj.conn; 31 | case isa(obj, 'dj.ERD') 32 | % copy constructor 33 | self.nodes = obj.nodes; 34 | self.conn = obj.conn; 35 | self.graph = obj.graph; 36 | otherwise 37 | error 'invalid ERD initalization' 38 | end 39 | end 40 | end 41 | 42 | function up(self) 43 | % add one layer of nodes upstream in hierarchy 44 | parents = cellfun(@(s) self.conn.parents(s), self.nodes, 'uni', false); 45 | self.nodes = union(self.nodes, cat(2, parents{:})); 46 | end 47 | 48 | function down(self) 49 | % add one layer of nodes downstream in hierarchy 50 | children = cellfun(@(s) self.conn.children(s), self.nodes, 'uni', false); 51 | self.nodes = union(self.nodes, cat(2,children{:})); 52 | end 53 | 54 | 55 | function ret = plus(self, obj) 56 | % union of ERD graphs 57 | % A + B is an ERD with all the nodes from A and B. 58 | % or when B is an integer, expand A by B levels upstream. 59 | 60 | ret = dj.ERD(self); % copy 61 | switch true 62 | case isa(obj, 'dj.ERD') 63 | if isempty(ret.nodes) 64 | ret = dj.ERD(obj); 65 | else 66 | ret.nodes = union(ret.nodes, obj.nodes); 67 | end 68 | case isnumeric(obj) 69 | n = length(ret.nodes); 70 | for i=1:obj 71 | ret.down 72 | if length(ret.nodes)==n 73 | break 74 | end 75 | n = length(ret.nodes); 76 | end 77 | otherwise 78 | error 'invalid ERD addition argument' 79 | end 80 | end 81 | 82 | 83 | function ret = minus(self, obj) 84 | % difference of ERD graphs 85 | % A - B is an ERD with all the nodes from A that are not in B. 86 | % or when B is an integer, expand A by B levels downstream. 87 | 88 | ret = dj.ERD(self); % copy 89 | switch true 90 | case isa(obj, 'dj.ERD') 91 | ret.nodes = setdiff(ret.nodes, obj.nodes); 92 | case isnumeric(obj) 93 | n = length(ret.nodes); 94 | for i=1:obj 95 | ret.up 96 | if length(ret.nodes)==n 97 | break 98 | end 99 | n = length(ret.nodes); 100 | end 101 | otherwise 102 | error 'invalid ERD difference argument' 103 | end 104 | end 105 | 106 | 107 | function display(self) 108 | self.draw 109 | end 110 | 111 | 112 | function draw(self) 113 | % draw the diagram 114 | 115 | % exclude auxiliary tables (job tables, etc.) 116 | j = cellfun(@isempty, regexp(self.nodes, '^`[a-zA-z0-9_-]+`\.`~\w+`$')); 117 | self.nodes = self.nodes(j); 118 | 119 | self.makeGraph 120 | 121 | rege = cellfun(@(s) sprintf('^`[a-zA-z0-9_-]+`\\.`%s[a-z]\\w*`$',s), ... 122 | dj.Schema.tierPrefixes, 'uni', false); 123 | rege{end+1} = '^`[a-zA-z0-9_-]+`\.`\W?\w+__\w+`$'; % for part tables 124 | rege{end+1} = '^\d+$'; % for numbered nodes 125 | tiers = cellfun(@(l) find(~cellfun(@isempty, regexp(l, rege)), 1, 'last'), ... 126 | self.graph.Nodes.Name); 127 | colormap(0.3+0.7*[ 128 | 0.3 0.3 0.3 129 | 0.0 0.5 0.0 130 | 0.0 0.0 1.0 131 | 1.0 0.0 0.0 132 | 1.0 1.0 1.0 133 | 0.0 0.0 0.0 134 | 1.0 0.0 0.0 135 | ]); 136 | marker = {'hexagram' 'square' 'o' 'pentagram' '.' '.' '.'}; 137 | self.graph.Nodes.marker = marker(tiers)'; 138 | h = self.graph.plot('layout', 'layered', 'NodeLabel', []); 139 | h.NodeCData = tiers; 140 | caxis([0.5 7.5]) 141 | h.MarkerSize = 12; 142 | h.Marker = self.graph.Nodes.marker; 143 | h.EdgeColor = [1 1 1]*0; 144 | h.EdgeAlpha = 0.25; 145 | axis off 146 | for i=1:self.graph.numnodes 147 | if tiers(i)<7 % ignore jobs, logs, etc. 148 | isPart = tiers(i)==6; 149 | fs = dj.config('displayDiagram_font_size')*(1 - 0.3*isPart); 150 | fc = isPart*0.3*[1 1 1]; 151 | name = self.conn.tableToClass(self.graph.Nodes.Name{i}); 152 | text(h.XData(i)+0.1, h.YData(i), name, ... 153 | 'fontsize', fs, 'rotation', -16, 'color', fc, ... 154 | 'Interpreter', 'none'); 155 | end 156 | end 157 | if self.graph.numedges 158 | line_widths = [1 2]; 159 | h.LineWidth = line_widths(2-self.graph.Edges.primary); 160 | line_styles = {'-', ':'}; 161 | h.LineStyle = line_styles(2-self.graph.Edges.primary); 162 | ee = cellfun(@(e) find(strcmp(e, self.graph.Nodes.Name), 1, 'first'), ... 163 | self.graph.Edges.EndNodes(~self.graph.Edges.multi,:)); 164 | highlight(h, ee(:,1), ee(:,2), 'LineWidth', 3) 165 | ee = cellfun(@(e) find(strcmp(e, self.graph.Nodes.Name), 1, 'first'), ... 166 | self.graph.Edges.EndNodes(self.graph.Edges.aliased,:)); 167 | highlight(h, ee(:,1), ee(:,2), 'EdgeColor', 'r') 168 | end 169 | figure(gcf) % bring figure to foreground 170 | end 171 | end 172 | 173 | 174 | methods(Access=protected) 175 | 176 | function makeGraph(self) 177 | % take foreign key and construct a digraph including all the 178 | % nodes from the list 179 | 180 | list = self.nodes; 181 | if isempty(self.conn.foreignKeys) 182 | ref = []; 183 | from = []; 184 | else 185 | from = arrayfun(@(item) find(strcmp(item.from, list)), ... 186 | self.conn.foreignKeys, 'uni', false); 187 | ref = arrayfun(@(item) find(strcmp(item.ref, list)), ... 188 | self.conn.foreignKeys, 'uni', false); 189 | ix = ~cellfun(@isempty, from) & ~cellfun(@isempty, ref); 190 | if ~isempty(ref) 191 | primary = [self.conn.foreignKeys(ix).primary]; 192 | aliased = [self.conn.foreignKeys(ix).aliased]; 193 | multi = [self.conn.foreignKeys(ix).multi]; 194 | ref = [ref{ix}]; 195 | from = [from{ix}]; 196 | % for every renamed edge, introduce a new node 197 | for m = find(aliased) 198 | t = length(list)+1; 199 | list{t} = sprintf('%d',t); 200 | q = length(ref)+1; 201 | ref(q) = ref(m); 202 | from(q) = t; 203 | ref(m) = t; 204 | primary(q) = primary(m); 205 | aliased(q) = aliased(m); 206 | multi(q) = multi(m); 207 | end 208 | end 209 | end 210 | 211 | self.graph = digraph(ref, from, 1:length(ref), list); 212 | if self.graph.numedges 213 | self.graph.Edges.primary = primary(self.graph.Edges.Weight)'; 214 | self.graph.Edges.aliased = aliased(self.graph.Edges.Weight)'; 215 | self.graph.Edges.multi = multi(self.graph.Edges.Weight)'; 216 | end 217 | end 218 | end 219 | methods(Static) 220 | function tier = getTier(tableName) 221 | tier = []; 222 | for pattern = cellfun(@(x) dj.(x(4:end)).tierRegexp, dj.Schema.tierClasses, ... 223 | 'uni', false) 224 | fieldInfo = regexp(tableName, pattern{1}, 'names'); 225 | if ~isempty(fieldInfo) 226 | tier = fieldnames(fieldInfo); 227 | tier = tier{1}; 228 | break 229 | end 230 | end 231 | end 232 | end 233 | end 234 | -------------------------------------------------------------------------------- /+dj/Imported.m: -------------------------------------------------------------------------------- 1 | classdef Imported < dj.internal.AutoPopulate 2 | % defines an imported table 3 | properties(Constant) 4 | tierRegexp = sprintf('(?%s%s)', ... 5 | dj.Schema.tierPrefixes{strcmp(dj.Schema.allowedTiers, 'imported')}, ... 6 | dj.Schema.baseRegexp) 7 | end 8 | methods 9 | function self = Imported(varargin) 10 | self@dj.internal.AutoPopulate(varargin{:}) 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /+dj/Jobs.m: -------------------------------------------------------------------------------- 1 | classdef Jobs < dj.Relvar 2 | properties(Constant) 3 | tierRegexp = sprintf('(?%s%s)', ... 4 | dj.Schema.tierPrefixes{strcmp(dj.Schema.allowedTiers, 'job')}, ... 5 | dj.Schema.baseRegexp) 6 | end 7 | methods 8 | function self = Jobs(varargin) 9 | self@dj.Relvar(varargin{:}) 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /+dj/Lookup.m: -------------------------------------------------------------------------------- 1 | classdef Lookup < dj.internal.UserRelation 2 | % defines a lookup table 3 | properties(Constant) 4 | tierRegexp = sprintf('(?%s%s)', ... 5 | dj.Schema.tierPrefixes{strcmp(dj.Schema.allowedTiers, 'lookup')}, ... 6 | dj.Schema.baseRegexp) 7 | end 8 | methods 9 | function self = Lookup(varargin) 10 | self@dj.internal.UserRelation(varargin{:}) 11 | if isprop(self, 'contents') 12 | if length(self.contents) > count(self) 13 | self.inserti(self.contents) 14 | end 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /+dj/Manual.m: -------------------------------------------------------------------------------- 1 | classdef Manual < dj.internal.UserRelation 2 | % Defines a manual table 3 | properties(Constant) 4 | tierRegexp = sprintf('(?%s%s)', ... 5 | dj.Schema.tierPrefixes{strcmp(dj.Schema.allowedTiers, 'manual')}, ... 6 | dj.Schema.baseRegexp) 7 | end 8 | methods 9 | function self = Manual(varargin) 10 | self@dj.internal.UserRelation(varargin{:}) 11 | end 12 | end 13 | end -------------------------------------------------------------------------------- /+dj/Part.m: -------------------------------------------------------------------------------- 1 | classdef Part < dj.Relvar 2 | 3 | properties(Abstract, SetAccess=protected) 4 | master 5 | end 6 | 7 | methods 8 | function self = Part 9 | assert(isa(self.master, 'dj.internal.Master'),... 10 | 'The property master should be a UserRelation') 11 | assert(~isempty(regexp(class(self), sprintf('^%s[A-Z]', class(self.master)), 'once')), ... 12 | 'The part class %s must be prefixed with its master %s', ... 13 | class(self), class(self.master)) 14 | end 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /+dj/Table.m: -------------------------------------------------------------------------------- 1 | classdef Table < dj.internal.Table 2 | 3 | methods 4 | function self = Table(varargin) 5 | warning('DataJoint:deprecate', ... 6 | 'dj.Table has been deprecated and will be removed in a future version') 7 | self = self@dj.internal.Table(varargin{:}); 8 | end 9 | end 10 | 11 | end -------------------------------------------------------------------------------- /+dj/close.m: -------------------------------------------------------------------------------- 1 | % dj.close - close the connection (rolls back any ongoing transactions) 2 | 3 | function close 4 | close(dj.conn) -------------------------------------------------------------------------------- /+dj/config.m: -------------------------------------------------------------------------------- 1 | function res = config(name, value) 2 | % CONFIG(name, value) 3 | % Description: 4 | % Manage DataJoint configuration. 5 | % Inputs: 6 | % name[optional]: (string) Dot-based address to desired setting. 7 | % value[optional]: (string) New value to be set. 8 | % Examples: 9 | % dj.config 10 | % dj.config('safemode') 11 | % dj.config('safemode', true) 12 | % previous_value = dj.config('safemode', false) 13 | % dj.config('stores.external_raw', struct(... 14 | % 'datajoint_type', 'blob', ... 15 | % 'protocol', 'file', ... 16 | % 'location', '/net/djblobs/myschema' ... 17 | % )) 18 | % dj.config('stores.external', struct(... 19 | % 'datajoint_type', 'blob', ... 20 | % 'protocol', 's3', ... 21 | % 'endpoint', 's3.amazonaws.com:9000', ... 22 | % 'bucket', 'testbucket', ... 23 | % 'location', 'datajoint-projects/lab1', ... 24 | % 'access_key', '1234567', ... 25 | % 'secret_key', 'foaf1234'... 26 | % )) 27 | % dj.config('blobCache', '/net/djcache') 28 | switch nargin 29 | case 0 30 | out = dj.internal.Settings; 31 | res = out.result; 32 | case 1 33 | out = dj.internal.Settings(name); 34 | res = out.result; 35 | case 2 36 | switch nargout 37 | case 0 38 | dj.internal.Settings(name, value); 39 | case 1 40 | out = dj.internal.Settings(name, value); 41 | res = out.result; 42 | otherwise 43 | error('Exceeded 1 output limit.') 44 | end 45 | otherwise 46 | error('Exceeded 2 input limit.') 47 | end 48 | end -------------------------------------------------------------------------------- /+dj/conn.m: -------------------------------------------------------------------------------- 1 | % dj.conn - constructs and returns a persistent dj.Connection object. 2 | % 3 | % This function can be used in cases when all datajoint schemas connect to 4 | % the same database server with the same credentials. If this is not so, 5 | % you may wish to create multiple functions patterned after dj.conn to 6 | % manage multiple persistent connections. 7 | % 8 | % The first time dj.conn is called, it must establish a conection. The 9 | % connection parameters may be specified by input arguments. Any values omitted 10 | % from the input arguments will be taken from the environment variables 11 | % DJ_HOST, DJ_USER, DJ_PASS, and DJ_INIT are used. 12 | % Finally, if the required parameters are still missing, dj.conn will prompt the 13 | % user to enter them manually. 14 | % 15 | % The last parameter, initQuery (or the environemnt variable DJ_INIT) specify 16 | % the query to be executed everytime a new connection session is established. 17 | % 18 | % Once established during the first invocation, the connection object cannot 19 | % be changed. To reset the connection, use 'clear functions' or 'clear classes'. 20 | 21 | function connObj = conn(host, user, pass, initQuery, reset, use_tls, nogui) 22 | persistent CONN 23 | 24 | if nargin < 5 || isempty(reset) 25 | reset = false; 26 | end 27 | 28 | % get password prompt option 29 | if nargin < 7 || isempty(nogui) 30 | nogui = ~usejava('desktop'); 31 | end 32 | 33 | if nargin<1 || isempty(host) 34 | host = dj.config('databaseHost'); 35 | end 36 | if ~contains(host, ':') 37 | host = [host ':' num2str(dj.config('databasePort'))]; 38 | end 39 | 40 | if isa(CONN, 'dj.Connection') && ~reset 41 | if nargin>0 42 | if strcmp(CONN.host,host) 43 | warning('DataJoint:Connection:AlreadyInstantiated', sprintf([... 44 | 'Connection already instantiated.\n' ... 45 | 'Will use existing connection to "' CONN.host '".\n' ... 46 | 'To reconnect, set reset to true'])); 47 | else 48 | error('DataJoint:Connection:AlreadyInstantiated', sprintf([... 49 | 'Connection already instantiated to "' CONN.host '".\n' ... 50 | 'To reconnect, set reset to true'])); 51 | end 52 | end 53 | else 54 | % get host address 55 | if isempty(host) 56 | host = input('Enter datajoint host address> ','s'); 57 | end 58 | 59 | % get username 60 | if nargin<2 || isempty(user) 61 | user = dj.config('databaseUser'); 62 | end 63 | if isempty(user) 64 | user = input('Enter datajoint username> ', 's'); 65 | end 66 | 67 | % get password 68 | if nargin<3 || isempty(pass) 69 | pass = dj.config('databasePassword'); 70 | end 71 | if isempty(pass) 72 | if nogui 73 | pass = input('Enter datajoint password> ','s'); 74 | else 75 | pass = dj.lib.getpass('Enter datajoint password'); 76 | end 77 | end 78 | 79 | % get initial query (if any) to execute when a connection is (re)established 80 | if nargin<4 || isempty(initQuery) 81 | initQuery = dj.config('connectionInit_function'); 82 | end 83 | 84 | % get tls option 85 | if nargin<6 || isempty(use_tls) 86 | use_tls = dj.config('databaseUse_tls'); 87 | end 88 | 89 | if islogical(use_tls) && ~use_tls 90 | use_tls = 'false'; 91 | elseif islogical(use_tls) && use_tls 92 | use_tls = 'true'; 93 | elseif ~isstruct(use_tls) 94 | use_tls = 'none'; 95 | else 96 | use_tls = jsonencode(use_tls); 97 | end 98 | 99 | CONN = dj.Connection(host, user, pass, initQuery, use_tls); 100 | end 101 | 102 | connObj = CONN; 103 | 104 | % if connection fails to establish, remove the persistent connection object 105 | if ~connObj.isConnected 106 | try 107 | query(connObj, 'status') 108 | catch e 109 | CONN = []; 110 | rethrow(e) 111 | end 112 | end 113 | 114 | if nargout==0 115 | fprintf('database connection id: %d\n', connObj.serverId); 116 | end 117 | 118 | end 119 | -------------------------------------------------------------------------------- /+dj/createSchema.m: -------------------------------------------------------------------------------- 1 | function createSchema(package, parentdir, db) 2 | % DJ.CREATESCHEMA - interactively create a new DataJoint schema 3 | % 4 | % INPUT: 5 | % (optional) package - name of the package to be associated with the schema 6 | % (optional) parentdir - name of the dirctory where to create new package 7 | % (optional) db - database name to associate with the schema 8 | 9 | if nargin < 3 10 | dbname = input('Enter database name >> ','s'); 11 | else 12 | dbname = db; 13 | end 14 | 15 | if ~dbname 16 | disp 'No database name entered. Quitting.' 17 | elseif isempty(regexp(dbname,'^[a-z][a-z0-9_\-]*$','once')) 18 | error(['Invalid database name. Valid name must begin with a letter and include ' ... 19 | 'only lowercase alphanumerical, dashes, and underscores.']) 20 | else 21 | % create database 22 | s = query(dj.conn, ... 23 | sprintf(['SELECT schema_name as `schema_name` ' ... 24 | 'FROM information_schema.schemata WHERE schema_name = "%s"'], dbname)); 25 | 26 | if ~isempty(s.schema_name) 27 | disp 'database already exists' 28 | else 29 | query(dj.conn, sprintf('CREATE SCHEMA `%s`',dbname)) 30 | disp 'database created' 31 | end 32 | 33 | if nargin < 1 34 | if usejava('desktop') 35 | disp 'Please select a package folder. Opening UI...' 36 | folder = uigetdir('./','Select a package folder'); 37 | else 38 | folder = input('Enter package folder path >> ','s'); 39 | end 40 | else 41 | if nargin < 3 42 | if usejava('desktop') 43 | fprintf('Please select folder to create package %s in. Opening UI...\n', ... 44 | ['+', package]) 45 | folder = uigetdir('./', sprintf('Select folder to create package %s in', ... 46 | ['+', package])); 47 | else 48 | folder = input('Enter parent folder path >> ','s'); 49 | end 50 | else 51 | folder = parentdir; 52 | end 53 | 54 | if folder 55 | folder = fullfile(folder, ['+', package]); 56 | mkdir(folder) 57 | end 58 | end 59 | 60 | if ~folder 61 | disp 'No package selected. Cancelled.' 62 | else 63 | [filepath,package] = fileparts(folder); 64 | if package(1)~='+' 65 | error 'Package folders must start with a +' 66 | end 67 | package = package(2:end); % discard + 68 | 69 | % create the getSchema function 70 | schemaFile = fullfile(folder,'getSchema.m'); 71 | if exist(schemaFile,'file') 72 | fprintf('%s.getSchema.m already exists\n', package) 73 | else 74 | f = fopen(schemaFile,'wt'); 75 | assert(-1 ~= f, 'Could not open %s', f) 76 | 77 | fprintf(f,'function obj = getSchema\n'); 78 | fprintf(f,'persistent schemaObject\n'); 79 | fprintf(f,'if isempty(schemaObject)\n'); 80 | fprintf(f,' schemaObject = dj.Schema(dj.conn, ''%s'', ''%s'');\n', ... 81 | package, dbname); 82 | fprintf(f,'end\n'); 83 | fprintf(f,'obj = schemaObject;\n'); 84 | fprintf(f,'end\n'); 85 | fclose(f); 86 | end 87 | 88 | % test that getSchema is on the path 89 | whichpath = which(sprintf('%s.getSchema',package)); 90 | if isempty(whichpath) 91 | warning('Could not open %s.getSchema. Ensure that %s is on the path', ... 92 | package, filepath) 93 | end 94 | end 95 | end 96 | end -------------------------------------------------------------------------------- /+dj/kill.m: -------------------------------------------------------------------------------- 1 | % Show MySQL all connections and prompt to kill a connection. 2 | % dj.kill() lists all connections and prompts the user to enter an id to 3 | % kill. 4 | % 5 | % dj.kill(restriction) lists all connections satisfying the specified 6 | % restriction. Restrictions are specified as strings and can involve any 7 | % of the attributes of information_schema.processlist: ID, USER, HOST, 8 | % DB, COMMAND, TIME, STATE, INFO. 9 | % 10 | % dj.kill(restriction, connection) allows specifying the target connection. 11 | % will use default connection (dj.conn) if not specified. 12 | % 13 | % dj.kill(restriction, connection, order_by) allows providing an order_by 14 | % argument. By default, output is lited by ID in ascending order. 15 | % 16 | % Examples: 17 | % dj.kill('HOST LIKE "%at-compute%"') lists only connections from 18 | % at-compute. 19 | % 20 | % dj.kill('TIME > 600') lists only connections older than 10 minutes. 21 | % 22 | % dj.kill('', dj.conn, 'time') will display no restrictions for the 23 | % default connection ordered by TIME. 24 | % 25 | 26 | 27 | function kill(restriction, connection, order_by) 28 | 29 | if nargin < 3 30 | order_by = {}; 31 | end 32 | 33 | if nargin < 2 34 | connection = dj.conn; 35 | end 36 | 37 | qstr = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()'; 38 | 39 | if nargin && ~isempty(restriction) 40 | qstr = sprintf('%s AND (%s)', qstr, restriction); 41 | end 42 | 43 | if isempty(order_by) 44 | qstr = sprintf('%s ORDER BY id', qstr); 45 | else 46 | if iscell(order_by) 47 | qstr = sprintf('%s ORDER BY %s', qstr, strjoin(order_by, ',')); 48 | else 49 | qstr = sprintf('%s ORDER BY %s', qstr, order_by); 50 | end 51 | end 52 | 53 | while true 54 | query(connection, qstr) 55 | id = input('process to kill (''q''-quit, ''a''-all) > ', 's'); 56 | if ischar(id) && strncmpi(id, 'q', 1) 57 | break 58 | elseif ischar(id) && strncmpi(id, 'a', 1) 59 | res = query(connection, qstr); 60 | 61 | res = cell2struct(struct2cell(res), lower(fieldnames(res))); 62 | 63 | id = double(res.id)'; 64 | for i = id 65 | query(connection, 'kill {Si}', i) 66 | end 67 | break 68 | end 69 | id = sscanf(id,'%d'); 70 | if ~isempty(id) 71 | query(connection, 'kill {Si}', id(1)) 72 | end 73 | end 74 | 75 | end 76 | -------------------------------------------------------------------------------- /+dj/kill_quick.m: -------------------------------------------------------------------------------- 1 | % Kill database connections without prompting. 2 | % dj.kill_quick() kills MySQL server connections matching 'restriction', 3 | % returning the number of terminated connections. 4 | % 5 | % Restrictions are specified as strings and can involve any of the attributes 6 | % of information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, 7 | % STATE, INFO. 8 | % 9 | % Examples: 10 | % dj.kill_quick('HOST LIKE "%compute%"') terminates connections from hosts 11 | % containing "compute" in their hostname. 12 | % 13 | % dj.kill_quick('TIME > 600') terminates all connections older than 10 14 | % minutes. 15 | 16 | function nkill = kill_quick(restriction, connection) 17 | 18 | if nargin < 2 19 | connection = dj.conn; 20 | end 21 | 22 | qstr = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()'; 23 | 24 | if nargin && ~isempty(restriction) 25 | qstr = sprintf('%s AND (%s)', qstr, restriction); 26 | end 27 | 28 | res = query(connection, qstr); 29 | 30 | res = cell2struct(struct2cell(res), lower(fieldnames(res))); 31 | 32 | nkill = 0; 33 | for id = double(res.id)' 34 | query(connection, 'kill {Si}', id); 35 | nkill = nkill + 1; 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /+dj/new.m: -------------------------------------------------------------------------------- 1 | function new(className, tableTierChoice, varargin) 2 | % DJ.NEW - interactively create a new DataJoint table/class 3 | % 4 | % INPUT: 5 | % className in the format 'package.ClassName' 6 | 7 | if nargin<1 8 | className = input('Enter .: ','s'); 9 | end 10 | 11 | p = find(className == '.', 1, 'last'); 12 | if isempty(p) 13 | throwAsCaller(MException('DataJoint:makeClass', 'dj.new requires package.ClassName"')); 14 | end 15 | 16 | schemaFunction = [className(1:p-1) '.getSchema']; 17 | if isempty(which(schemaFunction)) 18 | fprintf('Package %s is missing. Calling dj.createSchema...\n', className(1:p-1)); 19 | % this wouldn't work well if nested package is given 20 | dj.createSchema(className(1:p-1), varargin{:}); 21 | if isempty(which(schemaFunction)) 22 | throwAsCaller(MException('DataJoint:makeClass', 'Cannot find %s', schemaFunction)); 23 | end 24 | end 25 | 26 | if nargin < 2 27 | makeClass(eval(schemaFunction), className(p+1:end)); 28 | else 29 | makeClass(eval(schemaFunction), className(p+1:end), tableTierChoice); 30 | end 31 | -------------------------------------------------------------------------------- /+dj/reload.m: -------------------------------------------------------------------------------- 1 | function reload 2 | % reload all active schemas 3 | reload(dj.conn) -------------------------------------------------------------------------------- /+dj/setPassword.m: -------------------------------------------------------------------------------- 1 | function setPassword(newPassword) 2 | query(dj.conn, 'SET PASSWORD = PASSWORD("{S}")', newPassword) 3 | disp done 4 | end -------------------------------------------------------------------------------- /+dj/setup.m: -------------------------------------------------------------------------------- 1 | function setup(varargin) 2 | p = inputParser; 3 | addOptional(p, 'force', false); 4 | addOptional(p, 'prompt', true); 5 | parse(p, varargin{:}); 6 | force = p.Results.force; 7 | prompt = p.Results.prompt; 8 | persistent INVOKED 9 | if ~isempty(INVOKED) && ~force 10 | return 11 | end 12 | % check MATLAB 13 | if verLessThan('matlab', '9.1') 14 | error('DataJoint:System:UnsupportedMatlabVersion', ... 15 | 'MATLAB version 9.1 (R2016b) or greater is required'); 16 | end 17 | % require certain toolboxes 18 | requiredToolboxes = {... 19 | struct(... 20 | 'Name', 'GHToolbox', ... 21 | 'ResolveTarget', 'datajoint/GHToolbox'... 22 | ), ... 23 | struct(... 24 | 'Name', 'mym', ... 25 | 'ResolveTarget', 'datajoint/mym', ... 26 | 'Version', @(v) compareVersions(v, '2.8.2', @(v_actual,v_ref) v_actual>=v_ref)... 27 | )... 28 | }; 29 | try 30 | ghtb.require(requiredToolboxes, 'prompt', prompt); 31 | catch ME 32 | installPromptMsg = { 33 | 'Toolbox ''%s'' did not meet the minimum requirements.' 34 | 'Would you like to proceed with an upgrade?' 35 | }; 36 | if strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') && (~prompt || strcmpi('yes',... 37 | dj.internal.ask(sprintf(sprintf('%s\n', installPromptMsg{:}), 'GHToolbox')))) 38 | % fetch 39 | tmp_toolbox = [tempname '.mltbx']; 40 | websave(tmp_toolbox, ['https://github.com/' requiredToolboxes{1}.ResolveTarget ... 41 | '/releases/download/' ... 42 | subsref(webread(['https://api.github.com/repos/' ... 43 | requiredToolboxes{1}.ResolveTarget ... 44 | '/releases/latest'], ... 45 | weboptions('Timeout', 60)), ... 46 | substruct('.', 'tag_name')) ... 47 | '/GHToolbox.mltbx'], weboptions('Timeout', 60)); 48 | % install 49 | try 50 | matlab.addons.install(tmp_toolbox, 'overwrite'); 51 | catch ME 52 | if strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') 53 | matlab.addons.toolbox.installToolbox(tmp_toolbox); 54 | else 55 | rethrow(ME); 56 | end 57 | end 58 | % remove temp toolbox file 59 | delete(tmp_toolbox); 60 | % retrigger dependency validation 61 | ghtb.require(requiredToolboxes, 'prompt', prompt); 62 | elseif strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') 63 | GHToolboxMsg = { 64 | 'Toolbox ''GHToolbox'' did not meet the minimum requirements.' 65 | 'Please proceed to install it.' 66 | }; 67 | error('DataJoint:verifyGHToolbox:Failed', ... 68 | sprintf('%s\n', GHToolboxMsg{:})); 69 | else 70 | rethrow(ME) 71 | end 72 | end 73 | % check mym 74 | mymVersion = mym('version'); 75 | assert(mymVersion.major > 2 || mymVersion.major==2 && mymVersion.minor>=6, ... 76 | 'DataJoint:System:mYmIncompatible', ... 77 | 'Outdated version of mYm. Please upgrade to version 2.6 or later'); 78 | % set cache 79 | INVOKED = true; 80 | end -------------------------------------------------------------------------------- /+dj/version.m: -------------------------------------------------------------------------------- 1 | function varargout = version 2 | % report DataJoint version 3 | 4 | v = struct('major', 3, 'minor', 5, 'bugfix', 1); 5 | 6 | if nargout 7 | varargout{1}=v; 8 | else 9 | fprintf('\nDataJoint version %d.%d.%d\n\n', v.major, v.minor, v.bugfix) 10 | end 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: 'bug, awaiting-triage' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Bug Report 11 | 12 | ### Description 13 | A clear and concise description of what is the overall operation that is intended to be performed that resulted in an error. 14 | 15 | ### Reproducibility 16 | Include: 17 | - OS (WIN | MACOS | Linux) 18 | - MATLAB Version 19 | - MySQL Version 20 | - MySQL Deployment Strategy (local-native | local-docker | remote) 21 | - DataJoint Version 22 | - Minimum number of steps to reliably reproduce the issue 23 | - Complete error stack as a result of evaluating the above steps 24 | 25 | ### Expected Behavior 26 | A clear and concise description of what you expected to happen. 27 | 28 | ### Screenshots 29 | If applicable, add screenshots to help explain your problem. 30 | 31 | ### Additional Research and Context 32 | Add any additional research or context that was conducted in creating this report. 33 | 34 | For example: 35 | - Related GitHub issues and PR's either within this repository or in other relevant repositories. 36 | - Specific links to specific lines or a focus within source code. 37 | - Relevant summary of Maintainers development meetings, milestones, projects, etc. 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | # contact_links: 3 | # - name: DataJoint Community Engagement Policy 4 | # url: https://docs.datajoint.io/python/intro/05-Community.html 5 | # about: Please make sure to review the DataJoint Community Engagement Policy. 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for a new feature 4 | title: '' 5 | labels: 'enhancement, awaiting-triage' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Feature Request 11 | 12 | ### Problem 13 | A clear and concise description how this idea has manifested and the context. Elaborate on the need for this feature and/or what could be improved. Ex. I'm always frustrated when [...] 14 | 15 | ### Requirements 16 | A clear and concise description of the requirements to satisfy the new feature. Detail what you expect from a successful implementation of the feature. Ex. When using this feature, it should [...] 17 | 18 | ### Justification 19 | Provide the key benefits in making this a supported feature. Ex. Adding support for this feature would ensure [...] 20 | 21 | ### Alternative Considerations 22 | Do you currently have a work-around for this? Provide any alternative solutions or features you've considered. 23 | 24 | ### Related Errors 25 | Add any errors as a direct result of not exposing this feature. 26 | 27 | Please include steps to reproduce provided errors as follows: 28 | - OS (WIN | MACOS | Linux) 29 | - MATLAB Version 30 | - MySQL Version 31 | - MySQL Deployment Strategy (local-native | local-docker | remote) 32 | - DataJoint Version 33 | - Minimum number of steps to reliably reproduce the issue 34 | - Complete error stack as a result of evaluating the above steps 35 | 36 | ### Screenshots 37 | If applicable, add screenshots to help explain your feature. 38 | 39 | ### Additional Research and Context 40 | Add any additional research or context that was conducted in creating this feature request. 41 | 42 | For example: 43 | - Related GitHub issues and PR's either within this repository or in other relevant repositories. 44 | - Specific links to specific lines or a focus within source code. 45 | - Relevant summary of Maintainers development meetings, milestones, projects, etc. 46 | - Any additional supplemental web references or links that would further justify this feature request. 47 | -------------------------------------------------------------------------------- /.github/workflows/development.yaml: -------------------------------------------------------------------------------- 1 | name: Development 2 | on: 3 | push: 4 | branches: 5 | - '**' # every branch 6 | - '!stage*' # exclude branches beginning with stage 7 | pull_request: 8 | branches: 9 | - '**' # every branch 10 | - '!stage*' # exclude branches beginning with stage 11 | jobs: 12 | test: 13 | if: github.event_name == 'push' || github.event_name == 'pull_request' 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | matlab_version: ["R2019a"] 18 | mysql_version: ["8.0.18", "5.7", "5.6"] 19 | # include: 20 | # - matlab_version: "R2018b" 21 | # mysql_version: "5.7" 22 | # - matlab_version: "R2016b" 23 | # mysql_version: "5.7" 24 | steps: 25 | - uses: actions/checkout@v2 26 | - name: Run primary tests 27 | env: 28 | MATLAB_UID: "1001" 29 | MATLAB_GID: "116" 30 | MATLAB_USER: ${{ secrets.matlab_user }} 31 | MATLAB_HOSTID: ${{ secrets.matlab_hostid }} 32 | MATLAB_VERSION: ${{ matrix.matlab_version }} 33 | MYSQL_TAG: ${{ matrix.mysql_version }} 34 | MINIO_VER: RELEASE.2022-01-03T18-22-58Z 35 | MATLAB_LICENSE: ${{ secrets[format('matlab_license_{0}', matrix.matlab_version)] }} 36 | DOCKER_CLIENT_TIMEOUT: "120" 37 | COMPOSE_HTTP_TIMEOUT: "120" 38 | run: | 39 | docker-compose -f LNX-docker-compose.yaml up --build --exit-code-from app 40 | - name: Add toolbox artifact 41 | uses: actions/upload-artifact@v3 42 | with: 43 | name: dj-toolbox-${{matrix.matlab_version}} 44 | path: DataJoint.mltbx 45 | retention-days: 1 46 | publish-docs: 47 | if: | 48 | github.event_name == 'push' && 49 | startsWith(github.ref, 'refs/tags') 50 | needs: test 51 | runs-on: ubuntu-latest 52 | env: 53 | DOCKER_CLIENT_TIMEOUT: "120" 54 | COMPOSE_HTTP_TIMEOUT: "120" 55 | steps: 56 | - uses: actions/checkout@v2 57 | - name: Deploy docs 58 | run: | 59 | export MODE=BUILD 60 | export PACKAGE=datajoint 61 | export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git 62 | export HOST_UID=$(id -u) 63 | docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build 64 | git push origin gh-pages 65 | -------------------------------------------------------------------------------- /.github/workflows/publish-docs.yaml: -------------------------------------------------------------------------------- 1 | name: Publish Docs 2 | on: 3 | push: 4 | tags: 5 | - 'test*.*.*' 6 | workflow_dispatch: 7 | jobs: 8 | publish-docs: 9 | runs-on: ubuntu-latest 10 | env: 11 | DOCKER_CLIENT_TIMEOUT: "120" 12 | COMPOSE_HTTP_TIMEOUT: "120" 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Deploy docs 16 | run: | 17 | export MODE=BUILD 18 | export PACKAGE=datajoint 19 | export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git 20 | export HOST_UID=$(id -u) 21 | docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build 22 | git push origin gh-pages 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.m~ 2 | mym/ 3 | *.env 4 | notebook 5 | *getSchema.m 6 | docker-compose*.y*ml 7 | !docs/docker-compose.yaml 8 | .vscode 9 | matlab.prf 10 | win.* 11 | macos.* 12 | *.prj 13 | *.mltbx 14 | Jobs.m 15 | .DS_Store -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and 4 | [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. 5 | 6 | ## [3.5.1] - 2023-03-27 7 | 8 | + Bugfix: Allow schemas named with dashes 9 | 10 | ## [3.5.0] - 2022-03-21 11 | 12 | + Bugfix: Cascading delete for renamed foreign keys (#379) PR #386 13 | + Minor: Add renaming the same attribute multiple times within a single projection PR #386 14 | + Minor: Add config for reading values with 32-bit dimensions (datajoint/mym#86) PR #395 15 | 16 | [3.5.1]: https://github.com/datajoint/datajoint-matlab/compare/v3.5.0...v3.5.1 17 | [3.5.0]: https://github.com/datajoint/datajoint-matlab/releases/tag/v3.5.0 18 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009-2012, Dimitri Yatsenko 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /LNX-docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # docker compose -f LNX-docker-compose.yaml --env-file LNX.env up --exit-code-from app --build 2 | version: '2.2' 3 | x-net: &net 4 | networks: 5 | - main 6 | services: 7 | db: 8 | <<: *net 9 | image: datajoint/mysql:${MYSQL_TAG} 10 | environment: 11 | - MYSQL_ROOT_PASSWORD=simple 12 | minio: 13 | <<: *net 14 | environment: 15 | - MINIO_ACCESS_KEY=datajoint 16 | - MINIO_SECRET_KEY=datajoint 17 | image: minio/minio:$MINIO_VER 18 | # ports: 19 | # - "9000:9000" 20 | # To persist MinIO data and config 21 | # volumes: 22 | # - ./minio/data:/data 23 | # - ./minio/config:/root/.minio 24 | command: server /data 25 | healthcheck: 26 | test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"] 27 | timeout: 5s 28 | retries: 60 29 | interval: 1s 30 | fakeservices.datajoint.io: 31 | <<: *net 32 | image: datajoint/nginx:v0.2.4 33 | environment: 34 | - ADD_db_TYPE=DATABASE 35 | - ADD_db_ENDPOINT=db:3306 36 | - ADD_minio_TYPE=MINIO 37 | - ADD_minio_ENDPOINT=minio:9000 38 | - ADD_minio_PORT=80 # allow unencrypted connections 39 | - ADD_minio_PREFIX=/datajoint 40 | - ADD_browser_TYPE=MINIOADMIN 41 | - ADD_browser_ENDPOINT=minio:9000 42 | - ADD_browser_PORT=80 # allow unencrypted connections 43 | depends_on: 44 | db: 45 | condition: service_healthy 46 | minio: 47 | condition: service_healthy 48 | app: 49 | <<: *net 50 | environment: 51 | - MATLAB_LICENSE 52 | - MATLAB_USER 53 | - DJ_HOST=fakeservices.datajoint.io 54 | - DJ_USER=root 55 | - DJ_PASS=simple 56 | - DJ_TEST_HOST=fakeservices.datajoint.io 57 | - DJ_TEST_USER=datajoint 58 | - DJ_TEST_PASSWORD=datajoint 59 | - S3_ENDPOINT=fakeservices.datajoint.io 60 | - S3_ACCESS_KEY=datajoint 61 | - S3_SECRET_KEY=datajoint 62 | - S3_BUCKET=datajoint.test 63 | image: raphaelguzman/matlab:${MATLAB_VERSION}-MIN 64 | depends_on: 65 | fakeservices.datajoint.io: 66 | condition: service_healthy 67 | user: ${MATLAB_UID}:${MATLAB_GID} 68 | working_dir: /tmp 69 | command: 70 | - /bin/bash 71 | - -c 72 | - | 73 | set -e 74 | export ORIG_DIR=$$(pwd) 75 | mkdir ~/Documents 76 | cd /src 77 | # package into toolbox, and install 78 | matlab -nodisplay -r " 79 | try\ 80 | websave([tempdir 'GHToolbox.mltbx'],\ 81 | ['https://github.com/datajoint/GHToolbox' \ 82 | '/releases/download/' subsref(webread(['https://api.github.com/repos' \ 83 | '/datajoint/GHToolbox' \ 84 | '/releases/latest']),\ 85 | substruct('.', 'tag_name')) \ 86 | '/GHToolbox.mltbx']);\ 87 | matlab.addons.toolbox.installToolbox([tempdir 'GHToolbox.mltbx']);\ 88 | fid = fopen('README.md', 'r');\ 89 | docs = fread(fid, '*char')';\ 90 | fclose(fid);\ 91 | ghtb.package('DataJoint',\ 92 | 'Raphael Guzman',\ 93 | 'raphael.h.guzman@gmail.com',\ 94 | ['Scientific workflow management framework built on top of a ' \ 95 | 'relational database.'],\ 96 | docs,\ 97 | {'.vscode', '.git*', '*.env', '*.yaml', 'tests', 'mym', 'docs-parts',\ 98 | '*.txt', '*.prf', '*.md', 'notebook'},\ 99 | @() strjoin(arrayfun(@(x) num2str(x),\ 100 | cell2mat(struct2cell(dj.version)),\ 101 | 'uni', false),\ 102 | '.'),\ 103 | {'+dj', 'erd.m'});\ 104 | matlab.addons.toolbox.installToolbox('DataJoint.mltbx');\ 105 | cd(getenv('ORIG_DIR'));\ 106 | addpath('tests');\ 107 | dir(fileparts(which('erd')));\ 108 | res=run(Main);\ 109 | disp(res);\ 110 | if all([res.Passed]) exit, else exit(1), end;\ 111 | ,\ 112 | catch ME,\ 113 | disp(getReport(ME, 'extended'));\ 114 | exit(1);\ 115 | ,\ 116 | end;\ 117 | " 118 | mac_address: $MATLAB_HOSTID 119 | volumes: 120 | - ./tests:/tmp/tests 121 | - .:/src 122 | networks: 123 | main: 124 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![View DataJoint on File Exchange](https://www.mathworks.com/matlabcentral/images/matlab-file-exchange.svg)](https://www.mathworks.com/matlabcentral/fileexchange/63218-datajoint) 2 | 3 | # Welcome to DataJoint for MATLAB! 4 | 5 | DataJoint for MATLAB is a high-level programming interface for relational databases designed to support data processing chains in science labs. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, and querying data. 6 | 7 | For more information, see our 8 | [general DataJoint docs](https://datajoint.com/docs/) and 9 | [DataJoint MATLAB docs](https://datajoint.com/docs/additional-resources/). 10 | 11 | ## For Developers: Running Tests Locally 12 | 13 |
14 | Click to expand details 15 | 16 | + Create an `.env` with desired development environment values e.g. 17 | 18 | ``` sh 19 | MATLAB_USER=rguzman 20 | MATLAB_LICENSE=IyBCRUd... # For image usage instructions see https://github.com/guzman-raphael/matlab, https://hub.docker.com/r/raphaelguzman/matlab 21 | MATLAB_VERSION=R2019a 22 | MATLAB_HOSTID=XX:XX:XX:XX:XX:XX 23 | MATLAB_UID=1000 24 | MATLAB_GID=1000 25 | MYSQL_TAG=5.7 26 | MINIO_VER=RELEASE.2022-01-03T18-22-58Z 27 | ``` 28 | 29 | + `cp local-docker-compose.yaml docker-compose.yaml` 30 | + `docker-compose up` (Note configured `JUPYTER_PASSWORD`) 31 | + Select a means of running MATLAB e.g. Jupyter Notebook, GUI, or Terminal (see bottom) 32 | + Add `tests` directory to path e.g. in MATLAB, `addpath('tests')` 33 | + Run desired tests. Some examples are as follows: 34 | 35 | | Use Case | MATLAB Code | 36 | | ---------------------------- | ------------------------------------------------------------------------------ | 37 | | Run all tests | `run(Main)` | 38 | | Run one class of tests | `run(TestTls)` | 39 | | Run one specific test | `runtests('TestTls/TestTls_testInsecureConn')` | 40 | | Run tests based on test name | `import matlab.unittest.TestSuite;`
`import matlab.unittest.selectors.HasName;`
`import matlab.unittest.constraints.ContainsSubstring;`
`suite = TestSuite.fromClass(?Main, ... `
    `HasName(ContainsSubstring('Conn')));`
`run(suite)`| 41 | 42 | ### Launch Jupyter Notebook 43 | 44 | + Navigate to `localhost:8888` 45 | + Input Jupyter password 46 | + Launch a notebook i.e. `New > MATLAB` 47 | 48 | ### Launch MATLAB GUI (supports remote interactive debugger) 49 | 50 | + Shell into `datajoint-matlab_app_1` i.e. `docker exec -it datajoint-matlab_app_1 bash` 51 | + Launch Matlab by running command `matlab` 52 | 53 | ### Launch MATLAB Terminal 54 | 55 | + Shell into `datajoint-matlab_app_1` i.e. `docker exec -it datajoint-matlab_app_1 bash` 56 | + Launch Matlab with no GUI by running command `matlab -nodisplay` 57 | 58 |
59 | -------------------------------------------------------------------------------- /docs/.docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM datajoint/miniconda3:4.10.3-py3.9-alpine 2 | ARG PACKAGE 3 | WORKDIR /main 4 | COPY --chown=anaconda:anaconda ./docs/.docker/apk_requirements.txt ${APK_REQUIREMENTS} 5 | COPY --chown=anaconda:anaconda ./docs/.docker/pip_requirements.txt ${PIP_REQUIREMENTS} 6 | RUN \ 7 | /entrypoint.sh echo "Dependencies installed" && \ 8 | git config --global user.name "GitHub Action" && \ 9 | git config --global user.email "action@github.com"&& \ 10 | git config --global pull.rebase false && \ 11 | git init 12 | # COPY --chown=anaconda:anaconda ./${PACKAGE} /main/${PACKAGE} 13 | COPY --chown=anaconda:anaconda ./docs/mkdocs.yaml /main/docs/mkdocs.yaml 14 | COPY --chown=anaconda:anaconda ./docs/src /main/docs/src 15 | COPY --chown=anaconda:anaconda ./CHANGELOG.md /main/docs/src/about/changelog.md -------------------------------------------------------------------------------- /docs/.docker/apk_requirements.txt: -------------------------------------------------------------------------------- 1 | git 2 | -------------------------------------------------------------------------------- /docs/.docker/pip_requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs-material 2 | mkdocs-redirects 3 | mike 4 | mdx-truly-sane-lists 5 | mkdocs-literate-nav 6 | mkdocs-exclude-search 7 | -------------------------------------------------------------------------------- /docs/.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | # https://github.com/DavidAnson/markdownlint 2 | # https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md 3 | MD009: false # permit trailing spaces 4 | MD013: 5 | line_length: "88" # Line length limits 6 | tables: false # disable for tables 7 | headings: false # disable for headings 8 | code_blocks: false # disable for code blocks 9 | MD030: false # Number of spaces after a list 10 | MD033: # HTML elements allowed 11 | allowed_elements: 12 | - "div" 13 | - "span" 14 | - "a" 15 | - "br" 16 | - "sup" 17 | - "figure" 18 | MD034: false # Bare URLs OK 19 | MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling 20 | MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling 21 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Docs Contributions 2 | 3 | Docs contributors should be aware of the following extensions, with the corresponding 4 | settings files, that were used in developing these docs: 5 | 6 | - [MarkdownLinter](https://github.com/DavidAnson/markdownlint): 7 | - `.markdownlint.yaml` establishes settings for various 8 | [linter rules](https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md) 9 | - `.vscode/settings.json` formatting settings 10 | 11 | ```json 12 | { 13 | "[markdown]" : { 14 | "editor.rulers": [88], 15 | "editor.formatOnPaste": true, 16 | "editor.formatOnSave": true, 17 | // https://github.com/stkb/Rewrap/ 18 | // Toggle via command prompt per file 19 | // Default paragraph rewrap key: alt+q or option+q 20 | "rewrap.autoWrap.enabled": true, 21 | "rewrap.wrappingColumn": 88 22 | }, 23 | // https://github.com/DavidAnson/markdownlint 24 | "editor.codeActionsOnSave": { 25 | "source.fixAll.markdownlint":true 26 | }, 27 | "markdownlint.focusMode": 5, // ignore issues around the cursor 28 | } 29 | ``` 30 | 31 | - [CSpell](https://github.com/streetsidesoftware/vscode-spell-checker): `cspell.json` 32 | has various ignored words. 33 | 34 | - [ReWrap](https://github.com/stkb/Rewrap/): `.vscode/settings.json` allows toggling 35 | automated hard wrapping for files at 88 characters. This can also be keymapped to be 36 | performed on individual paragraphs, see documentation. 37 | -------------------------------------------------------------------------------- /docs/cspell.json: -------------------------------------------------------------------------------- 1 | // cSpell Settings 2 | //https://github.com/streetsidesoftware/vscode-spell-checker 3 | { 4 | "version": "0.2", // Version of the setting file. Always 0.2 5 | "language": "en", // language - current active spelling language 6 | "enabledLanguageIds": [ 7 | "markdown" 8 | ], 9 | // flagWords - list of words to be always considered incorrect 10 | // This is useful for offensive words and common spelling errors. 11 | // For example "hte" should be "the" 12 | "flagWords": [], 13 | "allowCompoundWords": true, 14 | "ignorePaths": [ 15 | "src/archive" 16 | ], 17 | "words": [ 18 | "ghtb", 19 | "disp", 20 | "mltbx", 21 | "setenv", 22 | "isempty", 23 | "fetchn", 24 | "struct", 25 | "linenums", 26 | "repr", 27 | "conda", 28 | "numpy", 29 | "sess", 30 | "cond", 31 | "aggr", 32 | "Aggr", 33 | "cajal", 34 | "Cajal", 35 | "hubel", 36 | "Hubel" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /docs/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # MODE="LIVE" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-matlab.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build 2 | # 3 | # navigate to http://localhost/ 4 | version: "2.4" 5 | services: 6 | docs: 7 | build: 8 | dockerfile: docs/.docker/Dockerfile 9 | context: ../ 10 | args: 11 | - PACKAGE 12 | image: ${PACKAGE}-docs 13 | environment: 14 | - PACKAGE 15 | - UPSTREAM_REPO 16 | - MODE 17 | volumes: 18 | - ..:/main 19 | user: ${HOST_UID}:anaconda 20 | ports: 21 | - 80:80 22 | command: 23 | - sh 24 | - -c 25 | - | 26 | set -e 27 | export PATCH_VERSION=$$(cat /main/+dj/version.m | grep 'v =' | sed 's/[^0-9]*/./g' | cut -c 2-6) 28 | if echo "$${MODE}" | grep -i live &>/dev/null; then 29 | mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 30 | elif echo "$${MODE}" | grep -iE "qa|build" &>/dev/null; then 31 | git branch -D gh-pages || true 32 | git fetch $${UPSTREAM_REPO} gh-pages:gh-pages || true 33 | mike deploy --config-file ./docs/mkdocs.yaml -u $PATCH_VERSION latest 34 | mike set-default --config-file ./docs/mkdocs.yaml latest 35 | if echo "$${MODE}" | grep -i qa &>/dev/null; then 36 | mike serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 37 | fi 38 | else 39 | echo "Unexpected mode..." 40 | exit 1 41 | fi 42 | -------------------------------------------------------------------------------- /docs/mkdocs.yaml: -------------------------------------------------------------------------------- 1 | # ---------------------- PROJECT SPECIFIC --------------------------- 2 | 3 | site_name: DataJoint Matlab 4 | repo_url: https://github.com/datajoint/datajoint-matlab 5 | repo_name: datajoint/datajoint-matlab 6 | nav: 7 | - DataJoint Matlab: getting-started/index.md 8 | - Getting Started: getting-started/index.md 9 | - Existing Pipelines: concepts/existing-pipelines.md 10 | - Query Language: 11 | - Common Commands: query-lang/common-commands.md 12 | - Operators: query-lang/operators.md 13 | - Iteration: query-lang/iteration.md 14 | - Reproducibility: 15 | - Table Tiers: reproduce/table-tiers.md 16 | - Make Method: reproduce/make-method.md 17 | # - Tutorials: tutorials.md # Commented out pending viable draft 18 | - Changelog: changelog.md 19 | 20 | # ---------------------------- STANDARD ----------------------------- 21 | 22 | edit_uri: ./edit/master/docs/src 23 | docs_dir: ./src 24 | theme: 25 | font: 26 | text: Roboto Slab 27 | code: Source Code Pro 28 | name: material 29 | custom_dir: src/.overrides 30 | icon: 31 | logo: main/company-logo 32 | favicon: assets/images/company-logo-blue.png 33 | features: 34 | - toc.integrate 35 | - content.code.annotate # Add codeblock annotations 36 | palette: 37 | - media: "(prefers-color-scheme: light)" 38 | scheme: datajoint 39 | toggle: 40 | icon: material/brightness-7 41 | name: Switch to dark mode 42 | - media: "(prefers-color-scheme: dark)" 43 | scheme: slate 44 | toggle: 45 | icon: material/brightness-4 46 | name: Switch to light mode 47 | plugins: 48 | - search 49 | - redirects: 50 | redirect_maps: 51 | "index.md": "getting-started/index.md" 52 | - literate-nav: 53 | nav_file: navigation.md 54 | - exclude-search: 55 | exclude: 56 | - "*/navigation.md" 57 | - "*/archive/*md" 58 | markdown_extensions: 59 | - attr_list 60 | - toc: 61 | permalink: true 62 | - pymdownx.emoji: 63 | emoji_index: !!python/name:materialx.emoji.twemoji 64 | emoji_generator: !!python/name:materialx.emoji.to_svg 65 | options: 66 | custom_icons: 67 | - .overrides/.icons 68 | - mdx_truly_sane_lists 69 | - pymdownx.tabbed: 70 | alternate_style: true 71 | - admonition 72 | - pymdownx.details 73 | - pymdownx.superfences: 74 | custom_fences: 75 | - name: mermaid 76 | class: mermaid 77 | format: !!python/name:pymdownx.superfences.fence_code_format 78 | extra: 79 | generator: false # Disable watermark 80 | analytics: 81 | provider: google 82 | property: !ENV GOOGLE_ANALYTICS_KEY 83 | version: 84 | provider: mike 85 | social: 86 | - icon: main/company-logo 87 | link: https://www.datajoint.com 88 | name: DataJoint 89 | - icon: fontawesome/brands/slack 90 | link: https://datajoint.slack.com 91 | name: Slack 92 | - icon: fontawesome/brands/linkedin 93 | link: https://www.linkedin.com/company/datajoint 94 | name: LinkedIn 95 | - icon: fontawesome/brands/twitter 96 | link: https://twitter.com/datajoint 97 | name: Twitter 98 | - icon: fontawesome/brands/github 99 | link: https://github.com/datajoint 100 | name: GitHub 101 | - icon: fontawesome/brands/docker 102 | link: https://hub.docker.com/u/datajoint 103 | name: DockerHub 104 | - icon: fontawesome/brands/python 105 | link: https://pypi.org/user/datajointbot 106 | name: PyPI 107 | - icon: fontawesome/brands/stack-overflow 108 | link: https://stackoverflow.com/questions/tagged/datajoint 109 | name: StackOverflow 110 | - icon: fontawesome/brands/youtube 111 | link: https://www.youtube.com/channel/UCdeCuFOTCXlVMRzh6Wk-lGg 112 | name: YouTube 113 | extra_css: 114 | - assets/stylesheets/extra.css 115 | -------------------------------------------------------------------------------- /docs/src/.overrides/.icons/main/company-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/src/.overrides/assets/images/company-logo-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datajoint/datajoint-matlab/ff4889628fdc7268b47b6e9bce0feaf0011f7f91/docs/src/.overrides/assets/images/company-logo-blue.png -------------------------------------------------------------------------------- /docs/src/.overrides/assets/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --dj-primary: #00a0df; 3 | --dj-secondary: #ff5113; 4 | --dj-background: #808285; 5 | --dj-black: #000000; 6 | --dj-white: #ffffff; 7 | } 8 | 9 | /* footer previous/next navigation */ 10 | .md-footer__inner:not([hidden]) { 11 | display: none 12 | } 13 | 14 | /* footer social icons */ 15 | html a[title="DataJoint"].md-social__link svg { 16 | color: var(--dj-primary); 17 | } 18 | html a[title="Slack"].md-social__link svg { 19 | color: var(--dj-primary); 20 | } 21 | html a[title="LinkedIn"].md-social__link svg { 22 | color: var(--dj-primary); 23 | } 24 | html a[title="Twitter"].md-social__link svg { 25 | color: var(--dj-primary); 26 | } 27 | html a[title="GitHub"].md-social__link svg { 28 | color: var(--dj-primary); 29 | } 30 | html a[title="DockerHub"].md-social__link svg { 31 | color: var(--dj-primary); 32 | } 33 | html a[title="PyPI"].md-social__link svg { 34 | color: var(--dj-primary); 35 | } 36 | html a[title="StackOverflow"].md-social__link svg { 37 | color: var(--dj-primary); 38 | } 39 | html a[title="YouTube"].md-social__link svg { 40 | color: var(--dj-primary); 41 | } 42 | 43 | [data-md-color-scheme="datajoint"] { 44 | /* ribbon */ 45 | /* ribbon + markdown heading expansion */ 46 | --md-primary-fg-color: var(--dj-black); 47 | /* ribbon text */ 48 | --md-primary-bg-color: var(--dj-primary); 49 | 50 | /* navigation */ 51 | /* navigation header + links */ 52 | --md-typeset-a-color: var(--dj-primary); 53 | /* navigation on hover + diagram outline */ 54 | --md-accent-fg-color: var(--dj-secondary); 55 | 56 | /* main */ 57 | /* main header + already viewed*/ 58 | --md-default-fg-color--light: var(--dj-background); 59 | /* primary text */ 60 | --md-typeset-color: var(--dj-black); 61 | /* code comments + diagram text */ 62 | --md-code-fg-color: var(--dj-primary); 63 | 64 | /* footer */ 65 | /* previous/next text */ 66 | /* --md-footer-fg-color: var(--dj-primary); */ 67 | } 68 | 69 | [data-md-color-scheme="slate"] { 70 | /* ribbon */ 71 | /* ribbon + markdown heading expansion */ 72 | --md-primary-fg-color: var(--dj-primary); 73 | /* ribbon text */ 74 | --md-primary-bg-color: var(--dj-white); 75 | 76 | /* navigation */ 77 | /* navigation header + links */ 78 | --md-typeset-a-color: var(--dj-primary); 79 | /* navigation on hover + diagram outline */ 80 | --md-accent-fg-color: var(--dj-secondary); 81 | 82 | /* main */ 83 | /* main header + already viewed*/ 84 | /* --md-default-fg-color--light: var(--dj-background); */ 85 | /* primary text */ 86 | --md-typeset-color: var(--dj-white); 87 | /* code comments + diagram text */ 88 | --md-code-fg-color: var(--dj-primary); 89 | 90 | /* footer */ 91 | /* previous/next text */ 92 | /* --md-footer-fg-color: var(--dj-white); */ 93 | } 94 | -------------------------------------------------------------------------------- /docs/src/.overrides/partials/nav.html: -------------------------------------------------------------------------------- 1 | {% set class = "md-nav md-nav--primary" %} 2 | {% if "navigation.tabs" in features %} 3 | {% set class = class ~ " md-nav--lifted" %} 4 | {% endif %} 5 | {% if "toc.integrate" in features %} 6 | {% set class = class ~ " md-nav--integrated" %} 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /docs/src/changelog.md: -------------------------------------------------------------------------------- 1 | ../../CHANGELOG.md -------------------------------------------------------------------------------- /docs/src/concepts/existing-pipelines.md: -------------------------------------------------------------------------------- 1 | # Existing Pipelines 2 | 3 | This section describes how to work with database schemas without access to the original 4 | code that generated the schema. These situations often arise when the database is 5 | created by another user who has not shared the generating code yet or when the database 6 | schema is created from a programming language other than Matlab. 7 | 8 | ## Creating a virtual class 9 | 10 | DataJoint MATLAB creates a `TableAccessor` property in each schema object. The 11 | `TableAccessor` property, a *virtual class generator*, is available as `schema.v`, and 12 | allows listing and querying of the tables defined on the server without needing to 13 | create the MATLAB table definitions locally. For example, creating a scratch 14 | `experiment` schema package and querying an existing `my_experiment.Session` table on 15 | the server can be done as follows: 16 | 17 | ``` matlab 18 | dj.createSchema('experiment', '/scratch', 'my_experiment') 19 | addpath('/scratch') 20 | experiment_schema = experiment.getSchema(); 21 | experiment_schema.v.Session() & 'session_id=1234'; 22 | ``` 23 | 24 | ???+ Note 25 | 26 | You can view the available tables in a schema by using tab completion on 27 | the `schema.v` property. 28 | 29 | To visualize an unfamiliar schema, see commands for generating [diagrams](../../getting-started/#diagram). 30 | -------------------------------------------------------------------------------- /docs/src/getting-started/index.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | ## Installation 4 | 5 | First, please install DataJoint via one of the following: 6 | 7 | === "Matlab ≥ R2016b (recommended)" 8 | 9 | 1. Utilize MATLAB built-in GUI i.e. *Top Ribbon -> Add-Ons -> Get Add-Ons* 10 | 2. Search and Select `DataJoint` 11 | 3. Select *Add from GitHub* 12 | 13 | === "Matlab < R2016" 14 | 15 | 1. Utilize MATLAB built-in GUI i.e. *Top Ribbon -> Add-Ons -> Get Add-Ons* 16 | 2. Search and Select `DataJoint` 17 | 3. Select *Download from GitHub* 18 | 4. Save `DataJoint.mltbx` locally 19 | 5. Navigate in MATLAB tree browser to saved toolbox file 20 | 6. Right-Click and Select *Install* 21 | 7. Select *Install* 22 | 23 | === "GHToolbox" 24 | 25 | 1. Install *GHToolbox* using using an appropriate method in https://github.com/datajoint/GHToolbox 26 | 2. run: `ghtb.install('datajoint/datajoint-matlab')` 27 | 28 | === "From source" 29 | 30 | 1. Download `DataJoint.mltbx` locally 31 | 2. Navigate in MATLAB tree browser to saved toolbox file 32 | 3. Right-Click and Select *Install* 33 | 4. Select *Install* 34 | 35 | After installing, that you have the latest version from Matlab: 36 | 37 | ```matlab 38 | >> dj.version 39 | DataJoint version 3.5.0 40 | ``` 41 | 42 | ## Connection 43 | 44 | At the MATLAB command prompt, assign the environment variables with 45 | the database credentials. For example, if you are connection to the 46 | server `tutorial-db.datajoint.io` with username `alice` and password 47 | `fake-password`, execute the following commands: 48 | 49 | ```matlab 50 | setenv DJ_USER alice 51 | setenv DJ_HOST tutorial-db.datajoint.io 52 | setenv DJ_PASS 'fake-password' 53 | ``` 54 | 55 | !!! note 56 | 57 | Although you may connect to any MySQL server of your choice, the DataJoint company 58 | offers an online tutorial environment at `tutorial-db.datajoint.io`. Simply sign up 59 | for a free [DataJoint account](https://accounts.datajoint.io). You will be granted 60 | privileges to create schemas that are prefixed as `{user}_`. 61 | 62 | You will need to execute these commands at the beginning of each DataJoint work session. 63 | To automate this process, you might add these items to the Matlab 64 | [startup.m](https://www.mathworks.com/help/matlab/ref/startup.html) script. 65 | 66 | However, be careful not to share this file or commit it to a public directory (a common 67 | mistake), as it contains a your login credentials in plain text. If you are not sure, 68 | it is better not to set `DJ_PASS`, in which case DataJoint will prompt to enter the 69 | password when connecting to the database. 70 | 71 | To change the database password, use the following command 72 | 73 | ```matlab 74 | >> dj.setPassword('my#cool!new*password') 75 | ``` 76 | 77 | And update your credentials in your startup script for the next session. 78 | 79 | For more information on various settings, access help via `help('dj.config')` or review 80 | it online 81 | [here](https://github.com/datajoint/datajoint-matlab/blob/c2bd6b3e195dfeef773d4e12bad5573c461193b0/%2Bdj/config.m#L2-L27). 82 | 83 | ## Creating Schemas 84 | 85 | A schema can be created either automatically using the `dj.createSchema` 86 | script or manually. While `dj.createSchema` simplifies the process, we'll also highlight 87 | the manual approach to demonstrate each step. 88 | 89 | ### Manual 90 | 91 | We can create the database schema using the following command: 92 | 93 | ``` matlab 94 | query(dj.conn, 'CREATE SCHEMA `{user}_my_schema`') 95 | ``` 96 | 97 | ??? Note "Server privileges" 98 | 99 | You must have create privileges for the schema name pattern. It is a common practice 100 | to grant all privileges to users for schemas that begin with the username, in 101 | addition to some shared schemas. Thus the user `alice` would be able to perform any 102 | work in any schema that begins with `alice_`. 103 | 104 | Next, we can create the MATLAB package. 105 | 106 | DataJoint organizes schemas as MATLAB **packages**. If you are not 107 | familiar with packages, please review: 108 | 109 | - [How to work with MATLAB packages](https://www.mathworks.com/help/matlab/matlab_oop/scoping-classes-with-packages.html) 110 | - [How to manage MATLAB's search paths](https://www.mathworks.com/help/matlab/search-path.html) 111 | 112 | In your project directory, create the package folder, which must begin with a `+` sign. 113 | For example, for the schema called `my_schema`, you would create the folder 114 | `+my_schema`. Make sure that your project directory (the parent directory of your 115 | package folder) is added to the MATLAB search path. 116 | 117 | Finally, we'll associate the package with the database schema. 118 | 119 | This step tells DataJoint that all classes in the package folder `+my_schema` will work 120 | with tables in the database schema `{user}_my_schema`. Each package corresponds to 121 | exactly one schema. In some special cases, multiple packages may all relate to a single 122 | database schema, but in most cases there will be a one-to-one relationship between 123 | packages and schemas. 124 | 125 | In the `+my_schema` folder, create the file `getSchema.m` with the 126 | following contents: 127 | 128 | ``` matlab 129 | function obj = getSchema 130 | persistent OBJ 131 | if isempty(OBJ) 132 | OBJ = dj.Schema(dj.conn, 'experiment', 'alice_experiment'); 133 | end 134 | obj = OBJ; 135 | end 136 | ``` 137 | 138 | This function returns a persistent object of type `dj.Schema`, 139 | establishing the link between the `my_schema` package in MATLAB and the 140 | schema `{user}_my_schema` on the database server. 141 | 142 | ## Automatic 143 | 144 | Alternatively, we can execute 145 | 146 | ``` matlab 147 | >> dj.createSchema 148 | ``` 149 | 150 | This automated script will walk you through the steps above and will create the schema, 151 | the package folder, and the `getSchema` function in that folder. 152 | 153 | ## Defining Tables 154 | 155 | DataJoint provides the interactive script `dj.new` for creating a new table. It will 156 | prompt to enter the new table's class name in the form `package.ClassName`. This will 157 | create the file `+package/ClassName.m`. 158 | 159 | For example, define the table `my_schema.Rectangle` 160 | 161 | ``` matlab 162 | >> dj.new 163 | Enter .: my_schema.Rectangle 164 | 165 | Choose table tier: 166 | L=lookup 167 | M=manual 168 | I=imported 169 | C=computed 170 | P=part 171 | (L/M/I/C/P) > M 172 | ``` 173 | 174 | This will create the file `+my_schema.Rectangle.m` with the following 175 | contents: 176 | 177 | ``` matlab 178 | %{ 179 | # my newest table 180 | # add primary key here 181 | ----- 182 | # add additional attributes 183 | %} 184 | classdef Rectangle < dj.Manual 185 | end 186 | ``` 187 | 188 | While `dj.new` adds a little bit of convenience, we can also create the classes from 189 | scratch manually. Each newly created class must inherit from the DataJoint class 190 | corresponding to the correct [data tier](../../reproduce/table-tiers): `dj.Lookup`, 191 | `dj.Manual`, `dj.Imported` or `dj.Computed`. 192 | 193 | The most important part of the table definition is the comment preceding the `classdef`. 194 | DataJoint will parse this comment to define the table. The class will become usable 195 | after you edit this comment. For example: 196 | 197 | File `+my_schema/Rectangle.m` 198 | 199 | ```matlab 200 | %{ 201 | shape_id: int 202 | --- 203 | shape_height: float 204 | shape_width: float 205 | %} 206 | classdef Rectangle < dj.Manual 207 | end 208 | ``` 209 | 210 | File `+my_schema/Area.m` 211 | 212 | ```matlab 213 | %{ 214 | -> my_schema.Rectangle 215 | --- 216 | shape_area: float 217 | %} 218 | classdef Area < dj.Computed 219 | end 220 | ``` 221 | 222 | The table definition is contained in the first block comment in the class definition 223 | file. Note that although it looks like a mere comment, the table definition is parsed 224 | by DataJoint. 225 | 226 | Users do not need to do anything special to have the table created in the database. The 227 | table is created upon the first attempt to use the class for manipulating its data 228 | (e.g. inserting or fetching entities). 229 | 230 | Furthermore, DataJoint provides the `syncDef` method to update the `classdef` file 231 | definition string for the table with the definition in the actual table: 232 | 233 | ``` matlab 234 | syncDef(my_schema.Area) % updates the table definition in file +my_schema/Area.m 235 | ``` 236 | 237 | ## Diagram 238 | 239 | ### Display 240 | 241 | The diagram displays the relationship of the data model in the data pipeline. 242 | 243 | This can be done for an entire schema, or multiple schema: 244 | 245 | ``` matlab 246 | draw(dj.ERD(my_schema.getSchema)) 247 | % OR 248 | erd my_schema 249 | erd my_schema my_other_schema 250 | ``` 251 | 252 | Or for individual or sets of tables: 253 | ```python 254 | erd my_schema.Rectangle 255 | draw(dj.ERD(my_schema.Rectangle) + dj.ERD(my_schema.Area)) 256 | ``` 257 | 258 | ![Shapes Pipeline](../images/shapes_pipeline.svg) 259 | 260 | ### Customize 261 | 262 | Adding or substracting a number to a diagram object adds nodes downstream or upstream, 263 | respectively, in the pipeline. 264 | 265 | ``` python 266 | draw(dj.ERD(my_schema.Rectangle)+1) # (1) 267 | ``` 268 | 269 | 1. Plot all the tables directly downstream from `my_schema.Rectangle` 270 | 271 | ```python 272 | draw(dj.ERD(my_schema)-1+1) # (1) 273 | ``` 274 | 275 | 1. Plot all tables directly downstream of those directly upstream of this schema. 276 | 277 | ## Add data 278 | 279 | Let's add data for a rectangle: 280 | 281 | ```matlab 282 | insert(my_schema.Rectangle, {1, 2, 4}) 283 | insert(my_schema.Rectangle, [{2, 2, 3},{3, 4, 2}]) 284 | ``` 285 | 286 | ## Run computation 287 | 288 | Let's start the computations on our entity: `Area`. 289 | 290 | ```python 291 | populate(my_schema.Rectangle) 292 | ``` 293 | 294 | ## Query 295 | 296 | Let's inspect the results. 297 | 298 | ```python 299 | Area & 'shape_area >= 8' 300 | ``` 301 | -------------------------------------------------------------------------------- /docs/src/images/shapes_pipeline.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | %3 4 | 5 | 6 | 7 | Area 8 | 12 | 13 | Area 14 | 15 | 16 | 17 | 18 | 19 | Rectangle 20 | 25 | 26 | Rectangle 27 | 28 | 29 | 30 | 31 | 32 | Rectangle->Area 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /docs/src/query-lang/common-commands.md: -------------------------------------------------------------------------------- 1 | # Common Commands 2 | 3 | 4 | 5 | ## Make 6 | 7 | See the article on [`make` methods](../../reproduce/make-method/) 8 | 9 | ## Fetch 10 | 11 | DataJoint for MATLAB provides three distinct fetch methods: `fetch`, `fetch1`, and 12 | `fetchn`. The three methods differ by the type and number of their returned variables. 13 | 14 | - `query.fetch` returns the result in the form of an *n* ⨉ 1 [struct array](https://www.mathworks.com/help/matlab/ref/struct.html) 15 | where *n* is the number of records matching the query expression. 16 | 17 | - `query.fetch1` and `query.fetchn` split the result into separate output arguments, one 18 | for each attribute of the query. 19 | 20 | The types of the variables returned by `fetch1` and `fetchn` depend on the datatypes of 21 | the attributes. `query.fetchn` will enclose any attributes of char and blob types in 22 | [cell arrays](https://www.mathworks.com/help/matlab/cell-arrays.html) whereas 23 | `query.fetch1` will unpack them. 24 | 25 | MATLAB has two alternative forms of invoking a method on an object: using the dot 26 | notation or passing the object as the first argument. The following two notations 27 | produce an equivalent result: 28 | 29 | ``` matlab 30 | result = query.fetch(query, 'attr1') 31 | result = fetch(query, 'attr1') 32 | ``` 33 | 34 | However, the dot syntax only works when the query object is already assigned to a 35 | variable. The second syntax is more commonly used to avoid extra variables. 36 | 37 | For example, the two methods below are equivalent although the second 38 | method creates an extra variable. 39 | 40 | ``` matlab 41 | result = fetch(my_schema.Rectangle, '*'); 42 | 43 | query = my_schema.Rectangle; 44 | result = query.fetch() 45 | ``` 46 | 47 | ### The primary key 48 | 49 | Without any arguments, the `fetch` method retrieves the primary key values of the table 50 | in the form of a single column `struct`. The attribute names become the fieldnames of 51 | the `struct`. 52 | 53 | ``` matlab 54 | keys = query.fetch; 55 | keys = fetch(my_schema.Rectangle & my_schema.Area); 56 | ``` 57 | 58 | Note that MATLAB allows calling functions without the parentheses `()`. 59 | 60 | ### An entire query 61 | 62 | With a single-quoted asterisk (`'*'`) as the input argument, the `fetch` command 63 | retrieves the entire result as a struct array. 64 | 65 | ``` matlab 66 | data = query.fetch('*'); 67 | data = fetch(my_schema.Rectangle & my_schema.Area, '*'); 68 | ``` 69 | 70 | ??? Note "For very large tables..." 71 | 72 | In some cases, the amount of data returned by fetch can be quite large. When `query` 73 | is a table object rather than a query expression, `query.sizeOnDisk()` reports the 74 | estimated size of the entire table. It can be used to assess whether running 75 | `query.fetch('*')` would be wise. Please note that it is only currently possible to 76 | query the size of entire tables stored directly in the database . 77 | 78 | ### Separate variables 79 | 80 | The `fetch1` and `fetchn` methods are used to retrieve each attribute into a separate 81 | variable. DataJoint needs two different methods to tell MATLAB whether the result 82 | should be in array or scalar form; for numerical fields it does not matter 83 | (because scalars are still matrices in MATLAB) but non-uniform collections of values 84 | must be enclosed in cell arrays. 85 | 86 | `query.fetch1` is used when `query` contains exactly one entity, otherwise `fetch1` will 87 | raise an error. 88 | 89 | `query.fetchn` returns an arbitrary number of elements with character arrays and blobs 90 | returned in the form of cell arrays, even when `query` happens to contain a single 91 | entity. 92 | 93 | ``` matlab 94 | [shape_id, height] = query.fetch1('shape_id', 'shape_height'); % (1) 95 | [shape_ids, heights] = query.fetchn('shape_id', 'shape_height'); % (2) 96 | ``` 97 | 98 | 1. When the table has exactly one entity. 99 | 2. When the table has any number of entities: 100 | 101 | ### The primary key and individual values 102 | 103 | It is often convenient to know the primary key values corresponding to attribute values 104 | retrieved by `fetchn`. This can be done by adding a special input argument indicating 105 | the request and another output argument to receive the key values: 106 | 107 | ``` matlab 108 | [shape_ids, heights, keys] = query.fetchn('shape_id', 'shape_height', 'KEY'); 109 | ``` 110 | 111 | The resulting value of `keys` will be a column array of type `struct`. 112 | This mechanism is only implemented for `fetchn`. 113 | 114 | ### Rename and calculate 115 | 116 | In DataJoint for MATLAB, all `fetch` methods have all the same capability as the 117 | [`proj` operator](../operators#proj). For example, renaming an attribute can be 118 | accomplished using the syntax below. 119 | 120 | ``` matlab 121 | [shape_ids, perimeter] = query.fetchn('shape_id', ... 122 | '2*(shape_height+shape_width) -> perimeter'); 123 | ``` 124 | 125 | See [`proj`](../operators#proj) for an in-depth description of projection. 126 | 127 | ### Sorting results 128 | 129 | To sort the result, add the additional `ORDER BY` argument in `fetch` and `fetchn` 130 | methods as the last argument. 131 | 132 | The following command retrieves the field `shape_id` from rectangles with height greater 133 | than 2, sorted by width. 134 | 135 | ``` matlab 136 | notes = fetchn(my_schema.Rectangle & 'shape_height>2"', 'shape_id' ... 137 | 'ORDER BY shape_width'); 138 | ``` 139 | 140 | The ORDER BY argument is passed directly to SQL and follows the same syntax as the 141 | [ORDER BY clause](https://dev.mysql.com/doc/refman/5.7/en/order-by-optimization.html) 142 | 143 | Similarly, the LIMIT and OFFSET clauses can be used to limit the result to a subset of 144 | entities. For example, to return the five rectangles with largest area, one could do the 145 | following: 146 | 147 | ``` matlab 148 | s = fetch(my_schema.Area, '*', 'ORDER BY shape_area DESC LIMIT 5') 149 | ``` 150 | 151 | The limit clause is passed directly to SQL and follows the same 152 | [rules](https://dev.mysql.com/doc/refman/5.7/en/select.html) 153 | 154 | 155 | -------------------------------------------------------------------------------- /docs/src/query-lang/iteration.md: -------------------------------------------------------------------------------- 1 | # Iteration 2 | 3 | The DataJoint model primarily handles data as sets, in the form of tables. However, it 4 | can sometimes be useful to access or to perform actions such as visualization upon 5 | individual entities sequentially. In DataJoint this is accomplished through iteration. 6 | 7 | In the simple example below, iteration is used to display the names and values of the 8 | primary key attributes of each entity in the simple table or table expression `tab`. 9 | 10 | ``` matlab 11 | for key = tab.fetch()' 12 | disp(key) 13 | end 14 | ``` 15 | 16 | Note that the results returned by `fetch` must be transposed. MATLAB iterates across 17 | columns, so the single column `struct` returned by `fetch` must be transposed into a 18 | single row. 19 | -------------------------------------------------------------------------------- /docs/src/query-lang/operators.md: -------------------------------------------------------------------------------- 1 | # Operators 2 | 3 | The examples below will use the table definitions in [table tiers](../../reproduce/table-tiers). 4 | 5 | 6 | 7 | ## Restriction 8 | 9 | `&` and `-` operators permit restriction. 10 | 11 | ### By a mapping 12 | 13 | For a [Session table](../../reproduce/table-tiers#manual-tables), that has the attribute 14 | `session_date`, we can restrict to sessions from January 1st, 2022: 15 | 16 | ```matlab 17 | Session & struct('session_date', '2018-01-01') 18 | ``` 19 | 20 | If there were any typos (e.g., using `sess_date` instead of `session_date`), our query 21 | will return all of the entities of `Session`. 22 | 23 | ### By a string 24 | 25 | Conditions may include arithmetic operations, functions, range tests, etc. Restriction 26 | of table `A` by a string containing an attribute not found in table `A` produces an 27 | error. 28 | 29 | ```matlab 30 | Session & 'user = "Alice"' % (1) 31 | Session & 'session_date >= "2022-01-01"' % (2) 32 | ``` 33 | 34 | 1. All the sessions performed by Alice 35 | 2. All of the sessions on or after January 1st, 2022 36 | 37 | ### By a collection 38 | 39 | When `cond` is a collection of conditions, the conditions are applied by logical 40 | disjunction (logical OR). Restricting a table by a collection will return all entities 41 | that meet *any* of the conditions in the collection. 42 | 43 | For example, if we restrict the `Session` table by a collection containing two 44 | conditions, one for user and one for date, the query will return any sessions with a 45 | matching user *or* date. 46 | 47 | ```matlab 48 | cond_cell = {'user = "Alice"', 'session_date = "2022-01-01"'} % (1) 49 | cond_struct = struct('user', 'Alice', 'session_date', '2022-01-01') % (2) 50 | cond_struct(2) = struct('user', 'Jerry', 'session_date', '2022-01-01') 51 | 52 | Session() & cond_struct % (3) 53 | ``` 54 | 55 | 1. A cell array 56 | 2. A structure array 57 | 3. This command will show all the sessions that either Alice or Jerry conducted on the 58 | given day. 59 | 60 | ### By a query 61 | 62 | Restriction by a query object is a generalization of restriction by a table. The example 63 | below creates a query object corresponding to all the users named Alice. The `Session` 64 | table is then restricted by the query object, returning all the sessions performed by 65 | Alice. The `Experiment` table is then restricted by the query object, returning all the 66 | experiments that are part of sessions performed by Alice. 67 | 68 | ``` matlab 69 | query = Session & 'user = "Alice"' 70 | Experiment & query 71 | ``` 72 | 73 | ## Proj 74 | 75 | Renaming an attribute in python can be done via keyword arguments: 76 | 77 | ```matlab 78 | table('old_attr->new_attr') 79 | ``` 80 | 81 | This can be done in the context of a table definition: 82 | 83 | ``` matlab 84 | %{ 85 | # Experiment Session 86 | -> experiment.Animal 87 | session : smallint # session number for the animal 88 | --- 89 | session_date : date # YYYY-MM-DD 90 | session_start_time : float # seconds relative to session_datetime 91 | session_end_time : float # seconds relative to session_datetime 92 | -> User.proj(experimenter='username') 93 | -> User.proj(supervisor='username') 94 | %} 95 | classdef Session < dj.Manual 96 | end 97 | ``` 98 | 99 | Or as part of a query 100 | 101 | ```matlab 102 | Session * Session.proj('session->other') 103 | ``` 104 | 105 | Projection can also be used to to compute new attributes from existing ones. 106 | 107 | ```matlab 108 | Session.proj('session_end_time-session_start_time -> duration') & 'duration > 10' 109 | ``` 110 | 111 | ## Aggr 112 | 113 | For more complicated calculations, we can use aggregation. 114 | 115 | ```matlab 116 | Subject.aggr(Session,'count(*)->n') % (1) 117 | Subject.aggr(Session,'avg(session_start_time)->average_start') % (2) 118 | ``` 119 | 120 | 1. Number of sessions per subject. 121 | 2. Average `session_start_time` for each subject 122 | 123 | 124 | 125 | ## Universal set 126 | 127 | !!! Warning 128 | 129 | `dj.U` is not yet implemented in MATLAB. The feature will be added in an 130 | upcoming release. You can track progress with 131 | [this GitHub issue](https://github.com/datajoint/datajoint-matlab/issues/144). 132 | 133 | Universal sets offer the complete list of combinations of attributes. 134 | 135 | ```matlab 136 | dj.U('laser_wavelength', 'laser_power') & Scan % (1) 137 | dj.U('laser_wavelength', 'laser_power').aggr(Scan, 'count(*)->n') % (2) 138 | dj.U().aggr(Session, 'max(session)->n') % (3) 139 | ``` 140 | 141 | 1. All combinations of wavelength and power. 142 | 2. Total number of scans for each combination. 143 | 3. Largest session number. 144 | 145 | `dj.U()`, as shown in the last example above, is often useful for integer IDs. 146 | For an example of this process, see the source code for 147 | [Element Array Electrophysiology's `insert_new_params`](https://datajoint.com/docs/elements/element-array-ephys/latest/api/element_array_ephys/ephys_acute/#element_array_ephys.ephys_acute.ClusteringParamSet.insert_new_params). 148 | -------------------------------------------------------------------------------- /docs/src/reproduce/make-method.md: -------------------------------------------------------------------------------- 1 | # Make Method 2 | 3 | Consider the following table definition from the article on 4 | [table tiers](../table-tiers): 5 | 6 | ```matlab 7 | %{ Filtered image 8 | -> test.Image 9 | --- 10 | filtered_image : longblob 11 | %} 12 | 13 | classdef FilteredImage < dj.Computed 14 | methods(Access=protected) 15 | function makeTuples(self, key) 16 | img = fetch1(test.Image & key, 'image'); 17 | key.filtered_image = my_filter(img); 18 | self.insert(key) 19 | end 20 | end 21 | end 22 | ``` 23 | 24 | The `FilteredImage` table can be populated as 25 | 26 | ``` matlab 27 | populate(test.FilteredImage) 28 | ``` 29 | 30 | Note that it is not necessary to specify which data needs to be computed. DataJoint will 31 | call `make`, one-by-one, for every key in `Image` for which `FilteredImage` has not yet 32 | been computed. 33 | 34 | The `make` method receives one argument: the dict `key` containing the primary key value 35 | of an element of `key source` to be worked on. 36 | 37 | ## Optional Arguments 38 | 39 | Behavior of the `populate` method depends on the number of output arguments requested in 40 | the function call. When no output arguments are requested, errors will halt population. 41 | With two output arguments(`failedKeys` and `errors`), `populate` will catch any 42 | encountered errors and return them along with the offending keys. 43 | 44 | ## Progress 45 | 46 | The function `parpopulate` works identically to `populate` except that it uses a job 47 | reservation mechanism to allow multiple processes to populate the same table in 48 | parallel without collision. When running `parpopulate` for the first time, DataJoint 49 | will create a job reservation table and its class `.Jobs` with the following 50 | declaration: 51 | 52 | ``` matlab 53 | {% 54 | # the job reservation table 55 | table_name : varchar(255) # className of the table 56 | key_hash : char(32) # key hash 57 | --- 58 | status : enum('reserved','error','ignore')# if tuple is missing, the job is available 59 | key=null : blob # structure containing the key 60 | error_message="" : varchar(1023) # error message returned if failed 61 | error_stack=null : blob # error stack if failed 62 | host="" : varchar(255) # system hostname 63 | pid=0 : int unsigned # system process id 64 | timestamp=CURRENT_TIMESTAMP : timestamp # automatic timestamp 65 | %} 66 | ``` 67 | 68 | A job is considered to be available when `.Jobs` contains no matching entry. 69 | 70 | For each `make` call, `parpopulate` sets the job status to `reserved`. When the job is 71 | completed, the record is removed. If the job results in error, the job record is left 72 | in place with the status set to `error` and the error message and error stacks saved. 73 | Consequently, jobs that ended in error during the last execution will not be attempted 74 | again until you delete the corresponding entities from `.Jobs`. 75 | 76 | The primary key of the jobs table comprises the name of the class and a 32-character 77 | hash of the job's primary key. However, the key is saved in a separate field for error 78 | debugging purposes. 79 | -------------------------------------------------------------------------------- /docs/src/reproduce/table-tiers.md: -------------------------------------------------------------------------------- 1 | # Table Tiers 2 | 3 | To define a DataJoint table in Matlab: 4 | 5 | 1. Define the table via multi-line comment. 6 | 2. Define a class inheriting from the appropriate DataJoint class: 7 | `dj.Lookup`, `dj.Manual`, `dj.Imported` or `dj.Computed`. 8 | 9 | ## Manual Tables 10 | 11 | The following code defines two manual tables, `Animal` and `Session`: 12 | 13 | File `+experiment/Animal.m` 14 | 15 | ``` matlab 16 | %{ 17 | # information about animal 18 | animal_id : int # animal id assigned by the lab 19 | --- 20 | -> experiment.Species 21 | date_of_birth=null : date # YYYY-MM-DD optional 22 | sex='' : enum('M', 'F', '') # leave empty if unspecified 23 | %} 24 | classdef Animal < dj.Manual 25 | end 26 | ``` 27 | 28 | File `+experiment/Session.m` 29 | 30 | ``` matlab 31 | %{ 32 | # Experiment Session 33 | -> experiment.Animal 34 | session : smallint # session number for the animal 35 | --- 36 | session_date : date # YYYY-MM-DD 37 | session_start_time : float # seconds relative to session_datetime 38 | session_end_time : float # seconds relative to session_datetime 39 | -> [nullable] experiment.User 40 | %} 41 | classdef Session < dj.Manual 42 | end 43 | ``` 44 | 45 | Note that the notation to permit null entries differs for attributes versus foreign 46 | key references. 47 | 48 | ## Lookup Tables 49 | 50 | Lookup tables are commonly populated from their `contents` property. 51 | 52 | The table below is declared as a lookup table with its contents property 53 | provided to generate entities. 54 | 55 | File `+lab/User.m` 56 | 57 | ```matlab 58 | %{ 59 | # users in the lab 60 | username : varchar(20) # user in the lab 61 | --- 62 | first_name : varchar(20) # user first name 63 | last_name : varchar(20) # user last name 64 | %} 65 | classdef User < dj.Lookup 66 | properties 67 | contents = { 68 | 'cajal' 'Santiago' 'Cajal' 69 | 'hubel' 'David' 'Hubel' 70 | 'wiesel' 'Torsten' 'Wiesel' 71 | } 72 | end 73 | end 74 | ``` 75 | 76 | ## Imported and Computed Tables 77 | 78 | Imported and Computed tables provide [`make` methods](./make-method) to determine how 79 | they are populated, either from files or other tables. 80 | 81 | Imagine that there is a table `test.Image` that contains 2D grayscale images in its 82 | `image` attribute. We can define the Computed table, `test.FilteredImage` that filters 83 | the image in some way and saves the result in its `filtered_image` attribute. 84 | 85 | ```matlab 86 | %{ Filtered image 87 | -> test.Image 88 | --- 89 | filtered_image : longblob 90 | %} 91 | 92 | classdef FilteredImage < dj.Computed 93 | methods(Access=protected) 94 | function makeTuples(self, key) 95 | img = fetch1(test.Image & key, 'image'); 96 | key.filtered_image = my_filter(img); 97 | self.insert(key) 98 | end 99 | end 100 | end 101 | ``` 102 | 103 | ??? Note "`makeTuples` vs. `make`" 104 | 105 | Currently matlab uses `makeTuples` rather than `make`. This will be 106 | fixed in an upcoming release. You can monitor the discussion 107 | [here](https://github.com/datajoint/datajoint-matlab/issues/141) 108 | 109 | ## Part Tables 110 | 111 | The following code defines a Imported table with an associated part table. In MATLAB, 112 | the master and part tables are declared in a separate `classdef` file. The name of the 113 | part table must begin with the name of the master table. The part table must declare the 114 | property `master` containing an object of the master. 115 | 116 | `+image/Scan.m` 117 | 118 | ``` matlab 119 | %{ 120 | # Two-photon imaging scan 121 | -> Session 122 | scan : smallint # scan number within the session 123 | --- 124 | -> Lens 125 | laser_wavelength : decimal(5,1) # um 126 | laser_power : decimal(4,1) # mW 127 | %} 128 | classdef Scan < dj.Computed 129 | methods(Access=protected) 130 | function make(self, key) 131 | self.insert(key) 132 | make(image.ScanField, key) 133 | end 134 | end 135 | end 136 | ``` 137 | 138 | `+image/ScanField.m` 139 | 140 | ``` matlab 141 | %{ 142 | # Region of interest resulting from segmentation 143 | -> image.Scan 144 | mask : smallint 145 | --- 146 | ROI : longblob # Region of interest 147 | %} 148 | 149 | classdef ScanField < dj.Part 150 | properties(SetAccess=protected) 151 | master = image.Scan 152 | end 153 | methods 154 | function make(self, key) 155 | ... 156 | self.insert(entity) 157 | end 158 | end 159 | end 160 | ``` 161 | -------------------------------------------------------------------------------- /docs/src/tutorials.md: -------------------------------------------------------------------------------- 1 | # Tutorials 2 | 3 | Coming soon! 4 | -------------------------------------------------------------------------------- /erd.m: -------------------------------------------------------------------------------- 1 | function erd(varargin) 2 | % ERD -- plot the entity relationship diagram of a DataJoint package. 3 | % 4 | % See also dj.Schema/erd, dj.Table.erd 5 | 6 | if ~nargin 7 | disp 'nothing to plot' 8 | return 9 | end 10 | 11 | ret = dj.ERD(); 12 | for entity = varargin 13 | if exist(entity{1}, 'class') 14 | obj = dj.ERD(feval(entity{1})); 15 | r = dj.config('displayDiagram_hierarchy_radius'); 16 | while min(r)>0 17 | if r(1)>0 18 | obj.up 19 | r(1) = r(1)-1; 20 | end 21 | if r(2)>0 22 | obj.down 23 | r(2) = r(2)-1; 24 | end 25 | end 26 | else 27 | obj = dj.ERD(feval([entity{1} '.getSchema'])); 28 | end 29 | ret = ret + obj; 30 | end 31 | ret.draw 32 | end -------------------------------------------------------------------------------- /local-docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # docker compose -f local-docker-compose.yaml --env-file LNX.env up --build 2 | version: '2.4' 3 | x-net: &net 4 | networks: 5 | - main 6 | services: 7 | db: 8 | <<: *net 9 | image: datajoint/mysql:${MYSQL_TAG} 10 | environment: 11 | - MYSQL_ROOT_PASSWORD=simple 12 | # ports: 13 | # - "3306:3306" 14 | ## To persist MySQL data 15 | # volumes: 16 | # - ./mysql/data:/var/lib/mysql 17 | minio: 18 | <<: *net 19 | environment: 20 | - MINIO_ACCESS_KEY=datajoint 21 | - MINIO_SECRET_KEY=datajoint 22 | image: minio/minio:$MINIO_VER 23 | # ports: 24 | # - "9000:9000" 25 | # To persist MinIO data and config 26 | # volumes: 27 | # - ./minio/data:/data 28 | # - ./minio/config:/root/.minio 29 | command: server /data 30 | healthcheck: 31 | test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"] 32 | timeout: 5s 33 | retries: 60 34 | interval: 1s 35 | fakeservices.datajoint.io: 36 | <<: *net 37 | image: datajoint/nginx:v0.2.4 38 | environment: 39 | - ADD_db_TYPE=DATABASE 40 | - ADD_db_ENDPOINT=db:3306 41 | - ADD_minio_TYPE=MINIO 42 | - ADD_minio_ENDPOINT=minio:9000 43 | - ADD_minio_PORT=80 # allow unencrypted connections 44 | - ADD_minio_PREFIX=/datajoint 45 | - ADD_browser_TYPE=MINIOADMIN 46 | - ADD_browser_ENDPOINT=minio:9000 47 | - ADD_browser_PORT=80 # allow unencrypted connections 48 | ports: 49 | - "80:80" 50 | - "443:443" 51 | - "3306:3306" 52 | depends_on: 53 | db: 54 | condition: service_healthy 55 | minio: 56 | condition: service_healthy 57 | app: 58 | <<: *net 59 | environment: 60 | - DJ_HOST=fakeservices.datajoint.io 61 | - DJ_USER=root 62 | - DJ_PASS=simple 63 | - DJ_TEST_HOST=fakeservices.datajoint.io 64 | - DJ_TEST_USER=datajoint 65 | - DJ_TEST_PASSWORD=datajoint 66 | - S3_ENDPOINT=fakeservices.datajoint.io 67 | - S3_ACCESS_KEY=datajoint 68 | - S3_SECRET_KEY=datajoint 69 | - S3_BUCKET=datajoint.test 70 | - MATLAB_USER 71 | - MATLAB_LICENSE 72 | - JUPYTER_PASSWORD=datajoint 73 | - DISPLAY 74 | image: raphaelguzman/matlab:${MATLAB_VERSION}-GUI 75 | depends_on: 76 | fakeservices.datajoint.io: 77 | condition: service_healthy 78 | ports: 79 | - "8888:8888" 80 | user: ${MATLAB_UID}:${MATLAB_GID} 81 | working_dir: /home/muser 82 | command: 83 | - /bin/bash 84 | - -c 85 | - | 86 | set -e 87 | ORIG_DIR=$$(pwd) 88 | mkdir ~/Documents 89 | cd /src 90 | # package into toolbox, and install 91 | matlab -nodisplay -r " 92 | websave([tempdir 'GHToolbox.mltbx'],\ 93 | ['https://github.com/datajoint/GHToolbox' \ 94 | '/releases/download/' subsref(webread(['https://api.github.com/repos' \ 95 | '/datajoint/GHToolbox' \ 96 | '/releases/latest']),\ 97 | substruct('.', 'tag_name')) \ 98 | '/GHToolbox.mltbx']);\ 99 | matlab.addons.toolbox.installToolbox([tempdir 'GHToolbox.mltbx']);\ 100 | fid = fopen('README.md', 'r');\ 101 | docs = fread(fid, '*char')';\ 102 | fclose(fid);\ 103 | ghtb.package('DataJoint',\ 104 | 'Raphael Guzman',\ 105 | 'raphael.h.guzman@gmail.com',\ 106 | ['Scientific workflow management framework built on top of a ' \ 107 | 'relational database.'],\ 108 | docs,\ 109 | {'.vscode', '.git*', '*.env', '*.yaml', 'tests', 'mym', 'docs-parts',\ 110 | '*.txt', '*.prf', '*.md', 'notebook'},\ 111 | @() strjoin(arrayfun(@(x) num2str(x),\ 112 | cell2mat(struct2cell(dj.version)),\ 113 | 'uni', false),\ 114 | '.'),\ 115 | {'+dj', 'erd.m'});\ 116 | matlab.addons.toolbox.installToolbox('DataJoint.mltbx');\ 117 | addpath('tests');\ 118 | savepath;\ 119 | cd(tempdir);\ 120 | dir(fileparts(which('erd')));\ 121 | disp(dj.version);\ 122 | " 123 | cd "$${ORIG_DIR}" 124 | # Copy preferences 125 | # cp /tmp/matlab.prf /home/muser/.matlab/${MATLAB_VERSION}/matlab.prf 126 | # Interactive Jupyter Notebook environment 127 | jupyter notebook 128 | mac_address: $MATLAB_HOSTID 129 | volumes: 130 | ## Dev mounts 131 | - .:/src 132 | - /tmp/.X11-unix:/tmp/.X11-unix:rw 133 | ## Additional mounts may go here 134 | # - ./notebook:/home/muser/notebooks 135 | # - ./matlab.prf:/tmp/matlab.prf 136 | networks: 137 | main: -------------------------------------------------------------------------------- /tests/Main.m: -------------------------------------------------------------------------------- 1 | classdef Main < ... 2 | TestConfig & ... 3 | TestConnection & ... 4 | TestDelete & ... 5 | TestDeclaration & ... 6 | TestERD & ... 7 | TestExternalFile & ... 8 | TestExternalS3 & ... 9 | TestFetch & ... 10 | TestPopulate & ... 11 | TestProjection & ... 12 | TestRelationalOperand & ... 13 | TestSchema & ... 14 | TestTls & ... 15 | TestUuid & ... 16 | TestBlob 17 | end -------------------------------------------------------------------------------- /tests/Prep.m: -------------------------------------------------------------------------------- 1 | classdef Prep < matlab.unittest.TestCase 2 | % Setup and teardown for tests. 3 | properties (Constant) 4 | CONN_INFO_ROOT = struct(... 5 | 'host', getenv('DJ_HOST'), ... 6 | 'user', getenv('DJ_USER'), ... 7 | 'password', getenv('DJ_PASS')); 8 | CONN_INFO = struct(... 9 | 'host', getenv('DJ_TEST_HOST'), ... 10 | 'user', getenv('DJ_TEST_USER'), ... 11 | 'password', getenv('DJ_TEST_PASSWORD')); 12 | S3_CONN_INFO = struct(... 13 | 'endpoint', getenv('S3_ENDPOINT'), ... 14 | 'access_key', getenv('S3_ACCESS_KEY'), ... 15 | 'secret_key', getenv('S3_SECRET_KEY'), ... 16 | 'bucket', getenv('S3_BUCKET')); 17 | PREFIX = 'djtest'; 18 | end 19 | properties 20 | test_root; 21 | external_file_store_root; 22 | end 23 | methods 24 | function obj = Prep() 25 | % Initialize test_root 26 | % obj.test_root = [pwd '/tests']; 27 | obj.test_root = fileparts(which('Main')); 28 | if ispc 29 | obj.external_file_store_root = [getenv('TEMP') '\root']; 30 | else 31 | obj.external_file_store_root = '/tmp/root'; 32 | end 33 | end 34 | end 35 | methods (TestClassSetup) 36 | function init(testCase) 37 | disp('---------------INIT---------------'); 38 | clear functions; 39 | addpath([testCase.test_root '/test_schemas']); 40 | dj.config('safemode', false); 41 | disp(dj.version); 42 | curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... 43 | testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password,'',true); 44 | % create test users 45 | ver = curr_conn.query('select @@version as version').version; 46 | if compareVersions(ver,'5.8') 47 | cmd = {... 48 | 'CREATE USER IF NOT EXISTS ''datajoint''@''%%'' ' 49 | 'IDENTIFIED BY ''datajoint'';' 50 | }; 51 | curr_conn.query(sprintf('%s',cmd{:})); 52 | 53 | cmd = {... 54 | 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'';' 55 | }; 56 | curr_conn.query(sprintf('%s',cmd{:})); 57 | 58 | cmd = {... 59 | 'CREATE USER IF NOT EXISTS ''djview''@''%%'' ' 60 | 'IDENTIFIED BY ''djview'';' 61 | }; 62 | curr_conn.query(sprintf('%s',cmd{:})); 63 | 64 | cmd = {... 65 | 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'';' 66 | }; 67 | curr_conn.query(sprintf('%s',cmd{:})); 68 | 69 | cmd = {... 70 | 'CREATE USER IF NOT EXISTS ''djssl''@''%%'' ' 71 | 'IDENTIFIED BY ''djssl'' ' 72 | 'REQUIRE SSL;' 73 | }; 74 | curr_conn.query(sprintf('%s',cmd{:})); 75 | 76 | cmd = {... 77 | 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'';' 78 | }; 79 | curr_conn.query(sprintf('%s',cmd{:})); 80 | else 81 | cmd = {... 82 | 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'' ' 83 | 'IDENTIFIED BY ''datajoint'';' 84 | }; 85 | curr_conn.query(sprintf('%s',cmd{:})); 86 | 87 | cmd = {... 88 | 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'' ' 89 | 'IDENTIFIED BY ''djview'';' 90 | }; 91 | curr_conn.query(sprintf('%s',cmd{:})); 92 | 93 | cmd = {... 94 | 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'' ' 95 | 'IDENTIFIED BY ''djssl'' ' 96 | 'REQUIRE SSL;' 97 | }; 98 | curr_conn.query(sprintf('%s',cmd{:})); 99 | end 100 | % create test bucket 101 | dj.store_plugins.S3.RESTCallAWSSigned('http://', ... 102 | testCase.S3_CONN_INFO.endpoint, ['/' testCase.S3_CONN_INFO.bucket], ... 103 | uint8(''), testCase.S3_CONN_INFO.access_key, ... 104 | testCase.S3_CONN_INFO.secret_key, 'put'); 105 | end 106 | end 107 | methods (TestClassTeardown) 108 | function dispose(testCase) 109 | disp('---------------DISP---------------'); 110 | dj.config('safemode', false); 111 | curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... 112 | testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password, '',true); 113 | 114 | % remove databases 115 | curr_conn.query('SET FOREIGN_KEY_CHECKS=0;'); 116 | res = curr_conn.query(['SHOW DATABASES LIKE "' testCase.PREFIX '_%";']); 117 | for i = 1:length(res.(['Database (' testCase.PREFIX '_%)'])) 118 | curr_conn.query(['DROP DATABASE `' ... 119 | res.(['Database (' testCase.PREFIX '_%)']){i} '`;']); 120 | end 121 | curr_conn.query('SET FOREIGN_KEY_CHECKS=1;'); 122 | 123 | % remove external storage content 124 | if ispc 125 | [status,cmdout] = system(['rmdir /Q /s "' ... 126 | testCase.external_file_store_root '"']); 127 | else 128 | [status,cmdout] = system(['rm -R ' ... 129 | testCase.external_file_store_root]); 130 | end 131 | % remove test bucket 132 | dj.store_plugins.S3.RESTCallAWSSigned('http://', ... 133 | testCase.S3_CONN_INFO.endpoint, ['/' testCase.S3_CONN_INFO.bucket], ... 134 | uint8(''), testCase.S3_CONN_INFO.access_key, ... 135 | testCase.S3_CONN_INFO.secret_key, 'delete'); 136 | 137 | % remove users 138 | cmd = {... 139 | 'DROP USER ''datajoint''@''%%'';' 140 | 'DROP USER ''djview''@''%%'';' 141 | 'DROP USER ''djssl''@''%%'';' 142 | }; 143 | res = curr_conn.query(sprintf('%s',cmd{:})); 144 | curr_conn.delete; 145 | 146 | % Remove getSchemas to ensure they are created by tests. 147 | files = dir([testCase.test_root '/test_schemas']); 148 | dirFlags = [files.isdir] & ~strcmp({files.name},'.') & ~strcmp({files.name},'..'); 149 | subFolders = files(dirFlags); 150 | for k = 1 : length(subFolders) 151 | delete(sprintf('%s/test_schemas/%s/getSchema.m', ... 152 | testCase.test_root, subFolders(k).name)); 153 | if strcmp(subFolders(k).name, '+Lab') 154 | delete(sprintf('%s/test_schemas/%s/Jobs.m', ... 155 | testCase.test_root, subFolders(k).name)); 156 | end 157 | % delete(['test_schemas/+University/getSchema.m']) 158 | end 159 | warning('off','MATLAB:RMDIR:RemovedFromPath'); 160 | rmpath([testCase.test_root '/test_schemas']); 161 | warning('on','MATLAB:RMDIR:RemovedFromPath'); 162 | end 163 | end 164 | end -------------------------------------------------------------------------------- /tests/TestBlob.m: -------------------------------------------------------------------------------- 1 | classdef TestBlob < Prep 2 | methods (Test) 3 | function TestBlob_test32BitRead(testCase) 4 | st = dbstack; 5 | disp(['---------------' st(1).name '---------------']); 6 | value = ['6D596D005302000000010000000100000004000000686974730073696465730074' ... 7 | '61736B73007374616765004D00000041020000000100000007000000060000000000000' ... 8 | '0000000000000F8FF000000000000F03F000000000000F03F0000000000000000000000' ... 9 | '000000F03F0000000000000000000000000000F8FF23000000410200000001000000070' ... 10 | '0000004000000000000006C006C006C006C00720072006C002300000041020000000100' ... 11 | '00000700000004000000000000006400640064006400640064006400250000004102000' ... 12 | '0000100000008000000040000000000000053007400610067006500200031003000']; 13 | hexstring = value'; 14 | reshapedString = reshape(hexstring,2,length(value)/2); 15 | hexMtx = reshapedString.'; 16 | decMtx = hex2dec(hexMtx); 17 | packed = uint8(decMtx); 18 | 19 | data = struct; 20 | data.stage = 'Stage 10'; 21 | data.tasks = 'ddddddd'; 22 | data.sides = 'llllrrl'; 23 | data.hits = [NaN,1,1,0,1,0,NaN]; 24 | 25 | dj.config('use_32bit_dims', true); 26 | unpacked = mym('deserialize', packed); 27 | dj.config('use_32bit_dims', false); 28 | 29 | testCase.verifyEqual(unpacked, data); 30 | end 31 | end 32 | end -------------------------------------------------------------------------------- /tests/TestConnection.m: -------------------------------------------------------------------------------- 1 | classdef TestConnection < Prep 2 | % TestConnection tests typical connection scenarios. 3 | methods (Test) 4 | function TestConnection_testConnection(testCase) 5 | st = dbstack; 6 | disp(['---------------' st(1).name '---------------']); 7 | testCase.verifyTrue(dj.conn(... 8 | testCase.CONN_INFO.host,... 9 | testCase.CONN_INFO.user,... 10 | testCase.CONN_INFO.password,'',true).isConnected); 11 | end 12 | function TestConnection_testConnectionExists(testCase) 13 | % testConnectionExists tests that will not fail if connection open 14 | % to the same host. 15 | % Fix https://github.com/datajoint/datajoint-matlab/issues/160 16 | st = dbstack; 17 | disp(['---------------' st(1).name '---------------']); 18 | dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); 19 | dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); 20 | end 21 | function TestConnection_testConnectionDiffHost(testCase) 22 | % testConnectionDiffHost tests that will fail if connection open 23 | % to a different host. 24 | % Fix https://github.com/datajoint/datajoint-matlab/issues/160 25 | st = dbstack; 26 | disp(['---------------' st(1).name '---------------']); 27 | dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); 28 | 29 | testCase.verifyError(@() dj.conn(... 30 | 'anything', '', '', '', '', true), ... 31 | 'DataJoint:Connection:AlreadyInstantiated'); 32 | end 33 | function TestConnection_testPort(testCase) 34 | st = dbstack; 35 | disp(['---------------' st(1).name '---------------']); 36 | testCase.verifyError(@() dj.conn(... 37 | [testCase.CONN_INFO.host ':3307'], ... 38 | testCase.CONN_INFO.user,... 39 | testCase.CONN_INFO.password,'',true), ... 40 | 'MySQL:Error'); 41 | end 42 | function TestConnection_testTransactionRollback(testCase) 43 | st = dbstack; 44 | disp(['---------------' st(1).name '---------------']); 45 | package = 'University'; 46 | 47 | c1 = dj.conn(... 48 | testCase.CONN_INFO.host,... 49 | testCase.CONN_INFO.user,... 50 | testCase.CONN_INFO.password,'',true); 51 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 52 | [testCase.PREFIX '_university']); 53 | schema = University.getSchema; 54 | tmp = { 55 | 20 'Henry' 'Jupyter' '2020-11-25 12:34:56' 56 | 21 'Lacy' 'Mars' '2017-11-25 12:34:56' 57 | }; 58 | 59 | insert(University.Student, tmp(1, :)); 60 | 61 | schema.conn.startTransaction 62 | try 63 | insert(University.Student, tmp(2, :)); 64 | assert(false, 'Customer:Error', 'Message') 65 | catch ME 66 | schema.conn.cancelTransaction 67 | if ~strcmp(ME.identifier,'Customer:Error') 68 | rethrow(ME); 69 | end 70 | end 71 | 72 | q = University.Student & 'student_id in (20,21)'; 73 | testCase.verifyEqual(q.count, 1); 74 | testCase.verifyEqual(q.fetch1('student_id'), 20); 75 | end 76 | end 77 | end -------------------------------------------------------------------------------- /tests/TestDeclaration.m: -------------------------------------------------------------------------------- 1 | classdef TestDeclaration < Prep 2 | methods (Test) 3 | %{ Function to test if a table can be inserted with curley brackets in the comments 4 | %} 5 | function TestDeclaration_testCurlyBracketComment(testCase) 6 | packageName = 'Lab'; 7 | lowerPackageName = lower(packageName); 8 | % Create the connection 9 | c1 = dj.conn(... 10 | testCase.CONN_INFO.host,... 11 | testCase.CONN_INFO.user,... 12 | testCase.CONN_INFO.password, '', true); 13 | 14 | % Create the schema 15 | dj.createSchema(packageName, [testCase.test_root '/test_schemas'], ... 16 | [testCase.PREFIX '_' lowerPackageName]); 17 | 18 | % Initialize the table 19 | Lab.Subject 20 | 21 | % Check that the comment is correct 22 | table = Lab.Subject(); 23 | firstAttributeComment = table.header.attributes.comment; 24 | assert(strcmp( ... 25 | firstAttributeComment, ... 26 | '{subject_id} Comment to test curly bracket'), ... 27 | 'Comment did not get inserted correctly'... 28 | ); 29 | end 30 | end 31 | end -------------------------------------------------------------------------------- /tests/TestDelete.m: -------------------------------------------------------------------------------- 1 | classdef TestDelete < Prep 2 | % TestDelete tests delete operations. 3 | methods (Test) 4 | function TestDelete_testRenamedDelete(testCase) 5 | st = dbstack; 6 | disp(['---------------' st(1).name '---------------']); 7 | % https://github.com/datajoint/datajoint-matlab/issues/362 8 | dj.config('safemode', false); 9 | package = 'Company'; 10 | 11 | c1 = dj.conn(... 12 | testCase.CONN_INFO.host,... 13 | testCase.CONN_INFO.user,... 14 | testCase.CONN_INFO.password,'',true); 15 | 16 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 17 | [testCase.PREFIX '_company']); 18 | 19 | inserti(Company.Employee, {'raphael', 2019; 'shan', 2018; 'chris', 2018; ... 20 | 'thinh', 2019}); 21 | inserti(Company.Duty, {'schedule1', 'shan', 2018; 'schedule2', 'raphael', 2019}); 22 | inserti(Company.Machine, {'shan', 2018, 'abc1023'; 'raphael', 2019, 'xyz9876'}); 23 | testCase.verifyEqual(length(fetch(Company.Employee)), 4); 24 | testCase.verifyEqual(length(fetch(Company.Duty)), 2); 25 | testCase.verifyEqual(length(fetch(Company.Machine)), 2); 26 | 27 | del(Company.Employee & 'employee_id="shan"'); 28 | 29 | testCase.verifyEqual(length(fetch(Company.Employee)), 3); 30 | testCase.verifyEqual(... 31 | length(fetch(Company.Employee & struct('employee_id', 'shan'))), 0); 32 | testCase.verifyEqual(length(fetch(Company.Duty)), 1); 33 | testCase.verifyEqual(... 34 | length(fetch(Company.Duty & struct('monday_on_call', 'shan'))), 0); 35 | testCase.verifyEqual(length(fetch(Company.Machine)), 1); 36 | testCase.verifyEqual(... 37 | length(fetch(Company.Machine & struct('employee_id', 'shan'))), 0); 38 | end 39 | function TestDelete_testThreeFKOnePK(testCase) 40 | st = dbstack; 41 | disp(['---------------' st(1).name '---------------']); 42 | % https:%github.com/datajoint/datajoint-matlab/issues/379 43 | dj.config('safemode', false); 44 | package = 'TestLab'; 45 | 46 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 47 | [testCase.PREFIX '_testlab']); 48 | 49 | users = [{'user0'; 'user1'; 'user2'; 'user3'}]; 50 | 51 | insert(TestLab.User, users); 52 | 53 | duty = [{'2020-01-01','user0','user2', 'user1'}, 54 | {'2020-01-02','user0','user1', 'user2'}, 55 | {'2020-12-30','user1','user2', 'user0'}, 56 | {'2020-12-31','user0','user2', 'user3'}]; 57 | 58 | insert(TestLab.Duty, duty); 59 | 60 | key.user_id = 'user1'; 61 | del(TestLab.User & key); 62 | 63 | testCase.verifyEqual(length(fetch(TestLab.User)), 3); 64 | testCase.verifyEqual(length(fetch(TestLab.Duty)), 1); 65 | testCase.verifyEqual(length(fetch(TestLab.User & 'user_id = "user1"')), 0); 66 | testCase.verifyEqual(length(fetch(TestLab.Duty & 'duty_first = "user1" OR duty_second = "user1" OR duty_third = "user1"')), 0); 67 | end 68 | end 69 | end -------------------------------------------------------------------------------- /tests/TestERD.m: -------------------------------------------------------------------------------- 1 | classdef TestERD < Prep 2 | % TestERD tests unusual ERD scenarios. 3 | methods (Test) 4 | function TestERD_testDraw(testCase) 5 | st = dbstack; 6 | disp(['---------------' st(1).name '---------------']); 7 | package = 'University'; 8 | 9 | c1 = dj.conn(... 10 | testCase.CONN_INFO.host,... 11 | testCase.CONN_INFO.user,... 12 | testCase.CONN_INFO.password,'',true); 13 | 14 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 15 | [testCase.PREFIX '_university']); 16 | 17 | insert(University.Student, { 18 | 0 'John' 'Smith' '2019-09-19 16:50' 19 | 1 'Phil' 'Howard' '2019-04-30 12:34:56' 20 | 2 'Ben' 'Goyle' '2019-05-11' 21 | }); 22 | 23 | dj.ERD(University.Student) 24 | end 25 | end 26 | end -------------------------------------------------------------------------------- /tests/TestExternalFile.m: -------------------------------------------------------------------------------- 1 | classdef TestExternalFile < Prep 2 | % TestExternalFile tests scenarios related to external file store. 3 | methods (Static) 4 | function TestExternalFile_checks(test_instance, store, cache) 5 | % load config 6 | pkg_path = test_instance.test_root; 7 | ext_root = strrep(test_instance.external_file_store_root, '\', '/'); 8 | dj.config.load([strrep(pkg_path, '\', '/') '/test_schemas/store_config.json']); 9 | dj.config(['stores.' store '.location'], strrep(dj.config(... 10 | ['stores.' store '.location']), '{{external_file_store_root}}', ... 11 | ext_root)); 12 | dj.config('stores.main', dj.config(['stores.' store])); 13 | dj.config(cache, [ext_root '/cache']); 14 | % create location/cache directories 15 | mkdir(dj.config(cache)); 16 | if strcmp(dj.config('stores.main.protocol'), 'file') 17 | mkdir(dj.config('stores.main.location')); 18 | elseif strcmp(dj.config('stores.main.protocol'), 's3') 19 | if any(strcmp('secure', fieldnames(dj.config('stores.main')))) && ... 20 | dj.config('stores.main.secure') 21 | dj.config('stores.main.endpoint', strrep(... 22 | test_instance.S3_CONN_INFO.endpoint, ':9000', ':443')); 23 | else 24 | dj.config('stores.main.endpoint', test_instance.S3_CONN_INFO.endpoint); 25 | end 26 | dj.config('stores.main.access_key', test_instance.S3_CONN_INFO.access_key); 27 | dj.config('stores.main.secret_key', test_instance.S3_CONN_INFO.secret_key); 28 | dj.config('stores.main.bucket', test_instance.S3_CONN_INFO.bucket); 29 | end 30 | % create schema 31 | package = 'External'; 32 | dj.createSchema(package,[test_instance.test_root '/test_schemas'], ... 33 | [test_instance.PREFIX '_external']); 34 | % test value 35 | rng(5); 36 | test_val1 = floor(rand(1,3)*100); 37 | % insert 38 | insert(External.Dimension, struct( ... 39 | 'dimension_id', 4, ... 40 | 'dimension', test_val1 ... 41 | )); 42 | % check that external tables are loaded on new schema objs if they already exist 43 | delete([test_instance.test_root '/test_schemas' '/+' package '/getSchema.m']); 44 | dj.createSchema(package,[test_instance.test_root '/test_schemas'], ... 45 | [test_instance.PREFIX '_external']); 46 | schema = External.getSchema; 47 | % fetch 48 | schema.tableNames.remove('External.Dimension'); 49 | q = External.Dimension & 'dimension_id=4'; 50 | res = q.fetch('dimension'); 51 | value_check = res(1).dimension; 52 | test_instance.verifyEqual(value_check, test_val1); 53 | % check subfolding 54 | packed_cell = mym('serialize {M}', test_val1); 55 | uuid = dj.lib.DataHash(packed_cell{1}, 'bin', 'hex', 'MD5'); 56 | uuid_path = schema.external.table('main').make_uuid_path(uuid, ''); 57 | if strcmp(dj.config('stores.main.protocol'), 'file') 58 | subfold_path = strrep(uuid_path, dj.config('stores.main.location'), ''); 59 | elseif strcmp(dj.config('stores.main.protocol'), 's3') 60 | subfold_path = strrep(uuid_path, ['/' dj.config('stores.main.bucket') ... 61 | '/' dj.config('stores.main.location')], ''); 62 | end 63 | subfold_path = strrep(subfold_path, ['/' schema.dbname '/'], ''); 64 | subfold_cell = strsplit(subfold_path, '/'); 65 | if length(subfold_cell) > 1 66 | subfold_cell = subfold_cell(1:end-1); 67 | subfold_path = ['/' strjoin(subfold_cell, '/')]; 68 | else 69 | subfold_cell = {}; 70 | subfold_path = ''; 71 | end 72 | test_instance.verifyEqual(cellfun(@(x) length(x), subfold_cell)', ... 73 | schema.external.table('main').spec.type_config.subfolding); 74 | % delete value to rely on cache 75 | schema.external.table('main').spec.remove_object(uuid_path); 76 | res = q.fetchn('dimension'); 77 | value_check = res{1}; 78 | test_instance.verifyEqual(value_check, test_val1); 79 | % populate 80 | populate(External.Image); 81 | q = External.Image & 'dimension_id=4'; 82 | res = q.fetch1('img'); 83 | value_check = res; 84 | test_instance.verifyEqual(size(value_check), test_val1); 85 | % check used and unused 86 | test_instance.verifyTrue(schema.external.table('main').used.count==2); 87 | test_instance.verifyTrue(schema.external.table('main').unused.count==0); 88 | % delete from Dimension 89 | del(External.Dimension); 90 | % check children 91 | q = External.Image; 92 | test_instance.verifyTrue(q.count==0); 93 | % check used and unused 94 | test_instance.verifyTrue(schema.external.table('main').used.count==0); 95 | test_instance.verifyTrue(schema.external.table('main').unused.count==2); 96 | % check delete from external 97 | schema.external.table('main').delete(true, ''); 98 | if strcmp(dj.config('stores.main.protocol'), 'file') 99 | test_instance.verifyEqual(lastwarn, ['File ''' ... 100 | dj.config('stores.main.location') '/' schema.dbname subfold_path '/' ... 101 | uuid ''' not found.']); 102 | end 103 | % reverse engineer 104 | q = External.Dimension; 105 | raw_def = dj.internal.Declare.getDefinition(q); 106 | assembled_def = describe(q); 107 | [raw_sql, ~] = dj.internal.Declare.declare(q, raw_def); 108 | [assembled_sql, ~] = dj.internal.Declare.declare(q, assembled_def); 109 | test_instance.verifyEqual(assembled_sql, raw_sql); 110 | % drop table 111 | drop(External.Dimension); 112 | % check used and unused 113 | test_instance.verifyTrue(schema.external.table('main').used.count==0); 114 | test_instance.verifyTrue(schema.external.table('main').unused.count==0); 115 | % remove external storage content 116 | if ispc 117 | [status,cmdout] = system(['rmdir /Q /s "' ... 118 | test_instance.external_file_store_root '"']); 119 | else 120 | [status,cmdout] = system(['rm -R ' ... 121 | test_instance.external_file_store_root]); 122 | end 123 | % Remove previous mapping 124 | schema.external.tables = struct(); 125 | % drop database 126 | schema.conn.query(['DROP DATABASE `' test_instance.PREFIX '_external`']); 127 | dj.config.restore; 128 | end 129 | end 130 | methods (Test) 131 | function TestExternalFile_testLocal(testCase) 132 | st = dbstack; 133 | disp(['---------------' st(1).name '---------------']); 134 | TestExternalFile.TestExternalFile_checks(testCase, 'new_local', 'blobCache'); 135 | end 136 | function TestExternalFile_testLocalDefault(testCase) 137 | st = dbstack; 138 | disp(['---------------' st(1).name '---------------']); 139 | TestExternalFile.TestExternalFile_checks(testCase, 'new_local_default', ... 140 | 'blobCache'); 141 | end 142 | function TestExternalFile_testBackward(testCase) 143 | st = dbstack; 144 | disp(['---------------' st(1).name '---------------']); 145 | TestExternalFile.TestExternalFile_checks(testCase, 'local', 'cache'); 146 | end 147 | function TestExternalFile_testBackwardDefault(testCase) 148 | st = dbstack; 149 | disp(['---------------' st(1).name '---------------']); 150 | TestExternalFile.TestExternalFile_checks(testCase, 'local_default', 'cache'); 151 | end 152 | function TestExternalFile_testMD5Hash(testCase) 153 | st = dbstack; 154 | disp(['---------------' st(1).name '---------------']); 155 | v = int64([1;2]); 156 | packed_cell = mym('serialize {M}', v); 157 | uuid = dj.lib.DataHash(packed_cell{1}, 'bin', 'hex', 'MD5'); 158 | testCase.verifyEqual(uuid, '1d751e2e1e74faf84ab485fde8ef72be'); 159 | end 160 | end 161 | end -------------------------------------------------------------------------------- /tests/TestExternalS3.m: -------------------------------------------------------------------------------- 1 | classdef TestExternalS3 < Prep 2 | % TestExternalS3 tests scenarios related to external S3 store. 3 | methods (Test) 4 | % function TestExternalS3_testRemote(testCase) 5 | % st = dbstack; 6 | % disp(['---------------' st(1).name '---------------']); 7 | % TestExternalFile.TestExternalFile_checks(testCase, 'new_remote', ... 8 | % 'blobCache'); 9 | % end 10 | function TestExternalS3_testRemoteDefault(testCase) 11 | st = dbstack; 12 | disp(['---------------' st(1).name '---------------']); 13 | TestExternalFile.TestExternalFile_checks(testCase, 'new_remote_default', ... 14 | 'blobCache'); 15 | end 16 | function TestExternalS3_testBackward(testCase) 17 | st = dbstack; 18 | disp(['---------------' st(1).name '---------------']); 19 | TestExternalFile.TestExternalFile_checks(testCase, 'remote', 'cache'); 20 | end 21 | function TestExternalS3_testBackwardDefault(testCase) 22 | st = dbstack; 23 | disp(['---------------' st(1).name '---------------']); 24 | TestExternalFile.TestExternalFile_checks(testCase, 'remote_default', ... 25 | 'cache'); 26 | end 27 | function TestExternalS3_testLocationFlexibility(testCase) 28 | st = dbstack; 29 | disp(['---------------' st(1).name '---------------']); 30 | location_values = {... 31 | 'leading/slash/test',... 32 | '/leading/slash/test',... 33 | 'leading\slash\test',... 34 | 'f:\leading\slash\test',... 35 | 'f:\leading/slash\test',... 36 | '/',... 37 | 'C:\',... 38 | ''... 39 | }; 40 | rel_path = 'does/this/work'; 41 | for v = location_values 42 | s3Obj = dj.store_plugins.S3(struct(... 43 | 'datajoint_type', 'blob', ... 44 | 'protocol', 's3', ... 45 | 'location', v, ... 46 | 'endpoint', testCase.S3_CONN_INFO.endpoint, ... 47 | 'bucket', testCase.S3_CONN_INFO.bucket, ... 48 | 'access_key', testCase.S3_CONN_INFO.access_key, ... 49 | 'secret_key', testCase.S3_CONN_INFO.secret_key, ... 50 | 'secure', false, ... 51 | 'subfolding', [1;1] ... 52 | )); 53 | if length(v{1}) > 6 54 | testCase.verifyEqual(s3Obj.make_external_filepath(rel_path), ... 55 | ['/' testCase.S3_CONN_INFO.bucket '/leading/slash/test' '/' rel_path]); 56 | else 57 | testCase.verifyEqual(s3Obj.make_external_filepath(rel_path), ... 58 | ['/' testCase.S3_CONN_INFO.bucket '/' rel_path]); 59 | end 60 | end 61 | end 62 | end 63 | end -------------------------------------------------------------------------------- /tests/TestPopulate.m: -------------------------------------------------------------------------------- 1 | classdef TestPopulate < Prep 2 | methods(Test) 3 | function TestPopulate_testPopulate(testCase) 4 | st = dbstack; 5 | disp(['---------------' st(1).name '---------------']); 6 | package = 'Lab'; 7 | 8 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 9 | [testCase.PREFIX '_lab']); 10 | 11 | lab_schema = Lab.getSchema; % we need schema's connection id 12 | sid = lab_schema.conn.serverId; 13 | 14 | insert(Lab.Subject, { 15 | 100, '2010-04-02'; 16 | }); 17 | 18 | insert(Lab.Rig, struct( ... 19 | 'rig_manufacturer', 'FooLab', ... 20 | 'rig_model', '1.0', ... 21 | 'rig_note', 'FooLab Frobnicator v1.0' ... 22 | )); 23 | 24 | % parallel populate of 1 record 25 | % .. (SessionAnalysis logs session ID as session_analysis data) 26 | % NOTE: need to call parpopulate 1st to ensure Jobs table 27 | % exists 28 | 29 | insert(Lab.Session, struct( ... 30 | 'session_id', 0, ... 31 | 'subject_id', 100, ... 32 | 'rig_manufacturer', 'FooLab', ... 33 | 'rig_model', '1.0' ... 34 | )); 35 | 36 | parpopulate(Lab.SessionAnalysis); 37 | a_result = fetch(Lab.SessionAnalysis & 'session_id = 0', '*'); 38 | testCase.verifyEqual(a_result.session_analysis.connection_id, sid); 39 | 40 | % regular populate of 1 record 41 | % .. (SessionAnalysis logs jobs record as session_analysis data) 42 | 43 | insert(Lab.Session, struct( ... 44 | 'session_id', 1, ... 45 | 'subject_id', 100, ... 46 | 'rig_manufacturer', 'FooLab', ... 47 | 'rig_model', '1.0' ... 48 | )); 49 | 50 | populate(Lab.SessionAnalysis); 51 | a_result = fetch(Lab.SessionAnalysis & 'session_id = 1', '*'); 52 | testCase.verifyEqual(a_result.session_analysis, 1); 53 | 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /tests/TestProjection.m: -------------------------------------------------------------------------------- 1 | classdef TestProjection < Prep 2 | % TestProjection tests use of q.proj(...). 3 | methods (Test) 4 | function TestProjection_testDateConversion(testCase) 5 | st = dbstack; 6 | disp(['---------------' st(1).name '---------------']); 7 | package = 'University'; 8 | 9 | c1 = dj.conn(... 10 | testCase.CONN_INFO.host,... 11 | testCase.CONN_INFO.user,... 12 | testCase.CONN_INFO.password,'',true); 13 | 14 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 15 | [testCase.PREFIX '_university']); 16 | 17 | insert(University.Student, { 18 | 10 'Raphael' 'Guzman' datestr(datetime, 'yyyy-mm-dd HH:MM:SS') 19 | 11 'Shan' 'Shen' '2019-11-25 12:34:56' 20 | 12 'Joe' 'Schmoe' '2018-01-24 14:34:16' 21 | }); 22 | 23 | q = proj(University.Student, 'date(enrolled)->enrolled_date') & ... 24 | 'enrolled_date="2018-01-24"'; 25 | 26 | res = q.fetch1('enrolled_date'); 27 | testCase.verifyEqual(res, '2018-01-24'); 28 | 29 | dj.config('safemode', 0); 30 | drop(University.Student); 31 | end 32 | function TestProjection_testRenameSameKey(testCase) 33 | st = dbstack; 34 | disp(['---------------' st(1).name '---------------']); 35 | package = 'University'; 36 | 37 | c1 = dj.conn(... 38 | testCase.CONN_INFO.host,... 39 | testCase.CONN_INFO.user,... 40 | testCase.CONN_INFO.password,'',true); 41 | 42 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 43 | [testCase.PREFIX '_university']); 44 | 45 | insert(University.Student, { 46 | 10 'Raphael' 'Guzman' datestr(datetime, 'yyyy-mm-dd HH:MM:SS') 47 | 11 'Shan' 'Shen' '2019-11-25 12:34:56' 48 | 12 'Joe' 'Schmoe' '2018-01-24 14:34:16' 49 | }); 50 | 51 | q = proj(University.Student & 'first_name = "Raphael"', 'student_id->faculty_id', 'student_id->school_id'); 52 | testCase.verifyEqual(q.fetch('faculty_id', 'school_id'), struct('faculty_id', 10, 'school_id', 10)); 53 | 54 | dj.config('safemode', 0); 55 | drop(University.Student); 56 | end 57 | end 58 | end -------------------------------------------------------------------------------- /tests/TestRelationalOperand.m: -------------------------------------------------------------------------------- 1 | classdef TestRelationalOperand < Prep 2 | % TestRelationalOperand tests relational operations. 3 | methods (Test) 4 | function TestRelationalOperand_testUpdateDate(testCase) 5 | st = dbstack; 6 | disp(['---------------' st(1).name '---------------']); 7 | % https://github.com/datajoint/datajoint-matlab/issues/211 8 | package = 'University'; 9 | 10 | c1 = dj.conn(... 11 | testCase.CONN_INFO.host,... 12 | testCase.CONN_INFO.user,... 13 | testCase.CONN_INFO.password,'',true); 14 | 15 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 16 | [testCase.PREFIX '_university']); 17 | 18 | insert(University.A, struct( ... 19 | 'id', 20, ... 20 | 'date', '2019-12-20' ... 21 | )); 22 | q = University.A & 'id=20'; 23 | 24 | new_value = []; 25 | q.update('date', new_value); 26 | res = mym(['select date from `' testCase.PREFIX ... 27 | '_university`.`a` where id=20 and date is null;']); 28 | testCase.verifyEqual(length(res.date), 1); 29 | 30 | new_value = '2020-04-14'; 31 | q.update('date', new_value); 32 | res = mym(['select date from `' testCase.PREFIX ... 33 | '_university`.`a` where id=20 and date like ''' new_value ''';']); 34 | testCase.verifyEqual(length(res.date), 1); 35 | 36 | q.update('date'); 37 | res = mym(['select date from `' testCase.PREFIX ... 38 | '_university`.`a` where id=20 and date is null;']); 39 | testCase.verifyEqual(length(res.date), 1); 40 | end 41 | function TestRelationalOperand_testUpdateString(testCase) 42 | st = dbstack; 43 | disp(['---------------' st(1).name '---------------']); 44 | % related https://github.com/datajoint/datajoint-matlab/issues/211 45 | package = 'University'; 46 | 47 | c1 = dj.conn(... 48 | testCase.CONN_INFO.host,... 49 | testCase.CONN_INFO.user,... 50 | testCase.CONN_INFO.password,'',true); 51 | 52 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 53 | [testCase.PREFIX '_university']); 54 | 55 | insert(University.A, struct( ... 56 | 'id', 30, ... 57 | 'string', 'normal' ... 58 | )); 59 | q = University.A & 'id=30'; 60 | 61 | new_value = ''; 62 | q.update('string', new_value); 63 | res = mym(['select string from `' testCase.PREFIX ... 64 | '_university`.`a` where id=30 and string like ''' new_value ''';']); 65 | testCase.verifyEqual(length(res.string), 1); 66 | 67 | new_value = ' '; 68 | q.update('string', new_value); 69 | res = mym(['select string from `' testCase.PREFIX ... 70 | '_university`.`a` where id=30 and string like ''' new_value ''';']); 71 | testCase.verifyEqual(length(res.string), 1); 72 | 73 | new_value = []; 74 | q.update('string', new_value); 75 | res = mym(['select string from `' testCase.PREFIX ... 76 | '_university`.`a` where id=30 and string is null;']); 77 | testCase.verifyEqual(length(res.string), 1); 78 | 79 | new_value = 'diff'; 80 | q.update('string', new_value); 81 | res = mym(['select string from `' testCase.PREFIX ... 82 | '_university`.`a` where id=30 and string like ''' new_value ''';']); 83 | testCase.verifyEqual(length(res.string), 1); 84 | 85 | q.update('string'); 86 | res = mym(['select string from `' testCase.PREFIX ... 87 | '_university`.`a` where id=30 and string is null;']); 88 | testCase.verifyEqual(length(res.string), 1); 89 | end 90 | function TestRelationalOperand_testFkOptions(testCase) 91 | st = dbstack; 92 | disp(['---------------' st(1).name '---------------']); 93 | % https://github.com/datajoint/datajoint-matlab/issues/110 94 | package = 'Lab'; 95 | 96 | c1 = dj.conn(... 97 | testCase.CONN_INFO.host,... 98 | testCase.CONN_INFO.user,... 99 | testCase.CONN_INFO.password,'',true); 100 | 101 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 102 | [testCase.PREFIX '_lab']); 103 | 104 | insert(Lab.Subject, { 105 | 0, '2020-04-02'; 106 | 1, '2020-05-03'; 107 | 2, '2020-04-22'; 108 | }); 109 | insert(Lab.Rig, struct( ... 110 | 'rig_manufacturer', 'Lenovo', ... 111 | 'rig_model', 'ThinkPad', ... 112 | 'rig_note', 'blah' ... 113 | )); 114 | % insert as renamed foreign keys 115 | insert(Lab.ActiveSession, struct( ... 116 | 'subject_id', 0, ... 117 | 'session_rig_class', 'Lenovo', ... 118 | 'session_rig_id', 'ThinkPad' ... 119 | )); 120 | testCase.verifyEqual(fetch1(Lab.ActiveSession, 'session_rig_class'), 'Lenovo'); 121 | % insert null for rig (subject reserved, awaiting rig assignment) 122 | insert(Lab.ActiveSession, struct( ... 123 | 'subject_id', 1 ... 124 | )); 125 | testCase.verifyTrue(isempty(fetch1(Lab.ActiveSession & 'subject_id=1', ... 126 | 'session_rig_class'))); 127 | % insert duplicate rig (rigs should only be active once per 128 | % subject) 129 | try 130 | insert(Lab.ActiveSession, struct( ... 131 | 'subject_id', 2, ... 132 | 'session_rig_class', 'Lenovo', ... 133 | 'session_rig_id', 'ThinkPad' ... 134 | )); 135 | error('Unique index fail...'); 136 | catch ME 137 | if ~contains(ME.message, 'Duplicate entry') 138 | rethrow(ME); 139 | end 140 | end 141 | % verify reverse engineering 142 | % (pending https://github.com/datajoint/datajoint-matlab/issues/305 solution) 143 | q = Lab.ActiveSession; 144 | raw_def = dj.internal.Declare.getDefinition(q); 145 | assembled_def = describe(q); 146 | [raw_sql, ~] = dj.internal.Declare.declare(q, raw_def); 147 | % [assembled_sql, ~] = dj.internal.Declare.declare(q, assembled_def); 148 | % testCase.verifyEqual(raw_sql, assembled_sql); 149 | end 150 | end 151 | end -------------------------------------------------------------------------------- /tests/TestSchema.m: -------------------------------------------------------------------------------- 1 | classdef TestSchema < Prep 2 | % TestSchema tests related to schemas. 3 | methods (Test) 4 | % https://github.com/datajoint/datajoint-matlab/issues/254 5 | function TestSchema_testUnsupportedDJTypes(testCase) 6 | st = dbstack; 7 | disp(['---------------' st(1).name '---------------']); 8 | % setup 9 | dj.config.restore; 10 | package = 'External'; 11 | c1 = dj.conn(... 12 | testCase.CONN_INFO.host,... 13 | testCase.CONN_INFO.user,... 14 | testCase.CONN_INFO.password,'',true); 15 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 16 | [testCase.PREFIX '_' lower(package)]); 17 | store_dir = '/tmp/fake'; 18 | store_name = 'main'; 19 | mkdir(store_dir); 20 | dj.config('safemode', false); 21 | dj.config('stores',struct(store_name, struct('protocol', 'file', 'location', ... 22 | store_dir))); 23 | id = 2; 24 | External.Document 25 | c1.query(['insert into `' testCase.PREFIX '_' lower(package) '`.`~external_' ... 26 | store_name '`(hash,size,attachment_name,filepath,contents_hash) values ' ... 27 | '(X''1d751e2e1e74faf84ab485fde8ef72be'', 1, ''attach_name'', ''filepath'',' ... 28 | ' X''1d751e2e1e74faf84ab485fde8ef72ca''),' ... 29 | '(X''1d751e2e1e74faf84ab485fde8ef72bf'', 2, ''attach_name'', ''filepath'',' ... 30 | 'X''1d751e2e1e74faf84ab485fde8ef72cb'')']); 31 | c1.query(['insert into `' testCase.PREFIX '_' lower(package) '`.`#document` ' ... 32 | 'values (' num2str(id) ', ''raphael'', ''hello'',' ... 33 | 'X''1d751e2e1e74faf84ab485fde8ef72be'',' ... 34 | 'X''1d751e2e1e74faf84ab485fde8ef72bf'')']); 35 | delete([testCase.test_root '/test_schemas/+' package '/getSchema.m']); 36 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 37 | [testCase.PREFIX '_' lower(package)]); 38 | schema = External.getSchema; 39 | q = schema.v.Document & ['document_id=' num2str(id)]; 40 | % display (be careful with MxPrint's b/c this test relies on stdout) 41 | queryPreview = evalc('q'); 42 | queryPreview = splitlines(queryPreview); 43 | recordPreview = queryPreview(end-4); 44 | recordPreview = strsplit(recordPreview{1}); 45 | testCase.verifyTrue(all(cellfun(@(x) strcmp(x,'''=BLOB='''), ... 46 | recordPreview(4:end-1)))); 47 | % reverse engineering 48 | raw_def = dj.internal.Declare.getDefinition(q); 49 | assembled_def = describe(q); 50 | [raw_sql, ~] = dj.internal.Declare.declare(q, raw_def); 51 | [assembled_sql, ~] = dj.internal.Declare.declare(q, assembled_def); 52 | testCase.verifyEqual(raw_sql, assembled_sql); 53 | % fetch good 54 | testCase.verifyEqual(q.fetch1('document_id'), id); 55 | % fetch bad 56 | for c = {'document_data1', 'document_data2', 'document_data3'} 57 | try 58 | res = q.fetch(char(c{1})); 59 | catch ME 60 | if ~strcmp(ME.identifier,'DataJoint:DataType:NotYetSupported') 61 | rethrow(ME); 62 | end 63 | end 64 | end 65 | % insert bad 66 | try 67 | insert(q, struct( ... 68 | 'document_id', 3, ... 69 | 'document_name', 'john', ... 70 | 'document_data1', 'this', ... 71 | 'document_data2', 'won''t', ... 72 | 'document_data3', 'work' ... 73 | )); 74 | catch ME 75 | if ~strcmp(ME.identifier,'DataJoint:DataType:NotYetSupported') 76 | rethrow(ME); 77 | end 78 | end 79 | % update good 80 | new_name = 'peter'; 81 | update(q, 'document_name', new_name); 82 | testCase.verifyEqual(q.fetch1('document_name'), new_name); 83 | % update bad 84 | for c = {'document_data1', 'document_data2', 'document_data3'} 85 | try 86 | update(q, char(c{1}), 'this'); 87 | catch ME 88 | if ~strcmp(ME.identifier,'DataJoint:DataType:NotYetSupported') 89 | rethrow(ME); 90 | end 91 | end 92 | end 93 | % clean up 94 | schema.dropQuick; 95 | rmdir(store_dir, 's'); 96 | end 97 | function TestSchema_testNew(testCase) 98 | st = dbstack; 99 | disp(['---------------' st(1).name '---------------']); 100 | % setup 101 | dj.config.restore; 102 | dj.config('safemode', false); 103 | dj.new('new_space.Student', 'M', pwd , 'djtest_new-space'); 104 | rmdir('+new_space', 's'); 105 | end 106 | function TestSchema_testVirtual(testCase) 107 | st = dbstack; 108 | disp(['---------------' st(1).name '---------------']); 109 | % setup 110 | dj.config.restore; 111 | package = 'Lab'; 112 | c1 = dj.conn(... 113 | testCase.CONN_INFO.host, ... 114 | testCase.CONN_INFO.user, ... 115 | testCase.CONN_INFO.password,'',true); 116 | dj.createSchema(package, sprintf('%s/test_schemas', testCase.test_root), ... 117 | sprintf('%s_%s', testCase.PREFIX, lower(package))); 118 | Lab.SessionAnalysis() 119 | schema = dj.Schema(c1, package, sprintf('%s_%s', testCase.PREFIX, lower(package))); 120 | schema.v.SessionAnalysis() 121 | end 122 | end 123 | end -------------------------------------------------------------------------------- /tests/TestTls.m: -------------------------------------------------------------------------------- 1 | classdef TestTls < Prep 2 | % TestTls tests TLS connection scenarios. 3 | methods (Test) 4 | function TestTls_testSecureConn(testCase) 5 | % secure connection test 6 | st = dbstack; 7 | disp(['---------------' st(1).name '---------------']); 8 | testCase.verifyTrue(length(dj.conn(... 9 | testCase.CONN_INFO.host, ... 10 | testCase.CONN_INFO.user, ... 11 | testCase.CONN_INFO.password, ... 12 | '',true,true).query(... 13 | 'SHOW STATUS LIKE ''Ssl_cipher''').Value{1}) > 0); 14 | end 15 | function TestTls_testInsecureConn(testCase) 16 | % insecure connection test 17 | st = dbstack; 18 | disp(['---------------' st(1).name '---------------']); 19 | testCase.verifyEqual(dj.conn(... 20 | testCase.CONN_INFO.host, ... 21 | testCase.CONN_INFO.user, ... 22 | testCase.CONN_INFO.password, ... 23 | '',true,false).query(... 24 | 'SHOW STATUS LIKE ''Ssl_cipher''').Value{1}, ... 25 | ''); 26 | end 27 | function TestTls_testPreferredConn(testCase) 28 | % preferred connection test 29 | st = dbstack; 30 | disp(['---------------' st(1).name '---------------']); 31 | testCase.verifyTrue(length(dj.conn(... 32 | testCase.CONN_INFO.host, ... 33 | testCase.CONN_INFO.user, ... 34 | testCase.CONN_INFO.password, ... 35 | '',true).query(... 36 | 'SHOW STATUS LIKE ''Ssl_cipher''').Value{1}) > 0); 37 | end 38 | function TestTls_testRejectException(testCase) 39 | % test exception on require TLS 40 | st = dbstack; 41 | disp(['---------------' st(1).name '---------------']); 42 | 43 | try 44 | curr_conn = dj.conn(... 45 | testCase.CONN_INFO.host, ... 46 | 'djssl', ... 47 | 'djssl', ... 48 | '',true,false); 49 | testCase.verifyTrue(false); 50 | catch ME 51 | testCase.verifyEqual(ME.identifier, 'MySQL:Error'); 52 | testCase.verifyTrue(contains(ME.message,'requires secure connection') || ... 53 | contains(ME.message,'Access denied')); %MySQL8 or MySQL5 54 | end 55 | end 56 | function TestTls_testStructException(testCase) 57 | % test exception on TLS struct 58 | st = dbstack; 59 | disp(['---------------' st(1).name '---------------']); 60 | testCase.verifyError(@() dj.conn(... 61 | testCase.CONN_INFO.host, ... 62 | testCase.CONN_INFO.user, ... 63 | testCase.CONN_INFO.password, ... 64 | '',true,struct('ca','fake/path/some/where')), ... 65 | 'mYm:TLS:InvalidStruct'); 66 | end 67 | end 68 | end -------------------------------------------------------------------------------- /tests/TestUuid.m: -------------------------------------------------------------------------------- 1 | classdef TestUuid < Prep 2 | % TestUuid tests uuid scenarios. 3 | methods (Test) 4 | function TestUuid_testInsertFetch(testCase) 5 | st = dbstack; 6 | disp(['---------------' st(1).name '---------------']); 7 | package = 'University'; 8 | 9 | c1 = dj.conn(... 10 | testCase.CONN_INFO.host,... 11 | testCase.CONN_INFO.user,... 12 | testCase.CONN_INFO.password,'',true); 13 | 14 | dj.createSchema(package,[testCase.test_root '/test_schemas'], ... 15 | [testCase.PREFIX '_university']); 16 | 17 | test_val1 = '1d751e2e-1e74-faf8-4ab4-85fde8ef72be'; 18 | test_val2 = '03aaa83d-289d-4f7e-96a7-bf91c2d8f5a4'; 19 | insert(University.Message, struct( ... 20 | 'msg_id', test_val1, ... 21 | 'body', 'Great campus!', ... 22 | 'dep_id', test_val2 ... 23 | )); 24 | 25 | test_val1 = '12321346-1312-4123-1234-312739283795'; 26 | insert(University.Message, struct( ... 27 | 'msg_id', strrep(test_val1, '-', ''), ... 28 | 'body', 'Where can I find the gym?' ... 29 | )); 30 | 31 | q = University.Message; 32 | res = q.fetch('*'); 33 | testCase.verifyEqual(res(1).msg_id, test_val1); 34 | testCase.verifyEqual(res(1).dep_id, uint8.empty(1, 0)); 35 | end 36 | function TestUuid_testQuery(testCase) 37 | st = dbstack; 38 | disp(['---------------' st(1).name '---------------']); 39 | package = 'University'; 40 | 41 | c1 = dj.conn(... 42 | testCase.CONN_INFO.host,... 43 | testCase.CONN_INFO.user,... 44 | testCase.CONN_INFO.password,'',true); 45 | 46 | test_val1 = '1d751e2e-1e74-faf8-4ab4-85fde8ef72be'; 47 | test_val2 = '12321346-1312-4123-1234-312739283795'; 48 | 49 | q = University.Message & [struct('msg_id',test_val1),struct('msg_id',test_val2)]; 50 | res = q.fetch('msg_id'); 51 | value_check = res(2).msg_id; 52 | 53 | testCase.verifyEqual(value_check, test_val1); 54 | end 55 | function TestUuid_testReverseEngineering(testCase) 56 | st = dbstack; 57 | disp(['---------------' st(1).name '---------------']); 58 | q = University.Message; 59 | raw_def = dj.internal.Declare.getDefinition(q); 60 | assembled_def = describe(q); 61 | [raw_sql, ~] = dj.internal.Declare.declare(q, raw_def); 62 | [assembled_sql, ~] = dj.internal.Declare.declare(q, assembled_def); 63 | testCase.verifyEqual(raw_sql, assembled_sql); 64 | end 65 | end 66 | end -------------------------------------------------------------------------------- /tests/test_schemas/+Company/Duty.m: -------------------------------------------------------------------------------- 1 | %{ 2 | schedule: varchar(32) 3 | --- 4 | (monday_on_call) -> Company.Employee(employee_id) 5 | %} 6 | classdef Duty < dj.Manual 7 | end 8 | -------------------------------------------------------------------------------- /tests/test_schemas/+Company/Employee.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # 3 | employee_id : varchar(12) # 4 | employment_year : int # 5 | %} 6 | classdef Employee < dj.Manual 7 | end 8 | -------------------------------------------------------------------------------- /tests/test_schemas/+Company/Machine.m: -------------------------------------------------------------------------------- 1 | %{ 2 | -> Company.Employee 3 | machine_id: varchar(10) 4 | %} 5 | classdef Machine < dj.Manual 6 | end 7 | -------------------------------------------------------------------------------- /tests/test_schemas/+External/Dimension.m: -------------------------------------------------------------------------------- 1 | %{ 2 | dimension_id : int 3 | --- 4 | dimension=null : blob@main 5 | %} 6 | classdef Dimension < dj.Manual 7 | end -------------------------------------------------------------------------------- /tests/test_schemas/+External/Document.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # Document 3 | document_id : int 4 | --- 5 | document_name : varchar(30) 6 | document_data1 : attach 7 | document_data2 : attach@main 8 | document_data3 : filepath@main 9 | %} 10 | classdef Document < dj.Lookup 11 | end -------------------------------------------------------------------------------- /tests/test_schemas/+External/Image.m: -------------------------------------------------------------------------------- 1 | %{ 2 | -> External.Dimension 3 | --- 4 | img=null : blob@main 5 | %} 6 | classdef Image < dj.Computed 7 | methods(Access=protected) 8 | function makeTuples(self, key) 9 | dim = num2cell(fetch1(External.Dimension & key, 'dimension')); 10 | rng(5); 11 | key.img = rand(dim{:}); 12 | self.insert(key) 13 | end 14 | end 15 | end -------------------------------------------------------------------------------- /tests/test_schemas/+Lab/ActiveSession.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # ActiveSession 3 | -> [unique] Lab.Subject 4 | --- 5 | (session_rig_class, session_rig_id) -> [nullable, unique] Lab.Rig(rig_manufacturer, rig_model) 6 | %} 7 | classdef ActiveSession < dj.Manual 8 | end -------------------------------------------------------------------------------- /tests/test_schemas/+Lab/Rig.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # Rig 3 | rig_manufacturer: varchar(50) 4 | rig_model: varchar(30) 5 | --- 6 | rig_note : varchar(100) 7 | %} 8 | classdef Rig < dj.Manual 9 | end -------------------------------------------------------------------------------- /tests/test_schemas/+Lab/Session.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # Session 3 | session_id: int 4 | --- 5 | -> Lab.Subject 6 | -> Lab.Rig 7 | %} 8 | classdef Session < dj.Manual 9 | end 10 | -------------------------------------------------------------------------------- /tests/test_schemas/+Lab/SessionAnalysis.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # SessionAnalysis 3 | -> Lab.Session 4 | --- 5 | session_analysis: longblob 6 | %} 7 | classdef SessionAnalysis < dj.Computed 8 | methods(Access=protected) 9 | function makeTuples(self,key) 10 | 11 | c = self.schema.conn; 12 | r = sprintf('connection_id = %d', c.serverId); 13 | 14 | j = fetch(Lab.Jobs() & r, '*'); 15 | 16 | if isempty(j) 17 | key.session_analysis = key.session_id; 18 | else 19 | key.session_analysis = j; 20 | end 21 | 22 | insert(self, key); 23 | 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /tests/test_schemas/+Lab/Subject.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # Subject 3 | subject_id : int # {subject_id} Comment to test curly bracket 4 | --- 5 | subject_dob : date 6 | unique index(subject_dob) 7 | %} 8 | classdef Subject < dj.Manual 9 | end -------------------------------------------------------------------------------- /tests/test_schemas/+TestLab/Duty.m: -------------------------------------------------------------------------------- 1 | %{ 2 | duty_date: date 3 | ----- 4 | (duty_first) -> TestLab.User(user_id) 5 | (duty_second) -> TestLab.User(user_id) 6 | (duty_third) -> TestLab.User(user_id) 7 | %} 8 | 9 | classdef Duty < dj.Manual 10 | end -------------------------------------------------------------------------------- /tests/test_schemas/+TestLab/User.m: -------------------------------------------------------------------------------- 1 | %{ 2 | user_id: varchar(32) 3 | %} 4 | 5 | classdef User < dj.Manual 6 | end -------------------------------------------------------------------------------- /tests/test_schemas/+University/A.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # A 3 | id : int 4 | --- 5 | string=null : varchar(30) 6 | datetime=null : datetime 7 | date=null : date 8 | number=null : float 9 | blob=null : longblob 10 | %} 11 | classdef A < dj.Manual 12 | end -------------------------------------------------------------------------------- /tests/test_schemas/+University/Date.m: -------------------------------------------------------------------------------- 1 | %{ 2 | id : int 3 | --- 4 | date_nodef_nounq : date 5 | date_nulldef_nounq=null : date 6 | #date_cordef1_nounq="" : date 7 | date_cordef2_nounq="2020-10-20" : date 8 | #date_wrgdef_nounq=4 : date 9 | date_nodef_unq : date 10 | date_nulldef_unq=null : date 11 | #date_cordef1_unq="" : date 12 | date_cordef2_unq="2020-10-20" : date 13 | #date_wrgdef_unq=4 : date 14 | unique index (date_nodef_unq) 15 | unique index (date_nulldef_unq) 16 | #unique index (date_cordef1_unq) 17 | unique index (date_cordef2_unq) 18 | #unique index (date_wrgdef_unq) 19 | %} 20 | classdef Date < dj.Manual 21 | end -------------------------------------------------------------------------------- /tests/test_schemas/+University/Float.m: -------------------------------------------------------------------------------- 1 | %{ 2 | id : int 3 | --- 4 | float_nodef_nounq : float 5 | float_nulldef_nounq=null : float 6 | float_cordef1_nounq=1.2 : float 7 | #float_wrgdef_nounq="hi" : float 8 | float_nodef_unq : float 9 | float_nulldef_unq=null : float 10 | float_cordef1_unq=1.2 : float 11 | #float_wrgdef_unq="hi" : float 12 | unique index (float_nodef_unq) 13 | unique index (float_nulldef_unq) 14 | unique index (float_cordef1_unq) 15 | #unique index (float_wrgdef_unq) 16 | %} 17 | classdef Float < dj.Manual 18 | end -------------------------------------------------------------------------------- /tests/test_schemas/+University/Integer.m: -------------------------------------------------------------------------------- 1 | %{ 2 | id : int 3 | --- 4 | int_nodef_nounq : int 5 | int_nulldef_nounq=null : int 6 | int_cordef1_nounq=4 : int 7 | #int_wrgdef_nounq="hi" : int 8 | int_nodef_unq : int 9 | int_nulldef_unq=null : int 10 | int_cordef1_unq=4 : int 11 | #int_wrgdef_unq="hi" : int 12 | unique index (int_nodef_unq) 13 | unique index (int_nulldef_unq) 14 | unique index (int_cordef1_unq) 15 | #unique index (int_wrgdef_unq) 16 | %} 17 | classdef Integer < dj.Manual 18 | end -------------------------------------------------------------------------------- /tests/test_schemas/+University/Message.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # Message 3 | msg_id : uuid # (this) is: a test of 'an' (elaborate), /type: of "comment" (good luck?) 4 | --- 5 | body : varchar(30) # (try) with: double colon (to), throw: it (off) 6 | dep_id=null : uuid 7 | %} 8 | classdef Message < dj.Manual 9 | end -------------------------------------------------------------------------------- /tests/test_schemas/+University/String.m: -------------------------------------------------------------------------------- 1 | %{ 2 | id : int 3 | --- 4 | string_nodef_nounq : varchar(50) 5 | string_nulldef_nounq=null : varchar(50) 6 | #string_cordef1_nounq="" : varchar(50) 7 | string_cordef2_nounq="hi" : varchar(50) 8 | #string_wrgdef_nounq=4 : varchar(50) 9 | string_nodef_unq : varchar(50) 10 | string_nulldef_unq=null : varchar(50) 11 | #string_cordef1_unq="" : varchar(50) 12 | string_cordef2_unq="hi" : varchar(50) 13 | #string_wrgdef_unq=4 : varchar(50) 14 | unique index (string_nodef_unq) 15 | unique index (string_nulldef_unq) 16 | #unique index (string_cordef1_unq) 17 | unique index (string_cordef2_unq) 18 | #unique index (string_wrgdef_unq) 19 | %} 20 | classdef String < dj.Manual 21 | end -------------------------------------------------------------------------------- /tests/test_schemas/+University/Student.m: -------------------------------------------------------------------------------- 1 | %{ 2 | # Student 3 | student_id : int 4 | --- 5 | first_name : varchar(30) 6 | last_name : varchar(30) 7 | enrolled : datetime 8 | %} 9 | classdef Student < dj.Manual 10 | end -------------------------------------------------------------------------------- /tests/test_schemas/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "database.host": "", 3 | "database.password": "", 4 | "database.user": "", 5 | "database.port": 3306, 6 | "database.reconnect": true, 7 | "connection.init_function": null, 8 | "connection.charset": "", 9 | "loglevel": "INFO", 10 | "safemode": true, 11 | "fetch_format": "array", 12 | "display.limit": [12, 5], 13 | "display.width": 14, 14 | "display.show_tuple_count": true, 15 | "database.use_tls": null, 16 | "enable_python_native_blobs": false, 17 | "custom.accessories" : { 18 | "item.type": "keyboard", 19 | "location" : "office" 20 | }, 21 | "stores.options" : [ 22 | { 23 | "protocol.type": "file", 24 | "location" : "/tmp" 25 | }, 26 | { 27 | "protocol.type": "s3", 28 | "location" : "/home" 29 | } 30 | ] 31 | } -------------------------------------------------------------------------------- /tests/test_schemas/config_lite.json: -------------------------------------------------------------------------------- 1 | { 2 | "database.use_tls" : true 3 | } -------------------------------------------------------------------------------- /tests/test_schemas/store_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "database.host": "env", 3 | "database.password": "var", 4 | "database.user": "override", 5 | "database.port": 3306, 6 | "database.reconnect": true, 7 | "connection.init_function": null, 8 | "connection.charset": "", 9 | "loglevel": "DEBUG", 10 | "safemode": false, 11 | "fetch_format": "array", 12 | "display.limit": 12, 13 | "display.width": 14, 14 | "display.show_tuple_count": true, 15 | "database.use_tls": null, 16 | "enable_python_native_blobs": false, 17 | "stores": { 18 | "local": { 19 | "protocol": "file", 20 | "location": "{{external_file_store_root}}/base", 21 | "subfolding": [ 22 | 3, 23 | 4 24 | ] 25 | }, 26 | "new_local": { 27 | "datajoint_type": "blob", 28 | "protocol": "file", 29 | "location": "{{external_file_store_root}}/base", 30 | "subfolding": [ 31 | 3, 32 | 4 33 | ] 34 | }, 35 | "local_default": { 36 | "protocol": "file", 37 | "location": "{{external_file_store_root}}/base" 38 | }, 39 | "new_local_default": { 40 | "datajoint_type": "blob", 41 | "protocol": "file", 42 | "location": "{{external_file_store_root}}/base" 43 | }, 44 | "remote": { 45 | "protocol": "s3", 46 | "location": "lab1/base", 47 | "endpoint": "example:1234", 48 | "bucket": "dep1", 49 | "access_key": "key", 50 | "secret_key": "secret", 51 | "secure": false, 52 | "subfolding": [ 53 | 3, 54 | 4 55 | ] 56 | }, 57 | "new_remote": { 58 | "datajoint_type": "blob", 59 | "protocol": "s3", 60 | "location": "lab1/base", 61 | "endpoint": "example:1234", 62 | "bucket": "dep1", 63 | "access_key": "key", 64 | "secret_key": "secret", 65 | "secure": true, 66 | "subfolding": [] 67 | }, 68 | "remote_default": { 69 | "protocol": "s3", 70 | "location": "lab1/base", 71 | "endpoint": "example:1234", 72 | "bucket": "dep1", 73 | "access_key": "key", 74 | "secret_key": "secret" 75 | }, 76 | "new_remote_default": { 77 | "datajoint_type": "blob", 78 | "protocol": "s3", 79 | "location": "lab1/base", 80 | "endpoint": "example:1234", 81 | "bucket": "dep1", 82 | "access_key": "key", 83 | "secret_key": "secret" 84 | } 85 | } 86 | } --------------------------------------------------------------------------------