├── .gitignore ├── api ├── api.ijs ├── api_adm.ijs ├── api_blob.ijs ├── api_change.ijs ├── api_create.ijs ├── api_csv.ijs ├── api_csvcdefs.ijs ├── api_drop.ijs ├── api_gen.ijs ├── api_info.ijs ├── api_misc.ijs ├── api_read.ijs ├── api_rename.ijs ├── api_replicate.ijs ├── api_sort.ijs ├── api_table.ijs ├── client.ijs └── jjd.ijs ├── base ├── column.ijs ├── common.ijs ├── constants.ijs ├── database.ijs ├── fixpairs.ijs ├── folder.ijs ├── jmfx.ijs ├── keyindex.ijs ├── lock.ijs ├── log.ijs ├── parse.ijs ├── pm.ijs ├── read.ijs ├── table.ijs ├── tests.ijs ├── testtuts.ijs ├── util.ijs ├── util_epoch.ijs ├── util_epoch_901.ijs ├── util_ptable.ijs ├── validate.ijs ├── where.ijs └── zutil.ijs ├── bug ├── csvwr.ijs └── empty_join_where.ijs ├── cd ├── jd-arm64.dll ├── jd.dll ├── jpcre-arm64.dll ├── jpcre.dll ├── libjd.dylib ├── libjd.so ├── libjpcre.dylib ├── libjpcre.so └── rpi │ ├── libjd.so │ └── libjpcre.so ├── config └── server_default.ijs ├── csv ├── csv.ijs ├── csvinstall.ijs └── csvtest.ijs ├── demo ├── common.ijs ├── jhs │ └── jdapp1.ijs ├── northwind │ └── csv │ │ ├── Categories.cdefs │ │ ├── Categories.csv │ │ ├── Customers.cdefs │ │ ├── Customers.csv │ │ ├── Employees.cdefs │ │ ├── Employees.csv │ │ ├── OrderDetails.cdefs │ │ ├── OrderDetails.csv │ │ ├── Orders.cdefs │ │ ├── Orders.csv │ │ ├── Products.cdefs │ │ ├── Products.csv │ │ ├── Shippers.cdefs │ │ ├── Shippers.csv │ │ ├── Suppliers.cdefs │ │ ├── Suppliers.csv │ │ ├── jdclass │ │ └── jdcsvrefs.txt ├── sandp │ └── csv │ │ ├── j.cdefs │ │ ├── j.csv │ │ ├── jdclass │ │ ├── jdcsvrefs.txt │ │ ├── p.cdefs │ │ ├── p.csv │ │ ├── s.cdefs │ │ ├── s.csv │ │ ├── sp.cdefs │ │ ├── sp.csv │ │ ├── spj.cdefs │ │ └── spj.csv └── sed │ └── csv │ ├── e.cdefs │ ├── e.csv │ ├── jdclass │ ├── jdcsvrefs.txt │ ├── r.cdefs │ ├── r.csv │ ├── s.cdefs │ ├── s.csv │ ├── t.cdefs │ └── t.csv ├── doc ├── Admin.htm ├── Docs.htm ├── General.htm ├── Guide.htm ├── Index.htm ├── License.htm ├── Ops.htm ├── Ops_change.htm ├── Ops_common.htm ├── Ops_create.htm ├── Ops_csv.htm ├── Ops_damaged.htm ├── Ops_drop.htm ├── Ops_info.htm ├── Ops_join.htm ├── Ops_misc.htm ├── Ops_raw.htm ├── Ops_read.htm ├── Ops_rename.htm ├── Ops_table-table.htm ├── Overview.htm ├── Release.htm ├── Replicate.htm ├── Support.htm ├── Technical.htm └── jd.css ├── dynamic ├── base.ijs └── ref.ijs ├── jd.ijs ├── jdkey.txt ├── manifest.ijs ├── mtm ├── json.txt ├── mtm.ijs ├── mtm_man.ijs ├── mtm_server.ijs └── mtm_server_tools.ijs ├── nopacman └── jddev.ijs ├── pm ├── bench.ijs ├── csvtest.ijs ├── pm.ijs ├── pm_1.ijs ├── pm_base.ijs ├── pm_disk.ijs ├── pm_insert_1.ijs ├── pm_map.ijs ├── pm_read_avg_by.ijs ├── pm_tlen_damage_repair.ijs ├── pmb.ijs ├── pmc.ijs ├── pmx.ijs └── replicate.ijs ├── server ├── fork.ijs ├── jcs_server.ijs ├── jctask.ijs ├── jds │ ├── jds_server.ijs │ └── jds_tools.ijs ├── jds_client │ ├── examples.ijs │ ├── jds_client.ijs │ └── jds_client_test.ijs ├── node │ ├── cert.napem │ ├── config.js │ ├── custom.ijs │ ├── jds.js │ ├── key.napem │ ├── node_tools.ijs │ ├── server.html │ └── server.js └── port.ijs ├── test ├── admin_test.ijs ├── alloc_test.ijs ├── basic_test.ijs ├── byten_test.ijs ├── cd_query_test.ijs ├── createcol_derived_test.ijs ├── createcol_test.ijs ├── createtable_test.ijs ├── csv_test.ijs ├── csvcdefs_test.ijs ├── csvrd_types_test.ijs ├── delete_test.ijs ├── drop_rename_test.ijs ├── droptable_test.ijs ├── epoch_test.ijs ├── error_test.ijs ├── fixpairs_test.ijs ├── flush.ijs ├── info_test.ijs ├── insert_revert_test.ijs ├── insert_test.ijs ├── intx_test.ijs ├── jdindex_test.ijs ├── jds_test.ijs ├── join_test.ijs ├── join_type_test.ijs ├── joinorder_test.ijs ├── joinx_test.ijs ├── keyindex_test.ijs ├── log_test.ijs ├── modify_test.ijs ├── multicol_test.ijs ├── names_test.ijs ├── northwind_test.ijs ├── orderby_test.ijs ├── osfna_test.ijs ├── pm_aggby_test.ijs ├── ptable_csv_test.ijs ├── ptable_delete_test.ijs ├── ptable_insert_test.ijs ├── ptable_modify_test.ijs ├── ptable_read_test.ijs ├── ptable_update_test.ijs ├── query_test.ijs ├── read_types_test.ijs ├── readx_test.ijs ├── ref_dirty_test.ijs ├── ref_test.ijs ├── refx_test.ijs ├── repair_test.ijs ├── replicate_test.ijs ├── rmdir_test.ijs ├── sandp_test.ijs ├── sed_test.ijs ├── shape_test.ijs ├── sort_test.ijs ├── stress_test.ijs ├── table_append_test.ijs ├── update_test.ijs ├── upsert_test.ijs ├── user_test.ijs ├── utf8_test.ijs ├── valid_data_test.ijs ├── validate_test.ijs └── where_test.ijs ├── todo.txt ├── tools ├── csv_load.ijs ├── fix_tlen.ijs ├── locreport.ijs ├── pandas │ ├── clean.ijs │ ├── load.py │ ├── pandas.ijs │ ├── util.py │ └── write.py ├── ptable.ijs ├── quandl.ijs ├── repair.ijs ├── report_copath_names.ijs ├── setscriptlists.ijs ├── tests.ijs ├── ts.ijs └── tut.ijs ├── tutorial ├── admin_tut.ijs ├── blob_tut.ijs ├── bus_lic_tut.ijs ├── createcol_derived_mapped_tut.ijs ├── createcol_derived_tut.ijs ├── createcol_tut.ijs ├── createtable_tut.ijs ├── csv_advanced_tut.ijs ├── csv_intro_tut.ijs ├── csv_load_tut.ijs ├── custom_tut.ijs ├── delete_tut.ijs ├── dropstop_tut.ijs ├── epochdt_tut.ijs ├── gen_tut.ijs ├── info_tut.ijs ├── insert_tut.ijs ├── intro_a_tut.ijs ├── intro_b_tut.ijs ├── intro_c_tut.ijs ├── intx_tut.ijs ├── jcs_tut.ijs ├── jctask_tut.ijs ├── jds_client_tut.ijs ├── jds_tut.ijs ├── jhs_tut.ijs ├── json_tut.ijs ├── key_tut.ijs ├── link_tut.ijs ├── log_tut.ijs ├── mtm_tut.ijs ├── node_tut.ijs ├── northwind_tut.ijs ├── pairs_tut.ijs ├── pandas_clean_tut.ijs ├── pandas_install_tut.ijs ├── pandas_load_tut.ijs ├── pandas_write_tut.ijs ├── performance_tut.ijs ├── ptable_tut.ijs ├── quandl_ibm_tut.ijs ├── read_tut.ijs ├── reads_aggregation_tut.ijs ├── reads_basic_tut.ijs ├── reads_clauses_tut.ijs ├── reads_from_tut.ijs ├── reads_join_tut.ijs ├── reads_option_table_tut.ijs ├── replicate_tut.ijs ├── sandp_tut.ijs ├── sed_tut.ijs ├── setget_tut.ijs ├── sort_tut.ijs ├── stock_data_tut.ijs ├── table_from_pairs_tut.ijs ├── table_table_tut.ijs ├── tablecopy_tut.ijs ├── tableinsert_tut.ijs ├── tablemove_tut.ijs ├── taxi_tut.ijs ├── update_advanced_tut.ijs ├── update_basic_tut.ijs ├── upsert_advanced_tut.ijs ├── upsert_basic_tut.ijs └── vr_tut.ijs └── types ├── autoindex.ijs ├── base.ijs ├── byte.ijs ├── datetimes.ijs ├── epoch.ijs ├── numeric.ijs └── varbyte.ijs /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /api/api_blob.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | 3 | coclass'jd' 4 | 5 | NB. blank delimited names up to * 6 | bdn=: 3 : 0 7 | 'must not be boxed' assert 0=L.y 8 | i=. y i.'*' 9 | '* not found'assert i<#y 10 | t=. i{.y 11 | (<(bdnames i{.y)),<(>:i)}.y 12 | ) 13 | 14 | jd_blobwrite=: 3 : 0 15 | 'must be boxed'assert 1=L.y 16 | ECOUNT assert (#y)e.2 3 4 17 | data=. >{:y 18 | h=. }:y 19 | name=. ;{:h 20 | vcname name 21 | d=. }:h 22 | p=. (dbpath DB),'/',;d,each'/' 23 | 'not a jd folder' assert fexist p,'jdstate' 24 | p=. p,'jdblob' 25 | jdcreatefolder p 26 | 'blob write failed' assert _1-.@-:data fwrite p,'/',name,'.blob' 27 | JDOK 28 | ) 29 | 30 | jd_blobread=: 3 : 0 31 | d=. bdnames y 32 | 'bad arg count'assert (#d)e.1 2 3 33 | name=. ;{:d 34 | d=. }:d 35 | p=. (dbpath DB),'/',;d,each'/' 36 | p=. 'jdblob',~(dbpath DB),'/',;d,each'/' 37 | r=. fread p,'/',name,'.blob' 38 | 'blob read failed'assert _1-.@-:r 39 | NB. ((;d,each'/'),name);r 40 | 'blob';r 41 | ) 42 | 43 | jd_bloberase=: 3 : 0 44 | d=. bdnames y 45 | 'bad arg count'assert (#d)e.1 2 3 46 | name=. ;{:d 47 | d=. }:d 48 | p=. 'jdblob',~(dbpath DB),'/',;d,each'/' 49 | r=. ferase p,'/',name,'.blob' 50 | 'blob erase failed'assert _1-.@-:r 51 | JDOK 52 | ) 53 | -------------------------------------------------------------------------------- /api/api_rename.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2018, Jsoftware Inc. All rights reserved. 2 | 3 | coclass'jd' 4 | 5 | NB. should be reworked to not use jd_close 6 | jd_renametable=: 3 : 0 7 | a=. bdnames y 8 | ECOUNT assert 2=#a 9 | assertnoref ;{.a 10 | 11 | ns=. getparttables ;{.a 12 | for_i. i.#ns do. 13 | tab=. i{ns 14 | t=. jdgl tab 15 | old=. }:PATH__t 16 | part=. (old i: PTM)}.old 17 | new=. ((->:#NAME__t)}.PATH__t),(;{:a),part 18 | jd_close'' 19 | new frename old 20 | getdb'' NB. required because of close 21 | end. 22 | JDOK 23 | ) 24 | 25 | NB. should be reworked to not use jd_close 26 | NB. cco is not adjusted - confusing - new is at end unless already there 27 | jd_renamecol=: 3 : 0 28 | a=. bdnames y 29 | ECOUNT assert 3=#a 30 | t=. jdgl {.a 31 | 'col not found' assert (1{a)e.NAMES__t 32 | 'col already exists' assert -.(2{a)e.NAMES__t 33 | notjd_assert }.a 34 | vcname ;2{a 35 | assertnodynamic 2{.a 36 | ns=. getparttables ;{.a 37 | for_i. i.#ns do. 38 | if. i=1 do. continue. end. NB. ignore f~ 39 | t=. jdgl (i{ns),1{a 40 | 41 | oldname=. ord 20 | ord=. jdremq each ifdesc (_4 stripsp@}. ])^:[&.> ord 21 | 'duplicate col'assert (#ord)=#~.ord 22 | 1 validtc__dbl ({.q 29 | ndx=. i.#ord 30 | for_nd. |. ndx,.ifdesc do. 31 | 'n d'=.nd 32 | q=. q {&.>~ < /:`\:@.d n{::q 33 | end. 34 | s=. >{:q 35 | if. s-:i.#s do. JDOK return. end. 36 | t=. jdgl tab 37 | update_subscr__t'' 38 | for_c. CHILDREN__t do. 39 | select. typ__c 40 | case.'autodindex' do. continue. 41 | case.'ref' do. 42 | if. dirty__c do. continue. end. 43 | if. left__c do. setdirty__c 1 continue. end. NB. could handle this 44 | end. 45 | if. derived__c do. setderiveddirty__c'' else. dat__c=: s{dat__c end. 46 | end. 47 | JDOK 48 | ) 49 | -------------------------------------------------------------------------------- /api/jjd.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | NB. JHS jjd server for wget clients 3 | 4 | 0 : 0 5 | JD server is JHS app jjd 6 | 7 | JD server serves URL jjd on a 'known' port 8 | for security the server normally only binds to localhost 9 | client must have ssh tunnel to get to servers localhost 10 | 11 | JHS gets an http reqest for jjd and calls jev_post_raw 12 | jev_post_raw processes the request in the post data and sends a response 13 | 14 | post data has following format (bytes): 15 | header * jdargs 16 | header is fin;fout;dbname;user;pswd; 17 | jdargs is jd arguments in fin format 18 | 19 | fin/fout is one of: TEXT/JBIN/JSON/HTML (type support incomplete) 20 | 21 | *** 22 | 23 | Clients 24 | 25 | 1. J client - api/client.ijs 26 | 2. non J client - wget or equivalent - same as J client but without J 27 | 3. browser (ajax) -> apache -> cgi J task -> jwget from JD server 28 | ) 29 | 30 | coclass'jjd' 31 | coinsert'jhs' 32 | 33 | jev_post_raw=: 3 : 0 34 | try. 35 | i=. NV i. '*' 36 | 'fin fout dbx user pswd'=. <;._2 i{.NV 37 | DB_jd_=: dbx 38 | UP_jd_=: user NB. user field is user/pswd and pswd field is not used 39 | SERVER_jd_=: 'intask' 40 | d=. (>:i)}.NV 41 | select. fin 42 | case.'JBIN' do. d=. 3!:2 d 43 | case.'TEXT' do. d=. d NB. nothing to do 44 | case. do. assert. 0['unknown input format' 45 | end. 46 | if. 1=#d do. d=.;d end. 47 | d=. (dbx;user;'intask') jdx_jd_ d 48 | select. fout 49 | case.'JBIN' do. 'application/octet-stream' gsrcf 3!:1 d 50 | case.'TEXT' do. 'text/plain' gsrcf totext_jd_ d 51 | case. do. 'unknown output format' assert 0 52 | end. 53 | catch. 54 | 'text/plain' gsrcf 13!:12'' 55 | end. 56 | i.0 0 57 | ) 58 | 59 | OKURL_jhs_=: a:-.~~.'jjd';OKURL 60 | -------------------------------------------------------------------------------- /base/constants.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2016, Jsoftware Inc. All rights reserved. 2 | 3 | coclass'jd' 4 | 5 | JDOK=: ,.<'Jd OK' 6 | 7 | IMAX=: 9223372036854775807 8 | IMIN=: _1+-IMAX 9 | IBAD=: IMIN 10 | 11 | I1MAX=: <:<.2^7 12 | I1MIN=: _1+-I1MAX 13 | I1BAD=: I1MIN 14 | 15 | I2MAX=: <:<.2^15 16 | I2MIN=: _1+-I2MAX 17 | I2BAD=: I2MIN 18 | 19 | I4MAX=: <:<.2^31 20 | I4MIN=: _1+-I4MAX 21 | I4BAD=: I4MIN 22 | 23 | PTM=: '^' NB. ptable filename 24 | 25 | NB. chars not allowed in Jd names because they are used as file names 26 | NB. also to avoid parsing problems 27 | NB. ` unix shell script 28 | NB. ~ vi etc. temp file 29 | RESERVEDCHARS=: '/\ *.,:?<>|"''`~-=' NB. illegal in dan/table/col names - -= conflict with joins 30 | RESERVEDWORDS=: ;:'by from where order' 31 | MAXNAMECHARS=: 201 NB. includes terminating blank 32 | 33 | t=. 0 : 0 NB. map colname RESERVEDCHARS 34 | / _fs_ 35 | \ _bs_ 36 | _ 37 | * _star_ 38 | . _dot_ 39 | , _comma_ 40 | : _colon_ 41 | ? _query_ 42 | < _lt_ 43 | > _gt_ 44 | | _bar_ 45 | " _dquote_ 46 | ' _quote_ 47 | ` _tie_ 48 | ~ _tilda_ 49 | - _minus_ 50 | = _equal_ 51 | ) 52 | 53 | 'bad COLNRPLC'assert RESERVEDCHARS-:;{.each <;._2 t 54 | COLNRPLC=: (<"0 RESERVEDCHARS_jd_) ,. 2}.each<;._2 t 55 | 56 | 57 | ECOUNT=: 'incorrect arg count' 58 | EDNONE=: 'bad SUBSCR' 59 | EEPOCH=: 'bad epoch data' 60 | ESHAPE=: 'bad shape' 61 | EBTS=: 'bad trailing shape' 62 | ETALLY=: 'bad count' 63 | ETYPE=: 'bad type' 64 | EALLOC=: 'bad alloc' 65 | EOPTION=: 'bad option' 66 | EOPTIONV=: 'bad option value' 67 | EDUPLICATE=: 'duplicate col' 68 | EUNKNOWN=: 'unknown col' 69 | ENOTJD=: 'jd prefix not allowed' 70 | EMISSING=: 'missing col' 71 | EDELETE=: 'jddeletefolder failed' 72 | EINDEX=: 'bad index' 73 | EDROPSTOP=: 'dropstop' 74 | EUNIQUE=: 'warning: deleted N rows to stay unique' 75 | EBOOLEAN=: 'bad boolean data' 76 | EFLOAT=: 'bad float data' 77 | EINT=: 'bad int data' 78 | EINT1=: 'bad int1 data' 79 | EINT2=: 'bad int2 data' 80 | EINT4=: 'bad int4 data' 81 | EDERIVED=: 'derived not allowed' 82 | ECSVBYTE0=: 'byte col trailing shape 0 not allowed' 83 | EPTABLE=: 'ptable not allowed' 84 | ETABLEFILE=: '/table and /file are mutually exclusive' 85 | 86 | LOCALE=: CLASS=: <'jd' 87 | CHILD=: <'jdfolder' 88 | -------------------------------------------------------------------------------- /base/folder.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | coclass 'jdfolder' 3 | coinsert 'jd' 4 | 5 | CLASS=: <'jdfolder' 6 | CHILD=: <'jddatabase' 7 | STATE=: '' 8 | 9 | testcreate =: ] 10 | open =: create =: 3 : 0 11 | locDBF=: coname'' 12 | ) 13 | close =: ] 14 | -------------------------------------------------------------------------------- /base/pm.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | coclass'jd' 3 | 4 | pmhelp=: 0 : 0 5 | pmclear_jd_'' NB. clears times and start recording 6 | pmclear_jd_ 0 NB. stop recording 7 | pmr_jd_'' NB. reports times for last PMMR_jd_ ops 8 | pmtotal_jd_'' NB. returns total time 9 | 10 | Non-Jd times can be included. 11 | 12 | pm'name' NB. start timer 13 | pmz'' NB. end timer 14 | pmz_jd_''[doit data[pm_jd_'zxcv' 15 | ) 16 | 17 | pmclear=: 3 : 0 18 | if. 0-:y do. PMON=: 0 return. end. 19 | PMON=: 1 20 | PMN=: PMT=: '' 21 | i.0 0 22 | ) 23 | 24 | NB. record section start 25 | pm=: 3 : 0 26 | PMNAME=: y 27 | PMSTART=: 6!:9'' 28 | i.0 0 29 | ) 30 | 31 | pmfmt=: 3 : 0 32 | if. 0=#y do. '' else. ' ',y end. 33 | ) 34 | 35 | NB. record section end 36 | pmz=: 3 : 0 37 | if. _1=nc<'PMON' do. PMON=: 0 end. 38 | if. -.PMON do. return. end. 39 | t=. 0>.(#PMN)-<:PMMR 40 | PMN=: t}.PMN,<(14{.PMNAME),fmtsummary jdlasty 41 | PMT=: t}.PMT,PMSTART-~6!:9'' 42 | i.0 0 43 | ) 44 | 45 | pmnb=: 3 : 0 46 | t=. 0>.(#PMN)-<:PMMR 47 | PMN=: t}.PMN,PMN,<'total' 56 | ) 57 | 58 | pmtotal=: 3 : 0 59 | +/<.0.5+1000*PMT%6!:8'' 60 | ) 61 | 62 | fmtsummary=: 3 : 0 63 | if. 0=L.y do. 64 | deb y 65 | else. 66 | y=. (<'-') ((2~:;3!:0 each y)#i.#y)}y 67 | y=. (<'-') ((1<;$@$each y)#i.#y)}y 68 | y=. ;(<' '),~each y 69 | end. 70 | if. 120<#y do. y=. '...',~120{.y end. 71 | deb y 72 | ) 73 | -------------------------------------------------------------------------------- /base/tests.ijs: -------------------------------------------------------------------------------- 1 | tests=: <;._2 [ 0 : 0 2 | test/admin_test.ijs 3 | test/alloc_test.ijs 4 | test/basic_test.ijs 5 | test/byten_test.ijs 6 | test/cd_query_test.ijs 7 | test/createcol_derived_test.ijs 8 | test/createcol_test.ijs 9 | test/createtable_test.ijs 10 | test/csv_test.ijs 11 | test/csvcdefs_test.ijs 12 | test/csvrd_types_test.ijs 13 | test/delete_test.ijs 14 | test/drop_rename_test.ijs 15 | test/droptable_test.ijs 16 | test/epoch_test.ijs 17 | test/error_test.ijs 18 | test/fixpairs_test.ijs 19 | test/info_test.ijs 20 | test/insert_revert_test.ijs 21 | test/insert_test.ijs 22 | test/intx_test.ijs 23 | test/jdindex_test.ijs 24 | test/jds_test.ijs 25 | test/join_test.ijs 26 | test/join_type_test.ijs 27 | test/joinorder_test.ijs 28 | test/joinx_test.ijs 29 | test/keyindex_test.ijs 30 | test/log_test.ijs 31 | test/modify_test.ijs 32 | test/multicol_test.ijs 33 | test/names_test.ijs 34 | test/northwind_test.ijs 35 | test/orderby_test.ijs 36 | test/osfna_test.ijs 37 | test/pm_aggby_test.ijs 38 | test/ptable_csv_test.ijs 39 | test/ptable_delete_test.ijs 40 | test/ptable_insert_test.ijs 41 | test/ptable_modify_test.ijs 42 | test/ptable_read_test.ijs 43 | test/ptable_update_test.ijs 44 | test/query_test.ijs 45 | test/read_types_test.ijs 46 | test/readx_test.ijs 47 | test/ref_dirty_test.ijs 48 | test/ref_test.ijs 49 | test/refx_test.ijs 50 | test/repair_test.ijs 51 | test/replicate_test.ijs 52 | test/rmdir_test.ijs 53 | test/sandp_test.ijs 54 | test/sed_test.ijs 55 | test/shape_test.ijs 56 | test/sort_test.ijs 57 | test/stress_test.ijs 58 | test/table_append_test.ijs 59 | test/update_test.ijs 60 | test/upsert_test.ijs 61 | test/user_test.ijs 62 | test/utf8_test.ijs 63 | test/valid_data_test.ijs 64 | test/validate_test.ijs 65 | test/where_test.ijs 66 | ) -------------------------------------------------------------------------------- /base/testtuts.ijs: -------------------------------------------------------------------------------- 1 | testtuts=: <;._2 [ 0 : 0 2 | tutorial/admin_tut.ijs 3 | tutorial/blob_tut.ijs 4 | tutorial/bus_lic_tut.ijs 5 | tutorial/createcol_derived_mapped_tut.ijs 6 | tutorial/createcol_derived_tut.ijs 7 | tutorial/createcol_tut.ijs 8 | tutorial/createtable_tut.ijs 9 | tutorial/csv_advanced_tut.ijs 10 | tutorial/csv_intro_tut.ijs 11 | tutorial/csv_load_tut.ijs 12 | tutorial/custom_tut.ijs 13 | tutorial/delete_tut.ijs 14 | tutorial/dropstop_tut.ijs 15 | tutorial/epochdt_tut.ijs 16 | tutorial/gen_tut.ijs 17 | tutorial/info_tut.ijs 18 | tutorial/insert_tut.ijs 19 | tutorial/intro_a_tut.ijs 20 | tutorial/intro_b_tut.ijs 21 | tutorial/intro_c_tut.ijs 22 | tutorial/intx_tut.ijs 23 | tutorial/jcs_tut.ijs 24 | tutorial/jctask_tut.ijs 25 | tutorial/jds_client_tut.ijs 26 | tutorial/jds_tut.ijs 27 | tutorial/jhs_tut.ijs 28 | tutorial/json_tut.ijs 29 | tutorial/key_tut.ijs 30 | tutorial/link_tut.ijs 31 | tutorial/log_tut.ijs 32 | tutorial/mtm_tut.ijs 33 | tutorial/node_tut.ijs 34 | tutorial/northwind_tut.ijs 35 | tutorial/pairs_tut.ijs 36 | tutorial/pandas_clean_tut.ijs 37 | tutorial/pandas_install_tut.ijs 38 | tutorial/pandas_load_tut.ijs 39 | tutorial/pandas_write_tut.ijs 40 | tutorial/performance_tut.ijs 41 | tutorial/ptable_tut.ijs 42 | tutorial/quandl_ibm_tut.ijs 43 | tutorial/read_tut.ijs 44 | tutorial/reads_aggregation_tut.ijs 45 | tutorial/reads_basic_tut.ijs 46 | tutorial/reads_clauses_tut.ijs 47 | tutorial/reads_from_tut.ijs 48 | tutorial/reads_join_tut.ijs 49 | tutorial/reads_option_table_tut.ijs 50 | tutorial/replicate_tut.ijs 51 | tutorial/sandp_tut.ijs 52 | tutorial/sed_tut.ijs 53 | tutorial/setget_tut.ijs 54 | tutorial/sort_tut.ijs 55 | tutorial/stock_data_tut.ijs 56 | tutorial/table_from_pairs_tut.ijs 57 | tutorial/table_table_tut.ijs 58 | tutorial/tablecopy_tut.ijs 59 | tutorial/tableinsert_tut.ijs 60 | tutorial/tablemove_tut.ijs 61 | tutorial/taxi_tut.ijs 62 | tutorial/update_advanced_tut.ijs 63 | tutorial/update_basic_tut.ijs 64 | tutorial/upsert_advanced_tut.ijs 65 | tutorial/upsert_basic_tut.ijs 66 | tutorial/vr_tut.ijs 67 | ) -------------------------------------------------------------------------------- /base/util_epoch.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2019, Jsoftware Inc. All rights reserved. 2 | coclass 'jd' 3 | 4 | NB. datetime epoch 5 | 6 | NB. e unix style epoch but in nanoseconds with 2000-01-01 base 7 | NB. s iso 8601 (relaxed/stricter rules) 8 | 9 | NB. conversion utils preserve rank 10 | 11 | NB. e from s 12 | NB. efs '2014-06-07T08:09:10.123456789+05' 13 | NB. efs '2014-06-07T08:09:10.123Z' 14 | NB. efs '2014-06-07T08:09:10.123' 15 | NB. efs '2014-06-07T08:09:10+05' 16 | NB. efs '2014-06-07' 17 | NB. delimiter (-T:+-Z or blank) ends field, but is not validated 18 | NB. 7:8:9 treated same as 07:08:09 19 | NB. x is allow errors, ignore offset, return offset 20 | 21 | NB. x is from d039 22 | efs=: 3 : 0 23 | '9' efs y NB. changed (with 6!:17) to allow errors 24 | : 25 | s=. $y 26 | p=. }:s 27 | rows=. */p 28 | cols=. {:s 29 | r=. (LIBJD_jd_,' efs x x x *c *x *x x')cd rows;cols;((rows,cols) ($,) y);(rows$-1);(<0);0 30 | d=. p$>4{r 31 | b=. d~:IMIN NB. don't adjust bad values 32 | select. x 33 | case. 'd' do. d=. d-b*(86400*1e9)|d 34 | case. '0' do. d=. d-b*1e9|d 35 | case. '3' do. d=. d-b*1e6|d 36 | case. '9' do. 37 | end. 38 | d 39 | ) 40 | 41 | NB. s from e - efs invers 42 | NB. 0{x is ',' or '.' for hh:mm:ss,nnnnnnnnn 43 | NB. 1{x is 'Z' for a final Z 44 | NB. 2{x is d for date, t or 0 for time, m or 3 for millis, n or 9 for nanos 45 | sfe=: 3 : 0 46 | ', 9'sfe y 47 | : 48 | s=. $y 49 | r=. #@,y 50 | c=. ('Z'=1{x)+10 19 23 29{~'d039'i.'d039d039'{~'d039dtmn'i.2{x NB. old and new style 51 | y=. ,y 52 | 53 | NB. kludge display of min/max aggregation of epchdt cols 54 | if. (1=#y) *. +./_ __ e. y do. 55 | s$>3{(LIBJD_jd_,' sfe x x x *c *x *c')cd A__=: r;c;((r,c)$' ');(,(_=y){0 6342969599999999999);x 56 | else. 57 | s$>3{(LIBJD_jd_,' sfe x x x *c *x *c')cd r;c;((r,c)$' ');y;x 58 | end. 59 | ) 60 | 61 | NB. stuff for old style datetime yyyymmddhhmmss 62 | 63 | NB. e from yyyymmddhhmmss 64 | eft=: 3 : 0 65 | r=. #y 66 | t=. 0=$$y 67 | r=. >2{(LIBJD_jd_,' eft x x *x *x')cd r;(r$2-2);,y 68 | if. t do. r=. ''$r end. 69 | ) 70 | 71 | 3 : 0'' 72 | try. load JDP,'base/util_epoch_901.ijs'['d'6!:17'2000' catch. end. 73 | ) 74 | -------------------------------------------------------------------------------- /base/util_epoch_901.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2019, Jsoftware Inc. All rights reserved. 2 | NB. 90x has efs/sfe as foreigns 3 | 4 | coclass 'jd' 5 | 6 | efs=: 6!:17 7 | 8 | sfe=: 3 : 0 9 | ', 9'sfe y 10 | : 11 | t=. (2{.x),'d039d039'{~'d039dtmn'i.2{x NB. new and old precision 12 | if. (1=#@,y) *. +./_ __="0 1 ,y do. 13 | t 6!:16 ,(_=,y){0 6342969599999999999 NB. kludge min/max agg empty 14 | else. 15 | t 6!:16 y 16 | end. 17 | ) 18 | -------------------------------------------------------------------------------- /base/util_ptable.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | 3 | coclass 'jd' 4 | 5 | isptable=: 3 : 0 6 | if. '.'e.y do. 0 return. end. NB.!!! ptable must be first in from and by itself - f and not f.g 7 | -.0-:jdgl :: 0: y,PTM NB. ispatable table_name 8 | ) 9 | 10 | NB. return sorted ptable suffixes ; Tlens 11 | getparts=: 3 : 0 12 | a=. y,PTM 13 | n=. NAMES__dbl 14 | b=. (:#y)}.each 2 }.getparttables y 29 | ) 30 | 31 | NB. sorted partition table names (f, f~, f~..., ...) 32 | NB. returns single table name if not partition table 33 | NB. returns single table name if y is already a partition name (has a ~) 34 | getparttables=: 3 : 0 35 | if. PTM e.y do. d 86 | ) 87 | -------------------------------------------------------------------------------- /base/validate.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | 3 | coclass'jd' 4 | 5 | valsize=: 4 : 0 6 | mr=. mappings_jmf_{~({."1 mappings_jmf_)i.m 32 | mn=. om,'__c' 33 | tabs=. tabs,NAME__PARENT__c 34 | cols=. cols,NAME__c 35 | tlen=. tlen,Tlen__c 36 | map=. map,om 37 | type=. type,typ__c 38 | s=. countdat__c'' 39 | count=. count,s 40 | t=. 0~:Tlen__c-s NB. 1 if too many or too few rows 41 | a=. fsize PATH__c,'/',om 42 | b=. HS_jmf_+getmsize_jmf_ om,Cloc__c 43 | filesize=. filesize,a 44 | jsize=. jsize,b 45 | if. (a~:_1)*.0~:s do. NB. J data type agreement 46 | ta=. (((i{"1{:R 23 | if. -.shape-:$d do. 24 | smoutput 'bad shape: ',col,' ',":$d 25 | smoutput ,d 26 | assert 0 27 | end. 28 | d=. ,d 29 | if. val-:(#val){.d do. i.0 0 return. end. 30 | smoutput 'bad value: ',col 31 | smoutput val 32 | smoutput d 33 | assert 0 34 | ) 35 | 36 | drd=: 3 : 0 37 | R=: jd 'reads ',y 38 | ALLR=: ALLR,' 14 | 'out' jhdiv '' 15 | ) 16 | 17 | create=: 3 : 0 18 | 'jdapp1'jhr'' 19 | ) 20 | 21 | ev_jd_click=: 3 : 0 22 | 'db arg'=. getvs'db arg' 23 | jhrajax tohtml_jd_ (db;'u/p';'intask') jdx_jd_ arg 24 | ) 25 | 26 | jev_get=: create NB. browser get request 27 | 28 | CSS=: 0 : 0 29 | td{border:1px solid red} 30 | ) 31 | 32 | JS=: 0 : 0 33 | function ev_jd_click() {jdoajax(["db","arg"],"");} 34 | function ev_db_enter() {jscdo("jd");} 35 | function ev_arg_enter(){jscdo("jd");} 36 | 37 | function ajax(ts) 38 | { 39 | jbyid("out").innerHTML=ts[0]; 40 | } 41 | ) 42 | 43 | desc=: 0 : 0 44 | Run jd commands on a database.

45 | ) 46 | 47 | OKURL_jhs_=: a:-.~~.'jdapp1';OKURL 48 | -------------------------------------------------------------------------------- /demo/northwind/csv/Categories.cdefs: -------------------------------------------------------------------------------- 1 | 1 CategoryID int 2 | 2 CategoryName varbyte 3 | 3 Description varbyte 4 | 4 Picture varbyte 5 | options TAB LF NO \ 0 6 | -------------------------------------------------------------------------------- /demo/northwind/csv/Categories.csv: -------------------------------------------------------------------------------- 1 | 1 Beverages Soft drinks, coffees, teas, beers, and ales 2 | 2 Condiments Sweet and savory sauces, relishes, spreads, and seasonings 3 | 3 Confections Desserts, candies, and sweet breads 4 | 4 Dairy Products Cheeses 5 | 5 Grains/Cereals Breads, crackers, pasta, and cereal 6 | 6 Meat/Poultry Prepared meats 7 | 7 Produce Dried fruit and bean curd 8 | 8 Seafood Seaweed and fish 9 | -------------------------------------------------------------------------------- /demo/northwind/csv/Customers.cdefs: -------------------------------------------------------------------------------- 1 | 1 Address varbyte 2 | 2 City varbyte 3 | 3 CompanyName varbyte 4 | 4 ContactName varbyte 5 | 5 ContactTitle varbyte 6 | 6 Country varbyte 7 | 7 CustomerID byte 10 8 | 8 Fax varbyte 9 | 9 Phone varbyte 10 | 10 PostalCode varbyte 11 | 11 Region varbyte 12 | options TAB LF NO \ 0 13 | -------------------------------------------------------------------------------- /demo/northwind/csv/Employees.cdefs: -------------------------------------------------------------------------------- 1 | 1 Address varbyte 2 | 2 BirthDate datetime 3 | 3 City varbyte 4 | 4 Country varbyte 5 | 5 EmployeeID int 6 | 6 Extension varbyte 7 | 7 FirstName varbyte 8 | 8 HireDate datetime 9 | 9 HomePhone varbyte 10 | 10 LastName varbyte 11 | 11 Notes varbyte 12 | 12 Photo varbyte 13 | 13 PostalCode varbyte 14 | 14 Region varbyte 15 | 15 ReportsTo int 16 | 16 Title varbyte 17 | 17 TitleOfCourtesy varbyte 18 | options TAB LF NO \ 0 19 | -------------------------------------------------------------------------------- /demo/northwind/csv/OrderDetails.cdefs: -------------------------------------------------------------------------------- 1 | 1 Discount float 2 | 2 OrderID int 3 | 3 ProductID int 4 | 4 Quantity int 5 | 5 UnitPrice float 6 | options TAB LF NO \ 0 7 | -------------------------------------------------------------------------------- /demo/northwind/csv/Orders.cdefs: -------------------------------------------------------------------------------- 1 | 1 CustomerID byte 10 2 | 2 EmployeeID int 3 | 3 Freight float 4 | 4 OrderDate datetime 5 | 5 OrderID int 6 | 6 RequiredDate datetime 7 | 7 ShipAddress varbyte 8 | 8 ShipCity varbyte 9 | 9 ShipCountry varbyte 10 | 10 ShipName varbyte 11 | 11 ShippedDate datetime 12 | 12 ShipPostalCode varbyte 13 | 13 ShipRegion varbyte 14 | 14 ShipVia int 15 | options TAB LF NO \ 0 16 | -------------------------------------------------------------------------------- /demo/northwind/csv/Products.cdefs: -------------------------------------------------------------------------------- 1 | 1 CategoryID int 2 | 2 Discontinued byte 10 3 | 3 ProductID int 4 | 4 ProductName varbyte 5 | 5 QuantityPerUnit varbyte 6 | 6 ReorderLevel int 7 | 7 SupplierID int 8 | 8 UnitPrice float 9 | 9 UnitsInStock int 10 | 10 UnitsOnOrder int 11 | options TAB LF NO \ 0 12 | -------------------------------------------------------------------------------- /demo/northwind/csv/Shippers.cdefs: -------------------------------------------------------------------------------- 1 | 1 CompanyName varbyte 2 | 2 Phone varbyte 3 | 3 ShipperID int 4 | options TAB LF NO \ 0 5 | -------------------------------------------------------------------------------- /demo/northwind/csv/Shippers.csv: -------------------------------------------------------------------------------- 1 | Speedy Express (503) 555-9831 1 2 | United Package (503) 555-3199 2 3 | Federal Shipping (503) 555-9931 3 4 | -------------------------------------------------------------------------------- /demo/northwind/csv/Suppliers.cdefs: -------------------------------------------------------------------------------- 1 | 1 Address varbyte 2 | 2 City varbyte 3 | 3 CompanyName varbyte 4 | 4 ContactName varbyte 5 | 5 ContactTitle varbyte 6 | 6 Country varbyte 7 | 7 Fax varbyte 8 | 8 HomePage varbyte 9 | 9 Phone varbyte 10 | 10 PostalCode varbyte 11 | 11 Region varbyte 12 | 12 SupplierID int 13 | options TAB LF NO \ 0 14 | -------------------------------------------------------------------------------- /demo/northwind/csv/jdclass: -------------------------------------------------------------------------------- 1 | csv -------------------------------------------------------------------------------- /demo/northwind/csv/jdcsvrefs.txt: -------------------------------------------------------------------------------- 1 | OrderDetails OrderID Orders OrderID 2 | OrderDetails ProductID Products ProductID 3 | Orders CustomerID Customers CustomerID 4 | Orders EmployeeID Employees EmployeeID 5 | Orders ShipVia Shippers ShipperID 6 | Products SupplierID Suppliers SupplierID 7 | Products CategoryID Categories CategoryID 8 | -------------------------------------------------------------------------------- /demo/sandp/csv/j.cdefs: -------------------------------------------------------------------------------- 1 | 1 jid byte 3 2 | 2 jname byte 10 3 | 3 city byte 10 4 | options TAB LF NO \ 0 5 | -------------------------------------------------------------------------------- /demo/sandp/csv/j.csv: -------------------------------------------------------------------------------- 1 | J1 Sorter Paris 2 | J2 Display Rome 3 | J3 OCR Athens 4 | J4 Console Athens 5 | J5 RAID London 6 | J6 EDS Oslo 7 | J7 Tape London 8 | -------------------------------------------------------------------------------- /demo/sandp/csv/jdclass: -------------------------------------------------------------------------------- 1 | csv -------------------------------------------------------------------------------- /demo/sandp/csv/jdcsvrefs.txt: -------------------------------------------------------------------------------- 1 | sp sid s sid 2 | sp pid p pid 3 | spj sid s sid 4 | spj pid p pid 5 | -------------------------------------------------------------------------------- /demo/sandp/csv/p.cdefs: -------------------------------------------------------------------------------- 1 | 1 pid byte 3 2 | 2 pname byte 10 3 | 3 color byte 10 4 | 4 weight int 5 | 5 city byte 10 6 | options TAB LF NO \ 0 7 | -------------------------------------------------------------------------------- /demo/sandp/csv/p.csv: -------------------------------------------------------------------------------- 1 | P1 Nut Red 12 London 2 | P2 Bolt Green 17 Paris 3 | P3 Screw Blue 17 Oslo 4 | P4 Screw Red 14 London 5 | P5 Cam Blue 12 Paris 6 | P6 Cog Red 19 London 7 | -------------------------------------------------------------------------------- /demo/sandp/csv/s.cdefs: -------------------------------------------------------------------------------- 1 | 1 sid byte 3 2 | 2 sname byte 10 3 | 3 status int 4 | 4 city byte 10 5 | options TAB LF NO \ 0 6 | -------------------------------------------------------------------------------- /demo/sandp/csv/s.csv: -------------------------------------------------------------------------------- 1 | S1 Smith 20 London 2 | S2 Jones 10 Paris 3 | S3 Blake 30 Paris 4 | S4 Clark 20 London 5 | S5 Adams 30 Athens 6 | -------------------------------------------------------------------------------- /demo/sandp/csv/sp.cdefs: -------------------------------------------------------------------------------- 1 | 1 sid byte 3 2 | 2 pid byte 3 3 | 3 qty int 4 | options TAB LF NO \ 0 5 | -------------------------------------------------------------------------------- /demo/sandp/csv/sp.csv: -------------------------------------------------------------------------------- 1 | S1 P1 300 2 | S1 P2 200 3 | S1 P3 400 4 | S1 P4 200 5 | S1 P5 100 6 | S1 P6 100 7 | S2 P1 300 8 | S2 P2 400 9 | S3 P2 200 10 | S4 P2 200 11 | S4 P4 300 12 | S4 P5 400 13 | -------------------------------------------------------------------------------- /demo/sandp/csv/spj.cdefs: -------------------------------------------------------------------------------- 1 | 1 sid byte 3 2 | 2 pid byte 3 3 | 3 jid byte 3 4 | 4 qty int 5 | options TAB LF NO \ 0 6 | -------------------------------------------------------------------------------- /demo/sandp/csv/spj.csv: -------------------------------------------------------------------------------- 1 | S1 P1 J1 200 2 | S1 P1 J4 700 3 | S2 P3 J1 400 4 | S2 P3 J2 200 5 | S2 P3 J3 200 6 | S2 P3 J4 500 7 | S2 P3 J5 600 8 | S2 P3 J6 400 9 | S2 P3 J7 800 10 | S2 P5 J2 100 11 | S3 P3 J1 200 12 | S3 P4 J2 500 13 | S4 P6 J3 300 14 | S4 P6 J7 300 15 | S5 P2 J2 200 16 | S5 P2 J4 100 17 | S5 P5 J5 500 18 | S5 P5 J7 100 19 | S5 P6 J2 200 20 | S5 P1 J4 100 21 | S5 P3 J4 200 22 | S5 P4 J4 800 23 | S5 P5 J4 400 24 | S5 P6 J4 500 25 | -------------------------------------------------------------------------------- /demo/sed/csv/e.cdefs: -------------------------------------------------------------------------------- 1 | 1 ename byte 12 2 | 2 sno int 3 | 3 sname byte 12 4 | 4 rno int 5 | 5 rname byte 12 6 | options TAB LF NO \ 0 7 | -------------------------------------------------------------------------------- /demo/sed/csv/e.csv: -------------------------------------------------------------------------------- 1 | Smith 34 Auto 6 Naples 2 | Jones 33 Truck 5 Rome 3 | Robinson 34 Auto 6 Naples 4 | Jasper 33 Truck 5 Rome 5 | Steinberg 33 Truck 5 Rome 6 | Rafferty 31 Auto 4 Naples 7 | -------------------------------------------------------------------------------- /demo/sed/csv/jdclass: -------------------------------------------------------------------------------- 1 | csv -------------------------------------------------------------------------------- /demo/sed/csv/jdcsvrefs.txt: -------------------------------------------------------------------------------- 1 | s rno rname r rno rname 2 | e rno rname sno sname s rno rname sno sname 3 | t ename e ename 4 | 5 | -------------------------------------------------------------------------------- /demo/sed/csv/r.cdefs: -------------------------------------------------------------------------------- 1 | 1 rno int 2 | 2 rname byte 12 3 | 3 popul int 4 | options TAB LF NO \ 0 5 | -------------------------------------------------------------------------------- /demo/sed/csv/r.csv: -------------------------------------------------------------------------------- 1 | 4 Naples 1000 2 | 5 Rome 2000 3 | 6 Naples 2000 4 | -------------------------------------------------------------------------------- /demo/sed/csv/s.cdefs: -------------------------------------------------------------------------------- 1 | 1 sno int 2 | 2 sname byte 12 3 | 3 rno int 4 | 4 rname byte 12 5 | 5 name byte 12 6 | options TAB LF NO \ 0 7 | -------------------------------------------------------------------------------- /demo/sed/csv/s.csv: -------------------------------------------------------------------------------- 1 | 31 Auto 4 Naples Sales 2 | 31 Truck 6 Naples Clerical 3 | 33 Truck 5 Rome Engineering 4 | 34 Auto 6 Naples Clerical 5 | 35 Auto 4 Naples Marketing 6 | -------------------------------------------------------------------------------- /demo/sed/csv/t.cdefs: -------------------------------------------------------------------------------- 1 | 1 tno int 2 | 2 ename byte 12 3 | 3 tname byte 12 4 | 4 tsize byte 12 5 | 5 tcat byte 12 6 | options TAB LF NO \ 0 7 | -------------------------------------------------------------------------------- /demo/sed/csv/t.csv: -------------------------------------------------------------------------------- 1 | 11 Jones Desk small A 2 | 12 Jasper Cabinet medium B 3 | 13 Smith Chair small A 4 | 14 Jones Cabinet medium R 5 | -------------------------------------------------------------------------------- /doc/Docs.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

Jd wiki is the Jd reference.

17 |

Best way to learn and explore is to install Jd and run the interactive tutorials.

18 |

A listing of a run of all tutorials is at tutorials and browsing that gives a peak before doing an install.

19 | 20 | 21 | -------------------------------------------------------------------------------- /doc/General.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

Install

17 |

Install Jd (addon data/jd) with JAL/Package Manager. Jd requires J64 (807 or later) on Windows, Linux, macOS, and RPI.

18 |

Run Casual

19 | 
   load'jd' NB. '~addons/data/jd/jd.ijs'
20 |

Run Production

21 |

Recommended to NOT run a Jd production system from the ~addons/data/jd folder. Package Manager update could update this folder and your production system would be running a new code base that you had not tested.

22 |

A Jd production system should have its own folder. For example, copy your current jd folder from ~addons/data to create ~/production/jd.

23 | 
   load'~/production/jd/jd.ijs' NB. load production Jd
24 | 25 | 26 | -------------------------------------------------------------------------------- /doc/Ops_drop.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

dropcol

17 |

dropcol table col
18 | OK if col does not exist
19 | error if col is used in a ref

20 |

drop a ref col by name as you would any other col
21 | get ref col names with:

22 |

jd'info ref' 

23 |

dropdb

24 |

drops each col of each table
25 | releases lock
26 | deletes filefolder

27 |

dropfilesize

28 |

dropfilesize

29 |

files tend to be kept larger than required
30 | data in the file can grow without having to resize/remap

31 |

dropfilesize frees up unused file space - use if a db is static

32 |

dropfilesize returns filenames, old size, new size

33 |

droptable

34 |

droptable [/reset] table

35 |

/reset preserves schema
36 |   drops rows
37 |   drops dynamic hash/unique/... cols
38 |   keeps filesize

39 |

OK if table does not exist
40 | error if table has reference (refers to or referred by)

41 | 42 | 43 | -------------------------------------------------------------------------------- /doc/Ops_info.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

info

17 |

info type [...] - info about current DB

18 |

agg                         - aggregators
19 | derived     [table[column]] - derived cols
20 | last                        - cmd time space parts
21 | ref         [table[column]] - ref cols
22 | schema      [table[column]] - normal cols (non jd...)
23 | summary     [table^*]       - tables or tables that make up the ptable
24 | table       [table^*]       - tables or tables that make up the ptable
25 | validate    [table[column]]
26 | validatebad [table[column]]
27 | varbyte     [table[column]]

28 |

Tables are sorted and cols are in default selection order

29 |

Schema and similar types treat ptable as a single table
30 | validate and validatebad list ptable parts

31 |

list

32 |

list version - Jd version
33 | list open - open databases

34 | 35 | 36 | -------------------------------------------------------------------------------- /doc/Ops_join.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

ref

17 |

ref [option] tab1 col1a ... tab2 col2a ...

18 |

creates ref col in tab1 that has data necessary to do joins

19 |

col types int1/int2/int4/float/varbyte not allowed

20 |

elided option creates simple data for a left1 join

21 |

/left option creates more complicated data necessary for a left join

22 |

/left ref only works with single joins - f,f-g works and f,f-g,g-h fails

23 |

data for left1 or inner join can be derived from the left data

24 |

insert/delete/update to either table sets ref col dirty flag

25 |

ref col recalculated (dirty) as required

26 |

get ref col names with:

27 |

jd'info ref' 

28 |

drop a ref col by name as you would any other col
29 |

30 | 31 | 32 | -------------------------------------------------------------------------------- /doc/Ops_raw.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

get

17 |

get table col

18 |

return raw col data
19 | varbyte - indexes;values
20 | can be used with set to efficiently adjust col data

21 |

set

22 |

set table col;data

23 |

set raw col data
24 | varbyte - indexes;values
25 | can be used with get to efficiently adjust col data

26 | 27 | 28 | -------------------------------------------------------------------------------- /doc/Ops_rename.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

renamecol

17 |

renamecol table col newname

18 |

error if col is used in ref

19 |

renametable

20 |

renametable table newname

21 |

error if table is used in ref

22 | 23 | 24 | -------------------------------------------------------------------------------- /doc/Ops_table-table.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

Table operations

17 |

See tutorial table_table

18 |

tablecopy

19 |

tablecopy snktable srctable srcdan - implicit jdaccess'srcdan'

20 |

copies a table from srcdan to the current db
21 | uses host file copy and runs as fast as hardware allows

22 |

tableinsert

23 |

tableinsert snktable srctable srcdan - implicit jdaccess'srcdan'

24 |

implemented as an efficient loop over cols with an append of new data

25 |

affected ref cols are marked dirty

26 |

tablemove

27 |

tablemove snktable srctable srcdan - implicit jdaccess'srcdan'

28 |

moves a table from srcdan to the current db
29 | uses host file move and runs fast as it just updates pointers

30 |

srcdan must be on the same file hardware for move to work

31 | 32 | 33 | -------------------------------------------------------------------------------- /doc/Replicate.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

An application can require a DB to be available for quick updates and slow queries would interfere with this. A solution is to replicate the DB for queries.

17 |

Run tutorial for more info:

18 | 
   jdrt'replicate'
19 |

The rlog arg is the folder that holds info from the src DB required to replicate the DB in the snk DB.

20 |

jdrepinfo

21 |

jdrepinfo

22 |

report db rep status

23 |

jdrepkill

24 |

jdrepkill

25 |

stops db (either src or snk) use of rlog folder

26 |

jdrepsrc

27 |

jdrepsrc_jd_ rlog

28 |

initializes rlog folder with copy of db and sets up for logging new change ops

29 |

jdrepsnk

30 |

jdrepsnk_jd_ rlog

31 |

db ops first update as required from rlog file updates db from folder

32 | 33 | 34 | -------------------------------------------------------------------------------- /doc/Support.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 |

16 |

All Jd users get free support through the J forums. For some users, the excellent and prompt forum support is probably all that is required. All fees are in USD.

17 |

Jd Prepaid Support

18 |

Jd Prepaid Support provides high-priority access to Jd developers/programmers for up to 100 hours for an annual fee of 10,000.

19 |

Jd Support

20 |

Jd Support provides the same support as Jd Prepaid Support but at mutually scheduled times for 300/hour.

21 |

Jd Application Development and Training

22 |

Jsoftware provides a range of consulting services for J/Jd application development and training.

23 | 24 | 25 | -------------------------------------------------------------------------------- /doc/jd.css: -------------------------------------------------------------------------------- 1 | /* Jd.css */ 2 | 3 | body {margin:1em 5% 1em 5%} 4 | 5 | a {color:#0000d0} 6 | b,code,pre,tt {color:#000080} 7 | 8 | pre { 9 | background:#fffaf4; 10 | border:1pt solid #aebdcc; 11 | padding:5pt; 12 | } 13 | 14 | h1,h2,h3 { 15 | color:#015293; 16 | font-family:sans-serif; 17 | border-bottom:2px solid silver; 18 | margin-top:1.2em; 19 | margin-bottom:0.5em; 20 | line-height:1.3; 21 | } 22 | 23 | img { 24 | display:block; 25 | margin-left:auto; 26 | margin-right:auto; 27 | } 28 | 29 | #nav { 30 | background:#ddd; 31 | padding:2px 5px 2px 5px; 32 | border:1pt solid #aebdcc; 33 | display:inline-block; 34 | } 35 | 36 | #nav a { 37 | text-decoration:none; 38 | } 39 | 40 | #nav a.act { 41 | color:#000; 42 | } 43 | -------------------------------------------------------------------------------- /dynamic/base.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | coclass 'jddbase' 3 | coinsert 'jdcolumn' 4 | 5 | NB. ref is a dynamic col 6 | 7 | visible=: 0 8 | static=: 0 9 | 10 | STATE=: (STATE_jdcolumn_-.<'shape'),<'subscriptions' 11 | subscriptions=: 0 2$a: 12 | 13 | countdat=: 3 : '(dirty+.left){Tlen,~#dat' NB. left jams Tlen as it can't be calculated 14 | 15 | NB. y is (table);(list of column names) 16 | subscribe =: 3 : 0 17 | cols=.,&.>,boxxopen cols [ 'tab cols' =. ,&.> y 18 | t=. getloctab tab 19 | np=.NAME__PARENT 20 | relpath =. ((tab-.@-:np) # '^.',np,'.'),NAME NB. path from t 21 | writestate__t SUBSCR__t=: SUBSCR__t , relpath;(#subscriptions),&2{"1{:jd'info schema' 12 | 'int2 not allowed' assert -.(<'int2')e.d 13 | 'int4 not allowed' assert -.(<'int4')e.d 14 | 'ref not allowed' assert 0 0=>$each {.{:jd'info ref' 15 | jd'info summary' 16 | ) 17 | 18 | NB. RO task init - y is DB 19 | rinit=: 3 : 0 20 | jdadmin 0 21 | jdadmin y;MTRO_jmf_ 22 | NB. (;{.{.jdadminop_jd_'')jdadminop_jd_ 'read reads info list' NB. only allow read type ops 23 | JDMTMRO_jd_=: 1 NB. mark as mtm ro task 24 | jd'info summary' NB. critical that RO tasks has correct and stable locale info 25 | ) 26 | -------------------------------------------------------------------------------- /nopacman/jddev.ijs: -------------------------------------------------------------------------------- 1 | man=: 0 : 0 2 | Jd development 3 | 4 | ~addons -> j9.4/addons 5 | ~Addons -> git/addons 6 | 7 | editing/testing done with git/addons/data/jd 8 | 9 | load'~addons/data/jd/jd.ijs' NB. from ~addons 10 | load'~Addons/data/jd/jd.ijs' NB. from git/addons 11 | 12 | initial load sets JDP_z_ as path to jd folder and this 13 | is used for all other file references 14 | 15 | *** pacman release build (from ~Addons/ide/jhs): 16 | ensure jd'list version' matches jd wiki release notes 17 | bupx jd'list version' 18 | 19 | ...$ cd git/addons/data/jd 20 | ...$ git pull 21 | ...$ git status - resolve problems 22 | 23 | start J 24 | load'~Addons/ide/jhs/nopacman/dev.ijs' 25 | setp'data/jd' 26 | manifest_status'' NB. edit manifest to resolve problems 27 | bump_version'' 28 | 29 | ...$ git status 30 | ...$ git commit -a -m "pacman release ..." 31 | ...$ git push 32 | -------------------------------------------------------------------------------- /pm/pm_disk.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | measure ms/gb sustained disk transfer rates 3 | ) 4 | 5 | GBC=: 1e9 6 | D=: GBC$'a' 7 | fn=: jpath'~temp/jnk.0' 8 | 9 | 10 | NB. y loop indexed fwrite - report ms/gb 11 | ms_gb_ndx_fwrite=: 3 : 0 12 | assert (y>0)*.y<30 13 | ''fwrite fn 14 | PMA'' 15 | s=. 0 16 | for. i.y do. 17 | D 1!:12 fn;s 18 | s=. s+GBC 19 | end. 20 | ''PM'' 21 | <.0.5+y%~PMT'' 22 | ) 23 | 24 | NB. y loop indexed fread - report ms/gb 25 | ms_gb_ndx_fread=: 3 : 0 26 | assert y<:<.GBC%~fsize fn 27 | s=. 0 28 | t=. 0 29 | PMA'' 30 | for. i.y do. 31 | t=. +/'a'=1!:11 fn;s,GBC 32 | s=. s+GBC 33 | end. 34 | ''PM'' 35 | <.0.5+y%~PMT'' 36 | ) 37 | 38 | ms_gb_map_read=: 3 : 0 39 | JCHAR map_jmf_ 'd';fn 40 | b=. <.GBC%~#d 41 | PMA'' 42 | t=. +'a'=/d 43 | ''PM'' 44 | unmap_jmf_ 'd' 45 | <.0.5+b%~PMT'' 46 | ) 47 | 48 | cu=: 3 : 0 49 | -------------------------------------------------------------------------------- /pm/pm_insert_1.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | 3 | man=: 0 : 0 4 | bld rows,cols NB. cols , rows 5 | faster '' NB. current bench'' is x % faster that original 6 | 7 | runpm'' NB. run test under perfmon 8 | show'' NB. show perfmon results 9 | 10 | record' NB. record last perfmon results in benchfile 11 | seerecord'' NB. show last record in benchfile 12 | seerecord 3 NB. show benchfile record 3 13 | 14 | 15 | jpm - j performance monitor - run in jconsole or Jqt 16 | showtotal_jpm_'' 17 | showdetail_jpm_'jdx' 18 | ) 19 | 20 | require'jfiles' 21 | require'jpm' 22 | 23 | old=: 0.00152047 NB. time from before project started 24 | 25 | 26 | reload=: 3 : 0 27 | load JDP,'pm/pm_insert_1.ijs' 28 | ) 29 | 30 | benchfile=: 'bench.ijf' 31 | 32 | bld=: 3 : 0 33 | 'cols rows'=: y 34 | 'new'jdadmin'ins1' 35 | d=. i.rows 36 | jd'createtable f' 37 | for_n. i.cols do. 38 | jd('createcol f ',('c',":n),' int');d 39 | end. 40 | newdata=: ,('c',each":each<"0 i.cols),.<23 41 | i.0 0 42 | ) 43 | 44 | bench=: 3 : 0 45 | y timex 'jd''insert f'';newdata' 46 | ) 47 | 48 | old=: 0.00152047 NB. bld 50 300000 49 | 50 | faster=: 3 : '(old-bench'''')%old' NB. faster bench'' 51 | 52 | runpm=: 3 : 0 53 | start_jpm_'' 54 | jd'insert f';newdata 55 | show'' 56 | ) 57 | 58 | show=: 3 : 0 59 | 0 1 showtotal_jpm_'' 60 | ) 61 | 62 | record=: 3 : 0 63 | if. -.fexist benchfile do. jcreate benchfile end. 64 | (<(isotimestamp 6!:0'');jread benchfile;y 70 | echo t 71 | d 72 | ) 73 | -------------------------------------------------------------------------------- /pm/pm_map.ijs: -------------------------------------------------------------------------------- 1 | NB. time map related things. 2 | 3 | map_creates=: 3 : 0 4 | jd'createtable f' 5 | for_i. i.y do. 6 | jd'createcol f c',(":i),' int' 7 | end. 8 | ) 9 | 10 | map_times=: 3 : 0 11 | jdadminx 'test' 12 | echo 'create';timex'map_creates 1000' 13 | echo 'close ';timex'jd''close''' 14 | echo 'schema';timex'jd''info schema''' 15 | ) 16 | 17 | NB. from a complete shutdown and restart 18 | map_map_time=: 3 : 0 19 | jdadmin'test' 20 | echo'map';timex'jd''info schema''' 21 | ) 22 | 23 | 0 : 0 24 | times on fast linux with ssd 25 | map_times'' 26 | ┌──────┬────────┐ 27 | │create│0.830701│ 28 | └──────┴────────┘ 29 | ┌──────┬────────┐ 30 | │close │0.081012│ 31 | └──────┴────────┘ 32 | ┌──────┬───────┐ 33 | │schema│0.20704│ 34 | └──────┴───────┘ 35 | map_map_time'' 36 | ┌───┬────────┐ 37 | │map│0.821654│ 38 | └───┴────────┘ 39 | 40 | times on older windows laptop 41 | map_times'' 42 | ┌──────┬───────┐ 43 | │create│10.5645│ 44 | └──────┴───────┘ 45 | ┌──────┬────────┐ 46 | │close │0.309289│ 47 | └──────┴────────┘ 48 | ┌──────┬───────┐ 49 | │schema│1.34713│ 50 | └──────┴───────┘ 51 | map_map_time'' 52 | ┌───┬───────┐ 53 | │map│13.8546│ 54 | └───┴───────┘ 55 | ) 56 | -------------------------------------------------------------------------------- /pm/pm_read_avg_by.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | 3 | man=: 0 : 0 4 | bld '' NB. cols , rows 5 | faster '' NB. current bench'' is x % faster that original 6 | 7 | runpm y NB. run test under perfmon 8 | show'' NB. show perfmon results 9 | 10 | record' NB. record last perfmon results in benchfile 11 | seerecord'' NB. show last record in benchfile 12 | seerecord 3 NB. show benchfile record 3 13 | 14 | 15 | jpm - j performance monitor - run in jconsole or Jqt 16 | showtotal_jpm_'' 17 | showdetail_jpm_'jdx' 18 | ) 19 | 20 | require'jfiles' 21 | require'jpm' 22 | 23 | sentence=: 'read avg fare_amount by passenger_count from yellow_tripdata' 24 | 25 | old=: 0.316146 NB. time from before project started 26 | 27 | reload=: 3 : 0 28 | load JDP,'pm/pm_read_avg_by.ijs' 29 | ) 30 | 31 | 32 | 33 | foo=: 3 : 0 34 | fare=. jd 'get yellow_tripdata fare_amount' 35 | pcnt=. jd 'get yellow_tripdata passenger_count' 36 | (~.pcnt),.pcnt (+/ % #)/. fare 37 | ) 38 | 39 | 40 | benchfile=: 'bench.ijf' 41 | 42 | jdadmin'taxi' NB. tripdata table from jdrt 43 | 44 | faster=: 3 : '(old-jdtx sentence)%old' NB. faster bench'' 45 | 46 | NB. sentence 47 | runpm=: 3 : 0 48 | start_jpm_'' 49 | jd y 50 | show'' 51 | ) 52 | 53 | show=: 3 : 0 54 | 0 1 showtotal_jpm_'' 55 | ) 56 | 57 | record=: 3 : 0 58 | if. -.fexist benchfile do. jcreate benchfile end. 59 | (<(isotimestamp 6!:0'');jread benchfile;y 65 | echo t 66 | d 67 | ) 68 | 69 | 70 | coclass 'jdquery' 71 | coinsert 'jddatabase' 72 | 73 | Query=: 3 : 0 74 | q__=: inds;indices;nby;agg;cloc 75 | if. 0 do. 76 | r=. '' 77 | for_n. cloc do. 78 | r=. r, read 88 | end. 89 | ) 90 | -------------------------------------------------------------------------------- /pm/pm_tlen_damage_repair.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | 4.41 removed getdb fread to validate database class 3 | and fexist for jddamage and jdrepair files 4 | 5 | 4.42 replaced insert writestate for Tlen with jdtlen mapped file 6 | 7 | git/jd_old/jd_4.40 and jd_4.41 used for performance comparisons 8 | ) 9 | 10 | bld=: 3 : 0 11 | 'new'jdadmin'test' 12 | jd'createtable f' 13 | for_i. i.y do. 14 | jd'createcol f c',(":i),' int' 15 | end. 16 | pairs=: ,(,.'c',each ":each i.y),.<23 17 | ) 18 | 19 | run=: 3 : 0 20 | bld 10 21 | r=. '' 22 | r=. r, timex'jd''insert f'';pairs' 23 | r=. r, timex'jd''insert f'';pairs' 24 | r=. r,3 timex'jd''insert f'';pairs' 25 | r=. r, timex'0[jd''read c0 from f''' 26 | r=. r, timex'0[jd''read c0 from f''' 27 | r=. r,3 timex'0[jd''read c0 from f''' 28 | r=. ('jd','.'-.~jd'list version'),'=: ',":r 29 | ".r 30 | r 31 | ) 32 | 33 | -------------------------------------------------------------------------------- /pm/pmb.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | 3 | NB. timings for ref2 tables csv dump/restore 4 | 5 | 0 : 0 6 | test arows acols brows 7 | test 10 1 5 8 | 9 | jd'gen ref2 arows acols brows' create tables a and b 10 | tables are csv dumped and restored 11 | operations are timed and reported 12 | useful in stress testing larger data against hardware (particularly ram) 13 | 14 | test 10 2 3 NB. creats small tables and runs quickly 15 | test 50e6 10 10000 NB. runs in a few minutes with 6gb 16 | test 300e6 10 10000 NB. runs in a few minutes with 24gb - thrashes with less 17 | ) 18 | 19 | f=: 3 : 'y,~'' '',~6j1 ":timex ''jd y''' 20 | 21 | CSVFOLDER=: '~temp/jd/csv/junk' 22 | 23 | test=: 3 : 0 24 | jddeletefolder_jd_ CSVFOLDER 25 | assert 3=#y 26 | assert y>:10 0 3 27 | assert ({.y)>:{:y 28 | 'arows acols brows'=. 10":each y 29 | jd'close' 30 | jdadmin 0 31 | jdadminx'test' 32 | r=. 0 2$'' 33 | r=. r,f 'gen ref2 a ',arows,' ',acols,' b ',brows 34 | r=. r,f s NB. first may be slow as akey/adata/aref need to be read from disk 35 | r=. r,f s 36 | r=. r,f s 37 | assert 12 4-:,;{:jd s 38 | r=. r,f 'csvdump' 39 | jdadminx'test' 40 | r=. r,f 'csvrestore' 41 | r=. r,f s NB. first may be slow as akey/adata/aref need to be read from disk 42 | r=. r,f s 43 | r=. r,f s 44 | assert 12 4-:,;{:jd s NB. first may be slow as akey/adata/aref need to be read from disk 45 | r=. r,(6j1":+/;".each 6{.each <"1 r),' total' 46 | r=. r,' ' 47 | r=. r,(14j9 ": 1e9%~fsize CSVFOLDER,'a.csv'),' GB - fsize a.csv' 48 | r=. r,(14j9 ": 1e9%~fsize CSVFOLDER,'b.csv'),' GB - fsize b.csv' 49 | r 50 | ) 51 | 52 | -------------------------------------------------------------------------------- /pm/pmc.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | 3 | NB. performance measurement for large tables and multiple drives 4 | 5 | NB. 'f 1000 10' tm tm_csvcreate 6 | tm_csvcreate=: 0 : 0 7 | *jdadminx'pmcsv' 8 | gen one A0 A1 A2 9 | *jdflush_jd_'' 10 | ) 11 | 12 | NB. '~temp/csv' tm tm_csvdump 13 | tm_csvdump=: 0 : 0 14 | *jdadminx'pmcsv' 15 | *CSVFOLDER=: 'A0' 16 | *deletefolder CSVFOLDER 17 | *jdflush_jd_'' 18 | csvdump 19 | *jdflush_jd_'' 20 | ) 21 | 22 | NB. '~temp/csv' tm tm_csvdump 23 | tm_csvrestore=: 0 : 0 24 | *jdadminx'pmcsv' 25 | *CSVFOLDER=: 'A0' 26 | *jdflush_jd_'' 27 | csvrestore 28 | *jdflush_jd_'' 29 | ) 30 | -------------------------------------------------------------------------------- /pm/pmx.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | 3 | 0 : 0 4 | 5 | linux cr ref unique 6 | 100e6 33 9 37 7 | 500e6 178 77 186 8 | 1000e6 379 158 495 9 | 10 | win 11 | 50e6 35 8 29 12 | 100e6 65 20 72 13 | 500e6 142 75 426 14 | ) 15 | 16 | FLUSH=: 1 17 | 18 | jdf=: 3 : 0 19 | jd y 20 | if. FLUSH do. jd'flush' end. 21 | ) 22 | 23 | tn=: 3 : 0 24 | ;(y>:1e6){(":y);(":<.y%1e6),'e6' 25 | ) 26 | 27 | admin=: 3 : 0 28 | jdadmin'~temp/jdpm/','d',tn y 29 | ) 30 | 31 | NB. create tables for preformance tests 32 | NB. cr 10 - create table h10 with 10 rows 33 | NB. cr 100e6 - create table h100e6 with 100e6 rows 34 | cr=: 3 : 0 35 | assert 19.<.n%1 43 | jd'createtable f' 44 | jdf'createcol f a int _';3*i.n 45 | jdf'createcol f b int _';n$b NB. ref f b g b - many to one 46 | jdf'createcol f c int _';3*i.n NB. data (no dups) 47 | jdf'createcol f d int _';n$i.c NB. data (dups) 48 | 49 | jd'createtable g' 50 | jdf'createcol g b int _';|.b 51 | pmr_jd_'' 52 | ) 53 | 54 | ref=: 3 : 0 55 | pmclear_jd_'' 56 | jdf'ref f b g b' 57 | pmr_jd_'' 58 | ) 59 | 60 | hash=: 3 : 0 61 | pmclear_jd_'' 62 | jdf'createhash f d' 63 | pmr_jd_'' 64 | ) 65 | 66 | unique=: 3 : 0 67 | pmclear_jd_'' 68 | jdf'createunique f c' 69 | pmr_jd_'' 70 | ) 71 | 72 | rd=: 3 : 0 73 | jd'reads from f,f.h where c=7' 74 | jd'reads from f,f.h where c=7' 75 | 76 | jd'reads from f,f.g where cu=7' 77 | jd'reads from f,f.g where cu=7' 78 | jd'reads from f,f.h where cu=7' 79 | jd'reads from f,f.h where cu=7' 80 | 81 | jd'reads count a from f,f.g where dh=3' 82 | jd'reads count a from f,f.g where dh=3' 83 | jd'reads count a from f,f.h where dh=3' 84 | jd'reads from f,f.g where c=7' 85 | jd'reads from f,f.g where c=7' 86 | jd'reads count a from f,f.h where dh=3' 87 | ) 88 | -------------------------------------------------------------------------------- /server/fork.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | try to fork/execl in J in order to get pid from fork result 3 | trouble is the pid is for /bin/sh 4 | and can't figure out how to get from there to the pid for the jconsole task 5 | ) 6 | 7 | libc=: unxlib'c' 8 | fork=: libc,' fork >x' 9 | execl=: libc,' execl i *c *c *c *c *' 10 | 11 | start=: 0 : 0 12 | echo i.2 3 13 | echo 6!:0'' 14 | a=: i.23 15 | echo a 16 | ) 17 | 18 | start fwrite '~/start.ijs' 19 | 20 | NB. redirect (no terminal) task 21 | rdofork=: 3 : 0 22 | r=. fork cd '' 23 | if. r=0 do. 24 | try. 25 | arg=. '/bin/sh';'sh';'-c';'/home/eric/j901/bin/jconsole start.ijs < /dev/null > jnk.jnk';<<0 26 | a=. execl cd arg 27 | 'dofork execl failed'fwrite'jnk.jnk' 28 | catch. 29 | 'dofork cd failed'fwrite'jnk.jnk' 30 | end. 31 | exit'' NB. only get here if cd or execl failed 32 | end. 33 | r 34 | ) 35 | 36 | NB. terminal task 37 | tdofork=: 3 : 0 38 | r=. fork cd '' 39 | if. r=0 do. 40 | try. 41 | cmd=. 'x-terminal-emulator -e ','"/home/eric/j901/bin/jconsole start.ijs"' 42 | arg=. '/bin/sh';'sh';'-c';cmd;<<0 43 | a=. execl cd arg 44 | 'dofork execl failed'fwrite'jnk.jnk' 45 | catch. 46 | 'dofork cd failed'fwrite'jnk.jnk' 47 | end. 48 | exit'' NB. only get here if cd or execl failed 49 | end. 50 | r 51 | ) 52 | -------------------------------------------------------------------------------- /server/jcs_server.ijs: -------------------------------------------------------------------------------- 1 | NB. script template to start server 2 | jcs_start_template=: 0 : 0 3 | load'~addons/net/jcs/jcs.ijs' 4 | load'jd' 5 | ADMIN 6 | SERVER=: jcss 'PORT' 7 | su__SERVER=: 'SU' 8 | rpc__SERVER=: rpcjd 9 | echo'OK' 10 | runserver__SERVER'' 11 | ) 12 | 13 | NB. create server start script 14 | NB. file;port;su;admin 15 | NB. port 65200 or '65200' is localhost only - '*:65200' is bind any 16 | jcs_start_fix=: 3 : 0 17 | 'file port su admin'=. y 18 | echo(('*'={.port)*.0~:#su)#'superuser and non-localhost has security implications!' 19 | t=. jcs_start_template rplc 'PORT';":port 20 | t=. t rplc 'SU';su 21 | admin=. (-LF={:admin)}.admin 22 | t=. t rplc 'ADMIN';admin 23 | t fwrite file 24 | i.0 0 25 | ) 26 | 27 | NB. start'~temp/sa.ijs' 28 | jcs_start=: 3 : 0 29 | t=. jpath'~temp/jcs_start.txt' 30 | if. FHS*.IFUNIX do. 31 | fork_jtask_ 'ijconsole "',y,'" > "',t,'"' 32 | else. 33 | fork_jtask_ '"',(jpath'~bin/jconsole'),'" "',y,'" > "',t,'"' 34 | end. 35 | 6!:3[0.5 NB. kludge to let task get started 36 | fread t 37 | ) 38 | -------------------------------------------------------------------------------- /server/jds/jds_tools.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | NB. jdrt'jds' 3 | 4 | require JDP,'server/port.ijs' 5 | 6 | jds_server_config_template=: 0 : 0 7 | load 'jd.ijs' 8 | load JDP,'server/jds/jds_server.ijs' 9 | PORT=: 10 | LOGFILE_z_=: '' 11 | LOGLEVEL_z_=: 12 | DBS=: jdremq_jd_ each ',' strsplit_jd_'' 13 | init'' 14 | ) 15 | 16 | NB. path;65220;0;'test,"foo,bar",~temp/jd/mum' - NB. PJDS global 17 | create_jds=: 3 : 0 18 | 'path port logfile loglevel dbs'=. y 19 | sport=. ":port 20 | f=. jpath path,('/'#~'/'~:{:path),'jds/',sport,'/' 21 | mkdir_j_ f 22 | log=. f,'log.log' 23 | logstd=. f,'logstd.log' NB. stdout/stderr 24 | ferase log;logstd 25 | loglevel=. 0 26 | 27 | t=. jds_server_config_template rplc '';JDP;'';sport;'';logfile;'';(":loglevel);'';dbs 28 | t fwrite f,'run.ijs' 29 | 30 | select. UNAME 31 | case. 'Linux';'FreeBSD';'OpenBSD';'Darwin' do. 32 | t=. '#!/bin/bash',LF 33 | t=. t,'"BINPATH/jconsole" "SCRIPT"' rplc 'BINPATH';(jpath'~bin');'SCRIPT';f,'run.ijs' 34 | if. FHS do. t=. t rplc 'jconsole';'ijconsole' end. 35 | t fwrite f,'/run.sh' 36 | shell'chmod +x ',f,'run.sh' 37 | cmd=. ('nohup "PATHrun.sh" > "LOG" 2>&1') rplc 'PATH';f;'LOG';logstd 38 | cmd fwrite f,'run.txt' 39 | case. 'Win' do. 40 | t=. 'for /f "tokens=5" %%a in (''netstat -aon ^| find ":',sport,'" ^| find "LISTENING"'') do taskkill /f /pid %%a' 41 | t=. t,LF 42 | t=. ('/';'\')rplc~'"BINPATH/jconsole" "SCRIPT"' rplc 'BINPATH';(jpath'~bin');'SCRIPT';f,'run.ijs' 43 | t fwrite f,'run.bat' 44 | cmd=. ('"PATHrun.bat" > "LOG" 2>&1') rplc 'PATH';f;'LOG';logstd 45 | (cmd rplc '/';'\') fwrite f,'run.txt' 46 | case. 'Darwin' do. 47 | end. 48 | f 49 | ) 50 | -------------------------------------------------------------------------------- /server/jds_client/examples.ijs: -------------------------------------------------------------------------------- 1 | CONTEXT=: 'jbin jbin http u/p;' NB. default x 2 | msr'dropdb' 3 | msr'createdb' 4 | msr'createtable f' 5 | msr'createcol f a int' 6 | msr'insert f';'a';2 3 7 | msr'read from f' 8 | 'json json http u/p;' msr 'read from f' 9 | wget'read from f' 10 | wget'insert f';'a';6 11 | wget'read from f' 12 | curl'read from f' 13 | -------------------------------------------------------------------------------- /server/node/cert.napem: -------------------------------------------------------------------------------- 1 | -----BEGIN XXXXXX----- 2 | MIIDPjCCAiYCCQChs6/LGvY9VjANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQGEwJO 3 | QTELMAkGA1UECAwCTkExCzAJBgNVBAcMAk5BMQswCQYDVQQKDAJOQTELMAkGA1UE 4 | CwwCTkExCzAJBgNVBAMMAk5BMREwDwYJKoZIhvcNAQkBFgJOQTAeFw0yMTAxMTAx 5 | NzIwMzFaFw00ODA1MjcxNzIwMzFaMGExCzAJBgNVBAYTAk5BMQswCQYDVQQIDAJO 6 | QTELMAkGA1UEBwwCTkExCzAJBgNVBAoMAk5BMQswCQYDVQQLDAJOQTELMAkGA1UE 7 | AwwCTkExETAPBgkqhkiG9w0BCQEWAk5BMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A 8 | MIIBCgKCAQEAzwzBm6EG/QYYghbQw+euJEkVIncP82iOJewQEOTx7Pwcm/JR6yD+ 9 | tNSWBNMXWraQaaUBmfdY4oprcllnJg5xflzu318ggt6zRvVPSW95vjm12fUXIJAR 10 | qRg/uJKT9E4WyAQ4K7jGfAwsaWeW68FwZ66QqKbVaXn4QvG8c0DZ6nUYtZ4YL4FR 11 | PmTYfOuN7vd12zY6pSVJzm1861sN+5CeHsgR6qRZtZcVlgawg2Yn9c2lYpUsv9Aj 12 | lfVJy+QzozIyn9LgzpCq/xfqC/gpWohELleXAX1BMBQTEdxZ1F14s5tzXXexTStZ 13 | TdReRG0v+0bhtesDyEuk3tcmghaoo57fpwIDAQABMA0GCSqGSIb3DQEBCwUAA4IB 14 | AQAXd6XD1+3BrkntMJWF9C9kvZjtdl8pma1z9wsZf0KIGDhH6t0Ye5M03WLHNc7v 15 | kPZmYighz/o/azPIqFhOsjH7ci0rBHqwwdNZwWHjn/4KyY1IKewexBp/3a4sG0Lt 16 | t/9fWsoUEClQ6D8XtDvWwE97NxTQ2R7+LMjSArD90pSjE30KjTRbDpGyFjTPR0Fq 17 | PU1cfOYar5tsbndLnhvH4+VeSL9zaykOPF7+esRkyH1WXj8ulAcdNvbGo3xXUrGM 18 | B9vz45WU6kn9Lg3ZTZ8vP3YKqR8CgXrqArFmAL0zCH/Y+PkVGRgs7CDYeTkSTLs/ 19 | 5lu6EdwUxSLwgDhbtkAYSVcN 20 | -----END XXXXXX----- 21 | -------------------------------------------------------------------------------- /server/node/config.js: -------------------------------------------------------------------------------- 1 | // node jdserver config 2 | 3 | exports.port= 3000; // port to serve 4 | 5 | // services provided: name,host,port,dan 6 | exports.svc= 7 | '[\ 8 | ["s1" ,"s2" ,"s3" ,"s4"],\ 9 | ["localhost","localhost","localhost","localhost"],\ 10 | [65220 ,65220 ,65221 ,65221],\ 11 | ["jds_db_a" ,"jds_db_b" ,"jds_db_c" ,"jds_db_d"]\ 12 | ]' 13 | 14 | // valid user/pswd list 15 | exports.up= ["u/p","test/dummy"]; 16 | -------------------------------------------------------------------------------- /server/node/custom.ijs: -------------------------------------------------------------------------------- 1 | NB. jd_x... verbs for jdrt'node' 2 | 3 | jd_xdo=: 3 : 0 4 | try. 1 2$'result'; 9 | 10 | BODY 11 | ) 12 | 13 | jd_xget=: 3 : 0 14 | try. 15 | data=. ,LF,.~":i.>:?10 10 16 | body=. '

Report for URL: ',y,'

',(timestamp''),'

' 17 | body=. body,'
',data,'
' 18 | catch. 19 | body=. '
request failed: ',(13!:12''),'
' 20 | end. 21 | 1 2$'*text/html';custom_html rplc 'BODY';body 22 | ) -------------------------------------------------------------------------------- /server/node/jds.js: -------------------------------------------------------------------------------- 1 | // forward client request to jds server and return result (html or json) to client 2 | 3 | const http = require('http'); 4 | 5 | // p is string (json table) result from jds request 6 | // {\n"*... indicates key data is Content-Type ... and is returned to client 7 | // otherwise p is returned to client as text/plain 8 | function reply(code,res,p) 9 | { 10 | if('{\n"*'==p.substring(0,4)) // *text/html indicates content-type and string 11 | { 12 | var q= JSON.parse(p); 13 | var k = Object.keys(q); 14 | res.writeHead(code, "OK", {'Content-Type': k[0].substring(1)}); 15 | res.end(q[k[0]]); 16 | } 17 | else 18 | { 19 | res.writeHead(code, "OK", {'Content-Type': 'text/plain'}); 20 | res.end(p); 21 | } 22 | } 23 | 24 | async function jdsreq(up,host,port,dan,s,res) 25 | { 26 | s= "json json "+dan+" "+up+';'+s; 27 | let promise= dorequest(host,port,s); 28 | promise.then(good,bad); 29 | function good(data){reply(200,res,data);} 30 | function bad(data) {reply(200,res,data);} 31 | } 32 | 33 | exports.jdsreq= jdsreq; 34 | 35 | function dorequest(host,port,body){ 36 | return new Promise(function(resolve, reject) { 37 | i= body.indexOf('#'); 38 | if(i!=-1){body= body.substring(0,i)+"\n"+body.substring(i+1);} 39 | let options = {hostname: host,port: port,path: "/",method: "POST", 40 | headers: { 41 | "Content-Type": "application/json", 42 | "Content-Length": Buffer.byteLength(body) 43 | } 44 | } 45 | http 46 | .request(options, res => { 47 | var data = "" 48 | res.on("data", d => {data += d}) 49 | res.on("end", () => {resolve(data);}) 50 | }) 51 | .on("error", (error) => {reject(JSON.stringify(error));}) 52 | .end(body) 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /server/node/key.napem: -------------------------------------------------------------------------------- 1 | -----BEGIN XXXXXX----- 2 | MIIEogIBAAKCAQEAzwzBm6EG/QYYghbQw+euJEkVIncP82iOJewQEOTx7Pwcm/JR 3 | 6yD+tNSWBNMXWraQaaUBmfdY4oprcllnJg5xflzu318ggt6zRvVPSW95vjm12fUX 4 | IJARqRg/uJKT9E4WyAQ4K7jGfAwsaWeW68FwZ66QqKbVaXn4QvG8c0DZ6nUYtZ4Y 5 | L4FRPmTYfOuN7vd12zY6pSVJzm1861sN+5CeHsgR6qRZtZcVlgawg2Yn9c2lYpUs 6 | v9AjlfVJy+QzozIyn9LgzpCq/xfqC/gpWohELleXAX1BMBQTEdxZ1F14s5tzXXex 7 | TStZTdReRG0v+0bhtesDyEuk3tcmghaoo57fpwIDAQABAoIBADC/ah4ALasI8mA+ 8 | iux08bUtV8KDkcfXSBB5oda4cJVBTmMRDGR1okJlAuUj5g2/igS1isj7SXwc5utH 9 | TVfScrozmzIQLZFiaYc9XmuFyX4WDFDCua06/k13nhkQubD9abOx+p+go2q1Wwd+ 10 | EjRzmMlwejKbBQgOZXstexcPn77pYhxrPwdLfub6LaLRrEH8D41yHAxtCXcxbhX4 11 | YLdmNn4z+Y70iG9zsml6/5fVlpebO0LrtWlaU2HKRoC0+ssC1lsYT9upUpr9s3fF 12 | Y5mij3w0v+iDSLcsVpL80SDDpuWSc25a0rluWU11GBwovCYkppSS/NtxTZrbCVI5 13 | nC6bsZECgYEA+jjrfvUF0nOXIf7VzRnLUqKb3IUot8m543krI1DYWtzGf9D1ps9K 14 | 0gmcHnE0ToYbw+k2TkzkDaM3ePrauEg2GKfYnxjZb6hz4TGUk1J+MAYFyDVd5EzL 15 | s4kuiFWf8nlVPSzpHZSu3pLRVp2TdDDspWJAfojdiSE/yk0SFW0oiM0CgYEA09Sk 16 | Eu50F6eDV3FBRTOrdwwchC0K1yqDqcgWhV2W0SM9EfgARUQifWYIzFu06WCIlnrh 17 | gFwp5E5L7gigK411/WZlhMV1KWo2jyZcoNTvri0YCS1LCxYZqC5EkBEgeqBz64Q8 18 | 7l61+MrmRKcimKtQXlnx1ppprYHJVFrt+KWXWkMCgYBp8VekNVH1vQMsXZ1gbZ9s 19 | qXScdbS7vCl8an0dQsmYDwnsbCjWK8ZeD7uOGBgxdw6icEeR93RHYefGzU913v0o 20 | i0hodDj70bLekgN6LN9IrgQctJvAYdONfHO7PaWvIDg64p0A7Ubs7fS8W5NnCp/n 21 | liYAbSp/ORitKtyRPdp/sQKBgBg6kim4LH8oIdW2loibyFjtgbV8kWja7ARXZnwK 22 | oF17cljDzQQfLvK44Pi+5IptCDJuC6zq3b/HVNf9FJTmsUAynhn5m9UI2QS2lpGu 23 | yVtavQ0PqS2zcWvTHvR7gQOFpryhe3aoTpgOXRgtJb/gmGiyUP8C2mOGHMkRPXbb 24 | OdyRAoGAd9PRu2epxKNPQyC2DLuLESlh/aJ2IX/vmZOc02PDK1hKx606aFi5SQH6 25 | hL/sQ18gV2PnCAmVT4IXRih0dlRKtjB9teOwi+Q5a4Jp5Z/Y9TUWzZ+1dFgTdaqm 26 | /ajDNO+axg7mxMqgE6VkqW83mktnd3mJjuTW01b+6PGimCtaX5Q= 27 | -----END XXXXXX----- 28 | -------------------------------------------------------------------------------- /server/node/node_tools.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | NB. node server tools 3 | 4 | require JDP,'server/port.ijs' 5 | 6 | NB. PATH;PORT;NODEBIN 7 | NB. create node files in folder PATH/PORT 8 | create_node=: 3 : 0 9 | 'path port nodebin'=. y 10 | sport=. ":port 11 | p=. jpath path,('/'#~'/'~:{:path),'node/',sport,'/' 12 | mkdir_j_ p 13 | 14 | if. IFWIN do. 15 | NB. create run.bat and run.txt 16 | t=. '"NODEBIN" "JS" "CONFIG"' 17 | t=. t rplc 'NODEBIN';nodebin;'JS';(p,'server.js');'CONFIG';p,'config.js' 18 | f=. p,'run.bat' 19 | r=. t fwrite p,'run.bat' 20 | pw=. hostpathsep p 21 | ('"PATHrun.bat" > "LOG" 2>&1' rplc 'PATH';pw;'LOG';pw,'logstd.log') fwrite p,'run.txt' 22 | else. 23 | NB. create run.sh and run.txt 24 | t=. '#!/bin/bash' 25 | t=. t,LF,'"NODEBIN" "JS" "CONFIG"' 26 | t=. t rplc 'NODEBIN';nodebin;'JS';(p,'server.js');'CONFIG';p,'config.js' 27 | f=. p,'run.sh' 28 | r=. t fwrite f 29 | shell'chmod +x "',f,'"' 30 | ('nohup "PATHrun.sh" > "LOG" 2>&1' rplc 'PATH';p;'LOG';p,'logstd.log') fwrite p,'run.txt' 31 | end. 32 | 33 | NB. create cert.pem and key.pem - mangled to avoid github warning 34 | ((fread JDP,'server/node/cert.napem')rplc 'XXXXXX';'CERTIFICATE') fwrite p,'cert.pem' 35 | ((fread JDP,'server/node/key.napem') rplc 'XXXXXX';'RSA PRIVATE KEY') fwrite p,'key.pem' 36 | 37 | NB. copy node js source files 38 | n=. 'server.js';'jds.js';'config.js';'server.html' 39 | d=. fread each (3#<'1990-12-06' 26 | jd'createtable /types /pairs f';,jd'read /types from g' [fd'' 27 | assert (jd'read from f')-:jd'read from g' 28 | 29 | jd'createtable /pairs f';'a';23;'b';3 3$'abcdefghi' [fd'' 30 | 31 | -------------------------------------------------------------------------------- /test/csvcdefs_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | 3 | NB. csvdefs set cdefs types from data 4 | 5 | 0 : 0 6 | dateX 7 | all dateX fields must be of same format 8 | 9 | 8 20001212 10 | 10 2000-12-12 11 | 12 | 8 12122000 13 | 10 12-12-2000 14 | 15 | 14 20001212151515 16 | 19 2000-12-12-15-15-15 17 | 18 | 14 12122000151515 19 | 19 12-12-2000-15-15-15 20 | ) 21 | 22 | NB. byte abc;xxxxxxxxxxxxxxxxxxxxxxxxxxxxx 23 | 24 | 25 | tests=: <;._2 [0 : 0 26 | varbyte abc;xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 27 | byte_30 abc;xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 28 | byte ;;; 29 | byte ; ;; 30 | boolean 0;1;0;1 31 | boolean 0;1;1;0;true;false;TRUE;FALSE 32 | int 2;+23;-123 33 | float 2;2.3;+2.3;2.3e7;-2.3E7;.23 34 | int 20001212;20001213 35 | int 12122000;12132000 36 | int 20001212101010;20001213101010 37 | int 12122000101010;12132000101010 38 | edate 2001-10-10;2002-12-12 39 | edatetime 2001-10-10T10;2002-12-13T10:10:10 40 | edatetimem 2001-10-10T10:10:10.123;2002-12-13T10:10:10.123 41 | edatetimen 2001-10-10T10:10:10.1234;2002-12-13T10:10:10.1234 42 | ) 43 | 44 | run=: 3 : 0 45 | r=. '' 46 | a=. y 47 | for_i. i.#a do. 48 | n=. ;i{a 49 | b=. n i. ' ' 50 | type=. b{.n 51 | d=. dlb b}.n 52 | d=. <;._1 ';',d 53 | d=. >,each d 54 | t=. 40 gettypex_jd_ d 55 | t=. t rplc ' ';'_' 56 | if. -.type-:t do. echo 'failed: ',(2":i),' - ',n end. 57 | r=. r,type-:t 58 | end. 59 | assert r 60 | ) 61 | 62 | run'' 63 | 64 | CSVFOLDER=: jpath '~temp' 65 | fcsv=: CSVFOLDER,'/t1.csv' 66 | fcdefs=: CSVFOLDER,'/t1.cdefs' 67 | 68 | csv=: 0 : 0 69 | name,num,price,text 70 | "cog",30,12.34 71 | "nut",,18.75,jkl mno 72 | "pin",32,,qewrqwerqewr 73 | ) 74 | 75 | csv fwrite fcsv 76 | 77 | rd=: 3 : 0 78 | 'new' jdadmin 'test' 79 | jd 'csvcdefs /replace /h 1 t1.csv' 80 | jd 'csvrd t1.csv tab' 81 | jd 'reads /types from tab' 82 | ) 83 | 84 | assert ('name(byte 3)';'num(int)';'price(float)';'text(byte 12)')-:{.rd'' -------------------------------------------------------------------------------- /test/delete_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2017, Jsoftware Inc. All rights reserved. 2 | 3 | jdadminx'test' 4 | jd'gen test f 6' 5 | t=: jdgl_jd_'f' 6 | assert(i.6)-:,'x'jdfroms_jd_ jd'reads from f' 7 | assert 6=Tlen__t 8 | jd'delete f';'x=3' 9 | assert(3-.~i.6)-:,'x'jdfroms_jd_ jd'reads from f' 10 | assert 5=Tlen__t 11 | jd'delete f';'x>1' 12 | assert(0 1)-:,'x'jdfroms_jd_ jd'reads from f' 13 | assert 2=Tlen__t 14 | -------------------------------------------------------------------------------- /test/drop_rename_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | 3 | NB. test droptable dropcol renametable renamecol 4 | 5 | test=: 3 : 0 6 | jdadminx'test' 7 | jd'createtable f' 8 | jd'createcol f a int _';i.5 9 | jd'createcol f b int _';i.5 10 | jd'createtable g' 11 | jd'createcol g a int _';i.5 12 | jd'createcol g b int _';i.5 13 | ) 14 | 15 | NB. droptable 16 | test'' 17 | jd'droptable f' 18 | 19 | test'' 20 | jd'ref f a g a' 21 | assert 'domain error'-:jd etx'droptable f' 22 | 23 | NB. dropcol 24 | test'' 25 | jd'dropcol f a' 26 | 27 | test'' 28 | jd'ref f a g a' 29 | 'domain error'-:jd etx'dropcol f a' NB. fails with ref 30 | 31 | NB. renametable 32 | test'' 33 | jd'renametable f foo' 34 | t=. jdgl_jd_'foo' 35 | assert 'foo'-:NAME__t 36 | jd'renametable foo boo' 37 | t=. jdgl_jd_'boo' 38 | assert 'boo'-:NAME__t 39 | 40 | test'' 41 | jd'renametable f foo' NB. works with hash 42 | jd'renametable g goo' 43 | 44 | test'' 45 | jd'ref f a g a' 46 | 'domain error'-:jd etx'renametable f foo' NB. fails with ref 47 | 48 | NB. renamecol 49 | test'' 50 | jd'renamecol f a aaa' 51 | jd'renamecol f aaa x' 52 | jd'renamecol f x aaaa' 53 | jd'reads aaaa from f' 54 | 55 | 'jd'jdae'renamecol f jdindex adsf' 56 | 'jd'jdae'renamecol f aaaa jdadsf' 57 | 'not found'jdae'renamecol f asdf qewr' 58 | 'already exists'jdae'renamecol f aaaa b' 59 | 60 | test'' 61 | jd'ref f a g a' 62 | assert'domain error'-:jd etx'renamecol f a aaa' NB. fails with ref 63 | 64 | NB. test drop jdref col 65 | jdadminx'test' 66 | jd'gen ref2 f 6 3 g 3' 67 | assert (,'f')-:,;{.{:jd'info ref' 68 | jd'reads from f,f.g' 69 | jd'dropcol f jdref_aref_g_bref' 70 | assert ''-:,;{.{:jd'info ref' 71 | 'not find'jdae'reads from f,f.g' 72 | 73 | jdadminx'test' 74 | jd'gen ref2 f 6 3 g 3' 75 | jd'ref g bref f akey' 76 | assert 'fg'-:,;{.{:jd'info ref' 77 | jd'reads from f,f.g' 78 | jd'dropcol f jdref_aref_g_bref' 79 | assert (,'g')-:,;{.{:jd'info ref' 80 | 'not find'jdae'reads from f,f.g' 81 | jd'dropcol g jdref_bref_f_akey' 82 | assert ''-:,;{.{:jd'info ref' 83 | 'not find'jdae'reads from g,g.f' 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /test/droptable_test.ijs: -------------------------------------------------------------------------------- 1 | jdadminx'test' 2 | 3 | NB. droptable 4 | jd'gen test f 3' 5 | jd'droptable f' 6 | jd'gen test f 3' 7 | jd'gen test f 3' 8 | jd'gen test g 3' 9 | jd'ref f int g int' 10 | 'ref'jdae'droptable f' 11 | 'ref'jdae'droptable g' 12 | jd'dropcol f ',,;{:{:jd'info ref' 13 | jd'droptable f' 14 | jd'droptable g' 15 | 16 | NB. droptable /reset 17 | jd'gen test f 0' 18 | d0=: jd'read from f' 19 | jd'gen test f 3' 20 | d3=: jd'read from f' 21 | jd'droptable /reset f' 22 | assert d0-:jd'read from f' 23 | jd'insert f';,d3 24 | assert d3-:jd'read from f' 25 | 26 | jd'gen test g 3' 27 | jd'ref f int g int' 28 | 'ref'jdae'droptable f' 29 | 'ref'jdae'droptable g' 30 | 'ref'jdae'droptable /reset f' 31 | 'ref'jdae'droptable /reset g' 32 | 33 | jd'dropcol f ',,;{:{:jd'info ref' 34 | jd'droptable /reset f' 35 | assert d0-:jd'read from f' 36 | 37 | NB. ptable 38 | jdadminx'test' 39 | jd'createtable f' 40 | jd'createcol f set int' 41 | jd'createcol f a int' 42 | jd'createptable f set' 43 | jd'insert f';'set';(6$2012 2013);'a';i.6 44 | jd'droptable f' 45 | assert 0=#dir'~temp/jd/test/f*' 46 | -------------------------------------------------------------------------------- /test/epoch_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2019, Jsoftware Inc. All rights reserved. 2 | 3 | NB. verify extra precision is dropped 4 | jdadminx'test' 5 | jd'createtable f' 6 | jd'createcol f edt edatetime' 7 | jd'insert f edt';'2014-09-10T00:00:00,000000000' 8 | jd'insert f edt';'2014-09-10T01:01:01,000000000' 9 | jd'insert f edt';'2014-09-10T01:01:01,123400000' 10 | jd'insert f edt';'2014-09-10T01:01:01,123456789' 11 | jd'reads from f' 12 | assert ( ;{:{:jd'read /e from f')-:463622400000000000 463626061000000000 463626061000000000 463626061000000000 13 | 14 | jdadminx'test' 15 | jd'createtable f' 16 | jd'createcol f e edate' 17 | jd'insert f e';'2000' 18 | jd'insert f e';'2000' 19 | jd'reads from f' 20 | jd'insert f e';0 21 | jd'insert f e';'1700' NB. bad allowed from sfe 22 | jd'insert f e';IMIN_jd_ NB. bad allowed 23 | 24 | NB. bad allowed in csv 25 | CSVFOLDER=: '~temp/jd/csv' 26 | jd'csvwr f.csv f' 27 | jd'csvrd f.csv g' 28 | -------------------------------------------------------------------------------- /test/error_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | NB. tests for expected errors 3 | NB. in general other test scripts do not test for expected errors 4 | 5 | jdadminx'test' 6 | jd'createtable f a int,b int' 7 | 8 | 'duplicate'jdae'createtable f a int,a int' 9 | 10 | 'duplicate'jdae'insert f';'a';2;'b';3;'a';4;'b';5 11 | 'unknown' jdae'insert f qqq';23 12 | 'missing' jdae'insert f';'a';23 13 | 14 | 'duplicate'jdae'update f';'a=0';'a';2;'a';3 15 | 'unknown' jdae'update f';'a=0';'qqq';23 16 | 17 | 18 | NB. csv 19 | CSVFOLDER=: F=: '~temp/jd/csv/junk/' 20 | jdadminx'test' 21 | jdcreatefolder_jd_ F[jddeletefolder_jd_ F 22 | ('2,3',LF) fwrite F,'a.csv' 23 | ('abc',LF,'abc',LF) fwrite F,'a.cnames' 24 | 'not found'jdae'csvrd a.csv a' 25 | 'duplicate'jdae'csvcdefs /replace /h 0 /c a.csv' 26 | -------------------------------------------------------------------------------- /test/fixpairs_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2017, Jsoftware Inc. All rights reserved. 2 | NB. test scalar extension 3 | jdadminx'test' 4 | jd'createtable f a int,b byte 4' 5 | 6 | jd'insert f';'a';(0);'b';'b' 7 | jd'insert f';'a';(,1);'b';,'b' 8 | 9 | jd'insert f';'a';2 3;'b';'b' 10 | jd'insert f';'a';4 5;'b';,'b' 11 | jd'insert f';'a';6 7;'b';'abc' 12 | jd'insert f';'a';8 9;'b';1 3$'abc' 13 | jd'insert f';'a';10 11;'b';2 3$'abc' 14 | 15 | jd'insert f';'a';12 13;'b';'abcd' 16 | jd'insert f';'a';14 15;'b';2 4$'abcd' 17 | 18 | 'bad shape'jdae'insert f';'a';12 13;'b';'abcde' 19 | 'bad shape'jdae'insert f';'a';14 15;'b';2 5$'abcde' 20 | jd'reads from f' 21 | 22 | assert (i.16)=,'a'jdfroms_jd_ jd'reads from f' 23 | assert((6#,:'b '),(6#,:'abc'),4#,:'abcd')-:'b'jdfroms_jd_ jd'reads from f' 24 | 25 | jdadminx'test' 26 | jd'createtable g a int' 27 | 'shape'jdae'insert g';'a';i.2 3 4 28 | -------------------------------------------------------------------------------- /test/flush.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | 3 | NB. test host flush of memory mapped dirty pages 4 | 5 | jd'close' 6 | jdadminx'test' 7 | jd'createtable f' 8 | jd'createcol f a int _';i.10e6 9 | 10 | jd'flush' NB. may have already been flushed if createcol does flush 11 | c=: jdgl_jd_'f a' 12 | dat__c=: >:dat__c NB. lots of dirty pages 13 | t=: timex 'jd''flush''' 14 | 'dirty buffer flush too fast'assert (t>0.1)+.UNAME-:'Darwin' NB. mac does not wait for flush to finish 15 | -------------------------------------------------------------------------------- /test/info_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | 0 : 0 3 | agg - select aggregators 4 | last - cmd time space 5 | schema [table [column]] - normal cols (non jd...) 6 | summary [table] 7 | table 8 | validate [table][column]] 9 | validatebad [table][column]] 10 | varbyte [table [column]] 11 | ref [table] - ref cols 12 | ) 13 | 14 | jdadminx'test' 15 | jd'gen test gg 3' 16 | jd'gen test f 3' 17 | jd'ref f x gg x' 18 | jd'ref f boolean gg boolean' 19 | 20 | f=: 3 : '#;{."1 {:r=:jd ''info '',y' 21 | 22 | assert 8=f'agg' 23 | 24 | assert 1=f'last' 25 | 26 | assert 16=f'schema' 27 | assert 8=f'schema gg' 28 | assert 1=f'schema f float' 29 | 30 | assert 2=f'summary' 31 | assert 1=f'summary gg' 32 | 33 | assert 2=f'table' 34 | 35 | assert 20=f'validate' 36 | assert 0=f'validatebad' 37 | 38 | assert 2=f'varbyte' 39 | assert 1=f'varbyte gg' 40 | jd'close' 41 | assert 2=f'varbyte' 42 | assert 1=f'varbyte gg' 43 | 44 | assert 2=f'ref' 45 | jd'close' 46 | assert 2=f'ref' NB. verify works without mapping 47 | 48 | jd'dropcol f jdref_x_gg_x' 49 | jd'dropcol f jdref_boolean_gg_boolean' 50 | 51 | t=. jdgl_jd_'f' 52 | assert 3=Tlen__t 53 | 54 | jd'delete f';'jdindex=1' 55 | assert 2=Tlen__t 56 | 57 | jd'insert f';,jd'read from f' 58 | assert 4=Tlen__t 59 | 60 | jd'droptable /reset f' 61 | t=. jdgl_jd_'f' 62 | assert 0=Tlen__t 63 | 64 | NB. ptable 65 | require JDP,'tools/ptable.ijs' 66 | ptablebld'int' 67 | assert 3=f'summary' 68 | assert 1=f'summary f' 69 | assert ((,'f');,.27)={:jd'info summary f' 70 | assert 2=ttally_jd_ {:jd'reads from f where p=2016 and p2=1' 71 | jd'delete f';'p=2016 and p2=1' 72 | assert ((,'f');,.25)={:jd'info summary f' 73 | assert (}.{:jd'info schema ptab')-:}.{:jd'info schema f' 74 | assert (}.{:jd'info schema f',PTM_jd_,'2016')-:}.{:jd'info schema f' 75 | assert 0 6 3 1 2 5 10 4-:,;}.{:jd'info summary f',PTM_jd_,'*' 76 | assert 3=f'table' 77 | assert 8=f'table f',PTM_jd_,'*' 78 | -------------------------------------------------------------------------------- /test/insert_revert_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2017, Jsoftware Inc. All rights reserved. 2 | FORCEREVERT_jd_=: 0 3 | jdadminx'test' 4 | jd'gen test f 4' 5 | a=. jd'read from f' 6 | FORCEREVERT_jd_=: 1 7 | 'insert failed'jdae'insert f';,a 8 | jdrepair_jd_'fixing it now' 9 | repair_jd_'' 10 | jddamage_jd_'' 11 | assert 4='count'jdfroms_jd_ jd'reads count:count x from f' 12 | jdadmin 0 13 | jdadmin'test' 14 | assert 4='count'jdfroms_jd_ jd'reads count:count x from f' 15 | -------------------------------------------------------------------------------- /test/insert_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | bldnums=: 3 : 0 3 | jdadminx'test' 4 | jd'createtable';T;'a boolean,b int,c float' 5 | ) 6 | 7 | ins=: 3 : 0 8 | jd'insert';T;,N,.y 9 | ) 10 | 11 | insx=: 4 : 0 12 | x jdae 'insert';T;,N,.y 13 | ) 14 | 15 | N=: ;:'a b c' 16 | T=: 'num' 17 | 18 | bldnums'' 19 | ins 0;2;2.2 20 | 1=;{:,jd'reads count a from ',T 21 | 22 | ins 0 1;4 5;4.2 4.3 23 | 3=;{:,jd'reads count a from ',T 24 | 25 | 26 | N=: ;:'a c' 27 | 'missing' insx 0 1;4.4 5 28 | 29 | N=: ;:'a b c c' 30 | 'duplicate' insx 0 1;4 5;4.2 4.3;4.4 5.4 31 | 32 | N=: ;:'a b c d e' 33 | 'unknown' insx 0 1;4 5;4.2 4.3;4 4;23 34 | 35 | 36 | N=: ;:'a b c' 37 | 'count' insx 0 1 ; 4 5 6 ; 4.2 4.3 38 | ins 0 1 ; 4 5 ; 4.2 4.3 39 | 'data' insx 0 ; 2.5 ; 2.2 40 | 41 | d =: 201212010 42 | t =: 101010 43 | dt=: 20121010111111 44 | 45 | blddt=: 3 : 0 46 | jdadminx'test' 47 | jd'createtable';'dt';'a datetime,b date,c int' NB. time 48 | ) 49 | 50 | N=: ;:'a b c' 51 | T=: 'dt' 52 | blddt'' 53 | ins 1 ; 2 ; 3 54 | ins 1 1 ; 2 2 ; 3 3 55 | 3=;{:,jd'reads count a from dt' 56 | 57 | blddt2=: 3 : 0 58 | jdadminx'test' 59 | jd'createtable';'dt2';'a datetime,b datetime' 60 | ) 61 | 62 | N=: ;:'a b' 63 | T=: 'dt2' 64 | blddt2'' 65 | ins 2;23 66 | 67 | N=: ;:'a b c' 68 | T=: 'ch' 69 | bldch=: 3 : 0 70 | jdadminx'test' 71 | jd'createtable';'ch';'a byte,b byte 2,c varbyte' 72 | ) 73 | bldch'' 74 | ins 'a' ; (1 2$'ab') ; <<'abd' 75 | ins 'ab' ; (2 2$'ab') ; <'abd';'qwert' 76 | 77 | jd'reads from ch' 78 | assert 5=;{:,jd'reads count a from ch' 79 | 80 | ins 'ab' ; (1 2$'ab') ; <<'abd' 81 | ins 'a' ; (2 2$'ab') ; <<'abd' 82 | 'shape' insx 'ab' ; (2 3$'ab') ; <'abd';'qwert' 83 | 84 | N=: ;:'a b' 85 | T=: 'chx' 86 | bldchx=: 3 : 0 87 | jdadminx'test' 88 | jd'createtable';T;'a byte,b byte 2' 89 | ) 90 | bldchx'' 91 | ins 'a' ; (1 2$'ab') 92 | ins 'ab' ; (2 2$'ab') 93 | 94 | jd'reads from chx' 95 | 3=;{:,jd'reads count a from chx' 96 | 97 | ins 'ab' ; 1 2$'ab' 98 | ins 'a' ; 2 2$'ab' 99 | ins 'ab' ; 1 2$'ab' 100 | 'shape' insx 'ab' ; 2 3$'ab' 101 | -------------------------------------------------------------------------------- /test/jdindex_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | NB. Test queries on the jdindex column. 3 | 4 | jdadminx 'test' 5 | 6 | jd 'createtable t index int' 7 | jd 'insert t index';i.10 8 | 9 | test =: 'jdindex'&, -:&jd&('read from t where '&,) 'index'&, 10 | 11 | NB. Ignore the first test (equality with negative index) 12 | assert *./ }. , test@:;@> { ' = < <= > >=' ,&<&(<;.1) ' -3 6 12' 13 | 14 | assert test ' range 1,4,6,7' 15 | assert test ' range -2,4,6,12' 16 | assert test ' range 4,4,8' 17 | -------------------------------------------------------------------------------- /test/jds_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | 3 | 0 : 0 4 | jds is a cover on jd that has extra parameters 5 | it is intended for use by an http client 6 | see jds=: defn in client.ijs for details 7 | ) 8 | 9 | jsonok=: }:0 : 0 10 | { 11 | "Jd OK":0 12 | } 13 | ) 14 | 15 | show=: echo 16 | show=: [ NB. use echo to see stuff 17 | 18 | 'new'jdadmin'test' 19 | assert jsonok-:jds'json json test u/p;createtable f' 20 | assert jsonok-:jds'json json test u/p;createcol f a int' 21 | assert jsonok-:jds'json json test u/p;createcol f b byte' 22 | assert jsonok-:jds'json json test u/p;createcol f c byte 4' 23 | assert jsonok-:jds'json json test u/p;insert f',LF,jsonenc 'a';23;'b';'x';'c';'asdf' 24 | assert (jd'read from f')-:>each jsondec jds'json json test u/p;read from f' 25 | assert +./'Jd server error'E. jds'json json test u/p;reads from f' 26 | 27 | tests=: <;._2 [ 0 : 0 28 | XXXXdec jds'XXXX XXXX test u/p;read from f' 29 | XXXXdec jds'XXXX XXXX test u/p;info summary' 30 | XXXXdec jds'XXXX XXXX test u/p;insert f',LF,XXXXenc'a';666;'b';'q';'c';'qewr' 31 | XXXXdec jds'XXXX XXXX test u/p;insert f',LF,XXXXenc'a';5 6 6;'b';'fgh';'c';3 4$'qewrasdfzxcv' 32 | XXXXdec jds'XXXX XXXX test u/p;insert f',LF,XXXXenc 'a';666 33 | XXXXdec jds'XXXX XXXX test u/p;update f',LF,XXXXenc 'a=666';'a';777 34 | XXXXdec jds'XXXX XXXX test u/p;infox summary' 35 | XXXXdec jds'XXXX XXXX test u/p;info xsummary' 36 | XXXXdec jds'XXXX XXXX test u/p;list version' 37 | XXXXdec jds'XXXX XXXX test u/p;csvreport' 38 | ) 39 | 40 | run=: 3 : 0 41 | for_n. y do. 42 | show ' ',t=. (;n) rplc 'XXXX';'jbin' 43 | show a=. ".t 44 | show ' ',t=. (;n) rplc 'XXXX';'json' 45 | show b=. ".t 46 | 'jbin and json differ' assert (a-:b)+.(a-:>each b)+.(a-:>each |:b)+.'Jd server error'-:;{.{.a 47 | end. 48 | ) 49 | 50 | run tests 51 | -------------------------------------------------------------------------------- /test/join_type_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2018, Jsoftware Inc. All rights reserved. 2 | 3 | NB. tests that might help understand join types and how they work 4 | 5 | bld=: 3 : 0 6 | d=. i.3 7 | 'new'jdadmin'join' 8 | jd'createtable f' 9 | jd'createcol f a int' 10 | jd'insert f';'a';>:d,d 11 | jd'createtable g' 12 | jd'createcol g b int' 13 | jd'insert g';'b';>:i.3 14 | ) 15 | 16 | show=: 3 : 0 17 | c=. jdgl_jd_ 'f jdref_a_g_b' 18 | if. 2=#dat__c do. 19 | end. 20 | i.0 0 21 | ) 22 | 23 | bld'' 24 | jd'ref f a g b' 25 | show'' 26 | 27 | jd'insert f';'a';4 28 | show'' 29 | 30 | jd'insert g';'b';5 31 | show'' 32 | 33 | jd'insert f';'a';5 34 | show'' 35 | 36 | bld'' 37 | jd'ref /left f a g b' 38 | 39 | show'' 40 | 41 | jd'insert f';'a';4 42 | show'' 43 | 44 | jd'insert g';'b';5 45 | show'' 46 | 47 | 48 | -------------------------------------------------------------------------------- /test/joinorder_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | NB. Test that joins are performed in an optimal order 3 | 4 | 0 : 0 5 | this test has serious problems 6 | ref /left join does not work with 3 tables 7 | f,f.g,g.h - fails 8 | f,f-g,g-h - fails 9 | joinorder stuff doesn't seem to give any real benefit 10 | ) 11 | 12 | tc=: 4 : 0 13 | jd'droptable';x 14 | jd'createtable';x;'a int' 15 | jd'insert';x;'a';y 16 | ) 17 | 18 | NB. y is a list of boxed integer lists. 19 | NB. For each make a table with the list in column a, and link them in order. 20 | NB. Table names are f,g,... 21 | tcs =: 3 : 0 22 | jdadminx'test' 23 | getname =. a.{~ (a.i.'f')&+ 24 | for_i. i.#y do. 25 | (getname i) tc i{::y 26 | if. i>0 do. jd 'ref ',;:^:_1 , (getname&.> (,~<:)i),.<'a' end. 27 | end. 28 | ) 29 | 30 | spjoin =: [: 7!:2@> (' ';'exact ') ('jd ',[:quote'read from ',,)&.> < 31 | 32 | tcs 3#<2#i.1e4 33 | NB. assert 4 < %~/ spjoin 'f,f.g,g.h where h.a<100' 34 | NB. assert 10< %~/ spjoin 'f,f.g,g.h where h.a<100' 35 | NB. assert 6< %~/ spjoin 'f,f.g,g.h where h.a<100' 36 | NB. assert 4 < %~/ spjoin 'f,f-g,g-h where h.a<100' 37 | NB. assert 10< %~/ spjoin 'f,f-g,g-h where h.a<100' 38 | NB. assert 6< %~/ spjoin 'f,f-g,g-h where h.a<100' 39 | 40 | tcs 500 10 1 #&.> 3#a 12 | ) 13 | 14 | bld=: 3 : 0 15 | jdadminx'test' 16 | jd'createtable f' 17 | jd'createcol f a int _';i.3 18 | p=: jdpath_jd_'' 19 | plog=: p,'log.txt' 20 | ) 21 | 22 | bld'' 23 | 24 | FEER_jd_=: 'logijf test' 25 | 0 logijf_jd_ 'fubar' 26 | assert 1=#jdlogijfshow_jd_'' 27 | a=. chk 0 28 | assert 'fubar'-:;{:{.a 29 | assert 'logijf test'-:;{:(({."1 a)i.<'FEER'){a 30 | assert ''-:;;{:{:a 31 | assert +./'logijf' E. fread plog 32 | 33 | 34 | 0 logijf_jd_ 'fubar';'asdf' 35 | a=. chk 1 36 | assert 'asdf'-:;;{:{:a 37 | 38 | 0 logijf_jd_ 'fubar';'asdf';123 39 | a=. chk 2 40 | assert (<'asdf';123)-:{:{:a 41 | 42 | assert 'assertion failure'-:logijfdamage_jd_ etx 'fubar' 43 | assert de-:(#de){.}.13!:12'' 44 | 45 | NB. logtxt 46 | LOGOPS_jd_=: 1 47 | jdadminx'test' 48 | jd'createtable f' 49 | jd'createcol f a int' 50 | assert 2=+/LF=fread plog 51 | 52 | 'fubar'logtxt_jd_ 'adsf';i.3 4 53 | jd'insert f a';i.20000 54 | assert 7=+/LF=fread plog 55 | LOGOPS_jd_=: 0 56 | 57 | bld'' 58 | c=. jdgl_jd_'f a' 59 | abc=: dat__c NB. bump refcount 60 | NB. 'failed'jdae'reads from f' 61 | erase'abc' NB. kill the refs time bomb 62 | NB. assert +./'refs' E. fread plog 63 | jd'reads from f' NB. logtxt but not damaged 64 | 65 | bld'' 66 | 'assertion failure'-:jddamage_jd_ etx'really messed up' 67 | jddamage_jd_'' 68 | assert +./'really messed up' E. fread plog 69 | -------------------------------------------------------------------------------- /test/multicol_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | NB. Test multicolumn queries. 3 | 4 | NB. following is from old core tests 5 | TYPES_CORE=. TYPES_jd_-.;:'edate edatetime edatetimem edatetimen int1 int2 int4' NB. core tests not adapted to new datatypes 6 | 7 | toloc =. 'jdt'&,&.> 8 | 9 | NB. x is type, y is length 10 | gen =: 4 :0 11 | ('r_',({.~i.&' ')x)~ y , ".(}.~i.&' ')x 12 | ) 13 | 14 | NB. random generators 15 | NB. y is shape 16 | r_boolean =: ?@$&2 17 | r_int =: ?@$&1e5 18 | r_float =: 1e5 * ?@$&0 19 | r_byte =: a. {~ 97+?@$&26 20 | r_enum =: r_byte 21 | r_varbyte =: [:r_byte&.> ?@$&10 22 | r_date =: 1e7 + ?@$&9e7 23 | r_datetime=: 1e13 + ?@$&9e13 24 | r_time =: 1e5 + ?@$&9e5 25 | 26 | NUMERIC =: (#~ ((<'jdtnumeric') e. copath@toloc)"0) TYPES_CORE 27 | isnum =: NUMERIC e.~ <@:({.~i.&' ') 28 | 29 | ind =: (2#0,i.5) , (3#i.4) ,: (6#5 6) 30 | cols =: ('c',":)&.> i.3 31 | 32 | read =: [: >@{:"1@jd 'read ' , ] 33 | read1=: [: read 'from a' , (' where ' #~ *@#) , ] 34 | 35 | testtype =: 3 : 0 36 | arg=. y 37 | jdadminx 'test' 38 | jd 'createtable';'a'; ,&(' ',arg)&.> cols 39 | jd 'insert';'a';, cols ,. <"_1 dat=. ind { arg gen 7 40 | testqueries arg;/(>@:) 66 | toq =. ([: ; '"' (,,[)&.:>"_1 boxopen"1@])`(":@]) @. (isnum@[) 67 | toq_m =. [: l@, ([: <@('(',')',~l)^:(1<#) y&toq(<@)"_1)"_1 68 | q =. [: read1 ([:l cols{~[),(('=';' in '){::~1<#@]),[:toq_m [:|:dat{~<@; 69 | r =. dat #"1~ ] (]e.{) [:|:dat{~[ 70 | testinds =. (r -: q)&, 71 | assert testinds&>/"1 INDS 72 | 73 | 9!:11 old 74 | ) 75 | 76 | testtype@> TYPES_CORE -. <'varbyte' 77 | -------------------------------------------------------------------------------- /test/names_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | NB.test db, dan, table, col name validation 3 | 4 | 'new'jdadmin'jdtest' NB. jd prefix allowed in DB 5 | e=. 'invalid name' 6 | 7 | e jdae'createtable a/b' 8 | e jdae'createtable ab',1{a. 9 | e jdae'createtable "ab"' 10 | e jdae'createtable "a b"' 11 | e jdae'createtable from' 12 | e jdae'createtable jdfoo' 13 | 14 | jd'createtable f' 15 | e jdae'createcol f jdfoo int' 16 | e jdae'createcol f foo',(0{a.),' int' 17 | -------------------------------------------------------------------------------- /test/orderby_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | 3 | NB. copied from tutorial reads with asserts added 4 | jdadmin'sandp' 5 | r=. jd 'reads weight,city from p order by weight,city' 6 | assert 12 12 14 17 17 19-:,'weight'jdfroms_jd_ r 7 | assert 'LPLOPL'-:{."1 'city'jdfroms_jd_ r 8 | 9 | r=. jd 'reads weight,city from p order by weight,city desc' 10 | assert 12 12 14 17 17 19-:,'weight'jdfroms_jd_ r 11 | assert'PLLPOL'-:{."1 'city'jdfroms_jd_ r 12 | 13 | r=. jd 'reads weight,city from p order by weight desc,city' 14 | assert 19 17 17 14 12 12-:,'weight'jdfroms_jd_ r 15 | assert'LOPLLP'-:{."1 'city'jdfroms_jd_ r 16 | 17 | r=. jd 'reads weight,city from p order by weight desc,city desc' 18 | assert 19 17 17 14 12 12-:,'weight'jdfroms_jd_ r 19 | assert'LPOLPL'-:{."1 'city'jdfroms_jd_ r 20 | 21 | 22 | -------------------------------------------------------------------------------- /test/pm_aggby_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | 3 | NB. aggregate by timings - thanks to Joe Bogner database email 4 | NB. compare aggregate by with J 5 | NB. hash does not help Jd performance 6 | 7 | jdadminx 'timing' 8 | jd'createtable t';'a int';'b byte 8';'c byte 4';'d byte 12' 9 | N=:20e6 5e6{~IFIOS+.IFRASPI+.UNAME-:'Android' NB. arm64 10 | group1=.10 8 $ 8 # a. {~ 97+(i.10) 11 | group1v=. group1 {~ ?. N#10 12 | group2=.10 4 $ 4 # a. {~ 97+(i.5) 13 | group2v=. group2 {~ ?. N#5 14 | jd'insert t';'a';(N#1);'b';group1v;'c';group2v;'d';group1v,.group2v 15 | 16 | tbc=: 6!:2 'jdresult=:jd ''reads sum a by b,c from t''' 17 | td=: 6!:2 'r=: jd ''reads sum a by d from t''' 18 | assert ({:"1{:jdresult)-:{:"1{:r 19 | 20 | aloc=: jdgl_jd_'t a' 21 | bloc=: jdgl_jd_'t b' 22 | cloc=: jdgl_jd_'t c' 23 | 24 | doj=: 3 : 0 25 | groupidx=:i.~ |: i.~ every (dat__bloc;dat__cloc) 26 | jresult=: ((~.groupidx){dat__bloc);((~.groupidx){dat__cloc);,.(groupidx +//. dat__aloc) 27 | ) 28 | 29 | tj=: 6!:2'doj 0' 30 | assert jresult-:{:jdresult 31 | 32 | assert JDTIMING_jd_+.tbc>td NB. b,c vs d 33 | assert JDTIMING_jd_+.(0.2*tj)>|td-tj NB. d vs j - should be close 34 | assert JDTIMING_jd_+.td<1.4*tj NB. not too much worse than j time 35 | 36 | -------------------------------------------------------------------------------- /test/ptable_delete_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2016, Jsoftware Inc. All rights reserved. 2 | 3 | require JDP,'tools/ptable.ijs' 4 | 5 | gc=: 3 : 0 6 | >{:{:jd'read count p from f' 7 | ) 8 | 9 | gu=: 3 : 0 10 | >{:{:jd'read u from f' 11 | ) 12 | 13 | guw=: 3 : 0 14 | if. 0=L.y do. 15 | >{:{:jd'read u from f where ',y 16 | else. 17 | r=. jd'key /in f';y 18 | >{:{:jd'read u from f where jdindex in (',((":r)rplc' ';','),')' 19 | end. 20 | ) 21 | 22 | chk=: 3 : 0 23 | v=. (gu'')-.guw y 24 | jd'delete ptab';y 25 | jd'delete f';y 26 | assert (jd'reads from ptab')-:jd'reads from f' 27 | assert v-:gu'' 28 | assert (#v)=gc'' 29 | ) 30 | 31 | ptablebld'int' 32 | 33 | chk'val=14' 34 | chk'jdindex in (4,21)' 35 | chk'p';2015;'u';12 16 36 | chk'p';2015 2015 2016 2016;'u';13 15 22 25 37 | chk'jdindex in (2,6,9,12)' 38 | 39 | ptablebld'int' 40 | chk'val=14 or val=8' 41 | chk'jdindex in (21,4)' 42 | chk'p';2015;'u';16 12 43 | chk'p';2016 2016 2015 2015;'u';25 22 15 13 44 | chk'jdindex in (12,9,6,2)' 45 | 46 | ptablebld'int' 47 | chk'p';2015;'val';8 NB. delete multiple rows - keyin 48 | chk'p';2016 2015;'val';6 8 49 | chk'p';2014 2015;'val';14 8 50 | 51 | 52 | -------------------------------------------------------------------------------- /test/ptable_insert_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2016, Jsoftware Inc. All rights reserved. 2 | 3 | NB. y is pcol type 4 | bld=: 3 : 0 5 | jdadminx'test' 6 | 7 | jd'createtable j' 8 | jd'createcol j a int _';i.4 9 | jd'createcol j v int _';1000+i.4 10 | 11 | jd'createtable f' 12 | jd'createcol f set ',y 13 | jd'createcol f val int' 14 | jd'createcol f a int' 15 | jd'ref f a j a' 16 | 17 | jd'createptable f set' 18 | 19 | jd'createtable base' 20 | jd'createcol base set ',y 21 | jd'createcol base val int' 22 | jd'createcol base a int' 23 | jd'ref base a j a' 24 | ) 25 | 26 | nv=. 'set';2011 2011 2011 2012 2012 2012;'val';(6$i.2);'a';6$i.4 27 | 28 | bld'int' 29 | jd'insert base';nv 30 | jd'insert f';nv 31 | a=: jd'reads from base,base.j order by base.set' 32 | b=: jd'reads from f,f.j' 33 | assert ({:a)-:{:b 34 | 35 | assert 2011 2012-:;{:{:jd'read from f^' 36 | jd'insert f';'set';2016;'val';22;'a';23 37 | assert 2011 2012 2016-:;{:{:jd'read from f^' 38 | jd'insert f';'set';2009;'val';22;'a';23 39 | assert 2009 2011 2012 2016-:;{:{:jd'read from f^' NB. f; is sorted 40 | 41 | bld'edate' 42 | nv=. 'set';(sfe_jd_ efs_jd_ 6 4$'201120112011201220122012');'val';(6$i.2);'a';6$i.4 43 | jd'insert f';nv 44 | jd'insert base';nv 45 | assert (jd'reads from base')-:jd'reads from f' 46 | 47 | nv=. 'set';0;'val';23;'a';24 48 | jd'insert f';nv 49 | jd'insert base';nv 50 | assert (jd'reads from base order by set')-:jd'reads from f order by set' 51 | 52 | nv=. 'set';(efs_jd_ '2016',:'2017');'val';27 28;'a';29 30 53 | jd'insert f';nv 54 | jd'insert base';nv 55 | assert (jd'reads from base order by set')-:jd'reads from f order by set' 56 | 57 | nv=. 'set';(1 4$'2020');'val';30;'a';31 58 | jd'insert base';nv 59 | jd'insert f';nv 60 | assert (jd'reads from base order by set')-:jd'reads from f order by set' 61 | 62 | nv=. 'set';(2 4$'20302031');'val';30 31;'a';31 32 63 | jd'insert base';nv 64 | jd'insert f';nv 65 | assert (jd'reads from base order by set')-:jd'reads from f order by set' 66 | 67 | nv=. 'set';('2030');'val';30;'a';31 68 | jd'insert base';nv 69 | jd'insert f';nv 70 | assert (jd'reads from base order by set')-:jd'reads from f order by set' 71 | 72 | -------------------------------------------------------------------------------- /test/ptable_modify_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2016, Jsoftware Inc. All rights reserved. 2 | 3 | require JDP,'tools/ptable.ijs' 4 | 5 | chk=: 3 : 0 6 | jd'update ptab ';y 7 | jd'update f' ;y 8 | assert ({:jd'reads from ptab order by sort')-:{:jd'reads from f order by sort' 9 | ) 10 | 11 | ptablebld'int' 12 | 13 | chk'val=6';'p1';666 777 888 999 14 | chk'val=6 and p=2015';'p1';23 24 15 | chk'val=5 and p in (2015,2016)';'p1';1111 2222;'b';'zx' 16 | chk 12;'p1';555 17 | chk 12 20;'p1';555 777 18 | 19 | assert 'index'jdae'update f';_12;'p1';23 20 | assert 'index'jdae'update f';27;'p1';23 21 | 22 | -------------------------------------------------------------------------------- /test/ptable_update_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2016, Jsoftware Inc. All rights reserved. 2 | 3 | require JDP,'tools/ptable.ijs' 4 | 5 | chk=: 3 : 0 6 | jd 'update ptab ';y 7 | jd'update f' ;y 8 | assert ({:jd'reads from ptab order by sort')-:{:jd'reads from f order by sort' 9 | ) 10 | 11 | ptablebld'int' 12 | 13 | chk'val=6';'p1';666 777 888 999 14 | chk'val=6 and p=2015';'p1';23 24 15 | chk'val=5 and p in (2015,2016)';'p1';1111 2222;'p2';23 24 16 | 17 | 18 | -------------------------------------------------------------------------------- /test/readx_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | NB. tests for read/reads/readtc options 3 | 4 | a=: ><;._2 [0 : 0 5 | 2014-01-01 6 | 2015-01-01 7 | ) 8 | 9 | e=: efs_jd_ a 10 | 11 | jdadminx'test' 12 | jd'createtable f' 13 | jd'createcol f aa int _';i.2 14 | jd'createcol f bb edate _';e NB. createcol does not handle iso 8601 format 15 | 16 | NB. reads options 17 | assert (2 2$('aa';'bb';(,.0 1);a))-:jd'reads from f' 18 | assert (2 2$('aa';0 1;'bb';a))-:jd'reads /lr from f' 19 | assert (2 2$('aa';'bb';(,.0 1);,.e))-:jd'reads /e from f' 20 | assert (2 2$('aa';0 1;'bb';e))-:jd'reads /lr /e from f' 21 | 22 | NB. read options 23 | assert (2 2$('aa';0 1;'bb';a))-:jd'read from f' 24 | assert (2 2$('aa';0 1;'bb';a))-:jd'read /lr from f' NB. /lr has no effect 25 | assert (2 2$('aa';0 1;'bb';e))-:jd'read /e from f' 26 | assert (2 2$('aa';0 1;'bb';e))-:jd'read /lr /e from f' 27 | -------------------------------------------------------------------------------- /test/refx_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2015, Jsoftware Inc. All rights reserved. 2 | NB. verify ref dat set properly 3 | 4 | acr=: 3 : 0 5 | n=. y 6 | jdadminx'test' 7 | jd'createtable f' 8 | a=. 13*i.n 9 | b=. 89*i.1000<.-:n 10 | jd'createcol f a int _';a 11 | jd'createcol f b int _';n$b 12 | jd'createtable g' 13 | jd'createcol g a int _';|.a 14 | jd'createtable h' 15 | jd'createcol h b int _';|.b 16 | ) 17 | 18 | validate=: 3 : 0 19 | jd'ref f a g a' 20 | jd'reads from f,f.g where jdindex=0' NB. make dirty ref clean 21 | jd'ref f b h b' 22 | jd'reads from f,f.h where jdindex=0' NB. make dirty ref clean 23 | 24 | h=. jdgl_jd_'f jdref_a_g_a' 25 | a=. jdgl_jd_'f' 26 | refdata=: Tlen__a=#dat__h 27 | 28 | ra=: jd'reads from f,f.g,f.h' 29 | ) 30 | 31 | acr 100 32 | 33 | validate'' 34 | 35 | jdshuffle_jd_'f' 36 | jdshuffle_jd_'g' 37 | jdshuffle_jd_'h' 38 | 39 | validate'' 40 | 41 | NB. preliminary work for when ref dynamically handles delete/insert 42 | acr 10 43 | jd'ref f a g a' 44 | jd'delete f';'a=13' 45 | jd'delete g';'a=13' 46 | jd'insert f';'a';100;'b';200 47 | jd'insert g';'a';500 48 | 49 | 50 | -------------------------------------------------------------------------------- /test/repair_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2017, Jsoftware Inc. All rights reserved. 2 | 3 | jdadminx'test' 4 | jd'gen test f 5' 5 | c=. jdgl_jd_'f int' 6 | dat__c=: (datcount__c <:Tlen__c){.dat__c 7 | 'damaged'jdae'reads from f' 8 | jdrepair_jd_'fixing it now' 9 | repair_jd_'' 10 | 11 | jdadminx'test' 12 | jd'gen test f 5' 13 | c=. jdgl_jd_'f int' 14 | d=. jdgl_jd_'f byte4' 15 | dat__c=: (datcount__c <:Tlen__c){.dat__c 16 | dat__d=: (datcount__d <:Tlen__d){.dat__d 17 | 'damaged'jdae'reads from f' 18 | jdrepair_jd_'fixing it now' 19 | repair_jd_'' 20 | 21 | jdadminx'test' 22 | jd'gen ref2 f 5 2 g 3' 23 | jd'reads from f,f.g' 24 | c=. jdgl_jd_'f adata' 25 | dat__c=: (datcount__c <:Tlen__c){.dat__c 26 | 'damaged'jdae'reads from f' 27 | jdrepair_jd_'fixing it now' 28 | repair_jd_'' 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /test/sed_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | NB. section/employee/region database 3 | NB. Tables 4 | NB. r(rno,name;popul) 5 | NB. | 6 | NB. s(sno,sdiv,rnum;name) 7 | NB. | 8 | NB. e(ename,secnum) 9 | NB. | 10 | NB. t(tno,enum,tname,tsize) 11 | 12 | sed_tests=: 3 : 0 13 | drd'from r order by rno' 14 | drd'from s order by name,sname' 15 | drd'from e order by ename' 16 | drd'from t order by tno' 17 | drd'ename,s.sno,s.sname,r.rno,r.rname,s.name from e,e.s,s.r order by ename' 18 | NB. dassert'ename';6 12 fixr 'Smith Jones Robinson Jasper Steinberg Rafferty ' 19 | dassert's.sno';33 33 31 34 34 33;6 1 20 | NB. dassert'r.rno';6 5 6 5 5 4;6 1 21 | NB. dassert'r.rname';6 12 fixr 'Naples Rome Naples Rome Rome Naples' 22 | NB. dassert's.name';6 12 fixr 'Clerical Engineering Clerical Engineering Engineering Sales' 23 | drd'ename,s.sno,s.sname,r.rno,r.rname,s.name from e,e.s,s.r where r.rname="Rome" order by ename' 24 | NB. dassert 'ename';'Jones Jasper Steinberg' 25 | NB. dassert 's.sno';33 33 33;3 1 26 | NB. dassert 's.sname';3 12 fixr 'Truck Truck Truck' 27 | NB. dassert 'r.rno';5 5 5;3 1 28 | NB. dassert 'r.rname';3 12 fixr 'Rome Rome Rome' 29 | NB. dassert 's.name';3 12 fixr 'Engineering Engineering Engineering' 30 | drd'from e,e.s,s.r where r.rname="Rome" order by e.ename' 31 | NB. dassert 'e.ename';3 12 fixr 'Jones Jasper Steinberg' 32 | NB. drd'from e,e.s where s.sname="Auto"' 33 | NB. dassert 'e.ename';3 12 fixr 'Smith Robinson Rafferty' 34 | drd'tno,ename,r.rname from t,t.e,e.s,s.r order by tno' 35 | NB. dassert 'ename';4 12 fixr 'Jones Jasper Smith Jones' 36 | NB. dassert 'tno';11 12 13 14;4 1 37 | ) 38 | 39 | jdadmin'sed' 40 | ALLR=: '' 41 | sed_tests'' 42 | ALLRSED=: ALLR 43 | 44 | jdadmin'sed_shuffle' 45 | 46 | ALLR=: '' 47 | sed_tests'' 48 | ALLRSEDSHUFFLE=: ALLR 49 | 50 | assert ALLRSED-:ALLRSEDSHUFFLE 51 | -------------------------------------------------------------------------------- /test/sort_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2017, Jsoftware Inc. All rights reserved. 2 | 3 | f=: 3 : 0 4 | jdadminnew'test' 5 | jd'createtable /pairs f';'a';((y?.y){y$i:3);'b';y$'abbcas' 6 | ) 7 | 8 | tst=: 3 : 0 9 | f 30 10 | d=: jd'reads from f order by ',y 11 | jd'sort f ',y 12 | assert d-:jd'reads from f' 13 | ) 14 | 15 | tst'a' 16 | tst'a desc' 17 | tst'b' 18 | tst'b desc' 19 | 20 | tst'a,b' 21 | tst'a desc,b' 22 | tst'a,b desc' 23 | tst'a desc,b desc' 24 | 25 | tst'b,a' 26 | tst'b desc,a' 27 | tst'b,a desc' 28 | tst'b desc,a desc' 29 | 30 | f 30 31 | jd'sort f a,b' 32 | d=. jd'reads from f' 33 | jd'sort f a,b desc' 34 | assert -.d-:jd'reads from f' 35 | 36 | -------------------------------------------------------------------------------- /test/stress_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | NB. test ref with gen ref2 tables and random deletes/inserts 3 | 4 | sortit=: 3 : 0 5 | a=. jd'read from a,a.b' 6 | i=.({."1 a)i.<'a.akey' 7 | akeys=. /:;{:1{a 8 | v=. (= 20' 7 | s3=. jd'read * from s where status >= 30' 8 | s=. jd'read * from s order by sid' 9 | s=. s /:{."1 s 10 | 11 | COLDEFS=. 'city byte 10';'sid byte 3';'sname byte 10';'status int' 12 | 13 | jdadmin 0 14 | 15 | jdadminx'snkdb' 16 | jd'createtable';'snkt';COLDEFS 17 | jd'insert';'snkt';,s1 18 | 19 | jdadminx'srcdb' 20 | jd'createtable';'srct';COLDEFS 21 | jd'insert';'srct';,s2 22 | 23 | jdadminx'~temp/jd/deeperdir/srcdb2' 24 | jd'createtable';'srct';COLDEFS 25 | jd'insert';'srct';,s3 26 | 27 | jdaccess'snkdb' 28 | 29 | t=. DB_jd_ 30 | 'invalid srcdb'jdae'tableinsert snkt srct foo' 31 | assert t-:DB_jd_ 32 | 33 | 'invalid srcdb'jdae'tableinsert snkt srct snkdb' 34 | 35 | jd'tableinsert snkt srct srcdb' 36 | 37 | jd'tableinsert snkt srct srcdb2' 38 | z=. jd'read * from snkt order by sid' 39 | z=. s /:{."1 z 40 | assert s-:z 41 | 42 | jdadminx'a' 43 | jd'gen ref2 f 4 0 g 2' 44 | jdadminx'b' 45 | jd'gen ref2 j 4 0 k 2' 46 | 47 | c=. jdgl_jd_'j jdref_aref_k_bref' 48 | assert 1=dirty__c 49 | jd'reads from j,j.k' 50 | assert 0=dirty__c 51 | 52 | assert 4='count'jdfroms_jd_ jd'reads count:count akey from j' 53 | jd'tableinsert j f a' 54 | assert 8='count'jdfroms_jd_ jd'reads count:count akey from j' 55 | 56 | c=. jdgl_jd_'j jdref_aref_k_bref' 57 | assert 1=dirty__c 58 | assert 8='count'jdfroms_jd_ jd'reads count:k.bref from j,j.k' 59 | assert 0=dirty__c 60 | 61 | NB. test map as required 62 | jdadminx'testf' 63 | jd'gen test f 3' 64 | jdadminx'testg' 65 | jd'gen test g 3' 66 | jd'close' 67 | jd'tableinsert g f testf' 68 | assert 0 1 2 0 1 2-:,;{:jd'reads x from g' 69 | -------------------------------------------------------------------------------- /test/update_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | ins=: 3 : 0 3 | jd'insert';T;,N,.y 4 | ) 5 | 6 | insx=: 4 : 0 7 | try. 8 | jd'insert';T;,N,.y 9 | assert 0 10 | catch. 11 | assert (;1{jdlast 12 | end. 13 | ) 14 | 15 | N=: ;:'A B C' 16 | T=: 'ch' 17 | bldch=: 3 : 0 18 | jdadminx'test' 19 | jd'createtable';'ch';'A byte,B byte 2,C varbyte' 20 | ) 21 | 22 | bldch'' 23 | ins 'q' ; (1 2$'qw') ; <<'qwe' 24 | ins 'Bx' ; (2 2$'zxpo') ; <'ghi';'jklmn' 25 | 26 | jd'reads from ch' 27 | 3=;{:,jd'reads count A from ch' 28 | 29 | jd'update';T;'B="qw"';'A';'z' 30 | jd'reads from ch' 31 | jd'close' 32 | 33 | jd'update';T;'B="qw"';'A';'z' 34 | jd'reads from ch' 35 | jd'close' 36 | 37 | jd'update';T;'B="qw"';'A';'z' 38 | assert 'zBx'-:;{:{:jd'read A from ch' 39 | jd'update ch';_;'A';'a' 40 | assert 'aaa'-:;{:{:jd'read A from ch' 41 | 42 | NB. bug - an error in update deleted the record 43 | jdadminx 'test' 44 | jd 'createtable tab' 45 | jd 'createcol tab size int _';3 5 7 46 | jd 'createcol tab name byte 3';3 3$'bobsamtom' 47 | assert'bobsamtom'-:,'name'jdfrom_jd_ jd'read from tab' 48 | jd 'reads jdindex,* from tab' 49 | 'bad shape' jdae 'update tab';'jdindex=1';'name';,:'ellen' 50 | assert'bobsamtom'-:,'name'jdfrom_jd_ jd'read from tab' NB. verify row not deleted 51 | 52 | -------------------------------------------------------------------------------- /test/upsert_test.ijs: -------------------------------------------------------------------------------- 1 | bld=: 3 : 0 2 | jdadminx'test' 3 | jd'createtable f' 4 | jd'createcol f a int';i.5 5 | jd'createcol f b int';i.5 6 | jd'createcol f c int';i.5 7 | jd'createcol f d byte';'abcde' 8 | jd'createcol f e byte 2';5 2$'aabbccddee' 9 | jd'reads from f' 10 | ) 11 | 12 | NB. int key 13 | bld'' 14 | [newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'x';'e';'y' 15 | jd'upsert f';'a';newdata 16 | [d=. jd'reads from f' NB. updated 2 rows and inserted 2 rows 17 | assert 7=>{:jd'reads count a from f' 18 | assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f' 19 | 20 | NB. same as previous but with byte key col 21 | bld'' 22 | [newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'crbt';'e';'y' 23 | jd'upsert f';'d';newdata 24 | [d=. jd'reads from f' NB. updated 2 rows and inserted 2 rows 25 | assert 7=>{:jd'reads count a from f' 26 | assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f' 27 | 28 | NB. same as previous but with byteN key col 29 | bld'' 30 | [newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'x';'e';4 2$'ccxxbbqq' 31 | jd'upsert f';'e';newdata 32 | [d=. jd'reads from f' NB. updated 2 rows and inserted 2 rows 33 | assert 7=>{:jd'reads count a from f' 34 | assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f' 35 | 36 | bld'' 37 | [newdata=: 'a';2 1;'b';1000 1002;'c';2000 2002;'d';'x';'e';2 2$'ccbb' 38 | jd'upsert f';'a';newdata NB. update 2 rows 39 | [newdata=: 'a';5 6;'b';1001 1003;'c';2001 2003;'d';'x';'e';2 2$'xxqq' 40 | jd'upsert f';'a';newdata NB. insert 2 rows 41 | jd'reads from f' 42 | assert d-:jd'reads from f' 43 | 44 | 45 | NB. multiple keys 46 | bld'' 47 | newdata=: 'a';2 5 1 6;'b';2 1001 1 1003;'c';2000 2001 2002 2003;'d';'w';'e';'q' 48 | [d=. jd'upsert f';'a b';newdata 49 | assert 7=>{:jd'reads count a from f' 50 | assert 0 1 2 3 4 1001 1003= >{:{.jd'read b from f' 51 | 52 | NB. multiple keys int,byte 53 | bld'' 54 | [a=. jd'upsert f';'a d';newdata 55 | assert a-:d 56 | 57 | NB. multiple keys byte,byteN 58 | bld'' 59 | [a=. jd'upsert f';'a d';newdata 60 | assert a-:d 61 | -------------------------------------------------------------------------------- /test/user_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2018, Jsoftware Inc. All rights reserved. 2 | 3 | len=: 10 4 | bgn=: efs_jd_ '2015-11-20' 5 | 6 | jdadminnew'test' 7 | 8 | f=: 3 : 0 9 | jd 'createtable tab' 10 | jd 'createcol tab Measure int _';len $2 3 4 11 | jd 'createcol tab Usage float _';len $o. 2 3 4 12 | jd 'createcol tab RecordTS edatetimem _';bgn + 864e11 * len$0 1 2 13 | jd 'createcol tab RE byte 10';>len$;:'one two three four' 14 | ) 15 | f'' 16 | jd 'reads from tab' 17 | NB. dates are all the same so use in following order by does not make a difference 18 | jd 'reads sum Usage by RecordTS,RE from tab where Measure=3' 19 | jd 'reads sum Usage by RE,RecordTS from tab where Measure=3' 20 | 21 | NB. enhance this test with different dates are validation of results 22 | -------------------------------------------------------------------------------- /test/utf8_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2017, Jsoftware Inc. All rights reserved. 2 | 3 | 0 : 0 4 | OSX pre catalina stored filenames in decomposed form 5 | following accented a chars look the same, but have different values. 6 | a.i.'ä' NB. utf8 from J 7 | 195 164 8 | a.i.'ä' NB. after 'corruption' from file system 9 | 97 204 136 10 | ) 11 | 12 | jdadminx'test' 13 | jd'createtable Käyf' 14 | jd'createcol Käyf Käyfc int' 15 | jd'insert Käyf';'Käyfc';i.5 16 | 17 | jd'createtable Käyg' 18 | jd'createcol Käyg Käygc int' 19 | jd'insert Käyg';'Käygc';i.5 20 | 21 | jd'ref Käyf Käyfc Käyg Käygc' 22 | jd'reads from Käyf,Käyf.Käyg' 23 | assert (2#<,.i.5)-:{:jd'reads from Käyf,Käyf.Käyg' 24 | 25 | jd'dropcol Käyf jdref_Käyfc_Käyg_Käygc' 26 | jd'renamecol Käyf Käyfc Käyfcxxx' 27 | jd'reads Käyfcxxx from Käyf' 28 | 29 | jd'renametable Käyf Käyfxxx' 30 | jd'reads Käyfcxxx from Käyfxxx' 31 | 32 | jd'droptable Käyfxxx' 33 | 'found'jdae'reads from Käyfxxx' 34 | jd'droptable /reset Käyg' 35 | jd'reads from Käyg' 36 | -------------------------------------------------------------------------------- /test/where_test.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | f =. fixwhere_jdtable_ 3 | m =. <@:(<;._1)@[ -: f@] 4 | 5 | assert '|a|qequal|" a b "' m 'a=" a b "' 6 | 7 | assert '|a|qequal|"a and i=23"' m 'a="a and i=23"' 8 | 9 | assert 0:@:f :: 1: 'a="abc' 10 | 11 | assert '|a|qequal|"abc"' m 'a="abc"' 12 | 13 | NB. f'a eq"abc"' NB. should this be allowed (that is blank not required before quote)? 14 | 15 | 16 | NB. test not 17 | NB. not prior to 3.2 (2016 feb 20) gave wrong answers 18 | 19 | g=: 4 : 0 20 | assert x ='a'jdfrom_jd_ jd'read from f where ',y 21 | assert (t-.x)='a'jdfrom_jd_ jd'read from f where not ',y 22 | ) 23 | 24 | t=: i.10 NB. all rows 25 | 26 | jdadminx'test' 27 | jd'createtable f' 28 | jd'createcol f a int _';i.10 29 | 30 | 5 g 'a = 5' 31 | 5 7 g 'a in (5,7)' 32 | 4 5 6 g 'a range (4,6)' 33 | 34 | jd'delete f';'a=5' 35 | t=: t -. 5 36 | 4 6 g 'a range (4,6)' 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /todo.txt: -------------------------------------------------------------------------------- 1 | in progress: 2 | 3 | zip/unzip ops 4 | 5 | ********************************************* 6 | not started: 7 | 8 | * mark derived col dirty - slow as it walks over all col locales 9 | 10 | * jcs server should have jbin support 11 | 12 | * derived cols 13 | csvwr/csvdump need option to not dump derived cols 14 | csvrestore should detect derived cols are missing and do createdcol 15 | 16 | * admin.ijs '' - not hardwired 17 | 18 | * CSVFOLDER parameter 19 | csvwr colsep,rowsep,quotes,escape parameters 20 | easier automated csv load 21 | 22 | * epoch - flag to allow mm/dd/yy order 23 | 24 | * insert /missing to allow missing cols 25 | 26 | * read /nub to return only unique rows 27 | 28 | -------------------------------------------------------------------------------- /tools/csv_load.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2020, Jsoftware Inc. All rights reserved. 2 | NB. utils for getting csv files into Jd and J 3 | 4 | coclass'jd' 5 | 6 | csvadmin=: 3 : 'jdadmin :: (''new''&jdadmin) y' 7 | 8 | csvprepare=: 3 : 0 NB. tab ; csv_file 9 | t=. 'arg must be: ''table'';''file''' 10 | t assert 1=L.y 11 | t assert 2=#y 12 | t assert 2=;3!:0 each y 13 | 'table file'=. y 14 | t assert fexist file 15 | vtname table 16 | csvadmin 'csvload' 17 | jdcsvfolder'' 18 | file fwrite CSVFOLDER__,table,'.csvlink' NB. CSVFOLDER link to csv file 19 | d=. jd'csvprobe /replace ',table,'.csvlink' NB. read first few rows as byte data 20 | r=. 'first few rows:' 21 | r=. r,,LF,.":{:d 22 | r=. r,LF,'first row could be column headers or just data' 23 | r=. r,LF,'run appropriate line from the following:' 24 | r=. r,LF,' csvload_jd_ ''',table,''';0 NB. if first row looks like data' 25 | r=. r,LF,' csvload_jd_ ''',table,''';1 NB. if first row looks like col headers' 26 | ) 27 | 28 | csvload=: 3 : 0 NB. tab;header 29 | 'table header'=. y 30 | csvadmin'csvload' 31 | if. fexist CSVFOLDER__,table,'.cnames' do. 32 | header=. ' /c ' 33 | else. 34 | header=. ;header{' /u ';' /h 1 ' 35 | end. 36 | jd'droptable ',table NB. delete table if it exists 37 | jd'csvcdefs /replace ',header,table,'.csvlink' NB. create metadata - /u - c1,c2,... col names 38 | jd'csvscan ',table,'.csvlink' NB. scan entire file to adjust cdefs max byte col widths 39 | jd'csvrd ',table,'.csvlink ',table NB. using metadata, load csv file into Jd table 40 | jd'csvreport /f ',table NB. loader full report for table tab 41 | ) 42 | 43 | csvrename=: 3 : 0 NB. table ; oldn ; start J - do not load Jd 12 | load ' path to Jd 4.42 or later folder ','/tools/fix_tlen.ijs' 13 | fix_tlen '' NB. display this help 14 | fix_tlen ' path to folder with tables to fix' NB. for example, '~temp/jd' 15 | >shutdown J 16 | >start J and load Jd (version prior to 4.42) 17 | > should be OK now 18 | > databases marked damaged because of this problem can be fixed 19 | jdrepair 'tlen was fixed' 20 | jddamage '' NB. db is probably OK now 21 | ) 22 | 23 | fix_tlen=:3 : 0 24 | if. ''-:y do. fix_tlen_man return. end. 25 | 'y used as path to jd folders to fix'assert 0~:#y 26 | if. (<'jd')e.conl 0 do. jdadmin 0[jd'close' end. NB. avoid conflict if Jd is loaded 27 | d=. {."1 dirtree y 28 | k=. '/jdtlen' 29 | ck=. -#k 30 | d=. ((ck{.each d)=<.omax*avg 16 | clean=: 3 : 0 17 | 'tab omin omax change'=. y 18 | 'invalid change'assert (1{"1 {:s 21 | typs=. <"1 >2{"1 {:s 22 | a=. (0,#cleanlabs)$'' 23 | for_col. cols do. 24 | c=. jdgl_jd_ tab,' ',;col 25 | select. typ__c 26 | case.'int';'edatetimen';'float' do. 27 | r=. cleansub c;omin;omax;change 28 | case. 'byte' do. 29 | r=. (+/(0{a.)={."1 ,.dat__c),(<:#cleanlabs)#0 30 | case. do. r=. _1 31 | end. 32 | a=. a,r 33 | end. 34 | if. 'change'-:change do. clean tab;omin;omax;'' return. end. 35 | (,.' ';cols),.(,.' ';typs),.cleanlabs,<"0 a 36 | ) 37 | 38 | NB. if change - replace Na, Out-, out+ with median 39 | cleansub=: 3 : 0 40 | 'c omin omax change'=. y 41 | nav=. >('float'-:typ__c){IMIN_jd_;__ 42 | d=. forcecopy dat__c 43 | r=. +/na=. d=nav 44 | r=. r,100*fmt r%#d 45 | r=. r,mid=. median d 46 | 'median is Na'assert mid~:nav 47 | d=. mid (I.na)}d NB. replace Na now so other calcs make sense 48 | r=. r,avg=. fmt (+/d)%#d 49 | r=. r,+/neg=. 0>d 50 | r=. r,<./d 51 | r=. r,>./d 52 | r=. r,+/outm=. d<<.avg%omin NB. mid could be 0 so use avg 53 | r=. r,+/outp=. d><.omax*avg NB. mid could be 0 so use avg 54 | 55 | if. change-:'change' do. dat__c=: mid (I.outm+.outp)}d end. NB. replace Na,outm,outp with mid 56 | 57 | r 58 | ) 59 | -------------------------------------------------------------------------------- /tools/pandas/load.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import sys 3 | 4 | jmfheader= 568 # bytes at start of jmf file 5 | 6 | def tobin(file, dfcol): 7 | if 'object'==str(dfcol.dtype): 8 | dfcol.fillna('\x00',inplace=True) # NONE replaced by '\x00' 9 | f= open(file,'wb') 10 | f.write(b' '*jmfheader) 11 | n= 0 12 | for c in dfcol: # get max byte length 13 | n= max(n,len(c.encode())) 14 | for c in dfcol: 15 | f.write(c.encode().ljust(n)) 16 | f.close() 17 | else: 18 | f= open(file,'w') 19 | f.write(' '*jmfheader) 20 | dfcol.values.tofile(f) 21 | f.close() 22 | 23 | def doit(sink,source,cnt): 24 | df= eval("pd."+source) 25 | 26 | if cnt != 'all': 27 | cnt= int(cnt) 28 | if cnt<0: 29 | df= df.tail(-cnt) 30 | else: 31 | df= df.head(cnt) 32 | 33 | cols= df.columns 34 | types= df.dtypes 35 | 36 | for idx,col in enumerate(cols): 37 | print(str(idx).rjust(4,' ')+" "+str(col)) 38 | # build meta data col+type+rows 39 | m= str(types[[col]]) 40 | m= m.split('\n')[0] 41 | t= m.split(' ')[-1] # datatype 42 | m= t+'\t'+str(df.shape[0]) 43 | 44 | n= str(idx).rjust(4,'0') 45 | f= open(sink+n+'.pandasmeta','w') 46 | f.write(str(col)+"\t"+m+'\t') # write metadata to pandasmeta file 47 | f.close() 48 | 49 | tobin(sink+n+'.dat',df[col]) # write data bytes to dat file 50 | 51 | sink= str(sys.argv[1]) # "j9.4-user/temp/jd/pandas_db/pandas_table/jdpandas_raw/" 52 | source= str(sys.argv[2]) # "read_csv(\"t0.csv\")" 53 | cnt= str(sys.argv[3]) # "all" 54 | 55 | doit(sink,source,cnt) 56 | 57 | # debug 58 | # >python3 -i .../addons/data/jd/tools/pandas/load.py 59 | # >>> doit("tmp/","read_csv('t0.csv')","all") 60 | -------------------------------------------------------------------------------- /tools/pandas/util.py: -------------------------------------------------------------------------------- 1 | # exec(open('git/addons/data/jd/tools/pandas/util.py').read()) 2 | 3 | import pandas as pd 4 | import sys 5 | 6 | jmfheader= 568 # bytes at start of jmf file 7 | 8 | def bld(file): 9 | df= pd.read_csv(file) 10 | return df 11 | 12 | def tobin(file, dfcol): 13 | if 'object'==str(dfcol.dtype): 14 | print('object') 15 | dfcol.fillna('\x00',inplace=True) # NONE replaced by '\x00' 16 | f= open(file,'wb') 17 | f.write(b' '*jmfheader) 18 | n= 0 19 | for c in dfcol: # get max byte length 20 | n= max(n,len(c.encode())) 21 | for c in dfcol: 22 | f.write(c.encode().ljust(n)) 23 | f.close() 24 | else: 25 | f= open(file,'w') 26 | f.write(' '*jmfheader) 27 | dfcol.values.tofile(f) 28 | f.close() 29 | 30 | 31 | def get(file,shape,cnt): 32 | r= [] 33 | f= open(file,'rb') 34 | sk= jmfheader 35 | for i in range(cnt): 36 | f.seek(sk) 37 | b= f.read(int(shape)) 38 | sk= sk+shape 39 | r.append(b.decode()) 40 | return r 41 | -------------------------------------------------------------------------------- /tools/pandas/write.py: -------------------------------------------------------------------------------- 1 | # write Jd table cols to csv/parquet/... file 2 | import pandas as pd 3 | import numpy as np 4 | import sys 5 | from numpy import fromfile 6 | 7 | jmfheader= 568 # bytes at start of jmf file 8 | 9 | # get col array of strings from file 10 | def getcol(file,shape,cnt): 11 | r= [] 12 | f= open(file,'rb') 13 | sk= jmfheader 14 | for i in range(cnt): 15 | f.seek(sk) 16 | b= f.read(int(shape)) 17 | sk= sk+shape 18 | r.append(b.decode()) 19 | return r 20 | 21 | table= str(sys.argv[1]) 22 | op= str(sys.argv[2]) 23 | cnt= int(str(sys.argv[3])) 24 | 25 | f= open(table+"/jd_pandas_write.txt", 'r') 26 | s= f.read() 27 | names= s.split('\n') # names[0] is colname type shape 28 | df = pd.DataFrame() 29 | for i,nm in enumerate(names): 30 | p= str(names[i]) 31 | a= p.rsplit(' ',1) 32 | shape= str(a[1]) 33 | b= a[0].rsplit(' ',1) 34 | type= str(b[1]) 35 | colname= str(b[0]) 36 | s= table+colname+"/dat" 37 | if type!='object': 38 | if type=='datetime64[ns]': 39 | a= fromfile(s, dtype=type, count=cnt, offset=jmfheader) 40 | for i in range(cnt): 41 | if ~np.isnan(a[i]): 42 | a[i]= a[i]+946684800000000000 # adjust to epoch 1970 43 | df[colname]= a 44 | else: 45 | df[colname]= fromfile(s, dtype=type, count=cnt, offset=jmfheader) 46 | else: 47 | shape= 1 if shape=='_' else int(shape) 48 | df[colname]= getcol(s,shape,cnt) 49 | 50 | eval("df."+op) 51 | -------------------------------------------------------------------------------- /tools/quandl.ijs: -------------------------------------------------------------------------------- 1 | require'pacman' NB. httpget 2 | 3 | NB. quandl_get 'ibm' 4 | quandl_get=: 3 : 0 5 | jd'droptable ',y 6 | k=. '?api_key=',fread'~config/quandl_apikey.txt' 7 | 'rc fn'=. httpget_jpacman_ 'https://www.quandl.com/api/v3/datasets/EOD/',(toupper y),'.csv',k 8 | 'httpget failed'assert 0=rc 9 | (fread fn)fwrite CSVFOLDER,y,'.csv' 10 | y,'.csv' 11 | ) 12 | 13 | NB. quandl_cdefs 'ibm' - build cdefs metadata file 14 | quandl_cdefs=: 3 : 0 15 | fcsv=. y,'.csv' 16 | jd'csvprobe /replace ',fcsv 17 | jd'csvcdefs /replace /h 1 ',fcsv 18 | jd'csvscan ',fcsv 19 | CDEFSFILE=: CSVFOLDER,'eod.cdefs' 20 | CDEFSFILE frename CSVFOLDER,y,'.cdefs' 21 | 'eod.cdefs' 22 | ) 23 | 24 | NB. quandl_load 'ibm' 25 | quandl_load=: 3 : 0 26 | jd'droptable ',y 27 | jd'csvrd /cdefs F.csv F'rplc'F';y 28 | ) 29 | 30 | quandl_plot=: 4 : 0 31 | t=. 'title ',(toupper y),' ',x,,' ',.;{:jd'reads last Date,first Date from ',y 32 | t plot |.>{:{.jd'read ',x,' from ',y 33 | ) 34 | -------------------------------------------------------------------------------- /tools/repair.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | coclass 'jd' 3 | 4 | 0 : 0 5 | Tlen set as <./Tlen,col counts 6 | each col count set to Tlen 7 | refs marked dirty if not already dirty 8 | ) 9 | 10 | repair=: 3 : 0 11 | getdb'' NB. set dbl 12 | jdrepair'fixing it now' NB. insists on damaged 13 | for_tn. NAMES__dbl do. 14 | tn=. ;tn 15 | t=. jdgl tn 16 | for_c. NAMES__t do. 17 | a=. 1 jdclocs_jd_ tn;'' NB. map cols 18 | count=. '' 19 | for_aa. a do. 20 | if. -.'jd'-:2{.NAME__aa do. count=. count,countdat__aa dat__aa end. 21 | end. 22 | end. 23 | m=. <./count 24 | techo tn,' Tlen min-count max-count ',":Tlen__t,m,>./count 25 | if. Tlen__t>m do. setTlen__t <./count end. 26 | for_c. NAMES__t do. 27 | c=. getloc__t c 28 | cn=. NAME__c 29 | if. 'jdref_'-:3{.cn do. setdirty__c 1 continue. end. 30 | if. 'jd'-:2{.cn do. continue. end. 31 | if. Tlen__t~:countdat__c dat__c do. 32 | techo' repair count - ',cn 33 | dat__c=: (datcount__c Tlen__t){.dat__c 34 | end. 35 | end. 36 | end. 37 | jddamage_jd_'' NB. remove damage and repair mark 38 | i.0 0 39 | ) 40 | -------------------------------------------------------------------------------- /tools/report_copath_names.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | report copath name counts 3 | report jdgl_jd_'f a' 4 | ) 5 | 6 | foo=: 3 : 0 " 0 7 | #each names__y each '';<"0[0 1 2 3 8 | ) 9 | 10 | report=: 3 : 0 11 | cs=. y,copath y 12 | d=. foo cs 13 | d=. cs,.d 14 | d,~;:'locale total noun adv conj verb' 15 | ) -------------------------------------------------------------------------------- /tools/setscriptlists.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2019 Jsoftware Inc. All rights reserved. 2 | 3 | coclass'jd' 4 | 5 | NB. setscriptlists creates/runs script to create tests/tuts script lists 6 | NB. tests and tuts scripts to define script lists 7 | NB. leading digits in folders and files set order 8 | NB. 999_ guys are sorted by name 9 | NB. tut names must be unique 10 | setscriptlists=: 3 : 0 11 | NB. tests 12 | p=. JDP 13 | t=. 1 dir p,'test/*_test.ijs' 14 | t=. /:~(#p)}.each t 15 | t=. ;t,each LF 16 | t=. 'tests=: <;._2 [ 0 : 0',LF,t,')' 17 | f=. JDP,'base/tests.ijs' 18 | t fwrite f 19 | load f 20 | 21 | NB. tuts 22 | t=. 1 dir p,'tutorial/*_tut.ijs' 23 | t=. /:~(#p)}.each t 24 | t=. ;t,each LF 25 | t=. 'testtuts=: <;._2 [ 0 : 0',LF,t,')' 26 | f=. JDP,'base/testtuts.ijs' 27 | t fwrite f 28 | load f 29 | 30 | tutchk'' 31 | ) 32 | 33 | tutcats=: 3 : 0 34 | load JDP,'tools/tut.ijs' 35 | a=. ;(' '=each tuts)i.each 0 36 | >(a<}.a,0)#tuts 37 | ) 38 | 39 | tutchk=: 3 : 0 40 | load JDP,'tools/tut.ijs' 41 | a=. ;(' '=each tuts)i.each 0 42 | t=. (a>:}.a,0)#tuts 43 | t=. dltb each t 44 | f=. 1 dir JDP,'tutorial' 45 | f=. _8}.each (>:;f i:each'/')}.each f 46 | if. #t-.f do. 47 | echo 'tuts without files' 48 | echo >' ',each t-.f 49 | end. 50 | if. #f-.t do. 51 | echo 'files without tuts' 52 | echo >' ',each f-.t 53 | end. 54 | 'tuts files mismatch'assert 0=#(f-.t),t-.f 55 | 'tuts duplicats'assert(#~.dltb each tuts_jd_)=#dltb each tuts_jd_ 56 | i.0 0 57 | ) 58 | -------------------------------------------------------------------------------- /tools/ts.ijs: -------------------------------------------------------------------------------- 1 | NB. used in win vs jddll.dll debug 2 | 3 | DLLDEBUG=: 1 4 | load'~Jddev/jd.ijs' 5 | load'~addons/ide/jhs/core.ijs' 6 | 7 | echo'~Jddev and C code in git !!!!!!!!!!!!!!!!!!' 8 | 9 | 3 : 0'' 10 | if. _1=nc<'SKLISTEN_jhs_' do. init_jhs_'' end. 11 | ) 12 | 13 | -------------------------------------------------------------------------------- /tools/tut.ijs: -------------------------------------------------------------------------------- 1 | tuts=: <;._2 [ 0 : 0 2 | basic 3 | intro_a 4 | intro_b 5 | intro_c 6 | pairs 7 | stock_data 8 | table_from_pairs 9 | epochdt 10 | admin 11 | op 12 | createcol 13 | createcol_derived 14 | createcol_derived_mapped 15 | createtable 16 | delete 17 | gen 18 | info 19 | insert 20 | intx 21 | key 22 | read 23 | reads 24 | reads_basic 25 | reads_clauses 26 | reads_from 27 | reads_join 28 | reads_aggregation 29 | reads_option_table 30 | setget 31 | sort 32 | tablecopy 33 | tableinsert 34 | tablemove 35 | update 36 | update_basic 37 | update_advanced 38 | upsert 39 | upsert_basic 40 | upsert_advanced 41 | csv 42 | csv_load 43 | csv_intro 44 | bus_lic 45 | taxi 46 | quandl_ibm 47 | csv_advanced 48 | pandas 49 | pandas_install 50 | pandas_load 51 | pandas_clean 52 | pandas_write 53 | demo 54 | northwind 55 | sandp 56 | sed 57 | vr 58 | server 59 | jcs 60 | jds 61 | jds_client 62 | jhs 63 | mtm 64 | node 65 | json 66 | jctask 67 | xtra 68 | blob 69 | custom 70 | dropstop 71 | link 72 | log 73 | performance 74 | ptable 75 | replicate 76 | table_table 77 | ) -------------------------------------------------------------------------------- /tutorial/admin_tut.ijs: -------------------------------------------------------------------------------- 1 | jdadmin 0 NB. remove all admin 2 | jdadminx'test' NB. create new db test (~temp/jd/test) 3 | jdadmin'' NB. admin report 4 | NB. [w] ...test - lock prevents interference from other tasks 5 | NB. test ~temp/jd/test - DAN test uses this database folder 6 | NB. test u/p - user/pswds allowed 7 | NB. test * - ops allowed (* for all) 8 | 9 | admin=: 0 : 0 10 | 'all' jdadminfp '' NB. all DAN uses DB that contains this script 11 | 'all' jdadminup 'u/p' NB. user/pswds allowed access 12 | 'all' jdadminop '*' NB. ops allowed 13 | 14 | 'ro' jdadminfp '' 15 | 'ro' jdadminup 'abc/def ghi/jkl' 16 | 'ro' jdadminop 'read reads' 17 | ) 18 | 19 | i.0 0[admin fwrite '~temp/jd/test/admin.ijs' 20 | 21 | jdadmin 0 NB. remove all admin 22 | jdadmin'test' NB. admin for ~temp/test (load ~temp/jd/test/admin.ijs) 23 | jdadmin'' 24 | 25 | jdaccess'all u/p intask' NB. DAN all uses admin rows with all 26 | jd'createtable';'f';'a int' 27 | jd'insert';'f';'a';23 24 28 | jd'read from f' NB. all ops allowed 29 | 30 | jdaccess 'ro x/y intask' NB. jd ops: ro with x/y intask 31 | jd etx'read from f' NB. fails - ro does not allow user x/y 32 | jdlast 33 | 34 | jdaccess 'ro ghi/jkl intask' 35 | jd'read from f' 36 | jd etx'delete';'j';'jid=J1' NB. fails - ro does not allow delete 37 | jdlast 38 | -------------------------------------------------------------------------------- /tutorial/blob_tut.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | preliminary support for blobs - subject to change! 3 | 4 | blob - binary large object 5 | a blob is stored as a file in the database folder structure 6 | a blob can be stored at DB folder, table folder, or column folder 7 | ) 8 | 9 | jdadmin 0 10 | 'new'jdadmin'test' 11 | jd'createtable f' 12 | jd'createcol f a int' 13 | 14 | NB. blobs in the DB folder 15 | jd'blobwrite name';'db data' 16 | jd'blobwrite crudstuff';'db different data' 17 | jd'blobread name' 18 | jd'blobread crudstuff' 19 | jd'bloberase name' 20 | 'failed'jdae'blobread name' 21 | 22 | NB. blobs in the table folder 23 | jd'blobwrite f name';'table data' 24 | jd'blobwrite f crudstuff';'table different data' 25 | jd'blobread f name' 26 | jd'blobread f crudstuff' 27 | 28 | NB. blobs in the col folder 29 | jd'blobwrite f a name';'col data' 30 | jd'blobwrite f a crudstuff';'col different data' 31 | jd'blobread f a name' 32 | jd'blobread f a crudstuff' 33 | 34 | jd'info blob' 35 | 36 | 0 : 0 37 | use custom ops to connect a blob to a table row 38 | see jd_myblobread_jd_ below 39 | ) 40 | 41 | 'new'jdadmin'test' 42 | jd'createtable f' 43 | jd'createcol f num int' 44 | jd'createcol f name byte 5' 45 | jd'insert f';'num';44 55 66;'name';3 5$'abc de fghi ' 46 | jd'read from f' 47 | 48 | jd'blobwrite f abc';'this is the data for name abc' 49 | jd'blobwrite f de';'dededede' 50 | jd'blobwrite f fghi';'yet another blob' 51 | 52 | jd_myblobread_jd_=: 3 : 0 53 | a=. jdi_read'from f where num=',y 54 | d=. 'name' jdfrom_jd_ a 55 | 'can only read on blob at a time'assert 1=#d 56 | a,jd_blobread'f ',,d 57 | ) 58 | 59 | jd'myblobread 44' 60 | jd'myblobread 55' 61 | jd'myblobread 55' 62 | -------------------------------------------------------------------------------- /tutorial/bus_lic_tut.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | load Chicago business_licenses.csv - >250 MB, 35 cols, and >860e6 rows 3 | 4 | original source: https://catalog.data.gov/dataset?res_format=CSV&tags=licenses&organization=city-of-chicago 5 | 6 | put at www.jsoftware.com/download/jdcsv/business_licenses.zip 7 | 8 | download (only done first time) can take a few minutes 9 | 10 | load csv to Jd table can take 20 seconds on a slow machine 11 | and 1 second on a fast machine with ssd 12 | ) 13 | 14 | CSVFOLDER=: '~temp/jd/csv/buslic/' NB. folder for csv files 15 | jdcreatefolder_jd_ CSVFOLDER NB. ensure folder exists 16 | jdadminx'buslic' NB. new db ~temp/jd/test/ 17 | fcsv=: 'business_licenses.csv' 18 | 19 | NB. next advance does download (if required) and takes minutes 20 | NB. zip downloaded from www.jsoftware.com/jdcsv and unzipped in CSVFOLDER 21 | getcsv_jd_ fcsv 22 | fsize CSVFOLDER,fcsv 23 | 24 | NB. csvprobe will set initial .cdefs metadata and read first 20 rows 25 | jd'csvprobe /replace ',fcsv 26 | NB. looking at data it is clear first row is column names 27 | NB. subsequent rows look like data (rather than more headers) 28 | 29 | NB. csvcdefs will set .cdefs metadata - /h 1 iindicates first row has col names 30 | jd'csvcdefs /replace /h 1 ',fcsv 31 | 32 | NB. csvcdefs sets metadata but based on only first 5000 rows 33 | NB. csvscan wll scan entire file to get max col widths from all rows 34 | jd'csvscan ',fcsv 35 | 36 | fread CSVFOLDER,'business_licenses.cdefs' NB. final metadata 37 | 38 | NB. csvrd will load csv file into Jd table buslic 39 | jd'csvrd ',fcsv,' buslic' 40 | jd'csvreport' 41 | 42 | jd'reads count ID from buslic' NB. more than 860 thousand rows 43 | jd'reads count:count ID by "APPLICATION TYPE" from buslic' 44 | jd'reads first "LICENSE DESCRIPTION",count:count ID by "LICENSE CODE" from buslic order by count desc' 45 | 10{.>1{ {:jd'reads first "LICENSE DESCRIPTION",count:count ID by "LICENSE CODE" from buslic order by count desc' 46 | jd'reads ID,"APPLICATION TYPE","LEGAL NAME" from buslic where "LEGAL NAME" = "CHICAGO GAME CO"' 47 | d=: 'APPLICATION TYPE'jdfrom_jd_ jd'read "APPLICATION TYPE" from buslic' 48 | $d 49 | 10{.d 50 | 51 | -------------------------------------------------------------------------------- /tutorial/createcol_derived_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. derived col 2 | 3 | 0 : 0 4 | derived col 5 | values calculated = usually from another col 6 | not mapped 7 | allowed in ref 8 | not allowed in a ptable 9 | calculated when first referenced 10 | not allowed in insert/upsert/update/... pairs 11 | 12 | calculated with verb derived_mapped_xxxx defined in custom.ijs 13 | xxxx is from jd'createcol /derived col type [trailing shape] xxxx' 14 | 15 | csv dumps data (not derived_xxxx verb) 16 | jdloadcustom loads changes to custom.ijs to db locale 17 | 18 | mark dirty whenever a dependency might have changed 19 | mark dirty is agressive - e.g. in perhaps most case sort would be ok 20 | but instead sort does not sort derived and marks them dirty 21 | sort/dropcol/delete all mark derived dirty 22 | ) 23 | 24 | jdadminnew'test' 25 | jd'createtable f' 26 | jd'createcol f b byte 4' 27 | jd'createcol f e edate' 28 | jd'insert f';'b';(3 4$'abcdef');'e';'2014-10-12','2015-10-13',:'2016-10-14' 29 | jd'reads from f' 30 | 31 | jd'createcol /derived f catg byte 2 vcatg ' NB. vcatg verb used to derive col 32 | jd'createcol /derived f year int vyear' NB. vyear verb used to derive col 33 | 'value error'jdae'reads from f' NB. derive verbs not defined 34 | jdlast NB. last error indicates table and col 35 | 36 | custom=: 0 : 0 rplc'RPAREN';')' NB. defns for derive verbs 37 | derived_vcatg=: 3 : 0 38 | 2{."1 jd_get'f b' 39 | RPAREN 40 | 41 | derived_vyear=: 3 : 0 42 | 0".4{."1 sfe jd_get'f e' 43 | RPAREN 44 | ) 45 | custom fappend jdpath_jd_'custom.ijs' NB. add derive verbs to custom.ijs 46 | jdloadcustom_jd_'' NB. load changes 47 | 48 | jd'reads from f' 49 | 50 | CSVFOLDER=: '~temp/jd/csv' 51 | jd'csvwr f.csv f' 52 | jd'csvrd f.csv g' 53 | assert (jd'reads from f')-:jd'reads from g' 54 | jd'info derived' 55 | jd'info derived f' 56 | jd'info derived f catg' 57 | -------------------------------------------------------------------------------- /tutorial/createcol_tut.ijs: -------------------------------------------------------------------------------- 1 | jdadminnew'tutorial' 2 | jd'createtable f' 3 | jd'createcol f a int' 4 | jd'insert f a';12 13 14 5 | jd'createcol f b int' NB. default values 6 | jd'createcol f d int';88 89 90 NB. values 7 | jd'createcol f e int';101 NB. value 8 | jd'createcol f "a/ =\" \b" int';23 24 25 NB. all chars are allowed 9 | jd'reads from f' 10 | jd'reads from f where "a/ =\" \b"=24' 11 | -------------------------------------------------------------------------------- /tutorial/createtable_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'droptable f' NB. droptable if it exists 4 | jd'createtable f' 5 | 'already exists'jdae'createtable f' NB. accept expected error 6 | jd'createtable /replace f' NB. droptable first if it exists 7 | jd'createtable /replace f' NB. no coldefs 8 | jd'createtable /replace f a int' NB. one coldef 9 | jd'createtable /replace f a int,b byte 3' NB. , delimited coldefs 10 | jd'createtable /replace f a int',LF,'b byte 3' NB. LF delimited coldefs 11 | jd'createtable /replace f';'a int';'b byte 3' NB. boxed coldefs 12 | jd'createtable /replace /a 10000 2 0 f' NB. resize allocation values 13 | jd'createtable /replace /pairs f';'a';2 3;'b';2 3$'abcdef' NB. pairs - coldefs and data 14 | jd'createtable /replace /types /pairs f';'a(edate)';'1990-12-02';'b(float)';23 15 | 'access'jdadmin'' NB. tutorial end - you may want to access another dan 16 | -------------------------------------------------------------------------------- /tutorial/csv_load_tut.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | this tutorial will help you load csv files 3 | you load a csv into Jd and, for free, it is in J 4 | maximize your ability to work with csv files 5 | with minimal Jd knowledge 6 | 7 | utils in this tutorial provide basic support 8 | if you need more, you'll need to dig into Jd! 9 | ) 10 | 11 | (0 : 0) fwrite '~temp/f.csv' NB. create csv file 12 | 123,nut 13 | 678,bolt 14 | ) 15 | 16 | csvprepare_jd_ 'abc';'~temp/f.csv' NB. table_name ; csv_file 17 | csvload_jd_ 'abc';0 NB. first row looks like data 18 | jd'reads from abc' NB. labeled columns 19 | jd'read from abc' NB. labeled rows 20 | 1{::0{jd'read from abc' 21 | jd'read c2 from abc' 22 | 23 | (3!:1 jd'read from abc')fwrite '~temp/abc.dat' NB. write binary rep file 24 | 3!:2 fread '~temp/abc.dat' NB. get the data - perhaps in another session 25 | 26 | (0 : 0) fwrite '~temp/g.csv' 27 | part/number,"part/common,name" 28 | 123,"wing,nut" 29 | 678,bolt 30 | ) 31 | 32 | csvprepare_jd_ 'def';'~temp/g.csv' 33 | csvload_jd_ 'def';1 NB. first row looks like column headers 34 | jd'reads from def' NB. labeled columns 35 | jd'reads from abc' NB. table abc still there 36 | 37 | csvprepare_jd_ 'abc';JDP,'/demo/northwind/csv/Customers.csv' 38 | csvload_jd_ 'abc';0 NB. default names 39 | jd'reads from abc where jdindex<5' NB. first 5 rows 40 | jd'reads c1,c6 from abc where jdindex<5' NB. 2 cols from first 5 rows 41 | 42 | NB. you can rename cols to have your names 43 | newn=: ;:'Address City CompanyName ContactName ContactTitle Country CustomerID Fax Phone PostalCode Region' 44 | oldn=: {."1 jd'read from abc' NB. current names 45 | csvrename_jd_ 'abc';oldn;{:"1 r 23 | RPAREN 24 | ) 25 | 26 | jdadminx'test' NB. new admin, new db, no custom.ijs 27 | custom fwrite '~temp/jd/test/custom.ijs' NB. create custom.ijs 28 | jdloadcustom_jd_'' NB. load changes 29 | jd'createtable';'f';'cola int,colb int' 30 | jd'insert';'f';'cola';23 24 25;'colb';33 34 35 31 | jd'read from f' 32 | jd'xins';55;66 33 | jd'read from f' 34 | jd'xra' 35 | jd'xsum' 36 | -------------------------------------------------------------------------------- /tutorial/delete_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'gen test f 10' 4 | jd'reads from f' 5 | jd'delete f';3 5 6 | jd'reads from f' 7 | jd'delete f';'x<3' 8 | jd'reads from f' 9 | jd'key /in f';'int';104 107 NB. key selects 2 rows 10 | jd'delete f';'int';104 107 11 | jd'reads from f' 12 | jd'key /in f';'int';108 109;'boolean';1 1 NB. key selects 1 row 13 | jd'delete f';'int';108 109;'boolean';1 1 NB. delete row(s) selected by key 14 | jd'reads from f' 15 | 'access'jdadmin'' 16 | -------------------------------------------------------------------------------- /tutorial/dropstop_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. jddropstop - prevent drop of db/table/col 2 | 3 | NB. admin can mark db/table/col so that drops fail 4 | NB. dropstop can prevent inadvertant drops 5 | NB. that might be difficult to recover from 6 | NB. see jddropstop documentation 7 | 8 | NB. [1] jddropstop writes empty jddroptstop files in the path 9 | NB. 0 jddropstop erases all jddropstop files in the path 10 | 11 | NB. jddropstop file prevents a drop of a db/table/col 12 | 13 | de=: 'domain error' 14 | 15 | jdadminx'test' 16 | jd'createtable f a int,b int' 17 | jd'createtable g a int,b int' 18 | 19 | jddropstop_jd_'' NB. mark db, all tables, and all cols as dropstop 20 | assert de-:jd etx 'dropdb' 21 | assert de=:jd etx 'droptable f' 22 | assert de=:jd etx 'dropcol f a' 23 | 24 | 0 jddropstop_jd_ 'f a' NB. allow drop of f a 25 | jd'dropcol f a' 26 | assert de=:jd etx 'dropcol f b' NB. f b still protected 27 | 28 | 0 jddropstop_jd_ 'f' NB. allow drop of f and all cols in f 29 | jd'dropcol f b' 30 | jd'droptable f' 31 | 32 | 0 jddropstop_jd_ '' NB. allow drop of db and all tables and all cols 33 | jd'dropdb' 34 | -------------------------------------------------------------------------------- /tutorial/gen_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'gen test f 3' NB. f with 3 rows 4 | jd'reads from f' 5 | jd'gen ref2 g 5 2 h 3' NB. g 5 rows and 2 extra cols, h 3 rows 6 | jd'reads from g,g.h' 7 | jd'gen one i 3 2' 8 | jd'reads from h' 9 | jd'gen two j 6 k 3' 10 | jd'reads from j' 11 | jd'reads from k' 12 | -------------------------------------------------------------------------------- /tutorial/info_tut.ijs: -------------------------------------------------------------------------------- 1 | jdadminnew'test' 2 | jd'gen ref2 a 3 0 b 2' NB. gen ref2 tables a and b jointed 3 | jd'reads from a,a.b' 4 | jd'info table' 5 | jd'info summary' 6 | jd'info summary b' 7 | jd'info schema' 8 | jd'info schema a' 9 | jd'info schema a aref' 10 | jd'info ref' NB. table a reference - joins aref to b bref 11 | jd'info agg' NB. aggregations 12 | jd'reads from a' 13 | jd'info last' NB. performance data 14 | jd'info validate a' NB. validate table 15 | jd'info validate a aref' NB. validate table col 16 | jd'info validatebad a' NB. report problems 17 | -------------------------------------------------------------------------------- /tutorial/insert_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'createtable f a int,b byte,b4 byte 4' 4 | jd'insert f';'a';23;'b';'a';'b4';1 4$'abcd' 5 | jd'reads from f' 6 | jd'insert f';'a';24 25;'b';'bc';'b4';2 4$'aaaabbbb' 7 | jd'reads from f' 8 | 'unknown' jdae'insert f';'a';24 25;'xxx';'bc';'b4';2 4$'aaaabbbb' 9 | 'missing' jdae'insert f';'a';2 3 4;'b';'abc' 10 | 'count' jdae'insert f';'a';2 3 4;'b';'ab';'b4';3 2$'a' 11 | 'odd number'jdae'insert f';'a';2;'b' 12 | -------------------------------------------------------------------------------- /tutorial/intro_a_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. create db, table, columns 2 | NB. read - select, where, aggregation 3 | jdadminnew'test' NB. create new db (~temp/jd/test) 4 | c=: ' year int, sales int, product byte 5,team byte 4' 5 | jd'createtable t1 ',c 6 | jd'reads from t1' 7 | d=: (4#2002 2003 2001);(?12$1000);(>12$2#'shoes';'hats');>12$'blue';'red' 8 | d=: ('year';'sales';'product';'team'),.d 9 | ,d NB. list of pairs 10 | jd'insert t1';,d NB. insert list of pairs 11 | jd'reads from t1' NB. names on top 12 | jd'read from t1' NB. names on left 13 | jd'info table' 14 | jd'info schema' 15 | 16 | ''jdae'reads from t7' NB. jdae - accept expected error 17 | jdlast NB. last error 18 | jdlasty NB. last jd arg 19 | 20 | jd'reads from t1' 21 | jd'reads year,sales from t1' 22 | jd'reads from t1 order by year' 23 | jd'reads from t1 where year<2003' 24 | jd'reads from t1 where year<2003 order by year' 25 | jd'reads from t1 where year<2003 and team="blue"' 26 | jd'reads from t1 where year<2003 and team="blue" or product="hats"' 27 | jd'reads from t1 where year<2000' 28 | 29 | jd'reads sum sales from t1' 30 | jd'reads alias:sum sales from t1' 31 | jd'reads sum over sales:sum sales from t1' 32 | jd'reads sum sales by year from t1 order by year' 33 | jd'reads sum sales by team from t1' 34 | jd'reads sum sales by product from t1' 35 | jd'reads sum sales by team,product from t1' 36 | jd'reads sum sales by year,team from t1 order by year' 37 | jd'reads sum sales by year from t1' 38 | jd'reads sum sales,max sales by year from t1' 39 | s=: 'reads sum over sales:sum sales , max over sales:max sales by year from t1' 40 | jd s 41 | jd s,' order by sum over sales' NB. order by uses alias 42 | jd s,' order by max over sales' 43 | 44 | NB. worthwhile skimming other basic tutorials 45 | NB. do advanced tutorials as questions come up 46 | NB. ptable tutorial for tables with many rows (300e6 to billions) 47 | -------------------------------------------------------------------------------- /tutorial/intro_b_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. update/upsert/delete/join 2 | jdadminnew'test' NB. create new db (~tmep/jd/test) 3 | d=: (4#2002 2003 2001);(?12$1000);(>12$2#'shoes';'hats');>12$'blue';'red' 4 | d=: ('year';'sales';'product';'team'),.d 5 | jd'createtable /pairs t1 ';,d NB. col names, types, shapes from pairs 6 | jd'reads from t1' 7 | jd'insert t1';'year';2004;'sales';999;'product';'socks';'team';'red' 8 | jd'reads from t1 where year>=2003' 9 | jd'reads *,jdindex from t1' NB. jdindex is table row index 10 | d=: jd'read jdindex,sales from t1 where year=2002' 11 | 'index sales'=: ,{:"1 d 12 | index 13 | sales 14 | jd'reads from t1 where year=2002' 15 | jd'update t1';index;'sales';sales+1 16 | jd'reads from t1 where year=2002' 17 | 18 | jd'reads from t1 where year in (2002,2005)' 19 | p=: 'year';2002 2005;'sales';5000 6000;'product';(>'shoes';'hats');'team';>'blue';'blue' 20 | NB. upsert uses key (year product team) to update existing row or insert new row 21 | jd'upsert t1';'year product team';p 22 | jd'reads from t1 where year in (2002,2005)' 23 | 24 | jd'reads from t1 order by year' 25 | jd'delete t1 year=2001 or product="socks"' 26 | jd'reads from t1 order by year' 27 | 28 | jd'createtable t2 team byte 4,contact byte 10' 29 | jd'insert t2';'team'; (>'blue';'red');'contact';>'4161231234';'7051231235' 30 | jd'reads from t2' 31 | jd'ref t1 team t2 team' NB. join t1 to t2 32 | jd'info table' 33 | jd'info ref' NB. table t1 col jdref_team_t2_team joins t1 to t2 34 | jd'reads from t1,t1.t2' 35 | jd'reads sum sales,first t2.contact by year,team from t1,t1.t2 order by sales' 36 | jd'reads from t1,t1.t2 where t2.contact="7051231235"' 37 | -------------------------------------------------------------------------------- /tutorial/intro_c_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. admin/create/drop 2 | jdadminnew'test' NB. create new db (~temp/jd/test) 3 | jd'createtable f' 4 | jd'createcol f a int' 5 | jd'insert f';'a';i.3 6 | jd'reads /lr from f' NB. reads with /lr labeled row option 7 | jdadmin 0 NB. db unavailable 8 | ''jdae'read from f' 9 | 10 | NB. you could exit J, shutdown, and relax 11 | NB. then start it all up and access the db 12 | 13 | jdadmin'test' NB. admin for db test that must already exist 14 | jd'read from f' 15 | jdadminnew'fubar' NB. create new db (~temp/jd/fubar) 16 | jd'createtable /pairs f';'b';'zxcv';'c';i.4 17 | jd'read from f' 18 | jdadmin'test' NB. access test again 19 | jd'read from f' 20 | jdadmin'fubar' 21 | jd'read from f' 22 | jd'dropcol f c' 23 | jd'read from f' 24 | jd'info table' 25 | jd'droptable f' 26 | jd'info table' 27 | jd'dropdb' 28 | jdadmin'test' 29 | jd'reads from f' 30 | jdadmin etx 'fubar' NB. error as it does not exist 31 | 13!:12'' 32 | jdadmin 0 NB. remove all admin - no db available 33 | -------------------------------------------------------------------------------- /tutorial/intx_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'gen test f 5' 4 | jd'info schema' 5 | jd'intx f int intx' NB. convert int col to int1 col 6 | jd'info schema' 7 | -------------------------------------------------------------------------------- /tutorial/jcs_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. Jd server based on jcs - assumes familiarity with jcs and Jd 2 | NB. see: Client/Server - parallel each - parallel jobs : ~addons/net/jcs/jcs.ijt 3 | load JDP,'server/jcs_server.ijs' NB. jcs server utilities 4 | require'~addons/net/jcs/jcs.ijs' 5 | '~addons/net/jcs must be updated'assert 0~:nct_jcs_ :: 0: '' 6 | 7 | version_jcs_'' NB. zmq version - error if problems with zmq installation 8 | 9 | f=. '~temp/sa.ijs' 10 | jcs_start_fix f;65200;'su:1234';'jdadminnew''sa_test''' 11 | fread f 12 | NB. 65200 binds localhost - '*:65200' binds any 13 | jcs_start f 14 | 15 | sa=. jcsc 65200 16 | jdaccess__sa 'sa_test u/p' 17 | jd__sa'gen test f 2' 18 | jd__sa'reads from f' 19 | 'unsupported'jdae__sa'info xxx' 20 | jdlast 21 | jdlasty 22 | 23 | 0 : 0 24 | start a jconsole task to connect to this server 25 | load'~addons/net/jcs/jcs.ijs' 26 | sa=. jcsc 65200 27 | jdaccess__sa'sa_test u/p' 28 | jd__sa'reads from f' 29 | destroy__sa'' 30 | ) 31 | 32 | destroy__sa'' 33 | sa=. jcsc 65200 NB. connect to the server again 34 | jdaccess__sa 'sa_test u/p' 35 | jd__sa'reads from f' 36 | 37 | su__sa=: 'su:1234' NB. must match su in server - superuser 38 | runsu__sa 'jdadminnew''fubar''' NB. create new db in server 39 | 40 | jdaccess__sa'fubar u/p' 41 | jd__sa'gen test f 4' 42 | jd__sa'reads from f' 43 | 44 | jdaccess__sa'sa_test u/p' 45 | jd__sa'reads from f' 46 | 47 | kill__sa'' NB. exit server (superuser) 48 | -------------------------------------------------------------------------------- /tutorial/jctask_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. Lab: run jconsole tasks 2 | 3 | require JDP,'server/jctask.ijs' 4 | 5 | report_jctask_'' NB. report of jc tasks 6 | 7 | NB. if tasks are listed that you care about 8 | NB. do not do the next step as it kills all jc tasks 9 | 10 | clean_jctask_'all' 11 | 12 | NB. jconsole terminal task 13 | NB. new jconsole terminal task will have focus 14 | NB. click tutorial window to recover focus 15 | tid=. run_jctask_ 't';'test1';'echo i.3 4' NB. type;description;sentences 16 | 17 | tid NB. unique jc task identifer 18 | dir '~temp/jctask','/',tid NB. tid is folder with task info 19 | tid get_jctask_ 'start.ijs' NB. script used to start the task 20 | tid get_jctask_ 'type' 21 | tid get_jctask_ 'description' 22 | tid get_jctask_ 'pid' NB. task host pid 23 | report_jctask_'' 24 | 25 | NB. in jconsole terminal window created in a previous step, run sentence: exit'' 26 | report_jctask_'' NB. note red (read/exit/date) changed from run to exit 27 | 28 | tid=. run_jctask_ 't';'test2';'echo i.23' NB. type;description;sentences 29 | report_jctask_'' 30 | 31 | NB. close the new jconsole terminal by clicking the x close button 32 | report_jctask_'' NB. note red changed from run to dead 33 | 34 | tid=: run_jctask_ 't';'test3';'a=: i.23',LF,'echo a' NB. LF delimited list of sentences 35 | report_jctask_'' 36 | killtid_jctask_ tid 37 | report_jctask_'' 38 | 39 | tid=. run_jctask_ 't';'test4';<'i.5';'exit''''' NB. boxed list of sentences 40 | report_jctask_'' NB. note test4 red is exit 41 | 42 | NB. jconsole redirect task (no terminal) 43 | tid=. run_jctask_ 'r';'test5';'echo i.5' 44 | tid get_jctask_ 'start.ijs' NB. r type has last sentence of: exit'' 45 | report_jctask_'' 46 | tid get_jctask_ 'out' NB. redirected task outputt 47 | 48 | report_jctask_'' 49 | clean_jctask_'all' NB. kill run tasks and remove all jctask folders 50 | -------------------------------------------------------------------------------- /tutorial/jds_client_tut.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | accessing a jds server from J/wget/curl 3 | jdrt'jds' NB. prerequisite tutorial 4 | ) 5 | 6 | load JDP,'server/jds_client/jds_client.ijs' 7 | 8 | 9 | PORT=: 65220 NB. port with jds service 10 | 11 | check_jds PORT 12 | pidport_jport_'' 13 | pidfromport_jport_ PORT 14 | 15 | 0 : 0 16 | config client to use server: host port fin fout dan u/p 17 | fin - format for input arrays - json or jbin (3!:1) 18 | fout - format for outut arrays - json or jbin 19 | dan - data access name (jdaccess) 20 | u/p - user/pswd 21 | ) 22 | jds_client_config 'localhost';PORT;'jbin';'jbin';'jds_db_a';'u/p' 23 | 24 | msrx'info summary' NB. http data to send to jds server 25 | msr'info summary' 26 | msr'droptable f' 27 | msr'createtable f' 28 | msr'createcol f a int' 29 | msr'read from f' 30 | msrx'insert f';'a';777 31 | msr'insert f';'a';777 32 | msr'insert f';'a';888 999 33 | msr'read from f' 34 | 35 | 0 : 0 36 | try the access from another J task 37 | this task does load Jd and only loads jds_client.ijs 38 | 39 | start jconsole 40 | load '~addons/data/jd/server/jds_client/jds_client.ijs' 41 | jds_client_config 'localhost';65220;'jbin';'jbin';'jds_db_a';'u/p' 42 | msr'read from f' 43 | ) 44 | 45 | NB. change fin/fout to json 46 | jds_client_config 'localhost';PORT;'json';'json';'jds_db_a';'u/p' 47 | msr'info schema' 48 | msr'read from f' 49 | msr'insert f';'a';5 4 3 NB. data is sent to jds server in json format 50 | msr'read from f' 51 | 52 | NB. the jds server can be accessed by tools such as wget and curl 53 | NB. wget or curl will fail (interface error) if not installed 54 | NB. if wget fails, try curl and vice versa 55 | wgetx'read from f' NB. host command that will be run 56 | wget etx 'read from f' 57 | wget etx 'insert f';'a';999 888 777 58 | 59 | wget etx 'read from f' 60 | 61 | curlx'info summary' 62 | curl etx 'info summary' 63 | -------------------------------------------------------------------------------- /tutorial/jds_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. how to run a Jd server task with jds 2 | 3 | require JDP,'server/jds/jds_tools.ijs' 4 | 5 | spath=: '~temp/jdserver' NB. path to server folders 6 | PORT=: 65220 7 | LOGFILE=: spath,'/jds/',(":PORT),'/log.log' 8 | LOGLEVEL=: 0 NB. 0 for all, 1 for most, ..., 9 for only important 9 | DBS=: 'jds_db_a,jds_db_b' NB. dbs to jdadmin - "s around as required 10 | NB. server init does jdadmin for each db - 'new'jdadmin if db does not exist 11 | 12 | NB. next step creates scripts for managing the jds server on PORT 13 | [path=: create_jds spath;PORT;LOGFILE;LOGLEVEL;DBS 14 | run_sh_bat=: ;('Win'-:UNAME){'run.sh';'run.bat' 15 | dir path 16 | fread path,'run.ijs' NB. ijs script to start this server 17 | fread path,run_sh_bat NB. host shell script to run this server 18 | fread path,'run.txt' NB. fork_jtask_ arg to start this server 19 | 20 | killport_jport_ PORT NB. kill task (if any) serving port 21 | check_jds PORT NB. start jds server on PORT 22 | pidport_jport_'' NB. table of pids and ports 23 | pidfromport_jport_ PORT NB. pid of server task - _1 if start failed 24 | fread path,'logstd.log' NB. stdout/stderr log 25 | fread path,'log.log' NB. event log 26 | killport_jport_ PORT NB. kill the server 27 | 28 | 0 :0 29 | you might want to run the jds server in a terminal window 30 | so you see log messages that will help in debugging 31 | to run the jds server in a terminal window: 32 | 33 | killport PORT NB. kill currrent server 34 | path,runit NB. command to paste into terminal to run server 35 | ) 36 | 37 | NB. setup 65221 server for use by node tutorial 38 | create_jds spath;65221;(spath,'/jds/65221/log.log');LOGLEVEL;'jds_db_c,jds_db_d' 39 | 40 | 0 : 0 41 | jdrt'jds_client' NB. how a client can use a server 42 | jdrt'node' NB. how to set up a node server to access a server 43 | ) 44 | -------------------------------------------------------------------------------- /tutorial/json_tut.ijs: -------------------------------------------------------------------------------- 1 | jdjson=: 3 : 0 2 | if. L.y do. 3 | jds'json json;',(;{.y),LF,jsonenc}.y 4 | else. 5 | jds'json json;',y 6 | end. 7 | ) 8 | 9 | 'new'jdadmin'json' 10 | jdjson 'createtable f' NB. {} json empty dictionary - instead of J empty array i.0 0 11 | jdjson 'createcol f i int' 12 | jdjson 'createcol f b byte' 13 | jdjson 'createcol f b4 byte 4' 14 | 15 | jdjson 'insert f';'i';2;'b';'z';'b4';'abc' 16 | jdjson 'read from f' 17 | 18 | jdjson 'insert f';'i';2;'b';'z';'b4';'abc' 19 | jdjson 'insert f';'i';3 4 5;'b';'abc';'b4';3 4$'abcdefghijkl' 20 | jdjson 'read from f' NB. json always returns a dictionary 21 | jdjson 'reads from f' NB. reads is an error 22 | 23 | jdjson'insert f';'i';2;'b';'z';'b4';'abcde' NB. error returned as json dictionary 24 | jdlast 25 | 26 | jdjson'delete f';'jdindex>_1' 27 | jdjson'info summary' 28 | 29 | jdjson'insert f';'i';3 4;'b';'qw';'b4';2 3$'dgg' 30 | jdjson'read from f' 31 | jdjson'insert f';'i';2;'b';'z';'b4';2 5$'x' NB. error bad shape 32 | 33 | jdjson'delete f';'jdindex>_1' 34 | jdjson'createcol f v varbyte' 35 | jdjson'insert f';'i';23;'b';'x';'b4';'abc';'v';<<'abc' 36 | jdjson'read from f' 37 | jdjson'insert f';'i';23 24;'b';'xy';'b4';(2 3$'abcdef');'v';<'qwer';'asdfasdf' 38 | jdjson'read from f' 39 | jdjson'info summary' 40 | 41 | jdjson'createtable g' 42 | jdjson'read from g' 43 | 44 | NB. json result is a dictionary 45 | NB. dictionary can be converted to a list 46 | NB. sometimes you need the list encoding and not the dictionary 47 | NB. e.g if you want to feed the json result of a read to an insert 48 | 49 | lfromd=: 3 : 0 NB. json - list from dictionary 50 | d=. }.}:<;.2 y,LF 51 | '[',']',~;(d i.each ':') (','"_`[`])} each d 52 | ) 53 | 54 | d=. jdjson 'read from f' 55 | [lfromd d 56 | NB. t already has the json encoded pairs so can't use jdjson helper verb 57 | jds 'json json;insert f',LF,lfromd d 58 | jdjson 'read from f' 59 | 60 | jdjson'update f';'i=23';'b';'+' 61 | jdjson'read from f' 62 | 63 | jdjson'delete f';'i=23' 64 | jdjson'read from f' 65 | 66 | jdjson'list version' 67 | -------------------------------------------------------------------------------- /tutorial/key_tut.ijs: -------------------------------------------------------------------------------- 1 | jdadminnew'tutorial' 2 | jd'gen test f 5' 3 | jd'key f';'int';104 102 4 | jd'key f';'int';104 102 120 5 | jd'key f';'int';104 102 120;'byte';'ECx' NB. _1 for not found 6 | jd'key /in f';'int';104 102 120;'byte';'ECx' NB. sorted - no _1 7 | jd'key f';'boolean';1 NB. last row that matches 8 | jd'key /in f';'boolean';1 NB. all rows that match 9 | -------------------------------------------------------------------------------- /tutorial/mtm_tut.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | preliminary version of mtm - multiple task manager - subject to change! 3 | 4 | this tutorial has some pointers, but is primarily a place holder 5 | ) 6 | 7 | load JDP,'mtm/mtm_man.ijs' 8 | jd_mtm_overview_jman_ 9 | jd_mtm_demo_jman_ 10 | -------------------------------------------------------------------------------- /tutorial/northwind_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadmin'northwind' 3 | jd'reads Description from Categories' 4 | jd'reads Address,City from Suppliers where Country="UK"' 5 | jd'reads ProductName,Categories.CategoryName from Products,Products.Categories where Categories.CategoryName="Beverages"' 6 | jd'reads Products.ProductName,Suppliers.City from Products,Products.Suppliers where Suppliers.CompanyName="Exotic Liquids"' 7 | jd'reads ProductName,UnitPrice from Products where UnitPrice<10' 8 | jd'reads Suppliers.Country,Categories.CategoryName from Products,Products.Categories,Products.Suppliers where Suppliers.Country="UK"' 9 | jd'reads CustomerID from Customers where Country="UK"' 10 | jd'reads Country,HireDate from Employees where Country<>"UK" and HireDate>19930000' 11 | jd'reads from Shippers' 12 | jd'reads Customers.Country,Employees.Country,Shippers.ShipperID from Orders,Orders.Customers,Orders.Employees,Orders.Shippers where Customers.Country="UK" and Employees.Country="UK" and Shippers.ShipperID=1' 13 | jd'reads Customers.Country,OrderDetails.Quantity from OrderDetails,OrderDetails.Orders.Customers where Customers.Country="UK" and OrderDetails.Quantity>60' 14 | jd'reads count:count Country by Country from Suppliers' 15 | jd'reads min:min UnitsInStock,max:max UnitsInStock by Suppliers.Country from Products,Products.Suppliers' 16 | -------------------------------------------------------------------------------- /tutorial/pairs_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'gen test f 2' 4 | jd'reads from f' NB. labeled cols 5 | jd'read from f' NB. labeled rows (same as reads /lr) 6 | 7 | 0 : 0 8 | labeled cols can be convenient for display to the user 9 | but loses some info about the actual data 10 | everything is a matrix even if the actual data is a list 11 | varbyte data is opened 12 | ) 13 | 14 | $each{:jd'reads from f' NB. reads data is 2 by N 15 | $each{:"1 jd'reads /lr from f' NB. read data has actual shape 16 | 17 | 0 : 0 18 | pairs - list of name,value pairs 19 | pairs are args to insert, update, and other ops 20 | ) 21 | 22 | [d=: 'a';2 3;'b';2 3$'abcdef' NB. list of name,value pairs 23 | jd'createtable /replace g a int,b byte 3' 24 | jd'insert g';d 25 | [r=: jd'read from g' NB. result 26 | ,r NB. ravel of read result is pairs 27 | 28 | jd'insert g';'a';23;'b';3 3$'z' NB. data extends 29 | jd'read from g' 30 | jd'insert g';'a';23;'b';3 2$'a' NB. byte N col extends with blanks 31 | jd'read from g' 32 | 'bad shape'jdae'insert g';'a';23;'b';3 5$'a' NB. byte N data not discarded 33 | -------------------------------------------------------------------------------- /tutorial/pandas_clean_tut.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | a major problem with using a table is if the data is not clean 3 | missing values or bad values (data entry finger slip) 4 | garbage in, garbage out 5 | ) 6 | 7 | NB. study tutorial pandas_load first 8 | 9 | load JDP,'tools/pandas/pandas.ijs' 10 | pystatus_jd_'' 11 | getjdfile_jd_'yellow_tripdata.parquet' 12 | [file=: jpath'~temp/jdfiles/yellow_tripdata.parquet' 13 | pandas_load_jd_ 'pandas_db';'pandas_table';'all';'read_parquet';file;'' 14 | 15 | jd'info summary pandas_table' 16 | jd'reads from pandas_table where jdindex<3' 17 | 18 | jd'read max fare:max fare_amount , avg fare:avg fare_amount from pandas_table' 19 | NB. it seems unlikely that the max is valid - and it will have affected the avg 20 | 21 | jd'read max riders:max passenger_count , min riders: min passenger_count from pandas_table' 22 | NB. max 9 makes sense, but min is the value (IMIN_jd) that Jd uses for missing int data 23 | 24 | 0 : 0 25 | cleaning the table is necessary for it to be useful and is a hard problem 26 | script JDP,tools/pandas/clean.ijs has a basic tool that is adequate 27 | for this table and could be a starting point for other tables 28 | requirements can vary widely between tables 29 | ) 30 | 31 | load JDP,'tools/pandas/clean.ijs' 32 | 33 | 0 : 0 34 | report from next sentence has a line for each table column 35 | Na column reports the count of missing data 36 | passenger_count has 71503 missing values 37 | mid col has the median value (value from the middle of the sorted data) 38 | out- col has how many values were <<.avg%200 39 | fare_amount has 12733 values that are less than that threshold 40 | note that this is the same value as neg values in that column 41 | out+ col has count of values ><.300*avg 42 | trip_distance has max 306159 and out+ 68 - likely data entry problems 43 | ) 44 | 45 | clean'pandas_table';200;300;'' 46 | 47 | NB. the next step updates the table by replacing Na,out-,out+ data with the mid value 48 | NB. note that store_and_fwd_flag byte data Na values are not adjusted 49 | clean'pandas_table';200;300;'change' 50 | jd'read max fare:max fare_amount , avg fare:avg fare_amount from pandas_table' 51 | jd'read max riders:max passenger_count , min riders: min passenger_count from pandas_table' 52 | -------------------------------------------------------------------------------- /tutorial/pandas_install_tut.ijs: -------------------------------------------------------------------------------- 1 | load JDP,'tools/pandas/pandas.ijs' 2 | 3 | pystatus_jd_'' NB. error is OK 4 | 5 | 0 : 0 6 | if previous step ran without error, your install is OK and 7 | you can skip this tutorial and run the pandas_load tutorial 8 | ) 9 | 10 | 0 : 0 11 | linux distributions can usually install with apt (or yum) 12 | with something like the following: 13 | > sudo apt update 14 | > sudo apt install python3 15 | > sudo apt install python3-pip 16 | 17 | windows app store version slim and you need a full install 18 | windows should install the full python3 package from: 19 | https://www.python.org/downloads/ 20 | 21 | mac should install from the web site or with homebrew 22 | ) 23 | 24 | 0 : 0 25 | value in '~config/python3.cfg' is the python3 binary to run 26 | default value of python3 is set for unix 27 | default value of py is set for windows 28 | if the default value is not right, you will have set it manually 29 | ) 30 | 31 | pystatus_jd_'' NB. you have work to do if there is an error 32 | -------------------------------------------------------------------------------- /tutorial/pandas_write_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. jdrt'pandas_load' NB. prerequisite tutorial 2 | 3 | load JDP,'tools/pandas/pandas.ijs' 4 | 5 | jdadmin :: ('new'&jdadmin) 'pandas_db' 6 | 7 | bld=: 3 : 0 8 | jd'droptable foo' 9 | jd'createtable foo' 10 | jd'createcol foo i int' 11 | jd'createcol foo b byte' 12 | jd'createcol foo b3 byte 3' 13 | jd'createcol foo f float' 14 | jd'createcol foo tn edatetimen' 15 | jd'insert foo';'i';1 2 3;'b';'abc';'b3';(3 3$'abcd');'f';(2.3+i.3);'tn';694312540000000000 694312423000000000 694313601000000000 16 | jd'read from foo' 17 | ) 18 | 19 | bld'' 20 | 21 | NB. database ; table ; count ; op ; file ; parms 22 | NB. database - jdadmin arg 23 | NB. table - Jd table to write to file 24 | NB. count - rows to write - 'all' for all rows 25 | NB. op - pandas op 26 | NB. file - file to write 27 | NB. parms - parameters to pandas op 28 | 29 | [pandas_write_ops_jd_ 30 | 31 | pandas_write_jd_ 'pandas_db';'foo';'all';'to_csv';'~temp/tst.csv';'' 32 | fread'~temp/tst.csv' 33 | 34 | pandas_write_jd_ 'pandas_db';'foo';'all';'to_parquet';'~temp/tst.parquet';'' 35 | 100{.fread'~temp/tst.parquet' 36 | -------------------------------------------------------------------------------- /tutorial/quandl_ibm_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | 0 : 0 3 | www.quandl.com is a financial data source 4 | with a free signup you can easily download interesting sample csv files 5 | with a paid subscription, you have whatever you want at your fingertips 6 | 7 | for this tutorial the EOD-IBM.csv file was downloaded and put at: 8 | www.jsoftware.com/download/jdcsv/eod-ibm.zip 9 | 10 | download (only done first time) can take a minutes 11 | ) 12 | 13 | require'plot' 14 | 15 | CSVFOLDER=: '~temp/jd/csv/ibm/' NB. folder for csv files 16 | jdcreatefolder_jd_ CSVFOLDER NB. ensure folder exists 17 | jdadminx'ibm' NB. new db ~temp/jd/ibm 18 | fcsv=: 'EOD-IBM.csv' 19 | 20 | NB. next advance does download (if required) and takes minutes 21 | NB. zip downloaded from www.jsoftware.com/jdcsv and unzipped in CSVFOLDER 22 | getcsv_jd_ fcsv 23 | fsize CSVFOLDER,fcsv 24 | 25 | NB. csvprobe sets set initial .cdefs metadata and reads first 20 rows 26 | jd'csvprobe /replace ',fcsv 27 | NB. looking at data it is clear first row is column names 28 | NB. subsequent rows look like data (rather than more headers) 29 | 30 | NB. csvcdefs will set .cdefs metadata - /h 1 iindicates first row has col names 31 | jd'csvcdefs /replace /h 1 ',fcsv 32 | 33 | NB. csvcdefs sets metadata but based on only first 5000 rows 34 | NB. csvscan wll scan entire file to get max col widths from all rows 35 | jd'csvscan ',fcsv 36 | 37 | fread CSVFOLDER,'EOD-IBM.cdefs' NB. final metadata 38 | 39 | NB. csvrd will load csv file into Jd table buslic 40 | jd'csvrd ',fcsv,' ibm' 41 | jd'csvreport' 42 | 43 | jd'reads count Date from ibm' 44 | [minmax=.,;{:jd'reads min Open,max Open from ibm' 45 | [t=. (":minmax)rplc' ';',' 46 | jd'reads Date,Open from ibm where Open in (',t,')' 47 | 48 | stockplot=: 3 : 0 49 | if. IFTESTS_jd_ do. return. end. NB. avoid plot in jdtests_jd_'' 50 | title=. 'title IBM Open prices',,' ',.;{:jd'reads min Date,max Date from ibm' 51 | title plot |.>{:{.jd'read Open from ibm' 52 | ) 53 | 54 | stockplot'' -------------------------------------------------------------------------------- /tutorial/read_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | 0 : 0 3 | read result is table data with row labels 4 | read is the same as reads /lr 5 | see reads tutorial 6 | ) -------------------------------------------------------------------------------- /tutorial/reads_aggregation_tut.ijs: -------------------------------------------------------------------------------- 1 | jdadmin'sandp' 2 | 3 | jd'reads from j' 4 | jd'reads count jname from j' 5 | jd'reads count jname by city from j' NB. by col 6 | jd'reads cnt:count jname by city from j' NB. alias 7 | 8 | jd'reads from p' 9 | jd'reads avg weight by color from p' 10 | jd'reads avg_wt:avg weight by color from p' 11 | jd'reads avg_wt:avg weight,max_wt:max weight by color from p' 12 | jd'reads avg_wt:avg weight,max_wt:max weight by city,color from p' 13 | 14 | NB. aliases are particularly useful with different aggregations to a column 15 | 16 | NB. full syntax for an aggregation is: 17 | NB. Aggregation(,Aggregation)* by column(,column)* 18 | NB. Aggregation: [alias:] aggregator column 19 | 20 | jd'info agg' NB. db aggregation functions 21 | 22 | NB. first and last are the first and last occurrences in the table 23 | NB. sorting is after aggregation, so not affected by "order by" 24 | 25 | jdadminx'test' 26 | jd'createtable f a int' 27 | jd'insert f a';_1,(i.4),_1 28 | jd'reads from f' 29 | jd'reads count a from f' 30 | 31 | NB. db custom aggregations fns are defined in db custom.ijs 32 | NB. define avgnonneg to ignore negative values (null) in getting the average 33 | 34 | custom=: 0 : 0 35 | aggavgx=: 3 : '(+/t)%#t=. (y>:0)#y' 36 | aggavgx addagg 'avgnonneg' 37 | ) 38 | 39 | custom fwrite '~temp/jd/test/custom.ijs' 40 | jdloadcustom_jd_'' NB. load changes 41 | 42 | jd'info agg' 43 | jd'reads avg-non-neg:avgnonneg a,avg:avg a from f' 44 | -------------------------------------------------------------------------------- /tutorial/reads_basic_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadmin'sandp' NB. use sandp demo database 3 | 4 | jd'reads from p' 5 | 6 | NB. reads is complicated and this tutorial has only simple examples 7 | NB. run reads_... for details on specific features 8 | 9 | NB. reads [OPTIONS] [SELECT] from FROM [where WHERE] [order by ORDER BY] 10 | 11 | NB. OPTIONS 12 | jd'reads /lr from p' NB. labeled rows 13 | 14 | NB. SELECT 15 | jd'reads from p' 16 | jd'reads color,weight from p' 17 | jd'reads avg weight from p' NB. aggregation 18 | jd'reads avg wt:avg weight from p' NB. alias 19 | jd'reads avg wt:avg weight by color from p' NB. by 20 | 21 | 22 | NB. FROM 23 | jd'reads from p' 24 | jd'reads from sp' 25 | jd'reads from sp,sp.p' NB. sp has been joined to p (sp.pid -> p.pid) 26 | 27 | NB. WHERE 28 | jd'reads from p' 29 | jd'reads from p where color="Red"' 30 | jd'reads from p where color="Red" and weight>12' 31 | 32 | NB. ORDER BY 33 | jd'reads from p' 34 | jd'reads from p order by city' 35 | jd'reads from p order by city desc' 36 | -------------------------------------------------------------------------------- /tutorial/reads_clauses_tut.ijs: -------------------------------------------------------------------------------- 1 | jdadmin'sandp' 2 | 3 | jd'reads from j' 4 | jd'reads from p' 5 | jd'reads from s' 6 | jd'read from s' NB. read is reads flipped 7 | 8 | NB. a query can have SELECT, FROM, WHERE, and ORDER 9 | NB. [SELECT] from FROM [where WHERE] [order by ORDER] 10 | NB. everything but FROM is optional 11 | 12 | jd'reads * from j' NB. SELECT * is all cols 13 | jd'reads city from j' NB. SELECT city 14 | jd'reads city,color from p' NB. SELECT city and color 15 | 16 | NB. WHERE is a predicate for the rows to satisfy 17 | NB. it consists of relations like qty>100 or city="Paris" 18 | NB. which can be preceded by "not" and combined by "and" and "or" 19 | NB. operations execute as in J (right to left and can be parenthesized) 20 | NB. cols of compatible type can be compared 21 | 22 | jd'reads from p where pname="Screw" or weight<14 and city="London"' 23 | jd'reads from p where pname="Screw" or (weight<14 and city="London")' 24 | jd'reads from p where (pname="Screw" or weight<14) and city="London"' 25 | jd'reads from s where city in ("Athens","London")' 26 | jd'reads from p where color is "Red"' 27 | jd'reads from p where color = "Red"' 28 | jd'reads from p where color eq "Red"' 29 | [r=. jd'reads pname from p where pname like "C"' NB. regex 30 | assert 'CamCog'-:' '-.~,>{:r 31 | [r=. jd'reads pname from p where pname likeci "C"' NB. regex - case insensitives 32 | assert 'ScrewScrewCamCog'-:' '-.~,>{:r 33 | [r=. jd'reads pname from p where pname unlike "C"' NB. regex 34 | assert 'NutBoltScrewScrew'-:' '-.~,>{:r 35 | [r=. jd'reads pname from p where pname unlikeci "C"' NB. regex 36 | assert 'NutBolt'-:' '-.~,>{:r 37 | jd'reads from p where weight range (11,14)' NB. In range, inclusive 38 | jd'reads from p where weight range (11,12,16,18)' NB. Multiple ranges 39 | 40 | NB. ORDER is col(s) to order by 41 | NB. col order ascending unless followed by "desc" 42 | 43 | jd 'reads weight,city from p order by weight,city' 44 | jd 'reads weight,city from p order by weight,city desc' 45 | jd 'reads weight,city from p order by weight desc,city' 46 | jd 'reads weight,city from p order by weight desc,city' 47 | 48 | jd 'reads weight,city from p where weight>=17 or city is "London" order by city, weight desc' 49 | -------------------------------------------------------------------------------- /tutorial/reads_option_table_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. reads result can be used to create a table 2 | jdadmin'sandp' 3 | jd'reads from sp' 4 | jd'reads /table abc from sp where sid="S2"' NB. create table abc (replace if already exists) 5 | jd'reads from abc' 6 | jd'reads from sp,sp.p' 7 | jd'reads from sp,sp.p where sp.sid="S1" and p.color ne "Green"' 8 | jd'reads /table viewall from sp,sp.p where sp.sid="S1" and p.color ne "Green"' 9 | jd'reads from viewall' NB. . in col names replaced by __ 10 | jd'reads /table viewsome sp.sid,p.color,p.city from sp,sp.p where sp.sid="S1" and p.color ne "Green"' 11 | jd'reads from viewsome' 12 | -------------------------------------------------------------------------------- /tutorial/sandp_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. C.J.Date's suppliers and parts database (sandp) 2 | NB. data matches that in the SQLite addon 3 | 4 | jdadmin'sandp' 5 | jd'reads from j' 6 | jd'reads from p' 7 | jd'reads from s' 8 | jd'reads from sp' 9 | jd'reads from spj' 10 | jd'reads from sp where sid="S1" and qty > 250' 11 | jd'reads pid,qty from sp where pid="P2"' 12 | jd'reads sp.sid,sp.pid,sp.qty,s.city,p.city from sp,sp.s,sp.p where s.city=p.city' 13 | jd'reads job_cnt:count jname by city from j' 14 | jd'reads sum weight by color from p where city<>"Paris"' 15 | jd'reads sum weight by color,city from p' 16 | jd'reads avg weight by city,color from p' 17 | jd'reads avg weight,pone:first pid by where:city,color from p' 18 | jd'reads min_st:min status by city from s' 19 | jd'reads sum p.weight,min s.status by supplier:s.city from sp,sp.s,sp.p' 20 | jd'reads sum p.weight by supp:s.city,part:p.city from sp,sp.s,sp.p' 21 | jd'reads avg_qty:avg qty by supp:s.sname,part:p.city from spj,spj.s,spj.p order by avg_qty' 22 | jd'reads avg_qty:avg qty by part_col:p.color from spj,spj.p' 23 | -------------------------------------------------------------------------------- /tutorial/sed_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. section/employee/region database 2 | NB. Tables 3 | NB. r(rno,name;popul) 4 | NB. | 5 | NB. s(sno,sdiv,rnum;name) 6 | NB. | 7 | NB. e(ename,secnum) 8 | NB. | 9 | NB. t(tno,enum,tname,tsize) 10 | 11 | jdadmin'sed' 12 | jd'reads from r' 13 | jd'reads from s' 14 | jd'reads from e' 15 | jd'reads from t' 16 | jd'reads ename,s.sno,s.sname,r.rno,r.rname,s.name from e,e.s,s.r' 17 | jd'reads ename,s.sno,s.sname,r.rno,r.rname,s.name from e,e.s,s.r where r.rname="Rome"' 18 | jd'reads from e,e.s,s.r where r.rname="Rome"' 19 | jd'reads from e,e.s where s.sname="Auto"' 20 | jd'reads tno,ename,r.rname from t,t.e,e.s,s.r' 21 | -------------------------------------------------------------------------------- /tutorial/setget_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | NB. create new date col from existing datetime col 3 | 4 | NB. set does not handle dynamic cols that need updates 5 | NB. this is not a problem when set is done to a newly created col 6 | NB. but is a serious problem if used on a col in a ref 7 | 8 | jdadminx'test' 9 | jd'createtable';'f';'dt datetime' 10 | jd'insert';'f';'dt';20121212101010 20121111101010 11 | jd'createcol f d date' 12 | jd'reads from f' 13 | jd'set';'f';'d';<.10e6%~jd'get f dt' 14 | jd'reads from f' 15 | t=. jd'read from f' 16 | ('d'jdfrom_jd_ t)-:<.10e6%~'dt'jdfrom_jd_ t 17 | 18 | -------------------------------------------------------------------------------- /tutorial/sort_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | f=: 3 : 0 3 | jdadminnew'test' 4 | jd'createtable /pairs f';'a';((y?.y){y$i:3);'b';y$'abbcas' 5 | ) 6 | 7 | jd'reads from f' [ f 10 8 | jd'reads from f' [ jd'sort f a' 9 | jd'reads from f' [ jd'sort f a desc' 10 | jd'reads from f' [ jd'sort f a,b' 11 | jd'reads from f' [ jd'sort f a,b desc' 12 | -------------------------------------------------------------------------------- /tutorial/stock_data_tut.ijs: -------------------------------------------------------------------------------- 1 | require 'plot' 2 | require JDP,'tools/quandl.ijs' 3 | 4 | 0 : 0 5 | www.quandl.com is a source for free, historical, stock data 6 | 7 | you need a free quandl account and apikey to download data 8 | 9 | www.quandl.com 10 | top right: SIGN IN 11 | bottom right: CREATE ONE 12 | fill in form: SIGN UP FOR FREE 13 | 14 | welcome gives your apikey (also in your account settings) 15 | 16 | run following sentence (with your apikey) to put your key in config 17 | 18 | '????' fwrite '~config/quandl_apikey.txt' 19 | ) 20 | 21 | 'you need to set your apikey'assert 10<#fread'~config/quandl_apikey.txt' 22 | 23 | jdadminx'quandl' NB. create empty quandl db 24 | CSVFOLDER=: '~temp/jd/quandl/csv/' NB. folder for quandl csv files 25 | jdcreatefolder_jd_ CSVFOLDER 26 | 27 | NB. next step downloads the Proctor & Gamble csv file - can take a minute 28 | quandl_get'pg' 29 | fsize CSVFOLDER,'pg.csv' 30 | 31 | quandl_cdefs'pg' NB. analyze csv file to build cdefs metadata file 32 | fread CSVFOLDER,'eod.cdefs' 33 | 34 | quandl_load'pg' NB. load pg.csv into table pg 35 | jd'info table' 36 | jd'info schema' 37 | jd'reads #:count Date from pg' NB. number of rows in table 38 | jd'reads latest:first Date,oldest:last Date from pg' 39 | 'Open' quandl_plot'pg' 40 | 'Split'quandl_plot'pg' 41 | NB. note how the splits line up with open drops 42 | 43 | NB. next step downloads the IBM csv file - can take a minute 44 | quandl_get'ibm' 45 | quandl_load'ibm' NB. uses eod.cdefs file built earlier 46 | jd'info table' 47 | 'Volume'quandl_plot'ibm' 48 | -------------------------------------------------------------------------------- /tutorial/table_from_pairs_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. create table from pairs 2 | jdadminx'test' 3 | jd'createtable /pairs f';'a';'abc';'b';1 2 3;'c';3 4$'qwerty' 4 | jd'reads from f' 5 | 6 | NB. type implicit from simple examination of data 7 | jdadminx'test' 8 | jd'createtable /pairs f';'a';1 2;'b';'1990-06-02',:'1990-07-12' 9 | jd'reads from f' 10 | jd'info schema f' NB. b is byte 10 11 | 12 | NB. type explicit from name(type) 13 | jdadminx'test' 14 | jd'createtable /types /pairs f';'a(int)';1 2;'b(edate)';'1990-06-02',:'1990-07-12' 15 | jd'reads from f' 16 | jd'info schema f' NB. b is edate 17 | 18 | NB. read[s] can provide explicit type 19 | [jd'read /types from f' 20 | 21 | NB. create table from pairs read from another table 22 | NB. read result is pairs (just needs ,) 23 | NB. reads result is less convenient and the format has lost info (shape and varbyte) 24 | jdadmin'sandp' 25 | [d=. jd'read /types sp.sid,sp.pid,sp.qty,s.city,p.city from sp,sp.s,sp.p where s.city=p.city' 26 | jd'createtable /types /pairs test';,d 27 | jd'reads from test' NB. . replaced by __ 28 | jd'droptable test' 29 | -------------------------------------------------------------------------------- /tutorial/table_table_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. table-table tutorial 2 | NB. tableinsert - tablecopy - tablemove 3 | 4 | bld=: 3 : 0 5 | jdadmin 0 6 | jdadminx'gdb' 7 | jd'gen test ga 2' 8 | jd'gen test gb 2' 9 | jdadminx'fdb' 10 | jd'gen test fa 3' 11 | jd'gen test fb 3' 12 | ) 13 | 14 | NB. tableinsert requires conformable tables - same col names/types 15 | bld '' 16 | jd'tableinsert fa ga gdb' 17 | assert 0 1 1 0 1-:,>{:jd'reads boolean from fa' 18 | 19 | bld'' 20 | jd'tablecopy new ga gdb' 21 | jd'tablecopy again ga gdb' NB. show that table ga is still in gdb ((just copied) 22 | jd'tablemove moved ga gdb' 23 | jd etx 'tablecopy nope ga gdb' 24 | assert 'invalid srcdb'-:,;1{jdlast 25 | t=. jd'reads from new' 26 | assert t-:jd'reads from again' 27 | assert t-:jd'reads from moved' 28 | 29 | NB. move table from src db to new table current db 30 | jdadmin 0 31 | jdadminx'src' 32 | jd'gen test b 2' 33 | jdadminx'snk' 34 | jd'gen test a 3' 35 | jd'tablemove new b src' NB. move table b in db src to db snk 36 | jd'reads from a' 37 | jd'reads from new' 38 | jdadmin'src' 39 | jd etx'reads from b' NB. assertion failure - table doesn't exist (it was moved) 40 | 41 | NB. copy table from src db to new table current db 42 | jdadmin 0 43 | jdadminx'src' 44 | jd'gen test b 2' 45 | jdadminx'snk' 46 | jd'gen test a 3' 47 | jd'tablecopy new b src' NB. move table b in db src to db snk 48 | jd'reads from a' 49 | jd'reads from new' 50 | jdadmin'src' 51 | jd'reads from b' 52 | 53 | NB. insert table from src db to table in current db 54 | jdadmin 0 55 | jdadminx'src' 56 | jd'gen test b 2' 57 | jdadminx'snk' 58 | jd'gen test a 3' 59 | jd'tableinsert a b src' NB. insert table b in src to table a 60 | jd'reads from a' 61 | -------------------------------------------------------------------------------- /tutorial/tablecopy_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jdadminnew'tutorial2' 4 | jd'createtable f a int' 5 | jd'insert f';'a';i.2 6 | jdaccess'tutorial' 7 | jd'tablecopy new f tutorial2' 8 | jd'reads from new' 9 | -------------------------------------------------------------------------------- /tutorial/tableinsert_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'createtable f a int' 4 | jd'insert f';'a';i.2 5 | jdadminnew'tutorial2' 6 | jd'createtable g a int' 7 | jd'insert g';'a';100+i.2 8 | jdaccess'tutorial' 9 | jd'tableinsert f g tutorial2' 10 | jd'reads from f' 11 | -------------------------------------------------------------------------------- /tutorial/tablemove_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jdadminnew'tutorial2' 4 | jd'createtable f a int' 5 | jd'insert f';'a';i.2 6 | jdaccess'tutorial' 7 | jd'tablemove new f tutorial2' 8 | jd'reads from new' NB. table f in dan example2 no longer exists 9 | -------------------------------------------------------------------------------- /tutorial/taxi_tut.ijs: -------------------------------------------------------------------------------- 1 | 0 : 0 2 | create table with data from downloaded csv file with taxi trip data 3 | 4 | page about the data is here: 5 | https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page 6 | 7 | download csv file: 8 | 'https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2018-01.csv' 9 | 10 | download (only done first time tutorial is run) can take a few minutes 11 | ) 12 | 13 | require'web/gethttp' 14 | 15 | PATH=: '~temp/jd/csv/yellow_tripdata/' 16 | NAME=: 'yellow_tripdata_2019-01.csv' 17 | 18 | taxi_get=: 3 : 0 19 | if. fexist PATH,NAME do. (PATH,NAME,' already downloaded') return. end. 20 | fn=: jpath '~temp/',NAME 21 | ('file';fn) gethttp 'https://s3.amazonaws.com/nyc-tlc/trip+data/',NAME 22 | jdcreatefolder_jd_ PATH 23 | (fread fn) fwrite PATH,NAME 24 | dir PATH 25 | ) 26 | 27 | NB. next step can take a few minutes if file is not already downloaded 28 | taxi_get'' NB. download csv file if not already downloaded 29 | 30 | NB. use csv_load utils to load csv 31 | csvprepare_jd_ 'yellow_tripdata';PATH,NAME 32 | 33 | NB. next step can take a few minutes 34 | csvload_jd_ 'yellow_tripdata';1 35 | 36 | jd'info summary' 37 | jd'info schema yellow_tripdata' 38 | 39 | jd'reads avg fare_amount by passenger_count from yellow_tripdata' 40 | -------------------------------------------------------------------------------- /tutorial/update_advanced_tut.ijs: -------------------------------------------------------------------------------- 1 | NB. how to update several records at once 2 | 3 | jdadminx'test' 4 | 5 | jd'droptable GC' 6 | jd'createtable GC';'ID int';'Name byte 20';'Town byte 30' 7 | jd'insert GC';'ID';10 14 17 18 22 42;'Name';(20{."1>'Porsche';'Mercedes';'BWM';'VW';'Opel';'Maybach');'Town';30{."1>'Stuttgart';'Stuttgart';'Munich';'Wolfsburg';'Ruesselsheim';'Where?' 8 | 9 | jd'reads from GC' 10 | 11 | NB. Single update of Town for Mercedes, ID=14 12 | jd'update GC';'ID=14';'Town';,:30{.'Stuttgart-Untertuerkheim' 13 | 14 | jd'reads from GC' 15 | 16 | jd'reads from GC order by ID' 17 | 18 | NB. Let us update the single column "town" for three records at once 19 | NB. IDs are, respectively: 42, 10, and 18 for Maybach, Porsche, VW 20 | ]upd_town=:'Town';3 30{.>'Sindelfingen';'Stuttgart-Zuffenhausen';'Wolfsburg-Autostadt' 21 | 22 | jd'update GC';'ID in 42, 10, 18';upd_town 23 | 24 | jd'reads from GC' 25 | 26 | empty 0 : 0 27 | Oops, that is not what we had in mind! 28 | when doing a bulk update it is mandatory to pass 29 | data to the update according to the 30 | sequence in which they show up in an unsorted read 31 | ) 32 | 33 | NB. Delete the whole table 34 | jd'delete GC';'ID<>0' 35 | 36 | NB. Build it again 37 | jd'insert GC';'ID';10 14 17 18 22 42;'Name';(20{."1>'Porsche';'Mercedes';'BWM';'VW';'Opel';'Maybach');'Town';30{."1>'Stuttgart';'Stuttgart';'Munich';'Wolfsburg';'Ruesselsheim';'Where?' 38 | jd'reads from GC' 39 | 40 | empty 0 : 0 41 | prior to updating data, read the affected records 42 | order of IDs in where does not affect order of records in result 43 | ) 44 | 45 | ]db=.jd 'read ID from GC where ID in 18,10,42' 46 | ]db=.jd 'read ID from GC where ID in 42,10,18' 47 | 48 | NB. Update records as before 49 | ]upd_town=:'Town';3 30{.>'Sindelfingen';'Stuttgart-Zuffenhausen';'Wolfsburg-Autostadt' 50 | 51 | NB. The update IDs are, respectively: 42, 10, and 18 52 | upd_IDs=.42 10 18 53 | 54 | sort_order=.upd_IDs i. 0 1{::db 55 | 56 | jd'update GC';'ID in 42, 10, 18';,(0{upd_town),.(({:jd'reads count a from f' 18 | assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f' 19 | 20 | NB. same as previous but with byte key col 21 | bld'' 22 | [newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'crbt';'e';'y' 23 | jd'upsert f';'d';newdata 24 | [d=. jd'reads from f' NB. updated 2 rows and inserted 2 rows 25 | assert 7=>{:jd'reads count a from f' 26 | assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f' 27 | 28 | NB. same as previous but with byteN key col 29 | bld'' 30 | [newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'x';'e';4 2$'ccxxbbqq' 31 | jd'upsert f';'e';newdata 32 | [d=. jd'reads from f' NB. updated 2 rows and inserted 2 rows 33 | assert 7=>{:jd'reads count a from f' 34 | assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f' 35 | 36 | bld'' 37 | [newdata=: 'a';2 1;'b';1000 1002;'c';2000 2002;'d';'x';'e';2 2$'ccbb' 38 | jd'upsert f';'a';newdata NB. update 2 rows 39 | [newdata=: 'a';5 6;'b';1001 1003;'c';2001 2003;'d';'x';'e';2 2$'xxqq' 40 | jd'upsert f';'a';newdata NB. insert 2 rows 41 | jd'reads from f' 42 | assert d-:jd'reads from f' 43 | 44 | 45 | NB. multiple keys 46 | bld'' 47 | newdata=: 'a';2 5 1 6;'b';2 1001 1 1003;'c';2000 2001 2002 2003;'d';'w';'e';'q' 48 | [d=. jd'upsert f';'a b';newdata 49 | assert 7=>{:jd'reads count a from f' 50 | assert 0 1 2 3 4 1001 1003= >{:{.jd'read b from f' 51 | 52 | NB. multiple keys int,byte 53 | bld'' 54 | [a=. jd'upsert f';'a d';newdata 55 | assert a-:d 56 | 57 | NB. multiple keys byte,byteN 58 | bld'' 59 | [a=. jd'upsert f';'a d';newdata 60 | assert a-:d 61 | -------------------------------------------------------------------------------- /tutorial/upsert_basic_tut.ijs: -------------------------------------------------------------------------------- 1 | 2 | jdadminnew'tutorial' 3 | jd'createtable /pairs f';'a';0 1 2 3 4;'b';23 24 25 26 27 4 | jd'reads from f' 5 | NB. upsert based on key a 6 | jd'upsert f';'a';'a';2 5 6;'b';66 77 88 NB. 1 update and 2 inserts 7 | jd'reads from f' 8 | jd'createcol f c byte';'abcdefg' 9 | jd'reads from f' 10 | NB. upsert based on key a b 11 | jd'upsert f';'a b';'a';2 5 6;'b';66 _100 1000;'c';'xyx' NB. 1 update and 2 inserts 12 | jd'reads from f' 13 | -------------------------------------------------------------------------------- /types/byte.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | coclass 'jdtbase' 3 | 4 | NB. ========================================================= 5 | coclass deftype 'byte' 6 | DATASIZE =: 1 7 | DATAFILL =: ' ' 8 | fixtype=: ,@boxopen [ throwif@:(2 ~: 3!:0) 9 | fixtext=: >@:fixstring_jdcolumn_ 10 | 11 | fixtypex=: 3 : 0 12 | ETYPE assert 2=3!:0 y 13 | y 14 | ) 15 | -------------------------------------------------------------------------------- /types/datetimes.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | NB. date, datetime, and time types 3 | coclass 'jdtint' 4 | coclass deftype 'timelike' 5 | 6 | DATAFILL =: -~2 7 | 8 | fixdatetime=: 4 :0 9 | fixtype_jdtint_ y 10 | ) 11 | 12 | NB. ========================================================= 13 | coclass deftype 'date' 14 | fixtype =: 'yyyymmdd'&fixdatetime 15 | 16 | NB. ========================================================= 17 | coclass deftype 'datetime' 18 | fixtype =: 'yyyymmdd.hhmmss'&fixdatetime 19 | 20 | -------------------------------------------------------------------------------- /types/epoch.ijs: -------------------------------------------------------------------------------- 1 | NB. Copyright 2014, Jsoftware Inc. All rights reserved. 2 | NB. epoch datetime 3 | coclass 'jdtint' 4 | 5 | coclass deftype 'elike' 6 | 7 | STATE=: STATE_jdcolumn_,;:'sep utc' 8 | 9 | sep=: ',' 10 | utc=: ' ' 11 | 12 | DATAFILL=: _9223372036854775808 NB. efs_jd_'1700' NB. invalid 13 | 14 | fixinsert=: 3 : 0 15 | if. JCHAR=3!:0 y do. ('d039'{~TYPES_E i.