├── tests ├── __init__.py ├── reference_parses │ ├── QUERY_MS2DATA_WHERE_MS2PROD_226.18___ba59129fc3ca9db0a463480fd413f880.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_RTMIN_50___ea275091f8beb5dc52e5013e2a0b38bd.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_CHARGE_1___c3f3cfbd42a396813e18b743b41ffa23.json │ ├── QUERY_scanmz_MS2DATA__WHERE_MS2PROD_226.18___1be0b3783ef46849f8be2a041dc7a3f8.json │ ├── QUERY_scannum_MS2DATA__WHERE_MS2NL_163___4c06210393d721e52082566c4910e179.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MOBILITY_range_min_1___53fd66ff28048399de8e74796382da10.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_157.0857+10___18bc388c403057bfcdaf4429d31c4bbe.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_formula_C10____43e7a8234211481785e7b5087b076430.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_POLARITY_Negative___856798678fb46ad031afbf6c1c18a9e3.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_POLARITY_Positive___0db9167846d408f02e831cf233b8b563.json │ ├── Query_scaninfo_MS1DATA__Where_POLARITY_Negative___316a62728a9e41b2093691da736a05ff.json │ ├── query_scaninfo_MS1DATA__where_POLARITY_Negative___2d2873c1ba10ba537f8f4d89e6741c9a.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_MS1MZ__X_OR_X+2_OR_X___c9c7ec9c134050419dd101e92775db20.json │ ├── QUERY_MS2DATA_WHERE_MS2PROD_226.18_AND_MS2PREC_226___7a3ae5ca424175eb968641a06e191be8.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_SCANMIN_100_AND_SCAN___d83553d176ee4bac735b23927f3477fe.json │ ├── QUERY_scansum_MS2DATA__WHERE_MS2PROD_271_AND_MS2PR___0aa157e0adeb2407305cfe3cb431eba3.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_AND_MS2PRO___8ee2287663d6ad552d20963061530a3b.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_And_MS2PRO___4fe070795d41b0ea705398a6e5d940ab.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_and_MS2PRO___09ccf26b267546b40a6acd472bbea037.json │ ├── QUERY_MS2DATA_WHERE_MS2PROD_226.18TOLERANCEPPM_5___e52fbecfe84d774889d7b5f536a3363e.json │ ├── QUERY_scansum_MS1DATA__WHERE_MS1MZ_100TOLERANCEMZ____ba08671aada7c435d266b095c78c7443.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_226.18TOLERA___183ad9892736b0d8642ccec4aa131c40.json │ ├── QUERY_scannum_MS2DATA__WHERE_MS2PREC_85.0282TOLERA___134f5926097a502b8c1ed113d63bce43.json │ ├── QUERY_scannum_MS2DATA__WHERE_MS2PROD_85.0284TOLERA___4f5faa5c621cc16a76cfaebbb0441dae.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_226.18TOLERA___765966e9867749f2dee435696d06aca4.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PREC_X_AND_MOBILI___899195142a35d40a2b2dc735894c737a.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_425.2898TOLERA___1f23674928bf7c61db1bede091d016ac.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_425.2898TOLERA___106d511f8de4ae41364fbece4838e2c8.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_MS2PROD_309.2TOLERAN___ca35f95cb92bd1fbd9a06d74463ecc45.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PREC_403.15TOLERA___787f67aec200ae337b0e3df9031db8ca.json │ ├── QUERY_scansum_MS1DATA__FILTER_MS1MZ_515TOLERANCEMZ___41ea32589a61e1e93c2e53c6e188d73a.json │ ├── QUERY_scannum_MS2DATA__WHERE_MS2PROD_88TOLERANCEMZ___6d988b4f5914b1ec60cff898439e7816.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_167.0857TOLE___17c6443350bc76ae1462d4e492139426.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_660.2TOLERAN___534180716b576e89874a50f7245c8f30.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_TOLERANCEM___de1efdec2d9f13fba1556258838fc30f.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS1MZ_X-2INTENSITYMA___61a2da302d53487ba89667bb7b7433a3.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_X_range_min_100,_max___ce48d94145b8fe9dd12d4257ffda9ad1.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_TOLERANCEM___a5ce05df2b600cc86a0a76d43dd61ff6.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_341.28TOLERA___76665345ac8dbd664ab054e37eced641.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___10cc51d88921a4f32ca1f0a38ea3b4f2.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD__58.06513_OR___914cae9424e4492a2d88e8e1d0128f6e.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_184.0739TOLE___208cb60e1979a0a3d01cf4ea7a398d27.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_X-1.993INTENSI___c11b8d3c85dd33cbb8a73699d7f26efb.json │ ├── QUERY_scaninfo_MS2DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___8211c5bd2dcbba27b8676e891ccdf078.json │ ├── QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___9008f814f26ce5a1835e900d1e9c659f.json │ └── QUERY_scaninfo_MS2DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___6c21ac388a474d5ebc9485f44f67972e.json ├── test_advanced_queries.txt ├── test_data │ ├── top_down.txt │ └── top_down.mgf ├── test_translate.py ├── test_visualize.py ├── test_file_loading.py ├── get_data.sh ├── test_extraction.py ├── test_queries.txt └── test_parse.py ├── MANIFEST.in ├── requirements_deploy.txt ├── massql ├── __init__.py ├── msql.ebnf └── msql_cmd.py ├── workflow ├── .nextflow │ ├── cache │ │ ├── 0144b051-9fc7-4715-9899-5093dfaf80f6 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── 000003.log │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.condescending_plateau │ │ ├── 0bc4b07e-570c-44d6-af81-6c8162ead2fd │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.admiring_kirch │ │ ├── 0ecca33d-d6ef-4f54-b93d-3d4f24ea6ac8 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.fervent_roentgen │ │ ├── 14f27b53-5a2d-4720-bfa6-d01541f3d9e8 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.compassionate_lumiere │ │ ├── 23505b1e-0ef3-41a2-9d58-161eab2ade49 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.ridiculous_payne │ │ ├── 28a5681c-8724-46e0-aaea-d4d46ace730d │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.clever_poincare │ │ ├── 514e5b26-3728-4d42-9293-85ab6c38e470 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.desperate_elion │ │ ├── 601e2974-9388-4979-8d9e-e38deb8da9cb │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.distracted_faraday │ │ ├── 636d7f3e-d7c4-47ab-bf3f-b264282d416c │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── 000003.log │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.mighty_solvay │ │ ├── 82349cae-2a67-4e19-9235-3d1ac0695aa8 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── 000003.log │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.big_pauling │ │ ├── 86505b83-a56d-4179-8498-3987c26f2c05 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.compassionate_bartik │ │ ├── aa7fd51f-b34b-4692-a396-d56f916e7fff │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.chaotic_minsky │ │ ├── b98d1746-9c1a-4e3a-a4a6-4e6317e8052b │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.mighty_goodall │ │ ├── d24bf493-6303-4706-973e-b60c310a82ec │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── 000003.log │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.exotic_pauling │ │ ├── ddd73f2b-3938-4c1c-8d16-6304d0196e78 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.special_pasteur │ │ ├── dde8c0e1-778e-4e7c-a123-a26d751526b7 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.mighty_galileo │ │ ├── dff43999-fd46-4b77-aa98-7b10c8e8cce2 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.scruffy_moriondo │ │ ├── e922605c-0877-4299-b626-d45dedbef936 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.fervent_carlsson │ │ ├── edba329c-a7dc-4757-93a8-40740444e360 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.cranky_noyce │ │ ├── f9eeffa0-390d-4389-9a8b-5921ed3db901 │ │ │ ├── db │ │ │ │ ├── LOCK │ │ │ │ ├── CURRENT │ │ │ │ ├── 000003.log │ │ │ │ └── MANIFEST-000002 │ │ │ └── index.angry_torvalds │ │ └── fae3991d-c833-4ecc-921e-2ab73bb95746 │ │ │ ├── db │ │ │ ├── LOCK │ │ │ ├── CURRENT │ │ │ └── MANIFEST-000002 │ │ │ └── index.stoic_brown │ └── history ├── .nextflow.log.2 ├── .nextflow.log.9 ├── .nextflow.log.8 ├── .nextflow.log.7 └── .nextflow.log.4 ├── requirements_test.txt ├── .gitignore ├── requirements.txt ├── .gitmodules ├── utilities └── motifdb_converter.py ├── .github └── workflows │ ├── publish.yml │ ├── test-workflow.yml │ ├── test-unit.yml │ └── test-package.yml ├── LICENSE.txt ├── Makefile ├── setup.py └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include massql/msql.ebnf -------------------------------------------------------------------------------- /requirements_deploy.txt: -------------------------------------------------------------------------------- 1 | twine 2 | build -------------------------------------------------------------------------------- /massql/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | __all__ = ['msql_parser', 'msql_engine'] 3 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0144b051-9fc7-4715-9899-5093dfaf80f6/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0ecca33d-d6ef-4f54-b93d-3d4f24ea6ac8/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/28a5681c-8724-46e0-aaea-d4d46ace730d/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/601e2974-9388-4979-8d9e-e38deb8da9cb/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/636d7f3e-d7c4-47ab-bf3f-b264282d416c/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/82349cae-2a67-4e19-9235-3d1ac0695aa8/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/d24bf493-6303-4706-973e-b60c310a82ec/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dff43999-fd46-4b77-aa98-7b10c8e8cce2/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/e922605c-0877-4299-b626-d45dedbef936/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/fae3991d-c833-4ecc-921e-2ab73bb95746/db/LOCK: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0144b051-9fc7-4715-9899-5093dfaf80f6/db/000003.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/636d7f3e-d7c4-47ab-bf3f-b264282d416c/db/000003.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/82349cae-2a67-4e19-9235-3d1ac0695aa8/db/000003.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/d24bf493-6303-4706-973e-b60c310a82ec/db/000003.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/28a5681c-8724-46e0-aaea-d4d46ace730d/index.clever_poincare: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/636d7f3e-d7c4-47ab-bf3f-b264282d416c/index.mighty_solvay: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/82349cae-2a67-4e19-9235-3d1ac0695aa8/index.big_pauling: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/d24bf493-6303-4706-973e-b60c310a82ec/index.exotic_pauling: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/fae3991d-c833-4ecc-921e-2ab73bb95746/index.stoic_brown: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0ecca33d-d6ef-4f54-b93d-3d4f24ea6ac8/index.fervent_roentgen: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/601e2974-9388-4979-8d9e-e38deb8da9cb/index.distracted_faraday: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dff43999-fd46-4b77-aa98-7b10c8e8cce2/index.scruffy_moriondo: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/e922605c-0877-4299-b626-d45dedbef936/index.fervent_carlsson: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-xdist 3 | pathvalidate 4 | requests 5 | pytest-cov -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0144b051-9fc7-4715-9899-5093dfaf80f6/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0144b051-9fc7-4715-9899-5093dfaf80f6/index.condescending_plateau: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0ecca33d-d6ef-4f54-b93d-3d4f24ea6ac8/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/28a5681c-8724-46e0-aaea-d4d46ace730d/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/601e2974-9388-4979-8d9e-e38deb8da9cb/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/636d7f3e-d7c4-47ab-bf3f-b264282d416c/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/82349cae-2a67-4e19-9235-3d1ac0695aa8/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/d24bf493-6303-4706-973e-b60c310a82ec/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dff43999-fd46-4b77-aa98-7b10c8e8cce2/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/e922605c-0877-4299-b626-d45dedbef936/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /workflow/.nextflow/cache/fae3991d-c833-4ecc-921e-2ab73bb95746/db/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000002 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | workflow/work 3 | *__pycache__* 4 | tests/data 5 | tests/test_data/*.feather 6 | tests/test_parses 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pymzml 2 | lark>=1.0 3 | pandas 4 | pyarrow 5 | tqdm 6 | py_expression_eval 7 | pyteomics 8 | psims 9 | plotly 10 | kaleido 11 | pydot 12 | pyyaml -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/db/000003.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/db/000003.log -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0144b051-9fc7-4715-9899-5093dfaf80f6/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/0144b051-9fc7-4715-9899-5093dfaf80f6/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0ecca33d-d6ef-4f54-b93d-3d4f24ea6ac8/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/0ecca33d-d6ef-4f54-b93d-3d4f24ea6ac8/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/28a5681c-8724-46e0-aaea-d4d46ace730d/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/28a5681c-8724-46e0-aaea-d4d46ace730d/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/601e2974-9388-4979-8d9e-e38deb8da9cb/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/601e2974-9388-4979-8d9e-e38deb8da9cb/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/636d7f3e-d7c4-47ab-bf3f-b264282d416c/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/636d7f3e-d7c4-47ab-bf3f-b264282d416c/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/82349cae-2a67-4e19-9235-3d1ac0695aa8/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/82349cae-2a67-4e19-9235-3d1ac0695aa8/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/d24bf493-6303-4706-973e-b60c310a82ec/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/d24bf493-6303-4706-973e-b60c310a82ec/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dff43999-fd46-4b77-aa98-7b10c8e8cce2/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/dff43999-fd46-4b77-aa98-7b10c8e8cce2/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/e922605c-0877-4299-b626-d45dedbef936/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/e922605c-0877-4299-b626-d45dedbef936/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/index.cranky_noyce: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/edba329c-a7dc-4757-93a8-40740444e360/index.cranky_noyce -------------------------------------------------------------------------------- /workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/fae3991d-c833-4ecc-921e-2ab73bb95746/db/MANIFEST-000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/fae3991d-c833-4ecc-921e-2ab73bb95746/db/MANIFEST-000002 -------------------------------------------------------------------------------- /workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/index.admiring_kirch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/0bc4b07e-570c-44d6-af81-6c8162ead2fd/index.admiring_kirch -------------------------------------------------------------------------------- /workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/index.desperate_elion: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/514e5b26-3728-4d42-9293-85ab6c38e470/index.desperate_elion -------------------------------------------------------------------------------- /workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/index.chaotic_minsky: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/aa7fd51f-b34b-4692-a396-d56f916e7fff/index.chaotic_minsky -------------------------------------------------------------------------------- /workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/index.mighty_goodall: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/b98d1746-9c1a-4e3a-a4a6-4e6317e8052b/index.mighty_goodall -------------------------------------------------------------------------------- /workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/index.special_pasteur: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/ddd73f2b-3938-4c1c-8d16-6304d0196e78/index.special_pasteur -------------------------------------------------------------------------------- /workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/index.mighty_galileo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/dde8c0e1-778e-4e7c-a123-a26d751526b7/index.mighty_galileo -------------------------------------------------------------------------------- /workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/index.angry_torvalds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/f9eeffa0-390d-4389-9a8b-5921ed3db901/index.angry_torvalds -------------------------------------------------------------------------------- /workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/index.ridiculous_payne: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/23505b1e-0ef3-41a2-9d58-161eab2ade49/index.ridiculous_payne -------------------------------------------------------------------------------- /workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/index.compassionate_bartik: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/86505b83-a56d-4179-8498-3987c26f2c05/index.compassionate_bartik -------------------------------------------------------------------------------- /workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/index.compassionate_lumiere: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwang87/MassQueryLanguage/HEAD/workflow/.nextflow/cache/14f27b53-5a2d-4720-bfa6-d01541f3d9e8/index.compassionate_lumiere -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_MS2DATA_WHERE_MS2PROD_226.18___ba59129fc3ca9db0a463480fd413f880.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | 226.18 8 | ] 9 | } 10 | ], 11 | "query": "QUERY MS2DATA WHERE MS2PROD=226.18", 12 | "querytype": { 13 | "datatype": "datams2data", 14 | "function": null 15 | } 16 | } -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "proteosafe/CCMSDeployments"] 2 | path = proteosafe/CCMSDeployments 3 | url = https://github.com/CCMS-UCSD/CCMSDeployments.git 4 | [submodule "proteosafe/CCMS-Integration-Tests"] 5 | path = proteosafe/CCMS-Integration-Tests 6 | url = https://github.com/CCMS-UCSD/CCMS-Integration-Tests.git 7 | [submodule "workflow/GNPS2_DeploymentTooling"] 8 | path = workflow/GNPS2_DeploymentTooling 9 | url = https://github.com/Wang-Bioinformatics-Lab/GNPS2_DeploymentTooling.git 10 | -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_RTMIN_50___ea275091f8beb5dc52e5013e2a0b38bd.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "rtmincondition", 6 | "value": [ 7 | 50.0 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scaninfo(MS1DATA) WHERE RTMIN=50", 12 | "querytype": { 13 | "datatype": "datams1data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_CHARGE_1___c3f3cfbd42a396813e18b743b41ffa23.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "chargecondition", 6 | "value": [ 7 | 1.0 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scaninfo(MS2DATA) WHERE CHARGE=1", 12 | "querytype": { 13 | "datatype": "datams2data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scanmz_MS2DATA__WHERE_MS2PROD_226.18___1be0b3783ef46849f8be2a041dc7a3f8.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | 226.18 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scanmz(MS2DATA) WHERE MS2PROD=226.18", 12 | "querytype": { 13 | "datatype": "datams2data", 14 | "function": "functionscanmz" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scannum_MS2DATA__WHERE_MS2NL_163___4c06210393d721e52082566c4910e179.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2neutrallosscondition", 6 | "value": [ 7 | 163.0 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scannum(MS2DATA) WHERE MS2NL=163", 12 | "querytype": { 13 | "datatype": "datams2data", 14 | "function": "functionscannum" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MOBILITY_range_min_1___53fd66ff28048399de8e74796382da10.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "max": 500.0, 6 | "min": 100.0, 7 | "type": "mobilitycondition" 8 | } 9 | ], 10 | "query": "QUERY scaninfo(MS2DATA) WHERE MOBILITY=range(min=100, max=500)", 11 | "querytype": { 12 | "datatype": "datams2data", 13 | "function": "functionscaninfo" 14 | } 15 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_157.0857+10___18bc388c403057bfcdaf4429d31c4bbe.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | 167.0857 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=157.0857+10", 12 | "querytype": { 13 | "datatype": "datams2data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_formula_C10____43e7a8234211481785e7b5087b076430.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | 120.0 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=formula(C10)", 12 | "querytype": { 13 | "datatype": "datams2data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_POLARITY_Negative___856798678fb46ad031afbf6c1c18a9e3.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "polaritycondition", 6 | "value": [ 7 | "negativepolarity" 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scaninfo(MS1DATA) WHERE POLARITY=Negative", 12 | "querytype": { 13 | "datatype": "datams1data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_POLARITY_Positive___0db9167846d408f02e831cf233b8b563.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "polaritycondition", 6 | "value": [ 7 | "positivepolarity" 8 | ] 9 | } 10 | ], 11 | "query": "QUERY scaninfo(MS1DATA) WHERE POLARITY=Positive", 12 | "querytype": { 13 | "datatype": "datams1data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/Query_scaninfo_MS1DATA__Where_POLARITY_Negative___316a62728a9e41b2093691da736a05ff.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "polaritycondition", 6 | "value": [ 7 | "negativepolarity" 8 | ] 9 | } 10 | ], 11 | "query": "Query scaninfo(MS1DATA) Where POLARITY=Negative", 12 | "querytype": { 13 | "datatype": "datams1data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/query_scaninfo_MS1DATA__where_POLARITY_Negative___2d2873c1ba10ba537f8f4d89e6741c9a.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "polaritycondition", 6 | "value": [ 7 | "negativepolarity" 8 | ] 9 | } 10 | ], 11 | "query": "query scaninfo(MS1DATA) where POLARITY=Negative", 12 | "querytype": { 13 | "datatype": "datams1data", 14 | "function": "functionscaninfo" 15 | } 16 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_MS1MZ__X_OR_X+2_OR_X___c9c7ec9c134050419dd101e92775db20.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms1mzcondition", 6 | "value": [ 7 | "X", 8 | "X+2.0", 9 | "X+4.0", 10 | "X+6.0" 11 | ] 12 | } 13 | ], 14 | "query": "QUERY scaninfo(MS1DATA) WHERE MS1MZ=(X OR X+2 OR X+4 OR X+6)", 15 | "querytype": { 16 | "datatype": "datams1data", 17 | "function": "functionscaninfo" 18 | } 19 | } -------------------------------------------------------------------------------- /tests/test_advanced_queries.txt: -------------------------------------------------------------------------------- 1 | QUERY scannum(MS2DATA) WHERE MS1MZ=XX 2 | QUERY scannum(MS2DATA) WHERE MS1MZ=XX:INTENSITYVALUE=1 AND MS1MZ=XX+2:INTENSITYVALUE=1 AND MS2PREC=XX AND MS2PROD=160.5 AND SIMILARITY>0.9 3 | QUERY scanrangesum(MS1DATA, TOLERANCE=0.1) WHERE MS1MZ=(QUERY scanmz(MS2DATA) WHERE MS2PROD=85.02820:MZDELTA=0.01 AND MS2NL=59.07350:MZDELTA=0.01):MZDELTA=0.01 4 | QUERY scannum(MS2DATA) WHERE MS1MZ=X AND MS2PREC=X AND MS2PROD=160.5 5 | QUERY scaninfo(MS2DATA) WHERE MS1MZ=X-2:INTENSITYMATCH=Y*(0.0608+(.000002*X)):INTENSITYMATCHPERCENT=25 AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 AND MS2PREC=X -------------------------------------------------------------------------------- /utilities/motifdb_converter.py: -------------------------------------------------------------------------------- 1 | from lib2to3.pytree import convert 2 | import requests 3 | 4 | def convert_motifdb_to_massql(usi_motifdb): 5 | url = "https://metabolomics-usi.ucsd.edu/json/" 6 | r = requests.get(url, params={'usi1': usi_motifdb}) 7 | 8 | peaks = r.json()['peaks'] 9 | 10 | print(peaks) 11 | 12 | query = "QUERY scaninfo(MS2DATA) WHERE " 13 | 14 | conditions = " AND ".join(["MS2PROD={}".format(peak[0]) for peak in peaks]) 15 | query += conditions 16 | 17 | return query 18 | 19 | 20 | def main(): 21 | convert_motifdb_to_massql("mzspec:MOTIFDB::accession:151065") 22 | 23 | if __name__ == "__main__": 24 | main() -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_MS2DATA_WHERE_MS2PROD_226.18_AND_MS2PREC_226___7a3ae5ca424175eb968641a06e191be8.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | 226.18 8 | ] 9 | }, 10 | { 11 | "conditiontype": "where", 12 | "type": "ms2precursorcondition", 13 | "value": [ 14 | 226.1797 15 | ] 16 | } 17 | ], 18 | "query": "QUERY MS2DATA WHERE MS2PROD=226.18 AND MS2PREC=226.1797", 19 | "querytype": { 20 | "datatype": "datams2data", 21 | "function": null 22 | } 23 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_SCANMIN_100_AND_SCAN___d83553d176ee4bac735b23927f3477fe.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "scanmincondition", 6 | "value": [ 7 | 100.0 8 | ] 9 | }, 10 | { 11 | "conditiontype": "where", 12 | "type": "scanmaxcondition", 13 | "value": [ 14 | 105.0 15 | ] 16 | } 17 | ], 18 | "query": "QUERY scaninfo(MS1DATA) WHERE SCANMIN=100 AND SCANMAX=105", 19 | "querytype": { 20 | "datatype": "datams1data", 21 | "function": "functionscaninfo" 22 | } 23 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scansum_MS2DATA__WHERE_MS2PROD_271_AND_MS2PR___0aa157e0adeb2407305cfe3cb431eba3.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | 271.0 8 | ] 9 | }, 10 | { 11 | "conditiontype": "where", 12 | "type": "ms2precursorcondition", 13 | "value": [ 14 | 500.0 15 | ] 16 | } 17 | ], 18 | "query": "QUERY scansum(MS2DATA) WHERE MS2PROD=271 AND MS2PREC=500", 19 | "querytype": { 20 | "datatype": "datams2data", 21 | "function": "functionscansum" 22 | } 23 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_AND_MS2PRO___8ee2287663d6ad552d20963061530a3b.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | "X" 8 | ] 9 | }, 10 | { 11 | "conditiontype": "where", 12 | "type": "ms2productcondition", 13 | "value": [ 14 | "2.0*X-55.9349375" 15 | ] 16 | } 17 | ], 18 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X AND MS2PROD=2.0*(X - formula(Fe))", 19 | "querytype": { 20 | "datatype": "datams2data", 21 | "function": "functionscaninfo" 22 | } 23 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_And_MS2PRO___4fe070795d41b0ea705398a6e5d940ab.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | "X" 8 | ] 9 | }, 10 | { 11 | "conditiontype": "where", 12 | "type": "ms2productcondition", 13 | "value": [ 14 | "2.0*X-55.9349375" 15 | ] 16 | } 17 | ], 18 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X And MS2PROD=2.0*(X - formula(Fe))", 19 | "querytype": { 20 | "datatype": "datams2data", 21 | "function": "functionscaninfo" 22 | } 23 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_and_MS2PRO___09ccf26b267546b40a6acd472bbea037.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2productcondition", 6 | "value": [ 7 | "X" 8 | ] 9 | }, 10 | { 11 | "conditiontype": "where", 12 | "type": "ms2productcondition", 13 | "value": [ 14 | "2.0*X-55.9349375" 15 | ] 16 | } 17 | ], 18 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X and MS2PROD=2.0*(X - formula(Fe))", 19 | "querytype": { 20 | "datatype": "datams2data", 21 | "function": "functionscaninfo" 22 | } 23 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_MS2DATA_WHERE_MS2PROD_226.18TOLERANCEPPM_5___e52fbecfe84d774889d7b5f536a3363e.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierppmtolerance": { 7 | "name": "qualifierppmtolerance", 8 | "unit": "ppm", 9 | "value": 5.0 10 | }, 11 | "type": "qualifier" 12 | }, 13 | "type": "ms2productcondition", 14 | "value": [ 15 | 226.18 16 | ] 17 | } 18 | ], 19 | "query": "QUERY MS2DATA WHERE MS2PROD=226.18:TOLERANCEPPM=5", 20 | "querytype": { 21 | "datatype": "datams2data", 22 | "function": null 23 | } 24 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scansum_MS1DATA__WHERE_MS1MZ_100TOLERANCEMZ____ba08671aada7c435d266b095c78c7443.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifiermztolerance": { 7 | "name": "qualifiermztolerance", 8 | "unit": "mz", 9 | "value": 0.1 10 | }, 11 | "type": "qualifier" 12 | }, 13 | "type": "ms1mzcondition", 14 | "value": [ 15 | 100.0 16 | ] 17 | } 18 | ], 19 | "query": "QUERY scansum(MS1DATA) WHERE MS1MZ=100:TOLERANCEMZ=0.1", 20 | "querytype": { 21 | "datatype": "datams1data", 22 | "function": "functionscansum" 23 | } 24 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_226.18TOLERA___183ad9892736b0d8642ccec4aa131c40.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierppmtolerance": { 7 | "name": "qualifierppmtolerance", 8 | "unit": "ppm", 9 | "value": 5.0 10 | }, 11 | "type": "qualifier" 12 | }, 13 | "type": "ms2productcondition", 14 | "value": [ 15 | 226.18 16 | ] 17 | } 18 | ], 19 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=226.18:TOLERANCEPPM=5", 20 | "querytype": { 21 | "datatype": "datams2data", 22 | "function": "functionscaninfo" 23 | } 24 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scannum_MS2DATA__WHERE_MS2PREC_85.0282TOLERA___134f5926097a502b8c1ed113d63bce43.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifiermztolerance": { 7 | "name": "qualifiermztolerance", 8 | "unit": "mz", 9 | "value": 0.1 10 | }, 11 | "type": "qualifier" 12 | }, 13 | "type": "ms2precursorcondition", 14 | "value": [ 15 | 85.0282 16 | ] 17 | } 18 | ], 19 | "query": "QUERY scannum(MS2DATA) WHERE MS2PREC=85.0282:TOLERANCEMZ=0.1", 20 | "querytype": { 21 | "datatype": "datams2data", 22 | "function": "functionscannum" 23 | } 24 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scannum_MS2DATA__WHERE_MS2PROD_85.0284TOLERA___4f5faa5c621cc16a76cfaebbb0441dae.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifiermztolerance": { 7 | "name": "qualifiermztolerance", 8 | "unit": "mz", 9 | "value": 0.005 10 | }, 11 | "type": "qualifier" 12 | }, 13 | "type": "ms2productcondition", 14 | "value": [ 15 | 85.0284 16 | ] 17 | } 18 | ], 19 | "query": "QUERY scannum(MS2DATA) WHERE MS2PROD=85.0284:TOLERANCEMZ=0.005", 20 | "querytype": { 21 | "datatype": "datams2data", 22 | "function": "functionscannum" 23 | } 24 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_226.18TOLERA___765966e9867749f2dee435696d06aca4.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierexcluded": { 7 | "name": "qualifierexcluded" 8 | }, 9 | "qualifierppmtolerance": { 10 | "name": "qualifierppmtolerance", 11 | "unit": "ppm", 12 | "value": 5.0 13 | }, 14 | "type": "qualifier" 15 | }, 16 | "type": "ms2productcondition", 17 | "value": [ 18 | 226.18 19 | ] 20 | } 21 | ], 22 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=226.18:TOLERANCEPPM=5:EXCLUDED", 23 | "querytype": { 24 | "datatype": "datams2data", 25 | "function": "functionscaninfo" 26 | } 27 | } -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is 2 | # created. For more information see: 3 | # https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 4 | 5 | name: PyPI 6 | 7 | on: 8 | release: 9 | types: [created] 10 | 11 | jobs: 12 | deploy: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine build 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | python -m build --sdist --wheel . 31 | twine upload dist/* -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PREC_X_AND_MOBILI___899195142a35d40a2b2dc735894c737a.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "type": "ms2precursorcondition", 6 | "value": [ 7 | "X" 8 | ] 9 | }, 10 | { 11 | "conditiontype": "where", 12 | "max": "X*0.00078231+0.48817", 13 | "min": "X*0.0006775+0.40557", 14 | "type": "mobilitycondition" 15 | }, 16 | { 17 | "conditiontype": "where", 18 | "maxdefect": 0.99, 19 | "mindefect": 0.9, 20 | "type": "xcondition" 21 | } 22 | ], 23 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PREC=X AND MOBILITY=range(min=X*0.0006775+0.40557, max=X*0.00078231+0.48817) AND X=massdefect(min=0.9, max=0.99)", 24 | "querytype": { 25 | "datatype": "datams2data", 26 | "function": "functionscaninfo" 27 | } 28 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_425.2898TOLERA___1f23674928bf7c61db1bede091d016ac.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 1.0 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.1 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms1mzcondition", 18 | "value": [ 19 | 425.2898 20 | ] 21 | } 22 | ], 23 | "query": "QUERY scaninfo(MS1DATA) WHERE MS1MZ=425.2898:TOLERANCEMZ=0.1:INTENSITYPERCENT>1", 24 | "querytype": { 25 | "datatype": "datams1data", 26 | "function": "functionscaninfo" 27 | } 28 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_425.2898TOLERA___106d511f8de4ae41364fbece4838e2c8.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 0.1 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.1 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms1mzcondition", 18 | "value": [ 19 | 425.2898 20 | ] 21 | } 22 | ], 23 | "query": "QUERY scaninfo(MS1DATA) WHERE MS1MZ=425.2898:TOLERANCEMZ=0.1:INTENSITYPERCENT>0.1", 24 | "querytype": { 25 | "datatype": "datams1data", 26 | "function": "functionscaninfo" 27 | } 28 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_MS2PROD_309.2TOLERAN___ca35f95cb92bd1fbd9a06d74463ecc45.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensityticpercent": { 7 | "name": "qualifierintensityticpercent", 8 | "value": 10.0 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.1 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms2productcondition", 18 | "value": [ 19 | 309.2 20 | ] 21 | } 22 | ], 23 | "query": "QUERY scaninfo(MS1DATA) WHERE MS2PROD=309.2:TOLERANCEMZ=0.1:INTENSITYTICPERCENT=10", 24 | "querytype": { 25 | "datatype": "datams1data", 26 | "function": "functionscaninfo" 27 | } 28 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PREC_403.15TOLERA___787f67aec200ae337b0e3df9031db8ca.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifiermztolerance": { 7 | "name": "qualifiermztolerance", 8 | "unit": "mz", 9 | "value": 0.1 10 | }, 11 | "type": "qualifier" 12 | }, 13 | "type": "ms2precursorcondition", 14 | "value": [ 15 | 403.15 16 | ] 17 | }, 18 | { 19 | "conditiontype": "where", 20 | "max": 0.94, 21 | "min": 0.91, 22 | "type": "mobilitycondition" 23 | } 24 | ], 25 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PREC=403.15:TOLERANCEMZ=0.1 AND MOBILITY=range(min=0.910, max=0.940)", 26 | "querytype": { 27 | "datatype": "datams2data", 28 | "function": "functionscaninfo" 29 | } 30 | } -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2021 Mingxun Wang 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scansum_MS1DATA__FILTER_MS1MZ_515TOLERANCEMZ___41ea32589a61e1e93c2e53c6e188d73a.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "filter", 5 | "qualifiers": { 6 | "qualifiermassdefect": { 7 | "max": 0.2112, 8 | "min": 0.1332, 9 | "name": "qualifiermassdefect" 10 | }, 11 | "qualifiermztolerance": { 12 | "name": "qualifiermztolerance", 13 | "unit": "mz", 14 | "value": 35.0 15 | }, 16 | "type": "qualifier" 17 | }, 18 | "type": "ms1mzcondition", 19 | "value": [ 20 | 515.0 21 | ] 22 | } 23 | ], 24 | "query": "QUERY scansum(MS1DATA) FILTER MS1MZ=515:TOLERANCEMZ=35:MASSDEFECT=massdefect(min=0.1332, max=0.2112)", 25 | "querytype": { 26 | "datatype": "datams1data", 27 | "function": "functionscansum" 28 | } 29 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | test_parse: 4 | pytest -vv --cov=massql ./tests/test_parse.py 5 | 6 | test_translate: 7 | pytest -vv --cov=massql ./tests/test_translate.py 8 | 9 | test_extraction: 10 | pytest -vv --cov=massql ./tests/test_extraction.py 11 | 12 | test_visualize: 13 | pytest -vv --cov=massql ./tests/test_visualize.py 14 | 15 | test_fileloading: 16 | pytest -vv --cov=massql ./tests/test_file_loading.py 17 | 18 | test_query: 19 | pytest -vv --cov=massql ./tests/test_query.py -n 4 20 | 21 | test_full: 22 | pytest -vv --cov=massql ./tests/ -n 8 23 | 24 | # test_full_parallel: 25 | # pytest -vv test.py test_parse.py test_extraction.py -n 6 26 | 27 | # test_specific: 28 | # pytest --capture=tee-sys -vv test.py::test_min_intensitypercent 29 | # pytest --capture=tee-sys -vv test.py::test_query 30 | 31 | deploy_clean: 32 | rm build/ dist/ massql.egg-info -rf 33 | 34 | deploy_pypi: 35 | python -m build --sdist --wheel . 36 | twine upload dist/* 37 | 38 | specific_pytest: 39 | #pytest -vv --cov=massql ./tests/test_extraction.py::test_extract_MGF 40 | python ./tests/test_extraction.py -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scannum_MS2DATA__WHERE_MS2PROD_88TOLERANCEMZ___6d988b4f5914b1ec60cff898439e7816.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 10.0 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.1 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms2productcondition", 18 | "value": [ 19 | 88.0 20 | ] 21 | }, 22 | { 23 | "conditiontype": "where", 24 | "type": "ms2precursorcondition", 25 | "value": [ 26 | 600.0 27 | ] 28 | } 29 | ], 30 | "query": "QUERY scannum(MS2DATA) WHERE MS2PROD=88:TOLERANCEMZ=0.1:INTENSITYPERCENT>10 AND MS2PREC=600", 31 | "querytype": { 32 | "datatype": "datams2data", 33 | "function": "functionscannum" 34 | } 35 | } -------------------------------------------------------------------------------- /.github/workflows/test-workflow.yml: -------------------------------------------------------------------------------- 1 | # # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | # name: NF Workflow Testing 5 | 6 | # on: 7 | # push: 8 | # branches: [ master ] 9 | # pull_request: 10 | # branches: [ master ] 11 | 12 | # jobs: 13 | # nextflow-test: 14 | 15 | # runs-on: ubuntu-latest 16 | 17 | # steps: 18 | # - uses: actions/checkout@v2 19 | # #- name: Set up Python 3.9 20 | # # uses: actions/setup-python@v2 21 | # # with: 22 | # # python-version: 3.9 23 | # - name: Install dependencies 24 | # run: | 25 | # #python -m pip install --upgrade pip 26 | # #pip install flake8 pytest 27 | # #if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 28 | # wget -qO- get.nextflow.io | bash 29 | # sudo mv nextflow /usr/local/bin/ 30 | # - name: Download Data 31 | # run: | 32 | # cd tests && sh ./get_data.sh && cd .. 33 | # - name: Workflow Testing 34 | # run: | 35 | # cd workflow 36 | # make run_test 37 | # make run_test_no_extract 38 | -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_167.0857TOLE___17c6443350bc76ae1462d4e492139426.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierppmtolerance": { 7 | "name": "qualifierppmtolerance", 8 | "unit": "ppm", 9 | "value": 5.0 10 | }, 11 | "type": "qualifier" 12 | }, 13 | "type": "ms2productcondition", 14 | "value": [ 15 | 167.0857 16 | ] 17 | }, 18 | { 19 | "conditiontype": "where", 20 | "qualifiers": { 21 | "qualifierppmtolerance": { 22 | "name": "qualifierppmtolerance", 23 | "unit": "ppm", 24 | "value": 5.0 25 | }, 26 | "type": "qualifier" 27 | }, 28 | "type": "ms2neutrallosscondition", 29 | "value": [ 30 | 176.0321 31 | ] 32 | } 33 | ], 34 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=167.0857:TOLERANCEPPM=5 AND MS2NL=176.0321:TOLERANCEPPM=5", 35 | "querytype": { 36 | "datatype": "datams2data", 37 | "function": "functionscaninfo" 38 | } 39 | } -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | import setuptools 3 | 4 | with open("README.md", "r", encoding="utf-8") as fh: 5 | long_description = fh.read() 6 | 7 | setuptools.setup( 8 | name="massql", 9 | version="2025.12.10", 10 | author="Mingxun Wang", 11 | author_email="mwang87@gmail.com", 12 | description="Mass spectrometry query language python implementation", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | url="https://github.com/mwang87/MassQueryLanguage", 16 | project_urls={ 17 | "Bug Tracker": "https://github.com/mwang87/MassQueryLanguage/issues", 18 | "Documentation": "https://mwang87.github.io/MassQueryLanguage_Documentation/" 19 | }, 20 | scripts=['massql/msql_cmd.py'], 21 | entry_points = { 22 | 'console_scripts': ['massql=massql.msql_cmd:main'], 23 | }, 24 | classifiers=[ 25 | "Programming Language :: Python :: 3", 26 | "License :: OSI Approved :: MIT License", 27 | "Operating System :: OS Independent", 28 | ], 29 | packages=["massql"], 30 | install_requires=[ 31 | "pymzml", 32 | "lark-parser", 33 | "pandas", 34 | "pyarrow", 35 | "tqdm", 36 | "py_expression_eval", 37 | "pyteomics", 38 | "psims", 39 | "plotly", 40 | "kaleido", 41 | "pydot", 42 | "pyyaml" 43 | ], 44 | python_requires=">=3.9", 45 | include_package_data=True 46 | ) 47 | -------------------------------------------------------------------------------- /tests/test_data/top_down.txt: -------------------------------------------------------------------------------- 1 | 132465.09 12.546343 2 | 133225.4931 12.19392 3 | 133989.6146 18.382821 4 | 134744.8918 16.622169 5 | 135508.8453 29.15272 6 | 135709.4353 12.315019 7 | 136269.7202 23.290751 8 | 136474.0863 16.300753 9 | 136823.8683 13.401818 10 | 137025.2584 42.788894 11 | 137234.0651 14.446877 12 | 137584.0828 13.276068 13 | 137785.3407 36.485754 14 | 137995.0349 14.735787 15 | 138340.1478 20.745298 16 | 138549.298 62.669336 17 | 138754.2691 23.062884 18 | 139105.4871 23.903191 19 | 139304.9367 54.209214 20 | 139509.8068 24.567254 21 | 139864.7173 23.295769 22 | 140065.2515 62.811609 23 | 140270.0596 30.497242 24 | 140623.8754 29.028855 25 | 140826.022 61.716949 26 | 141035.2999 20.697356 27 | 141383.9161 30.244086 28 | 141585.5889 76.422158 29 | 141794.0949 34.943156 30 | 142144.3674 54.074194 31 | 142345.3573 100 32 | 142553.5582 42.638963 33 | 142904.3269 35.756392 34 | 143105.3558 66.716483 35 | 143310.0193 33.103042 36 | 143665.4048 35.090121 37 | 143865.6924 70.924197 38 | 144073.8344 34.686524 39 | 144424.0522 27.382454 40 | 144625.9266 69.20136 41 | 144833.5925 26.57023 42 | 145185.3049 31.549792 43 | 145385.8676 84.545703 44 | 145589.8953 39.743775 45 | 145944.8025 28.254461 46 | 146149.9511 71.468122 47 | 146355.289 24.295305 48 | 146704.5286 19.559481 49 | 146905.8959 62.213523 50 | 147106.5088 29.916714 51 | 147669.7064 43.913333 52 | 147873.7765 17.187839 53 | 148426.2304 39.17724 54 | 148634.4224 20.539458 55 | 149190.0231 19.643502 56 | 149395.1611 10.928549 57 | 149953.621 15.454649 58 | 150715.2934 10.336184 -------------------------------------------------------------------------------- /tests/test_data/top_down.mgf: -------------------------------------------------------------------------------- 1 | BEGIN IONS 2 | SCANS=1 3 | 132465.09 12.546343 4 | 133225.4931 12.19392 5 | 133989.6146 18.382821 6 | 134744.8918 16.622169 7 | 135508.8453 29.15272 8 | 135709.4353 12.315019 9 | 136269.7202 23.290751 10 | 136474.0863 16.300753 11 | 136823.8683 13.401818 12 | 137025.2584 42.788894 13 | 137234.0651 14.446877 14 | 137584.0828 13.276068 15 | 137785.3407 36.485754 16 | 137995.0349 14.735787 17 | 138340.1478 20.745298 18 | 138549.298 62.669336 19 | 138754.2691 23.062884 20 | 139105.4871 23.903191 21 | 139304.9367 54.209214 22 | 139509.8068 24.567254 23 | 139864.7173 23.295769 24 | 140065.2515 62.811609 25 | 140270.0596 30.497242 26 | 140623.8754 29.028855 27 | 140826.022 61.716949 28 | 141035.2999 20.697356 29 | 141383.9161 30.244086 30 | 141585.5889 76.422158 31 | 141794.0949 34.943156 32 | 142144.3674 54.074194 33 | 142345.3573 100 34 | 142553.5582 42.638963 35 | 142904.3269 35.756392 36 | 143105.3558 66.716483 37 | 143310.0193 33.103042 38 | 143665.4048 35.090121 39 | 143865.6924 70.924197 40 | 144073.8344 34.686524 41 | 144424.0522 27.382454 42 | 144625.9266 69.20136 43 | 144833.5925 26.57023 44 | 145185.3049 31.549792 45 | 145385.8676 84.545703 46 | 145589.8953 39.743775 47 | 145944.8025 28.254461 48 | 146149.9511 71.468122 49 | 146355.289 24.295305 50 | 146704.5286 19.559481 51 | 146905.8959 62.213523 52 | 147106.5088 29.916714 53 | 147669.7064 43.913333 54 | 147873.7765 17.187839 55 | 148426.2304 39.17724 56 | 148634.4224 20.539458 57 | 149190.0231 19.643502 58 | 149395.1611 10.928549 59 | 149953.621 15.454649 60 | 150715.2934 10.336184 61 | END IONS -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_660.2TOLERAN___534180716b576e89874a50f7245c8f30.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 1.0 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.1 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms2productcondition", 18 | "value": [ 19 | 660.2 20 | ] 21 | }, 22 | { 23 | "conditiontype": "where", 24 | "qualifiers": { 25 | "qualifierintensitypercent": { 26 | "name": "qualifierintensitypercent", 27 | "value": 1.0 28 | }, 29 | "qualifiermztolerance": { 30 | "name": "qualifiermztolerance", 31 | "unit": "mz", 32 | "value": 0.1 33 | }, 34 | "type": "qualifier" 35 | }, 36 | "type": "ms2productcondition", 37 | "value": [ 38 | 468.2 39 | ] 40 | } 41 | ], 42 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=660.2:TOLERANCEMZ=0.1:INTENSITYPERCENT=1 AND MS2PROD=468.2:TOLERANCEMZ=0.1:INTENSITYPERCENT=1", 43 | "querytype": { 44 | "datatype": "datams2data", 45 | "function": "functionscaninfo" 46 | } 47 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_TOLERANCEM___de1efdec2d9f13fba1556258838fc30f.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 5.0 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.1 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms2productcondition", 18 | "value": [ 19 | "X" 20 | ] 21 | }, 22 | { 23 | "conditiontype": "where", 24 | "qualifiers": { 25 | "qualifierintensitypercent": { 26 | "name": "qualifierintensitypercent", 27 | "value": 5.0 28 | }, 29 | "qualifiermztolerance": { 30 | "name": "qualifiermztolerance", 31 | "unit": "mz", 32 | "value": 0.1 33 | }, 34 | "type": "qualifier" 35 | }, 36 | "type": "ms2productcondition", 37 | "value": [ 38 | "X+119.1" 39 | ] 40 | } 41 | ], 42 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+119.1 :TOLERANCEMZ=0.1:INTENSITYPERCENT=5", 43 | "querytype": { 44 | "datatype": "datams2data", 45 | "function": "functionscaninfo" 46 | } 47 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS1MZ_X-2INTENSITYMA___61a2da302d53487ba89667bb7b7433a3.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitymatch": { 7 | "name": "qualifierintensitymatch", 8 | "value": "Y*0.0608+2e-06*X" 9 | }, 10 | "qualifierintensitytolpercent": { 11 | "name": "qualifierintensitytolpercent", 12 | "value": 25.0 13 | }, 14 | "type": "qualifier" 15 | }, 16 | "type": "ms1mzcondition", 17 | "value": [ 18 | "X-2.0" 19 | ] 20 | }, 21 | { 22 | "conditiontype": "where", 23 | "qualifiers": { 24 | "qualifierintensitymatch": { 25 | "name": "qualifierintensitymatch", 26 | "value": "Y" 27 | }, 28 | "qualifierintensitypercent": { 29 | "name": "qualifierintensitypercent", 30 | "value": 5.0 31 | }, 32 | "qualifierintensityreference": { 33 | "name": "qualifierintensityreference" 34 | }, 35 | "type": "qualifier" 36 | }, 37 | "type": "ms1mzcondition", 38 | "value": [ 39 | "X" 40 | ] 41 | }, 42 | { 43 | "conditiontype": "where", 44 | "type": "ms2precursorcondition", 45 | "value": [ 46 | "X" 47 | ] 48 | } 49 | ], 50 | "query": "QUERY scaninfo(MS2DATA) WHERE MS1MZ=X-2:INTENSITYMATCH=Y*(0.0608+(.000002*X)):INTENSITYMATCHPERCENT=25 AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 AND MS2PREC=X", 51 | "querytype": { 52 | "datatype": "datams2data", 53 | "function": "functionscaninfo" 54 | } 55 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_X_range_min_100,_max___ce48d94145b8fe9dd12d4257ffda9ad1.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "max": 500.0, 6 | "min": 100.0, 7 | "type": "xcondition" 8 | }, 9 | { 10 | "conditiontype": "where", 11 | "qualifiers": { 12 | "qualifierintensitymatch": { 13 | "name": "qualifierintensitymatch", 14 | "value": "Y*0.0608+2e-06*X" 15 | }, 16 | "qualifierintensitytolpercent": { 17 | "name": "qualifierintensitytolpercent", 18 | "value": 25.0 19 | }, 20 | "type": "qualifier" 21 | }, 22 | "type": "ms1mzcondition", 23 | "value": [ 24 | "X-2.0" 25 | ] 26 | }, 27 | { 28 | "conditiontype": "where", 29 | "qualifiers": { 30 | "qualifierintensitymatch": { 31 | "name": "qualifierintensitymatch", 32 | "value": "Y" 33 | }, 34 | "qualifierintensitypercent": { 35 | "name": "qualifierintensitypercent", 36 | "value": 5.0 37 | }, 38 | "qualifierintensityreference": { 39 | "name": "qualifierintensityreference" 40 | }, 41 | "type": "qualifier" 42 | }, 43 | "type": "ms1mzcondition", 44 | "value": [ 45 | "X" 46 | ] 47 | }, 48 | { 49 | "conditiontype": "where", 50 | "type": "ms2precursorcondition", 51 | "value": [ 52 | "X" 53 | ] 54 | } 55 | ], 56 | "query": "QUERY scaninfo(MS2DATA) WHERE X=range(min=100, max=500) AND MS1MZ=X-2:INTENSITYMATCH=Y*(0.0608+(.000002*X)):INTENSITYMATCHPERCENT=25 AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 AND MS2PREC=X", 57 | "querytype": { 58 | "datatype": "datams2data", 59 | "function": "functionscaninfo" 60 | } 61 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_X_TOLERANCEM___a5ce05df2b600cc86a0a76d43dd61ff6.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 5.0 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.1 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms2productcondition", 18 | "value": [ 19 | "X" 20 | ] 21 | }, 22 | { 23 | "conditiontype": "where", 24 | "qualifiers": { 25 | "qualifierintensitypercent": { 26 | "name": "qualifierintensitypercent", 27 | "value": 5.0 28 | }, 29 | "qualifiermztolerance": { 30 | "name": "qualifiermztolerance", 31 | "unit": "mz", 32 | "value": 0.1 33 | }, 34 | "type": "qualifier" 35 | }, 36 | "type": "ms2productcondition", 37 | "value": [ 38 | "X+164.9" 39 | ] 40 | }, 41 | { 42 | "conditiontype": "where", 43 | "qualifiers": { 44 | "qualifierintensitypercent": { 45 | "name": "qualifierintensitypercent", 46 | "value": 5.0 47 | }, 48 | "qualifiermztolerance": { 49 | "name": "qualifiermztolerance", 50 | "unit": "mz", 51 | "value": 0.1 52 | }, 53 | "type": "qualifier" 54 | }, 55 | "type": "ms2productcondition", 56 | "value": [ 57 | "X+329.8" 58 | ] 59 | } 60 | ], 61 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+164.9 :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+329.8:TOLERANCEMZ=0.1:INTENSITYPERCENT=5", 62 | "querytype": { 63 | "datatype": "datams2data", 64 | "function": "functionscaninfo" 65 | } 66 | } -------------------------------------------------------------------------------- /tests/test_translate.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | import os 4 | 5 | # Making sure the root is in the path, kind of a hack 6 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) 7 | 8 | from massql import msql_translator 9 | 10 | import json 11 | import pytest 12 | 13 | def test_translate(): 14 | languages = ["english", "korean", "chinese", "french", "german", "spanish", "portuguese", "japanese"] 15 | 16 | for language in languages: 17 | # Writing out the queries and comparing 18 | test_queries_filename = os.path.join(os.path.dirname(__file__), "test_queries.txt") 19 | for line in open(test_queries_filename): 20 | test_query = line.rstrip() 21 | translation = msql_translator.translate_query(test_query, language=language) 22 | if "contact Ming" in translation: 23 | print("Not Implemented ", language, test_query) 24 | print(translation) 25 | 26 | 27 | def test_translate2(): 28 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=184.0739:TOLERANCEMZ=0.01:INTENSITYPERCENT=30:INTENSITYVALUE=500 AND \ 29 | MS2PROD=125.0004:TOLERANCEMZ=0.01:INTENSITYPERCENT=10:INTENSITYVALUE=1500 AND \ 30 | MS2PROD=104.1075:TOLERANCEMZ=0.01 AND \ 31 | MS2PROD=86.09697:TOLERANCEMZ=0.01:INTENSITYPERCENT=10:INTENSITYVALUE=2000 \ 32 | " 33 | translated = msql_translator.translate_query(query, language="english") 34 | print(translated) 35 | 36 | def test_or_translate(): 37 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=(184.0739 OR 190)" 38 | translated = msql_translator.translate_query(query, language="english") 39 | print(translated) 40 | 41 | def test_cardinality(): 42 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=(58.06513 OR 60.04439 OR 70.06513 OR 72.08078 OR 74.06004 OR 84.04439 OR 84.08078 OR 86.09643 OR 87.05529 OR 88.0393 OR 88.07569 OR 100.11208 OR 101.07094 OR 101.10732 OR 102.05495 OR 102.09134 OR 104.05285 OR 110.07127 OR 114.12773 OR 115.08659 OR 115.12297 OR 116.0706 OR 118.0685 OR 120.08078 OR 124.08692 OR 129.10224 OR 129.11347 OR 129.13862 OR 130.08625 OR 132.08415 OR 134.09643 OR 136.07569 OR 138.10257 OR 143.12912 OR 148.11208 OR 150.09134 OR 157.14477 OR 159.09167 OR 164.10699 OR 173.10732 OR 187.12297):CARDINALITY=range(min=2,max=5):TOLERANCEPPM=10:INTENSITYPERCENT=5" 43 | translated = msql_translator.translate_query(query, language="english") 44 | print(translated) 45 | 46 | 47 | def main(): 48 | test_cardinality() 49 | #test_translate() 50 | #test_translate2() 51 | #test_or_translate() 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_341.28TOLERA___76665345ac8dbd664ab054e37eced641.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 2.0 9 | }, 10 | "qualifiermztolerance": { 11 | "name": "qualifiermztolerance", 12 | "unit": "mz", 13 | "value": 0.01 14 | }, 15 | "type": "qualifier" 16 | }, 17 | "type": "ms2productcondition", 18 | "value": [ 19 | 341.28 20 | ] 21 | }, 22 | { 23 | "conditiontype": "where", 24 | "qualifiers": { 25 | "qualifierintensitypercent": { 26 | "name": "qualifierintensitypercent", 27 | "value": 2.0 28 | }, 29 | "qualifiermztolerance": { 30 | "name": "qualifiermztolerance", 31 | "unit": "mz", 32 | "value": 0.01 33 | }, 34 | "type": "qualifier" 35 | }, 36 | "type": "ms2productcondition", 37 | "value": [ 38 | 323.27 39 | ] 40 | }, 41 | { 42 | "conditiontype": "where", 43 | "type": "ms2precursorcondition", 44 | "value": [ 45 | "X" 46 | ] 47 | }, 48 | { 49 | "conditiontype": "where", 50 | "qualifiers": { 51 | "qualifierintensitypercent": { 52 | "name": "qualifierintensitypercent", 53 | "value": 2.0 54 | }, 55 | "qualifiermztolerance": { 56 | "name": "qualifiermztolerance", 57 | "unit": "mz", 58 | "value": 0.01 59 | }, 60 | "type": "qualifier" 61 | }, 62 | "type": "ms2productcondition", 63 | "value": [ 64 | "X-358.2871" 65 | ] 66 | } 67 | ], 68 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=341.28:TOLERANCEMZ=0.01:INTENSITYPERCENT=2 AND MS2PROD=323.27:TOLERANCEMZ=0.01:INTENSITYPERCENT=2 AND MS2PREC=X AND MS2PROD=X-358.2871:TOLERANCEMZ=0.01:INTENSITYPERCENT=2", 69 | "querytype": { 70 | "datatype": "datams2data", 71 | "function": "functionscaninfo" 72 | } 73 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___10cc51d88921a4f32ca1f0a38ea3b4f2.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitymatch": { 7 | "name": "qualifierintensitymatch", 8 | "value": "Y" 9 | }, 10 | "qualifierintensitypercent": { 11 | "name": "qualifierintensitypercent", 12 | "value": 25.0 13 | }, 14 | "qualifierintensityreference": { 15 | "name": "qualifierintensityreference" 16 | }, 17 | "qualifiermztolerance": { 18 | "name": "qualifiermztolerance", 19 | "unit": "mz", 20 | "value": 0.1 21 | }, 22 | "type": "qualifier" 23 | }, 24 | "type": "ms1mzcondition", 25 | "value": [ 26 | "X" 27 | ] 28 | }, 29 | { 30 | "conditiontype": "where", 31 | "qualifiers": { 32 | "qualifierintensitymatch": { 33 | "name": "qualifierintensitymatch", 34 | "value": "Y*0.33" 35 | }, 36 | "qualifierintensitytolpercent": { 37 | "name": "qualifierintensitytolpercent", 38 | "value": 30.0 39 | }, 40 | "qualifiermztolerance": { 41 | "name": "qualifiermztolerance", 42 | "unit": "mz", 43 | "value": 0.1 44 | }, 45 | "type": "qualifier" 46 | }, 47 | "type": "ms1mzcondition", 48 | "value": [ 49 | "X+2.0" 50 | ] 51 | }, 52 | { 53 | "conditiontype": "where", 54 | "type": "ms2precursorcondition", 55 | "value": [ 56 | "X" 57 | ] 58 | }, 59 | { 60 | "conditiontype": "filter", 61 | "type": "ms1mzcondition", 62 | "value": [ 63 | "X" 64 | ] 65 | } 66 | ], 67 | "query": "QUERY scaninfo(MS1DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=25:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.33:INTENSITYMATCHPERCENT=30 AND MS2PREC=X FILTER MS1MZ=X", 68 | "querytype": { 69 | "datatype": "datams1data", 70 | "function": "functionscaninfo" 71 | } 72 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD__58.06513_OR___914cae9424e4492a2d88e8e1d0128f6e.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifiercardinality": { 7 | "max": 5.0, 8 | "min": 2.0, 9 | "name": "qualifiercardinality" 10 | }, 11 | "qualifierintensitypercent": { 12 | "name": "qualifierintensitypercent", 13 | "value": 5.0 14 | }, 15 | "qualifierppmtolerance": { 16 | "name": "qualifierppmtolerance", 17 | "unit": "ppm", 18 | "value": 10.0 19 | }, 20 | "type": "qualifier" 21 | }, 22 | "type": "ms2productcondition", 23 | "value": [ 24 | 58.06513, 25 | 60.04439, 26 | 70.06513, 27 | 72.08078, 28 | 74.06004, 29 | 84.04439, 30 | 84.08078, 31 | 86.09643, 32 | 87.05529, 33 | 88.0393, 34 | 88.07569, 35 | 100.11208, 36 | 101.07094, 37 | 101.10732, 38 | 102.05495, 39 | 102.09134, 40 | 104.05285, 41 | 110.07127, 42 | 114.12773, 43 | 115.08659, 44 | 115.12297, 45 | 116.0706, 46 | 118.0685, 47 | 120.08078, 48 | 124.08692, 49 | 129.10224, 50 | 129.11347, 51 | 129.13862, 52 | 130.08625, 53 | 132.08415, 54 | 134.09643, 55 | 136.07569, 56 | 138.10257, 57 | 143.12912, 58 | 148.11208, 59 | 150.09134, 60 | 157.14477, 61 | 159.09167, 62 | 164.10699, 63 | 173.10732, 64 | 187.12297 65 | ] 66 | } 67 | ], 68 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=(58.06513 OR 60.04439 OR 70.06513 OR 72.08078 OR 74.06004 OR 84.04439 OR 84.08078 OR 86.09643 OR 87.05529 OR 88.0393 OR 88.07569 OR 100.11208 OR 101.07094 OR 101.10732 OR 102.05495 OR 102.09134 OR 104.05285 OR 110.07127 OR 114.12773 OR 115.08659 OR 115.12297 OR 116.0706 OR 118.0685 OR 120.08078 OR 124.08692 OR 129.10224 OR 129.11347 OR 129.13862 OR 130.08625 OR 132.08415 OR 134.09643 OR 136.07569 OR 138.10257 OR 143.12912 OR 148.11208 OR 150.09134 OR 157.14477 OR 159.09167 OR 164.10699 OR 173.10732 OR 187.12297):CARDINALITY=range(min=2,max=5):TOLERANCEPPM=10:INTENSITYPERCENT=5", 69 | "querytype": { 70 | "datatype": "datams2data", 71 | "function": "functionscaninfo" 72 | } 73 | } -------------------------------------------------------------------------------- /.github/workflows/test-unit.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Unit Testing 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | unit-testing-parse: 14 | # Use a matrix strategy to test against multiple Python versions 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | # Define the Python versions you want to test 19 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 20 | steps: 21 | - uses: actions/checkout@v2 22 | 23 | # Step to set up Python using the version from the matrix 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v2 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | 29 | # Added step to explicitly output the Python version for clarity in logs 30 | - name: 🐍 Display Python version 31 | run: python -V 32 | 33 | - name: Setup Graphviz 34 | uses: ts-graphviz/setup-graphviz@v1 35 | 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip 39 | # Check if the pip install -r requirements_test.txt line is correct 40 | # It's common to name the test requirements file 'requirements_dev.txt' or 'requirements_test.txt' 41 | # Assuming your file is 'requirements_test.txt' 42 | pip install flake8 pytest 43 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 44 | if [ -f requirements_test.txt ]; then pip install -r requirements_test.txt; fi 45 | 46 | - name: Testing Parse 47 | run: | 48 | # Use python -m pytest to ensure the correct environment's pytest is used 49 | python -m pytest -vv --cov=massql ./tests/test_parse.py 50 | 51 | - name: Run test_translate 52 | run: | 53 | pytest -vv --cov=massql ./tests/test_translate.py 54 | 55 | test-data-processing: 56 | runs-on: ubuntu-latest 57 | 58 | steps: 59 | - uses: actions/checkout@v2 60 | - uses: actions/setup-python@v2 61 | with: 62 | python-version: 3.9 63 | - uses: ts-graphviz/setup-graphviz@v1 64 | 65 | - name: Download Test Data 66 | run: | 67 | cd tests && sh ./get_data.sh 68 | cd .. 69 | 70 | - name: Install dependencies 71 | run: | 72 | python -m pip install --upgrade pip 73 | pip install flake8 pytest pytest-cov 74 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 75 | if [ -f requirements_test.txt ]; then pip install -r requirements_test.txt; fi 76 | 77 | - name: Run test_extraction 78 | run: | 79 | pytest -vv --cov=massql ./tests/test_extraction.py 80 | 81 | - name: Run test_visualize 82 | run: | 83 | pytest -vv --cov=massql ./tests/test_visualize.py 84 | 85 | - name: Run test_file_loading 86 | run: | 87 | pytest -vv --cov=massql ./tests/test_file_loading.py 88 | 89 | - name: Run test_query 90 | run: | 91 | pytest -vv --cov=massql ./tests/test_query.py -n 4 92 | -------------------------------------------------------------------------------- /tests/test_visualize.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | # Making sure the root is in the path, kind of a hack 5 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) 6 | 7 | from massql import msql_visualizer 8 | 9 | import json 10 | import pytest 11 | 12 | def test_visualize(): 13 | # Writing out the queries and comparing 14 | current_dir = os.path.dirname(__file__) 15 | test_queries_filename = os.path.join(current_dir, "test_queries.txt") 16 | for line in open(test_queries_filename): 17 | test_query = line.rstrip() 18 | print(test_query) 19 | ms1_fig, ms2_fig = msql_visualizer.visualize_query(test_query) 20 | 21 | def test_visualize_basic(): 22 | #query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=177 AND MS2PROD=270 AND MS2NL=163" 23 | #query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X AND MS2PROD=X+14" 24 | #query = "QUERY scaninfo(MS2DATA) WHERE MS1MZ=X AND MS1MZ=X+14" 25 | query = "QUERY scaninfo(MS1DATA) WHERE \ 26 | MS1MZ=X-2:INTENSITYMATCH=Y*0.063:INTENSITYMATCHPERCENT=25 \ 27 | AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE" 28 | 29 | ms1_fig, ms2_fig = msql_visualizer.visualize_query(query) 30 | ms2_fig.write_image("test_ms2_visualize.png", engine="kaleido") 31 | ms1_fig.write_image("test_ms1_visualize.png", engine="kaleido") 32 | 33 | 34 | def test_visualize_xrange(): 35 | query = "QUERY scaninfo(MS1DATA) WHERE \ 36 | MS1MZ=X-2:INTENSITYMATCH=Y*0.063:INTENSITYMATCHPERCENT=25 \ 37 | AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND \ 38 | X=range(min=100, max=500)" 39 | 40 | ms1_fig, ms2_fig = msql_visualizer.visualize_query(query) 41 | ms2_fig.write_image("test_ms2_visualize.png", engine="kaleido") 42 | ms1_fig.write_image("test_ms1_visualize.png", engine="kaleido") 43 | 44 | def test_visualize_y_set(): 45 | query = "QUERY scaninfo(MS1DATA) WHERE \ 46 | MS1MZ=X-2:INTENSITYMATCH=Y*0.063:INTENSITYMATCHPERCENT=25 \ 47 | AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE" 48 | 49 | ms1_fig, ms2_fig = msql_visualizer.visualize_query(query, variable_y=0.1) 50 | ms1_fig.write_image("test_ms1_visualize.png", engine="kaleido") 51 | open("test_ms1_visualize.html", 'w').write(ms1_fig.to_html(full_html=False, include_plotlyjs='cdn')) 52 | 53 | def test_visualize_usi(): 54 | query = "QUERY scaninfo(MS1DATA) WHERE MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 \ 55 | AND \ 56 | MS1MZ=X+1:INTENSITYMATCH=Y*0.4:INTENSITYMATCHPERCENT=70 AND MS1MZ=X+1.998:INTENSITYMATCH=Y*0.446:INTENSITYMATCHPERCENT=30:TOLERANCEPPM=10 AND MS2PREC=X" 57 | 58 | ms1_usi = "mzspec:MSV000085669:ccms_peak/mzML_ph5/std_mix_100-mixed-metal_2_ph8.mzML:scan:1380" 59 | 60 | import requests 61 | r = requests.get("https://metabolomics-usi.gnps2.org/json/?usi1={}".format(ms1_usi)) 62 | ms1_peaks = r.json()["peaks"] 63 | 64 | 65 | ms1_fig, ms2_fig = msql_visualizer.visualize_query(query, variable_x=662.27, variable_y=0.0436, ms1_peaks=ms1_peaks) 66 | ms1_fig.write_image("test_ms1_visualize.png", engine="kaleido") 67 | open("test_ms1_visualize.html", 'w').write(ms1_fig.to_html(full_html=False, include_plotlyjs='cdn')) 68 | 69 | def main(): 70 | #test_visualize_y_set() 71 | test_visualize_xrange() 72 | #test_visualize_usi() 73 | 74 | if __name__ == "__main__": 75 | main() 76 | -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS2PROD_184.0739TOLE___208cb60e1979a0a3d01cf4ea7a398d27.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitypercent": { 7 | "name": "qualifierintensitypercent", 8 | "value": 30.0 9 | }, 10 | "qualifierintensityvalue": { 11 | "name": "qualifierintensityvalue", 12 | "value": 500.0 13 | }, 14 | "qualifiermztolerance": { 15 | "name": "qualifiermztolerance", 16 | "unit": "mz", 17 | "value": 0.01 18 | }, 19 | "type": "qualifier" 20 | }, 21 | "type": "ms2productcondition", 22 | "value": [ 23 | 184.0739 24 | ] 25 | }, 26 | { 27 | "conditiontype": "where", 28 | "qualifiers": { 29 | "qualifierintensitypercent": { 30 | "name": "qualifierintensitypercent", 31 | "value": 10.0 32 | }, 33 | "qualifierintensityvalue": { 34 | "name": "qualifierintensityvalue", 35 | "value": 1500.0 36 | }, 37 | "qualifiermztolerance": { 38 | "name": "qualifiermztolerance", 39 | "unit": "mz", 40 | "value": 0.01 41 | }, 42 | "type": "qualifier" 43 | }, 44 | "type": "ms2productcondition", 45 | "value": [ 46 | 125.0004 47 | ] 48 | }, 49 | { 50 | "conditiontype": "where", 51 | "qualifiers": { 52 | "qualifiermztolerance": { 53 | "name": "qualifiermztolerance", 54 | "unit": "mz", 55 | "value": 0.01 56 | }, 57 | "type": "qualifier" 58 | }, 59 | "type": "ms2productcondition", 60 | "value": [ 61 | 104.1075 62 | ] 63 | }, 64 | { 65 | "conditiontype": "where", 66 | "qualifiers": { 67 | "qualifierintensitypercent": { 68 | "name": "qualifierintensitypercent", 69 | "value": 10.0 70 | }, 71 | "qualifierintensityvalue": { 72 | "name": "qualifierintensityvalue", 73 | "value": 2000.0 74 | }, 75 | "qualifiermztolerance": { 76 | "name": "qualifiermztolerance", 77 | "unit": "mz", 78 | "value": 0.01 79 | }, 80 | "type": "qualifier" 81 | }, 82 | "type": "ms2productcondition", 83 | "value": [ 84 | 86.09697 85 | ] 86 | } 87 | ], 88 | "query": "QUERY scaninfo(MS2DATA) WHERE MS2PROD=184.0739:TOLERANCEMZ=0.01:INTENSITYPERCENT>30:INTENSITYVALUE>500 AND MS2PROD=125.0004:TOLERANCEMZ=0.01:INTENSITYPERCENT>10:INTENSITYVALUE>1500 AND MS2PROD=104.1075:TOLERANCEMZ=0.01 AND MS2PROD=86.09697:TOLERANCEMZ=0.01:INTENSITYPERCENT>10:INTENSITYVALUE>2000", 89 | "querytype": { 90 | "datatype": "datams2data", 91 | "function": "functionscaninfo" 92 | } 93 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_X-1.993INTENSI___c11b8d3c85dd33cbb8a73699d7f26efb.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitymatch": { 7 | "name": "qualifierintensitymatch", 8 | "value": "Y*0.063" 9 | }, 10 | "qualifierintensitytolpercent": { 11 | "name": "qualifierintensitytolpercent", 12 | "value": 25.0 13 | }, 14 | "qualifierppmtolerance": { 15 | "name": "qualifierppmtolerance", 16 | "unit": "ppm", 17 | "value": 10.0 18 | }, 19 | "type": "qualifier" 20 | }, 21 | "type": "ms1mzcondition", 22 | "value": [ 23 | "X-1.993" 24 | ] 25 | }, 26 | { 27 | "conditiontype": "where", 28 | "qualifiers": { 29 | "qualifierintensitymatch": { 30 | "name": "qualifierintensitymatch", 31 | "value": "Y" 32 | }, 33 | "qualifierintensitypercent": { 34 | "name": "qualifierintensitypercent", 35 | "value": 5.0 36 | }, 37 | "qualifierintensityreference": { 38 | "name": "qualifierintensityreference" 39 | }, 40 | "type": "qualifier" 41 | }, 42 | "type": "ms1mzcondition", 43 | "value": [ 44 | "X" 45 | ] 46 | }, 47 | { 48 | "conditiontype": "where", 49 | "qualifiers": { 50 | "qualifierintensitymatch": { 51 | "name": "qualifierintensitymatch", 52 | "value": "Y*0.5" 53 | }, 54 | "qualifierintensitytolpercent": { 55 | "name": "qualifierintensitytolpercent", 56 | "value": 60.0 57 | }, 58 | "type": "qualifier" 59 | }, 60 | "type": "ms1mzcondition", 61 | "value": [ 62 | "X+1.0" 63 | ] 64 | }, 65 | { 66 | "conditiontype": "where", 67 | "qualifiers": { 68 | "qualifierppmtolerance": { 69 | "name": "qualifierppmtolerance", 70 | "unit": "ppm", 71 | "value": 10.0 72 | }, 73 | "type": "qualifier" 74 | }, 75 | "type": "ms1mzcondition", 76 | "value": [ 77 | "X-52.91" 78 | ] 79 | }, 80 | { 81 | "conditiontype": "where", 82 | "type": "ms2precursorcondition", 83 | "value": [ 84 | "X" 85 | ] 86 | }, 87 | { 88 | "conditiontype": "filter", 89 | "type": "ms1mzcondition", 90 | "value": [ 91 | "X" 92 | ] 93 | } 94 | ], 95 | "query": "QUERY scaninfo(MS1DATA) WHERE MS1MZ=X-1.993:INTENSITYMATCH=Y*0.063:INTENSITYMATCHPERCENT=25:TOLERANCEPPM=10 AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 AND MS1MZ=X+1:INTENSITYMATCH=Y*0.5:INTENSITYMATCHPERCENT=60 AND MS1MZ=X-52.91:TOLERANCEPPM=10 AND MS2PREC=X FILTER MS1MZ=X", 96 | "querytype": { 97 | "datatype": "datams1data", 98 | "function": "functionscaninfo" 99 | } 100 | } -------------------------------------------------------------------------------- /.github/workflows/test-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Periodic Testing of Pypi Package 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | schedule: 10 | # Runs every Monday at 08:00 UTC 11 | - cron: '0 8 * * 1' 12 | 13 | jobs: 14 | package-testing-parse: 15 | # Use a matrix strategy to test against multiple Python versions 16 | runs-on: ubuntu-latest 17 | strategy: 18 | matrix: 19 | # Define the Python versions you want to test 20 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 21 | steps: 22 | - uses: actions/checkout@v2 23 | 24 | # Step to set up Python using the version from the matrix 25 | - name: Set up Python ${{ matrix.python-version }} 26 | uses: actions/setup-python@v2 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | 30 | # Added step to explicitly output the Python version for clarity in logs 31 | - name: 🐍 Display Python version 32 | run: python -V 33 | 34 | - name: Setup Graphviz 35 | uses: ts-graphviz/setup-graphviz@v1 36 | 37 | - name: Install dependencies 38 | run: | 39 | python -m pip install --upgrade pip 40 | pip install massql 41 | if [ -f requirements_test.txt ]; then pip install -r requirements_test.txt; fi 42 | 43 | - name: Removing MassQL source 44 | run: | 45 | rm massql -r 46 | 47 | - name: Testing Parse 48 | run: | 49 | python -m pytest -vv --cov=massql ./tests/test_parse.py 50 | 51 | - name: Run test_translate 52 | run: | 53 | pytest -vv --cov=massql ./tests/test_translate.py 54 | 55 | package-testing-data-processing: 56 | # Use a matrix strategy to test against multiple Python versions 57 | runs-on: ubuntu-latest 58 | strategy: 59 | matrix: 60 | # Define the Python versions you want to test 61 | python-version: ["3.9"] 62 | steps: 63 | - uses: actions/checkout@v2 64 | 65 | # Step to set up Python using the version from the matrix 66 | - name: Set up Python ${{ matrix.python-version }} 67 | uses: actions/setup-python@v2 68 | with: 69 | python-version: ${{ matrix.python-version }} 70 | 71 | # Added step to explicitly output the Python version for clarity in logs 72 | - name: 🐍 Display Python version 73 | run: python -V 74 | 75 | - name: Setup Graphviz 76 | uses: ts-graphviz/setup-graphviz@v1 77 | 78 | - name: Download Test Data 79 | run: | 80 | cd tests && sh ./get_data.sh 81 | cd .. 82 | 83 | - name: Install dependencies 84 | run: | 85 | python -m pip install --upgrade pip 86 | pip install massql 87 | if [ -f requirements_test.txt ]; then pip install -r requirements_test.txt; fi 88 | 89 | - name: Removing MassQL source 90 | run: | 91 | rm massql -r 92 | 93 | - name: Run test_extraction 94 | run: | 95 | pytest -vv --cov=massql ./tests/test_extraction.py 96 | 97 | - name: Run test_visualize 98 | run: | 99 | pytest -vv --cov=massql ./tests/test_visualize.py 100 | 101 | - name: Run test_file_loading 102 | run: | 103 | pytest -vv --cov=massql ./tests/test_file_loading.py 104 | 105 | - name: Run test_query 106 | run: | 107 | pytest -vv --cov=massql ./tests/test_query.py -n 4 108 | 109 | 110 | -------------------------------------------------------------------------------- /tests/test_file_loading.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | # Making sure the root is in the path, kind of a hack 5 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) 6 | 7 | from massql import msql_fileloading 8 | 9 | import json 10 | import pytest 11 | import time 12 | 13 | def test_improper_file(): 14 | with pytest.raises(Exception): 15 | msql_fileloading.load_file('file.cdf') 16 | 17 | def test_gnps_library_loading(): 18 | ms1_df, ms2_df = msql_fileloading.load_data("tests/data/gnps-library.json") 19 | print(ms2_df[ms2_df["scan"] == "CCMSLIB00000072227"]) 20 | assert(len(ms2_df[ms2_df["scan"] == "CCMSLIB00000072227"]) > 300) 21 | 22 | def test_mzml_load(): 23 | print("Loading pymzML") 24 | ms1_df, ms2_df = msql_fileloading._load_data_mzML2("tests/data/JB_182_2_fe.mzML") 25 | print(ms2_df) 26 | 27 | assert(max(ms2_df["rt"]) > 0) 28 | 29 | print("Loading Pyteomics") 30 | ms1_df, ms2_df = msql_fileloading._load_data_mzML_pyteomics("tests/data/JB_182_2_fe.mzML") 31 | print(ms2_df) 32 | 33 | assert(max(ms2_df["rt"]) > 0) 34 | 35 | 36 | def test_mzxml_load(): 37 | ms1_df, ms2_df = msql_fileloading.load_data("tests/data/T04251505.mzXML", cache=None) 38 | 39 | def test_mgf_load(): 40 | ms1_df, ms2_df = msql_fileloading.load_data("tests/data/specs_ms.mgf", cache=None) 41 | 42 | def test_mzml_mobility_load(): 43 | ms1_df, ms2_df = msql_fileloading.load_data("tests/data/meoh_water_ms2_1_31_1_395.mzML", cache=None) 44 | assert("mobility" in ms2_df) 45 | assert(max(ms2_df["rt"]) > 0) 46 | 47 | def test_mzml_rt_seconds(): 48 | ms1_df, ms2_df = msql_fileloading.load_data("tests/data/1810E-II.mzML", cache=None) 49 | print(ms2_df) 50 | assert(max(ms2_df["rt"]) < 60) 51 | 52 | def test_waters_load(): 53 | # This has UV spectra in it, we need to handle that gracefully 54 | ms1_df, ms2_df = msql_fileloading.load_data("tests/data/GT15A.mzML", cache=None) 55 | 56 | 57 | def test_cache_feather(): 58 | # Measure start time 59 | start_time = time.time() 60 | msql_fileloading.load_data("tests/data/JB_182_2_fe.mzML", cache="feather") 61 | msql_fileloading.load_data("tests/data/T04251505.mzXML", cache="feather") 62 | msql_fileloading.load_data("tests/data/meoh_water_ms2_1_31_1_395.mzML", cache="feather") 63 | # Measure end time 64 | end_time = time.time() 65 | print("Feather time: ", end_time - start_time) 66 | 67 | def test_nocache(): 68 | # Measure start time 69 | start_time = time.time() 70 | msql_fileloading.load_data("tests/data/JB_182_2_fe.mzML", cache=None) 71 | msql_fileloading.load_data("tests/data/T04251505.mzXML", cache=None) 72 | msql_fileloading.load_data("tests/data/meoh_water_ms2_1_31_1_395.mzML", cache=None) 73 | # Measure end time 74 | end_time = time.time() 75 | print("No cache time: ", end_time - start_time) 76 | 77 | def test_cache_filename(): 78 | input_filename = "tests/data/JB_182_2_fe.mzML" 79 | ms1_filename, ms2_filename = msql_fileloading._determine_feather_cache_filename(input_filename) 80 | assert(ms1_filename == "tests/data/JB_182_2_fe.mzML_ms1.msql.feather") 81 | 82 | ms1_filename, ms2_filename = msql_fileloading._determine_feather_cache_filename(input_filename, cache_file="cache_file") 83 | 84 | ms1_filename, ms2_filename = msql_fileloading._determine_feather_cache_filename(input_filename, cache_dir="cache_file") 85 | print(ms1_filename, ms2_filename) 86 | 87 | 88 | def main(): 89 | #test_mzml_load() 90 | #test_mzml_mobility_load() 91 | #test_mzml_rt_seconds() 92 | #test_waters_load() 93 | #test_cache_feather() 94 | #test_nocache() 95 | test_cache_filename() 96 | 97 | 98 | if __name__ == "__main__": 99 | main() 100 | -------------------------------------------------------------------------------- /workflow/.nextflow/history: -------------------------------------------------------------------------------- 1 | 2023-06-29 17:27:14 - clever_poincare - 970777586803ac5cf3829da36228416b 28a5681c-8724-46e0-aaea-d4d46ace730d nextflow run workflow.nf -c test.config 2 | 2023-06-29 17:28:07 - fervent_roentgen - 39a1d0251ee21699f787b323bc30ec78 0ecca33d-d6ef-4f54-b93d-3d4f24ea6ac8 nextflow run workflow.nf -c test.config 3 | 2023-06-29 17:28:27 - scruffy_moriondo - ad24abe470e05583572d9283de7926ce dff43999-fd46-4b77-aa98-7b10c8e8cce2 nextflow run workflow.nf -c test.config 4 | 2023-06-29 17:28:58 - distracted_faraday - c1c1ec0d1157f96997e994439f10cc9f 601e2974-9388-4979-8d9e-e38deb8da9cb nextflow run workflow.nf -c test.config 5 | 2023-06-29 17:29:29 - fervent_carlsson - ba4e4cdee6217b25063bdc022644bef5 e922605c-0877-4299-b626-d45dedbef936 nextflow run workflow.nf -c test.config 6 | 2023-06-29 17:29:45 4m 7s compassionate_bartik ERR 7576a5a9f2d43c5a3f9487b46ddca565 86505b83-a56d-4179-8498-3987c26f2c05 nextflow run workflow.nf -c test.config 7 | 2023-06-29 17:33:58 4m 6s angry_torvalds OK bc9cb50db6b442959f1d11b63085e64e f9eeffa0-390d-4389-9a8b-5921ed3db901 nextflow run workflow.nf -c test.config --parallel_files YES 8 | 2023-06-29 17:38:12 7.3s mighty_galileo ERR bc9cb50db6b442959f1d11b63085e64e dde8c0e1-778e-4e7c-a123-a26d751526b7 nextflow run workflow.nf -c test.config --parallel_files YES 9 | 2023-06-29 17:38:26 - peaceful_hilbert - a4021aecb5535bf5ac80f668dcfbf0bc b75a5f59-d24c-4b83-a8a3-b8af416d00b8 nextflow run workflow.nf -c test.config --parallel_files YES 10 | 2023-06-29 17:38:42 - stoic_brown - a4021aecb5535bf5ac80f668dcfbf0bc fae3991d-c833-4ecc-921e-2ab73bb95746 nextflow run workflow.nf -c test.config --parallel_files YES --resume 11 | 2023-06-29 17:39:15 4m 44s admiring_kirch OK bed422ff9b8c096939f08df30c250f05 0bc4b07e-570c-44d6-af81-6c8162ead2fd nextflow run workflow.nf -c test.config --parallel_files YES --resume 12 | 2023-06-29 17:44:17 4m 48s ridiculous_payne OK 0cda8d56eb164dcd180c04688e673f35 23505b1e-0ef3-41a2-9d58-161eab2ade49 nextflow run workflow.nf -c test.config --parallel_files YES --resume 13 | 2023-07-23 10:04:33 33.9s mighty_solvay ERR bdcb50bad8e1371f8a5e118592eb7865 636d7f3e-d7c4-47ab-bf3f-b264282d416c nextflow run workflow.nf -c test.config --parallel_files YES --resume 14 | 2023-07-23 11:33:28 13s exotic_pauling ERR b399ae278185a91451e63827f80a0818 d24bf493-6303-4706-973e-b60c310a82ec nextflow run workflow.nf -c test.config --parallel_files YES --resume 15 | 2023-07-23 12:00:21 7m 59s big_pauling ERR b399ae278185a91451e63827f80a0818 82349cae-2a67-4e19-9235-3d1ac0695aa8 nextflow run workflow.nf -c test.config --parallel_files YES --resume 16 | 2023-07-23 12:08:28 13.5s desperate_elion OK b399ae278185a91451e63827f80a0818 514e5b26-3728-4d42-9293-85ab6c38e470 nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 17 | 2023-07-23 13:22:39 8.5s mighty_goodall ERR cf0e0c8002bca83be33254242463e2b6 b98d1746-9c1a-4e3a-a4a6-4e6317e8052b nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 18 | 2023-07-23 13:27:41 7.6s compassionate_lumiere ERR cf0e0c8002bca83be33254242463e2b6 14f27b53-5a2d-4720-bfa6-d01541f3d9e8 nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 19 | 2023-07-23 13:29:06 2m 35s cranky_noyce ERR cf0e0c8002bca83be33254242463e2b6 edba329c-a7dc-4757-93a8-40740444e360 nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 20 | 2023-07-23 13:34:55 3.9s condescending_plateau OK 593bbe495d0ab259db5a5df48fa2a798 0144b051-9fc7-4715-9899-5093dfaf80f6 nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 21 | 2023-07-23 13:35:21 4m 35s chaotic_minsky ERR e8137b4fbeab6fd97849066a7fa34e68 aa7fd51f-b34b-4692-a396-d56f916e7fff nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 22 | 2023-07-23 13:41:37 5m 31s special_pasteur OK 85aa13ec71093fff899b816e843e75bb ddd73f2b-3938-4c1c-8d16-6304d0196e78 nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 23 | -------------------------------------------------------------------------------- /tests/get_data.sh: -------------------------------------------------------------------------------- 1 | mkdir -p data 2 | cd data 3 | wget --no-verbose --output-document=GNPS00002_A3_p.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000084494/ccms_peak/raw/GNPS00002_A3_p.mzML" 4 | wget --no-verbose --output-document=GNPS00002_A3_p.mzml "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000084494/ccms_peak/raw/GNPS00002_A3_p.mzML" 5 | wget --no-verbose --output-document=GNPS00002_A10_n.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000084494/ccms_peak/raw/GNPS00002_A10_n.mzML" 6 | wget --no-verbose --output-document=QC_0.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000085852/ccms_peak/QC_raw/QC_0.mzML" 7 | wget --no-verbose --output-document=bld_plt1_07_120_1.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000085944/ccms_peak/raw_data/bld_plt1_07_120_1.mzML" 8 | wget --no-verbose --output-document=NS_1x_test.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000087352/updates/2021-05-03_allegraaron_95d2215b/peak/NS_1x_test.mzML" 9 | wget --no-verbose --output-document=JB_182_2_fe.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000084289/ccms_peak/JB_182_2_fe.mzML" 10 | wget --no-verbose --output-document=S_N2_neutral_Zn.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000083387/updates/2019-11-12_allegraaron_e893cb7e/peak/S_N2_neutral_Zn.mzML" 11 | wget --no-verbose --output-document=gnps-library.json "https://external.gnps2.org/gnpslibrary/GNPS-LIBRARY.json" 12 | wget --no-verbose --output-document=specs_ms.mgf "http://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?task=5ecfcf81cb3c471698995b194d8246a0&block=main&file=spectra/specs_ms.mgf" 13 | wget --no-verbose --output-document=1810E-II.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?file=f.MSV000084691/ccms_peak/1810E-II.mzML&forceDownload=true" 14 | wget --no-verbose --output-document=T04251505.mzXML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?file=f.MSV000082797/ccms_peak/raw/MTBLS368/T04251505.mzXML&forceDownload=true" 15 | wget --no-verbose --output-document=isa_9_fe.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?file=f.MSV000084030/ccms_peak/isa_9_fe.mzML&forceDownload=true" 16 | wget --no-verbose --output-document=01308_H02_P013387_B00_N16_R1.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000083508/ccms_peak/colon/Trypsin_HCD_QExactiveplus/01308_H02_P013387_B00_N16_R1.mzML" 17 | wget --no-verbose --output-document=119A-24.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?file=f.MSV000083461/ccms_peak/mzXML/119A-24.mzML" 18 | wget --no-verbose --output-document=Hui_N2_fe.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000084628/ccms_peak/Hui_N2_fe.mzML" 19 | wget --no-verbose --output-document=meoh_water_ms2_1_31_1_395.mzML "https://proteomics2.ucsd.edu/ProteoSAFe/DownloadResultFile?task=2eb041215fdd4f89a2ef91be70752e16&file=workflow_results/spectra/meoh_water_ms2_1_31_1_395.mzML&block=main&process_html=false" 20 | wget --no-verbose --output-document=MMSRG_027.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.MSV000088268/peak/Anelize%20and%20Hector/MMSRG_027.mzML" 21 | wget --no-verbose --output-document=featurelist_pos.mgf "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?file=f.MSV000086995/updates/2022-01-18_mwang87_e619431a/peak/bahbobeh/featurelist_pos.mgf&forceDownload=true" 22 | wget --no-verbose --output-document=GT15A.mzML "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?file=f.MSV000087048/ccms_peak/Green_Tea_manuscript_data/GT15A.mzML" 23 | wget --no-verbose --output-document=PLT2_B1.mzML "http://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?task=9e5e3f5a23754d16a981fd009ae51e82&block=main&file=f.MSV000088800/ccms_peak/NRRL_PLT2_czapek_solid_raw/PLT2_B1.mzML" -------------------------------------------------------------------------------- /tests/test_extraction.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | # Making sure the root is in the path, kind of a hack 5 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) 6 | 7 | from massql import msql_extract 8 | from massql import msql_engine 9 | 10 | import json 11 | import pytest 12 | 13 | def test_extract_mzML(): 14 | query = "QUERY scaninfo(MS2DATA)" 15 | results_df = msql_engine.process_query(query, "tests/data/GNPS00002_A3_p.mzML") 16 | 17 | assert(len(results_df) > 1) 18 | results_df["filename"] = "GNPS00002_A3_p.mzML" 19 | 20 | merged_summary_df = msql_extract._extract_spectra(results_df, "tests/data/", 21 | output_json_filename="test.json", 22 | output_summary="summary.tsv", 23 | output_mzML_filename="test.mzML") 24 | assert(len(merged_summary_df) == 79) 25 | 26 | def test_extract_mzmz(): 27 | query = "QUERY scaninfo(MS2DATA)" 28 | results_df = msql_engine.process_query(query, "tests/data/GNPS00002_A3_p.mzml") 29 | 30 | assert(len(results_df) > 1) 31 | results_df["filename"] = "GNPS00002_A3_p.mzml" 32 | 33 | merged_summary_df = msql_extract._extract_spectra(results_df, "tests/data/", 34 | output_json_filename="test.json", 35 | output_summary="summary.tsv", 36 | output_mzML_filename="test.mzML") 37 | assert(len(merged_summary_df) == 79) 38 | 39 | def test_extract_mzXML(): 40 | query = "QUERY scaninfo(MS1DATA)" 41 | results_df = msql_engine.process_query(query, "tests/data/T04251505.mzXML") 42 | print(results_df) 43 | 44 | assert(len(results_df) > 1) 45 | results_df = results_df[:5] 46 | 47 | results_df["filename"] = "T04251505.mzXML" 48 | 49 | print("Extracting", len(results_df)) 50 | merged_summary_df = msql_extract._extract_spectra(results_df, "tests/data/", output_json_filename="test.json") 51 | assert(len(merged_summary_df) == 5) 52 | 53 | def test_extract_MGF(): 54 | query = "QUERY scaninfo(MS2DATA)" 55 | results_df = msql_engine.process_query(query, "tests/data/specs_ms.mgf") 56 | print(results_df) 57 | 58 | assert(len(results_df) > 1) 59 | results_df = results_df[:5] 60 | 61 | results_df["filename"] = "specs_ms.mgf" 62 | 63 | print("Extracting", len(results_df)) 64 | print(results_df) 65 | 66 | # Extracting now 67 | merged_summary_df = msql_extract._extract_spectra(results_df, "tests/data/", output_json_filename="test.json") 68 | assert(len(merged_summary_df) == 5) 69 | 70 | def test_gnps_library_extract(): 71 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=175:INTENSITYPERCENT=20" 72 | results_df = msql_engine.process_query(query, "tests/data/gnps-library.json") 73 | 74 | results_df["filename"] = "gnps-library.json" 75 | 76 | merged_summary_df = msql_extract._extract_spectra(results_df, 77 | "tests/data/", 78 | output_mzML_filename="extracted.mzML", 79 | output_summary="extracted.tsv") 80 | 81 | def test_waters_uv_extract(): 82 | query = "QUERY scaninfo(MS2DATA)" 83 | results_df = msql_engine.process_query(query, "tests/data/MMSRG_027.mzML") 84 | 85 | results_df["filename"] = "MMSRG_027.mzML" 86 | results_df = results_df.head() 87 | 88 | merged_summary_df = msql_extract._extract_spectra(results_df, 89 | "tests/data/", 90 | output_mzML_filename="extracted.mzML", 91 | output_summary="extracted.tsv") 92 | 93 | def main(): 94 | #test_extract_mzML() 95 | #test_extract_mzXML() 96 | test_extract_MGF() 97 | #test_gnps_library_extract() 98 | #test_waters_uv_extract() 99 | 100 | 101 | if __name__ == "__main__": 102 | main() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mass Spec Query Language 2 | 3 | [![Unit Testing](https://github.com/mwang87/MassQueryLanguage/actions/workflows/test-unit.yml/badge.svg)](https://github.com/mwang87/MassQueryLanguage/actions/workflows/test-unit.yml) 4 | [![Periodic Testing of Package](https://github.com/mwang87/MassQueryLanguage/actions/workflows/test-package.yml/badge.svg)](https://github.com/mwang87/MassQueryLanguage/actions/workflows/test-package.yml) 5 | 6 | 7 | The Mass Spec Query Language (MassQL) is a domain specific language meant to be a succinct way to 8 | express a query in a mass spectrometry centric fashion. It is inspired by SQL, 9 | but it attempts to bake in assumptions of mass spectrometry to make querying much more 10 | natural for mass spectrometry users. Broadly we attempt to design it according to several principles: 11 | 12 | 1. Expressiveness - Capture complex mass spectrometry patterns that the community would like to look for 13 | 1. Precision - Exactly prescribe how to find data without ambiguity 14 | 2. Scalable - Easily facilitating the querying of one spectrum all the way up to entire repositories of data 15 | 3. Relatively Natural - MassQL should be relatively easy to read and write and even use to communicate ideas about mass 16 | spectrometry, you know like a language. 17 | 18 | ## Repository Structure 19 | 20 | This is the repository to define the language and reference implementation. This contains several parts 21 | 22 | 1. Language Grammar 23 | 1. Reference Implementation Python API 24 | 1. Command line Utility to execute 25 | 1. NextFlow Workflow For Large Scale Analysis 26 | 1. ProteoSAFe workflow 27 | 1. Dash interactive exploration 28 | 29 | ## Developers/Contact 30 | 31 | Mingxun Wang is the main creator and developer of MassQL. Contact me for contributing or using it! 32 | 33 | ## Language Specification/Documentation 34 | 35 | Checkout specifics for the language, examples, and design patterns at the documentation. 36 | 37 | [Documentation Link](https://mwang87.github.io/MassQueryLanguage_Documentation/) 38 | 39 | ## Python API 40 | 41 | We currently test massql in python 3.9, but are figuring out other versions if they work or not. 42 | 43 | To install massql 44 | 45 | ``` 46 | pip install massql 47 | ``` 48 | 49 | Here is the most basic operation you can do 50 | 51 | ``` 52 | from massql import msql_engine 53 | 54 | results_df = msql_engine.process_query(input_query, input_filename) 55 | ``` 56 | 57 | If you want to push in a data frame you already have, you can specify it 58 | 59 | ``` 60 | from massql import msql_engine 61 | from massql import msql_fileloading 62 | 63 | # Loading Data 64 | ms1_df, ms2_df = msql_fileloading.load_data(input_filename) 65 | 66 | # Executing Query 67 | results_df = msql_engine.process_query(input_query, input_filename, ms1_df=ms1_df, ms2_df=ms2_df) 68 | ``` 69 | 70 | ## Command Line Tool 71 | 72 | You can use the command line tool ```massql``` to query things or put things into a pipeline. 73 | 74 | A few examples of what you can do 75 | 76 | ``` 77 | massql test.mzML "QUERY scaninfo(MS2DATA)" --output_file results.tsv 78 | ``` 79 | 80 | ## Web API 81 | 82 | ### API Version 83 | 84 | ```/api``` 85 | 86 | ### Parsing query into intermediate JSON string 87 | 88 | ```/parse?query=``` 89 | 90 | [Example Link](https://msql.ucsd.edu/parse?query=QUERY%20MS2DATA%20WHERE%20MS1MZ=100) 91 | 92 | ### Visualization of Query 93 | 94 | Visualization image of MS1 spectra 95 | 96 | ```/visualize/ms1``` 97 | 98 | [Example Link](https://msql.ucsd.edu/visualize/ms1?query=QUERY+scaninfo%28MS1DATA%29+WHERE+MS1MZ%3DX%3ATOLERANCEMZ%3D0.1%3AINTENSITYPERCENT%3D25%3AINTENSITYMATCH%3DY%3AINTENSITYMATCHREFERENCE+AND+%0AMS1MZ%3DX%2B2%3ATOLERANCEMZ%3D0.1%3AINTENSITYMATCH%3DY%2A0.66%3AINTENSITYMATCHPERCENT%3D30+AND+%0AMS1MZ%3DX-2%3ATOLERANCEMZ%3D0.1%3AINTENSITYMATCH%3DY%2A0.66%3AINTENSITYMATCHPERCENT%3D30+AND+MS1MZ%3DX%2B4%3ATOLERANCEMZ%3D0.2%3AINTENSITYMATCH%3DY%2A0.17%3AINTENSITYMATCHPERCENT%3D40+AND+%0AMS1MZ%3DX-4%3ATOLERANCEMZ%3D0.2%3AINTENSITYMATCH%3DY%2A0.17%3AINTENSITYMATCHPERCENT%3D40+AND+%0AMS2PREC%3DX&filename=GNPS00002_A3_p.mzML&x_axis=&y_axis=&facet_column=&scan=&x_value=572.828&y_value=0.64&ms1_usi=mzspec%3AGNPS%3ATASK-f6e8346934904399ae6742723762b2cb-f.MSV000084691%2Fccms_peak%2F1810E-II.mzML%3Ascan%3A474&ms2_usi=) 99 | 100 | Visualization image of MS2 spectra 101 | 102 | ```/visualize/ms2``` 103 | 104 | ## Testing 105 | 106 | To run tests, you'll need to first fetch some fixtures that are not bundled with the git repo: 107 | ```cd tests && sh ./get_data.sh``` 108 | 109 | You will also want to install the extra requirements for the test suite: 110 | ```pip install -r requirements_test.txt``` 111 | 112 | ## License 113 | 114 | MIT License 115 | -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___8211c5bd2dcbba27b8676e891ccdf078.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitymatch": { 7 | "name": "qualifierintensitymatch", 8 | "value": "Y" 9 | }, 10 | "qualifierintensitypercent": { 11 | "name": "qualifierintensitypercent", 12 | "value": 5.0 13 | }, 14 | "qualifierintensityreference": { 15 | "name": "qualifierintensityreference" 16 | }, 17 | "qualifiermztolerance": { 18 | "name": "qualifiermztolerance", 19 | "unit": "mz", 20 | "value": 0.1 21 | }, 22 | "type": "qualifier" 23 | }, 24 | "type": "ms1mzcondition", 25 | "value": [ 26 | "X" 27 | ] 28 | }, 29 | { 30 | "conditiontype": "where", 31 | "qualifiers": { 32 | "qualifierintensitymatch": { 33 | "name": "qualifierintensitymatch", 34 | "value": "Y*0.97" 35 | }, 36 | "qualifierintensitytolpercent": { 37 | "name": "qualifierintensitytolpercent", 38 | "value": 30.0 39 | }, 40 | "qualifiermztolerance": { 41 | "name": "qualifiermztolerance", 42 | "unit": "mz", 43 | "value": 0.1 44 | }, 45 | "type": "qualifier" 46 | }, 47 | "type": "ms1mzcondition", 48 | "value": [ 49 | "X+2.0" 50 | ] 51 | }, 52 | { 53 | "conditiontype": "where", 54 | "qualifiers": { 55 | "qualifierintensitymatch": { 56 | "name": "qualifierintensitymatch", 57 | "value": "Y*0.34" 58 | }, 59 | "qualifierintensitytolpercent": { 60 | "name": "qualifierintensitytolpercent", 61 | "value": 30.0 62 | }, 63 | "qualifiermztolerance": { 64 | "name": "qualifiermztolerance", 65 | "unit": "mz", 66 | "value": 0.1 67 | }, 68 | "type": "qualifier" 69 | }, 70 | "type": "ms1mzcondition", 71 | "value": [ 72 | "X-2.0" 73 | ] 74 | }, 75 | { 76 | "conditiontype": "where", 77 | "qualifiers": { 78 | "qualifierintensitymatch": { 79 | "name": "qualifierintensitymatch", 80 | "value": "Y*0.32" 81 | }, 82 | "qualifierintensitytolpercent": { 83 | "name": "qualifierintensitytolpercent", 84 | "value": 40.0 85 | }, 86 | "qualifiermztolerance": { 87 | "name": "qualifiermztolerance", 88 | "unit": "mz", 89 | "value": 0.2 90 | }, 91 | "type": "qualifier" 92 | }, 93 | "type": "ms1mzcondition", 94 | "value": [ 95 | "X+4.0" 96 | ] 97 | }, 98 | { 99 | "conditiontype": "where", 100 | "qualifiers": { 101 | "qualifiermztolerance": { 102 | "name": "qualifiermztolerance", 103 | "unit": "mz", 104 | "value": 2.0 105 | }, 106 | "type": "qualifier" 107 | }, 108 | "type": "ms2precursorcondition", 109 | "value": [ 110 | "X" 111 | ] 112 | }, 113 | { 114 | "conditiontype": "where", 115 | "max": 900.0, 116 | "min": 300.0, 117 | "type": "xcondition" 118 | } 119 | ], 120 | "query": "QUERY scaninfo(MS2DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=5:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.97:INTENSITYMATCHPERCENT=30 AND MS1MZ=X-2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.34:INTENSITYMATCHPERCENT=30 AND MS1MZ=X+4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.32:INTENSITYMATCHPERCENT=40 AND MS2PREC=X:TOLERANCEMZ=2 AND X=range(min=300, max=900)", 121 | "querytype": { 122 | "datatype": "datams2data", 123 | "function": "functionscaninfo" 124 | } 125 | } -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS1DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___9008f814f26ce5a1835e900d1e9c659f.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitymatch": { 7 | "name": "qualifierintensitymatch", 8 | "value": "Y" 9 | }, 10 | "qualifierintensitypercent": { 11 | "name": "qualifierintensitypercent", 12 | "value": 25.0 13 | }, 14 | "qualifierintensityreference": { 15 | "name": "qualifierintensityreference" 16 | }, 17 | "qualifiermztolerance": { 18 | "name": "qualifiermztolerance", 19 | "unit": "mz", 20 | "value": 0.1 21 | }, 22 | "type": "qualifier" 23 | }, 24 | "type": "ms1mzcondition", 25 | "value": [ 26 | "X" 27 | ] 28 | }, 29 | { 30 | "conditiontype": "where", 31 | "qualifiers": { 32 | "qualifierintensitymatch": { 33 | "name": "qualifierintensitymatch", 34 | "value": "Y*0.66" 35 | }, 36 | "qualifierintensitytolpercent": { 37 | "name": "qualifierintensitytolpercent", 38 | "value": 30.0 39 | }, 40 | "qualifiermztolerance": { 41 | "name": "qualifiermztolerance", 42 | "unit": "mz", 43 | "value": 0.1 44 | }, 45 | "type": "qualifier" 46 | }, 47 | "type": "ms1mzcondition", 48 | "value": [ 49 | "X+2.0" 50 | ] 51 | }, 52 | { 53 | "conditiontype": "where", 54 | "qualifiers": { 55 | "qualifierintensitymatch": { 56 | "name": "qualifierintensitymatch", 57 | "value": "Y*0.66" 58 | }, 59 | "qualifierintensitytolpercent": { 60 | "name": "qualifierintensitytolpercent", 61 | "value": 30.0 62 | }, 63 | "qualifiermztolerance": { 64 | "name": "qualifiermztolerance", 65 | "unit": "mz", 66 | "value": 0.1 67 | }, 68 | "type": "qualifier" 69 | }, 70 | "type": "ms1mzcondition", 71 | "value": [ 72 | "X-2.0" 73 | ] 74 | }, 75 | { 76 | "conditiontype": "where", 77 | "qualifiers": { 78 | "qualifierintensitymatch": { 79 | "name": "qualifierintensitymatch", 80 | "value": "Y*0.17" 81 | }, 82 | "qualifierintensitytolpercent": { 83 | "name": "qualifierintensitytolpercent", 84 | "value": 40.0 85 | }, 86 | "qualifiermztolerance": { 87 | "name": "qualifiermztolerance", 88 | "unit": "mz", 89 | "value": 0.2 90 | }, 91 | "type": "qualifier" 92 | }, 93 | "type": "ms1mzcondition", 94 | "value": [ 95 | "X+4.0" 96 | ] 97 | }, 98 | { 99 | "conditiontype": "where", 100 | "qualifiers": { 101 | "qualifierintensitymatch": { 102 | "name": "qualifierintensitymatch", 103 | "value": "Y*0.17" 104 | }, 105 | "qualifierintensitytolpercent": { 106 | "name": "qualifierintensitytolpercent", 107 | "value": 40.0 108 | }, 109 | "qualifiermztolerance": { 110 | "name": "qualifiermztolerance", 111 | "unit": "mz", 112 | "value": 0.2 113 | }, 114 | "type": "qualifier" 115 | }, 116 | "type": "ms1mzcondition", 117 | "value": [ 118 | "X-4.0" 119 | ] 120 | }, 121 | { 122 | "conditiontype": "where", 123 | "type": "ms2precursorcondition", 124 | "value": [ 125 | "X" 126 | ] 127 | }, 128 | { 129 | "conditiontype": "filter", 130 | "type": "ms1mzcondition", 131 | "value": [ 132 | "X" 133 | ] 134 | } 135 | ], 136 | "query": "QUERY scaninfo(MS1DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=25:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X-2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X+4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS1MZ=X-4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS2PREC=X FILTER MS1MZ=X", 137 | "querytype": { 138 | "datatype": "datams1data", 139 | "function": "functionscaninfo" 140 | } 141 | } -------------------------------------------------------------------------------- /massql/msql.ebnf: -------------------------------------------------------------------------------- 1 | statement: querykeyword querytype wherekeyword wherefullcondition+ "FILTER" filterfullcondition+ 2 | | querykeyword querytype "FILTER" filterfullcondition+ 3 | | querykeyword querytype wherekeyword wherefullcondition+ 4 | | querykeyword querytype 5 | 6 | querytype: datams1data 7 | | datams2data 8 | | function"(" datams1data ")" 9 | | function "(" datams2data ")" 10 | | function"(" datams1data "," param "=" floating ")" 11 | 12 | datams1data: "MS1DATA" | "ms1data" | "Ms1Data" 13 | datams2data: "MS2DATA" | "ms2data" | "Ms2Data" 14 | wherekeyword: "WHERE" | "where" | "Where" 15 | querykeyword: "QUERY" | "query" | "Query" 16 | 17 | wherefullcondition: wherefullcondition booleanandconjunction wherefullcondition 18 | | condition ":" qualifier 19 | | condition 20 | 21 | filterfullcondition: filterfullcondition booleanandconjunction filterfullcondition 22 | | condition ":" qualifier 23 | | condition 24 | 25 | condition: conditionfields equal numericalexpression 26 | | conditionfields equal "(" statement ")" 27 | | conditionfields equal "(" numericalexpressionwithor ")" 28 | | conditionfields equal wildcard 29 | | polaritycondition equal positivepolarity 30 | | polaritycondition equal negativepolarity 31 | | xcondition equal xfunction "(min=" numericalexpression "," "max=" numericalexpression ")" 32 | | mobilitycondition equal mobilityfunction "(min=" numericalexpression "," "max=" numericalexpression ")" 33 | 34 | qualifier: qualifier ":" qualifier 35 | | qualifierfields equal numericalexpression 36 | | qualifierfields greaterthan numericalexpression 37 | | qualifierfields lessthan numericalexpression 38 | | qualifierintensityreference 39 | | qualifiermassdefect equal xdefect "(min=" numericalexpression "," "max=" numericalexpression ")" 40 | | qualifierexclude 41 | | qualifiercardinality equal xrange "(min=" numericalexpression "," "max=" numericalexpression ")" 42 | | qualifierotherscan equal otherscanrtrange "(left=" numericalexpression "," "right=" numericalexpression ")" 43 | 44 | equal: "=" 45 | lessthan: "<" 46 | greaterthan: ">" 47 | positivepolarity: "POSITIVE" 48 | | "positive" 49 | | "Positive" 50 | negativepolarity: "NEGATIVE" 51 | | "negative" 52 | | "Negative" 53 | 54 | conditionfields: ms2productcondition 55 | | ms2precursorcondition 56 | | ms2neutrallosscondition 57 | | ms1mzcondition 58 | | rtmincondition 59 | | rtmaxcondition 60 | | scanmincondition 61 | | scanmaxcondition 62 | | chargecondition 63 | 64 | ms2productcondition: "MS2PROD" | "MS2MZ" 65 | ms2precursorcondition: "MS2PREC" 66 | ms2neutrallosscondition: "MS2NL" 67 | ms1mzcondition: "MS1MZ" 68 | rtmincondition: "RTMIN" 69 | rtmaxcondition: "RTMAX" 70 | scanmincondition: "SCANMIN" 71 | scanmaxcondition: "SCANMAX" 72 | polaritycondition: "POLARITY" 73 | chargecondition: "CHARGE" 74 | 75 | mobilitycondition: "MOBILITY" 76 | mobilityfunction: mobilityrange 77 | mobilityrange: "range" 78 | 79 | xcondition: "X" 80 | xfunction: xrange | xdefect 81 | xrange: "range" 82 | xdefect: "massdefect" 83 | 84 | otherscanrtrange: "rtrange" 85 | 86 | 87 | qualifierfields: qualifiermztolerance 88 | | qualifierppmtolerance 89 | | qualifierintensitypercent 90 | | qualifierintensityticpercent 91 | | qualifierintensityvalue 92 | | qualifierintensitymatch 93 | | qualifierintensitytolpercent 94 | 95 | qualifiermztolerance: "TOLERANCEMZ" 96 | qualifierppmtolerance: "TOLERANCEPPM" 97 | qualifierintensitypercent: "INTENSITYPERCENT" 98 | qualifierintensityticpercent: "INTENSITYTICPERCENT" 99 | qualifierintensityvalue: "INTENSITYVALUE" 100 | qualifierintensitymatch: "INTENSITYMATCH" 101 | qualifierintensitytolpercent: "INTENSITYMATCHPERCENT" 102 | qualifierintensityreference: "INTENSITYMATCHREFERENCE" 103 | qualifiermassdefect: "MASSDEFECT" 104 | qualifierexclude: "EXCLUDED" 105 | qualifiercardinality: "CARDINALITY" | "MATCHCOUNT" 106 | qualifierotherscan: "OTHERSCAN" 107 | 108 | function: functionscannum 109 | | functionscansum 110 | | functionscanrangesum 111 | | functionscanmz 112 | | functionscaninfo 113 | | functionscanmaxint 114 | 115 | functionscannum: "scannum" 116 | functionscansum: "scansum" 117 | functionscanrangesum: "scanrangesum" 118 | functionscanmz: "scanmz" 119 | functionscaninfo: "scaninfo" 120 | functionscanmaxint: "scanmaxint" 121 | 122 | booleanandconjunction: "AND" 123 | | "and" 124 | | "And" 125 | 126 | booleanorconjunction: "OR" 127 | 128 | param: "TOLERANCE" 129 | floating: /[-+]?([0-9]*\.[0-9]+|[0-9]+)/ 130 | variable: /[XY]/ 131 | wildcard: "ANY" 132 | 133 | // OR Numerical Expression 134 | numericalexpressionwithor: numericalexpression 135 | | numericalexpressionwithor booleanorconjunction numericalexpressionwithor 136 | 137 | // Numerical Expression 138 | numericalexpression: term 139 | | numericalexpression plus term 140 | | numericalexpression minus term 141 | term: factor 142 | | term multiply factor 143 | | term divide factor 144 | factor: floating 145 | | variable 146 | | "formula(" moleculeformula ")" 147 | | "aminoaciddelta(" aminoacids ")" 148 | | peptidefunction 149 | | "(" numericalexpression ")" 150 | peptidefunction: "peptide(" peptide "," "charge=" peptidecharge "," "ion=" peptideion ")" 151 | multiply: "*" 152 | divide: "/" 153 | plus: "+" 154 | minus: "-" 155 | 156 | moleculeformula: /[A-Z][A-Za-z0-9]*/ 157 | aminoacids: /[A-Z][A-Z]*/ 158 | peptide: /[A-Z][A-Z]*/ 159 | peptidecharge: /[1-9]/ 160 | peptideion: /[A-Za-z]/ 161 | 162 | // Boiler Plate 163 | %import common.ESCAPED_STRING -> STRING 164 | %import common.SIGNED_NUMBER -> NUMBER 165 | %import common.WS 166 | %ignore WS -------------------------------------------------------------------------------- /tests/reference_parses/QUERY_scaninfo_MS2DATA__WHERE_MS1MZ_XTOLERANCEMZ_0___6c21ac388a474d5ebc9485f44f67972e.json: -------------------------------------------------------------------------------- 1 | { 2 | "conditions": [ 3 | { 4 | "conditiontype": "where", 5 | "qualifiers": { 6 | "qualifierintensitymatch": { 7 | "name": "qualifierintensitymatch", 8 | "value": "Y" 9 | }, 10 | "qualifierintensitypercent": { 11 | "name": "qualifierintensitypercent", 12 | "value": 5.0 13 | }, 14 | "qualifierintensityreference": { 15 | "name": "qualifierintensityreference" 16 | }, 17 | "qualifiermztolerance": { 18 | "name": "qualifiermztolerance", 19 | "unit": "mz", 20 | "value": 0.1 21 | }, 22 | "type": "qualifier" 23 | }, 24 | "type": "ms1mzcondition", 25 | "value": [ 26 | "X" 27 | ] 28 | }, 29 | { 30 | "conditiontype": "where", 31 | "qualifiers": { 32 | "qualifierintensitymatch": { 33 | "name": "qualifierintensitymatch", 34 | "value": "Y*0.66" 35 | }, 36 | "qualifierintensitytolpercent": { 37 | "name": "qualifierintensitytolpercent", 38 | "value": 30.0 39 | }, 40 | "qualifiermztolerance": { 41 | "name": "qualifiermztolerance", 42 | "unit": "mz", 43 | "value": 0.1 44 | }, 45 | "type": "qualifier" 46 | }, 47 | "type": "ms1mzcondition", 48 | "value": [ 49 | "X+2.0" 50 | ] 51 | }, 52 | { 53 | "conditiontype": "where", 54 | "qualifiers": { 55 | "qualifierintensitymatch": { 56 | "name": "qualifierintensitymatch", 57 | "value": "Y*0.66" 58 | }, 59 | "qualifierintensitytolpercent": { 60 | "name": "qualifierintensitytolpercent", 61 | "value": 30.0 62 | }, 63 | "qualifiermztolerance": { 64 | "name": "qualifiermztolerance", 65 | "unit": "mz", 66 | "value": 0.1 67 | }, 68 | "type": "qualifier" 69 | }, 70 | "type": "ms1mzcondition", 71 | "value": [ 72 | "X-2.0" 73 | ] 74 | }, 75 | { 76 | "conditiontype": "where", 77 | "qualifiers": { 78 | "qualifierintensitymatch": { 79 | "name": "qualifierintensitymatch", 80 | "value": "Y*0.17" 81 | }, 82 | "qualifierintensitytolpercent": { 83 | "name": "qualifierintensitytolpercent", 84 | "value": 40.0 85 | }, 86 | "qualifiermztolerance": { 87 | "name": "qualifiermztolerance", 88 | "unit": "mz", 89 | "value": 0.2 90 | }, 91 | "type": "qualifier" 92 | }, 93 | "type": "ms1mzcondition", 94 | "value": [ 95 | "X+4.0" 96 | ] 97 | }, 98 | { 99 | "conditiontype": "where", 100 | "qualifiers": { 101 | "qualifierintensitymatch": { 102 | "name": "qualifierintensitymatch", 103 | "value": "Y*0.17" 104 | }, 105 | "qualifierintensitytolpercent": { 106 | "name": "qualifierintensitytolpercent", 107 | "value": 40.0 108 | }, 109 | "qualifiermztolerance": { 110 | "name": "qualifiermztolerance", 111 | "unit": "mz", 112 | "value": 0.2 113 | }, 114 | "type": "qualifier" 115 | }, 116 | "type": "ms1mzcondition", 117 | "value": [ 118 | "X-4.0" 119 | ] 120 | }, 121 | { 122 | "conditiontype": "where", 123 | "qualifiers": { 124 | "qualifiermztolerance": { 125 | "name": "qualifiermztolerance", 126 | "unit": "mz", 127 | "value": 4.0 128 | }, 129 | "type": "qualifier" 130 | }, 131 | "type": "ms2precursorcondition", 132 | "value": [ 133 | "X" 134 | ] 135 | }, 136 | { 137 | "conditiontype": "where", 138 | "max": 900.0, 139 | "min": 400.0, 140 | "type": "xcondition" 141 | } 142 | ], 143 | "query": "QUERY scaninfo(MS2DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=5:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X-2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X+4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS1MZ=X-4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS2PREC=X:TOLERANCEMZ=4 AND X=range(min=400, max=900)", 144 | "querytype": { 145 | "datatype": "datams2data", 146 | "function": "functionscaninfo" 147 | } 148 | } -------------------------------------------------------------------------------- /tests/test_queries.txt: -------------------------------------------------------------------------------- 1 | QUERY MS2DATA WHERE MS2PROD=226.18 2 | QUERY MS2DATA WHERE MS2PROD=226.18:TOLERANCEPPM=5 3 | QUERY scanmz(MS2DATA) WHERE MS2PROD=226.18 4 | QUERY MS2DATA WHERE MS2PROD=226.18 AND MS2PREC=226.1797 5 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=226.18:TOLERANCEPPM=5 6 | QUERY scansum(MS1DATA) WHERE MS1MZ=100:TOLERANCEMZ=0.1 7 | QUERY scannum(MS2DATA) WHERE MS2PREC=85.0282:TOLERANCEMZ=0.1 8 | QUERY scansum(MS2DATA) WHERE MS2PROD=271 AND MS2PREC=500 9 | QUERY scannum(MS2DATA) WHERE MS2PROD=88:TOLERANCEMZ=0.1:INTENSITYPERCENT>10 AND MS2PREC=600 10 | QUERY scannum(MS2DATA) WHERE MS2NL=163 11 | QUERY scaninfo(MS1DATA) WHERE MS1MZ=425.2898:TOLERANCEMZ=0.1:INTENSITYPERCENT>0.1 12 | QUERY scaninfo(MS1DATA) WHERE MS1MZ=425.2898:TOLERANCEMZ=0.1:INTENSITYPERCENT>1 13 | QUERY scaninfo(MS1DATA) WHERE RTMIN=50 14 | QUERY scaninfo(MS1DATA) WHERE SCANMIN=100 AND SCANMAX=105 15 | QUERY scaninfo(MS1DATA) WHERE POLARITY=Positive 16 | QUERY scaninfo(MS2DATA) WHERE CHARGE=1 17 | QUERY scaninfo(MS1DATA) WHERE MS2PROD=309.2:TOLERANCEMZ=0.1:INTENSITYTICPERCENT=10 18 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=X :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+164.9 :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+329.8:TOLERANCEMZ=0.1:INTENSITYPERCENT=5 19 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=X :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+119.1 :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 20 | QUERY scaninfo(MS1DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=25:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X-2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X+4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS1MZ=X-4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS2PREC=X FILTER MS1MZ=X 21 | QUERY scaninfo(MS1DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=25:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.33:INTENSITYMATCHPERCENT=30 AND MS2PREC=X FILTER MS1MZ=X 22 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=167.0857:TOLERANCEPPM=5 AND MS2NL=176.0321:TOLERANCEPPM=5 23 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=157.0857+10 24 | QUERY scaninfo(MS1DATA) WHERE POLARITY=Negative 25 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=X AND MS2PROD=2.0*(X - formula(Fe)) 26 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=X and MS2PROD=2.0*(X - formula(Fe)) 27 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=X And MS2PROD=2.0*(X - formula(Fe)) 28 | QUERY scaninfo(MS1DATA) WHERE MS1MZ=X-1.993:INTENSITYMATCH=Y*0.063:INTENSITYMATCHPERCENT=25:TOLERANCEPPM=10 AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 AND MS1MZ=X+1:INTENSITYMATCH=Y*0.5:INTENSITYMATCHPERCENT=60 AND MS1MZ=X-52.91:TOLERANCEPPM=10 AND MS2PREC=X FILTER MS1MZ=X 29 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=660.2:TOLERANCEMZ=0.1:INTENSITYPERCENT=1 AND MS2PROD=468.2:TOLERANCEMZ=0.1:INTENSITYPERCENT=1 30 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=X :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+164.9 :TOLERANCEMZ=0.1:INTENSITYPERCENT=5 AND MS2PROD=X+329.8:TOLERANCEMZ=0.1:INTENSITYPERCENT=5 31 | QUERY scaninfo(MS1DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=25:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.33:INTENSITYMATCHPERCENT=30 AND MS2PREC=X FILTER MS1MZ=X 32 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=184.0739:TOLERANCEMZ=0.01:INTENSITYPERCENT>30:INTENSITYVALUE>500 AND MS2PROD=125.0004:TOLERANCEMZ=0.01:INTENSITYPERCENT>10:INTENSITYVALUE>1500 AND MS2PROD=104.1075:TOLERANCEMZ=0.01 AND MS2PROD=86.09697:TOLERANCEMZ=0.01:INTENSITYPERCENT>10:INTENSITYVALUE>2000 33 | QUERY scaninfo(MS2DATA) WHERE MS1MZ=X-2:INTENSITYMATCH=Y*(0.0608+(.000002*X)):INTENSITYMATCHPERCENT=25 AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 AND MS2PREC=X 34 | QUERY scaninfo(MS2DATA) WHERE X=range(min=100, max=500) AND MS1MZ=X-2:INTENSITYMATCH=Y*(0.0608+(.000002*X)):INTENSITYMATCHPERCENT=25 AND MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYPERCENT=5 AND MS2PREC=X 35 | query scaninfo(MS1DATA) where POLARITY=Negative 36 | Query scaninfo(MS1DATA) Where POLARITY=Negative 37 | QUERY scaninfo(MS1DATA) WHERE MS1MZ=(X OR X+2 OR X+4 OR X+6) 38 | QUERY scaninfo(MS2DATA) WHERE MOBILITY=range(min=100, max=500) 39 | QUERY scaninfo(MS2DATA) WHERE MS2PREC=403.15:TOLERANCEMZ=0.1 AND MOBILITY=range(min=0.910, max=0.940) 40 | QUERY scansum(MS1DATA) FILTER MS1MZ=515:TOLERANCEMZ=35:MASSDEFECT=massdefect(min=0.1332, max=0.2112) 41 | QUERY scaninfo(MS2DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=5:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X-2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.66:INTENSITYMATCHPERCENT=30 AND MS1MZ=X+4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS1MZ=X-4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.17:INTENSITYMATCHPERCENT=40 AND MS2PREC=X:TOLERANCEMZ=4 AND X=range(min=400, max=900) 42 | QUERY scaninfo(MS2DATA) WHERE MS1MZ=X:TOLERANCEMZ=0.1:INTENSITYPERCENT=5:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE AND MS1MZ=X+2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.97:INTENSITYMATCHPERCENT=30 AND MS1MZ=X-2:TOLERANCEMZ=0.1:INTENSITYMATCH=Y*0.34:INTENSITYMATCHPERCENT=30 AND MS1MZ=X+4:TOLERANCEMZ=0.2:INTENSITYMATCH=Y*0.32:INTENSITYMATCHPERCENT=40 AND MS2PREC=X:TOLERANCEMZ=2 AND X=range(min=300, max=900) 43 | QUERY scannum(MS2DATA) WHERE MS2PROD=85.0284:TOLERANCEMZ=0.005 44 | QUERY scaninfo(MS2DATA) WHERE MS2PREC=X AND MOBILITY=range(min=X*0.0006775+0.40557, max=X*0.00078231+0.48817) AND X=massdefect(min=0.9, max=0.99) 45 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=(58.06513 OR 60.04439 OR 70.06513 OR 72.08078 OR 74.06004 OR 84.04439 OR 84.08078 OR 86.09643 OR 87.05529 OR 88.0393 OR 88.07569 OR 100.11208 OR 101.07094 OR 101.10732 OR 102.05495 OR 102.09134 OR 104.05285 OR 110.07127 OR 114.12773 OR 115.08659 OR 115.12297 OR 116.0706 OR 118.0685 OR 120.08078 OR 124.08692 OR 129.10224 OR 129.11347 OR 129.13862 OR 130.08625 OR 132.08415 OR 134.09643 OR 136.07569 OR 138.10257 OR 143.12912 OR 148.11208 OR 150.09134 OR 157.14477 OR 159.09167 OR 164.10699 OR 173.10732 OR 187.12297):CARDINALITY=range(min=2,max=5):TOLERANCEPPM=10:INTENSITYPERCENT=5 46 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=226.18:TOLERANCEPPM=5:EXCLUDED 47 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=formula(C10) 48 | QUERY scaninfo(MS2DATA) WHERE MS2PROD=341.28:TOLERANCEMZ=0.01:INTENSITYPERCENT=2 AND MS2PROD=323.27:TOLERANCEMZ=0.01:INTENSITYPERCENT=2 AND MS2PREC=X AND MS2PROD=X-358.2871:TOLERANCEMZ=0.01:INTENSITYPERCENT=2 -------------------------------------------------------------------------------- /workflow/.nextflow.log.2: -------------------------------------------------------------------------------- 1 | Jul-23 13:34:52.008 [main] DEBUG nextflow.cli.Launcher - $> nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 2 | Jul-23 13:34:52.425 [main] INFO nextflow.cli.CmdRun - N E X T F L O W ~ version 21.10.0 3 | Jul-23 13:34:52.488 [main] INFO nextflow.cli.CmdRun - Launching `workflow.nf` [condescending_plateau] - revision: 593bbe495d 4 | Jul-23 13:34:52.528 [main] DEBUG nextflow.config.ConfigBuilder - User config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 5 | Jul-23 13:34:52.529 [main] DEBUG nextflow.config.ConfigBuilder - Parsing config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 6 | Jul-23 13:34:52.593 [main] DEBUG nextflow.config.ConfigBuilder - Applying config profile: `standard` 7 | Jul-23 13:34:55.025 [main] DEBUG nextflow.plugin.PluginsFacade - Setting up plugin manager > mode=prod; plugins-dir=/home/user/.nextflow/plugins 8 | Jul-23 13:34:55.028 [main] DEBUG nextflow.plugin.PluginsFacade - Plugins default=[] 9 | Jul-23 13:34:55.051 [main] INFO org.pf4j.DefaultPluginStatusProvider - Enabled plugins: [] 10 | Jul-23 13:34:55.054 [main] INFO org.pf4j.DefaultPluginStatusProvider - Disabled plugins: [] 11 | Jul-23 13:34:55.063 [main] INFO org.pf4j.DefaultPluginManager - PF4J version 3.4.1 in 'deployment' mode 12 | Jul-23 13:34:55.085 [main] INFO org.pf4j.AbstractPluginManager - No plugins 13 | Jul-23 13:34:55.228 [main] DEBUG nextflow.Session - Session uuid: 0144b051-9fc7-4715-9899-5093dfaf80f6 14 | Jul-23 13:34:55.229 [main] DEBUG nextflow.Session - Run name: condescending_plateau 15 | Jul-23 13:34:55.230 [main] DEBUG nextflow.Session - Executor pool size: 16 16 | Jul-23 13:34:55.306 [main] DEBUG nextflow.cli.CmdRun - 17 | Version: 21.10.0 build 5640 18 | Created: 11-11-2021 18:27 UTC (10:27 PDT) 19 | System: Linux 5.15.0-53-generic 20 | Runtime: Groovy 3.0.9 on OpenJDK 64-Bit Server VM 1.8.0_152-release-1056-b12 21 | Encoding: UTF-8 (UTF-8) 22 | Process: 4179540@mingwangucr [169.235.26.140] 23 | CPUs: 16 - Mem: 61.8 GB (1.3 GB) - Swap: 735.3 MB (0) 24 | Jul-23 13:34:55.366 [main] DEBUG nextflow.Session - Work-dir: /home/user/SourceCode/MassQueryLanguage/workflow/work [ext2/ext3] 25 | Jul-23 13:34:55.423 [main] DEBUG nextflow.executor.ExecutorFactory - Extension executors providers=[] 26 | Jul-23 13:34:55.459 [main] DEBUG nextflow.Session - Observer factory: DefaultObserverFactory 27 | Jul-23 13:34:55.561 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 28 | Jul-23 13:34:55.772 [main] DEBUG nextflow.Session - Session start invoked 29 | Jul-23 13:34:57.518 [main] DEBUG nextflow.script.ScriptRunner - > Launching execution 30 | Jul-23 13:34:57.759 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 31 | Jul-23 13:34:58.067 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 32 | Jul-23 13:34:58.068 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 33 | Jul-23 13:34:58.081 [main] DEBUG nextflow.executor.Executor - [warm up] executor > local 34 | Jul-23 13:34:58.093 [main] DEBUG n.processor.LocalPollingMonitor - Creating local task monitor for executor 'local' > cpus=16; memory=61.8 GB; capacity=16; pollInterval=100ms; dumpInterval=5m 35 | Jul-23 13:34:58.340 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 36 | Jul-23 13:34:58.340 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 37 | Jul-23 13:34:58.418 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 38 | Jul-23 13:34:58.418 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 39 | Jul-23 13:34:58.437 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 40 | Jul-23 13:34:58.437 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 41 | Jul-23 13:34:58.451 [main] DEBUG nextflow.Session - Workflow process names [dsl2]: summarizeResults, formatResultsMergeRounds, queryData2, queryData, formatExtractedSpectraRounds 42 | Jul-23 13:34:58.458 [main] DEBUG nextflow.extension.CH - Bridging dataflow queue=DataflowQueue(queue=[]) 43 | Jul-23 13:34:58.464 [main] DEBUG nextflow.Session - Ignite dataflow network (11) 44 | Jul-23 13:34:58.505 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > queryData 45 | Jul-23 13:34:58.507 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatResultsMergeRounds 46 | Jul-23 13:34:58.511 [PathVisitor-3] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **/*.mzXML; options: [:] 47 | Jul-23 13:34:58.512 [PathVisitor-4] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **/*.mzxml; options: [:] 48 | Jul-23 13:34:58.511 [PathVisitor-6] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **/*.mgf; options: [:] 49 | Jul-23 13:34:58.511 [PathVisitor-1] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **/*.mzML; options: [:] 50 | Jul-23 13:34:58.511 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatExtractedSpectraRounds 51 | Jul-23 13:34:58.511 [PathVisitor-2] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **/*.mzml; options: [:] 52 | Jul-23 13:34:58.529 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > summarizeResults 53 | Jul-23 13:34:58.530 [main] DEBUG nextflow.script.ScriptRunner - > Await termination 54 | Jul-23 13:34:58.511 [PathVisitor-5] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **/*.MGF; options: [:] 55 | Jul-23 13:34:58.534 [main] DEBUG nextflow.Session - Session await 56 | Jul-23 13:34:58.667 [Actor Thread 14] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 57 | Jul-23 13:34:58.670 [Actor Thread 17] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 58 | Jul-23 13:34:58.670 [main] DEBUG nextflow.Session - Session await > all process finished 59 | Jul-23 13:34:58.735 [main] DEBUG nextflow.Session - Session await > all barriers passed 60 | Jul-23 13:34:58.761 [main] DEBUG nextflow.trace.WorkflowStatsObserver - Workflow completed > WorkflowStats[succeededCount=0; failedCount=0; ignoredCount=0; cachedCount=0; pendingCount=0; submittedCount=0; runningCount=0; retriesCount=0; abortedCount=0; succeedDuration=0ms; failedDuration=0ms; cachedDuration=0ms;loadCpus=0; loadMemory=0; peakRunning=0; peakCpus=0; peakMemory=0; ] 61 | Jul-23 13:34:58.882 [main] DEBUG nextflow.CacheDB - Closing CacheDB done 62 | Jul-23 13:34:58.930 [main] DEBUG nextflow.script.ScriptRunner - > Execution complete -- Goodbye 63 | -------------------------------------------------------------------------------- /workflow/.nextflow.log.9: -------------------------------------------------------------------------------- 1 | Jul-23 10:04:24.787 [main] DEBUG nextflow.cli.Launcher - $> nextflow run workflow.nf -c test.config --parallel_files YES --resume 2 | Jul-23 10:04:25.196 [main] INFO nextflow.cli.CmdRun - N E X T F L O W ~ version 21.10.0 3 | Jul-23 10:04:25.308 [main] INFO nextflow.cli.CmdRun - Launching `workflow.nf` [mighty_solvay] - revision: bdcb50bad8 4 | Jul-23 10:04:25.373 [main] DEBUG nextflow.config.ConfigBuilder - User config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 5 | Jul-23 10:04:25.374 [main] DEBUG nextflow.config.ConfigBuilder - Parsing config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 6 | Jul-23 10:04:25.497 [main] DEBUG nextflow.config.ConfigBuilder - Applying config profile: `standard` 7 | Jul-23 10:04:28.280 [main] DEBUG nextflow.plugin.PluginsFacade - Setting up plugin manager > mode=prod; plugins-dir=/home/user/.nextflow/plugins 8 | Jul-23 10:04:28.284 [main] DEBUG nextflow.plugin.PluginsFacade - Plugins default=[] 9 | Jul-23 10:04:28.320 [main] INFO org.pf4j.DefaultPluginStatusProvider - Enabled plugins: [] 10 | Jul-23 10:04:28.324 [main] INFO org.pf4j.DefaultPluginStatusProvider - Disabled plugins: [] 11 | Jul-23 10:04:28.334 [main] INFO org.pf4j.DefaultPluginManager - PF4J version 3.4.1 in 'deployment' mode 12 | Jul-23 10:04:28.364 [main] INFO org.pf4j.AbstractPluginManager - No plugins 13 | Jul-23 10:04:28.566 [main] DEBUG nextflow.Session - Session uuid: 636d7f3e-d7c4-47ab-bf3f-b264282d416c 14 | Jul-23 10:04:28.567 [main] DEBUG nextflow.Session - Run name: mighty_solvay 15 | Jul-23 10:04:28.568 [main] DEBUG nextflow.Session - Executor pool size: 16 16 | Jul-23 10:04:33.792 [main] DEBUG nextflow.cli.CmdRun - 17 | Version: 21.10.0 build 5640 18 | Created: 11-11-2021 18:27 UTC (10:27 PDT) 19 | System: Linux 5.15.0-53-generic 20 | Runtime: Groovy 3.0.9 on OpenJDK 64-Bit Server VM 1.8.0_152-release-1056-b12 21 | Encoding: UTF-8 (UTF-8) 22 | Process: 4123733@mingwangucr [169.235.26.140] 23 | CPUs: 16 - Mem: 61.8 GB (1.4 GB) - Swap: 735.3 MB (0) 24 | Jul-23 10:04:33.865 [main] DEBUG nextflow.Session - Work-dir: /home/user/SourceCode/MassQueryLanguage/workflow/work [ext2/ext3] 25 | Jul-23 10:04:33.917 [main] DEBUG nextflow.executor.ExecutorFactory - Extension executors providers=[] 26 | Jul-23 10:04:33.965 [main] DEBUG nextflow.Session - Observer factory: DefaultObserverFactory 27 | Jul-23 10:04:34.114 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 28 | Jul-23 10:04:34.361 [main] DEBUG nextflow.Session - Session start invoked 29 | Jul-23 10:04:36.606 [main] DEBUG nextflow.script.ScriptRunner - > Launching execution 30 | Jul-23 10:04:36.880 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 31 | Jul-23 10:04:37.063 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 32 | Jul-23 10:04:37.063 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 33 | Jul-23 10:04:37.084 [main] DEBUG nextflow.executor.Executor - [warm up] executor > local 34 | Jul-23 10:04:37.098 [main] DEBUG n.processor.LocalPollingMonitor - Creating local task monitor for executor 'local' > cpus=16; memory=61.8 GB; capacity=16; pollInterval=100ms; dumpInterval=5m 35 | Jul-23 10:04:37.338 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 36 | Jul-23 10:04:37.338 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 37 | Jul-23 10:04:37.449 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 38 | Jul-23 10:04:37.449 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 39 | Jul-23 10:04:37.500 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 40 | Jul-23 10:04:37.500 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 41 | Jul-23 10:04:37.513 [main] DEBUG nextflow.Session - Workflow process names [dsl2]: summarizeResults, formatResultsMergeRounds, queryData2, queryData, formatExtractedSpectraRounds 42 | Jul-23 10:04:37.514 [main] DEBUG nextflow.Session - Ignite dataflow network (5) 43 | Jul-23 10:04:37.558 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > queryData 44 | Jul-23 10:04:37.563 [PathVisitor-1] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **; options: [:] 45 | Jul-23 10:04:37.565 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatResultsMergeRounds 46 | Jul-23 10:04:37.587 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatExtractedSpectraRounds 47 | Jul-23 10:04:37.588 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > summarizeResults 48 | Jul-23 10:04:37.593 [main] DEBUG nextflow.script.ScriptRunner - > Await termination 49 | Jul-23 10:04:37.594 [main] DEBUG nextflow.Session - Session await 50 | Jul-23 10:04:38.781 [Actor Thread 23] INFO nextflow.conda.CondaCache - Creating env using conda: /home/user/SourceCode/MassQueryLanguage/workflow/bin/conda_env.yml [cache /home/user/SourceCode/MassQueryLanguage/workflow/work/conda/conda_env-1a73718c9f06f7c6630eeee45ccbfd68] 51 | Jul-23 10:05:06.717 [SIGINT handler] DEBUG nextflow.Session - Session aborted -- Cause: SIGINT 52 | Jul-23 10:05:06.762 [SIGINT handler] DEBUG nextflow.Session - The following nodes are still active: 53 | [process] queryData 54 | status=ACTIVE 55 | port 0: (queue) OPEN ; channel: - 56 | port 1: (cntrl) - ; channel: $ 57 | 58 | [process] formatResultsMergeRounds 59 | status=ACTIVE 60 | port 0: (queue) OPEN ; channel: results/* 61 | port 1: (cntrl) - ; channel: $ 62 | 63 | [process] formatExtractedSpectraRounds 64 | status=ACTIVE 65 | port 0: (queue) OPEN ; channel: json/* 66 | port 1: (cntrl) - ; channel: $ 67 | 68 | [process] summarizeResults 69 | status=ACTIVE 70 | port 0: (queue) OPEN ; channel: merged_results 71 | port 1: (cntrl) - ; channel: $ 72 | 73 | Jul-23 10:05:06.775 [main] DEBUG nextflow.Session - Session await > all process finished 74 | Jul-23 10:05:06.776 [main] DEBUG nextflow.Session - Session await > all barriers passed 75 | Jul-23 10:05:06.799 [Actor Thread 26] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 76 | Jul-23 10:05:06.799 [Actor Thread 25] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 77 | Jul-23 10:05:06.812 [main] DEBUG nextflow.trace.WorkflowStatsObserver - Workflow completed > WorkflowStats[succeededCount=0; failedCount=0; ignoredCount=0; cachedCount=0; pendingCount=0; submittedCount=0; runningCount=0; retriesCount=0; abortedCount=0; succeedDuration=0ms; failedDuration=0ms; cachedDuration=0ms;loadCpus=0; loadMemory=0; peakRunning=0; peakCpus=0; peakMemory=0; ] 78 | Jul-23 10:05:06.928 [main] DEBUG nextflow.CacheDB - Closing CacheDB done 79 | Jul-23 10:05:06.969 [main] DEBUG nextflow.script.ScriptRunner - > Execution complete -- Goodbye 80 | -------------------------------------------------------------------------------- /workflow/.nextflow.log.8: -------------------------------------------------------------------------------- 1 | Jul-23 11:33:24.401 [main] DEBUG nextflow.cli.Launcher - $> nextflow run workflow.nf -c test.config --parallel_files YES --resume 2 | Jul-23 11:33:24.843 [main] INFO nextflow.cli.CmdRun - N E X T F L O W ~ version 21.10.0 3 | Jul-23 11:33:24.958 [main] INFO nextflow.cli.CmdRun - Launching `workflow.nf` [exotic_pauling] - revision: b399ae2781 4 | Jul-23 11:33:25.020 [main] DEBUG nextflow.config.ConfigBuilder - User config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 5 | Jul-23 11:33:25.021 [main] DEBUG nextflow.config.ConfigBuilder - Parsing config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 6 | Jul-23 11:33:25.100 [main] DEBUG nextflow.config.ConfigBuilder - Applying config profile: `standard` 7 | Jul-23 11:33:27.774 [main] DEBUG nextflow.plugin.PluginsFacade - Setting up plugin manager > mode=prod; plugins-dir=/home/user/.nextflow/plugins 8 | Jul-23 11:33:27.785 [main] DEBUG nextflow.plugin.PluginsFacade - Plugins default=[] 9 | Jul-23 11:33:27.830 [main] INFO org.pf4j.DefaultPluginStatusProvider - Enabled plugins: [] 10 | Jul-23 11:33:27.833 [main] INFO org.pf4j.DefaultPluginStatusProvider - Disabled plugins: [] 11 | Jul-23 11:33:27.841 [main] INFO org.pf4j.DefaultPluginManager - PF4J version 3.4.1 in 'deployment' mode 12 | Jul-23 11:33:27.860 [main] INFO org.pf4j.AbstractPluginManager - No plugins 13 | Jul-23 11:33:28.065 [main] DEBUG nextflow.Session - Session uuid: d24bf493-6303-4706-973e-b60c310a82ec 14 | Jul-23 11:33:28.066 [main] DEBUG nextflow.Session - Run name: exotic_pauling 15 | Jul-23 11:33:28.067 [main] DEBUG nextflow.Session - Executor pool size: 16 16 | Jul-23 11:33:28.173 [main] DEBUG nextflow.cli.CmdRun - 17 | Version: 21.10.0 build 5640 18 | Created: 11-11-2021 18:27 UTC (10:27 PDT) 19 | System: Linux 5.15.0-53-generic 20 | Runtime: Groovy 3.0.9 on OpenJDK 64-Bit Server VM 1.8.0_152-release-1056-b12 21 | Encoding: UTF-8 (UTF-8) 22 | Process: 4143788@mingwangucr [169.235.26.140] 23 | CPUs: 16 - Mem: 61.8 GB (1.4 GB) - Swap: 735.3 MB (0) 24 | Jul-23 11:33:28.240 [main] DEBUG nextflow.Session - Work-dir: /home/user/SourceCode/MassQueryLanguage/workflow/work [ext2/ext3] 25 | Jul-23 11:33:28.302 [main] DEBUG nextflow.executor.ExecutorFactory - Extension executors providers=[] 26 | Jul-23 11:33:28.331 [main] DEBUG nextflow.Session - Observer factory: DefaultObserverFactory 27 | Jul-23 11:33:28.457 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 28 | Jul-23 11:33:28.719 [main] DEBUG nextflow.Session - Session start invoked 29 | Jul-23 11:33:30.793 [main] DEBUG nextflow.script.ScriptRunner - > Launching execution 30 | Jul-23 11:33:31.105 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 31 | Jul-23 11:33:31.367 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 32 | Jul-23 11:33:31.367 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 33 | Jul-23 11:33:31.376 [main] DEBUG nextflow.executor.Executor - [warm up] executor > local 34 | Jul-23 11:33:31.388 [main] DEBUG n.processor.LocalPollingMonitor - Creating local task monitor for executor 'local' > cpus=16; memory=61.8 GB; capacity=16; pollInterval=100ms; dumpInterval=5m 35 | Jul-23 11:33:31.673 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 36 | Jul-23 11:33:31.674 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 37 | Jul-23 11:33:31.790 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 38 | Jul-23 11:33:31.791 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 39 | Jul-23 11:33:31.818 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 40 | Jul-23 11:33:31.818 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 41 | Jul-23 11:33:31.838 [main] DEBUG nextflow.Session - Workflow process names [dsl2]: summarizeResults, formatResultsMergeRounds, queryData2, queryData, formatExtractedSpectraRounds 42 | Jul-23 11:33:31.840 [main] DEBUG nextflow.Session - Ignite dataflow network (5) 43 | Jul-23 11:33:31.872 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > queryData 44 | Jul-23 11:33:31.874 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatResultsMergeRounds 45 | Jul-23 11:33:31.876 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatExtractedSpectraRounds 46 | Jul-23 11:33:31.877 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > summarizeResults 47 | Jul-23 11:33:31.885 [main] DEBUG nextflow.script.ScriptRunner - > Await termination 48 | Jul-23 11:33:31.903 [main] DEBUG nextflow.Session - Session await 49 | Jul-23 11:33:31.922 [PathVisitor-1] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **; options: [:] 50 | Jul-23 11:33:32.982 [Actor Thread 25] INFO nextflow.conda.CondaCache - Creating env using conda: /home/user/SourceCode/MassQueryLanguage/workflow/bin/conda_env.yml [cache /home/user/SourceCode/MassQueryLanguage/workflow/work/conda/conda_env-1a73718c9f06f7c6630eeee45ccbfd68] 51 | Jul-23 11:33:40.830 [SIGINT handler] DEBUG nextflow.Session - Session aborted -- Cause: SIGINT 52 | Jul-23 11:33:40.882 [SIGINT handler] DEBUG nextflow.Session - The following nodes are still active: 53 | [process] queryData 54 | status=ACTIVE 55 | port 0: (queue) OPEN ; channel: - 56 | port 1: (cntrl) - ; channel: $ 57 | 58 | [process] formatResultsMergeRounds 59 | status=ACTIVE 60 | port 0: (queue) OPEN ; channel: results/* 61 | port 1: (cntrl) - ; channel: $ 62 | 63 | [process] formatExtractedSpectraRounds 64 | status=ACTIVE 65 | port 0: (queue) OPEN ; channel: json/* 66 | port 1: (cntrl) - ; channel: $ 67 | 68 | [process] summarizeResults 69 | status=ACTIVE 70 | port 0: (queue) OPEN ; channel: merged_results 71 | port 1: (cntrl) - ; channel: $ 72 | 73 | Jul-23 11:33:40.890 [main] DEBUG nextflow.Session - Session await > all process finished 74 | Jul-23 11:33:40.893 [main] DEBUG nextflow.Session - Session await > all barriers passed 75 | Jul-23 11:33:40.921 [Actor Thread 28] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 76 | Jul-23 11:33:40.926 [main] DEBUG nextflow.trace.WorkflowStatsObserver - Workflow completed > WorkflowStats[succeededCount=0; failedCount=0; ignoredCount=0; cachedCount=0; pendingCount=0; submittedCount=0; runningCount=0; retriesCount=0; abortedCount=0; succeedDuration=0ms; failedDuration=0ms; cachedDuration=0ms;loadCpus=0; loadMemory=0; peakRunning=0; peakCpus=0; peakMemory=0; ] 77 | Jul-23 11:33:40.928 [Actor Thread 27] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 78 | Jul-23 11:33:41.009 [main] DEBUG nextflow.CacheDB - Closing CacheDB done 79 | Jul-23 11:33:41.050 [main] DEBUG nextflow.script.ScriptRunner - > Execution complete -- Goodbye 80 | -------------------------------------------------------------------------------- /workflow/.nextflow.log.7: -------------------------------------------------------------------------------- 1 | Jul-23 12:00:17.540 [main] DEBUG nextflow.cli.Launcher - $> nextflow run workflow.nf -c test.config --parallel_files YES --resume 2 | Jul-23 12:00:17.879 [main] INFO nextflow.cli.CmdRun - N E X T F L O W ~ version 21.10.0 3 | Jul-23 12:00:17.965 [main] INFO nextflow.cli.CmdRun - Launching `workflow.nf` [big_pauling] - revision: b399ae2781 4 | Jul-23 12:00:18.019 [main] DEBUG nextflow.config.ConfigBuilder - User config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 5 | Jul-23 12:00:18.020 [main] DEBUG nextflow.config.ConfigBuilder - Parsing config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 6 | Jul-23 12:00:18.111 [main] DEBUG nextflow.config.ConfigBuilder - Applying config profile: `standard` 7 | Jul-23 12:00:20.842 [main] DEBUG nextflow.plugin.PluginsFacade - Setting up plugin manager > mode=prod; plugins-dir=/home/user/.nextflow/plugins 8 | Jul-23 12:00:20.846 [main] DEBUG nextflow.plugin.PluginsFacade - Plugins default=[] 9 | Jul-23 12:00:20.871 [main] INFO org.pf4j.DefaultPluginStatusProvider - Enabled plugins: [] 10 | Jul-23 12:00:20.874 [main] INFO org.pf4j.DefaultPluginStatusProvider - Disabled plugins: [] 11 | Jul-23 12:00:20.884 [main] INFO org.pf4j.DefaultPluginManager - PF4J version 3.4.1 in 'deployment' mode 12 | Jul-23 12:00:20.914 [main] INFO org.pf4j.AbstractPluginManager - No plugins 13 | Jul-23 12:00:21.138 [main] DEBUG nextflow.Session - Session uuid: 82349cae-2a67-4e19-9235-3d1ac0695aa8 14 | Jul-23 12:00:21.139 [main] DEBUG nextflow.Session - Run name: big_pauling 15 | Jul-23 12:00:21.142 [main] DEBUG nextflow.Session - Executor pool size: 16 16 | Jul-23 12:00:21.254 [main] DEBUG nextflow.cli.CmdRun - 17 | Version: 21.10.0 build 5640 18 | Created: 11-11-2021 18:27 UTC (10:27 PDT) 19 | System: Linux 5.15.0-53-generic 20 | Runtime: Groovy 3.0.9 on OpenJDK 64-Bit Server VM 1.8.0_152-release-1056-b12 21 | Encoding: UTF-8 (UTF-8) 22 | Process: 4150305@mingwangucr [169.235.26.140] 23 | CPUs: 16 - Mem: 61.8 GB (1.4 GB) - Swap: 735.3 MB (0) 24 | Jul-23 12:00:21.346 [main] DEBUG nextflow.Session - Work-dir: /home/user/SourceCode/MassQueryLanguage/workflow/work [ext2/ext3] 25 | Jul-23 12:00:21.396 [main] DEBUG nextflow.executor.ExecutorFactory - Extension executors providers=[] 26 | Jul-23 12:00:21.428 [main] DEBUG nextflow.Session - Observer factory: DefaultObserverFactory 27 | Jul-23 12:00:21.556 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 28 | Jul-23 12:00:21.762 [main] DEBUG nextflow.Session - Session start invoked 29 | Jul-23 12:00:23.674 [main] DEBUG nextflow.script.ScriptRunner - > Launching execution 30 | Jul-23 12:00:24.035 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 31 | Jul-23 12:00:24.292 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 32 | Jul-23 12:00:24.292 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 33 | Jul-23 12:00:24.309 [main] DEBUG nextflow.executor.Executor - [warm up] executor > local 34 | Jul-23 12:00:24.326 [main] DEBUG n.processor.LocalPollingMonitor - Creating local task monitor for executor 'local' > cpus=16; memory=61.8 GB; capacity=16; pollInterval=100ms; dumpInterval=5m 35 | Jul-23 12:00:24.671 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 36 | Jul-23 12:00:24.672 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 37 | Jul-23 12:00:24.771 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 38 | Jul-23 12:00:24.771 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 39 | Jul-23 12:00:24.814 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 40 | Jul-23 12:00:24.814 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 41 | Jul-23 12:00:24.834 [main] DEBUG nextflow.Session - Workflow process names [dsl2]: summarizeResults, formatResultsMergeRounds, queryData2, queryData, formatExtractedSpectraRounds 42 | Jul-23 12:00:24.836 [main] DEBUG nextflow.Session - Ignite dataflow network (5) 43 | Jul-23 12:00:24.880 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > queryData 44 | Jul-23 12:00:24.893 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatResultsMergeRounds 45 | Jul-23 12:00:24.894 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatExtractedSpectraRounds 46 | Jul-23 12:00:24.895 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > summarizeResults 47 | Jul-23 12:00:24.895 [main] DEBUG nextflow.script.ScriptRunner - > Await termination 48 | Jul-23 12:00:24.896 [main] DEBUG nextflow.Session - Session await 49 | Jul-23 12:00:24.901 [PathVisitor-1] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **; options: [:] 50 | Jul-23 12:00:26.107 [Actor Thread 24] INFO nextflow.conda.CondaCache - Creating env using conda: /home/user/SourceCode/MassQueryLanguage/workflow/bin/conda_env.yml [cache /home/user/SourceCode/MassQueryLanguage/workflow/work/conda/conda_env-a7c7dacd5204cfe814b4763f1b0accf4] 51 | Jul-23 12:05:24.547 [Task monitor] DEBUG n.processor.TaskPollingMonitor - No more task to compute -- The following nodes are still active: 52 | [process] queryData 53 | status=ACTIVE 54 | port 0: (queue) OPEN ; channel: - 55 | port 1: (cntrl) - ; channel: $ 56 | 57 | [process] formatResultsMergeRounds 58 | status=ACTIVE 59 | port 0: (queue) OPEN ; channel: results/* 60 | port 1: (cntrl) - ; channel: $ 61 | 62 | [process] formatExtractedSpectraRounds 63 | status=ACTIVE 64 | port 0: (queue) OPEN ; channel: json/* 65 | port 1: (cntrl) - ; channel: $ 66 | 67 | [process] summarizeResults 68 | status=ACTIVE 69 | port 0: (queue) OPEN ; channel: merged_results 70 | port 1: (cntrl) - ; channel: $ 71 | 72 | Jul-23 12:08:19.708 [SIGINT handler] DEBUG nextflow.Session - Session aborted -- Cause: SIGINT 73 | Jul-23 12:08:19.721 [SIGINT handler] DEBUG nextflow.Session - The following nodes are still active: 74 | [process] queryData 75 | status=ACTIVE 76 | port 0: (queue) OPEN ; channel: - 77 | port 1: (cntrl) - ; channel: $ 78 | 79 | [process] formatResultsMergeRounds 80 | status=ACTIVE 81 | port 0: (queue) OPEN ; channel: results/* 82 | port 1: (cntrl) - ; channel: $ 83 | 84 | [process] formatExtractedSpectraRounds 85 | status=ACTIVE 86 | port 0: (queue) OPEN ; channel: json/* 87 | port 1: (cntrl) - ; channel: $ 88 | 89 | [process] summarizeResults 90 | status=ACTIVE 91 | port 0: (queue) OPEN ; channel: merged_results 92 | port 1: (cntrl) - ; channel: $ 93 | 94 | Jul-23 12:08:19.732 [main] DEBUG nextflow.Session - Session await > all process finished 95 | Jul-23 12:08:19.733 [main] DEBUG nextflow.Session - Session await > all barriers passed 96 | Jul-23 12:08:19.756 [Actor Thread 27] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 97 | Jul-23 12:08:19.762 [Actor Thread 25] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 98 | Jul-23 12:08:19.791 [main] DEBUG nextflow.trace.WorkflowStatsObserver - Workflow completed > WorkflowStats[succeededCount=0; failedCount=0; ignoredCount=0; cachedCount=0; pendingCount=0; submittedCount=0; runningCount=0; retriesCount=0; abortedCount=0; succeedDuration=0ms; failedDuration=0ms; cachedDuration=0ms;loadCpus=0; loadMemory=0; peakRunning=0; peakCpus=0; peakMemory=0; ] 99 | Jul-23 12:08:19.807 [main] DEBUG nextflow.CacheDB - Closing CacheDB done 100 | Jul-23 12:08:19.829 [main] DEBUG nextflow.script.ScriptRunner - > Execution complete -- Goodbye 101 | -------------------------------------------------------------------------------- /massql/msql_cmd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import os 5 | import sys 6 | import json 7 | import uuid 8 | import pandas as pd 9 | 10 | # Making sure the root is in the path, kind of a hack 11 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) 12 | sys.path.insert(0, os.path.join(os.path.dirname(__file__))) 13 | 14 | from massql import msql_parser 15 | from massql import msql_engine 16 | from massql import msql_extract 17 | 18 | def main(): 19 | parser = argparse.ArgumentParser(description="MSQL CMD") 20 | parser.add_argument('filename', help='Input filename') 21 | parser.add_argument('query', help='Input Query') 22 | parser.add_argument('--output_file', default=None, help='output results filename, if filename is too long will truncate filename') 23 | parser.add_argument('--parallel_query', default="NO", help='YES to make it parallel with ray locally, NO is default') 24 | 25 | parser.add_argument('--cache', default=None, help='feather cache with feather, no caching is the default') 26 | parser.add_argument('--cache_file', default=None, help='Path to explicit cache filename, must have cache set to YES') 27 | parser.add_argument('--cache_dir', default=None, help='Path to cache directory, must have cache set to YES. Mutually exclusive to cache_file. Additionally, caching filename will automatically be hashed') 28 | 29 | parser.add_argument('--original_path', default=None, help='Original absolute path for the filename, useful in proteosafe') 30 | parser.add_argument('--extract_mzML', default=None, help='Extracting spectra found as mzML file, if filename is too long will truncate filename') 31 | parser.add_argument('--extract_json', default=None, help='Extracting spectra found as json file, each spectrum is a line, if filename is too long, wiil truncate',) 32 | parser.add_argument('--maxfilesize', default=None, help='Maximum file size in MB') 33 | 34 | args = parser.parse_args() 35 | 36 | print(args) 37 | 38 | PARALLEL = args.parallel_query == "YES" 39 | 40 | if PARALLEL: 41 | msql_engine.init_ray() 42 | 43 | # Massaging the query on input, we have a system to enable multiple queries to be entered, where results are merged 44 | # The delimeter is specified as ||| 45 | all_queries = args.query.split("|||") 46 | 47 | # Let's parse first 48 | for query in all_queries: 49 | parsed_query = msql_parser.parse_msql(query) 50 | print(json.dumps(parsed_query, indent=4)) 51 | 52 | # Checking the input file for size 53 | if args.maxfilesize is not None: 54 | if os.path.isfile(args.filename): 55 | if os.path.getsize(args.filename) / 1024 / 1024 > int(args.maxfilesize): 56 | print("File is too big, exiting") 57 | exit(0) 58 | 59 | # Executing 60 | all_results_list = [] 61 | for i, query in enumerate(all_queries): 62 | results_df = msql_engine.process_query(query, 63 | args.filename, 64 | cache=args.cache, 65 | cache_file=args.cache_file, 66 | cache_dir=args.cache_dir) 67 | 68 | results_df["query_index"] = i 69 | all_results_list.append(results_df) 70 | 71 | # Merging 72 | results_df = pd.concat(all_results_list) 73 | 74 | print("#############################") 75 | print("MassQL Found {} results".format(len(results_df))) 76 | print("#############################") 77 | 78 | # Setting mzupper and mzlower 79 | try: 80 | if "comment" in results_df: 81 | results_df["mz_lower"] = results_df["comment"].astype(float) - 10 82 | results_df["mz_upper"] = results_df["comment"].astype(float) + 10 83 | except: 84 | pass 85 | 86 | # Handling output filename 87 | output_results_filename = args.output_file 88 | output_file_toolong_unique_prefix = str(uuid.uuid4()).replace("-", "") + "_" 89 | 90 | if output_results_filename is not None: 91 | if len(os.path.basename(output_results_filename)) > 80: 92 | output_results_filename = os.path.join(os.path.split(output_results_filename)[0], 93 | output_file_toolong_unique_prefix + os.path.basename(output_results_filename)[-80:]) 94 | 95 | if output_results_filename and len(results_df) > 0: 96 | results_df["filename"] = os.path.basename(args.filename) 97 | 98 | # This is for GNPS and GNPS2 99 | if args.original_path is not None: 100 | useful_filename = args.original_path 101 | # TODO: Clean up for ProteoSAFe 102 | useful_filename = useful_filename.split("demangled_spectra/")[-1] 103 | 104 | # Cleanup for repository search 105 | useful_filename = useful_filename.replace("/data/ccms-data/uploads/", "") 106 | 107 | # This works for paths in nf_tasks 108 | if "/data/nf_data/server/nf_tasks/" in useful_filename: 109 | useful_filename = useful_filename.replace("/data/nf_data/server/nf_tasks/", "") 110 | # Get the second folder name from path with python PathLib 111 | import pathlib 112 | task = pathlib.Path(useful_filename).parts[0] 113 | useful_filename = useful_filename.replace(task + "/", "") 114 | 115 | # This works for nf_work becuase of getting data from the nf_work directory 116 | if "/data/nf_data/server/nf_work/" in useful_filename: 117 | useful_filename = useful_filename.replace("/data/nf_data/server/nf_work/", "") 118 | # Get the second folder name from path with python PathLib 119 | import pathlib 120 | task = pathlib.Path(useful_filename).parts[0] 121 | useful_filename = useful_filename.replace(task + "/", "") 122 | 123 | hash1 = pathlib.Path(useful_filename).parts[0] 124 | useful_filename = useful_filename.replace(hash1 + "/", "") 125 | 126 | hash2 = pathlib.Path(useful_filename).parts[0] 127 | useful_filename = useful_filename.replace(hash2 + "/", "") 128 | 129 | # Saving output 130 | results_df["original_path"] = useful_filename 131 | 132 | # Forcing Types 133 | try: 134 | results_df["scan"] = results_df["scan"].astype(int) 135 | except: 136 | pass 137 | 138 | try: 139 | results_df["ms1scan"] = results_df["ms1scan"].astype(int) 140 | except: 141 | pass 142 | 143 | try: 144 | results_df["charge"] = results_df["charge"].astype(int) 145 | except: 146 | pass 147 | 148 | try: 149 | results_df["mslevel"] = results_df["mslevel"].astype(int) 150 | except: 151 | pass 152 | 153 | # Forcing the column order 154 | columns = list(results_df.columns) 155 | columns.sort() 156 | 157 | if ".tsv" in output_results_filename: 158 | results_df.to_csv(output_results_filename, index=False, sep="\t", columns=columns) 159 | else: 160 | results_df.to_csv(output_results_filename, index=False, columns=columns) 161 | 162 | # Extracting 163 | if args.extract_json is not None and len(results_df) > 0: 164 | print("Extracting {} spectra".format(len(results_df))) 165 | try: 166 | output_json_filename = args.extract_json 167 | if output_json_filename is not None: 168 | if len(os.path.basename(output_json_filename)) > 80: 169 | output_json_filename = os.path.join(os.path.split(output_json_filename)[0], 170 | output_file_toolong_unique_prefix + os.path.basename(output_json_filename)[-80:]) 171 | 172 | msql_extract._extract_spectra(results_df, os.path.dirname(args.filename), output_json_filename=output_json_filename) 173 | except: 174 | print("Extraction Failed") 175 | 176 | 177 | if __name__ == "__main__": 178 | main() 179 | -------------------------------------------------------------------------------- /tests/test_parse.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | # Making sure the root is in the path, kind of a hack 5 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) 6 | 7 | from massql import msql_parser 8 | import json 9 | import pytest 10 | 11 | def test_parse(): 12 | # This test not only performs a parse, but also tries to compare it against a reference parse 13 | 14 | from pathvalidate import sanitize_filename 15 | import os 16 | import hashlib 17 | 18 | # Writing out the queries and comparing 19 | current_dir = os.path.dirname(__file__) 20 | test_parses_dir = os.path.join(current_dir, "test_parses") 21 | reference_parses_dir = os.path.join(current_dir, "reference_parses") 22 | 23 | os.makedirs(test_parses_dir, exist_ok=True) 24 | 25 | test_queries_filename = os.path.join(current_dir, "test_queries.txt") 26 | for line in open(test_queries_filename): 27 | test_query = line.rstrip() 28 | print(test_query) 29 | output_parse = msql_parser.parse_msql(test_query) 30 | 31 | hash_object = hashlib.md5(test_query.encode("ascii")) 32 | hash_output = hash_object.hexdigest() 33 | 34 | json_filename = sanitize_filename(test_query).replace(" ", "_").replace("=", "_").replace("(", "_").replace(")", "_")[:50] + "___" + hash_output + ".json" 35 | 36 | output_filename = os.path.join(test_parses_dir, json_filename) 37 | output_json_str = json.dumps(output_parse, sort_keys=True, indent=4) 38 | 39 | with open(output_filename, "w") as o: 40 | o.write(output_json_str) 41 | 42 | reference_filename = os.path.join(reference_parses_dir, json_filename) 43 | reference_string = open(reference_filename).read() 44 | 45 | try: 46 | assert(output_json_str == reference_string) 47 | except: 48 | print("Assertion error", reference_filename) 49 | raise 50 | 51 | def test_comment_parse(): 52 | query = """ 53 | # COMMENT 54 | QUERY scaninfo(MS1DATA) # COMMENT2 55 | """ 56 | 57 | parsed_output = msql_parser.parse_msql(query) 58 | 59 | print(parsed_output) 60 | 61 | def test_number_expression_parse(): 62 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=157.0857+10" 63 | parsed_output = msql_parser.parse_msql(query) 64 | print(json.dumps(parsed_output, indent=4)) 65 | 66 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=16.70857*10 +(0)" 67 | parsed_output = msql_parser.parse_msql(query) 68 | print(json.dumps(parsed_output, indent=4)) 69 | 70 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=1670.857/10 +(0)" 71 | parsed_output = msql_parser.parse_msql(query) 72 | print(json.dumps(parsed_output, indent=4)) 73 | 74 | def test_formula_expression_parse(): 75 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=formula(CH2)+14" 76 | parsed_output = msql_parser.parse_msql(query) 77 | print(json.dumps(parsed_output, indent=4)) 78 | assert(parsed_output["conditions"][0]["value"][0] > 28) 79 | 80 | def test_formula2_expression_parse(): 81 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=formula(Fe)" 82 | parsed_output = msql_parser.parse_msql(query) 83 | print(json.dumps(parsed_output, indent=4)) 84 | assert(parsed_output["conditions"][0]["value"][0] > 28) 85 | 86 | def test_aminoacids_expression_parse(): 87 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=aminoaciddelta(G)" 88 | parsed_output = msql_parser.parse_msql(query) 89 | print(parsed_output) 90 | print(json.dumps(parsed_output, indent=4)) 91 | assert(parsed_output["conditions"][0]["value"][0] > 57) 92 | assert(parsed_output["conditions"][0]["value"][0] < 58) 93 | 94 | def test_peptide_expression_parse(): 95 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=peptide(G, charge=1, ion=y)" 96 | parsed_output = msql_parser.parse_msql(query) 97 | print(parsed_output) 98 | print(json.dumps(parsed_output, indent=4)) 99 | 100 | assert(parsed_output["conditions"][0]["value"][0] > 76) 101 | assert(parsed_output["conditions"][0]["value"][0] < 77) 102 | 103 | def test_variable_formula_parse(): 104 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X AND MS2PROD=X-formula(Fe)*2" 105 | 106 | parsed_output = msql_parser.parse_msql(query) 107 | print(json.dumps(parsed_output, indent=4)) 108 | 109 | def test_variable_formula_parse2(): 110 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X AND MS2PROD=2.0*(X - formula(Fe))" 111 | 112 | parsed_output = msql_parser.parse_msql(query) 113 | print(json.dumps(parsed_output, indent=4)) 114 | 115 | def test_xrange_parse(): 116 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X AND MS2PROD=2.0*(X - formula(Fe)) AND X=range(min=5, max=100)" 117 | 118 | parsed_output = msql_parser.parse_msql(query) 119 | print(json.dumps(parsed_output, indent=4)) 120 | 121 | query = "QUERY scaninfo(MS2DATA) WHERE MS1MZ=X:INTENSITYMATCH=Y:INTENSITYMATCHREFERENCE:INTENSITYVALUE=10000 \ 122 | AND \ 123 | MS1MZ=X+1:INTENSITYMATCH=Y*0.4:INTENSITYMATCHPERCENT=50:TOLERANCEPPM=10 AND MS1MZ=X+1.998:INTENSITYMATCH=Y*0.446:INTENSITYMATCHPERCENT=50:TOLERANCEPPM=10 AND X=range(min=300,max=900)" 124 | 125 | parsed_output = msql_parser.parse_msql(query) 126 | print(json.dumps(parsed_output, indent=4)) 127 | 128 | def test_xdefect_parse(): 129 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=X AND MS2PROD=2.0*(X - formula(Fe)) AND \ 130 | X=massdefect(min=0.1, max=0.2) AND X=range(min=5, max=100)" 131 | 132 | parsed_output = msql_parser.parse_msql(query) 133 | print(json.dumps(parsed_output, indent=4)) 134 | 135 | def test_ms2_synonyms(): 136 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=100" 137 | query2 = "QUERY scaninfo(MS2DATA) WHERE MS2MZ=100" 138 | 139 | parsed_output = msql_parser.parse_msql(query) 140 | parsed_output2 = msql_parser.parse_msql(query2) 141 | 142 | parsed_output["query"] = "" 143 | parsed_output2["query"] = "" 144 | 145 | assert(json.dumps(parsed_output) == json.dumps(parsed_output2)) 146 | 147 | def test_variable_parse(): 148 | # This finds the sum of the MS1 of the MS2 spectrum with 149 | query = "QUERY scaninfo(MS2DATA) WHERE MS1MZ=X AND MS2PREC=X AND MS2PROD=119.09" 150 | parse_obj = msql_parser.parse_msql(query) 151 | print(json.dumps(parse_obj, indent=4)) 152 | 153 | def test_ms2_or(): 154 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=(100 OR 104)" 155 | 156 | parsed_output = msql_parser.parse_msql(query) 157 | print(parsed_output) 158 | 159 | def test_ms1_or(): 160 | query = "QUERY scaninfo(MS1DATA) WHERE MS1MZ=(100 OR 104)" 161 | 162 | parsed_output = msql_parser.parse_msql(query) 163 | print(parsed_output) 164 | assert(len(parsed_output["conditions"][0]["value"]) == 2) 165 | 166 | def test_ms1_multiple_or(): 167 | query = "QUERY scaninfo(MS1DATA) WHERE MS1MZ=(100 OR 104 OR 106 OR 108)" 168 | 169 | parsed_output = msql_parser.parse_msql(query) 170 | print(parsed_output) 171 | assert(len(parsed_output["conditions"][0]["value"]) == 4) 172 | 173 | def test_ms1_multiple_or_with_variable(): 174 | query = "QUERY scaninfo(MS1DATA) WHERE MS1MZ=(X OR X+2 OR X+4 OR X+6)" 175 | 176 | parsed_output = msql_parser.parse_msql(query) 177 | print(parsed_output) 178 | assert(len(parsed_output["conditions"][0]["value"]) == 4) 179 | 180 | def test_mobility(): 181 | query = "QUERY scaninfo(MS2DATA) WHERE MOBILITY=range(min=100, max=500)" 182 | parsed_output = msql_parser.parse_msql(query) 183 | print(parsed_output) 184 | 185 | assert(parsed_output["conditions"][0]["min"] == 100) 186 | 187 | def test_mobility_variables(): 188 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PREC=X AND MOBILITY=range(min=X/100, max=2*X/100)" 189 | parsed_output = msql_parser.parse_msql(query) 190 | print(parsed_output) 191 | 192 | 193 | def test_negation(): 194 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PREC=227:EXCLUDED" 195 | parsed_output = msql_parser.parse_msql(query) 196 | print(parsed_output) 197 | 198 | assert(parsed_output["conditions"][0]["qualifiers"]["qualifierexcluded"]["name"] == "qualifierexcluded") 199 | 200 | def test_wildcard(): 201 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=ANY:MASSDEFECT=massdefect(min=0.8, max=0.9)" 202 | parsed_output = msql_parser.parse_msql(query) 203 | print(parsed_output) 204 | assert(parsed_output["conditions"][0]["qualifiers"]["qualifiermassdefect"]["min"] == 0.8) 205 | 206 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PREC=X:MASSDEFECT=massdefect(min=0.9, max=0.99) AND MOBILITY=range(min=X*0.0006775+0.40557, max=X*0.00078231+0.48817) AND X=massdefect(min=0.9, max=0.99)" 207 | parsed_output = msql_parser.parse_msql(query) 208 | print(parsed_output) 209 | 210 | 211 | def test_cardinality(): 212 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=(100 OR 104):CARDINALITY=range(min=1, max=50)" 213 | parsed_output = msql_parser.parse_msql(query) 214 | print(parsed_output) 215 | assert(parsed_output["conditions"][0]["qualifiers"]["qualifiercardinality"]["min"] == 1.0) 216 | 217 | query = "QUERY scaninfo(MS2DATA) WHERE MS2PROD=(100 OR 104):MATCHCOUNT=range(min=1, max=50)" 218 | parsed_output = msql_parser.parse_msql(query) 219 | print(parsed_output) 220 | assert(parsed_output["conditions"][0]["qualifiers"]["qualifiercardinality"]["min"] == 1.0) 221 | 222 | def test_or_cardinality(): 223 | query = """ 224 | QUERY scaninfo(MS2DATA) WHERE 225 | MS2PROD=(120.08078 OR 86.09643 OR 70.06513 OR 72.08078 OR 159.09167 OR 136.07569):CARDINALITY=range(min=2, max=2) 226 | """ 227 | 228 | parsed_output = msql_parser.parse_msql(query) 229 | print(parsed_output) 230 | 231 | def test_other_rt(): 232 | query = """QUERY scaninfo(MS1DATA) WHERE 233 | MS1MZ=X AND 234 | MS1MZ=X+52:OTHERSCAN=rtrange(left=-40, right=0)""" 235 | 236 | parsed_output = msql_parser.parse_msql(query) 237 | print(parsed_output) 238 | 239 | 240 | 241 | 242 | def main(): 243 | test_other_rt() 244 | #test_xrange_parse() 245 | #test_parse() 246 | #test_comment_parse() 247 | #test_number_expression_parse() 248 | #test_formula_expression_parse() 249 | #test_aminoacids_expression_parse() 250 | #test_peptide_expression_parse() 251 | #test_formula2_expression_parse() 252 | #test_variable_formula_parse() 253 | #test_variable_formula_parse2() 254 | #test_ms2_synonyms() 255 | #test_visualize_parse() 256 | #test_xdefect_parse() 257 | #test_ms2_or() 258 | #test_ms1_multiple_or() 259 | #test_ms1_multiple_or_with_variable() 260 | #test_mobility() 261 | #test_mobility_variables() 262 | #test_negation() 263 | #test_wildcard() 264 | #test_ms1_multiple_or() 265 | #test_cardinality() 266 | #test_or_cardinality() 267 | 268 | 269 | 270 | if __name__ == "__main__": 271 | main() -------------------------------------------------------------------------------- /workflow/.nextflow.log.4: -------------------------------------------------------------------------------- 1 | Jul-23 13:27:38.153 [main] DEBUG nextflow.cli.Launcher - $> nextflow run workflow.nf -c test.config --parallel_files YES --resume --maxfilesize 100 2 | Jul-23 13:27:38.536 [main] INFO nextflow.cli.CmdRun - N E X T F L O W ~ version 21.10.0 3 | Jul-23 13:27:38.608 [main] INFO nextflow.cli.CmdRun - Launching `workflow.nf` [compassionate_lumiere] - revision: cf0e0c8002 4 | Jul-23 13:27:38.660 [main] DEBUG nextflow.config.ConfigBuilder - User config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 5 | Jul-23 13:27:38.661 [main] DEBUG nextflow.config.ConfigBuilder - Parsing config file: /home/user/SourceCode/MassQueryLanguage/workflow/test.config 6 | Jul-23 13:27:38.768 [main] DEBUG nextflow.config.ConfigBuilder - Applying config profile: `standard` 7 | Jul-23 13:27:41.292 [main] DEBUG nextflow.plugin.PluginsFacade - Setting up plugin manager > mode=prod; plugins-dir=/home/user/.nextflow/plugins 8 | Jul-23 13:27:41.296 [main] DEBUG nextflow.plugin.PluginsFacade - Plugins default=[] 9 | Jul-23 13:27:41.318 [main] INFO org.pf4j.DefaultPluginStatusProvider - Enabled plugins: [] 10 | Jul-23 13:27:41.320 [main] INFO org.pf4j.DefaultPluginStatusProvider - Disabled plugins: [] 11 | Jul-23 13:27:41.328 [main] INFO org.pf4j.DefaultPluginManager - PF4J version 3.4.1 in 'deployment' mode 12 | Jul-23 13:27:41.348 [main] INFO org.pf4j.AbstractPluginManager - No plugins 13 | Jul-23 13:27:41.473 [main] DEBUG nextflow.Session - Session uuid: 14f27b53-5a2d-4720-bfa6-d01541f3d9e8 14 | Jul-23 13:27:41.473 [main] DEBUG nextflow.Session - Run name: compassionate_lumiere 15 | Jul-23 13:27:41.474 [main] DEBUG nextflow.Session - Executor pool size: 16 16 | Jul-23 13:27:41.589 [main] DEBUG nextflow.cli.CmdRun - 17 | Version: 21.10.0 build 5640 18 | Created: 11-11-2021 18:27 UTC (10:27 PDT) 19 | System: Linux 5.15.0-53-generic 20 | Runtime: Groovy 3.0.9 on OpenJDK 64-Bit Server VM 1.8.0_152-release-1056-b12 21 | Encoding: UTF-8 (UTF-8) 22 | Process: 4174469@mingwangucr [169.235.26.140] 23 | CPUs: 16 - Mem: 61.8 GB (1.3 GB) - Swap: 735.3 MB (0) 24 | Jul-23 13:27:41.668 [main] DEBUG nextflow.Session - Work-dir: /home/user/SourceCode/MassQueryLanguage/workflow/work [ext2/ext3] 25 | Jul-23 13:27:41.749 [main] DEBUG nextflow.executor.ExecutorFactory - Extension executors providers=[] 26 | Jul-23 13:27:41.798 [main] DEBUG nextflow.Session - Observer factory: DefaultObserverFactory 27 | Jul-23 13:27:41.891 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 28 | Jul-23 13:27:42.106 [main] DEBUG nextflow.Session - Session start invoked 29 | Jul-23 13:27:43.936 [main] DEBUG nextflow.script.ScriptRunner - > Launching execution 30 | Jul-23 13:27:44.230 [main] DEBUG nextflow.util.CustomThreadPool - Creating default thread pool > poolSize: 17; maxThreads: 1000 31 | Jul-23 13:27:44.430 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 32 | Jul-23 13:27:44.432 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 33 | Jul-23 13:27:44.440 [main] DEBUG nextflow.executor.Executor - [warm up] executor > local 34 | Jul-23 13:27:44.449 [main] DEBUG n.processor.LocalPollingMonitor - Creating local task monitor for executor 'local' > cpus=16; memory=61.8 GB; capacity=16; pollInterval=100ms; dumpInterval=5m 35 | Jul-23 13:27:44.725 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 36 | Jul-23 13:27:44.725 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 37 | Jul-23 13:27:44.858 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 38 | Jul-23 13:27:44.859 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 39 | Jul-23 13:27:44.889 [main] DEBUG nextflow.executor.ExecutorFactory - << taskConfig executor: null 40 | Jul-23 13:27:44.890 [main] DEBUG nextflow.executor.ExecutorFactory - >> processorType: 'local' 41 | Jul-23 13:27:44.904 [main] DEBUG nextflow.Session - Workflow process names [dsl2]: summarizeResults, formatResultsMergeRounds, queryData2, queryData, formatExtractedSpectraRounds 42 | Jul-23 13:27:44.905 [main] DEBUG nextflow.Session - Ignite dataflow network (5) 43 | Jul-23 13:27:44.936 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > queryData 44 | Jul-23 13:27:44.940 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatResultsMergeRounds 45 | Jul-23 13:27:44.941 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > formatExtractedSpectraRounds 46 | Jul-23 13:27:44.944 [main] DEBUG nextflow.processor.TaskProcessor - Starting process > summarizeResults 47 | Jul-23 13:27:44.945 [main] DEBUG nextflow.script.ScriptRunner - > Await termination 48 | Jul-23 13:27:44.965 [main] DEBUG nextflow.Session - Session await 49 | Jul-23 13:27:44.964 [PathVisitor-1] DEBUG nextflow.file.PathVisitor - files for syntax: glob; folder: data/; pattern: **; options: [:] 50 | Jul-23 13:27:46.194 [Actor Thread 26] DEBUG nextflow.conda.CondaCache - mamba found local env for environment=/home/user/SourceCode/MassQueryLanguage/workflow/bin/conda_env.yml; path=/home/user/SourceCode/MassQueryLanguage/workflow/work/conda/conda_env-a7c7dacd5204cfe814b4763f1b0accf4 51 | Jul-23 13:27:46.405 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 52 | Jul-23 13:27:46.427 [Task submitter] INFO nextflow.Session - [d2/0d3fcb] Submitted process > queryData (1) 53 | Jul-23 13:27:46.484 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 54 | Jul-23 13:27:46.485 [Task submitter] INFO nextflow.Session - [47/01554a] Submitted process > queryData (2) 55 | Jul-23 13:27:46.500 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 56 | Jul-23 13:27:46.501 [Task submitter] INFO nextflow.Session - [a2/7a5daf] Submitted process > queryData (16) 57 | Jul-23 13:27:46.519 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 58 | Jul-23 13:27:46.520 [Task submitter] INFO nextflow.Session - [ab/600324] Submitted process > queryData (5) 59 | Jul-23 13:27:46.529 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 60 | Jul-23 13:27:46.530 [Task submitter] INFO nextflow.Session - [a1/46b6d3] Submitted process > queryData (3) 61 | Jul-23 13:27:46.538 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 62 | Jul-23 13:27:46.539 [Task submitter] INFO nextflow.Session - [fb/9d92a3] Submitted process > queryData (11) 63 | Jul-23 13:27:46.551 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 64 | Jul-23 13:27:46.568 [Task submitter] INFO nextflow.Session - [72/6aa7b7] Submitted process > queryData (13) 65 | Jul-23 13:27:46.601 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 66 | Jul-23 13:27:46.607 [Task submitter] INFO nextflow.Session - [0e/f15e60] Submitted process > queryData (14) 67 | Jul-23 13:27:46.625 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 68 | Jul-23 13:27:46.626 [Task submitter] INFO nextflow.Session - [28/27f283] Submitted process > queryData (15) 69 | Jul-23 13:27:46.635 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 70 | Jul-23 13:27:46.643 [Task submitter] INFO nextflow.Session - [d4/37988d] Submitted process > queryData (12) 71 | Jul-23 13:27:46.653 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 72 | Jul-23 13:27:46.668 [Task submitter] INFO nextflow.Session - [07/fd08d0] Submitted process > queryData (10) 73 | Jul-23 13:27:46.678 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 74 | Jul-23 13:27:46.688 [Task submitter] INFO nextflow.Session - [a0/8f6375] Submitted process > queryData (6) 75 | Jul-23 13:27:46.709 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 76 | Jul-23 13:27:46.718 [Task submitter] INFO nextflow.Session - [0b/d64029] Submitted process > queryData (9) 77 | Jul-23 13:27:46.733 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 78 | Jul-23 13:27:46.744 [Task submitter] INFO nextflow.Session - [8e/d1378e] Submitted process > queryData (4) 79 | Jul-23 13:27:46.762 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 80 | Jul-23 13:27:46.773 [Task submitter] INFO nextflow.Session - [0f/438a44] Submitted process > queryData (7) 81 | Jul-23 13:27:46.785 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 82 | Jul-23 13:27:46.796 [Task submitter] INFO nextflow.Session - [44/7f909b] Submitted process > queryData (8) 83 | Jul-23 13:27:48.014 [Task monitor] DEBUG n.processor.TaskPollingMonitor - Task completed > TaskHandler[id: 3; name: queryData (5); status: COMPLETED; exit: 127; error: -; workDir: /home/user/SourceCode/MassQueryLanguage/workflow/work/ab/60032476e2a5aea1547d8d98c48e56] 84 | Jul-23 13:27:48.018 [Task submitter] DEBUG nextflow.executor.LocalTaskHandler - Launch cmd line: /bin/bash -ue .command.run 85 | Jul-23 13:27:48.019 [Task submitter] INFO nextflow.Session - [ec/11faf0] Submitted process > queryData (25) 86 | Jul-23 13:27:48.113 [Task monitor] ERROR nextflow.processor.TaskProcessor - Error executing process > 'queryData (5)' 87 | 88 | Caused by: 89 | Process `queryData (5)` terminated with an error exit status (127) 90 | 91 | Command executed: 92 | 93 | python /home/user/SourceCode/MassQueryLanguage/workflow/bin/msql_cmd.py "featurelist_pos.mgf" "QUERY scaninfo(MS2DATA)" --output_file "_home_user_SourceCode_MassQueryLanguage_workflow_data_featurelist_pos.mgf_output.tsv" --parallel_query NO --cache NO --original_path "/home/user/SourceCode/MassQueryLanguage/workflow/data/featurelist_pos.mgf" --extract_json _home_user_SourceCode_MassQueryLanguage_workflow_data_featurelist_pos.mgf_extract.json --maxfilesize 100 94 | 95 | Command exit status: 96 | 127 97 | 98 | Command output: 99 | (empty) 100 | 101 | Command error: 102 | .command.sh: line 2: python: command not found 103 | 104 | Work dir: 105 | /home/user/SourceCode/MassQueryLanguage/workflow/work/ab/60032476e2a5aea1547d8d98c48e56 106 | 107 | Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` 108 | Jul-23 13:27:48.142 [Task monitor] DEBUG nextflow.Session - Session aborted -- Cause: Process `queryData (5)` terminated with an error exit status (127) 109 | Jul-23 13:27:48.178 [Task monitor] DEBUG nextflow.Session - The following nodes are still active: 110 | [process] formatResultsMergeRounds 111 | status=ACTIVE 112 | port 0: (queue) OPEN ; channel: results/* 113 | port 1: (cntrl) - ; channel: $ 114 | 115 | [process] formatExtractedSpectraRounds 116 | status=ACTIVE 117 | port 0: (queue) OPEN ; channel: json/* 118 | port 1: (cntrl) - ; channel: $ 119 | 120 | [process] summarizeResults 121 | status=ACTIVE 122 | port 0: (queue) OPEN ; channel: merged_results 123 | port 1: (cntrl) - ; channel: $ 124 | 125 | Jul-23 13:27:48.182 [main] DEBUG nextflow.Session - Session await > all process finished 126 | Jul-23 13:27:48.183 [main] DEBUG nextflow.Session - Session await > all barriers passed 127 | Jul-23 13:27:48.208 [main] WARN n.processor.TaskPollingMonitor - Killing pending tasks (16) 128 | Jul-23 13:27:48.221 [Actor Thread 14] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 129 | Jul-23 13:27:48.253 [Actor Thread 8] DEBUG nextflow.file.SortFileCollector - FileCollector temp dir not removed: null 130 | Jul-23 13:27:48.260 [main] DEBUG nextflow.executor.LocalTaskHandler - Unable to kill queryData (1) -- command: kill -TERM 4174860; exit: 1 131 | bash: line 1: kill: (4174860) - No such process 132 | 133 | Jul-23 13:27:48.274 [main] DEBUG nextflow.executor.LocalTaskHandler - Unable to kill queryData (2) -- command: kill -TERM 4174866; exit: 1 134 | bash: line 1: kill: (4174866) - No such process 135 | 136 | Jul-23 13:27:48.313 [main] DEBUG nextflow.executor.LocalTaskHandler - Unable to kill queryData (11) -- command: kill -TERM 4174894; exit: 1 137 | bash: line 1: kill: (4174894) - No such process 138 | 139 | Jul-23 13:27:48.358 [main] DEBUG nextflow.executor.LocalTaskHandler - Unable to kill queryData (15) -- command: kill -TERM 4174913; exit: 1 140 | bash: line 1: kill: (4174913) - No such process 141 | 142 | Jul-23 13:27:48.401 [main] DEBUG nextflow.executor.LocalTaskHandler - Unable to kill queryData (8) -- command: kill -TERM 4174960; exit: 1 143 | bash: line 1: kill: (4174960) - No such process 144 | 145 | Jul-23 13:27:48.422 [main] DEBUG nextflow.trace.WorkflowStatsObserver - Workflow completed > WorkflowStats[succeededCount=0; failedCount=1; ignoredCount=0; cachedCount=0; pendingCount=35; submittedCount=1; runningCount=-1; retriesCount=0; abortedCount=16; succeedDuration=0ms; failedDuration=1.4s; cachedDuration=0ms;loadCpus=-1; loadMemory=0; peakRunning=16; peakCpus=16; peakMemory=0; ] 146 | Jul-23 13:27:48.544 [main] DEBUG nextflow.CacheDB - Closing CacheDB done 147 | Jul-23 13:27:48.569 [main] DEBUG nextflow.script.ScriptRunner - > Execution complete -- Goodbye 148 | --------------------------------------------------------------------------------