├── .dockerignore ├── .flake8 ├── .github └── FUNDING.yml ├── .gitignore ├── .style.yapf ├── LICENSE ├── README.md ├── docs ├── Makefile ├── Providers.md ├── README.md ├── Useful.md ├── conf.py ├── create_api.sh ├── filesync.rst ├── grid_search.rst ├── imgs │ ├── code.png │ ├── computers.png │ ├── computers_rsync.png │ ├── console_rsync.png │ ├── dags.png │ ├── describe.png │ ├── graph.png │ ├── hierarchical_logging.png │ ├── layouts.png │ ├── logs.png │ ├── logs_rsync.png │ ├── project_settings.png │ ├── projects.png │ ├── reports.png │ ├── slack.svg │ ├── tasks.png │ └── telegram.svg ├── index.rst ├── installation.rst ├── layout.rst ├── requirements.txt ├── screenshots.md └── usage.rst ├── examples ├── Readme.md ├── __init__.py ├── bash │ ├── Readme.md │ ├── config.yml │ ├── config_error.yml │ └── config_grid.yml ├── cifar_simple │ ├── .ignore │ ├── Readme.md │ ├── __init__.py │ ├── catalyst.yml │ ├── config.yml │ ├── experiment.py │ └── model.py ├── click │ ├── Readme.md │ ├── config.yml │ └── main.py ├── digit-recognizer │ ├── Readme.md │ ├── __init__.py │ ├── all.yml │ ├── catalyst-stage.yml │ ├── catalyst.yml │ ├── dataset.py │ ├── download.yml │ ├── executors │ │ ├── __init__.py │ │ ├── infer.py │ │ └── valid.py │ ├── experiment.py │ ├── grid.yml │ ├── model.py │ ├── pipe.yml │ ├── train-distr-stage.yml │ ├── train-distr.yml │ └── train.yml ├── grid │ ├── Readme.md │ ├── dag.yml │ └── task.yml ├── hierarchical_logging │ ├── Readme.md │ ├── config.yml │ └── executors.py ├── kaggle_submit │ ├── folder │ │ └── file.txt │ ├── predict.ipynb │ └── submit.yml └── progress_bar │ ├── config.yml │ └── executor.py ├── mlcomp ├── __init__.py ├── __main__.py ├── __version__.py ├── bin │ ├── redis-License.md │ └── redis-server ├── contrib │ ├── __init__.py │ ├── __main__.py │ ├── catalyst │ │ ├── __init__.py │ │ ├── callbacks │ │ │ ├── __init__.py │ │ │ └── inference.py │ │ ├── configs │ │ │ └── classify │ │ │ │ ├── bninception.yml │ │ │ │ ├── cafferesnet101.yml │ │ │ │ ├── densenet121.yml │ │ │ │ ├── densenet161.yml │ │ │ │ ├── densenet169.yml │ │ │ │ ├── densenet201.yml │ │ │ │ ├── dpn107.yml │ │ │ │ ├── dpn131.yml │ │ │ │ ├── dpn68.yml │ │ │ │ ├── dpn68b.yml │ │ │ │ ├── dpn92.yml │ │ │ │ ├── dpn98.yml │ │ │ │ ├── fbresnet152.yml │ │ │ │ ├── inceptionresnetv2.yml │ │ │ │ ├── inceptionv3.yml │ │ │ │ ├── inceptionv4.yml │ │ │ │ ├── nasnetalarge.yml │ │ │ │ ├── nasnetamobile.yml │ │ │ │ ├── pnasnet5large.yml │ │ │ │ ├── polynet.yml │ │ │ │ ├── resnet101.yml │ │ │ │ ├── resnet152.yml │ │ │ │ ├── resnet34.yml │ │ │ │ ├── resnet50.yml │ │ │ │ ├── resnext101_32x4d.yml │ │ │ │ ├── resnext101_64x4d.yml │ │ │ │ ├── se_resnet101.yml │ │ │ │ ├── se_resnet152.yml │ │ │ │ ├── se_resnet50.yml │ │ │ │ ├── se_resnext101_32x4d.yml │ │ │ │ ├── se_resnext50_32x4d.yml │ │ │ │ ├── senet154.yml │ │ │ │ └── xception.yml │ │ ├── optim │ │ │ ├── __init__.py │ │ │ └── cosineanneal.py │ │ └── register.py │ ├── complex │ │ ├── __init__.py │ │ └── infer.py │ ├── criterion │ │ ├── __init__.py │ │ ├── ce.py │ │ ├── ring.py │ │ └── triplet.py │ ├── dataset │ │ ├── __init__.py │ │ ├── classify.py │ │ ├── segment.py │ │ └── video.py │ ├── metrics │ │ ├── __init__.py │ │ └── dice.py │ ├── model │ │ ├── __init__.py │ │ ├── efficientnet.py │ │ ├── pretrained.py │ │ ├── segmentation_model_pytorch.py │ │ ├── timm.py │ │ └── video │ │ │ ├── __init__.py │ │ │ └── resnext3d │ │ │ ├── __init__.py │ │ │ ├── r2plus1_util.py │ │ │ ├── resnext3d.py │ │ │ ├── resnext3d_block.py │ │ │ ├── resnext3d_stage.py │ │ │ └── resnext3d_stem.py │ ├── sampler │ │ ├── __init__.py │ │ ├── balanced.py │ │ ├── distributed.py │ │ └── hard_negative.py │ ├── scripts │ │ ├── __init__.py │ │ └── split.py │ ├── search │ │ ├── __init__.py │ │ └── grid.py │ ├── segmentation │ │ ├── Readme.md │ │ ├── __init__.py │ │ ├── base │ │ │ ├── __init__.py │ │ │ ├── encoder_decoder.py │ │ │ └── model.py │ │ ├── common │ │ │ ├── __init__.py │ │ │ └── blocks.py │ │ ├── deeplabv3 │ │ │ ├── Readme.md │ │ │ ├── __init__.py │ │ │ ├── aspp.py │ │ │ ├── backbone │ │ │ │ ├── __init__.py │ │ │ │ ├── drn.py │ │ │ │ ├── mobilenet.py │ │ │ │ ├── resnet.py │ │ │ │ └── xception.py │ │ │ ├── decoder.py │ │ │ └── deeplab.py │ │ ├── encoders │ │ │ ├── __init__.py │ │ │ ├── _preprocessing.py │ │ │ ├── densenet.py │ │ │ ├── dpn.py │ │ │ ├── inceptionresnetv2.py │ │ │ ├── resnet.py │ │ │ ├── senet.py │ │ │ └── vgg.py │ │ ├── fpn │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ └── model.py │ │ ├── linknet │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ └── model.py │ │ ├── pspnet │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ └── model.py │ │ └── unet │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ └── model.py │ ├── split │ │ ├── __init__.py │ │ └── frame.py │ ├── torch │ │ ├── __init__.py │ │ ├── layers.py │ │ └── tensors.py │ └── transform │ │ ├── __init__.py │ │ ├── albumentations.py │ │ ├── rle.py │ │ └── tta.py ├── db │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── db.py │ │ └── options.py │ ├── enums.py │ ├── models │ │ ├── __init__.py │ │ ├── auxilary.py │ │ ├── base.py │ │ ├── computer.py │ │ ├── dag.py │ │ ├── dag_storage.py │ │ ├── docker.py │ │ ├── file.py │ │ ├── log.py │ │ ├── memory.py │ │ ├── model.py │ │ ├── project.py │ │ ├── report.py │ │ ├── space.py │ │ ├── step.py │ │ └── task.py │ ├── providers │ │ ├── __init__.py │ │ ├── auxiliary.py │ │ ├── base.py │ │ ├── computer.py │ │ ├── dag.py │ │ ├── dag_storage.py │ │ ├── docker.py │ │ ├── file.py │ │ ├── log.py │ │ ├── memory.py │ │ ├── model.py │ │ ├── project.py │ │ ├── report │ │ │ ├── __init__.py │ │ │ ├── img.py │ │ │ ├── layout.py │ │ │ ├── report.py │ │ │ ├── series.py │ │ │ └── task.py │ │ ├── space.py │ │ ├── step.py │ │ ├── task.py │ │ └── task_synced.py │ ├── report_info │ │ ├── __init__.py │ │ ├── f1.py │ │ ├── img_classify.py │ │ ├── img_segment.py │ │ ├── info.py │ │ ├── item.py │ │ ├── metric.py │ │ ├── precision_recall.py │ │ └── series.py │ ├── signals.py │ └── tests │ │ ├── __init__.py │ │ └── test_project.py ├── docker │ ├── .env │ ├── Readme.md │ ├── server │ ├── server-compose.yml │ ├── worker │ └── worker-compose.yml ├── migration │ ├── Readme.md │ ├── __init__.py │ ├── manage.py │ ├── migrate.cfg │ └── versions │ │ ├── 001_init.py │ │ ├── 002 │ │ └── report_layout │ │ │ ├── base.yml │ │ │ ├── base_time.yml │ │ │ ├── classify.yml │ │ │ ├── img-classify.yml │ │ │ └── segment.yml │ │ ├── 002_data.py │ │ ├── 003_task_loss.py │ │ ├── 004_task_continued.py │ │ ├── 005_project_sync_folders.py │ │ ├── 006_space.py │ │ ├── 007_space_relation.py │ │ ├── 008_space_tag.py │ │ ├── 009_dag_tag.py │ │ └── 010_memory.py ├── report.py ├── server │ ├── __init__.py │ ├── __main__.py │ ├── back │ │ ├── __init__.py │ │ ├── app.py │ │ ├── create_dags │ │ │ ├── __init__.py │ │ │ ├── copy.py │ │ │ ├── model_add.py │ │ │ ├── model_start.py │ │ │ ├── pipe.py │ │ │ └── standard.py │ │ └── supervisor.py │ ├── front │ │ ├── README.md │ │ ├── angular.json │ │ ├── e2e │ │ │ ├── protractor.conf.js │ │ │ ├── src │ │ │ │ ├── app.e2e-spec.ts │ │ │ │ └── app.po.ts │ │ │ └── tsconfig.e2e.json │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── src │ │ │ ├── app │ │ │ │ ├── _helpers │ │ │ │ │ ├── auth.gaurd.ts │ │ │ │ │ ├── error.interceptor.ts │ │ │ │ │ └── jwt.intercepter.ts │ │ │ │ ├── animations.ts │ │ │ │ ├── app-routing.module.ts │ │ │ │ ├── app-settings.ts │ │ │ │ ├── app.component.css │ │ │ │ ├── app.component.html │ │ │ │ ├── app.component.ts │ │ │ │ ├── app.module.ts │ │ │ │ ├── auxiliary │ │ │ │ │ ├── auxiliary.component.css │ │ │ │ │ ├── auxiliary.component.html │ │ │ │ │ ├── auxiliary.component.ts │ │ │ │ │ ├── auxiliary.service.ts │ │ │ │ │ └── supervisor │ │ │ │ │ │ ├── supervisor.component.css │ │ │ │ │ │ ├── supervisor.component.html │ │ │ │ │ │ └── supervisor.component.ts │ │ │ │ ├── base.service.ts │ │ │ │ ├── computer │ │ │ │ │ ├── computer.component.css │ │ │ │ │ ├── computer.component.html │ │ │ │ │ ├── computer.component.spec.ts │ │ │ │ │ ├── computer.component.ts │ │ │ │ │ ├── computer.service.ts │ │ │ │ │ ├── sync-dialog.html │ │ │ │ │ └── sync-dialog.ts │ │ │ │ ├── dag │ │ │ │ │ ├── dag-detail │ │ │ │ │ │ ├── code │ │ │ │ │ │ │ ├── code.component.css │ │ │ │ │ │ │ ├── code.component.html │ │ │ │ │ │ │ ├── code.component.spec.ts │ │ │ │ │ │ │ └── code.component.ts │ │ │ │ │ │ ├── config │ │ │ │ │ │ │ ├── config.component.css │ │ │ │ │ │ │ ├── config.component.html │ │ │ │ │ │ │ ├── config.component.spec.ts │ │ │ │ │ │ │ └── config.component.ts │ │ │ │ │ │ ├── dag-detail-routing.module.ts │ │ │ │ │ │ ├── dag-detail.module.ts │ │ │ │ │ │ ├── dag-detail │ │ │ │ │ │ │ ├── dag-detail.component.css │ │ │ │ │ │ │ ├── dag-detail.component.html │ │ │ │ │ │ │ ├── dag-detail.component.spec.ts │ │ │ │ │ │ │ ├── dag-detail.component.ts │ │ │ │ │ │ │ └── dag-detail.service.ts │ │ │ │ │ │ └── graph │ │ │ │ │ │ │ ├── graph.component.css │ │ │ │ │ │ │ ├── graph.component.html │ │ │ │ │ │ │ ├── graph.component.spec.ts │ │ │ │ │ │ │ └── graph.component.ts │ │ │ │ │ ├── dag-routing.module.ts │ │ │ │ │ ├── dag.module.ts │ │ │ │ │ ├── dag.service.ts │ │ │ │ │ ├── dag │ │ │ │ │ │ ├── dag.component.css │ │ │ │ │ │ ├── dag.component.html │ │ │ │ │ │ ├── dag.component.spec.ts │ │ │ │ │ │ └── dag.component.ts │ │ │ │ │ └── dags │ │ │ │ │ │ ├── dag-restart-dialog.html │ │ │ │ │ │ ├── dag-restart-dialog.ts │ │ │ │ │ │ ├── dags.component.css │ │ │ │ │ │ ├── dags.component.html │ │ │ │ │ │ ├── dags.component.spec.ts │ │ │ │ │ │ └── dags.component.ts │ │ │ │ ├── dialog │ │ │ │ │ ├── dialog.component.html │ │ │ │ │ └── dialog.component.ts │ │ │ │ ├── dynamicresource.service.ts │ │ │ │ ├── helpers.ts │ │ │ │ ├── log │ │ │ │ │ ├── log.component.css │ │ │ │ │ ├── log.component.html │ │ │ │ │ ├── log.component.spec.ts │ │ │ │ │ ├── log.component.ts │ │ │ │ │ └── log.service.ts │ │ │ │ ├── login │ │ │ │ │ ├── login.component.html │ │ │ │ │ └── login.component.ts │ │ │ │ ├── message.service.ts │ │ │ │ ├── messages │ │ │ │ │ ├── messages.component.css │ │ │ │ │ ├── messages.component.html │ │ │ │ │ ├── messages.component.spec.ts │ │ │ │ │ └── messages.component.ts │ │ │ │ ├── model │ │ │ │ │ ├── model-add-dialog.component.ts │ │ │ │ │ ├── model-add-dialog.html │ │ │ │ │ ├── model-start-dialog.component.ts │ │ │ │ │ ├── model-start-dialog.css │ │ │ │ │ ├── model-start-dialog.html │ │ │ │ │ ├── model.component.css │ │ │ │ │ ├── model.component.html │ │ │ │ │ ├── model.component.ts │ │ │ │ │ └── model.service.ts │ │ │ │ ├── models.ts │ │ │ │ ├── paginator.ts │ │ │ │ ├── project │ │ │ │ │ ├── project-add-dialog.html │ │ │ │ │ ├── project-add-dialog.ts │ │ │ │ │ ├── project.component.css │ │ │ │ │ ├── project.component.html │ │ │ │ │ ├── project.component.spec.ts │ │ │ │ │ ├── project.component.ts │ │ │ │ │ └── project.service.ts │ │ │ │ ├── report │ │ │ │ │ ├── internal │ │ │ │ │ │ ├── img-classify │ │ │ │ │ │ │ ├── img-classify.component.css │ │ │ │ │ │ │ ├── img-classify.component.html │ │ │ │ │ │ │ ├── img-classify.component.ts │ │ │ │ │ │ │ └── img-classify.service.ts │ │ │ │ │ │ ├── img-segment │ │ │ │ │ │ │ ├── img-segment.component.css │ │ │ │ │ │ │ ├── img-segment.component.html │ │ │ │ │ │ │ ├── img-segment.component.ts │ │ │ │ │ │ │ └── img-segment.service.ts │ │ │ │ │ │ ├── img │ │ │ │ │ │ │ ├── img.component.css │ │ │ │ │ │ │ ├── img.component.html │ │ │ │ │ │ │ ├── img.component.spec.ts │ │ │ │ │ │ │ └── img.component.ts │ │ │ │ │ │ ├── layout │ │ │ │ │ │ │ ├── layout.component.css │ │ │ │ │ │ │ ├── layout.component.html │ │ │ │ │ │ │ ├── layout.component.spec.ts │ │ │ │ │ │ │ ├── layout.component.ts │ │ │ │ │ │ │ └── layout.service.ts │ │ │ │ │ │ ├── series │ │ │ │ │ │ │ ├── series.component.css │ │ │ │ │ │ │ ├── series.component.html │ │ │ │ │ │ │ ├── series.component.spec.ts │ │ │ │ │ │ │ └── series.component.ts │ │ │ │ │ │ └── table │ │ │ │ │ │ │ ├── table.component.css │ │ │ │ │ │ │ ├── table.component.html │ │ │ │ │ │ │ └── table.component.ts │ │ │ │ │ ├── layouts │ │ │ │ │ │ ├── layout-add-dialog.html │ │ │ │ │ │ ├── layout-add-dialog.ts │ │ │ │ │ │ ├── layouts.component.css │ │ │ │ │ │ ├── layouts.component.html │ │ │ │ │ │ ├── layouts.component.spec.ts │ │ │ │ │ │ ├── layouts.component.ts │ │ │ │ │ │ └── layouts.service.ts │ │ │ │ │ ├── report-detail │ │ │ │ │ │ ├── report-detail.component.css │ │ │ │ │ │ ├── report-detail.component.html │ │ │ │ │ │ ├── report-detail.component.spec.ts │ │ │ │ │ │ ├── report-detail.component.ts │ │ │ │ │ │ ├── report-update-dialog.component.css │ │ │ │ │ │ ├── report-update-dialog.component.html │ │ │ │ │ │ └── report-update-dialog.component.ts │ │ │ │ │ ├── report-routing.module.ts │ │ │ │ │ ├── report.module.ts │ │ │ │ │ ├── report.service.ts │ │ │ │ │ ├── report │ │ │ │ │ │ ├── report.component.css │ │ │ │ │ │ ├── report.component.html │ │ │ │ │ │ ├── report.component.spec.ts │ │ │ │ │ │ └── report.component.ts │ │ │ │ │ └── reports │ │ │ │ │ │ ├── report-add-dialog.html │ │ │ │ │ │ ├── report-add-dialog.ts │ │ │ │ │ │ ├── reports.component.css │ │ │ │ │ │ ├── reports.component.html │ │ │ │ │ │ ├── reports.component.spec.ts │ │ │ │ │ │ └── reports.component.ts │ │ │ │ ├── shared.module.ts │ │ │ │ ├── skynet │ │ │ │ │ ├── memory │ │ │ │ │ │ ├── memory-add-dialog.html │ │ │ │ │ │ ├── memory-add-dialog.ts │ │ │ │ │ │ ├── memory.component.css │ │ │ │ │ │ ├── memory.component.html │ │ │ │ │ │ ├── memory.component.spec.ts │ │ │ │ │ │ ├── memory.component.ts │ │ │ │ │ │ └── memory.service.ts │ │ │ │ │ ├── skynet-routing.module.ts │ │ │ │ │ ├── skynet.module.ts │ │ │ │ │ ├── skynet │ │ │ │ │ │ ├── skynet.component.css │ │ │ │ │ │ ├── skynet.component.html │ │ │ │ │ │ ├── skynet.component.spec.ts │ │ │ │ │ │ └── skynet.component.ts │ │ │ │ │ └── space │ │ │ │ │ │ ├── space-add-dialog.html │ │ │ │ │ │ ├── space-add-dialog.ts │ │ │ │ │ │ ├── space-run-dialog.html │ │ │ │ │ │ ├── space-run-dialog.ts │ │ │ │ │ │ ├── space.component.css │ │ │ │ │ │ ├── space.component.html │ │ │ │ │ │ ├── space.component.spec.ts │ │ │ │ │ │ ├── space.component.ts │ │ │ │ │ │ └── space.service.ts │ │ │ │ └── task │ │ │ │ │ ├── task-detail │ │ │ │ │ ├── step │ │ │ │ │ │ ├── step.component.css │ │ │ │ │ │ ├── step.component.html │ │ │ │ │ │ ├── step.component.spec.ts │ │ │ │ │ │ └── step.component.ts │ │ │ │ │ ├── task-detail-routing.module.ts │ │ │ │ │ ├── task-detail.module.ts │ │ │ │ │ └── task-detail │ │ │ │ │ │ ├── task-detail.component.css │ │ │ │ │ │ ├── task-detail.component.html │ │ │ │ │ │ ├── task-detail.component.spec.ts │ │ │ │ │ │ └── task-detail.component.ts │ │ │ │ │ ├── task-routing.module.ts │ │ │ │ │ ├── task-table │ │ │ │ │ ├── task-info-dialog.component.css │ │ │ │ │ ├── task-info-dialog.component.html │ │ │ │ │ ├── task-info-dialog.component.ts │ │ │ │ │ ├── task-table.component.css │ │ │ │ │ ├── task-table.component.html │ │ │ │ │ └── task-table.component.ts │ │ │ │ │ ├── task.module.ts │ │ │ │ │ ├── task.service.ts │ │ │ │ │ ├── task │ │ │ │ │ ├── task.component.css │ │ │ │ │ ├── task.component.html │ │ │ │ │ ├── task.component.spec.ts │ │ │ │ │ └── task.component.ts │ │ │ │ │ └── tasks │ │ │ │ │ ├── tasks.component.css │ │ │ │ │ ├── tasks.component.html │ │ │ │ │ ├── tasks.component.spec.ts │ │ │ │ │ └── tasks.component.ts │ │ │ ├── assets │ │ │ │ ├── img │ │ │ │ │ ├── back.svg │ │ │ │ │ ├── code-signs.svg │ │ │ │ │ ├── config.svg │ │ │ │ │ ├── delete.svg │ │ │ │ │ ├── edit.svg │ │ │ │ │ ├── forward.svg │ │ │ │ │ ├── info.svg │ │ │ │ │ ├── mlcomp_icon.jpg │ │ │ │ │ ├── mlcomp_logo.png │ │ │ │ │ ├── model.svg │ │ │ │ │ ├── network.svg │ │ │ │ │ ├── play-button.svg │ │ │ │ │ ├── programming-code.svg │ │ │ │ │ ├── report.svg │ │ │ │ │ ├── restart.svg │ │ │ │ │ ├── stop.svg │ │ │ │ │ └── trash.svg │ │ │ │ ├── plotly.min.js │ │ │ │ ├── prettify │ │ │ │ │ ├── lang-Splus.js │ │ │ │ │ ├── lang-aea.js │ │ │ │ │ ├── lang-agc.js │ │ │ │ │ ├── lang-apollo.js │ │ │ │ │ ├── lang-basic.js │ │ │ │ │ ├── lang-cbm.js │ │ │ │ │ ├── lang-cl.js │ │ │ │ │ ├── lang-clj.js │ │ │ │ │ ├── lang-css.js │ │ │ │ │ ├── lang-dart.js │ │ │ │ │ ├── lang-el.js │ │ │ │ │ ├── lang-erl.js │ │ │ │ │ ├── lang-erlang.js │ │ │ │ │ ├── lang-ex.js │ │ │ │ │ ├── lang-exs.js │ │ │ │ │ ├── lang-fs.js │ │ │ │ │ ├── lang-go.js │ │ │ │ │ ├── lang-hs.js │ │ │ │ │ ├── lang-kotlin.js │ │ │ │ │ ├── lang-lasso.js │ │ │ │ │ ├── lang-lassoscript.js │ │ │ │ │ ├── lang-latex.js │ │ │ │ │ ├── lang-lgt.js │ │ │ │ │ ├── lang-lisp.js │ │ │ │ │ ├── lang-ll.js │ │ │ │ │ ├── lang-llvm.js │ │ │ │ │ ├── lang-logtalk.js │ │ │ │ │ ├── lang-ls.js │ │ │ │ │ ├── lang-lsp.js │ │ │ │ │ ├── lang-lua.js │ │ │ │ │ ├── lang-matlab.js │ │ │ │ │ ├── lang-ml.js │ │ │ │ │ ├── lang-mumps.js │ │ │ │ │ ├── lang-n.js │ │ │ │ │ ├── lang-nemerle.js │ │ │ │ │ ├── lang-pascal.js │ │ │ │ │ ├── lang-proto.js │ │ │ │ │ ├── lang-r.js │ │ │ │ │ ├── lang-rd.js │ │ │ │ │ ├── lang-rkt.js │ │ │ │ │ ├── lang-rust.js │ │ │ │ │ ├── lang-s.js │ │ │ │ │ ├── lang-scala.js │ │ │ │ │ ├── lang-scm.js │ │ │ │ │ ├── lang-sql.js │ │ │ │ │ ├── lang-ss.js │ │ │ │ │ ├── lang-swift.js │ │ │ │ │ ├── lang-tcl.js │ │ │ │ │ ├── lang-tex.js │ │ │ │ │ ├── lang-vb.js │ │ │ │ │ ├── lang-vbs.js │ │ │ │ │ ├── lang-vhd.js │ │ │ │ │ ├── lang-vhdl.js │ │ │ │ │ ├── lang-wiki.js │ │ │ │ │ ├── lang-xq.js │ │ │ │ │ ├── lang-xquery.js │ │ │ │ │ ├── lang-yaml.js │ │ │ │ │ ├── lang-yml.js │ │ │ │ │ ├── prettify.css │ │ │ │ │ ├── prettify.js │ │ │ │ │ ├── run_prettify.js │ │ │ │ │ └── skins │ │ │ │ │ │ ├── desert.css │ │ │ │ │ │ ├── doxy.css │ │ │ │ │ │ ├── sons-of-obsidian.css │ │ │ │ │ │ └── sunburst.css │ │ │ │ └── visjs │ │ │ │ │ ├── vis.min.css │ │ │ │ │ └── vis.min.js │ │ │ ├── browserslist │ │ │ ├── environments │ │ │ │ ├── environment.prod.ts │ │ │ │ └── environment.ts │ │ │ ├── index.html │ │ │ ├── karma.conf.js │ │ │ ├── main.ts │ │ │ ├── polyfills.ts │ │ │ ├── styles.css │ │ │ ├── test.ts │ │ │ ├── tsconfig.app.json │ │ │ ├── tsconfig.spec.json │ │ │ └── tslint.json │ │ ├── tsconfig.json │ │ └── tslint.json │ └── tests │ │ └── __init__.py ├── utils │ ├── __init__.py │ ├── config.py │ ├── describe.py │ ├── img.py │ ├── io.py │ ├── logging.py │ ├── misc.py │ ├── plot.py │ ├── req.py │ ├── req_stdlib │ ├── schedule.py │ ├── tests.py │ └── torch.py └── worker │ ├── __init__.py │ ├── __main__.py │ ├── app.py │ ├── executors │ ├── __init__.py │ ├── base │ │ ├── __init__.py │ │ ├── executor.py │ │ └── step.py │ ├── bash.py │ ├── catalyst_ │ │ ├── __init__.py │ │ └── catalyst_.py │ ├── click.py │ ├── kaggle.py │ ├── model.py │ └── split.py │ ├── reports │ ├── __init__.py │ ├── classification.py │ └── segmenation.py │ ├── storage.py │ ├── sync.py │ ├── tasks.py │ └── tests │ └── __init__.py ├── pytest.ini ├── requirements.txt ├── setup.py ├── teamcity ├── deploy.sh └── tests.sh └── yapf.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | *.egg-ingo 4 | examples 5 | .git 6 | mlcomp/server/front/node_modules -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git,__pycache__,docs,build,dist,mlcomp/server/front -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: catalyst_team 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style=pep8 3 | allow_split_before_dict_value=False 4 | join_multiple_lines=False 5 | split_before_named_assigns=True 6 | dedent_closing_brackets=True -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | Docs are available [here](https://catalyst-team.github.io/mlcomp/index.html) -------------------------------------------------------------------------------- /docs/Useful.md: -------------------------------------------------------------------------------- 1 | export MKL_NUM_THREADS=1 2 | export OMP_NUM_THREADS=1 -------------------------------------------------------------------------------- /docs/create_api.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | sphinx-apidoc -o ./_modules ../ -M -f 3 | 4 | -------------------------------------------------------------------------------- /docs/imgs/code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/code.png -------------------------------------------------------------------------------- /docs/imgs/computers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/computers.png -------------------------------------------------------------------------------- /docs/imgs/computers_rsync.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/computers_rsync.png -------------------------------------------------------------------------------- /docs/imgs/console_rsync.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/console_rsync.png -------------------------------------------------------------------------------- /docs/imgs/dags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/dags.png -------------------------------------------------------------------------------- /docs/imgs/describe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/describe.png -------------------------------------------------------------------------------- /docs/imgs/graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/graph.png -------------------------------------------------------------------------------- /docs/imgs/hierarchical_logging.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/hierarchical_logging.png -------------------------------------------------------------------------------- /docs/imgs/layouts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/layouts.png -------------------------------------------------------------------------------- /docs/imgs/logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/logs.png -------------------------------------------------------------------------------- /docs/imgs/logs_rsync.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/logs_rsync.png -------------------------------------------------------------------------------- /docs/imgs/project_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/project_settings.png -------------------------------------------------------------------------------- /docs/imgs/projects.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/projects.png -------------------------------------------------------------------------------- /docs/imgs/reports.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/reports.png -------------------------------------------------------------------------------- /docs/imgs/slack.svg: -------------------------------------------------------------------------------- 1 | ODSODSslackslack -------------------------------------------------------------------------------- /docs/imgs/tasks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/docs/imgs/tasks.png -------------------------------------------------------------------------------- /docs/imgs/telegram.svg: -------------------------------------------------------------------------------- 1 | newsnewson telegramon telegram -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinx-autobuild 3 | sphinx_rtd_theme 4 | -------------------------------------------------------------------------------- /docs/screenshots.md: -------------------------------------------------------------------------------- 1 | Projects 2 | 3 | ![projects](imgs/projects.png) 4 | 5 | Dags 6 | 7 | ![dags](imgs/dags.png) 8 | 9 | Code 10 | 11 | ![code](imgs/code.png) 12 | 13 | Graph 14 | 15 | ![code](imgs/graph.png) 16 | 17 | Tasks 18 | 19 | ![tasks](imgs/tasks.png) 20 | 21 | Logs 22 | 23 | ![logs](imgs/logs.png) 24 | 25 | Hierarchical logging 26 | 27 | ![hierarchical_logging](imgs/hierarchical_logging.png) 28 | 29 | Layouts 30 | 31 | ![layouts](imgs/layouts.png) 32 | 33 | Reports 34 | 35 | ![reports](imgs/reports.png) -------------------------------------------------------------------------------- /examples/Readme.md: -------------------------------------------------------------------------------- 1 | - [cifar_simple](cifar_simple/Readme.md) is a example extended of [Catalyst cifar_simple](https://github.com/catalyst-team/catalyst/tree/master/examples/cifar_simple) 2 | - [digit-recognizer](digit-recognizer/Readme.md) Kaggle digit recognizer competition 3 | 4 | -download from/upload to Kaggle 5 | 6 | -Catalyst multistage 7 | 8 | -grid search 9 | 10 | -distributed learning 11 | 12 | -all pipeline within the single command! 13 | 14 | - [hierarchical_logging](hierarchical_logging/Readme.md) Hierarchical logging. 15 | - [bash](bash/Readme.md) You can run any bash command. 16 | - [click](click/Readme.md) Click is a Python package for creating beautiful command line interfaces. 17 | You can debug the code as usual, then run it in MLComp's cluster. -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/examples/__init__.py -------------------------------------------------------------------------------- /examples/bash/Readme.md: -------------------------------------------------------------------------------- 1 | Run 2 | 3 | ```bash 4 | mlcomp dag examples/bash/config.yml 5 | ``` 6 | 7 | You can run any bash command. -------------------------------------------------------------------------------- /examples/bash/config.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: bash 3 | project: examples 4 | expdir: . 5 | executors: 6 | bash: 7 | type: bash 8 | command: ls -------------------------------------------------------------------------------- /examples/bash/config_error.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: bash 3 | project: examples 4 | expdir: . 5 | executors: 6 | bash: 7 | type: bash 8 | command: python -c "print(10); import time; time.sleep(1); raise Exception()" -------------------------------------------------------------------------------- /examples/bash/config_grid.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: bash 3 | project: examples 4 | expdir: . 5 | executors: 6 | bash: 7 | type: bash 8 | command: echo $variable 9 | grid: 10 | - variable: 0-10 -------------------------------------------------------------------------------- /examples/cifar_simple/.ignore: -------------------------------------------------------------------------------- 1 | # like a usual .gitignore file -------------------------------------------------------------------------------- /examples/cifar_simple/Readme.md: -------------------------------------------------------------------------------- 1 | A minimal extended example of [Catalyst cifar_simple](https://github.com/catalyst-team/catalyst/tree/master/examples/cifar_simple) 2 | 3 | Run 4 | 5 | ```bash 6 | cd examples/cifar_simple 7 | mlcomp dag config.yml 8 | ``` 9 | 10 | Just config.yml is added. 11 | 12 | The overhead is minimal. -------------------------------------------------------------------------------- /examples/cifar_simple/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from catalyst.dl import SupervisedRunner as Runner 3 | from experiment import Experiment 4 | from model import Net 5 | -------------------------------------------------------------------------------- /examples/cifar_simple/catalyst.yml: -------------------------------------------------------------------------------- 1 | model_params: 2 | model: Net 3 | 4 | 5 | args: 6 | expdir: "." 7 | logdir: "log" 8 | 9 | stages: 10 | 11 | data_params: 12 | batch_size: 32 13 | num_workers: 0 14 | 15 | state_params: 16 | num_epochs: 1 17 | main_metric: &reduce_metric accuracy01 18 | minimize_metric: False 19 | 20 | criterion_params: 21 | criterion: CrossEntropyLoss 22 | 23 | scheduler_params: 24 | scheduler: MultiStepLR 25 | milestones: [10] 26 | gamma: 0.3 27 | 28 | callbacks_params: 29 | loss: 30 | callback: CriterionCallback 31 | optimizer: 32 | callback: OptimizerCallback 33 | accuracy: 34 | callback: AccuracyCallback 35 | accuracy_args: [1] 36 | scheduler: 37 | callback: SchedulerCallback 38 | reduced_metric: *reduce_metric 39 | saver: 40 | callback: CheckpointCallback 41 | 42 | stage1: 43 | 44 | optimizer_params: 45 | optimizer: Adam 46 | lr: 0.001 47 | weight_decay: 0.0001 -------------------------------------------------------------------------------- /examples/cifar_simple/config.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: cifar 3 | project: examples 4 | expdir: . 5 | layout: classify 6 | executors: 7 | train: 8 | type: catalyst 9 | gpu: 1 10 | args: 11 | config: catalyst.yml 12 | -------------------------------------------------------------------------------- /examples/cifar_simple/model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | 4 | from catalyst.contrib import registry 5 | 6 | 7 | @registry.Model 8 | class Net(nn.Module): 9 | def __init__(self): 10 | super().__init__() 11 | self.conv1 = nn.Conv2d(3, 6, 5) 12 | self.pool = nn.MaxPool2d(2, 2) 13 | self.conv2 = nn.Conv2d(6, 16, 5) 14 | self.fc1 = nn.Linear(16 * 5 * 5, 120) 15 | self.fc2 = nn.Linear(120, 84) 16 | self.fc3 = nn.Linear(84, 10) 17 | 18 | def forward(self, x): 19 | x = self.pool(F.relu(self.conv1(x))) 20 | x = self.pool(F.relu(self.conv2(x))) 21 | x = x.view(-1, 16 * 5 * 5) 22 | x = F.relu(self.fc1(x)) 23 | x = F.relu(self.fc2(x)) 24 | x = self.fc3(x) 25 | return x 26 | -------------------------------------------------------------------------------- /examples/click/Readme.md: -------------------------------------------------------------------------------- 1 | Run 2 | 3 | ```bash 4 | mlcomp dag examples/click/config.yml 5 | ``` 6 | 7 | Click is a Python package for creating beautiful command line interfaces. 8 | You can debug the code as usual, then run it in MLComp's cluster. -------------------------------------------------------------------------------- /examples/click/config.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: click 3 | project: examples 4 | expdir: . 5 | executors: 6 | click: 7 | type: click 8 | module: main 9 | command: work -------------------------------------------------------------------------------- /examples/click/main.py: -------------------------------------------------------------------------------- 1 | import click 2 | import time 3 | from tqdm import tqdm 4 | 5 | 6 | @click.group() 7 | def base(): 8 | pass 9 | 10 | 11 | @base.command() 12 | @click.option('--count', type=int, default=100) 13 | def work(count: int): 14 | print('start') 15 | items = list(range(count)) 16 | bar = tqdm(items) 17 | for item in bar: 18 | bar.set_description(f'item={item}') 19 | time.sleep(0.01) 20 | print('end') 21 | 22 | 23 | if __name__ == '__main__': 24 | base() 25 | -------------------------------------------------------------------------------- /examples/digit-recognizer/Readme.md: -------------------------------------------------------------------------------- 1 | [Kaggle digit-recognizer competition](https://www.kaggle.com/c/digit-recognizer) 2 | 3 | Install [Kaggle API](https://github.com/Kaggle/kaggle-api). 4 | 5 | Ensure you filled your key in ~/.kaggle/kaggle.json 6 | 7 | ```bash 8 | cd examples/digit-recognizer 9 | ``` 10 | 11 | Download dataset and split on folds 12 | 13 | ```bash 14 | mlcomp dag download.yml 15 | ``` 16 | 17 | Distributed training. Please fill your gpu count 18 | 19 | ```bash 20 | mlcomp dag train-distr.yml 21 | ``` 22 | 23 | Distributed training with stages. 24 | 25 | ```bash 26 | mlcomp dag train-distr-stage.yml 27 | ``` 28 | 29 | Grid search 30 | 31 | ```bash 32 | mlcomp dag grid.yml 33 | ``` 34 | 35 | All together with an upload of the submission to Kaggle! 36 | 37 | ```bash 38 | mlcomp dag all.yml 39 | ``` 40 | 41 | You can check your [submissions page](https://www.kaggle.com/c/digit-recognizer/submissions) -------------------------------------------------------------------------------- /examples/digit-recognizer/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from catalyst.dl import SupervisedRunner as Runner 3 | from experiment import Experiment 4 | from model import Net -------------------------------------------------------------------------------- /examples/digit-recognizer/all.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: all 3 | project: examples 4 | layout: classify 5 | executors: 6 | download: 7 | type: download 8 | competition: digit-recognizer 9 | split: 10 | type: split 11 | variant: frame 12 | label: label 13 | file: train.csv 14 | depends: download 15 | train: 16 | type: catalyst 17 | gpu: 1 18 | args: 19 | config: catalyst.yml 20 | trace: models/net.pth 21 | depends: split 22 | infer: 23 | type: infer_mnist 24 | gpu: 1 25 | y: torch(x, file='net.pth', batch_size=128) 26 | test: True 27 | suffix: test 28 | prepare_submit: True 29 | model_name: net 30 | depends: train 31 | submit: 32 | type: Submit 33 | depends: infer 34 | suffix: test 35 | model_name: net 36 | competition: digit-recognizer 37 | column: Label -------------------------------------------------------------------------------- /examples/digit-recognizer/catalyst-stage.yml: -------------------------------------------------------------------------------- 1 | model_params: 2 | model: Net 3 | 4 | 5 | args: 6 | expdir: "." 7 | logdir: "log" 8 | 9 | stages: 10 | 11 | data_params: 12 | batch_size: 256 13 | num_workers: 1 14 | 15 | state_params: 16 | main_metric: accuracy01 17 | minimize_metric: False 18 | checkpoint_data: 19 | exp: "thin is my best experiments" 20 | date: "today" 21 | 22 | criterion_params: 23 | criterion: CrossEntropyLoss 24 | 25 | callbacks_params: 26 | loss: 27 | callback: CriterionCallback 28 | optimizer: 29 | callback: OptimizerCallback 30 | accuracy: 31 | callback: AccuracyCallback 32 | accuracy_args: [1] 33 | saver: 34 | callback: CheckpointCallback 35 | 36 | stage1: 37 | state_params: 38 | num_epochs: 10 39 | 40 | optimizer_params: 41 | optimizer: Adam 42 | lr: 0.001 43 | weight_decay: 0.0001 44 | 45 | stage2: 46 | state_params: 47 | num_epochs: 10 48 | 49 | optimizer_params: 50 | optimizer: Adam 51 | lr: 0.001 52 | weight_decay: 0.00001 -------------------------------------------------------------------------------- /examples/digit-recognizer/catalyst.yml: -------------------------------------------------------------------------------- 1 | model_params: 2 | model: Net 3 | 4 | 5 | args: 6 | expdir: "." 7 | logdir: "log" 8 | 9 | stages: 10 | 11 | data_params: 12 | batch_size: 256 13 | num_workers: 1 14 | 15 | state_params: 16 | num_epochs: 10 17 | main_metric: accuracy01 18 | minimize_metric: False 19 | checkpoint_data: 20 | exp: "thin is my best experiments" 21 | date: "today" 22 | 23 | criterion_params: 24 | criterion: CrossEntropyLoss 25 | 26 | callbacks_params: 27 | loss: 28 | callback: CriterionCallback 29 | optimizer: 30 | callback: OptimizerCallback 31 | accuracy: 32 | callback: AccuracyCallback 33 | accuracy_args: [1] 34 | saver: 35 | callback: CheckpointCallback 36 | 37 | stage1: 38 | 39 | optimizer_params: 40 | optimizer: Adam 41 | lr: 0.001 42 | weight_decay: 0.0001 -------------------------------------------------------------------------------- /examples/digit-recognizer/download.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: mnist_download 3 | project: examples 4 | executors: 5 | download: 6 | type: download 7 | competition: digit-recognizer 8 | split: 9 | type: split 10 | variant: frame 11 | label: label 12 | file: train.csv 13 | depends: download 14 | -------------------------------------------------------------------------------- /examples/digit-recognizer/executors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/examples/digit-recognizer/executors/__init__.py -------------------------------------------------------------------------------- /examples/digit-recognizer/experiment.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import albumentations as A 4 | 5 | from catalyst.dl import ConfigExperiment 6 | 7 | from dataset import MnistDataset 8 | 9 | 10 | class Experiment(ConfigExperiment): 11 | def get_datasets(self, stage: str, **kwargs): 12 | datasets = OrderedDict() 13 | 14 | train = MnistDataset( 15 | 'data/train.csv', 16 | fold_csv='data/fold.csv', 17 | transforms=Experiment.get_test_transforms() 18 | ) 19 | 20 | valid = MnistDataset( 21 | 'data/train.csv', 22 | fold_csv='data/fold.csv', 23 | is_test=True, 24 | transforms=Experiment.get_test_transforms() 25 | ) 26 | 27 | datasets['train'] = train 28 | datasets['valid'] = valid 29 | 30 | return datasets 31 | 32 | @staticmethod 33 | def get_test_transforms(): 34 | return A.Compose([A.Normalize(mean=(0.485, ), std=(0.229, ))]) 35 | -------------------------------------------------------------------------------- /examples/digit-recognizer/grid.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: mnist_train 3 | project: examples 4 | layout: classify 5 | expdir: . 6 | executors: 7 | train: 8 | type: catalyst 9 | gpu: 1 10 | args: 11 | config: catalyst.yml 12 | grid: 13 | - batch_size: [20, 40, 80] 14 | - - num_workers: 2 15 | lr: 0.01 16 | - num_workers: 3 17 | lr: 0.1 18 | -------------------------------------------------------------------------------- /examples/digit-recognizer/model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | 4 | from catalyst.contrib import registry 5 | 6 | 7 | @registry.Model 8 | class Net(nn.Module): 9 | def __init__(self): 10 | super().__init__() 11 | self.conv1 = nn.Conv2d(1, 20, 5, 1) 12 | self.conv2 = nn.Conv2d(20, 50, 5, 1) 13 | self.fc1 = nn.Linear(4 * 4 * 50, 500) 14 | self.fc2 = nn.Linear(500, 10) 15 | 16 | def forward(self, x): 17 | x = F.relu(self.conv1(x)) 18 | x = F.max_pool2d(x, 2, 2) 19 | x = F.relu(self.conv2(x)) 20 | x = F.max_pool2d(x, 2, 2) 21 | x = x.view(-1, 4 * 4 * 50) 22 | x = F.relu(self.fc1(x)) 23 | x = self.fc2(x) 24 | return F.log_softmax(x, dim=1) 25 | -------------------------------------------------------------------------------- /examples/digit-recognizer/pipe.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: mnist_pipe 3 | project: examples 4 | type: pipe 5 | expdir: . 6 | pipes: 7 | valid: 8 | valid: 9 | type: valid_mnist 10 | gpu: 1 11 | infer_valid: 12 | infer: 13 | type: infer_mnist 14 | suffix: valid 15 | gpu: 1 16 | infer_test: 17 | infer: 18 | type: infer_mnist 19 | suffix: test 20 | test: true 21 | gpu: 1 22 | submit: 23 | submit_mnist: 24 | type: submit 25 | competition: digit-recognizer 26 | predict_column: Label 27 | suffix: test 28 | -------------------------------------------------------------------------------- /examples/digit-recognizer/train-distr-stage.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: mnist_train_stage 3 | project: examples 4 | layout: classify 5 | expdir: . 6 | executors: 7 | train: 8 | type: catalyst 9 | gpu: 2-4 10 | distr: True 11 | single_node: True 12 | args: 13 | config: catalyst-stage.yml 14 | -------------------------------------------------------------------------------- /examples/digit-recognizer/train-distr.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: mnist_train 3 | project: examples 4 | layout: classify 5 | expdir: . 6 | executors: 7 | train: 8 | type: catalyst 9 | gpu: 2-4 10 | distr: True 11 | single_node: True 12 | args: 13 | config: catalyst.yml 14 | -------------------------------------------------------------------------------- /examples/digit-recognizer/train.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: mnist_train 3 | project: examples 4 | layout: classify 5 | expdir: . 6 | executors: 7 | train: 8 | type: catalyst 9 | gpu: 1 10 | args: 11 | config: catalyst.yml -------------------------------------------------------------------------------- /examples/grid/Readme.md: -------------------------------------------------------------------------------- 1 | Run 2 | 3 | ```bash 4 | mlcomp dag examples/grid/dag.yml 5 | mlcomp dag examples/grid/task.yml 6 | ``` 7 | 8 | Both dag and task can apply grids. 9 | 10 | Grid applied to DAG: there are multiple dags are being created. 11 | 12 | Grid applied to Task: the dag is the same. There will be many instances of this task. -------------------------------------------------------------------------------- /examples/grid/dag.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: dag_grid 3 | project: examples 4 | expdir: . 5 | executors: 6 | bash: 7 | type: bash 8 | command: echo $variable 9 | bash2: 10 | type: bash 11 | command: echo $variable 12 | depends: bash 13 | grid: 14 | - variable: 0-2 -------------------------------------------------------------------------------- /examples/grid/task.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: bash 3 | project: examples 4 | expdir: . 5 | executors: 6 | bash: 7 | type: bash 8 | command: echo $variable 9 | grid: 10 | - variable: 0-4 -------------------------------------------------------------------------------- /examples/hierarchical_logging/Readme.md: -------------------------------------------------------------------------------- 1 | Run 2 | 3 | ```bash 4 | cd examples/hierarchical_logging 5 | mlcomp dag config.yml 6 | ``` 7 | 8 | Navigate to the tasks panel. Open it and find a new task. 9 | 10 | Click it and open Report tab. You should see something like that: 11 | 12 | ![hierarchical_logging](../../docs/imgs/hierarchical_logging.png) -------------------------------------------------------------------------------- /examples/hierarchical_logging/config.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: step 3 | project: examples 4 | expdir: . 5 | executors: 6 | step: 7 | type: step -------------------------------------------------------------------------------- /examples/hierarchical_logging/executors.py: -------------------------------------------------------------------------------- 1 | from mlcomp.worker.executors import Executor 2 | 3 | 4 | @Executor.register 5 | class Step(Executor): 6 | def work(self): 7 | self.step.start(1, 'step 1') 8 | 9 | self.step.start(1, 'step 2') 10 | 11 | self.step.start(2, 'step 2.1') 12 | 13 | self.step.start(3, 'step 2.1.1') 14 | 15 | self.step.start(3, 'step 2.1.2') 16 | 17 | self.step.start(2, 'step 2.2') 18 | 19 | self.step.end(0) 20 | -------------------------------------------------------------------------------- /examples/kaggle_submit/folder/file.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/examples/kaggle_submit/folder/file.txt -------------------------------------------------------------------------------- /examples/kaggle_submit/submit.yml: -------------------------------------------------------------------------------- 1 | folders: 2 | - folder 3 | competition: deepfake-detection-challenge 4 | max_size: 1 -------------------------------------------------------------------------------- /examples/progress_bar/config.yml: -------------------------------------------------------------------------------- 1 | info: 2 | name: progress_bar 3 | project: examples 4 | expdir: . 5 | executors: 6 | bash: 7 | type: progress -------------------------------------------------------------------------------- /examples/progress_bar/executor.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from mlcomp.worker.executors import Executor 4 | 5 | 6 | @Executor.register 7 | class Progress(Executor): 8 | def work(self): 9 | items = list(range(1000)) 10 | for i in self.tqdm(items, interval=1): 11 | time.sleep(0.01) 12 | -------------------------------------------------------------------------------- /mlcomp/__version__.py: -------------------------------------------------------------------------------- 1 | __version__ = '20.3.1.1' 2 | -------------------------------------------------------------------------------- /mlcomp/bin/redis-server: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/bin/redis-server -------------------------------------------------------------------------------- /mlcomp/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/__init__.py: -------------------------------------------------------------------------------- 1 | from .register import register 2 | 3 | __all__ = ['register'] 4 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/callbacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/catalyst/callbacks/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/bninception.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: bninception} 2 | stages: 3 | data_params: 4 | batch_size: 208 5 | height: 224 6 | mean: [104, 117, 128] 7 | std: [1, 1, 1] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/cafferesnet101.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: cafferesnet101} 2 | stages: 3 | data_params: 4 | batch_size: 64 5 | height: 224 6 | mean: [102.9801, 115.9465, 122.7717] 7 | std: [1, 1, 1] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/densenet121.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: densenet121} 2 | stages: 3 | data_params: 4 | batch_size: 64 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/densenet161.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: densenet161} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/densenet169.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: densenet169} 2 | stages: 3 | data_params: 4 | batch_size: 48 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/densenet201.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: densenet201} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/dpn107.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: dpn107} 2 | stages: 3 | data_params: 4 | batch_size: 16 5 | height: 224 6 | mean: [0.48627450980392156, 0.4588235294117647, 0.40784313725490196] 7 | std: [0.23482446870963955, 0.23482446870963955, 0.23482446870963955] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/dpn131.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: dpn131} 2 | stages: 3 | data_params: 4 | batch_size: 16 5 | height: 224 6 | mean: [0.48627450980392156, 0.4588235294117647, 0.40784313725490196] 7 | std: [0.23482446870963955, 0.23482446870963955, 0.23482446870963955] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/dpn68.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: dpn68} 2 | stages: 3 | data_params: 4 | batch_size: 64 5 | height: 224 6 | mean: [0.48627450980392156, 0.4588235294117647, 0.40784313725490196] 7 | std: [0.23482446870963955, 0.23482446870963955, 0.23482446870963955] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/dpn68b.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: dpn68b} 2 | stages: 3 | data_params: 4 | batch_size: 64 5 | height: 224 6 | mean: [0.48627450980392156, 0.4588235294117647, 0.40784313725490196] 7 | std: [0.23482446870963955, 0.23482446870963955, 0.23482446870963955] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/dpn92.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: dpn92} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 224 6 | mean: [0.48627450980392156, 0.4588235294117647, 0.40784313725490196] 7 | std: [0.23482446870963955, 0.23482446870963955, 0.23482446870963955] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/dpn98.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: dpn98} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 224 6 | mean: [0.48627450980392156, 0.4588235294117647, 0.40784313725490196] 7 | std: [0.23482446870963955, 0.23482446870963955, 0.23482446870963955] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/fbresnet152.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: fbresnet152} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/inceptionresnetv2.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: inceptionresnetv2} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 299 6 | mean: [0.5, 0.5, 0.5] 7 | std: [0.5, 0.5, 0.5] 8 | width: 299 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/inceptionv3.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: inceptionv3} 2 | stages: 3 | data_params: 4 | batch_size: 80 5 | height: 299 6 | mean: [0.5, 0.5, 0.5] 7 | std: [0.5, 0.5, 0.5] 8 | width: 299 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/inceptionv4.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: inceptionv4} 2 | stages: 3 | data_params: 4 | batch_size: 48 5 | height: 299 6 | mean: [0.5, 0.5, 0.5] 7 | std: [0.5, 0.5, 0.5] 8 | width: 299 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/nasnetalarge.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: nasnetalarge} 2 | stages: 3 | data_params: 4 | batch_size: 8 5 | height: 331 6 | mean: [0.5, 0.5, 0.5] 7 | std: [0.5, 0.5, 0.5] 8 | width: 331 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/nasnetamobile.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: nasnetamobile} 2 | stages: 3 | data_params: 4 | batch_size: 96 5 | height: 224 6 | mean: [0.5, 0.5, 0.5] 7 | std: [0.5, 0.5, 0.5] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/pnasnet5large.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: pnasnet5large} 2 | stages: 3 | data_params: 4 | batch_size: 8 5 | height: 331 6 | mean: [0.5, 0.5, 0.5] 7 | std: [0.5, 0.5, 0.5] 8 | width: 331 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/polynet.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: polynet} 2 | stages: 3 | data_params: 4 | batch_size: 8 5 | height: 331 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 331 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/resnet101.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: resnet101} 2 | stages: 3 | data_params: 4 | batch_size: 64 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/resnet152.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: resnet152} 2 | stages: 3 | data_params: 4 | batch_size: 48 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/resnet34.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: resnet34} 2 | stages: 3 | data_params: 4 | batch_size: 232 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/resnet50.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: resnet50} 2 | stages: 3 | data_params: 4 | batch_size: 56 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/resnext101_32x4d.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: resnext101_32x4d} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/resnext101_64x4d.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: resnext101_64x4d} 2 | stages: 3 | data_params: 4 | batch_size: 16 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/se_resnet101.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: se_resnet101} 2 | stages: 3 | data_params: 4 | batch_size: 40 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/se_resnet152.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: se_resnet152} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/se_resnet50.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: se_resnet50} 2 | stages: 3 | data_params: 4 | batch_size: 80 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/se_resnext101_32x4d.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: se_resnext101_32x4d} 2 | stages: 3 | data_params: 4 | batch_size: 40 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/se_resnext50_32x4d.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: se_resnext50_32x4d} 2 | stages: 3 | data_params: 4 | batch_size: 64 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/senet154.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: senet154} 2 | stages: 3 | data_params: 4 | batch_size: 16 5 | height: 224 6 | mean: [0.485, 0.456, 0.406] 7 | std: [0.229, 0.224, 0.225] 8 | width: 224 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/configs/classify/xception.yml: -------------------------------------------------------------------------------- 1 | model_params: {variant: xception} 2 | stages: 3 | data_params: 4 | batch_size: 32 5 | height: 299 6 | mean: [0.5, 0.5, 0.5] 7 | std: [0.5, 0.5, 0.5] 8 | width: 299 9 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/optim/__init__.py: -------------------------------------------------------------------------------- 1 | from .cosineanneal import OneCycleCosineAnnealLR 2 | 3 | __all__ = ['OneCycleCosineAnnealLR'] 4 | -------------------------------------------------------------------------------- /mlcomp/contrib/catalyst/optim/cosineanneal.py: -------------------------------------------------------------------------------- 1 | from torch.optim.lr_scheduler import CosineAnnealingLR 2 | 3 | 4 | class OneCycleCosineAnnealLR(CosineAnnealingLR): 5 | def __init__(self, *args, **kwargs): 6 | self.start_epoch = None 7 | self.last_epoch = None 8 | super().__init__(*args, **kwargs) 9 | 10 | def step(self, epoch=None): 11 | if self.last_epoch is not None: 12 | if self.start_epoch is None: 13 | self.start_epoch = self.last_epoch 14 | self.last_epoch = 0 15 | for i in range(len(self.base_lrs)): 16 | self.optimizer.param_groups[i]['lr'] = self.base_lrs[0] 17 | 18 | if self.last_epoch >= self.T_max - 1: 19 | self.start_epoch = self.last_epoch 20 | self.last_epoch = -1 21 | for i in range(len(self.base_lrs)): 22 | self.optimizer.param_groups[i]['lr'] = self.base_lrs[0] 23 | 24 | super().step(epoch) 25 | 26 | 27 | __all__ = ['OneCycleCosineAnnealLR'] 28 | -------------------------------------------------------------------------------- /mlcomp/contrib/complex/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/complex/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/criterion/__init__.py: -------------------------------------------------------------------------------- 1 | from .ring import RingLoss 2 | 3 | __all__ = ['RingLoss'] 4 | -------------------------------------------------------------------------------- /mlcomp/contrib/criterion/ce.py: -------------------------------------------------------------------------------- 1 | from torch.nn import CrossEntropyLoss 2 | from torch.nn.functional import nll_loss, log_softmax 3 | 4 | 5 | class LabelSmoothingCrossEntropy(CrossEntropyLoss): 6 | def __init__(self, eps: float = 0.1, *args, **kwargs): 7 | super().__init__(*args, **kwargs) 8 | self.eps = eps 9 | 10 | def forward(self, output, target): 11 | c = output.size()[-1] 12 | log_preds = log_softmax(output, dim=-1) 13 | if self.reduction == 'sum': 14 | loss = -log_preds.sum() 15 | else: 16 | loss = -log_preds.sum(dim=-1) 17 | if self.reduction == 'mean': 18 | loss = loss.mean() 19 | nl = nll_loss(log_preds, target, reduction=self.reduction) 20 | return loss * self.eps / c + (1 - self.eps) * nl 21 | 22 | 23 | __all__ = ['LabelSmoothingCrossEntropy'] 24 | -------------------------------------------------------------------------------- /mlcomp/contrib/dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/dataset/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/metrics/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/metrics/dice.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def dice_numpy(targets, outputs, threshold=None, min_area=None, 5 | empty_one: bool = True, eps=1e-6): 6 | if threshold is not None: 7 | # noinspection PyUnresolvedReferences 8 | outputs = (outputs >= threshold).astype(np.uint8) 9 | 10 | targets_sum = targets.sum() 11 | outputs_sum = outputs.sum() 12 | 13 | if min_area and outputs_sum < min_area: 14 | outputs = np.zeros(outputs.shape, dtype=np.uint8) 15 | outputs_sum = 0 16 | 17 | if empty_one and targets_sum == 0 and outputs_sum == 0: 18 | return 1 19 | 20 | intersection = (targets * outputs).sum() 21 | union = targets_sum + outputs_sum 22 | dice = 2 * intersection / (union + eps) 23 | return dice 24 | -------------------------------------------------------------------------------- /mlcomp/contrib/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .pretrained import Pretrained 2 | 3 | __all__ = ['Pretrained'] 4 | -------------------------------------------------------------------------------- /mlcomp/contrib/model/segmentation_model_pytorch.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | 3 | import mlcomp.contrib.segmentation as smb 4 | 5 | 6 | class SegmentationModelPytorch(nn.Module): 7 | def __init__( 8 | self, 9 | arch: str, 10 | encoder: str, 11 | num_classes: int = 1, 12 | encoder_weights: str = 'imagenet', 13 | activation=None, 14 | **kwargs 15 | ): 16 | super().__init__() 17 | 18 | model = getattr(smb, arch) 19 | self.model = model( 20 | encoder_name=encoder, 21 | classes=num_classes, 22 | encoder_weights=encoder_weights, 23 | activation=activation, 24 | **kwargs 25 | ) 26 | 27 | def forward(self, x): 28 | res = self.model.forward(x) 29 | if self.model.activation: 30 | res = self.model.activation(res) 31 | return res 32 | 33 | 34 | __all__ = ['SegmentationModelPytorch'] 35 | -------------------------------------------------------------------------------- /mlcomp/contrib/model/video/__init__.py: -------------------------------------------------------------------------------- 1 | from .resnext3d import ResNeXt3D 2 | -------------------------------------------------------------------------------- /mlcomp/contrib/model/video/resnext3d/__init__.py: -------------------------------------------------------------------------------- 1 | from .resnext3d import ResNeXt3D -------------------------------------------------------------------------------- /mlcomp/contrib/sampler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/sampler/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/sampler/distributed.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from torch.utils.data import DistributedSampler 4 | 5 | 6 | class DistributedSamplerIndices(DistributedSampler): 7 | def __init__(self, sampler, *args, **kwargs): 8 | super().__init__(sampler, num_replicas=None, rank=None, shuffle=True) 9 | self.sampler = sampler 10 | 11 | def get_indices(self): 12 | return list(self.sampler.__iter__()) 13 | 14 | def __iter__(self): 15 | # deterministically shuffle based on epoch 16 | np.random.seed(self.epoch) 17 | 18 | indices = self.get_indices() 19 | 20 | if self.shuffle: 21 | np.random.shuffle(indices) 22 | 23 | # add extra samples to make it evenly divisible 24 | indices += indices[:(self.total_size - len(indices))] 25 | assert len(indices) == self.total_size 26 | 27 | # subsample 28 | indices = indices[self.rank:self.total_size:self.num_replicas] 29 | assert len(indices) == self.num_samples 30 | 31 | return iter(indices) 32 | 33 | 34 | __all__ = ['DistributedSamplerIndices'] 35 | -------------------------------------------------------------------------------- /mlcomp/contrib/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/scripts/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/search/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/search/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/Readme.md: -------------------------------------------------------------------------------- 1 | Source: https://github.com/qubvel/segmentation_models.pytorch -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .unet import Unet 3 | from .linknet import Linknet 4 | from .fpn import FPN 5 | from .pspnet import PSPNet 6 | 7 | from . import encoders -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/base/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .encoder_decoder import EncoderDecoder -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/base/model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class Model(nn.Module): 5 | 6 | def __init__(self): 7 | super().__init__() 8 | 9 | def initialize(self): 10 | for m in self.modules(): 11 | if isinstance(m, nn.Conv2d): 12 | nn.init.kaiming_normal_(m.weight, mode='fan_out', 13 | nonlinearity='relu') 14 | elif isinstance(m, nn.BatchNorm2d): 15 | nn.init.constant_(m.weight, 1) 16 | nn.init.constant_(m.bias, 0) 17 | -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/segmentation/common/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/deeplabv3/Readme.md: -------------------------------------------------------------------------------- 1 | Source: https://github.com/jfzhang95/pytorch-deeplab-xception -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/deeplabv3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/segmentation/deeplabv3/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/deeplabv3/backbone/__init__.py: -------------------------------------------------------------------------------- 1 | from mlcomp.contrib.segmentation.deeplabv3.backbone import resnet, \ 2 | xception, drn, mobilenet 3 | 4 | 5 | def build_backbone(backbone, output_stride, BatchNorm): 6 | if backbone == 'resnet': 7 | return resnet.ResNet101(output_stride, BatchNorm) 8 | elif backbone == 'xception': 9 | return xception.AlignedXception(output_stride, BatchNorm) 10 | elif backbone == 'drn': 11 | return drn.drn_d_54(BatchNorm) 12 | elif backbone == 'mobilenet': 13 | return mobilenet.MobileNetV2(output_stride, BatchNorm) 14 | else: 15 | raise NotImplementedError 16 | -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/encoders/_preprocessing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def preprocess_input(x, mean=None, std=None, input_space='RGB', 5 | input_range=None, **kwargs): 6 | if input_space == 'BGR': 7 | x = x[..., ::-1].copy() 8 | 9 | if input_range is not None: 10 | if x.max() > 1 and input_range[1] == 1: 11 | x = x / 255. 12 | 13 | if mean is not None: 14 | mean = np.array(mean) 15 | x = x - mean 16 | 17 | if std is not None: 18 | std = np.array(std) 19 | x = x / std 20 | 21 | return x 22 | -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/fpn/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .model import FPN -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/linknet/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .model import Linknet -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/pspnet/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .model import PSPNet -------------------------------------------------------------------------------- /mlcomp/contrib/segmentation/unet/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .model import Unet -------------------------------------------------------------------------------- /mlcomp/contrib/split/__init__.py: -------------------------------------------------------------------------------- 1 | from .frame import stratified_group_k_fold, stratified_k_fold 2 | 3 | __all__ = ['stratified_group_k_fold', 'stratified_k_fold'] 4 | -------------------------------------------------------------------------------- /mlcomp/contrib/torch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/torch/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/torch/layers.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class LambdaLayer(nn.Module): 5 | def __init__(self, lambd): 6 | super(LambdaLayer, self).__init__() 7 | self.lambd = lambd 8 | 9 | def forward(self, x): 10 | return self.lambd(x) 11 | 12 | 13 | __all__ = ['LambdaLayer'] 14 | -------------------------------------------------------------------------------- /mlcomp/contrib/torch/tensors.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def flip(x, dim): 5 | indices = [slice(None)] * x.dim() 6 | indices[dim] = torch.arange(x.size(dim) - 1, -1, -1, 7 | dtype=torch.long, device=x.device) 8 | return x[tuple(indices)] 9 | 10 | 11 | __all__ = ['flip'] 12 | -------------------------------------------------------------------------------- /mlcomp/contrib/transform/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/contrib/transform/__init__.py -------------------------------------------------------------------------------- /mlcomp/contrib/transform/albumentations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from albumentations import ImageOnlyTransform 3 | 4 | 5 | class ChannelTranspose(ImageOnlyTransform): 6 | def get_transform_init_args_names(self): 7 | return () 8 | 9 | def get_params_dependent_on_targets(self, params): 10 | pass 11 | 12 | def __init__(self, axes=(2, 0, 1)): 13 | super().__init__(always_apply=True) 14 | self.axes = axes 15 | 16 | def apply(self, img, **params): 17 | return np.transpose(img, self.axes) 18 | 19 | 20 | class Ensure4d(ImageOnlyTransform): 21 | def get_transform_init_args_names(self): 22 | return () 23 | 24 | def get_params_dependent_on_targets(self, params): 25 | pass 26 | 27 | def __init__(self): 28 | super().__init__(always_apply=True) 29 | 30 | def apply(self, img, **params): 31 | while len(img.shape) < 4: 32 | img = img[None] 33 | return img 34 | 35 | 36 | __all__ = ['ChannelTranspose', 'Ensure4d'] 37 | -------------------------------------------------------------------------------- /mlcomp/contrib/transform/rle.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def mask2rle(img): 5 | """ 6 | img: numpy array, 1 - mask, 0 - background 7 | Returns run length as string formatted 8 | """ 9 | pixels = img.T.flatten() 10 | pixels = np.concatenate([[0], pixels, [0]]) 11 | runs = np.where(pixels[1:] != pixels[:-1])[0] + 1 12 | runs[1::2] -= runs[::2] 13 | return ' '.join(str(x) for x in runs) 14 | 15 | 16 | def rle2mask(mask_rle, shape): 17 | """ 18 | mask_rle: run-length as string formatted (start length) 19 | shape: (width,height) of array to return 20 | Returns numpy array, 1 - mask, 0 - background 21 | 22 | """ 23 | s = mask_rle.split() 24 | starts, lengths = [np.asarray(x, dtype=int) for x in 25 | (s[0:][::2], s[1:][::2])] 26 | starts -= 1 27 | ends = starts + lengths 28 | img = np.zeros(shape[0] * shape[1], dtype=np.uint8) 29 | for lo, hi in zip(starts, ends): 30 | img[lo:hi] = 1 31 | return img.reshape(shape).T 32 | 33 | 34 | __all__ = ['mask2rle', 'rle2mask'] 35 | -------------------------------------------------------------------------------- /mlcomp/contrib/transform/tta.py: -------------------------------------------------------------------------------- 1 | import albumentations as A 2 | import numpy as np 3 | 4 | from torch.utils.data import Dataset 5 | 6 | from mlcomp.contrib.torch.tensors import flip 7 | 8 | 9 | class TtaWrap(Dataset): 10 | def __init__(self, dataset: Dataset, tfms=()): 11 | self.dataset = dataset 12 | self.tfms = tfms 13 | 14 | def __getitem__(self, item): 15 | return self.dataset[item] 16 | 17 | def __len__(self): 18 | return len(self.dataset) 19 | 20 | def inverse(self, a: np.array): 21 | last_dim = len(a.shape) - 1 22 | for t in self.tfms: 23 | if isinstance(t, A.HorizontalFlip): 24 | a = flip(a, last_dim) 25 | elif isinstance(t, A.VerticalFlip): 26 | a = flip(a, last_dim - 1) 27 | elif isinstance(t, A.Transpose): 28 | axis = (0, 1, 3, 2) if len(a.shape) == 4 else (0, 2, 1) 29 | a = a.permute(*axis) 30 | 31 | return a 32 | 33 | 34 | __all__ = ['TtaWrap'] 35 | -------------------------------------------------------------------------------- /mlcomp/db/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .signals import * -------------------------------------------------------------------------------- /mlcomp/db/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .db import Session 2 | from .options import PaginatorOptions 3 | 4 | __all__ = ['Session', 'PaginatorOptions'] 5 | -------------------------------------------------------------------------------- /mlcomp/db/core/options.py: -------------------------------------------------------------------------------- 1 | class PaginatorOptions: 2 | def __init__( 3 | self, 4 | page_number: int, 5 | page_size: int, 6 | sort_column: str = None, 7 | sort_descending: bool = None 8 | ): 9 | self.sort_column = sort_column 10 | self.sort_descending = sort_descending 11 | self.page_number = page_number 12 | self.page_size = page_size 13 | 14 | assert (page_number is not None and page_size) \ 15 | or (page_number is not None and not page_size), \ 16 | 'Specify both page_number and page_size' 17 | 18 | if not sort_column: 19 | self.sort_column = 'id' 20 | self.sort_descending = True 21 | 22 | 23 | __all__ = ['PaginatorOptions'] 24 | -------------------------------------------------------------------------------- /mlcomp/db/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .project import Project 2 | from .task import Task, TaskDependence, TaskSynced 3 | from .file import File 4 | from .dag_storage import DagStorage, DagLibrary 5 | from .computer import Computer, ComputerUsage 6 | from .log import Log 7 | from .step import Step 8 | from .dag import Dag, DagTag 9 | from .report import ReportSeries, ReportImg, ReportTasks, Report, ReportLayout 10 | from .docker import Docker 11 | from .model import Model 12 | from .auxilary import Auxiliary 13 | from .memory import Memory 14 | from .space import Space, SpaceTag 15 | 16 | __all__ = [ 17 | 'Project', 'Task', 'TaskDependence', 'File', 'DagStorage', 'DagLibrary', 18 | 'Computer', 'ComputerUsage', 'Log', 'Step', 'Dag', 'ReportSeries', 19 | 'ReportImg', 'ReportTasks', 'Report', 'ReportLayout', 'Docker', 'Model', 20 | 'Auxiliary', 'TaskSynced', 'Memory', 'Space', 'DagTag' 21 | ] 22 | -------------------------------------------------------------------------------- /mlcomp/db/models/auxilary.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | 3 | from mlcomp.db.models.base import Base 4 | 5 | 6 | class Auxiliary(Base): 7 | __tablename__ = 'auxiliary' 8 | 9 | name = sa.Column(sa.String, primary_key=True) 10 | data = sa.Column(sa.String) 11 | 12 | 13 | __all__ = ['Auxiliary'] 14 | -------------------------------------------------------------------------------- /mlcomp/db/models/base.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.ext.declarative import declarative_base 2 | from sqlalchemy_serializer import SerializerMixin 3 | 4 | Base = declarative_base(cls=SerializerMixin) 5 | -------------------------------------------------------------------------------- /mlcomp/db/models/dag_storage.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | from sqlalchemy import ForeignKey 3 | 4 | from mlcomp.db.models.base import Base 5 | 6 | 7 | class DagStorage(Base): 8 | __tablename__ = 'dag_storage' 9 | 10 | id = sa.Column(sa.Integer, primary_key=True) 11 | dag = sa.Column(sa.Integer, ForeignKey('dag.id')) 12 | file = sa.Column(sa.Integer, ForeignKey('file.id')) 13 | path = sa.Column(sa.String) 14 | is_dir = sa.Column(sa.Boolean) 15 | 16 | 17 | class DagLibrary(Base): 18 | __tablename__ = 'dag_library' 19 | 20 | id = sa.Column(sa.Integer, primary_key=True) 21 | dag = sa.Column(sa.Integer, ForeignKey('dag.id')) 22 | library = sa.Column(sa.String) 23 | version = sa.Column(sa.String) 24 | 25 | 26 | __all__ = ['DagStorage', 'DagLibrary'] 27 | -------------------------------------------------------------------------------- /mlcomp/db/models/docker.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | from sqlalchemy import ForeignKey 3 | 4 | from mlcomp.db.models.base import Base 5 | 6 | 7 | class Docker(Base): 8 | __tablename__ = 'docker' 9 | 10 | name = sa.Column(sa.String, primary_key=True) 11 | computer = sa.Column(sa.String, 12 | ForeignKey('computer.name'), 13 | primary_key=True) 14 | last_activity = sa.Column(sa.DateTime, nullable=False) 15 | ports = sa.Column(sa.String, nullable=False) 16 | 17 | 18 | __all__ = ['Docker'] 19 | -------------------------------------------------------------------------------- /mlcomp/db/models/file.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import sqlalchemy as sa 4 | from sqlalchemy import ForeignKey 5 | 6 | from mlcomp.db.models.base import Base 7 | 8 | 9 | class File(Base): 10 | __tablename__ = 'file' 11 | 12 | id = sa.Column(sa.Integer, primary_key=True) 13 | md5 = sa.Column(sa.String) 14 | created = sa.Column(sa.DateTime, default='Now()') 15 | content = sa.Column(sa.LargeBinary) 16 | project = sa.Column(sa.Integer, ForeignKey('project.id')) 17 | dag = sa.Column(sa.Integer, ForeignKey('dag.id')) 18 | size = sa.Column(sa.BigInteger, nullable=False, default=0) 19 | 20 | def __init__(self, **kwargs): 21 | super().__init__(**kwargs) 22 | self.size = sys.getsizeof(self.content) 23 | 24 | 25 | __all__ = ['File'] 26 | -------------------------------------------------------------------------------- /mlcomp/db/models/log.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | from sqlalchemy import ForeignKey 3 | 4 | from mlcomp.db.models.base import Base 5 | 6 | 7 | class Log(Base): 8 | __tablename__ = 'log' 9 | 10 | id = sa.Column(sa.Integer, primary_key=True) 11 | step = sa.Column(sa.Integer, ForeignKey('step.id')) 12 | message = sa.Column(sa.String) 13 | time = sa.Column(sa.DateTime) 14 | level = sa.Column(sa.Integer) 15 | component = sa.Column(sa.Integer) 16 | module = sa.Column(sa.String) 17 | line = sa.Column(sa.Integer) 18 | task = sa.Column(sa.Integer, ForeignKey('task.id')) 19 | computer = sa.Column(sa.String, ForeignKey('computer.name')) 20 | 21 | 22 | __all__ = ['Log'] 23 | -------------------------------------------------------------------------------- /mlcomp/db/models/memory.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | 3 | from mlcomp.db.models.base import Base 4 | 5 | 6 | class Memory(Base): 7 | __tablename__ = 'memory' 8 | 9 | id = sa.Column(sa.Integer, primary_key=True) 10 | model = sa.Column(sa.String, nullable=False) 11 | variant = sa.Column(sa.String) 12 | num_classes = sa.Column(sa.Integer) 13 | img_size = sa.Column(sa.Integer) 14 | batch_size = sa.Column(sa.Integer, nullable=False) 15 | memory = sa.Column(sa.Float, nullable=False) 16 | 17 | 18 | __all__ = ['Memory'] 19 | -------------------------------------------------------------------------------- /mlcomp/db/models/model.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | from sqlalchemy import ForeignKey 3 | from sqlalchemy.orm import relationship 4 | 5 | from mlcomp.db.models.base import Base 6 | 7 | 8 | class Model(Base): 9 | __tablename__ = 'model' 10 | 11 | id = sa.Column(sa.Integer, primary_key=True) 12 | name = sa.Column(sa.String) 13 | score_local = sa.Column(sa.Float) 14 | score_public = sa.Column(sa.Float) 15 | project = sa.Column(sa.Integer, ForeignKey('project.id')) 16 | dag = sa.Column(sa.Integer, ForeignKey('dag.id')) 17 | created = sa.Column(sa.DateTime) 18 | equations = sa.Column(sa.String) 19 | fold = sa.Column(sa.Integer) 20 | 21 | dag_rel = relationship('Dag', lazy='noload') 22 | project_rel = relationship('Project', lazy='noload') 23 | 24 | 25 | __all__ = ['Model'] 26 | -------------------------------------------------------------------------------- /mlcomp/db/models/project.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | 3 | from mlcomp.db.models.base import Base 4 | 5 | 6 | class Project(Base): 7 | __tablename__ = 'project' 8 | 9 | id = sa.Column(sa.Integer, primary_key=True) 10 | name = sa.Column(sa.String, nullable=False) 11 | class_names = sa.Column(sa.String, nullable=False) 12 | sync_folders = sa.Column(sa.String, nullable=False) 13 | ignore_folders = sa.Column(sa.String, nullable=False) 14 | 15 | 16 | __all__ = ['Project'] 17 | -------------------------------------------------------------------------------- /mlcomp/db/models/space.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | from sqlalchemy import ForeignKey 3 | 4 | from mlcomp.db.models.base import Base 5 | 6 | 7 | class Space(Base): 8 | __tablename__ = 'space' 9 | 10 | name = sa.Column(sa.String, nullable=False, primary_key=True) 11 | created = sa.Column(sa.DateTime, nullable=False) 12 | changed = sa.Column(sa.DateTime, nullable=False) 13 | content = sa.Column(sa.String, nullable=False) 14 | 15 | 16 | class SpaceRelation(Base): 17 | __tablename__ = 'space_relation' 18 | 19 | parent = sa.Column(sa.String, ForeignKey('space.name'), 20 | primary_key=True) 21 | child = sa.Column(sa.String, ForeignKey('space.name'), 22 | primary_key=True) 23 | 24 | 25 | class SpaceTag(Base): 26 | __tablename__ = 'space_tag' 27 | 28 | space = sa.Column(sa.String, ForeignKey('space.name'), primary_key=True) 29 | tag = sa.Column(sa.String, primary_key=True) 30 | 31 | 32 | __all__ = ['Space', 'SpaceRelation', 'SpaceTag'] 33 | -------------------------------------------------------------------------------- /mlcomp/db/models/step.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | from sqlalchemy import ForeignKey 3 | from sqlalchemy.orm import relationship 4 | 5 | from mlcomp.db.models.base import Base 6 | 7 | 8 | class Step(Base): 9 | __tablename__ = 'step' 10 | 11 | id = sa.Column(sa.Integer, primary_key=True) 12 | level = sa.Column(sa.Integer) 13 | task = sa.Column(sa.Integer, ForeignKey('task.id')) 14 | started = sa.Column(sa.DateTime) 15 | finished = sa.Column(sa.DateTime) 16 | name = sa.Column(sa.String) 17 | task_rel = relationship('Task', lazy='noload') 18 | index = sa.Column(sa.Integer) 19 | 20 | 21 | __all__ = ['Step'] 22 | -------------------------------------------------------------------------------- /mlcomp/db/providers/auxiliary.py: -------------------------------------------------------------------------------- 1 | from mlcomp.db.models import Auxiliary 2 | from mlcomp.db.providers.base import BaseDataProvider 3 | from mlcomp.utils.io import yaml_load 4 | 5 | 6 | class AuxiliaryProvider(BaseDataProvider): 7 | model = Auxiliary 8 | 9 | def get(self): 10 | query = self.query(self.model) 11 | res = dict() 12 | for r in query.all(): 13 | res[r.name] = yaml_load(r.data) 14 | res[r.name] = self.serializer(res[r.name]) 15 | return res 16 | 17 | 18 | __all__ = ['AuxiliaryProvider'] 19 | -------------------------------------------------------------------------------- /mlcomp/db/providers/dag_storage.py: -------------------------------------------------------------------------------- 1 | from mlcomp.db.models import DagStorage, File, DagLibrary 2 | from mlcomp.db.providers.base import BaseDataProvider 3 | 4 | 5 | class DagStorageProvider(BaseDataProvider): 6 | model = DagStorage 7 | 8 | def by_dag(self, dag: int): 9 | query = self.query(DagStorage, File).join(File, isouter=True). \ 10 | filter(DagStorage.dag == dag). \ 11 | order_by(DagStorage.path) 12 | return query.all() 13 | 14 | 15 | class DagLibraryProvider(BaseDataProvider): 16 | model = DagLibrary 17 | 18 | def dag(self, dag: int): 19 | return self.query(DagLibrary.library, DagLibrary.version). \ 20 | filter(DagLibrary.dag == dag).all() 21 | 22 | 23 | __all__ = ['DagStorageProvider', 'DagLibraryProvider'] 24 | -------------------------------------------------------------------------------- /mlcomp/db/providers/docker.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from mlcomp.db.models import Docker 4 | from mlcomp.db.providers.base import BaseDataProvider 5 | from mlcomp.utils.misc import now 6 | 7 | 8 | class DockerProvider(BaseDataProvider): 9 | model = Docker 10 | 11 | def get(self, computer: str, name: str): 12 | return self.query(Docker). \ 13 | filter(Docker.computer == computer). \ 14 | filter(Docker.name == name). \ 15 | one() 16 | 17 | def get_online(self): 18 | min_activity = now() - datetime.timedelta(seconds=30) 19 | return self.query(Docker). \ 20 | filter(Docker.last_activity >= min_activity). \ 21 | all() 22 | 23 | def queues_online(self): 24 | res = [] 25 | for docker in self.get_online(): 26 | res.append((docker.computer, 27 | f'{docker.computer}_{docker.name}_supervisor')) 28 | return res 29 | 30 | 31 | __all__ = ['DockerProvider'] 32 | -------------------------------------------------------------------------------- /mlcomp/db/providers/report/__init__.py: -------------------------------------------------------------------------------- 1 | from .img import ReportImgProvider 2 | from .report import ReportProvider 3 | from .layout import ReportLayoutProvider 4 | from .series import ReportSeriesProvider 5 | from .task import ReportTasksProvider 6 | 7 | __all__ = [ 8 | 'ReportImgProvider', 'ReportProvider', 'ReportLayoutProvider', 9 | 'ReportSeriesProvider', 'ReportTasksProvider' 10 | ] 11 | -------------------------------------------------------------------------------- /mlcomp/db/providers/report/task.py: -------------------------------------------------------------------------------- 1 | from mlcomp.db.models import ReportTasks 2 | from mlcomp.db.providers.base import BaseDataProvider 3 | 4 | 5 | class ReportTasksProvider(BaseDataProvider): 6 | model = ReportTasks 7 | 8 | 9 | __all__ = ['ReportTasksProvider'] 10 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/__init__.py: -------------------------------------------------------------------------------- 1 | from .f1 import ReportLayoutF1 2 | from .img_classify import ReportLayoutImgClassify 3 | from .info import ReportLayoutInfo 4 | from .metric import ReportLayoutMetric 5 | from .precision_recall import ReportLayoutPrecisionRecall 6 | from .series import ReportLayoutSeries 7 | from .item import ReportLayoutItem 8 | 9 | __all__ = [ 10 | 'ReportLayoutF1', 'ReportLayoutImgClassify', 'ReportLayoutInfo', 11 | 'ReportLayoutMetric', 'ReportLayoutPrecisionRecall', 'ReportLayoutSeries', 12 | 'ReportLayoutItem' 13 | ] 14 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/f1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from sklearn.metrics import classification_report 4 | 5 | from mlcomp.db.report_info.item import ReportLayoutItem 6 | from mlcomp.utils.plot import figure_to_binary, plot_classification_report 7 | 8 | 9 | class ReportLayoutF1(ReportLayoutItem): 10 | def plot(self, y: np.array, pred: np.array): 11 | report = classification_report(y, pred) 12 | fig = plot_classification_report(report) 13 | return figure_to_binary(fig, dpi=70) 14 | 15 | 16 | __all__ = ['ReportLayoutF1'] 17 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/img_classify.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from mlcomp.db.report_info.item import ReportLayoutItem 4 | 5 | 6 | class ReportLayoutImgClassify(ReportLayoutItem): 7 | def __init__( 8 | self, 9 | name: str, 10 | confusion_matrix: bool, 11 | ): 12 | super().__init__(name) 13 | 14 | self.confusion_matrix = confusion_matrix 15 | 16 | @classmethod 17 | def from_dict(cls, name: str, value: OrderedDict): 18 | value.pop('type') 19 | confusion_matrix = value.pop('confusion_matrix', False) 20 | 21 | assert len(value) == 0, f'Unknown parameter in ' \ 22 | f'report.img_classify={value.popitem()}' 23 | return cls( 24 | name, 25 | confusion_matrix=confusion_matrix 26 | ) 27 | 28 | 29 | __all__ = ['ReportLayoutImgClassify'] 30 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/img_segment.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from mlcomp.db.report_info.item import ReportLayoutItem 4 | 5 | 6 | class ReportLayoutImgSegment(ReportLayoutItem): 7 | def __init__( 8 | self, 9 | name: str, 10 | max_height: int, 11 | max_width: int 12 | ): 13 | super().__init__(name) 14 | 15 | self.max_height = max_height 16 | self.max_width = max_width 17 | 18 | @classmethod 19 | def from_dict(cls, name: str, value: OrderedDict): 20 | value.pop('type') 21 | max_height = value.pop('max_height', None) 22 | max_width = value.pop('max_width', None) 23 | 24 | assert len(value) == 0, f'Unknown parameter in ' \ 25 | f'report.img_segment={value.popitem()}' 26 | 27 | return cls( 28 | name, 29 | max_height=max_height, 30 | max_width=max_width 31 | ) 32 | 33 | 34 | __all__ = ['ReportLayoutImgSegment'] 35 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/item.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | 4 | class ReportLayoutItem: 5 | def __init__(self, name: str): 6 | self.name = name 7 | 8 | @classmethod 9 | def from_dict(cls, name: str, value: OrderedDict): 10 | value.pop('type') 11 | assert len(value) == 0, f'Unknown parameter in ' \ 12 | f'report info item = {name}: {value.popitem()}' 13 | return cls(name) 14 | 15 | 16 | __all__ = ['ReportLayoutItem'] 17 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/metric.py: -------------------------------------------------------------------------------- 1 | class ReportLayoutMetric: 2 | def __init__(self, name: str, minimize: bool): 3 | self.name = name 4 | self.minimize = minimize 5 | 6 | @staticmethod 7 | def from_dict(data: dict): 8 | name = data.pop('name') 9 | minimize = data.pop('minimize') 10 | assert len(data) == 0, f'Unknown parameter in ' \ 11 | f'report.metric={data.popitem()}' 12 | return ReportLayoutMetric(name, minimize) 13 | 14 | def serialize(self): 15 | return {'minimize': self.minimize, 'name': self.name} 16 | 17 | 18 | __all__ = ['ReportLayoutMetric'] 19 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/precision_recall.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from sklearn.metrics import precision_recall_curve 5 | 6 | from mlcomp.db.report_info.item import ReportLayoutItem 7 | from mlcomp.utils.plot import figure_to_binary 8 | 9 | 10 | class ReportLayoutPrecisionRecall(ReportLayoutItem): 11 | def plot(self, y: np.array, pred: np.array): 12 | p, r, t = precision_recall_curve(y, pred) 13 | fig, ax = plt.subplots(figsize=(4.2, 2.7)) 14 | ax2 = ax.twinx() 15 | 16 | t = np.hstack([t, t[-1]]) 17 | 18 | ax.plot(r, p) 19 | 20 | ax.set_xlabel('Recall') 21 | ax.set_ylabel('Precision') 22 | ax.set_ylim([0.0, 1.05]) 23 | ax.set_xlim([0.0, 1.0]) 24 | ax2.set_ylabel('Threashold') 25 | ax2.plot(r, t, c='red') 26 | return figure_to_binary(fig) 27 | 28 | 29 | __all__ = ['ReportLayoutPrecisionRecall'] 30 | -------------------------------------------------------------------------------- /mlcomp/db/report_info/series.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from mlcomp.db.report_info.item import ReportLayoutItem 4 | 5 | 6 | class ReportLayoutSeries(ReportLayoutItem): 7 | def __init__(self, name: str, key: str): 8 | super().__init__(name) 9 | 10 | self.key = key 11 | 12 | @classmethod 13 | def from_dict(cls, name: str, value: OrderedDict): 14 | assert 'key' in value, f'report.series={name}. key is required' 15 | value.pop('type') 16 | key = value.pop('key') 17 | 18 | assert len(value) == 0, f'Unknown parameter in ' \ 19 | f'report.series={name}: {value.popitem()}' 20 | return cls(name, key) 21 | 22 | 23 | __all__ = ['ReportLayoutSeries'] 24 | -------------------------------------------------------------------------------- /mlcomp/db/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/db/tests/__init__.py -------------------------------------------------------------------------------- /mlcomp/db/tests/test_project.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # noinspection PyUnresolvedReferences 3 | from mlcomp.utils.tests import session 4 | from mlcomp.db.core import Session 5 | from mlcomp.db.providers import ProjectProvider 6 | 7 | 8 | class TestProject(object): 9 | 10 | def _configure(self, session): 11 | provider = ProjectProvider(session) 12 | provider.add_project(name='test') 13 | return provider 14 | 15 | def test_add(self, session: Session): 16 | provider = ProjectProvider(session) 17 | project = provider.add_project(name='test') 18 | assert provider.by_id(project.id) 19 | 20 | def test_by_name(self, session: Session): 21 | provider = self._configure(session) 22 | project = provider.by_name('test') 23 | assert project is not None 24 | 25 | def test_get(self, session: Session): 26 | provider = self._configure(session) 27 | res = provider.get() 28 | assert len(res['data']) == 1 29 | assert res['total'] == 1 30 | -------------------------------------------------------------------------------- /mlcomp/docker/.env: -------------------------------------------------------------------------------- 1 | ROOT_FOLDER=~/mlcomp 2 | TOKEN=5d2a7f73-75f8-4304-98d1-93bf7a65dc8f 3 | DB_TYPE=SQLITE 4 | POSTGRES_DB=mlcomp 5 | POSTGRES_USER=mlcomp 6 | POSTGRES_PASSWORD=12345 7 | POSTGRES_HOST=localhost 8 | POSTGRES_PORT=5432 9 | PGDATA=/var/lib/postgresql/data/pgdata 10 | REDIS_HOST=localhost 11 | REDIS_PORT=6379 12 | REDIS_PASSWORD=12345 13 | WEB_HOST=0.0.0.0 14 | WEB_PORT=4201 15 | CONSOLE_LOG_LEVEL=INFO 16 | FILE_LOG_LEVEL=INFO 17 | DB_LOG_LEVEL=INFO 18 | IP=192.168.0.1 19 | PORT=22 20 | MASTER_PORT_RANGE=29500-29510 21 | NCCL_SOCKET_IFNAME=eth0 22 | FILE_SYNC_INTERVAL=0 23 | WORKER_USAGE_INTERVAL=10 24 | INSTALL_DEPENDENCIES=False 25 | SYNC_WITH_THIS_COMPUTER=True 26 | CAN_PROCESS_TASKS=True -------------------------------------------------------------------------------- /mlcomp/docker/Readme.md: -------------------------------------------------------------------------------- 1 | PYTHONPATH=. python mlcomp/worker/__main__.py supervisor 2 | 3 | PYTHONPATH=. python2 /usr/bin/supervisord -c docker/supervisord.conf -e INFO 4 | 5 | **Test it into docker**: 6 | 7 | Worker: 8 | 9 | docker build -f docker/worker -t mlcomp-worker . 10 | 11 | docker run --net=host -v /opt/mlcomp/:/opt/mlcomp -it mlcomp-worker /bin/bash 12 | 13 | docker-compose -f docker/worker-compose.yml up --build 14 | 15 | Server: 16 | 17 | docker build -f docker/server -t mlcomp-server . 18 | 19 | docker run --net=host -it mlcomp-server /bin/bash 20 | 21 | docker-compose -f docker/server-compose.yml up --build -------------------------------------------------------------------------------- /mlcomp/docker/server: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | RUN pip install mlcomp 4 | 5 | # Set the locale 6 | ENV LANG C.UTF-8 7 | ENV LC_ALL C.UTF-8 8 | 9 | CMD ["mlcomp-server", "start-site"] -------------------------------------------------------------------------------- /mlcomp/docker/server-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | postgres: 3 | container_name: postgres 4 | env_file: .env 5 | image: postgres:11.1 6 | restart: always 7 | volumes: [ 8 | '$ROOT_FOLDER/db/pgdata/data:/var/lib/postgresql/data/pgdata', 9 | '$ROOT_FOLDER/db/pgdata/backups:/backups' 10 | ] 11 | shm_size: 4G 12 | ports: 13 | - 5432:5432 14 | redis: 15 | command: 'redis-server --requirepass $REDIS_PASSWORD' 16 | container_name: redis 17 | image: redis:5.0.4 18 | restart: always 19 | ports: 20 | - 6379:6379 21 | mlcomp-server: 22 | container_name: mlcomp-server 23 | restart: always 24 | build: 25 | context: . 26 | dockerfile: server 27 | links: 28 | - postgres 29 | - redis 30 | environment: 31 | - ROOT_FOLDER=/root/mlcomp 32 | - EXTRA_ENV=REDIS_HOST=redis;POSTGRES_HOST=postgres;LOG_NAME=server 33 | volumes: [ 34 | '$ROOT_FOLDER:/root/mlcomp' 35 | ] 36 | depends_on: 37 | - postgres 38 | - redis 39 | ports: 40 | - 4201:4201 41 | version: '2.3' -------------------------------------------------------------------------------- /mlcomp/docker/worker: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | build-essential \ 5 | libsm6 \ 6 | libxext6 \ 7 | libfontconfig1 \ 8 | libxrender1 \ 9 | libswscale-dev \ 10 | libtbb2 \ 11 | libtbb-dev \ 12 | libjpeg-dev \ 13 | libpng-dev \ 14 | libtiff-dev \ 15 | libjasper-dev \ 16 | libavformat-dev \ 17 | libpq-dev \ 18 | libturbojpeg \ 19 | software-properties-common \ 20 | nano \ 21 | wget \ 22 | psmisc \ 23 | python3-dev \ 24 | python3-setuptools \ 25 | && apt-get clean \ 26 | && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 27 | 28 | RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && \ 29 | apt-get update && apt-get install -y python3.6 python3.6-dev python3-pip 30 | 31 | RUN ln -sfn /usr/bin/python3.6 /usr/bin/python3 && ln -sfn /usr/bin/python3 /usr/bin/python && ln -sfn /usr/bin/pip3 /usr/bin/pip 32 | 33 | RUN apt install git -y 34 | 35 | RUN pip install mlcomp 36 | 37 | # Set the locale 38 | ENV LANG C.UTF-8 39 | ENV LC_ALL C.UTF-8 40 | 41 | CMD ["mlcomp-worker", "start"] 42 | -------------------------------------------------------------------------------- /mlcomp/docker/worker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | mlcomp-worker: 3 | container_name: mlcomp-worker 4 | restart: always 5 | build: 6 | context: ../ 7 | dockerfile: docker/worker 8 | env_file: .env 9 | environment: 10 | - NVIDIA_VISIBLE_DEVICES=all 11 | runtime: nvidia 12 | volumes: ["/opt/mlcomp:/opt/mlcomp"] 13 | version: '2.3' -------------------------------------------------------------------------------- /mlcomp/migration/Readme.md: -------------------------------------------------------------------------------- 1 | This is a database migration repository. 2 | 3 | More information at 4 | http://code.google.com/p/sqlalchemy-migrate/ 5 | 6 | 7 | python manage.py version_control - To create the migration table 8 | python manage.py upgrade - To upgrade DB -------------------------------------------------------------------------------- /mlcomp/migration/__init__.py: -------------------------------------------------------------------------------- 1 | # template repository default module 2 | -------------------------------------------------------------------------------- /mlcomp/migration/manage.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import migrate.versioning.api as api 4 | from migrate.exceptions import DatabaseAlreadyControlledError 5 | 6 | from mlcomp import SA_CONNECTION_STRING 7 | 8 | 9 | def migrate(connection_string: str = None): 10 | folder = os.path.dirname(__file__) 11 | connection_string = connection_string or SA_CONNECTION_STRING 12 | try: 13 | api.version_control(url=connection_string, repository=folder) 14 | except DatabaseAlreadyControlledError: 15 | pass 16 | 17 | api.upgrade(url=connection_string, repository=folder) 18 | 19 | 20 | __all__ = ['migrate'] 21 | -------------------------------------------------------------------------------- /mlcomp/migration/versions/002/report_layout/base.yml: -------------------------------------------------------------------------------- 1 | extend: base_time 2 | 3 | layout: 4 | - type: panel 5 | title: main 6 | parent_cols: 1 7 | row_height: 300 8 | items: 9 | - type: table 10 | source: [loss] 11 | cols: 1 12 | - type: series 13 | source: _other -------------------------------------------------------------------------------- /mlcomp/migration/versions/002/report_layout/base_time.yml: -------------------------------------------------------------------------------- 1 | layout: 2 | - type: panel 3 | title: base 4 | expanded: False 5 | parent_cols: 2 6 | row_height: 400 7 | items: 8 | - type: series 9 | source: _timer/_fps 10 | - type: series 11 | source: _timer/data_time 12 | - type: series 13 | source: _timer/model_time 14 | - type: series 15 | source: _timer/batch_time -------------------------------------------------------------------------------- /mlcomp/migration/versions/002/report_layout/classify.yml: -------------------------------------------------------------------------------- 1 | extend: base_time 2 | 3 | metric: 4 | name: accuracy01 5 | minimize: False 6 | 7 | layout: 8 | - type: panel 9 | title: main 10 | parent_cols: 1 11 | row_height: 300 12 | items: 13 | - type: table 14 | source: [accuracy01, loss] 15 | cols: 1 16 | - type: series 17 | source: accuracy01 18 | multi: True 19 | - type: series 20 | source: loss 21 | multi: True 22 | - type: series 23 | source: _other -------------------------------------------------------------------------------- /mlcomp/migration/versions/002/report_layout/img-classify.yml: -------------------------------------------------------------------------------- 1 | metric: 2 | name: accuracy 3 | minimize: False 4 | 5 | items: 6 | accuracy: 7 | key: accuracy 8 | type: series 9 | img_classify: 10 | type: img_classify 11 | confusion_matrix: True 12 | 13 | layout: 14 | - type: table 15 | source: [accuracy] 16 | - type: img_classify 17 | source: img_classify 18 | attrs: 19 | - type: int 20 | source: attr1 21 | name: y_pred 22 | - type: int 23 | source: attr2 24 | name: y -------------------------------------------------------------------------------- /mlcomp/migration/versions/002/report_layout/segment.yml: -------------------------------------------------------------------------------- 1 | extend: base_time 2 | 3 | metric: 4 | name: dice 5 | minimize: False 6 | 7 | layout: 8 | - type: panel 9 | title: table 10 | table: True 11 | parent_cols: 1 12 | items: 13 | - type: table 14 | source: [loss, dice] 15 | 16 | - type: panel 17 | title: main 18 | parent_cols: 1 19 | row_height: 400 20 | items: 21 | - type: series 22 | source: loss 23 | multi: True 24 | - type: series 25 | source: dice 26 | multi: True 27 | - type: series 28 | source: _other -------------------------------------------------------------------------------- /mlcomp/migration/versions/003_task_loss.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, MetaData, Float 2 | 3 | meta = MetaData() 4 | 5 | 6 | def upgrade(migrate_engine): 7 | conn = migrate_engine.connect() 8 | trans = conn.begin() 9 | 10 | try: 11 | meta.bind = conn 12 | 13 | task = Table('task', meta, autoload=True) 14 | task_loss = Column('loss', Float) 15 | task_loss.create(task) 16 | except Exception: 17 | trans.rollback() 18 | raise 19 | else: 20 | trans.commit() 21 | 22 | 23 | def downgrade(migrate_engine): 24 | conn = migrate_engine.connect() 25 | trans = conn.begin() 26 | 27 | try: 28 | meta.bind = conn 29 | 30 | task = Table('task', meta, autoload=True) 31 | task.c.loss.drop() 32 | except Exception: 33 | trans.rollback() 34 | raise 35 | else: 36 | trans.commit() 37 | -------------------------------------------------------------------------------- /mlcomp/migration/versions/004_task_continued.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, MetaData, Boolean 2 | 3 | meta = MetaData() 4 | 5 | 6 | def upgrade(migrate_engine): 7 | conn = migrate_engine.connect() 8 | trans = conn.begin() 9 | 10 | try: 11 | meta.bind = conn 12 | 13 | task = Table('task', meta, autoload=True) 14 | col = Column('continued', Boolean) 15 | col.create(task) 16 | except Exception: 17 | trans.rollback() 18 | raise 19 | else: 20 | trans.commit() 21 | 22 | 23 | def downgrade(migrate_engine): 24 | conn = migrate_engine.connect() 25 | trans = conn.begin() 26 | 27 | try: 28 | meta.bind = conn 29 | 30 | task = Table('task', meta, autoload=True) 31 | task.c.continued.drop() 32 | except Exception: 33 | trans.rollback() 34 | raise 35 | else: 36 | trans.commit() 37 | -------------------------------------------------------------------------------- /mlcomp/migration/versions/005_project_sync_folders.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, MetaData, String 2 | 3 | meta = MetaData() 4 | 5 | 6 | def upgrade(migrate_engine): 7 | conn = migrate_engine.connect() 8 | trans = conn.begin() 9 | 10 | try: 11 | meta.bind = conn 12 | 13 | table = Table('project', meta, autoload=True) 14 | col = Column('sync_folders', String(8000)) 15 | col.create(table) 16 | except Exception: 17 | trans.rollback() 18 | raise 19 | else: 20 | trans.commit() 21 | 22 | 23 | def downgrade(migrate_engine): 24 | conn = migrate_engine.connect() 25 | trans = conn.begin() 26 | 27 | try: 28 | meta.bind = conn 29 | 30 | table = Table('project', meta, autoload=True) 31 | table.c.continued.drop() 32 | except Exception: 33 | trans.rollback() 34 | raise 35 | else: 36 | trans.commit() 37 | -------------------------------------------------------------------------------- /mlcomp/migration/versions/006_space.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, MetaData, String, TIMESTAMP 2 | 3 | meta = MetaData() 4 | 5 | 6 | table = Table( 7 | 'space', meta, 8 | Column('name', String(200), primary_key=True), 9 | Column('changed', TIMESTAMP, nullable=False), 10 | Column('created', TIMESTAMP, nullable=False), 11 | Column('content', String(10000), nullable=False), 12 | ) 13 | 14 | 15 | def upgrade(migrate_engine): 16 | conn = migrate_engine.connect() 17 | trans = conn.begin() 18 | 19 | try: 20 | meta.bind = conn 21 | table.create() 22 | except Exception: 23 | trans.rollback() 24 | raise 25 | else: 26 | trans.commit() 27 | 28 | 29 | def downgrade(migrate_engine): 30 | conn = migrate_engine.connect() 31 | trans = conn.begin() 32 | 33 | try: 34 | meta.bind = conn 35 | table.drop() 36 | except Exception: 37 | trans.rollback() 38 | raise 39 | else: 40 | trans.commit() 41 | -------------------------------------------------------------------------------- /mlcomp/migration/versions/008_space_tag.py: -------------------------------------------------------------------------------- 1 | from migrate import ForeignKeyConstraint 2 | from sqlalchemy import Table, Column, MetaData, String 3 | 4 | meta = MetaData() 5 | 6 | table = Table( 7 | 'space_tag', meta, 8 | Column('space', String(200), primary_key=True), 9 | Column('tag', String(100), primary_key=True), 10 | ) 11 | 12 | 13 | def upgrade(migrate_engine): 14 | conn = migrate_engine.connect() 15 | trans = conn.begin() 16 | 17 | try: 18 | meta.bind = conn 19 | table.create() 20 | 21 | space = Table('space', meta, autoload=True) 22 | ForeignKeyConstraint([table.c.space], [space.c.name], 23 | ondelete='CASCADE').create() 24 | except Exception: 25 | trans.rollback() 26 | raise 27 | else: 28 | trans.commit() 29 | 30 | 31 | def downgrade(migrate_engine): 32 | conn = migrate_engine.connect() 33 | trans = conn.begin() 34 | 35 | try: 36 | meta.bind = conn 37 | table.drop() 38 | except Exception: 39 | trans.rollback() 40 | raise 41 | else: 42 | trans.commit() 43 | -------------------------------------------------------------------------------- /mlcomp/migration/versions/009_dag_tag.py: -------------------------------------------------------------------------------- 1 | from migrate import ForeignKeyConstraint 2 | from sqlalchemy import Table, Column, MetaData, String, Integer 3 | 4 | meta = MetaData() 5 | 6 | table = Table( 7 | 'dag_tag', meta, 8 | Column('dag', Integer, primary_key=True), 9 | Column('tag', String(100), primary_key=True), 10 | ) 11 | 12 | 13 | def upgrade(migrate_engine): 14 | conn = migrate_engine.connect() 15 | trans = conn.begin() 16 | 17 | try: 18 | meta.bind = conn 19 | table.create() 20 | 21 | dag = Table('dag', meta, autoload=True) 22 | ForeignKeyConstraint([table.c.dag], [dag.c.id], 23 | ondelete='CASCADE').create() 24 | except Exception: 25 | trans.rollback() 26 | raise 27 | else: 28 | trans.commit() 29 | 30 | 31 | def downgrade(migrate_engine): 32 | conn = migrate_engine.connect() 33 | trans = conn.begin() 34 | 35 | try: 36 | meta.bind = conn 37 | table.drop() 38 | except Exception: 39 | trans.rollback() 40 | raise 41 | else: 42 | trans.commit() 43 | -------------------------------------------------------------------------------- /mlcomp/migration/versions/010_memory.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, MetaData, String, Integer, Float 2 | 3 | meta = MetaData() 4 | 5 | table = Table( 6 | 'memory', meta, 7 | Column('id', Integer, primary_key=True), 8 | Column('model', String(200), nullable=False), 9 | Column('memory', Float, nullable=False), 10 | Column('batch_size', Integer, nullable=False), 11 | Column('num_classes', Integer), 12 | Column('variant', String(200)), 13 | Column('img_size', Integer), 14 | ) 15 | 16 | 17 | def upgrade(migrate_engine): 18 | conn = migrate_engine.connect() 19 | trans = conn.begin() 20 | 21 | try: 22 | meta.bind = conn 23 | table.create() 24 | except Exception: 25 | trans.rollback() 26 | raise 27 | else: 28 | trans.commit() 29 | 30 | 31 | def downgrade(migrate_engine): 32 | conn = migrate_engine.connect() 33 | trans = conn.begin() 34 | 35 | try: 36 | meta.bind = conn 37 | table.drop() 38 | except Exception: 39 | trans.rollback() 40 | raise 41 | else: 42 | trans.commit() 43 | -------------------------------------------------------------------------------- /mlcomp/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/__init__.py -------------------------------------------------------------------------------- /mlcomp/server/back/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/back/__init__.py -------------------------------------------------------------------------------- /mlcomp/server/back/create_dags/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .standard import dag_standard 3 | from .pipe import dag_pipe 4 | from .model_add import dag_model_add 5 | from .model_start import dag_model_start 6 | -------------------------------------------------------------------------------- /mlcomp/server/back/create_dags/pipe.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from mlcomp.db.core import Session 4 | from mlcomp.db.enums import DagType 5 | from mlcomp.db.providers import DagProvider, ProjectProvider, ModelProvider 6 | from mlcomp.db.models import Dag 7 | from mlcomp.worker.storage import Storage 8 | 9 | 10 | def dag_pipe(session: Session, config: dict, config_text: str = None): 11 | assert 'pipes' in config, 'pipe missed' 12 | 13 | info = config['info'] 14 | 15 | storage = Storage(session) 16 | dag_provider = DagProvider(session) 17 | 18 | folder = os.getcwd() 19 | project = ProjectProvider(session).by_name(info['project']).id 20 | dag = dag_provider.add( 21 | Dag( 22 | config=config_text, 23 | project=project, 24 | name=info['name'], 25 | docker_img=info.get('docker_img'), 26 | type=DagType.Pipe.value 27 | ) 28 | ) 29 | storage.upload(folder, dag) 30 | 31 | # Change model dags which have the same name 32 | ModelProvider(session 33 | ).change_dag(project=project, name=info['name'], to=dag.id) 34 | 35 | 36 | __all__ = ['dag_pipe'] 37 | -------------------------------------------------------------------------------- /mlcomp/server/front/e2e/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // Protractor configuration file, see link for more information 2 | // https://github.com/angular/protractor/blob/master/lib/config.ts 3 | 4 | const { SpecReporter } = require('jasmine-spec-reporter'); 5 | 6 | exports.config = { 7 | allScriptsTimeout: 11000, 8 | specs: [ 9 | './src/**/*.e2e-spec.ts' 10 | ], 11 | capabilities: { 12 | 'browserName': 'chrome' 13 | }, 14 | directConnect: true, 15 | baseUrl: 'http://localhost:4200/', 16 | framework: 'jasmine', 17 | jasmineNodeOpts: { 18 | showColors: true, 19 | defaultTimeoutInterval: 30000, 20 | print: function() {} 21 | }, 22 | onPrepare() { 23 | require('ts-node').register({ 24 | project: require('path').join(__dirname, './tsconfig.e2e.json') 25 | }); 26 | jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } })); 27 | } 28 | }; -------------------------------------------------------------------------------- /mlcomp/server/front/e2e/src/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { AppPage } from './app.po'; 2 | import { browser, logging } from 'protractor'; 3 | 4 | describe('workspace-project App', () => { 5 | let page: AppPage; 6 | 7 | beforeEach(() => { 8 | page = new AppPage(); 9 | }); 10 | 11 | it('should display welcome message', () => { 12 | page.navigateTo(); 13 | expect(page.getTitleText()).toEqual('Welcome to mlcomp!'); 14 | }); 15 | 16 | afterEach(async () => { 17 | // Assert that there are no errors emitted from the browser 18 | const logs = await browser.manage().logs().get(logging.Type.BROWSER); 19 | expect(logs).not.toContain(jasmine.objectContaining({ 20 | level: logging.Level.SEVERE, 21 | } as logging.Entry)); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /mlcomp/server/front/e2e/src/app.po.ts: -------------------------------------------------------------------------------- 1 | import { browser, by, element } from 'protractor'; 2 | 3 | export class AppPage { 4 | navigateTo() { 5 | return browser.get(browser.baseUrl) as Promise; 6 | } 7 | 8 | getTitleText() { 9 | return element(by.css('app-root h1')).getText() as Promise; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /mlcomp/server/front/e2e/tsconfig.e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "module": "commonjs", 6 | "target": "es5", 7 | "types": [ 8 | "jasmine", 9 | "jasminewd2", 10 | "node" 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/_helpers/auth.gaurd.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import { Router, 3 | CanActivate, 4 | ActivatedRouteSnapshot, 5 | RouterStateSnapshot } from '@angular/router'; 6 | 7 | @Injectable({ providedIn: 'root' }) 8 | export class AuthGuard implements CanActivate { 9 | constructor( 10 | private router: Router 11 | ) {} 12 | 13 | canActivate(route: ActivatedRouteSnapshot, 14 | state: RouterStateSnapshot) { 15 | if (localStorage.getItem('token')) { 16 | // authorised so return true 17 | return true; 18 | } 19 | 20 | // not logged in so redirect to login page with the return url 21 | this.router.navigate(['/login'], 22 | { queryParams: { returnUrl: state.url }}); 23 | 24 | return false; 25 | } 26 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/_helpers/error.interceptor.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import { 3 | HttpRequest, 4 | HttpHandler, 5 | HttpEvent, 6 | HttpInterceptor 7 | } from '@angular/common/http'; 8 | import {Observable, throwError} from 'rxjs'; 9 | import {catchError} from 'rxjs/operators'; 10 | import {Router} from "@angular/router"; 11 | 12 | @Injectable() 13 | export class ErrorInterceptor implements HttpInterceptor { 14 | constructor( 15 | private router: Router 16 | ) { 17 | } 18 | 19 | 20 | intercept(request: HttpRequest, next: HttpHandler): 21 | Observable> { 22 | 23 | return next.handle(request).pipe(catchError(err => { 24 | if (err.status === 401) { 25 | localStorage.removeItem('token'); 26 | this.router.navigate(['/login'], 27 | { 28 | queryParams: { 29 | returnUrl: 30 | this.router.routerState.snapshot.url 31 | } 32 | }); 33 | 34 | } 35 | return throwError(err); 36 | })) 37 | } 38 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/_helpers/jwt.intercepter.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import { HttpRequest, 3 | HttpHandler, 4 | HttpEvent, 5 | HttpInterceptor } from '@angular/common/http'; 6 | import { Observable } from 'rxjs'; 7 | 8 | @Injectable() 9 | export class JwtInterceptor implements HttpInterceptor { 10 | intercept(request: HttpRequest, next: HttpHandler): 11 | Observable> { 12 | 13 | let token = localStorage.getItem('token'); 14 | if (token) { 15 | request = request.clone({ 16 | setHeaders: { 17 | Authorization: token 18 | } 19 | }); 20 | } 21 | 22 | return next.handle(request); 23 | } 24 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/animations.ts: -------------------------------------------------------------------------------- 1 | import { 2 | trigger, animateChild, group, 3 | transition, animate, style, query 4 | } from '@angular/animations'; 5 | 6 | 7 | // Routable animations 8 | export const slideInAnimation = 9 | trigger('routeAnimation', [ 10 | transition('heroes <=> hero', [ 11 | style({ position: 'relative' }), 12 | query(':enter, :leave', [ 13 | style({ 14 | position: 'absolute', 15 | top: 0, 16 | left: 0, 17 | width: '100%' 18 | }) 19 | ]), 20 | query(':enter', [ 21 | style({ left: '-100%'}) 22 | ]), 23 | query(':leave', animateChild()), 24 | group([ 25 | query(':leave', [ 26 | animate('300ms ease-out', style({ left: '100%'})) 27 | ]), 28 | query(':enter', [ 29 | animate('300ms ease-out', style({ left: '0%'})) 30 | ]) 31 | ]), 32 | query(':enter', animateChild()), 33 | ]) 34 | ]); 35 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/app-settings.ts: -------------------------------------------------------------------------------- 1 | import {environment} from "../environments/environment"; 2 | 3 | export class AppSettings { 4 | public static get API_ENDPOINT(): string { 5 | let port = parseInt(window.location.port); 6 | if(Number.isNaN(port)){ 7 | port = 80 8 | } 9 | if(!environment.production){ 10 | port += 1; 11 | } 12 | return `http://${window.location.hostname}:${port}/api/` 13 | } 14 | 15 | public static status_colors = { 16 | 'not_ran': 'gray', 17 | 'queued': 'lightblue', 18 | 'in_progress': 'lime', 19 | 'failed': '#e83217', 20 | 'stopped': '#cb88ea', 21 | 'skipped': 'orange', 22 | 'success': 'green' 23 | }; 24 | public static log_colors = { 25 | 'debug': 'gray', 26 | 'info': 'lightblue', 27 | 'warning': 'lightyellow', 28 | 'error': 'red' 29 | }; 30 | } 31 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/app.component.css: -------------------------------------------------------------------------------- 1 | /* AppComponent's private CSS styles */ 2 | h1 { 3 | font-size: 1.2em; 4 | color: #999; 5 | margin-bottom: 0; 6 | } 7 | h2 { 8 | font-size: 2em; 9 | margin-top: 0; 10 | padding-top: 0; 11 | } 12 | nav a { 13 | padding: 5px 10px; 14 | text-decoration: none; 15 | margin-top: 10px; 16 | display: inline-block; 17 | background-color: #eee; 18 | border-radius: 4px; 19 | } 20 | nav a:visited, a:link { 21 | color: #607D8B; 22 | } 23 | nav a:hover { 24 | color: #039be5; 25 | background-color: #CFD8DC; 26 | } 27 | nav a.active { 28 | color: #039be5; 29 | } 30 | 31 | .text-center { 32 | text-align:center; 33 | } 34 | 35 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/app.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |

7 | 8 | 19 | 20 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {Router, RouterOutlet} from "@angular/router"; 3 | import {slideInAnimation} from './animations'; 4 | import {Location} from "@angular/common"; 5 | import {MatIconRegistry} from "@angular/material"; 6 | import {DomSanitizer} from "@angular/platform-browser"; 7 | 8 | @Component({ 9 | selector: 'app-root', 10 | templateUrl: './app.component.html', 11 | styleUrls: ['./app.component.css'], 12 | animations: [slideInAnimation] 13 | }) 14 | export class AppComponent { 15 | title = 'ML comp dashboard'; 16 | } 17 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/auxiliary/auxiliary.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/auxiliary/auxiliary.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/auxiliary/auxiliary.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/auxiliary/auxiliary.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {BaseService} from "../base.service"; 3 | 4 | @Injectable({ 5 | providedIn: 'root' 6 | }) 7 | export class AuxiliaryService extends BaseService { 8 | protected collection_part: string; 9 | protected single_part: string = 'auxiliary'; 10 | 11 | 12 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/auxiliary/supervisor/supervisor.component.css: -------------------------------------------------------------------------------- 1 | a { 2 | color: blue !important; 3 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/auxiliary/supervisor/supervisor.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, Input, OnInit} from '@angular/core'; 2 | import {AuxiliarySupervisor} from "../../models"; 3 | 4 | @Component({ 5 | selector: 'app-auxiliary-supervisor', 6 | templateUrl: './supervisor.component.html', 7 | styleUrls: ['./supervisor.component.css'] 8 | }) 9 | export class AuxiliarySupervisorComponent{ 10 | @Input() data: AuxiliarySupervisor; 11 | } 12 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/computer/computer.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { ComputerComponent } from './computer.component'; 4 | 5 | describe('ComputerComponent', () => { 6 | let component: ComputerComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ ComputerComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(ComputerComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/computer/sync-dialog.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | import {SyncStart} from "../models"; 4 | 5 | @Component({ 6 | selector: 'sync-dialog', 7 | templateUrl: 'sync-dialog.html', 8 | }) 9 | export class SyncDialogComponent { 10 | 11 | constructor( 12 | public dialogRef: MatDialogRef, 13 | @Inject(MAT_DIALOG_DATA) public data: SyncStart) { 14 | if (data.projects && data.projects.length > 0) { 15 | data.project = data.projects[0]; 16 | } 17 | for (let p of data.projects) { 18 | if (!p.sync_folders) { 19 | p.sync_folders = '[\ndata,\nmodels\n]' 20 | } 21 | if (!p.ignore_folders) { 22 | p.ignore_folders = '[\n\n]' 23 | } 24 | } 25 | } 26 | 27 | onNoClick(): void { 28 | this.dialogRef.close(); 29 | } 30 | 31 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/code/code.component.css: -------------------------------------------------------------------------------- 1 | .mat-tree-node { min-height: 20px } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/code/code.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { CodeComponent } from './code.component'; 4 | 5 | describe('CodeComponent', () => { 6 | let component: CodeComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ CodeComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(CodeComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/config/config.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/dag/dag-detail/config/config.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/config/config.component.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/config/config.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { ConfigComponent } from './config.component'; 4 | 5 | describe('ConfigComponent', () => { 6 | let component: ConfigComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ ConfigComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(ConfigComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/dag-detail-routing.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {RouterModule, Routes} from '@angular/router'; 3 | 4 | import {CodeComponent} from './code/code.component'; 5 | import {ConfigComponent} from './config/config.component'; 6 | import {GraphComponent} from './graph/graph.component'; 7 | import {DagDetailComponent} from './dag-detail/dag-detail.component'; 8 | import {TasksComponent} from "../../task/tasks/tasks.component" 9 | 10 | 11 | const routes: Routes = [ 12 | { 13 | 14 | path: '', 15 | component: DagDetailComponent, 16 | children: [ 17 | {path: 'code', component: CodeComponent}, 18 | {path: 'config', component: ConfigComponent}, 19 | {path: 'graph', component: GraphComponent}, 20 | {path: 'tasks', component: TasksComponent} 21 | ] 22 | 23 | 24 | } 25 | ]; 26 | 27 | @NgModule({ 28 | imports: [ 29 | RouterModule.forChild(routes) 30 | ], 31 | exports: [ 32 | RouterModule 33 | ] 34 | }) 35 | export class DagDetailRoutingModule { 36 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/dag-detail.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {CodeComponent} from './code/code.component'; 3 | import {ConfigComponent} from './config/config.component'; 4 | import {GraphComponent} from './graph/graph.component'; 5 | 6 | import {DagDetailRoutingModule} from './dag-detail-routing.module'; 7 | import { DagDetailComponent } from './dag-detail/dag-detail.component'; 8 | import {SharedModule} from "../../shared.module"; 9 | 10 | @NgModule({ 11 | imports: [ 12 | DagDetailRoutingModule, 13 | SharedModule 14 | ], 15 | declarations: [ 16 | CodeComponent, 17 | ConfigComponent, 18 | GraphComponent, 19 | DagDetailComponent 20 | ] 21 | }) 22 | export class DagDetailModule { 23 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/dag-detail/dag-detail.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 100%; 3 | min-width: 1580px; 4 | } 5 | 6 | .mat-form-field { 7 | font-size: 14px; 8 | width: 100%; 9 | } 10 | 11 | td, th { 12 | width: 10%; 13 | 14 | } 15 | 16 | mat-icon{ 17 | cursor: pointer; 18 | } 19 | 20 | ::ng-deep .mat-sort-header-container{ 21 | display:flex; 22 | justify-content:center; 23 | text-align: center !important; 24 | margin-left: 12px !important; 25 | } 26 | 27 | .mat-header-cell{ 28 | text-align: center !important; 29 | } 30 | 31 | td{ 32 | text-align: center; 33 | } 34 | 35 | .checkbox-section { 36 | display: flex; 37 | align-content: center; 38 | align-items: center; 39 | height: 60px; 40 | } 41 | 42 | .checkbox-margin { 43 | margin: 0 10px; 44 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/dag-detail/dag-detail.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { DagDetailComponent } from './dag-detail.component'; 4 | 5 | describe('DagDetailComponent', () => { 6 | let component: DagDetailComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ DagDetailComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(DagDetailComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/dag-detail/dag-detail.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {DagsComponent} from "../../dags/dags.component"; 3 | 4 | @Component({ 5 | selector: 'app-dag-detail', 6 | templateUrl: './dag-detail.component.html', 7 | styleUrls: ['../../dags/dags.component.css'] 8 | }) 9 | export class DagDetailComponent extends DagsComponent { 10 | get_filter(): any { 11 | let res = super.get_filter(); 12 | res.id = parseInt(this.route.snapshot.paramMap.get('id')); 13 | return res; 14 | } 15 | 16 | onActivate(component) { 17 | component.dag = parseInt(this.route.snapshot.paramMap.get('id')); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/graph/graph.component.css: -------------------------------------------------------------------------------- 1 | #mynetwork { 2 | border: 1px solid black; 3 | background: white; 4 | display: inline-block; 5 | width: 100%; 6 | height: 800px; 7 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/graph/graph.component.html: -------------------------------------------------------------------------------- 1 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-detail/graph/graph.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { GraphComponent } from './graph.component'; 4 | 5 | describe('GraphComponent', () => { 6 | let component: GraphComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ GraphComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(GraphComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag-routing.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {RouterModule, Routes} from '@angular/router'; 3 | import {DagsComponent} from "./dags/dags.component"; 4 | import {DagComponent} from "./dag/dag.component"; 5 | 6 | const routes: Routes = [ 7 | { 8 | path: '', 9 | component: DagComponent, 10 | children: [ 11 | { 12 | path: 'dag-detail/:id', 13 | loadChildren: 14 | './dag-detail/dag-detail.module#DagDetailModule' 15 | }, 16 | {path: '', component: DagsComponent} 17 | ] 18 | } 19 | ]; 20 | 21 | @NgModule({ 22 | imports: [ 23 | RouterModule.forChild(routes) 24 | ], 25 | exports: [ 26 | RouterModule 27 | ] 28 | }) 29 | export class DagRoutingModule { 30 | } 31 | 32 | 33 | /* 34 | Copyright Google LLC. All Rights Reserved. 35 | Use of this source code is governed by an MIT-style license that 36 | can be found in the LICENSE file at http://angular.io/license 37 | */ -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {DagDetailModule} from './dag-detail/dag-detail.module'; 3 | import { DagComponent } from './dag/dag.component'; 4 | import {DagRoutingModule} from './dag-routing.module' 5 | import {SharedModule} from "../shared.module"; 6 | import {DagRestartDialogComponent} from "./dags/dag-restart-dialog"; 7 | 8 | @NgModule({ 9 | imports: [ 10 | DagRoutingModule, 11 | DagDetailModule, 12 | SharedModule 13 | ], 14 | declarations: [ 15 | DagComponent, 16 | DagRestartDialogComponent 17 | ], 18 | entryComponents:[ 19 | DagRestartDialogComponent 20 | ] 21 | }) 22 | export class DagModule { 23 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag/dag.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/dag/dag/dag.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag/dag.component.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag/dag.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { DagComponent } from './dag.component'; 4 | 5 | describe('DagComponent', () => { 6 | let component: DagComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ DagComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(DagComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dag/dag.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-dag', 5 | templateUrl: './dag.component.html', 6 | styleUrls: ['./dag.component.css'] 7 | }) 8 | export class DagComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit() { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dags/dag-restart-dialog.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | File changes 4 | 13 | 14 |
15 | 16 | 
17 |     Error:
18 | 
19 |     {{error}}
20 |
21 | 22 |
23 | 24 | 25 |
26 | 27 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dags/dags.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 100%; 3 | min-width: 1580px; 4 | } 5 | 6 | td, th { 7 | width: 10%; 8 | 9 | } 10 | 11 | .checkbox-section { 12 | display: flex; 13 | align-content: center; 14 | align-items: center; 15 | height: 60px; 16 | } 17 | 18 | .checkbox-margin { 19 | margin: 0 10px; 20 | } 21 | 22 | .mat-form-field { 23 | font-size: 14px; 24 | width: 100%; 25 | } 26 | 27 | td{ 28 | text-align: center; 29 | } 30 | 31 | ::ng-deep .mat-sort-header-container{ 32 | display:flex; 33 | justify-content:center; 34 | text-align: center; 35 | margin-left: 12px !important; 36 | } 37 | 38 | .mat-header-cell{ 39 | text-align: center !important; 40 | } 41 | 42 | mat-icon{ 43 | cursor: pointer; 44 | } 45 | 46 | .checkbox-section { 47 | display: flex; 48 | align-content: center; 49 | align-items: center; 50 | height: 60px; 51 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dag/dags/dags.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { DagsComponent } from './dags.component'; 4 | 5 | describe('DagComponent', () => { 6 | let component: DagsComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ DagsComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(DagsComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dialog/dialog.component.html: -------------------------------------------------------------------------------- 1 |

{{data.message}}

2 |
3 | 4 | 5 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/dialog/dialog.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | 4 | export interface DialogData { 5 | message: string 6 | } 7 | 8 | @Component({ 9 | selector: 'app-dialog', 10 | templateUrl: './dialog.component.html', 11 | }) 12 | export class DialogComponent { 13 | 14 | constructor(public dialogRef: MatDialogRef, 15 | @Inject(MAT_DIALOG_DATA) public data: DialogData) {} 16 | 17 | onNoClick(): void { 18 | this.dialogRef.close(); 19 | } 20 | 21 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/log/log.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 100%; 3 | } 4 | 5 | .mat-form-field { 6 | font-size: 14px; 7 | width: 100%; 8 | } 9 | 10 | mat-icon{ 11 | cursor: pointer; 12 | } 13 | 14 | ::ng-deep .mat-sort-header-container{ 15 | display:flex; 16 | justify-content:center; 17 | text-align: center; 18 | margin-left: 12px !important; 19 | } 20 | 21 | .mat-header-cell{ 22 | text-align: center !important; 23 | } 24 | 25 | td{ 26 | text-align: center; 27 | } 28 | 29 | 30 | .checkbox-section { 31 | display: flex; 32 | align-content: center; 33 | align-items: center; 34 | height: 60px; 35 | } 36 | 37 | .checkbox-margin { 38 | margin: 0 10px; 39 | } 40 | 41 | table.filter{ 42 | width: 30%; 43 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/log/log.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { LogComponent } from './log.component'; 4 | 5 | describe('LogComponent', () => { 6 | let component: LogComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ LogComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(LogComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/log/log.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import {BaseService} from "../base.service"; 3 | 4 | @Injectable({ 5 | providedIn: 'root' 6 | }) 7 | export class LogService extends BaseService{ 8 | protected collection_part: string = 'logs'; 9 | protected single_part: string = 'log'; 10 | 11 | 12 | } 13 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/message.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | 3 | @Injectable({ providedIn: 'root' }) 4 | export class MessageService { 5 | messages: string[] = []; 6 | 7 | add(message: string) { 8 | this.messages.push(message); 9 | } 10 | 11 | clear() { 12 | this.messages = []; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/messages/messages.component.css: -------------------------------------------------------------------------------- 1 | /* MessagesComponent's private CSS styles */ 2 | h2 { 3 | color: red; 4 | font-family: Arial, Helvetica, sans-serif; 5 | font-weight: lighter; 6 | } 7 | body { 8 | margin: 2em; 9 | } 10 | body, input[text], button { 11 | color: crimson; 12 | font-family: Cambria, Georgia; 13 | } 14 | 15 | button.clear { 16 | font-family: Arial; 17 | background-color: #eee; 18 | border: none; 19 | padding: 5px 10px; 20 | border-radius: 4px; 21 | cursor: pointer; 22 | cursor: hand; 23 | } 24 | button:hover { 25 | background-color: #cfd8dc; 26 | } 27 | button:disabled { 28 | background-color: #eee; 29 | color: #aaa; 30 | cursor: auto; 31 | } 32 | button.clear { 33 | color: #888; 34 | margin-bottom: 12px; 35 | } 36 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/messages/messages.component.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |

Messages

4 | 6 |
{{message}}
7 | 8 |
9 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/messages/messages.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { MessagesComponent } from './messages.component'; 4 | 5 | describe('MessagesComponent', () => { 6 | let component: MessagesComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ MessagesComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(MessagesComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should be created', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/messages/messages.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | import { MessageService } from '../message.service'; 3 | 4 | @Component({ 5 | selector: 'app-messages', 6 | templateUrl: './messages.component.html', 7 | styleUrls: ['./messages.component.css'] 8 | }) 9 | export class MessagesComponent implements OnInit { 10 | 11 | constructor(public messageService: MessageService) {} 12 | 13 | ngOnInit() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/model/model-add-dialog.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | import {ModelAddData} from "../models"; 4 | import {ModelService} from "./model.service"; 5 | 6 | @Component({ 7 | selector: 'model-add-dialog', 8 | templateUrl: 'model-add-dialog.html', 9 | }) 10 | export class ModelAddDialogComponent { 11 | error: string; 12 | 13 | constructor( 14 | public dialogRef: MatDialogRef, 15 | @Inject(MAT_DIALOG_DATA) public data: ModelAddData, 16 | public service: ModelService) { 17 | } 18 | 19 | on_ok_click(): void{ 20 | this.service.add(this.data).subscribe(res=>{ 21 | this.error = res.error; 22 | if(res.success){ 23 | this.dialogRef.close(); 24 | } 25 | }); 26 | } 27 | 28 | on_cancel_click(): void { 29 | this.dialogRef.close(); 30 | } 31 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/model/model.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 70%; 3 | min-width: 1200px; 4 | } 5 | 6 | .mat-form-field { 7 | font-size: 14px; 8 | width: 100%; 9 | } 10 | 11 | td, th { 12 | width: 10%; 13 | 14 | } 15 | 16 | mat-icon{ 17 | cursor: pointer; 18 | } 19 | 20 | ::ng-deep .mat-sort-header-container{ 21 | display:flex; 22 | justify-content:center; 23 | text-align: center; 24 | margin-left: 12px !important; 25 | } 26 | 27 | .mat-header-cell{ 28 | text-align: center !important; 29 | } 30 | 31 | td{ 32 | text-align: center; 33 | } 34 | 35 | .checkbox-section { 36 | display: flex; 37 | align-content: center; 38 | align-items: center; 39 | height: 60px; 40 | } 41 | 42 | .checkbox-margin { 43 | margin: 0 10px; 44 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/project/project-add-dialog.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | import {ProjectAddData} from "../models"; 4 | 5 | @Component({ 6 | selector: 'project-add-dialog', 7 | templateUrl: 'project-add-dialog.html', 8 | }) 9 | export class ProjectAddDialogComponent { 10 | 11 | constructor( 12 | public dialogRef: MatDialogRef, 13 | @Inject(MAT_DIALOG_DATA) public data: ProjectAddData) { 14 | if(!data.class_names) 15 | { 16 | data.class_names = 'default: [\n\n]'; 17 | } 18 | if(!data.sync_folders){ 19 | data.sync_folders = '[\ndata,\nmodels\n]' 20 | } 21 | if(!data.ignore_folders){ 22 | data.ignore_folders = '[\n\n]' 23 | } 24 | } 25 | 26 | onNoClick(): void { 27 | this.dialogRef.close(); 28 | } 29 | 30 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/project/project.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 80%; 3 | min-width: 1120px; 4 | } 5 | 6 | .mat-form-field { 7 | font-size: 14px; 8 | width: 100%; 9 | } 10 | 11 | td{ 12 | text-align: center; 13 | } 14 | 15 | ::ng-deep .mat-sort-header-container{ 16 | display:flex; 17 | justify-content:center; 18 | text-align: center; 19 | margin-left: 12px !important; 20 | } 21 | 22 | .mat-header-cell{ 23 | text-align: center !important; 24 | } 25 | 26 | mat-icon{ 27 | cursor: pointer; 28 | } 29 | 30 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img-classify/img-classify.component.css: -------------------------------------------------------------------------------- 1 | mat-button-toggle-group { 2 | flex-wrap: wrap; 3 | } 4 | 5 | img { 6 | height: auto; 7 | width: auto; 8 | } 9 | 10 | .section { 11 | display: flex; 12 | align-content: center; 13 | align-items: center; 14 | height: 12px; 15 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img-classify/img-classify.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import {BaseService} from "../../../base.service"; 3 | 4 | @Injectable({ 5 | providedIn: 'root' 6 | }) 7 | export class ImgClassifyService extends BaseService{ 8 | protected collection_part: string = 'img_classify'; 9 | protected single_part: string; 10 | 11 | } 12 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img-segment/img-segment.component.css: -------------------------------------------------------------------------------- 1 | mat-button-toggle-group { 2 | flex-wrap: wrap; 3 | } 4 | 5 | img { 6 | height: auto; 7 | width: auto; 8 | } 9 | 10 | .section { 11 | display: flex; 12 | align-content: center; 13 | align-items: center; 14 | height: 12px; 15 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img-segment/img-segment.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import {BaseService} from "../../../base.service"; 3 | 4 | @Injectable({ 5 | providedIn: 'root' 6 | }) 7 | export class ImgSegmentService extends BaseService{ 8 | protected collection_part: string = 'img_segment'; 9 | protected single_part: string; 10 | 11 | } 12 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img/img.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/report/internal/img/img.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img/img.component.html: -------------------------------------------------------------------------------- 1 |
{{item.name}}
2 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img/img.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { ImgComponent } from './img.component'; 4 | 5 | describe('ImgComponent', () => { 6 | let component: ImgComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ ImgComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(ImgComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/img/img.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, Input, OnInit} from '@angular/core'; 2 | import {ReportItem} from "../../../models"; 3 | import {LayoutService} from "../layout/layout.service"; 4 | 5 | @Component({ 6 | selector: 'app-img', 7 | templateUrl: './img.component.html', 8 | styleUrls: ['./img.component.css'] 9 | }) 10 | export class ImgComponent implements OnInit { 11 | 12 | @Input() item: ReportItem; 13 | @Input() data; 14 | 15 | constructor(protected layout_service: LayoutService) { 16 | } 17 | 18 | ngOnInit() { 19 | this.subscribe_data_changed(); 20 | } 21 | 22 | 23 | private subscribe_data_changed() { 24 | this.layout_service.data_updated.subscribe(event => { 25 | if (event.key != this.item.source) { 26 | return; 27 | } 28 | 29 | this.data = event.data[this.item.index]; 30 | 31 | }); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/layout/layout.component.css: -------------------------------------------------------------------------------- 1 | app-layout{ 2 | width: 100%; 3 | height: 100%; 4 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/layout/layout.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { LayoutComponent } from './layout.component'; 4 | 5 | describe('LayoutComponent', () => { 6 | let component: LayoutComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ LayoutComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(LayoutComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/layout/layout.service.ts: -------------------------------------------------------------------------------- 1 | import {EventEmitter, Injectable} from '@angular/core'; 2 | 3 | @Injectable({ 4 | providedIn: 'root' 5 | }) 6 | export class LayoutService { 7 | data_updated: EventEmitter = new EventEmitter(); 8 | full_updated: EventEmitter = new EventEmitter(); 9 | } 10 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/series/series.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/report/internal/series/series.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/series/series.component.html: -------------------------------------------------------------------------------- 1 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/series/series.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { SeriesComponent } from './series.component'; 4 | 5 | describe('SeriesComponent', () => { 6 | let component: SeriesComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ SeriesComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(SeriesComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/table/table.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | border-collapse: collapse; 3 | margin-left: auto; 4 | margin-right: auto; 5 | } 6 | 7 | table, th, td { 8 | border: 1px solid black; 9 | } 10 | 11 | th, td{ 12 | padding: 5px; 13 | } 14 | 15 | th.up:after{ 16 | padding-left: 5px; 17 | content: "\25B4"; 18 | } 19 | 20 | th.down:after{ 21 | padding-left: 5px; 22 | content: "\25BE"; 23 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/internal/table/table.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 10 | 11 | 12 | 13 | 14 | 15 | 23 | 24 | 25 | 26 |
8 | {{h}} 9 |
16 | 17 | {{c| number: '.0-4'}} 18 | 19 | 20 | {{c}} 21 | 22 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/layouts/layout-add-dialog.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | Name 4 | 5 | 6 |
7 | 8 |
9 | 10 | 11 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/layouts/layout-add-dialog.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | import {LayoutAddData} from "../../models"; 4 | 5 | @Component({ 6 | selector: 'layout-add-dialog', 7 | templateUrl: 'layout-add-dialog.html', 8 | }) 9 | export class LayoutAddDialogComponent { 10 | 11 | constructor( 12 | public dialogRef: MatDialogRef, 13 | @Inject(MAT_DIALOG_DATA) public data: LayoutAddData) { 14 | 15 | } 16 | 17 | onNoClick(): void { 18 | this.dialogRef.close(); 19 | } 20 | 21 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/layouts/layouts.component.css: -------------------------------------------------------------------------------- 1 | .mat-form-field { 2 | font-size: 14px; 3 | width: 100%; 4 | } 5 | 6 | mat-icon{ 7 | cursor: pointer; 8 | } 9 | 10 | ::ng-deep .mat-sort-header-container{ 11 | display:flex; 12 | justify-content:center; 13 | text-align: center; 14 | margin-left: 12px !important; 15 | } 16 | 17 | .mat-header-cell{ 18 | text-align: center !important; 19 | } 20 | 21 | td{ 22 | text-align: center; 23 | } 24 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/layouts/layouts.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { LayoutsComponent } from './layouts.component'; 4 | 5 | describe('LayoutsComponent', () => { 6 | let component: LayoutsComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ LayoutsComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(LayoutsComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report-detail/report-detail.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/report/report-detail/report-detail.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report-detail/report-detail.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { ReportDetailComponent } from './report-detail.component'; 4 | 5 | describe('ReportDetailComponent', () => { 6 | let component: ReportDetailComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ ReportDetailComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(ReportDetailComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report-detail/report-update-dialog.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/report/report-detail/report-update-dialog.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report-detail/report-update-dialog.component.html: -------------------------------------------------------------------------------- 1 | 2 | Layout 3 | 6 | 7 | {{layout}} 8 | 9 | 10 | 11 | 12 |
13 | 14 | 15 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report-detail/report-update-dialog.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | import {ReportUpdateData} from "../../models"; 4 | 5 | @Component({ 6 | selector: 'report-update-dialog', 7 | templateUrl: 'report-update-dialog.component.html', 8 | }) 9 | export class ReportUpdateDialogComponent { 10 | 11 | constructor( 12 | public dialogRef: MatDialogRef, 13 | @Inject(MAT_DIALOG_DATA) public data: ReportUpdateData 14 | ) { 15 | if(data.layouts.length>0){ 16 | data.layout = data.layouts[0]; 17 | } 18 | } 19 | 20 | on_no_click(): void { 21 | this.dialogRef.close(); 22 | } 23 | 24 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report-routing.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {RouterModule, Routes} from '@angular/router'; 3 | import {ReportComponent} from "./report/report.component"; 4 | import {ReportsComponent} from "./reports/reports.component"; 5 | import {ReportDetailComponent} from "./report-detail/report-detail.component"; 6 | import {LayoutsComponent} from "./layouts/layouts.component"; 7 | 8 | const routes: Routes = [ 9 | { 10 | path: '', 11 | component: ReportComponent, 12 | children: [ 13 | {path: 'list', component: ReportsComponent}, 14 | {path: 'layouts', component: LayoutsComponent}, 15 | {path: 'report-detail/:id', component: ReportDetailComponent}, 16 | ] 17 | } 18 | ]; 19 | 20 | @NgModule({ 21 | imports: [ 22 | RouterModule.forChild(routes) 23 | ], 24 | exports: [ 25 | RouterModule 26 | ] 27 | }) 28 | export class ReportRoutingModule { 29 | } 30 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report/report.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/report/report/report.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report/report.component.html: -------------------------------------------------------------------------------- 1 | 5 | 6 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report/report.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { ReportComponent } from './reports.component'; 4 | 5 | describe('ReportsComponent', () => { 6 | let component: ReportComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ ReportComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(ReportComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/report/report.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-reports', 5 | templateUrl: './report.component.html', 6 | styleUrls: ['./report.component.css'] 7 | }) 8 | export class ReportComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit() { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/reports/report-add-dialog.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | Name 4 | 5 | 6 |
7 | 8 | 9 | Project 10 | 12 | 15 | {{project.name}} 16 | 17 | 18 | 19 | 20 | 21 | Layout 22 | 24 | 27 | {{layout.name}} 28 | 29 | 30 | 31 | 32 |
33 | 34 | 35 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/reports/report-add-dialog.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | import {ReportAddData} from "../../models"; 4 | 5 | @Component({ 6 | selector: 'report-add-dialog', 7 | templateUrl: 'report-add-dialog.html', 8 | }) 9 | export class ReportAddDialogComponent { 10 | 11 | constructor( 12 | public dialogRef: MatDialogRef, 13 | @Inject(MAT_DIALOG_DATA) public data: ReportAddData) { 14 | 15 | if(data.projects.length>0){ 16 | data.project = data.projects[0].id; 17 | } 18 | if(data.layouts.length>0){ 19 | data.layout = data.layouts[0].name; 20 | } 21 | } 22 | 23 | onNoClick(): void { 24 | this.dialogRef.close(); 25 | } 26 | 27 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/reports/reports.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 50%; 3 | min-width: 800px; 4 | } 5 | 6 | .mat-header-cell{ 7 | text-align: center !important; 8 | } 9 | 10 | td{ 11 | text-align: center; 12 | } 13 | 14 | mat-icon{ 15 | cursor: pointer; 16 | } 17 | 18 | ::ng-deep .mat-sort-header-container{ 19 | display:flex; 20 | justify-content:center; 21 | text-align: center; 22 | margin-left: 12px !important; 23 | } 24 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/report/reports/reports.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { ReportsComponent } from './reports-list.component'; 4 | 5 | describe('ReportsListComponent', () => { 6 | let component: ReportsComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ ReportsComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(ReportsComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/memory/memory-add-dialog.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material"; 3 | import {Memory} from "../../models"; 4 | 5 | @Component({ 6 | selector: 'memory-add-dialog', 7 | templateUrl: 'memory-add-dialog.html', 8 | }) 9 | export class MemoryAddDialogComponent { 10 | 11 | constructor( 12 | public dialogRef: MatDialogRef, 13 | @Inject(MAT_DIALOG_DATA) public data: Memory) { 14 | 15 | } 16 | 17 | onNoClick(): void { 18 | this.dialogRef.close(); 19 | } 20 | 21 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/memory/memory.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/skynet/memory/memory.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/memory/memory.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { MemoryComponent } from './memory.component'; 4 | 5 | describe('MemoryComponent', () => { 6 | let component: MemoryComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ MemoryComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(MemoryComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/skynet-routing.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {RouterModule, Routes} from '@angular/router'; 3 | import {SkynetComponent} from "./skynet/skynet.component"; 4 | import {SpaceComponent} from "./space/space.component"; 5 | import {MemoryComponent} from "./memory/memory.component"; 6 | 7 | const routes: Routes = [ 8 | { 9 | path: '', 10 | component: SkynetComponent, 11 | children: [ 12 | {path: 'space', component: SpaceComponent}, 13 | {path: 'memory', component: MemoryComponent}, 14 | ] 15 | } 16 | ]; 17 | 18 | @NgModule({ 19 | imports: [ 20 | RouterModule.forChild(routes) 21 | ], 22 | exports: [ 23 | RouterModule 24 | ] 25 | }) 26 | export class SkynetRoutingModule { 27 | } 28 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/skynet.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {SharedModule} from "../shared.module"; 3 | import {SkynetRoutingModule} from "./skynet-routing.module"; 4 | import {SkynetComponent} from "./skynet/skynet.component"; 5 | import {SpaceComponent} from "./space/space.component"; 6 | import {MemoryComponent} from "./memory/memory.component"; 7 | import {MemoryAddDialogComponent} from "./memory/memory-add-dialog"; 8 | import {SpaceAddDialogComponent} from "./space/space-add-dialog"; 9 | import {SpaceRunDialogComponent} from "./space/space-run-dialog"; 10 | 11 | 12 | @NgModule({ 13 | imports: [ 14 | SkynetRoutingModule, 15 | SharedModule 16 | ], 17 | declarations: [ 18 | SkynetComponent, 19 | SpaceComponent, 20 | MemoryComponent, 21 | MemoryAddDialogComponent, 22 | SpaceAddDialogComponent, 23 | SpaceRunDialogComponent 24 | ], 25 | entryComponents: [ 26 | MemoryAddDialogComponent, 27 | SpaceAddDialogComponent, 28 | SpaceRunDialogComponent 29 | ] 30 | }) 31 | export class SkynetModule { 32 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/skynet/skynet.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/skynet/skynet/skynet.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/skynet/skynet.component.html: -------------------------------------------------------------------------------- 1 | 5 | 6 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/skynet/skynet.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { SkynetComponent } from './skynet.component'; 4 | 5 | describe('SkynetComponent', () => { 6 | let component: SkynetComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ SkynetComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(SkynetComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/skynet/skynet.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-skynet', 5 | templateUrl: './skynet.component.html', 6 | styleUrls: ['./skynet.component.css'] 7 | }) 8 | export class SkynetComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit() { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/space/space-add-dialog.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | Name 4 | 5 | 6 |
7 | 8 | 9 |
10 | 11 | Content 12 | 21 | 22 |
23 | 24 | 
25 |     Error:
26 | 
27 |     {{error}}
28 |
29 | 30 |
31 | 32 | 33 |
34 | 35 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/space/space-run-dialog.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
6 | 7 | File changes 8 | 17 | 18 |
19 | 20 | 
21 |     Error:
22 | 
23 |     {{error}}
24 |
25 | 26 |
27 | 28 | 31 |
32 | 33 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/space/space.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 100%; 3 | min-width: 800px; 4 | } 5 | 6 | .column { 7 | float: left; 8 | width: 45%; 9 | } 10 | 11 | /* Clear floats after the columns */ 12 | .row:after { 13 | content: ""; 14 | display: table; 15 | clear: both; 16 | } 17 | 18 | .mat-header-cell{ 19 | text-align: center !important; 20 | } 21 | 22 | .disable-text-selection { 23 | -webkit-touch-callout: none; 24 | -webkit-user-select: none; 25 | -khtml-user-select: none; 26 | -moz-user-select: none; 27 | -ms-user-select: none; 28 | user-select: none; 29 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/skynet/space/space.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { SpaceComponent } from './space.component'; 4 | 5 | describe('SpaceComponent', () => { 6 | let component: SpaceComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ SpaceComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(SpaceComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-detail/step/step.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/task/task-detail/step/step.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-detail/step/step.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { StepComponent } from './step.component'; 4 | 5 | describe('StepComponent', () => { 6 | let component: StepComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ StepComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(StepComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-detail/task-detail-routing.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {RouterModule, Routes} from '@angular/router'; 3 | 4 | import {TaskDetailComponent} from "./task-detail/task-detail.component"; 5 | import {LogComponent} from "../../log/log.component"; 6 | import {ReportsComponent} from "../../report/reports/reports.component"; 7 | import {StepComponent} from "./step/step.component"; 8 | 9 | const routes: Routes = [ 10 | { 11 | 12 | path: '', 13 | component: TaskDetailComponent, 14 | children: [ 15 | {path: 'report', component: ReportsComponent}, 16 | {path: 'step', component: StepComponent}, 17 | {path: 'logs', component: LogComponent} 18 | ] 19 | 20 | 21 | } 22 | ]; 23 | 24 | @NgModule({ 25 | imports: [ 26 | RouterModule.forChild(routes) 27 | ], 28 | exports: [ 29 | RouterModule 30 | ] 31 | }) 32 | export class TaskDetailRoutingModule { 33 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-detail/task-detail.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | 3 | import {TaskDetailRoutingModule} from './task-detail-routing.module'; 4 | import {SharedModule} from "../../shared.module"; 5 | import {TaskDetailComponent} from "./task-detail/task-detail.component"; 6 | import { StepComponent } from './step/step.component'; 7 | 8 | @NgModule({ 9 | imports: [ 10 | TaskDetailRoutingModule, 11 | SharedModule 12 | ], 13 | declarations: [ 14 | TaskDetailComponent, 15 | StepComponent 16 | ] 17 | }) 18 | export class TaskDetailModule { 19 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-detail/task-detail/task-detail.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/task/task-detail/task-detail/task-detail.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-detail/task-detail/task-detail.component.html: -------------------------------------------------------------------------------- 1 |

Task detail

2 | 3 | 7 | 8 | 9 | 10 |
11 |

Child tasks

12 | 13 | 16 | 17 |
18 | 19 | 20 | 26 | 27 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-detail/task-detail/task-detail.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { TaskDetailComponent } from './task-detail.component'; 4 | 5 | describe('TaskDetailComponent', () => { 6 | let component: TaskDetailComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ TaskDetailComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(TaskDetailComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-routing.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {RouterModule, Routes} from '@angular/router'; 3 | import {TasksComponent} from "./tasks/tasks.component"; 4 | import {TaskComponent} from "./task/task.component"; 5 | 6 | const routes: Routes = [ 7 | { 8 | path: '', 9 | component: TaskComponent, 10 | children: [ 11 | {path: 'task-detail/:id', loadChildren: './task-detail/task-detail.module#TaskDetailModule'}, 12 | {path: '', component: TasksComponent} 13 | ] 14 | } 15 | ]; 16 | 17 | @NgModule({ 18 | imports: [ 19 | RouterModule.forChild(routes) 20 | ], 21 | exports: [ 22 | RouterModule 23 | ] 24 | }) 25 | export class TaskRoutingModule { 26 | } 27 | 28 | 29 | /* 30 | Copyright Google LLC. All Rights Reserved. 31 | Use of this source code is governed by an MIT-style license that 32 | can be found in the LICENSE file at http://angular.io/license 33 | */ -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-table/task-info-dialog.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/task/task-table/task-info-dialog.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-table/task-info-dialog.component.html: -------------------------------------------------------------------------------- 1 |
2 | Pid: {{data.pid}} 3 |
4 | 5 |
6 | Worker index: {{data.worker_index}} 7 |
8 | 9 |
10 | GPU assigned: {{data.gpu_assigned}} 11 |
12 | 13 |
14 | Celery id: {{data.celery_id}} 15 |
16 | 17 | 18 |
19 | Additional info: 20 | 21 |
22 | 
{{data.additional_info}}
23 |
24 | 25 | 26 |
27 | 28 |
29 | Result: 30 | 31 |
32 | 
{{data.result}}
33 |
34 | 35 | 36 |
-------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-table/task-info-dialog.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, Inject} from "@angular/core"; 2 | import {MAT_DIALOG_DATA} from "@angular/material"; 3 | import {TaskService} from "../task.service"; 4 | import {TaskInfo} from "../../models"; 5 | 6 | @Component({ 7 | selector: 'task-info-dialog.component.css', 8 | templateUrl: 'task-info-dialog.component.html', 9 | }) 10 | export class TaskInfoDialogComponent { 11 | error: string; 12 | 13 | constructor( 14 | @Inject(MAT_DIALOG_DATA) public data: TaskInfo, 15 | public service: TaskService) { 16 | 17 | this.service.info(data.id).subscribe(res => { 18 | this.data = res; 19 | }); 20 | } 21 | 22 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task-table/task-table.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 100%; 3 | min-width: 1580px; 4 | } 5 | 6 | .mat-form-field { 7 | font-size: 14px; 8 | width: 100%; 9 | } 10 | 11 | td{ 12 | text-align: center; 13 | } 14 | 15 | ::ng-deep .mat-sort-header-container{ 16 | display:flex; 17 | justify-content:center; 18 | text-align: center; 19 | margin-left: 12px !important; 20 | } 21 | 22 | .mat-header-cell{ 23 | text-align: center !important; 24 | } 25 | 26 | mat-icon{ 27 | cursor: pointer; 28 | } 29 | 30 | .checkbox-section { 31 | display: flex; 32 | align-content: center; 33 | align-items: center; 34 | height: 60px; 35 | } 36 | 37 | .checkbox-margin { 38 | margin: 0 10px; 39 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {TaskDetailModule} from './task-detail/task-detail.module'; 3 | import {TaskRoutingModule} from './task-routing.module' 4 | import {SharedModule} from "../shared.module"; 5 | import {TaskComponent} from "./task/task.component"; 6 | 7 | @NgModule({ 8 | imports: [ 9 | TaskRoutingModule, 10 | TaskDetailModule, 11 | SharedModule 12 | ], 13 | declarations: [ 14 | TaskComponent 15 | ] 16 | }) 17 | export class TaskModule { 18 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task/task.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/task/task/task.component.css -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task/task.component.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task/task.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { TaskComponent } from './dag.component'; 4 | 5 | describe('DagComponent', () => { 6 | let component: TaskComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [ TaskComponent ] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(TaskComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/task/task.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-task', 5 | templateUrl: './task.component.html', 6 | styleUrls: ['./task.component.css'] 7 | }) 8 | export class TaskComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit() { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/tasks/tasks.component.css: -------------------------------------------------------------------------------- 1 | table { 2 | width: 100%; 3 | min-width: 1580px; 4 | } 5 | 6 | .mat-form-field { 7 | font-size: 14px; 8 | width: 100%; 9 | } 10 | 11 | td{ 12 | text-align: center; 13 | } 14 | 15 | ::ng-deep .mat-sort-header-container{ 16 | display:flex; 17 | justify-content:center; 18 | text-align: center; 19 | margin-left: 12px !important; 20 | } 21 | 22 | .mat-header-cell{ 23 | text-align: center !important; 24 | } 25 | 26 | mat-icon{ 27 | cursor: pointer; 28 | } 29 | 30 | .checkbox-section { 31 | display: flex; 32 | align-content: center; 33 | align-items: center; 34 | height: 60px; 35 | } 36 | 37 | .checkbox-margin { 38 | margin: 0 10px; 39 | } 40 | 41 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/app/task/tasks/tasks.component.spec.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/app/task/tasks/tasks.component.spec.ts -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/img/delete.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/img/mlcomp_icon.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/assets/img/mlcomp_icon.jpg -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/img/mlcomp_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/front/src/assets/img/mlcomp_logo.png -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/img/model.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/img/restart.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/img/stop.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-go.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2010 Google Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xA0]+/,null,"\t\n\r \u00a0"],["pln",/^(?:\"(?:[^\"\\]|\\[\s\S])*(?:\"|$)|\'(?:[^\'\\]|\\[\s\S])+(?:\'|$)|`[^`]*(?:`|$))/,null,"\"'"]],[["com",/^(?:\/\/[^\r\n]*|\/\*[\s\S]*?\*\/)/],["pln",/^(?:[^\/\"\'`]|\/(?![\/\*]))+/i]]),["go"]); 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-latex.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2011 Martin S. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xA0]+/,null,"\t\n\r \u00a0"],["com",/^%[^\r\n]*/,null,"%"]],[["kwd",/^\\[a-zA-Z@]+/],["kwd",/^\\./],["typ",/^[$&]/],["lit",/[+-]?(?:\.\d+|\d+(?:\.\d*)?)(cm|em|ex|in|pc|pt|bp|mm)/i],["pun",/^[{}()\[\]=]+/]]),["latex","tex"]); 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-ll.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2013 Nikhil Dabas 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xA0]+/,null,"\t\n\r \u00a0"],["str",/^!?\"(?:[^\"\\]|\\[\s\S])*(?:\"|$)/,null,'"'],["com",/^;[^\r\n]*/,null,";"]],[["pln",/^[%@!](?:[-a-zA-Z$._][-a-zA-Z$._0-9]*|\d+)/],["kwd",/^[A-Za-z_][0-9A-Za-z_]*/,null],["lit",/^\d+\.\d+/],["lit",/^(?:\d+|0[xX][a-fA-F0-9]+)/],["pun",/^[()\[\]{},=*<>:]|\.\.\.$/]]),["llvm","ll"]); 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-llvm.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2013 Nikhil Dabas 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xA0]+/,null,"\t\n\r \u00a0"],["str",/^!?\"(?:[^\"\\]|\\[\s\S])*(?:\"|$)/,null,'"'],["com",/^;[^\r\n]*/,null,";"]],[["pln",/^[%@!](?:[-a-zA-Z$._][-a-zA-Z$._0-9]*|\d+)/],["kwd",/^[A-Za-z_][0-9A-Za-z_]*/,null],["lit",/^\d+\.\d+/],["lit",/^(?:\d+|0[xX][a-fA-F0-9]+)/],["pun",/^[()\[\]{},=*<>:]|\.\.\.$/]]),["llvm","ll"]); 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-proto.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2006 Google Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.sourceDecorator({keywords:"bytes,default,double,enum,extend,extensions,false,group,import,max,message,option,optional,package,repeated,required,returns,rpc,service,syntax,to,true",types:/^(bool|(double|s?fixed|[su]?int)(32|64)|float|string)\b/,cStyleComments:!0}),["proto"]); 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-rd.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2012 Jeffrey Arnold 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xA0]+/,null,"\t\n\r \u00a0"],["com",/^%[^\r\n]*/,null,"%"]],[["lit",/^\\(?:cr|l?dots|R|tab)\b/],["kwd",/^\\[a-zA-Z@]+/],["kwd",/^#(?:ifn?def|endif)/],["pln",/^\\[{}]/],["pun",/^[{}()\[\]]+/]]),["Rd","rd"]); 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-tex.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2011 Martin S. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\r \xA0]+/,null,"\t\n\r \u00a0"],["com",/^%[^\r\n]*/,null,"%"]],[["kwd",/^\\[a-zA-Z@]+/],["kwd",/^\\./],["typ",/^[$&]/],["lit",/[+-]?(?:\.\d+|\d+(?:\.\d*)?)(cm|em|ex|in|pc|pt|bp|mm)/i],["pun",/^[{}()\[\]=]+/]]),["latex","tex"]); 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-yaml.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2015 ribrdb @ code.google.com 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pun",/^[:|>?]+/,null,":|>?"],["dec",/^%(?:YAML|TAG)[^#\r\n]+/,null,"%"],["typ",/^[&]\S+/,null,"&"],["typ",/^!\S*/,null,"!"],["str",/^"(?:[^\\"]|\\.)*(?:"|$)/,null,'"'],["str",/^'(?:[^']|'')*(?:'|$)/,null,"'"],["com",/^#[^\r\n]*/,null,"#"],["pln",/^\s+/,null," \t\r\n"]],[["dec",/^(?:---|\.\.\.)(?:[\r\n]|$)/],["pun",/^-/],["kwd",/^[\w-]+:[ \r\n]/],["pln", 18 | /^\w+/]]),["yaml","yml"]); 19 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/lang-yml.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (C) 2015 ribrdb @ code.google.com 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | PR.registerLangHandler(PR.createSimpleLexer([["pun",/^[:|>?]+/,null,":|>?"],["dec",/^%(?:YAML|TAG)[^#\r\n]+/,null,"%"],["typ",/^[&]\S+/,null,"&"],["typ",/^!\S*/,null,"!"],["str",/^"(?:[^\\"]|\\.)*(?:"|$)/,null,'"'],["str",/^'(?:[^']|'')*(?:'|$)/,null,"'"],["com",/^#[^\r\n]*/,null,"#"],["pln",/^\s+/,null," \t\r\n"]],[["dec",/^(?:---|\.\.\.)(?:[\r\n]|$)/],["pun",/^-/],["kwd",/^[\w-]+:[ \r\n]/],["pln", 18 | /^\w+/]]),["yaml","yml"]); 19 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/prettify.css: -------------------------------------------------------------------------------- 1 | .pln{color:#000}@media screen{.str{color:#080}.kwd{color:#008}.com{color:#800}.typ{color:#606}.lit{color:#066}.clo,.opn,.pun{color:#660}.tag{color:#008}.atn{color:#606}.atv{color:#080}.dec,.var{color:#606}.fun{color:red}}@media print,projection{.kwd,.tag,.typ{font-weight:700}.str{color:#060}.kwd{color:#006}.com{color:#600;font-style:italic}.typ{color:#404}.lit{color:#044}.clo,.opn,.pun{color:#440}.tag{color:#006}.atn{color:#404}.atv{color:#060}}pre.prettyprint{padding:2px;border:1px solid #888}ol.linenums{margin-top:0;margin-bottom:0}li.L0,li.L1,li.L2,li.L3,li.L5,li.L6,li.L7,li.L8{list-style-type:decimal !important}li.L1,li.L3,li.L5,li.L7,li.L9{background:#eee} -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/skins/desert.css: -------------------------------------------------------------------------------- 1 | pre .atn,pre .kwd,pre .tag{font-weight:700}pre.prettyprint{display:block;background-color:#333}pre .nocode{background-color:none;color:#000}pre .str{color:#ffa0a0}pre .kwd{color:khaki}pre .com{color:#87ceeb}pre .typ{color:#98fb98}pre .lit{color:#cd5c5c}pre .pln,pre .pun{color:#fff}pre .tag{color:khaki}pre .atn{color:#bdb76b}pre .atv{color:#ffa0a0}pre .dec{color:#98fb98}ol.linenums{margin-top:0;margin-bottom:0;color:#AEAEAE}li.L0,li.L1,li.L2,li.L3,li.L5,li.L6,li.L7,li.L8{list-style-type:none}@media print{pre.prettyprint{background-color:none}code .str,pre .str{color:#060}code .kwd,pre .kwd{color:#006;font-weight:700}code .com,pre .com{color:#600;font-style:italic}code .typ,pre .typ{color:#404;font-weight:700}code .lit,pre .lit{color:#044}code .pun,pre .pun{color:#440}code .pln,pre .pln{color:#000}code .tag,pre .tag{color:#006;font-weight:700}code .atn,pre .atn{color:#404}code .atv,pre .atv{color:#060}} -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/skins/sons-of-obsidian.css: -------------------------------------------------------------------------------- 1 | .str{color:#EC7600}.kwd{color:#93C763}.com{color:#66747B}.typ{color:#678CB1}.lit{color:#FACD22}.pln,.pun{color:#F1F2F3}.tag{color:#8AC763}.atn{color:#E0E2E4}.atv{color:#EC7600}.dec{color:purple}pre.prettyprint{border:0 solid #888}ol.linenums{margin-top:0;margin-bottom:0}.prettyprint{background:#000}li.L0,li.L1,li.L2,li.L3,li.L4,li.L5,li.L6,li.L7,li.L8,li.L9{color:#555;list-style-type:decimal}li.L1,li.L3,li.L5,li.L7,li.L9{background:#111}@media print{.kwd,.tag,.typ{font-weight:700}.str{color:#060}.kwd{color:#006}.com{color:#600;font-style:italic}.typ{color:#404}.lit{color:#044}.pun{color:#440}.pln{color:#000}.tag{color:#006}.atn{color:#404}.atv{color:#060}} -------------------------------------------------------------------------------- /mlcomp/server/front/src/assets/prettify/skins/sunburst.css: -------------------------------------------------------------------------------- 1 | code .str,pre .str{color:#65B042}code .kwd,pre .kwd{color:#E28964}code .com,pre .com{color:#AEAEAE;font-style:italic}code .typ,pre .typ{color:#89bdff}code .lit,pre .lit{color:#3387CC}code .pln,code .pun,pre .pln,pre .pun{color:#fff}code .tag,pre .tag{color:#89bdff}code .atn,pre .atn{color:#bdb76b}code .atv,pre .atv{color:#65B042}code .dec,pre .dec{color:#3387CC}code.prettyprint,pre.prettyprint{background-color:#000;border-radius:8px}pre.prettyprint{width:95%;margin:1em auto;padding:1em;white-space:pre-wrap}ol.linenums{margin-top:0;margin-bottom:0;color:#AEAEAE}li.L0,li.L1,li.L2,li.L3,li.L5,li.L6,li.L7,li.L8{list-style-type:none}@media print{code .str,pre .str{color:#060}code .kwd,pre .kwd{color:#006;font-weight:700}code .com,pre .com{color:#600;font-style:italic}code .typ,pre .typ{color:#404;font-weight:700}code .lit,pre .lit{color:#044}code .pun,pre .pun{color:#440}code .pln,pre .pln{color:#000}code .tag,pre .tag{color:#006;font-weight:700}code .atn,pre .atn{color:#404}code .atv,pre .atv{color:#060}} -------------------------------------------------------------------------------- /mlcomp/server/front/src/browserslist: -------------------------------------------------------------------------------- 1 | # This file is currently used by autoprefixer to adjust CSS to support the below specified browsers 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | # 5 | # For IE 9-11 support, please remove 'not' from the last line of the file and adjust as needed 6 | 7 | > 0.5% 8 | last 2 versions 9 | Firefox ESR 10 | not dead 11 | not IE 9-11 12 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true 3 | }; 4 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false 7 | }; 8 | 9 | /* 10 | * For easier debugging in development mode, you can import the following file 11 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. 12 | * 13 | * This import should be commented out in production mode because it will have a negative impact 14 | * on performance if an error is thrown. 15 | */ 16 | // import 'zone.js/dist/zone-error'; // Included with Angular CLI. 17 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | MLComp 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration file, see link for more information 2 | // https://karma-runner.github.io/1.0/config/configuration-file.html 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | basePath: '', 7 | frameworks: ['jasmine', '@angular-devkit/build-angular'], 8 | plugins: [ 9 | require('karma-jasmine'), 10 | require('karma-chrome-launcher'), 11 | require('karma-jasmine-html-reporter'), 12 | require('karma-coverage-istanbul-reporter'), 13 | require('@angular-devkit/build-angular/plugins/karma') 14 | ], 15 | client: { 16 | clearContext: false // leave Jasmine Spec Runner output visible in browser 17 | }, 18 | coverageIstanbulReporter: { 19 | dir: require('path').join(__dirname, '../coverage'), 20 | reports: ['html', 'lcovonly'], 21 | fixWebpackSourcePaths: true 22 | }, 23 | reporters: ['progress', 'kjhtml'], 24 | port: 9876, 25 | colors: true, 26 | logLevel: config.LOG_INFO, 27 | autoWatch: true, 28 | browsers: ['Chrome'], 29 | singleRun: false 30 | }); 31 | }; 32 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule); 12 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/test.ts: -------------------------------------------------------------------------------- 1 | // This file is required by karma.conf.js and loads recursively all the .spec and framework files 2 | 3 | import 'zone.js/dist/zone-testing'; 4 | import { getTestBed } from '@angular/core/testing'; 5 | import { 6 | BrowserDynamicTestingModule, 7 | platformBrowserDynamicTesting 8 | } from '@angular/platform-browser-dynamic/testing'; 9 | 10 | declare const require: any; 11 | 12 | // First, initialize the Angular testing environment. 13 | getTestBed().initTestEnvironment( 14 | BrowserDynamicTestingModule, 15 | platformBrowserDynamicTesting() 16 | ); 17 | // Then we find all the tests. 18 | const context = require.context('./', true, /\.spec\.ts$/); 19 | // And load the modules. 20 | context.keys().map(context); 21 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "types": [] 6 | }, 7 | "exclude": [ 8 | "test.ts", 9 | "**/*.spec.ts", 10 | "**/*.avoid.ts", 11 | "**/*.0.ts", 12 | "**/*.1.ts", 13 | "**/*.1b.ts", 14 | "**/*.2.ts", 15 | "**/*.3.ts", 16 | "**/*.4.ts", 17 | "**/*.5.ts", 18 | "**/*.6.ts", 19 | "**/*.7.ts" 20 | ] 21 | } -------------------------------------------------------------------------------- /mlcomp/server/front/src/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/spec", 5 | "types": [ 6 | "jasmine", 7 | "node" 8 | ] 9 | }, 10 | "files": [ 11 | "test.ts", 12 | "polyfills.ts" 13 | ], 14 | "include": [ 15 | "**/*.spec.ts", 16 | "**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /mlcomp/server/front/src/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tslint.json", 3 | "rules": { 4 | "directive-selector": [ 5 | true, 6 | "attribute", 7 | "app", 8 | "camelCase" 9 | ], 10 | "component-selector": [ 11 | true, 12 | "element", 13 | "app", 14 | "kebab-case" 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /mlcomp/server/front/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "baseUrl": "./", 5 | "outDir": "./dist/out-tsc", 6 | "sourceMap": true, 7 | "declaration": false, 8 | "module": "es2015", 9 | "moduleResolution": "node", 10 | "emitDecoratorMetadata": true, 11 | "experimentalDecorators": true, 12 | "importHelpers": true, 13 | "target": "es5", 14 | "typeRoots": [ 15 | "node_modules/@types" 16 | ], 17 | "lib": [ 18 | "es2018", 19 | "dom" 20 | ] 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /mlcomp/server/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/server/tests/__init__.py -------------------------------------------------------------------------------- /mlcomp/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/utils/__init__.py -------------------------------------------------------------------------------- /mlcomp/utils/img.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import numpy as np 4 | import cv2 5 | 6 | 7 | def resize_saving_ratio(img: np.array, size: Tuple[int, int]): 8 | if not size: 9 | return img 10 | if size[0] and img.shape[0] > size[0]: 11 | k = size[0] / img.shape[0] 12 | img = cv2.resize(img, (int(k * img.shape[1]), size[0])) 13 | if size[1] and img.shape[1] > size[1]: 14 | k = size[1] / img.shape[1] 15 | img = cv2.resize(img, (size[1], int(k * img.shape[0]))) 16 | return img 17 | 18 | 19 | __all__ = ['resize_saving_ratio'] 20 | -------------------------------------------------------------------------------- /mlcomp/utils/schedule.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | 3 | from apscheduler.schedulers.background import BackgroundScheduler 4 | 5 | 6 | def start_schedule(jobs): 7 | scheduler = BackgroundScheduler() 8 | for func, interval in jobs: 9 | scheduler.add_job(func=func, trigger='interval', seconds=interval, 10 | max_instances=1) 11 | scheduler.start() 12 | 13 | # Shut down the scheduler when exiting the app 14 | atexit.register(lambda: scheduler.shutdown()) 15 | -------------------------------------------------------------------------------- /mlcomp/utils/tests.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from importlib import reload 3 | import shutil 4 | 5 | import mlcomp 6 | from mlcomp import ROOT_FOLDER 7 | from mlcomp.db.core import Session 8 | from mlcomp.migration.manage import migrate 9 | 10 | 11 | @pytest.fixture() 12 | def session(): 13 | if ROOT_FOLDER: 14 | shutil.rmtree(ROOT_FOLDER) 15 | reload(mlcomp) 16 | 17 | migrate() 18 | res = Session.create_session() 19 | yield res 20 | -------------------------------------------------------------------------------- /mlcomp/worker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/worker/__init__.py -------------------------------------------------------------------------------- /mlcomp/worker/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | from celery import Celery 3 | import os 4 | import sys 5 | 6 | from mlcomp import REDIS_PASSWORD, REDIS_HOST, REDIS_PORT 7 | 8 | sys.path.insert(0, os.path.dirname(__file__)) 9 | 10 | broker = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/0' 11 | 12 | app = Celery( 13 | 'mlcomp', 14 | broker=broker, 15 | backend=broker, 16 | include=['mlcomp.worker.tasks'] 17 | ) 18 | __all__ = ['app'] 19 | -------------------------------------------------------------------------------- /mlcomp/worker/executors/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .base import StepWrap, Executor 3 | -------------------------------------------------------------------------------- /mlcomp/worker/executors/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .step import StepWrap 2 | from .executor import Executor 3 | 4 | __all__ = ['StepWrap', 'Executor'] 5 | -------------------------------------------------------------------------------- /mlcomp/worker/executors/catalyst_/__init__.py: -------------------------------------------------------------------------------- 1 | from .catalyst_ import Catalyst 2 | 3 | __all__ = ['Catalyst'] 4 | -------------------------------------------------------------------------------- /mlcomp/worker/reports/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/worker/reports/__init__.py -------------------------------------------------------------------------------- /mlcomp/worker/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/mlcomp/worker/tests/__init__.py -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | norecursedirs = build docker docs examples mlcomp/server/front dist 3 | addopts = -p no:warnings -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Pillow<7 2 | PyYAML>=5.1 3 | redis 4 | setuptools>=41.0.1 5 | click>=7.0 6 | psutil>=5.6.2 7 | GPUtil==1.4.0 8 | pathspec>=0.5.9 9 | apscheduler>=3.6.0 10 | sqlalchemy>=1.3.4 11 | celery>=4.3.0 12 | kaggle>=1.5.3 13 | scipy>=1.3.0 14 | flask>=1.0.2 15 | requests 16 | flask_cors>=3.0.6 17 | sqlalchemy_serializer==1.3.1 18 | scikit-learn>=0.21.2 19 | psycopg2-binary>=2.8.2 20 | tiffile 21 | albumentations>=0.2.3 22 | sqlalchemy-migrate>=0.12.0 23 | cython 24 | supervisor>=4.0.4 25 | torchvision>=0.4.2 26 | torch>=1.0.0 27 | numpy>=1.18.0 28 | pandas>=0.25.3 29 | tqdm>=4.39.0 30 | simplejson>=3.13.2 31 | 32 | # For describing 33 | jupyter 34 | networkx>=2.2 35 | 36 | # For tests 37 | pytest>=5.0.1 38 | pytest-xdist>=1.29.0 39 | 40 | # Application level 41 | pretrainedmodels>=0.7.4 42 | 43 | catalyst==20.3.1 -------------------------------------------------------------------------------- /teamcity/deploy.sh: -------------------------------------------------------------------------------- 1 | pip install sphinx sphinx-autobuild 2 | 3 | mkdir -p builds/ 4 | sphinx-build -b html ./docs/ builds/ -W --keep-going 5 | 6 | COMMENT=$(git log -1 --pretty=%B) 7 | 8 | cp -a builds $TEMP/builds 9 | 10 | cd $TEMP 11 | 12 | git clone --single-branch --branch gh-pages https://GH_TOKEN:$GH_TOKEN@github.com/catalyst-team/mlcomp.git 13 | 14 | cd mlcomp 15 | rm -rf * 16 | cp -a $TEMP/builds/* . 17 | 18 | if [ $GIT_BRANCH == 'refs/heads/master' ]; then 19 | git config --global user.email "teamcity@catalyst.github" 20 | git config --global user.name "Teamcity" 21 | git add . 22 | git commit -m "$COMMENT" 23 | git push 24 | fi -------------------------------------------------------------------------------- /teamcity/tests.sh: -------------------------------------------------------------------------------- 1 | pip install -r requirements.txt 2 | 3 | pip install flake8 flake8-quotes yapf 4 | 5 | pip install -U pytest pytest-xdist 6 | 7 | pytest -v --forked --numprocesses=auto -------------------------------------------------------------------------------- /yapf.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightforever/mlcomp/c78fdb77ec9c4ec8ff11beea50b90cab20903ad9/yapf.sh --------------------------------------------------------------------------------