├── .gitignore
├── .python-version
├── LICENSE
├── README.md
├── README_JA.md
├── demo.jpg
├── haipera_logo.jpg
├── pyproject.toml
├── requirements-dev.lock
├── requirements.lock
└── src
└── paramit
├── __init__.py
├── cli
├── __init__.py
└── __main__.py
├── constants.py
├── cuda.py
└── nb.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # python generated files
2 | __pycache__/
3 | *.py[oc]
4 | build/
5 | dist/
6 | wheels/
7 | *.egg-info
8 |
9 | # venv
10 | .venv
11 |
12 | # OSX
13 | .DS_Store
14 |
15 | # haipera
16 | reports
17 |
18 | .vscode
19 |
--------------------------------------------------------------------------------
/.python-version:
--------------------------------------------------------------------------------
1 | 3.9
2 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [2024-] [Haipera, Towaki Takikawa & Allen Wang]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## Paramit: Parameterize Python scripts/notebooks all from the command line
2 |
3 | [](https://github.com/haipera/paramit/blob/main/LICENSE)
4 | [](https://github.com/haipera/paramit/stargazers)
5 | [](https://colab.research.google.com/drive/12jY7Kr1Rupj-aJFjlIRgZf1x-nySQdoJ?usp=sharing)
6 | [](https://twitter.com/haipera_ai)
7 |
8 | [日本語のREADMEはこちら！](README_JA.md)
9 |
10 | **Paramit was recently renamed from Haipera to Paramit. Make sure you `pip uninstall haipera` first.**
11 |
12 | Automatically track hyperparameters for your ML models without the boilerplate, and run 100s of experiments all at once, with 1 command.
13 |
14 | Built by Haipera.
15 |
16 | [Sign up on our waitlist for updates!](https://docs.google.com/forms/d/e/1FAIpQLSer1jjQKapYnNbyBCnpMBB4Nv2kmm7MnFp7t25ISYA7mlH6WA/viewform)
17 |
18 | [Join our Discord server!](https://discord.gg/UtHcwJzW)
19 |
20 |
21 | 
22 |
23 |
24 | ## What is Paramit?
25 |
26 | Paramit is an open-source framework to take scripts _and_ notebooks and make them **production ready**.
27 |
28 | - 🦥 **Config files without any code.** Automatically probes the source code to generate reproducible config files.
29 | - 🤖 **Grid search from CLI.** Use the command line to directly iterate through hyperparameters.
30 | - 🪵 **Automatic experiment logging.** Automatically generates per-experiment output folders with reproducible configs.
31 | - ☁️ **Scale to the Cloud (coming soon!).** Run everything locally, or send your model to Haipera Cloud or your own Cloud for parallel experimentation.
32 |
33 | Other general features:
34 |
35 | - supports running `.ipynb` notebook files as scripts
36 | - supports running a notebook server (with configs)
37 | - debug as usual with `pdb`
38 | - supports Windows, Linux, OSX
39 | - saves console logs along with configs
40 | - artifacts (images, models, etc) are also saved to separate experiment folders
41 |
42 | #### What's next for Paramit?
43 |
44 | - bring-your-own-cloud GPU training infrastructure
45 | - automatic logging
46 | - automatic GPU profiling instrumentation
47 | - dashboard for GPU profile analytics w/ LLMs
48 | - experiment management web dashboard
49 |
50 | Let us know at info@haipera.com if you have opinions - or if you have dying problems or needs that you want us to hear! We're all ears.
51 |
52 | ## Getting Started
53 |
54 |
55 |
56 | Install Paramit:
57 |
58 | ```
59 | pip install paramit
60 | ```
61 |
62 | If you want to use the notebook hosting, you can do
63 |
64 | ```
65 | pip install "paramit[notebook]"
66 | ```
67 |
68 | On Linux, you'll have to install a `venv` package, like:
69 |
70 | ```
71 | apt install python3.10-venv
72 | ```
73 |
74 | Make sure you have a `requirements.txt` file where `script.py` or any Python script you want to run is (or alternatively, somewhere in the Git repo for the script).
75 |
76 | ## Example of using paramit
77 |
78 | In a typical project, you may set up a script like:
79 |
80 | ```python3
81 | import numpy
82 |
83 | num_apples = 100
84 | apple_price = 3.0
85 | print("# apples: ", num_apples)
86 | print("price of an apple: ", apple_price)
87 | price = num_apples * apple_price
88 | print("total: ", price)
89 | ```
90 |
91 | And in the same folder, you may have a `requirements.txt` that lists the dependencies:
92 |
93 | ```
94 | numpy
95 | ```
96 |
97 | Say you want to start experimenting with code like this. You'll probably adjust `num_apples` and `apple_price` manually at first, but eventually you'll lose track of what changes caused the differences in the results.
98 |
99 | To properly keep track of things, you may write code to load these variables from command line interfaces, set up a notebook, write dense JSON or YAML files, log the outputs in a logging service, save the outputs / configs in a separate experiment folder, etc. There's a lot of grunt work involved in making experimentation reproducible.
100 |
101 | Paramit is designed to solve this. With paramit you can edit variables on the fly, which you can view with:
102 |
103 | ```
104 | paramit run script.py --help
105 | ```
106 |
107 | By default, paramit will try to use the default `python3` interpreter to run your code. If you want to specify a speciifc Python interpreter to use, set the environment variable:
108 |
109 | ```
110 | PARAMIT_PYTHON_PATH=/path/to/your/python/interpreter
111 | ```
112 |
113 | When you run paramit, you can pass in arguments without ever setting up `argparse`:
114 | ```
115 | paramit run script.py --num-apples 30
116 | ```
117 |
118 | This will also generate a `script.toml` configuration file.
119 |
120 | You can run these generated config files directly:
121 |
122 | ```
123 | paramit run script.toml
124 | ```
125 |
126 | You can also set up grid searches over parameters by:
127 |
128 | ```
129 | paramit run script.py --num-apples 30,60 --apple-price 1.0,2.0
130 | ```
131 |
132 | Running `paramit` will also generate a `reports` folder where you run `paramit` from, with isolated experiment outputs in that folder.
133 |
134 | You can then re-run existing configs reproducibly with:
135 |
136 | ```
137 | paramit run reports/experiment/script.toml
138 | ```
139 |
140 | ## Using paramit with Jupyter Notebooks
141 |
142 | You can even run paramit with Jupyter notebooks! Using `paramit run` on a notebook file will run the notebook as a script. This is convenient when you want to develop your script inside a notebook environment, but then scale out your runs across a bunch of parameters.
143 |
144 | ```
145 | paramit run script.ipynb --num-apples 30,40,50
146 | ```
147 |
148 | If you instead want to spin up a notebook with your chosen config, and have it run in an isolated environment (inside the generated `reports` folder), you can simply run the notebook with `paramit notebook`:
149 |
150 | ```
151 | paramit notebook script.ipynb --num-apples 30
152 | ```
153 |
154 | This will start a notebook server as usual with the provided configs, inside a dedicated folder inside `reports`.
155 |
156 | This turns out to be a convenient way to do _versioning_ for notebooks- if you have a notebook that you want to use for different data or different examples, instead of cloning 8 versions of the same notebook, you can just have a single notebook and 8 different config files for those notebooks!
157 |
158 | You can also run a Python script as a notebook, although usually there are probably not great reasons to do this.
159 |
160 | ## Demo on Google Colab
161 | You can also try our Google Colab version which allows you to run Paramit in the cloud. Check out our Colab demo using the following notebook: [](https://colab.research.google.com/drive/12jY7Kr1Rupj-aJFjlIRgZf1x-nySQdoJ?usp=sharing)
162 |
163 |
164 | ## More examples
165 |
166 | See https://github.com/haipera/haipera-samples for more complex examples that you can try running paramit on.
167 |
168 |
169 | ## Have issues?
170 |
171 | Haipera is still in its early stages, so it'll likely to have bugs. We're actively developing haipera, so if you file a GitHub issue or comment in the Discord server or drop us a line at support@haipera.com we will try to resolve them ASAP!
172 |
--------------------------------------------------------------------------------
/README_JA.md:
--------------------------------------------------------------------------------
1 | ## paramit:コードを書かなくても始められるPythonスクリプトやNotebookの設定管理！
2 |
3 | [](https://github.com/haipera/paramit/blob/main/LICENSE)
4 | [](https://github.com/haipera/paramit/stargazers)
5 | [](https://colab.research.google.com/drive/12jY7Kr1Rupj-aJFjlIRgZf1x-nySQdoJ?usp=sharing)
6 | [](https://twitter.com/haipera_ai)
7 |
8 | えっボイラープレートを書かなくても設定管理ができる？再現性のあるスクリプトが書ける？NotebookをベースにCLIでハイパラサーチできる？勝手にフォルダ分けしてくれる？スクリプトから自動でGPUぶんまわそう！
9 |
10 | [ディスコード!](https://discord.gg/UtHcwJzW)
11 |
12 |
13 |
14 |
15 |
16 | ## paramitってなに？
17 |
18 | paramitは、スクリプトとノートブックからカスタムなコードを書かなくても「実験管理」を可能にするオープンソースフレームワークです。
19 |
20 | - 🦥 **コード不要のコンフィグファイル。** ソースコードを自動解析して再現可能な設定ファイルを生成。
21 | - 🐳 **仮想環境でデプロイして実験の再現性を確保。** 実験の再現性を最大化するため、仮想環境をすべて管理。
22 | - 🤖 **CLIからハイパラチューニング。** コマンドラインから直接ハイパーパラメータを調整したり、グリッドサーチすることが可能。
23 | - 🪵 **自動実験ログ記録。** 再現可能な設定を含む実験ごとの出力フォルダを自動生成。
24 | - ☁️ **クラウドホスティング（近日公開予定！）。** ローカルで全て実行するか、Haiperaクラウドやあなたのクラウドアカウントにモデルを送信して並列実験が可能。
25 |
26 |
27 | その他機能：
28 |
29 | - `pip install paramit` だけでインストール！
30 | - .ipynbノートブックファイルをスクリプトとして実行可能
31 | - ノートブックサーバーの実行をサポート（設定管理付き！）
32 | - 仮想環境のキャッシュ機能
33 | - 通常通りpdbでデバッグ可能
34 | - Windows、Linux、OSXをサポート
35 | - 設定と共にコンソールログを保存
36 | - 成果物（画像、モデルなど）も別の実験フォルダに保存
37 |
38 | #### 実装予定の機能
39 |
40 | - Bring-you-ownクラウドでのGPUトレーニングインフラ
41 | - 自動ログ設定（wandb的な何か）
42 | - GPUプロファイリングの自動計測
43 | - LLMを活用したGPUプロファイル分析ダッシュボード
44 | - 実験管理用Webダッシュボード
45 |
46 | ご意見などありましたら、info@haipera.comまでお知らせください。また、解決したい切実な問題やニーズがあれば、いつでもお聞かせください！Twitterなどでも大丈夫です[@yongyuanxi](https://x.com/yongyuanxi)。
47 |
48 | ## paramitのはじめかた
49 |
50 | インストール：
51 | ```
52 | pip install paramit
53 | ```
54 |
55 | ノートブック機能を使うには：
56 | ```
57 | pip install "paramit[notebook]"
58 | ```
59 |
60 | Linux環境だとVenv用のパッケージをインストールする必要があります。
61 | ```
62 | apt install python3.10-venv
63 | ```
64 |
65 | `script.py`や実行したいPythonスクリプトがある場所（または代替として、スクリプトのGitリポジトリ内のどこか）に`requirements.txt`ファイルがあることを確認してください。
66 |
67 | ## paramitの使い方
68 |
69 | Pythonで色々と実験してる時、以下のようなスクリプトを書くことがあります：
70 |
71 | ```python3
72 | import numpy
73 |
74 | num_apples = 100
75 | apple_price = 3.0
76 | print("# apples: ", num_apples)
77 | print("price of an apple: ", apple_price)
78 | price = num_apples * apple_price
79 | print("total: ", price)
80 | ```
81 |
82 | 同じフォルダに、Dependenciesをリストアップしたrequirements.txtがあるかもしれません：
83 |
84 | ```
85 | numpy
86 | ```
87 |
88 | あまり現実的ではない例ですが、このコードで実験を始めるとします。
89 |
90 | まず最初は`num_apples`と`apple_price`を手動で調整したりして、どう結果が変わるかをみたりします。ただ、スクリプトが複雑化して変数が増えてきたり、メトリックスなどが増えてくるとだんだんとどのパラメータがどの結果とCorrespondするのかがわけわからなくなってきます。
91 |
92 | これらの変数を追跡するには、これらの変数をコマンドラインインターフェースから編集できるようにしたり、ノートブックをセットアップしたり、これを追跡するためのJSONやYAMLファイルを設定したり、ログサービスに出力をログしたり、別々の実験フォルダに出力や設定を保存するなどが必要です。
93 |
94 | 実験を再現可能にするには**多くの作業**が必要で複雑なプロジェクトになると大変な量のボイラープレートが発生したりします。
95 |
96 | paramitはこれを解決するために設計されています。paramitを使用すると、変数を設定管理のフレームワークを使用しなくても編集できます。
97 |
98 | ```
99 | paramit run script.py --help
100 | ```
101 |
102 | paramitを実行すると、`argparse`を設定しなくても引数を渡すことができます：
103 |
104 | ```
105 | paramit run script.py --num-apples 30
106 | ```
107 |
108 | paramitを走らせると、コードを実行するための仮想環境のビルドが呼び出され、`script.toml`設定ファイルが生成されます。
109 |
110 | 生成された設定ファイルを直接実行することもできます：
111 |
112 | ```
113 | paramit run script.toml
114 | ```
115 |
116 | パラメータのグリッドサーチを設定することもできます（例えばこの例だと４つの実験がスケジュールされます）：
117 |
118 | ```
119 | paramit run script.py --num-apples 30,60 --apple-price 1.0,2.0
120 | ```
121 |
122 | paramitを実行すると、paramitを実行した場所にreportsフォルダが生成され、そのフォルダ内に独立した実験出力が保存されます。
123 |
124 | 既存の設定を再現可能に再実行するには：
125 |
126 | ```
127 | paramit run reports/experiment/script.toml
128 | ```
129 |
130 | ## Using paramit with Jupyter Notebooks
131 |
132 | Jupyterノートブックでもparamitを実行できます！
133 |
134 | ノートブックファイルで`paramit run`を使用すると、ノートブックをスクリプトとして実行します。
135 |
136 | これは、ノートブック環境でスクリプトを開発し、その後多くのパラメータにわたって実行をスケールアウトしたい場合に便利です。
137 |
138 | ```
139 | paramit run script.ipynb --num-apples 30,40,50
140 | ```
141 |
142 | CLIからの設定でノートブックを起動し、分離された環境（生成されたreportsフォルダ内）で実行したい場合は、`paramit notebook`を使うことでノートブックサーバーを実行できます：
143 |
144 | ```
145 | paramit notebook script.ipynb --num-apples 30
146 | ```
147 |
148 | これにより、提供された設定で通常通りノートブックサーバーが起動し、reports内の専用フォルダ内で実行されます。
149 |
150 | Reports内で生成された設定（Config）ファイルはノートブックのバージョン管理としても使えたりします。異なるデータや異なる例に使用したいノートブックがある場合、同じノートブックの8つのクローンを作成する代わりに、単一のノートブックと8つの異なる設定ファイルを用意するだけで済みます！
151 |
152 | ## Demo on Google Colab
153 |
154 | クラウドでparamitを実行できるGoogle Colabバージョンも試すことができます: [](https://colab.research.google.com/drive/12jY7Kr1Rupj-aJFjlIRgZf1x-nySQdoJ?usp=sharing)
155 |
156 | ## More examples
157 |
158 | paramitで実行できるより複雑な例については、https://github.com/haipera/haipera-samples をご覧ください。
159 |
160 | ## Have issues?
161 |
162 | paramitはまだ初期段階にあるため、バグがある可能性が高いです。GitHubでイシューを立てるか、Discordサーバーでコメントするか、support@haipera.comまでメールを送っていただければ、できるだけ早く解決するよう努めます！
163 |
--------------------------------------------------------------------------------
/demo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/outerport/paramit/3520957f96fe415c83d1790be4f44ecc36efbcb9/demo.jpg
--------------------------------------------------------------------------------
/haipera_logo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/outerport/paramit/3520957f96fe415c83d1790be4f44ecc36efbcb9/haipera_logo.jpg
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [project]
2 | name = "paramit"
3 | version = "0.2.4"
4 | description = "Parameterize Python scripts/notebooks all from the command line."
5 | authors = [
6 | { name = "Towaki Takikawa", email = "tovacinni@gmail.com" },
7 | { name = "Allen Wang", email = "allen.houze.wang@gmail.com" }
8 | ]
9 | dependencies = [
10 | "pydantic>=2.7.4",
11 | "tomli>=2.0.1",
12 | "tomli-w>=1.0.0",
13 | "gitpython>=3.1.43",
14 | "platformdirs>=4.2.2",
15 | "subprocess-tee>=0.4.2",
16 | "nbconvert>=7.16.4",
17 | "nbformat>=5.10.4",
18 | "ipython>=8.18.0",
19 | "libcst>=1.4.0",
20 | "jupytext>=1.16.3",
21 | ]
22 | readme = "README.md"
23 | requires-python = ">= 3.9"
24 |
25 | [project.scripts]
26 | haipera = "paramit.cli:main"
27 | paramit = "paramit.cli:main"
28 |
29 | [project.optional-dependencies]
30 | notebook = [
31 | "notebook>=7.2.1",
32 | ]
33 |
34 | [build-system]
35 | requires = ["hatchling"]
36 | build-backend = "hatchling.build"
37 |
38 | [tool.rye]
39 | managed = true
40 | dev-dependencies = [
41 | "pytest>=8.2.2",
42 | ]
43 |
44 | [tool.hatch.metadata]
45 | allow-direct-references = true
46 |
47 | [tool.hatch.build.targets.wheel]
48 | packages = ["src/paramit"]
49 |
--------------------------------------------------------------------------------
/requirements-dev.lock:
--------------------------------------------------------------------------------
1 | # generated by rye
2 | # use `rye lock` or `rye sync` to update this lockfile
3 | #
4 | # last locked with the following flags:
5 | # pre: false
6 | # features: []
7 | # all-features: false
8 | # with-sources: false
9 | # generate-hashes: false
10 |
11 | -e file:.
12 | annotated-types==0.7.0
13 | # via pydantic
14 | asttokens==2.4.1
15 | # via stack-data
16 | attrs==23.2.0
17 | # via jsonschema
18 | # via referencing
19 | beautifulsoup4==4.12.3
20 | # via nbconvert
21 | bleach==6.1.0
22 | # via nbconvert
23 | decorator==5.1.1
24 | # via ipython
25 | defusedxml==0.7.1
26 | # via nbconvert
27 | exceptiongroup==1.2.2
28 | # via ipython
29 | # via pytest
30 | executing==2.0.1
31 | # via stack-data
32 | fastjsonschema==2.20.0
33 | # via nbformat
34 | gitdb==4.0.11
35 | # via gitpython
36 | gitpython==3.1.43
37 | # via paramit
38 | importlib-metadata==8.2.0
39 | # via jupyter-client
40 | # via nbconvert
41 | iniconfig==2.0.0
42 | # via pytest
43 | ipython==8.18.1
44 | # via paramit
45 | jedi==0.19.1
46 | # via ipython
47 | jinja2==3.1.4
48 | # via nbconvert
49 | jsonschema==4.23.0
50 | # via nbformat
51 | jsonschema-specifications==2023.12.1
52 | # via jsonschema
53 | jupyter-client==8.6.2
54 | # via nbclient
55 | jupyter-core==5.7.2
56 | # via jupyter-client
57 | # via nbclient
58 | # via nbconvert
59 | # via nbformat
60 | jupyterlab-pygments==0.3.0
61 | # via nbconvert
62 | jupytext==1.16.3
63 | # via paramit
64 | libcst==1.4.0
65 | # via paramit
66 | markdown-it-py==3.0.0
67 | # via jupytext
68 | # via mdit-py-plugins
69 | markupsafe==2.1.5
70 | # via jinja2
71 | # via nbconvert
72 | matplotlib-inline==0.1.7
73 | # via ipython
74 | mdit-py-plugins==0.4.1
75 | # via jupytext
76 | mdurl==0.1.2
77 | # via markdown-it-py
78 | mistune==3.0.2
79 | # via nbconvert
80 | nbclient==0.10.0
81 | # via nbconvert
82 | nbconvert==7.16.4
83 | # via paramit
84 | nbformat==5.10.4
85 | # via jupytext
86 | # via nbclient
87 | # via nbconvert
88 | # via paramit
89 | packaging==24.1
90 | # via jupytext
91 | # via nbconvert
92 | # via pytest
93 | pandocfilters==1.5.1
94 | # via nbconvert
95 | parso==0.8.4
96 | # via jedi
97 | pexpect==4.9.0
98 | # via ipython
99 | platformdirs==4.2.2
100 | # via jupyter-core
101 | # via paramit
102 | pluggy==1.5.0
103 | # via pytest
104 | prompt-toolkit==3.0.47
105 | # via ipython
106 | ptyprocess==0.7.0
107 | # via pexpect
108 | pure-eval==0.2.3
109 | # via stack-data
110 | pydantic==2.8.2
111 | # via paramit
112 | pydantic-core==2.20.1
113 | # via pydantic
114 | pygments==2.18.0
115 | # via ipython
116 | # via nbconvert
117 | pytest==8.3.1
118 | python-dateutil==2.9.0.post0
119 | # via jupyter-client
120 | pyyaml==6.0.1
121 | # via jupytext
122 | # via libcst
123 | pyzmq==26.0.3
124 | # via jupyter-client
125 | referencing==0.35.1
126 | # via jsonschema
127 | # via jsonschema-specifications
128 | rpds-py==0.19.0
129 | # via jsonschema
130 | # via referencing
131 | six==1.16.0
132 | # via asttokens
133 | # via bleach
134 | # via python-dateutil
135 | smmap==5.0.1
136 | # via gitdb
137 | soupsieve==2.5
138 | # via beautifulsoup4
139 | stack-data==0.6.3
140 | # via ipython
141 | subprocess-tee==0.4.2
142 | # via paramit
143 | tinycss2==1.3.0
144 | # via nbconvert
145 | tomli==2.0.1
146 | # via jupytext
147 | # via paramit
148 | # via pytest
149 | tomli-w==1.0.0
150 | # via paramit
151 | tornado==6.4.1
152 | # via jupyter-client
153 | traitlets==5.14.3
154 | # via ipython
155 | # via jupyter-client
156 | # via jupyter-core
157 | # via matplotlib-inline
158 | # via nbclient
159 | # via nbconvert
160 | # via nbformat
161 | typing-extensions==4.12.2
162 | # via ipython
163 | # via pydantic
164 | # via pydantic-core
165 | wcwidth==0.2.13
166 | # via prompt-toolkit
167 | webencodings==0.5.1
168 | # via bleach
169 | # via tinycss2
170 | zipp==3.19.2
171 | # via importlib-metadata
172 |
--------------------------------------------------------------------------------
/requirements.lock:
--------------------------------------------------------------------------------
1 | # generated by rye
2 | # use `rye lock` or `rye sync` to update this lockfile
3 | #
4 | # last locked with the following flags:
5 | # pre: false
6 | # features: []
7 | # all-features: false
8 | # with-sources: false
9 | # generate-hashes: false
10 |
11 | -e file:.
12 | annotated-types==0.7.0
13 | # via pydantic
14 | asttokens==2.4.1
15 | # via stack-data
16 | attrs==23.2.0
17 | # via jsonschema
18 | # via referencing
19 | beautifulsoup4==4.12.3
20 | # via nbconvert
21 | bleach==6.1.0
22 | # via nbconvert
23 | decorator==5.1.1
24 | # via ipython
25 | defusedxml==0.7.1
26 | # via nbconvert
27 | exceptiongroup==1.2.2
28 | # via ipython
29 | executing==2.0.1
30 | # via stack-data
31 | fastjsonschema==2.20.0
32 | # via nbformat
33 | gitdb==4.0.11
34 | # via gitpython
35 | gitpython==3.1.43
36 | # via paramit
37 | importlib-metadata==8.2.0
38 | # via jupyter-client
39 | # via nbconvert
40 | ipython==8.18.1
41 | # via paramit
42 | jedi==0.19.1
43 | # via ipython
44 | jinja2==3.1.4
45 | # via nbconvert
46 | jsonschema==4.23.0
47 | # via nbformat
48 | jsonschema-specifications==2023.12.1
49 | # via jsonschema
50 | jupyter-client==8.6.2
51 | # via nbclient
52 | jupyter-core==5.7.2
53 | # via jupyter-client
54 | # via nbclient
55 | # via nbconvert
56 | # via nbformat
57 | jupyterlab-pygments==0.3.0
58 | # via nbconvert
59 | jupytext==1.16.3
60 | # via paramit
61 | libcst==1.4.0
62 | # via paramit
63 | markdown-it-py==3.0.0
64 | # via jupytext
65 | # via mdit-py-plugins
66 | markupsafe==2.1.5
67 | # via jinja2
68 | # via nbconvert
69 | matplotlib-inline==0.1.7
70 | # via ipython
71 | mdit-py-plugins==0.4.1
72 | # via jupytext
73 | mdurl==0.1.2
74 | # via markdown-it-py
75 | mistune==3.0.2
76 | # via nbconvert
77 | nbclient==0.10.0
78 | # via nbconvert
79 | nbconvert==7.16.4
80 | # via paramit
81 | nbformat==5.10.4
82 | # via jupytext
83 | # via nbclient
84 | # via nbconvert
85 | # via paramit
86 | packaging==24.1
87 | # via jupytext
88 | # via nbconvert
89 | pandocfilters==1.5.1
90 | # via nbconvert
91 | parso==0.8.4
92 | # via jedi
93 | pexpect==4.9.0
94 | # via ipython
95 | platformdirs==4.2.2
96 | # via jupyter-core
97 | # via paramit
98 | prompt-toolkit==3.0.47
99 | # via ipython
100 | ptyprocess==0.7.0
101 | # via pexpect
102 | pure-eval==0.2.3
103 | # via stack-data
104 | pydantic==2.8.2
105 | # via paramit
106 | pydantic-core==2.20.1
107 | # via pydantic
108 | pygments==2.18.0
109 | # via ipython
110 | # via nbconvert
111 | python-dateutil==2.9.0.post0
112 | # via jupyter-client
113 | pyyaml==6.0.1
114 | # via jupytext
115 | # via libcst
116 | pyzmq==26.0.3
117 | # via jupyter-client
118 | referencing==0.35.1
119 | # via jsonschema
120 | # via jsonschema-specifications
121 | rpds-py==0.19.0
122 | # via jsonschema
123 | # via referencing
124 | six==1.16.0
125 | # via asttokens
126 | # via bleach
127 | # via python-dateutil
128 | smmap==5.0.1
129 | # via gitdb
130 | soupsieve==2.5
131 | # via beautifulsoup4
132 | stack-data==0.6.3
133 | # via ipython
134 | subprocess-tee==0.4.2
135 | # via paramit
136 | tinycss2==1.3.0
137 | # via nbconvert
138 | tomli==2.0.1
139 | # via jupytext
140 | # via paramit
141 | tomli-w==1.0.0
142 | # via paramit
143 | tornado==6.4.1
144 | # via jupyter-client
145 | traitlets==5.14.3
146 | # via ipython
147 | # via jupyter-client
148 | # via jupyter-core
149 | # via matplotlib-inline
150 | # via nbclient
151 | # via nbconvert
152 | # via nbformat
153 | typing-extensions==4.12.2
154 | # via ipython
155 | # via pydantic
156 | # via pydantic-core
157 | wcwidth==0.2.13
158 | # via prompt-toolkit
159 | webencodings==0.5.1
160 | # via bleach
161 | # via tinycss2
162 | zipp==3.19.2
163 | # via importlib-metadata
164 |
--------------------------------------------------------------------------------
/src/paramit/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/outerport/paramit/3520957f96fe415c83d1790be4f44ecc36efbcb9/src/paramit/__init__.py
--------------------------------------------------------------------------------
/src/paramit/cli/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 | import ast
4 | import libcst as cst
5 | from libcst.metadata import PositionProvider
6 | import datetime
7 | from typing import List, Any, Dict, Union
8 | from pydantic import BaseModel
9 | import tomli
10 | import tomli_w
11 | import uuid
12 | import enum
13 | import subprocess
14 | from copy import deepcopy
15 | import subprocess_tee
16 | import tempfile
17 | import shutil
18 | from paramit.nb import (
19 | convert_ipynb_to_py,
20 | convert_source_code_to_ipynb,
21 | is_jupyter_kernel_installed,
22 | )
23 | from paramit.constants import YELLOW, MAGENTA, GREEN, RED, RESET
24 |
25 | sys.stdout.reconfigure(line_buffering=True)
26 |
27 |
28 | class ParamitMode(enum.Enum):
29 | RUN = "run"
30 | CLOUD = "cloud"
31 | NOTEBOOK = "notebook"
32 |
33 |
34 | class ParamitVariable(BaseModel):
35 | name: str
36 | value: Any
37 | type: str
38 | file_name: str
39 | line: int
40 | column: int
41 |
42 | def __str__(self):
43 | return (
44 | f"{self.name} = {self.value} ({self.type}) [{self.file_name}:{self.line}]"
45 | )
46 |
47 |
48 | class ParamitMetadata(BaseModel):
49 | version: str
50 | created_on: str
51 | script_path: str
52 | python_path: str
53 |
54 |
55 | class ParamitParameter(BaseModel):
56 | name: str
57 | type: str
58 | values: List[Any]
59 |
60 |
61 | class VariableVisitor(cst.CSTVisitor):
62 | METADATA_DEPENDENCIES = (PositionProvider,)
63 |
64 | def __init__(self, file_path: str):
65 | self.file_path = file_path
66 | self.variables = []
67 | self.current_context = []
68 |
69 | def visit_Assign(self, node: cst.Assign):
70 | for target in node.targets:
71 | if isinstance(target.target, cst.Name) and isinstance(
72 | node.value, cst.Integer
73 | ):
74 | pos = self.get_metadata(PositionProvider, node).start
75 | self.add_variable(
76 | target.target.value, node.value.value, pos.line, pos.column
77 | )
78 | elif isinstance(target.target, cst.Name) and isinstance(
79 | node.value, cst.Float
80 | ):
81 | pos = self.get_metadata(PositionProvider, node).start
82 | self.add_variable(
83 | target.target.value, node.value.value, pos.line, pos.column
84 | )
85 | elif isinstance(target.target, cst.Name) and isinstance(
86 | node.value, cst.SimpleString
87 | ):
88 | pos = self.get_metadata(PositionProvider, node).start
89 | self.add_variable(
90 | target.target.value,
91 | node.value.value.strip("'\""),
92 | pos.line,
93 | pos.column,
94 | )
95 | elif isinstance(target.target, cst.Name) and isinstance(
96 | node.value, cst.Name
97 | ):
98 | if node.value.value == "True" or node.value.value == "False":
99 | pos = self.get_metadata(PositionProvider, node).start
100 | value = True if node.value.value == "True" else False
101 | self.add_variable(target.target.value, value, pos.line, pos.column)
102 | elif (
103 | isinstance(target.target, cst.Attribute)
104 | and isinstance(node.value, cst.SimpleString)
105 | and isinstance(target.target.value, cst.Name)
106 | and target.target.value.value == "self"
107 | ):
108 | pass # Disabled for now
109 |
110 | def visit_Call(self, node: cst.Call):
111 | if isinstance(node.func, cst.Name):
112 | self.current_context.append(node.func.value)
113 | elif isinstance(node.func, cst.Attribute):
114 | self.current_context.append(node.func.attr.value)
115 |
116 | # Disabled for now
117 | """
118 | for arg in node.args:
119 | if isinstance(arg, cst.Arg) and isinstance(arg.value, cst.SimpleString):
120 | self.add_variable(arg.keyword.value if arg.keyword else None, arg.value)
121 | """
122 |
123 | def leave_Call(self, original_node: cst.Call):
124 | if self.current_context:
125 | self.current_context.pop()
126 |
127 | def visit_FunctionDef(self, node: cst.FunctionDef):
128 | pass
129 | """
130 | if node.name.value == "__init__":
131 | params = node.params
132 | if params.default_params:
133 | for param in params.default_params:
134 | if isinstance(param.default, cst.SimpleString):
135 | pass # Disabled for now
136 | # self.add_variable(param.name.value, param.default)
137 | """
138 |
139 | def visit_ClassDef(self, node: cst.ClassDef):
140 | self.current_context.append(node.name.value)
141 | for base in node.bases:
142 | if isinstance(base.value, cst.Call):
143 | self.visit_Call(base.value)
144 |
145 | def leave_ClassDef(self, original_node: cst.ClassDef):
146 | self.current_context.pop()
147 |
148 | def add_variable(self, name: str, value: Any, line: int, column: int):
149 | full_name = ".".join(self.current_context + [name]) if name else ""
150 | self.variables.append(
151 | ParamitVariable(
152 | name=full_name,
153 | value=value,
154 | type=type(value).__name__,
155 | file_name=os.path.basename(self.file_path),
156 | line=line,
157 | column=column,
158 | )
159 | )
160 |
161 |
162 | class VariableTransformer(cst.CSTTransformer):
163 | METADATA_DEPENDENCIES = (PositionProvider,)
164 |
165 | ## TODO: Clean all the transformation code
166 | ## Instead of taking in a config file, this should take in the same
167 | ## ParamitVariable objects that the VariableVisitor generates
168 | ## and check against line and column numbers
169 | def __init__(self, config: Dict[str, Union[str, int, float, bool]]):
170 | self.config = config
171 |
172 | def leave_Assign(
173 | self, original_node: cst.Assign, updated_node: cst.Assign
174 | ) -> cst.Assign:
175 | if len(original_node.targets) == 1:
176 | target = original_node.targets[0].target
177 | if isinstance(target, cst.Name):
178 | name = target.value
179 | # pos = self.get_metadata(PositionProvider, original_node).start
180 | if name in self.config:
181 | value = self.config[name]
182 | if isinstance(original_node.value, cst.SimpleString):
183 | value_node = cst.SimpleString(value=f"'{value}'")
184 | elif isinstance(
185 | original_node.value, cst.Name
186 | ) and original_node.value.value in ["True", "False"]:
187 | value_node = cst.Name(value="True" if value else "False")
188 | elif isinstance(original_node.value, cst.Integer):
189 | value_node = cst.Integer(str(value))
190 | elif isinstance(original_node.value, cst.Float):
191 | value_node = cst.Float(str(value))
192 | else:
193 | raise ValueError(f"Unsupported type {type(value)}")
194 | return updated_node.with_changes(value=value_node)
195 | return updated_node
196 |
197 |
198 | def find_variables(tree: cst.Module, path: str) -> List[ParamitVariable]:
199 | """Find all variables, their values, and types in the given CST tree."""
200 | visitor = VariableVisitor(file_path=path)
201 | wrapper = cst.MetadataWrapper(tree)
202 | wrapper.visit(visitor)
203 | return visitor.variables
204 |
205 |
206 | def expand_paths_in_global_variables(
207 | global_vars: List[ParamitVariable], script_path: str
208 | ) -> List[ParamitVariable]:
209 | """Expand the path in the given global variables using the script path."""
210 | expanded_vars = []
211 | for var in global_vars:
212 | if isinstance(var.value, str) and var.value != "":
213 | expanded_path = os.path.abspath(
214 | os.path.join(os.path.dirname(script_path), var.value)
215 | )
216 | if os.path.exists(expanded_path):
217 | expanded_vars.append(
218 | ParamitVariable(
219 | name=var.name,
220 | value=expanded_path,
221 | type=var.type,
222 | file_name=var.file_name,
223 | line=var.line,
224 | column=var.column,
225 | )
226 | )
227 | else:
228 | expanded_vars.append(var)
229 | else:
230 | expanded_vars.append(var)
231 | return expanded_vars
232 |
233 |
234 | def generate_config_file(
235 | tree: cst.Module,
236 | path: str,
237 | ) -> Dict[str, Any]:
238 | """Generate a TOML configuration file with the given global variables."""
239 | global_vars = find_variables(tree, path)
240 | global_vars = expand_paths_in_global_variables(global_vars, path)
241 | script_path = path.replace(".toml", ".py")
242 |
243 | config = {"global": {}, "meta": {}}
244 | for var in global_vars:
245 | parts = var.name.split(".")
246 | if len(parts) == 1:
247 | config["global"][parts[0]] = var.value
248 | else:
249 | current_dict = config["global"]
250 | for part in parts[:-1]:
251 | if part not in current_dict:
252 | current_dict[part] = {}
253 | current_dict = current_dict[part]
254 | current_dict[parts[-1]] = var.value
255 |
256 | python_path = get_python_path()
257 |
258 | metadata = ParamitMetadata(
259 | version="0.2.4",
260 | created_on=str(datetime.datetime.now()),
261 | script_path=os.path.abspath(script_path),
262 | python_path=python_path if python_path else "",
263 | )
264 |
265 | config["meta"] = metadata.model_dump()
266 |
267 | return config
268 |
269 |
270 | def load_config_file(path: str) -> dict:
271 | """Load a TOML configuration file from the given path
272 | and return it as a dictionary."""
273 |
274 | with open(path, "rb") as f:
275 | return tomli.load(f)
276 |
277 |
278 | def set_global_variables_from_config(
279 | tree: cst.Module, config: Dict[str, Dict[str, Union[str, int, float, bool]]]
280 | ) -> cst.Module:
281 | """Set global variables in the given CST tree using the values from the config dictionary."""
282 | transformer = VariableTransformer(config["global"])
283 | wrapper = cst.MetadataWrapper(tree)
284 | modified_tree = wrapper.visit(transformer)
285 | return modified_tree
286 |
287 |
288 | def help_in_args(args: List[str]) -> bool:
289 | """Check if the help flag is in the given list of arguments."""
290 | return any(arg in args for arg in ["-h", "--help"])
291 |
292 |
293 | def parse_args(args: List[str]) -> Dict[str, Any]:
294 | """Parse the given list of arguments into a dictionary."""
295 | args_dict = {}
296 | for arg_index, arg in enumerate(args):
297 | if arg in ["--help", "--h"]:
298 | continue
299 | if arg.startswith("--"):
300 | if "=" in arg:
301 | key, value = arg.split("=")
302 | else:
303 | key = arg
304 | if arg_index + 1 >= len(args):
305 | print(f"{RED}Error: Argument {arg} is missing a value{RESET}")
306 | sys.exit(1)
307 |
308 | value = ""
309 | while arg_index + 1 < len(args) and not args[arg_index + 1].startswith(
310 | "--"
311 | ):
312 | if value:
313 | value += ","
314 | value += args[arg_index + 1]
315 | arg_index += 1
316 |
317 | key = key[2:].replace("-", "_")
318 | args_dict[key] = value
319 | return args_dict
320 |
321 |
322 | def expand_args_dict(args_dict: Dict[str, str]) -> Dict[str, ParamitParameter]:
323 | """Parse the value in the args according to the special paramit syntax.
324 |
325 | The syntax is as follows:
326 | "123,126,128" -> [123, 126, 128]
327 | "blue, red" -> ["blue", "red"]
328 | "blue red" -> ["blue", "red"]
329 | """
330 |
331 | hyperparameters = {}
332 | for arg, value in args_dict.items():
333 | if "," in value:
334 | values = value.split(",")
335 | else:
336 | values = [value]
337 |
338 | if not values:
339 | print(f"{RED}Error: Argument {arg} must have at least one value{RESET}")
340 | sys.exit(1)
341 |
342 | value_type = None
343 | if all(v.isdigit() for v in values):
344 | value_type = int
345 | values = [value_type(v) for v in values]
346 | elif all(v.replace(".", "").replace("e", "").isdigit() for v in values):
347 | value_type = float
348 | values = [value_type(v) for v in values]
349 | elif all(v.lower() in ["true", "false"] for v in values):
350 | value_type = bool
351 |
352 | def str_to_bool(value):
353 | if value.lower() == "true":
354 | return True
355 | elif value.lower() == "false":
356 | return False
357 | else:
358 | print(f"{RED}Error: Bool argument must be True or False{RESET}")
359 | sys.exit(1)
360 |
361 | values = [str_to_bool(v) for v in values]
362 |
363 | else:
364 | value_type = str
365 | values = [value_type(v) for v in values]
366 | # Make paths absolute if the argument exists as a path
367 | for i, v in enumerate(values):
368 | if os.path.exists(v):
369 | values[i] = os.path.abspath(v)
370 |
371 |
372 | hyperparameters[arg] = ParamitParameter(
373 | name=arg, type=type(values[0]).__name__, values=values
374 | )
375 |
376 | return hyperparameters
377 |
378 |
379 | def pretty_print_config(config: Dict[str, Any]) -> None:
380 | """Pretty print the config as parameters that can be passed to the CLI."""
381 | print("Arguments:")
382 | for key, value in config["global"].items():
383 | print(f" --{key.replace('_', '-')}={value}")
384 | print("\nMetadata:")
385 | for key, value in config["meta"].items():
386 | print(f" {key}: {value}")
387 |
388 |
389 | def generate_configs_from_hyperparameters(
390 | base_config: Dict[str, Any], hyperparameters: Dict[str, ParamitParameter]
391 | ) -> List[Dict[str, Any]]:
392 | """Generate a list of configurations from the base config and hyperparameters."""
393 |
394 | hyperparameters_range: List[ParamitParameter] = []
395 | hyperparameters_single: List[ParamitParameter] = []
396 | for hyperparameter in hyperparameters:
397 | if len(hyperparameters[hyperparameter].values) > 1:
398 | hyperparameters_range.append(hyperparameters[hyperparameter])
399 | else:
400 | hyperparameters_single.append(hyperparameters[hyperparameter])
401 |
402 | for hyperparameter in hyperparameters_single:
403 | if hyperparameter.name in base_config["global"]:
404 | try:
405 | base_config["global"][hyperparameter.name] = type(
406 | base_config["global"][hyperparameter.name]
407 | )(hyperparameter.values[0])
408 | except ValueError:
409 | print(
410 | f"{RED}Error: Argument {hyperparameter.name} must be of type {type(base_config['global'][hyperparameter.name])}{RESET}"
411 | )
412 | sys.exit(1)
413 | else:
414 | print(
415 | f"{RED}Error: Argument {hyperparameter.name} not found in the code or config{RESET}"
416 | )
417 | # Print the available arguments
418 | pretty_print_config(base_config)
419 | sys.exit(1)
420 |
421 | if not hyperparameters_range:
422 | return [base_config]
423 |
424 | # Generate all possible combinations of hyperparameters
425 | hyperparameters_combinations = []
426 | for i in range(len(hyperparameters_range)):
427 | hyperparameter = hyperparameters_range[i]
428 | if not hyperparameters_combinations:
429 | for value in hyperparameter.values:
430 | hyperparameters_combinations.append({hyperparameter.name: value})
431 | else:
432 | new_combinations = []
433 | for combination in hyperparameters_combinations:
434 | for value in hyperparameter.values:
435 | new_combination = combination.copy()
436 | new_combination[hyperparameter.name] = value
437 | new_combinations.append(new_combination)
438 | hyperparameters_combinations = new_combinations
439 |
440 | configs: List[Dict[str, Any]] = []
441 |
442 | for combination in hyperparameters_combinations:
443 | config = deepcopy(base_config)
444 | for key, value in combination.items():
445 | if key in config["global"]:
446 | try:
447 | config["global"][key] = type(config["global"][key])(value)
448 | except ValueError:
449 | print(
450 | f"{RED}Error: Argument {key} must be of type {type(config['global'][key])}{RESET}"
451 | )
452 | sys.exit(1)
453 | else:
454 | print(
455 | f"{RED}Error: Argument {key} not found in the code or config{RESET}"
456 | )
457 | pretty_print_config(base_config)
458 | sys.exit(1)
459 |
460 | configs.append(config)
461 |
462 | return configs
463 |
464 |
465 | def print_usage():
466 | print(
467 | f"{MAGENTA}Usage: paramit [run | cloud | notebook] {RESET}"
468 | )
469 | print()
470 | print("commands")
471 | print(" run - Run the Python script or notebook")
472 | print(" cloud - Run the Python script or notebook on the cloud")
473 | print(" notebook - Start a Jupyter notebook server with the script or notebook")
474 |
475 |
476 | def get_python_path() -> str:
477 | # Check a paramit specific environment variable
478 | if "HAIPERA_PYTHON_PATH" in os.environ:
479 | return os.environ["HAIPERA_PYTHON_PATH"]
480 |
481 | # Check for VIRTUAL_ENV first (covers venv and conda environments)
482 | if "VIRTUAL_ENV" in os.environ:
483 | return os.path.join(os.environ["VIRTUAL_ENV"], "bin", "python")
484 |
485 | # Check for CONDA_PREFIX (specific to conda environments)
486 | if "CONDA_PREFIX" in os.environ:
487 | return os.path.join(os.environ["CONDA_PREFIX"], "bin", "python")
488 |
489 | # Look for python3 or python in PATH
490 | for cmd in ["python3", "python"]:
491 | python_path = shutil.which(cmd)
492 | if python_path:
493 | return python_path
494 |
495 | # If still not found, try common locations
496 | common_locations = [
497 | "/usr/bin/python3",
498 | "/usr/local/bin/python3",
499 | "/usr/bin/python",
500 | "/usr/local/bin/python",
501 | "C:\\Python\\python.exe",
502 | "C:\\Program Files\\Python\\python.exe",
503 | ]
504 | for location in common_locations:
505 | if os.path.exists(location):
506 | return location
507 | raise FileNotFoundError(
508 | "Could not find a Python interpreter. Please set the HAIPERA_PYTHON_PATH environment variable."
509 | )
510 |
511 |
512 | def is_package_installed(package_name: str) -> bool:
513 | python_path = get_python_path()
514 | try:
515 | result = subprocess.run(
516 | [python_path, "-m", "pip", "show", package_name],
517 | capture_output=True,
518 | text=True,
519 | )
520 | return result.returncode == 0
521 | except subprocess.CalledProcessError:
522 | return False
523 |
524 |
525 | def run_code(source_code: str, python_path: str, cwd: str, script_path: str) -> None:
526 | with tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
527 | # Write the __file__ variable at the top of the file to the original script path
528 | temp_file.write(f"__file__ = {repr(os.path.abspath(script_path))}\n")
529 |
530 | # Write the code to set the original directory as the working directory
531 | temp_file.write(f"import os\nos.chdir({repr(os.path.dirname(os.path.abspath(script_path)))})\n")
532 |
533 | temp_file.write(source_code)
534 | temp_file_path = temp_file.name
535 |
536 | log_file_path = os.path.join(cwd, "console.log")
537 | try:
538 | output = subprocess_tee.run(
539 | f"{python_path} -u {temp_file_path}",
540 | cwd=cwd,
541 | shell=True,
542 | )
543 |
544 | # Save
545 | with open(log_file_path, "w") as f:
546 | f.write(output.stdout)
547 |
548 | except subprocess.CalledProcessError as e:
549 | return e.stderr
550 | finally:
551 | os.unlink(temp_file_path)
552 |
553 |
554 | def main():
555 | if len(sys.argv) < 3:
556 | print_usage()
557 | sys.exit(1)
558 |
559 | try:
560 | mode = ParamitMode(sys.argv[1])
561 | except ValueError:
562 | print_usage()
563 | sys.exit(1)
564 |
565 | mode = ParamitMode(sys.argv[1])
566 |
567 | if mode == ParamitMode.CLOUD:
568 | print(
569 | f"{MAGENTA}Cloud runs are in development. Please sign up on the waitlist for updates at https://www.haipera.com{RESET}"
570 | )
571 | sys.exit(1)
572 |
573 | path = sys.argv[2]
574 | if not os.path.exists(path):
575 | print(f"{RED}Error: File {path} does not exist{RESET}")
576 | sys.exit(1)
577 |
578 | cli_args = parse_args(sys.argv[3:])
579 | hyperparameters = expand_args_dict(cli_args)
580 |
581 | if (
582 | not path.endswith(".py")
583 | and not path.endswith(".toml")
584 | and not path.endswith(".ipynb")
585 | ):
586 | print(
587 | f"{RED}Error: File {path} is not a Python or TOML or Notebook file{RESET}"
588 | )
589 | sys.exit(1)
590 |
591 | if path.endswith(".toml"):
592 | config = load_config_file(path)
593 |
594 | try:
595 | ParamitMetadata(**config["meta"])
596 | except Exception:
597 | print(
598 | f"{RED}Error: The config file is not a valid paramit config file{RESET}"
599 | )
600 |
601 | if not os.path.exists(config["meta"]["script_path"]):
602 | print(
603 | f"{RED}Error: Python file {config['meta']['script_path']} does not exist{RESET}"
604 | )
605 | sys.exit(1)
606 |
607 | with open(config["meta"]["script_path"], "r") as f:
608 | if config["meta"]["script_path"].endswith(".ipynb"):
609 | code = convert_ipynb_to_py(config["meta"]["script_path"])
610 | elif config["meta"]["script_path"].endswith(".py"):
611 | code = f.read()
612 | else:
613 | print(
614 | f"{RED}Error: Python file {config['meta']['script_path']} is not a Python or Notebook file{RESET}"
615 | )
616 | sys.exit(1)
617 |
618 | elif path.endswith(".py"):
619 | with open(path, "r") as f:
620 | code = f.read()
621 | elif path.endswith(".ipynb"):
622 | code = convert_ipynb_to_py(path)
623 |
624 | try:
625 | # We do an extra check here to catch syntax errors w/ helpful messages
626 | ast.parse(code)
627 | except SyntaxError as e:
628 | e.filename = path
629 | print(f"{RED}SyntaxError: {e}{RESET}")
630 | sys.exit(1)
631 |
632 | tree = cst.parse_module(code)
633 |
634 | config_path = path.replace(".py", ".toml").replace(".ipynb", ".toml")
635 |
636 | if help_in_args(sys.argv[3:]):
637 | generated_config_file = generate_config_file(tree, path)
638 | print(f"{MAGENTA}Usage: paramit run [args]{RESET}")
639 | pretty_print_config(generated_config_file)
640 | sys.exit(0)
641 |
642 | elif not os.path.exists(config_path):
643 | generated_config = generate_config_file(tree, path)
644 | with open(config_path, "wb") as f:
645 | tomli_w.dump(generated_config, f)
646 |
647 | elif not path.endswith(".toml"):
648 | print(
649 | f"{YELLOW}Warning: Configuration file {config_path} already exists{RESET}"
650 | )
651 | overwrite = input("Do you want to overwrite it? (y/n): ")
652 | if overwrite.lower() == "y":
653 | generated_config = generate_config_file(tree, path)
654 |
655 | with open(config_path, "wb") as f:
656 | tomli_w.dump(generated_config, f)
657 |
658 | config = load_config_file(config_path)
659 | python_path = config["meta"]["python_path"]
660 | orig_script_path = config["meta"]["script_path"]
661 |
662 | experiment_configs = generate_configs_from_hyperparameters(config, hyperparameters)
663 |
664 | if len(experiment_configs) > 100:
665 | print(f"{YELLOW}Warning: Running {len(experiment_configs)} experiments{RESET}")
666 | confirm = input("Do you want to continue? (y/n): ")
667 | if confirm.lower() != "y":
668 | sys.exit(0)
669 | elif len(experiment_configs) == 0:
670 | print(f"{YELLOW}Warning: No experiments to run{RESET}")
671 | sys.exit(0)
672 | elif len(experiment_configs) == 1:
673 | pass
674 | else:
675 | print(f"{GREEN}Running {len(experiment_configs)} experiments{RESET}")
676 |
677 | if mode == ParamitMode.NOTEBOOK and len(experiment_configs) > 1:
678 | print("Notebook mode only supports running a single experiment")
679 | sys.exit(1)
680 |
681 | for experiment_config in experiment_configs:
682 | experiment_id = (
683 | datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
684 | + "-"
685 | + str(uuid.uuid4())[0:8]
686 | )
687 | experiment_dir = os.path.join("reports", experiment_id)
688 | os.makedirs(experiment_dir, exist_ok=True)
689 | base_name = os.path.splitext(os.path.basename(path))[0]
690 |
691 | # Save the config file in the experiment directory
692 | with open(os.path.join(experiment_dir, base_name + ".toml"), "wb") as f:
693 | tomli_w.dump(experiment_config, f)
694 |
695 | modified_tree = set_global_variables_from_config(tree, experiment_config)
696 |
697 | source_code = modified_tree.code
698 |
699 | if mode == ParamitMode.RUN:
700 | with open(os.path.join(experiment_dir, base_name + ".py"), "w") as f:
701 | f.write(source_code)
702 |
703 | if path.endswith(".ipynb"):
704 | notebook_path = os.path.join(experiment_dir, base_name + ".ipynb")
705 | with open(notebook_path, "w") as f:
706 | f.write(convert_source_code_to_ipynb(source_code))
707 |
708 | print(f"Running with the Python interpreter at {python_path}")
709 | run_code(source_code, python_path, experiment_dir, orig_script_path)
710 |
711 | elif mode == ParamitMode.NOTEBOOK:
712 | ipykernel_is_installed = is_package_installed("ipykernel")
713 | if not ipykernel_is_installed:
714 | print(
715 | "ipykernel is not installed. Please install it to use notebook mode"
716 | )
717 | sys.exit(1)
718 |
719 | notebook_path = os.path.join(experiment_dir, base_name + ".ipynb")
720 | with open(notebook_path, "w") as f:
721 | f.write(convert_source_code_to_ipynb(source_code))
722 | print("Starting Jupyter notebook server!\n")
723 | kernel_name = os.path.basename(os.path.dirname(path))
724 | if not is_jupyter_kernel_installed(kernel_name):
725 | subprocess.run(
726 | [
727 | python_path,
728 | "-m",
729 | "ipykernel",
730 | "install",
731 | "--name",
732 | kernel_name,
733 | "--user",
734 | ],
735 | check=True,
736 | )
737 | subprocess.run(
738 | [
739 | "jupyter",
740 | "notebook",
741 | notebook_path,
742 | "--MultiKernelManager.default_kernel_name",
743 | kernel_name,
744 | "--notebook-dir",
745 | experiment_dir,
746 | ],
747 | check=True,
748 | )
749 |
--------------------------------------------------------------------------------
/src/paramit/cli/__main__.py:
--------------------------------------------------------------------------------
1 | from . import main
2 |
3 | if __name__ == "__main__":
4 | main()
--------------------------------------------------------------------------------
/src/paramit/constants.py:
--------------------------------------------------------------------------------
1 | MAGENTA = "\033[95m"
2 | YELLOW = "\033[33m"
3 | GREEN = "\033[32m"
4 | RED = "\033[91m"
5 | RESET = "\033[0m"
6 |
--------------------------------------------------------------------------------
/src/paramit/cuda.py:
--------------------------------------------------------------------------------
1 | import subprocess
2 | import re
3 | from typing import Optional, Tuple
4 |
5 | __all__ = ["get_cuda_version"]
6 |
7 |
8 | def get_cuda_version() -> Optional[Tuple[int, int]]:
9 | try:
10 | output = subprocess.check_output(["nvcc", "--version"]).decode("utf-8")
11 | version = re.search(r"release (\S+),", output)
12 | if version:
13 | version = version.group(1)
14 | major, minor = version.split(".")
15 | return int(major), int(minor)
16 | except Exception:
17 | return None
18 |
--------------------------------------------------------------------------------
/src/paramit/nb.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | import jupytext
4 | from jupyter_core.paths import jupyter_path
5 |
6 | __all__ = [
7 | "convert_ipynb_to_py",
8 | "convert_py_to_ipynb",
9 | "convert_source_code_to_ipynb",
10 | "is_jupyter_kernel_installed",
11 | ]
12 |
13 |
14 | def convert_ipynb_to_py(ipynb_file: str) -> str:
15 | notebook = jupytext.read(ipynb_file)
16 | py_contents = jupytext.writes(notebook, fmt="py:percent")
17 | return py_contents
18 |
19 |
20 | def convert_py_to_ipynb(py_file: str) -> str:
21 | notebook = jupytext.read(py_file)
22 | ipynb_contents = jupytext.writes(notebook, fmt="ipynb")
23 | return ipynb_contents
24 |
25 |
26 | def convert_source_code_to_ipynb(source_code: str) -> str:
27 | notebook = jupytext.reads(source_code, fmt="py:percent")
28 | ipynb_contents = jupytext.writes(notebook, fmt="ipynb")
29 | return ipynb_contents
30 |
31 |
32 | def is_jupyter_kernel_installed(kernel_name):
33 | for kernel_dir in jupyter_path("kernels"):
34 | spec_file = os.path.join(kernel_dir, kernel_name, "kernel.json")
35 | if os.path.exists(spec_file):
36 | return True
37 | return False
38 |
--------------------------------------------------------------------------------