├── .gitignore ├── LICENSE ├── MANIFEST ├── README.md ├── setup.py └── tensorboardcolab ├── __init__.py ├── callbacks.py └── core.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Tommy Tao 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | setup.py 3 | tensorboardcolab/__init__.py 4 | tensorboardcolab/callbacks.py 5 | tensorboardcolab/core.py 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorBoardColab 2 | 3 | A library make TensorBoard working in Colab Google 4 | 5 | ## Install 6 | 7 | pip install tensorboardcolab 8 | 9 | In Colab Google Jupyter, for auto install and ensure using latest version of TensorBoardColab, please add "!pip install -U tensorboardcolab" at the first line of Jupyter cell 10 | 11 | ## Requirements 12 | 13 | Tensorflow 14 | TensorBoard 15 | npm 16 | 17 | ## Import 18 | 19 | from tensorboardcolab import * 20 | 21 | ## Initialization 22 | 23 | tbc=TensorBoardColab() 24 | 25 | After initialization, TensorBoard link will be shown in Colab Google Juyter output 26 | 27 | PS: If Initialization failed and keep retrying forever, please increase startup_waiting_time larger than 8 seconds as below 28 | 29 | tbc=TensorBoardColab(startup_waiting_time=30) 30 | 31 | ## Add to Keras callback 32 | 33 | model.fit(x,y,epochs=100000,callbacks=[TensorBoardColabCallback(tbc)]) 34 | 35 | ## Save picture to TensorBoard 36 | 37 | tbc.save_image(title="test_title", image=image) 38 | 39 | ## Save a value to graph of TensorBoard 40 | 41 | tbc.save_value("graph_name", "line_name", epoch, value) 42 | . 43 | . 44 | . 45 | tbc.flush_line(line_name) 46 | tbc.close() 47 | 48 | ## Thanks 49 | 50 | ngrok ! 51 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='tensorboardcolab', 5 | version='0.0.21', 6 | packages=['tensorboardcolab'], 7 | url='https://github.com/taomanwai/tensorboardcolab', 8 | license='MIT', 9 | author='Tommy Tao', 10 | author_email='o_otaotao@hotmail.com', 11 | description='' 12 | ) 13 | -------------------------------------------------------------------------------- /tensorboardcolab/__init__.py: -------------------------------------------------------------------------------- 1 | from tensorboardcolab.core import * 2 | from tensorboardcolab.callbacks import * -------------------------------------------------------------------------------- /tensorboardcolab/callbacks.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from keras.callbacks import TensorBoard 3 | import time 4 | import os 5 | import io 6 | 7 | from tensorboardcolab.core import TensorBoardColab 8 | 9 | 10 | class TensorBoardColab: 11 | def __init__(self, port=6006, graph_path='./Graph', startup_waiting_time=8): 12 | self.port = port 13 | self.graph_path = graph_path 14 | self.writer = None 15 | self.deep_writers = {} 16 | self.eager_execution = None 17 | get_ipython().system_raw('npm i -s -q --unsafe-perm -g ngrok') # sudo npm i -s -q --unsafe-perm -g ngrok 18 | 19 | setup_passed = False 20 | retry_count = 0 21 | sleep_time = startup_waiting_time / 3.0 22 | while not setup_passed: 23 | get_ipython().system_raw('kill -9 $(sudo lsof -t -i:%d)' % port) 24 | get_ipython().system_raw('rm -Rf ' + graph_path) 25 | print('Wait for %d seconds...' % startup_waiting_time) 26 | time.sleep(sleep_time) 27 | get_ipython().system_raw('tensorboard --logdir %s --host 0.0.0.0 --port %d &' % (graph_path, port)) 28 | time.sleep(sleep_time) 29 | get_ipython().system_raw('ngrok http %d &' % port) 30 | time.sleep(sleep_time) 31 | try: 32 | tensorboard_link = get_ipython().getoutput( 33 | 'curl -s http://localhost:4040/api/tunnels | python3 -c "import sys, json; print(json.load(sys.stdin))"')[ 34 | 0] 35 | tensorboard_link = eval(tensorboard_link)['tunnels'][0]['public_url'] 36 | setup_passed = True 37 | except: 38 | setup_passed = False 39 | retry_count += 1 40 | print('Initialization failed, retry again (%d)' % retry_count) 41 | print('\n') 42 | 43 | print("TensorBoard link:") 44 | print(tensorboard_link) 45 | 46 | def get_graph_path(self): 47 | return self.graph_path 48 | 49 | def is_eager_execution(self): 50 | if self.eager_execution is None: 51 | try: 52 | tf.summary.FileWriter(self.graph_path) 53 | self.eager_execution = False 54 | except Exception as err: 55 | self.eager_execution = str( 56 | err) == 'tf.summary.FileWriter is not compatible with eager execution. Use tf.contrib.summary instead.' 57 | return self.eager_execution 58 | 59 | def get_writer(self): 60 | if self.writer is None: 61 | if self.is_eager_execution(): 62 | self.writer = tf.contrib.summary.create_file_writer(self.graph_path) 63 | else: 64 | self.writer = tf.summary.FileWriter(self.graph_path) 65 | 66 | return self.writer 67 | 68 | def get_deep_writers(self, name): 69 | if not (name in self.deep_writers): 70 | log_path = os.path.join(self.graph_path, name) 71 | if self.is_eager_execution(): 72 | self.deep_writers[name] = tf.contrib.summary.create_file_writer(log_path) 73 | else: 74 | self.deep_writers[name] = tf.summary.FileWriter(log_path) 75 | return self.deep_writers[name] 76 | 77 | def save_image(self, title, image): 78 | image_path = os.path.join(self.graph_path, 'images') 79 | if self.is_eager_execution(): 80 | print('Warning: save_image() is not supported in eager execution mode') 81 | # writer = tf.contrib.summary.create_file_writer(image_path) 82 | # writer.set_as_default() 83 | # with tf.contrib.summary.always_record_summaries(): 84 | # tf.contrib.summary.image( 85 | # title, 86 | # image_tensor 87 | # ) 88 | else: 89 | summary_op = tf.summary.image(title, image) 90 | with tf.Session() as sess: 91 | summary = sess.run(summary_op) 92 | writer = tf.summary.FileWriter(image_path) 93 | writer.add_summary(summary) 94 | writer.close() 95 | 96 | def save_value(self, graph_name, line_name, epoch, value): 97 | if self.is_eager_execution(): 98 | self.get_deep_writers(line_name).set_as_default() 99 | global_step = tf.train.get_or_create_global_step() 100 | global_step.assign(epoch) 101 | with tf.contrib.summary.always_record_summaries(): 102 | tf.contrib.summary.scalar(graph_name, value) 103 | else: 104 | summary = tf.Summary() 105 | summary_value = summary.value.add() 106 | summary_value.simple_value = value 107 | summary_value.tag = graph_name 108 | self.get_deep_writers(line_name).add_summary(summary, epoch) 109 | 110 | def flush_line(self, line_name): 111 | self.get_deep_writers(line_name).flush() 112 | 113 | def close(self): 114 | if self.writer is not None: 115 | self.writer.close() 116 | self.writer = None 117 | for key in self.deep_writers: 118 | self.deep_writers[key].close() 119 | self.deep_writers = {} 120 | 121 | 122 | class TensorBoardColabCallback(TensorBoard): 123 | def __init__(self, tbc=None, write_graph=True, **kwargs): 124 | # Make the original `TensorBoard` log to a subdirectory 'training' 125 | 126 | if tbc is None: 127 | return 128 | 129 | log_dir = tbc.get_graph_path() 130 | 131 | training_log_dir = os.path.join(log_dir, 'training') 132 | super(TensorBoardColabCallback, self).__init__(training_log_dir, **kwargs) 133 | 134 | # Log the validation metrics to a separate subdirectory 135 | self.val_log_dir = os.path.join(log_dir, 'validation') 136 | 137 | def set_model(self, model): 138 | # Setup writer for validation metrics 139 | if self.is_eager_execution(): 140 | self.val_writer = tf.contrib.summary.create_file_writer(self.val_log_dir) 141 | else: 142 | self.val_writer = tf.summary.FileWriter(self.val_log_dir) 143 | 144 | super(TensorBoardColabCallback, self).set_model(model) 145 | 146 | def on_epoch_end(self, epoch, logs=None): 147 | # Pop the validation logs and handle them separately with 148 | # `self.val_writer`. Also rename the keys so that they can 149 | # be plotted on the same figure with the training metrics 150 | logs = logs or {} 151 | val_logs = {k.replace('val_', ''): v for k, v in logs.items() if k.startswith('val_')} 152 | 153 | for name, value in val_logs.items(): 154 | if self.is_eager_execution(): 155 | self.val_writer.set_as_default() 156 | global_step = tf.train.get_or_create_global_step() 157 | global_step.assign(epoch) 158 | with tf.contrib.summary.always_record_summaries(): 159 | tf.contrib.summary.scalar(name, value.item()) 160 | else: 161 | summary = tf.Summary() 162 | summary_value = summary.value.add() 163 | summary_value.simple_value = value.item() 164 | summary_value.tag = name 165 | self.val_writer.add_summary(summary, epoch) 166 | 167 | self.val_writer.flush() 168 | 169 | # Pass the remaining logs to `TensorBoard.on_epoch_end` 170 | logs = {k: v for k, v in logs.items() if not k.startswith('val_')} 171 | super(TensorBoardColabCallback, self).on_epoch_end(epoch, logs) 172 | 173 | def on_train_end(self, logs=None): 174 | super(TensorBoardColabCallback, self).on_train_end(logs) 175 | self.val_writer.close() 176 | -------------------------------------------------------------------------------- /tensorboardcolab/core.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from keras.callbacks import TensorBoard 3 | import time 4 | import os 5 | import io 6 | 7 | class TensorBoardColab: 8 | def __init__(self, port=6006, graph_path='./Graph', startup_waiting_time=8): 9 | self.port = port 10 | self.graph_path = graph_path 11 | self.writer = None 12 | self.deep_writers = {} 13 | self.eager_execution = None 14 | get_ipython().system_raw('npm i -s -q --unsafe-perm -g ngrok') # sudo npm i -s -q --unsafe-perm -g ngrok 15 | 16 | setup_passed = False 17 | retry_count = 0 18 | sleep_time = startup_waiting_time / 3.0 19 | while not setup_passed: 20 | get_ipython().system_raw('kill -9 $(sudo lsof -t -i:%d)' % port) 21 | get_ipython().system_raw('rm -Rf ' + graph_path) 22 | print('Wait for %d seconds...' % startup_waiting_time) 23 | time.sleep(sleep_time) 24 | get_ipython().system_raw('tensorboard --logdir %s --host 0.0.0.0 --port %d &' % (graph_path, port)) 25 | time.sleep(sleep_time) 26 | get_ipython().system_raw('ngrok http %d &' % port) 27 | time.sleep(sleep_time) 28 | try: 29 | tensorboard_link = get_ipython().getoutput( 30 | 'curl -s http://localhost:4040/api/tunnels | python3 -c "import sys, json; print(json.load(sys.stdin))"')[ 31 | 0] 32 | tensorboard_link = eval(tensorboard_link)['tunnels'][0]['public_url'] 33 | setup_passed = True 34 | except: 35 | setup_passed = False 36 | retry_count += 1 37 | print('Initialization failed, retry again (%d)' % retry_count) 38 | print('\n') 39 | 40 | print("TensorBoard link:") 41 | print(tensorboard_link) 42 | 43 | def get_graph_path(self): 44 | return self.graph_path 45 | 46 | def is_eager_execution(self): 47 | if self.eager_execution is None: 48 | try: 49 | tf.summary.FileWriter(self.graph_path) 50 | self.eager_execution = False 51 | except Exception as err: 52 | self.eager_execution = str( 53 | err) == 'tf.summary.FileWriter is not compatible with eager execution. Use tf.contrib.summary instead.' 54 | return self.eager_execution 55 | 56 | def get_writer(self): 57 | if self.writer is None: 58 | if self.is_eager_execution(): 59 | self.writer = tf.contrib.summary.create_file_writer(self.graph_path) 60 | else: 61 | self.writer = tf.summary.FileWriter(self.graph_path) 62 | 63 | return self.writer 64 | 65 | def get_deep_writers(self, name): 66 | if not (name in self.deep_writers): 67 | log_path = os.path.join(self.graph_path, name) 68 | if self.is_eager_execution(): 69 | self.deep_writers[name] = tf.contrib.summary.create_file_writer(log_path) 70 | else: 71 | self.deep_writers[name] = tf.summary.FileWriter(log_path) 72 | return self.deep_writers[name] 73 | 74 | def save_image(self, title, image): 75 | image_path = os.path.join(self.graph_path, 'images') 76 | if self.is_eager_execution(): 77 | print('Warning: save_image() is not supported in eager execution mode') 78 | # writer = tf.contrib.summary.create_file_writer(image_path) 79 | # writer.set_as_default() 80 | # with tf.contrib.summary.always_record_summaries(): 81 | # tf.contrib.summary.image( 82 | # title, 83 | # image_tensor 84 | # ) 85 | else: 86 | summary_op = tf.summary.image(title, image) 87 | with tf.Session() as sess: 88 | summary = sess.run(summary_op) 89 | writer = tf.summary.FileWriter(image_path) 90 | writer.add_summary(summary) 91 | writer.close() 92 | 93 | def save_value(self, graph_name, line_name, epoch, value): 94 | if self.is_eager_execution(): 95 | self.get_deep_writers(line_name).set_as_default() 96 | global_step = tf.train.get_or_create_global_step() 97 | global_step.assign(epoch) 98 | with tf.contrib.summary.always_record_summaries(): 99 | tf.contrib.summary.scalar(graph_name, value) 100 | else: 101 | summary = tf.Summary() 102 | summary_value = summary.value.add() 103 | summary_value.simple_value = value 104 | summary_value.tag = graph_name 105 | self.get_deep_writers(line_name).add_summary(summary, epoch) 106 | 107 | def flush_line(self, line_name): 108 | self.get_deep_writers(line_name).flush() 109 | 110 | def close(self): 111 | if self.writer is not None: 112 | self.writer.close() 113 | self.writer = None 114 | for key in self.deep_writers: 115 | self.deep_writers[key].close() 116 | self.deep_writers = {} 117 | 118 | 119 | class TensorBoardColabCallback(TensorBoard): 120 | def __init__(self, tbc=None, write_graph=True, **kwargs): 121 | # Make the original `TensorBoard` log to a subdirectory 'training' 122 | 123 | if tbc is None: 124 | return 125 | 126 | self.tbc = tbc 127 | 128 | log_dir = tbc.get_graph_path() 129 | 130 | training_log_dir = os.path.join(log_dir, 'training') 131 | super(TensorBoardColabCallback, self).__init__(training_log_dir, **kwargs) 132 | 133 | # Log the validation metrics to a separate subdirectory 134 | self.val_log_dir = os.path.join(log_dir, 'validation') 135 | 136 | def set_model(self, model): 137 | # Setup writer for validation metrics 138 | if self.tbc.is_eager_execution(): 139 | self.val_writer = tf.contrib.summary.create_file_writer(self.val_log_dir) 140 | else: 141 | self.val_writer = tf.summary.FileWriter(self.val_log_dir) 142 | 143 | super(TensorBoardColabCallback, self).set_model(model) 144 | 145 | def on_epoch_end(self, epoch, logs=None): 146 | # Pop the validation logs and handle them separately with 147 | # `self.val_writer`. Also rename the keys so that they can 148 | # be plotted on the same figure with the training metrics 149 | logs = logs or {} 150 | val_logs = {k.replace('val_', ''): v for k, v in logs.items() if k.startswith('val_')} 151 | 152 | for name, value in val_logs.items(): 153 | if self.tbc.is_eager_execution(): 154 | self.val_writer.set_as_default() 155 | global_step = tf.train.get_or_create_global_step() 156 | global_step.assign(epoch) 157 | with tf.contrib.summary.always_record_summaries(): 158 | tf.contrib.summary.scalar(name, value.item()) 159 | else: 160 | summary = tf.Summary() 161 | summary_value = summary.value.add() 162 | summary_value.simple_value = value.item() 163 | summary_value.tag = name 164 | self.val_writer.add_summary(summary, epoch) 165 | 166 | self.val_writer.flush() 167 | 168 | # Pass the remaining logs to `TensorBoard.on_epoch_end` 169 | logs = {k: v for k, v in logs.items() if not k.startswith('val_')} 170 | super(TensorBoardColabCallback, self).on_epoch_end(epoch, logs) 171 | 172 | def on_train_end(self, logs=None): 173 | super(TensorBoardColabCallback, self).on_train_end(logs) 174 | self.val_writer.close() --------------------------------------------------------------------------------