├── .gitignore
├── DenseNet
├── densenet
│ ├── experiments
│ │ └── cifar-10
│ │ │ ├── conv-Ultimate-Tensorization
│ │ │ ├── eval.py
│ │ │ ├── hyper_parameters.py
│ │ │ ├── input_data.py
│ │ │ ├── nets
│ │ │ │ └── dense.py
│ │ │ └── train.py
│ │ │ └── data
│ │ │ ├── prepare_data.py
│ │ │ └── prepare_data_KA.py
│ └── tensornet
│ │ ├── __init__.py
│ │ ├── layers
│ │ ├── __init__.py
│ │ ├── auxx.py
│ │ ├── batch_normalization.py
│ │ ├── conv.py
│ │ ├── linear.py
│ │ ├── linear_dev.py
│ │ ├── tr.py
│ │ ├── tt.py
│ │ ├── tt_conv.py
│ │ ├── tt_conv1d_full.py
│ │ ├── tt_conv_direct.py
│ │ ├── tt_conv_full.py
│ │ ├── tt_dev.py
│ │ └── ttrelu.py
│ │ └── tt
│ │ ├── __init__.py
│ │ ├── matrix_svd.py
│ │ ├── max_ranks.py
│ │ └── svd.py
└── densent-mpo
│ ├── experiments
│ └── cifar-10
│ │ ├── conv-Ultimate-Tensorization
│ │ ├── eval.py
│ │ ├── hyper_parameters.py
│ │ ├── input_data.py
│ │ ├── nets
│ │ │ └── dense-mpo.py
│ │ └── train.py
│ │ └── data
│ │ ├── prepare_data.py
│ │ └── prepare_data_KA.py
│ └── tensornet
│ ├── __init__.py
│ ├── layers
│ ├── __init__.py
│ ├── auxx.py
│ ├── batch_normalization.py
│ ├── conv.py
│ ├── linear.py
│ ├── linear_dev.py
│ ├── tr.py
│ ├── tt.py
│ ├── tt_conv.py
│ ├── tt_conv1d_full.py
│ ├── tt_conv_direct.py
│ ├── tt_conv_full.py
│ ├── tt_dev.py
│ └── ttrelu.py
│ └── tt
│ ├── __init__.py
│ ├── matrix_svd.py
│ ├── max_ranks.py
│ └── svd.py
├── FC2
├── entropy.py
├── fc2-mpo1
│ ├── hyperprameter.py
│ ├── inference.py
│ ├── train.py
│ └── tt.py
└── fc2
│ ├── hyper_parameters.py
│ ├── inference.py
│ └── train.py
├── LICENSE
├── LeNet5
├── lenet5
│ ├── hyperprameter.py
│ ├── inference.py
│ ├── train.py
│ └── tt.py
└── mpo_lenet5
│ ├── hyperprameter.py
│ ├── inference.py
│ ├── train.py
│ └── tt.py
├── README.md
├── ResNet
├── resnet-mpo
│ ├── experiments
│ │ └── cifar-10
│ │ │ ├── conv-Ultimate-Tensorization
│ │ │ ├── eval.py
│ │ │ ├── hyper_parameters.py
│ │ │ ├── input_data.py
│ │ │ ├── nets
│ │ │ │ └── wideResNet-TT-ap.py
│ │ │ └── train.py
│ │ │ └── data
│ │ │ ├── prepare_data.py
│ │ │ └── prepare_data_KA.py
│ └── tensornet
│ │ ├── __init__.py
│ │ ├── layers
│ │ ├── __init__.py
│ │ ├── aux_py
│ │ ├── auxx.py
│ │ ├── batch_normalization.py
│ │ ├── conv.py
│ │ ├── linear.py
│ │ ├── linear_dev.py
│ │ ├── tr.py
│ │ ├── tt.py
│ │ ├── tt_conv.py
│ │ ├── tt_conv1d_full.py
│ │ ├── tt_conv_direct.py
│ │ ├── tt_conv_full.py
│ │ ├── tt_dev.py
│ │ └── ttrelu.py
│ │ └── tt
│ │ ├── __init__.py
│ │ ├── matrix_svd.py
│ │ ├── max_ranks.py
│ │ └── svd.py
└── resnet
│ ├── experiments
│ └── cifar-10
│ │ ├── conv-Ultimate-Tensorization
│ │ ├── eval.py
│ │ ├── hyper_parameters.py
│ │ ├── input_data.py
│ │ ├── nets
│ │ │ ├── __pycache__
│ │ │ │ └── wideResNet.cpython-36.pyc
│ │ │ └── wideResNet.py
│ │ └── train.py
│ │ └── data
│ │ ├── prepare_data.py
│ │ └── prepare_data_KA.py
│ └── tensornet
│ ├── __init__.py
│ ├── layers
│ ├── __init__.py
│ ├── aux_py
│ ├── auxx.py
│ ├── batch_normalization.py
│ ├── conv.py
│ ├── linear.py
│ ├── linear_dev.py
│ ├── tr.py
│ ├── tt.py
│ ├── tt_conv.py
│ ├── tt_conv1d_full.py
│ ├── tt_conv_direct.py
│ ├── tt_conv_full.py
│ ├── tt_dev.py
│ └── ttrelu.py
│ └── tt
│ ├── __init__.py
│ ├── matrix_svd.py
│ ├── max_ranks.py
│ └── svd.py
├── VGG-16
├── vgg-16-mpo
│ ├── experiments
│ │ └── cifar-10
│ │ │ ├── conv-Ultimate-Tensorization
│ │ │ ├── eval.py
│ │ │ ├── hyper_parameters.py
│ │ │ ├── input_data.py
│ │ │ ├── nets
│ │ │ │ └── vgg-16-mpo.py
│ │ │ └── train.py
│ │ │ └── data
│ │ │ ├── prepare_data.py
│ │ │ └── prepare_data_KA.py
│ └── tensornet
│ │ ├── __init__.py
│ │ ├── layers
│ │ ├── __init__.py
│ │ ├── auxx.py
│ │ ├── batch_normalization.py
│ │ ├── conv.py
│ │ ├── linear.py
│ │ ├── linear_dev.py
│ │ ├── tr.py
│ │ ├── tt.py
│ │ ├── tt_conv.py
│ │ ├── tt_conv1d_full.py
│ │ ├── tt_conv_direct.py
│ │ ├── tt_conv_full.py
│ │ ├── tt_dev.py
│ │ └── ttrelu.py
│ │ └── tt
│ │ ├── __init__.py
│ │ ├── matrix_svd.py
│ │ ├── max_ranks.py
│ │ └── svd.py
└── vgg-16
│ ├── experiments
│ └── cifar-10
│ │ ├── conv-Ultimate-Tensorization
│ │ ├── eval.py
│ │ ├── hyper_parameters.py
│ │ ├── input_data.py
│ │ ├── nets
│ │ │ └── vgg-16.py
│ │ └── train.py
│ │ └── data
│ │ ├── prepare_data.py
│ │ └── prepare_data_KA.py
│ └── tensornet
│ ├── __init__.py
│ ├── layers
│ ├── __init__.py
│ ├── auxx.py
│ ├── batch_normalization.py
│ ├── conv.py
│ ├── linear.py
│ ├── linear_dev.py
│ ├── tr.py
│ ├── tt.py
│ ├── tt_conv.py
│ ├── tt_conv1d_full.py
│ ├── tt_conv_direct.py
│ ├── tt_conv_full.py
│ ├── tt_dev.py
│ └── ttrelu.py
│ └── tt
│ ├── __init__.py
│ ├── matrix_svd.py
│ ├── max_ranks.py
│ └── svd.py
├── VGG-19
├── vgg-19-mpo
│ ├── experiments
│ │ └── cifar-10
│ │ │ ├── conv-Ultimate-Tensorization
│ │ │ ├── eval.py
│ │ │ ├── hyper_parameters.py
│ │ │ ├── input_data.py
│ │ │ ├── nets
│ │ │ │ └── vgg-19-mpo.py
│ │ │ └── train.py
│ │ │ └── data
│ │ │ ├── prepare_data.py
│ │ │ └── prepare_data_KA.py
│ └── tensornet
│ │ ├── __init__.py
│ │ ├── layers
│ │ ├── __init__.py
│ │ ├── aux_py
│ │ ├── auxx.py
│ │ ├── batch_normalization.py
│ │ ├── conv.py
│ │ ├── linear.py
│ │ ├── linear_dev.py
│ │ ├── tr.py
│ │ ├── tt.py
│ │ ├── tt_conv.py
│ │ ├── tt_conv1d_full.py
│ │ ├── tt_conv_direct.py
│ │ ├── tt_conv_full.py
│ │ ├── tt_dev.py
│ │ └── ttrelu.py
│ │ └── tt
│ │ ├── __init__.py
│ │ ├── matrix_svd.py
│ │ ├── max_ranks.py
│ │ └── svd.py
└── vgg-19
│ ├── experiments
│ └── cifar-10
│ │ ├── conv-Ultimate-Tensorization
│ │ ├── eval.py
│ │ ├── hyper_parameters.py
│ │ ├── input_data.py
│ │ ├── nets
│ │ │ └── vgg-19.py
│ │ └── train.py
│ │ └── data
│ │ ├── prepare_data.py
│ │ └── prepare_data_KA.py
│ └── tensornet
│ ├── __init__.py
│ ├── layers
│ ├── __init__.py
│ ├── aux_py
│ ├── auxx.py
│ ├── batch_normalization.py
│ ├── conv.py
│ ├── linear.py
│ ├── linear_dev.py
│ ├── tr.py
│ ├── tt.py
│ ├── tt_conv.py
│ ├── tt_conv1d_full.py
│ ├── tt_conv_direct.py
│ ├── tt_conv_full.py
│ ├── tt_dev.py
│ └── ttrelu.py
│ └── tt
│ ├── __init__.py
│ ├── matrix_svd.py
│ ├── max_ranks.py
│ └── svd.py
└── requirements.txt
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | MANIFEST
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 | .pytest_cache/
49 |
50 | # Translations
51 | *.mo
52 | *.pot
53 |
54 | # Django stuff:
55 | *.log
56 | local_settings.py
57 | db.sqlite3
58 |
59 | # Flask stuff:
60 | instance/
61 | .webassets-cache
62 |
63 | # Scrapy stuff:
64 | .scrapy
65 |
66 | # Sphinx documentation
67 | docs/_build/
68 |
69 | # PyBuilder
70 | target/
71 |
72 | # Jupyter Notebook
73 | .ipynb_checkpoints
74 |
75 | # pyenv
76 | .python-version
77 |
78 | # celery beat schedule file
79 | celerybeat-schedule
80 |
81 | # SageMath parsed files
82 | *.sage.py
83 |
84 | # Environments
85 | .env
86 | .venv
87 | env/
88 | venv/
89 | ENV/
90 | env.bak/
91 | venv.bak/
92 |
93 | # Spyder project settings
94 | .spyderproject
95 | .spyproject
96 |
97 | # Rope project settings
98 | .ropeproject
99 |
100 | # mkdocs documentation
101 | /site
102 |
103 | # mypy
104 | .mypy_cache/
105 |
--------------------------------------------------------------------------------
/DenseNet/densenet/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/DenseNet/densenet/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | in_ch,
6 | out_ch,
7 | window_size,
8 | strides=[1, 1],
9 | padding='SAME',
10 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
11 | filters_regularizer=None,
12 | biases_initializer=tf.zeros_initializer,
13 | biases_regularizer=None,
14 | trainable=True,
15 | cpu_variables=False,
16 | scope=None):
17 | """ convolutional layer
18 | Args:
19 | inp: input tensor, float - [batch_size, H, W, C]
20 | out_ch: output channels count count, int
21 | window_size: convolution window size, list [wH, wW]
22 | strides: strides, list [sx, sy]
23 | padding: 'SAME' or 'VALID', string
24 | filters_initializer: filters init function
25 | filters_regularizer: filters regularizer function
26 | biases_initializer: biases init function (if None then no biases will be used)
27 | biases_regularizer: biases regularizer function
28 | trainable: trainable variables flag, bool
29 | cpu_variables: cpu variables flag, bool
30 | scope: layer variable scope name, string
31 | Returns:
32 | out: output tensor, float - [batch_size, H', W', out_ch]
33 | """
34 |
35 | with tf.variable_scope(scope):
36 | shape = inp.get_shape().as_list()
37 | assert len(shape) == 4, "Not 4D input tensor"
38 |
39 | filters = get_var_wrap('filters',
40 | shape=window_size + [in_ch, out_ch],
41 | initializer=filters_initializer,
42 | regularizer=filters_regularizer,
43 | trainable=trainable,
44 | cpu_variable=cpu_variables)
45 |
46 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
47 |
48 | if biases_initializer is not None:
49 | biases = get_var_wrap('biases',
50 | shape=[out_ch],
51 | initializer=biases_initializer,
52 | regularizer=biases_regularizer,
53 | trainable=trainable,
54 | cpu_variable=cpu_variables)
55 | out = tf.add(out, biases, name='out')
56 | else:
57 | out = tf.identity(out, name='out')
58 | return out
59 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/DenseNet/densenet/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | in_ch,
6 | out_ch,
7 | window_size,
8 | strides=[1, 1],
9 | padding='SAME',
10 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
11 | filters_regularizer=None,
12 | biases_initializer=tf.zeros_initializer,
13 | biases_regularizer=None,
14 | trainable=True,
15 | cpu_variables=False,
16 | scope=None):
17 | """ convolutional layer
18 | Args:
19 | inp: input tensor, float - [batch_size, H, W, C]
20 | out_ch: output channels count count, int
21 | window_size: convolution window size, list [wH, wW]
22 | strides: strides, list [sx, sy]
23 | padding: 'SAME' or 'VALID', string
24 | filters_initializer: filters init function
25 | filters_regularizer: filters regularizer function
26 | biases_initializer: biases init function (if None then no biases will be used)
27 | biases_regularizer: biases regularizer function
28 | trainable: trainable variables flag, bool
29 | cpu_variables: cpu variables flag, bool
30 | scope: layer variable scope name, string
31 | Returns:
32 | out: output tensor, float - [batch_size, H', W', out_ch]
33 | """
34 |
35 | with tf.variable_scope(scope):
36 | shape = inp.get_shape().as_list()
37 | assert len(shape) == 4, "Not 4D input tensor"
38 |
39 | filters = get_var_wrap('filters',
40 | shape=window_size + [in_ch, out_ch],
41 | initializer=filters_initializer,
42 | regularizer=filters_regularizer,
43 | trainable=trainable,
44 | cpu_variable=cpu_variables)
45 |
46 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
47 |
48 | if biases_initializer is not None:
49 | biases = get_var_wrap('biases',
50 | shape=[out_ch],
51 | initializer=biases_initializer,
52 | regularizer=biases_regularizer,
53 | trainable=trainable,
54 | cpu_variable=cpu_variables)
55 | out = tf.add(out, biases, name='out')
56 | else:
57 | out = tf.identity(out, name='out')
58 | return out
59 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/DenseNet/densent-mpo/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/FC2/entropy.py:
--------------------------------------------------------------------------------
1 | # Calculate entanglement entropy from well-trained MPO tensors.
2 | # Available for FC2-TTO net.
3 |
4 | import tensorflow as tf
5 | import os
6 | flags = tf.app.flags
7 | FLAGS = flags.FLAGS
8 | flags.DEFINE_integer('tt_ranks_1', 10, 'ranks of the first tensor train')
9 | os.environ["CUDA_VISIBLE_DEVICES"]=""
10 | r_1 = FLAGS.tt_ranks_1
11 | dir_log = './logs/tt_rank1_%d/epoch_295000.ckpt'%r_1
12 | inp_mode = [4,7,7,4]
13 | out_mode = [4,4,4,4]
14 | mat_ranks = [1,r_1,r_1,r_1,1]
15 | mpo_mat = []
16 | mpo_tensor_l = []
17 | mpo_tensor_r = []
18 | density = []
19 | entropy = []
20 | for i in range(4):
21 | mpo_mat.append(tf.get_variable('tt_scope_1/mat_core_%d'%(i+1),shape=[out_mode[i] * mat_ranks[i + 1], mat_ranks[i] * inp_mode[i]]))
22 | saver = tf.train.Saver()
23 |
24 | mpo_tensor_l.append(tf.reshape(tf.transpose(mpo_mat[0]), [-1, mat_ranks[1]]))
25 | temp = tf.reshape(mpo_mat[1],[out_mode[1],mat_ranks[2],mat_ranks[1],inp_mode[1]])
26 | mpo_tensor_l.append(tf.reshape(tf.einsum('ij,lmjk->ilkm', mpo_tensor_l[0], temp), [-1, mat_ranks[2]]))
27 | temp = tf.reshape(mpo_mat[2],[out_mode[2],mat_ranks[3],mat_ranks[2],inp_mode[2]])
28 | mpo_tensor_l.append(tf.reshape(tf.einsum('ij,lmjk->ilkm', mpo_tensor_l[1], temp), [-1,mat_ranks[3]]))
29 |
30 | mpo_tensor_r.append(tf.reshape(tf.transpose(mpo_mat[3]), [mat_ranks[3], -1]))
31 | temp = tf.reshape(mpo_mat[2],[out_mode[2],mat_ranks[3],mat_ranks[2],inp_mode[2]])
32 | mpo_tensor_r.append(tf.reshape(tf.einsum('ijkl,jm->kilm',temp, mpo_tensor_r[0]),[mat_ranks[2],-1]))
33 | temp = tf.reshape(mpo_mat[1],[out_mode[1],mat_ranks[2],mat_ranks[1],inp_mode[1]])
34 | mpo_tensor_r.append(tf.reshape(tf.einsum('ijkl,jm->kilm',temp, mpo_tensor_r[1]),[mat_ranks[1],-1]))
35 |
36 |
37 |
38 | for i in range(3):
39 | density.append(tf.matmul(mpo_tensor_l[i], mpo_tensor_r[2-i]))
40 | for i in range(3):
41 | s = tf.svd(density[i],compute_uv=False)
42 | s = s * s
43 | s = s/tf.reduce_sum(s)
44 | num_nonzero = tf.count_nonzero(s)
45 | lamda = tf.zeros([tf.to_int32(num_nonzero),])
46 | lamda = s[:tf.to_int32(num_nonzero)]
47 |
48 | entropy.append(- tf.reduce_sum(lamda * tf.log(lamda)))
49 | with tf.Session() as sess:
50 | init = tf.global_variables_initializer()
51 | sess.run(init)
52 | saver.restore(sess, dir_log)
53 | sess.run(entropy)
54 | print(entropy[0].eval(),entropy[1].eval(),entropy[2].eval())
--------------------------------------------------------------------------------
/FC2/fc2-mpo1/hyperprameter.py:
--------------------------------------------------------------------------------
1 | # ==============================================================================
2 | import tensorflow as tf
3 | import sys
4 |
5 | # Basic model parameters as external flags.
6 | flags = tf.app.flags
7 | FLAGS = flags.FLAGS
8 |
9 | flags.DEFINE_integer('input_node', 784, 'picture size you want to input to the network')
10 | flags.DEFINE_integer('hidden_node', 256, 'number of hidden units')
11 | flags.DEFINE_integer('output_node', 10, 'labels number you want to input to the network')
12 | flags.DEFINE_integer('global_step', 300000, 'total step the network to train')
13 |
14 | flags.DEFINE_integer('tt_ranks_1', 4, 'ranks of the first tensor train')
15 | flags.DEFINE_integer('tt_ranks_2', 4, 'ranks of the second tensor train')
16 | flags.DEFINE_integer('batch_size', 100, 'Batch size. ')
17 | flags.DEFINE_float('REGULARIZER_RATE', 0.0001, 'L2 regularizer_rate')
18 | flags.DEFINE_float('LEARNING_RATE_BASE', 0.8, 'L2 learning_rate_base')
19 | flags.DEFINE_float('LEARNING_RATE_DECAY', 0.99, 'L2 learning_rate_decay')
20 |
21 | ##===========================================================================================================
22 |
23 | def print_hyper_parameters():
24 |
25 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size)
26 |
27 | sys.stdout.write('tt_ranks_1 %d\n' % FLAGS.tt_ranks_1)
28 |
29 | sys.stdout.write('tt_ranks_2 %d\n' % FLAGS.tt_ranks_2)
--------------------------------------------------------------------------------
/FC2/fc2-mpo1/inference.py:
--------------------------------------------------------------------------------
1 |
2 | import numpy as np
3 | import tensorflow as tf
4 | import tt
5 | from hyperprameter import *
6 |
7 |
8 | r_1 = FLAGS.tt_ranks_1
9 | r_2 = FLAGS.tt_ranks_2
10 | input_node=FLAGS.input_node
11 | output_node=FLAGS.output_node
12 | hidden1_node=FLAGS.hidden_node
13 | #TTO_layer1
14 | inp_modes1 = [4,7,7,4]
15 | out_modes1 = [4,4,4,4]
16 | mat_rank1 = [1,r_1,r_1,r_1,1]
17 |
18 | #TTO_layer2
19 | inp_modes2 = [4,4,4,4]
20 | out_modes2 = [1,10,1,1]
21 | mat_rank2 = [1,r_2,r_2,r_2,1]
22 |
23 |
24 |
25 | def inference(inputs):
26 | inputs = tt.tto(inputs,
27 | np.array(inp_modes1,dtype=np.int32),
28 | np.array(out_modes1,dtype=np.int32),
29 | np.array(mat_rank1,dtype=np.int32),
30 | scope='tt_scope_1')
31 | inputs = tf.nn.relu(inputs)
32 | inputs = tt.tto(inputs,
33 | np.array(inp_modes2, dtype=np.int32),
34 | np.array(out_modes2, dtype=np.int32),
35 | np.array(mat_rank2, dtype=np.int32),
36 | scope='tt_scope_2')
37 | return inputs
38 |
--------------------------------------------------------------------------------
/FC2/fc2-mpo1/train.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | @author: zfgao
4 | """
5 |
6 | import tensorflow as tf
7 | from tensorflow.examples.tutorials.mnist import input_data
8 | import inference
9 | from hyperprameter import *
10 |
11 | BATCH_SIZE=FLAGS.batch_size
12 | TRAINING_STEPS=FLAGS.global_step
13 | LEARNING_RATE_BASE=FLAGS.LEARNING_RATE_BASE
14 | LEARNING_RATE_DECAY=FLAGS.LEARNING_RATE_DECAY
15 | REGULARIZER_RATE=FLAGS.REGULARIZER_RATE
16 | MOVING_DECAY=0.99
17 | #seed =12345
18 | #tf.set_random_seed(seed)
19 |
20 | def mnist(inp):
21 | x=tf.placeholder(tf.float32,[None,inference.input_node],name='x-input')
22 | y_=tf.placeholder(tf.float32,[None,inference.output_node],name='y-input')
23 | # regularizer = tf.contrib.layers.l2_regularizer(REGULARIZER_RATE)
24 |
25 | y=inference.inference(x)
26 | global_step=tf.Variable(0,trainable=False)
27 |
28 | # ema = tf.train.ExponentialMovingAverage(MOVING_DECAY, global_step)
29 | # ema_op = ema.apply(tf.trainable_variables())
30 |
31 | ce=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
32 | loss=tf.reduce_mean(ce)
33 | loss += tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()]) * REGULARIZER_RATE
34 | # loss = loss + tf.add_n(tf.get_collection('losses'))
35 | learning_rate=tf.train.exponential_decay(LEARNING_RATE_BASE,
36 | global_step,
37 | inp.train.num_examples/BATCH_SIZE,
38 | LEARNING_RATE_DECAY)
39 | train_steps=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
40 | # with tf.control_dependencies([train_steps,ema_op]):
41 | # train_op=tf.no_op(name='train')
42 |
43 | correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
44 | accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
45 |
46 |
47 | with tf.Session() as sess:
48 | init_op = tf.global_variables_initializer()
49 | sess.run(init_op)
50 | best_acc = 0
51 | for i in range(TRAINING_STEPS):
52 | xs,ys = inp.train.next_batch(BATCH_SIZE)
53 | _,step,lr = sess.run([train_steps,global_step,learning_rate],feed_dict={x:xs,y_:ys})
54 | if i%1000 == 0:
55 | accuracy_score = sess.run(accuracy, feed_dict={x:inp.test.images,y_:inp.test.labels})
56 | print('step={},lr={}'.format(step,lr))
57 | if best_acc< accuracy_score:
58 | best_acc = accuracy_score
59 | print('Accuracy at step %s: %s' % (i, accuracy_score))
60 | accuracy_score=sess.run(accuracy,feed_dict={x:inp.test.images,y_:inp.test.labels})
61 | print("After %s trainning step(s),best accuracy=%g" %(step,best_acc))
62 |
63 |
64 |
65 | def main(argv=None):
66 | inp=input_data.read_data_sets("./data/",validation_size=0,one_hot=True)
67 | mnist(inp)
68 |
69 | if __name__=='__main__':
70 | tf.app.run()
71 |
--------------------------------------------------------------------------------
/FC2/fc2/hyper_parameters.py:
--------------------------------------------------------------------------------
1 |
2 | # ==============================================================================
3 | import tensorflow as tf
4 | import sys
5 |
6 | # Basic model parameters as external flags.
7 | flags = tf.app.flags
8 | FLAGS = flags.FLAGS
9 |
10 | flags.DEFINE_integer('input_node', 784, 'picture size you want to input to the network')
11 | flags.DEFINE_integer('hidden_node', 256, 'number of hidden units')
12 | flags.DEFINE_integer('output_node', 10, 'labels number you want to input to the network')
13 | flags.DEFINE_integer('global_step', 300000, 'total step the network to train')
14 |
15 | flags.DEFINE_integer('tt_ranks_1', 4, 'ranks of the first tensor train')
16 | flags.DEFINE_integer('tt_ranks_2', 4, 'ranks of the second tensor train')
17 | flags.DEFINE_integer('batch_size', 100, 'Batch size. ')
18 |
19 |
20 | ##===========================================================================================================
21 |
22 | def print_hyper_parameters():
23 |
24 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size)
25 |
26 | sys.stdout.write('tt_ranks_1 %d\n' % FLAGS.tt_ranks_1)
27 |
28 | sys.stdout.write('tt_ranks_2 %d\n' % FLAGS.tt_ranks_2)
--------------------------------------------------------------------------------
/FC2/fc2/inference.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | @author: zfgao
4 | """
5 |
6 | # FC2 network without biases
7 | import tensorflow as tf
8 | from hyper_parameters import *
9 | #define the parameters
10 | input_node=FLAGS.input_node
11 | output_node=FLAGS.output_node
12 | hidden1_node=FLAGS.hidden_node
13 | def get_weight_variable(shape):
14 | weights=tf.Variable(tf.truncated_normal(shape,stddev=0.1))
15 | tf.summary.histogram('weights', weights)
16 | return weights
17 |
18 | def get_biases_variable(shape):
19 | biases=tf.Variable(tf.zeros(shape))
20 | tf.summary.histogram('biases', biases)
21 | return biases
22 | # we set this full-connected layers without biases.
23 | def inference(input_tensor):
24 | w1=get_weight_variable([input_node,hidden1_node])
25 | y1=tf.matmul(input_tensor,w1)
26 | y1=tf.nn.relu(y1)
27 | w2=get_weight_variable([hidden1_node,output_node])
28 | y=tf.matmul(y1,w2)
29 | return y
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/FC2/fc2/train.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | @author: zfgao
4 | """
5 |
6 | import tensorflow as tf
7 | from tensorflow.examples.tutorials.mnist import input_data
8 | import inference
9 | from hyperprameter import *
10 |
11 | BATCH_SIZE=FLAGS.batch_size
12 | TRAINING_STEPS=FLAGS.global_step
13 | LEARNING_RATE_BASE=FLAGS.LEARNING_RATE_BASE
14 | LEARNING_RATE_DECAY=FLAGS.LEARNING_RATE_DECAY
15 | REGULARIZER_RATE=FLAGS.REGULARIZER_RATE
16 | MOVING_DECAY=0.99
17 | #seed =12345
18 | #tf.set_random_seed(seed)
19 |
20 | def mnist(inp):
21 | x=tf.placeholder(tf.float32,[None,inference.input_node],name='x-input')
22 | y_=tf.placeholder(tf.float32,[None,inference.output_node],name='y-input')
23 | # regularizer = tf.contrib.layers.l2_regularizer(REGULARIZER_RATE)
24 |
25 | y=inference.inference(x)
26 | global_step=tf.Variable(0,trainable=False)
27 |
28 | # ema = tf.train.ExponentialMovingAverage(MOVING_DECAY, global_step)
29 | # ema_op = ema.apply(tf.trainable_variables())
30 |
31 | ce=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
32 | loss=tf.reduce_mean(ce)
33 | loss += tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()]) * REGULARIZER_RATE
34 | # loss = loss + tf.add_n(tf.get_collection('losses'))
35 | learning_rate=tf.train.exponential_decay(LEARNING_RATE_BASE,
36 | global_step,
37 | inp.train.num_examples/BATCH_SIZE,
38 | LEARNING_RATE_DECAY)
39 | train_steps=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
40 | # with tf.control_dependencies([train_steps,ema_op]):
41 | # train_op=tf.no_op(name='train')
42 |
43 | correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
44 | accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
45 |
46 |
47 | with tf.Session() as sess:
48 | init_op = tf.global_variables_initializer()
49 | sess.run(init_op)
50 | best_acc = 0
51 | for i in range(TRAINING_STEPS):
52 | xs,ys = inp.train.next_batch(BATCH_SIZE)
53 | _,step,lr = sess.run([train_steps,global_step,learning_rate],feed_dict={x:xs,y_:ys})
54 | if i%1000 == 0:
55 | accuracy_score = sess.run(accuracy, feed_dict={x:inp.test.images,y_:inp.test.labels})
56 | print('step={},lr={}'.format(step,lr))
57 | if best_acc< accuracy_score:
58 | best_acc = accuracy_score
59 | print('Accuracy at step %s: %s' % (i, accuracy_score))
60 | accuracy_score=sess.run(accuracy,feed_dict={x:inp.test.images,y_:inp.test.labels})
61 | print("After %s trainning step(s),best accuracy=%g" %(step,best_acc))
62 |
63 |
64 |
65 | def main(argv=None):
66 | inp=input_data.read_data_sets("./data/",validation_size=0,one_hot=True)
67 | mnist(inp)
68 |
69 | if __name__=='__main__':
70 | tf.app.run()
71 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2019 zfgao66
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/LeNet5/lenet5/hyperprameter.py:
--------------------------------------------------------------------------------
1 | # ==============================================================================
2 | import tensorflow as tf
3 | import sys
4 |
5 | # Basic model parameters as external flags.
6 | flags = tf.app.flags
7 | FLAGS = flags.FLAGS
8 |
9 | flags.DEFINE_integer('input_node', 784, 'picture size you want to input to the network')
10 | flags.DEFINE_integer('hidden_node', 256, 'number of hidden units')
11 | flags.DEFINE_integer('output_node', 10, 'labels number you want to input to the network')
12 | flags.DEFINE_integer('global_step', 300000, 'total step the network to train')
13 |
14 | flags.DEFINE_integer('tt_ranks_1', 26, 'ranks of the first tensor train')
15 | flags.DEFINE_integer('tt_ranks_2', 2, 'ranks of the second tensor train')
16 | flags.DEFINE_integer('batch_size', 100, 'Batch size. ')
17 | flags.DEFINE_float('REGULARIZER_RATE', 0.0001, 'L2 regularizer_rate')
18 | flags.DEFINE_float('LEARNING_RATE_BASE', 0.01, 'L2 learning_rate_base')
19 | flags.DEFINE_float('LEARNING_RATE_DECAY', 0.99, 'L2 learning_rate_decay')
20 |
21 | ##===========================================================================================================
22 |
23 | def print_hyper_parameters():
24 |
25 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size*FLAGS.num_gpus)
26 |
27 | sys.stdout.write('tt_ranks_1 %d\n' % FLAGS.tt_ranks_1)
28 |
29 | sys.stdout.write('tt_ranks_2 %d\n' % FLAGS.tt_ranks_2)
--------------------------------------------------------------------------------
/LeNet5/lenet5/inference.py:
--------------------------------------------------------------------------------
1 |
2 | import numpy as np
3 | import tensorflow as tf
4 | import tt
5 | from hyperprameter import *
6 |
7 |
8 | input_node=FLAGS.input_node
9 | output_node=FLAGS.output_node
10 |
11 |
12 |
13 | def conv2d(x, W):
14 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID')
15 |
16 |
17 | def max_pool_2x2(x):
18 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
19 | strides=[1, 2, 2, 1], padding='SAME')
20 |
21 |
22 | def weight_variable(shape):
23 | weights = tf.Variable(tf.truncated_normal(shape, stddev=0.1))
24 | return weights
25 |
26 |
27 | def bias_variable(shape):
28 | initial = tf.constant(0.0, shape=shape)
29 | return tf.Variable(initial)
30 |
31 |
32 | def inference(inputs):
33 | W_conv1 = weight_variable([5, 5, 1, 6])
34 | b_conv1 = bias_variable([6])
35 |
36 | x_image = tf.reshape(inputs, [-1, 28, 28, 1])
37 | # 28x28x1
38 | h_conv1 = tf.nn.relu(tf.nn.conv2d(x_image, W_conv1, strides=[1, 1, 1, 1], padding='SAME') + b_conv1)
39 | # 28x28x6
40 | h_pool1 = max_pool_2x2(h_conv1)
41 | # 14x14x6
42 | W_conv2 = weight_variable([5, 5, 6, 16])
43 | b_conv2 = bias_variable([16])
44 | h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
45 | # 10x10x16
46 | h_pool2 = max_pool_2x2(h_conv2)
47 | # 5x5x16
48 | W_conv3 = weight_variable([5, 5, 16, 120])
49 | b_conv3 = bias_variable([120])
50 | h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3)
51 | # 1x1x120
52 | inputs = tf.reshape(h_conv3, [-1, 120])
53 | W_fc1 = weight_variable([120, 84])
54 | h_fc1 = tf.nn.relu(tf.matmul(inputs, W_fc1))
55 | W_fc2 = weight_variable([84, 10])
56 | output = tf.matmul(h_fc1, W_fc2)
57 |
58 |
59 | return output
60 |
--------------------------------------------------------------------------------
/LeNet5/lenet5/train.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | @author: zfgao
4 | """
5 |
6 | import tensorflow as tf
7 | from tensorflow.examples.tutorials.mnist import input_data
8 | import inference
9 | from hyperprameter import *
10 |
11 | BATCH_SIZE=FLAGS.batch_size
12 | TRAINING_STEPS=FLAGS.global_step
13 | LEARNING_RATE_BASE=FLAGS.LEARNING_RATE_BASE
14 | LEARNING_RATE_DECAY=FLAGS.LEARNING_RATE_DECAY
15 | REGULARIZER_RATE=FLAGS.REGULARIZER_RATE
16 | MOVING_DECAY=0.99
17 | #seed =12345
18 | #tf.set_random_seed(seed)
19 |
20 | def mnist(inp):
21 | x=tf.placeholder(tf.float32,[None,inference.input_node],name='x-input')
22 | y_=tf.placeholder(tf.float32,[None,inference.output_node],name='y-input')
23 | y=inference.inference(x)
24 | global_step=tf.Variable(0,trainable=False)
25 | ce=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
26 | loss=tf.reduce_mean(ce)
27 | loss += tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()]) * REGULARIZER_RATE
28 |
29 | learning_rate=tf.train.exponential_decay(LEARNING_RATE_BASE,
30 | global_step,
31 | inp.train.num_examples/BATCH_SIZE,
32 | LEARNING_RATE_DECAY)
33 | train_steps=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
34 |
35 | correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
36 | accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
37 |
38 |
39 | with tf.Session() as sess:
40 | init_op = tf.global_variables_initializer()
41 | sess.run(init_op)
42 | best_acc = 0
43 | for i in range(TRAINING_STEPS):
44 | xs,ys = inp.train.next_batch(BATCH_SIZE)
45 | _,step,lr = sess.run([train_steps,global_step,learning_rate],feed_dict={x:xs,y_:ys})
46 | if i%1000 == 0:
47 | accuracy_score = sess.run(accuracy, feed_dict={x:inp.test.images,y_:inp.test.labels})
48 | print('step={},lr={}'.format(step,lr))
49 | if best_acc< accuracy_score:
50 | best_acc = accuracy_score
51 | print('Accuracy at step %s: %s' % (i, accuracy_score))
52 | accuracy_score=sess.run(accuracy,feed_dict={x:inp.test.images,y_:inp.test.labels})
53 | print("After %s trainning step(s),best accuracy=%g" %(step,best_acc))
54 |
55 |
56 |
57 | def main(argv=None):
58 | inp=input_data.read_data_sets("./data/",validation_size=0,one_hot=True)
59 | mnist(inp)
60 |
61 | if __name__=='__main__':
62 | tf.app.run()
63 |
--------------------------------------------------------------------------------
/LeNet5/mpo_lenet5/hyperprameter.py:
--------------------------------------------------------------------------------
1 | # ==============================================================================
2 | import tensorflow as tf
3 | import sys
4 |
5 | # Basic model parameters as external flags.
6 | flags = tf.app.flags
7 | FLAGS = flags.FLAGS
8 |
9 | flags.DEFINE_integer('input_node', 784, 'picture size you want to input to the network')
10 | flags.DEFINE_integer('hidden_node', 256, 'number of hidden units')
11 | flags.DEFINE_integer('output_node', 10, 'labels number you want to input to the network')
12 | flags.DEFINE_integer('global_step', 300000, 'total step the network to train')
13 |
14 | flags.DEFINE_integer('tt_ranks_1', 4, 'ranks of the first tensor train')
15 | flags.DEFINE_integer('tt_ranks_2', 4, 'ranks of the second tensor train')
16 | flags.DEFINE_integer('tt_ranks_3', 2, 'ranks of the third tensor train')
17 |
18 | flags.DEFINE_integer('batch_size', 100, 'Batch size. ')
19 | flags.DEFINE_float('REGULARIZER_RATE', 0.0001, 'L2 regularizer_rate')
20 | flags.DEFINE_float('LEARNING_RATE_BASE', 0.01, 'L2 learning_rate_base')
21 | flags.DEFINE_float('LEARNING_RATE_DECAY', 0.99, 'L2 learning_rate_decay')
22 |
23 | ##===========================================================================================================
24 |
25 | def print_hyper_parameters():
26 |
27 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size)
28 |
29 | sys.stdout.write('tt_ranks_1 %d\n' % FLAGS.tt_ranks_1)
30 |
31 | sys.stdout.write('tt_ranks_2 %d\n' % FLAGS.tt_ranks_2)
32 |
33 | sys.stdout.write('tt_ranks_3 %d\n' % FLAGS.tt_ranks_3)
--------------------------------------------------------------------------------
/LeNet5/mpo_lenet5/inference.py:
--------------------------------------------------------------------------------
1 |
2 | import numpy as np
3 | import tensorflow as tf
4 | import tt
5 | from hyperprameter import *
6 |
7 | r_1 = FLAGS.tt_ranks_1
8 | r_2 = FLAGS.tt_ranks_2
9 | r_3 = FLAGS.tt_ranks_3
10 |
11 | input_node=FLAGS.input_node
12 | output_node=FLAGS.output_node
13 |
14 | # TTO_layer2
15 | inp_modes2 = [2, 10, 10, 2]
16 | out_modes2 = [2, 5, 6, 2]
17 | mat_rank2 = [1, r_1, r_1, r_1, 1]
18 |
19 | # TTO_layer3
20 | inp_modes3 = [2, 5, 6, 2]
21 | out_modes3 = [2, 3, 7, 2]
22 | mat_rank3 = [1, r_2, r_2, r_2, 1]
23 |
24 | # TTO_layer4
25 | inp_modes4 = [2, 3, 7, 2]
26 | out_modes4 = [1, 5, 2, 1]
27 | mat_rank4 = [1, r_3, r_3, r_3, 1]
28 |
29 |
30 | def conv2d(x, W):
31 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID')
32 |
33 |
34 | def max_pool_2x2(x):
35 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
36 | strides=[1, 2, 2, 1], padding='SAME')
37 |
38 |
39 | def weight_variable(shape):
40 | weights = tf.Variable(tf.truncated_normal(shape, stddev=0.1))
41 | return weights
42 |
43 |
44 | def bias_variable(shape):
45 | initial = tf.constant(0.0, shape=shape)
46 | return tf.Variable(initial)
47 |
48 |
49 | def inference(inputs):
50 | W_conv1 = weight_variable([5, 5, 1, 6])
51 | b_conv1 = bias_variable([6])
52 |
53 | x_image = tf.reshape(inputs, [-1, 28, 28, 1])
54 | # 28x28x1
55 | h_conv1 = tf.nn.relu(tf.nn.conv2d(x_image, W_conv1, strides=[1, 1, 1, 1], padding='SAME') + b_conv1)
56 | # 28x28x6
57 | h_pool1 = max_pool_2x2(h_conv1)
58 | # 14x14x6
59 | W_conv2 = weight_variable([5, 5, 6, 16])
60 | b_conv2 = bias_variable([16])
61 | h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
62 | # 10x10x16
63 | h_pool2 = max_pool_2x2(h_conv2)
64 | # 5x5x16
65 | inputs = tf.reshape(h_pool2,[-1,400])
66 | inputs = tf.nn.relu(tt.tto(inputs,
67 | np.array(inp_modes1, dtype=np.int32),
68 | np.array(out_modes1, dtype=np.int32),
69 | np.array(mat_rank1, dtype=np.int32),
70 | scope='tt_scope_1'))
71 | inputs = tf.nn.relu(tt.tto(inputs,
72 | np.array(inp_modes2, dtype=np.int32),
73 | np.array(out_modes2, dtype=np.int32),
74 | np.array(mat_rank2, dtype=np.int32),
75 | scope='tt_scope_2'))
76 |
77 | output = tt.tto(inputs,
78 | np.array(inp_modes3, dtype=np.int32),
79 | np.array(out_modes3, dtype=np.int32),
80 | np.array(mat_rank3, dtype=np.int32),
81 | scope='tt_scope_3')
82 |
83 |
84 | return output
85 |
--------------------------------------------------------------------------------
/LeNet5/mpo_lenet5/train.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | @author: zfgao
4 | """
5 |
6 | import tensorflow as tf
7 | from tensorflow.examples.tutorials.mnist import input_data
8 | import inference
9 | from hyperprameter import *
10 |
11 | BATCH_SIZE=FLAGS.batch_size
12 | TRAINING_STEPS=FLAGS.global_step
13 | LEARNING_RATE_BASE=FLAGS.LEARNING_RATE_BASE
14 | LEARNING_RATE_DECAY=FLAGS.LEARNING_RATE_DECAY
15 | REGULARIZER_RATE=FLAGS.REGULARIZER_RATE
16 | MOVING_DECAY=0.99
17 | #seed =12345
18 | #tf.set_random_seed(seed)
19 |
20 | def mnist(inp):
21 | x=tf.placeholder(tf.float32,[None,inference.input_node],name='x-input')
22 | y_=tf.placeholder(tf.float32,[None,inference.output_node],name='y-input')
23 | y=inference.inference(x)
24 | global_step=tf.Variable(0,trainable=False)
25 | ce=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
26 | loss=tf.reduce_mean(ce)
27 | loss += tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()]) * REGULARIZER_RATE
28 |
29 | learning_rate=tf.train.exponential_decay(LEARNING_RATE_BASE,
30 | global_step,
31 | inp.train.num_examples/BATCH_SIZE,
32 | LEARNING_RATE_DECAY)
33 | train_steps=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
34 |
35 | correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
36 | accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
37 |
38 |
39 | with tf.Session() as sess:
40 | init_op = tf.global_variables_initializer()
41 | sess.run(init_op)
42 | best_acc = 0
43 | for i in range(TRAINING_STEPS):
44 | xs,ys = inp.train.next_batch(BATCH_SIZE)
45 | _,step,lr = sess.run([train_steps,global_step,learning_rate],feed_dict={x:xs,y_:ys})
46 | if i%1000 == 0:
47 | accuracy_score = sess.run(accuracy, feed_dict={x:inp.test.images,y_:inp.test.labels})
48 | print('step={},lr={}'.format(step,lr))
49 | if best_acc< accuracy_score:
50 | best_acc = accuracy_score
51 | print('Accuracy at step %s: %s' % (i, accuracy_score))
52 | accuracy_score=sess.run(accuracy,feed_dict={x:inp.test.images,y_:inp.test.labels})
53 | print("After %s trainning step(s),best accuracy=%g" %(step,best_acc))
54 |
55 |
56 |
57 | def main(argv=None):
58 | inp=input_data.read_data_sets("./data/",validation_size=0,one_hot=True)
59 | mnist(inp)
60 |
61 | if __name__=='__main__':
62 | tf.app.run()
63 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # MPO-code
2 | This repository contains the code for using MPO structure to replace FC2, LeNet-5, VGG, Resnet, Densenet's full-connected layers introduced in the following paper
3 | "Compressing deep neural networks by matrix product operators"
4 | Ze-Feng Gao, Song Cheng, Rong-Qiang He, Hui-Hai Zhao, Z.Y.Xie, Zhong-Yi Lu, and Tao Xiang
5 | The code is built on Tensorflow
6 | # Requirement
7 | Python >=3.4.3
TensorFlow >=1.3.0
8 | # How to run
9 | `$ pip install --user -r requirements.txt`
10 | `$ python train.py --args=args-you-want-change `
11 | # Note
12 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/layers/aux_py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | out_ch,
6 | window_size,
7 | strides=[1, 1],
8 | padding='SAME',
9 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
10 | filters_regularizer=None,
11 | biases_initializer=tf.zeros_initializer,
12 | biases_regularizer=None,
13 | trainable=True,
14 | cpu_variables=False,
15 | scope=None):
16 | """ convolutional layer
17 | Args:
18 | inp: input tensor, float - [batch_size, H, W, C]
19 | out_ch: output channels count count, int
20 | window_size: convolution window size, list [wH, wW]
21 | strides: strides, list [sx, sy]
22 | padding: 'SAME' or 'VALID', string
23 | filters_initializer: filters init function
24 | filters_regularizer: filters regularizer function
25 | biases_initializer: biases init function (if None then no biases will be used)
26 | biases_regularizer: biases regularizer function
27 | trainable: trainable variables flag, bool
28 | cpu_variables: cpu variables flag, bool
29 | scope: layer variable scope name, string
30 | Returns:
31 | out: output tensor, float - [batch_size, H', W', out_ch]
32 | """
33 |
34 | with tf.variable_scope(scope):
35 | shape = inp.get_shape().as_list()
36 | assert len(shape) == 4, "Not 4D input tensor"
37 | in_ch = shape[-1]
38 |
39 | filters = get_var_wrap('filters',
40 | shape=window_size + [in_ch, out_ch],
41 | initializer=filters_initializer,
42 | regularizer=filters_regularizer,
43 | trainable=trainable,
44 | cpu_variable=cpu_variables)
45 |
46 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
47 |
48 | if biases_initializer is not None:
49 | biases = get_var_wrap('biases',
50 | shape=[out_ch],
51 | initializer=biases_initializer,
52 | regularizer=biases_regularizer,
53 | trainable=trainable,
54 | cpu_variable=cpu_variables)
55 | out = tf.add(out, biases, name='out')
56 | else:
57 | out = tf.identity(out, name='out')
58 | return out
59 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/ResNet/resnet-mpo/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/ResNet/resnet/experiments/cifar-10/conv-Ultimate-Tensorization/nets/__pycache__/wideResNet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zfgao66/deeplearning-mpo/c345b9fea79e16f98f9b50e0b4e0bcaf4ed4c8e6/ResNet/resnet/experiments/cifar-10/conv-Ultimate-Tensorization/nets/__pycache__/wideResNet.cpython-36.pyc
--------------------------------------------------------------------------------
/ResNet/resnet/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/ResNet/resnet/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/aux_py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | out_ch,
6 | window_size,
7 | strides=[1, 1],
8 | padding='SAME',
9 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
10 | filters_regularizer=None,
11 | biases_initializer=tf.zeros_initializer,
12 | biases_regularizer=None,
13 | trainable=True,
14 | cpu_variables=False,
15 | scope=None):
16 | """ convolutional layer
17 | Args:
18 | inp: input tensor, float - [batch_size, H, W, C]
19 | out_ch: output channels count count, int
20 | window_size: convolution window size, list [wH, wW]
21 | strides: strides, list [sx, sy]
22 | padding: 'SAME' or 'VALID', string
23 | filters_initializer: filters init function
24 | filters_regularizer: filters regularizer function
25 | biases_initializer: biases init function (if None then no biases will be used)
26 | biases_regularizer: biases regularizer function
27 | trainable: trainable variables flag, bool
28 | cpu_variables: cpu variables flag, bool
29 | scope: layer variable scope name, string
30 | Returns:
31 | out: output tensor, float - [batch_size, H', W', out_ch]
32 | """
33 |
34 | with tf.variable_scope(scope):
35 | shape = inp.get_shape().as_list()
36 | assert len(shape) == 4, "Not 4D input tensor"
37 | in_ch = shape[-1]
38 |
39 | filters = get_var_wrap('filters',
40 | shape=window_size + [in_ch, out_ch],
41 | initializer=filters_initializer,
42 | regularizer=filters_regularizer,
43 | trainable=trainable,
44 | cpu_variable=cpu_variables)
45 |
46 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
47 |
48 | if biases_initializer is not None:
49 | biases = get_var_wrap('biases',
50 | shape=[out_ch],
51 | initializer=biases_initializer,
52 | regularizer=biases_regularizer,
53 | trainable=trainable,
54 | cpu_variable=cpu_variables)
55 | out = tf.add(out, biases, name='out')
56 | else:
57 | out = tf.identity(out, name='out')
58 | return out
59 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/tt.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | from .auxx import get_var_wrap
4 |
5 | def tt(inp,
6 | inp_modes,
7 | out_modes,
8 | mat_ranks,
9 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
10 | cores_regularizer=None,
11 | biases_initializer=tf.zeros_initializer,
12 | biases_regularizer=None,
13 | trainable=True,
14 | cpu_variables=False,
15 | scope=None):
16 | """ tt-layer (tt-matrix by full tensor product)
17 | Args:
18 | inp: input tensor, float - [batch_size, prod(inp_modes)]
19 | inp_modes: input tensor modes
20 | out_modes: output tensor modes
21 | mat_ranks: tt-matrix ranks
22 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
23 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
24 | biases_initializer: biases init function (if None then no biases will be used)
25 | biases_regularizer: biases regularizer function
26 | trainable: trainable variables flag, bool
27 | cpu_variables: cpu variables flag, bool
28 | scope: layer variable scope name, string
29 | Returns:
30 | out: output tensor, float - [batch_size, prod(out_modes)]
31 | """
32 | with tf.variable_scope(scope):
33 | dim = inp_modes.size
34 |
35 | mat_cores = []
36 |
37 | for i in range(dim):
38 | if type(cores_initializer) == list:
39 | cinit = cores_initializer[i]
40 | else:
41 | cinit = cores_initializer
42 |
43 | if type(cores_regularizer) == list:
44 | creg = cores_regularizer[i]
45 | else:
46 | creg = cores_regularizer
47 |
48 | mat_cores.append(get_var_wrap('mat_core_%d' % (i + 1),
49 | shape=[out_modes[i] * mat_ranks[i + 1], mat_ranks[i] * inp_modes[i]],
50 | initializer=cinit,
51 | regularizer=creg,
52 | trainable=trainable,
53 | cpu_variable=cpu_variables))
54 |
55 |
56 | # inp(b,n0,n1,...,nL-1), b: batch_size
57 | out = tf.reshape(inp, [-1, np.prod(inp_modes)]) # out(b,(n0,n1,...,nL-1))
58 | out = tf.transpose(out, [1, 0]) # inp((n0,n1,...,nL-1),b)
59 |
60 | for i in range(dim):
61 | # out((ri,ni),(ni+1,...,nL-1,b,m0,...mi-1))
62 | out = tf.reshape(out, [mat_ranks[i] * inp_modes[i], -1])
63 | # out[(mi,ri+1),(ni+1,...,nL-1,m0,...mi-1,b)] =
64 | # mat_cores_i[(mi,ri+1),(ri,ni)] * out((ri,ni),(ni+1,...,nL-1,b,m0,...mi-1))
65 | out = tf.matmul(mat_cores[i], out)
66 | # out[mi,(ri+1,ni+1,...,nL-1,b,m0,...mi-1)]
67 | out = tf.reshape(out, [out_modes[i], -1])
68 | # out[(ri+1,ni+1,...,nL-1,b,m0,...mi-1),mi]
69 | out = tf.transpose(out, [1, 0])
70 |
71 | if biases_initializer is not None:
72 |
73 | biases = get_var_wrap('biases',
74 | shape=[np.prod(out_modes)],
75 | initializer=biases_initializer,
76 | regularizer=biases_regularizer,
77 | trainable=trainable,
78 | cpu_variable=cpu_variables)
79 |
80 | out = tf.add(tf.reshape(out, [-1, np.prod(out_modes)]), biases, name="out")
81 | else:
82 | out = tf.reshape(out, [-1, np.prod(out_modes)], name="out")
83 |
84 | return out
85 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/ResNet/resnet/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/experiments/cifar-10/conv-Ultimate-Tensorization/hyper_parameters.py:
--------------------------------------------------------------------------------
1 |
2 | # ==============================================================================
3 | import tensorflow as tf
4 | import sys
5 | import math
6 | # Basic model parameters as external flags.
7 | flags = tf.app.flags
8 | FLAGS = flags.FLAGS
9 |
10 | net_type = 'VGG-16'
11 |
12 |
13 | flags.DEFINE_boolean('norm_after_aug', True, 'whether normalization after augmentation or before')
14 | # by_sample: subtract mean of all images and divide each entry by its standard deviation
15 | # by_channel: subtract mean of every channel and divide each channel data by its standard deviation
16 | flags.DEFINE_string('normalization','by_sample','image normalization method')
17 |
18 | flags.DEFINE_integer('random_seed',12345, 'random seed.')
19 |
20 | flags.DEFINE_float('MOMENTUM', 0.9, 'The Nesterov momentum of Momentum optimizer')
21 | flags.DEFINE_float('WEIGHT_DECAY', 2e-4, 'weight decay for L2 regularization')
22 | flags.DEFINE_float('res_conv_drop_prob', 0.0, 'residual convolution block dropout probability. 0 for no dropout.')
23 | flags.DEFINE_float('res_TT_drop_prob', 0.0, 'residual TT block dropout probability. 0 for no dropout.')
24 | flags.DEFINE_float('vgg_conv_drop_prob', 0.0, 'vgg convolution block dropout probability. 0 for no dropout.')
25 | flags.DEFINE_float('initial_learning_rate', 0.1, 'initial learning rate')
26 | flags.DEFINE_integer('num_epochs_per_decay', 50, 'NEPD')
27 | flags.DEFINE_integer('learning_rate_decay_steps', 6, '.')
28 | # flags.DEFINE_integer('learning_rate_decay_factor', 0.5, '.')
29 | flags.DEFINE_float('learning_rate_decay_factor', math.pow(0.1, 0.5), '.')
30 |
31 |
32 | flags.DEFINE_integer('batch_size', 100, 'Batch size. '
33 | 'Must divide evenly into the dataset sizes.')
34 | flags.DEFINE_integer('log_steps', 100, 'Summary log steps')
35 | flags.DEFINE_integer('num_gpus', 1, 'Number of gpus for training')
36 |
37 | if net_type == 'VGG-16':
38 | flags.DEFINE_string('net_module', './nets/vgg-16-mpo.py', 'Module with architecture description.')
39 | flags.DEFINE_integer('tt_rank1', 4, 'mpo rank of 1st full-connected layer')
40 | flags.DEFINE_integer('tt_rank2', 4, 'mpo rank of 2nd full-connected layer')
41 | flags.DEFINE_integer('tt_rank3', 4, 'mpo rank of 3rd full-connected layer')
42 |
43 | log_dir = './log/VGG-16_wd%.1e_NEPD%d_LRDS%d' %\
44 | (FLAGS.WEIGHT_DECAY, FLAGS.num_epochs_per_decay,
45 | FLAGS.learning_rate_decay_steps)
46 |
47 | max_epochs = FLAGS.num_epochs_per_decay * FLAGS.learning_rate_decay_steps + FLAGS.num_epochs_per_decay/2
48 |
49 | learning_rate_decay_boundary = [100, 200, 250]
50 | learning_rate_decay_value = [1, 0.1, 0.01,0.001]
51 | flags.DEFINE_integer("max_epochs", int(max_epochs), 'Number of epochs to run trainer.')
52 |
53 |
54 | flags.DEFINE_string('log_dir', log_dir, 'Directory to put log files.')
55 | flags.DEFINE_string('data_dir', '../data/', 'Directory to put the training data.')
56 |
57 | flags.DEFINE_boolean('log_device_placement', False, """Whether to log device placement.""")
58 |
59 | ##===========================================================================================================
60 |
61 | def print_hyper_parameters():
62 | sys.stdout.write('random_seed %d\n' % FLAGS.random_seed)
63 |
64 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size*FLAGS.num_gpus)
65 | if net_type == 'VGG-16':
66 | sys.stdout.write('r1 %d\n' % FLAGS.tt_rank1)
67 | sys.stdout.write('r2 %d\n' % FLAGS.tt_rank2)
68 | sys.stdout.write('r3 %d\n' % FLAGS.tt_rank3)
69 |
70 | sys.stdout.write('MOMENTUM %.1f\n' % FLAGS.MOMENTUM)
71 | sys.stdout.write('WEIGHT_DECAY %.1e\n' % FLAGS.WEIGHT_DECAY)
72 |
73 | sys.stdout.write('net_module %s\n' % FLAGS.net_module)
74 | sys.stdout.write('log_dir %s\n' % FLAGS.log_dir)
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | in_ch,
6 | out_ch,
7 | window_size,
8 | strides=[1, 1],
9 | padding='SAME',
10 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
11 | filters_regularizer=None,
12 | biases_initializer=tf.zeros_initializer,
13 | biases_regularizer=None,
14 | trainable=True,
15 | cpu_variables=False,
16 | scope=None):
17 | """ convolutional layer
18 | Args:
19 | inp: input tensor, float - [batch_size, H, W, C]
20 | out_ch: output channels count count, int
21 | window_size: convolution window size, list [wH, wW]
22 | strides: strides, list [sx, sy]
23 | padding: 'SAME' or 'VALID', string
24 | filters_initializer: filters init function
25 | filters_regularizer: filters regularizer function
26 | biases_initializer: biases init function (if None then no biases will be used)
27 | biases_regularizer: biases regularizer function
28 | trainable: trainable variables flag, bool
29 | cpu_variables: cpu variables flag, bool
30 | scope: layer variable scope name, string
31 | Returns:
32 | out: output tensor, float - [batch_size, H', W', out_ch]
33 | """
34 |
35 | with tf.variable_scope(scope):
36 | shape = inp.get_shape().as_list()
37 | assert len(shape) == 4, "Not 4D input tensor"
38 |
39 |
40 | filters = get_var_wrap('filters',
41 | shape=window_size + [in_ch, out_ch],
42 | initializer=filters_initializer,
43 | regularizer=filters_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
48 |
49 | if biases_initializer is not None:
50 | biases = get_var_wrap('biases',
51 | shape=[out_ch],
52 | initializer=biases_initializer,
53 | regularizer=biases_regularizer,
54 | trainable=trainable,
55 | cpu_variable=cpu_variables)
56 | out = tf.add(out, biases, name='out')
57 | else:
58 | out = tf.identity(out, name='out')
59 | return out
60 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16-mpo/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/experiments/cifar-10/conv-Ultimate-Tensorization/hyper_parameters.py:
--------------------------------------------------------------------------------
1 |
2 | # ==============================================================================
3 | import tensorflow as tf
4 | import sys
5 | import math
6 | # Basic model parameters as external flags.
7 | flags = tf.app.flags
8 | FLAGS = flags.FLAGS
9 |
10 | net_type = 'VGG-16'
11 |
12 |
13 | flags.DEFINE_boolean('norm_after_aug', True, 'whether normalization after augmentation or before')
14 | # by_sample: subtract mean of all images and divide each entry by its standard deviation
15 | # by_channel: subtract mean of every channel and divide each channel data by its standard deviation
16 | flags.DEFINE_string('normalization','by_sample','image normalization method')
17 |
18 | flags.DEFINE_integer('random_seed',12345, 'random seed.')
19 |
20 | flags.DEFINE_float('MOMENTUM', 0.9, 'The Nesterov momentum of Momentum optimizer')
21 | flags.DEFINE_float('WEIGHT_DECAY', 2e-4, 'weight decay for L2 regularization')
22 |
23 | flags.DEFINE_float('vgg_conv_drop_prob', 0.0, 'vgg convolution block dropout probability. 0 for no dropout.')
24 | flags.DEFINE_float('initial_learning_rate', 0.1, 'initial learning rate')
25 | flags.DEFINE_integer('num_epochs_per_decay', 50, 'NEPD')
26 | flags.DEFINE_integer('learning_rate_decay_steps', 6, '.')
27 | # flags.DEFINE_integer('learning_rate_decay_factor', 0.5, '.')
28 | flags.DEFINE_float('learning_rate_decay_factor', math.pow(0.1, 0.5), '.')
29 |
30 |
31 | flags.DEFINE_integer('batch_size', 100, 'Batch size. '
32 | 'Must divide evenly into the dataset sizes.')
33 | flags.DEFINE_integer('log_steps', 100, 'Summary log steps')
34 | flags.DEFINE_integer('num_gpus', 1, 'Number of gpus for training')
35 |
36 | if net_type == 'VGG-16':
37 | flags.DEFINE_string('net_module', './nets/vgg-16.py', 'Module with architecture description.')
38 | log_dir = './log/VGG-16_wd%.1e_NEPD%d_LRDS%d' %\
39 | (FLAGS.WEIGHT_DECAY, FLAGS.num_epochs_per_decay,
40 | FLAGS.learning_rate_decay_steps)
41 |
42 | max_epochs = FLAGS.num_epochs_per_decay * FLAGS.learning_rate_decay_steps + FLAGS.num_epochs_per_decay/2
43 |
44 | learning_rate_decay_boundary = [100, 200, 250]
45 | learning_rate_decay_value = [1, 0.1, 0.01,0.001]
46 | flags.DEFINE_integer("max_epochs", int(max_epochs), 'Number of epochs to run trainer.')
47 |
48 |
49 | flags.DEFINE_string('log_dir', log_dir, 'Directory to put log files.')
50 | flags.DEFINE_string('data_dir', '../data/', 'Directory to put the training data.')
51 |
52 | flags.DEFINE_boolean('log_device_placement', False, """Whether to log device placement.""")
53 |
54 | ##===========================================================================================================
55 |
56 | def print_hyper_parameters():
57 | sys.stdout.write('random_seed %d\n' % FLAGS.random_seed)
58 |
59 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size*FLAGS.num_gpus)
60 |
61 | sys.stdout.write('MOMENTUM %.1f\n' % FLAGS.MOMENTUM)
62 | sys.stdout.write('WEIGHT_DECAY %.1e\n' % FLAGS.WEIGHT_DECAY)
63 |
64 | sys.stdout.write('net_module %s\n' % FLAGS.net_module)
65 | sys.stdout.write('log_dir %s\n' % FLAGS.log_dir)
--------------------------------------------------------------------------------
/VGG-16/vgg-16/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | in_ch,
6 | out_ch,
7 | window_size,
8 | strides=[1, 1],
9 | padding='SAME',
10 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
11 | filters_regularizer=None,
12 | biases_initializer=tf.zeros_initializer,
13 | biases_regularizer=None,
14 | trainable=True,
15 | cpu_variables=False,
16 | scope=None):
17 | """ convolutional layer
18 | Args:
19 | inp: input tensor, float - [batch_size, H, W, C]
20 | out_ch: output channels count count, int
21 | window_size: convolution window size, list [wH, wW]
22 | strides: strides, list [sx, sy]
23 | padding: 'SAME' or 'VALID', string
24 | filters_initializer: filters init function
25 | filters_regularizer: filters regularizer function
26 | biases_initializer: biases init function (if None then no biases will be used)
27 | biases_regularizer: biases regularizer function
28 | trainable: trainable variables flag, bool
29 | cpu_variables: cpu variables flag, bool
30 | scope: layer variable scope name, string
31 | Returns:
32 | out: output tensor, float - [batch_size, H', W', out_ch]
33 | """
34 |
35 | with tf.variable_scope(scope):
36 | shape = inp.get_shape().as_list()
37 | assert len(shape) == 4, "Not 4D input tensor"
38 |
39 |
40 | filters = get_var_wrap('filters',
41 | shape=window_size + [in_ch, out_ch],
42 | initializer=filters_initializer,
43 | regularizer=filters_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
48 |
49 | if biases_initializer is not None:
50 | biases = get_var_wrap('biases',
51 | shape=[out_ch],
52 | initializer=biases_initializer,
53 | regularizer=biases_regularizer,
54 | trainable=trainable,
55 | cpu_variable=cpu_variables)
56 | out = tf.add(out, biases, name='out')
57 | else:
58 | out = tf.identity(out, name='out')
59 | return out
60 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/layers/tt.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | from .auxx import get_var_wrap
4 |
5 | def tt(inp,
6 | inp_modes,
7 | out_modes,
8 | mat_ranks,
9 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
10 | cores_regularizer=None,
11 | biases_initializer=tf.zeros_initializer,
12 | biases_regularizer=None,
13 | trainable=True,
14 | cpu_variables=False,
15 | scope=None):
16 | """ tt-layer (tt-matrix by full tensor product)
17 | Args:
18 | inp: input tensor, float - [batch_size, prod(inp_modes)]
19 | inp_modes: input tensor modes
20 | out_modes: output tensor modes
21 | mat_ranks: tt-matrix ranks
22 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
23 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
24 | biases_initializer: biases init function (if None then no biases will be used)
25 | biases_regularizer: biases regularizer function
26 | trainable: trainable variables flag, bool
27 | cpu_variables: cpu variables flag, bool
28 | scope: layer variable scope name, string
29 | Returns:
30 | out: output tensor, float - [batch_size, prod(out_modes)]
31 | """
32 | with tf.variable_scope(scope):
33 | dim = inp_modes.size
34 |
35 | mat_cores = []
36 |
37 | for i in range(dim):
38 | if type(cores_initializer) == list:
39 | cinit = cores_initializer[i]
40 | else:
41 | cinit = cores_initializer
42 |
43 | if type(cores_regularizer) == list:
44 | creg = cores_regularizer[i]
45 | else:
46 | creg = cores_regularizer
47 |
48 | mat_cores.append(get_var_wrap('mat_core_%d' % (i + 1),
49 | shape=[out_modes[i] * mat_ranks[i + 1], mat_ranks[i] * inp_modes[i]],
50 | initializer=cinit,
51 | regularizer=creg,
52 | trainable=trainable,
53 | cpu_variable=cpu_variables))
54 |
55 |
56 | # inp(b,n0,n1,...,nL-1), b: batch_size
57 | out = tf.reshape(inp, [-1, np.prod(inp_modes)]) # out(b,(n0,n1,...,nL-1))
58 | out = tf.transpose(out, [1, 0]) # inp((n0,n1,...,nL-1),b)
59 |
60 | for i in range(dim):
61 | # out((ri,ni),(ni+1,...,nL-1,b,m0,...mi-1))
62 | out = tf.reshape(out, [mat_ranks[i] * inp_modes[i], -1])
63 | # out[(mi,ri+1),(ni+1,...,nL-1,m0,...mi-1,b)] =
64 | # mat_cores_i[(mi,ri+1),(ri,ni)] * out((ri,ni),(ni+1,...,nL-1,b,m0,...mi-1))
65 | out = tf.matmul(mat_cores[i], out)
66 | # out[mi,(ri+1,ni+1,...,nL-1,b,m0,...mi-1)]
67 | out = tf.reshape(out, [out_modes[i], -1])
68 | # out[(ri+1,ni+1,...,nL-1,b,m0,...mi-1),mi]
69 | out = tf.transpose(out, [1, 0])
70 |
71 | if biases_initializer is not None:
72 |
73 | biases = get_var_wrap('biases',
74 | shape=[np.prod(out_modes)],
75 | initializer=biases_initializer,
76 | regularizer=biases_regularizer,
77 | trainable=trainable,
78 | cpu_variable=cpu_variables)
79 |
80 | out = tf.add(tf.reshape(out, [-1, np.prod(out_modes)]), biases, name="out")
81 | else:
82 | out = tf.reshape(out, [-1, np.prod(out_modes)], name="out")
83 |
84 | return out
85 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/VGG-16/vgg-16/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/experiments/cifar-10/conv-Ultimate-Tensorization/hyper_parameters.py:
--------------------------------------------------------------------------------
1 |
2 | # ==============================================================================
3 | import tensorflow as tf
4 | import sys
5 | import math
6 | # Basic model parameters as external flags.
7 | flags = tf.app.flags
8 | FLAGS = flags.FLAGS
9 |
10 | net_type = 'VGG-19'
11 |
12 |
13 | flags.DEFINE_boolean('norm_after_aug', True, 'whether normalization after augmentation or before')
14 | # by_sample: subtract mean of all images and divide each entry by its standard deviation
15 | # by_channel: subtract mean of every channel and divide each channel data by its standard deviation
16 | flags.DEFINE_string('normalization','by_sample','image normalization method')
17 |
18 | flags.DEFINE_integer('random_seed',12345, 'random seed.')
19 |
20 | flags.DEFINE_float('MOMENTUM', 0.9, 'The Nesterov momentum of Momentum optimizer')
21 | flags.DEFINE_float('WEIGHT_DECAY', 2e-4, 'weight decay for L2 regularization')
22 | flags.DEFINE_float('res_conv_drop_prob', 0.0, 'residual convolution block dropout probability. 0 for no dropout.')
23 | flags.DEFINE_float('res_TT_drop_prob', 0.0, 'residual TT block dropout probability. 0 for no dropout.')
24 | flags.DEFINE_float('vgg_conv_drop_prob', 0.0, 'vgg convolution block dropout probability. 0 for no dropout.')
25 | flags.DEFINE_float('initial_learning_rate', 0.1, 'initial learning rate')
26 | flags.DEFINE_integer('num_epochs_per_decay', 50, 'NEPD')
27 | flags.DEFINE_integer('learning_rate_decay_steps', 6, '.')
28 | # flags.DEFINE_integer('learning_rate_decay_factor', 0.5, '.')
29 | flags.DEFINE_float('learning_rate_decay_factor', math.pow(0.1, 0.5), '.')
30 |
31 |
32 | flags.DEFINE_integer('batch_size', 100, 'Batch size. '
33 | 'Must divide evenly into the dataset sizes.')
34 | flags.DEFINE_integer('log_steps', 100, 'Summary log steps')
35 | flags.DEFINE_integer('num_gpus', 1, 'Number of gpus for training')
36 |
37 | if net_type == 'VGG-19':
38 | flags.DEFINE_string('net_module', './nets/vgg-19-mpo.py', 'Module with architecture description.')
39 | flags.DEFINE_integer('tt_rank1', 4, 'mpo rank of 1st full-connected layer')
40 | flags.DEFINE_integer('tt_rank2', 4, 'mpo rank of 2nd full-connected layer')
41 | flags.DEFINE_integer('tt_rank3', 4, 'mpo rank of 3rd full-connected layer')
42 |
43 | log_dir = './log/VGG-19_wd%.1e_NEPD%d_LRDS%d' %\
44 | (FLAGS.WEIGHT_DECAY, FLAGS.num_epochs_per_decay,
45 | FLAGS.learning_rate_decay_steps)
46 |
47 | max_epochs = FLAGS.num_epochs_per_decay * FLAGS.learning_rate_decay_steps + FLAGS.num_epochs_per_decay/2
48 |
49 | learning_rate_decay_boundary = [100, 200, 250]
50 | learning_rate_decay_value = [1, 0.1, 0.01,0.001]
51 | flags.DEFINE_integer("max_epochs", int(max_epochs), 'Number of epochs to run trainer.')
52 |
53 |
54 | flags.DEFINE_string('log_dir', log_dir, 'Directory to put log files.')
55 | flags.DEFINE_string('data_dir', '../data/', 'Directory to put the training data.')
56 |
57 | flags.DEFINE_boolean('log_device_placement', False, """Whether to log device placement.""")
58 |
59 | ##===========================================================================================================
60 |
61 | def print_hyper_parameters():
62 | sys.stdout.write('random_seed %d\n' % FLAGS.random_seed)
63 |
64 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size*FLAGS.num_gpus)
65 | if net_type == 'VGG-19':
66 | sys.stdout.write('r1 %d\n' % FLAGS.tt_rank1)
67 | sys.stdout.write('r2 %d\n' % FLAGS.tt_rank2)
68 | sys.stdout.write('r3 %d\n' % FLAGS.tt_rank3)
69 |
70 | sys.stdout.write('MOMENTUM %.1f\n' % FLAGS.MOMENTUM)
71 | sys.stdout.write('WEIGHT_DECAY %.1e\n' % FLAGS.WEIGHT_DECAY)
72 |
73 | sys.stdout.write('net_module %s\n' % FLAGS.net_module)
74 | sys.stdout.write('log_dir %s\n' % FLAGS.log_dir)
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/layers/aux_py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | in_ch,
6 | out_ch,
7 | window_size,
8 | strides=[1, 1],
9 | padding='SAME',
10 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
11 | filters_regularizer=None,
12 | biases_initializer=tf.zeros_initializer,
13 | biases_regularizer=None,
14 | trainable=True,
15 | cpu_variables=False,
16 | scope=None):
17 | """ convolutional layer
18 | Args:
19 | inp: input tensor, float - [batch_size, H, W, C]
20 | out_ch: output channels count count, int
21 | window_size: convolution window size, list [wH, wW]
22 | strides: strides, list [sx, sy]
23 | padding: 'SAME' or 'VALID', string
24 | filters_initializer: filters init function
25 | filters_regularizer: filters regularizer function
26 | biases_initializer: biases init function (if None then no biases will be used)
27 | biases_regularizer: biases regularizer function
28 | trainable: trainable variables flag, bool
29 | cpu_variables: cpu variables flag, bool
30 | scope: layer variable scope name, string
31 | Returns:
32 | out: output tensor, float - [batch_size, H', W', out_ch]
33 | """
34 |
35 | with tf.variable_scope(scope):
36 | shape = inp.get_shape().as_list()
37 | assert len(shape) == 4, "Not 4D input tensor"
38 |
39 |
40 | filters = get_var_wrap('filters',
41 | shape=window_size + [in_ch, out_ch],
42 | initializer=filters_initializer,
43 | regularizer=filters_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
48 |
49 | if biases_initializer is not None:
50 | biases = get_var_wrap('biases',
51 | shape=[out_ch],
52 | initializer=biases_initializer,
53 | regularizer=biases_regularizer,
54 | trainable=trainable,
55 | cpu_variable=cpu_variables)
56 | out = tf.add(out, biases, name='out')
57 | else:
58 | out = tf.identity(out, name='out')
59 | return out
60 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19-mpo/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/experiments/cifar-10/conv-Ultimate-Tensorization/hyper_parameters.py:
--------------------------------------------------------------------------------
1 |
2 | # ==============================================================================
3 | import tensorflow as tf
4 | import sys
5 | import math
6 | # Basic model parameters as external flags.
7 | flags = tf.app.flags
8 | FLAGS = flags.FLAGS
9 |
10 | net_type = 'VGG-19'
11 |
12 |
13 | flags.DEFINE_boolean('norm_after_aug', True, 'whether normalization after augmentation or before')
14 | # by_sample: subtract mean of all images and divide each entry by its standard deviation
15 | # by_channel: subtract mean of every channel and divide each channel data by its standard deviation
16 | flags.DEFINE_string('normalization','by_sample','image normalization method')
17 |
18 | flags.DEFINE_integer('random_seed',12345, 'random seed.')
19 |
20 | flags.DEFINE_float('MOMENTUM', 0.9, 'The Nesterov momentum of Momentum optimizer')
21 | flags.DEFINE_float('WEIGHT_DECAY', 2e-4, 'weight decay for L2 regularization')
22 |
23 | flags.DEFINE_float('vgg_conv_drop_prob', 0.0, 'vgg convolution block dropout probability. 0 for no dropout.')
24 | flags.DEFINE_float('initial_learning_rate', 0.1, 'initial learning rate')
25 | flags.DEFINE_integer('num_epochs_per_decay', 50, 'NEPD')
26 | flags.DEFINE_integer('learning_rate_decay_steps', 6, '.')
27 | # flags.DEFINE_integer('learning_rate_decay_factor', 0.5, '.')
28 | flags.DEFINE_float('learning_rate_decay_factor', math.pow(0.1, 0.5), '.')
29 |
30 |
31 | flags.DEFINE_integer('batch_size', 100, 'Batch size. '
32 | 'Must divide evenly into the dataset sizes.')
33 | flags.DEFINE_integer('log_steps', 100, 'Summary log steps')
34 | flags.DEFINE_integer('num_gpus', 1, 'Number of gpus for training')
35 |
36 | if net_type == 'VGG-19':
37 | flags.DEFINE_string('net_module', './nets/vgg-19.py', 'Module with architecture description.')
38 | log_dir = './log/VGG-19_wd%.1e_NEPD%d_LRDS%d' %\
39 | (FLAGS.WEIGHT_DECAY, FLAGS.num_epochs_per_decay,
40 | FLAGS.learning_rate_decay_steps)
41 |
42 | max_epochs = FLAGS.num_epochs_per_decay * FLAGS.learning_rate_decay_steps + FLAGS.num_epochs_per_decay/2
43 |
44 | learning_rate_decay_boundary = [100, 200, 250]
45 | learning_rate_decay_value = [1, 0.1, 0.01,0.001]
46 | flags.DEFINE_integer("max_epochs", int(max_epochs), 'Number of epochs to run trainer.')
47 |
48 |
49 | flags.DEFINE_string('log_dir', log_dir, 'Directory to put log files.')
50 | flags.DEFINE_string('data_dir', '../data/', 'Directory to put the training data.')
51 |
52 | flags.DEFINE_boolean('log_device_placement', False, """Whether to log device placement.""")
53 |
54 | ##===========================================================================================================
55 |
56 | def print_hyper_parameters():
57 | sys.stdout.write('random_seed %d\n' % FLAGS.random_seed)
58 |
59 | sys.stdout.write('batch_size %d\n' % FLAGS.batch_size*FLAGS.num_gpus)
60 |
61 | sys.stdout.write('MOMENTUM %.1f\n' % FLAGS.MOMENTUM)
62 | sys.stdout.write('WEIGHT_DECAY %.1e\n' % FLAGS.WEIGHT_DECAY)
63 |
64 | sys.stdout.write('net_module %s\n' % FLAGS.net_module)
65 | sys.stdout.write('log_dir %s\n' % FLAGS.log_dir)
--------------------------------------------------------------------------------
/VGG-19/vgg-19/experiments/cifar-10/data/prepare_data.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | for x in range(1, 6):
26 | fname = batches_dir + '/data_batch_' + str(x)
27 | data, labels = unpickle(fname)
28 | if x == 1:
29 | train_images = data
30 | train_labels = labels
31 | else:
32 | train_images = np.vstack((train_images, data))
33 | train_labels = np.concatenate((train_labels, labels))
34 |
35 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
36 |
37 | print(train_images.shape, validation_images.shape)
38 | print(train_labels.shape, validation_labels.shape)
39 | np.savez_compressed('cifar', train_images=train_images, validation_images=validation_images,
40 | train_labels=train_labels, validation_labels=validation_labels)
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/experiments/cifar-10/data/prepare_data_KA.py:
--------------------------------------------------------------------------------
1 | ################################################################
2 | # Load and unpack CIFAR-10 python version #
3 | # from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz #
4 | # #
5 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
6 | # Run this scipt with python3 only #
7 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #
8 | ################################################################
9 |
10 | import pickle
11 | import numpy as np
12 |
13 | batches_dir = 'cifar-10-batches-py'
14 |
15 | def unpickle(fname):
16 | fo = open(fname, 'rb')
17 | d = pickle.load(fo, encoding='bytes')
18 | fo.close()
19 | data = np.reshape(d[b'data'], [-1, 3, 32, 32])
20 | data = np.transpose(data, [0, 2, 3, 1]) # (10000,32,32,3)
21 | data = np.reshape(data, [-1, 32*32*3])
22 | labels = np.array(d[b'labels'], dtype='int8')
23 | return data, labels
24 |
25 | # image(N, 32x32x3)
26 | def ket_augmentation(images):
27 | # images_KA(n, y4,y3,y2,y1,y0, x4,x3,x2,x1,x0, j)
28 | # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11)
29 | images_KA = np.reshape(images, [-1, 2,2,2,2,2, 2,2,2,2,2, 3])
30 |
31 | # images_KA(n, y4,x4, y3,x3, y2,x2, y1,x1, y0,x0, j)
32 | images_KA = np.transpose(images_KA, [0, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 11])
33 | # images_KA = np.transpose(images_KA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
34 |
35 | images_KA = np.reshape(images_KA, [-1, 32*32*3])
36 | return images_KA
37 |
38 | for x in range(1, 6):
39 | fname = batches_dir + '/data_batch_' + str(x)
40 | data, labels = unpickle(fname)
41 | if x == 1:
42 | train_images = data
43 | train_labels = labels
44 | else:
45 | train_images = np.vstack((train_images, data))
46 | train_labels = np.concatenate((train_labels, labels))
47 | train_images = ket_augmentation(train_images)
48 | # train_images_1 = ket_augmentation(train_images)
49 | # train_diff = train_images - train_images_1
50 | # print(train_diff.max())
51 |
52 | validation_images, validation_labels = unpickle(batches_dir + '/test_batch')
53 | validation_images = ket_augmentation(validation_images)
54 |
55 | print(train_images.shape, validation_images.shape)
56 | print(train_labels.shape, validation_labels.shape)
57 | np.savez_compressed('cifar_KA', train_images=train_images, validation_images=validation_images,
58 | train_labels=train_labels, validation_labels=validation_labels)
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/__init__.py:
--------------------------------------------------------------------------------
1 | from . import layers
2 | from . import tt
3 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
2 | from .linear import *
3 | from .linear_dev import *
4 | from .batch_normalization import *
5 | from .tt import *
6 | from .tr import *
7 | from .ttrelu import *
8 | from .tt_dev import *
9 | from .conv import *
10 | from .tt_conv import *
11 | from .tt_conv_full import *
12 | from .tt_conv_direct import *
13 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/aux_py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/auxx.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 |
3 |
4 | def get_var_wrap(name,
5 | shape,
6 | initializer,
7 | regularizer,
8 | trainable,
9 | cpu_variable):
10 | if cpu_variable:
11 | with tf.device('/cpu:0'):
12 | return tf.get_variable(name,
13 | shape=shape,
14 | initializer=initializer,
15 | regularizer=regularizer,
16 | trainable=trainable)
17 | return tf.get_variable(name,
18 | shape=shape,
19 | initializer=initializer,
20 | regularizer=regularizer,
21 | trainable=trainable)
22 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/conv.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def conv(inp,
5 | in_ch,
6 | out_ch,
7 | window_size,
8 | strides=[1, 1],
9 | padding='SAME',
10 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
11 | filters_regularizer=None,
12 | biases_initializer=tf.zeros_initializer,
13 | biases_regularizer=None,
14 | trainable=True,
15 | cpu_variables=False,
16 | scope=None):
17 | """ convolutional layer
18 | Args:
19 | inp: input tensor, float - [batch_size, H, W, C]
20 | out_ch: output channels count count, int
21 | window_size: convolution window size, list [wH, wW]
22 | strides: strides, list [sx, sy]
23 | padding: 'SAME' or 'VALID', string
24 | filters_initializer: filters init function
25 | filters_regularizer: filters regularizer function
26 | biases_initializer: biases init function (if None then no biases will be used)
27 | biases_regularizer: biases regularizer function
28 | trainable: trainable variables flag, bool
29 | cpu_variables: cpu variables flag, bool
30 | scope: layer variable scope name, string
31 | Returns:
32 | out: output tensor, float - [batch_size, H', W', out_ch]
33 | """
34 |
35 | with tf.variable_scope(scope):
36 | shape = inp.get_shape().as_list()
37 | assert len(shape) == 4, "Not 4D input tensor"
38 |
39 |
40 | filters = get_var_wrap('filters',
41 | shape=window_size + [in_ch, out_ch],
42 | initializer=filters_initializer,
43 | regularizer=filters_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.nn.conv2d(inp, filters, [1] + strides + [1], padding, name='conv2d')
48 |
49 | if biases_initializer is not None:
50 | biases = get_var_wrap('biases',
51 | shape=[out_ch],
52 | initializer=biases_initializer,
53 | regularizer=biases_regularizer,
54 | trainable=trainable,
55 | cpu_variable=cpu_variables)
56 | out = tf.add(out, biases, name='out')
57 | else:
58 | out = tf.identity(out, name='out')
59 | return out
60 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/linear.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from .auxx import get_var_wrap
3 |
4 | def linear(inp,
5 | out_size,
6 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
7 | weights_regularizer=None,
8 | biases_initializer=tf.zeros_initializer,
9 | biases_regularizer=None,
10 | trainable=True,
11 | cpu_variables=False,
12 | scope=None):
13 | """ linear layer
14 | Args:
15 | inp: input tensor, float - [batch_size, inp_size]
16 | out_size: layer units count, int
17 | weights_initializer: weights init function
18 | weights_regularizer: weights regularizer function
19 | biases_initializer: biases init function (if None then no biases will be used)
20 | biases_regularizer: biases regularizer function
21 | trainable: trainable variables flag, bool
22 | cpu_variables: cpu variables flag, bool
23 | scope: layer variable scope name, string
24 | Returns:
25 | out: output tensor, float - [batch_size, out_size]
26 | """
27 | with tf.variable_scope(scope):
28 | shape = inp.get_shape().as_list()
29 | assert len(shape) == 2, 'Not 2D input tensor'
30 | inp_size = shape[-1]
31 |
32 | weights = get_var_wrap('weights',
33 | shape=[inp_size, out_size],
34 | initializer=weights_initializer,
35 | regularizer=weights_regularizer,
36 | trainable=trainable,
37 | cpu_variable=cpu_variables)
38 |
39 | if biases_initializer is not None:
40 | biases = get_var_wrap('biases',
41 | shape=[out_size],
42 | initializer=biases_initializer,
43 | regularizer=biases_regularizer,
44 | trainable=trainable,
45 | cpu_variable=cpu_variables)
46 |
47 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
48 | else:
49 | out = tf.matmul(inp, weights, name='out')
50 | return out
51 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/linear_dev.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 |
6 | def linear_dev(inp,
7 | out_size,
8 | weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
9 | weights_regularizer=None,
10 | biases_initializer=tf.zeros_initializer,
11 | biases_regularizer=None,
12 | trainable=True,
13 | cpu_variables=False,
14 | scope=None):
15 | """ linear layer
16 | Args:
17 | inp: input tensor, float - [batch_size, inp_size]
18 | out_size: layer units count, int
19 | weights_initializer: weights init function
20 | weights_regularizer: weights regularizer function
21 | biases_initializer: biases init function (if None then no biases will be used)
22 | biases_regularizer: biases regularizer function
23 | trainable: trainable variables flag, bool
24 | cpu_variables: cpu variables flag, bool
25 | scope: layer variable scope name, string
26 | Returns:
27 | out: output tensor, float - [batch_size, out_size]
28 | """
29 | with tf.variable_scope(scope):
30 | shape = inp.get_shape().as_list()
31 | assert len(shape) == 2, 'Not 2D input tensor'
32 | inp_size = shape[-1]
33 | x = math.sqrt(6.0 / (inp_size*out_size))
34 | sigma = math.sqrt(2.0 / (inp_size * out_size))
35 | # tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.uniform(low=-x,high=x,size=[inp_size, out_size])
36 | tempI = np.eye(inp_size, out_size, dtype='float32') + np.random.normal(0.0, sigma,[inp_size, out_size])
37 | weights = get_var_wrap('weights',
38 | shape=None,
39 | initializer=tempI.astype(dtype='float32'),
40 | regularizer=weights_regularizer,
41 | trainable=trainable,
42 | cpu_variable=cpu_variables)
43 |
44 | if biases_initializer is not None:
45 | biases = get_var_wrap('biases',
46 | shape=[out_size],
47 | initializer=biases_initializer,
48 | regularizer=biases_regularizer,
49 | trainable=trainable,
50 | cpu_variable=cpu_variables)
51 |
52 | out = tf.add(tf.matmul(inp, weights, name='matmul'), biases, name='out')
53 | else:
54 | out = tf.matmul(inp, weights, name='out')
55 | return out
56 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/tt.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | from .auxx import get_var_wrap
4 |
5 | def tt(inp,
6 | inp_modes,
7 | out_modes,
8 | mat_ranks,
9 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
10 | cores_regularizer=None,
11 | biases_initializer=tf.zeros_initializer,
12 | biases_regularizer=None,
13 | trainable=True,
14 | cpu_variables=False,
15 | scope=None):
16 | """ tt-layer (tt-matrix by full tensor product)
17 | Args:
18 | inp: input tensor, float - [batch_size, prod(inp_modes)]
19 | inp_modes: input tensor modes
20 | out_modes: output tensor modes
21 | mat_ranks: tt-matrix ranks
22 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
23 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
24 | biases_initializer: biases init function (if None then no biases will be used)
25 | biases_regularizer: biases regularizer function
26 | trainable: trainable variables flag, bool
27 | cpu_variables: cpu variables flag, bool
28 | scope: layer variable scope name, string
29 | Returns:
30 | out: output tensor, float - [batch_size, prod(out_modes)]
31 | """
32 | with tf.variable_scope(scope):
33 | dim = inp_modes.size
34 |
35 | mat_cores = []
36 |
37 | for i in range(dim):
38 | if type(cores_initializer) == list:
39 | cinit = cores_initializer[i]
40 | else:
41 | cinit = cores_initializer
42 |
43 | if type(cores_regularizer) == list:
44 | creg = cores_regularizer[i]
45 | else:
46 | creg = cores_regularizer
47 |
48 | mat_cores.append(get_var_wrap('mat_core_%d' % (i + 1),
49 | shape=[out_modes[i] * mat_ranks[i + 1], mat_ranks[i] * inp_modes[i]],
50 | initializer=cinit,
51 | regularizer=creg,
52 | trainable=trainable,
53 | cpu_variable=cpu_variables))
54 |
55 |
56 | # inp(b,n0,n1,...,nL-1), b: batch_size
57 | out = tf.reshape(inp, [-1, np.prod(inp_modes)]) # out(b,(n0,n1,...,nL-1))
58 | out = tf.transpose(out, [1, 0]) # inp((n0,n1,...,nL-1),b)
59 |
60 | for i in range(dim):
61 | # out((ri,ni),(ni+1,...,nL-1,b,m0,...mi-1))
62 | out = tf.reshape(out, [mat_ranks[i] * inp_modes[i], -1])
63 | # out[(mi,ri+1),(ni+1,...,nL-1,m0,...mi-1,b)] =
64 | # mat_cores_i[(mi,ri+1),(ri,ni)] * out((ri,ni),(ni+1,...,nL-1,b,m0,...mi-1))
65 | out = tf.matmul(mat_cores[i], out)
66 | # out[mi,(ri+1,ni+1,...,nL-1,b,m0,...mi-1)]
67 | out = tf.reshape(out, [out_modes[i], -1])
68 | # out[(ri+1,ni+1,...,nL-1,b,m0,...mi-1),mi]
69 | out = tf.transpose(out, [1, 0])
70 |
71 | if biases_initializer is not None:
72 |
73 | biases = get_var_wrap('biases',
74 | shape=[np.prod(out_modes)],
75 | initializer=biases_initializer,
76 | regularizer=biases_regularizer,
77 | trainable=trainable,
78 | cpu_variable=cpu_variables)
79 |
80 | out = tf.add(tf.reshape(out, [-1, np.prod(out_modes)]), biases, name="out")
81 | else:
82 | out = tf.reshape(out, [-1, np.prod(out_modes)], name="out")
83 |
84 | return out
85 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/layers/tt_conv1d_full.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | import numpy as np
3 | import math
4 | from .auxx import get_var_wrap
5 | import tt_conv_full
6 |
7 | def tt_conv1d_full(inp,
8 | window,
9 | inp_ch_modes,
10 | out_ch_modes,
11 | ranks,
12 | strides=[1, 1],
13 | padding='SAME',
14 | filters_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
15 | filters_regularizer=None,
16 | cores_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
17 | cores_regularizer=None,
18 | biases_initializer=tf.zeros_initializer,
19 | biases_regularizer=None,
20 | trainable=True,
21 | cpu_variables=False,
22 | scope=None):
23 | """
24 | conv1d wrapper for conv2d function. Internally tensorflow does a conv2d for its vanilla
25 | conv1d. Similarly, this process is applied here. Input is expanded by dim 1 and then output is simply squeezed.
26 |
27 | Note: window should be [1, w] where you insert your width
28 | strides should be [1, stride_width]
29 |
30 |
31 | tt-conv-layer (convolution of full input tensor with tt-filters (make tt full then use conv2d))
32 | Args:
33 | inp: input tensor, float - [batch_size, W, C]
34 | window: convolution window size, list [wH, wW]
35 | inp_ch_modes: input channels modes, np.array (int32) of size d
36 | out_ch_modes: output channels modes, np.array (int32) of size d
37 | ranks: tt-filters ranks, np.array (int32) of size (d + 1)
38 | strides: strides, list of 2 ints - [sx, sy]
39 | padding: 'SAME' or 'VALID', string
40 | filters_initializer: filters init function
41 | filters_regularizer: filters regularizer function
42 | cores_initializer: cores init function, could be a list of functions for specifying different function for each core
43 | cores_regularizer: cores regularizer function, could be a list of functions for specifying different function for each core
44 | biases_initializer: biases init function (if None then no biases will be used)
45 | biases_regularizer: biases regularizer function
46 | trainable: trainable variables flag, bool
47 | cpu_variables: cpu variables flag, bool
48 | scope: layer variable scope name, string
49 | Returns:
50 | out: output tensor, float - [batch_size, W, prod(out_modes)]
51 | """
52 | inp_expanded = tf.expand_dims(inp, dim = 1) # expand on height dim
53 |
54 | conv2d_output = tt_conv_full(inp,
55 | window,
56 | inp_ch_modes,
57 | out_ch_modes,
58 | ranks,
59 | strides=strides,
60 | padding=padding,
61 | filters_initializer=filters_initializer,
62 | filters_regularizer=filters_regularizer,
63 | cores_initializer=cores_initializer,
64 | cores_regularizer=cores_regularizer,
65 | biases_initializer=biases_initializer,
66 | biases_regularizer=biases_regularizer,
67 | trainable=trainable,
68 | cpu_variables=cpu_variables,
69 | scope=scope)
70 |
71 | return tf.squeeze(conv2d_output) # get rid of height dimension
72 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/tt/__init__.py:
--------------------------------------------------------------------------------
1 | from .svd import *
2 | from .max_ranks import *
3 | from .matrix_svd import *
4 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/tt/matrix_svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .svd import svd
3 |
4 | def matrix_svd(X, left_modes, right_modes, ranks):
5 | """ TT-SVD for matrix
6 | Args:
7 | X: input matrix, numpy array float32
8 | left_modes: tt-left-modes, numpy array int32
9 | right_modes: tt-right-modes, numpy array int32
10 | ranks: tt-ranks, numpy array int32
11 | Returns:
12 | core: tt-cores array, numpy 1D array float32
13 | """
14 | c = X.copy()
15 | d = left_modes.size
16 | c = np.reshape(c, np.concatenate((left_modes, right_modes)))
17 | order = np.repeat(np.arange(0, d), 2) + np.tile([0, d], d)
18 | c = np.transpose(c, axes=order)
19 | c = np.reshape(c, left_modes * right_modes)
20 | return svd(c, left_modes * right_modes, ranks)
21 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/tt/max_ranks.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def max_ranks(modes):
4 | """ Computation of maximal ranks for TT-SVD
5 | Args:
6 | modes: tt-modes, numpy array int32
7 | Returns:
8 | ranks: maximal tt-ranks, numpy array int32
9 | """
10 | d = modes.size
11 | ranks = np.zeros(d + 1, dtype='int32')
12 | ranks[0] = 1
13 | prod = np.prod(modes)
14 | for i in range(d):
15 | m = ranks[i] * modes[i]
16 | ranks[i + 1] = min(m, prod // m);
17 | prod = prod // m * ranks[i + 1]
18 | ranks[d] = 1
19 | return ranks
20 |
--------------------------------------------------------------------------------
/VGG-19/vgg-19/tensornet/tt/svd.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | def svd(X, modes, ranks):
4 | """ TT-SVD
5 | Args:
6 | X: input array, numpy array float32
7 | modes: tt-modes, numpy array int32
8 | ranks: tt-ranks, numpy array int32
9 | Returns:
10 | core: tt-cores array, numpy 1D array float32
11 | """
12 | c = X.copy()
13 | d = modes.size
14 | core = np.zeros(np.sum(ranks[:-1] * modes * ranks[1:]), dtype='float32')
15 | pos = 0
16 | for i in range(0, d-1):
17 | m = ranks[i] * modes[i]
18 | c = np.reshape(c, [m, -1])
19 | u, s, v = np.linalg.svd(c, full_matrices=False)
20 | u = u[:, 0:ranks[i + 1]]
21 | s = s[0:ranks[i + 1]]
22 | v = v[0:ranks[i + 1], :]
23 | core[pos:pos + ranks[i] * modes[i] * ranks[i + 1]] = u.ravel()
24 | pos += ranks[i] * modes[i] * ranks[i + 1]
25 | c = np.dot(np.diag(s), v)
26 | core[pos:pos + ranks[d - 1] * modes[d - 1] * ranks[d]] = c.ravel()
27 | return core
28 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | tensorflow>=1.3.0
2 | python>=3.4.3
3 |
--------------------------------------------------------------------------------