\n",
2898 | "This work is released under the MIT license.\n",
2899 | "
"
2900 | ]
2901 | }
2902 | ],
2903 | "metadata": {
2904 | "kernelspec": {
2905 | "display_name": "Python 3",
2906 | "language": "python",
2907 | "name": "python3"
2908 | },
2909 | "language_info": {
2910 | "codemirror_mode": {
2911 | "name": "ipython",
2912 | "version": 3
2913 | },
2914 | "file_extension": ".py",
2915 | "mimetype": "text/x-python",
2916 | "name": "python",
2917 | "nbconvert_exporter": "python",
2918 | "pygments_lexer": "ipython3",
2919 | "version": "3.8.3"
2920 | }
2921 | },
2922 | "nbformat": 4,
2923 | "nbformat_minor": 4
2924 | }
2925 |
--------------------------------------------------------------------------------
/Examples/test.ipynb:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/Figures/algorithm1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tongnie/tensorlib/1515a6307dea0c28c742d594c7eed013fb63cca1/Figures/algorithm1.png
--------------------------------------------------------------------------------
/Figures/algorithm2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tongnie/tensorlib/1515a6307dea0c28c742d594c7eed013fb63cca1/Figures/algorithm2.png
--------------------------------------------------------------------------------
/Figures/missing patterns.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tongnie/tensorlib/1515a6307dea0c28c742d594c7eed013fb63cca1/Figures/missing patterns.png
--------------------------------------------------------------------------------
/Figures/norm_compare.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tongnie/tensorlib/1515a6307dea0c28c742d594c7eed013fb63cca1/Figures/norm_compare.png
--------------------------------------------------------------------------------
/Figures/objective.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tongnie/tensorlib/1515a6307dea0c28c742d594c7eed013fb63cca1/Figures/objective.png
--------------------------------------------------------------------------------
/Helper.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 | """
4 | Tools for tensor operations
5 |
6 | @author: nietong
7 | """
8 |
9 | import numpy as np
10 | from collections import deque
11 | import matplotlib.pyplot as plt
12 | import random as random
13 |
14 |
15 | def shiftdim(array, n=None):
16 | if n is not None:
17 | if n >= 0:
18 | axes = tuple(range(len(array.shape)))
19 | new_axes = deque(axes)
20 | new_axes.rotate(n)
21 | return np.moveaxis(array, axes, tuple(new_axes))
22 | return np.expand_dims(array, axis=tuple(range(-n)))
23 | else:
24 | idx = 0
25 | for dim in array.shape:
26 | if dim == 1:
27 | idx += 1
28 | else:
29 | break
30 | axes = tuple(range(idx))
31 | # Note that this returns a tuple of 2 results
32 |
33 |
34 | def Fold(X, dim, i):
35 | #Fold a matrix into a tensor in mode i, dim is a tuple of the targeted tensor.
36 | dim = np.roll(dim, -i)
37 | X = shiftdim(np.reshape(X, dim,order='F'), len(dim)-i)
38 | return X
39 |
40 |
41 | def Unfold( X, dim, i ):
42 | #Unfold a tensor into a tensor in mode i.
43 | X_unfold = np.reshape(shiftdim(X,i), (dim[i],-1),order='F')
44 | return X_unfold
45 |
46 |
47 | def TensorFromMat(mat,dim):
48 | #Construct a 3D tensor from a matrix
49 | days_slice = [(start_i,start_i + dim[0]) for start_i in list(range(0,dim[0]*dim[2],dim[0]))]
50 | array_list = []
51 | for day_slice in days_slice:
52 | start_i,end_i = day_slice[0],day_slice[1]
53 | array_slice = mat[start_i:end_i,:]
54 | array_list.append(array_slice)
55 | tensor3d = np.array(np.stack(array_list,axis = 0).astype('float64'))
56 | tensor3d = np.moveaxis(tensor3d,0,-1)
57 |
58 | print(tensor3d.shape)
59 |
60 | return tensor3d
61 |
62 |
63 | def Tensor2Mat(tensor):
64 | #convert a tensor into a matrix by flattening the 'day' mode to 'time interval'.
65 | #The shape of given tensor should be 'time interval * locations * days'.
66 | #Note that this operation is slightly different from Unfold operation
67 | for k in range(np.shape(tensor)[-1]):
68 | if k == 0:
69 | stacked = np.vstack(tensor[:,:,k])
70 | else:
71 | stacked = np.vstack((stacked,tensor[:,:,k]))
72 | return stacked
73 |
74 |
75 | def compute_MAE(X_masked,X_true,X_hat): #Only calculate the errors on the masked and nonzero positions
76 | pos_test = np.where((X_true != 0) & (X_masked == 0))
77 | MAE = np.sum(abs(X_true[pos_test]-X_hat[pos_test]))/X_true[pos_test].shape[0]
78 |
79 | return MAE
80 |
81 |
82 | def compute_RMSE(X_masked,X_true,X_hat):
83 | pos_test = np.where((X_true != 0) & (X_masked == 0))
84 | RMSE = np.sqrt(((X_true[pos_test]-X_hat[pos_test])**2).sum()/X_true[pos_test].shape[0])
85 |
86 | return RMSE
87 |
88 |
89 | def compute_MAPE(X_masked,X_true,X_hat):
90 | pos_test = np.where((X_true != 0) & (X_masked == 0))
91 | MAPE = np.sum(np.abs(X_true[pos_test]-X_hat[pos_test]) / X_true[pos_test]) / X_true[pos_test].shape[0]
92 |
93 | return MAPE
94 |
95 |
96 | def get_missing_rate(X_lost):
97 | o_channel_num = (X_lost == 0).astype(int).sum().sum()
98 | matrix_miss_rate = o_channel_num/(X_lost.size)
99 |
100 | return matrix_miss_rate
101 |
102 |
103 | def generate_fiber_missing(tensor3d_true,lost_rate,mode:int):
104 | #three kinds of fiber-like missing cases, the original tensor structure is intervals*links*days.
105 | #mode0:links*days combination
106 | #mode1:intervals*days combination
107 | #mode2:intervals*links combination
108 | n = tensor3d_true.shape
109 | nn = np.delete(n,mode)
110 | S = np.ones(nn)
111 | coord = []
112 | for i in range(nn[0]):
113 | for j in range(nn[1]):
114 | coord.append((i,j))
115 | mask = random.sample(coord,int(lost_rate*len(coord)))
116 | for coord in mask:
117 | S[coord[0],coord[1]] = 0
118 | fai = np.expand_dims(S,mode).repeat(n[mode],axis=mode)
119 | tensor3d_lost_fiber = fai*tensor3d_true
120 | tensor_miss_rate = get_missing_rate(tensor3d_lost_fiber)
121 | print(f'fiber-mode{mode} missing rate of tensor is：{100*tensor_miss_rate:.2f}%')
122 |
123 | return tensor3d_lost_fiber
124 |
125 | def generate_tensor_random_missing(tensor3d_true,lost_rate):
126 | tensor3d_lost = tensor3d_true.copy()
127 | coord = []
128 | m,n,q = tensor3d_lost.shape
129 | for i in range(m):
130 | for j in range(n):
131 | for k in range(q):
132 | coord.append((i,j,k))
133 |
134 | mask = random.sample(coord,int(lost_rate*len(coord)))
135 | for coord in mask:
136 | tensor3d_lost[coord[0]][coord[1]][coord[2]] = 0
137 | return tensor3d_lost
138 |
139 |
--------------------------------------------------------------------------------
/Imputer.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 | """
4 | Truncated tensor Schatten p-norm based low-rank tensor completion, LRTC-TSpN
5 |
6 | @author: nietong
7 | """
8 |
9 | import numpy as np
10 | import matplotlib.pyplot as plt
11 | from Helper import Fold,Unfold,compute_MAE,compute_RMSE,compute_MAPE
12 |
13 | #Optiminize Truncated Schatten p-norm via ADMM
14 |
15 | def truncation(unfoldj,theta):
16 | #calculate the truncation of each unfolding
17 | dim = np.array(unfoldj.shape)
18 | wj = np.zeros(min(dim),)
19 | r = np.int(np.ceil(theta * min(dim)))
20 | wj[r:] = 1
21 |
22 | return wj
23 |
24 |
25 | def GST(sigma,w,p,J=5): #J is the ineer iterations of GST, J=5 is supposed to be enough
26 | #Generalized soft-thresholding algorithm
27 | if w == 0:
28 | Sp = sigma
29 | else:
30 | dt = np.zeros(J+1)
31 | tau = (2*w*(1-p))**(1/(2-p)) + w*p*(2*w*(1-p))**((p-1)/(2-p))
32 | if np.abs(sigma) <= tau:
33 | Sp = 0
34 | else:
35 | dt[0] = np.abs(sigma)
36 | for k in range(J):
37 | dt[k+1] = np.abs(sigma) - w*p*(dt[k])**(p-1)
38 | Sp = np.sign(sigma)*dt[k].item()
39 |
40 | return Sp
41 |
42 |
43 | def update_Mi(mat,alphai,beta,p,theta):
44 | #update M variable
45 | delta = []
46 | u,d,v = np.linalg.svd(mat,full_matrices=False)
47 | wi = truncation(mat,theta)
48 | for j in range(len(d)):
49 | deltaj = GST(d[j],(alphai/beta)*wi[j],p)
50 | delta.append(deltaj)
51 | delta = np.diag(delta)
52 | Mi = u@delta@v
53 |
54 | return Mi
55 |
56 |
57 | def TSpN_ADMM(X_true,X_missing,Omega,alpha,beta,incre,maxIter,epsilon,p,theta):
58 | X = X_missing.copy()
59 | X[Omega==False] = np.mean(X_missing[Omega]) #Initialize with mean values
60 | errList = []
61 | MAE_List = []
62 | RMSE_List = []
63 | dim = X_missing.shape
64 | M = np.zeros(np.insert(dim, 0, len(dim))) #M is a 4-th order tensor
65 | Q = np.zeros(np.insert(dim, 0, len(dim))) #Q is a 4-th order tensor
66 | print('TSp_ADMM Iteration: ')
67 |
68 | for k in range(maxIter):
69 | beta = beta * (1+incre) #Increase beta with given step
70 | print(f'\r Processing loop {k}',end = '',flush=True)
71 |
72 | #Update M variable
73 | for i in range(np.ndim(X_missing)):
74 | M[i] = Fold(update_Mi(Unfold(X+(1/beta)*Q[i],dim,i),alpha[i],beta,p,theta),dim,i) ##M为四维张量
75 |
76 | Xlast = X.copy()
77 | X = np.sum(beta*M-Q,axis=0)/(beta*(X_missing.ndim)) #Updata X variable
78 | X[Omega] = X_missing[Omega] #Observed data
79 |
80 | Q = Q + beta*(np.broadcast_to(X, np.insert(dim, 0, len(dim)))-M) #Update Q variable
81 |
82 | errList_k = np.linalg.norm(X-Xlast)/np.linalg.norm(Xlast)
83 | errList.append(errList_k)
84 | MAE_List.append(compute_MAE(X_missing,X_true,X))
85 | RMSE_List.append(compute_RMSE(X_missing,X_true,X))
86 |
87 | if errList_k < epsilon:
88 | break
89 |
90 | print(f'\n total iterations = {k} error={errList[-1]}')
91 |
92 | return X,MAE_List,RMSE_List,errList,k
93 |
94 |
95 | def LRTC_TSpN(complete_tensor,observed_tensor,theta=0.1,alpha=np.array([1,1,1]),p=0.5,beta=1e-5,incre=0.05,maxiter = 200,show_plot = True):
96 | X_true = complete_tensor.copy()
97 | X_missing = observed_tensor.copy()
98 | Omega = (X_missing != 0)
99 | alpha = alpha.reshape(-1,1)
100 | alpha = alpha / np.sum(alpha)
101 | epsilon = 1e-3
102 | X_hat,MAE_List,RMSE_List,errList,it = TSpN_ADMM(X_true,X_missing,Omega,alpha,beta,incre,maxiter,epsilon,p,theta)
103 | MAPE = compute_MAPE(X_missing,X_true,X_hat)
104 | print(f'LRTC-TSpN imptation MAE = {MAE_List[-1]:.3f}')
105 | print(f'LRTC-TSpN imputation RMSE = {RMSE_List[-1]:.3f}')
106 | print(f'LRTC-TSpN imputation MAPE = {MAPE:.3f}')
107 |
108 | if show_plot == True:
109 | plt.plot(range(len(MAE_List)),MAE_List)
110 | plt.xlabel('epoch')
111 | plt.ylabel('MAE')
112 | plt.title('Convergence curve of LRTC-TSpN')
113 |
114 | return it,X_hat,MAE_List,RMSE_List,errList
115 |
116 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 Tong Nie
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Low-rank tensor completion algorithm for spatiotemporal traffic data imputation
2 | --------------
3 | 
4 | [](https://opensource.org/licenses/MIT)
5 |
6 | **L**ow-**r**ank **t**ensor **c**ompletion using **T**runcated tensor **S**chatten **p**-**N**orm, LRTC-TSpN.
7 |
8 |
9 | > This is the code repository for paper 'Truncated tensor Schatten p-norm based approach for spatiotemporal traffic data
10 | imputation with complicated missing patterns' which is published on Transportation Research Part C: Emerging Technologies
11 |
12 | ## Overview
13 | This project provides some examples about how to use LRTC-TSpN to achieve efficient and accurate missing data imputation for transportation time series data. We aim at performing off-line data imputation tasks, with several realistic structural missing patterns. Missing data imputation problem is modelled as a low-rank tensor completion problem (low-rank tensor learning). The objective is to obtain a fully recovered tensor by minimizing a predefined tensor rank function, given the observations. We define a new **truncated tensor Schatten p-norm** to substitute for the traditional tensor nuclear norm. We recommend ones to refer to Kolda and Bader’s review [Tensor Decompositions and Applications](https://epubs.siam.org/doi/abs/10.1137/07070111x) for more basics about tensor algebra.
14 |
15 | ## Model description
16 | We organise the multiple input time series data into a third-order tensor structure of (time intervals × locations × days). Traditional methods resort to tensor nuclear norm (or sum of nuclear norm) to substitute for the tensor rank, however, these convex surrogates are not powerful in practice. Therefore, we use the newly emerging Schatten p-norm and its truncated version to approximate tensor rank in order to achieve more accurate traffic data imputation.
17 |
18 |
19 |
20 | 
21 |
28 | 
29 |
39 | 
40 |
44 | 
45 |