├── LICENSE ├── README.md ├── traj_management ├── traj_chunk_read │ ├── data │ │ └── 1.csv │ ├── example │ │ └── aisread.py │ ├── pyais │ │ ├── __pycache__ │ │ │ └── chunk_read.cpython-38.pyc │ │ └── chunk_read.py │ └── requirements.txt └── traj_similarity │ ├── data │ └── 1.csv │ ├── requirements.txt │ └── trajsimilarity.py ├── traj_mining ├── traffic_flow_predict │ ├── data │ │ ├── flowtc.csv │ │ ├── flowtj.csv │ │ └── result.csv │ ├── example.py │ ├── generate_dataset.py │ ├── lstm_encoder_decoder.py │ ├── model │ │ └── lstm-ed.pt │ ├── plotting.py │ └── requirements.txt ├── traj_anchor_berth │ ├── data │ │ └── 1.csv │ ├── pyais │ │ ├── __pycache__ │ │ │ ├── cleaning.cpython-38.pyc │ │ │ ├── geotostr.cpython-38.pyc │ │ │ ├── plotting.cpython-38.pyc │ │ │ ├── segmentation.cpython-38.pyc │ │ │ ├── traj_clean.cpython-38.pyc │ │ │ ├── traj_interpolation.cpython-38.pyc │ │ │ ├── traj_segment.cpython-38.pyc │ │ │ └── traj_stay.cpython-38.pyc │ │ ├── geotostr.py │ │ ├── ptinpolygon.py │ │ ├── traj_clean.py │ │ ├── traj_interpolation.py │ │ ├── traj_segment.py │ │ └── traj_stay.py │ ├── requirements.txt │ └── trajanchorberth.py └── traj_cluster │ ├── hdbscan │ └── myhdbscan.py │ └── requirements.txt ├── traj_preprocess ├── ais_clean │ ├── data │ │ └── 1.csv │ ├── example │ │ └── trajclean.py │ ├── pyais │ │ ├── __pycache__ │ │ │ ├── chunk_read.cpython-38.pyc │ │ │ └── traj_clean.cpython-38.pyc │ │ └── traj_clean.py │ └── requirements.txt ├── ais_compress │ ├── data │ │ └── 1.csv │ ├── example │ │ └── trajcompress.py │ ├── pyais │ │ ├── __pycache__ │ │ │ ├── traj_compress.cpython-38.pyc │ │ │ └── traj_interpolation.cpython-38.pyc │ │ └── traj_compress.py │ └── requirements.txt ├── ais_encoder_decoder │ ├── examples │ │ ├── binary_payload.py │ │ ├── communication_state.py │ │ ├── decode.py │ │ ├── encode_dict.py │ │ ├── encode_msg.py │ │ ├── file_stream.py │ │ ├── msg_to_csv.py │ │ ├── out_of_order.py │ │ ├── sample.ais │ │ └── single_message.py │ ├── pyais │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-38.pyc │ │ │ ├── constants.cpython-38.pyc │ │ │ ├── decode.cpython-38.pyc │ │ │ ├── encode.cpython-38.pyc │ │ │ ├── exceptions.cpython-38.pyc │ │ │ ├── messages.cpython-38.pyc │ │ │ ├── stream.cpython-38.pyc │ │ │ └── util.cpython-38.pyc │ │ ├── ais_types.py │ │ ├── constants.py │ │ ├── decode.py │ │ ├── encode.py │ │ ├── exceptions.py │ │ ├── messages.py │ │ ├── stream.py │ │ └── util.py │ └── requirements.txt ├── ais_interpolate │ ├── data │ │ └── 1.csv │ ├── example │ │ └── trajinterpolation.py │ ├── pyais │ │ ├── __pycache__ │ │ │ └── traj_interpolation.cpython-38.pyc │ │ └── traj_interpolation.py │ └── requirements.txt ├── ais_kalman_filter │ ├── data │ │ └── 1.csv │ ├── example │ │ └── trajkalmanfilter.py │ ├── pyais │ │ ├── __pycache__ │ │ │ ├── traj_clean.cpython-38.pyc │ │ │ ├── traj_interpolation.cpython-38.pyc │ │ │ ├── traj_kalmanfilter.cpython-38.pyc │ │ │ ├── traj_segment.cpython-38.pyc │ │ │ └── traj_stay.cpython-38.pyc │ │ └── traj_kalmanfilter.py │ └── requirements.txt ├── ais_segment │ ├── data │ │ └── 1.csv │ ├── example │ │ └── trajsegment.py │ ├── pyais │ │ ├── __pycache__ │ │ │ ├── traj_compress.cpython-38.pyc │ │ │ ├── traj_interpolation.cpython-38.pyc │ │ │ └── traj_segment.cpython-38.pyc │ │ └── traj_segment.py │ └── requirements.txt └── ais_stay │ ├── data │ └── 1.csv │ ├── example │ └── trajstay.py │ ├── pyais │ ├── __pycache__ │ │ ├── traj_clean.cpython-38.pyc │ │ ├── traj_interpolation.cpython-38.pyc │ │ ├── traj_segment.cpython-38.pyc │ │ └── traj_stay.cpython-38.pyc │ ├── traj_clean.py │ ├── traj_interpolation.py │ ├── traj_segment.py │ └── traj_stay.py │ └── requirements.txt ├── traj_show ├── __pycache__ │ └── folium.cpython-37.pyc ├── data │ └── 1.csv ├── draw │ └── show.html ├── requirements.txt ├── trajshow.py └── trajshow_matplotlib.py └── user guide └── Developer's Guide.pdf /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 leeyeah 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyVT: A Toolkit for Preprocessing and Analysis of Vessel Spatio-Temporal Trajectories 2 | 3 | ## Introduction 4 | 5 | PyVT, an open-source Python toolkit for preprocessing and analysis of vessel spatio-temporal trajectories. The toolkit realizes four functions: trajectory preprocessing, trajectory data management, trajectory mining, and trajectory visualization. The toolkit is simple to use, provides satisfactory default parameter settings, and makes it easy for basic users to use and replicate its functionality in proprietary applications. More advanced users can further adjust the functions to meet their needs by optimizing parameters. At the same time, its design focuses on composability and reusability, making it functional modular and maintainable, allowing for further development. 6 |
7 | 8 | ## Requirements 9 | 10 | PyVT is developed based on Python 3. The environment that each module depends on, see the requirements.txt. 11 | 12 | ## Developer's guide 13 | See the "Developer's Guide" 14 | 15 | ## License 16 | MIT 17 | -------------------------------------------------------------------------------- /traj_management/traj_chunk_read/example/aisread.py: -------------------------------------------------------------------------------- 1 | from pyais import chunk_read 2 | 3 | if __name__ == '__main__': 4 | # chunk_read.read_ais_data_region(input="../data/1.csv", output='../data/test.csv') # 此处填写csv路径 5 | chunk_read.read_ais_data_mmsi(input="../data/1.csv", output='../data/test.csv', mmsi='244726000') # 此处填写csv路径 6 | -------------------------------------------------------------------------------- /traj_management/traj_chunk_read/pyais/__pycache__/chunk_read.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_management/traj_chunk_read/pyais/__pycache__/chunk_read.cpython-38.pyc -------------------------------------------------------------------------------- /traj_management/traj_chunk_read/pyais/chunk_read.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import time 3 | 4 | 5 | def suply10(x): 6 | return x * 10.0 7 | 8 | 9 | def read_ais_data_region(input: str, output: str, logleftup: float = 122.0, logrightdown: float = 123.0, 10 | latleftup: float = 32.0, latrightdown: float = 31.0, dtst: str = '2019-01-01 00:00:01', 11 | dtend: str = '2020-01-01 00:00:01'): 12 | ''' 13 | :param input: filename of the input file 14 | :param output: filename of the output file 15 | :param logleftup: longitude of upper left corner 16 | :param logrightdown: longitude of lower right corner 17 | :param latleftup: longitude of upper left corner 18 | :param latrightdown: latitude of lower right corner 19 | :param dtst: start time 20 | :param dtend: end time 21 | :return: null 22 | ''' 23 | cols = read_data_nm(input, 5) 24 | inputfile = open(input, 'rb') 25 | data = pd.read_csv(inputfile, sep=',', iterator=True) 26 | timearrayst = time.strptime(dtst, "%Y-%m-%d %H:%M:%S") 27 | timearrayend = time.strptime(dtend, "%Y-%m-%d %H:%M:%S") 28 | timestampst = time.mktime(timearrayst) # timestamp 29 | timestampend = time.mktime(timearrayend) 30 | loop = True 31 | chunkSize = 10000 # The number of reads per chunk 32 | chunks = [] 33 | while loop: 34 | try: 35 | chunk = data.get_chunk(chunkSize) 36 | chunk.columns = cols 37 | chunked = chunk[(chunk['DRGPSTIME'] > timestampst) & (chunk['DRGPSTIME'] < timestampend) & ( 38 | chunk['DRLONGITUDE'] > logleftup) & (chunk['DRLONGITUDE'] < logrightdown) & ( 39 | chunk['DRLATITUDE'] < latleftup) & (chunk['DRLATITUDE'] > latrightdown)] 40 | cn = chunked.fillna(value=0) 41 | # cn['DRTRUEHEADING'] = cn['DRTRUEHEADING'].transform(suply10) 42 | chunks.append(cn) 43 | except StopIteration: 44 | loop = False 45 | print("Iteration is stopped.") 46 | rdata = pd.concat(chunks, ignore_index=True) 47 | rdata.to_csv(output, index=False) # 输出csv 48 | print("Finished.") 49 | return 50 | 51 | 52 | def read_ais_data_callsign(input: str, output: str, callsign: str = ''): 53 | ''' 54 | :param input: filename of the input file 55 | :param output: filename of the output file 56 | :param callsign: CALLSIGN 57 | :return: null 58 | ''' 59 | cols = read_data_nm(input, 5) 60 | inputfile = open(input, 'rb') 61 | data = pd.read_csv(inputfile, sep=',', iterator=True) 62 | loop = True 63 | chunkSize = 10000 # The number of reads per chunk 64 | chunks = [] 65 | while loop: 66 | try: 67 | chunk = data.get_chunk(chunkSize) 68 | chunk.columns = cols 69 | chunk['CALLSIGN'] = chunk['CALLSIGN'].map(lambda x: str(x)) 70 | chunked = chunk[(str(chunk['CALLSIGN']) == callsign)] 71 | cn = chunked.fillna(value=0) 72 | chunks.append(cn) 73 | except StopIteration: 74 | loop = False 75 | print("Iteration is stopped.") 76 | rdata = pd.concat(chunks, ignore_index=True) 77 | rdata.to_csv(output, index=False) # 输出csv 78 | print("Finished.") 79 | return 80 | 81 | 82 | def read_ais_data_mmsi(input: str, output: str, mmsi: str = ''): 83 | ''' 84 | :param input: filename of the input file 85 | :param output: filename of the output file 86 | :param mmsi: DRMMSI 87 | :return: null 88 | ''' 89 | cols = read_data_nm(input, 5) 90 | inputfile = open(input, 'rb') 91 | data = pd.read_csv(inputfile, sep=',', iterator=True) 92 | loop = True 93 | chunkSize = 10000 # The number of reads per chunk 94 | chunks = [] 95 | while loop: 96 | try: 97 | chunk = data.get_chunk(chunkSize) 98 | chunk.columns = cols 99 | chunk['DRMMSI'] = chunk['DRMMSI'].map(lambda x: str(x)) 100 | chunked = chunk[(chunk['DRMMSI'] == mmsi)] 101 | cn = chunked.fillna(value=0) 102 | chunks.append(cn) 103 | except StopIteration: 104 | loop = False 105 | print("Iteration is stopped.") 106 | rdata = pd.concat(chunks, ignore_index=True) 107 | rdata.to_csv(output, index=False) # 输出csv 108 | print("Finished.") 109 | return 110 | 111 | 112 | def read_data_nm(input_file, nm): 113 | ''' 114 | The function is to print the first * line of the file 115 | :param input_file: filename 116 | :param nm: row number 117 | :return: null 118 | ''' 119 | pd.set_option('display.max_columns', None) 120 | pd.set_option('display.max_rows', None) 121 | pd.set_option('max_colwidth', 100) 122 | data = pd.read_csv(input_file, nrows=nm) 123 | print(data) 124 | return data.columns 125 | -------------------------------------------------------------------------------- /traj_management/traj_chunk_read/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==1.4.1 2 | -------------------------------------------------------------------------------- /traj_management/traj_similarity/requirements.txt: -------------------------------------------------------------------------------- 1 | dtaidistance==2.3.9 2 | fastdtw==0.3.4 3 | haversine==2.6.0 4 | numpy==1.23.0 5 | pandas==1.4.3 6 | -------------------------------------------------------------------------------- /traj_management/traj_similarity/trajsimilarity.py: -------------------------------------------------------------------------------- 1 | from dtaidistance import dtw 2 | from dtaidistance import dtw_ndim 3 | import pandas as pd 4 | import numpy as np 5 | from fastdtw import fastdtw 6 | from haversine.haversine import haversine 7 | 8 | 9 | def load_data(file_path): 10 | """ 11 | import trajectory data 12 | :return: trajectory data 13 | """ 14 | data = pd.read_csv(file_path, usecols=['DRGPSTIME', 'DRMMSI', 'DRLATITUDE', 'DRLONGITUDE']) 15 | data.rename(columns={'DRLONGITUDE': 'long', 'DRLATITUDE': 'lat', 'DRGPSTIME': 't', 'DRMMSI': 'mmsi'}, inplace=True) 16 | trajectories = [] 17 | grouped = data[:].groupby('mmsi') 18 | for name, group in grouped: 19 | if len(group) > 1: 20 | group = group.sort_values(by='t', ascending=True) 21 | group['long'] = group['long'].apply(lambda x: x * (1 / 1.0)) # 以°为单位 22 | group['lat'] = group['lat'].apply(lambda x: x * (1 / 1.0)) 23 | loc = group[['lat', 'long']].values # 原始 24 | trajectories.append(loc) 25 | return trajectories 26 | 27 | 28 | def dist_computer(trajectories): 29 | distance_matrix = dtw_ndim.distance_matrix_fast(trajectories, 2) 30 | distance_matrix = np.array(distance_matrix) 31 | np.savetxt('1.txt', distance_matrix, fmt='%.5f') 32 | return distance_matrix 33 | 34 | 35 | def dist_computer_fastdtw(trajectories): 36 | distance_matrix = [] 37 | i = 0 38 | l = len(trajectories) 39 | for traj in trajectories: 40 | i = i + 1 41 | print(str(i) + "/" + str(l)) 42 | v = [] 43 | for q in trajectories: 44 | distance, path = fastdtw(traj, q, dist=haversine) 45 | v.append(distance) 46 | distance_matrix.append(v) 47 | distance_matrix = np.array(distance_matrix) 48 | np.savetxt('1.txt', distance_matrix, fmt='%.5f') 49 | return distance_matrix 50 | 51 | 52 | if __name__ == '__main__': 53 | trajectories = load_data('./data/1.csv') 54 | dist_matrix = dist_computer(trajectories) 55 | -------------------------------------------------------------------------------- /traj_mining/traffic_flow_predict/example.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import matplotlib 4 | import matplotlib.pyplot as plt 5 | from importlib import reload 6 | from matplotlib.ticker import MaxNLocator 7 | import generate_dataset 8 | import lstm_encoder_decoder 9 | import plotting 10 | import torch 11 | 12 | matplotlib.rcParams.update({'font.size': 14}) 13 | print(torch.cuda.is_available()) 14 | 15 | # generate dataset for LSTM 16 | t, y, xorigin, yorigin = generate_dataset.read_data('./data/result.csv', 'total') 17 | t_train, y_train, t_test, y_test = generate_dataset.train_test_split(t, y, split=0.80) 18 | 19 | fig1, ax1 = plt.subplots(1, figsize=(12, 6)) 20 | ax1.plot(t, y, linewidth=1, marker='o', ls='-') 21 | ax1.set_xlabel('Time') 22 | ax1.set_ylabel('Hourly traffic volume') 23 | ax1.yaxis.set_major_locator(MaxNLocator(10)) 24 | ax1.xaxis.set_major_locator(MaxNLocator(7)) 25 | 26 | fig1.autofmt_xdate() 27 | ax1.grid(True) 28 | plt.savefig('plots/time_series.png') 29 | 30 | # plot time series with train/test split 31 | fig2, ax2 = plt.subplots(1, figsize=(12, 6)) 32 | ax2.plot(t_train, y_train, linewidth=1, ls='-', marker='o', label='Train') 33 | ax2.plot(np.concatenate([[t_train[-1]], t_test]), np.concatenate([[y_train[-1]], y_test]), linewidth=1, ls='--', 34 | marker='*', label='Test') 35 | ax2.set_xlabel('Time') 36 | ax2.set_ylabel('Hourly traffic volume') 37 | plt.title('Time Series Split into Train and Test Sets') 38 | plt.legend() 39 | plt.tight_layout 40 | plt.savefig('plots/train_test_split.png') 41 | 42 | # set size of input/output windows 43 | iw = 24 44 | ow = 6 45 | s = 3 46 | 47 | # generate windowed training/test datasets 48 | Xtrain, Ytrain = generate_dataset.windowed_dataset(y_train, input_window=iw, output_window=ow, stride=s) 49 | Xtest, Ytest = generate_dataset.windowed_dataset(y_test, input_window=iw, output_window=ow, stride=s) 50 | # Xtimetrain, Ytimetrain = generate_dataset.windowed_dataset(t_train, input_window=iw, output_window=ow, stride=s) 51 | # Xtimetest, Ytimetest = generate_dataset.windowed_dataset(t_test, input_window=iw, output_window=ow, stride=s) 52 | 53 | # plot example of windowed data 54 | fig3, ax3 = plt.subplots(1, figsize=(12, 6)) 55 | ax3.plot(np.arange(0, iw), Xtrain[:, 0, 0], ls='-', linewidth=1.0, marker='o', label='Input') 56 | ax3.plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtrain[-1, 0, 0]], Ytrain[:, 0, 0]]), linewidth=1.0, 57 | ls='--', marker='o', label='Target') 58 | plt.xlim([0, iw + ow - 1]) 59 | # plt.yticks(np.arange(0, 24, 4)) 60 | ax2.set_xlabel('Time') 61 | ax2.set_ylabel('Hourly traffic volume') 62 | plt.title('Example of Windowed Training Data') 63 | plt.legend() 64 | plt.tight_layout() 65 | plt.savefig('plots/windowed_data.png') 66 | 67 | # LSTM encoder-decoder 68 | # convert windowed data from np.array to PyTorch tensor 69 | X_train, Y_train, X_test, Y_test = generate_dataset.numpy_to_torch(Xtrain, Ytrain, Xtest, Ytest) 70 | 71 | # specify model parameters and train 72 | import torch 73 | 74 | ispre = True # 是否预测 75 | if ispre: 76 | # 加载模型 77 | m_state_dict = torch.load('./model/lstm-ed.pt') 78 | new_m = lstm_encoder_decoder.lstm_seq2seq(input_size=X_train.shape[2], hidden_size=128) 79 | new_m.load_state_dict(m_state_dict) 80 | plotting.plot_train_test_results(new_m, Xtrain, Ytrain, Xtest, Ytest) 81 | plt.close('all') 82 | else: 83 | # specify model parameters and train 84 | # 保存模型 85 | model = lstm_encoder_decoder.lstm_seq2seq(input_size=X_train.shape[2], hidden_size=128) 86 | loss = model.train_model(input_tensor=X_train, target_tensor=Y_train, n_epochs=300, target_len=ow, batch_size=64, 87 | training_prediction='mixed_teacher_forcing', teacher_forcing_ratio=0.5, 88 | learning_rate=0.001, 89 | dynamic_tf=False) 90 | torch.save(model.state_dict(), './model/lstm-ed.pt') 91 | -------------------------------------------------------------------------------- /traj_mining/traffic_flow_predict/generate_dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import pandas as pd 4 | from sklearn.preprocessing import MinMaxScaler 5 | 6 | 7 | def synthetic_data(Nt=2000, tf=80 * np.pi): 8 | ''' 9 | create synthetic time series dataset 10 | : param Nt: number of time steps 11 | : param tf: final time 12 | : return t, y: time, feature arrays 13 | ''' 14 | t = np.linspace(0., tf, Nt) 15 | y = np.sin(2. * t) + 0.5 * np.cos(t) + np.random.normal(0., 0.2, Nt) 16 | return t, y 17 | 18 | 19 | def read_data(filenm, colnm): 20 | inputfile = open(filenm, 'rb') # 可打开含有中文的地址 21 | df = pd.read_csv(inputfile, usecols=['time', 'up', 'down', 'total']) 22 | t = df['time'].values 23 | y = df[colnm].values 24 | x = np.arange(t.shape[0]) 25 | y_norm = (y - y.min()) / (y.max() - y.min()) 26 | return x, y_norm, t, y 27 | 28 | 29 | def train_test_split(t, y, split=0.8): 30 | ''' 31 | 32 | split time series into train/test sets 33 | 34 | : param t: time array 35 | : para y: feature array 36 | : para split: percent of data to include in training set 37 | : return t_train, y_train: time/feature training and test sets; 38 | : t_test, y_test: (shape: [# samples, 1]) 39 | 40 | ''' 41 | 42 | indx_split = int(split * len(y)) 43 | indx_train = np.arange(0, indx_split) 44 | indx_test = np.arange(indx_split, len(y)) 45 | 46 | t_train = t[indx_train] 47 | y_train = y[indx_train] 48 | y_train = y_train.reshape(-1, 1) 49 | 50 | t_test = t[indx_test] 51 | y_test = y[indx_test] 52 | y_test = y_test.reshape(-1, 1) 53 | 54 | return t_train, y_train, t_test, y_test 55 | 56 | 57 | def windowed_dataset(y, input_window=5, output_window=1, stride=1, num_features=1): 58 | ''' 59 | create a windowed dataset 60 | 61 | : param y: time series feature (array) 62 | : param input_window: number of y samples to give model 63 | : param output_window: number of future y samples to predict 64 | : param stide: spacing between windows 65 | : param num_features: number of features (i.e., 1 for us, but we could have multiple features) 66 | : return X, Y: arrays with correct dimensions for LSTM 67 | : (i.e., [input/output window size # examples, # features]) 68 | ''' 69 | L = y.shape[0] 70 | num_samples = (L - input_window - output_window) // stride + 1 71 | 72 | X = np.zeros([input_window, num_samples, num_features]) 73 | Y = np.zeros([output_window, num_samples, num_features]) 74 | 75 | for ff in np.arange(num_features): 76 | for ii in np.arange(num_samples): 77 | start_x = stride * ii 78 | end_x = start_x + input_window 79 | X[:, ii, ff] = y[start_x:end_x, ff] 80 | 81 | start_y = stride * ii + input_window 82 | end_y = start_y + output_window 83 | Y[:, ii, ff] = y[start_y:end_y, ff] 84 | 85 | return X, Y 86 | 87 | 88 | def windowed_dataset_time(y, input_window=5, output_window=1, stride=1, num_features=1): 89 | ''' 90 | create a windowed dataset 91 | 92 | : param y: time series feature (array) 93 | : param input_window: number of y samples to give model 94 | : param output_window: number of future y samples to predict 95 | : param stide: spacing between windows 96 | : param num_features: number of features (i.e., 1 for us, but we could have multiple features) 97 | : return X, Y: arrays with correct dimensions for LSTM 98 | : (i.e., [input/output window size # examples, # features]) 99 | ''' 100 | L = y.shape[0] 101 | num_samples = (L - input_window - output_window) // stride + 1 102 | 103 | X = np.zeros([input_window, num_samples, num_features]) 104 | Y = np.zeros([output_window, num_samples, num_features]) 105 | 106 | for ff in np.arange(num_features): 107 | for ii in np.arange(num_samples): 108 | start_x = stride * ii 109 | end_x = start_x + input_window 110 | X[:, ii, ff] = y[start_x:end_x, ff] 111 | 112 | start_y = stride * ii + input_window 113 | end_y = start_y + output_window 114 | Y[:, ii, ff] = y[start_y:end_y, ff] 115 | 116 | return X, Y 117 | 118 | 119 | def numpy_to_torch(Xtrain, Ytrain, Xtest, Ytest): 120 | ''' 121 | convert numpy array to PyTorch tensor 122 | 123 | : param Xtrain: windowed training input data (input window size, # examples, # features); np.array 124 | : param Ytrain: windowed training target data (output window size, # examples, # features); np.array 125 | : param Xtest: windowed test input data (input window size, # examples, # features); np.array 126 | : param Ytest: windowed test target data (output window size, # examples, # features); np.array 127 | : return X_train_torch, Y_train_torch, 128 | : X_test_torch, Y_test_torch: all input np.arrays converted to PyTorch tensors 129 | 130 | ''' 131 | 132 | X_train_torch = torch.from_numpy(Xtrain).type(torch.Tensor) 133 | Y_train_torch = torch.from_numpy(Ytrain).type(torch.Tensor) 134 | 135 | X_test_torch = torch.from_numpy(Xtest).type(torch.Tensor) 136 | Y_test_torch = torch.from_numpy(Ytest).type(torch.Tensor) 137 | 138 | return X_train_torch, Y_train_torch, X_test_torch, Y_test_torch 139 | -------------------------------------------------------------------------------- /traj_mining/traffic_flow_predict/lstm_encoder_decoder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | import os, errno 4 | import sys 5 | from tqdm import trange 6 | 7 | import torch 8 | import torch.nn as nn 9 | from torch import optim 10 | import torch.nn.functional as F 11 | 12 | 13 | class lstm_encoder(nn.Module): 14 | ''' Encodes time-series sequence ''' 15 | 16 | def __init__(self, input_size, hidden_size, num_layers=3): 17 | ''' 18 | : param input_size: the number of features in the input X 19 | : param hidden_size: the number of features in the hidden state h 20 | : param num_layers: number of recurrent layers (i.e., 2 means there are 21 | : 2 stacked LSTMs) 22 | ''' 23 | 24 | super(lstm_encoder, self).__init__() 25 | self.input_size = input_size 26 | self.hidden_size = hidden_size 27 | self.num_layers = num_layers 28 | 29 | # define LSTM layer 30 | self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, 31 | num_layers=num_layers, dropout=0.3) 32 | 33 | def forward(self, x_input): 34 | ''' 35 | : param x_input: input of shape (seq_len, # in batch, input_size) 36 | : return lstm_out, hidden: lstm_out gives all the hidden states in the sequence; 37 | : hidden gives the hidden state and cell state for the last 38 | : element in the sequence 39 | ''' 40 | 41 | lstm_out, self.hidden = self.lstm(x_input.view(x_input.shape[0], x_input.shape[1], self.input_size)) 42 | 43 | return lstm_out, self.hidden 44 | 45 | def init_hidden(self, batch_size): 46 | ''' 47 | initialize hidden state 48 | : param batch_size: x_input.shape[1] 49 | : return: zeroed hidden state and cell state 50 | ''' 51 | 52 | return (torch.zeros(self.num_layers, batch_size, self.hidden_size), 53 | torch.zeros(self.num_layers, batch_size, self.hidden_size)) 54 | 55 | 56 | class lstm_decoder(nn.Module): 57 | ''' Decodes hidden state output by encoder ''' 58 | 59 | def __init__(self, input_size, hidden_size, num_layers=3): 60 | ''' 61 | : param input_size: the number of features in the input X 62 | : param hidden_size: the number of features in the hidden state h 63 | : param num_layers: number of recurrent layers (i.e., 2 means there are 64 | : 2 stacked LSTMs) 65 | ''' 66 | 67 | super(lstm_decoder, self).__init__() 68 | self.input_size = input_size 69 | self.hidden_size = hidden_size 70 | self.num_layers = num_layers 71 | 72 | self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, 73 | num_layers=num_layers, dropout=0.3) 74 | self.linear = nn.Linear(hidden_size, input_size) 75 | 76 | def forward(self, x_input, encoder_hidden_states): 77 | ''' 78 | : param x_input: should be 2D (batch_size, input_size) 79 | : param encoder_hidden_states: hidden states 80 | : return output, hidden: output gives all the hidden states in the sequence; 81 | : hidden gives the hidden state and cell state for the last 82 | : element in the sequence 83 | 84 | ''' 85 | 86 | lstm_out, self.hidden = self.lstm(x_input.unsqueeze(0), encoder_hidden_states) 87 | output = self.linear(lstm_out.squeeze(0)) 88 | 89 | return output, self.hidden 90 | 91 | 92 | class lstm_seq2seq(nn.Module): 93 | ''' train LSTM encoder-decoder and make predictions ''' 94 | 95 | def __init__(self, input_size, hidden_size): 96 | 97 | ''' 98 | : param input_size: the number of expected features in the input X 99 | : param hidden_size: the number of features in the hidden state h 100 | ''' 101 | 102 | super(lstm_seq2seq, self).__init__() 103 | 104 | self.input_size = input_size 105 | self.hidden_size = hidden_size 106 | 107 | self.encoder = lstm_encoder(input_size=input_size, hidden_size=hidden_size) 108 | self.decoder = lstm_decoder(input_size=input_size, hidden_size=hidden_size) 109 | 110 | def train_model(self, input_tensor, target_tensor, n_epochs, target_len, batch_size, 111 | training_prediction='recursive', teacher_forcing_ratio=0.5, learning_rate=0.01, dynamic_tf=False): 112 | 113 | ''' 114 | train lstm encoder-decoder 115 | 116 | : param input_tensor: input data with shape (seq_len, # in batch, number features); PyTorch tensor 117 | : param target_tensor: target data with shape (seq_len, # in batch, number features); PyTorch tensor 118 | : param n_epochs: number of epochs 119 | : param target_len: number of values to predict 120 | : param batch_size: number of samples per gradient update 121 | : param training_prediction: type of prediction to make during training ('recursive', 'teacher_forcing', or 122 | : 'mixed_teacher_forcing'); default is 'recursive' 123 | : param teacher_forcing_ratio: float [0, 1) indicating how much teacher forcing to use when 124 | : training_prediction = 'teacher_forcing.' For each batch in training, we generate a random 125 | : number. If the random number is less than teacher_forcing_ratio, we use teacher forcing. 126 | : Otherwise, we predict recursively. If teacher_forcing_ratio = 1, we train only using 127 | : teacher forcing. 128 | : param learning_rate: float >= 0; learning rate 129 | : param dynamic_tf: use dynamic teacher forcing (True/False); dynamic teacher forcing 130 | : reduces the amount of teacher forcing for each epoch 131 | : return losses: array of loss function for each epoch 132 | ''' 133 | 134 | # initialize array of losses 135 | losses = np.full(n_epochs, np.nan) 136 | 137 | optimizer = optim.Adam(self.parameters(), lr=learning_rate) 138 | criterion = nn.MSELoss() 139 | 140 | # calculate number of batch iterations 141 | n_batches = int(input_tensor.shape[1] / batch_size) 142 | 143 | with trange(n_epochs) as tr: 144 | for it in tr: 145 | 146 | batch_loss = 0. 147 | batch_loss_tf = 0. 148 | batch_loss_no_tf = 0. 149 | num_tf = 0 150 | num_no_tf = 0 151 | 152 | for b in range(n_batches): 153 | # select data 154 | input_batch = input_tensor[:, b: b + batch_size, :] 155 | target_batch = target_tensor[:, b: b + batch_size, :] 156 | 157 | # outputs tensor 158 | outputs = torch.zeros(target_len, batch_size, input_batch.shape[2]) 159 | 160 | # initialize hidden state 161 | encoder_hidden = self.encoder.init_hidden(batch_size) 162 | 163 | # zero the gradient 164 | optimizer.zero_grad() 165 | 166 | # encoder outputs 167 | encoder_output, encoder_hidden = self.encoder(input_batch) 168 | 169 | # decoder with teacher forcing 170 | decoder_input = input_batch[-1, :, :] # shape: (batch_size, input_size) 171 | decoder_hidden = encoder_hidden 172 | 173 | if training_prediction == 'recursive': 174 | # predict recursively 175 | for t in range(target_len): 176 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 177 | outputs[t] = decoder_output 178 | decoder_input = decoder_output 179 | 180 | if training_prediction == 'teacher_forcing': 181 | # use teacher forcing 182 | if random.random() < teacher_forcing_ratio: 183 | for t in range(target_len): 184 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 185 | outputs[t] = decoder_output 186 | decoder_input = target_batch[t, :, :] 187 | 188 | # predict recursively 189 | else: 190 | for t in range(target_len): 191 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 192 | outputs[t] = decoder_output 193 | decoder_input = decoder_output 194 | 195 | if training_prediction == 'mixed_teacher_forcing': 196 | # predict using mixed teacher forcing 197 | for t in range(target_len): 198 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 199 | outputs[t] = decoder_output 200 | 201 | # predict with teacher forcing 202 | if random.random() < teacher_forcing_ratio: 203 | decoder_input = target_batch[t, :, :] 204 | 205 | # predict recursively 206 | else: 207 | decoder_input = decoder_output 208 | 209 | # compute the loss 210 | loss = criterion(outputs, target_batch) 211 | batch_loss += loss.item() 212 | 213 | # backpropagation 214 | loss.backward() 215 | optimizer.step() 216 | 217 | # loss for epoch 218 | batch_loss /= n_batches 219 | losses[it] = batch_loss 220 | 221 | # dynamic teacher forcing 222 | if dynamic_tf and teacher_forcing_ratio > 0: 223 | teacher_forcing_ratio = teacher_forcing_ratio - 0.02 224 | 225 | # progress bar 226 | tr.set_postfix(loss="{0:.3f}".format(batch_loss)) 227 | 228 | return losses 229 | 230 | def predict(self, input_tensor, target_len): 231 | 232 | ''' 233 | : param input_tensor: input data (seq_len, input_size); PyTorch tensor 234 | : param target_len: number of target values to predict 235 | : return np_outputs: np.array containing predicted values; prediction done recursively 236 | ''' 237 | 238 | # encode input_tensor 239 | input_tensor = input_tensor.unsqueeze(1) # add in batch size of 1 240 | encoder_output, encoder_hidden = self.encoder(input_tensor) 241 | 242 | # initialize tensor for predictions 243 | outputs = torch.zeros(target_len, input_tensor.shape[2]) 244 | 245 | # decode input_tensor 246 | decoder_input = input_tensor[-1, :, :] 247 | decoder_hidden = encoder_hidden 248 | 249 | for t in range(target_len): 250 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden) 251 | outputs[t] = decoder_output.squeeze(0) 252 | decoder_input = decoder_output 253 | 254 | np_outputs = outputs.detach().numpy() 255 | 256 | return np_outputs 257 | -------------------------------------------------------------------------------- /traj_mining/traffic_flow_predict/model/lstm-ed.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traffic_flow_predict/model/lstm-ed.pt -------------------------------------------------------------------------------- /traj_mining/traffic_flow_predict/plotting.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import torch 4 | from matplotlib.ticker import MaxNLocator 5 | from matplotlib.dates import DayLocator, HourLocator, DateFormatter, drange 6 | import datetime 7 | import matplotlib.ticker as ticker 8 | import sklearn.metrics as metrics 9 | 10 | 11 | def plot_train_test_results(lstm_model, Xtrain, Ytrain, Xtest, Ytest, num_rows=3): 12 | ''' 13 | plot examples of the lstm encoder-decoder evaluated on the training/test data 14 | 15 | : param lstm_model: trained lstm encoder-decoder 16 | : param Xtrain: np.array of windowed training input data 17 | : param Ytrain: np.array of windowed training target data 18 | : param Xtest: np.array of windowed test input data 19 | : param Ytest: np.array of windowed test target data 20 | : param num_rows: number of training/test examples to plot 21 | : return: num_rows x 2 plots; first column is training data predictions, 22 | : second column is test data predictions 23 | ''' 24 | 25 | # input window size 26 | iw = Xtrain.shape[0] 27 | ow = Ytest.shape[0] 28 | 29 | # figure setup 30 | num_cols = 2 31 | num_plots = num_rows * num_cols 32 | 33 | fig, ax = plt.subplots(num_rows, num_cols, figsize=(13, 15)) 34 | 35 | # plot training/test predictions 36 | for ii in range(num_rows): 37 | # train set 38 | X_train_plt = Xtrain[:, ii, :] 39 | Y_train_pred = lstm_model.predict(torch.from_numpy(X_train_plt).type(torch.Tensor), target_len=ow) 40 | 41 | ax[ii, 0].plot(np.arange(0, iw), Xtrain[:, ii, 0], 'k', linewidth=2, label='Input') 42 | ax[ii, 0].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtrain[-1, ii, 0]], Ytrain[:, ii, 0]]), 43 | color=(0.2, 0.42, 0.72), linewidth=2, label='Target') 44 | ax[ii, 0].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtrain[-1, ii, 0]], Y_train_pred[:, 0]]), 45 | color=(0.76, 0.01, 0.01), linewidth=2, label='Prediction') 46 | ax[ii, 0].set_xlim([0, iw + ow - 1]) 47 | ax[ii, 0].set_xlabel('$t$') 48 | ax[ii, 0].set_ylabel('$y$') 49 | 50 | # test set 51 | X_test_plt = Xtest[:, ii, :] 52 | Y_test_pred = lstm_model.predict(torch.from_numpy(X_test_plt).type(torch.Tensor), target_len=ow) 53 | ax[ii, 1].plot(np.arange(0, iw), Xtest[:, ii, 0], 'k', linewidth=2, label='Input') 54 | ax[ii, 1].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Ytest[:, ii, 0]]), 55 | color=(0.2, 0.42, 0.72), linewidth=2, label='Target') 56 | ax[ii, 1].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Y_test_pred[:, 0]]), 57 | color=(0.76, 0.01, 0.01), linewidth=2, label='Prediction') 58 | ax[ii, 1].set_xlim([0, iw + ow - 1]) 59 | ax[ii, 1].set_xlabel('$t$') 60 | ax[ii, 1].set_ylabel('$y$') 61 | 62 | # plot_results(np.arange(0, iw), Xtrain[:, ii, 0], np.arange(iw - 1, iw + ow), 63 | # np.concatenate([[Xtrain[-1, ii, 0]], Ytrain[:, ii, 0]]), np.arange(iw - 1, iw + ow), 64 | # np.concatenate([[Xtrain[-1, ii, 0]], Y_train_pred[:, 0]]), 'train' + str(ii)) 65 | plot_results(np.arange(0, iw), Xtest[:, ii, 0], 66 | np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Ytest[:, ii, 0]]), 67 | np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Y_test_pred[:, 0]]), 68 | 'test' + str(ii)) 69 | 70 | if ii == 0: 71 | ax[ii, 0].set_title('Train') 72 | ax[ii, 1].legend(bbox_to_anchor=(1, 1)) 73 | ax[ii, 1].set_title('Test') 74 | 75 | plt.suptitle('LSTM Encoder-Decoder Predictions', x=0.445, y=1.) 76 | plt.tight_layout() 77 | plt.subplots_adjust(top=0.95) 78 | plt.savefig('plots/predictions.png') 79 | # plt.show() 80 | plt.close() 81 | 82 | return 83 | 84 | 85 | def plot_results(xi, yi, xt, yt, xp, yp, tpe): 86 | fig1, ax1 = plt.subplots(1, figsize=(12, 6)) 87 | yi = yi * 32 88 | yt = yt * 32 89 | yp = yp * 32 90 | ax1.plot(['2019-4-1 0:00', 91 | '2019-4-1 1:00', 92 | '2019-4-1 2:00', 93 | '2019-4-1 3:00', 94 | '2019-4-1 4:00', 95 | '2019-4-1 5:00', 96 | '2019-4-1 6:00', 97 | '2019-4-1 7:00', 98 | '2019-4-1 8:00', 99 | '2019-4-1 9:00', 100 | '2019-4-1 10:00', 101 | '2019-4-1 11:00', 102 | '2019-4-1 12:00', 103 | '2019-4-1 13:00', 104 | '2019-4-1 14:00', 105 | '2019-4-1 15:00', 106 | '2019-4-1 16:00', 107 | '2019-4-1 17:00', 108 | '2019-4-1 18:00', 109 | '2019-4-1 19:00', 110 | '2019-4-1 20:00', 111 | '2019-4-1 21:00', 112 | '2019-4-1 22:00', 113 | '2019-4-1 23:00'], yi, linewidth=1, marker='o', ls='-', label='Input') 114 | ax1.plot(['2019-4-1 23:00', 115 | '2019-4-2 0:00', 116 | '2019-4-2 1:00', 117 | '2019-4-2 2:00', 118 | '2019-4-2 3:00', 119 | '2019-4-2 4:00', 120 | '2019-4-2 5:00'], yt, linewidth=1, marker='o', ls='-', label='Target') 121 | ax1.plot(['2019-4-1 23:00', 122 | '2019-4-2 0:00', 123 | '2019-4-2 1:00', 124 | '2019-4-2 2:00', 125 | '2019-4-2 3:00', 126 | '2019-4-2 4:00', 127 | '2019-4-2 5:00'], yp, linewidth=1, marker='*', ls='--', label='Prediction') 128 | 129 | mape = metrics.mean_absolute_percentage_error(yt, yp) 130 | vs = metrics.explained_variance_score(yt, yp) 131 | mae = metrics.mean_absolute_error(yt, yp) 132 | mse = metrics.mean_squared_error(yt, yp) 133 | r2 = metrics.r2_score(yt, yp) 134 | rmse = np.sqrt(mse) 135 | print('mape:' + str(mape)) 136 | print('mae:' + str(mae)) 137 | print('mse:' + str(mse)) 138 | print('r2:' + str(r2)) 139 | print('rmse:' + str(rmse)) 140 | 141 | ax1.set_xlabel('Time') 142 | ax1.set_ylabel('Hourly traffic volume') 143 | ax1.yaxis.set_major_locator(MaxNLocator(10)) 144 | ax1.xaxis.set_major_locator(MaxNLocator(12)) 145 | ax1.yaxis.set_major_locator(ticker.MaxNLocator(integer=True)) 146 | ax1.set_ylim(0, 35) 147 | fig1.autofmt_xdate(rotation=30) 148 | ax1.grid(True) 149 | plt.legend() 150 | plt.title('Vessel traffic flow prediction') 151 | plt.tight_layout() 152 | plt.savefig('plots/' + str(tpe) + '.png') 153 | return 154 | -------------------------------------------------------------------------------- /traj_mining/traffic_flow_predict/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | numpy==1.23.0 3 | pandas==1.4.1 4 | scikit_learn==1.1.1 5 | torch==1.10.2+cu113 6 | tqdm==4.63.0 7 | -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/cleaning.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/cleaning.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/geotostr.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/geotostr.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/plotting.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/plotting.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/segmentation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/segmentation.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/traj_clean.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_clean.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/traj_interpolation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_interpolation.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/traj_segment.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_segment.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/__pycache__/traj_stay.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_stay.cpython-38.pyc -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/geotostr.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import math 3 | import numpy as np 4 | 5 | 6 | def latgeotostr(v): 7 | lbl = 'N' 8 | if float(v) >= 0: 9 | lbl = 'N' 10 | else: 11 | lbl = 'S' 12 | v = float(v) 13 | d = math.floor(v) 14 | m = (v - d) * 60 15 | lat = str(d) + '°' + str(np.round(m, 3)) + '′' + lbl 16 | return lat 17 | 18 | 19 | def longgeotostr(v): 20 | lbl = 'E' 21 | if float(v) >= 0: 22 | lbl = 'E' 23 | else: 24 | lbl = 'W' 25 | v = float(v) 26 | d = math.floor(v) 27 | m = (v - d) * 60 28 | longi = str(d) + '°' + str(np.round(m, 3)) + '′' + lbl 29 | return longi 30 | 31 | 32 | if __name__ == '__main__': 33 | L = 336 34 | s = 3.1415926 * (3 * 25 + 90 + L) * (3 * 25 + 90 + L) 35 | print(str(s)) 36 | -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/ptinpolygon.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | 4 | 5 | def isPointInPoly(point, poly): 6 | '''Determine whether a point is inside an arbitrary polygon(convex or non-convex) 7 | Input: 8 | point: tuple(x, y) 9 | poly: [x1, y1, x2, y2, ..., xn, yn] 10 | Output: 11 | True: inside 12 | False: outside 13 | ''' 14 | assert len(poly) % 2 == 0 15 | 16 | vector_list = [] 17 | angle_sum = 0 18 | poly = np.array(poly) 19 | 20 | for i in range(int(len(poly) / 2)): 21 | vector = [poly[i << 1] - point[0], poly[(i << 1) + 1] - point[1]] 22 | vector = np.array(vector) 23 | vector_list.append(vector) 24 | 25 | for i in range(len(vector_list)): 26 | if i == len(vector_list) - 1: 27 | cross = np.cross(vector_list[i], vector_list[0]) 28 | cos = np.dot(vector_list[i], vector_list[0]) / ( 29 | np.linalg.norm(vector_list[i]) * np.linalg.norm(vector_list[0])) 30 | else: 31 | cross = np.cross(vector_list[i], vector_list[i + 1]) 32 | cos = np.dot(vector_list[i], vector_list[i + 1]) / ( 33 | np.linalg.norm(vector_list[i]) * np.linalg.norm(vector_list[i + 1])) 34 | try: 35 | angle = math.acos(cos) 36 | if cross >= 0: 37 | angle_sum += angle 38 | else: 39 | angle_sum -= angle 40 | except: 41 | print(cos) 42 | 43 | if abs(angle_sum) > 6.283185307: 44 | return True 45 | else: 46 | return False 47 | 48 | # if __name__ == '__main__': 49 | # point = (9, 1.1) 50 | # poly = [1, 1, 1, 10, 10, 10, 5, 5, 10, 1] 51 | # print(isPointInPoly(point, poly)) 52 | -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/traj_clean.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from vincenty import vincenty 4 | import warnings 5 | 6 | warnings.filterwarnings("ignore") 7 | 8 | 9 | def heuristic_clean(data, vthreshold: float): 10 | ''' 11 | :param data: raw trajectory 12 | :param vthreshold: speed threshold 13 | :return: cleaned trajectory 14 | ''' 15 | tdf = pd.DataFrame(columns=data.columns, index=data.index) 16 | tdf.drop(tdf.index, inplace=True) 17 | 18 | data.drop_duplicates() 19 | data = data.reset_index(drop=True) 20 | for shipmmsi, dt in data.groupby('DRMMSI'): 21 | if len(str(shipmmsi)) > 5: 22 | dt_copy = dt.copy(deep=True) 23 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 24 | dt_copy = dt_copy.reset_index(drop=True) 25 | dt_copy['NOISE'] = 0 26 | for idx, di in dt_copy.iterrows(): 27 | if di['NOISE'] == 0: 28 | if idx < len(dt_copy) - 1: 29 | t1 = di['DRGPSTIME'] 30 | t2 = dt_copy.loc[idx + 1, 'DRGPSTIME'] 31 | Δt = (t2 - t1) / 3600.0 32 | pt1 = (di['DRLATITUDE'], di['DRLONGITUDE']) 33 | pt2 = (dt_copy.loc[idx + 1, 'DRLATITUDE'], dt_copy.loc[idx + 1, 'DRLONGITUDE']) 34 | Δd = vincenty(pt1, pt2) * 1000 / 1852.25 35 | Δv = Δd / Δt 36 | if Δv > vthreshold: 37 | dt_copy.loc[idx + 1, 'NOISE'] = 1 38 | print("1") 39 | else: 40 | continue 41 | dt_copy_new = dt_copy.query('NOISE==0') 42 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1) 43 | tdf = tdf.append(dt_copy_new, ignore_index=True) 44 | return tdf 45 | 46 | 47 | def sw_clean(data, sw: int): 48 | ''' 49 | :param data: raw trajectory 50 | :param sw: the size of sliding window 51 | :return: cleaned trajectory 52 | ''' 53 | tdf = pd.DataFrame(columns=data.columns, index=data.index) 54 | tdf.drop(tdf.index, inplace=True) 55 | 56 | data.drop_duplicates() 57 | data = data.reset_index(drop=True) 58 | for shipmmsi, dt in data.groupby('DRMMSI'): 59 | if len(str(shipmmsi)) > 6: 60 | dt_copy = dt.copy(deep=True) 61 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 62 | dt_copy = dt_copy.reset_index(drop=True) 63 | dt_copy['NOISE'] = 0 64 | copylength = len(dt_copy) 65 | num_samples = copylength // sw + 1 66 | for idx in np.arange(num_samples): 67 | start_x = sw * idx 68 | end_x = start_x + sw - 1 69 | end_x = (copylength - 1) if end_x > (copylength - 1) else end_x 70 | dt_temp = dt_copy.loc[start_x:end_x] 71 | lats = dt_temp.loc[:, "DRLATITUDE"] 72 | longs = dt_temp.loc[:, "DRLONGITUDE"] 73 | stdlat = lats.std() 74 | meanlat = lats.mean() 75 | stdlog = longs.std() 76 | meanlog = longs.mean() 77 | for jdx, di in dt_temp.iterrows(): 78 | if abs(di['DRLATITUDE'] - meanlat) > abs(1.5 * stdlat) or abs(di['DRLONGITUDE'] - meanlog) > abs( 79 | 1.5 * stdlog): 80 | dt_copy.loc[jdx, 'NOISE'] = 1 81 | print("1") 82 | pass 83 | 84 | dt_copy_new = dt_copy.query('NOISE==0') 85 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1) 86 | tdf = tdf.append(dt_copy_new, ignore_index=True) 87 | return tdf 88 | -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/traj_interpolation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from scipy.interpolate import pchip_interpolate 4 | from typing import Dict, List, Union, Optional 5 | from vincenty import vincenty 6 | 7 | # types 8 | TrajData = Dict[str, Union[List[float]]] 9 | 10 | 11 | def traj_load(traj_file: str) -> TrajData: 12 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []} 13 | df = pd.read_csv(traj_file) 14 | dt_c = df.copy(deep=True) 15 | dt_c['DRLONGITUDE'] = dt_c['DRLONGITUDE'].map(lambda x: x / 1.0) 16 | dt_c['DRLATITUDE'] = dt_c['DRLATITUDE'].map(lambda x: x / 1.0) 17 | dt_c['DRSPEED'] = dt_c['DRSPEED'].map(lambda x: x / 1.0) 18 | for index, row in dt_c.iterrows(): 19 | trajs_data['lat'].append(row['DRLATITUDE']) 20 | trajs_data['lon'].append(row['DRLONGITUDE']) 21 | trajs_data['speed'].append(row['DRSPEED']) 22 | trajs_data['tstamp'].append(row['DRGPSTIME']) 23 | return trajs_data 24 | 25 | 26 | def traj_calculate_distance(traj_data: TrajData) -> List[float]: 27 | traj_dist = np.zeros(len(traj_data['lat'])) 28 | for i in range(len(traj_dist) - 1): 29 | lat1 = traj_data['lat'][i] 30 | lon1 = traj_data['lon'][i] 31 | lat2 = traj_data['lat'][i + 1] 32 | lon2 = traj_data['lon'][i + 1] 33 | pts = (lat1, lon1) 34 | pte = (lat2, lon2) 35 | s = vincenty(pts, pte) * 1000 # unit meter 36 | traj_dist[i + 1] = s 37 | return traj_dist.tolist() 38 | 39 | 40 | def traj_interpolate(traj_file: str, res: float = 1.0, num: Optional[int] = None) -> TrajData: 41 | ''' 42 | :param traj_data: raw trajectory filename 43 | :param res: time resolution 44 | :param num: None 45 | :return: interpolated trajectory 46 | ''' 47 | traj_data = traj_load(traj_file) 48 | if res <= 0.0: 49 | raise ValueError('res must be > 0.0') 50 | if num is not None and num < 0: 51 | raise ValueError('num must be >= 0') 52 | _traj_dist = traj_calculate_distance(traj_data) 53 | xi = np.cumsum(_traj_dist) 54 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]]) 55 | num = num if num is not None else int(np.ceil(xi[-1] / res)) 56 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True) 57 | y = pchip_interpolate(xi, yi, x, axis=1) 58 | traj_data_interp = {'lat': list(y[0, :]), 'lon': list(y[1, :]), 'speed': list(y[2, :]), 'tstamp': list(y[-1, :])} 59 | return traj_data_interp 60 | 61 | 62 | def traj_calculate_distance_ts(traj_data: TrajData) -> List[float]: 63 | traj_dist = np.zeros(len(traj_data['lat'])) 64 | for i in range(len(traj_dist) - 1): 65 | s = int(traj_data['tstamp'][i + 1]) - int(traj_data['tstamp'][i]) 66 | traj_dist[i + 1] = s 67 | return traj_dist.tolist() 68 | 69 | 70 | def traj_interpolate_df(traj_data, res: float = 1.0, num: Optional[int] = None) -> TrajData: 71 | ''' 72 | :param traj_data: raw trajectory dataframe 73 | :param res: time resolution 74 | :param num: None 75 | :return: interpolated trajectory 76 | ''' 77 | if res <= 0.0: 78 | raise ValueError('res must be > 0.0') 79 | if num is not None and num < 0: 80 | raise ValueError('num must be >= 0') 81 | _traj_dist = traj_calculate_distance_ts(traj_data) 82 | xi = np.cumsum(_traj_dist) 83 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]]) 84 | num = num if num is not None else int(np.ceil(xi[-1] / res)) 85 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True) 86 | y = pchip_interpolate(xi, yi, x, axis=1) 87 | return y.T 88 | -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/traj_segment.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import warnings 3 | 4 | warnings.filterwarnings("ignore") 5 | 6 | 7 | def segment(df, tsinterval): 8 | """ 9 | Trajectory segmentation. 10 | :param df: an array 11 | :type df: DataFrame 12 | :param tsepsilon: time threshold 13 | :type tsepsilon: float 14 | :param tdf: return segmented trajectory 15 | :type tdf: DataFrame 16 | """ 17 | tdf = pd.DataFrame(columns=df.columns, index=df.index) 18 | tdf.drop(tdf.index, inplace=True) 19 | tdf['tdiff'] = 0 20 | 21 | for shipmmsi, dt in df.groupby('DRMMSI'): 22 | data = dt.copy(deep=True) 23 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 24 | data = data.reset_index(drop=True) 25 | data['tdiff'] = data['DRGPSTIME'].diff().fillna(0) 26 | i = 0 27 | lastindex = 0 28 | for idx, di in data.iterrows(): 29 | if di['tdiff'] > tsinterval and idx >= 1: 30 | data.loc[lastindex:idx - 1, 'DRMMSI'] = str(di['DRMMSI']) + '_' + str(i) 31 | i = i + 1 32 | lastindex = idx 33 | tdf = tdf.append(data, ignore_index=True) 34 | tdf = tdf.drop(['tdiff'], axis=1) 35 | return tdf -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/pyais/traj_stay.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import time 5 | import math 6 | from scipy.spatial.distance import pdist 7 | from scipy.spatial.distance import squareform 8 | from pyais import traj_clean 9 | from pyais import traj_interpolation 10 | from pyais import traj_segment 11 | 12 | import warnings 13 | 14 | warnings.filterwarnings("ignore") 15 | 16 | 17 | # Part1 18 | 19 | def compute_squared_EDM(X): 20 | return squareform(pdist(X, metric='euclidean')) 21 | 22 | 23 | def ST_DBSCAN(data, eps1: float, eps2: float, minPts: int): 24 | ''' 25 | :param data: interpolated trajectory 26 | :param eps1: spatial neighborhood 27 | :param eps2: time neighborhood 28 | :param minPts: the minimum number of points that satisfy the double-neighborhood 29 | :return: labels 30 | ''' 31 | n, m = data.shape 32 | timeDisMat = compute_squared_EDM(data[:, 0].reshape(n, 1)) 33 | disMat = compute_squared_EDM(data[:, 1:]) 34 | core_points_index = np.where(np.sum(np.where((disMat <= eps1) & (timeDisMat <= eps2), 1, 0), axis=1) >= minPts)[0] 35 | labels = np.full((n,), -1) 36 | clusterId = 0 37 | for pointId in core_points_index: 38 | if (labels[pointId] == -1): 39 | labels[pointId] = clusterId 40 | neighbour = np.where((disMat[:, pointId] <= eps1) & (timeDisMat[:, pointId] <= eps2) & (labels == -1))[0] 41 | seeds = set(neighbour) 42 | while len(seeds) > 0: 43 | newPoint = seeds.pop() 44 | labels[newPoint] = clusterId 45 | queryResults = set(np.where((disMat[:, newPoint] <= eps1) & (timeDisMat[:, newPoint] <= eps2))[0]) 46 | if len(queryResults) >= minPts: 47 | for resultPoint in queryResults: 48 | if labels[resultPoint] == -1: 49 | seeds.add(resultPoint) 50 | clusterId = clusterId + 1 51 | return labels 52 | 53 | 54 | def stay_st_detect(traj_file: str): 55 | ''' 56 | :param traj_data: raw trajectory filename 57 | :return: interpolated trajectory,labels 58 | ''' 59 | trajdf = pd.read_csv(traj_file, encoding='gbk') 60 | segdf = traj_segment.segment(trajdf, 1500) 61 | cleandf = traj_clean.heuristic_clean(segdf, 25) 62 | 63 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index) 64 | tdf.drop(tdf.index, inplace=True) 65 | tdf['SP_Status'] = 0 66 | 67 | for shipmmsi, dt in cleandf.groupby('DRMMSI'): 68 | data = dt.copy(deep=True) 69 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 70 | data = data.reset_index(drop=True) 71 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []} 72 | for index, row in data.iterrows(): 73 | trajs_data['lat'].append(row['DRLATITUDE']) 74 | trajs_data['lon'].append(row['DRLONGITUDE']) 75 | trajs_data['speed'].append(row['DRSPEED']) 76 | trajs_data['tstamp'].append(row['DRGPSTIME']) 77 | if len(trajs_data['lon']) < 4: 78 | continue 79 | res = 30.0 80 | my_traj_data_interp = traj_interpolation.traj_interpolate_df(trajs_data, res, None) 81 | dataDf = pd.DataFrame(my_traj_data_interp) 82 | dataDf.columns = ['lat', 'lon', 'spd', 'ts'] 83 | dataDf['ts'] = dataDf['ts'].map(lambda x: int(x)) 84 | datanew = dataDf[['ts', 'lat', 'lon']].values 85 | 86 | minpts = round(np.log(len(datanew))) 87 | if minpts < 4: 88 | minpts = 4 89 | ep2 = minpts / 2 * res 90 | ep1 = 2.2 * ep2 / 3600.0 / 60.0 91 | tj = pd.DataFrame(datanew) 92 | tj.columns = ['DRGPSTIME', 'DRLATITUDE', 'DRLONGITUDE'] 93 | labels = ST_DBSCAN(datanew, ep1, ep2, minpts) 94 | tj['SP_Status'] = labels 95 | tj['DRMMSI'] = shipmmsi 96 | tj['DRGPSTIME'] = tj['DRGPSTIME'].map(lambda x: int(x)) 97 | tdf = tdf.append(tj, ignore_index=True) 98 | tdf = tdf.fillna(0) 99 | return tdf 100 | 101 | 102 | # Part2 103 | def rad(d): 104 | return float(d) * math.pi / 180.0 105 | 106 | 107 | EARTH_RADIUS = 6378.137 108 | 109 | 110 | def GetDistance(lng1, lat1, lng2, lat2): 111 | radLat1 = rad(lat1) 112 | radLat2 = rad(lat2) 113 | a = radLat1 - radLat2 114 | b = rad(lng1) - rad(lng2) 115 | s = 2 * math.asin( 116 | math.sqrt(math.pow(math.sin(a / 2), 2) + math.cos(radLat1) * math.cos(radLat2) * math.pow(math.sin(b / 2), 2))) 117 | s = s * EARTH_RADIUS 118 | s = round(s * 10000, 2) / 10 119 | return s 120 | 121 | 122 | def ka(dis, dc): 123 | if (dis >= dc): 124 | return 0 125 | else: 126 | return 1 127 | 128 | 129 | def density(data, dc): 130 | dc = float(dc) 131 | latitude = list(data['DRLATITUDE']) 132 | longitude = list(data['DRLONGITUDE']) 133 | part_density = [] # 存储局部密度值 134 | scope = [] # 记录每个点计算局部密度的范围 135 | leftBoundary = 0 136 | rightBoundary = len(data) - 1 # 左边界与右边界 137 | for i in range(len(data)): 138 | trigger = True 139 | left = i - 1 140 | right = i + 1 141 | incrementLeft = 1 142 | incrementRight = 1 143 | while trigger: 144 | # 向左拓展 145 | if incrementLeft != 0: 146 | if left < 0: 147 | left = leftBoundary 148 | distanceLeft = GetDistance(longitude[left], latitude[left], longitude[i], latitude[i]) 149 | if (distanceLeft < dc) & (left > leftBoundary): 150 | left -= 1 151 | else: 152 | incrementLeft = 0 153 | # 向右拓展 154 | if incrementRight != 0: 155 | if right > rightBoundary: 156 | right = rightBoundary 157 | distanceRight = GetDistance(longitude[i], latitude[i], longitude[right], latitude[right]) 158 | if (distanceRight < dc) & (right < rightBoundary): 159 | right += 1 160 | else: 161 | incrementRight = 0 162 | # 若左右都停止了拓展，此点的局部密度计算结束 163 | if (incrementLeft == 0) & (incrementRight == 0): 164 | trigger = False 165 | if (left == leftBoundary) & (incrementRight == 0): 166 | trigger = False 167 | if (incrementLeft == 0) & (right == rightBoundary): 168 | trigger = False 169 | if left == leftBoundary: 170 | scope.append([left, right - 1]) 171 | part_density.append(right - left - 1) 172 | elif right == rightBoundary: 173 | scope.append([left + 1, right]) 174 | part_density.append(right - left - 1) 175 | else: 176 | scope.append([left + 1, right - 1]) 177 | part_density.append(right - left - 2) 178 | return part_density, scope 179 | 180 | 181 | # 反向更新的方法 182 | def SP_search(data, tc): 183 | tc = int(tc) 184 | SP = [] 185 | part_density = copy.deepcopy(list(data['part_density'])) 186 | scope = copy.deepcopy(list(data['scope'])) 187 | latitude = copy.deepcopy(list(data['DRLATITUDE'])) 188 | longitude = copy.deepcopy(list(data['DRLONGITUDE'])) 189 | timestamp = copy.deepcopy(list(data['DRGPSTIME'])) 190 | trigger = True 191 | used = [] 192 | while trigger: 193 | partD = max(part_density) 194 | index = part_density.index(partD) 195 | print('index:', index) 196 | start = scope[index][0] 197 | end = scope[index][1] 198 | if len(used) != 0: 199 | for i in used: 200 | if (scope[i][0] > start) & (scope[i][0] < end): 201 | part_density[index] = scope[i][0] - start - 1 202 | scope[index][1] = scope[i][0] - 1 203 | print("1_1") 204 | if (scope[i][1] > start) & (scope[i][1] < end): 205 | part_density[index] = end - scope[i][1] - 1 206 | scope[index][0] = scope[i][1] + 1 207 | print("1_2") 208 | if (scope[i][0] <= start) & (scope[i][1] >= end): 209 | part_density[index] = 0 210 | scope[index][0] = 0; 211 | scope[index][1] = 0 212 | print("1_3") 213 | start = scope[index][0]; 214 | end = scope[index][1] 215 | timeCross = timestamp[end] - timestamp[start] 216 | print('time:', timeCross) 217 | if timeCross > tc: 218 | SarvT = time.localtime(timestamp[start]) 219 | SlevT = time.localtime(timestamp[end]) 220 | SP.append([index, latitude[index], longitude[index], SarvT, SlevT, scope[index]]) 221 | used.append(index) 222 | for k in range(scope[index][0], scope[index][1] + 1): 223 | part_density[k] = 0 224 | part_density[index] = 0 225 | if max(part_density) == 0: 226 | trigger = False 227 | return SP 228 | 229 | 230 | # 通过代表的距离是否小于距离阈值来大致判断停留点是否一致 231 | def similar(sp, dc): 232 | dc = float(dc) 233 | latitude = copy.deepcopy(list(sp['latitude'])) 234 | longitude = copy.deepcopy(list(sp['longitude'])) 235 | i = 0; 236 | index = list(sp.index) 237 | for i in index: 238 | for j in index: 239 | if i != j: 240 | dist = GetDistance(longitude[i], latitude[i], longitude[j], latitude[j]) 241 | if dist < 1.5 * dc: 242 | sp = sp.drop(j, axis=0) 243 | index.remove(j) 244 | return sp 245 | 246 | 247 | def stay_spt_detect(traj_file: str): 248 | ''' 249 | :param traj_data: raw trajectory filename 250 | :return: interpolated trajectory,labels 251 | ''' 252 | trajdf = pd.read_csv(traj_file, encoding='gbk') 253 | segdf = traj_segment.segment(trajdf, 1500) 254 | cleandf = traj_clean.heuristic_clean(segdf, 25) 255 | 256 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index) 257 | tdf.drop(tdf.index, inplace=True) 258 | tdf['SP_Status'] = 0 259 | 260 | for shipmmsi, dt in cleandf.groupby('DRMMSI'): 261 | if len(str(shipmmsi)) > 5: 262 | if len(dt) > 10: 263 | data = dt.copy(deep=True) 264 | data = data.reset_index(drop=True) 265 | sj = data.copy(deep=True) 266 | sj['SP_Status'] = 0 267 | 268 | dc = 400 269 | tc = 600 270 | data['part_density'], data['scope'] = density(data, dc) 271 | SP = SP_search(data, tc) 272 | output = pd.DataFrame(SP) 273 | 274 | if output.empty == False: 275 | output.columns = ['index', 'longitude', 'latitude', 'arriveTime', 'leftTime', 'scope'] 276 | output = similar(output, dc) 277 | 278 | for sco in output['scope']: 279 | start, end = sco[0], sco[1] 280 | tcog = sj['DRDIRECTION'][start:end] 281 | ss = 0 # 1靠泊、2锚泊 282 | if tcog.var() < 800: 283 | ss = 1 284 | else: 285 | ss = 2 286 | sj.loc[start:end, 'SP_Status'] = ss 287 | tdf = tdf.append(sj, ignore_index=True) 288 | return tdf 289 | -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | numpy==1.23.0 3 | pandas==1.4.1 4 | scipy==1.8.1 5 | vincenty==0.1.4 6 | -------------------------------------------------------------------------------- /traj_mining/traj_anchor_berth/trajanchorberth.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import pandas as pd 4 | import matplotlib.pyplot as plt 5 | from pyais import geotostr 6 | from pyais import traj_stay 7 | 8 | 9 | # Function to know if we have a CCW turn 10 | def CCW(p1, p2, p3): 11 | if (p3[1] - p1[1]) * (p2[0] - p1[0]) >= (p2[1] - p1[1]) * (p3[0] - p1[0]): 12 | return True 13 | return False 14 | 15 | 16 | def GiftWrapping(S): 17 | ''' 18 | outline of polygon 19 | :param S: collection of points 20 | :return: outline of polygon 21 | ''' 22 | print(plt.style.available) # 查看可用风格 23 | index = 0 24 | n = len(S) 25 | P = [None] * n 26 | l = np.where(S[:, 0] == np.min(S[:, 0])) 27 | pointOnHull = S[l[0][0]] 28 | i = 0 29 | while True: 30 | P[i] = pointOnHull 31 | endpoint = S[0] 32 | for j in range(1, n): 33 | if (endpoint[0] == pointOnHull[0] and endpoint[1] == pointOnHull[1]) or not CCW(S[j], P[i], endpoint): 34 | endpoint = S[j] 35 | i = i + 1 36 | pointOnHull = endpoint 37 | index += 1 38 | if endpoint[0] == P[0][0] and endpoint[1] == P[0][1]: 39 | break 40 | for i in range(n): 41 | if P[-1] is None: 42 | del P[-1] 43 | P = np.array(P) 44 | # Plot final hull 45 | centriod = polygoncenterofmass(P.tolist()) 46 | print(geotostr.longgeotostr(centriod[0])) 47 | print(geotostr.latgeotostr(centriod[1])) 48 | plt.clf() 49 | plt.plot(P[:, 0], P[:, 1], 'b-', picker=5) 50 | plt.plot([P[-1, 0], P[0, 0]], [P[-1, 1], P[0, 1]], 'b-', picker=5, label='The boundary of anchor berth') 51 | plt.plot(S[:, 0], S[:, 1], ".g", label='The anchor points') 52 | plt.plot(centriod[0], centriod[1], "or", label='The approximate location of the anchor') 53 | plt.yticks(fontproperties='Times New Roman', size=12) 54 | plt.xticks(fontproperties='Times New Roman', size=12) 55 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14}) 56 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14}) 57 | plt.title('Calculation of Anchor Berth Area', fontdict={'family': 'Times New Roman', 'size': 16}) 58 | plt.ticklabel_format(useOffset=False, style='plain') 59 | plt.legend(loc='upper left') 60 | plt.grid(True) 61 | plt.tight_layout() 62 | plt.show(block=True) 63 | plt.pause(1) 64 | return P 65 | 66 | 67 | def shoelace(list): 68 | ''' 69 | shoelace formula, calculate area 70 | :param list: input in clockwise direction 71 | :return: area of polygon 72 | ''' 73 | list.append(list[0]) 74 | n = len(list) 75 | t1 = 0.0 76 | t2 = 0.0 77 | for i in range(n - 1): 78 | t1 += (list[i][0] * list[i + 1][1]) * 3600 # 平方海里，′ 79 | t2 += (list[i][1] * list[i + 1][0]) * 3600 80 | s = abs(t1 - t2) * 0.5 * 1852.25 * 1852.25 81 | print(str(s)) 82 | return s 83 | 84 | 85 | def polygonarea(pointLists): 86 | area = 0.0 87 | for i in range(len(pointLists)): 88 | j = (i + 1) % len(pointLists) 89 | area += pointLists[i][1] * pointLists[j][0] 90 | area -= pointLists[i][0] * pointLists[j][1] 91 | area /= 2.0 92 | return (abs(area)) 93 | 94 | 95 | def polygoncenterofmass(pointLists): 96 | ''' 97 | centroid of polygon 98 | :param pointLists: boundary points 99 | :return: centroid of polygon 100 | ''' 101 | if len(pointLists) < 3: 102 | return (0, 0) 103 | else: 104 | cx = 0 105 | cy = 0 106 | factor = 0 107 | j = 0 108 | a = polygonarea(pointLists) 109 | for i in range(len(pointLists)): 110 | j = (i + 1) % len(pointLists) 111 | factor = pointLists[i][1] * pointLists[j][0] - pointLists[j][1] * pointLists[i][0] 112 | cx += (pointLists[i][1] + pointLists[j][1]) * factor 113 | cy += (pointLists[i][0] + pointLists[j][0]) * factor 114 | factor = 1.0 / (6.0 * a) 115 | cx = cx * factor 116 | cy = cy * factor 117 | return [abs(cy), abs(cx)] 118 | 119 | 120 | def main(): 121 | font_legend = { 122 | 'family': 'Times New Roman', 123 | 'weight': 'normal', 124 | 'size': 14 125 | } 126 | trajdata = traj_stay.stay_st_detect('./data/1.csv') 127 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm'] 128 | for shipmmsi, dt in trajdata.groupby('DRMMSI'): 129 | labels = dt['SP_Status'].values 130 | lbs = set(labels) 131 | plt.plot(dt['DRLONGITUDE'].values, dt['DRLATITUDE'].values, marker='o', 132 | markeredgewidth=1.0, linewidth=0.75, label='Interpolated trajectory', 133 | markerfacecolor='white', markeredgecolor='m', ms=4, alpha=1.0, color='m', zorder=2) 134 | for i, item in enumerate(lbs): 135 | if item >= 0: 136 | colorSytle = scatterColors[i % len(scatterColors)] 137 | subCluster = dt.query("SP_Status==@item") 138 | plt.scatter(subCluster['DRLONGITUDE'].values, subCluster['DRLATITUDE'].values, marker='*', s=70, 139 | edgecolors=colorSytle, label='Stay point', 140 | c='white', linewidths=1.0, zorder=3) 141 | plt.title('Stay Point Identification', fontdict={'family': 'Times New Roman', 'size': 16}) 142 | plt.yticks(fontproperties='Times New Roman', size=12) 143 | plt.xticks(fontproperties='Times New Roman', size=12) 144 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14}) 145 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14}) 146 | plt.ticklabel_format(useOffset=False, style='plain') 147 | plt.legend(loc="best", prop=font_legend) 148 | plt.grid(True) 149 | plt.tight_layout() 150 | plt.show() 151 | 152 | subCluster.rename(columns={'DRLATITUDE': 'latitude', 'DRLONGITUDE': 'longitude'}, 153 | inplace=True) 154 | points = [] 155 | for idx, data in subCluster.iterrows(): 156 | p = [data['longitude'], data['latitude']] 157 | points.append(p) 158 | P = np.array(points) 159 | L = GiftWrapping(P) 160 | mj = shoelace(L.tolist()) 161 | print(str(mj)) 162 | else: 163 | continue 164 | 165 | 166 | if __name__ == '__main__': 167 | main() 168 | -------------------------------------------------------------------------------- /traj_mining/traj_cluster/hdbscan/myhdbscan.py: -------------------------------------------------------------------------------- 1 | from dtaidistance import dtw_ndim 2 | from dtaidistance import dtw 3 | from fastdtw import fastdtw 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import pandas as pd 7 | import hdbscan 8 | 9 | 10 | def load_data(file_path): 11 | """ 12 | import trajectory data 13 | :return: trajectory data 14 | """ 15 | data = pd.read_csv(file_path, usecols=['DRGPSTIME', 'DRMMSI', 'DRLATITUDE', 'DRLONGITUDE']) 16 | data.rename(columns={'DRLONGITUDE': 'long', 'DRLATITUDE': 'lat', 'DRGPSTIME': 't', 'DRMMSI': 'mmsi'}, inplace=True) 17 | trajectories = [] 18 | grouped = data[:].groupby('mmsi') 19 | for name, group in grouped: 20 | if len(group) > 1: 21 | group = group.sort_values(by='t', ascending=True) 22 | group['long'] = group['long'].apply(lambda x: x * (1 / 600000.0)) # 以°为单位 23 | group['lat'] = group['lat'].apply(lambda x: x * (1 / 600000.0)) 24 | loc = group[['lat', 'long']].values # 原始 25 | trajectories.append(loc) 26 | return trajectories 27 | 28 | 29 | def dist_computer(trajectories): 30 | distance_matrix = dtw_ndim.distance_matrix_fast(trajectories, 2) 31 | distance_matrix = np.array(distance_matrix) 32 | return distance_matrix 33 | 34 | 35 | if __name__ == '__main__': 36 | trajectories = load_data('../data/final1.csv') 37 | dist_matrix = dist_computer(trajectories) 38 | clusterer = hdbscan.HDBSCAN(min_cluster_size=4, cluster_selection_epsilon=0, metric='precomputed') 39 | clusterer.fit(dist_matrix) 40 | lbs = clusterer.labels_.copy() 41 | li = list(set(lbs)) 42 | 43 | params = {'axes.titlesize': 'large', 44 | 'legend.fontsize': 14, 45 | 'legend.handlelength': 3} 46 | plt.rcParams.update(params) 47 | plt.style.available 48 | # plt.style.use("dark_background") 49 | 50 | cls = ['red', 'green', 'brown', 'pink', 'yellow', 'black', 'rosybrown', 'royalblue', 'purple', 'tomato', 51 | 'cyan', 'mediumaquamarine', 'mediumblue', 'mediumorchid', 'mediumpurple', 'lightcyan', 'lavender', 52 | 'lavenderblush', 'mediumseagreen', 'mediumslateblue', 'teal', 'mediumspringgreen', 'mediumturquoise', 53 | 'mediumvioletred', 'maroon', 'yellowgreen', 54 | 'midnightblue', 'mintcream', 'mistyrose', 'moccasin', 'navajowhite', 'navy', 'oldlace', 'olive', 'olivedrab', 55 | 'orange', 'orangered', 'orchid', 'palegoldenrod', 'palegreen', 'darkturquoise', 'darkviolet', 'deeppink', 56 | 'deepskyblue', 'dimgray', 'dodgerblue', 'firebrick', 'floralwhite', 'forestgreen', 'fuchsia', 'gainsboro', 57 | 'ghostwhite', 'gold', 'goldenrod', 'gray', 'green', 'greenyellow', 'honeydew', 'hotpink', 'indianred', 58 | 'indigo', 'ivory', 'khaki', 'lemonchiffon', 'lightblue', 'aliceblue', 'lightcoral', 'lightgoldenrodyellow', 59 | 'lightgreen', 'lightgray', 'lightpink', 'lightsalmon', 'lightseagreen', 'lightskyblue', 'lightslategray', 60 | 'lightsteelblue', 'lightyellow', 'lime', 'limegreen', 'linen', 'magenta', 'maroon', 'paleturquoise', 61 | 'palevioletred', 'papayawhip', 'peachpuff', 'peru', 'plum', 'powderblue', 'saddlebrown', 'salmon', 62 | 'sandybrown', 'seagreen', 'seashell', 'sienna', 'silver', 'skyblue', 'slateblue', 'slategray', 63 | 'snow', 'springgreen', 'steelblue', 'tan', 'thistle', 'violet', 'wheat', 'white', 'whitesmoke', 64 | 'antiquewhite', 'aqua', 'aquamarine', 'azure', 'beige', 'bisque', 'blanchedalmond', 'turquoise', 65 | 'lawngreen', 'blue', 'blueviolet', 'burlywood', 'cadetblue', 'chartreuse', 'chocolate', 'coral', 66 | 'cornflowerblue', 'cornsilk', 'crimson', 'darkblue', 'darkcyan', 'darkgoldenrod', 'darkgray', 67 | 'darkgreen', 'darkkhaki', 'darkmagenta', 'darkolivegreen', 'darkorange', 'darkorchid', 'darkred', 68 | 'darksalmon', 'darkseagreen', 'darkslateblue', 'darkslategray'] 69 | for index, value in enumerate(lbs): 70 | if value >= 0: 71 | t = trajectories[index] 72 | x, y = t.T 73 | if len(t) > 0: 74 | plt.plot(y, x, color=cls[value], alpha=1.0, lw=0.4) 75 | else: 76 | t = trajectories[index] 77 | x, y = t.T 78 | plt.plot(y, x, color='red', alpha=0.0, lw=0.4) 79 | plt.yticks(fontproperties='Times New Roman', size=12) 80 | plt.xticks(fontproperties='Times New Roman', size=12) 81 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14}) 82 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14}) 83 | plt.title('Trajectory Clustering', fontdict={'family': 'Times New Roman', 'size': 14}) 84 | plt.ticklabel_format(useOffset=False, style='plain') 85 | plt.grid(True) 86 | plt.tight_layout() 87 | plt.show() 88 | -------------------------------------------------------------------------------- /traj_mining/traj_cluster/requirements.txt: -------------------------------------------------------------------------------- 1 | dtaidistance==2.3.9 2 | fastdtw==0.3.4 3 | matplotlib==3.5.2 4 | numpy==1.23.0 5 | pandas==1.4.3 6 | -------------------------------------------------------------------------------- /traj_preprocess/ais_clean/example/trajclean.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import matplotlib.pyplot as plt 3 | from pyais import traj_clean 4 | 5 | 6 | def plot(rawdf, cleandf, rl): 7 | fig, ax = plt.subplots() 8 | for shipmmsi, dt in rawdf.groupby('DRMMSI'): 9 | if len(dt) >= rl and len(str(shipmmsi)) > 5: 10 | dt.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 11 | ax.plot(dt.loc[:, 'DRLONGITUDE'].values, dt.loc[:, 'DRLATITUDE'].values, marker='*', linestyle='--', 12 | color='red', 13 | linewidth=0.5) 14 | for shipmmsi, dt in cleandf.groupby('DRMMSI'): 15 | if len(dt) >= rl and len(str(shipmmsi)) > 5: 16 | dt.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 17 | ax.plot(dt.loc[:, 'DRLONGITUDE'].values, dt.loc[:, 'DRLATITUDE'].values, marker='>', color='green', 18 | linewidth=0.75) 19 | plt.yticks(fontproperties='Times New Roman', size=14) 20 | plt.xticks(fontproperties='Times New Roman', size=14) 21 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16}) 22 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16}) 23 | plt.title('Trajectory Cleaning', fontdict={'family': 'Times New Roman', 'size': 16}) 24 | plt.ticklabel_format(useOffset=False, style='plain') 25 | plt.grid() 26 | plt.tight_layout() 27 | plt.show(block=True) 28 | 29 | 30 | if __name__ == '__main__': 31 | rawdf = pd.read_csv('../data/1.csv') 32 | cleandf = traj_clean.heuristic_clean(rawdf, 25) 33 | plot(rawdf, cleandf, 1) 34 | -------------------------------------------------------------------------------- /traj_preprocess/ais_clean/pyais/__pycache__/chunk_read.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_clean/pyais/__pycache__/chunk_read.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_clean/pyais/__pycache__/traj_clean.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_clean/pyais/__pycache__/traj_clean.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_clean/pyais/traj_clean.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from vincenty import vincenty 4 | import warnings 5 | 6 | warnings.filterwarnings("ignore") 7 | 8 | 9 | def heuristic_clean(data, vthreshold: float): 10 | ''' 11 | :param data: raw trajectory 12 | :param vthreshold: speed threshold 13 | :return: cleaned trajectory 14 | ''' 15 | tdf = pd.DataFrame(columns=data.columns, index=data.index) 16 | tdf.drop(tdf.index, inplace=True) 17 | 18 | data.drop_duplicates() 19 | data = data.reset_index(drop=True) 20 | for shipmmsi, dt in data.groupby('DRMMSI'): 21 | if len(str(shipmmsi)) > 5: 22 | dt_copy = dt.copy(deep=True) 23 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 24 | dt_copy = dt_copy.reset_index(drop=True) 25 | dt_copy['NOISE'] = 0 26 | for idx, di in dt_copy.iterrows(): 27 | if di['NOISE'] == 0: 28 | if idx < len(dt_copy) - 1: 29 | t1 = di['DRGPSTIME'] 30 | t2 = dt_copy.loc[idx + 1, 'DRGPSTIME'] 31 | Δt = (t2 - t1) / 3600.0 32 | pt1 = (di['DRLATITUDE'], di['DRLONGITUDE']) 33 | pt2 = (dt_copy.loc[idx + 1, 'DRLATITUDE'], dt_copy.loc[idx + 1, 'DRLONGITUDE']) 34 | Δd = vincenty(pt1, pt2) * 1000 / 1852.25 35 | Δv = Δd / Δt 36 | if Δv > vthreshold: 37 | dt_copy.loc[idx + 1, 'NOISE'] = 1 38 | print("1") 39 | else: 40 | continue 41 | dt_copy_new = dt_copy.query('NOISE==0') 42 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1) 43 | tdf = tdf.append(dt_copy_new, ignore_index=True) 44 | return tdf 45 | 46 | 47 | def sw_clean(data, sw: int): 48 | ''' 49 | :param data: raw trajectory 50 | :param sw: the size of sliding window 51 | :return: cleaned trajectory 52 | ''' 53 | tdf = pd.DataFrame(columns=data.columns, index=data.index) 54 | tdf.drop(tdf.index, inplace=True) 55 | 56 | data.drop_duplicates() 57 | data = data.reset_index(drop=True) 58 | for shipmmsi, dt in data.groupby('DRMMSI'): 59 | if len(str(shipmmsi)) > 6: 60 | dt_copy = dt.copy(deep=True) 61 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 62 | dt_copy = dt_copy.reset_index(drop=True) 63 | dt_copy['NOISE'] = 0 64 | copylength = len(dt_copy) 65 | num_samples = copylength // sw + 1 66 | for idx in np.arange(num_samples): 67 | start_x = sw * idx 68 | end_x = start_x + sw - 1 69 | end_x = (copylength - 1) if end_x > (copylength - 1) else end_x 70 | dt_temp = dt_copy.loc[start_x:end_x] 71 | lats = dt_temp.loc[:, "DRLATITUDE"] 72 | longs = dt_temp.loc[:, "DRLONGITUDE"] 73 | stdlat = lats.std() 74 | meanlat = lats.mean() 75 | stdlog = longs.std() 76 | meanlog = longs.mean() 77 | for jdx, di in dt_temp.iterrows(): 78 | if abs(di['DRLATITUDE'] - meanlat) > abs(1.5 * stdlat) or abs(di['DRLONGITUDE'] - meanlog) > abs( 79 | 1.5 * stdlog): 80 | dt_copy.loc[jdx, 'NOISE'] = 1 81 | print("1") 82 | pass 83 | 84 | dt_copy_new = dt_copy.query('NOISE==0') 85 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1) 86 | tdf = tdf.append(dt_copy_new, ignore_index=True) 87 | return tdf 88 | -------------------------------------------------------------------------------- /traj_preprocess/ais_clean/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | numpy==1.23.0 3 | pandas==1.4.1 4 | vincenty==0.1.4 5 | -------------------------------------------------------------------------------- /traj_preprocess/ais_compress/example/trajcompress.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import matplotlib.pyplot as plt 3 | from pyais import traj_compress 4 | from pyais.traj_compress import douglaspeucker 5 | 6 | 7 | def sbctest(): 8 | trajdata = pd.read_csv('../data/1.csv', encoding="gbk") 9 | compresseddata = traj_compress.sbc(trajdata, 0.025, 0.25) 10 | plt.plot(trajdata['DRLONGITUDE'], trajdata['DRLATITUDE'], marker='*', linestyle='--', color='blue', linewidth=0.5, 11 | label='Raw trajectory') 12 | plt.plot(compresseddata['DRLONGITUDE'], compresseddata['DRLATITUDE'], marker='o', ms=5, linestyle='-', color='g', 13 | linewidth=0.5, label='Compressed trajectory') 14 | plt.yticks(fontproperties='Times New Roman', size=14) 15 | plt.xticks(fontproperties='Times New Roman', size=14) 16 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16}) 17 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16}) 18 | plt.title('Trajectory Compression', fontdict={'family': 'Times New Roman', 'size': 16}) 19 | plt.ticklabel_format(useOffset=False, style='plain') 20 | plt.grid() 21 | plt.legend() 22 | plt.tight_layout() 23 | plt.show() 24 | 25 | 26 | def dptest(): 27 | data = pd.read_csv('../data/1.csv', usecols=['DRMMSI', 'DRGPSTIME', 'DRLONGITUDE', 'DRLATITUDE']) 28 | grouped = data[:].groupby('DRMMSI') 29 | for name, group in grouped: 30 | trajdata = group.sort_values(by='DRGPSTIME', ascending=True) 31 | dp = douglaspeucker(trajdata[['DRLATITUDE', 'DRLONGITUDE']].values) 32 | epsilon = dp.avg() 33 | mask = dp.rdp(group[['DRLATITUDE', 'DRLONGITUDE']].values, epsilon, algo="iter", return_mask=True) 34 | compresseddata = group[mask] 35 | plt.plot(trajdata['DRLONGITUDE'], trajdata['DRLATITUDE'], marker='*', linestyle='--', color='blue', 36 | linewidth=0.5, 37 | label='Raw trajectory') 38 | plt.plot(compresseddata['DRLONGITUDE'], compresseddata['DRLATITUDE'], marker='o', ms=5, linestyle='-', 39 | color='g', 40 | linewidth=0.5, label='Compressed trajectory') 41 | plt.yticks(fontproperties='Times New Roman', size=14) 42 | plt.xticks(fontproperties='Times New Roman', size=14) 43 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16}) 44 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16}) 45 | plt.title('Trajectory Compression', fontdict={'family': 'Times New Roman', 'size': 16}) 46 | plt.ticklabel_format(useOffset=False, style='plain') 47 | plt.grid() 48 | plt.legend() 49 | plt.tight_layout() 50 | plt.show() 51 | 52 | 53 | if __name__ == '__main__': 54 | sbctest() 55 | # dptest() 56 | -------------------------------------------------------------------------------- /traj_preprocess/ais_compress/pyais/__pycache__/traj_compress.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_compress/pyais/__pycache__/traj_compress.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_compress/pyais/__pycache__/traj_interpolation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_compress/pyais/__pycache__/traj_interpolation.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_compress/pyais/traj_compress.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from vincenty import vincenty 4 | from math import sqrt 5 | from functools import partial 6 | import sys 7 | 8 | 9 | def sbc(s, max_dist_threhold: float, max_spd_threhold: float): 10 | ''' 11 | :param s: raw trajectory 12 | :param max_dist_threhold: distance threshold 13 | :param max_spd_threhold: speed threshold 14 | :return: compressed trajectory 15 | ''' 16 | scopy = pd.DataFrame.copy(s, deep=True) 17 | s = scopy.reset_index(drop=True) 18 | if len(s) <= 2: 19 | return s 20 | else: 21 | is_halt = False 22 | e = 1 23 | while e < len(s) and not is_halt: 24 | i = 1 25 | while i < e and not is_halt: 26 | deltae = s.at[e, 'DRGPSTIME'] - s.at[0, 'DRGPSTIME'] 27 | deltai = s.at[i, 'DRGPSTIME'] - s.at[0, 'DRGPSTIME'] 28 | xp = s.at[0, 'DRLATITUDE'] + (s.at[e, 'DRLATITUDE'] - s.at[0, 'DRLATITUDE']) * deltai / deltae 29 | yp = s.at[0, 'DRLONGITUDE'] + (s.at[e, 'DRLONGITUDE'] - s.at[0, 'DRLONGITUDE']) * deltai / deltae 30 | ptp = (xp, yp) 31 | pta = (s.at[i, 'DRLATITUDE'], s.at[i, 'DRLONGITUDE']) 32 | Vi_1 = vincenty(pta, (s.at[i - 1, 'DRLATITUDE'], s.at[i - 1, 'DRLONGITUDE'])) / ( 33 | s.at[i, 'DRGPSTIME'] - s.at[i - 1, 'DRGPSTIME']) * 1000 / 1852.25 * 3600 34 | Vi = vincenty((s.at[i + 1, 'DRLATITUDE'], s.at[i + 1, 'DRLONGITUDE']), pta) / ( 35 | s.at[i + 1, 'DRGPSTIME'] - s.at[i, 'DRGPSTIME']) * 1000 / 1852.25 * 3600 36 | if vincenty(pta, ptp) * 1000 / 1852.25 > max_dist_threhold or abs(Vi_1 - Vi) > max_spd_threhold: 37 | is_halt = True 38 | else: 39 | i = i + 1 40 | 41 | if is_halt: 42 | return pd.concat([s[0:1], sbc(s[i:], max_dist_threhold, max_spd_threhold)], ignore_index=True) 43 | e = e + 1 44 | if not is_halt: 45 | return s.loc[[0, len(s) - 1]] 46 | 47 | 48 | class douglaspeucker: 49 | def __init__(self, points): 50 | """ 51 | Rarmer Douglas Peucker 52 | :param points: trajectory points 53 | """ 54 | self.points = points 55 | 56 | def point_distance_line(self, pt, point1, point2): 57 | vec1 = np.array(point1) - np.array(pt) 58 | vec2 = np.array(point2) - np.array(pt) 59 | distance = np.abs(np.cross(vec1, vec2)) / np.linalg.norm(np.array(point1) - np.array(point2)) 60 | return distance 61 | 62 | def __deviations(self): 63 | deviations = [] 64 | if len(self.points) > 2: 65 | for i in range(2, len(self.points)): 66 | p1 = self.points[i - 2] 67 | p2 = self.points[i - 1] 68 | p3 = self.points[i] 69 | dev = self.point_distance_line(p2, p1, p3) 70 | deviations.append(dev) 71 | return deviations 72 | 73 | def avg(self): 74 | values = self.__deviations() 75 | if len(values) > 0: 76 | mean = np.mean(values) 77 | else: 78 | mean = 0 79 | return mean 80 | 81 | def max(self): 82 | values = self.__deviations() 83 | mx = np.max(values) 84 | return mx 85 | 86 | def percent(self): 87 | values = self.__deviations() 88 | p = np.percentile(values, 75) 89 | return p 90 | 91 | def pldist(point, start, end): 92 | """ 93 | Calculates the distance from ``point`` to the line given 94 | by the points ``start`` and ``end``. 95 | :param point: a point 96 | :type point: numpy array 97 | :param start: a point of the line 98 | :type start: numpy array 99 | :param end: another point of the line 100 | :type end: numpy array 101 | """ 102 | if np.all(np.equal(start, end)): 103 | return np.linalg.norm(point - start) 104 | 105 | return np.divide( 106 | np.abs(np.linalg.norm(np.cross(end - start, start - point))), 107 | np.linalg.norm(end - start)) 108 | 109 | def rdp_rec(self, M, epsilon, dist=pldist): 110 | """ 111 | Simplifies a given array of points. 112 | Recursive version. 113 | :param M: an array 114 | :type M: numpy array 115 | :param epsilon: epsilon in the rdp algorithm 116 | :type epsilon: float 117 | :param dist: distance function 118 | :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist` 119 | """ 120 | dmax = 0.0 121 | index = -1 122 | 123 | for i in range(1, M.shape[0]): 124 | d = dist(M[i], M[0], M[-1]) 125 | 126 | if d > dmax: 127 | index = i 128 | dmax = d 129 | 130 | if dmax > epsilon: 131 | r1 = self.rdp_rec(M[:index + 1], epsilon, dist) 132 | r2 = self.rdp_rec(M[index:], epsilon, dist) 133 | 134 | return np.vstack((r1[:-1], r2)) 135 | else: 136 | return np.vstack((M[0], M[-1])) 137 | 138 | def _rdp_iter(self, M, start_index, last_index, epsilon, dist=pldist): 139 | stk = [] 140 | stk.append([start_index, last_index]) 141 | global_start_index = start_index 142 | indices = np.ones(last_index - start_index + 1, dtype=bool) 143 | 144 | while stk: 145 | start_index, last_index = stk.pop() 146 | 147 | dmax = 0.0 148 | index = start_index 149 | 150 | for i in range(index + 1, last_index): 151 | if indices[i - global_start_index]: 152 | d = dist(M[i], M[start_index], M[last_index]) 153 | if d > dmax: 154 | index = i 155 | dmax = d 156 | 157 | if dmax > epsilon: 158 | stk.append([start_index, index]) 159 | stk.append([index, last_index]) 160 | else: 161 | for i in range(start_index + 1, last_index): 162 | indices[i - global_start_index] = False 163 | 164 | return indices 165 | 166 | def rdp_iter(self, M, epsilon, dist=pldist, return_mask=False): 167 | """ 168 | Simplifies a given array of points. 169 | Iterative version. 170 | :param M: an array 171 | :type M: numpy array 172 | :param epsilon: epsilon in the rdp algorithm 173 | :type epsilon: float 174 | :param dist: distance function 175 | :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist` 176 | :param return_mask: return the mask of points to keep instead 177 | :type return_mask: bool 178 | """ 179 | mask = self._rdp_iter(M, 0, len(M) - 1, epsilon, dist) 180 | 181 | if return_mask: 182 | return mask 183 | 184 | return M[mask] 185 | 186 | def rdp(self, M, epsilon=0, dist=pldist, algo="iter", return_mask=False): 187 | """ 188 | Simplifies a given array of points using the Ramer-Douglas-Peucker 189 | algorithm. 190 | Example: 191 | >>> from rdp import rdp 192 | >>> rdp([[1, 1], [2, 2], [3, 3], [4, 4]]) 193 | [[1, 1], [4, 4]] 194 | This is a convenience wrapper around both :func:`rdp.rdp_iter` 195 | and :func:`rdp.rdp_rec` that detects if the input is a numpy array 196 | in order to adapt the output accordingly. This means that 197 | when it is called using a Python list as argument, a Python 198 | list is returned, and in case of an invocation using a numpy 199 | array, a NumPy array is returned. 200 | The parameter ``return_mask=True`` can be used in conjunction 201 | with ``algo="iter"`` to return only the mask of points to keep. Example: 202 | >>> from rdp import rdp 203 | >>> import numpy as np 204 | >>> arr = np.array([1, 1, 2, 2, 3, 3, 4, 4]).reshape(4, 2) 205 | >>> arr 206 | array([[1, 1], 207 | [2, 2], 208 | [3, 3], 209 | [4, 4]]) 210 | >>> mask = rdp(arr, algo="iter", return_mask=True) 211 | >>> mask 212 | array([ True, False, False, True], dtype=bool) 213 | >>> arr[mask] 214 | array([[1, 1], 215 | [4, 4]]) 216 | :param M: a series of points 217 | :type M: numpy array with shape ``(n,d)`` where ``n`` is the number of points and ``d`` their dimension 218 | :param epsilon: epsilon in the rdp algorithm 219 | :type epsilon: float 220 | :param dist: distance function 221 | :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist` 222 | :param algo: either ``iter`` for an iterative algorithm or ``rec`` for a recursive algorithm 223 | :type algo: string 224 | :param return_mask: return mask instead of simplified array 225 | :type return_mask: bool 226 | """ 227 | 228 | if algo == "iter": 229 | algo = partial(self.rdp_iter, return_mask=return_mask) 230 | elif algo == "rec": 231 | if return_mask: 232 | raise NotImplementedError("return_mask=True not supported with algo=\"rec\"") 233 | algo = self.rdp_rec 234 | 235 | if "numpy" in str(type(M)): 236 | return algo(M, epsilon, dist) 237 | 238 | return algo(np.array(M), epsilon, dist).tolist() 239 | -------------------------------------------------------------------------------- /traj_preprocess/ais_compress/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | numpy==1.23.0 3 | pandas==1.4.1 4 | vincenty==0.1.4 5 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/binary_payload.py: -------------------------------------------------------------------------------- 1 | import bitarray 2 | 3 | from pyais import decode 4 | from pyais.util import bits2bytes, bytes2bits 5 | 6 | # This is a message of type 6 which contains binary payload 7 | msg = decode(b"!AIVDM,1,1,,B,6B?n;be:cbapalgc;i6?Ow4,2*4A") 8 | 9 | assert msg.msg_type == 6 10 | 11 | # The payload is bytes by default 12 | assert msg.data == b'\xeb/\x11\x8f\x7f\xf1' 13 | 14 | 15 | # But using `bytes2bits` you can convert the bytes into a bitarray 16 | assert bytes2bits(msg.data) == bitarray.bitarray('111010110010111100010001100011110111111111110001') 17 | 18 | # Or to a bitstring using the bitarray to01() method 19 | assert bytes2bits(msg.data).to01() == '111010110010111100010001100011110111111111110001' 20 | 21 | # It is also possible to transform a set of bits back to bytes 22 | assert bits2bytes('111010110010111100010001100011110111111111110001') == b'\xeb/\x11\x8f\x7f\xf1' 23 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/communication_state.py: -------------------------------------------------------------------------------- 1 | """ 2 | The following example shows how you can get the communication state 3 | of a message. This works for message types 1, 2, 4, 9, 11 and 18. 4 | 5 | These messages contain diagnostic information for the radio system. 6 | """ 7 | from pyais import decode 8 | import json 9 | import functools 10 | 11 | msg = '!AIVDM,1,1,,A,B69Gk3h071tpI02lT2ek?wg61P06,0*1F' 12 | decoded = decode(msg) 13 | 14 | print("The raw radio value is:", decoded.radio) 15 | print("Communication state is SOTMDA:", decoded.is_sotdma) 16 | print("Communication state is ITDMA:", decoded.is_itdma) 17 | 18 | pretty_json = functools.partial(json.dumps, indent=4) 19 | print("Communication state:", pretty_json(decoded.get_communication_state())) 20 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/decode.py: -------------------------------------------------------------------------------- 1 | from pyais import decode, IterMessages 2 | 3 | # Decode a single part using decode 4 | decoded = decode(b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05") 5 | print(decoded) 6 | 7 | parts = [ 8 | b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*08", 9 | b"!AIVDM,2,2,4,A,000000000000000,2*20", 10 | ] 11 | 12 | # Decode a multipart message using decode 13 | decoded = decode(*parts) 14 | print(decoded) 15 | 16 | # Decode a string 17 | decoded = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05") 18 | print(decoded) 19 | 20 | # Decode to dict 21 | decoded = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05") 22 | as_dict = decoded.asdict() 23 | print(as_dict) 24 | 25 | # Decode to json 26 | decoded = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05") 27 | json = decoded.to_json() 28 | print(json) 29 | 30 | # It does not matter if you pass a string or bytes 31 | decoded_b = decode(b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05") 32 | decoded_s = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05") 33 | assert decoded_b == decoded_s 34 | 35 | # Lets say you have some kind of stream of messages. Than you can use `IterMessages` to decode the messages: 36 | fake_stream = [ 37 | b"!AIVDM,1,1,,A,13HOI:0P0000VOHLCnHQKwvL05Ip,0*23", 38 | b"!AIVDM,1,1,,A,133sVfPP00PD>hRMDH@jNOvN20S8,0*7F", 39 | b"!AIVDM,1,1,,B,100h00PP0@PHFV`Mg5gTH?vNPUIp,0*3B", 40 | b"!AIVDM,1,1,,B,13eaJF0P00Qd388Eew6aagvH85Ip,0*45", 41 | b"!AIVDM,1,1,,A,14eGrSPP00ncMJTO5C6aBwvP2D0?,0*7A", 42 | b"!AIVDM,1,1,,A,15MrVH0000KH<:V:NtBLoqFP2H9:,0*2F", 43 | ] 44 | for msg in IterMessages(fake_stream): 45 | print(msg.decode()) 46 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/encode_dict.py: -------------------------------------------------------------------------------- 1 | """ 2 | The following example shows how to create an AIS message using a dictionary of values. 3 | 4 | The full specification of the AIVDM/AIVDO protocol is out of the scope of this example. 5 | For a good overview of the AIVDM/AIVDO Sentence Layer please refer to this project: https://gpsd.gitlab.io/gpsd/AIVDM.html#_aivdmaivdo_sentence_layer 6 | 7 | But you should keep the following things in mind: 8 | 9 | - AIS messages are part of a two layer protocol 10 | - the outer layer is the NMEA 0183 data exchange format 11 | - the actual AIS message is part of the NMEA 0183’s 82-character payload 12 | - because some AIS messages are larger than 82 characters they need to be split across several fragments 13 | - there are 27 different types of AIS messages which differ in terms of fields 14 | 15 | Now to the actual encoding of messages: It is possible to encode a dictionary of values into an AIS message. 16 | To do so, you need some values that you want to encode. The keys need to match the interface of the actual message. 17 | You can call `.fields()` on any message class, to get glimpse on the available fields for each message type. 18 | Unknown keys in the dict are simply omitted by pyais. Most keys have default values and do not need to 19 | be passed explicitly. Only the keys `type` and `mmsi` are always required 20 | 21 | For the following example, let's assume that we want to create a type 1 AIS message. 22 | """ 23 | # Required imports 24 | from pyais.encode import encode_dict 25 | from pyais.messages import MessageType1 26 | 27 | # This statement tells us which fields can be set for messages of type 1 28 | print(MessageType1.fields()) 29 | 30 | # A dictionary of fields that we want to encode 31 | # Note that you can pass many more fields for type 1 messages, but we let pyais 32 | # use default values for those keys 33 | data = { 34 | 'course': 219.3, 35 | 'lat': 37.802, 36 | 'lon': -122.341, 37 | 'mmsi': '366053209', 38 | 'type': 1 39 | } 40 | 41 | # This creates an encoded AIS message 42 | # Note, that `encode_dict` returns always a list of fragments. 43 | # This is done, because you may never know if a message fits into the 82 character 44 | # size limit of payloads 45 | encoded = encode_dict(data) 46 | print(encoded) 47 | 48 | # You can also change the NMEA fields like the radio channel: 49 | print(encode_dict(data, radio_channel="B")) 50 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/encode_msg.py: -------------------------------------------------------------------------------- 1 | """ 2 | The following example shows how to create an AIS message using a dictionary of values. 3 | 4 | The full specification of the AIVDM/AIVDO protocol is out of the scope of this example. 5 | For a good overview of the AIVDM/AIVDO Sentence Layer please refer to this project: https://gpsd.gitlab.io/gpsd/AIVDM.html#_aivdmaivdo_sentence_layer 6 | 7 | But you should keep the following things in mind: 8 | 9 | - AIS messages are part of a two layer protocol 10 | - the outer layer is the NMEA 0183 data exchange format 11 | - the actual AIS message is part of the NMEA 0183’s 82-character payload 12 | - because some AIS messages are larger than 82 characters they need to be split across several fragments 13 | - there are 27 different types of AIS messages which differ in terms of fields 14 | 15 | Now to the actual encoding of messages: It is possible to create a payload class and encode it. 16 | 17 | For the following example, let's assume that we want to create a type 1 AIS message. 18 | """ 19 | # Required imports 20 | from pyais.encode import encode_msg 21 | from pyais.messages import MessageType1 22 | 23 | # You do not need to pass every attribute to the class. 24 | # All field other than `mmsi` do have default values. 25 | msg = MessageType1.create(course=219.3, lat=37.802, lon=-122.341, mmsi='366053209') 26 | 27 | # WARNING: If you try to instantiate the class directly (without using .create()) 28 | # you need to pass all attributes, as no default values are used. 29 | 30 | # This creates an encoded AIS message 31 | # Note, that `encode_msg` returns always a list of fragments. 32 | # This is done, because you may never know if a message fits into the 82 character 33 | # size limit of payloads 34 | encoded = encode_msg(msg) 35 | print(encoded) 36 | 37 | # You can also change the NMEA fields like the radio channel: 38 | print(encode_msg(msg, radio_channel="B")) 39 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/file_stream.py: -------------------------------------------------------------------------------- 1 | """ 2 | The following example shows how to read and parse AIS messages from a file. 3 | 4 | When reading a file, the following things are important to know: 5 | 6 | - lines that begin with a `#` are ignored 7 | - invalid messages are skipped 8 | - invalid lines are skipped 9 | """ 10 | import pathlib 11 | 12 | from pyais.stream import FileReaderStream 13 | 14 | filename = pathlib.Path(__file__).parent.joinpath('sample.ais') 15 | 16 | for msg in FileReaderStream(str(filename)): 17 | decoded = msg.decode() 18 | print(decoded) 19 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/msg_to_csv.py: -------------------------------------------------------------------------------- 1 | """ 2 | The following example shows how you could write the decoded data to CSV. 3 | You first need to decode the data into a dictionary and then write the 4 | dictionary to a CSV file using a `DictWriter`. 5 | """ 6 | import csv 7 | 8 | from pyais import decode 9 | 10 | ais_msg = "!AIVDO,1,1,,,B>qc:003wk?8mP=18D3Q3wgTiT;T,0*13" 11 | data_dict = decode(ais_msg).asdict() 12 | 13 | with open('decoded_message.csv', 'w') as f: 14 | w = csv.DictWriter(f, data_dict.keys()) 15 | w.writeheader() 16 | w.writerow(data_dict) 17 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/out_of_order.py: -------------------------------------------------------------------------------- 1 | """ 2 | The following example shows how to deal with messages that are out of order. 3 | 4 | In many cases it is not guaranteed that the messages arrive in the correct order. 5 | The most prominent example would be UDP. For this use case, there is the `OutOfOrderByteStream` 6 | class. You can pass any number of messages as an iterable into this class and it will 7 | handle the assembly of the messages. 8 | """ 9 | from pyais.stream import IterMessages 10 | 11 | messages = [ 12 | b'!AIVDM,2,1,1,A,538CQ>02A;h?D9QC800pu8@T>0P4l9E8L0000017Ah:;;5r50Ahm5;C0,0*07', 13 | b'!AIVDM,2,2,1,A,F@V@00000000000,2*35', 14 | b'!AIVDM,2,2,9,A,F@V@00000000000,2*3D', 15 | b'!AIVDM,2,1,9,A,538CQ>02A;h?D9QC800pu8@T>0P4l9E8L0000017Ah:;;5r50Ahm5;C0,0*0F', 16 | ] 17 | 18 | for msg in IterMessages(messages): 19 | print(msg.decode()) 20 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/sample.ais: -------------------------------------------------------------------------------- 1 | # taken from https://www.aishub.net/ais-dispatcher 2 | !AIVDM,1,1,,A,13HOI:0P0000VOHLCnHQKwvL05Ip,0*23 3 | !AIVDM,1,1,,A,133sVfPP00PD>hRMDH@jNOvN20S8,0*7F 4 | !AIVDM,1,1,,B,100h00PP0@PHFV`Mg5gTH?vNPUIp,0*3B 5 | !AIVDM,1,1,,B,13eaJF0P00Qd388Eew6aagvH85Ip,0*45 6 | !AIVDM,1,1,,A,14eGrSPP00ncMJTO5C6aBwvP2D0?,0*7A 7 | !AIVDM,1,1,,A,15MrVH0000KH<:V:NtBLoqFP2H9:,0*2F 8 | # Lines with a leading `#` are ignored 9 | # Also invalid lines or invalid messages are ignored 10 | IamNotAnAisMessage1111 -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/examples/single_message.py: -------------------------------------------------------------------------------- 1 | from pyais.messages import NMEAMessage 2 | 3 | message = NMEAMessage(b"!AIVDM,1,1,,B,15M67FC000G?ufbE`FepT@3n00Sa,0*5C") 4 | print(message.decode()) 5 | 6 | # or 7 | 8 | message = NMEAMessage.from_string("!AIVDM,1,1,,B,15M67FC000G?ufbE`FepT@3n00Sa,0*5C") 9 | print(message.decode()) 10 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__init__.py: -------------------------------------------------------------------------------- 1 | from pyais.messages import NMEAMessage, ANY_MESSAGE 2 | from pyais.stream import FileReaderStream, IterMessages 3 | from pyais.encode import encode_dict, encode_msg, ais_to_nmea_0183 4 | from pyais.decode import decode 5 | 6 | __license__ = 'MIT' 7 | __version__ = '2.1.2' 8 | __author__ = 'Leon Morten Richter' 9 | 10 | __all__ = ( 11 | 'encode_dict', 12 | 'encode_msg', 13 | 'ais_to_nmea_0183', 14 | 'NMEAMessage', 15 | 'ANY_MESSAGE', 16 | 'IterMessages', 17 | 'FileReaderStream', 18 | 'decode', 19 | ) 20 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/constants.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/constants.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/decode.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/decode.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/encode.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/encode.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/exceptions.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/exceptions.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/messages.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/messages.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/stream.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/stream.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/__pycache__/util.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/util.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/ais_types.py: -------------------------------------------------------------------------------- 1 | from enum import IntEnum 2 | 3 | 4 | class AISType(IntEnum): 5 | # Refer to https://gpsd.gitlab.io/gpsd/AIVDM.html 6 | NOT_IMPLEMENTED = 0 7 | POS_CLASS_A1 = 1 8 | POS_CLASS_A2 = 2 9 | POS_CLASS_A3 = 3 10 | BASE_STATION = 4 11 | STATIC_AND_VOYAGE = 5 12 | BINARY_ADDRESSED = 6 13 | BINARY_ACK = 7 14 | BINARY_BROADCAST = 8 15 | SAR_AIRCRAFT_POS = 9 16 | DATE_INQ = 10 17 | DATE_RESP = 11 18 | SAFETY_MSG = 12 19 | SAFETY_ACK = 13 20 | SAFETY_BROADCAST = 14 21 | INTERROGATE = 15 22 | ASSIGN_MODE = 16 23 | DGNSS = 17 24 | POS_CLASS_B = 18 25 | POS_CLASS_B_EXT = 19 26 | LINK_MGMT = 20 27 | AID_TO_NAV = 21 28 | CHANNEL_MGMT = 22 29 | GROUP_ASSIGN = 23 30 | STATIC = 24 31 | BINARY_SINGLE_SLOT = 25 32 | BINARY_MULTI_SLOT = 26 33 | LONG_RANGE_BROADCAST = 27 34 | 35 | @classmethod 36 | def _missing_(cls, value: object) -> int: 37 | return AISType.NOT_IMPLEMENTED 38 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/constants.py: -------------------------------------------------------------------------------- 1 | import typing 2 | from enum import IntEnum, Enum 3 | 4 | # Keywords 5 | UNDEFINED = 'Undefined' 6 | RESERVED = 'Reserved' 7 | NULL = 'N/A' 8 | ANSI_RED = '\x1b[31m' 9 | ANSI_RESET = '\x1b[0m' 10 | 11 | 12 | class TalkerID(str, Enum): 13 | """ Enum of all NMEA talker IDs. 14 | See: https://gpsd.gitlab.io/gpsd/AIVDM.html#_talker_ids""" 15 | Base_Station = "AB" 16 | Dependent_Base_Station = "AD" 17 | Mobile_Station = "AI" 18 | Navigation_Station = "AN" 19 | Receiving_Station = "AR" 20 | Limited_Base_Station = "AS" 21 | Transmitting_Station = "AT" 22 | Repeater_Station = "AX" 23 | Base_Station_Deprecated = "BS" 24 | Physical_Shore_Station = "SA" 25 | UNDEFINED = "UNDEFINED" 26 | 27 | @classmethod 28 | def _missing_(cls, value: typing.Any) -> str: 29 | return TalkerID.UNDEFINED 30 | 31 | @classmethod 32 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["TalkerID"]: 33 | return cls(v) if v is not None else None 34 | 35 | 36 | class NavigationStatus(IntEnum): 37 | UnderWayUsingEngine = 0 38 | AtAnchor = 1 39 | NotUnderCommand = 2 40 | RestrictedManoeuverability = 3 41 | ConstrainedByHerDraught = 4 42 | Moored = 5 43 | Aground = 6 44 | EngagedInFishing = 7 45 | UnderWaySailing = 8 46 | AISSARTActive = 14 47 | Undefined = 15 48 | 49 | @classmethod 50 | def _missing_(cls, value: object) -> int: 51 | return NavigationStatus.Undefined 52 | 53 | @classmethod 54 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["NavigationStatus"]: 55 | return cls(v) if v is not None else None 56 | 57 | 58 | class ManeuverIndicator(IntEnum): 59 | NotAvailable = 0 60 | NoSpecialManeuver = 1 61 | SpecialManeuver = 2 62 | UNDEFINED = 3 63 | 64 | @classmethod 65 | def _missing_(cls, value: object) -> int: 66 | return ManeuverIndicator.UNDEFINED 67 | 68 | @classmethod 69 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["ManeuverIndicator"]: 70 | return cls(v) if v is not None else None 71 | 72 | 73 | class EpfdType(IntEnum): 74 | Undefined = 0 75 | GPS = 1 76 | GLONASS = 2 77 | GPS_GLONASS = 3 78 | Loran_C = 4 79 | Chayka = 5 80 | IntegratedNavigationSystem = 6 81 | Surveyed = 7 82 | Galileo = 8 83 | 84 | @classmethod 85 | def _missing_(cls, value: object) -> int: 86 | return EpfdType.Undefined 87 | 88 | @classmethod 89 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["EpfdType"]: 90 | return cls(v) if v is not None else None 91 | 92 | 93 | class ShipType(IntEnum): 94 | NotAvailable = 0 95 | # 20's 96 | WIG = 20 97 | WIG_HazardousCategory_A = 21 98 | WIG_HazardousCategory_B = 22 99 | WIG_HazardousCategory_C = 23 100 | WIG_HazardousCategory_D = 24 101 | WIG_Reserved = 25 102 | # 30's 103 | Fishing = 30 104 | Towing = 31 105 | Towing_LengthOver200 = 32 106 | DredgingOrUnderwaterOps = 33 107 | DivingOps = 34 108 | MilitaryOps = 35 109 | Sailing = 36 110 | PleasureCraft = 37 111 | # 40's 112 | HSC = 40 113 | HSC_HazardousCategory_A = 41 114 | HSC_HazardousCategory_B = 42 115 | HSC_HazardousCategory_C = 43 116 | HSC_HazardousCategory_D = 44 117 | HSC_Reserved = 45 118 | HSC_NoAdditionalInformation = 49 119 | # 50's 120 | PilotVessel = 50 121 | SearchAndRescueVessel = 51 122 | Tug = 52 123 | PortTender = 53 124 | AntiPollutionEquipment = 54 125 | LawEnforcement = 55 126 | SPARE = 56 127 | MedicalTransport = 58 128 | NonCombatShip = 59 129 | # 60's 130 | Passenger = 60 131 | Passenger_HazardousCategory_A = 61 132 | Passenger_HazardousCategory_B = 62 133 | Passenger_HazardousCategory_C = 63 134 | Passenger_HazardousCategory_D = 64 135 | Passenger_Reserved = 65 136 | Passenger_NoAdditionalInformation = 69 137 | # 70's 138 | Cargo = 70 139 | Cargo_HazardousCategory_A = 71 140 | Cargo_HazardousCategory_B = 72 141 | Cargo_HazardousCategory_C = 73 142 | Cargo_HazardousCategory_D = 74 143 | Cargo_Reserved = 75 144 | Cargo_NoAdditionalInformation = 79 145 | # 80's 146 | Tanker = 80 147 | Tanker_HazardousCategory_A = 81 148 | Tanker_HazardousCategory_B = 82 149 | Tanker_HazardousCategory_C = 83 150 | Tanker_HazardousCategory_D = 84 151 | Tanker_Reserved = 85 152 | Tanker_NoAdditionalInformation = 89 153 | # 90's 154 | OtherType = 90 155 | OtherType_HazardousCategory_A = 91 156 | OtherType_HazardousCategory_B = 92 157 | OtherType_HazardousCategory_C = 93 158 | OtherType_HazardousCategory_D = 94 159 | OtherType_Reserved = 95 160 | OtherType_NoAdditionalInformation = 99 161 | 162 | @classmethod 163 | def _missing_(cls, value: object) -> int: 164 | if isinstance(value, int): 165 | if 24 < value < 30: 166 | return ShipType.WIG_Reserved 167 | 168 | if 44 < value < 49: 169 | return ShipType.HSC_Reserved 170 | 171 | if 55 < value < 58: 172 | return ShipType.SPARE 173 | 174 | if 64 < value < 69: 175 | return ShipType.Passenger_Reserved 176 | 177 | if 74 < value < 79: 178 | return ShipType.Cargo_Reserved 179 | 180 | if 84 < value < 89: 181 | return ShipType.Tanker_Reserved 182 | 183 | if 94 < value < 99: 184 | return ShipType.OtherType_Reserved 185 | 186 | return ShipType.NotAvailable 187 | 188 | @classmethod 189 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["ShipType"]: 190 | return cls(v) if v is not None else None 191 | 192 | 193 | class DacFid(IntEnum): 194 | DangerousCargoIndication = 13 195 | TidalWindow = 15 196 | NumPersonsOnBoard = 17 197 | ClearanceTimeToEnterPort = 19 198 | BerthingData = 21 199 | AreaNotice = 24 200 | RouteInfoAddressed = 29 201 | TextDescriptionAddressed = 31 202 | ETA = 221 203 | RTA = 222 204 | AtoN_MonitoringData_UK = 245 205 | AtoN_MonitoringData_ROI = 260 206 | 207 | @classmethod 208 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["DacFid"]: 209 | return cls(v) if v is not None else None 210 | 211 | 212 | class NavAid(IntEnum): 213 | DEFAULT = 0 214 | REFERENCE_POINT = 1 215 | RACON = 2 216 | FIXED = 3 217 | FITTED = 4 218 | SPARE = 5 219 | LIGHT_NO_SECTORS = 6 220 | LIGHT_SECTORS = 7 221 | LEADING_LIGHT_FRONT = 8 222 | LEADING_LIGHT_REAR = 9 223 | BEACON_CARDINAL_N = 10 224 | BEACON_CARDINAL_E = 11 225 | BEACON_CARDINAL_S = 12 226 | BEACON_CARDINAL_W = 13 227 | BEACON_STARBOARD = 14 228 | BEACON_CHANNEL_PORT_HAND = 15 229 | BEACON_CHANNEL_STARBOARD_HAND = 16 230 | BEACON_ISOLATED_DANGER = 17 231 | BEACON_SAFE_WATER = 18 232 | BEACON_SPECIAL_MARK = 19 233 | CARDINAL_MARK_N = 20 234 | CARDINAL_MARK_E = 21 235 | CARDINAL_MARK_S = 22 236 | CARDINAL_MARK_W = 23 237 | PORT_HAND_MARK = 24 238 | STARBOARD_HAND_MARK = 25 239 | PREFERRED_HAND_PORT_HAND = 26 240 | PREFERRED_HAND_STARBOARD_HAND = 27 241 | ISOLATED_DANGER = 28 242 | SAFE_WATER = 29 243 | SPECIAL_MARK = 30 244 | LIGHT_VESSEL = 31 245 | 246 | @classmethod 247 | def _missing_(cls, value: object) -> int: 248 | return NavAid.DEFAULT 249 | 250 | @classmethod 251 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["NavAid"]: 252 | return cls(v) if v is not None else None 253 | 254 | 255 | class TransmitMode(IntEnum): 256 | TXA_TXB_RXA_RXB = 0 # default 257 | TXA_RXA_RXB = 1 258 | TXB_RXA_RXB = 2 259 | RESERVED = 3 260 | 261 | @classmethod 262 | def _missing_(cls, value: object) -> int: 263 | return TransmitMode.TXA_TXB_RXA_RXB 264 | 265 | @classmethod 266 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["TransmitMode"]: 267 | return cls(v) if v is not None else None 268 | 269 | 270 | class StationType(IntEnum): 271 | ALL = 0 272 | RESERVED = 1 273 | CLASS_B_ALL = 2 274 | SAR_AIRBORNE = 3 275 | AID_NAV = 4 276 | CLASS_B_SHIPBORNE = 5 277 | REGIONAL = 6 278 | 279 | @classmethod 280 | def _missing_(cls, value: object) -> int: 281 | if isinstance(value, int): 282 | if 6 <= value <= 9: 283 | return StationType.REGIONAL 284 | if 10 <= value <= 15: 285 | return StationType.RESERVED 286 | return StationType.ALL 287 | 288 | @classmethod 289 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["StationType"]: 290 | return cls(v) if v is not None else None 291 | 292 | 293 | class StationIntervals(IntEnum): 294 | AUTONOMOUS_MODE = 0 295 | MINUTES_10 = 1 296 | MINUTES_6 = 2 297 | MINUTES_3 = 3 298 | MINUTES_1 = 4 299 | SECONDS_30 = 5 300 | SECONDS_15 = 6 301 | SECONDS_10 = 7 302 | SECONDS_5 = 8 303 | NEXT_SHORT_REPORTER_INTERVAL = 9 304 | NEXT_LONGER_REPORTING_INTERVAL = 10 305 | RESERVED = 11 306 | 307 | @classmethod 308 | def _missing_(cls, value: object) -> int: 309 | return StationIntervals.RESERVED 310 | 311 | @classmethod 312 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["StationIntervals"]: 313 | return cls(v) if v is not None else None 314 | 315 | 316 | class SyncState(IntEnum): 317 | """ 318 | https://www.navcen.uscg.gov/?pageName=AISMessagesA#Sync 319 | """ 320 | UTC_DIRECT = 0x00 321 | UTC_INDIRECT = 0x01 322 | BASE_DIRECT = 0x02 323 | BASE_INDIRECT = 0x03 324 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/decode.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from pyais.exceptions import TooManyMessagesException, MissingMultipartMessageException 4 | from pyais.messages import NMEAMessage, ANY_MESSAGE 5 | 6 | 7 | def _assemble_messages(*args: bytes) -> NMEAMessage: 8 | # Convert bytes into NMEAMessage and remember fragment_count and fragment_numbers 9 | temp: typing.List[NMEAMessage] = [] 10 | frags: typing.List[int] = [] 11 | frag_cnt: int = 1 12 | for msg in args: 13 | nmea = NMEAMessage(msg) 14 | temp.append(nmea) 15 | frags.append(nmea.frag_num) 16 | frag_cnt = nmea.fragment_count 17 | 18 | # Make sure provided parts assemble a single (multiline message) 19 | if len(args) > frag_cnt: 20 | raise TooManyMessagesException(f"Got {len(args)} messages, but fragment count is {frag_cnt}") 21 | 22 | # Make sure all parts of a multipart message are provided 23 | diff = [x for x in range(1, frag_cnt + 1) if x not in frags] 24 | if len(diff): 25 | raise MissingMultipartMessageException(f"Missing fragment numbers: {diff}") 26 | 27 | # Assemble temporary messages 28 | final = NMEAMessage.assemble_from_iterable(temp) 29 | return final 30 | 31 | 32 | def decode(*args: typing.Union[str, bytes]) -> ANY_MESSAGE: 33 | parts = tuple(msg.encode('utf-8') if isinstance(msg, str) else msg for msg in args) 34 | nmea = _assemble_messages(*parts) 35 | return nmea.decode() 36 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/encode.py: -------------------------------------------------------------------------------- 1 | import math 2 | import typing 3 | 4 | from pyais.messages import Payload, MSG_CLASS 5 | from pyais.util import chunks, compute_checksum 6 | 7 | # Types 8 | DATA_DICT = typing.Dict[str, typing.Union[str, int, float, bytes, bool]] 9 | AIS_SENTENCES = typing.List[str] 10 | 11 | 12 | def get_ais_type(data: DATA_DICT) -> int: 13 | """ 14 | Get the message type from a set of keyword arguments. The first occurence of either 15 | `type` or `msg_type` will be used. 16 | """ 17 | keys = ['type', 'msg_type'] 18 | length = len(keys) - 1 19 | for i, key in enumerate(keys): 20 | try: 21 | ais_type = data[key] 22 | return int(ais_type) 23 | except (KeyError, ValueError) as err: 24 | if i == length: 25 | raise ValueError("Missing or invalid AIS type. Must be a number.") from err 26 | raise ValueError("Missing type") 27 | 28 | 29 | def data_to_payload(ais_type: int, data: DATA_DICT) -> Payload: 30 | try: 31 | return MSG_CLASS[ais_type].create(**data) 32 | except KeyError as err: 33 | raise ValueError(f"AIS message type {ais_type} is not supported") from err 34 | 35 | 36 | def ais_to_nmea_0183(payload: str, ais_talker_id: str, radio_channel: str, fill_bits: int) -> AIS_SENTENCES: 37 | """ 38 | Splits the AIS payload into sentences, ASCII encodes the payload, creates 39 | and sends the relevant NMEA 0183 sentences. 40 | 41 | HINT: 42 | This method takes care of splitting large payloads (larger than 60 characters) 43 | into multiple sentences. With a total of 80 maximum chars excluding end of line 44 | per sentence, and 20 chars head + tail in the nmea 0183 carrier protocol, 60 45 | chars remain for the actual payload. 46 | 47 | @param payload: Armored AIs payload. 48 | @param ais_talker_id: AIS talker ID (AIVDO or AIVDM) 49 | @param radio_channel: Radio channel (either A or B) 50 | @param fill_bits: The number of fill bits requires to pad the data payload to a 6 bit boundary. 51 | @return: A list of relevant AIS sentences. 52 | """ 53 | messages = [] 54 | max_len = 61 55 | seq_id = '' 56 | frag_cnt = math.ceil(len(payload) / max_len) 57 | 58 | if len(ais_talker_id) != 5: 59 | raise ValueError("AIS talker is must have exactly 6 characters. E.g. AIVDO") 60 | 61 | if len(radio_channel) != 1: 62 | raise ValueError("Radio channel must be a single character") 63 | 64 | for frag_num, chunk in enumerate(chunks(payload, max_len), start=1): 65 | tpl = "!{},{},{},{},{},{},{}*{:02X}" 66 | dummy_message = tpl.format(ais_talker_id, frag_cnt, frag_num, seq_id, radio_channel, chunk, fill_bits, 0) 67 | checksum = compute_checksum(dummy_message) 68 | fill_bits_frag = fill_bits if frag_num == frag_cnt else 0 # Make sure we set fill bits only for last fragment 69 | msg = tpl.format(ais_talker_id, frag_cnt, frag_num, seq_id, radio_channel, chunk, fill_bits_frag, checksum) 70 | messages.append(msg) 71 | 72 | return messages 73 | 74 | 75 | def encode_dict(data: DATA_DICT, talker_id: str = "AIVDO", radio_channel: str = "A") -> AIS_SENTENCES: 76 | """ 77 | Takes a dictionary of data and some NMEA specific kwargs and returns the NMEA 0183 encoded AIS sentence. 78 | 79 | Notes: 80 | - the data dict should also contain the AIS message type (1-27) under the `type` key. 81 | - different messages take different keywords. Refer to the payload classes above to get a glimpse 82 | on what fields each AIS message can take. 83 | 84 | @param data: The AIS data as a dictionary. 85 | @param talker_id: AIS packets have the introducer "AIVDM" or "AIVDO"; 86 | AIVDM packets are reports from other ships and AIVDO packets are reports from your own ship. 87 | @param radio_channel: The radio channel. Can be either 'A' (default) or 'B'. 88 | @return: NMEA 0183 encoded AIS sentences. 89 | 90 | """ 91 | if talker_id not in ("AIVDM", "AIVDO"): 92 | raise ValueError("talker_id must be any of ['AIVDM', 'AIVDO']") 93 | 94 | if radio_channel not in ('A', 'B'): 95 | raise ValueError("radio_channel must be any of ['A', 'B']") 96 | 97 | ais_type = get_ais_type(data) 98 | payload = data_to_payload(ais_type, data) 99 | armored_payload, fill_bits = payload.encode() 100 | return ais_to_nmea_0183(armored_payload, talker_id, radio_channel, fill_bits) 101 | 102 | 103 | def encode_msg(msg: Payload, talker_id: str = "AIVDO", radio_channel: str = "A") -> AIS_SENTENCES: 104 | if talker_id not in ("AIVDM", "AIVDO"): 105 | raise ValueError("talker_id must be any of ['AIVDM', 'AIVDO']") 106 | 107 | if radio_channel not in ('A', 'B'): 108 | raise ValueError("radio_channel must be any of ['A', 'B']") 109 | 110 | armored_payload, fill_bits = msg.encode() 111 | return ais_to_nmea_0183(armored_payload, talker_id, radio_channel, fill_bits) 112 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/exceptions.py: -------------------------------------------------------------------------------- 1 | class AISBaseException(Exception): 2 | """The base exception for all exceptions""" 3 | 4 | 5 | class InvalidNMEAMessageException(AISBaseException): 6 | """Invalid NMEA Message""" 7 | pass 8 | 9 | 10 | class UnknownMessageException(AISBaseException): 11 | """Message not supported yet""" 12 | pass 13 | 14 | 15 | class MissingMultipartMessageException(AISBaseException): 16 | """Multipart message with missing parts provided""" 17 | 18 | 19 | class TooManyMessagesException(AISBaseException): 20 | """Too many messages""" 21 | 22 | 23 | class UnknownPartNoException(AISBaseException): 24 | """Unknown part number""" 25 | 26 | 27 | class InvalidDataTypeException(AISBaseException): 28 | """An Unknown data type was passed to an encoding/decoding function""" 29 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/stream.py: -------------------------------------------------------------------------------- 1 | import typing 2 | from abc import ABC, abstractmethod 3 | from socket import AF_INET, SOCK_DGRAM, SOCK_STREAM, socket 4 | from typing import ( 5 | BinaryIO, Generator, Generic, Iterable, List, TypeVar, cast 6 | ) 7 | 8 | from pyais.exceptions import InvalidNMEAMessageException 9 | from pyais.messages import NMEAMessage 10 | 11 | F = TypeVar("F", BinaryIO, socket, None) 12 | DOLLAR_SIGN = ord("$") 13 | EXCLAMATION_POINT = ord("!") 14 | 15 | 16 | def should_parse(byte_str: bytes) -> bool: 17 | """Return True if a given byte string seems to be NMEA message. 18 | This method does **NOT** validate the message, but uses a heuristic 19 | approach to check (or guess) if byte string is a valid nmea_message. 20 | """ 21 | # The byte sequence is not empty and starts with a $ or a ! and has 6 ',' 22 | return len(byte_str) > 0 and byte_str[0] in (DOLLAR_SIGN, EXCLAMATION_POINT) and byte_str.count(b",") == 6 23 | 24 | 25 | class AssembleMessages(ABC): 26 | """ 27 | Base class that assembles multiline messages. 28 | Offers a iterator like interface. 29 | 30 | This class comes without a __init__ method, because it should never be instantiated! 31 | """ 32 | 33 | def __enter__(self) -> "AssembleMessages": 34 | # Enables use of with statement 35 | return self 36 | 37 | def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: 38 | return None 39 | 40 | def __iter__(self) -> Generator[NMEAMessage, None, None]: 41 | return self._assemble_messages() 42 | 43 | def __next__(self) -> NMEAMessage: 44 | """Returns the next decoded NMEA message.""" 45 | return next(iter(self)) 46 | 47 | def _assemble_messages(self) -> Generator[NMEAMessage, None, None]: 48 | buffer: typing.Dict[typing.Tuple[int, str], typing.List[typing.Optional[NMEAMessage]]] = {} 49 | 50 | messages = self._iter_messages() 51 | for line in messages: 52 | 53 | try: 54 | msg: NMEAMessage = NMEAMessage(line) 55 | except InvalidNMEAMessageException: 56 | # Be gentle and just skip invalid messages 57 | continue 58 | 59 | if msg.is_single: 60 | yield msg 61 | else: 62 | # Instead of None use -1 as a seq_id 63 | seq_id = msg.seq_id 64 | if seq_id is None: 65 | seq_id = -1 66 | 67 | # seq_id and channel make a unique stream 68 | slot = (seq_id, msg.channel) 69 | 70 | if slot not in buffer: 71 | # Create a new array in the buffer that has enough space for all fragments 72 | buffer[slot] = [None, ] * max(msg.fragment_count, 0xff) 73 | 74 | buffer[slot][msg.frag_num - 1] = msg 75 | msg_parts = buffer[slot][0:msg.fragment_count] 76 | 77 | # Check if all fragments are found 78 | not_none_parts = [m for m in msg_parts if m is not None] 79 | if len(not_none_parts) == msg.fragment_count: 80 | yield NMEAMessage.assemble_from_iterable(not_none_parts) 81 | del buffer[slot] 82 | 83 | @abstractmethod 84 | def _iter_messages(self) -> Generator[bytes, None, None]: 85 | raise NotImplementedError("Implement me!") 86 | 87 | 88 | class IterMessages(AssembleMessages): 89 | 90 | def __init__(self, messages: Iterable[bytes]): 91 | # If the user passes a single byte string make it into a list 92 | if isinstance(messages, bytes): 93 | messages = [messages, ] 94 | self.messages: Iterable[bytes] = messages 95 | 96 | @classmethod 97 | def from_strings(cls, messages: Iterable[str], ignore_encoding_errors: bool = False, 98 | encoding: str = "utf-8") -> "IterMessages": 99 | # If the users passes a single message as string, make it a list 100 | if isinstance(messages, str): 101 | messages = [messages, ] 102 | 103 | encoded: List[bytes] = [] 104 | for message in messages: 105 | try: 106 | encoded.append(message.encode(encoding)) 107 | except UnicodeEncodeError as e: 108 | if ignore_encoding_errors: 109 | # Just skip and carry on 110 | continue 111 | raise e 112 | 113 | return IterMessages(encoded) 114 | 115 | def _iter_messages(self) -> Generator[bytes, None, None]: 116 | # Transform self.messages into a generator 117 | yield from (message for message in self.messages) 118 | 119 | 120 | class Stream(AssembleMessages, Generic[F], ABC): 121 | 122 | def __init__(self, fobj: F) -> None: 123 | """ 124 | Create a new Stream-like object. 125 | @param fobj: A file-like or socket object. 126 | """ 127 | self._fobj: F = fobj 128 | 129 | def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: 130 | if self._fobj is not None: 131 | self._fobj.close() 132 | 133 | def _iter_messages(self) -> Generator[bytes, None, None]: 134 | # Do not parse lines, that are obviously not NMEA messages 135 | yield from (line for line in self.read() if should_parse(line)) 136 | 137 | @abstractmethod 138 | def read(self) -> Generator[bytes, None, None]: 139 | raise NotImplementedError() 140 | 141 | 142 | class BinaryIOStream(Stream[BinaryIO]): 143 | """Read messages from a file-like object""" 144 | 145 | def __init__(self, file: BinaryIO) -> None: 146 | super().__init__(file) 147 | 148 | def read(self) -> Generator[bytes, None, None]: 149 | yield from self._fobj.readlines() 150 | 151 | 152 | class FileReaderStream(BinaryIOStream): 153 | """ 154 | Read NMEA messages from file 155 | """ 156 | 157 | def __init__(self, filename: str, mode: str = "rb") -> None: 158 | self.filename: str = filename 159 | self.mode: str = mode 160 | # Try to open file 161 | try: 162 | file = open(self.filename, mode=self.mode) 163 | file = cast(BinaryIO, file) 164 | except Exception as e: 165 | raise FileNotFoundError(f"Could not open file {self.filename}") from e 166 | super().__init__(file) 167 | 168 | 169 | class ByteStream(Stream[None]): 170 | """ 171 | Takes a iterable that contains ais messages as bytes and assembles them. 172 | """ 173 | 174 | def __init__(self, iterable: Iterable[bytes]) -> None: 175 | self.iterable: Iterable[bytes] = iterable 176 | super().__init__(None) 177 | 178 | def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: 179 | return 180 | 181 | def read(self) -> Generator[bytes, None, None]: 182 | yield from self.iterable 183 | 184 | 185 | class SocketStream(Stream[socket]): 186 | BUF_SIZE = 4096 187 | 188 | def recv(self) -> bytes: 189 | return b"" 190 | 191 | def read(self) -> Generator[bytes, None, None]: 192 | partial: bytes = b'' 193 | while True: 194 | body = self.recv() 195 | 196 | # Server closed connection 197 | if not body: 198 | return None 199 | 200 | lines = body.split(b'\r\n') 201 | 202 | line = partial + lines[0] 203 | if line: 204 | yield line 205 | 206 | yield from (line for line in lines[1:-1] if line) 207 | 208 | partial = lines[-1] 209 | 210 | 211 | class UDPReceiver(SocketStream): 212 | 213 | def __init__(self, host: str, port: int) -> None: 214 | sock: socket = socket(AF_INET, SOCK_DGRAM) 215 | sock.bind((host, port)) 216 | super().__init__(sock) 217 | 218 | def recv(self) -> bytes: 219 | return self._fobj.recvfrom(self.BUF_SIZE)[0] 220 | 221 | 222 | class TCPConnection(SocketStream): 223 | """ 224 | Read AIS data from a remote TCP server 225 | https://en.wikipedia.org/wiki/NMEA_0183 226 | """ 227 | 228 | def recv(self) -> bytes: 229 | return self._fobj.recv(self.BUF_SIZE) 230 | 231 | def __init__(self, host: str, port: int = 80) -> None: 232 | sock: socket = socket(AF_INET, SOCK_STREAM) 233 | try: 234 | sock.connect((host, port)) 235 | except ConnectionRefusedError as e: 236 | sock.close() 237 | raise ConnectionRefusedError(f"Failed to connect to {host}:{port}") from e 238 | super().__init__(sock) 239 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/pyais/util.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import typing 3 | from collections import OrderedDict 4 | from functools import partial, reduce 5 | from operator import xor 6 | from typing import Any, Generator, Hashable, TYPE_CHECKING, Union, Dict 7 | 8 | from bitarray import bitarray 9 | 10 | from pyais.constants import SyncState 11 | 12 | if TYPE_CHECKING: 13 | BaseDict = OrderedDict[Hashable, Any] 14 | else: 15 | BaseDict = OrderedDict 16 | 17 | from_bytes = partial(int.from_bytes, byteorder="big") 18 | from_bytes_signed = partial(int.from_bytes, byteorder="big", signed=True) 19 | 20 | T = typing.TypeVar('T') 21 | 22 | 23 | def decode_into_bit_array(data: bytes, fill_bits: int = 0) -> bitarray: 24 | """ 25 | Decodes a raw AIS message into a bitarray. 26 | :param data: Raw AIS message in bytes 27 | :param fill_bits: Number of trailing fill bits to be ignored 28 | :return: 29 | """ 30 | bit_arr = bitarray() 31 | length = len(data) 32 | for i, c in enumerate(data): 33 | if c < 0x30 or c > 0x77 or 0x57 < c < 0x6: 34 | raise ValueError(f"Invalid character: {chr(c)}") 35 | 36 | # Convert 8 bit binary to 6 bit binary 37 | c -= 0x30 if (c < 0x60) else 0x38 38 | c &= 0x3F 39 | 40 | if i == length - 1 and fill_bits: 41 | # The last part be shorter than 6 bits and contain fill bits 42 | c = c >> fill_bits 43 | bit_arr += bitarray(f'{c:b}'.zfill(6 - fill_bits)) 44 | else: 45 | bit_arr += bitarray(f'{c:06b}') 46 | 47 | return bit_arr 48 | 49 | 50 | def chunks(sequence: typing.Sequence[T], n: int) -> Generator[typing.Sequence[T], None, None]: 51 | """Yield successive n-sized chunks from sequence.""" 52 | return (sequence[i:i + n] for i in range(0, len(sequence), n)) 53 | 54 | 55 | def decode_bin_as_ascii6(bit_arr: bitarray) -> str: 56 | """ 57 | Decode binary data as 6 bit ASCII. 58 | :param bit_arr: array of bits 59 | :return: ASCII String 60 | """ 61 | string: str = "" 62 | c: bitarray 63 | for c in chunks(bit_arr, 6): # type:ignore 64 | n: int = from_bytes(c.tobytes()) >> 2 65 | 66 | # Last entry may not have 6 bits 67 | if len(c) != 6: 68 | n >> (6 - len(c)) 69 | 70 | if n < 0x20: 71 | n += 0x40 72 | 73 | # Break if there is an @ 74 | if n == 64: 75 | break 76 | 77 | string += chr(n) 78 | 79 | return string.strip() 80 | 81 | 82 | def get_int(data: bitarray, ix_low: int, ix_high: int, signed: bool = False) -> int: 83 | """ 84 | Cast a subarray of a bitarray into an integer. 85 | The bitarray module adds tailing zeros when calling tobytes(), if the bitarray is not a multiple of 8. 86 | So those need to be shifted away. 87 | :param data: some bitarray 88 | :param ix_low: the lower index of the sub-array 89 | :param ix_high: the upper index of the sub-array 90 | :param signed: True if the value should be interpreted as a signed integer 91 | :return: a normal integer (int) 92 | """ 93 | shift: int = (8 - ((ix_high - ix_low) % 8)) % 8 94 | data = data[ix_low:ix_high] 95 | i: int = from_bytes_signed(data) if signed else from_bytes(data) 96 | return i >> shift 97 | 98 | 99 | def compute_checksum(msg: Union[str, bytes]) -> int: 100 | """ 101 | Compute the checksum of a given message. 102 | This method takes the **whole** message including the leading `!`. 103 | 104 | >>> compute_checksum(b"!AIVDM,1,1,,B,15M67FC000G?ufbE`FepT@3n00Sa,0") 105 | 91 106 | 107 | :param msg: message 108 | :return: int value of the checksum. Format as hex with `f'{checksum:02x}'` 109 | """ 110 | if isinstance(msg, str): 111 | msg = msg.encode() 112 | 113 | msg = msg[1:].split(b'*', 1)[0] 114 | return reduce(xor, msg) 115 | 116 | 117 | # https://gpsd.gitlab.io/gpsd/AIVDM.html#_aivdmaivdo_payload_armoring 118 | PAYLOAD_ARMOR = { 119 | 0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: ':', 120 | 11: ';', 12: '<', 13: '=', 14: '>', 15: '?', 16: '@', 17: 'A', 18: 'B', 19: 'C', 20: 'D', 121 | 21: 'E', 22: 'F', 23: 'G', 24: 'H', 25: 'I', 26: 'J', 27: 'K', 28: 'L', 29: 'M', 30: 'N', 122 | 31: 'O', 32: 'P', 33: 'Q', 34: 'R', 35: 'S', 36: 'T', 37: 'U', 38: 'V', 39: 'W', 40: '`', 123 | 41: 'a', 42: 'b', 43: 'c', 44: 'd', 45: 'e', 46: 'f', 47: 'g', 48: 'h', 49: 'i', 50: 'j', 124 | 51: 'k', 52: 'l', 53: 'm', 54: 'n', 55: 'o', 56: 'p', 57: 'q', 58: 'r', 59: 's', 60: 't', 125 | 61: 'u', 62: 'v', 63: 'w' 126 | } 127 | 128 | # https://gpsd.gitlab.io/gpsd/AIVDM.html#_ais_payload_data_types 129 | SIX_BIT_ENCODING = { 130 | '@': 0, 'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7, 'H': 8, 'I': 9, 'J': 10, 131 | 'K': 11, 'L': 12, 'M': 13, 'N': 14, 'O': 15, 'P': 16, 'Q': 17, 'R': 18, 'S': 19, 'T': 20, 132 | 'U': 21, 'V': 22, 'W': 23, 'X': 24, 'Y': 25, 'Z': 26, '[': 27, '\\': 28, ']': 29, '^': 30, 133 | '_': 31, ' ': 32, '!': 33, '"': 34, '#': 35, '$': 36, '%': 37, '&': 38, '\'': 39, '(': 40, 134 | ')': 41, '*': 42, '+': 43, ',': 44, '-': 45, '.': 46, '/': 47, '0': 48, '1': 49, '2': 50, 135 | '3': 51, '4': 52, '5': 53, '6': 54, '7': 55, '8': 56, '9': 57, ':': 58, ';': 59, '<': 60, 136 | '=': 61, '>': 62, '?': 63 137 | } 138 | 139 | 140 | def to_six_bit(char: str) -> str: 141 | """ 142 | Encode a single character as six-bit bitstring. 143 | @param char: The character to encode 144 | @return: The six-bit representation as string 145 | """ 146 | char = char.upper() 147 | try: 148 | encoding = SIX_BIT_ENCODING[char] 149 | return f"{encoding:06b}" 150 | except KeyError: 151 | raise ValueError(f"received char '{char}' that cant be encoded") 152 | 153 | 154 | def encode_ascii_6(bits: bitarray) -> typing.Tuple[str, int]: 155 | """ 156 | Transform the bitarray to an ASCII-encoded bit vector. 157 | Each character represents six bits of data. 158 | @param bits: The bitarray to convert to an ASCII-encoded bit vector. 159 | @return: ASCII-encoded bit vector and the number of fill bits required to pad the data payload to a 6 bit boundary. 160 | """ 161 | out = "" 162 | chunk: bitarray 163 | padding = 0 164 | for chunk in chunks(bits, 6): # type:ignore 165 | padding = 6 - len(chunk) 166 | num = from_bytes(chunk.tobytes()) >> 2 167 | armor = PAYLOAD_ARMOR[num] 168 | out += armor 169 | return out, padding 170 | 171 | 172 | def int_to_bytes(val: typing.Union[int, bytes]) -> int: 173 | """ 174 | Convert a bytes object to an integer. Byteorder is big. 175 | 176 | @param val: A bytes object to convert to an int. If the value is already an int, this is a NO-OP. 177 | @return: Integer representation of `val` 178 | """ 179 | if isinstance(val, int): 180 | return val 181 | return int.from_bytes(val, 'big') 182 | 183 | 184 | def bits2bytes(bits: typing.Union[str, bitarray]) -> bytes: 185 | """ 186 | Convert a bitstring or a bitarray to bytes. 187 | >>> bits2bytes('00100110') 188 | b'&' 189 | """ 190 | bits = bitarray(bits) 191 | return bits.tobytes() 192 | 193 | 194 | def bytes2bits(in_bytes: bytes, default: typing.Optional[bitarray] = None) -> bitarray: 195 | """ 196 | Convert a bytes object to a bitarray. 197 | 198 | @param in_bytes : The bytes to encode 199 | @param default : A default value to return if `in_bytes` is *Falseish* 200 | 201 | >>> bytes2bits(b'&') 202 | bitarray('00100110') 203 | """ 204 | if default is not None and not in_bytes: 205 | return default 206 | bits = bitarray(endian='big') 207 | bits.frombytes(in_bytes) 208 | return bits 209 | 210 | 211 | def b64encode_str(val: bytes, encoding: str = 'utf-8') -> str: 212 | """BASE64 encoded a bytes string and returns the result as UTF-8 string""" 213 | return base64.b64encode(val).decode(encoding) 214 | 215 | 216 | def coerce_val(val: typing.Any, d_type: typing.Type[T]) -> T: 217 | """Forces a given value in a given datatype""" 218 | if d_type == bytes and not isinstance(val, bytes): 219 | raise ValueError(f"Expected bytes, but got: {type(val)}") 220 | 221 | return d_type(val) # type: ignore 222 | 223 | 224 | def int_to_bin(val: typing.Union[int, bool], width: int, signed: bool = True) -> bitarray: 225 | """ 226 | Convert an integer or boolean value to binary. If the value is too great to fit into 227 | `width` bits, the maximum possible number that still fits is used. 228 | 229 | @param val: Any integer or boolean value. 230 | @param width: The bit width. If less than width bits are required, leading zeros are added. 231 | @param signed: Set to True/False if the value is signed or not. 232 | @return: The binary representation of value with exactly width bits. Type is bitarray. 233 | """ 234 | # Compute the total number of bytes required to hold up to `width` bits. 235 | n_bytes, mod = divmod(width, 8) 236 | if mod > 0: 237 | n_bytes += 1 238 | 239 | # If the value is too big, return a bitarray of all 1's 240 | mask = (1 << width) - 1 241 | if val >= mask: 242 | return bitarray('1' * width) 243 | 244 | bits = bitarray(endian='big') 245 | bits.frombytes(val.to_bytes(n_bytes, 'big', signed=signed)) 246 | return bits[8 - mod if mod else 0:] 247 | 248 | 249 | def str_to_bin(val: str, width: int) -> bitarray: 250 | """ 251 | Convert a string value to binary using six-bit ASCII encoding up to `width` chars. 252 | 253 | @param val: The string to first convert to six-bit ASCII and then to binary. 254 | @param width: The width of the full string. If the string has fewer characters than width, trailing '@' are added. 255 | @return: The binary representation of value with exactly width bits. Type is bitarray. 256 | """ 257 | out = bitarray(endian='big') 258 | 259 | # Each char will be converted to a six-bit binary vector. 260 | # Therefore, the total number of chars is floor(WIDTH / 6). 261 | num_chars = int(width / 6) 262 | 263 | # Add trailing '@' if the string is shorter than `width` 264 | for _ in range(num_chars - len(val)): 265 | val += "@" 266 | 267 | # Encode AT MOST width characters 268 | for char in val[:num_chars]: 269 | # Covert each char to six-bit ASCII vector 270 | txt = to_six_bit(char) 271 | out += bitarray(txt) 272 | 273 | return out 274 | 275 | 276 | def chk_to_int(chk_str: bytes) -> typing.Tuple[int, int]: 277 | """ 278 | Converts a checksum string to a tuple of (fillbits, checksum). 279 | >>> chk_to_int(b"0*1B") 280 | (0, 27) 281 | """ 282 | if not len(chk_str): 283 | return 0, -1 284 | 285 | fill_bits: int = int(chr(chk_str[0])) 286 | try: 287 | checksum = int(chk_str[2:], 16) 288 | except (IndexError, ValueError): 289 | checksum = -1 290 | return fill_bits, checksum 291 | 292 | 293 | SYNC_MASK = 0x03 294 | TIMEOUT_MASK = 0x07 295 | MSG_MASK = 0x3fff 296 | SLOT_INCREMENT_MASK = 0x1fff 297 | 298 | 299 | def get_sotdma_comm_state(radio: int) -> Dict[str, typing.Optional[int]]: 300 | """ 301 | The SOTDMA communication state is structured as follows: 302 | +-------------------+----------------------+------------------------------------------------------------------------------------------------+ 303 | | Parameter | Number of bits | Description | 304 | +-------------------+----------------------+------------------------------------------------------------------------------------------------+ 305 | | Sync state | 2 | 0 UTC direct | 306 | | | | 1 UTC indirect | 307 | | | | 2 Station is synchronized to a base station | 308 | | | | 3 Station is synchronized to another station based on the highest number of received stations | 309 | | Slot time-out | 3 | Specifies frames remaining until a new slot is selected | 310 | | | | 0 means that this was the last transmission in this slot | 311 | | | | 1-7 means that 1 to 7 frames respectively are left until slot change | 312 | | Sub message | 14 | 14 The sub message depends on the current value in slot time-out | 313 | +-------------------+----------------------+------------------------------------------------------------------------------------------------+ 314 | 315 | The slot time-out defines how to interpret the sub message: 316 | +-----------------+---------------------------------------------------------------------------+ 317 | | Slot time-out | Description | 318 | +-----------------+---------------------------------------------------------------------------+ 319 | | 3, 5, 7 | Number of receiving stations (not own station) (between 0 and 16 383) | 320 | | 2, 4, 6 | Slot number Slot number used for this transmission (between 0 and 2 249) | 321 | | 1 | UTC hour (bits 13 to 9) and minute (bits 8 to 2) | 322 | | 0 | Next frame | 323 | +-----------------+---------------------------------------------------------------------------+ 324 | 325 | You may refer to: 326 | - https://github.com/M0r13n/pyais/issues/17 327 | - https://www.itu.int/dms_pubrec/itu-r/rec/m/R-REC-M.1371-1-200108-S!!PDF-E.pdf 328 | - https://www.navcen.uscg.gov/?pageName=AISMessagesA#Sync 329 | """ 330 | result = { 331 | 'received_stations': None, 332 | 'slot_number': None, 333 | 'utc_hour': None, 334 | 'utc_minute': None, 335 | 'slot_offset': None, 336 | 'slot_timeout': 0, 337 | 'sync_state': 0, 338 | } 339 | 340 | sync_state = (radio >> 17) & SYNC_MASK # First two (2) bits 341 | slot_timeout = (radio >> 14) & TIMEOUT_MASK # Next three (3) bits 342 | sub_msg = radio & MSG_MASK # Last 14 bits 343 | 344 | if slot_timeout == 0: 345 | result['slot_offset'] = sub_msg 346 | elif slot_timeout == 1: 347 | result['utc_hour'] = (sub_msg >> 9) & 0xf 348 | result['utc_minute'] = (sub_msg >> 2) & 0x3f 349 | elif slot_timeout in (2, 4, 6): 350 | result['slot_number'] = sub_msg 351 | elif slot_timeout in (3, 5, 7): 352 | result['received_stations'] = sub_msg 353 | else: 354 | raise ValueError("Slot timeout can only be an integer between 0 and 7") 355 | 356 | result['sync_state'] = SyncState(sync_state) 357 | result['slot_timeout'] = slot_timeout 358 | return result 359 | 360 | 361 | def get_itdma_comm_state(radio: int) -> Dict[str, typing.Optional[int]]: 362 | """ 363 | +-----------------+------+--------------------------------------------------------------------------------+ 364 | | Parameter | Bits | Description | 365 | +-----------------+------+--------------------------------------------------------------------------------+ 366 | | Sync state | 2 | 0 UTC direct | 367 | | | | 1 UTC indirec | 368 | | | | 2 Station is synchronized to a base station | 369 | | | | 3 Station is synchronized to another station | 370 | | Slot increment | 13 | Offset to next slot to be used, or zero (0) if no more transmissions | 371 | | Number of slots | 3 | Number of consecutive slots to allocate. (0 = 1 slot, 1 = 2 slots,2 = 3 slots, | 372 | | | | 3 = 4 slots, 4 = 5 slots) | 373 | | Keep flag | 1 | Set to TRUE = 1 if the slot remains allocated for one additional frame | 374 | +-----------------+------+--------------------------------------------------------------------------------+ 375 | 376 | You may refer to: 377 | - https://github.com/M0r13n/pyais/issues/17 378 | - https://www.itu.int/dms_pubrec/itu-r/rec/m/R-REC-M.1371-1-200108-S!!PDF-E.pdf 379 | - https://www.navcen.uscg.gov/?pageName=AISMessagesA#Sync 380 | """ 381 | 382 | sync_state = (radio >> 17) & SYNC_MASK # First two (2) bits 383 | slot_increment = (radio >> 4) & SLOT_INCREMENT_MASK # Next 13 bits 384 | num_slots = (radio >> 1) & TIMEOUT_MASK # Next three (3) bits 385 | keep_flag = radio & 0x01 # Last bit 386 | 387 | return { 388 | 'keep_flag': keep_flag, 389 | 'sync_state': sync_state, 390 | 'slot_increment': slot_increment, 391 | 'num_slots': num_slots, 392 | 'keep_flag': keep_flag, 393 | } 394 | -------------------------------------------------------------------------------- /traj_preprocess/ais_encoder_decoder/requirements.txt: -------------------------------------------------------------------------------- 1 | attr==0.3.2 2 | attrs==21.4.0 3 | bitarray==2.5.1 4 | -------------------------------------------------------------------------------- /traj_preprocess/ais_interpolate/example/trajinterpolation.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import matplotlib.pyplot as plt 3 | from pyais import traj_interpolation 4 | 5 | 6 | def main(): 7 | traj_file = '../data/1.csv' 8 | res = 30 9 | num = None 10 | gpx_data = traj_interpolation.traj_load(traj_file) 11 | traj_data_interp = traj_interpolation.traj_interpolate(traj_file, res, num) 12 | 13 | plt.plot(gpx_data['lon'], gpx_data['lat'], marker='*', ms=5, linestyle='--', color='red', linewidth=0.5, 14 | label='Raw trajectory') 15 | plt.plot(traj_data_interp['lon'], traj_data_interp['lat'], marker='o', ms=2, linestyle='-', color='g', 16 | linewidth=0.5, label='Interpolated trajectory') 17 | plt.yticks(fontproperties='Times New Roman', size=12) 18 | plt.xticks(fontproperties='Times New Roman', size=12) 19 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14}) 20 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14}) 21 | plt.title('Trajectory Interpolation', fontdict={'family': 'Times New Roman', 'size': 16}) 22 | plt.ticklabel_format(useOffset=False, style='plain') 23 | plt.grid() 24 | plt.legend() 25 | plt.tight_layout() 26 | plt.show() 27 | 28 | 29 | if __name__ == '__main__': 30 | main() 31 | -------------------------------------------------------------------------------- /traj_preprocess/ais_interpolate/pyais/__pycache__/traj_interpolation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_interpolate/pyais/__pycache__/traj_interpolation.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_interpolate/pyais/traj_interpolation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from scipy.interpolate import pchip_interpolate 4 | from typing import Dict, List, Union, Optional 5 | from vincenty import vincenty 6 | 7 | # types 8 | TrajData = Dict[str, Union[List[float]]] 9 | 10 | 11 | def traj_load(traj_file: str) -> TrajData: 12 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []} 13 | df = pd.read_csv(traj_file) 14 | dt_c = df.copy(deep=True) 15 | dt_c['DRLONGITUDE'] = dt_c['DRLONGITUDE'].map(lambda x: x / 1.0) 16 | dt_c['DRLATITUDE'] = dt_c['DRLATITUDE'].map(lambda x: x / 1.0) 17 | dt_c['DRSPEED'] = dt_c['DRSPEED'].map(lambda x: x / 1.0) 18 | for index, row in dt_c.iterrows(): 19 | trajs_data['lat'].append(row['DRLATITUDE']) 20 | trajs_data['lon'].append(row['DRLONGITUDE']) 21 | trajs_data['speed'].append(row['DRSPEED']) 22 | trajs_data['tstamp'].append(row['DRGPSTIME']) 23 | return trajs_data 24 | 25 | 26 | def traj_calculate_distance(traj_data: TrajData) -> List[float]: 27 | traj_dist = np.zeros(len(traj_data['lat'])) 28 | for i in range(len(traj_dist) - 1): 29 | lat1 = traj_data['lat'][i] 30 | lon1 = traj_data['lon'][i] 31 | lat2 = traj_data['lat'][i + 1] 32 | lon2 = traj_data['lon'][i + 1] 33 | pts = (lat1, lon1) 34 | pte = (lat2, lon2) 35 | s = vincenty(pts, pte) * 1000 # unit meter 36 | traj_dist[i + 1] = s 37 | return traj_dist.tolist() 38 | 39 | 40 | def traj_interpolate(traj_file: str, res: float = 1.0, num: Optional[int] = None) -> TrajData: 41 | ''' 42 | :param traj_data: raw trajectory filename 43 | :param res: time resolution 44 | :param num: None 45 | :return: interpolated trajectory 46 | ''' 47 | traj_data = traj_load(traj_file) 48 | if res <= 0.0: 49 | raise ValueError('res must be > 0.0') 50 | if num is not None and num < 0: 51 | raise ValueError('num must be >= 0') 52 | _traj_dist = traj_calculate_distance(traj_data) 53 | xi = np.cumsum(_traj_dist) 54 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]]) 55 | num = num if num is not None else int(np.ceil(xi[-1] / res)) 56 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True) 57 | y = pchip_interpolate(xi, yi, x, axis=1) 58 | traj_data_interp = {'lat': list(y[0, :]), 'lon': list(y[1, :]), 'speed': list(y[2, :]), 'tstamp': list(y[-1, :])} 59 | return traj_data_interp 60 | 61 | 62 | def traj_calculate_distance_ts(traj_data: TrajData) -> List[float]: 63 | traj_dist = np.zeros(len(traj_data['lat'])) 64 | for i in range(len(traj_dist) - 1): 65 | s = int(traj_data['tstamp'][i + 1]) - int(traj_data['tstamp'][i]) 66 | traj_dist[i + 1] = s 67 | return traj_dist.tolist() 68 | 69 | 70 | def traj_interpolate_df(traj_data, res: float = 1.0, num: Optional[int] = None) -> TrajData: 71 | ''' 72 | :param traj_data: raw trajectory dataframe 73 | :param res: time resolution 74 | :param num: None 75 | :return: interpolated trajectory 76 | ''' 77 | if res <= 0.0: 78 | raise ValueError('res must be > 0.0') 79 | if num is not None and num < 0: 80 | raise ValueError('num must be >= 0') 81 | _traj_dist = traj_calculate_distance_ts(traj_data) 82 | xi = np.cumsum(_traj_dist) 83 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]]) 84 | num = num if num is not None else int(np.ceil(xi[-1] / res)) 85 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True) 86 | y = pchip_interpolate(xi, yi, x, axis=1) 87 | return y.T 88 | -------------------------------------------------------------------------------- /traj_preprocess/ais_interpolate/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | numpy==1.23.0 3 | pandas==1.4.1 4 | scipy==1.8.1 5 | vincenty==0.1.4 6 | -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/example/trajkalmanfilter.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import time 4 | from decimal import Decimal 5 | import matplotlib.pyplot as plt 6 | from pyais import traj_kalmanfilter 7 | 8 | if __name__ == '__main__': 9 | df = pd.read_csv("../data/1.csv") 10 | ais_origin = [] 11 | ais_data = [] 12 | ais_kalman = traj_kalmanfilter.KalmanFilter() 13 | 14 | for index, row in df.iterrows(): 15 | spd = Decimal(row['SPEED'] * 1852.25 / 3600) 16 | tm = str(row['Date']) + ' ' + str(row['Time']) 17 | timeArray = time.strptime(tm, "%Y-%m-%d %H:%M:%S") 18 | timeStamp = int(time.mktime(timeArray)) 19 | ais_origin.append([row['Longitude'], row['Latitude']]) 20 | coords = str(row['Longitude']) + ',' + str(row['Latitude']) 21 | pt = ais_kalman.process(speed=spd, coordinate=coords, time_stamp=timeStamp, accuracy=10.0) 22 | ais_data.append(ais_kalman._split_coordinate(pt)) 23 | res = list(filter(None, ais_data)) 24 | x, y = np.array(res).T 25 | reso = list(filter(None, ais_origin)) 26 | xo, yo = np.array(reso).T 27 | plt.plot(xo, yo, color='b', marker='*', linestyle='--', linewidth=0.5, label='Raw trajectory') 28 | plt.plot(x, y, color='r', marker='o', ms=5, linestyle='-', linewidth=0.5, label='Filtered trajectory') 29 | plt.yticks(fontproperties='Times New Roman', size=14) 30 | plt.xticks(fontproperties='Times New Roman', size=14) 31 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16}) 32 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16}) 33 | plt.title('Kalman Filter', fontdict={'family': 'Times New Roman', 'size': 16}) 34 | plt.ticklabel_format(useOffset=False, style='plain') 35 | plt.grid() 36 | plt.legend() 37 | plt.tight_layout() 38 | plt.show() 39 | -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_clean.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_clean.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_interpolation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_interpolation.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_kalmanfilter.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_kalmanfilter.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_segment.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_segment.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_stay.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_stay.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/pyais/traj_kalmanfilter.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | 4 | class KalmanFilter(object): 5 | def __init__(self): 6 | self.variance = Decimal('-1') 7 | self.longitude, self.latitude = None, None 8 | self.time_stamp = None 9 | self.accuracy = None 10 | 11 | def set_state(self, coordinate: str, time_stamp: float, accuracy: Decimal): 12 | self.longitude, self.latitude = self._split_coordinate(coordinate) 13 | self.time_stamp = time_stamp 14 | self.accuracy = Decimal(accuracy) 15 | self.variance = Decimal(accuracy * accuracy) 16 | 17 | def process(self, speed: Decimal, coordinate: str, time_stamp: float, accuracy: Decimal): 18 | longitude, latitude = self._split_coordinate(coordinate) 19 | if self.variance < 0: 20 | # 初始状态 21 | self.set_state(coordinate, time_stamp, accuracy) 22 | else: 23 | duration = Decimal(time_stamp - self.time_stamp) 24 | if duration > 0: 25 | self.variance += duration * speed * speed / Decimal('1000') 26 | self.time_stamp = time_stamp 27 | k = self.variance / (self.variance + Decimal(accuracy * accuracy)) 28 | # 计算位置 29 | self.latitude += k * (latitude - self.latitude) 30 | self.longitude += k * (longitude - self.longitude) 31 | # 方差 32 | self.variance = (1 - k) * self.variance 33 | coordinate = self._join_longitude_latitude(self.longitude, self.latitude) 34 | return coordinate 35 | 36 | @staticmethod 37 | def _split_coordinate(coordinate: str): 38 | if not coordinate is None: 39 | longitude, latitude = coordinate.split(',') 40 | return Decimal(longitude), Decimal(latitude) 41 | 42 | @staticmethod 43 | def _join_longitude_latitude(longitude, latitude): 44 | return f'{round(longitude, 6)},{round(latitude, 6)}' 45 | -------------------------------------------------------------------------------- /traj_preprocess/ais_kalman_filter/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | numpy==1.23.0 3 | pandas==1.4.1 4 | -------------------------------------------------------------------------------- /traj_preprocess/ais_segment/data/1.csv: -------------------------------------------------------------------------------- 1 | INDEX,DRMMSI,DRLATITUDE,DRLONGITUDE,DRDIRECTION,DRSPEED,DRGPSTIME,STATUS,DRTRUEHEADING,DIRECTION 2 | 0,244726000,31.172785,122.6683333,278.6,12.9,1556415580,0,0,0 3 | 1,244726000,31.17298333,122.6670317,280.8,12.9,1556415599,0,0,2.2 4 | 2,244726000,31.173205,122.6654467,279.3,12.9,1556415623,0,0,-1.5 5 | 3,244726000,31.17359667,122.6629683,280.3,12.9,1556415659,0,0,1 6 | 4,244726000,31.17378333,122.6617317,280.4,13,1556415677,0,0,0.1 7 | 5,244726000,31.17418167,122.65882,277.4,12.9,1556415719,0,0,-3 8 | 6,244726000,31.17444,122.6566783,278.8,12.9,1556415750,0,0,1.4 9 | 7,244726000,31.17461833,122.6551717,278.2,12.8,1556415771,0,0,-0.6 10 | 8,244726000,31.17479,122.6538267,278.2,12.5,1556415791,0,0,0 11 | 9,244726000,31.17507833,122.6512433,276.1,11.8,1556415831,0,0,-2.1 12 | 10,244726000,31.17588,122.6472183,283.9,10.5,1556415900,0,0,7.8 13 | 11,244726000,31.17622833,122.6456733,287.5,10,1556415930,0,0,3.6 14 | 12,244726000,31.17657833,122.6445783,291,9.9,1556415951,0,0,3.5 15 | 13,244726000,31.17699333,122.643175,289,10,1556415980,0,0,-2 16 | 14,244726000,31.17731833,122.6421333,289.6,10.3,1556416000,0,0,0.6 17 | 15,244726000,31.17766167,122.64099,289,10.6,1556416020,0,0,-0.6 18 | 16,244726000,31.177975,122.63993,289.1,11,1556416040,0,0,0.1 19 | 17,244726000,31.178305,122.63878,287.5,11.2,1556416060,0,0,-1.6 20 | 18,244726000,31.17875667,122.637145,290.3,11.3,1556416087,0,0,2.8 21 | 19,244726000,31.18002167,122.63505,315.3,11.1,1556416130,0,0,25 22 | 20,244726000,31.18141833,122.6333667,315,10.8,1556416170,0,0,-0.3 23 | 21,244726000,31.18208167,122.6327417,323.2,10.8,1556416188,0,0,8.2 24 | 22,244726000,31.18317,122.6320933,333.7,11,1556416211,0,0,10.5 25 | 23,244726000,31.183995,122.6315867,331.7,10.5,1556416230,0,0,-2 26 | 24,244726000,31.18483167,122.63106,331.2,10.1,1556416251,0,0,-0.5 27 | 25,244726000,31.18672667,122.6299767,334.9,8.7,1556416300,0,0,3.7 28 | 26,244726000,31.18746167,122.6296117,337.4,8.1,1556416320,1,0,2.5 29 | 27,244726000,31.18939,122.6286283,333.2,6.3,1556416385,1,0,-4.2 30 | 28,244726000,31.190035,122.6281667,326.8,5.3,1556416414,1,0,-6.4 31 | 29,244726000,31.19035833,122.6278967,323.5,4.8,1556416430,1,0,-3.3 32 | 30,244726000,31.19063167,122.627635,319.5,4.3,1556416448,1,0,-4 33 | 31,244726000,31.19089,122.6273467,315.7,3.7,1556416468,1,0,-3.8 34 | 32,244726000,31.19153333,122.6264433,311,1.9,1556416551,1,0,-4.7 35 | 33,244726000,31.19167667,122.62627,318,1.2,1556416581,1,0,7 36 | 34,244726000,31.19176333,122.6262217,330.4,1,1556416601,1,0,12.4 37 | 35,244726000,31.19184,122.6262133,348.8,0.8,1556416622,1,0,18.4 38 | 36,244726000,31.19207,122.6263383,39.4,0.7,1556416696,1,0,-309.4 39 | 37,244726000,31.192615,122.627115,53.5,1,1556416877,1,0,14.1 40 | 38,244726000,31.19303333,122.6278433,69.8,0.8,1556417057,1,0,16.3 41 | 39,244726000,31.19284667,122.62879,132.3,0.4,1556417417,1,0,62.5 42 | 40,244726000,31.19254833,122.628945,156,0.2,1556417597,1,0,23.7 43 | 41,244726000,31.19245,122.6290183,136.7,0.2,1556417777,1,0,-19.3 44 | 42,244726000,31.19241333,122.629095,164.1,0.2,1556418317,1,0,27.4 45 | 43,244726000,31.192395,122.6291083,215,0.1,1556418497,1,0,50.9 46 | 44,244726000,31.19236667,122.6291217,199.5,0.1,1556418677,1,0,-15.5 47 | 45,244726000,31.19232,122.6291883,29.3,0,1556419937,1,0,-170.2 48 | 46,244726000,31.192305,122.6291817,29.3,0.1,1556420117,1,0,0 49 | 47,244726000,31.19226333,122.6291617,233.5,0.1,1556420297,1,0,204.2 50 | 48,244726000,31.192155,122.6291817,181.5,0.1,1556420477,1,0,-52 51 | 49,244726000,31.19193833,122.62923,191.8,0.1,1556420837,1,0,10.3 52 | 50,244726000,31.19194,122.62923,191.8,0.1,1556421017,1,0,0 53 | 51,244726000,31.19192,122.6292483,237.6,0.2,1556421377,1,0,45.8 54 | 52,244726000,31.19188667,122.62926,192.2,0.1,1556421737,1,0,-45.4 55 | 53,244726000,31.19190333,122.629275,76.3,0.1,1556421917,1,0,-115.9 56 | 54,244726000,31.19190167,122.6292833,87.6,0.1,1556422097,1,0,11.3 57 | 55,244726000,31.19176,122.6292183,192.9,0.2,1556422818,1,0,105.3 58 | 56,244726000,31.19143167,122.6291633,198.7,0.1,1556423357,1,0,5.8 59 | 57,244726000,31.19148833,122.629175,227,0.2,1556424437,1,0,28.3 60 | 58,244726000,31.19138333,122.6291567,208.8,0.1,1556424977,1,0,-18.2 61 | 59,244726000,31.19141333,122.6291467,247,0.1,1556425157,1,0,38.2 62 | 60,244726000,31.19141333,122.6291133,247,0.1,1556425337,1,0,0 63 | 61,244726000,31.19128833,122.6291067,189.9,0.1,1556425877,1,0,-57.1 64 | 62,244726000,31.19130833,122.6290617,206.9,0.1,1556426057,1,0,17 65 | 63,244726000,31.19127,122.629065,204.5,0.1,1556426236,1,0,-2.4 66 | 64,244726000,31.191155,122.629035,179.6,0.2,1556426597,1,0,-24.9 67 | 65,244726000,31.19113,122.6290133,211,0.2,1556426778,1,0,31.4 68 | 66,244726000,31.19105167,122.6289983,233.1,0.1,1556427137,1,0,22.1 69 | 67,244726000,31.19101667,122.628975,148.8,0.1,1556427317,1,0,-84.3 70 | 68,244726000,31.19103667,122.6289517,208.2,0.1,1556427677,1,0,59.4 71 | 69,244726000,31.19099667,122.6288967,214.3,0.1,1556428037,1,0,6.1 72 | 70,244726000,31.19101,122.6288883,239.9,0.1,1556428217,1,0,25.6 73 | 71,244726000,31.19092,122.6289083,237.2,0.2,1556428577,1,0,-2.7 74 | 72,244726000,31.19089667,122.62891,226.9,0.1,1556428757,1,0,-10.3 75 | 73,244726000,31.19094167,122.62889,334,0.1,1556428937,1,0,107.1 76 | 74,244726000,31.19096833,122.6288567,305.7,0.1,1556429117,1,0,-28.3 77 | 75,244726000,31.19095333,122.6288417,251.4,0.1,1556429297,1,0,-54.3 78 | 76,244726000,31.19092,122.6288183,231.3,0.1,1556429478,1,0,-20.1 79 | 77,244726000,31.19084167,122.62873,218.3,0.1,1556429837,1,0,-13 80 | 78,244726000,31.19090167,122.6287067,313.7,0.1,1556430017,1,0,95.4 81 | 79,244726000,31.190845,122.62873,188.4,0.1,1556430198,1,0,-125.3 82 | 80,244726000,31.19087,122.6287283,223,0.1,1556430557,1,0,34.6 83 | 81,244726000,31.19089,122.6287583,307.3,0.1,1556430737,1,0,84.3 84 | 82,244726000,31.19090667,122.62876,270,0.2,1556431097,1,0,-37.3 85 | 83,244726000,31.19094333,122.628705,315.6,0.1,1556431277,1,0,45.6 86 | 84,244726000,31.19071333,122.62854,321.4,0.3,1556431817,1,0,5.8 87 | 85,244726000,31.190705,122.6284683,193.3,0.5,1556431997,1,0,-128.1 88 | 86,244726000,31.19053333,122.62833,179.9,0.3,1556432537,1,0,-13.4 89 | 87,244726000,31.19037833,122.628255,210.9,0.3,1556432717,1,0,31 90 | 88,244726000,31.190565,122.6281317,357.2,0.2,1556433077,1,0,146.3 91 | 89,244726000,31.19035333,122.628095,192.2,0.2,1556433257,1,0,-165 92 | 90,244726000,31.190375,122.628035,297.3,0.1,1556433437,1,0,105.1 93 | 91,244726000,31.190415,122.6279567,325.4,0.1,1556433617,1,0,28.1 94 | 92,244726000,31.19039667,122.62799,84.2,0.1,1556433977,1,0,-241.2 95 | 93,244726000,31.1903,122.6278533,242.9,0.2,1556434517,1,0,158.7 96 | 94,244726000,31.19015,122.627535,237.5,0.2,1556435597,1,0,-5.4 97 | 95,244726000,31.19009167,122.62746,230.5,0.2,1556435777,1,0,-7 98 | 96,244726000,31.19009833,122.6274317,246.4,0.1,1556435958,1,0,15.9 99 | 97,244726000,31.18984333,122.6272583,162.5,0.4,1556437217,1,0,-83.9 100 | 98,244726000,31.18970333,122.62704,356.9,0.4,1556437937,1,0,194.4 101 | 99,244726000,31.19007167,122.6268883,345,0.5,1556438117,1,0,-11.9 102 | 100,244726000,31.19001167,122.626515,208.3,0.4,1556438477,1,0,-136.7 103 | 101,244726000,31.18962,122.6262583,207.2,0.4,1556438658,1,0,-1.1 104 | 102,244726000,31.18986333,122.6258967,275,0.3,1556439018,1,0,67.8 105 | 103,244726000,31.19031,122.6251183,354.9,0.6,1556439553,1,0,79.9 106 | 104,244726000,31.19036667,122.6251167,0.2,0.6,1556439573,1,0,-354.7 107 | 105,244726000,31.19041833,122.6251267,7.3,0.6,1556439593,1,0,7.1 108 | 106,244726000,31.19052833,122.6251533,9.9,0.6,1556439633,1,0,2.6 109 | 107,244726000,31.19058833,122.6251667,15.3,0.6,1556439653,1,0,5.4 110 | 108,244726000,31.19065,122.6251767,17.9,0.6,1556439673,1,0,2.6 111 | 109,244726000,31.19070333,122.6252033,17.6,0.6,1556439693,1,0,-0.3 112 | 110,244726000,31.19075,122.625245,14.3,0.6,1556439714,1,0,-3.3 113 | 111,244726000,31.19082333,122.6252317,13.8,0.6,1556439733,1,0,-0.5 114 | 112,244726000,31.19086667,122.6252567,15.5,0.5,1556439753,1,0,1.7 115 | 113,244726000,31.19091333,122.62528,20.2,0.5,1556439773,1,0,4.7 116 | 114,244726000,31.191015,122.6253167,25.7,0.5,1556439813,1,0,5.5 117 | 115,244726000,31.19105333,122.6253233,27.6,0.4,1556439833,1,0,1.9 118 | 116,244726000,31.19108,122.62534,27.3,0.4,1556439853,1,0,-0.3 119 | 117,244726000,31.19109833,122.62538,46,0.4,1556439873,1,0,18.7 120 | 118,244726000,31.19111,122.6254417,74.5,0.6,1556439893,1,0,28.5 121 | 119,244726000,31.191125,122.6255117,78.1,0.7,1556439913,1,0,3.6 122 | 120,244726000,31.19117833,122.6260867,78,1.8,1556439993,1,0,-0.1 123 | 121,244726000,31.19122167,122.6262833,71.1,2.1,1556440011,1,0,-6.9 124 | 122,244726000,31.19132333,122.6265083,63.4,2.4,1556440030,1,0,-7.7 125 | 123,244726000,31.19143167,122.62669,56.7,2.6,1556440046,1,0,-6.7 126 | 124,244726000,31.191565,122.6268633,49.5,2.7,1556440063,1,0,-7.2 127 | 125,244726000,31.191715,122.62703,43.5,2.9,1556440078,1,0,-6 128 | 126,244726000,31.191895,122.62718,36.9,2.9,1556440093,1,0,-6.6 129 | 127,244726000,31.19211167,122.6273267,29.1,3.3,1556440109,1,0,-7.8 130 | 128,244726000,31.19241333,122.62746,18.4,3.6,1556440130,1,0,-10.7 131 | 129,244726000,31.19272,122.6275233,9.8,3.8,1556440146,1,0,-8.6 132 | 130,244726000,31.19296833,122.6275267,1,3.8,1556440161,1,0,-8.8 133 | 131,244726000,31.19361833,122.6273483,339.5,4.3,1556440196,1,0,338.5 134 | 132,244726000,31.19394333,122.6271217,328.2,4.6,1556440213,1,0,-11.3 135 | 133,244726000,31.19421667,122.626755,306.2,4.4,1556440233,1,0,-22 136 | 134,244726000,31.19426833,122.6263883,276.4,4.3,1556440250,1,0,-29.8 137 | 135,244726000,31.19428667,122.6254517,271.4,5.7,1556440282,1,0,-5 138 | 136,244726000,31.19411167,122.623925,258.6,7.6,1556440324,1,0,-12.8 139 | 137,244726000,31.19387,122.62316,248.5,8.1,1556440342,1,0,-10.1 140 | 138,244726000,31.19327167,122.6212483,250.8,9.7,1556440384,1,0,2.3 141 | 139,244726000,31.19296333,122.6202767,249.3,10.3,1556440403,1,0,-1.5 142 | 140,244726000,31.19261833,122.6191267,250.7,10.9,1556440424,0,0,1.4 143 | 141,244726000,31.19189167,122.6168567,249.4,11.8,1556440462,0,0,-1.3 144 | 142,244726000,31.191475,122.6155767,249.6,12.1,1556440484,0,0,0.2 145 | 143,244726000,31.19066333,122.6131333,246.3,12.5,1556440523,0,0,-3.3 146 | 144,244726000,31.19001667,122.6111317,249.1,12.9,1556440553,0,0,2.8 147 | 145,244726000,31.18934833,122.609255,251.3,13,1556440583,0,0,2.2 148 | 146,244726000,31.18888167,122.6078633,248.2,13.2,1556440603,0,0,-3.1 149 | 147,244726000,31.187485,122.60382,248.6,13.6,1556440664,0,0,0.4 150 | 148,244726000,31.18697,122.60248,246.9,13.6,1556440684,0,0,-1.7 151 | 149,244726000,31.18648167,122.6010267,248.6,13.8,1556440704,0,0,1.7 152 | 150,244726000,31.18605167,122.5996867,249.3,13.9,1556440724,0,0,0.7 153 | 151,244726000,31.185585,122.5982717,248.3,14,1556440744,0,0,-1 154 | 152,244726000,31.18512833,122.5970067,247.5,14.1,1556440761,0,0,-0.8 155 | 153,244726000,31.184575,122.5954417,248.6,14.3,1556440784,0,0,1.1 156 | 154,244726000,31.18409,122.594,249,14.4,1556440804,0,0,0.4 157 | 155,244726000,31.183605,122.5927167,245.2,14.4,1556440821,0,0,-3.8 158 | 156,244726000,31.18302667,122.5910467,248.8,14.5,1556440845,0,0,3.6 159 | 157,244726000,31.18226833,122.5887767,246.7,14.6,1556440876,0,0,-2.1 160 | 158,244726000,31.18147833,122.586575,249,14.7,1556440905,0,0,2.3 161 | 159,244726000,31.18049333,122.5839383,247.5,14.8,1556440941,0,0,-1.5 162 | 160,244726000,31.17986667,122.5821517,248.1,14.8,1556440965,0,0,0.6 163 | 161,244726000,31.17908167,122.579905,247.4,15,1556440996,0,0,-0.7 164 | 162,244726000,31.178285,122.5776467,248.7,15.1,1556441025,0,0,1.3 165 | 163,244726000,31.17430167,122.5691883,241,15,1556441145,0,0,-7.7 166 | 164,244726000,31.17350667,122.56747,242.4,15.1,1556441169,0,0,1.4 167 | 165,244726000,31.17269167,122.5657583,240.4,15.1,1556441194,0,0,-2 168 | 166,244726000,31.172075,122.564475,240.3,15.1,1556441211,0,0,-0.1 169 | 167,244726000,31.171495,122.5632533,241.4,15.2,1556441229,0,0,1.1 170 | 168,244726000,31.17068333,122.5615333,239.9,15,1556441253,0,0,-1.5 171 | 169,244726000,31.17005667,122.560195,242.3,14.8,1556441271,0,0,2.4 172 | 170,244726000,31.16887833,122.5573467,245.2,14,1556441313,0,0,2.9 173 | 171,244726000,31.168105,122.55528,247.5,13.2,1556441343,0,0,2.3 174 | 172,244726000,31.16749833,122.5537867,245.5,12.6,1556441368,0,0,-2 175 | 173,244726000,31.16696167,122.552325,247.5,12.1,1556441391,0,0,2 176 | 174,244726000,31.16662167,122.5513167,248.5,11.8,1556441409,0,0,1 177 | 175,244726000,31.16625167,122.55022,248.2,11.4,1556441428,0,0,-0.3 178 | 176,244726000,31.16592167,122.5491867,249.9,11.1,1556441445,0,0,1.7 179 | 177,244726000,31.16556667,122.5482167,246.7,10.7,1556441463,0,0,-3.2 180 | 178,244726000,31.165115,122.546975,249.4,10.3,1556441488,0,0,2.7 181 | 179,244726000,31.16478833,122.5460683,248.3,10,1556441505,0,0,-1.1 182 | 180,244726000,31.16447167,122.5451733,248.2,9.9,1556441523,0,0,-0.1 183 | 181,244726000,31.164155,122.5442667,248,10.1,1556441541,0,0,-0.2 184 | 182,244726000,31.16380833,122.5433467,246.4,10.4,1556441559,0,0,-1.6 185 | 183,244726000,31.163445,122.5422717,249.6,10.8,1556441580,0,0,3.2 186 | 184,244726000,31.1631,122.5412033,247.8,11.1,1556441598,0,0,-1.8 187 | 185,244726000,31.16226,122.53889,246.1,11.5,1556441640,0,0,-1.7 188 | 186,244726000,31.16185833,122.53778,246.5,11.8,1556441658,0,0,0.4 189 | 187,244726000,31.16125333,122.53595,248.1,12.1,1556441689,0,0,1.6 190 | 188,244726000,31.16079,122.5346517,247.6,12.4,1556441709,0,0,-0.5 191 | 189,244726000,31.16029667,122.5333517,245.8,12.6,1556441730,0,0,-1.8 192 | 190,244726000,31.15988333,122.5322083,247.8,12.7,1556441749,0,0,2 193 | 191,244726000,31.15911667,122.5294367,253.3,12.9,1556441790,0,0,5.5 194 | 192,244726000,31.15864167,122.527495,252.7,12.9,1556441820,0,0,-0.6 195 | 193,244726000,31.15826167,122.5261117,252.3,12.9,1556441840,0,0,-0.4 196 | 194,244726000,31.15793,122.5248617,253.1,12.7,1556441859,0,0,0.8 197 | 195,244726000,31.15759,122.5235767,252.6,12.6,1556441880,0,0,-0.5 198 | 196,244726000,31.157225,122.522215,253,12.5,1556441900,0,0,0.4 199 | 197,244726000,31.15674667,122.5204383,253.2,12.1,1556441929,0,0,0.2 200 | 198,244726000,31.1564,122.5192117,251.3,11.9,1556441949,0,0,-1.9 201 | 199,244726000,31.15562333,122.5168383,249.2,11.3,1556441990,0,0,-2.1 202 | 200,244726000,31.15507667,122.51511,249.2,11,1556442020,0,0,0 203 | 201,244726000,31.15458333,122.5135183,251.1,10.8,1556442049,0,0,1.9 204 | 202,244726000,31.15428667,122.51247,251.5,10.7,1556442069,0,0,0.4 205 | 203,244726000,31.15396333,122.511395,250.5,10.6,1556442089,0,0,-1 206 | 204,244726000,31.15333,122.5092467,251.2,10.5,1556442129,0,0,0.7 207 | 205,244726000,31.15300833,122.5081683,250.2,10.5,1556442149,0,0,-1 208 | 206,244726000,31.15239167,122.5060083,253.1,10.6,1556442189,0,0,2.9 209 | 207,244726000,31.15189667,122.5044967,247.5,10.9,1556442219,0,0,-5.6 210 | 208,244726000,31.15105167,122.5031517,228,10.5,1556442246,0,0,-19.5 211 | 209,244726000,31.14994,122.50166,231.2,10.9,1556442279,0,0,3.2 212 | 210,244726000,31.14791,122.4988667,230.1,11.4,1556442339,0,0,-1.1 213 | 211,244726000,31.14584833,122.4959583,229.5,11.7,1556442399,0,0,-0.6 214 | 212,244726000,31.14414167,122.4935017,230,11.7,1556442450,0,0,0.5 215 | 213,244726000,31.141885,122.49002,234.9,11.4,1556442519,0,0,4.9 216 | 214,244726000,31.14028833,122.4876183,230.1,11.1,1556442570,0,0,-4.8 217 | 215,244726000,31.13971833,122.4866417,236.8,10.9,1556442590,0,0,6.7 218 | 216,244726000,31.13865833,122.48474,234.2,10.2,1556442630,0,0,-2.6 219 | 217,244726000,31.13825333,122.484075,235.9,10,1556442650,0,0,1.7 220 | 218,244726000,31.13648833,122.4807867,237.1,9.5,1556442719,0,0,1.2 221 | 219,244726000,31.13539667,122.4785617,239.8,9.2,1556442770,0,0,2.7 222 | 220,244726000,31.13498667,122.4777567,238.2,9.1,1556442789,0,0,-1.6 223 | 221,244726000,31.13421167,122.4760033,244.9,8.7,1556442830,0,0,6.7 224 | 222,244726000,31.13383667,122.4752283,240.6,8.5,1556442848,0,0,-4.3 225 | 223,244726000,31.13314,122.4736267,243,7.9,1556442890,0,0,2.4 226 | 224,244726000,31.13284333,122.472905,243.5,7.7,1556442909,0,0,0.5 227 | 225,244726000,31.13142,122.46935,251.2,6.9,1556443010,0,0,7.7 228 | 226,244726000,31.13004667,122.465165,248.6,6,1556443139,0,0,-2.6 229 | 227,244726000,31.12956333,122.4636533,250.1,5.7,1556443190,0,0,1.5 230 | 228,244726000,31.12910833,122.46227,251.7,5.3,1556443239,0,0,1.6 231 | 229,244726000,31.12898167,122.4617467,255.2,5.3,1556443259,0,0,3.5 232 | 230,244726000,31.1287,122.46064,252.9,5.2,1556443299,0,0,-2.3 233 | 231,244726000,31.12856833,122.460095,253.5,5.3,1556443319,0,0,0.6 234 | 232,244726000,31.12825167,122.4589583,253,5.6,1556443359,0,0,-0.5 235 | 233,244726000,31.12808833,122.4583633,253.4,5.8,1556443379,0,0,0.4 236 | 234,244726000,31.12775833,122.45713,253,6,1556443419,0,0,-0.4 237 | 235,244726000,31.127585,122.4565067,251.7,6.1,1556443439,0,0,-1.3 238 | 236,244726000,31.12739833,122.455815,252.5,6.6,1556443460,0,0,0.8 239 | 237,244726000,31.12719833,122.4551267,251.2,7.1,1556443479,0,0,-1.3 240 | 238,244726000,31.12667667,122.4532333,248.8,8.5,1556443529,0,0,-2.4 241 | 239,244726000,31.12585167,122.4512667,243.1,9.3,1556443569,0,0,-5.7 242 | 240,244726000,31.12376167,122.4465083,242.2,10.2,1556443669,0,0,-0.9 243 | 285,244726000,31.10294333,122.3658517,271.1,10,1556445219,0,0,-0.9 244 | 286,244726000,31.10296167,122.3642933,271.6,9.9,1556445249,0,0,0.5 245 | 287,244726000,31.10299333,122.36318,271.9,9.9,1556445270,0,0,0.3 246 | 288,244726000,31.10304833,122.35998,272.9,9.9,1556445330,0,0,1 247 | 289,244726000,31.10313667,122.3562283,270,9.9,1556445399,0,0,-2.9 248 | 290,244726000,31.103215,122.3535617,272.2,9.9,1556445450,0,0,2.2 249 | 291,244726000,31.10323833,122.35256,271.5,9.9,1556445469,0,0,-0.7 250 | 292,244726000,31.103335,122.349805,272.6,9.9,1556445520,0,0,1.1 251 | 293,244726000,31.10331167,122.347185,268.5,9.8,1556445570,0,0,-4.1 252 | 294,244726000,31.103295,122.34453,269.2,9.8,1556445620,0,0,0.7 253 | 295,244726000,31.10328167,122.3434733,269.1,9.8,1556445640,0,0,-0.1 254 | 296,244726000,31.10328667,122.3424183,271.1,9.8,1556445659,0,0,2 255 | 297,244726000,31.10326,122.339815,269.9,9.9,1556445709,0,0,-1.2 256 | 298,244726000,31.10325,122.33769,270,9.9,1556445749,0,0,0.1 257 | 299,244726000,31.10323,122.33657,270,9.9,1556445770,0,0,0 258 | 300,244726000,31.103225,122.3355533,269.9,9.9,1556445789,0,0,-0.1 259 | 301,244726000,31.10321333,122.3338933,270.5,9.9,1556445820,0,0,0.6 260 | 302,244726000,31.10320333,122.3328283,269.7,9.9,1556445839,0,0,-0.8 261 | 303,244726000,31.103215,122.3312617,271.2,9.9,1556445869,0,0,1.5 262 | 304,244726000,31.103185,122.3296033,268.8,9.9,1556445899,0,0,-2.4 263 | 305,244726000,31.10317333,122.328535,269.5,9.9,1556445920,0,0,0.7 264 | 306,244726000,31.10315333,122.3269217,268.6,9.9,1556445950,0,0,-0.9 265 | 307,244726000,31.103155,122.3259017,268.7,9.9,1556445969,0,0,0.1 266 | -------------------------------------------------------------------------------- /traj_preprocess/ais_segment/example/trajsegment.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import matplotlib.pyplot as plt 3 | from pyais import traj_segment 4 | 5 | 6 | def main(): 7 | trajdata = pd.read_csv('../data/1.csv', encoding="gbk", 8 | usecols=['DRMMSI', 'DRGPSTIME', 'DRLONGITUDE', 'DRLATITUDE']) 9 | scattercolors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm'] 10 | rawgrouped = trajdata[:].groupby('DRMMSI') 11 | for name, group in rawgrouped: 12 | plt.plot(group['DRLONGITUDE'], group['DRLATITUDE'], marker='*', ms=8, linestyle='--', 13 | color='blue', linewidth=0.75) 14 | segmenteddata = traj_segment.segment(trajdata, 1500) 15 | seggrouped = segmenteddata[:].groupby('DRMMSI') 16 | i = 0 17 | for name, group in seggrouped: 18 | i = i + 1 19 | colorSytle = scattercolors[i % len(scattercolors)] 20 | plt.plot(group['DRLONGITUDE'], group['DRLATITUDE'], marker='o', ms=5, linestyle='-', 21 | color=colorSytle, linewidth=0.5) 22 | plt.yticks(fontproperties='Times New Roman', size=14) 23 | plt.xticks(fontproperties='Times New Roman', size=14) 24 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16}) 25 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16}) 26 | plt.title('Trajectory Segmentation', fontdict={'family': 'Times New Roman', 'size': 16}) 27 | plt.ticklabel_format(useOffset=False, style='plain') 28 | plt.grid() 29 | plt.tight_layout() 30 | plt.show() 31 | 32 | 33 | if __name__ == '__main__': 34 | main() 35 | -------------------------------------------------------------------------------- /traj_preprocess/ais_segment/pyais/__pycache__/traj_compress.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_segment/pyais/__pycache__/traj_compress.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_segment/pyais/__pycache__/traj_interpolation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_segment/pyais/__pycache__/traj_interpolation.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_segment/pyais/__pycache__/traj_segment.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_segment/pyais/__pycache__/traj_segment.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_segment/pyais/traj_segment.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import warnings 3 | 4 | warnings.filterwarnings("ignore") 5 | 6 | 7 | def segment(df, tsinterval): 8 | """ 9 | Trajectory segmentation. 10 | :param df: an array 11 | :type df: DataFrame 12 | :param tsepsilon: time threshold 13 | :type tsepsilon: float 14 | :param tdf: return segmented trajectory 15 | :type tdf: DataFrame 16 | """ 17 | tdf = pd.DataFrame(columns=df.columns, index=df.index) 18 | tdf.drop(tdf.index, inplace=True) 19 | tdf['tdiff'] = 0 20 | 21 | for shipmmsi, dt in df.groupby('DRMMSI'): 22 | data = dt.copy(deep=True) 23 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 24 | data = data.reset_index(drop=True) 25 | data['tdiff'] = data['DRGPSTIME'].diff().fillna(0) 26 | i = 0 27 | lastindex = 0 28 | for idx, di in data.iterrows(): 29 | if di['tdiff'] > tsinterval and idx >= 1: 30 | data.loc[lastindex:idx - 1, 'DRMMSI'] = str(di['DRMMSI']) + '_' + str(i) 31 | i = i + 1 32 | lastindex = idx 33 | tdf = tdf.append(data, ignore_index=True) 34 | tdf = tdf.drop(['tdiff'], axis=1) 35 | return tdf -------------------------------------------------------------------------------- /traj_preprocess/ais_segment/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | pandas==1.4.1 3 | -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/example/trajstay.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | from pyais import traj_stay 5 | 6 | 7 | def stay_st_test(traj_file: str): 8 | params = {'axes.titlesize': 'large', 9 | 'xtick.labelsize': 16, 10 | 'ytick.labelsize': 16, 11 | 'legend.fontsize': 16, 12 | 'legend.handlelength': 3} 13 | plt.rcParams.update(params) 14 | 15 | trajdata = traj_stay.stay_st_detect(traj_file) 16 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm'] 17 | for shipmmsi, dt in trajdata.groupby('DRMMSI'): 18 | labels = dt['SP_Status'].values 19 | clusterNum = len(set(labels)) 20 | plt.plot(dt['DRLONGITUDE'].values, dt['DRLATITUDE'].values, marker='o', markeredgewidth=1.0, linewidth=0.75, 21 | markerfacecolor='white', markeredgecolor='m', ms=4, alpha=1.0, color='m', zorder=2) 22 | for i in range(-1, clusterNum): 23 | colorSytle = scatterColors[i % len(scatterColors)] 24 | subCluster = dt.query('SP_Status == @i') 25 | if i >= 0: 26 | plt.scatter(subCluster['DRLONGITUDE'].values, subCluster['DRLATITUDE'].values, marker='*', s=70, 27 | edgecolors=colorSytle, 28 | c='white', linewidths=1.0, zorder=3) 29 | else: 30 | continue 31 | plt.title('Stay Point Identification', fontsize=16) 32 | plt.xlabel('Longitude', fontsize=16) 33 | plt.ylabel('Latitude', fontsize=16) 34 | plt.grid(True) 35 | plt.tight_layout() 36 | plt.show() 37 | 38 | 39 | def stay_spt_test(traj_file: str): 40 | params = {'axes.titlesize': 'large', 41 | 'xtick.labelsize': 14, 42 | 'ytick.labelsize': 14, 43 | 'legend.fontsize': 14, 44 | 'legend.handlelength': 3} 45 | plt.rcParams.update(params) 46 | 47 | trajdata = traj_stay.stay_spt_detect(traj_file) 48 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'm'] 49 | for shipmmsi, dt in trajdata.groupby('DRMMSI'): 50 | plt.plot(dt['DRLONGITUDE'].values, dt['DRLATITUDE'].values, marker='o', markeredgewidth=1.0, linewidth=0.75, 51 | markerfacecolor='white', markeredgecolor='m', ms=4, alpha=1.0, color='m', zorder=2) 52 | lbs = set(dt['SP_Status'].values) 53 | for index, v in enumerate(lbs): 54 | if v > 0: 55 | colorSytle = scatterColors[index % len(scatterColors)] 56 | subCluster = dt.query('SP_Status == @v') 57 | plt.scatter(subCluster['DRLONGITUDE'].values, subCluster['DRLATITUDE'].values, marker='*', s=70, 58 | edgecolors=colorSytle, 59 | c='white', linewidths=1.0, zorder=3) 60 | 61 | plt.title('Stay Point Identification', fontdict={'family': 'Times New Roman', 'size': 16}) 62 | plt.yticks(fontproperties='Times New Roman', size=14) 63 | plt.xticks(fontproperties='Times New Roman', size=14) 64 | plt.ticklabel_format(useOffset=False, style='plain') 65 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16}) 66 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16}) 67 | plt.grid(True) 68 | plt.tight_layout() 69 | plt.show() 70 | 71 | 72 | if __name__ == '__main__': 73 | # stay_spt_test("../data/1.csv") 74 | stay_st_test("../data/1.csv") 75 | -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/__pycache__/traj_clean.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_clean.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/__pycache__/traj_interpolation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_interpolation.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/__pycache__/traj_segment.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_segment.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/__pycache__/traj_stay.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_stay.cpython-38.pyc -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/traj_clean.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from vincenty import vincenty 4 | import warnings 5 | 6 | warnings.filterwarnings("ignore") 7 | 8 | 9 | def heuristic_clean(data, vthreshold: float): 10 | ''' 11 | :param data: raw trajectory 12 | :param vthreshold: speed threshold 13 | :return: cleaned trajectory 14 | ''' 15 | tdf = pd.DataFrame(columns=data.columns, index=data.index) 16 | tdf.drop(tdf.index, inplace=True) 17 | 18 | data.drop_duplicates() 19 | data = data.reset_index(drop=True) 20 | for shipmmsi, dt in data.groupby('DRMMSI'): 21 | if len(str(shipmmsi)) > 5: 22 | dt_copy = dt.copy(deep=True) 23 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 24 | dt_copy = dt_copy.reset_index(drop=True) 25 | dt_copy['NOISE'] = 0 26 | for idx, di in dt_copy.iterrows(): 27 | if di['NOISE'] == 0: 28 | if idx < len(dt_copy) - 1: 29 | t1 = di['DRGPSTIME'] 30 | t2 = dt_copy.loc[idx + 1, 'DRGPSTIME'] 31 | Δt = (t2 - t1) / 3600.0 32 | pt1 = (di['DRLATITUDE'], di['DRLONGITUDE']) 33 | pt2 = (dt_copy.loc[idx + 1, 'DRLATITUDE'], dt_copy.loc[idx + 1, 'DRLONGITUDE']) 34 | Δd = vincenty(pt1, pt2) * 1000 / 1852.25 35 | Δv = Δd / Δt 36 | if Δv > vthreshold: 37 | dt_copy.loc[idx + 1, 'NOISE'] = 1 38 | print("1") 39 | else: 40 | continue 41 | dt_copy_new = dt_copy.query('NOISE==0') 42 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1) 43 | tdf = tdf.append(dt_copy_new, ignore_index=True) 44 | return tdf 45 | 46 | 47 | def sw_clean(data, sw: int): 48 | ''' 49 | :param data: raw trajectory 50 | :param sw: the size of sliding window 51 | :return: cleaned trajectory 52 | ''' 53 | tdf = pd.DataFrame(columns=data.columns, index=data.index) 54 | tdf.drop(tdf.index, inplace=True) 55 | 56 | data.drop_duplicates() 57 | data = data.reset_index(drop=True) 58 | for shipmmsi, dt in data.groupby('DRMMSI'): 59 | if len(str(shipmmsi)) > 6: 60 | dt_copy = dt.copy(deep=True) 61 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 62 | dt_copy = dt_copy.reset_index(drop=True) 63 | dt_copy['NOISE'] = 0 64 | copylength = len(dt_copy) 65 | num_samples = copylength // sw + 1 66 | for idx in np.arange(num_samples): 67 | start_x = sw * idx 68 | end_x = start_x + sw - 1 69 | end_x = (copylength - 1) if end_x > (copylength - 1) else end_x 70 | dt_temp = dt_copy.loc[start_x:end_x] 71 | lats = dt_temp.loc[:, "DRLATITUDE"] 72 | longs = dt_temp.loc[:, "DRLONGITUDE"] 73 | stdlat = lats.std() 74 | meanlat = lats.mean() 75 | stdlog = longs.std() 76 | meanlog = longs.mean() 77 | for jdx, di in dt_temp.iterrows(): 78 | if abs(di['DRLATITUDE'] - meanlat) > abs(1.5 * stdlat) or abs(di['DRLONGITUDE'] - meanlog) > abs( 79 | 1.5 * stdlog): 80 | dt_copy.loc[jdx, 'NOISE'] = 1 81 | print("1") 82 | pass 83 | 84 | dt_copy_new = dt_copy.query('NOISE==0') 85 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1) 86 | tdf = tdf.append(dt_copy_new, ignore_index=True) 87 | return tdf 88 | -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/traj_interpolation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from scipy.interpolate import pchip_interpolate 4 | from typing import Dict, List, Union, Optional 5 | from vincenty import vincenty 6 | 7 | # types 8 | TrajData = Dict[str, Union[List[float]]] 9 | 10 | 11 | def traj_load(traj_file: str) -> TrajData: 12 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []} 13 | df = pd.read_csv(traj_file) 14 | dt_c = df.copy(deep=True) 15 | dt_c['DRLONGITUDE'] = dt_c['DRLONGITUDE'].map(lambda x: x / 1.0) 16 | dt_c['DRLATITUDE'] = dt_c['DRLATITUDE'].map(lambda x: x / 1.0) 17 | dt_c['DRSPEED'] = dt_c['DRSPEED'].map(lambda x: x / 1.0) 18 | for index, row in dt_c.iterrows(): 19 | trajs_data['lat'].append(row['DRLATITUDE']) 20 | trajs_data['lon'].append(row['DRLONGITUDE']) 21 | trajs_data['speed'].append(row['DRSPEED']) 22 | trajs_data['tstamp'].append(row['DRGPSTIME']) 23 | return trajs_data 24 | 25 | 26 | def traj_calculate_distance(traj_data: TrajData) -> List[float]: 27 | traj_dist = np.zeros(len(traj_data['lat'])) 28 | for i in range(len(traj_dist) - 1): 29 | lat1 = traj_data['lat'][i] 30 | lon1 = traj_data['lon'][i] 31 | lat2 = traj_data['lat'][i + 1] 32 | lon2 = traj_data['lon'][i + 1] 33 | pts = (lat1, lon1) 34 | pte = (lat2, lon2) 35 | s = vincenty(pts, pte) * 1000 # unit meter 36 | traj_dist[i + 1] = s 37 | return traj_dist.tolist() 38 | 39 | 40 | def traj_interpolate(traj_file: str, res: float = 1.0, num: Optional[int] = None) -> TrajData: 41 | ''' 42 | :param traj_data: raw trajectory filename 43 | :param res: time resolution 44 | :param num: None 45 | :return: interpolated trajectory 46 | ''' 47 | traj_data = traj_load(traj_file) 48 | if res <= 0.0: 49 | raise ValueError('res must be > 0.0') 50 | if num is not None and num < 0: 51 | raise ValueError('num must be >= 0') 52 | _traj_dist = traj_calculate_distance(traj_data) 53 | xi = np.cumsum(_traj_dist) 54 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]]) 55 | num = num if num is not None else int(np.ceil(xi[-1] / res)) 56 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True) 57 | y = pchip_interpolate(xi, yi, x, axis=1) 58 | traj_data_interp = {'lat': list(y[0, :]), 'lon': list(y[1, :]), 'speed': list(y[2, :]), 'tstamp': list(y[-1, :])} 59 | return traj_data_interp 60 | 61 | 62 | def traj_calculate_distance_ts(traj_data: TrajData) -> List[float]: 63 | traj_dist = np.zeros(len(traj_data['lat'])) 64 | for i in range(len(traj_dist) - 1): 65 | s = int(traj_data['tstamp'][i + 1]) - int(traj_data['tstamp'][i]) 66 | traj_dist[i + 1] = s 67 | return traj_dist.tolist() 68 | 69 | 70 | def traj_interpolate_df(traj_data, res: float = 1.0, num: Optional[int] = None) -> TrajData: 71 | ''' 72 | :param traj_data: raw trajectory dataframe 73 | :param res: time resolution 74 | :param num: None 75 | :return: interpolated trajectory 76 | ''' 77 | if res <= 0.0: 78 | raise ValueError('res must be > 0.0') 79 | if num is not None and num < 0: 80 | raise ValueError('num must be >= 0') 81 | _traj_dist = traj_calculate_distance_ts(traj_data) 82 | xi = np.cumsum(_traj_dist) 83 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]]) 84 | num = num if num is not None else int(np.ceil(xi[-1] / res)) 85 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True) 86 | y = pchip_interpolate(xi, yi, x, axis=1) 87 | return y.T 88 | -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/traj_segment.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import warnings 3 | 4 | warnings.filterwarnings("ignore") 5 | 6 | 7 | def segment(df, tsinterval): 8 | """ 9 | Trajectory segmentation. 10 | :param df: an array 11 | :type df: DataFrame 12 | :param tsepsilon: time threshold 13 | :type tsepsilon: float 14 | :param tdf: return segmented trajectory 15 | :type tdf: DataFrame 16 | """ 17 | tdf = pd.DataFrame(columns=df.columns, index=df.index) 18 | tdf.drop(tdf.index, inplace=True) 19 | tdf['tdiff'] = 0 20 | 21 | for shipmmsi, dt in df.groupby('DRMMSI'): 22 | data = dt.copy(deep=True) 23 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 24 | data = data.reset_index(drop=True) 25 | data['tdiff'] = data['DRGPSTIME'].diff().fillna(0) 26 | i = 0 27 | lastindex = 0 28 | for idx, di in data.iterrows(): 29 | if di['tdiff'] > tsinterval and idx >= 1: 30 | data.loc[lastindex:idx - 1, 'DRMMSI'] = str(di['DRMMSI']) + '_' + str(i) 31 | i = i + 1 32 | lastindex = idx 33 | tdf = tdf.append(data, ignore_index=True) 34 | tdf = tdf.drop(['tdiff'], axis=1) 35 | return tdf -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/pyais/traj_stay.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import time 5 | import math 6 | from scipy.spatial.distance import pdist 7 | from scipy.spatial.distance import squareform 8 | from pyais import traj_clean 9 | from pyais import traj_interpolation 10 | from pyais import traj_segment 11 | 12 | import warnings 13 | 14 | warnings.filterwarnings("ignore") 15 | 16 | 17 | # Part1 18 | 19 | def compute_squared_EDM(X): 20 | return squareform(pdist(X, metric='euclidean')) 21 | 22 | 23 | def ST_DBSCAN(data, eps1: float, eps2: float, minPts: int): 24 | ''' 25 | :param data: interpolated trajectory 26 | :param eps1: spatial neighborhood 27 | :param eps2: time neighborhood 28 | :param minPts: the minimum number of points that satisfy the double-neighborhood 29 | :return: labels 30 | ''' 31 | n, m = data.shape 32 | timeDisMat = compute_squared_EDM(data[:, 0].reshape(n, 1)) 33 | disMat = compute_squared_EDM(data[:, 1:]) 34 | core_points_index = np.where(np.sum(np.where((disMat <= eps1) & (timeDisMat <= eps2), 1, 0), axis=1) >= minPts)[0] 35 | labels = np.full((n,), -1) 36 | clusterId = 0 37 | for pointId in core_points_index: 38 | if (labels[pointId] == -1): 39 | labels[pointId] = clusterId 40 | neighbour = np.where((disMat[:, pointId] <= eps1) & (timeDisMat[:, pointId] <= eps2) & (labels == -1))[0] 41 | seeds = set(neighbour) 42 | while len(seeds) > 0: 43 | newPoint = seeds.pop() 44 | labels[newPoint] = clusterId 45 | queryResults = set(np.where((disMat[:, newPoint] <= eps1) & (timeDisMat[:, newPoint] <= eps2))[0]) 46 | if len(queryResults) >= minPts: 47 | for resultPoint in queryResults: 48 | if labels[resultPoint] == -1: 49 | seeds.add(resultPoint) 50 | clusterId = clusterId + 1 51 | return labels 52 | 53 | 54 | def stay_st_detect(traj_file: str): 55 | ''' 56 | :param traj_data: raw trajectory filename 57 | :return: interpolated trajectory,labels 58 | ''' 59 | trajdf = pd.read_csv(traj_file, encoding='gbk') 60 | segdf = traj_segment.segment(trajdf, 1500) 61 | cleandf = traj_clean.heuristic_clean(segdf, 25) 62 | 63 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index) 64 | tdf.drop(tdf.index, inplace=True) 65 | tdf['SP_Status'] = 0 66 | 67 | for shipmmsi, dt in cleandf.groupby('DRMMSI'): 68 | data = dt.copy(deep=True) 69 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 70 | data = data.reset_index(drop=True) 71 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []} 72 | for index, row in data.iterrows(): 73 | trajs_data['lat'].append(row['DRLATITUDE']) 74 | trajs_data['lon'].append(row['DRLONGITUDE']) 75 | trajs_data['speed'].append(row['DRSPEED']) 76 | trajs_data['tstamp'].append(row['DRGPSTIME']) 77 | if len(trajs_data['lon']) < 4: 78 | continue 79 | res = 30.0 80 | my_traj_data_interp = traj_interpolation.traj_interpolate_df(trajs_data, res, None) 81 | dataDf = pd.DataFrame(my_traj_data_interp) 82 | dataDf.columns = ['lat', 'lon', 'spd', 'ts'] 83 | dataDf['ts'] = dataDf['ts'].map(lambda x: int(x)) 84 | datanew = dataDf[['ts', 'lat', 'lon']].values 85 | 86 | minpts = round(np.log(len(datanew))) 87 | if minpts < 4: 88 | minpts = 4 89 | ep2 = minpts / 2 * res 90 | ep1 = 2.2 * ep2 / 3600.0 / 60.0 91 | tj = pd.DataFrame(datanew) 92 | tj.columns = ['DRGPSTIME', 'DRLATITUDE', 'DRLONGITUDE'] 93 | labels = ST_DBSCAN(datanew, ep1, ep2, minpts) 94 | tj['SP_Status'] = labels 95 | tj['DRMMSI'] = shipmmsi 96 | tj['DRGPSTIME'] = tj['DRGPSTIME'].map(lambda x: int(x)) 97 | tdf = tdf.append(tj, ignore_index=True) 98 | tdf = tdf.fillna(0) 99 | return tdf 100 | 101 | 102 | # Part2 103 | def rad(d): 104 | return float(d) * math.pi / 180.0 105 | 106 | 107 | EARTH_RADIUS = 6378.137 108 | 109 | 110 | def GetDistance(lng1, lat1, lng2, lat2): 111 | radLat1 = rad(lat1) 112 | radLat2 = rad(lat2) 113 | a = radLat1 - radLat2 114 | b = rad(lng1) - rad(lng2) 115 | s = 2 * math.asin( 116 | math.sqrt(math.pow(math.sin(a / 2), 2) + math.cos(radLat1) * math.cos(radLat2) * math.pow(math.sin(b / 2), 2))) 117 | s = s * EARTH_RADIUS 118 | s = round(s * 10000, 2) / 10 119 | return s 120 | 121 | 122 | def ka(dis, dc): 123 | if (dis >= dc): 124 | return 0 125 | else: 126 | return 1 127 | 128 | 129 | def density(data, dc): 130 | dc = float(dc) 131 | latitude = list(data['DRLATITUDE']) 132 | longitude = list(data['DRLONGITUDE']) 133 | part_density = [] # 存储局部密度值 134 | scope = [] # 记录每个点计算局部密度的范围 135 | leftBoundary = 0 136 | rightBoundary = len(data) - 1 # 左边界与右边界 137 | for i in range(len(data)): 138 | trigger = True 139 | left = i - 1 140 | right = i + 1 141 | incrementLeft = 1 142 | incrementRight = 1 143 | while trigger: 144 | # 向左拓展 145 | if incrementLeft != 0: 146 | if left < 0: 147 | left = leftBoundary 148 | distanceLeft = GetDistance(longitude[left], latitude[left], longitude[i], latitude[i]) 149 | if (distanceLeft < dc) & (left > leftBoundary): 150 | left -= 1 151 | else: 152 | incrementLeft = 0 153 | # 向右拓展 154 | if incrementRight != 0: 155 | if right > rightBoundary: 156 | right = rightBoundary 157 | distanceRight = GetDistance(longitude[i], latitude[i], longitude[right], latitude[right]) 158 | if (distanceRight < dc) & (right < rightBoundary): 159 | right += 1 160 | else: 161 | incrementRight = 0 162 | # 若左右都停止了拓展，此点的局部密度计算结束 163 | if (incrementLeft == 0) & (incrementRight == 0): 164 | trigger = False 165 | if (left == leftBoundary) & (incrementRight == 0): 166 | trigger = False 167 | if (incrementLeft == 0) & (right == rightBoundary): 168 | trigger = False 169 | if left == leftBoundary: 170 | scope.append([left, right - 1]) 171 | part_density.append(right - left - 1) 172 | elif right == rightBoundary: 173 | scope.append([left + 1, right]) 174 | part_density.append(right - left - 1) 175 | else: 176 | scope.append([left + 1, right - 1]) 177 | part_density.append(right - left - 2) 178 | return part_density, scope 179 | 180 | 181 | # 反向更新的方法 182 | def SP_search(data, tc): 183 | tc = int(tc) 184 | SP = [] 185 | part_density = copy.deepcopy(list(data['part_density'])) 186 | scope = copy.deepcopy(list(data['scope'])) 187 | latitude = copy.deepcopy(list(data['DRLATITUDE'])) 188 | longitude = copy.deepcopy(list(data['DRLONGITUDE'])) 189 | timestamp = copy.deepcopy(list(data['DRGPSTIME'])) 190 | trigger = True 191 | used = [] 192 | while trigger: 193 | partD = max(part_density) 194 | index = part_density.index(partD) 195 | print('index:', index) 196 | start = scope[index][0] 197 | end = scope[index][1] 198 | if len(used) != 0: 199 | for i in used: 200 | if (scope[i][0] > start) & (scope[i][0] < end): 201 | part_density[index] = scope[i][0] - start - 1 202 | scope[index][1] = scope[i][0] - 1 203 | print("1_1") 204 | if (scope[i][1] > start) & (scope[i][1] < end): 205 | part_density[index] = end - scope[i][1] - 1 206 | scope[index][0] = scope[i][1] + 1 207 | print("1_2") 208 | if (scope[i][0] <= start) & (scope[i][1] >= end): 209 | part_density[index] = 0 210 | scope[index][0] = 0; 211 | scope[index][1] = 0 212 | print("1_3") 213 | start = scope[index][0]; 214 | end = scope[index][1] 215 | timeCross = timestamp[end] - timestamp[start] 216 | print('time:', timeCross) 217 | if timeCross > tc: 218 | SarvT = time.localtime(timestamp[start]) 219 | SlevT = time.localtime(timestamp[end]) 220 | SP.append([index, latitude[index], longitude[index], SarvT, SlevT, scope[index]]) 221 | used.append(index) 222 | for k in range(scope[index][0], scope[index][1] + 1): 223 | part_density[k] = 0 224 | part_density[index] = 0 225 | if max(part_density) == 0: 226 | trigger = False 227 | return SP 228 | 229 | 230 | # 通过代表的距离是否小于距离阈值来大致判断停留点是否一致 231 | def similar(sp, dc): 232 | dc = float(dc) 233 | latitude = copy.deepcopy(list(sp['latitude'])) 234 | longitude = copy.deepcopy(list(sp['longitude'])) 235 | i = 0; 236 | index = list(sp.index) 237 | for i in index: 238 | for j in index: 239 | if i != j: 240 | dist = GetDistance(longitude[i], latitude[i], longitude[j], latitude[j]) 241 | if dist < 1.5 * dc: 242 | sp = sp.drop(j, axis=0) 243 | index.remove(j) 244 | return sp 245 | 246 | 247 | def stay_spt_detect(traj_file: str): 248 | ''' 249 | :param traj_data: raw trajectory filename 250 | :return: interpolated trajectory,labels 251 | ''' 252 | trajdf = pd.read_csv(traj_file, encoding='gbk') 253 | segdf = traj_segment.segment(trajdf, 1500) 254 | cleandf = traj_clean.heuristic_clean(segdf, 25) 255 | 256 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index) 257 | tdf.drop(tdf.index, inplace=True) 258 | tdf['SP_Status'] = 0 259 | 260 | for shipmmsi, dt in cleandf.groupby('DRMMSI'): 261 | if len(str(shipmmsi)) > 5: 262 | if len(dt) > 10: 263 | data = dt.copy(deep=True) 264 | data = data.reset_index(drop=True) 265 | sj = data.copy(deep=True) 266 | sj['SP_Status'] = 0 267 | 268 | dc = 400 269 | tc = 600 270 | data['part_density'], data['scope'] = density(data, dc) 271 | SP = SP_search(data, tc) 272 | output = pd.DataFrame(SP) 273 | 274 | if output.empty == False: 275 | output.columns = ['index', 'longitude', 'latitude', 'arriveTime', 'leftTime', 'scope'] 276 | output = similar(output, dc) 277 | 278 | for sco in output['scope']: 279 | start, end = sco[0], sco[1] 280 | tcog = sj['DRDIRECTION'][start:end] 281 | ss = 0 # 1靠泊、2锚泊 282 | if tcog.var() < 800: 283 | ss = 1 284 | else: 285 | ss = 2 286 | sj.loc[start:end, 'SP_Status'] = ss 287 | tdf = tdf.append(sj, ignore_index=True) 288 | return tdf 289 | -------------------------------------------------------------------------------- /traj_preprocess/ais_stay/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.1 2 | numpy==1.23.0 3 | pandas==1.4.1 4 | scipy==1.8.1 5 | vincenty==0.1.4 6 | -------------------------------------------------------------------------------- /traj_show/__pycache__/folium.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_show/__pycache__/folium.cpython-37.pyc -------------------------------------------------------------------------------- /traj_show/requirements.txt: -------------------------------------------------------------------------------- 1 | folium==0.12.1.post1 2 | matplotlib==3.5.1 3 | numpy==1.21.6 4 | pandas==1.3.5 5 | -------------------------------------------------------------------------------- /traj_show/trajshow.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import numpy as np 4 | import webbrowser as wb 5 | import folium 6 | from folium.plugins import HeatMap, MiniMap, MarkerCluster 7 | 8 | 9 | # draw a heatmap 10 | def draw_heatmap(map): 11 | data = ( 12 | np.random.normal(size=(100, 3)) * 13 | np.array([[1, 1, 1]]) + 14 | np.array([[30.9, 122.52, 1]]) 15 | ).tolist() 16 | HeatMap(data).add_to(map) 17 | 18 | 19 | # add minimap 20 | def draw_minimap(map): 21 | minimap = MiniMap(toggle_display=True, 22 | tile_layer='Stamen Watercolor', 23 | position='topleft', 24 | width=100, 25 | height=100) 26 | map.add_child(minimap) 27 | 28 | 29 | def draw_circlemarker(loc, spd, cog, map): 30 | tip = 'Coordinates:' + str(loc) + "\t" + 'Speed:' + str(spd) + '\t' + 'COG:' + str(cog) 31 | folium.CircleMarker( 32 | location=loc, 33 | radius=3.6, 34 | color="blueviolet", 35 | stroke=True, 36 | fill_color='white', 37 | fill=True, 38 | weight=1.5, 39 | fill_opacity=1.0, 40 | opacity=1, 41 | tooltip=tip 42 | ).add_to(map) 43 | 44 | 45 | # draw a small information marker on the map 46 | def draw_icon(map, loc): 47 | mk = folium.features.Marker(loc) 48 | pp = folium.Popup(str(loc)) 49 | ic = folium.features.Icon(color="blue") 50 | mk.add_child(ic) 51 | mk.add_child(pp) 52 | map.add_child(mk) 53 | 54 | 55 | # draw a stop marker on the map 56 | def draw_stop_icon(map, loc): 57 | # mk = folium.features.Marker(loc) 58 | # pp = folium.Popup(str(loc)) 59 | # ic = folium.features.Icon(color='red', icon='anchor', prefix='fa') 60 | # mk.add_child(ic) 61 | # mk.add_child(pp) 62 | # map.add_child(mk) 63 | folium.Marker(loc).add_to(map) 64 | 65 | 66 | def draw_line(map, loc1, loc2): 67 | kw = {"opacity": 1.0, "weight": 6} 68 | folium.PolyLine( 69 | smooth_factor=10, 70 | locations=[loc1, loc2], 71 | color="red", 72 | tooltip="Trajectory", 73 | **kw, 74 | ).add_to(map) 75 | 76 | 77 | def draw_lines(map, coordinates, c): 78 | folium.PolyLine( 79 | smooth_factor=0, 80 | locations=coordinates, 81 | color=c, 82 | weight=0.5 83 | ).add_to(map) 84 | 85 | 86 | # save the result as HTML to the specified path 87 | def open_html(map, htmlpath): 88 | map.save(htmlpath) 89 | search_text = 'cdn.jsdelivr.net' 90 | replace_text = 'gcore.jsdelivr.net' 91 | with open(htmlpath, 'r', encoding='UTF-8') as file: 92 | data = file.read() 93 | data = data.replace(search_text, replace_text) 94 | with open(htmlpath, 'w', encoding='UTF-8') as file: 95 | file.write(data) 96 | chromepath = "C:\Program Files (x86)\Google\Chrome\Application\chrome.exe" 97 | wb.register('chrome', None, wb.BackgroundBrowser(chromepath)) 98 | wb.get('chrome').open(htmlpath, autoraise=1) 99 | 100 | 101 | # read .csv file 102 | def read_traj_data(path): 103 | P = pd.read_csv(path, dtype={'DRLATITUDE': float, 'DRLONGITUDE': float}) 104 | locations_total = P.loc[:, ['DRLATITUDE', 'DRLONGITUDE']].values.tolist() 105 | speed_total = P.loc[:, ['DRSPEED']].values.tolist() 106 | cog_total = P.loc[:, ['DRDIRECTION']].values.tolist() 107 | locations_stay = P.loc[P['STATUS'] == 1, ['DRLATITUDE', 'DRLONGITUDE']].values.tolist() 108 | lct = [P['DRLATITUDE'].mean(), P['DRLONGITUDE'].mean()] 109 | return locations_total, speed_total, cog_total, locations_stay, lct 110 | 111 | 112 | def draw_single_traj(csv_path): 113 | ''' 114 | draw a single trajectory 115 | :param data: file path 116 | :return: null 117 | ''' 118 | locations, spds, cogs, stays, ct = read_traj_data(csv_path) 119 | m = folium.Map(ct, zoom_start=15, attr='default') # 中心区域的确定 120 | folium.PolyLine( # polyline方法为将坐标用实线形式连接起来 121 | locations, # 将坐标点连接起来 122 | weight=1.0, # 线的大小为1 123 | color='blueviolet', # 线的颜色 124 | opacity=0.8, # 线的透明度 125 | ).add_to(m) # 将这条线添加到刚才的区域map内 126 | num = len(locations) 127 | for i in range(num): 128 | draw_circlemarker(locations[i], spds[i], cogs[i], m) 129 | for i in iter(stays): 130 | draw_stop_icon(m, i) 131 | output_path = os.getcwd() + './draw/show.html' 132 | open_html(m, output_path) 133 | 134 | 135 | def draw_trajs(file_path): 136 | ''' 137 | draw multiple trajectories 138 | :param data: file path 139 | :return: null 140 | ''' 141 | map = folium.Map([31.1, 122.5], zoom_start=10, attr='default') # 中心区域的确定 142 | draw_minimap(map) 143 | fls = os.listdir(file_path) 144 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm'] 145 | i = 0 146 | for x in fls: 147 | i = i + 1 148 | colorSytle = scatterColors[i % len(scatterColors)] 149 | df = pd.read_csv(file_path + "/" + x, encoding="gbk") 150 | df['DRMMSI'] = df['DRMMSI'].apply(lambda _: str(_)) 151 | df['DRLONGITUDE'] = df['DRLONGITUDE'].map(lambda x: x / 1.0) 152 | df['DRLATITUDE'] = df['DRLATITUDE'].map(lambda x: x / 1.0) 153 | for shipmmsi, dt in df.groupby('DRMMSI'): 154 | if len(dt) > 2: 155 | dt_copy = dt.copy(deep=True) 156 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True) 157 | locations = dt_copy.loc[:, ['DRLATITUDE', 'DRLONGITUDE']].values.tolist() 158 | draw_lines(map, locations, colorSytle) 159 | output_path = os.getcwd() + './draw/show.html' 160 | open_html(map, output_path) 161 | 162 | 163 | if __name__ == '__main__': 164 | csv_path = r'./data/1.csv' 165 | draw_single_traj(csv_path) 166 | # csv_path = r'./data' 167 | # draw_trajs(csv_path) 168 | -------------------------------------------------------------------------------- /traj_show/trajshow_matplotlib.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | 5 | 6 | def load_data(file_path): 7 | """ 8 | import trajectory data 9 | :return: trajectory data 10 | """ 11 | data = pd.read_csv(file_path, usecols=['DRGPSTIME', 'DRMMSI', 'DRLATITUDE', 'DRLONGITUDE']) 12 | data.rename(columns={'DRLONGITUDE': 'long', 'DRLATITUDE': 'lat', 'DRGPSTIME': 't', 'DRMMSI': 'mmsi'}, inplace=True) 13 | data['long'] = data['long'].map(lambda x: x / 600000.0) 14 | data['lat'] = data['lat'].map(lambda x: x / 600000.0) 15 | return data 16 | 17 | 18 | if __name__ == '__main__': 19 | trajectories = load_data('./data/1.csv') 20 | params = {'axes.titlesize': 'large', 21 | 'legend.fontsize': 14, 22 | 'legend.handlelength': 3} 23 | plt.rcParams.update(params) 24 | 25 | for shipmmsi, dt in trajectories.groupby('mmsi'): 26 | plt.plot(dt['long'].values, dt['lat'].values, color='green', linewidth=0.5) 27 | 28 | plt.yticks(fontproperties='Times New Roman', size=12) 29 | plt.xticks(fontproperties='Times New Roman', size=12) 30 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14}) 31 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14}) 32 | plt.title('Preprocessed Trajectories', fontdict={'family': 'Times New Roman', 'size': 14}) 33 | plt.ticklabel_format(useOffset=False, style='plain') 34 | plt.grid(True) 35 | plt.tight_layout() 36 | plt.show() 37 | -------------------------------------------------------------------------------- /user guide/Developer's Guide.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/user guide/Developer's Guide.pdf --------------------------------------------------------------------------------