├── LICENSE
├── README.md
├── traj_management
├── traj_chunk_read
│ ├── data
│ │ └── 1.csv
│ ├── example
│ │ └── aisread.py
│ ├── pyais
│ │ ├── __pycache__
│ │ │ └── chunk_read.cpython-38.pyc
│ │ └── chunk_read.py
│ └── requirements.txt
└── traj_similarity
│ ├── data
│ └── 1.csv
│ ├── requirements.txt
│ └── trajsimilarity.py
├── traj_mining
├── traffic_flow_predict
│ ├── data
│ │ ├── flowtc.csv
│ │ ├── flowtj.csv
│ │ └── result.csv
│ ├── example.py
│ ├── generate_dataset.py
│ ├── lstm_encoder_decoder.py
│ ├── model
│ │ └── lstm-ed.pt
│ ├── plotting.py
│ └── requirements.txt
├── traj_anchor_berth
│ ├── data
│ │ └── 1.csv
│ ├── pyais
│ │ ├── __pycache__
│ │ │ ├── cleaning.cpython-38.pyc
│ │ │ ├── geotostr.cpython-38.pyc
│ │ │ ├── plotting.cpython-38.pyc
│ │ │ ├── segmentation.cpython-38.pyc
│ │ │ ├── traj_clean.cpython-38.pyc
│ │ │ ├── traj_interpolation.cpython-38.pyc
│ │ │ ├── traj_segment.cpython-38.pyc
│ │ │ └── traj_stay.cpython-38.pyc
│ │ ├── geotostr.py
│ │ ├── ptinpolygon.py
│ │ ├── traj_clean.py
│ │ ├── traj_interpolation.py
│ │ ├── traj_segment.py
│ │ └── traj_stay.py
│ ├── requirements.txt
│ └── trajanchorberth.py
└── traj_cluster
│ ├── hdbscan
│ └── myhdbscan.py
│ └── requirements.txt
├── traj_preprocess
├── ais_clean
│ ├── data
│ │ └── 1.csv
│ ├── example
│ │ └── trajclean.py
│ ├── pyais
│ │ ├── __pycache__
│ │ │ ├── chunk_read.cpython-38.pyc
│ │ │ └── traj_clean.cpython-38.pyc
│ │ └── traj_clean.py
│ └── requirements.txt
├── ais_compress
│ ├── data
│ │ └── 1.csv
│ ├── example
│ │ └── trajcompress.py
│ ├── pyais
│ │ ├── __pycache__
│ │ │ ├── traj_compress.cpython-38.pyc
│ │ │ └── traj_interpolation.cpython-38.pyc
│ │ └── traj_compress.py
│ └── requirements.txt
├── ais_encoder_decoder
│ ├── examples
│ │ ├── binary_payload.py
│ │ ├── communication_state.py
│ │ ├── decode.py
│ │ ├── encode_dict.py
│ │ ├── encode_msg.py
│ │ ├── file_stream.py
│ │ ├── msg_to_csv.py
│ │ ├── out_of_order.py
│ │ ├── sample.ais
│ │ └── single_message.py
│ ├── pyais
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ ├── __init__.cpython-38.pyc
│ │ │ ├── constants.cpython-38.pyc
│ │ │ ├── decode.cpython-38.pyc
│ │ │ ├── encode.cpython-38.pyc
│ │ │ ├── exceptions.cpython-38.pyc
│ │ │ ├── messages.cpython-38.pyc
│ │ │ ├── stream.cpython-38.pyc
│ │ │ └── util.cpython-38.pyc
│ │ ├── ais_types.py
│ │ ├── constants.py
│ │ ├── decode.py
│ │ ├── encode.py
│ │ ├── exceptions.py
│ │ ├── messages.py
│ │ ├── stream.py
│ │ └── util.py
│ └── requirements.txt
├── ais_interpolate
│ ├── data
│ │ └── 1.csv
│ ├── example
│ │ └── trajinterpolation.py
│ ├── pyais
│ │ ├── __pycache__
│ │ │ └── traj_interpolation.cpython-38.pyc
│ │ └── traj_interpolation.py
│ └── requirements.txt
├── ais_kalman_filter
│ ├── data
│ │ └── 1.csv
│ ├── example
│ │ └── trajkalmanfilter.py
│ ├── pyais
│ │ ├── __pycache__
│ │ │ ├── traj_clean.cpython-38.pyc
│ │ │ ├── traj_interpolation.cpython-38.pyc
│ │ │ ├── traj_kalmanfilter.cpython-38.pyc
│ │ │ ├── traj_segment.cpython-38.pyc
│ │ │ └── traj_stay.cpython-38.pyc
│ │ └── traj_kalmanfilter.py
│ └── requirements.txt
├── ais_segment
│ ├── data
│ │ └── 1.csv
│ ├── example
│ │ └── trajsegment.py
│ ├── pyais
│ │ ├── __pycache__
│ │ │ ├── traj_compress.cpython-38.pyc
│ │ │ ├── traj_interpolation.cpython-38.pyc
│ │ │ └── traj_segment.cpython-38.pyc
│ │ └── traj_segment.py
│ └── requirements.txt
└── ais_stay
│ ├── data
│ └── 1.csv
│ ├── example
│ └── trajstay.py
│ ├── pyais
│ ├── __pycache__
│ │ ├── traj_clean.cpython-38.pyc
│ │ ├── traj_interpolation.cpython-38.pyc
│ │ ├── traj_segment.cpython-38.pyc
│ │ └── traj_stay.cpython-38.pyc
│ ├── traj_clean.py
│ ├── traj_interpolation.py
│ ├── traj_segment.py
│ └── traj_stay.py
│ └── requirements.txt
├── traj_show
├── __pycache__
│ └── folium.cpython-37.pyc
├── data
│ └── 1.csv
├── draw
│ └── show.html
├── requirements.txt
├── trajshow.py
└── trajshow_matplotlib.py
└── user guide
└── Developer's Guide.pdf
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 leeyeah
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # PyVT: A Toolkit for Preprocessing and Analysis of Vessel Spatio-Temporal Trajectories
2 |
3 | ## Introduction
4 |
5 | PyVT, an open-source Python toolkit for preprocessing and analysis of vessel spatio-temporal trajectories. The toolkit realizes four functions: trajectory preprocessing, trajectory data management, trajectory mining, and trajectory visualization. The toolkit is simple to use, provides satisfactory default parameter settings, and makes it easy for basic users to use and replicate its functionality in proprietary applications. More advanced users can further adjust the functions to meet their needs by optimizing parameters. At the same time, its design focuses on composability and reusability, making it functional modular and maintainable, allowing for further development.
6 |

7 |
8 | ## Requirements
9 |
10 | PyVT is developed based on Python 3. The environment that each module depends on, see the requirements.txt.
11 |
12 | ## Developer's guide
13 | See the "Developer's Guide"
14 |
15 | ## License
16 | MIT
17 |
--------------------------------------------------------------------------------
/traj_management/traj_chunk_read/example/aisread.py:
--------------------------------------------------------------------------------
1 | from pyais import chunk_read
2 |
3 | if __name__ == '__main__':
4 | # chunk_read.read_ais_data_region(input="../data/1.csv", output='../data/test.csv') # 此处填写csv路径
5 | chunk_read.read_ais_data_mmsi(input="../data/1.csv", output='../data/test.csv', mmsi='244726000') # 此处填写csv路径
6 |
--------------------------------------------------------------------------------
/traj_management/traj_chunk_read/pyais/__pycache__/chunk_read.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_management/traj_chunk_read/pyais/__pycache__/chunk_read.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_management/traj_chunk_read/pyais/chunk_read.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import time
3 |
4 |
5 | def suply10(x):
6 | return x * 10.0
7 |
8 |
9 | def read_ais_data_region(input: str, output: str, logleftup: float = 122.0, logrightdown: float = 123.0,
10 | latleftup: float = 32.0, latrightdown: float = 31.0, dtst: str = '2019-01-01 00:00:01',
11 | dtend: str = '2020-01-01 00:00:01'):
12 | '''
13 | :param input: filename of the input file
14 | :param output: filename of the output file
15 | :param logleftup: longitude of upper left corner
16 | :param logrightdown: longitude of lower right corner
17 | :param latleftup: longitude of upper left corner
18 | :param latrightdown: latitude of lower right corner
19 | :param dtst: start time
20 | :param dtend: end time
21 | :return: null
22 | '''
23 | cols = read_data_nm(input, 5)
24 | inputfile = open(input, 'rb')
25 | data = pd.read_csv(inputfile, sep=',', iterator=True)
26 | timearrayst = time.strptime(dtst, "%Y-%m-%d %H:%M:%S")
27 | timearrayend = time.strptime(dtend, "%Y-%m-%d %H:%M:%S")
28 | timestampst = time.mktime(timearrayst) # timestamp
29 | timestampend = time.mktime(timearrayend)
30 | loop = True
31 | chunkSize = 10000 # The number of reads per chunk
32 | chunks = []
33 | while loop:
34 | try:
35 | chunk = data.get_chunk(chunkSize)
36 | chunk.columns = cols
37 | chunked = chunk[(chunk['DRGPSTIME'] > timestampst) & (chunk['DRGPSTIME'] < timestampend) & (
38 | chunk['DRLONGITUDE'] > logleftup) & (chunk['DRLONGITUDE'] < logrightdown) & (
39 | chunk['DRLATITUDE'] < latleftup) & (chunk['DRLATITUDE'] > latrightdown)]
40 | cn = chunked.fillna(value=0)
41 | # cn['DRTRUEHEADING'] = cn['DRTRUEHEADING'].transform(suply10)
42 | chunks.append(cn)
43 | except StopIteration:
44 | loop = False
45 | print("Iteration is stopped.")
46 | rdata = pd.concat(chunks, ignore_index=True)
47 | rdata.to_csv(output, index=False) # 输出csv
48 | print("Finished.")
49 | return
50 |
51 |
52 | def read_ais_data_callsign(input: str, output: str, callsign: str = ''):
53 | '''
54 | :param input: filename of the input file
55 | :param output: filename of the output file
56 | :param callsign: CALLSIGN
57 | :return: null
58 | '''
59 | cols = read_data_nm(input, 5)
60 | inputfile = open(input, 'rb')
61 | data = pd.read_csv(inputfile, sep=',', iterator=True)
62 | loop = True
63 | chunkSize = 10000 # The number of reads per chunk
64 | chunks = []
65 | while loop:
66 | try:
67 | chunk = data.get_chunk(chunkSize)
68 | chunk.columns = cols
69 | chunk['CALLSIGN'] = chunk['CALLSIGN'].map(lambda x: str(x))
70 | chunked = chunk[(str(chunk['CALLSIGN']) == callsign)]
71 | cn = chunked.fillna(value=0)
72 | chunks.append(cn)
73 | except StopIteration:
74 | loop = False
75 | print("Iteration is stopped.")
76 | rdata = pd.concat(chunks, ignore_index=True)
77 | rdata.to_csv(output, index=False) # 输出csv
78 | print("Finished.")
79 | return
80 |
81 |
82 | def read_ais_data_mmsi(input: str, output: str, mmsi: str = ''):
83 | '''
84 | :param input: filename of the input file
85 | :param output: filename of the output file
86 | :param mmsi: DRMMSI
87 | :return: null
88 | '''
89 | cols = read_data_nm(input, 5)
90 | inputfile = open(input, 'rb')
91 | data = pd.read_csv(inputfile, sep=',', iterator=True)
92 | loop = True
93 | chunkSize = 10000 # The number of reads per chunk
94 | chunks = []
95 | while loop:
96 | try:
97 | chunk = data.get_chunk(chunkSize)
98 | chunk.columns = cols
99 | chunk['DRMMSI'] = chunk['DRMMSI'].map(lambda x: str(x))
100 | chunked = chunk[(chunk['DRMMSI'] == mmsi)]
101 | cn = chunked.fillna(value=0)
102 | chunks.append(cn)
103 | except StopIteration:
104 | loop = False
105 | print("Iteration is stopped.")
106 | rdata = pd.concat(chunks, ignore_index=True)
107 | rdata.to_csv(output, index=False) # 输出csv
108 | print("Finished.")
109 | return
110 |
111 |
112 | def read_data_nm(input_file, nm):
113 | '''
114 | The function is to print the first * line of the file
115 | :param input_file: filename
116 | :param nm: row number
117 | :return: null
118 | '''
119 | pd.set_option('display.max_columns', None)
120 | pd.set_option('display.max_rows', None)
121 | pd.set_option('max_colwidth', 100)
122 | data = pd.read_csv(input_file, nrows=nm)
123 | print(data)
124 | return data.columns
125 |
--------------------------------------------------------------------------------
/traj_management/traj_chunk_read/requirements.txt:
--------------------------------------------------------------------------------
1 | pandas==1.4.1
2 |
--------------------------------------------------------------------------------
/traj_management/traj_similarity/requirements.txt:
--------------------------------------------------------------------------------
1 | dtaidistance==2.3.9
2 | fastdtw==0.3.4
3 | haversine==2.6.0
4 | numpy==1.23.0
5 | pandas==1.4.3
6 |
--------------------------------------------------------------------------------
/traj_management/traj_similarity/trajsimilarity.py:
--------------------------------------------------------------------------------
1 | from dtaidistance import dtw
2 | from dtaidistance import dtw_ndim
3 | import pandas as pd
4 | import numpy as np
5 | from fastdtw import fastdtw
6 | from haversine.haversine import haversine
7 |
8 |
9 | def load_data(file_path):
10 | """
11 | import trajectory data
12 | :return: trajectory data
13 | """
14 | data = pd.read_csv(file_path, usecols=['DRGPSTIME', 'DRMMSI', 'DRLATITUDE', 'DRLONGITUDE'])
15 | data.rename(columns={'DRLONGITUDE': 'long', 'DRLATITUDE': 'lat', 'DRGPSTIME': 't', 'DRMMSI': 'mmsi'}, inplace=True)
16 | trajectories = []
17 | grouped = data[:].groupby('mmsi')
18 | for name, group in grouped:
19 | if len(group) > 1:
20 | group = group.sort_values(by='t', ascending=True)
21 | group['long'] = group['long'].apply(lambda x: x * (1 / 1.0)) # 以°为单位
22 | group['lat'] = group['lat'].apply(lambda x: x * (1 / 1.0))
23 | loc = group[['lat', 'long']].values # 原始
24 | trajectories.append(loc)
25 | return trajectories
26 |
27 |
28 | def dist_computer(trajectories):
29 | distance_matrix = dtw_ndim.distance_matrix_fast(trajectories, 2)
30 | distance_matrix = np.array(distance_matrix)
31 | np.savetxt('1.txt', distance_matrix, fmt='%.5f')
32 | return distance_matrix
33 |
34 |
35 | def dist_computer_fastdtw(trajectories):
36 | distance_matrix = []
37 | i = 0
38 | l = len(trajectories)
39 | for traj in trajectories:
40 | i = i + 1
41 | print(str(i) + "/" + str(l))
42 | v = []
43 | for q in trajectories:
44 | distance, path = fastdtw(traj, q, dist=haversine)
45 | v.append(distance)
46 | distance_matrix.append(v)
47 | distance_matrix = np.array(distance_matrix)
48 | np.savetxt('1.txt', distance_matrix, fmt='%.5f')
49 | return distance_matrix
50 |
51 |
52 | if __name__ == '__main__':
53 | trajectories = load_data('./data/1.csv')
54 | dist_matrix = dist_computer(trajectories)
55 |
--------------------------------------------------------------------------------
/traj_mining/traffic_flow_predict/example.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import numpy as np
3 | import matplotlib
4 | import matplotlib.pyplot as plt
5 | from importlib import reload
6 | from matplotlib.ticker import MaxNLocator
7 | import generate_dataset
8 | import lstm_encoder_decoder
9 | import plotting
10 | import torch
11 |
12 | matplotlib.rcParams.update({'font.size': 14})
13 | print(torch.cuda.is_available())
14 |
15 | # generate dataset for LSTM
16 | t, y, xorigin, yorigin = generate_dataset.read_data('./data/result.csv', 'total')
17 | t_train, y_train, t_test, y_test = generate_dataset.train_test_split(t, y, split=0.80)
18 |
19 | fig1, ax1 = plt.subplots(1, figsize=(12, 6))
20 | ax1.plot(t, y, linewidth=1, marker='o', ls='-')
21 | ax1.set_xlabel('Time')
22 | ax1.set_ylabel('Hourly traffic volume')
23 | ax1.yaxis.set_major_locator(MaxNLocator(10))
24 | ax1.xaxis.set_major_locator(MaxNLocator(7))
25 |
26 | fig1.autofmt_xdate()
27 | ax1.grid(True)
28 | plt.savefig('plots/time_series.png')
29 |
30 | # plot time series with train/test split
31 | fig2, ax2 = plt.subplots(1, figsize=(12, 6))
32 | ax2.plot(t_train, y_train, linewidth=1, ls='-', marker='o', label='Train')
33 | ax2.plot(np.concatenate([[t_train[-1]], t_test]), np.concatenate([[y_train[-1]], y_test]), linewidth=1, ls='--',
34 | marker='*', label='Test')
35 | ax2.set_xlabel('Time')
36 | ax2.set_ylabel('Hourly traffic volume')
37 | plt.title('Time Series Split into Train and Test Sets')
38 | plt.legend()
39 | plt.tight_layout
40 | plt.savefig('plots/train_test_split.png')
41 |
42 | # set size of input/output windows
43 | iw = 24
44 | ow = 6
45 | s = 3
46 |
47 | # generate windowed training/test datasets
48 | Xtrain, Ytrain = generate_dataset.windowed_dataset(y_train, input_window=iw, output_window=ow, stride=s)
49 | Xtest, Ytest = generate_dataset.windowed_dataset(y_test, input_window=iw, output_window=ow, stride=s)
50 | # Xtimetrain, Ytimetrain = generate_dataset.windowed_dataset(t_train, input_window=iw, output_window=ow, stride=s)
51 | # Xtimetest, Ytimetest = generate_dataset.windowed_dataset(t_test, input_window=iw, output_window=ow, stride=s)
52 |
53 | # plot example of windowed data
54 | fig3, ax3 = plt.subplots(1, figsize=(12, 6))
55 | ax3.plot(np.arange(0, iw), Xtrain[:, 0, 0], ls='-', linewidth=1.0, marker='o', label='Input')
56 | ax3.plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtrain[-1, 0, 0]], Ytrain[:, 0, 0]]), linewidth=1.0,
57 | ls='--', marker='o', label='Target')
58 | plt.xlim([0, iw + ow - 1])
59 | # plt.yticks(np.arange(0, 24, 4))
60 | ax2.set_xlabel('Time')
61 | ax2.set_ylabel('Hourly traffic volume')
62 | plt.title('Example of Windowed Training Data')
63 | plt.legend()
64 | plt.tight_layout()
65 | plt.savefig('plots/windowed_data.png')
66 |
67 | # LSTM encoder-decoder
68 | # convert windowed data from np.array to PyTorch tensor
69 | X_train, Y_train, X_test, Y_test = generate_dataset.numpy_to_torch(Xtrain, Ytrain, Xtest, Ytest)
70 |
71 | # specify model parameters and train
72 | import torch
73 |
74 | ispre = True # 是否预测
75 | if ispre:
76 | # 加载模型
77 | m_state_dict = torch.load('./model/lstm-ed.pt')
78 | new_m = lstm_encoder_decoder.lstm_seq2seq(input_size=X_train.shape[2], hidden_size=128)
79 | new_m.load_state_dict(m_state_dict)
80 | plotting.plot_train_test_results(new_m, Xtrain, Ytrain, Xtest, Ytest)
81 | plt.close('all')
82 | else:
83 | # specify model parameters and train
84 | # 保存模型
85 | model = lstm_encoder_decoder.lstm_seq2seq(input_size=X_train.shape[2], hidden_size=128)
86 | loss = model.train_model(input_tensor=X_train, target_tensor=Y_train, n_epochs=300, target_len=ow, batch_size=64,
87 | training_prediction='mixed_teacher_forcing', teacher_forcing_ratio=0.5,
88 | learning_rate=0.001,
89 | dynamic_tf=False)
90 | torch.save(model.state_dict(), './model/lstm-ed.pt')
91 |
--------------------------------------------------------------------------------
/traj_mining/traffic_flow_predict/generate_dataset.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch
3 | import pandas as pd
4 | from sklearn.preprocessing import MinMaxScaler
5 |
6 |
7 | def synthetic_data(Nt=2000, tf=80 * np.pi):
8 | '''
9 | create synthetic time series dataset
10 | : param Nt: number of time steps
11 | : param tf: final time
12 | : return t, y: time, feature arrays
13 | '''
14 | t = np.linspace(0., tf, Nt)
15 | y = np.sin(2. * t) + 0.5 * np.cos(t) + np.random.normal(0., 0.2, Nt)
16 | return t, y
17 |
18 |
19 | def read_data(filenm, colnm):
20 | inputfile = open(filenm, 'rb') # 可打开含有中文的地址
21 | df = pd.read_csv(inputfile, usecols=['time', 'up', 'down', 'total'])
22 | t = df['time'].values
23 | y = df[colnm].values
24 | x = np.arange(t.shape[0])
25 | y_norm = (y - y.min()) / (y.max() - y.min())
26 | return x, y_norm, t, y
27 |
28 |
29 | def train_test_split(t, y, split=0.8):
30 | '''
31 |
32 | split time series into train/test sets
33 |
34 | : param t: time array
35 | : para y: feature array
36 | : para split: percent of data to include in training set
37 | : return t_train, y_train: time/feature training and test sets;
38 | : t_test, y_test: (shape: [# samples, 1])
39 |
40 | '''
41 |
42 | indx_split = int(split * len(y))
43 | indx_train = np.arange(0, indx_split)
44 | indx_test = np.arange(indx_split, len(y))
45 |
46 | t_train = t[indx_train]
47 | y_train = y[indx_train]
48 | y_train = y_train.reshape(-1, 1)
49 |
50 | t_test = t[indx_test]
51 | y_test = y[indx_test]
52 | y_test = y_test.reshape(-1, 1)
53 |
54 | return t_train, y_train, t_test, y_test
55 |
56 |
57 | def windowed_dataset(y, input_window=5, output_window=1, stride=1, num_features=1):
58 | '''
59 | create a windowed dataset
60 |
61 | : param y: time series feature (array)
62 | : param input_window: number of y samples to give model
63 | : param output_window: number of future y samples to predict
64 | : param stide: spacing between windows
65 | : param num_features: number of features (i.e., 1 for us, but we could have multiple features)
66 | : return X, Y: arrays with correct dimensions for LSTM
67 | : (i.e., [input/output window size # examples, # features])
68 | '''
69 | L = y.shape[0]
70 | num_samples = (L - input_window - output_window) // stride + 1
71 |
72 | X = np.zeros([input_window, num_samples, num_features])
73 | Y = np.zeros([output_window, num_samples, num_features])
74 |
75 | for ff in np.arange(num_features):
76 | for ii in np.arange(num_samples):
77 | start_x = stride * ii
78 | end_x = start_x + input_window
79 | X[:, ii, ff] = y[start_x:end_x, ff]
80 |
81 | start_y = stride * ii + input_window
82 | end_y = start_y + output_window
83 | Y[:, ii, ff] = y[start_y:end_y, ff]
84 |
85 | return X, Y
86 |
87 |
88 | def windowed_dataset_time(y, input_window=5, output_window=1, stride=1, num_features=1):
89 | '''
90 | create a windowed dataset
91 |
92 | : param y: time series feature (array)
93 | : param input_window: number of y samples to give model
94 | : param output_window: number of future y samples to predict
95 | : param stide: spacing between windows
96 | : param num_features: number of features (i.e., 1 for us, but we could have multiple features)
97 | : return X, Y: arrays with correct dimensions for LSTM
98 | : (i.e., [input/output window size # examples, # features])
99 | '''
100 | L = y.shape[0]
101 | num_samples = (L - input_window - output_window) // stride + 1
102 |
103 | X = np.zeros([input_window, num_samples, num_features])
104 | Y = np.zeros([output_window, num_samples, num_features])
105 |
106 | for ff in np.arange(num_features):
107 | for ii in np.arange(num_samples):
108 | start_x = stride * ii
109 | end_x = start_x + input_window
110 | X[:, ii, ff] = y[start_x:end_x, ff]
111 |
112 | start_y = stride * ii + input_window
113 | end_y = start_y + output_window
114 | Y[:, ii, ff] = y[start_y:end_y, ff]
115 |
116 | return X, Y
117 |
118 |
119 | def numpy_to_torch(Xtrain, Ytrain, Xtest, Ytest):
120 | '''
121 | convert numpy array to PyTorch tensor
122 |
123 | : param Xtrain: windowed training input data (input window size, # examples, # features); np.array
124 | : param Ytrain: windowed training target data (output window size, # examples, # features); np.array
125 | : param Xtest: windowed test input data (input window size, # examples, # features); np.array
126 | : param Ytest: windowed test target data (output window size, # examples, # features); np.array
127 | : return X_train_torch, Y_train_torch,
128 | : X_test_torch, Y_test_torch: all input np.arrays converted to PyTorch tensors
129 |
130 | '''
131 |
132 | X_train_torch = torch.from_numpy(Xtrain).type(torch.Tensor)
133 | Y_train_torch = torch.from_numpy(Ytrain).type(torch.Tensor)
134 |
135 | X_test_torch = torch.from_numpy(Xtest).type(torch.Tensor)
136 | Y_test_torch = torch.from_numpy(Ytest).type(torch.Tensor)
137 |
138 | return X_train_torch, Y_train_torch, X_test_torch, Y_test_torch
139 |
--------------------------------------------------------------------------------
/traj_mining/traffic_flow_predict/lstm_encoder_decoder.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import random
3 | import os, errno
4 | import sys
5 | from tqdm import trange
6 |
7 | import torch
8 | import torch.nn as nn
9 | from torch import optim
10 | import torch.nn.functional as F
11 |
12 |
13 | class lstm_encoder(nn.Module):
14 | ''' Encodes time-series sequence '''
15 |
16 | def __init__(self, input_size, hidden_size, num_layers=3):
17 | '''
18 | : param input_size: the number of features in the input X
19 | : param hidden_size: the number of features in the hidden state h
20 | : param num_layers: number of recurrent layers (i.e., 2 means there are
21 | : 2 stacked LSTMs)
22 | '''
23 |
24 | super(lstm_encoder, self).__init__()
25 | self.input_size = input_size
26 | self.hidden_size = hidden_size
27 | self.num_layers = num_layers
28 |
29 | # define LSTM layer
30 | self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,
31 | num_layers=num_layers, dropout=0.3)
32 |
33 | def forward(self, x_input):
34 | '''
35 | : param x_input: input of shape (seq_len, # in batch, input_size)
36 | : return lstm_out, hidden: lstm_out gives all the hidden states in the sequence;
37 | : hidden gives the hidden state and cell state for the last
38 | : element in the sequence
39 | '''
40 |
41 | lstm_out, self.hidden = self.lstm(x_input.view(x_input.shape[0], x_input.shape[1], self.input_size))
42 |
43 | return lstm_out, self.hidden
44 |
45 | def init_hidden(self, batch_size):
46 | '''
47 | initialize hidden state
48 | : param batch_size: x_input.shape[1]
49 | : return: zeroed hidden state and cell state
50 | '''
51 |
52 | return (torch.zeros(self.num_layers, batch_size, self.hidden_size),
53 | torch.zeros(self.num_layers, batch_size, self.hidden_size))
54 |
55 |
56 | class lstm_decoder(nn.Module):
57 | ''' Decodes hidden state output by encoder '''
58 |
59 | def __init__(self, input_size, hidden_size, num_layers=3):
60 | '''
61 | : param input_size: the number of features in the input X
62 | : param hidden_size: the number of features in the hidden state h
63 | : param num_layers: number of recurrent layers (i.e., 2 means there are
64 | : 2 stacked LSTMs)
65 | '''
66 |
67 | super(lstm_decoder, self).__init__()
68 | self.input_size = input_size
69 | self.hidden_size = hidden_size
70 | self.num_layers = num_layers
71 |
72 | self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,
73 | num_layers=num_layers, dropout=0.3)
74 | self.linear = nn.Linear(hidden_size, input_size)
75 |
76 | def forward(self, x_input, encoder_hidden_states):
77 | '''
78 | : param x_input: should be 2D (batch_size, input_size)
79 | : param encoder_hidden_states: hidden states
80 | : return output, hidden: output gives all the hidden states in the sequence;
81 | : hidden gives the hidden state and cell state for the last
82 | : element in the sequence
83 |
84 | '''
85 |
86 | lstm_out, self.hidden = self.lstm(x_input.unsqueeze(0), encoder_hidden_states)
87 | output = self.linear(lstm_out.squeeze(0))
88 |
89 | return output, self.hidden
90 |
91 |
92 | class lstm_seq2seq(nn.Module):
93 | ''' train LSTM encoder-decoder and make predictions '''
94 |
95 | def __init__(self, input_size, hidden_size):
96 |
97 | '''
98 | : param input_size: the number of expected features in the input X
99 | : param hidden_size: the number of features in the hidden state h
100 | '''
101 |
102 | super(lstm_seq2seq, self).__init__()
103 |
104 | self.input_size = input_size
105 | self.hidden_size = hidden_size
106 |
107 | self.encoder = lstm_encoder(input_size=input_size, hidden_size=hidden_size)
108 | self.decoder = lstm_decoder(input_size=input_size, hidden_size=hidden_size)
109 |
110 | def train_model(self, input_tensor, target_tensor, n_epochs, target_len, batch_size,
111 | training_prediction='recursive', teacher_forcing_ratio=0.5, learning_rate=0.01, dynamic_tf=False):
112 |
113 | '''
114 | train lstm encoder-decoder
115 |
116 | : param input_tensor: input data with shape (seq_len, # in batch, number features); PyTorch tensor
117 | : param target_tensor: target data with shape (seq_len, # in batch, number features); PyTorch tensor
118 | : param n_epochs: number of epochs
119 | : param target_len: number of values to predict
120 | : param batch_size: number of samples per gradient update
121 | : param training_prediction: type of prediction to make during training ('recursive', 'teacher_forcing', or
122 | : 'mixed_teacher_forcing'); default is 'recursive'
123 | : param teacher_forcing_ratio: float [0, 1) indicating how much teacher forcing to use when
124 | : training_prediction = 'teacher_forcing.' For each batch in training, we generate a random
125 | : number. If the random number is less than teacher_forcing_ratio, we use teacher forcing.
126 | : Otherwise, we predict recursively. If teacher_forcing_ratio = 1, we train only using
127 | : teacher forcing.
128 | : param learning_rate: float >= 0; learning rate
129 | : param dynamic_tf: use dynamic teacher forcing (True/False); dynamic teacher forcing
130 | : reduces the amount of teacher forcing for each epoch
131 | : return losses: array of loss function for each epoch
132 | '''
133 |
134 | # initialize array of losses
135 | losses = np.full(n_epochs, np.nan)
136 |
137 | optimizer = optim.Adam(self.parameters(), lr=learning_rate)
138 | criterion = nn.MSELoss()
139 |
140 | # calculate number of batch iterations
141 | n_batches = int(input_tensor.shape[1] / batch_size)
142 |
143 | with trange(n_epochs) as tr:
144 | for it in tr:
145 |
146 | batch_loss = 0.
147 | batch_loss_tf = 0.
148 | batch_loss_no_tf = 0.
149 | num_tf = 0
150 | num_no_tf = 0
151 |
152 | for b in range(n_batches):
153 | # select data
154 | input_batch = input_tensor[:, b: b + batch_size, :]
155 | target_batch = target_tensor[:, b: b + batch_size, :]
156 |
157 | # outputs tensor
158 | outputs = torch.zeros(target_len, batch_size, input_batch.shape[2])
159 |
160 | # initialize hidden state
161 | encoder_hidden = self.encoder.init_hidden(batch_size)
162 |
163 | # zero the gradient
164 | optimizer.zero_grad()
165 |
166 | # encoder outputs
167 | encoder_output, encoder_hidden = self.encoder(input_batch)
168 |
169 | # decoder with teacher forcing
170 | decoder_input = input_batch[-1, :, :] # shape: (batch_size, input_size)
171 | decoder_hidden = encoder_hidden
172 |
173 | if training_prediction == 'recursive':
174 | # predict recursively
175 | for t in range(target_len):
176 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)
177 | outputs[t] = decoder_output
178 | decoder_input = decoder_output
179 |
180 | if training_prediction == 'teacher_forcing':
181 | # use teacher forcing
182 | if random.random() < teacher_forcing_ratio:
183 | for t in range(target_len):
184 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)
185 | outputs[t] = decoder_output
186 | decoder_input = target_batch[t, :, :]
187 |
188 | # predict recursively
189 | else:
190 | for t in range(target_len):
191 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)
192 | outputs[t] = decoder_output
193 | decoder_input = decoder_output
194 |
195 | if training_prediction == 'mixed_teacher_forcing':
196 | # predict using mixed teacher forcing
197 | for t in range(target_len):
198 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)
199 | outputs[t] = decoder_output
200 |
201 | # predict with teacher forcing
202 | if random.random() < teacher_forcing_ratio:
203 | decoder_input = target_batch[t, :, :]
204 |
205 | # predict recursively
206 | else:
207 | decoder_input = decoder_output
208 |
209 | # compute the loss
210 | loss = criterion(outputs, target_batch)
211 | batch_loss += loss.item()
212 |
213 | # backpropagation
214 | loss.backward()
215 | optimizer.step()
216 |
217 | # loss for epoch
218 | batch_loss /= n_batches
219 | losses[it] = batch_loss
220 |
221 | # dynamic teacher forcing
222 | if dynamic_tf and teacher_forcing_ratio > 0:
223 | teacher_forcing_ratio = teacher_forcing_ratio - 0.02
224 |
225 | # progress bar
226 | tr.set_postfix(loss="{0:.3f}".format(batch_loss))
227 |
228 | return losses
229 |
230 | def predict(self, input_tensor, target_len):
231 |
232 | '''
233 | : param input_tensor: input data (seq_len, input_size); PyTorch tensor
234 | : param target_len: number of target values to predict
235 | : return np_outputs: np.array containing predicted values; prediction done recursively
236 | '''
237 |
238 | # encode input_tensor
239 | input_tensor = input_tensor.unsqueeze(1) # add in batch size of 1
240 | encoder_output, encoder_hidden = self.encoder(input_tensor)
241 |
242 | # initialize tensor for predictions
243 | outputs = torch.zeros(target_len, input_tensor.shape[2])
244 |
245 | # decode input_tensor
246 | decoder_input = input_tensor[-1, :, :]
247 | decoder_hidden = encoder_hidden
248 |
249 | for t in range(target_len):
250 | decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden)
251 | outputs[t] = decoder_output.squeeze(0)
252 | decoder_input = decoder_output
253 |
254 | np_outputs = outputs.detach().numpy()
255 |
256 | return np_outputs
257 |
--------------------------------------------------------------------------------
/traj_mining/traffic_flow_predict/model/lstm-ed.pt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traffic_flow_predict/model/lstm-ed.pt
--------------------------------------------------------------------------------
/traj_mining/traffic_flow_predict/plotting.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import matplotlib.pyplot as plt
3 | import torch
4 | from matplotlib.ticker import MaxNLocator
5 | from matplotlib.dates import DayLocator, HourLocator, DateFormatter, drange
6 | import datetime
7 | import matplotlib.ticker as ticker
8 | import sklearn.metrics as metrics
9 |
10 |
11 | def plot_train_test_results(lstm_model, Xtrain, Ytrain, Xtest, Ytest, num_rows=3):
12 | '''
13 | plot examples of the lstm encoder-decoder evaluated on the training/test data
14 |
15 | : param lstm_model: trained lstm encoder-decoder
16 | : param Xtrain: np.array of windowed training input data
17 | : param Ytrain: np.array of windowed training target data
18 | : param Xtest: np.array of windowed test input data
19 | : param Ytest: np.array of windowed test target data
20 | : param num_rows: number of training/test examples to plot
21 | : return: num_rows x 2 plots; first column is training data predictions,
22 | : second column is test data predictions
23 | '''
24 |
25 | # input window size
26 | iw = Xtrain.shape[0]
27 | ow = Ytest.shape[0]
28 |
29 | # figure setup
30 | num_cols = 2
31 | num_plots = num_rows * num_cols
32 |
33 | fig, ax = plt.subplots(num_rows, num_cols, figsize=(13, 15))
34 |
35 | # plot training/test predictions
36 | for ii in range(num_rows):
37 | # train set
38 | X_train_plt = Xtrain[:, ii, :]
39 | Y_train_pred = lstm_model.predict(torch.from_numpy(X_train_plt).type(torch.Tensor), target_len=ow)
40 |
41 | ax[ii, 0].plot(np.arange(0, iw), Xtrain[:, ii, 0], 'k', linewidth=2, label='Input')
42 | ax[ii, 0].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtrain[-1, ii, 0]], Ytrain[:, ii, 0]]),
43 | color=(0.2, 0.42, 0.72), linewidth=2, label='Target')
44 | ax[ii, 0].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtrain[-1, ii, 0]], Y_train_pred[:, 0]]),
45 | color=(0.76, 0.01, 0.01), linewidth=2, label='Prediction')
46 | ax[ii, 0].set_xlim([0, iw + ow - 1])
47 | ax[ii, 0].set_xlabel('$t$')
48 | ax[ii, 0].set_ylabel('$y$')
49 |
50 | # test set
51 | X_test_plt = Xtest[:, ii, :]
52 | Y_test_pred = lstm_model.predict(torch.from_numpy(X_test_plt).type(torch.Tensor), target_len=ow)
53 | ax[ii, 1].plot(np.arange(0, iw), Xtest[:, ii, 0], 'k', linewidth=2, label='Input')
54 | ax[ii, 1].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Ytest[:, ii, 0]]),
55 | color=(0.2, 0.42, 0.72), linewidth=2, label='Target')
56 | ax[ii, 1].plot(np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Y_test_pred[:, 0]]),
57 | color=(0.76, 0.01, 0.01), linewidth=2, label='Prediction')
58 | ax[ii, 1].set_xlim([0, iw + ow - 1])
59 | ax[ii, 1].set_xlabel('$t$')
60 | ax[ii, 1].set_ylabel('$y$')
61 |
62 | # plot_results(np.arange(0, iw), Xtrain[:, ii, 0], np.arange(iw - 1, iw + ow),
63 | # np.concatenate([[Xtrain[-1, ii, 0]], Ytrain[:, ii, 0]]), np.arange(iw - 1, iw + ow),
64 | # np.concatenate([[Xtrain[-1, ii, 0]], Y_train_pred[:, 0]]), 'train' + str(ii))
65 | plot_results(np.arange(0, iw), Xtest[:, ii, 0],
66 | np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Ytest[:, ii, 0]]),
67 | np.arange(iw - 1, iw + ow), np.concatenate([[Xtest[-1, ii, 0]], Y_test_pred[:, 0]]),
68 | 'test' + str(ii))
69 |
70 | if ii == 0:
71 | ax[ii, 0].set_title('Train')
72 | ax[ii, 1].legend(bbox_to_anchor=(1, 1))
73 | ax[ii, 1].set_title('Test')
74 |
75 | plt.suptitle('LSTM Encoder-Decoder Predictions', x=0.445, y=1.)
76 | plt.tight_layout()
77 | plt.subplots_adjust(top=0.95)
78 | plt.savefig('plots/predictions.png')
79 | # plt.show()
80 | plt.close()
81 |
82 | return
83 |
84 |
85 | def plot_results(xi, yi, xt, yt, xp, yp, tpe):
86 | fig1, ax1 = plt.subplots(1, figsize=(12, 6))
87 | yi = yi * 32
88 | yt = yt * 32
89 | yp = yp * 32
90 | ax1.plot(['2019-4-1 0:00',
91 | '2019-4-1 1:00',
92 | '2019-4-1 2:00',
93 | '2019-4-1 3:00',
94 | '2019-4-1 4:00',
95 | '2019-4-1 5:00',
96 | '2019-4-1 6:00',
97 | '2019-4-1 7:00',
98 | '2019-4-1 8:00',
99 | '2019-4-1 9:00',
100 | '2019-4-1 10:00',
101 | '2019-4-1 11:00',
102 | '2019-4-1 12:00',
103 | '2019-4-1 13:00',
104 | '2019-4-1 14:00',
105 | '2019-4-1 15:00',
106 | '2019-4-1 16:00',
107 | '2019-4-1 17:00',
108 | '2019-4-1 18:00',
109 | '2019-4-1 19:00',
110 | '2019-4-1 20:00',
111 | '2019-4-1 21:00',
112 | '2019-4-1 22:00',
113 | '2019-4-1 23:00'], yi, linewidth=1, marker='o', ls='-', label='Input')
114 | ax1.plot(['2019-4-1 23:00',
115 | '2019-4-2 0:00',
116 | '2019-4-2 1:00',
117 | '2019-4-2 2:00',
118 | '2019-4-2 3:00',
119 | '2019-4-2 4:00',
120 | '2019-4-2 5:00'], yt, linewidth=1, marker='o', ls='-', label='Target')
121 | ax1.plot(['2019-4-1 23:00',
122 | '2019-4-2 0:00',
123 | '2019-4-2 1:00',
124 | '2019-4-2 2:00',
125 | '2019-4-2 3:00',
126 | '2019-4-2 4:00',
127 | '2019-4-2 5:00'], yp, linewidth=1, marker='*', ls='--', label='Prediction')
128 |
129 | mape = metrics.mean_absolute_percentage_error(yt, yp)
130 | vs = metrics.explained_variance_score(yt, yp)
131 | mae = metrics.mean_absolute_error(yt, yp)
132 | mse = metrics.mean_squared_error(yt, yp)
133 | r2 = metrics.r2_score(yt, yp)
134 | rmse = np.sqrt(mse)
135 | print('mape:' + str(mape))
136 | print('mae:' + str(mae))
137 | print('mse:' + str(mse))
138 | print('r2:' + str(r2))
139 | print('rmse:' + str(rmse))
140 |
141 | ax1.set_xlabel('Time')
142 | ax1.set_ylabel('Hourly traffic volume')
143 | ax1.yaxis.set_major_locator(MaxNLocator(10))
144 | ax1.xaxis.set_major_locator(MaxNLocator(12))
145 | ax1.yaxis.set_major_locator(ticker.MaxNLocator(integer=True))
146 | ax1.set_ylim(0, 35)
147 | fig1.autofmt_xdate(rotation=30)
148 | ax1.grid(True)
149 | plt.legend()
150 | plt.title('Vessel traffic flow prediction')
151 | plt.tight_layout()
152 | plt.savefig('plots/' + str(tpe) + '.png')
153 | return
154 |
--------------------------------------------------------------------------------
/traj_mining/traffic_flow_predict/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | numpy==1.23.0
3 | pandas==1.4.1
4 | scikit_learn==1.1.1
5 | torch==1.10.2+cu113
6 | tqdm==4.63.0
7 |
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/cleaning.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/cleaning.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/geotostr.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/geotostr.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/plotting.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/plotting.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/segmentation.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/segmentation.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_clean.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_clean.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_interpolation.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_interpolation.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_segment.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_segment.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_stay.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_mining/traj_anchor_berth/pyais/__pycache__/traj_stay.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/geotostr.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import math
3 | import numpy as np
4 |
5 |
6 | def latgeotostr(v):
7 | lbl = 'N'
8 | if float(v) >= 0:
9 | lbl = 'N'
10 | else:
11 | lbl = 'S'
12 | v = float(v)
13 | d = math.floor(v)
14 | m = (v - d) * 60
15 | lat = str(d) + '°' + str(np.round(m, 3)) + '′' + lbl
16 | return lat
17 |
18 |
19 | def longgeotostr(v):
20 | lbl = 'E'
21 | if float(v) >= 0:
22 | lbl = 'E'
23 | else:
24 | lbl = 'W'
25 | v = float(v)
26 | d = math.floor(v)
27 | m = (v - d) * 60
28 | longi = str(d) + '°' + str(np.round(m, 3)) + '′' + lbl
29 | return longi
30 |
31 |
32 | if __name__ == '__main__':
33 | L = 336
34 | s = 3.1415926 * (3 * 25 + 90 + L) * (3 * 25 + 90 + L)
35 | print(str(s))
36 |
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/ptinpolygon.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import math
3 |
4 |
5 | def isPointInPoly(point, poly):
6 | '''Determine whether a point is inside an arbitrary polygon(convex or non-convex)
7 | Input:
8 | point: tuple(x, y)
9 | poly: [x1, y1, x2, y2, ..., xn, yn]
10 | Output:
11 | True: inside
12 | False: outside
13 | '''
14 | assert len(poly) % 2 == 0
15 |
16 | vector_list = []
17 | angle_sum = 0
18 | poly = np.array(poly)
19 |
20 | for i in range(int(len(poly) / 2)):
21 | vector = [poly[i << 1] - point[0], poly[(i << 1) + 1] - point[1]]
22 | vector = np.array(vector)
23 | vector_list.append(vector)
24 |
25 | for i in range(len(vector_list)):
26 | if i == len(vector_list) - 1:
27 | cross = np.cross(vector_list[i], vector_list[0])
28 | cos = np.dot(vector_list[i], vector_list[0]) / (
29 | np.linalg.norm(vector_list[i]) * np.linalg.norm(vector_list[0]))
30 | else:
31 | cross = np.cross(vector_list[i], vector_list[i + 1])
32 | cos = np.dot(vector_list[i], vector_list[i + 1]) / (
33 | np.linalg.norm(vector_list[i]) * np.linalg.norm(vector_list[i + 1]))
34 | try:
35 | angle = math.acos(cos)
36 | if cross >= 0:
37 | angle_sum += angle
38 | else:
39 | angle_sum -= angle
40 | except:
41 | print(cos)
42 |
43 | if abs(angle_sum) > 6.283185307:
44 | return True
45 | else:
46 | return False
47 |
48 | # if __name__ == '__main__':
49 | # point = (9, 1.1)
50 | # poly = [1, 1, 1, 10, 10, 10, 5, 5, 10, 1]
51 | # print(isPointInPoly(point, poly))
52 |
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/traj_clean.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | from vincenty import vincenty
4 | import warnings
5 |
6 | warnings.filterwarnings("ignore")
7 |
8 |
9 | def heuristic_clean(data, vthreshold: float):
10 | '''
11 | :param data: raw trajectory
12 | :param vthreshold: speed threshold
13 | :return: cleaned trajectory
14 | '''
15 | tdf = pd.DataFrame(columns=data.columns, index=data.index)
16 | tdf.drop(tdf.index, inplace=True)
17 |
18 | data.drop_duplicates()
19 | data = data.reset_index(drop=True)
20 | for shipmmsi, dt in data.groupby('DRMMSI'):
21 | if len(str(shipmmsi)) > 5:
22 | dt_copy = dt.copy(deep=True)
23 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
24 | dt_copy = dt_copy.reset_index(drop=True)
25 | dt_copy['NOISE'] = 0
26 | for idx, di in dt_copy.iterrows():
27 | if di['NOISE'] == 0:
28 | if idx < len(dt_copy) - 1:
29 | t1 = di['DRGPSTIME']
30 | t2 = dt_copy.loc[idx + 1, 'DRGPSTIME']
31 | Δt = (t2 - t1) / 3600.0
32 | pt1 = (di['DRLATITUDE'], di['DRLONGITUDE'])
33 | pt2 = (dt_copy.loc[idx + 1, 'DRLATITUDE'], dt_copy.loc[idx + 1, 'DRLONGITUDE'])
34 | Δd = vincenty(pt1, pt2) * 1000 / 1852.25
35 | Δv = Δd / Δt
36 | if Δv > vthreshold:
37 | dt_copy.loc[idx + 1, 'NOISE'] = 1
38 | print("1")
39 | else:
40 | continue
41 | dt_copy_new = dt_copy.query('NOISE==0')
42 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1)
43 | tdf = tdf.append(dt_copy_new, ignore_index=True)
44 | return tdf
45 |
46 |
47 | def sw_clean(data, sw: int):
48 | '''
49 | :param data: raw trajectory
50 | :param sw: the size of sliding window
51 | :return: cleaned trajectory
52 | '''
53 | tdf = pd.DataFrame(columns=data.columns, index=data.index)
54 | tdf.drop(tdf.index, inplace=True)
55 |
56 | data.drop_duplicates()
57 | data = data.reset_index(drop=True)
58 | for shipmmsi, dt in data.groupby('DRMMSI'):
59 | if len(str(shipmmsi)) > 6:
60 | dt_copy = dt.copy(deep=True)
61 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
62 | dt_copy = dt_copy.reset_index(drop=True)
63 | dt_copy['NOISE'] = 0
64 | copylength = len(dt_copy)
65 | num_samples = copylength // sw + 1
66 | for idx in np.arange(num_samples):
67 | start_x = sw * idx
68 | end_x = start_x + sw - 1
69 | end_x = (copylength - 1) if end_x > (copylength - 1) else end_x
70 | dt_temp = dt_copy.loc[start_x:end_x]
71 | lats = dt_temp.loc[:, "DRLATITUDE"]
72 | longs = dt_temp.loc[:, "DRLONGITUDE"]
73 | stdlat = lats.std()
74 | meanlat = lats.mean()
75 | stdlog = longs.std()
76 | meanlog = longs.mean()
77 | for jdx, di in dt_temp.iterrows():
78 | if abs(di['DRLATITUDE'] - meanlat) > abs(1.5 * stdlat) or abs(di['DRLONGITUDE'] - meanlog) > abs(
79 | 1.5 * stdlog):
80 | dt_copy.loc[jdx, 'NOISE'] = 1
81 | print("1")
82 | pass
83 |
84 | dt_copy_new = dt_copy.query('NOISE==0')
85 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1)
86 | tdf = tdf.append(dt_copy_new, ignore_index=True)
87 | return tdf
88 |
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/traj_interpolation.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | from scipy.interpolate import pchip_interpolate
4 | from typing import Dict, List, Union, Optional
5 | from vincenty import vincenty
6 |
7 | # types
8 | TrajData = Dict[str, Union[List[float]]]
9 |
10 |
11 | def traj_load(traj_file: str) -> TrajData:
12 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []}
13 | df = pd.read_csv(traj_file)
14 | dt_c = df.copy(deep=True)
15 | dt_c['DRLONGITUDE'] = dt_c['DRLONGITUDE'].map(lambda x: x / 1.0)
16 | dt_c['DRLATITUDE'] = dt_c['DRLATITUDE'].map(lambda x: x / 1.0)
17 | dt_c['DRSPEED'] = dt_c['DRSPEED'].map(lambda x: x / 1.0)
18 | for index, row in dt_c.iterrows():
19 | trajs_data['lat'].append(row['DRLATITUDE'])
20 | trajs_data['lon'].append(row['DRLONGITUDE'])
21 | trajs_data['speed'].append(row['DRSPEED'])
22 | trajs_data['tstamp'].append(row['DRGPSTIME'])
23 | return trajs_data
24 |
25 |
26 | def traj_calculate_distance(traj_data: TrajData) -> List[float]:
27 | traj_dist = np.zeros(len(traj_data['lat']))
28 | for i in range(len(traj_dist) - 1):
29 | lat1 = traj_data['lat'][i]
30 | lon1 = traj_data['lon'][i]
31 | lat2 = traj_data['lat'][i + 1]
32 | lon2 = traj_data['lon'][i + 1]
33 | pts = (lat1, lon1)
34 | pte = (lat2, lon2)
35 | s = vincenty(pts, pte) * 1000 # unit meter
36 | traj_dist[i + 1] = s
37 | return traj_dist.tolist()
38 |
39 |
40 | def traj_interpolate(traj_file: str, res: float = 1.0, num: Optional[int] = None) -> TrajData:
41 | '''
42 | :param traj_data: raw trajectory filename
43 | :param res: time resolution
44 | :param num: None
45 | :return: interpolated trajectory
46 | '''
47 | traj_data = traj_load(traj_file)
48 | if res <= 0.0:
49 | raise ValueError('res must be > 0.0')
50 | if num is not None and num < 0:
51 | raise ValueError('num must be >= 0')
52 | _traj_dist = traj_calculate_distance(traj_data)
53 | xi = np.cumsum(_traj_dist)
54 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]])
55 | num = num if num is not None else int(np.ceil(xi[-1] / res))
56 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True)
57 | y = pchip_interpolate(xi, yi, x, axis=1)
58 | traj_data_interp = {'lat': list(y[0, :]), 'lon': list(y[1, :]), 'speed': list(y[2, :]), 'tstamp': list(y[-1, :])}
59 | return traj_data_interp
60 |
61 |
62 | def traj_calculate_distance_ts(traj_data: TrajData) -> List[float]:
63 | traj_dist = np.zeros(len(traj_data['lat']))
64 | for i in range(len(traj_dist) - 1):
65 | s = int(traj_data['tstamp'][i + 1]) - int(traj_data['tstamp'][i])
66 | traj_dist[i + 1] = s
67 | return traj_dist.tolist()
68 |
69 |
70 | def traj_interpolate_df(traj_data, res: float = 1.0, num: Optional[int] = None) -> TrajData:
71 | '''
72 | :param traj_data: raw trajectory dataframe
73 | :param res: time resolution
74 | :param num: None
75 | :return: interpolated trajectory
76 | '''
77 | if res <= 0.0:
78 | raise ValueError('res must be > 0.0')
79 | if num is not None and num < 0:
80 | raise ValueError('num must be >= 0')
81 | _traj_dist = traj_calculate_distance_ts(traj_data)
82 | xi = np.cumsum(_traj_dist)
83 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]])
84 | num = num if num is not None else int(np.ceil(xi[-1] / res))
85 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True)
86 | y = pchip_interpolate(xi, yi, x, axis=1)
87 | return y.T
88 |
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/traj_segment.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import warnings
3 |
4 | warnings.filterwarnings("ignore")
5 |
6 |
7 | def segment(df, tsinterval):
8 | """
9 | Trajectory segmentation.
10 | :param df: an array
11 | :type df: DataFrame
12 | :param tsepsilon: time threshold
13 | :type tsepsilon: float
14 | :param tdf: return segmented trajectory
15 | :type tdf: DataFrame
16 | """
17 | tdf = pd.DataFrame(columns=df.columns, index=df.index)
18 | tdf.drop(tdf.index, inplace=True)
19 | tdf['tdiff'] = 0
20 |
21 | for shipmmsi, dt in df.groupby('DRMMSI'):
22 | data = dt.copy(deep=True)
23 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
24 | data = data.reset_index(drop=True)
25 | data['tdiff'] = data['DRGPSTIME'].diff().fillna(0)
26 | i = 0
27 | lastindex = 0
28 | for idx, di in data.iterrows():
29 | if di['tdiff'] > tsinterval and idx >= 1:
30 | data.loc[lastindex:idx - 1, 'DRMMSI'] = str(di['DRMMSI']) + '_' + str(i)
31 | i = i + 1
32 | lastindex = idx
33 | tdf = tdf.append(data, ignore_index=True)
34 | tdf = tdf.drop(['tdiff'], axis=1)
35 | return tdf
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/pyais/traj_stay.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import copy
4 | import time
5 | import math
6 | from scipy.spatial.distance import pdist
7 | from scipy.spatial.distance import squareform
8 | from pyais import traj_clean
9 | from pyais import traj_interpolation
10 | from pyais import traj_segment
11 |
12 | import warnings
13 |
14 | warnings.filterwarnings("ignore")
15 |
16 |
17 | # Part1
18 |
19 | def compute_squared_EDM(X):
20 | return squareform(pdist(X, metric='euclidean'))
21 |
22 |
23 | def ST_DBSCAN(data, eps1: float, eps2: float, minPts: int):
24 | '''
25 | :param data: interpolated trajectory
26 | :param eps1: spatial neighborhood
27 | :param eps2: time neighborhood
28 | :param minPts: the minimum number of points that satisfy the double-neighborhood
29 | :return: labels
30 | '''
31 | n, m = data.shape
32 | timeDisMat = compute_squared_EDM(data[:, 0].reshape(n, 1))
33 | disMat = compute_squared_EDM(data[:, 1:])
34 | core_points_index = np.where(np.sum(np.where((disMat <= eps1) & (timeDisMat <= eps2), 1, 0), axis=1) >= minPts)[0]
35 | labels = np.full((n,), -1)
36 | clusterId = 0
37 | for pointId in core_points_index:
38 | if (labels[pointId] == -1):
39 | labels[pointId] = clusterId
40 | neighbour = np.where((disMat[:, pointId] <= eps1) & (timeDisMat[:, pointId] <= eps2) & (labels == -1))[0]
41 | seeds = set(neighbour)
42 | while len(seeds) > 0:
43 | newPoint = seeds.pop()
44 | labels[newPoint] = clusterId
45 | queryResults = set(np.where((disMat[:, newPoint] <= eps1) & (timeDisMat[:, newPoint] <= eps2))[0])
46 | if len(queryResults) >= minPts:
47 | for resultPoint in queryResults:
48 | if labels[resultPoint] == -1:
49 | seeds.add(resultPoint)
50 | clusterId = clusterId + 1
51 | return labels
52 |
53 |
54 | def stay_st_detect(traj_file: str):
55 | '''
56 | :param traj_data: raw trajectory filename
57 | :return: interpolated trajectory,labels
58 | '''
59 | trajdf = pd.read_csv(traj_file, encoding='gbk')
60 | segdf = traj_segment.segment(trajdf, 1500)
61 | cleandf = traj_clean.heuristic_clean(segdf, 25)
62 |
63 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index)
64 | tdf.drop(tdf.index, inplace=True)
65 | tdf['SP_Status'] = 0
66 |
67 | for shipmmsi, dt in cleandf.groupby('DRMMSI'):
68 | data = dt.copy(deep=True)
69 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
70 | data = data.reset_index(drop=True)
71 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []}
72 | for index, row in data.iterrows():
73 | trajs_data['lat'].append(row['DRLATITUDE'])
74 | trajs_data['lon'].append(row['DRLONGITUDE'])
75 | trajs_data['speed'].append(row['DRSPEED'])
76 | trajs_data['tstamp'].append(row['DRGPSTIME'])
77 | if len(trajs_data['lon']) < 4:
78 | continue
79 | res = 30.0
80 | my_traj_data_interp = traj_interpolation.traj_interpolate_df(trajs_data, res, None)
81 | dataDf = pd.DataFrame(my_traj_data_interp)
82 | dataDf.columns = ['lat', 'lon', 'spd', 'ts']
83 | dataDf['ts'] = dataDf['ts'].map(lambda x: int(x))
84 | datanew = dataDf[['ts', 'lat', 'lon']].values
85 |
86 | minpts = round(np.log(len(datanew)))
87 | if minpts < 4:
88 | minpts = 4
89 | ep2 = minpts / 2 * res
90 | ep1 = 2.2 * ep2 / 3600.0 / 60.0
91 | tj = pd.DataFrame(datanew)
92 | tj.columns = ['DRGPSTIME', 'DRLATITUDE', 'DRLONGITUDE']
93 | labels = ST_DBSCAN(datanew, ep1, ep2, minpts)
94 | tj['SP_Status'] = labels
95 | tj['DRMMSI'] = shipmmsi
96 | tj['DRGPSTIME'] = tj['DRGPSTIME'].map(lambda x: int(x))
97 | tdf = tdf.append(tj, ignore_index=True)
98 | tdf = tdf.fillna(0)
99 | return tdf
100 |
101 |
102 | # Part2
103 | def rad(d):
104 | return float(d) * math.pi / 180.0
105 |
106 |
107 | EARTH_RADIUS = 6378.137
108 |
109 |
110 | def GetDistance(lng1, lat1, lng2, lat2):
111 | radLat1 = rad(lat1)
112 | radLat2 = rad(lat2)
113 | a = radLat1 - radLat2
114 | b = rad(lng1) - rad(lng2)
115 | s = 2 * math.asin(
116 | math.sqrt(math.pow(math.sin(a / 2), 2) + math.cos(radLat1) * math.cos(radLat2) * math.pow(math.sin(b / 2), 2)))
117 | s = s * EARTH_RADIUS
118 | s = round(s * 10000, 2) / 10
119 | return s
120 |
121 |
122 | def ka(dis, dc):
123 | if (dis >= dc):
124 | return 0
125 | else:
126 | return 1
127 |
128 |
129 | def density(data, dc):
130 | dc = float(dc)
131 | latitude = list(data['DRLATITUDE'])
132 | longitude = list(data['DRLONGITUDE'])
133 | part_density = [] # 存储局部密度值
134 | scope = [] # 记录每个点计算局部密度的范围
135 | leftBoundary = 0
136 | rightBoundary = len(data) - 1 # 左边界与右边界
137 | for i in range(len(data)):
138 | trigger = True
139 | left = i - 1
140 | right = i + 1
141 | incrementLeft = 1
142 | incrementRight = 1
143 | while trigger:
144 | # 向左拓展
145 | if incrementLeft != 0:
146 | if left < 0:
147 | left = leftBoundary
148 | distanceLeft = GetDistance(longitude[left], latitude[left], longitude[i], latitude[i])
149 | if (distanceLeft < dc) & (left > leftBoundary):
150 | left -= 1
151 | else:
152 | incrementLeft = 0
153 | # 向右拓展
154 | if incrementRight != 0:
155 | if right > rightBoundary:
156 | right = rightBoundary
157 | distanceRight = GetDistance(longitude[i], latitude[i], longitude[right], latitude[right])
158 | if (distanceRight < dc) & (right < rightBoundary):
159 | right += 1
160 | else:
161 | incrementRight = 0
162 | # 若左右都停止了拓展,此点的局部密度计算结束
163 | if (incrementLeft == 0) & (incrementRight == 0):
164 | trigger = False
165 | if (left == leftBoundary) & (incrementRight == 0):
166 | trigger = False
167 | if (incrementLeft == 0) & (right == rightBoundary):
168 | trigger = False
169 | if left == leftBoundary:
170 | scope.append([left, right - 1])
171 | part_density.append(right - left - 1)
172 | elif right == rightBoundary:
173 | scope.append([left + 1, right])
174 | part_density.append(right - left - 1)
175 | else:
176 | scope.append([left + 1, right - 1])
177 | part_density.append(right - left - 2)
178 | return part_density, scope
179 |
180 |
181 | # 反向更新的方法
182 | def SP_search(data, tc):
183 | tc = int(tc)
184 | SP = []
185 | part_density = copy.deepcopy(list(data['part_density']))
186 | scope = copy.deepcopy(list(data['scope']))
187 | latitude = copy.deepcopy(list(data['DRLATITUDE']))
188 | longitude = copy.deepcopy(list(data['DRLONGITUDE']))
189 | timestamp = copy.deepcopy(list(data['DRGPSTIME']))
190 | trigger = True
191 | used = []
192 | while trigger:
193 | partD = max(part_density)
194 | index = part_density.index(partD)
195 | print('index:', index)
196 | start = scope[index][0]
197 | end = scope[index][1]
198 | if len(used) != 0:
199 | for i in used:
200 | if (scope[i][0] > start) & (scope[i][0] < end):
201 | part_density[index] = scope[i][0] - start - 1
202 | scope[index][1] = scope[i][0] - 1
203 | print("1_1")
204 | if (scope[i][1] > start) & (scope[i][1] < end):
205 | part_density[index] = end - scope[i][1] - 1
206 | scope[index][0] = scope[i][1] + 1
207 | print("1_2")
208 | if (scope[i][0] <= start) & (scope[i][1] >= end):
209 | part_density[index] = 0
210 | scope[index][0] = 0;
211 | scope[index][1] = 0
212 | print("1_3")
213 | start = scope[index][0];
214 | end = scope[index][1]
215 | timeCross = timestamp[end] - timestamp[start]
216 | print('time:', timeCross)
217 | if timeCross > tc:
218 | SarvT = time.localtime(timestamp[start])
219 | SlevT = time.localtime(timestamp[end])
220 | SP.append([index, latitude[index], longitude[index], SarvT, SlevT, scope[index]])
221 | used.append(index)
222 | for k in range(scope[index][0], scope[index][1] + 1):
223 | part_density[k] = 0
224 | part_density[index] = 0
225 | if max(part_density) == 0:
226 | trigger = False
227 | return SP
228 |
229 |
230 | # 通过代表的距离是否小于距离阈值来大致判断停留点是否一致
231 | def similar(sp, dc):
232 | dc = float(dc)
233 | latitude = copy.deepcopy(list(sp['latitude']))
234 | longitude = copy.deepcopy(list(sp['longitude']))
235 | i = 0;
236 | index = list(sp.index)
237 | for i in index:
238 | for j in index:
239 | if i != j:
240 | dist = GetDistance(longitude[i], latitude[i], longitude[j], latitude[j])
241 | if dist < 1.5 * dc:
242 | sp = sp.drop(j, axis=0)
243 | index.remove(j)
244 | return sp
245 |
246 |
247 | def stay_spt_detect(traj_file: str):
248 | '''
249 | :param traj_data: raw trajectory filename
250 | :return: interpolated trajectory,labels
251 | '''
252 | trajdf = pd.read_csv(traj_file, encoding='gbk')
253 | segdf = traj_segment.segment(trajdf, 1500)
254 | cleandf = traj_clean.heuristic_clean(segdf, 25)
255 |
256 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index)
257 | tdf.drop(tdf.index, inplace=True)
258 | tdf['SP_Status'] = 0
259 |
260 | for shipmmsi, dt in cleandf.groupby('DRMMSI'):
261 | if len(str(shipmmsi)) > 5:
262 | if len(dt) > 10:
263 | data = dt.copy(deep=True)
264 | data = data.reset_index(drop=True)
265 | sj = data.copy(deep=True)
266 | sj['SP_Status'] = 0
267 |
268 | dc = 400
269 | tc = 600
270 | data['part_density'], data['scope'] = density(data, dc)
271 | SP = SP_search(data, tc)
272 | output = pd.DataFrame(SP)
273 |
274 | if output.empty == False:
275 | output.columns = ['index', 'longitude', 'latitude', 'arriveTime', 'leftTime', 'scope']
276 | output = similar(output, dc)
277 |
278 | for sco in output['scope']:
279 | start, end = sco[0], sco[1]
280 | tcog = sj['DRDIRECTION'][start:end]
281 | ss = 0 # 1靠泊、2锚泊
282 | if tcog.var() < 800:
283 | ss = 1
284 | else:
285 | ss = 2
286 | sj.loc[start:end, 'SP_Status'] = ss
287 | tdf = tdf.append(sj, ignore_index=True)
288 | return tdf
289 |
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | numpy==1.23.0
3 | pandas==1.4.1
4 | scipy==1.8.1
5 | vincenty==0.1.4
6 |
--------------------------------------------------------------------------------
/traj_mining/traj_anchor_berth/trajanchorberth.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import numpy as np
3 | import pandas as pd
4 | import matplotlib.pyplot as plt
5 | from pyais import geotostr
6 | from pyais import traj_stay
7 |
8 |
9 | # Function to know if we have a CCW turn
10 | def CCW(p1, p2, p3):
11 | if (p3[1] - p1[1]) * (p2[0] - p1[0]) >= (p2[1] - p1[1]) * (p3[0] - p1[0]):
12 | return True
13 | return False
14 |
15 |
16 | def GiftWrapping(S):
17 | '''
18 | outline of polygon
19 | :param S: collection of points
20 | :return: outline of polygon
21 | '''
22 | print(plt.style.available) # 查看可用风格
23 | index = 0
24 | n = len(S)
25 | P = [None] * n
26 | l = np.where(S[:, 0] == np.min(S[:, 0]))
27 | pointOnHull = S[l[0][0]]
28 | i = 0
29 | while True:
30 | P[i] = pointOnHull
31 | endpoint = S[0]
32 | for j in range(1, n):
33 | if (endpoint[0] == pointOnHull[0] and endpoint[1] == pointOnHull[1]) or not CCW(S[j], P[i], endpoint):
34 | endpoint = S[j]
35 | i = i + 1
36 | pointOnHull = endpoint
37 | index += 1
38 | if endpoint[0] == P[0][0] and endpoint[1] == P[0][1]:
39 | break
40 | for i in range(n):
41 | if P[-1] is None:
42 | del P[-1]
43 | P = np.array(P)
44 | # Plot final hull
45 | centriod = polygoncenterofmass(P.tolist())
46 | print(geotostr.longgeotostr(centriod[0]))
47 | print(geotostr.latgeotostr(centriod[1]))
48 | plt.clf()
49 | plt.plot(P[:, 0], P[:, 1], 'b-', picker=5)
50 | plt.plot([P[-1, 0], P[0, 0]], [P[-1, 1], P[0, 1]], 'b-', picker=5, label='The boundary of anchor berth')
51 | plt.plot(S[:, 0], S[:, 1], ".g", label='The anchor points')
52 | plt.plot(centriod[0], centriod[1], "or", label='The approximate location of the anchor')
53 | plt.yticks(fontproperties='Times New Roman', size=12)
54 | plt.xticks(fontproperties='Times New Roman', size=12)
55 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14})
56 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14})
57 | plt.title('Calculation of Anchor Berth Area', fontdict={'family': 'Times New Roman', 'size': 16})
58 | plt.ticklabel_format(useOffset=False, style='plain')
59 | plt.legend(loc='upper left')
60 | plt.grid(True)
61 | plt.tight_layout()
62 | plt.show(block=True)
63 | plt.pause(1)
64 | return P
65 |
66 |
67 | def shoelace(list):
68 | '''
69 | shoelace formula, calculate area
70 | :param list: input in clockwise direction
71 | :return: area of polygon
72 | '''
73 | list.append(list[0])
74 | n = len(list)
75 | t1 = 0.0
76 | t2 = 0.0
77 | for i in range(n - 1):
78 | t1 += (list[i][0] * list[i + 1][1]) * 3600 # 平方海里,′
79 | t2 += (list[i][1] * list[i + 1][0]) * 3600
80 | s = abs(t1 - t2) * 0.5 * 1852.25 * 1852.25
81 | print(str(s))
82 | return s
83 |
84 |
85 | def polygonarea(pointLists):
86 | area = 0.0
87 | for i in range(len(pointLists)):
88 | j = (i + 1) % len(pointLists)
89 | area += pointLists[i][1] * pointLists[j][0]
90 | area -= pointLists[i][0] * pointLists[j][1]
91 | area /= 2.0
92 | return (abs(area))
93 |
94 |
95 | def polygoncenterofmass(pointLists):
96 | '''
97 | centroid of polygon
98 | :param pointLists: boundary points
99 | :return: centroid of polygon
100 | '''
101 | if len(pointLists) < 3:
102 | return (0, 0)
103 | else:
104 | cx = 0
105 | cy = 0
106 | factor = 0
107 | j = 0
108 | a = polygonarea(pointLists)
109 | for i in range(len(pointLists)):
110 | j = (i + 1) % len(pointLists)
111 | factor = pointLists[i][1] * pointLists[j][0] - pointLists[j][1] * pointLists[i][0]
112 | cx += (pointLists[i][1] + pointLists[j][1]) * factor
113 | cy += (pointLists[i][0] + pointLists[j][0]) * factor
114 | factor = 1.0 / (6.0 * a)
115 | cx = cx * factor
116 | cy = cy * factor
117 | return [abs(cy), abs(cx)]
118 |
119 |
120 | def main():
121 | font_legend = {
122 | 'family': 'Times New Roman',
123 | 'weight': 'normal',
124 | 'size': 14
125 | }
126 | trajdata = traj_stay.stay_st_detect('./data/1.csv')
127 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm']
128 | for shipmmsi, dt in trajdata.groupby('DRMMSI'):
129 | labels = dt['SP_Status'].values
130 | lbs = set(labels)
131 | plt.plot(dt['DRLONGITUDE'].values, dt['DRLATITUDE'].values, marker='o',
132 | markeredgewidth=1.0, linewidth=0.75, label='Interpolated trajectory',
133 | markerfacecolor='white', markeredgecolor='m', ms=4, alpha=1.0, color='m', zorder=2)
134 | for i, item in enumerate(lbs):
135 | if item >= 0:
136 | colorSytle = scatterColors[i % len(scatterColors)]
137 | subCluster = dt.query("SP_Status==@item")
138 | plt.scatter(subCluster['DRLONGITUDE'].values, subCluster['DRLATITUDE'].values, marker='*', s=70,
139 | edgecolors=colorSytle, label='Stay point',
140 | c='white', linewidths=1.0, zorder=3)
141 | plt.title('Stay Point Identification', fontdict={'family': 'Times New Roman', 'size': 16})
142 | plt.yticks(fontproperties='Times New Roman', size=12)
143 | plt.xticks(fontproperties='Times New Roman', size=12)
144 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14})
145 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14})
146 | plt.ticklabel_format(useOffset=False, style='plain')
147 | plt.legend(loc="best", prop=font_legend)
148 | plt.grid(True)
149 | plt.tight_layout()
150 | plt.show()
151 |
152 | subCluster.rename(columns={'DRLATITUDE': 'latitude', 'DRLONGITUDE': 'longitude'},
153 | inplace=True)
154 | points = []
155 | for idx, data in subCluster.iterrows():
156 | p = [data['longitude'], data['latitude']]
157 | points.append(p)
158 | P = np.array(points)
159 | L = GiftWrapping(P)
160 | mj = shoelace(L.tolist())
161 | print(str(mj))
162 | else:
163 | continue
164 |
165 |
166 | if __name__ == '__main__':
167 | main()
168 |
--------------------------------------------------------------------------------
/traj_mining/traj_cluster/hdbscan/myhdbscan.py:
--------------------------------------------------------------------------------
1 | from dtaidistance import dtw_ndim
2 | from dtaidistance import dtw
3 | from fastdtw import fastdtw
4 | import matplotlib.pyplot as plt
5 | import numpy as np
6 | import pandas as pd
7 | import hdbscan
8 |
9 |
10 | def load_data(file_path):
11 | """
12 | import trajectory data
13 | :return: trajectory data
14 | """
15 | data = pd.read_csv(file_path, usecols=['DRGPSTIME', 'DRMMSI', 'DRLATITUDE', 'DRLONGITUDE'])
16 | data.rename(columns={'DRLONGITUDE': 'long', 'DRLATITUDE': 'lat', 'DRGPSTIME': 't', 'DRMMSI': 'mmsi'}, inplace=True)
17 | trajectories = []
18 | grouped = data[:].groupby('mmsi')
19 | for name, group in grouped:
20 | if len(group) > 1:
21 | group = group.sort_values(by='t', ascending=True)
22 | group['long'] = group['long'].apply(lambda x: x * (1 / 600000.0)) # 以°为单位
23 | group['lat'] = group['lat'].apply(lambda x: x * (1 / 600000.0))
24 | loc = group[['lat', 'long']].values # 原始
25 | trajectories.append(loc)
26 | return trajectories
27 |
28 |
29 | def dist_computer(trajectories):
30 | distance_matrix = dtw_ndim.distance_matrix_fast(trajectories, 2)
31 | distance_matrix = np.array(distance_matrix)
32 | return distance_matrix
33 |
34 |
35 | if __name__ == '__main__':
36 | trajectories = load_data('../data/final1.csv')
37 | dist_matrix = dist_computer(trajectories)
38 | clusterer = hdbscan.HDBSCAN(min_cluster_size=4, cluster_selection_epsilon=0, metric='precomputed')
39 | clusterer.fit(dist_matrix)
40 | lbs = clusterer.labels_.copy()
41 | li = list(set(lbs))
42 |
43 | params = {'axes.titlesize': 'large',
44 | 'legend.fontsize': 14,
45 | 'legend.handlelength': 3}
46 | plt.rcParams.update(params)
47 | plt.style.available
48 | # plt.style.use("dark_background")
49 |
50 | cls = ['red', 'green', 'brown', 'pink', 'yellow', 'black', 'rosybrown', 'royalblue', 'purple', 'tomato',
51 | 'cyan', 'mediumaquamarine', 'mediumblue', 'mediumorchid', 'mediumpurple', 'lightcyan', 'lavender',
52 | 'lavenderblush', 'mediumseagreen', 'mediumslateblue', 'teal', 'mediumspringgreen', 'mediumturquoise',
53 | 'mediumvioletred', 'maroon', 'yellowgreen',
54 | 'midnightblue', 'mintcream', 'mistyrose', 'moccasin', 'navajowhite', 'navy', 'oldlace', 'olive', 'olivedrab',
55 | 'orange', 'orangered', 'orchid', 'palegoldenrod', 'palegreen', 'darkturquoise', 'darkviolet', 'deeppink',
56 | 'deepskyblue', 'dimgray', 'dodgerblue', 'firebrick', 'floralwhite', 'forestgreen', 'fuchsia', 'gainsboro',
57 | 'ghostwhite', 'gold', 'goldenrod', 'gray', 'green', 'greenyellow', 'honeydew', 'hotpink', 'indianred',
58 | 'indigo', 'ivory', 'khaki', 'lemonchiffon', 'lightblue', 'aliceblue', 'lightcoral', 'lightgoldenrodyellow',
59 | 'lightgreen', 'lightgray', 'lightpink', 'lightsalmon', 'lightseagreen', 'lightskyblue', 'lightslategray',
60 | 'lightsteelblue', 'lightyellow', 'lime', 'limegreen', 'linen', 'magenta', 'maroon', 'paleturquoise',
61 | 'palevioletred', 'papayawhip', 'peachpuff', 'peru', 'plum', 'powderblue', 'saddlebrown', 'salmon',
62 | 'sandybrown', 'seagreen', 'seashell', 'sienna', 'silver', 'skyblue', 'slateblue', 'slategray',
63 | 'snow', 'springgreen', 'steelblue', 'tan', 'thistle', 'violet', 'wheat', 'white', 'whitesmoke',
64 | 'antiquewhite', 'aqua', 'aquamarine', 'azure', 'beige', 'bisque', 'blanchedalmond', 'turquoise',
65 | 'lawngreen', 'blue', 'blueviolet', 'burlywood', 'cadetblue', 'chartreuse', 'chocolate', 'coral',
66 | 'cornflowerblue', 'cornsilk', 'crimson', 'darkblue', 'darkcyan', 'darkgoldenrod', 'darkgray',
67 | 'darkgreen', 'darkkhaki', 'darkmagenta', 'darkolivegreen', 'darkorange', 'darkorchid', 'darkred',
68 | 'darksalmon', 'darkseagreen', 'darkslateblue', 'darkslategray']
69 | for index, value in enumerate(lbs):
70 | if value >= 0:
71 | t = trajectories[index]
72 | x, y = t.T
73 | if len(t) > 0:
74 | plt.plot(y, x, color=cls[value], alpha=1.0, lw=0.4)
75 | else:
76 | t = trajectories[index]
77 | x, y = t.T
78 | plt.plot(y, x, color='red', alpha=0.0, lw=0.4)
79 | plt.yticks(fontproperties='Times New Roman', size=12)
80 | plt.xticks(fontproperties='Times New Roman', size=12)
81 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14})
82 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14})
83 | plt.title('Trajectory Clustering', fontdict={'family': 'Times New Roman', 'size': 14})
84 | plt.ticklabel_format(useOffset=False, style='plain')
85 | plt.grid(True)
86 | plt.tight_layout()
87 | plt.show()
88 |
--------------------------------------------------------------------------------
/traj_mining/traj_cluster/requirements.txt:
--------------------------------------------------------------------------------
1 | dtaidistance==2.3.9
2 | fastdtw==0.3.4
3 | matplotlib==3.5.2
4 | numpy==1.23.0
5 | pandas==1.4.3
6 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_clean/example/trajclean.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | from pyais import traj_clean
4 |
5 |
6 | def plot(rawdf, cleandf, rl):
7 | fig, ax = plt.subplots()
8 | for shipmmsi, dt in rawdf.groupby('DRMMSI'):
9 | if len(dt) >= rl and len(str(shipmmsi)) > 5:
10 | dt.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
11 | ax.plot(dt.loc[:, 'DRLONGITUDE'].values, dt.loc[:, 'DRLATITUDE'].values, marker='*', linestyle='--',
12 | color='red',
13 | linewidth=0.5)
14 | for shipmmsi, dt in cleandf.groupby('DRMMSI'):
15 | if len(dt) >= rl and len(str(shipmmsi)) > 5:
16 | dt.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
17 | ax.plot(dt.loc[:, 'DRLONGITUDE'].values, dt.loc[:, 'DRLATITUDE'].values, marker='>', color='green',
18 | linewidth=0.75)
19 | plt.yticks(fontproperties='Times New Roman', size=14)
20 | plt.xticks(fontproperties='Times New Roman', size=14)
21 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16})
22 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16})
23 | plt.title('Trajectory Cleaning', fontdict={'family': 'Times New Roman', 'size': 16})
24 | plt.ticklabel_format(useOffset=False, style='plain')
25 | plt.grid()
26 | plt.tight_layout()
27 | plt.show(block=True)
28 |
29 |
30 | if __name__ == '__main__':
31 | rawdf = pd.read_csv('../data/1.csv')
32 | cleandf = traj_clean.heuristic_clean(rawdf, 25)
33 | plot(rawdf, cleandf, 1)
34 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_clean/pyais/__pycache__/chunk_read.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_clean/pyais/__pycache__/chunk_read.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_clean/pyais/__pycache__/traj_clean.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_clean/pyais/__pycache__/traj_clean.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_clean/pyais/traj_clean.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | from vincenty import vincenty
4 | import warnings
5 |
6 | warnings.filterwarnings("ignore")
7 |
8 |
9 | def heuristic_clean(data, vthreshold: float):
10 | '''
11 | :param data: raw trajectory
12 | :param vthreshold: speed threshold
13 | :return: cleaned trajectory
14 | '''
15 | tdf = pd.DataFrame(columns=data.columns, index=data.index)
16 | tdf.drop(tdf.index, inplace=True)
17 |
18 | data.drop_duplicates()
19 | data = data.reset_index(drop=True)
20 | for shipmmsi, dt in data.groupby('DRMMSI'):
21 | if len(str(shipmmsi)) > 5:
22 | dt_copy = dt.copy(deep=True)
23 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
24 | dt_copy = dt_copy.reset_index(drop=True)
25 | dt_copy['NOISE'] = 0
26 | for idx, di in dt_copy.iterrows():
27 | if di['NOISE'] == 0:
28 | if idx < len(dt_copy) - 1:
29 | t1 = di['DRGPSTIME']
30 | t2 = dt_copy.loc[idx + 1, 'DRGPSTIME']
31 | Δt = (t2 - t1) / 3600.0
32 | pt1 = (di['DRLATITUDE'], di['DRLONGITUDE'])
33 | pt2 = (dt_copy.loc[idx + 1, 'DRLATITUDE'], dt_copy.loc[idx + 1, 'DRLONGITUDE'])
34 | Δd = vincenty(pt1, pt2) * 1000 / 1852.25
35 | Δv = Δd / Δt
36 | if Δv > vthreshold:
37 | dt_copy.loc[idx + 1, 'NOISE'] = 1
38 | print("1")
39 | else:
40 | continue
41 | dt_copy_new = dt_copy.query('NOISE==0')
42 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1)
43 | tdf = tdf.append(dt_copy_new, ignore_index=True)
44 | return tdf
45 |
46 |
47 | def sw_clean(data, sw: int):
48 | '''
49 | :param data: raw trajectory
50 | :param sw: the size of sliding window
51 | :return: cleaned trajectory
52 | '''
53 | tdf = pd.DataFrame(columns=data.columns, index=data.index)
54 | tdf.drop(tdf.index, inplace=True)
55 |
56 | data.drop_duplicates()
57 | data = data.reset_index(drop=True)
58 | for shipmmsi, dt in data.groupby('DRMMSI'):
59 | if len(str(shipmmsi)) > 6:
60 | dt_copy = dt.copy(deep=True)
61 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
62 | dt_copy = dt_copy.reset_index(drop=True)
63 | dt_copy['NOISE'] = 0
64 | copylength = len(dt_copy)
65 | num_samples = copylength // sw + 1
66 | for idx in np.arange(num_samples):
67 | start_x = sw * idx
68 | end_x = start_x + sw - 1
69 | end_x = (copylength - 1) if end_x > (copylength - 1) else end_x
70 | dt_temp = dt_copy.loc[start_x:end_x]
71 | lats = dt_temp.loc[:, "DRLATITUDE"]
72 | longs = dt_temp.loc[:, "DRLONGITUDE"]
73 | stdlat = lats.std()
74 | meanlat = lats.mean()
75 | stdlog = longs.std()
76 | meanlog = longs.mean()
77 | for jdx, di in dt_temp.iterrows():
78 | if abs(di['DRLATITUDE'] - meanlat) > abs(1.5 * stdlat) or abs(di['DRLONGITUDE'] - meanlog) > abs(
79 | 1.5 * stdlog):
80 | dt_copy.loc[jdx, 'NOISE'] = 1
81 | print("1")
82 | pass
83 |
84 | dt_copy_new = dt_copy.query('NOISE==0')
85 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1)
86 | tdf = tdf.append(dt_copy_new, ignore_index=True)
87 | return tdf
88 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_clean/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | numpy==1.23.0
3 | pandas==1.4.1
4 | vincenty==0.1.4
5 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_compress/example/trajcompress.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | from pyais import traj_compress
4 | from pyais.traj_compress import douglaspeucker
5 |
6 |
7 | def sbctest():
8 | trajdata = pd.read_csv('../data/1.csv', encoding="gbk")
9 | compresseddata = traj_compress.sbc(trajdata, 0.025, 0.25)
10 | plt.plot(trajdata['DRLONGITUDE'], trajdata['DRLATITUDE'], marker='*', linestyle='--', color='blue', linewidth=0.5,
11 | label='Raw trajectory')
12 | plt.plot(compresseddata['DRLONGITUDE'], compresseddata['DRLATITUDE'], marker='o', ms=5, linestyle='-', color='g',
13 | linewidth=0.5, label='Compressed trajectory')
14 | plt.yticks(fontproperties='Times New Roman', size=14)
15 | plt.xticks(fontproperties='Times New Roman', size=14)
16 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16})
17 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16})
18 | plt.title('Trajectory Compression', fontdict={'family': 'Times New Roman', 'size': 16})
19 | plt.ticklabel_format(useOffset=False, style='plain')
20 | plt.grid()
21 | plt.legend()
22 | plt.tight_layout()
23 | plt.show()
24 |
25 |
26 | def dptest():
27 | data = pd.read_csv('../data/1.csv', usecols=['DRMMSI', 'DRGPSTIME', 'DRLONGITUDE', 'DRLATITUDE'])
28 | grouped = data[:].groupby('DRMMSI')
29 | for name, group in grouped:
30 | trajdata = group.sort_values(by='DRGPSTIME', ascending=True)
31 | dp = douglaspeucker(trajdata[['DRLATITUDE', 'DRLONGITUDE']].values)
32 | epsilon = dp.avg()
33 | mask = dp.rdp(group[['DRLATITUDE', 'DRLONGITUDE']].values, epsilon, algo="iter", return_mask=True)
34 | compresseddata = group[mask]
35 | plt.plot(trajdata['DRLONGITUDE'], trajdata['DRLATITUDE'], marker='*', linestyle='--', color='blue',
36 | linewidth=0.5,
37 | label='Raw trajectory')
38 | plt.plot(compresseddata['DRLONGITUDE'], compresseddata['DRLATITUDE'], marker='o', ms=5, linestyle='-',
39 | color='g',
40 | linewidth=0.5, label='Compressed trajectory')
41 | plt.yticks(fontproperties='Times New Roman', size=14)
42 | plt.xticks(fontproperties='Times New Roman', size=14)
43 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16})
44 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16})
45 | plt.title('Trajectory Compression', fontdict={'family': 'Times New Roman', 'size': 16})
46 | plt.ticklabel_format(useOffset=False, style='plain')
47 | plt.grid()
48 | plt.legend()
49 | plt.tight_layout()
50 | plt.show()
51 |
52 |
53 | if __name__ == '__main__':
54 | sbctest()
55 | # dptest()
56 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_compress/pyais/__pycache__/traj_compress.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_compress/pyais/__pycache__/traj_compress.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_compress/pyais/__pycache__/traj_interpolation.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_compress/pyais/__pycache__/traj_interpolation.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_compress/pyais/traj_compress.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | from vincenty import vincenty
4 | from math import sqrt
5 | from functools import partial
6 | import sys
7 |
8 |
9 | def sbc(s, max_dist_threhold: float, max_spd_threhold: float):
10 | '''
11 | :param s: raw trajectory
12 | :param max_dist_threhold: distance threshold
13 | :param max_spd_threhold: speed threshold
14 | :return: compressed trajectory
15 | '''
16 | scopy = pd.DataFrame.copy(s, deep=True)
17 | s = scopy.reset_index(drop=True)
18 | if len(s) <= 2:
19 | return s
20 | else:
21 | is_halt = False
22 | e = 1
23 | while e < len(s) and not is_halt:
24 | i = 1
25 | while i < e and not is_halt:
26 | deltae = s.at[e, 'DRGPSTIME'] - s.at[0, 'DRGPSTIME']
27 | deltai = s.at[i, 'DRGPSTIME'] - s.at[0, 'DRGPSTIME']
28 | xp = s.at[0, 'DRLATITUDE'] + (s.at[e, 'DRLATITUDE'] - s.at[0, 'DRLATITUDE']) * deltai / deltae
29 | yp = s.at[0, 'DRLONGITUDE'] + (s.at[e, 'DRLONGITUDE'] - s.at[0, 'DRLONGITUDE']) * deltai / deltae
30 | ptp = (xp, yp)
31 | pta = (s.at[i, 'DRLATITUDE'], s.at[i, 'DRLONGITUDE'])
32 | Vi_1 = vincenty(pta, (s.at[i - 1, 'DRLATITUDE'], s.at[i - 1, 'DRLONGITUDE'])) / (
33 | s.at[i, 'DRGPSTIME'] - s.at[i - 1, 'DRGPSTIME']) * 1000 / 1852.25 * 3600
34 | Vi = vincenty((s.at[i + 1, 'DRLATITUDE'], s.at[i + 1, 'DRLONGITUDE']), pta) / (
35 | s.at[i + 1, 'DRGPSTIME'] - s.at[i, 'DRGPSTIME']) * 1000 / 1852.25 * 3600
36 | if vincenty(pta, ptp) * 1000 / 1852.25 > max_dist_threhold or abs(Vi_1 - Vi) > max_spd_threhold:
37 | is_halt = True
38 | else:
39 | i = i + 1
40 |
41 | if is_halt:
42 | return pd.concat([s[0:1], sbc(s[i:], max_dist_threhold, max_spd_threhold)], ignore_index=True)
43 | e = e + 1
44 | if not is_halt:
45 | return s.loc[[0, len(s) - 1]]
46 |
47 |
48 | class douglaspeucker:
49 | def __init__(self, points):
50 | """
51 | Rarmer Douglas Peucker
52 | :param points: trajectory points
53 | """
54 | self.points = points
55 |
56 | def point_distance_line(self, pt, point1, point2):
57 | vec1 = np.array(point1) - np.array(pt)
58 | vec2 = np.array(point2) - np.array(pt)
59 | distance = np.abs(np.cross(vec1, vec2)) / np.linalg.norm(np.array(point1) - np.array(point2))
60 | return distance
61 |
62 | def __deviations(self):
63 | deviations = []
64 | if len(self.points) > 2:
65 | for i in range(2, len(self.points)):
66 | p1 = self.points[i - 2]
67 | p2 = self.points[i - 1]
68 | p3 = self.points[i]
69 | dev = self.point_distance_line(p2, p1, p3)
70 | deviations.append(dev)
71 | return deviations
72 |
73 | def avg(self):
74 | values = self.__deviations()
75 | if len(values) > 0:
76 | mean = np.mean(values)
77 | else:
78 | mean = 0
79 | return mean
80 |
81 | def max(self):
82 | values = self.__deviations()
83 | mx = np.max(values)
84 | return mx
85 |
86 | def percent(self):
87 | values = self.__deviations()
88 | p = np.percentile(values, 75)
89 | return p
90 |
91 | def pldist(point, start, end):
92 | """
93 | Calculates the distance from ``point`` to the line given
94 | by the points ``start`` and ``end``.
95 | :param point: a point
96 | :type point: numpy array
97 | :param start: a point of the line
98 | :type start: numpy array
99 | :param end: another point of the line
100 | :type end: numpy array
101 | """
102 | if np.all(np.equal(start, end)):
103 | return np.linalg.norm(point - start)
104 |
105 | return np.divide(
106 | np.abs(np.linalg.norm(np.cross(end - start, start - point))),
107 | np.linalg.norm(end - start))
108 |
109 | def rdp_rec(self, M, epsilon, dist=pldist):
110 | """
111 | Simplifies a given array of points.
112 | Recursive version.
113 | :param M: an array
114 | :type M: numpy array
115 | :param epsilon: epsilon in the rdp algorithm
116 | :type epsilon: float
117 | :param dist: distance function
118 | :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist`
119 | """
120 | dmax = 0.0
121 | index = -1
122 |
123 | for i in range(1, M.shape[0]):
124 | d = dist(M[i], M[0], M[-1])
125 |
126 | if d > dmax:
127 | index = i
128 | dmax = d
129 |
130 | if dmax > epsilon:
131 | r1 = self.rdp_rec(M[:index + 1], epsilon, dist)
132 | r2 = self.rdp_rec(M[index:], epsilon, dist)
133 |
134 | return np.vstack((r1[:-1], r2))
135 | else:
136 | return np.vstack((M[0], M[-1]))
137 |
138 | def _rdp_iter(self, M, start_index, last_index, epsilon, dist=pldist):
139 | stk = []
140 | stk.append([start_index, last_index])
141 | global_start_index = start_index
142 | indices = np.ones(last_index - start_index + 1, dtype=bool)
143 |
144 | while stk:
145 | start_index, last_index = stk.pop()
146 |
147 | dmax = 0.0
148 | index = start_index
149 |
150 | for i in range(index + 1, last_index):
151 | if indices[i - global_start_index]:
152 | d = dist(M[i], M[start_index], M[last_index])
153 | if d > dmax:
154 | index = i
155 | dmax = d
156 |
157 | if dmax > epsilon:
158 | stk.append([start_index, index])
159 | stk.append([index, last_index])
160 | else:
161 | for i in range(start_index + 1, last_index):
162 | indices[i - global_start_index] = False
163 |
164 | return indices
165 |
166 | def rdp_iter(self, M, epsilon, dist=pldist, return_mask=False):
167 | """
168 | Simplifies a given array of points.
169 | Iterative version.
170 | :param M: an array
171 | :type M: numpy array
172 | :param epsilon: epsilon in the rdp algorithm
173 | :type epsilon: float
174 | :param dist: distance function
175 | :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist`
176 | :param return_mask: return the mask of points to keep instead
177 | :type return_mask: bool
178 | """
179 | mask = self._rdp_iter(M, 0, len(M) - 1, epsilon, dist)
180 |
181 | if return_mask:
182 | return mask
183 |
184 | return M[mask]
185 |
186 | def rdp(self, M, epsilon=0, dist=pldist, algo="iter", return_mask=False):
187 | """
188 | Simplifies a given array of points using the Ramer-Douglas-Peucker
189 | algorithm.
190 | Example:
191 | >>> from rdp import rdp
192 | >>> rdp([[1, 1], [2, 2], [3, 3], [4, 4]])
193 | [[1, 1], [4, 4]]
194 | This is a convenience wrapper around both :func:`rdp.rdp_iter`
195 | and :func:`rdp.rdp_rec` that detects if the input is a numpy array
196 | in order to adapt the output accordingly. This means that
197 | when it is called using a Python list as argument, a Python
198 | list is returned, and in case of an invocation using a numpy
199 | array, a NumPy array is returned.
200 | The parameter ``return_mask=True`` can be used in conjunction
201 | with ``algo="iter"`` to return only the mask of points to keep. Example:
202 | >>> from rdp import rdp
203 | >>> import numpy as np
204 | >>> arr = np.array([1, 1, 2, 2, 3, 3, 4, 4]).reshape(4, 2)
205 | >>> arr
206 | array([[1, 1],
207 | [2, 2],
208 | [3, 3],
209 | [4, 4]])
210 | >>> mask = rdp(arr, algo="iter", return_mask=True)
211 | >>> mask
212 | array([ True, False, False, True], dtype=bool)
213 | >>> arr[mask]
214 | array([[1, 1],
215 | [4, 4]])
216 | :param M: a series of points
217 | :type M: numpy array with shape ``(n,d)`` where ``n`` is the number of points and ``d`` their dimension
218 | :param epsilon: epsilon in the rdp algorithm
219 | :type epsilon: float
220 | :param dist: distance function
221 | :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist`
222 | :param algo: either ``iter`` for an iterative algorithm or ``rec`` for a recursive algorithm
223 | :type algo: string
224 | :param return_mask: return mask instead of simplified array
225 | :type return_mask: bool
226 | """
227 |
228 | if algo == "iter":
229 | algo = partial(self.rdp_iter, return_mask=return_mask)
230 | elif algo == "rec":
231 | if return_mask:
232 | raise NotImplementedError("return_mask=True not supported with algo=\"rec\"")
233 | algo = self.rdp_rec
234 |
235 | if "numpy" in str(type(M)):
236 | return algo(M, epsilon, dist)
237 |
238 | return algo(np.array(M), epsilon, dist).tolist()
239 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_compress/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | numpy==1.23.0
3 | pandas==1.4.1
4 | vincenty==0.1.4
5 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/binary_payload.py:
--------------------------------------------------------------------------------
1 | import bitarray
2 |
3 | from pyais import decode
4 | from pyais.util import bits2bytes, bytes2bits
5 |
6 | # This is a message of type 6 which contains binary payload
7 | msg = decode(b"!AIVDM,1,1,,B,6B?n;be:cbapalgc;i6?Ow4,2*4A")
8 |
9 | assert msg.msg_type == 6
10 |
11 | # The payload is bytes by default
12 | assert msg.data == b'\xeb/\x11\x8f\x7f\xf1'
13 |
14 |
15 | # But using `bytes2bits` you can convert the bytes into a bitarray
16 | assert bytes2bits(msg.data) == bitarray.bitarray('111010110010111100010001100011110111111111110001')
17 |
18 | # Or to a bitstring using the bitarray to01() method
19 | assert bytes2bits(msg.data).to01() == '111010110010111100010001100011110111111111110001'
20 |
21 | # It is also possible to transform a set of bits back to bytes
22 | assert bits2bytes('111010110010111100010001100011110111111111110001') == b'\xeb/\x11\x8f\x7f\xf1'
23 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/communication_state.py:
--------------------------------------------------------------------------------
1 | """
2 | The following example shows how you can get the communication state
3 | of a message. This works for message types 1, 2, 4, 9, 11 and 18.
4 |
5 | These messages contain diagnostic information for the radio system.
6 | """
7 | from pyais import decode
8 | import json
9 | import functools
10 |
11 | msg = '!AIVDM,1,1,,A,B69Gk3h071tpI02lT2ek?wg61P06,0*1F'
12 | decoded = decode(msg)
13 |
14 | print("The raw radio value is:", decoded.radio)
15 | print("Communication state is SOTMDA:", decoded.is_sotdma)
16 | print("Communication state is ITDMA:", decoded.is_itdma)
17 |
18 | pretty_json = functools.partial(json.dumps, indent=4)
19 | print("Communication state:", pretty_json(decoded.get_communication_state()))
20 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/decode.py:
--------------------------------------------------------------------------------
1 | from pyais import decode, IterMessages
2 |
3 | # Decode a single part using decode
4 | decoded = decode(b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05")
5 | print(decoded)
6 |
7 | parts = [
8 | b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*08",
9 | b"!AIVDM,2,2,4,A,000000000000000,2*20",
10 | ]
11 |
12 | # Decode a multipart message using decode
13 | decoded = decode(*parts)
14 | print(decoded)
15 |
16 | # Decode a string
17 | decoded = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05")
18 | print(decoded)
19 |
20 | # Decode to dict
21 | decoded = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05")
22 | as_dict = decoded.asdict()
23 | print(as_dict)
24 |
25 | # Decode to json
26 | decoded = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05")
27 | json = decoded.to_json()
28 | print(json)
29 |
30 | # It does not matter if you pass a string or bytes
31 | decoded_b = decode(b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05")
32 | decoded_s = decode("!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05")
33 | assert decoded_b == decoded_s
34 |
35 | # Lets say you have some kind of stream of messages. Than you can use `IterMessages` to decode the messages:
36 | fake_stream = [
37 | b"!AIVDM,1,1,,A,13HOI:0P0000VOHLCnHQKwvL05Ip,0*23",
38 | b"!AIVDM,1,1,,A,133sVfPP00PD>hRMDH@jNOvN20S8,0*7F",
39 | b"!AIVDM,1,1,,B,100h00PP0@PHFV`Mg5gTH?vNPUIp,0*3B",
40 | b"!AIVDM,1,1,,B,13eaJF0P00Qd388Eew6aagvH85Ip,0*45",
41 | b"!AIVDM,1,1,,A,14eGrSPP00ncMJTO5C6aBwvP2D0?,0*7A",
42 | b"!AIVDM,1,1,,A,15MrVH0000KH<:V:NtBLoqFP2H9:,0*2F",
43 | ]
44 | for msg in IterMessages(fake_stream):
45 | print(msg.decode())
46 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/encode_dict.py:
--------------------------------------------------------------------------------
1 | """
2 | The following example shows how to create an AIS message using a dictionary of values.
3 |
4 | The full specification of the AIVDM/AIVDO protocol is out of the scope of this example.
5 | For a good overview of the AIVDM/AIVDO Sentence Layer please refer to this project: https://gpsd.gitlab.io/gpsd/AIVDM.html#_aivdmaivdo_sentence_layer
6 |
7 | But you should keep the following things in mind:
8 |
9 | - AIS messages are part of a two layer protocol
10 | - the outer layer is the NMEA 0183 data exchange format
11 | - the actual AIS message is part of the NMEA 0183’s 82-character payload
12 | - because some AIS messages are larger than 82 characters they need to be split across several fragments
13 | - there are 27 different types of AIS messages which differ in terms of fields
14 |
15 | Now to the actual encoding of messages: It is possible to encode a dictionary of values into an AIS message.
16 | To do so, you need some values that you want to encode. The keys need to match the interface of the actual message.
17 | You can call `.fields()` on any message class, to get glimpse on the available fields for each message type.
18 | Unknown keys in the dict are simply omitted by pyais. Most keys have default values and do not need to
19 | be passed explicitly. Only the keys `type` and `mmsi` are always required
20 |
21 | For the following example, let's assume that we want to create a type 1 AIS message.
22 | """
23 | # Required imports
24 | from pyais.encode import encode_dict
25 | from pyais.messages import MessageType1
26 |
27 | # This statement tells us which fields can be set for messages of type 1
28 | print(MessageType1.fields())
29 |
30 | # A dictionary of fields that we want to encode
31 | # Note that you can pass many more fields for type 1 messages, but we let pyais
32 | # use default values for those keys
33 | data = {
34 | 'course': 219.3,
35 | 'lat': 37.802,
36 | 'lon': -122.341,
37 | 'mmsi': '366053209',
38 | 'type': 1
39 | }
40 |
41 | # This creates an encoded AIS message
42 | # Note, that `encode_dict` returns always a list of fragments.
43 | # This is done, because you may never know if a message fits into the 82 character
44 | # size limit of payloads
45 | encoded = encode_dict(data)
46 | print(encoded)
47 |
48 | # You can also change the NMEA fields like the radio channel:
49 | print(encode_dict(data, radio_channel="B"))
50 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/encode_msg.py:
--------------------------------------------------------------------------------
1 | """
2 | The following example shows how to create an AIS message using a dictionary of values.
3 |
4 | The full specification of the AIVDM/AIVDO protocol is out of the scope of this example.
5 | For a good overview of the AIVDM/AIVDO Sentence Layer please refer to this project: https://gpsd.gitlab.io/gpsd/AIVDM.html#_aivdmaivdo_sentence_layer
6 |
7 | But you should keep the following things in mind:
8 |
9 | - AIS messages are part of a two layer protocol
10 | - the outer layer is the NMEA 0183 data exchange format
11 | - the actual AIS message is part of the NMEA 0183’s 82-character payload
12 | - because some AIS messages are larger than 82 characters they need to be split across several fragments
13 | - there are 27 different types of AIS messages which differ in terms of fields
14 |
15 | Now to the actual encoding of messages: It is possible to create a payload class and encode it.
16 |
17 | For the following example, let's assume that we want to create a type 1 AIS message.
18 | """
19 | # Required imports
20 | from pyais.encode import encode_msg
21 | from pyais.messages import MessageType1
22 |
23 | # You do not need to pass every attribute to the class.
24 | # All field other than `mmsi` do have default values.
25 | msg = MessageType1.create(course=219.3, lat=37.802, lon=-122.341, mmsi='366053209')
26 |
27 | # WARNING: If you try to instantiate the class directly (without using .create())
28 | # you need to pass all attributes, as no default values are used.
29 |
30 | # This creates an encoded AIS message
31 | # Note, that `encode_msg` returns always a list of fragments.
32 | # This is done, because you may never know if a message fits into the 82 character
33 | # size limit of payloads
34 | encoded = encode_msg(msg)
35 | print(encoded)
36 |
37 | # You can also change the NMEA fields like the radio channel:
38 | print(encode_msg(msg, radio_channel="B"))
39 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/file_stream.py:
--------------------------------------------------------------------------------
1 | """
2 | The following example shows how to read and parse AIS messages from a file.
3 |
4 | When reading a file, the following things are important to know:
5 |
6 | - lines that begin with a `#` are ignored
7 | - invalid messages are skipped
8 | - invalid lines are skipped
9 | """
10 | import pathlib
11 |
12 | from pyais.stream import FileReaderStream
13 |
14 | filename = pathlib.Path(__file__).parent.joinpath('sample.ais')
15 |
16 | for msg in FileReaderStream(str(filename)):
17 | decoded = msg.decode()
18 | print(decoded)
19 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/msg_to_csv.py:
--------------------------------------------------------------------------------
1 | """
2 | The following example shows how you could write the decoded data to CSV.
3 | You first need to decode the data into a dictionary and then write the
4 | dictionary to a CSV file using a `DictWriter`.
5 | """
6 | import csv
7 |
8 | from pyais import decode
9 |
10 | ais_msg = "!AIVDO,1,1,,,B>qc:003wk?8mP=18D3Q3wgTiT;T,0*13"
11 | data_dict = decode(ais_msg).asdict()
12 |
13 | with open('decoded_message.csv', 'w') as f:
14 | w = csv.DictWriter(f, data_dict.keys())
15 | w.writeheader()
16 | w.writerow(data_dict)
17 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/out_of_order.py:
--------------------------------------------------------------------------------
1 | """
2 | The following example shows how to deal with messages that are out of order.
3 |
4 | In many cases it is not guaranteed that the messages arrive in the correct order.
5 | The most prominent example would be UDP. For this use case, there is the `OutOfOrderByteStream`
6 | class. You can pass any number of messages as an iterable into this class and it will
7 | handle the assembly of the messages.
8 | """
9 | from pyais.stream import IterMessages
10 |
11 | messages = [
12 | b'!AIVDM,2,1,1,A,538CQ>02A;h?D9QC800pu8@T>0P4l9E8L0000017Ah:;;5r50Ahm5;C0,0*07',
13 | b'!AIVDM,2,2,1,A,F@V@00000000000,2*35',
14 | b'!AIVDM,2,2,9,A,F@V@00000000000,2*3D',
15 | b'!AIVDM,2,1,9,A,538CQ>02A;h?D9QC800pu8@T>0P4l9E8L0000017Ah:;;5r50Ahm5;C0,0*0F',
16 | ]
17 |
18 | for msg in IterMessages(messages):
19 | print(msg.decode())
20 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/sample.ais:
--------------------------------------------------------------------------------
1 | # taken from https://www.aishub.net/ais-dispatcher
2 | !AIVDM,1,1,,A,13HOI:0P0000VOHLCnHQKwvL05Ip,0*23
3 | !AIVDM,1,1,,A,133sVfPP00PD>hRMDH@jNOvN20S8,0*7F
4 | !AIVDM,1,1,,B,100h00PP0@PHFV`Mg5gTH?vNPUIp,0*3B
5 | !AIVDM,1,1,,B,13eaJF0P00Qd388Eew6aagvH85Ip,0*45
6 | !AIVDM,1,1,,A,14eGrSPP00ncMJTO5C6aBwvP2D0?,0*7A
7 | !AIVDM,1,1,,A,15MrVH0000KH<:V:NtBLoqFP2H9:,0*2F
8 | # Lines with a leading `#` are ignored
9 | # Also invalid lines or invalid messages are ignored
10 | IamNotAnAisMessage1111
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/examples/single_message.py:
--------------------------------------------------------------------------------
1 | from pyais.messages import NMEAMessage
2 |
3 | message = NMEAMessage(b"!AIVDM,1,1,,B,15M67FC000G?ufbE`FepT@3n00Sa,0*5C")
4 | print(message.decode())
5 |
6 | # or
7 |
8 | message = NMEAMessage.from_string("!AIVDM,1,1,,B,15M67FC000G?ufbE`FepT@3n00Sa,0*5C")
9 | print(message.decode())
10 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__init__.py:
--------------------------------------------------------------------------------
1 | from pyais.messages import NMEAMessage, ANY_MESSAGE
2 | from pyais.stream import FileReaderStream, IterMessages
3 | from pyais.encode import encode_dict, encode_msg, ais_to_nmea_0183
4 | from pyais.decode import decode
5 |
6 | __license__ = 'MIT'
7 | __version__ = '2.1.2'
8 | __author__ = 'Leon Morten Richter'
9 |
10 | __all__ = (
11 | 'encode_dict',
12 | 'encode_msg',
13 | 'ais_to_nmea_0183',
14 | 'NMEAMessage',
15 | 'ANY_MESSAGE',
16 | 'IterMessages',
17 | 'FileReaderStream',
18 | 'decode',
19 | )
20 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/__init__.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/__init__.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/constants.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/constants.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/decode.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/decode.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/encode.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/encode.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/exceptions.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/exceptions.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/messages.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/messages.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/stream.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/stream.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/util.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_encoder_decoder/pyais/__pycache__/util.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/ais_types.py:
--------------------------------------------------------------------------------
1 | from enum import IntEnum
2 |
3 |
4 | class AISType(IntEnum):
5 | # Refer to https://gpsd.gitlab.io/gpsd/AIVDM.html
6 | NOT_IMPLEMENTED = 0
7 | POS_CLASS_A1 = 1
8 | POS_CLASS_A2 = 2
9 | POS_CLASS_A3 = 3
10 | BASE_STATION = 4
11 | STATIC_AND_VOYAGE = 5
12 | BINARY_ADDRESSED = 6
13 | BINARY_ACK = 7
14 | BINARY_BROADCAST = 8
15 | SAR_AIRCRAFT_POS = 9
16 | DATE_INQ = 10
17 | DATE_RESP = 11
18 | SAFETY_MSG = 12
19 | SAFETY_ACK = 13
20 | SAFETY_BROADCAST = 14
21 | INTERROGATE = 15
22 | ASSIGN_MODE = 16
23 | DGNSS = 17
24 | POS_CLASS_B = 18
25 | POS_CLASS_B_EXT = 19
26 | LINK_MGMT = 20
27 | AID_TO_NAV = 21
28 | CHANNEL_MGMT = 22
29 | GROUP_ASSIGN = 23
30 | STATIC = 24
31 | BINARY_SINGLE_SLOT = 25
32 | BINARY_MULTI_SLOT = 26
33 | LONG_RANGE_BROADCAST = 27
34 |
35 | @classmethod
36 | def _missing_(cls, value: object) -> int:
37 | return AISType.NOT_IMPLEMENTED
38 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/constants.py:
--------------------------------------------------------------------------------
1 | import typing
2 | from enum import IntEnum, Enum
3 |
4 | # Keywords
5 | UNDEFINED = 'Undefined'
6 | RESERVED = 'Reserved'
7 | NULL = 'N/A'
8 | ANSI_RED = '\x1b[31m'
9 | ANSI_RESET = '\x1b[0m'
10 |
11 |
12 | class TalkerID(str, Enum):
13 | """ Enum of all NMEA talker IDs.
14 | See: https://gpsd.gitlab.io/gpsd/AIVDM.html#_talker_ids"""
15 | Base_Station = "AB"
16 | Dependent_Base_Station = "AD"
17 | Mobile_Station = "AI"
18 | Navigation_Station = "AN"
19 | Receiving_Station = "AR"
20 | Limited_Base_Station = "AS"
21 | Transmitting_Station = "AT"
22 | Repeater_Station = "AX"
23 | Base_Station_Deprecated = "BS"
24 | Physical_Shore_Station = "SA"
25 | UNDEFINED = "UNDEFINED"
26 |
27 | @classmethod
28 | def _missing_(cls, value: typing.Any) -> str:
29 | return TalkerID.UNDEFINED
30 |
31 | @classmethod
32 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["TalkerID"]:
33 | return cls(v) if v is not None else None
34 |
35 |
36 | class NavigationStatus(IntEnum):
37 | UnderWayUsingEngine = 0
38 | AtAnchor = 1
39 | NotUnderCommand = 2
40 | RestrictedManoeuverability = 3
41 | ConstrainedByHerDraught = 4
42 | Moored = 5
43 | Aground = 6
44 | EngagedInFishing = 7
45 | UnderWaySailing = 8
46 | AISSARTActive = 14
47 | Undefined = 15
48 |
49 | @classmethod
50 | def _missing_(cls, value: object) -> int:
51 | return NavigationStatus.Undefined
52 |
53 | @classmethod
54 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["NavigationStatus"]:
55 | return cls(v) if v is not None else None
56 |
57 |
58 | class ManeuverIndicator(IntEnum):
59 | NotAvailable = 0
60 | NoSpecialManeuver = 1
61 | SpecialManeuver = 2
62 | UNDEFINED = 3
63 |
64 | @classmethod
65 | def _missing_(cls, value: object) -> int:
66 | return ManeuverIndicator.UNDEFINED
67 |
68 | @classmethod
69 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["ManeuverIndicator"]:
70 | return cls(v) if v is not None else None
71 |
72 |
73 | class EpfdType(IntEnum):
74 | Undefined = 0
75 | GPS = 1
76 | GLONASS = 2
77 | GPS_GLONASS = 3
78 | Loran_C = 4
79 | Chayka = 5
80 | IntegratedNavigationSystem = 6
81 | Surveyed = 7
82 | Galileo = 8
83 |
84 | @classmethod
85 | def _missing_(cls, value: object) -> int:
86 | return EpfdType.Undefined
87 |
88 | @classmethod
89 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["EpfdType"]:
90 | return cls(v) if v is not None else None
91 |
92 |
93 | class ShipType(IntEnum):
94 | NotAvailable = 0
95 | # 20's
96 | WIG = 20
97 | WIG_HazardousCategory_A = 21
98 | WIG_HazardousCategory_B = 22
99 | WIG_HazardousCategory_C = 23
100 | WIG_HazardousCategory_D = 24
101 | WIG_Reserved = 25
102 | # 30's
103 | Fishing = 30
104 | Towing = 31
105 | Towing_LengthOver200 = 32
106 | DredgingOrUnderwaterOps = 33
107 | DivingOps = 34
108 | MilitaryOps = 35
109 | Sailing = 36
110 | PleasureCraft = 37
111 | # 40's
112 | HSC = 40
113 | HSC_HazardousCategory_A = 41
114 | HSC_HazardousCategory_B = 42
115 | HSC_HazardousCategory_C = 43
116 | HSC_HazardousCategory_D = 44
117 | HSC_Reserved = 45
118 | HSC_NoAdditionalInformation = 49
119 | # 50's
120 | PilotVessel = 50
121 | SearchAndRescueVessel = 51
122 | Tug = 52
123 | PortTender = 53
124 | AntiPollutionEquipment = 54
125 | LawEnforcement = 55
126 | SPARE = 56
127 | MedicalTransport = 58
128 | NonCombatShip = 59
129 | # 60's
130 | Passenger = 60
131 | Passenger_HazardousCategory_A = 61
132 | Passenger_HazardousCategory_B = 62
133 | Passenger_HazardousCategory_C = 63
134 | Passenger_HazardousCategory_D = 64
135 | Passenger_Reserved = 65
136 | Passenger_NoAdditionalInformation = 69
137 | # 70's
138 | Cargo = 70
139 | Cargo_HazardousCategory_A = 71
140 | Cargo_HazardousCategory_B = 72
141 | Cargo_HazardousCategory_C = 73
142 | Cargo_HazardousCategory_D = 74
143 | Cargo_Reserved = 75
144 | Cargo_NoAdditionalInformation = 79
145 | # 80's
146 | Tanker = 80
147 | Tanker_HazardousCategory_A = 81
148 | Tanker_HazardousCategory_B = 82
149 | Tanker_HazardousCategory_C = 83
150 | Tanker_HazardousCategory_D = 84
151 | Tanker_Reserved = 85
152 | Tanker_NoAdditionalInformation = 89
153 | # 90's
154 | OtherType = 90
155 | OtherType_HazardousCategory_A = 91
156 | OtherType_HazardousCategory_B = 92
157 | OtherType_HazardousCategory_C = 93
158 | OtherType_HazardousCategory_D = 94
159 | OtherType_Reserved = 95
160 | OtherType_NoAdditionalInformation = 99
161 |
162 | @classmethod
163 | def _missing_(cls, value: object) -> int:
164 | if isinstance(value, int):
165 | if 24 < value < 30:
166 | return ShipType.WIG_Reserved
167 |
168 | if 44 < value < 49:
169 | return ShipType.HSC_Reserved
170 |
171 | if 55 < value < 58:
172 | return ShipType.SPARE
173 |
174 | if 64 < value < 69:
175 | return ShipType.Passenger_Reserved
176 |
177 | if 74 < value < 79:
178 | return ShipType.Cargo_Reserved
179 |
180 | if 84 < value < 89:
181 | return ShipType.Tanker_Reserved
182 |
183 | if 94 < value < 99:
184 | return ShipType.OtherType_Reserved
185 |
186 | return ShipType.NotAvailable
187 |
188 | @classmethod
189 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["ShipType"]:
190 | return cls(v) if v is not None else None
191 |
192 |
193 | class DacFid(IntEnum):
194 | DangerousCargoIndication = 13
195 | TidalWindow = 15
196 | NumPersonsOnBoard = 17
197 | ClearanceTimeToEnterPort = 19
198 | BerthingData = 21
199 | AreaNotice = 24
200 | RouteInfoAddressed = 29
201 | TextDescriptionAddressed = 31
202 | ETA = 221
203 | RTA = 222
204 | AtoN_MonitoringData_UK = 245
205 | AtoN_MonitoringData_ROI = 260
206 |
207 | @classmethod
208 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["DacFid"]:
209 | return cls(v) if v is not None else None
210 |
211 |
212 | class NavAid(IntEnum):
213 | DEFAULT = 0
214 | REFERENCE_POINT = 1
215 | RACON = 2
216 | FIXED = 3
217 | FITTED = 4
218 | SPARE = 5
219 | LIGHT_NO_SECTORS = 6
220 | LIGHT_SECTORS = 7
221 | LEADING_LIGHT_FRONT = 8
222 | LEADING_LIGHT_REAR = 9
223 | BEACON_CARDINAL_N = 10
224 | BEACON_CARDINAL_E = 11
225 | BEACON_CARDINAL_S = 12
226 | BEACON_CARDINAL_W = 13
227 | BEACON_STARBOARD = 14
228 | BEACON_CHANNEL_PORT_HAND = 15
229 | BEACON_CHANNEL_STARBOARD_HAND = 16
230 | BEACON_ISOLATED_DANGER = 17
231 | BEACON_SAFE_WATER = 18
232 | BEACON_SPECIAL_MARK = 19
233 | CARDINAL_MARK_N = 20
234 | CARDINAL_MARK_E = 21
235 | CARDINAL_MARK_S = 22
236 | CARDINAL_MARK_W = 23
237 | PORT_HAND_MARK = 24
238 | STARBOARD_HAND_MARK = 25
239 | PREFERRED_HAND_PORT_HAND = 26
240 | PREFERRED_HAND_STARBOARD_HAND = 27
241 | ISOLATED_DANGER = 28
242 | SAFE_WATER = 29
243 | SPECIAL_MARK = 30
244 | LIGHT_VESSEL = 31
245 |
246 | @classmethod
247 | def _missing_(cls, value: object) -> int:
248 | return NavAid.DEFAULT
249 |
250 | @classmethod
251 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["NavAid"]:
252 | return cls(v) if v is not None else None
253 |
254 |
255 | class TransmitMode(IntEnum):
256 | TXA_TXB_RXA_RXB = 0 # default
257 | TXA_RXA_RXB = 1
258 | TXB_RXA_RXB = 2
259 | RESERVED = 3
260 |
261 | @classmethod
262 | def _missing_(cls, value: object) -> int:
263 | return TransmitMode.TXA_TXB_RXA_RXB
264 |
265 | @classmethod
266 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["TransmitMode"]:
267 | return cls(v) if v is not None else None
268 |
269 |
270 | class StationType(IntEnum):
271 | ALL = 0
272 | RESERVED = 1
273 | CLASS_B_ALL = 2
274 | SAR_AIRBORNE = 3
275 | AID_NAV = 4
276 | CLASS_B_SHIPBORNE = 5
277 | REGIONAL = 6
278 |
279 | @classmethod
280 | def _missing_(cls, value: object) -> int:
281 | if isinstance(value, int):
282 | if 6 <= value <= 9:
283 | return StationType.REGIONAL
284 | if 10 <= value <= 15:
285 | return StationType.RESERVED
286 | return StationType.ALL
287 |
288 | @classmethod
289 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["StationType"]:
290 | return cls(v) if v is not None else None
291 |
292 |
293 | class StationIntervals(IntEnum):
294 | AUTONOMOUS_MODE = 0
295 | MINUTES_10 = 1
296 | MINUTES_6 = 2
297 | MINUTES_3 = 3
298 | MINUTES_1 = 4
299 | SECONDS_30 = 5
300 | SECONDS_15 = 6
301 | SECONDS_10 = 7
302 | SECONDS_5 = 8
303 | NEXT_SHORT_REPORTER_INTERVAL = 9
304 | NEXT_LONGER_REPORTING_INTERVAL = 10
305 | RESERVED = 11
306 |
307 | @classmethod
308 | def _missing_(cls, value: object) -> int:
309 | return StationIntervals.RESERVED
310 |
311 | @classmethod
312 | def from_value(cls, v: typing.Optional[typing.Any]) -> typing.Optional["StationIntervals"]:
313 | return cls(v) if v is not None else None
314 |
315 |
316 | class SyncState(IntEnum):
317 | """
318 | https://www.navcen.uscg.gov/?pageName=AISMessagesA#Sync
319 | """
320 | UTC_DIRECT = 0x00
321 | UTC_INDIRECT = 0x01
322 | BASE_DIRECT = 0x02
323 | BASE_INDIRECT = 0x03
324 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/decode.py:
--------------------------------------------------------------------------------
1 | import typing
2 |
3 | from pyais.exceptions import TooManyMessagesException, MissingMultipartMessageException
4 | from pyais.messages import NMEAMessage, ANY_MESSAGE
5 |
6 |
7 | def _assemble_messages(*args: bytes) -> NMEAMessage:
8 | # Convert bytes into NMEAMessage and remember fragment_count and fragment_numbers
9 | temp: typing.List[NMEAMessage] = []
10 | frags: typing.List[int] = []
11 | frag_cnt: int = 1
12 | for msg in args:
13 | nmea = NMEAMessage(msg)
14 | temp.append(nmea)
15 | frags.append(nmea.frag_num)
16 | frag_cnt = nmea.fragment_count
17 |
18 | # Make sure provided parts assemble a single (multiline message)
19 | if len(args) > frag_cnt:
20 | raise TooManyMessagesException(f"Got {len(args)} messages, but fragment count is {frag_cnt}")
21 |
22 | # Make sure all parts of a multipart message are provided
23 | diff = [x for x in range(1, frag_cnt + 1) if x not in frags]
24 | if len(diff):
25 | raise MissingMultipartMessageException(f"Missing fragment numbers: {diff}")
26 |
27 | # Assemble temporary messages
28 | final = NMEAMessage.assemble_from_iterable(temp)
29 | return final
30 |
31 |
32 | def decode(*args: typing.Union[str, bytes]) -> ANY_MESSAGE:
33 | parts = tuple(msg.encode('utf-8') if isinstance(msg, str) else msg for msg in args)
34 | nmea = _assemble_messages(*parts)
35 | return nmea.decode()
36 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/encode.py:
--------------------------------------------------------------------------------
1 | import math
2 | import typing
3 |
4 | from pyais.messages import Payload, MSG_CLASS
5 | from pyais.util import chunks, compute_checksum
6 |
7 | # Types
8 | DATA_DICT = typing.Dict[str, typing.Union[str, int, float, bytes, bool]]
9 | AIS_SENTENCES = typing.List[str]
10 |
11 |
12 | def get_ais_type(data: DATA_DICT) -> int:
13 | """
14 | Get the message type from a set of keyword arguments. The first occurence of either
15 | `type` or `msg_type` will be used.
16 | """
17 | keys = ['type', 'msg_type']
18 | length = len(keys) - 1
19 | for i, key in enumerate(keys):
20 | try:
21 | ais_type = data[key]
22 | return int(ais_type)
23 | except (KeyError, ValueError) as err:
24 | if i == length:
25 | raise ValueError("Missing or invalid AIS type. Must be a number.") from err
26 | raise ValueError("Missing type")
27 |
28 |
29 | def data_to_payload(ais_type: int, data: DATA_DICT) -> Payload:
30 | try:
31 | return MSG_CLASS[ais_type].create(**data)
32 | except KeyError as err:
33 | raise ValueError(f"AIS message type {ais_type} is not supported") from err
34 |
35 |
36 | def ais_to_nmea_0183(payload: str, ais_talker_id: str, radio_channel: str, fill_bits: int) -> AIS_SENTENCES:
37 | """
38 | Splits the AIS payload into sentences, ASCII encodes the payload, creates
39 | and sends the relevant NMEA 0183 sentences.
40 |
41 | HINT:
42 | This method takes care of splitting large payloads (larger than 60 characters)
43 | into multiple sentences. With a total of 80 maximum chars excluding end of line
44 | per sentence, and 20 chars head + tail in the nmea 0183 carrier protocol, 60
45 | chars remain for the actual payload.
46 |
47 | @param payload: Armored AIs payload.
48 | @param ais_talker_id: AIS talker ID (AIVDO or AIVDM)
49 | @param radio_channel: Radio channel (either A or B)
50 | @param fill_bits: The number of fill bits requires to pad the data payload to a 6 bit boundary.
51 | @return: A list of relevant AIS sentences.
52 | """
53 | messages = []
54 | max_len = 61
55 | seq_id = ''
56 | frag_cnt = math.ceil(len(payload) / max_len)
57 |
58 | if len(ais_talker_id) != 5:
59 | raise ValueError("AIS talker is must have exactly 6 characters. E.g. AIVDO")
60 |
61 | if len(radio_channel) != 1:
62 | raise ValueError("Radio channel must be a single character")
63 |
64 | for frag_num, chunk in enumerate(chunks(payload, max_len), start=1):
65 | tpl = "!{},{},{},{},{},{},{}*{:02X}"
66 | dummy_message = tpl.format(ais_talker_id, frag_cnt, frag_num, seq_id, radio_channel, chunk, fill_bits, 0)
67 | checksum = compute_checksum(dummy_message)
68 | fill_bits_frag = fill_bits if frag_num == frag_cnt else 0 # Make sure we set fill bits only for last fragment
69 | msg = tpl.format(ais_talker_id, frag_cnt, frag_num, seq_id, radio_channel, chunk, fill_bits_frag, checksum)
70 | messages.append(msg)
71 |
72 | return messages
73 |
74 |
75 | def encode_dict(data: DATA_DICT, talker_id: str = "AIVDO", radio_channel: str = "A") -> AIS_SENTENCES:
76 | """
77 | Takes a dictionary of data and some NMEA specific kwargs and returns the NMEA 0183 encoded AIS sentence.
78 |
79 | Notes:
80 | - the data dict should also contain the AIS message type (1-27) under the `type` key.
81 | - different messages take different keywords. Refer to the payload classes above to get a glimpse
82 | on what fields each AIS message can take.
83 |
84 | @param data: The AIS data as a dictionary.
85 | @param talker_id: AIS packets have the introducer "AIVDM" or "AIVDO";
86 | AIVDM packets are reports from other ships and AIVDO packets are reports from your own ship.
87 | @param radio_channel: The radio channel. Can be either 'A' (default) or 'B'.
88 | @return: NMEA 0183 encoded AIS sentences.
89 |
90 | """
91 | if talker_id not in ("AIVDM", "AIVDO"):
92 | raise ValueError("talker_id must be any of ['AIVDM', 'AIVDO']")
93 |
94 | if radio_channel not in ('A', 'B'):
95 | raise ValueError("radio_channel must be any of ['A', 'B']")
96 |
97 | ais_type = get_ais_type(data)
98 | payload = data_to_payload(ais_type, data)
99 | armored_payload, fill_bits = payload.encode()
100 | return ais_to_nmea_0183(armored_payload, talker_id, radio_channel, fill_bits)
101 |
102 |
103 | def encode_msg(msg: Payload, talker_id: str = "AIVDO", radio_channel: str = "A") -> AIS_SENTENCES:
104 | if talker_id not in ("AIVDM", "AIVDO"):
105 | raise ValueError("talker_id must be any of ['AIVDM', 'AIVDO']")
106 |
107 | if radio_channel not in ('A', 'B'):
108 | raise ValueError("radio_channel must be any of ['A', 'B']")
109 |
110 | armored_payload, fill_bits = msg.encode()
111 | return ais_to_nmea_0183(armored_payload, talker_id, radio_channel, fill_bits)
112 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/exceptions.py:
--------------------------------------------------------------------------------
1 | class AISBaseException(Exception):
2 | """The base exception for all exceptions"""
3 |
4 |
5 | class InvalidNMEAMessageException(AISBaseException):
6 | """Invalid NMEA Message"""
7 | pass
8 |
9 |
10 | class UnknownMessageException(AISBaseException):
11 | """Message not supported yet"""
12 | pass
13 |
14 |
15 | class MissingMultipartMessageException(AISBaseException):
16 | """Multipart message with missing parts provided"""
17 |
18 |
19 | class TooManyMessagesException(AISBaseException):
20 | """Too many messages"""
21 |
22 |
23 | class UnknownPartNoException(AISBaseException):
24 | """Unknown part number"""
25 |
26 |
27 | class InvalidDataTypeException(AISBaseException):
28 | """An Unknown data type was passed to an encoding/decoding function"""
29 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/stream.py:
--------------------------------------------------------------------------------
1 | import typing
2 | from abc import ABC, abstractmethod
3 | from socket import AF_INET, SOCK_DGRAM, SOCK_STREAM, socket
4 | from typing import (
5 | BinaryIO, Generator, Generic, Iterable, List, TypeVar, cast
6 | )
7 |
8 | from pyais.exceptions import InvalidNMEAMessageException
9 | from pyais.messages import NMEAMessage
10 |
11 | F = TypeVar("F", BinaryIO, socket, None)
12 | DOLLAR_SIGN = ord("$")
13 | EXCLAMATION_POINT = ord("!")
14 |
15 |
16 | def should_parse(byte_str: bytes) -> bool:
17 | """Return True if a given byte string seems to be NMEA message.
18 | This method does **NOT** validate the message, but uses a heuristic
19 | approach to check (or guess) if byte string is a valid nmea_message.
20 | """
21 | # The byte sequence is not empty and starts with a $ or a ! and has 6 ','
22 | return len(byte_str) > 0 and byte_str[0] in (DOLLAR_SIGN, EXCLAMATION_POINT) and byte_str.count(b",") == 6
23 |
24 |
25 | class AssembleMessages(ABC):
26 | """
27 | Base class that assembles multiline messages.
28 | Offers a iterator like interface.
29 |
30 | This class comes without a __init__ method, because it should never be instantiated!
31 | """
32 |
33 | def __enter__(self) -> "AssembleMessages":
34 | # Enables use of with statement
35 | return self
36 |
37 | def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
38 | return None
39 |
40 | def __iter__(self) -> Generator[NMEAMessage, None, None]:
41 | return self._assemble_messages()
42 |
43 | def __next__(self) -> NMEAMessage:
44 | """Returns the next decoded NMEA message."""
45 | return next(iter(self))
46 |
47 | def _assemble_messages(self) -> Generator[NMEAMessage, None, None]:
48 | buffer: typing.Dict[typing.Tuple[int, str], typing.List[typing.Optional[NMEAMessage]]] = {}
49 |
50 | messages = self._iter_messages()
51 | for line in messages:
52 |
53 | try:
54 | msg: NMEAMessage = NMEAMessage(line)
55 | except InvalidNMEAMessageException:
56 | # Be gentle and just skip invalid messages
57 | continue
58 |
59 | if msg.is_single:
60 | yield msg
61 | else:
62 | # Instead of None use -1 as a seq_id
63 | seq_id = msg.seq_id
64 | if seq_id is None:
65 | seq_id = -1
66 |
67 | # seq_id and channel make a unique stream
68 | slot = (seq_id, msg.channel)
69 |
70 | if slot not in buffer:
71 | # Create a new array in the buffer that has enough space for all fragments
72 | buffer[slot] = [None, ] * max(msg.fragment_count, 0xff)
73 |
74 | buffer[slot][msg.frag_num - 1] = msg
75 | msg_parts = buffer[slot][0:msg.fragment_count]
76 |
77 | # Check if all fragments are found
78 | not_none_parts = [m for m in msg_parts if m is not None]
79 | if len(not_none_parts) == msg.fragment_count:
80 | yield NMEAMessage.assemble_from_iterable(not_none_parts)
81 | del buffer[slot]
82 |
83 | @abstractmethod
84 | def _iter_messages(self) -> Generator[bytes, None, None]:
85 | raise NotImplementedError("Implement me!")
86 |
87 |
88 | class IterMessages(AssembleMessages):
89 |
90 | def __init__(self, messages: Iterable[bytes]):
91 | # If the user passes a single byte string make it into a list
92 | if isinstance(messages, bytes):
93 | messages = [messages, ]
94 | self.messages: Iterable[bytes] = messages
95 |
96 | @classmethod
97 | def from_strings(cls, messages: Iterable[str], ignore_encoding_errors: bool = False,
98 | encoding: str = "utf-8") -> "IterMessages":
99 | # If the users passes a single message as string, make it a list
100 | if isinstance(messages, str):
101 | messages = [messages, ]
102 |
103 | encoded: List[bytes] = []
104 | for message in messages:
105 | try:
106 | encoded.append(message.encode(encoding))
107 | except UnicodeEncodeError as e:
108 | if ignore_encoding_errors:
109 | # Just skip and carry on
110 | continue
111 | raise e
112 |
113 | return IterMessages(encoded)
114 |
115 | def _iter_messages(self) -> Generator[bytes, None, None]:
116 | # Transform self.messages into a generator
117 | yield from (message for message in self.messages)
118 |
119 |
120 | class Stream(AssembleMessages, Generic[F], ABC):
121 |
122 | def __init__(self, fobj: F) -> None:
123 | """
124 | Create a new Stream-like object.
125 | @param fobj: A file-like or socket object.
126 | """
127 | self._fobj: F = fobj
128 |
129 | def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
130 | if self._fobj is not None:
131 | self._fobj.close()
132 |
133 | def _iter_messages(self) -> Generator[bytes, None, None]:
134 | # Do not parse lines, that are obviously not NMEA messages
135 | yield from (line for line in self.read() if should_parse(line))
136 |
137 | @abstractmethod
138 | def read(self) -> Generator[bytes, None, None]:
139 | raise NotImplementedError()
140 |
141 |
142 | class BinaryIOStream(Stream[BinaryIO]):
143 | """Read messages from a file-like object"""
144 |
145 | def __init__(self, file: BinaryIO) -> None:
146 | super().__init__(file)
147 |
148 | def read(self) -> Generator[bytes, None, None]:
149 | yield from self._fobj.readlines()
150 |
151 |
152 | class FileReaderStream(BinaryIOStream):
153 | """
154 | Read NMEA messages from file
155 | """
156 |
157 | def __init__(self, filename: str, mode: str = "rb") -> None:
158 | self.filename: str = filename
159 | self.mode: str = mode
160 | # Try to open file
161 | try:
162 | file = open(self.filename, mode=self.mode)
163 | file = cast(BinaryIO, file)
164 | except Exception as e:
165 | raise FileNotFoundError(f"Could not open file {self.filename}") from e
166 | super().__init__(file)
167 |
168 |
169 | class ByteStream(Stream[None]):
170 | """
171 | Takes a iterable that contains ais messages as bytes and assembles them.
172 | """
173 |
174 | def __init__(self, iterable: Iterable[bytes]) -> None:
175 | self.iterable: Iterable[bytes] = iterable
176 | super().__init__(None)
177 |
178 | def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
179 | return
180 |
181 | def read(self) -> Generator[bytes, None, None]:
182 | yield from self.iterable
183 |
184 |
185 | class SocketStream(Stream[socket]):
186 | BUF_SIZE = 4096
187 |
188 | def recv(self) -> bytes:
189 | return b""
190 |
191 | def read(self) -> Generator[bytes, None, None]:
192 | partial: bytes = b''
193 | while True:
194 | body = self.recv()
195 |
196 | # Server closed connection
197 | if not body:
198 | return None
199 |
200 | lines = body.split(b'\r\n')
201 |
202 | line = partial + lines[0]
203 | if line:
204 | yield line
205 |
206 | yield from (line for line in lines[1:-1] if line)
207 |
208 | partial = lines[-1]
209 |
210 |
211 | class UDPReceiver(SocketStream):
212 |
213 | def __init__(self, host: str, port: int) -> None:
214 | sock: socket = socket(AF_INET, SOCK_DGRAM)
215 | sock.bind((host, port))
216 | super().__init__(sock)
217 |
218 | def recv(self) -> bytes:
219 | return self._fobj.recvfrom(self.BUF_SIZE)[0]
220 |
221 |
222 | class TCPConnection(SocketStream):
223 | """
224 | Read AIS data from a remote TCP server
225 | https://en.wikipedia.org/wiki/NMEA_0183
226 | """
227 |
228 | def recv(self) -> bytes:
229 | return self._fobj.recv(self.BUF_SIZE)
230 |
231 | def __init__(self, host: str, port: int = 80) -> None:
232 | sock: socket = socket(AF_INET, SOCK_STREAM)
233 | try:
234 | sock.connect((host, port))
235 | except ConnectionRefusedError as e:
236 | sock.close()
237 | raise ConnectionRefusedError(f"Failed to connect to {host}:{port}") from e
238 | super().__init__(sock)
239 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/pyais/util.py:
--------------------------------------------------------------------------------
1 | import base64
2 | import typing
3 | from collections import OrderedDict
4 | from functools import partial, reduce
5 | from operator import xor
6 | from typing import Any, Generator, Hashable, TYPE_CHECKING, Union, Dict
7 |
8 | from bitarray import bitarray
9 |
10 | from pyais.constants import SyncState
11 |
12 | if TYPE_CHECKING:
13 | BaseDict = OrderedDict[Hashable, Any]
14 | else:
15 | BaseDict = OrderedDict
16 |
17 | from_bytes = partial(int.from_bytes, byteorder="big")
18 | from_bytes_signed = partial(int.from_bytes, byteorder="big", signed=True)
19 |
20 | T = typing.TypeVar('T')
21 |
22 |
23 | def decode_into_bit_array(data: bytes, fill_bits: int = 0) -> bitarray:
24 | """
25 | Decodes a raw AIS message into a bitarray.
26 | :param data: Raw AIS message in bytes
27 | :param fill_bits: Number of trailing fill bits to be ignored
28 | :return:
29 | """
30 | bit_arr = bitarray()
31 | length = len(data)
32 | for i, c in enumerate(data):
33 | if c < 0x30 or c > 0x77 or 0x57 < c < 0x6:
34 | raise ValueError(f"Invalid character: {chr(c)}")
35 |
36 | # Convert 8 bit binary to 6 bit binary
37 | c -= 0x30 if (c < 0x60) else 0x38
38 | c &= 0x3F
39 |
40 | if i == length - 1 and fill_bits:
41 | # The last part be shorter than 6 bits and contain fill bits
42 | c = c >> fill_bits
43 | bit_arr += bitarray(f'{c:b}'.zfill(6 - fill_bits))
44 | else:
45 | bit_arr += bitarray(f'{c:06b}')
46 |
47 | return bit_arr
48 |
49 |
50 | def chunks(sequence: typing.Sequence[T], n: int) -> Generator[typing.Sequence[T], None, None]:
51 | """Yield successive n-sized chunks from sequence."""
52 | return (sequence[i:i + n] for i in range(0, len(sequence), n))
53 |
54 |
55 | def decode_bin_as_ascii6(bit_arr: bitarray) -> str:
56 | """
57 | Decode binary data as 6 bit ASCII.
58 | :param bit_arr: array of bits
59 | :return: ASCII String
60 | """
61 | string: str = ""
62 | c: bitarray
63 | for c in chunks(bit_arr, 6): # type:ignore
64 | n: int = from_bytes(c.tobytes()) >> 2
65 |
66 | # Last entry may not have 6 bits
67 | if len(c) != 6:
68 | n >> (6 - len(c))
69 |
70 | if n < 0x20:
71 | n += 0x40
72 |
73 | # Break if there is an @
74 | if n == 64:
75 | break
76 |
77 | string += chr(n)
78 |
79 | return string.strip()
80 |
81 |
82 | def get_int(data: bitarray, ix_low: int, ix_high: int, signed: bool = False) -> int:
83 | """
84 | Cast a subarray of a bitarray into an integer.
85 | The bitarray module adds tailing zeros when calling tobytes(), if the bitarray is not a multiple of 8.
86 | So those need to be shifted away.
87 | :param data: some bitarray
88 | :param ix_low: the lower index of the sub-array
89 | :param ix_high: the upper index of the sub-array
90 | :param signed: True if the value should be interpreted as a signed integer
91 | :return: a normal integer (int)
92 | """
93 | shift: int = (8 - ((ix_high - ix_low) % 8)) % 8
94 | data = data[ix_low:ix_high]
95 | i: int = from_bytes_signed(data) if signed else from_bytes(data)
96 | return i >> shift
97 |
98 |
99 | def compute_checksum(msg: Union[str, bytes]) -> int:
100 | """
101 | Compute the checksum of a given message.
102 | This method takes the **whole** message including the leading `!`.
103 |
104 | >>> compute_checksum(b"!AIVDM,1,1,,B,15M67FC000G?ufbE`FepT@3n00Sa,0")
105 | 91
106 |
107 | :param msg: message
108 | :return: int value of the checksum. Format as hex with `f'{checksum:02x}'`
109 | """
110 | if isinstance(msg, str):
111 | msg = msg.encode()
112 |
113 | msg = msg[1:].split(b'*', 1)[0]
114 | return reduce(xor, msg)
115 |
116 |
117 | # https://gpsd.gitlab.io/gpsd/AIVDM.html#_aivdmaivdo_payload_armoring
118 | PAYLOAD_ARMOR = {
119 | 0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: ':',
120 | 11: ';', 12: '<', 13: '=', 14: '>', 15: '?', 16: '@', 17: 'A', 18: 'B', 19: 'C', 20: 'D',
121 | 21: 'E', 22: 'F', 23: 'G', 24: 'H', 25: 'I', 26: 'J', 27: 'K', 28: 'L', 29: 'M', 30: 'N',
122 | 31: 'O', 32: 'P', 33: 'Q', 34: 'R', 35: 'S', 36: 'T', 37: 'U', 38: 'V', 39: 'W', 40: '`',
123 | 41: 'a', 42: 'b', 43: 'c', 44: 'd', 45: 'e', 46: 'f', 47: 'g', 48: 'h', 49: 'i', 50: 'j',
124 | 51: 'k', 52: 'l', 53: 'm', 54: 'n', 55: 'o', 56: 'p', 57: 'q', 58: 'r', 59: 's', 60: 't',
125 | 61: 'u', 62: 'v', 63: 'w'
126 | }
127 |
128 | # https://gpsd.gitlab.io/gpsd/AIVDM.html#_ais_payload_data_types
129 | SIX_BIT_ENCODING = {
130 | '@': 0, 'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7, 'H': 8, 'I': 9, 'J': 10,
131 | 'K': 11, 'L': 12, 'M': 13, 'N': 14, 'O': 15, 'P': 16, 'Q': 17, 'R': 18, 'S': 19, 'T': 20,
132 | 'U': 21, 'V': 22, 'W': 23, 'X': 24, 'Y': 25, 'Z': 26, '[': 27, '\\': 28, ']': 29, '^': 30,
133 | '_': 31, ' ': 32, '!': 33, '"': 34, '#': 35, '$': 36, '%': 37, '&': 38, '\'': 39, '(': 40,
134 | ')': 41, '*': 42, '+': 43, ',': 44, '-': 45, '.': 46, '/': 47, '0': 48, '1': 49, '2': 50,
135 | '3': 51, '4': 52, '5': 53, '6': 54, '7': 55, '8': 56, '9': 57, ':': 58, ';': 59, '<': 60,
136 | '=': 61, '>': 62, '?': 63
137 | }
138 |
139 |
140 | def to_six_bit(char: str) -> str:
141 | """
142 | Encode a single character as six-bit bitstring.
143 | @param char: The character to encode
144 | @return: The six-bit representation as string
145 | """
146 | char = char.upper()
147 | try:
148 | encoding = SIX_BIT_ENCODING[char]
149 | return f"{encoding:06b}"
150 | except KeyError:
151 | raise ValueError(f"received char '{char}' that cant be encoded")
152 |
153 |
154 | def encode_ascii_6(bits: bitarray) -> typing.Tuple[str, int]:
155 | """
156 | Transform the bitarray to an ASCII-encoded bit vector.
157 | Each character represents six bits of data.
158 | @param bits: The bitarray to convert to an ASCII-encoded bit vector.
159 | @return: ASCII-encoded bit vector and the number of fill bits required to pad the data payload to a 6 bit boundary.
160 | """
161 | out = ""
162 | chunk: bitarray
163 | padding = 0
164 | for chunk in chunks(bits, 6): # type:ignore
165 | padding = 6 - len(chunk)
166 | num = from_bytes(chunk.tobytes()) >> 2
167 | armor = PAYLOAD_ARMOR[num]
168 | out += armor
169 | return out, padding
170 |
171 |
172 | def int_to_bytes(val: typing.Union[int, bytes]) -> int:
173 | """
174 | Convert a bytes object to an integer. Byteorder is big.
175 |
176 | @param val: A bytes object to convert to an int. If the value is already an int, this is a NO-OP.
177 | @return: Integer representation of `val`
178 | """
179 | if isinstance(val, int):
180 | return val
181 | return int.from_bytes(val, 'big')
182 |
183 |
184 | def bits2bytes(bits: typing.Union[str, bitarray]) -> bytes:
185 | """
186 | Convert a bitstring or a bitarray to bytes.
187 | >>> bits2bytes('00100110')
188 | b'&'
189 | """
190 | bits = bitarray(bits)
191 | return bits.tobytes()
192 |
193 |
194 | def bytes2bits(in_bytes: bytes, default: typing.Optional[bitarray] = None) -> bitarray:
195 | """
196 | Convert a bytes object to a bitarray.
197 |
198 | @param in_bytes : The bytes to encode
199 | @param default : A default value to return if `in_bytes` is *Falseish*
200 |
201 | >>> bytes2bits(b'&')
202 | bitarray('00100110')
203 | """
204 | if default is not None and not in_bytes:
205 | return default
206 | bits = bitarray(endian='big')
207 | bits.frombytes(in_bytes)
208 | return bits
209 |
210 |
211 | def b64encode_str(val: bytes, encoding: str = 'utf-8') -> str:
212 | """BASE64 encoded a bytes string and returns the result as UTF-8 string"""
213 | return base64.b64encode(val).decode(encoding)
214 |
215 |
216 | def coerce_val(val: typing.Any, d_type: typing.Type[T]) -> T:
217 | """Forces a given value in a given datatype"""
218 | if d_type == bytes and not isinstance(val, bytes):
219 | raise ValueError(f"Expected bytes, but got: {type(val)}")
220 |
221 | return d_type(val) # type: ignore
222 |
223 |
224 | def int_to_bin(val: typing.Union[int, bool], width: int, signed: bool = True) -> bitarray:
225 | """
226 | Convert an integer or boolean value to binary. If the value is too great to fit into
227 | `width` bits, the maximum possible number that still fits is used.
228 |
229 | @param val: Any integer or boolean value.
230 | @param width: The bit width. If less than width bits are required, leading zeros are added.
231 | @param signed: Set to True/False if the value is signed or not.
232 | @return: The binary representation of value with exactly width bits. Type is bitarray.
233 | """
234 | # Compute the total number of bytes required to hold up to `width` bits.
235 | n_bytes, mod = divmod(width, 8)
236 | if mod > 0:
237 | n_bytes += 1
238 |
239 | # If the value is too big, return a bitarray of all 1's
240 | mask = (1 << width) - 1
241 | if val >= mask:
242 | return bitarray('1' * width)
243 |
244 | bits = bitarray(endian='big')
245 | bits.frombytes(val.to_bytes(n_bytes, 'big', signed=signed))
246 | return bits[8 - mod if mod else 0:]
247 |
248 |
249 | def str_to_bin(val: str, width: int) -> bitarray:
250 | """
251 | Convert a string value to binary using six-bit ASCII encoding up to `width` chars.
252 |
253 | @param val: The string to first convert to six-bit ASCII and then to binary.
254 | @param width: The width of the full string. If the string has fewer characters than width, trailing '@' are added.
255 | @return: The binary representation of value with exactly width bits. Type is bitarray.
256 | """
257 | out = bitarray(endian='big')
258 |
259 | # Each char will be converted to a six-bit binary vector.
260 | # Therefore, the total number of chars is floor(WIDTH / 6).
261 | num_chars = int(width / 6)
262 |
263 | # Add trailing '@' if the string is shorter than `width`
264 | for _ in range(num_chars - len(val)):
265 | val += "@"
266 |
267 | # Encode AT MOST width characters
268 | for char in val[:num_chars]:
269 | # Covert each char to six-bit ASCII vector
270 | txt = to_six_bit(char)
271 | out += bitarray(txt)
272 |
273 | return out
274 |
275 |
276 | def chk_to_int(chk_str: bytes) -> typing.Tuple[int, int]:
277 | """
278 | Converts a checksum string to a tuple of (fillbits, checksum).
279 | >>> chk_to_int(b"0*1B")
280 | (0, 27)
281 | """
282 | if not len(chk_str):
283 | return 0, -1
284 |
285 | fill_bits: int = int(chr(chk_str[0]))
286 | try:
287 | checksum = int(chk_str[2:], 16)
288 | except (IndexError, ValueError):
289 | checksum = -1
290 | return fill_bits, checksum
291 |
292 |
293 | SYNC_MASK = 0x03
294 | TIMEOUT_MASK = 0x07
295 | MSG_MASK = 0x3fff
296 | SLOT_INCREMENT_MASK = 0x1fff
297 |
298 |
299 | def get_sotdma_comm_state(radio: int) -> Dict[str, typing.Optional[int]]:
300 | """
301 | The SOTDMA communication state is structured as follows:
302 | +-------------------+----------------------+------------------------------------------------------------------------------------------------+
303 | | Parameter | Number of bits | Description |
304 | +-------------------+----------------------+------------------------------------------------------------------------------------------------+
305 | | Sync state | 2 | 0 UTC direct |
306 | | | | 1 UTC indirect |
307 | | | | 2 Station is synchronized to a base station |
308 | | | | 3 Station is synchronized to another station based on the highest number of received stations |
309 | | Slot time-out | 3 | Specifies frames remaining until a new slot is selected |
310 | | | | 0 means that this was the last transmission in this slot |
311 | | | | 1-7 means that 1 to 7 frames respectively are left until slot change |
312 | | Sub message | 14 | 14 The sub message depends on the current value in slot time-out |
313 | +-------------------+----------------------+------------------------------------------------------------------------------------------------+
314 |
315 | The slot time-out defines how to interpret the sub message:
316 | +-----------------+---------------------------------------------------------------------------+
317 | | Slot time-out | Description |
318 | +-----------------+---------------------------------------------------------------------------+
319 | | 3, 5, 7 | Number of receiving stations (not own station) (between 0 and 16 383) |
320 | | 2, 4, 6 | Slot number Slot number used for this transmission (between 0 and 2 249) |
321 | | 1 | UTC hour (bits 13 to 9) and minute (bits 8 to 2) |
322 | | 0 | Next frame |
323 | +-----------------+---------------------------------------------------------------------------+
324 |
325 | You may refer to:
326 | - https://github.com/M0r13n/pyais/issues/17
327 | - https://www.itu.int/dms_pubrec/itu-r/rec/m/R-REC-M.1371-1-200108-S!!PDF-E.pdf
328 | - https://www.navcen.uscg.gov/?pageName=AISMessagesA#Sync
329 | """
330 | result = {
331 | 'received_stations': None,
332 | 'slot_number': None,
333 | 'utc_hour': None,
334 | 'utc_minute': None,
335 | 'slot_offset': None,
336 | 'slot_timeout': 0,
337 | 'sync_state': 0,
338 | }
339 |
340 | sync_state = (radio >> 17) & SYNC_MASK # First two (2) bits
341 | slot_timeout = (radio >> 14) & TIMEOUT_MASK # Next three (3) bits
342 | sub_msg = radio & MSG_MASK # Last 14 bits
343 |
344 | if slot_timeout == 0:
345 | result['slot_offset'] = sub_msg
346 | elif slot_timeout == 1:
347 | result['utc_hour'] = (sub_msg >> 9) & 0xf
348 | result['utc_minute'] = (sub_msg >> 2) & 0x3f
349 | elif slot_timeout in (2, 4, 6):
350 | result['slot_number'] = sub_msg
351 | elif slot_timeout in (3, 5, 7):
352 | result['received_stations'] = sub_msg
353 | else:
354 | raise ValueError("Slot timeout can only be an integer between 0 and 7")
355 |
356 | result['sync_state'] = SyncState(sync_state)
357 | result['slot_timeout'] = slot_timeout
358 | return result
359 |
360 |
361 | def get_itdma_comm_state(radio: int) -> Dict[str, typing.Optional[int]]:
362 | """
363 | +-----------------+------+--------------------------------------------------------------------------------+
364 | | Parameter | Bits | Description |
365 | +-----------------+------+--------------------------------------------------------------------------------+
366 | | Sync state | 2 | 0 UTC direct |
367 | | | | 1 UTC indirec |
368 | | | | 2 Station is synchronized to a base station |
369 | | | | 3 Station is synchronized to another station |
370 | | Slot increment | 13 | Offset to next slot to be used, or zero (0) if no more transmissions |
371 | | Number of slots | 3 | Number of consecutive slots to allocate. (0 = 1 slot, 1 = 2 slots,2 = 3 slots, |
372 | | | | 3 = 4 slots, 4 = 5 slots) |
373 | | Keep flag | 1 | Set to TRUE = 1 if the slot remains allocated for one additional frame |
374 | +-----------------+------+--------------------------------------------------------------------------------+
375 |
376 | You may refer to:
377 | - https://github.com/M0r13n/pyais/issues/17
378 | - https://www.itu.int/dms_pubrec/itu-r/rec/m/R-REC-M.1371-1-200108-S!!PDF-E.pdf
379 | - https://www.navcen.uscg.gov/?pageName=AISMessagesA#Sync
380 | """
381 |
382 | sync_state = (radio >> 17) & SYNC_MASK # First two (2) bits
383 | slot_increment = (radio >> 4) & SLOT_INCREMENT_MASK # Next 13 bits
384 | num_slots = (radio >> 1) & TIMEOUT_MASK # Next three (3) bits
385 | keep_flag = radio & 0x01 # Last bit
386 |
387 | return {
388 | 'keep_flag': keep_flag,
389 | 'sync_state': sync_state,
390 | 'slot_increment': slot_increment,
391 | 'num_slots': num_slots,
392 | 'keep_flag': keep_flag,
393 | }
394 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_encoder_decoder/requirements.txt:
--------------------------------------------------------------------------------
1 | attr==0.3.2
2 | attrs==21.4.0
3 | bitarray==2.5.1
4 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_interpolate/example/trajinterpolation.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | from pyais import traj_interpolation
4 |
5 |
6 | def main():
7 | traj_file = '../data/1.csv'
8 | res = 30
9 | num = None
10 | gpx_data = traj_interpolation.traj_load(traj_file)
11 | traj_data_interp = traj_interpolation.traj_interpolate(traj_file, res, num)
12 |
13 | plt.plot(gpx_data['lon'], gpx_data['lat'], marker='*', ms=5, linestyle='--', color='red', linewidth=0.5,
14 | label='Raw trajectory')
15 | plt.plot(traj_data_interp['lon'], traj_data_interp['lat'], marker='o', ms=2, linestyle='-', color='g',
16 | linewidth=0.5, label='Interpolated trajectory')
17 | plt.yticks(fontproperties='Times New Roman', size=12)
18 | plt.xticks(fontproperties='Times New Roman', size=12)
19 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14})
20 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14})
21 | plt.title('Trajectory Interpolation', fontdict={'family': 'Times New Roman', 'size': 16})
22 | plt.ticklabel_format(useOffset=False, style='plain')
23 | plt.grid()
24 | plt.legend()
25 | plt.tight_layout()
26 | plt.show()
27 |
28 |
29 | if __name__ == '__main__':
30 | main()
31 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_interpolate/pyais/__pycache__/traj_interpolation.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_interpolate/pyais/__pycache__/traj_interpolation.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_interpolate/pyais/traj_interpolation.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | from scipy.interpolate import pchip_interpolate
4 | from typing import Dict, List, Union, Optional
5 | from vincenty import vincenty
6 |
7 | # types
8 | TrajData = Dict[str, Union[List[float]]]
9 |
10 |
11 | def traj_load(traj_file: str) -> TrajData:
12 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []}
13 | df = pd.read_csv(traj_file)
14 | dt_c = df.copy(deep=True)
15 | dt_c['DRLONGITUDE'] = dt_c['DRLONGITUDE'].map(lambda x: x / 1.0)
16 | dt_c['DRLATITUDE'] = dt_c['DRLATITUDE'].map(lambda x: x / 1.0)
17 | dt_c['DRSPEED'] = dt_c['DRSPEED'].map(lambda x: x / 1.0)
18 | for index, row in dt_c.iterrows():
19 | trajs_data['lat'].append(row['DRLATITUDE'])
20 | trajs_data['lon'].append(row['DRLONGITUDE'])
21 | trajs_data['speed'].append(row['DRSPEED'])
22 | trajs_data['tstamp'].append(row['DRGPSTIME'])
23 | return trajs_data
24 |
25 |
26 | def traj_calculate_distance(traj_data: TrajData) -> List[float]:
27 | traj_dist = np.zeros(len(traj_data['lat']))
28 | for i in range(len(traj_dist) - 1):
29 | lat1 = traj_data['lat'][i]
30 | lon1 = traj_data['lon'][i]
31 | lat2 = traj_data['lat'][i + 1]
32 | lon2 = traj_data['lon'][i + 1]
33 | pts = (lat1, lon1)
34 | pte = (lat2, lon2)
35 | s = vincenty(pts, pte) * 1000 # unit meter
36 | traj_dist[i + 1] = s
37 | return traj_dist.tolist()
38 |
39 |
40 | def traj_interpolate(traj_file: str, res: float = 1.0, num: Optional[int] = None) -> TrajData:
41 | '''
42 | :param traj_data: raw trajectory filename
43 | :param res: time resolution
44 | :param num: None
45 | :return: interpolated trajectory
46 | '''
47 | traj_data = traj_load(traj_file)
48 | if res <= 0.0:
49 | raise ValueError('res must be > 0.0')
50 | if num is not None and num < 0:
51 | raise ValueError('num must be >= 0')
52 | _traj_dist = traj_calculate_distance(traj_data)
53 | xi = np.cumsum(_traj_dist)
54 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]])
55 | num = num if num is not None else int(np.ceil(xi[-1] / res))
56 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True)
57 | y = pchip_interpolate(xi, yi, x, axis=1)
58 | traj_data_interp = {'lat': list(y[0, :]), 'lon': list(y[1, :]), 'speed': list(y[2, :]), 'tstamp': list(y[-1, :])}
59 | return traj_data_interp
60 |
61 |
62 | def traj_calculate_distance_ts(traj_data: TrajData) -> List[float]:
63 | traj_dist = np.zeros(len(traj_data['lat']))
64 | for i in range(len(traj_dist) - 1):
65 | s = int(traj_data['tstamp'][i + 1]) - int(traj_data['tstamp'][i])
66 | traj_dist[i + 1] = s
67 | return traj_dist.tolist()
68 |
69 |
70 | def traj_interpolate_df(traj_data, res: float = 1.0, num: Optional[int] = None) -> TrajData:
71 | '''
72 | :param traj_data: raw trajectory dataframe
73 | :param res: time resolution
74 | :param num: None
75 | :return: interpolated trajectory
76 | '''
77 | if res <= 0.0:
78 | raise ValueError('res must be > 0.0')
79 | if num is not None and num < 0:
80 | raise ValueError('num must be >= 0')
81 | _traj_dist = traj_calculate_distance_ts(traj_data)
82 | xi = np.cumsum(_traj_dist)
83 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]])
84 | num = num if num is not None else int(np.ceil(xi[-1] / res))
85 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True)
86 | y = pchip_interpolate(xi, yi, x, axis=1)
87 | return y.T
88 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_interpolate/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | numpy==1.23.0
3 | pandas==1.4.1
4 | scipy==1.8.1
5 | vincenty==0.1.4
6 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/example/trajkalmanfilter.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | import time
4 | from decimal import Decimal
5 | import matplotlib.pyplot as plt
6 | from pyais import traj_kalmanfilter
7 |
8 | if __name__ == '__main__':
9 | df = pd.read_csv("../data/1.csv")
10 | ais_origin = []
11 | ais_data = []
12 | ais_kalman = traj_kalmanfilter.KalmanFilter()
13 |
14 | for index, row in df.iterrows():
15 | spd = Decimal(row['SPEED'] * 1852.25 / 3600)
16 | tm = str(row['Date']) + ' ' + str(row['Time'])
17 | timeArray = time.strptime(tm, "%Y-%m-%d %H:%M:%S")
18 | timeStamp = int(time.mktime(timeArray))
19 | ais_origin.append([row['Longitude'], row['Latitude']])
20 | coords = str(row['Longitude']) + ',' + str(row['Latitude'])
21 | pt = ais_kalman.process(speed=spd, coordinate=coords, time_stamp=timeStamp, accuracy=10.0)
22 | ais_data.append(ais_kalman._split_coordinate(pt))
23 | res = list(filter(None, ais_data))
24 | x, y = np.array(res).T
25 | reso = list(filter(None, ais_origin))
26 | xo, yo = np.array(reso).T
27 | plt.plot(xo, yo, color='b', marker='*', linestyle='--', linewidth=0.5, label='Raw trajectory')
28 | plt.plot(x, y, color='r', marker='o', ms=5, linestyle='-', linewidth=0.5, label='Filtered trajectory')
29 | plt.yticks(fontproperties='Times New Roman', size=14)
30 | plt.xticks(fontproperties='Times New Roman', size=14)
31 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16})
32 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16})
33 | plt.title('Kalman Filter', fontdict={'family': 'Times New Roman', 'size': 16})
34 | plt.ticklabel_format(useOffset=False, style='plain')
35 | plt.grid()
36 | plt.legend()
37 | plt.tight_layout()
38 | plt.show()
39 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_clean.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_clean.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_interpolation.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_interpolation.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_kalmanfilter.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_kalmanfilter.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_segment.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_segment.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_stay.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_kalman_filter/pyais/__pycache__/traj_stay.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/pyais/traj_kalmanfilter.py:
--------------------------------------------------------------------------------
1 | from decimal import Decimal
2 |
3 |
4 | class KalmanFilter(object):
5 | def __init__(self):
6 | self.variance = Decimal('-1')
7 | self.longitude, self.latitude = None, None
8 | self.time_stamp = None
9 | self.accuracy = None
10 |
11 | def set_state(self, coordinate: str, time_stamp: float, accuracy: Decimal):
12 | self.longitude, self.latitude = self._split_coordinate(coordinate)
13 | self.time_stamp = time_stamp
14 | self.accuracy = Decimal(accuracy)
15 | self.variance = Decimal(accuracy * accuracy)
16 |
17 | def process(self, speed: Decimal, coordinate: str, time_stamp: float, accuracy: Decimal):
18 | longitude, latitude = self._split_coordinate(coordinate)
19 | if self.variance < 0:
20 | # 初始状态
21 | self.set_state(coordinate, time_stamp, accuracy)
22 | else:
23 | duration = Decimal(time_stamp - self.time_stamp)
24 | if duration > 0:
25 | self.variance += duration * speed * speed / Decimal('1000')
26 | self.time_stamp = time_stamp
27 | k = self.variance / (self.variance + Decimal(accuracy * accuracy))
28 | # 计算位置
29 | self.latitude += k * (latitude - self.latitude)
30 | self.longitude += k * (longitude - self.longitude)
31 | # 方差
32 | self.variance = (1 - k) * self.variance
33 | coordinate = self._join_longitude_latitude(self.longitude, self.latitude)
34 | return coordinate
35 |
36 | @staticmethod
37 | def _split_coordinate(coordinate: str):
38 | if not coordinate is None:
39 | longitude, latitude = coordinate.split(',')
40 | return Decimal(longitude), Decimal(latitude)
41 |
42 | @staticmethod
43 | def _join_longitude_latitude(longitude, latitude):
44 | return f'{round(longitude, 6)},{round(latitude, 6)}'
45 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_kalman_filter/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | numpy==1.23.0
3 | pandas==1.4.1
4 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_segment/data/1.csv:
--------------------------------------------------------------------------------
1 | INDEX,DRMMSI,DRLATITUDE,DRLONGITUDE,DRDIRECTION,DRSPEED,DRGPSTIME,STATUS,DRTRUEHEADING,DIRECTION
2 | 0,244726000,31.172785,122.6683333,278.6,12.9,1556415580,0,0,0
3 | 1,244726000,31.17298333,122.6670317,280.8,12.9,1556415599,0,0,2.2
4 | 2,244726000,31.173205,122.6654467,279.3,12.9,1556415623,0,0,-1.5
5 | 3,244726000,31.17359667,122.6629683,280.3,12.9,1556415659,0,0,1
6 | 4,244726000,31.17378333,122.6617317,280.4,13,1556415677,0,0,0.1
7 | 5,244726000,31.17418167,122.65882,277.4,12.9,1556415719,0,0,-3
8 | 6,244726000,31.17444,122.6566783,278.8,12.9,1556415750,0,0,1.4
9 | 7,244726000,31.17461833,122.6551717,278.2,12.8,1556415771,0,0,-0.6
10 | 8,244726000,31.17479,122.6538267,278.2,12.5,1556415791,0,0,0
11 | 9,244726000,31.17507833,122.6512433,276.1,11.8,1556415831,0,0,-2.1
12 | 10,244726000,31.17588,122.6472183,283.9,10.5,1556415900,0,0,7.8
13 | 11,244726000,31.17622833,122.6456733,287.5,10,1556415930,0,0,3.6
14 | 12,244726000,31.17657833,122.6445783,291,9.9,1556415951,0,0,3.5
15 | 13,244726000,31.17699333,122.643175,289,10,1556415980,0,0,-2
16 | 14,244726000,31.17731833,122.6421333,289.6,10.3,1556416000,0,0,0.6
17 | 15,244726000,31.17766167,122.64099,289,10.6,1556416020,0,0,-0.6
18 | 16,244726000,31.177975,122.63993,289.1,11,1556416040,0,0,0.1
19 | 17,244726000,31.178305,122.63878,287.5,11.2,1556416060,0,0,-1.6
20 | 18,244726000,31.17875667,122.637145,290.3,11.3,1556416087,0,0,2.8
21 | 19,244726000,31.18002167,122.63505,315.3,11.1,1556416130,0,0,25
22 | 20,244726000,31.18141833,122.6333667,315,10.8,1556416170,0,0,-0.3
23 | 21,244726000,31.18208167,122.6327417,323.2,10.8,1556416188,0,0,8.2
24 | 22,244726000,31.18317,122.6320933,333.7,11,1556416211,0,0,10.5
25 | 23,244726000,31.183995,122.6315867,331.7,10.5,1556416230,0,0,-2
26 | 24,244726000,31.18483167,122.63106,331.2,10.1,1556416251,0,0,-0.5
27 | 25,244726000,31.18672667,122.6299767,334.9,8.7,1556416300,0,0,3.7
28 | 26,244726000,31.18746167,122.6296117,337.4,8.1,1556416320,1,0,2.5
29 | 27,244726000,31.18939,122.6286283,333.2,6.3,1556416385,1,0,-4.2
30 | 28,244726000,31.190035,122.6281667,326.8,5.3,1556416414,1,0,-6.4
31 | 29,244726000,31.19035833,122.6278967,323.5,4.8,1556416430,1,0,-3.3
32 | 30,244726000,31.19063167,122.627635,319.5,4.3,1556416448,1,0,-4
33 | 31,244726000,31.19089,122.6273467,315.7,3.7,1556416468,1,0,-3.8
34 | 32,244726000,31.19153333,122.6264433,311,1.9,1556416551,1,0,-4.7
35 | 33,244726000,31.19167667,122.62627,318,1.2,1556416581,1,0,7
36 | 34,244726000,31.19176333,122.6262217,330.4,1,1556416601,1,0,12.4
37 | 35,244726000,31.19184,122.6262133,348.8,0.8,1556416622,1,0,18.4
38 | 36,244726000,31.19207,122.6263383,39.4,0.7,1556416696,1,0,-309.4
39 | 37,244726000,31.192615,122.627115,53.5,1,1556416877,1,0,14.1
40 | 38,244726000,31.19303333,122.6278433,69.8,0.8,1556417057,1,0,16.3
41 | 39,244726000,31.19284667,122.62879,132.3,0.4,1556417417,1,0,62.5
42 | 40,244726000,31.19254833,122.628945,156,0.2,1556417597,1,0,23.7
43 | 41,244726000,31.19245,122.6290183,136.7,0.2,1556417777,1,0,-19.3
44 | 42,244726000,31.19241333,122.629095,164.1,0.2,1556418317,1,0,27.4
45 | 43,244726000,31.192395,122.6291083,215,0.1,1556418497,1,0,50.9
46 | 44,244726000,31.19236667,122.6291217,199.5,0.1,1556418677,1,0,-15.5
47 | 45,244726000,31.19232,122.6291883,29.3,0,1556419937,1,0,-170.2
48 | 46,244726000,31.192305,122.6291817,29.3,0.1,1556420117,1,0,0
49 | 47,244726000,31.19226333,122.6291617,233.5,0.1,1556420297,1,0,204.2
50 | 48,244726000,31.192155,122.6291817,181.5,0.1,1556420477,1,0,-52
51 | 49,244726000,31.19193833,122.62923,191.8,0.1,1556420837,1,0,10.3
52 | 50,244726000,31.19194,122.62923,191.8,0.1,1556421017,1,0,0
53 | 51,244726000,31.19192,122.6292483,237.6,0.2,1556421377,1,0,45.8
54 | 52,244726000,31.19188667,122.62926,192.2,0.1,1556421737,1,0,-45.4
55 | 53,244726000,31.19190333,122.629275,76.3,0.1,1556421917,1,0,-115.9
56 | 54,244726000,31.19190167,122.6292833,87.6,0.1,1556422097,1,0,11.3
57 | 55,244726000,31.19176,122.6292183,192.9,0.2,1556422818,1,0,105.3
58 | 56,244726000,31.19143167,122.6291633,198.7,0.1,1556423357,1,0,5.8
59 | 57,244726000,31.19148833,122.629175,227,0.2,1556424437,1,0,28.3
60 | 58,244726000,31.19138333,122.6291567,208.8,0.1,1556424977,1,0,-18.2
61 | 59,244726000,31.19141333,122.6291467,247,0.1,1556425157,1,0,38.2
62 | 60,244726000,31.19141333,122.6291133,247,0.1,1556425337,1,0,0
63 | 61,244726000,31.19128833,122.6291067,189.9,0.1,1556425877,1,0,-57.1
64 | 62,244726000,31.19130833,122.6290617,206.9,0.1,1556426057,1,0,17
65 | 63,244726000,31.19127,122.629065,204.5,0.1,1556426236,1,0,-2.4
66 | 64,244726000,31.191155,122.629035,179.6,0.2,1556426597,1,0,-24.9
67 | 65,244726000,31.19113,122.6290133,211,0.2,1556426778,1,0,31.4
68 | 66,244726000,31.19105167,122.6289983,233.1,0.1,1556427137,1,0,22.1
69 | 67,244726000,31.19101667,122.628975,148.8,0.1,1556427317,1,0,-84.3
70 | 68,244726000,31.19103667,122.6289517,208.2,0.1,1556427677,1,0,59.4
71 | 69,244726000,31.19099667,122.6288967,214.3,0.1,1556428037,1,0,6.1
72 | 70,244726000,31.19101,122.6288883,239.9,0.1,1556428217,1,0,25.6
73 | 71,244726000,31.19092,122.6289083,237.2,0.2,1556428577,1,0,-2.7
74 | 72,244726000,31.19089667,122.62891,226.9,0.1,1556428757,1,0,-10.3
75 | 73,244726000,31.19094167,122.62889,334,0.1,1556428937,1,0,107.1
76 | 74,244726000,31.19096833,122.6288567,305.7,0.1,1556429117,1,0,-28.3
77 | 75,244726000,31.19095333,122.6288417,251.4,0.1,1556429297,1,0,-54.3
78 | 76,244726000,31.19092,122.6288183,231.3,0.1,1556429478,1,0,-20.1
79 | 77,244726000,31.19084167,122.62873,218.3,0.1,1556429837,1,0,-13
80 | 78,244726000,31.19090167,122.6287067,313.7,0.1,1556430017,1,0,95.4
81 | 79,244726000,31.190845,122.62873,188.4,0.1,1556430198,1,0,-125.3
82 | 80,244726000,31.19087,122.6287283,223,0.1,1556430557,1,0,34.6
83 | 81,244726000,31.19089,122.6287583,307.3,0.1,1556430737,1,0,84.3
84 | 82,244726000,31.19090667,122.62876,270,0.2,1556431097,1,0,-37.3
85 | 83,244726000,31.19094333,122.628705,315.6,0.1,1556431277,1,0,45.6
86 | 84,244726000,31.19071333,122.62854,321.4,0.3,1556431817,1,0,5.8
87 | 85,244726000,31.190705,122.6284683,193.3,0.5,1556431997,1,0,-128.1
88 | 86,244726000,31.19053333,122.62833,179.9,0.3,1556432537,1,0,-13.4
89 | 87,244726000,31.19037833,122.628255,210.9,0.3,1556432717,1,0,31
90 | 88,244726000,31.190565,122.6281317,357.2,0.2,1556433077,1,0,146.3
91 | 89,244726000,31.19035333,122.628095,192.2,0.2,1556433257,1,0,-165
92 | 90,244726000,31.190375,122.628035,297.3,0.1,1556433437,1,0,105.1
93 | 91,244726000,31.190415,122.6279567,325.4,0.1,1556433617,1,0,28.1
94 | 92,244726000,31.19039667,122.62799,84.2,0.1,1556433977,1,0,-241.2
95 | 93,244726000,31.1903,122.6278533,242.9,0.2,1556434517,1,0,158.7
96 | 94,244726000,31.19015,122.627535,237.5,0.2,1556435597,1,0,-5.4
97 | 95,244726000,31.19009167,122.62746,230.5,0.2,1556435777,1,0,-7
98 | 96,244726000,31.19009833,122.6274317,246.4,0.1,1556435958,1,0,15.9
99 | 97,244726000,31.18984333,122.6272583,162.5,0.4,1556437217,1,0,-83.9
100 | 98,244726000,31.18970333,122.62704,356.9,0.4,1556437937,1,0,194.4
101 | 99,244726000,31.19007167,122.6268883,345,0.5,1556438117,1,0,-11.9
102 | 100,244726000,31.19001167,122.626515,208.3,0.4,1556438477,1,0,-136.7
103 | 101,244726000,31.18962,122.6262583,207.2,0.4,1556438658,1,0,-1.1
104 | 102,244726000,31.18986333,122.6258967,275,0.3,1556439018,1,0,67.8
105 | 103,244726000,31.19031,122.6251183,354.9,0.6,1556439553,1,0,79.9
106 | 104,244726000,31.19036667,122.6251167,0.2,0.6,1556439573,1,0,-354.7
107 | 105,244726000,31.19041833,122.6251267,7.3,0.6,1556439593,1,0,7.1
108 | 106,244726000,31.19052833,122.6251533,9.9,0.6,1556439633,1,0,2.6
109 | 107,244726000,31.19058833,122.6251667,15.3,0.6,1556439653,1,0,5.4
110 | 108,244726000,31.19065,122.6251767,17.9,0.6,1556439673,1,0,2.6
111 | 109,244726000,31.19070333,122.6252033,17.6,0.6,1556439693,1,0,-0.3
112 | 110,244726000,31.19075,122.625245,14.3,0.6,1556439714,1,0,-3.3
113 | 111,244726000,31.19082333,122.6252317,13.8,0.6,1556439733,1,0,-0.5
114 | 112,244726000,31.19086667,122.6252567,15.5,0.5,1556439753,1,0,1.7
115 | 113,244726000,31.19091333,122.62528,20.2,0.5,1556439773,1,0,4.7
116 | 114,244726000,31.191015,122.6253167,25.7,0.5,1556439813,1,0,5.5
117 | 115,244726000,31.19105333,122.6253233,27.6,0.4,1556439833,1,0,1.9
118 | 116,244726000,31.19108,122.62534,27.3,0.4,1556439853,1,0,-0.3
119 | 117,244726000,31.19109833,122.62538,46,0.4,1556439873,1,0,18.7
120 | 118,244726000,31.19111,122.6254417,74.5,0.6,1556439893,1,0,28.5
121 | 119,244726000,31.191125,122.6255117,78.1,0.7,1556439913,1,0,3.6
122 | 120,244726000,31.19117833,122.6260867,78,1.8,1556439993,1,0,-0.1
123 | 121,244726000,31.19122167,122.6262833,71.1,2.1,1556440011,1,0,-6.9
124 | 122,244726000,31.19132333,122.6265083,63.4,2.4,1556440030,1,0,-7.7
125 | 123,244726000,31.19143167,122.62669,56.7,2.6,1556440046,1,0,-6.7
126 | 124,244726000,31.191565,122.6268633,49.5,2.7,1556440063,1,0,-7.2
127 | 125,244726000,31.191715,122.62703,43.5,2.9,1556440078,1,0,-6
128 | 126,244726000,31.191895,122.62718,36.9,2.9,1556440093,1,0,-6.6
129 | 127,244726000,31.19211167,122.6273267,29.1,3.3,1556440109,1,0,-7.8
130 | 128,244726000,31.19241333,122.62746,18.4,3.6,1556440130,1,0,-10.7
131 | 129,244726000,31.19272,122.6275233,9.8,3.8,1556440146,1,0,-8.6
132 | 130,244726000,31.19296833,122.6275267,1,3.8,1556440161,1,0,-8.8
133 | 131,244726000,31.19361833,122.6273483,339.5,4.3,1556440196,1,0,338.5
134 | 132,244726000,31.19394333,122.6271217,328.2,4.6,1556440213,1,0,-11.3
135 | 133,244726000,31.19421667,122.626755,306.2,4.4,1556440233,1,0,-22
136 | 134,244726000,31.19426833,122.6263883,276.4,4.3,1556440250,1,0,-29.8
137 | 135,244726000,31.19428667,122.6254517,271.4,5.7,1556440282,1,0,-5
138 | 136,244726000,31.19411167,122.623925,258.6,7.6,1556440324,1,0,-12.8
139 | 137,244726000,31.19387,122.62316,248.5,8.1,1556440342,1,0,-10.1
140 | 138,244726000,31.19327167,122.6212483,250.8,9.7,1556440384,1,0,2.3
141 | 139,244726000,31.19296333,122.6202767,249.3,10.3,1556440403,1,0,-1.5
142 | 140,244726000,31.19261833,122.6191267,250.7,10.9,1556440424,0,0,1.4
143 | 141,244726000,31.19189167,122.6168567,249.4,11.8,1556440462,0,0,-1.3
144 | 142,244726000,31.191475,122.6155767,249.6,12.1,1556440484,0,0,0.2
145 | 143,244726000,31.19066333,122.6131333,246.3,12.5,1556440523,0,0,-3.3
146 | 144,244726000,31.19001667,122.6111317,249.1,12.9,1556440553,0,0,2.8
147 | 145,244726000,31.18934833,122.609255,251.3,13,1556440583,0,0,2.2
148 | 146,244726000,31.18888167,122.6078633,248.2,13.2,1556440603,0,0,-3.1
149 | 147,244726000,31.187485,122.60382,248.6,13.6,1556440664,0,0,0.4
150 | 148,244726000,31.18697,122.60248,246.9,13.6,1556440684,0,0,-1.7
151 | 149,244726000,31.18648167,122.6010267,248.6,13.8,1556440704,0,0,1.7
152 | 150,244726000,31.18605167,122.5996867,249.3,13.9,1556440724,0,0,0.7
153 | 151,244726000,31.185585,122.5982717,248.3,14,1556440744,0,0,-1
154 | 152,244726000,31.18512833,122.5970067,247.5,14.1,1556440761,0,0,-0.8
155 | 153,244726000,31.184575,122.5954417,248.6,14.3,1556440784,0,0,1.1
156 | 154,244726000,31.18409,122.594,249,14.4,1556440804,0,0,0.4
157 | 155,244726000,31.183605,122.5927167,245.2,14.4,1556440821,0,0,-3.8
158 | 156,244726000,31.18302667,122.5910467,248.8,14.5,1556440845,0,0,3.6
159 | 157,244726000,31.18226833,122.5887767,246.7,14.6,1556440876,0,0,-2.1
160 | 158,244726000,31.18147833,122.586575,249,14.7,1556440905,0,0,2.3
161 | 159,244726000,31.18049333,122.5839383,247.5,14.8,1556440941,0,0,-1.5
162 | 160,244726000,31.17986667,122.5821517,248.1,14.8,1556440965,0,0,0.6
163 | 161,244726000,31.17908167,122.579905,247.4,15,1556440996,0,0,-0.7
164 | 162,244726000,31.178285,122.5776467,248.7,15.1,1556441025,0,0,1.3
165 | 163,244726000,31.17430167,122.5691883,241,15,1556441145,0,0,-7.7
166 | 164,244726000,31.17350667,122.56747,242.4,15.1,1556441169,0,0,1.4
167 | 165,244726000,31.17269167,122.5657583,240.4,15.1,1556441194,0,0,-2
168 | 166,244726000,31.172075,122.564475,240.3,15.1,1556441211,0,0,-0.1
169 | 167,244726000,31.171495,122.5632533,241.4,15.2,1556441229,0,0,1.1
170 | 168,244726000,31.17068333,122.5615333,239.9,15,1556441253,0,0,-1.5
171 | 169,244726000,31.17005667,122.560195,242.3,14.8,1556441271,0,0,2.4
172 | 170,244726000,31.16887833,122.5573467,245.2,14,1556441313,0,0,2.9
173 | 171,244726000,31.168105,122.55528,247.5,13.2,1556441343,0,0,2.3
174 | 172,244726000,31.16749833,122.5537867,245.5,12.6,1556441368,0,0,-2
175 | 173,244726000,31.16696167,122.552325,247.5,12.1,1556441391,0,0,2
176 | 174,244726000,31.16662167,122.5513167,248.5,11.8,1556441409,0,0,1
177 | 175,244726000,31.16625167,122.55022,248.2,11.4,1556441428,0,0,-0.3
178 | 176,244726000,31.16592167,122.5491867,249.9,11.1,1556441445,0,0,1.7
179 | 177,244726000,31.16556667,122.5482167,246.7,10.7,1556441463,0,0,-3.2
180 | 178,244726000,31.165115,122.546975,249.4,10.3,1556441488,0,0,2.7
181 | 179,244726000,31.16478833,122.5460683,248.3,10,1556441505,0,0,-1.1
182 | 180,244726000,31.16447167,122.5451733,248.2,9.9,1556441523,0,0,-0.1
183 | 181,244726000,31.164155,122.5442667,248,10.1,1556441541,0,0,-0.2
184 | 182,244726000,31.16380833,122.5433467,246.4,10.4,1556441559,0,0,-1.6
185 | 183,244726000,31.163445,122.5422717,249.6,10.8,1556441580,0,0,3.2
186 | 184,244726000,31.1631,122.5412033,247.8,11.1,1556441598,0,0,-1.8
187 | 185,244726000,31.16226,122.53889,246.1,11.5,1556441640,0,0,-1.7
188 | 186,244726000,31.16185833,122.53778,246.5,11.8,1556441658,0,0,0.4
189 | 187,244726000,31.16125333,122.53595,248.1,12.1,1556441689,0,0,1.6
190 | 188,244726000,31.16079,122.5346517,247.6,12.4,1556441709,0,0,-0.5
191 | 189,244726000,31.16029667,122.5333517,245.8,12.6,1556441730,0,0,-1.8
192 | 190,244726000,31.15988333,122.5322083,247.8,12.7,1556441749,0,0,2
193 | 191,244726000,31.15911667,122.5294367,253.3,12.9,1556441790,0,0,5.5
194 | 192,244726000,31.15864167,122.527495,252.7,12.9,1556441820,0,0,-0.6
195 | 193,244726000,31.15826167,122.5261117,252.3,12.9,1556441840,0,0,-0.4
196 | 194,244726000,31.15793,122.5248617,253.1,12.7,1556441859,0,0,0.8
197 | 195,244726000,31.15759,122.5235767,252.6,12.6,1556441880,0,0,-0.5
198 | 196,244726000,31.157225,122.522215,253,12.5,1556441900,0,0,0.4
199 | 197,244726000,31.15674667,122.5204383,253.2,12.1,1556441929,0,0,0.2
200 | 198,244726000,31.1564,122.5192117,251.3,11.9,1556441949,0,0,-1.9
201 | 199,244726000,31.15562333,122.5168383,249.2,11.3,1556441990,0,0,-2.1
202 | 200,244726000,31.15507667,122.51511,249.2,11,1556442020,0,0,0
203 | 201,244726000,31.15458333,122.5135183,251.1,10.8,1556442049,0,0,1.9
204 | 202,244726000,31.15428667,122.51247,251.5,10.7,1556442069,0,0,0.4
205 | 203,244726000,31.15396333,122.511395,250.5,10.6,1556442089,0,0,-1
206 | 204,244726000,31.15333,122.5092467,251.2,10.5,1556442129,0,0,0.7
207 | 205,244726000,31.15300833,122.5081683,250.2,10.5,1556442149,0,0,-1
208 | 206,244726000,31.15239167,122.5060083,253.1,10.6,1556442189,0,0,2.9
209 | 207,244726000,31.15189667,122.5044967,247.5,10.9,1556442219,0,0,-5.6
210 | 208,244726000,31.15105167,122.5031517,228,10.5,1556442246,0,0,-19.5
211 | 209,244726000,31.14994,122.50166,231.2,10.9,1556442279,0,0,3.2
212 | 210,244726000,31.14791,122.4988667,230.1,11.4,1556442339,0,0,-1.1
213 | 211,244726000,31.14584833,122.4959583,229.5,11.7,1556442399,0,0,-0.6
214 | 212,244726000,31.14414167,122.4935017,230,11.7,1556442450,0,0,0.5
215 | 213,244726000,31.141885,122.49002,234.9,11.4,1556442519,0,0,4.9
216 | 214,244726000,31.14028833,122.4876183,230.1,11.1,1556442570,0,0,-4.8
217 | 215,244726000,31.13971833,122.4866417,236.8,10.9,1556442590,0,0,6.7
218 | 216,244726000,31.13865833,122.48474,234.2,10.2,1556442630,0,0,-2.6
219 | 217,244726000,31.13825333,122.484075,235.9,10,1556442650,0,0,1.7
220 | 218,244726000,31.13648833,122.4807867,237.1,9.5,1556442719,0,0,1.2
221 | 219,244726000,31.13539667,122.4785617,239.8,9.2,1556442770,0,0,2.7
222 | 220,244726000,31.13498667,122.4777567,238.2,9.1,1556442789,0,0,-1.6
223 | 221,244726000,31.13421167,122.4760033,244.9,8.7,1556442830,0,0,6.7
224 | 222,244726000,31.13383667,122.4752283,240.6,8.5,1556442848,0,0,-4.3
225 | 223,244726000,31.13314,122.4736267,243,7.9,1556442890,0,0,2.4
226 | 224,244726000,31.13284333,122.472905,243.5,7.7,1556442909,0,0,0.5
227 | 225,244726000,31.13142,122.46935,251.2,6.9,1556443010,0,0,7.7
228 | 226,244726000,31.13004667,122.465165,248.6,6,1556443139,0,0,-2.6
229 | 227,244726000,31.12956333,122.4636533,250.1,5.7,1556443190,0,0,1.5
230 | 228,244726000,31.12910833,122.46227,251.7,5.3,1556443239,0,0,1.6
231 | 229,244726000,31.12898167,122.4617467,255.2,5.3,1556443259,0,0,3.5
232 | 230,244726000,31.1287,122.46064,252.9,5.2,1556443299,0,0,-2.3
233 | 231,244726000,31.12856833,122.460095,253.5,5.3,1556443319,0,0,0.6
234 | 232,244726000,31.12825167,122.4589583,253,5.6,1556443359,0,0,-0.5
235 | 233,244726000,31.12808833,122.4583633,253.4,5.8,1556443379,0,0,0.4
236 | 234,244726000,31.12775833,122.45713,253,6,1556443419,0,0,-0.4
237 | 235,244726000,31.127585,122.4565067,251.7,6.1,1556443439,0,0,-1.3
238 | 236,244726000,31.12739833,122.455815,252.5,6.6,1556443460,0,0,0.8
239 | 237,244726000,31.12719833,122.4551267,251.2,7.1,1556443479,0,0,-1.3
240 | 238,244726000,31.12667667,122.4532333,248.8,8.5,1556443529,0,0,-2.4
241 | 239,244726000,31.12585167,122.4512667,243.1,9.3,1556443569,0,0,-5.7
242 | 240,244726000,31.12376167,122.4465083,242.2,10.2,1556443669,0,0,-0.9
243 | 285,244726000,31.10294333,122.3658517,271.1,10,1556445219,0,0,-0.9
244 | 286,244726000,31.10296167,122.3642933,271.6,9.9,1556445249,0,0,0.5
245 | 287,244726000,31.10299333,122.36318,271.9,9.9,1556445270,0,0,0.3
246 | 288,244726000,31.10304833,122.35998,272.9,9.9,1556445330,0,0,1
247 | 289,244726000,31.10313667,122.3562283,270,9.9,1556445399,0,0,-2.9
248 | 290,244726000,31.103215,122.3535617,272.2,9.9,1556445450,0,0,2.2
249 | 291,244726000,31.10323833,122.35256,271.5,9.9,1556445469,0,0,-0.7
250 | 292,244726000,31.103335,122.349805,272.6,9.9,1556445520,0,0,1.1
251 | 293,244726000,31.10331167,122.347185,268.5,9.8,1556445570,0,0,-4.1
252 | 294,244726000,31.103295,122.34453,269.2,9.8,1556445620,0,0,0.7
253 | 295,244726000,31.10328167,122.3434733,269.1,9.8,1556445640,0,0,-0.1
254 | 296,244726000,31.10328667,122.3424183,271.1,9.8,1556445659,0,0,2
255 | 297,244726000,31.10326,122.339815,269.9,9.9,1556445709,0,0,-1.2
256 | 298,244726000,31.10325,122.33769,270,9.9,1556445749,0,0,0.1
257 | 299,244726000,31.10323,122.33657,270,9.9,1556445770,0,0,0
258 | 300,244726000,31.103225,122.3355533,269.9,9.9,1556445789,0,0,-0.1
259 | 301,244726000,31.10321333,122.3338933,270.5,9.9,1556445820,0,0,0.6
260 | 302,244726000,31.10320333,122.3328283,269.7,9.9,1556445839,0,0,-0.8
261 | 303,244726000,31.103215,122.3312617,271.2,9.9,1556445869,0,0,1.5
262 | 304,244726000,31.103185,122.3296033,268.8,9.9,1556445899,0,0,-2.4
263 | 305,244726000,31.10317333,122.328535,269.5,9.9,1556445920,0,0,0.7
264 | 306,244726000,31.10315333,122.3269217,268.6,9.9,1556445950,0,0,-0.9
265 | 307,244726000,31.103155,122.3259017,268.7,9.9,1556445969,0,0,0.1
266 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_segment/example/trajsegment.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | from pyais import traj_segment
4 |
5 |
6 | def main():
7 | trajdata = pd.read_csv('../data/1.csv', encoding="gbk",
8 | usecols=['DRMMSI', 'DRGPSTIME', 'DRLONGITUDE', 'DRLATITUDE'])
9 | scattercolors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm']
10 | rawgrouped = trajdata[:].groupby('DRMMSI')
11 | for name, group in rawgrouped:
12 | plt.plot(group['DRLONGITUDE'], group['DRLATITUDE'], marker='*', ms=8, linestyle='--',
13 | color='blue', linewidth=0.75)
14 | segmenteddata = traj_segment.segment(trajdata, 1500)
15 | seggrouped = segmenteddata[:].groupby('DRMMSI')
16 | i = 0
17 | for name, group in seggrouped:
18 | i = i + 1
19 | colorSytle = scattercolors[i % len(scattercolors)]
20 | plt.plot(group['DRLONGITUDE'], group['DRLATITUDE'], marker='o', ms=5, linestyle='-',
21 | color=colorSytle, linewidth=0.5)
22 | plt.yticks(fontproperties='Times New Roman', size=14)
23 | plt.xticks(fontproperties='Times New Roman', size=14)
24 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16})
25 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16})
26 | plt.title('Trajectory Segmentation', fontdict={'family': 'Times New Roman', 'size': 16})
27 | plt.ticklabel_format(useOffset=False, style='plain')
28 | plt.grid()
29 | plt.tight_layout()
30 | plt.show()
31 |
32 |
33 | if __name__ == '__main__':
34 | main()
35 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_segment/pyais/__pycache__/traj_compress.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_segment/pyais/__pycache__/traj_compress.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_segment/pyais/__pycache__/traj_interpolation.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_segment/pyais/__pycache__/traj_interpolation.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_segment/pyais/__pycache__/traj_segment.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_segment/pyais/__pycache__/traj_segment.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_segment/pyais/traj_segment.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import warnings
3 |
4 | warnings.filterwarnings("ignore")
5 |
6 |
7 | def segment(df, tsinterval):
8 | """
9 | Trajectory segmentation.
10 | :param df: an array
11 | :type df: DataFrame
12 | :param tsepsilon: time threshold
13 | :type tsepsilon: float
14 | :param tdf: return segmented trajectory
15 | :type tdf: DataFrame
16 | """
17 | tdf = pd.DataFrame(columns=df.columns, index=df.index)
18 | tdf.drop(tdf.index, inplace=True)
19 | tdf['tdiff'] = 0
20 |
21 | for shipmmsi, dt in df.groupby('DRMMSI'):
22 | data = dt.copy(deep=True)
23 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
24 | data = data.reset_index(drop=True)
25 | data['tdiff'] = data['DRGPSTIME'].diff().fillna(0)
26 | i = 0
27 | lastindex = 0
28 | for idx, di in data.iterrows():
29 | if di['tdiff'] > tsinterval and idx >= 1:
30 | data.loc[lastindex:idx - 1, 'DRMMSI'] = str(di['DRMMSI']) + '_' + str(i)
31 | i = i + 1
32 | lastindex = idx
33 | tdf = tdf.append(data, ignore_index=True)
34 | tdf = tdf.drop(['tdiff'], axis=1)
35 | return tdf
--------------------------------------------------------------------------------
/traj_preprocess/ais_segment/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | pandas==1.4.1
3 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/example/trajstay.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 | from pyais import traj_stay
5 |
6 |
7 | def stay_st_test(traj_file: str):
8 | params = {'axes.titlesize': 'large',
9 | 'xtick.labelsize': 16,
10 | 'ytick.labelsize': 16,
11 | 'legend.fontsize': 16,
12 | 'legend.handlelength': 3}
13 | plt.rcParams.update(params)
14 |
15 | trajdata = traj_stay.stay_st_detect(traj_file)
16 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm']
17 | for shipmmsi, dt in trajdata.groupby('DRMMSI'):
18 | labels = dt['SP_Status'].values
19 | clusterNum = len(set(labels))
20 | plt.plot(dt['DRLONGITUDE'].values, dt['DRLATITUDE'].values, marker='o', markeredgewidth=1.0, linewidth=0.75,
21 | markerfacecolor='white', markeredgecolor='m', ms=4, alpha=1.0, color='m', zorder=2)
22 | for i in range(-1, clusterNum):
23 | colorSytle = scatterColors[i % len(scatterColors)]
24 | subCluster = dt.query('SP_Status == @i')
25 | if i >= 0:
26 | plt.scatter(subCluster['DRLONGITUDE'].values, subCluster['DRLATITUDE'].values, marker='*', s=70,
27 | edgecolors=colorSytle,
28 | c='white', linewidths=1.0, zorder=3)
29 | else:
30 | continue
31 | plt.title('Stay Point Identification', fontsize=16)
32 | plt.xlabel('Longitude', fontsize=16)
33 | plt.ylabel('Latitude', fontsize=16)
34 | plt.grid(True)
35 | plt.tight_layout()
36 | plt.show()
37 |
38 |
39 | def stay_spt_test(traj_file: str):
40 | params = {'axes.titlesize': 'large',
41 | 'xtick.labelsize': 14,
42 | 'ytick.labelsize': 14,
43 | 'legend.fontsize': 14,
44 | 'legend.handlelength': 3}
45 | plt.rcParams.update(params)
46 |
47 | trajdata = traj_stay.stay_spt_detect(traj_file)
48 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'm']
49 | for shipmmsi, dt in trajdata.groupby('DRMMSI'):
50 | plt.plot(dt['DRLONGITUDE'].values, dt['DRLATITUDE'].values, marker='o', markeredgewidth=1.0, linewidth=0.75,
51 | markerfacecolor='white', markeredgecolor='m', ms=4, alpha=1.0, color='m', zorder=2)
52 | lbs = set(dt['SP_Status'].values)
53 | for index, v in enumerate(lbs):
54 | if v > 0:
55 | colorSytle = scatterColors[index % len(scatterColors)]
56 | subCluster = dt.query('SP_Status == @v')
57 | plt.scatter(subCluster['DRLONGITUDE'].values, subCluster['DRLATITUDE'].values, marker='*', s=70,
58 | edgecolors=colorSytle,
59 | c='white', linewidths=1.0, zorder=3)
60 |
61 | plt.title('Stay Point Identification', fontdict={'family': 'Times New Roman', 'size': 16})
62 | plt.yticks(fontproperties='Times New Roman', size=14)
63 | plt.xticks(fontproperties='Times New Roman', size=14)
64 | plt.ticklabel_format(useOffset=False, style='plain')
65 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 16})
66 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 16})
67 | plt.grid(True)
68 | plt.tight_layout()
69 | plt.show()
70 |
71 |
72 | if __name__ == '__main__':
73 | # stay_spt_test("../data/1.csv")
74 | stay_st_test("../data/1.csv")
75 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/__pycache__/traj_clean.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_clean.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/__pycache__/traj_interpolation.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_interpolation.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/__pycache__/traj_segment.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_segment.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/__pycache__/traj_stay.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_preprocess/ais_stay/pyais/__pycache__/traj_stay.cpython-38.pyc
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/traj_clean.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | from vincenty import vincenty
4 | import warnings
5 |
6 | warnings.filterwarnings("ignore")
7 |
8 |
9 | def heuristic_clean(data, vthreshold: float):
10 | '''
11 | :param data: raw trajectory
12 | :param vthreshold: speed threshold
13 | :return: cleaned trajectory
14 | '''
15 | tdf = pd.DataFrame(columns=data.columns, index=data.index)
16 | tdf.drop(tdf.index, inplace=True)
17 |
18 | data.drop_duplicates()
19 | data = data.reset_index(drop=True)
20 | for shipmmsi, dt in data.groupby('DRMMSI'):
21 | if len(str(shipmmsi)) > 5:
22 | dt_copy = dt.copy(deep=True)
23 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
24 | dt_copy = dt_copy.reset_index(drop=True)
25 | dt_copy['NOISE'] = 0
26 | for idx, di in dt_copy.iterrows():
27 | if di['NOISE'] == 0:
28 | if idx < len(dt_copy) - 1:
29 | t1 = di['DRGPSTIME']
30 | t2 = dt_copy.loc[idx + 1, 'DRGPSTIME']
31 | Δt = (t2 - t1) / 3600.0
32 | pt1 = (di['DRLATITUDE'], di['DRLONGITUDE'])
33 | pt2 = (dt_copy.loc[idx + 1, 'DRLATITUDE'], dt_copy.loc[idx + 1, 'DRLONGITUDE'])
34 | Δd = vincenty(pt1, pt2) * 1000 / 1852.25
35 | Δv = Δd / Δt
36 | if Δv > vthreshold:
37 | dt_copy.loc[idx + 1, 'NOISE'] = 1
38 | print("1")
39 | else:
40 | continue
41 | dt_copy_new = dt_copy.query('NOISE==0')
42 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1)
43 | tdf = tdf.append(dt_copy_new, ignore_index=True)
44 | return tdf
45 |
46 |
47 | def sw_clean(data, sw: int):
48 | '''
49 | :param data: raw trajectory
50 | :param sw: the size of sliding window
51 | :return: cleaned trajectory
52 | '''
53 | tdf = pd.DataFrame(columns=data.columns, index=data.index)
54 | tdf.drop(tdf.index, inplace=True)
55 |
56 | data.drop_duplicates()
57 | data = data.reset_index(drop=True)
58 | for shipmmsi, dt in data.groupby('DRMMSI'):
59 | if len(str(shipmmsi)) > 6:
60 | dt_copy = dt.copy(deep=True)
61 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
62 | dt_copy = dt_copy.reset_index(drop=True)
63 | dt_copy['NOISE'] = 0
64 | copylength = len(dt_copy)
65 | num_samples = copylength // sw + 1
66 | for idx in np.arange(num_samples):
67 | start_x = sw * idx
68 | end_x = start_x + sw - 1
69 | end_x = (copylength - 1) if end_x > (copylength - 1) else end_x
70 | dt_temp = dt_copy.loc[start_x:end_x]
71 | lats = dt_temp.loc[:, "DRLATITUDE"]
72 | longs = dt_temp.loc[:, "DRLONGITUDE"]
73 | stdlat = lats.std()
74 | meanlat = lats.mean()
75 | stdlog = longs.std()
76 | meanlog = longs.mean()
77 | for jdx, di in dt_temp.iterrows():
78 | if abs(di['DRLATITUDE'] - meanlat) > abs(1.5 * stdlat) or abs(di['DRLONGITUDE'] - meanlog) > abs(
79 | 1.5 * stdlog):
80 | dt_copy.loc[jdx, 'NOISE'] = 1
81 | print("1")
82 | pass
83 |
84 | dt_copy_new = dt_copy.query('NOISE==0')
85 | dt_copy_new = dt_copy_new.drop(['NOISE'], axis=1)
86 | tdf = tdf.append(dt_copy_new, ignore_index=True)
87 | return tdf
88 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/traj_interpolation.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | from scipy.interpolate import pchip_interpolate
4 | from typing import Dict, List, Union, Optional
5 | from vincenty import vincenty
6 |
7 | # types
8 | TrajData = Dict[str, Union[List[float]]]
9 |
10 |
11 | def traj_load(traj_file: str) -> TrajData:
12 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []}
13 | df = pd.read_csv(traj_file)
14 | dt_c = df.copy(deep=True)
15 | dt_c['DRLONGITUDE'] = dt_c['DRLONGITUDE'].map(lambda x: x / 1.0)
16 | dt_c['DRLATITUDE'] = dt_c['DRLATITUDE'].map(lambda x: x / 1.0)
17 | dt_c['DRSPEED'] = dt_c['DRSPEED'].map(lambda x: x / 1.0)
18 | for index, row in dt_c.iterrows():
19 | trajs_data['lat'].append(row['DRLATITUDE'])
20 | trajs_data['lon'].append(row['DRLONGITUDE'])
21 | trajs_data['speed'].append(row['DRSPEED'])
22 | trajs_data['tstamp'].append(row['DRGPSTIME'])
23 | return trajs_data
24 |
25 |
26 | def traj_calculate_distance(traj_data: TrajData) -> List[float]:
27 | traj_dist = np.zeros(len(traj_data['lat']))
28 | for i in range(len(traj_dist) - 1):
29 | lat1 = traj_data['lat'][i]
30 | lon1 = traj_data['lon'][i]
31 | lat2 = traj_data['lat'][i + 1]
32 | lon2 = traj_data['lon'][i + 1]
33 | pts = (lat1, lon1)
34 | pte = (lat2, lon2)
35 | s = vincenty(pts, pte) * 1000 # unit meter
36 | traj_dist[i + 1] = s
37 | return traj_dist.tolist()
38 |
39 |
40 | def traj_interpolate(traj_file: str, res: float = 1.0, num: Optional[int] = None) -> TrajData:
41 | '''
42 | :param traj_data: raw trajectory filename
43 | :param res: time resolution
44 | :param num: None
45 | :return: interpolated trajectory
46 | '''
47 | traj_data = traj_load(traj_file)
48 | if res <= 0.0:
49 | raise ValueError('res must be > 0.0')
50 | if num is not None and num < 0:
51 | raise ValueError('num must be >= 0')
52 | _traj_dist = traj_calculate_distance(traj_data)
53 | xi = np.cumsum(_traj_dist)
54 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]])
55 | num = num if num is not None else int(np.ceil(xi[-1] / res))
56 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True)
57 | y = pchip_interpolate(xi, yi, x, axis=1)
58 | traj_data_interp = {'lat': list(y[0, :]), 'lon': list(y[1, :]), 'speed': list(y[2, :]), 'tstamp': list(y[-1, :])}
59 | return traj_data_interp
60 |
61 |
62 | def traj_calculate_distance_ts(traj_data: TrajData) -> List[float]:
63 | traj_dist = np.zeros(len(traj_data['lat']))
64 | for i in range(len(traj_dist) - 1):
65 | s = int(traj_data['tstamp'][i + 1]) - int(traj_data['tstamp'][i])
66 | traj_dist[i + 1] = s
67 | return traj_dist.tolist()
68 |
69 |
70 | def traj_interpolate_df(traj_data, res: float = 1.0, num: Optional[int] = None) -> TrajData:
71 | '''
72 | :param traj_data: raw trajectory dataframe
73 | :param res: time resolution
74 | :param num: None
75 | :return: interpolated trajectory
76 | '''
77 | if res <= 0.0:
78 | raise ValueError('res must be > 0.0')
79 | if num is not None and num < 0:
80 | raise ValueError('num must be >= 0')
81 | _traj_dist = traj_calculate_distance_ts(traj_data)
82 | xi = np.cumsum(_traj_dist)
83 | yi = np.array([traj_data[i] for i in ('lat', 'lon', 'speed', 'tstamp') if traj_data[i]])
84 | num = num if num is not None else int(np.ceil(xi[-1] / res))
85 | x = np.linspace(xi[0], xi[-1], num=num, endpoint=True)
86 | y = pchip_interpolate(xi, yi, x, axis=1)
87 | return y.T
88 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/traj_segment.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import warnings
3 |
4 | warnings.filterwarnings("ignore")
5 |
6 |
7 | def segment(df, tsinterval):
8 | """
9 | Trajectory segmentation.
10 | :param df: an array
11 | :type df: DataFrame
12 | :param tsepsilon: time threshold
13 | :type tsepsilon: float
14 | :param tdf: return segmented trajectory
15 | :type tdf: DataFrame
16 | """
17 | tdf = pd.DataFrame(columns=df.columns, index=df.index)
18 | tdf.drop(tdf.index, inplace=True)
19 | tdf['tdiff'] = 0
20 |
21 | for shipmmsi, dt in df.groupby('DRMMSI'):
22 | data = dt.copy(deep=True)
23 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
24 | data = data.reset_index(drop=True)
25 | data['tdiff'] = data['DRGPSTIME'].diff().fillna(0)
26 | i = 0
27 | lastindex = 0
28 | for idx, di in data.iterrows():
29 | if di['tdiff'] > tsinterval and idx >= 1:
30 | data.loc[lastindex:idx - 1, 'DRMMSI'] = str(di['DRMMSI']) + '_' + str(i)
31 | i = i + 1
32 | lastindex = idx
33 | tdf = tdf.append(data, ignore_index=True)
34 | tdf = tdf.drop(['tdiff'], axis=1)
35 | return tdf
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/pyais/traj_stay.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import copy
4 | import time
5 | import math
6 | from scipy.spatial.distance import pdist
7 | from scipy.spatial.distance import squareform
8 | from pyais import traj_clean
9 | from pyais import traj_interpolation
10 | from pyais import traj_segment
11 |
12 | import warnings
13 |
14 | warnings.filterwarnings("ignore")
15 |
16 |
17 | # Part1
18 |
19 | def compute_squared_EDM(X):
20 | return squareform(pdist(X, metric='euclidean'))
21 |
22 |
23 | def ST_DBSCAN(data, eps1: float, eps2: float, minPts: int):
24 | '''
25 | :param data: interpolated trajectory
26 | :param eps1: spatial neighborhood
27 | :param eps2: time neighborhood
28 | :param minPts: the minimum number of points that satisfy the double-neighborhood
29 | :return: labels
30 | '''
31 | n, m = data.shape
32 | timeDisMat = compute_squared_EDM(data[:, 0].reshape(n, 1))
33 | disMat = compute_squared_EDM(data[:, 1:])
34 | core_points_index = np.where(np.sum(np.where((disMat <= eps1) & (timeDisMat <= eps2), 1, 0), axis=1) >= minPts)[0]
35 | labels = np.full((n,), -1)
36 | clusterId = 0
37 | for pointId in core_points_index:
38 | if (labels[pointId] == -1):
39 | labels[pointId] = clusterId
40 | neighbour = np.where((disMat[:, pointId] <= eps1) & (timeDisMat[:, pointId] <= eps2) & (labels == -1))[0]
41 | seeds = set(neighbour)
42 | while len(seeds) > 0:
43 | newPoint = seeds.pop()
44 | labels[newPoint] = clusterId
45 | queryResults = set(np.where((disMat[:, newPoint] <= eps1) & (timeDisMat[:, newPoint] <= eps2))[0])
46 | if len(queryResults) >= minPts:
47 | for resultPoint in queryResults:
48 | if labels[resultPoint] == -1:
49 | seeds.add(resultPoint)
50 | clusterId = clusterId + 1
51 | return labels
52 |
53 |
54 | def stay_st_detect(traj_file: str):
55 | '''
56 | :param traj_data: raw trajectory filename
57 | :return: interpolated trajectory,labels
58 | '''
59 | trajdf = pd.read_csv(traj_file, encoding='gbk')
60 | segdf = traj_segment.segment(trajdf, 1500)
61 | cleandf = traj_clean.heuristic_clean(segdf, 25)
62 |
63 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index)
64 | tdf.drop(tdf.index, inplace=True)
65 | tdf['SP_Status'] = 0
66 |
67 | for shipmmsi, dt in cleandf.groupby('DRMMSI'):
68 | data = dt.copy(deep=True)
69 | data.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
70 | data = data.reset_index(drop=True)
71 | trajs_data = {'lat': [], 'lon': [], 'tstamp': [], 'speed': []}
72 | for index, row in data.iterrows():
73 | trajs_data['lat'].append(row['DRLATITUDE'])
74 | trajs_data['lon'].append(row['DRLONGITUDE'])
75 | trajs_data['speed'].append(row['DRSPEED'])
76 | trajs_data['tstamp'].append(row['DRGPSTIME'])
77 | if len(trajs_data['lon']) < 4:
78 | continue
79 | res = 30.0
80 | my_traj_data_interp = traj_interpolation.traj_interpolate_df(trajs_data, res, None)
81 | dataDf = pd.DataFrame(my_traj_data_interp)
82 | dataDf.columns = ['lat', 'lon', 'spd', 'ts']
83 | dataDf['ts'] = dataDf['ts'].map(lambda x: int(x))
84 | datanew = dataDf[['ts', 'lat', 'lon']].values
85 |
86 | minpts = round(np.log(len(datanew)))
87 | if minpts < 4:
88 | minpts = 4
89 | ep2 = minpts / 2 * res
90 | ep1 = 2.2 * ep2 / 3600.0 / 60.0
91 | tj = pd.DataFrame(datanew)
92 | tj.columns = ['DRGPSTIME', 'DRLATITUDE', 'DRLONGITUDE']
93 | labels = ST_DBSCAN(datanew, ep1, ep2, minpts)
94 | tj['SP_Status'] = labels
95 | tj['DRMMSI'] = shipmmsi
96 | tj['DRGPSTIME'] = tj['DRGPSTIME'].map(lambda x: int(x))
97 | tdf = tdf.append(tj, ignore_index=True)
98 | tdf = tdf.fillna(0)
99 | return tdf
100 |
101 |
102 | # Part2
103 | def rad(d):
104 | return float(d) * math.pi / 180.0
105 |
106 |
107 | EARTH_RADIUS = 6378.137
108 |
109 |
110 | def GetDistance(lng1, lat1, lng2, lat2):
111 | radLat1 = rad(lat1)
112 | radLat2 = rad(lat2)
113 | a = radLat1 - radLat2
114 | b = rad(lng1) - rad(lng2)
115 | s = 2 * math.asin(
116 | math.sqrt(math.pow(math.sin(a / 2), 2) + math.cos(radLat1) * math.cos(radLat2) * math.pow(math.sin(b / 2), 2)))
117 | s = s * EARTH_RADIUS
118 | s = round(s * 10000, 2) / 10
119 | return s
120 |
121 |
122 | def ka(dis, dc):
123 | if (dis >= dc):
124 | return 0
125 | else:
126 | return 1
127 |
128 |
129 | def density(data, dc):
130 | dc = float(dc)
131 | latitude = list(data['DRLATITUDE'])
132 | longitude = list(data['DRLONGITUDE'])
133 | part_density = [] # 存储局部密度值
134 | scope = [] # 记录每个点计算局部密度的范围
135 | leftBoundary = 0
136 | rightBoundary = len(data) - 1 # 左边界与右边界
137 | for i in range(len(data)):
138 | trigger = True
139 | left = i - 1
140 | right = i + 1
141 | incrementLeft = 1
142 | incrementRight = 1
143 | while trigger:
144 | # 向左拓展
145 | if incrementLeft != 0:
146 | if left < 0:
147 | left = leftBoundary
148 | distanceLeft = GetDistance(longitude[left], latitude[left], longitude[i], latitude[i])
149 | if (distanceLeft < dc) & (left > leftBoundary):
150 | left -= 1
151 | else:
152 | incrementLeft = 0
153 | # 向右拓展
154 | if incrementRight != 0:
155 | if right > rightBoundary:
156 | right = rightBoundary
157 | distanceRight = GetDistance(longitude[i], latitude[i], longitude[right], latitude[right])
158 | if (distanceRight < dc) & (right < rightBoundary):
159 | right += 1
160 | else:
161 | incrementRight = 0
162 | # 若左右都停止了拓展,此点的局部密度计算结束
163 | if (incrementLeft == 0) & (incrementRight == 0):
164 | trigger = False
165 | if (left == leftBoundary) & (incrementRight == 0):
166 | trigger = False
167 | if (incrementLeft == 0) & (right == rightBoundary):
168 | trigger = False
169 | if left == leftBoundary:
170 | scope.append([left, right - 1])
171 | part_density.append(right - left - 1)
172 | elif right == rightBoundary:
173 | scope.append([left + 1, right])
174 | part_density.append(right - left - 1)
175 | else:
176 | scope.append([left + 1, right - 1])
177 | part_density.append(right - left - 2)
178 | return part_density, scope
179 |
180 |
181 | # 反向更新的方法
182 | def SP_search(data, tc):
183 | tc = int(tc)
184 | SP = []
185 | part_density = copy.deepcopy(list(data['part_density']))
186 | scope = copy.deepcopy(list(data['scope']))
187 | latitude = copy.deepcopy(list(data['DRLATITUDE']))
188 | longitude = copy.deepcopy(list(data['DRLONGITUDE']))
189 | timestamp = copy.deepcopy(list(data['DRGPSTIME']))
190 | trigger = True
191 | used = []
192 | while trigger:
193 | partD = max(part_density)
194 | index = part_density.index(partD)
195 | print('index:', index)
196 | start = scope[index][0]
197 | end = scope[index][1]
198 | if len(used) != 0:
199 | for i in used:
200 | if (scope[i][0] > start) & (scope[i][0] < end):
201 | part_density[index] = scope[i][0] - start - 1
202 | scope[index][1] = scope[i][0] - 1
203 | print("1_1")
204 | if (scope[i][1] > start) & (scope[i][1] < end):
205 | part_density[index] = end - scope[i][1] - 1
206 | scope[index][0] = scope[i][1] + 1
207 | print("1_2")
208 | if (scope[i][0] <= start) & (scope[i][1] >= end):
209 | part_density[index] = 0
210 | scope[index][0] = 0;
211 | scope[index][1] = 0
212 | print("1_3")
213 | start = scope[index][0];
214 | end = scope[index][1]
215 | timeCross = timestamp[end] - timestamp[start]
216 | print('time:', timeCross)
217 | if timeCross > tc:
218 | SarvT = time.localtime(timestamp[start])
219 | SlevT = time.localtime(timestamp[end])
220 | SP.append([index, latitude[index], longitude[index], SarvT, SlevT, scope[index]])
221 | used.append(index)
222 | for k in range(scope[index][0], scope[index][1] + 1):
223 | part_density[k] = 0
224 | part_density[index] = 0
225 | if max(part_density) == 0:
226 | trigger = False
227 | return SP
228 |
229 |
230 | # 通过代表的距离是否小于距离阈值来大致判断停留点是否一致
231 | def similar(sp, dc):
232 | dc = float(dc)
233 | latitude = copy.deepcopy(list(sp['latitude']))
234 | longitude = copy.deepcopy(list(sp['longitude']))
235 | i = 0;
236 | index = list(sp.index)
237 | for i in index:
238 | for j in index:
239 | if i != j:
240 | dist = GetDistance(longitude[i], latitude[i], longitude[j], latitude[j])
241 | if dist < 1.5 * dc:
242 | sp = sp.drop(j, axis=0)
243 | index.remove(j)
244 | return sp
245 |
246 |
247 | def stay_spt_detect(traj_file: str):
248 | '''
249 | :param traj_data: raw trajectory filename
250 | :return: interpolated trajectory,labels
251 | '''
252 | trajdf = pd.read_csv(traj_file, encoding='gbk')
253 | segdf = traj_segment.segment(trajdf, 1500)
254 | cleandf = traj_clean.heuristic_clean(segdf, 25)
255 |
256 | tdf = pd.DataFrame(columns=cleandf.columns, index=cleandf.index)
257 | tdf.drop(tdf.index, inplace=True)
258 | tdf['SP_Status'] = 0
259 |
260 | for shipmmsi, dt in cleandf.groupby('DRMMSI'):
261 | if len(str(shipmmsi)) > 5:
262 | if len(dt) > 10:
263 | data = dt.copy(deep=True)
264 | data = data.reset_index(drop=True)
265 | sj = data.copy(deep=True)
266 | sj['SP_Status'] = 0
267 |
268 | dc = 400
269 | tc = 600
270 | data['part_density'], data['scope'] = density(data, dc)
271 | SP = SP_search(data, tc)
272 | output = pd.DataFrame(SP)
273 |
274 | if output.empty == False:
275 | output.columns = ['index', 'longitude', 'latitude', 'arriveTime', 'leftTime', 'scope']
276 | output = similar(output, dc)
277 |
278 | for sco in output['scope']:
279 | start, end = sco[0], sco[1]
280 | tcog = sj['DRDIRECTION'][start:end]
281 | ss = 0 # 1靠泊、2锚泊
282 | if tcog.var() < 800:
283 | ss = 1
284 | else:
285 | ss = 2
286 | sj.loc[start:end, 'SP_Status'] = ss
287 | tdf = tdf.append(sj, ignore_index=True)
288 | return tdf
289 |
--------------------------------------------------------------------------------
/traj_preprocess/ais_stay/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib==3.5.1
2 | numpy==1.23.0
3 | pandas==1.4.1
4 | scipy==1.8.1
5 | vincenty==0.1.4
6 |
--------------------------------------------------------------------------------
/traj_show/__pycache__/folium.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/traj_show/__pycache__/folium.cpython-37.pyc
--------------------------------------------------------------------------------
/traj_show/requirements.txt:
--------------------------------------------------------------------------------
1 | folium==0.12.1.post1
2 | matplotlib==3.5.1
3 | numpy==1.21.6
4 | pandas==1.3.5
5 |
--------------------------------------------------------------------------------
/traj_show/trajshow.py:
--------------------------------------------------------------------------------
1 | import os
2 | import pandas as pd
3 | import numpy as np
4 | import webbrowser as wb
5 | import folium
6 | from folium.plugins import HeatMap, MiniMap, MarkerCluster
7 |
8 |
9 | # draw a heatmap
10 | def draw_heatmap(map):
11 | data = (
12 | np.random.normal(size=(100, 3)) *
13 | np.array([[1, 1, 1]]) +
14 | np.array([[30.9, 122.52, 1]])
15 | ).tolist()
16 | HeatMap(data).add_to(map)
17 |
18 |
19 | # add minimap
20 | def draw_minimap(map):
21 | minimap = MiniMap(toggle_display=True,
22 | tile_layer='Stamen Watercolor',
23 | position='topleft',
24 | width=100,
25 | height=100)
26 | map.add_child(minimap)
27 |
28 |
29 | def draw_circlemarker(loc, spd, cog, map):
30 | tip = 'Coordinates:' + str(loc) + "\t" + 'Speed:' + str(spd) + '\t' + 'COG:' + str(cog)
31 | folium.CircleMarker(
32 | location=loc,
33 | radius=3.6,
34 | color="blueviolet",
35 | stroke=True,
36 | fill_color='white',
37 | fill=True,
38 | weight=1.5,
39 | fill_opacity=1.0,
40 | opacity=1,
41 | tooltip=tip
42 | ).add_to(map)
43 |
44 |
45 | # draw a small information marker on the map
46 | def draw_icon(map, loc):
47 | mk = folium.features.Marker(loc)
48 | pp = folium.Popup(str(loc))
49 | ic = folium.features.Icon(color="blue")
50 | mk.add_child(ic)
51 | mk.add_child(pp)
52 | map.add_child(mk)
53 |
54 |
55 | # draw a stop marker on the map
56 | def draw_stop_icon(map, loc):
57 | # mk = folium.features.Marker(loc)
58 | # pp = folium.Popup(str(loc))
59 | # ic = folium.features.Icon(color='red', icon='anchor', prefix='fa')
60 | # mk.add_child(ic)
61 | # mk.add_child(pp)
62 | # map.add_child(mk)
63 | folium.Marker(loc).add_to(map)
64 |
65 |
66 | def draw_line(map, loc1, loc2):
67 | kw = {"opacity": 1.0, "weight": 6}
68 | folium.PolyLine(
69 | smooth_factor=10,
70 | locations=[loc1, loc2],
71 | color="red",
72 | tooltip="Trajectory",
73 | **kw,
74 | ).add_to(map)
75 |
76 |
77 | def draw_lines(map, coordinates, c):
78 | folium.PolyLine(
79 | smooth_factor=0,
80 | locations=coordinates,
81 | color=c,
82 | weight=0.5
83 | ).add_to(map)
84 |
85 |
86 | # save the result as HTML to the specified path
87 | def open_html(map, htmlpath):
88 | map.save(htmlpath)
89 | search_text = 'cdn.jsdelivr.net'
90 | replace_text = 'gcore.jsdelivr.net'
91 | with open(htmlpath, 'r', encoding='UTF-8') as file:
92 | data = file.read()
93 | data = data.replace(search_text, replace_text)
94 | with open(htmlpath, 'w', encoding='UTF-8') as file:
95 | file.write(data)
96 | chromepath = "C:\Program Files (x86)\Google\Chrome\Application\chrome.exe"
97 | wb.register('chrome', None, wb.BackgroundBrowser(chromepath))
98 | wb.get('chrome').open(htmlpath, autoraise=1)
99 |
100 |
101 | # read .csv file
102 | def read_traj_data(path):
103 | P = pd.read_csv(path, dtype={'DRLATITUDE': float, 'DRLONGITUDE': float})
104 | locations_total = P.loc[:, ['DRLATITUDE', 'DRLONGITUDE']].values.tolist()
105 | speed_total = P.loc[:, ['DRSPEED']].values.tolist()
106 | cog_total = P.loc[:, ['DRDIRECTION']].values.tolist()
107 | locations_stay = P.loc[P['STATUS'] == 1, ['DRLATITUDE', 'DRLONGITUDE']].values.tolist()
108 | lct = [P['DRLATITUDE'].mean(), P['DRLONGITUDE'].mean()]
109 | return locations_total, speed_total, cog_total, locations_stay, lct
110 |
111 |
112 | def draw_single_traj(csv_path):
113 | '''
114 | draw a single trajectory
115 | :param data: file path
116 | :return: null
117 | '''
118 | locations, spds, cogs, stays, ct = read_traj_data(csv_path)
119 | m = folium.Map(ct, zoom_start=15, attr='default') # 中心区域的确定
120 | folium.PolyLine( # polyline方法为将坐标用实线形式连接起来
121 | locations, # 将坐标点连接起来
122 | weight=1.0, # 线的大小为1
123 | color='blueviolet', # 线的颜色
124 | opacity=0.8, # 线的透明度
125 | ).add_to(m) # 将这条线添加到刚才的区域map内
126 | num = len(locations)
127 | for i in range(num):
128 | draw_circlemarker(locations[i], spds[i], cogs[i], m)
129 | for i in iter(stays):
130 | draw_stop_icon(m, i)
131 | output_path = os.getcwd() + './draw/show.html'
132 | open_html(m, output_path)
133 |
134 |
135 | def draw_trajs(file_path):
136 | '''
137 | draw multiple trajectories
138 | :param data: file path
139 | :return: null
140 | '''
141 | map = folium.Map([31.1, 122.5], zoom_start=10, attr='default') # 中心区域的确定
142 | draw_minimap(map)
143 | fls = os.listdir(file_path)
144 | scatterColors = ['blue', 'red', 'yellow', 'cyan', 'purple', 'orange', 'olive', 'brown', 'black', 'm']
145 | i = 0
146 | for x in fls:
147 | i = i + 1
148 | colorSytle = scatterColors[i % len(scatterColors)]
149 | df = pd.read_csv(file_path + "/" + x, encoding="gbk")
150 | df['DRMMSI'] = df['DRMMSI'].apply(lambda _: str(_))
151 | df['DRLONGITUDE'] = df['DRLONGITUDE'].map(lambda x: x / 1.0)
152 | df['DRLATITUDE'] = df['DRLATITUDE'].map(lambda x: x / 1.0)
153 | for shipmmsi, dt in df.groupby('DRMMSI'):
154 | if len(dt) > 2:
155 | dt_copy = dt.copy(deep=True)
156 | dt_copy.sort_values(by='DRGPSTIME', ascending=True, inplace=True)
157 | locations = dt_copy.loc[:, ['DRLATITUDE', 'DRLONGITUDE']].values.tolist()
158 | draw_lines(map, locations, colorSytle)
159 | output_path = os.getcwd() + './draw/show.html'
160 | open_html(map, output_path)
161 |
162 |
163 | if __name__ == '__main__':
164 | csv_path = r'./data/1.csv'
165 | draw_single_traj(csv_path)
166 | # csv_path = r'./data'
167 | # draw_trajs(csv_path)
168 |
--------------------------------------------------------------------------------
/traj_show/trajshow_matplotlib.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 | import numpy as np
3 | import pandas as pd
4 |
5 |
6 | def load_data(file_path):
7 | """
8 | import trajectory data
9 | :return: trajectory data
10 | """
11 | data = pd.read_csv(file_path, usecols=['DRGPSTIME', 'DRMMSI', 'DRLATITUDE', 'DRLONGITUDE'])
12 | data.rename(columns={'DRLONGITUDE': 'long', 'DRLATITUDE': 'lat', 'DRGPSTIME': 't', 'DRMMSI': 'mmsi'}, inplace=True)
13 | data['long'] = data['long'].map(lambda x: x / 600000.0)
14 | data['lat'] = data['lat'].map(lambda x: x / 600000.0)
15 | return data
16 |
17 |
18 | if __name__ == '__main__':
19 | trajectories = load_data('./data/1.csv')
20 | params = {'axes.titlesize': 'large',
21 | 'legend.fontsize': 14,
22 | 'legend.handlelength': 3}
23 | plt.rcParams.update(params)
24 |
25 | for shipmmsi, dt in trajectories.groupby('mmsi'):
26 | plt.plot(dt['long'].values, dt['lat'].values, color='green', linewidth=0.5)
27 |
28 | plt.yticks(fontproperties='Times New Roman', size=12)
29 | plt.xticks(fontproperties='Times New Roman', size=12)
30 | plt.xlabel('Longitude', fontdict={'family': 'Times New Roman', 'size': 14})
31 | plt.ylabel('Latitude', fontdict={'family': 'Times New Roman', 'size': 14})
32 | plt.title('Preprocessed Trajectories', fontdict={'family': 'Times New Roman', 'size': 14})
33 | plt.ticklabel_format(useOffset=False, style='plain')
34 | plt.grid(True)
35 | plt.tight_layout()
36 | plt.show()
37 |
--------------------------------------------------------------------------------
/user guide/Developer's Guide.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PilotLeaf/PyVT/4e5070d95d37a56df8ae5cd06f565b77d9016113/user guide/Developer's Guide.pdf
--------------------------------------------------------------------------------