├── libcity ├── __init__.py ├── model │ ├── __init__.py │ ├── traffic_flow_prediction │ │ ├── layers │ │ │ ├── silearn │ │ │ │ ├── model │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── __pycache__ │ │ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ │ │ ├── __init__.cpython-310.pyc │ │ │ │ │ │ ├── batched_graph.cpython-38.pyc │ │ │ │ │ │ ├── encoding_tree.cpython-38.pyc │ │ │ │ │ │ └── encoding_tree.cpython-310.pyc │ │ │ │ │ ├── dynamic_graphs.py │ │ │ │ │ ├── batched_graph.py │ │ │ │ │ └── encoding_tree.py │ │ │ │ ├── test │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── __pycache__ │ │ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ │ │ ├── test_graph.cpython-38.pyc │ │ │ │ │ │ ├── test_scipy_ops.cpython-38.pyc │ │ │ │ │ │ ├── test_torch_ops.cpython-38.pyc │ │ │ │ │ │ └── test_torch_ops.cpython-38-pytest-6.2.5.pyc │ │ │ │ │ ├── test_scipy_ops.py │ │ │ │ │ ├── test_graph.py │ │ │ │ │ └── test_torch_ops.py │ │ │ │ ├── __init__.py │ │ │ │ ├── optimizer │ │ │ │ │ └── enc │ │ │ │ │ │ ├── partitioning │ │ │ │ │ │ ├── node_switch.py │ │ │ │ │ │ ├── __pycache__ │ │ │ │ │ │ │ ├── propagation.cpython-310.pyc │ │ │ │ │ │ │ └── propagation.cpython-38.pyc │ │ │ │ │ │ └── merging.py │ │ │ │ │ │ ├── __pycache__ │ │ │ │ │ │ ├── operator.cpython-310.pyc │ │ │ │ │ │ └── operator.cpython-38.pyc │ │ │ │ │ │ └── operator.py │ │ │ │ ├── backends │ │ │ │ │ ├── scipy_ops │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── __pycache__ │ │ │ │ │ │ │ ├── __init__.cpython-310.pyc │ │ │ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ │ │ │ ├── graph_ops.cpython-310.pyc │ │ │ │ │ │ │ ├── graph_ops.cpython-38.pyc │ │ │ │ │ │ │ ├── matrix_ops.cpython-38.pyc │ │ │ │ │ │ │ └── matrix_ops.cpython-310.pyc │ │ │ │ │ │ ├── graph_ops.py │ │ │ │ │ │ └── matrix_ops.py │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── __pycache__ │ │ │ │ │ │ ├── __init__.cpython-310.pyc │ │ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ │ │ ├── functional.cpython-38.pyc │ │ │ │ │ │ └── functional.cpython-310.pyc │ │ │ │ │ ├── torch_ops │ │ │ │ │ │ ├── __pycache__ │ │ │ │ │ │ │ ├── __init__.cpython-310.pyc │ │ │ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ │ │ │ ├── graph_ops.cpython-310.pyc │ │ │ │ │ │ │ ├── graph_ops.cpython-38.pyc │ │ │ │ │ │ │ ├── matrix_ops.cpython-38.pyc │ │ │ │ │ │ │ └── matrix_ops.cpython-310.pyc │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── matrix_ops.py │ │ │ │ │ └── functional.py │ │ │ │ ├── __pycache__ │ │ │ │ │ ├── graph.cpython-310.pyc │ │ │ │ │ ├── graph.cpython-38.pyc │ │ │ │ │ ├── __init__.cpython-310.pyc │ │ │ │ │ └── __init__.cpython-38.pyc │ │ │ │ ├── utils │ │ │ │ │ └── external.py │ │ │ │ └── graph.py │ │ │ ├── __pycache__ │ │ │ │ ├── layer_utils.cpython-310.pyc │ │ │ │ ├── mask_layers.cpython-310.pyc │ │ │ │ ├── pe_layers.cpython-310.pyc │ │ │ │ ├── patch_layers.cpython-310.pyc │ │ │ │ └── STTransformer_layers.cpython-310.pyc │ │ │ ├── layer_utils.py │ │ │ ├── pe_layers.py │ │ │ └── mask_layers.py │ │ ├── __init__.py │ │ └── ST_transformer.py │ ├── abstract_traffic_tradition_model.py │ ├── abstract_model.py │ ├── abstract_traffic_state_model.py │ ├── utils.py │ └── loss.py ├── data │ ├── __init__.py │ ├── list_dataset.py │ ├── dataset │ │ ├── __init__.py │ │ ├── abstract_dataset.py │ │ └── traffic_state_point_dataset.py │ ├── utils.py │ └── batch.py ├── config │ ├── __init__.py │ ├── evaluator │ │ └── TrafficStateEvaluator.json │ ├── data │ │ ├── TrafficStateDataset.json │ │ └── TrafficStatePointDataset.json │ ├── model │ │ └── traffic_state_pred │ │ │ └── STTransformer.json │ ├── executor │ │ ├── TrafficStateExecutor.json │ │ └── TrafficStateExecutorTest.json │ ├── task_config.json │ └── config_parser.py ├── evaluator │ ├── __init__.py │ ├── abstract_evaluator.py │ ├── eval_funcs.py │ ├── utils.py │ └── traffic_state_evaluator.py ├── pipeline │ └── __init__.py ├── executor │ ├── __init__.py │ ├── abstract_executor.py │ ├── abstract_tradition_executor.py │ ├── scheduler.py │ └── hyper_tuning.py └── utils │ ├── __init__.py │ ├── GPS_utils.py │ ├── normalization.py │ ├── argument_list.py │ ├── dataset.py │ └── utils.py ├── README.md └── run_model.py /libcity/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /libcity/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/__init__.py: -------------------------------------------------------------------------------- 1 | from silearn.backends.torch_ops import * 2 | -------------------------------------------------------------------------------- /libcity/data/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.data.utils import get_dataset 2 | 3 | __all__ = [ 4 | "get_dataset" 5 | ] 6 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/partitioning/node_switch.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /libcity/config/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.config.config_parser import ConfigParser 2 | 3 | __all__ = [ 4 | 'ConfigParser' 5 | ] 6 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from .graph_ops import * 4 | from .matrix_ops import * 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /libcity/evaluator/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.evaluator.traffic_state_evaluator import TrafficStateEvaluator 2 | 3 | 4 | __all__ = [ 5 | "TrafficStateEvaluator", 6 | ] 7 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.model.traffic_flow_prediction.ST_transformer import STTransformer 2 | 3 | __all__ = [ 4 | "STTransformer" 5 | ] 6 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # from .functional import * 2 | # from .torch_ops import torch_available 3 | from .torch_ops import * 4 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/__pycache__/layer_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/__pycache__/layer_utils.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/__pycache__/mask_layers.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/__pycache__/mask_layers.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/__pycache__/pe_layers.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/__pycache__/pe_layers.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.pipeline.pipeline import run_model, hyper_parameter, objective_function 2 | 3 | __all__ = [ 4 | "run_model", 5 | "hyper_parameter", 6 | "objective_function" 7 | ] 8 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/__pycache__/patch_layers.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/__pycache__/patch_layers.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/graph.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/graph.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/graph.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/graph.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/__pycache__/STTransformer_layers.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/__pycache__/STTransformer_layers.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/config/evaluator/TrafficStateEvaluator.json: -------------------------------------------------------------------------------- 1 | { 2 | "metrics": ["MAE", "MAPE", "MSE", "RMSE", "masked_MAE", "masked_MAPE", "masked_MSE", "masked_RMSE", "R2", "EVAR"], 3 | "evaluator_mode": "average", 4 | "save_mode": ["csv"] 5 | } 6 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_graph.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_graph.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/functional.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/functional.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/batched_graph.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/batched_graph.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/encoding_tree.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/encoding_tree.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_scipy_ops.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_scipy_ops.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_torch_ops.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_torch_ops.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/functional.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/__pycache__/functional.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/encoding_tree.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/model/__pycache__/encoding_tree.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/__pycache__/operator.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/__pycache__/operator.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/__pycache__/operator.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/__pycache__/operator.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/graph_ops.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/graph_ops.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/graph_ops.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/graph_ops.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/matrix_ops.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/matrix_ops.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/graph_ops.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/graph_ops.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/graph_ops.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/graph_ops.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/matrix_ops.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/matrix_ops.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/matrix_ops.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/__pycache__/matrix_ops.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/matrix_ops.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__pycache__/matrix_ops.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_torch_ops.cpython-38-pytest-6.2.5.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/test/__pycache__/test_torch_ops.cpython-38-pytest-6.2.5.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/partitioning/__pycache__/propagation.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/partitioning/__pycache__/propagation.cpython-310.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/partitioning/__pycache__/propagation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SELGroup/MultiSPANS/HEAD/libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/partitioning/__pycache__/propagation.cpython-38.pyc -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib.util 2 | 3 | if importlib.util.find_spec("torch") is None: 4 | torch_available = False 5 | else: 6 | torch_available = True 7 | from .graph_ops import * 8 | from .matrix_ops import * 9 | -------------------------------------------------------------------------------- /libcity/executor/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.executor.hyper_tuning import HyperTuning 2 | from libcity.executor.traffic_state_executor import TrafficStateExecutor 3 | from libcity.executor.traffic_state_executor_evaltest import TrafficStateExecutorTest 4 | 5 | __all__ = [ 6 | "TrafficStateExecutor", 7 | "HyperTuning", 8 | "TrafficStateExecutorTest" 9 | ] 10 | -------------------------------------------------------------------------------- /libcity/data/list_dataset.py: -------------------------------------------------------------------------------- 1 | from torch.utils.data import Dataset 2 | 3 | 4 | class ListDataset(Dataset): 5 | def __init__(self, data): 6 | """ 7 | data: 必须是一个 list 8 | """ 9 | self.data = data 10 | 11 | def __getitem__(self, index): 12 | return self.data[index] 13 | 14 | def __len__(self): 15 | return len(self.data) 16 | -------------------------------------------------------------------------------- /libcity/model/abstract_traffic_tradition_model.py: -------------------------------------------------------------------------------- 1 | class AbstractTraditionModel: 2 | 3 | def __init__(self, config, data_feature): 4 | self.data_feature = data_feature 5 | 6 | def run(self, data): 7 | """ 8 | Args: 9 | data : input of tradition model 10 | 11 | Returns: 12 | output of tradition model 13 | """ 14 | -------------------------------------------------------------------------------- /libcity/data/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.data.dataset.abstract_dataset import AbstractDataset 2 | from libcity.data.dataset.traffic_state_datatset import TrafficStateDataset 3 | from libcity.data.dataset.traffic_state_point_dataset import TrafficStatePointDataset 4 | 5 | 6 | __all__ = [ 7 | "AbstractDataset", 8 | "TrafficStateDataset", 9 | "TrafficStatePointDataset", 10 | ] 11 | -------------------------------------------------------------------------------- /libcity/config/data/TrafficStateDataset.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size": 32, 3 | "cache_dataset": true, 4 | "num_workers": 0, 5 | "pad_with_last_sample": true, 6 | "train_rate": 0.6, 7 | "eval_rate": 0.2, 8 | "scaler": "none", 9 | "load_external": false, 10 | "normal_external": false, 11 | "ext_scaler": "none", 12 | "input_window": 12, 13 | "output_window": 12, 14 | "add_time_in_day": false, 15 | "add_day_in_week": false 16 | } 17 | -------------------------------------------------------------------------------- /libcity/config/data/TrafficStatePointDataset.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size": 32, 3 | "cache_dataset": true, 4 | "num_workers": 0, 5 | "pad_with_last_sample": true, 6 | "train_rate": 0.6, 7 | "eval_rate": 0.2, 8 | "scaler": "standardindependc", 9 | "load_external": false, 10 | "normal_external": false, 11 | "ext_scaler": "none", 12 | "input_window": 12, 13 | "output_window": 12, 14 | "add_time_in_day": false, 15 | "add_day_in_week": false 16 | } 17 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/graph_ops.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy as np 3 | 4 | 5 | from silearn.graph import GraphSparse, GraphDense 6 | 7 | 8 | def vertex_reduce_sparse(g:GraphSparse): 9 | 10 | pass 11 | 12 | 13 | def nonzero_idx_dense(g: GraphDense, return_weights = True): 14 | es, et = np.nonzero(g.adj, as_tuple=True) 15 | 16 | if return_weights: 17 | return es, et, g.adj[es][et] 18 | else: 19 | return es, et -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MultiSPANS 2 | 3 | The code of MultiSPANS: A Multi-range Spatial-Temporal Transformer Network for Traffic Forecast via Structural Entropy Optimization, which is accepted by WSDM2024. 4 | 5 | The core code is located at ./TrafficForecast/MultiSPANS/libcity/model/traffic_flow_prediction 6 | The project can be fully adapted to Libcity benckmark. 7 | 8 | 9 | ## Requirements 10 | The implementation of MultiSPANS is under Python 3.10, with the following key packages installed: 11 | 12 | pytorch == 1.12.0 13 | 14 | torch-geometric == 2.3.0 15 | -------------------------------------------------------------------------------- /libcity/model/abstract_model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class AbstractModel(nn.Module): 5 | 6 | def __init__(self, config, data_feature): 7 | nn.Module.__init__(self) 8 | 9 | def predict(self, batch): 10 | """ 11 | Args: 12 | batch (Batch): a batch of input 13 | 14 | Returns: 15 | torch.tensor: predict result of this batch 16 | """ 17 | 18 | def calculate_loss(self, batch): 19 | """ 20 | Args: 21 | batch (Batch): a batch of input 22 | 23 | Returns: 24 | torch.tensor: return training loss 25 | """ 26 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/operator.py: -------------------------------------------------------------------------------- 1 | 2 | from silearn.model.encoding_tree import GraphEncoding 3 | 4 | 5 | class Operator(): 6 | """ 7 | Map the encoding tree node to intermediate values in computing 8 | """ 9 | cache_Vol: None 10 | cache_G: None 11 | cache_Eij: None 12 | 13 | 14 | def __init__(self, enc: GraphEncoding): 15 | self.enc = enc 16 | 17 | def perform(self, re_compute=True): 18 | """ 19 | Perform Optimization on Graph 20 | @:returns the variance of structural information 21 | """ 22 | pass 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/layer_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch import Tensor 4 | 5 | 6 | class Permution(nn.Module): 7 | def __init__(self, *dims, contiguous=False): 8 | super().__init__() 9 | self.dims, self.contiguous = dims, contiguous 10 | def forward(self, x:Tensor): 11 | if self.contiguous: return x.permute(*self.dims).contiguous() 12 | else: return x.permute(*self.dims) 13 | 14 | def Norm(norm,embed_dim): 15 | if "batch" in norm.lower(): 16 | return nn.Sequential(Permution(0,3,1,2), nn.BatchNorm2d(embed_dim), Permution(0,2,3,1)) 17 | else: 18 | return nn.LayerNorm(embed_dim) -------------------------------------------------------------------------------- /libcity/config/model/traffic_state_pred/STTransformer.json: -------------------------------------------------------------------------------- 1 | { 2 | "embed_dim":64, 3 | "skip_conv_flag" : false, 4 | "residual_conv_flag" : false, 5 | "skip_dim":64, 6 | "num_layers":3, 7 | "num_heads": 8, 8 | 9 | "conv_kernels":[1,2,3,6], 10 | "conv_stride":1, 11 | "conv_if_gc":true, 12 | "norm_type":"BatchNorm", 13 | 14 | "gconv_hop_num" : 3, 15 | "gconv_alpha" : 0, 16 | 17 | "att_dropout":0.1, 18 | "ffn_dropout":0.1, 19 | "Satt_pe_type":"laplacian", 20 | "Spe_learnable":false, 21 | "Tatt_pe_type":"sincos", 22 | "Tpe_learnable":false, 23 | "Smask_flag":true, 24 | "block_forward_mode":0, 25 | "sstore_attn":false 26 | } 27 | -------------------------------------------------------------------------------- /libcity/config/executor/TrafficStateExecutor.json: -------------------------------------------------------------------------------- 1 | { 2 | "gpu": true, 3 | "gpu_id": 0, 4 | "max_epoch": 20, 5 | "train_loss": "none", 6 | "epoch": 0, 7 | "learner": "adam", 8 | "learning_rate": 0.01, 9 | "weight_decay": 0, 10 | "lr_epsilon": 1e-8, 11 | "lr_beta1": 0.9, 12 | "lr_beta2": 0.999, 13 | "lr_alpha": 0.99, 14 | "lr_momentum": 0, 15 | "lr_decay": false, 16 | "lr_scheduler": "multisteplr", 17 | "lr_decay_ratio": 0.1, 18 | "steps": [5, 20, 40, 70], 19 | "step_size": 10, 20 | "lr_T_max": 30, 21 | "lr_eta_min": 0, 22 | "lr_patience": 10, 23 | "lr_threshold": 1e-4, 24 | "clip_grad_norm": false, 25 | "max_grad_norm": 1.0, 26 | "use_early_stop": false, 27 | "patience": 50, 28 | "log_level": "INFO", 29 | "log_every": 1, 30 | "saved_model": true, 31 | "load_best_epoch": true, 32 | "hyper_tune": false 33 | } 34 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/utils/external.py: -------------------------------------------------------------------------------- 1 | import silearn.backends 2 | 3 | def from_networkx(): 4 | 5 | pass 6 | 7 | def graph_from_torch_sparse(): 8 | pass 9 | 10 | def enc_from_torch_sparse(): 11 | pass 12 | 13 | def from_cugraph(): 14 | pass 15 | 16 | def to_cugraph(): 17 | pass 18 | 19 | 20 | def create_cugraph(es, et, w): 21 | import cudf, cupy, cugraph 22 | df = cudf.DataFrame() 23 | df["es"]=cupy.array(es) 24 | df["et"]=cupy.array(et) 25 | df["w"]=cupy.array(w) 26 | return cugraph.from_cudf_edgelist(df, "es", "et", "w", create_using=cugraph.MultiGraph, renumber=False) 27 | 28 | def create_cupy_partitioning(com): 29 | import cudf, cupy, cugraph 30 | df = cudf.DataFrame() 31 | df["vertex"] = cupy.arange(com.shape[0]) 32 | df["cluster"] = cupy.array(com) 33 | return df -------------------------------------------------------------------------------- /libcity/data/dataset/abstract_dataset.py: -------------------------------------------------------------------------------- 1 | class AbstractDataset(object): 2 | 3 | def __init__(self, config): 4 | raise NotImplementedError("Dataset not implemented") 5 | 6 | def get_data(self): 7 | """ 8 | 返回数据的DataLoader，包括训练数据、测试数据、验证数据 9 | 10 | Returns: 11 | tuple: tuple contains: 12 | train_dataloader: Dataloader composed of Batch (class) \n 13 | eval_dataloader: Dataloader composed of Batch (class) \n 14 | test_dataloader: Dataloader composed of Batch (class) 15 | """ 16 | raise NotImplementedError("get_data not implemented") 17 | 18 | def get_data_feature(self): 19 | """ 20 | 返回一个 dict，包含数据集的相关特征 21 | 22 | Returns: 23 | dict: 包含数据集的相关特征的字典 24 | """ 25 | raise NotImplementedError("get_data_feature not implemented") 26 | -------------------------------------------------------------------------------- /libcity/model/abstract_traffic_state_model.py: -------------------------------------------------------------------------------- 1 | from libcity.model.abstract_model import AbstractModel 2 | 3 | 4 | class AbstractTrafficStateModel(AbstractModel): 5 | 6 | def __init__(self, config, data_feature): 7 | self.data_feature = data_feature 8 | super().__init__(config, data_feature) 9 | 10 | def predict(self, batch): 11 | """ 12 | 输入一个batch的数据，返回对应的预测值，一般应该是**多步预测**的结果，一般会调用nn.Moudle的forward()方法 13 | 14 | Args: 15 | batch (Batch): a batch of input 16 | 17 | Returns: 18 | torch.tensor: predict result of this batch 19 | """ 20 | 21 | def calculate_loss(self, batch): 22 | """ 23 | 输入一个batch的数据，返回训练过程的loss，也就是需要定义一个loss函数 24 | 25 | Args: 26 | batch (Batch): a batch of input 27 | 28 | Returns: 29 | torch.tensor: return training loss 30 | """ 31 | -------------------------------------------------------------------------------- /libcity/config/executor/TrafficStateExecutorTest.json: -------------------------------------------------------------------------------- 1 | { 2 | "gpu": true, 3 | "gpu_id": 0, 4 | "max_epoch": 100, 5 | "train_loss": "masked_mae", 6 | "epoch": 0, 7 | "learner": "adam", 8 | "learning_rate": 0.01, 9 | "weight_decay": 0, 10 | "lr_epsilon": 1e-8, 11 | "lr_beta1": 0.9, 12 | "lr_beta2": 0.999, 13 | "lr_alpha": 0.99, 14 | "lr_momentum": 0, 15 | "lr_decay": false, 16 | "lr_scheduler": "multisteplr", 17 | "lr_decay_ratio": 0.1, 18 | "steps": [5, 20, 40, 70], 19 | "step_size": 10, 20 | "lr_T_max": 30, 21 | "lr_eta_min": 0, 22 | "lr_patience": 10, 23 | "lr_threshold": 1e-4, 24 | "clip_grad_norm": false, 25 | "max_grad_norm": 1.0, 26 | "use_early_stop": false, 27 | "patience": 50, 28 | "log_level": "INFO", 29 | "log_every": 1, 30 | "saved_model": true, 31 | "load_best_epoch": true, 32 | "hyper_tune": false, 33 | "if_test_epoch": true, 34 | "if_inv_loss": true, 35 | "if_inv_eval": true, 36 | "pred_channel_idx":[0], 37 | "outfeat_dim":1 38 | } 39 | -------------------------------------------------------------------------------- /libcity/evaluator/abstract_evaluator.py: -------------------------------------------------------------------------------- 1 | class AbstractEvaluator(object): 2 | 3 | def __init__(self, config): 4 | raise NotImplementedError('evaluator not implemented') 5 | 6 | def collect(self, batch): 7 | """ 8 | 收集一 batch 的评估输入 9 | 10 | Args: 11 | batch(dict): 输入数据 12 | """ 13 | raise NotImplementedError('evaluator collect not implemented') 14 | 15 | def evaluate(self): 16 | """ 17 | 返回之前收集到的所有 batch 的评估结果 18 | """ 19 | raise NotImplementedError('evaluator evaluate not implemented') 20 | 21 | def save_result(self, save_path, filename=None): 22 | """ 23 | 将评估结果保存到 save_path 文件夹下的 filename 文件中 24 | 25 | Args: 26 | save_path: 保存路径 27 | filename: 保存文件名 28 | """ 29 | raise NotImplementedError('evaluator save_result not implemented') 30 | 31 | def clear(self): 32 | """ 33 | 清除之前收集到的 batch 的评估信息，适用于每次评估开始时进行一次清空，排除之前的评估输入的影响。 34 | """ 35 | raise NotImplementedError('evaluator clear not implemented') 36 | -------------------------------------------------------------------------------- /libcity/config/task_config.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | 4 | "traffic_state_pred": { 5 | "allowed_model": [ 6 | "STTransformer" 7 | ], 8 | "allowed_dataset": [ 9 | "METR_LA", "PEMS_BAY", "PEMSD3", "PEMSD4", "PEMSD7", "PEMSD8", "PEMSD7(M)", 10 | "LOOP_SEATTLE", "LOS_LOOP", "LOS_LOOP_SMALL", "Q_TRAFFIC", "SZ_TAXI", 11 | "NYCBike20140409", "NYCBike20160708", "NYCBike20160809", "NYCTaxi20140112", 12 | "NYCTaxi20150103", "NYCTaxi20160102", "TAXIBJ", "T_DRIVE20150206", 13 | "BEIJING_SUBWAY_10MIN", "BEIJING_SUBWAY_15MIN", "BEIJING_SUBWAY_30MIN", 14 | "ROTTERDAM", "HZMETRO", "SHMETRO", "M_DENSE", "PORTO", "NYCTAXI_DYNA", 15 | "NYCTAXI_OD", "NYCTAXI_GRID", "T_DRIVE_SMALL", "NYCBIKE", "AUSTINRIDE", 16 | "BIKEDC", "BIKECHI", "NYC_RISK", "CHICAGO_RISK" 17 | ], 18 | "STTransformer": { 19 | "dataset_class": "TrafficStatePointDataset", 20 | "executor": "TrafficStateExecutorTest", 21 | "evaluator": "TrafficStateEvaluator" 22 | } 23 | 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /libcity/executor/abstract_executor.py: -------------------------------------------------------------------------------- 1 | class AbstractExecutor(object): 2 | 3 | def __init__(self, config, model, data_feature): 4 | raise NotImplementedError("Executor not implemented") 5 | 6 | def train(self, train_dataloader, eval_dataloader): 7 | """ 8 | use data to train model with config 9 | 10 | Args: 11 | train_dataloader(torch.Dataloader): Dataloader 12 | eval_dataloader(torch.Dataloader): Dataloader 13 | """ 14 | raise NotImplementedError("Executor train not implemented") 15 | 16 | def evaluate(self, test_dataloader): 17 | """ 18 | use model to test data 19 | 20 | Args: 21 | test_dataloader(torch.Dataloader): Dataloader 22 | """ 23 | raise NotImplementedError("Executor evaluate not implemented") 24 | 25 | def load_model(self, cache_name): 26 | """ 27 | 加载对应模型的 cache 28 | 29 | Args: 30 | cache_name(str): 保存的文件名 31 | """ 32 | raise NotImplementedError("Executor load cache not implemented") 33 | 34 | def save_model(self, cache_name): 35 | """ 36 | 将当前的模型保存到文件 37 | 38 | Args: 39 | cache_name(str): 保存的文件名 40 | """ 41 | raise NotImplementedError("Executor save cache not implemented") 42 | -------------------------------------------------------------------------------- /libcity/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from libcity.utils.utils import get_executor, get_model, get_evaluator, \ 2 | get_logger, get_local_time, ensure_dir, trans_naming_rule, preprocess_data, set_random_seed 3 | from libcity.utils.dataset import parse_time, cal_basetime, cal_timeoff, \ 4 | caculate_time_sim, parse_coordinate, string2timestamp, timestamp2array, \ 5 | timestamp2vec_origin 6 | from libcity.utils.argument_list import general_arguments, str2bool, \ 7 | str2float, hyper_arguments, add_general_args, add_hyper_args 8 | from libcity.utils.normalization import Scaler, NoneScaler, NormalScaler, \ 9 | StandardScaler, MinMax01Scaler, MinMax11Scaler, LogScaler, StandardIndependCScaler 10 | 11 | __all__ = [ 12 | "get_executor", 13 | "get_model", 14 | "get_evaluator", 15 | "get_logger", 16 | "get_local_time", 17 | "ensure_dir", 18 | "trans_naming_rule", 19 | "preprocess_data", 20 | "parse_time", 21 | "cal_basetime", 22 | "cal_timeoff", 23 | "caculate_time_sim", 24 | "parse_coordinate", 25 | "string2timestamp", 26 | "timestamp2array", 27 | "timestamp2vec_origin", 28 | "general_arguments", 29 | "hyper_arguments", 30 | "str2bool", 31 | "str2float", 32 | "Scaler", 33 | "NoneScaler", 34 | "NormalScaler", 35 | "StandardScaler", 36 | "MinMax01Scaler", 37 | "MinMax11Scaler", 38 | "LogScaler", 39 | "StandardIndependCScaler", 40 | "set_random_seed", 41 | "add_general_args", 42 | "add_hyper_args" 43 | ] 44 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/test_scipy_ops.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | import numpy as np 5 | import torch 6 | from numpy.testing import assert_equal 7 | from torch_geometric.seed import seed_everything 8 | 9 | from silearn.backends.scipy_ops import * 10 | 11 | # Sets the seed for generating random numbers in PyTorch, numpy and Python. 12 | seed_everything(42) 13 | 14 | class Testbackends(unittest.TestCase): 15 | def setUp(self): 16 | # self.adj = 17 | 18 | self.src = np.array([[1, 2], [5, 6], [3, 4], [7, 8], [9, 10], [11, 12]]) 19 | self.idx = np.array([0, 1, 0, 1, 1, 3]) 20 | self.cnt = np.array([2,3,0,1]) 21 | self.sum = np.array([[4, 6], [21, 24], [0, 0], [11, 12]]) 22 | self.max = np.array([[3, 4], [9, 10], [0, 0], [11, 12]]) 23 | 24 | self.p = np.array([1, 0, 3, 1]) 25 | self.q = np.array([1, 1/2, 1/4, 0]) 26 | 27 | def test_nonzero_idx_dense(self): 28 | pass 29 | 30 | def test_vertex_reduce(self): 31 | pass 32 | 33 | def test_scatter_sum(self): 34 | assert_equal(scatter_sum(self.src, self.idx), self.sum) 35 | 36 | def test_scatter_cnt(self): 37 | assert_equal(scatter_cnt(self.idx), self.cnt) 38 | 39 | def test_entropy(self): 40 | ground_truth = np.array([0, 0, 6, np.log2(1e36)]) 41 | assert_equal(entropy(self.p, self.q), ground_truth) 42 | 43 | def test_uncertainty(self): 44 | ground_truth = np.array([0, 1, 2, np.log2(1e36)]) 45 | assert_equal(uncertainty(self.q), ground_truth) 46 | 47 | 48 | if __name__ == '__main__': 49 | unittest.main() -------------------------------------------------------------------------------- /run_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | 训练并评估单一模型的脚本 3 | """ 4 | 5 | import argparse 6 | 7 | from libcity.pipeline import run_model 8 | from libcity.utils import str2bool, add_general_args 9 | 10 | 11 | if __name__ == '__main__': 12 | parser = argparse.ArgumentParser() 13 | # 增加指定的参数 14 | parser.add_argument('--task', type=str, 15 | default='traffic_state_pred', help='the name of task') 16 | parser.add_argument('--model', type=str, 17 | default='STTransformer', help='the name of model') 18 | # parser.add_argument('--model', type=str, 19 | # default='STTN', help='the name of model') 20 | parser.add_argument('--dataset', type=str, 21 | default='METR_LA', help='the name of dataset') 22 | parser.add_argument('--config_file', type=str, 23 | default=None, help='the file name of config file') 24 | parser.add_argument('--saved_model', type=str2bool, 25 | default=True, help='whether save the trained model') 26 | parser.add_argument('--train', type=str2bool, default=True, 27 | help='whether re-train model if the model is trained before') 28 | parser.add_argument('--exp_id', type=str, default=None, help='id of experiment') 29 | parser.add_argument('--scalar', type=str, default='standard', help='normalization of dataset') 30 | parser.add_argument('--seed', type=int, default=0, help='random seed') 31 | # 增加其他可选的参数 32 | add_general_args(parser) 33 | # 解析参数 34 | args = parser.parse_args() 35 | args.cache_dataset = False 36 | args.load_external = True 37 | args.add_day_in_week = True 38 | args.add_time_in_day = True 39 | dict_args = vars(args) 40 | other_args = {key: val for key, val in dict_args.items() if key not in [ 41 | 'task', 'model', 'dataset', 'config_file', 'saved_model', 'train'] and 42 | val is not None} 43 | run_model(task=args.task, model_name=args.model, dataset_name=args.dataset, 44 | config_file=args.config_file, saved_model=args.saved_model, 45 | train=args.train, other_args=other_args) -------------------------------------------------------------------------------- /libcity/data/dataset/traffic_state_point_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from libcity.data.dataset import TrafficStateDataset 4 | 5 | 6 | class TrafficStatePointDataset(TrafficStateDataset): 7 | 8 | def __init__(self, config): 9 | super().__init__(config) 10 | self.cache_file_name = os.path.join('./libcity/cache/dataset_cache/', 11 | 'point_based_{}.npz'.format(self.parameters_str)) 12 | 13 | def _load_geo(self): 14 | """ 15 | 加载.geo文件，格式[geo_id, type, coordinates, properties(若干列)] 16 | """ 17 | super()._load_geo() 18 | 19 | def _load_rel(self): 20 | """ 21 | 加载.rel文件，格式[rel_id, type, origin_id, destination_id, properties(若干列)] 22 | 23 | Returns: 24 | np.ndarray: self.adj_mx, N*N的邻接矩阵 25 | """ 26 | super()._load_rel() 27 | 28 | def _load_dyna(self, filename): 29 | """ 30 | 加载.dyna文件，格式[dyna_id, type, time, entity_id, properties(若干列)] 31 | 其中全局参数`data_col`用于指定需要加载的数据的列，不设置则默认全部加载 32 | 33 | Args: 34 | filename(str): 数据文件名，不包含后缀 35 | 36 | Returns: 37 | np.ndarray: 数据数组, 3d-array (len_time, num_nodes, feature_dim) 38 | """ 39 | return super()._load_dyna_3d(filename) 40 | 41 | def _add_external_information(self, df, ext_data=None): 42 | """ 43 | 增加外部信息（一周中的星期几/day of week，一天中的某个时刻/time of day，外部数据） 44 | 45 | Args: 46 | df(np.ndarray): 交通状态数据多维数组, (len_time, num_nodes, feature_dim) 47 | ext_data(np.ndarray): 外部数据 48 | 49 | Returns: 50 | np.ndarray: 融合后的外部数据和交通状态数据, (len_time, num_nodes, feature_dim_plus) 51 | """ 52 | return super()._add_external_information_3d(df, ext_data) 53 | 54 | def get_data_feature(self): 55 | """ 56 | 返回数据集特征，scaler是归一化方法，adj_mx是邻接矩阵，num_nodes是点的个数， 57 | feature_dim是输入数据的维度，output_dim是模型输出的维度 58 | 59 | Returns: 60 | dict: 包含数据集的相关特征的字典 61 | """ 62 | return {"scaler": self.scaler, "adj_mx": self.adj_mx, "ext_dim": self.ext_dim, 63 | "num_nodes": self.num_nodes, "feature_dim": self.feature_dim, 64 | "output_dim": self.output_dim, "num_batches": self.num_batches} 65 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/test_graph.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import torch 4 | from torch_geometric.seed import seed_everything 5 | from silearn.graph import * 6 | 7 | import silearn 8 | 9 | # Sets the seed for generating random numbers in PyTorch, numpy and Python. 10 | seed_everything(42) 11 | 12 | class TestGraphEncoding(unittest.TestCase): 13 | def setUp(self): 14 | self.adj = torch.tensor([[0, 0, 1, 0],[0, 0, 4, 3], [1, 4, 0, 2], [0, 3, 2, 0]]) 15 | self.adj = self.adj / self.adj.sum() 16 | self.edges = torch.nonzero(self.adj) 17 | self.p = self.adj[self.adj!=0] 18 | self.dist = torch.mm(torch.tensor([[1.0,1.0,1.0,1.0]]),torch.inverse((self.adj / self.adj.sum(dim=1, keepdim=True)) - torch.eye(4) + torch.ones((4,4)))).reshape(-1) 19 | self.graph = GraphSparse(self.edges, self.p, dist=self.dist) 20 | 21 | def test_parameters(self): 22 | # print(self.graph.) 23 | assert (hasattr(self.graph, 'device') and self.graph.device is not None) 24 | assert (hasattr(self.graph, 'backend') and self.graph.backend is not None) 25 | assert (hasattr(self.graph, 'directed') and self.graph.directed is not None) 26 | assert (hasattr(self.graph, 'num_vertices') and self.graph.num_vertices is not None) 27 | assert (hasattr(self.graph, 'num_edges') and self.graph.num_edges is not None) 28 | assert (hasattr(self.graph, 'stationary_dist') and self.graph.stationary_dist is not None) 29 | assert (hasattr(self.graph, 'edges') and self.graph.edges is not None) 30 | assert (hasattr(self.graph, 'vertex_weight_es') and self.graph.vertex_weight_es is not None) 31 | 32 | def test_device(self): 33 | assert self.graph.device == torch.device('cpu') 34 | 35 | def test_num_vertices(self): 36 | assert self.graph.num_vertices == 4 37 | 38 | def test_num_edges(self): 39 | assert self.graph.num_edges == 8 40 | 41 | def test_vertex_weight_es(self): 42 | assert torch.all(torch.isclose(self.graph.vertex_weight_es, torch.tensor([0.05, 0.35, 0.35, 0.25]))) 43 | 44 | def test_edges(self): 45 | edge_index, p = self.graph.edges 46 | assert torch.all(edge_index == self.edges) and torch.all(p == self.p) 47 | 48 | def test_stationary_dist(self): 49 | assert torch.all(self.graph.stationary_dist==self.dist) 50 | 51 | def test_query_probability(self): 52 | assert self.graph.query_probability(1, 2) == 0.2 53 | 54 | 55 | 56 | if __name__ == '__main__': 57 | unittest.main() -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/dynamic_graphs.py: -------------------------------------------------------------------------------- 1 | import silearn 2 | from silearn.graph import Graph, GraphSparse, GraphDense 3 | from encoding_tree import Partitioning 4 | # from .dynamic_nodes import UpdateMethod, DynamicNode, UpdateType 5 | 6 | 7 | # class IncreSparseGraph(GraphSparse, DynamicNode): 8 | class IncreSparseGraph(GraphSparse): 9 | r""" 10 | [Undirected] graph model with helper functions of modifying edge weights and edge sets. 11 | Weighted denotes the relative probability of random walking. 12 | """ 13 | tot_weights = 0 14 | dup_edges = False 15 | 16 | def __init__(self, edges, weights, dist=None, n_vertices=None): 17 | super().__init__(edges, weights, dist, n_vertices) 18 | self.original_tot_weight = weights.sum() 19 | self.directed = False 20 | 21 | @staticmethod 22 | def as_incremental(g: GraphSparse): 23 | return IncreSparseGraph(g._edges, 24 | g._p, 25 | g._dist, 26 | n_vertices=g.n_vertices) 27 | 28 | # @UpdateMethod(UpdateType.incre_edge_insertion) 29 | def insert_edges(self, edges, weights): 30 | r""" 31 | edges[i][0], edges[i][1]: the start / target node-id of the i-th inserted edge 32 | """ 33 | self.tot_weights += weights.sum() 34 | self._edges = silearn.concat((self._edges, edges), dim=0) 35 | self._p = silearn.concat(self._p, weights) 36 | 37 | @property 38 | def stationary_dist(self): 39 | return self._dist / self.tot_weights 40 | 41 | def refresh_stat_dist(self): 42 | self._dist = silearn.scatter_sum(self._p, self.edges[0]) 43 | 44 | def clean_dup_edges(self): 45 | if not self.dup_edges: 46 | return False 47 | self.dup_edges = False 48 | self._edges, self._p = silearn.sumup_duplicates(self._edges, self._p) 49 | 50 | def clone(self): 51 | return IncreSparseGraph(silearn.clone(self._edges), 52 | silearn.clone(self._p), 53 | n_vertices=self.n_vertices) 54 | 55 | def combine_graph(self, g: GraphSparse): 56 | e, p = g.edges 57 | self.insert_edges(e, p) 58 | 59 | 60 | def vertex_reduction(self, graph: Graph, encoding: Partitioning): 61 | r""" 62 | Vertex reduction maintaining the structural entropy 63 | """ 64 | if isinstance(graph, GraphSparse): 65 | edges, p = graph.edges 66 | dist = graph.stationary_dist 67 | edges = encoding.node_id[edges] 68 | dist = silearn.scatter_sum(dist, encoding) 69 | edges, p = silearn.sumup_duplicates(edges, p) 70 | return GraphSparse(edges, p, dist, n_vertices=dist.shape[-1]) 71 | else: 72 | # todo: implement GraphDense 73 | raise NotImplementedError("Not implemented yet") 74 | 75 | 76 | # todo: implementation 77 | # class IncrementalSparseDiGraph(IncrementalSparseGraph): 78 | # -------------------------------------------------------------------------------- /libcity/executor/abstract_tradition_executor.py: -------------------------------------------------------------------------------- 1 | from libcity.executor.abstract_executor import AbstractExecutor 2 | from logging import getLogger 3 | from libcity.utils import get_evaluator, ensure_dir 4 | import numpy as np 5 | import torch 6 | import time 7 | import os 8 | 9 | 10 | class AbstractTraditionExecutor(AbstractExecutor): 11 | def __init__(self, config, model, data_feature): 12 | self.evaluator = get_evaluator(config) 13 | self.config = config 14 | self.data_feature = data_feature 15 | self.device = self.config.get('device', torch.device('cpu')) 16 | self.model = model 17 | self.exp_id = self.config.get('exp_id', None) 18 | 19 | self.cache_dir = './libcity/cache/{}/model_cache'.format(self.exp_id) 20 | self.evaluate_res_dir = './libcity/cache/{}/evaluate_cache'.format(self.exp_id) 21 | 22 | ensure_dir(self.cache_dir) 23 | ensure_dir(self.evaluate_res_dir) 24 | 25 | self._logger = getLogger() 26 | self._scaler = self.data_feature.get('scaler') 27 | 28 | self.output_dim = self.config.get('output_dim', 1) 29 | 30 | def evaluate(self, test_dataloader): 31 | """ 32 | use model to test data 33 | 34 | Args: 35 | test_dataloader(torch.Dataloader): Dataloader 36 | """ 37 | self._logger.info('Start evaluating ...') 38 | 39 | y_truths = [] 40 | y_preds = [] 41 | for batch in test_dataloader: 42 | batch.to_ndarray() 43 | output = self.model.run(batch) 44 | y_true = self._scaler.inverse_transform(batch['y'][..., :self.output_dim]) 45 | y_pred = self._scaler.inverse_transform(output[..., :self.output_dim]) 46 | y_truths.append(y_true) 47 | y_preds.append(y_pred) 48 | 49 | y_preds = np.concatenate(y_preds, axis=0) 50 | y_truths = np.concatenate(y_truths, axis=0) # concatenate on batch 51 | outputs = {'prediction': y_preds, 'truth': y_truths} 52 | filename = \ 53 | time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime(time.time())) + '_' \ 54 | + self.config['model'] + '_' + self.config['dataset'] + '_predictions.npz' 55 | np.savez_compressed(os.path.join(self.evaluate_res_dir, filename), **outputs) 56 | self.evaluator.clear() 57 | self.evaluator.collect({'y_true': torch.tensor(y_truths), 'y_pred': torch.tensor(y_preds)}) 58 | test_result = self.evaluator.save_result(self.evaluate_res_dir) 59 | return test_result 60 | 61 | def train(self, train_dataloader, eval_dataloader): 62 | """ 63 | train model 64 | 65 | Args: 66 | train_dataloader(torch.Dataloader): Dataloader 67 | eval_dataloader(torch.Dataloader): Dataloader 68 | """ 69 | raise NotImplementedError 70 | 71 | def save_model(self, cache_name): 72 | """ 73 | 对于传统模型，不需要模型保存 74 | 75 | Args: 76 | cache_name(str): 保存的文件名 77 | """ 78 | assert True # do nothing 79 | 80 | def load_model(self, cache_name): 81 | """ 82 | 对于传统模型，不需要模型加载 83 | 84 | Args: 85 | cache_name(str): 保存的文件名 86 | """ 87 | assert True # do nothing 88 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/optimizer/enc/partitioning/merging.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import torch 5 | 6 | import silearn.optimizer.enc.operator as op 7 | 8 | import silearn 9 | cpp_path = os.path.dirname(silearn.__file__)+ "/../../libsilearn/lib/" 10 | dirs = os.listdir(cpp_path) 11 | import ctypes 12 | try: 13 | libSI = None 14 | import platform 15 | if platform.system() == 'Linux': 16 | exec=".so" 17 | else: 18 | exec=".dll" 19 | for x in dirs: 20 | if x.endswith(exec): 21 | libSI=ctypes.CDLL(cpp_path+x) 22 | native_si = libSI.si 23 | # native_si = libSI.si_hyper 24 | except Exception as e: 25 | # pass 26 | print(e) 27 | raise Exception("SI lib is not correctly compiled") 28 | 29 | 30 | class OperatorMerge(op.Operator): 31 | 32 | def perform(self, erase_loop=True, adj_cover=None, target_cluster = 0, m_scale =0): 33 | 34 | 35 | edges, trans_prob = self.enc.graph.edges 36 | edge_s = edges[:, 0].cpu().numpy().astype(np.uint32).ctypes.data_as( 37 | ctypes.POINTER(ctypes.c_uint32)) 38 | edge_t = edges[:, 1].cpu().numpy().astype(np.uint32).ctypes.data_as( 39 | ctypes.POINTER(ctypes.c_uint32)) 40 | edge_w = trans_prob.cpu().numpy().astype(np.double).ctypes.data_as( 41 | ctypes.POINTER(ctypes.c_double)) 42 | 43 | 44 | # assert self.graph.edge_index[0].shape[0] == self.graph.num_edges and \ 45 | # self.graph.edge_index[1].shape[0] == self.graph.num_edges and \ 46 | # self.graph.edge_weight.shape[0] == self.graph.num_edges 47 | # 48 | # assert self.graph.edge_index[0].max() < self.node_cnt and self.graph.edge_index[1].max() < self.node_cnt 49 | # assert self.graph.edge_index[0].min() >= 0 and self.graph.edge_index[1].min() >= 0 50 | 51 | node_cnt = self.enc.graph.num_vertices 52 | edge_cnt = edges.shape[0] 53 | result = np.zeros(node_cnt, np.uint32) 54 | result_pt = result.ctypes.data_as(ctypes.POINTER(ctypes.c_uint32)) 55 | 56 | if erase_loop: 57 | if target_cluster > 0: 58 | if adj_cover != None: 59 | adj_cover = (adj_cover > 0).cpu().numpy().astype(np.uint32).ctypes.data_as( 60 | ctypes.POINTER(ctypes.c_uint32)) 61 | libSI.si_tgt_adj(node_cnt, edge_cnt, edge_s, edge_t, edge_w, result_pt, 62 | ctypes.c_double(m_scale), adj_cover) 63 | else: 64 | libSI.si_tgt(node_cnt, edge_cnt, edge_s, edge_t, edge_w, result_pt, 65 | ctypes.c_double(m_scale)) 66 | 67 | 68 | else: 69 | libSI.si(node_cnt, edge_cnt, edge_s, edge_t, edge_w, result_pt) 70 | elif adj_cover is not None: 71 | adj_cover = (adj_cover > 0).cpu().numpy().astype(np.uint32).ctypes.data_as( 72 | ctypes.POINTER(ctypes.c_uint32)) 73 | libSI.si_hyper_adj(node_cnt, edge_cnt, edge_s, edge_t, edge_w, adj_cover, result_pt) 74 | else: 75 | libSI.si_hyper(node_cnt, edge_cnt, edge_s, edge_t, edge_w, result_pt) 76 | 77 | self.enc.node_id = torch.LongTensor(result.astype(np.int64)).to(self.enc.graph.device) 78 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/torch_ops/matrix_ops.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch_scatter 4 | 5 | 6 | def scatter_sum(src: torch.Tensor, idx: torch.LongTensor, clip_length=0): 7 | r""" 8 | Reduce sum all values from the :obj:`src` tensor at the indices specified in the :obj:`idx` tensor along axis 9 | :obj:`dim=0`. 10 | 11 | Args: 12 | src(torch.Tensor): The source tensor 13 | idx(torch.LongTensor): the indices of elements to scatter 14 | clip_length(int): if :obj:`clip_length` is not given, a minimal sized output tensor according to :obj:`idx.max()+1` is returned 15 | 16 | :rtype: :class:`Tensor` 17 | """ 18 | sz = torch.max(idx) + 1 if clip_length == 0 else clip_length 19 | # assert b.shape[0] >= sz 20 | if len(src.shape) == 1: 21 | return torch.zeros(sz, dtype=src.dtype, 22 | device=src.device).scatter_add_(0, idx, src) 23 | else: 24 | sp = list(src.shape) 25 | sp[0] = sz 26 | return torch.zeros(sp, dtype=src.dtype, 27 | device=src.device).scatter_add_( 28 | 0, 29 | idx.unsqueeze(-1).expand_as(src), src) 30 | 31 | 32 | def scatter_cnt(idx: torch.LongTensor,clip_length=0, dtype=torch.float): 33 | r""" 34 | Count the occurrence of each element in the :obj:`idx`. 35 | 36 | Args: 37 | idx(torch.LongTensor): the indices of elements to scatter 38 | dtype(torch.dtype): specify the type of returned tensor 39 | clip_length(int): if :obj:`clip_length` is not given, a minimal sized output tensor according to :obj:`idx.max()+1` is returned 40 | 41 | :rtype: :class:`Tensor` 42 | """ 43 | sz = torch.max(idx) + 1 if clip_length == 0 else clip_length 44 | return torch.zeros(sz, dtype=dtype, device=idx.device).scatter_add_( 45 | 0, idx, torch.ones(idx.shape[0], dtype=dtype, device=idx.device)) 46 | 47 | 48 | eps_dtype = { 49 | torch.float64: 1e-306, 50 | torch.double: 1e-306, 51 | torch.float32: 1e-36, 52 | torch.bfloat16: 1e-36, 53 | torch.float16: 1e-7, 54 | } 55 | 56 | 57 | # p * log_2 q 58 | def entropy(p: torch.Tensor, q: torch.Tensor): 59 | dtype = p.dtype 60 | eps = eps_dtype[dtype] if eps_dtype.keys().__contains__(dtype) else 1e-36 61 | return -p * torch.log2(torch.clip(q, min=eps)) 62 | 63 | 64 | def uncertainty(q: torch.Tensor): 65 | dtype = q.dtype 66 | eps = eps_dtype[dtype] if eps_dtype.keys().__contains__(dtype) else 1e-36 67 | return -torch.log2(torch.clip(q, min=eps)) 68 | 69 | 70 | scatter_max = torch_scatter.scatter_max 71 | logical_or = torch.logical_or 72 | logical_and = torch.logical_or 73 | logical_not = torch.logical_not 74 | log2 = torch.log2 75 | log_e = torch.log 76 | stack = torch.stack 77 | clone = torch.clone 78 | 79 | 80 | def concat(q, dim=-1): 81 | return torch.cat(q, dim) 82 | 83 | 84 | import torch.utils.dlpack 85 | 86 | cupy_available = False 87 | try: 88 | import cupy 89 | cupy_available = True 90 | except: 91 | pass 92 | 93 | 94 | def convert_backend(p: torch.Tensor, backend: str): 95 | if backend == "numpy": 96 | # force 97 | return p.cpu().detach().numpy() 98 | elif backend == "cupy": 99 | if not cupy_available: 100 | raise Exception("Target backend cupy is not available") 101 | # https://docs.cupy.dev/en/stable/user_guide/interoperability.html 102 | return cupy.asarray(p) 103 | elif backend == "dlpack": 104 | # noinspection PyUnresolvedReferences 105 | return torch.utils.dlpack.to_dlpack(p) 106 | else: 107 | raise NotImplementedError( 108 | f"convert_backend is not implemented for (torch.Tensor, {str(backend)})" 109 | ) 110 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/test/test_torch_ops.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import torch 4 | from torch_geometric.seed import seed_everything 5 | 6 | from silearn.backends.torch_ops import * 7 | 8 | # Sets the seed for generating random numbers in PyTorch, numpy and Python. 9 | seed_everything(42) 10 | 11 | 12 | class Testbackends(unittest.TestCase): 13 | 14 | def setUp(self): 15 | # self.adj = 16 | self.edges = torch.tensor([[0, 1], [2, 3], [0, 3], [1, 3], [0, 2], 17 | [2, 3]]) 18 | self.weights = [ 19 | torch.tensor([1, 1, 1, 1, 1, 1]), 20 | torch.tensor([2, 2, 2, 2, 1, 1]) 21 | ] 22 | self.operation_ptrs = torch.tensor([0, 1, 3, 5]) 23 | 24 | self.src = torch.tensor([[1, 2], [5, 6], [3, 4], [7, 8], [9, 10], 25 | [11, 12]]) 26 | self.idx = torch.LongTensor([0, 1, 0, 1, 1, 3]) 27 | self.cnt = torch.tensor([2, 3, 0, 1]) 28 | self.sum = torch.tensor([[4, 6], [21, 24], [0, 0], [11, 12]]) 29 | self.max = torch.tensor([[3, 4], [9, 10], [0, 0], [11, 12]]) 30 | 31 | self.p = torch.tensor([1, 0, 3, 1]) 32 | self.q = torch.tensor([1, 1 / 2, 1 / 4, 0]) 33 | 34 | def test_nonzero(self): 35 | pass 36 | 37 | def test_scatter_sum(self): 38 | assert torch.all(scatter_sum(self.src, self.idx) == self.sum) 39 | 40 | def test_scatter_cnt(self): 41 | assert torch.all(scatter_cnt(self.idx) == self.cnt) 42 | 43 | def test_entropy(self): 44 | ground_truth = torch.tensor([0, 0, 6, torch.log2(torch.tensor(1e36))]) 45 | assert torch.all(entropy(self.p, self.q) == ground_truth) 46 | 47 | def test_uncertainty(self): 48 | ground_truth = torch.tensor([0, 1, 2, torch.log2(torch.tensor(1e36))]) 49 | assert torch.all(uncertainty(self.q) == ground_truth) 50 | 51 | def test_concat(self): 52 | x = torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]]) 53 | ground_truth1 = torch.tensor([[1, 2, 3, 4, 1, 2, 3, 4], 54 | [5, 6, 7, 8, 5, 6, 7, 8]]) 55 | ground_truth2 = torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8], [1, 2, 3, 4], 56 | [5, 6, 7, 8]]) 57 | assert torch.all(concat([x, x]) == ground_truth1) 58 | assert torch.all(concat([x, x], 0) == ground_truth2) 59 | 60 | def test_edgeredu_get_edge_transform(self): 61 | ground_truth = torch.tensor([0, 4, 2, 3, 1, 4]) 62 | assert torch.all( 63 | EdgeRedu._get_edge_transform(self.edges) == ground_truth) 64 | 65 | def test_edgeredu_reduction_edge(self): 66 | ground_truth = (torch.tensor([[0, 1], [0, 2], [0, 3], [1, 3], 67 | [2, 3]]), torch.tensor([1, 1, 1, 1, 2]), 68 | torch.tensor([2, 1, 2, 2, 3])) 69 | edge_transform = EdgeRedu._get_edge_transform(self.edges) 70 | edge, weight1, weight2 = EdgeRedu._reduction_edge( 71 | self.edges, edge_transform, *self.weights) 72 | assert torch.all(edge == ground_truth[0]) 73 | assert torch.all(weight1 == ground_truth[1]) 74 | assert torch.all(weight2 == ground_truth[2]) 75 | 76 | def test_sumup_duplicates(self): 77 | # todo： operation_ptrs condition 78 | ground_truth = (torch.tensor([[0, 1], [0, 2], [0, 3], [1, 3], 79 | [2, 3]]), torch.tensor([1, 1, 1, 1, 2]), 80 | torch.tensor([2, 1, 2, 2, 3])) 81 | edge, weight1, weight2 = sumup_duplicates(self.edges, *self.weights) 82 | assert torch.all(edge == ground_truth[0]) 83 | assert torch.all(weight1 == ground_truth[1]) 84 | assert torch.all(weight2 == ground_truth[2]) 85 | 86 | 87 | if __name__ == '__main__': 88 | # unittest.main() 89 | edges = torch.tensor([[0, 1], [2, 3], [0, 3], [1, 3], [0, 2], [2, 3]]) 90 | weights = [ 91 | torch.tensor([1, 1, 1, 1, 1, 1]), 92 | torch.tensor([2, 2, 2, 2, 1, 1]) 93 | ] 94 | operation_ptrs = torch.tensor([1, 2, 3, 5]) 95 | print(sumup_duplicates(edges, *weights)) -------------------------------------------------------------------------------- /libcity/utils/GPS_utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | R_EARTH = 6371000 # meter 4 | 5 | 6 | def angle2radian(angle): 7 | """ 8 | convert from an angle to a radian 9 | :param angle: (float) 10 | :return: radian (float) 11 | """ 12 | return math.radians(angle) 13 | 14 | 15 | def radian2angle(radian): 16 | return math.degrees(radian) 17 | 18 | 19 | def spherical_law_of_cosines(phi1, lambda1, phi2, lambda2): 20 | """ 21 | calculate great circle distance with spherical law of cosines 22 | phi/lambda for latitude/longitude in radians 23 | :param phi1: point one's latitude in radians 24 | :param lambda1: point one's longitude in radians 25 | :param phi2: point two's latitude in radians 26 | :param lambda2: point two's longitude in radians 27 | :return: 28 | """ 29 | d_lambda = lambda2 - lambda1 30 | return math.acos(math.sin(phi1) * math.sin(phi2) + math.cos(phi1) * math.cos(phi2) * math.cos(d_lambda)) 31 | 32 | 33 | def haversine(phi1, lambda1, phi2, lambda2): 34 | """ 35 | calculate angular great circle distance with haversine formula 36 | see parameters in spherical_law_of_cosines 37 | """ 38 | d_phi = phi2 - phi1 39 | d_lambda = lambda2 - lambda1 40 | a = math.pow(math.sin(d_phi / 2), 2) + \ 41 | math.cos(phi1) * math.cos(phi2) * math.pow(math.sin(d_lambda / 2), 2) 42 | c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) 43 | return c 44 | 45 | 46 | def equirectangular_approximation(phi1, lambda1, phi2, lambda2): 47 | """ 48 | calculate angular great circle distance with Pythagoras’ theorem performed on an equirectangular projection 49 | see parameters in spherical_law_of_cosines 50 | """ 51 | x = (lambda2 - lambda1) * math.cos((phi1 + phi2) / 2) 52 | y = phi2 - phi1 53 | return math.sqrt(math.pow(x, 2) + math.pow(y, 2)) 54 | 55 | 56 | def dist(phi1, lambda1, phi2, lambda2, r=R_EARTH, method='hav'): 57 | """ 58 | calculate great circle distance with given latitude and longitude, 59 | :param phi1: point one's latitude in angle 60 | :param lambda1: point one's longitude in angle 61 | :param phi2: point two's latitude in angle 62 | :param lambda2: point two's longitude in angle 63 | :param r: earth radius(m) 64 | :param method: 'hav' means haversine, 65 | 'LoC' means Spherical Law of Cosines, 66 | 'approx' means Pythagoras’ theorem performed on an equirectangular projection 67 | :return: distance (m) 68 | """ 69 | return angular_dist(phi1, lambda1, phi2, lambda2, method) * r 70 | 71 | 72 | def angular_dist(phi1, lambda1, phi2, lambda2, method='hav'): 73 | """ 74 | calculate angular great circle distance with given latitude and longitude 75 | :return: angle 76 | """ 77 | if method.lower() == 'hav': 78 | return haversine(phi1, lambda1, phi2, lambda2) 79 | elif method.lower() == 'loc': 80 | return spherical_law_of_cosines(phi1, lambda1, phi2, lambda2) 81 | elif method.lower() == 'approx': 82 | return equirectangular_approximation(phi1, lambda1, phi2, lambda2) 83 | else: 84 | assert False 85 | 86 | 87 | def destination(phi1, lambda1, brng, distance, r=R_EARTH): 88 | """ 89 | 90 | :param phi1: 91 | :param lambda1: 92 | :param brng: 93 | :param distance: 94 | :return: 95 | """ 96 | delta = distance / r 97 | phi2 = math.asin(math.sin(phi1) * math.cos(delta) + math.cos(phi1) * math.sin(delta) * math.cos(brng)) 98 | lambda2 = lambda1 + math.atan2( 99 | math.sin(brng) * math.sin(delta) * math.cos(phi1), math.cos(delta) - math.sin(phi1) * math.sin(phi2) 100 | ) 101 | return phi2, lambda2 102 | 103 | 104 | def init_bearing(phi1, lambda1, phi2, lambda2): 105 | """ 106 | initial bearing of a great circle route 107 | :return: 0~360 108 | """ 109 | y = math.sin(lambda2 - lambda1) * math.cos(phi2) 110 | x = math.cos(phi1) * math.sin(phi2) - math.sin(phi1) * math.cos(phi2) * math.cos(lambda2 - lambda1) 111 | theta = math.atan2(y, x) 112 | brng = (theta * 180 / math.pi + 360) % 360 113 | return brng -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/scipy_ops/matrix_ops.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy 3 | 4 | from numpy.lib.shape_base import _make_along_axis_idx 5 | import functools 6 | from numpy.core import overrides 7 | from numpy.core.multiarray import normalize_axis_index 8 | array_function_dispatch = functools.partial( 9 | overrides.array_function_dispatch, module='numpy') 10 | 11 | def _add_along_axis_dispatcher(arr, indices, values, axis): 12 | return (arr, indices, values) 13 | 14 | @array_function_dispatch(_add_along_axis_dispatcher) 15 | def add_along_axis(arr, indices, values, axis): 16 | """ 17 | numpy scatter_add operation 18 | 19 | @param: 20 | arr: target result array 21 | indices: indices to change along each 1d slice of arr. 22 | values: values to insert at those indices. Its shape and dimension are broadcast to match that of indices. 23 | axis: the axis to take 1d slices along. 24 | """ 25 | # normalize inputs 26 | if axis is None: 27 | arr = arr.flat 28 | axis = 0 29 | arr_shape = (len(arr),) # flatiter has no .shape 30 | else: 31 | axis = normalize_axis_index(axis, arr.ndim) 32 | arr_shape = arr.shape 33 | # use the fancy index 34 | np.add.at(arr, _make_along_axis_idx(arr_shape, indices, axis), values) 35 | 36 | def scatter_sum(src : np.ndarray, idx : np.ndarray, clip_length = 0): 37 | """ 38 | @comment(xiang): only support cpu? 39 | """ 40 | sz = np.max(idx) + 1 if clip_length == 0 else clip_length 41 | # arr = scipy.sparse.coo() 42 | # assert b.shape[0] >= sz 43 | if len(src.shape) == 1: 44 | result = np.zeros(sz, dtype=src.dtype) 45 | # return np.zeros(sz, dtype=b.dtype, device=b.device).scatter_add_(0, a, b) 46 | add_along_axis(result, idx, src, axis=0) 47 | else: 48 | sp = list(src.shape) 49 | sp[0] = sz 50 | # return torch.zeros(sp, dtype=b.dtype, device=b.device).scatter_add_(0, a.unsqueeze(-1).expand_as(b), b) 51 | result = np.zeros(sp, dtype=src.dtype) 52 | add_along_axis(result, np.broadcast_to(np.expand_dims(idx, -1), src.shape), src, axis=0) 53 | return result 54 | 55 | def scatter_cnt(idx : np.ndarray, dtype = np.float, clip_length = 0): 56 | sz = np.max(idx) + 1 if clip_length == 0 else clip_length 57 | # return torch.zeros(sz, dtype = dtype, device=a.device).scatter_add_(0, a, torch.ones(a.shape[0], dtype = dtype, device=a.device)) 58 | result = np.zeros(sz, dtype=idx.dtype) 59 | add_along_axis(result, idx, np.ones(idx.shape[0], dtype=dtype), axis=0) 60 | return result 61 | 62 | def scatter_max(src : np.ndarray, idx : np.ndarray, clip_length = 0): 63 | raise NotImplementedError('Not Implemented') 64 | 65 | eps_dtype = { 66 | np.float64: 1e-306, 67 | np.double: 1e-306, 68 | np.float32: 1e-36, 69 | np.float16: 1e-7, 70 | } 71 | 72 | 73 | # p * log_2 q 74 | def entropy(p: np.ndarray, q:np.ndarray): 75 | dtype = p.dtype 76 | eps = eps_dtype[dtype] if eps_dtype.keys().__contains__(dtype) else 1e-36 77 | return -p * np.log2(np.clip(q, a_min = eps, a_max=None)) 78 | 79 | 80 | def uncertainty(q: np.ndarray): 81 | dtype = q.dtype 82 | eps = eps_dtype[dtype] if eps_dtype.keys().__contains__(dtype) else 1e-36 83 | return -np.log2(np.clip(q, a_min = eps, a_max=None)) 84 | 85 | 86 | # scatter_max = torch_scatter.scatter_max 87 | 88 | 89 | logical_or = np.logical_or 90 | logical_and = np.logical_or 91 | logical_not = np.logical_not 92 | log2 = np.log2 93 | log_e = np.log 94 | 95 | def concat(q, dim = -1): 96 | return torch.cat(q, dim) 97 | 98 | 99 | import torch.utils.dlpack 100 | 101 | try: 102 | import cupy 103 | except: 104 | pass 105 | 106 | def convert_backend(p : np.ndarray, backend: str): 107 | if backend == "numpy": 108 | # force 109 | return p.numpy(force = True) 110 | elif backend == "cupy": 111 | # https://docs.cupy.dev/en/stable/user_guide/interoperability.html 112 | return cupy.asarray(p) 113 | elif backend == "dlpack": 114 | # noinspection PyUnresolvedReferences 115 | return torch.utils.dlpack.to_dlpack(p) 116 | else: 117 | raise NotImplementedError(f"convert_backend is not implemented for (np.ndarray, {str(backend)})") 118 | 119 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/graph.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | 3 | import scipy 4 | import torch 5 | import torch_scatter 6 | 7 | import silearn 8 | 9 | import networkx 10 | 11 | 12 | 13 | # Graph Model for Random Walk Process 14 | # 15 | class Graph(object): 16 | # adj : torch.Tensor 17 | backend = "torch" 18 | directed = True 19 | 20 | def __init__(self): 21 | pass 22 | 23 | 24 | @property 25 | def device(self): 26 | return "cpu" 27 | 28 | @property 29 | def num_vertices(self): 30 | return 0 31 | 32 | @property 33 | def num_edges(self): 34 | return 0 35 | 36 | # def vertex_reduce(self, partition): 37 | # silearn.vertex_reduce(self, partition) 38 | 39 | @abstractmethod 40 | def to_networkx(self, create_using = networkx.DiGraph()): 41 | raise NotImplementedError("Not Implemented") 42 | 43 | @property 44 | @abstractmethod 45 | def stationary_dist(self): 46 | raise NotImplementedError("Not Implemented") 47 | 48 | @property 49 | @abstractmethod 50 | def vertex_weight_es(self): 51 | raise NotImplementedError("Not Implemented") 52 | 53 | @property 54 | @abstractmethod 55 | def edges(self): 56 | raise NotImplementedError("Not Implemented") 57 | @abstractmethod 58 | def query_probability(self, es, et): 59 | raise NotImplementedError("Not Implemented") 60 | 61 | @abstractmethod 62 | def clone(self): 63 | raise NotImplementedError("Not Implemented") 64 | 65 | 66 | 67 | 68 | class GraphSparse(Graph): 69 | 70 | """ 71 | E x 2 72 | """ 73 | _edges: None 74 | # E 75 | _p: None 76 | _dist = None 77 | n_vertices = 0 78 | 79 | tot_weights = 1 80 | 81 | def __init__(self, edges, p, dist = None, n_vertices = None): 82 | super().__init__() 83 | if n_vertices is None and dist is not None: 84 | self.n_vertices = dist.shape[-1] 85 | else: 86 | self.n_vertices = n_vertices 87 | self._edges, self._p, self._dist = edges, p, dist 88 | 89 | 90 | @property 91 | def device(self): 92 | return self._edges.device 93 | 94 | @property 95 | def num_vertices(self): 96 | return self.n_vertices 97 | 98 | @property 99 | def num_edges(self): 100 | return self._edges.shape[0] 101 | 102 | @property 103 | def vertex_weight_es(self): 104 | return silearn.scatter_sum(self._p, self.edges[0][:, 0]) 105 | 106 | 107 | @property 108 | def edges(self): 109 | return self._edges, self._p 110 | 111 | @property 112 | def stationary_dist(self): 113 | return self._dist 114 | 115 | def query_probability(self, es, et): 116 | edge_index = torch.tensor([es, et]) 117 | if torch.where((self._edges==edge_index).all(dim=1))[0].shape[0] == 0: 118 | raise ValueError('Edge not found') 119 | return self._p[torch.where((self._edges==edge_index).all(dim=1))[0].shape[0]] 120 | 121 | def to_networkx(self, create_using = networkx.DiGraph()): 122 | edges = silearn.convert_backend(self._edges, "numpy") 123 | weights = silearn.convert_backend(self._p, "numpy") 124 | scipy.sparse.coo.coo_matrix((weights, (edges[:, 0], edges[:, 1])), (self.n_vertices, self.n_vertices)) 125 | networkx.from_scipy_sparse_array(edges, create_using=create_using) 126 | 127 | # def query_weight(self, es, et): 128 | # 129 | def clone(self): 130 | return GraphSparse(silearn.clone(self._edges), silearn.clone(self._p), 131 | silearn.clone(self._dist), self.n_vertices) 132 | 133 | 134 | 135 | 136 | 137 | class GraphDense(Graph): 138 | adj: None 139 | dist: None 140 | 141 | 142 | def num_vertices(self): 143 | return self.adj.shape[0] 144 | 145 | def num_edges(self): 146 | return self.adj.shape[0] ** 2 147 | 148 | def stationary_dist(self): 149 | return self.dist 150 | 151 | def to_sparse(self): 152 | raise NotImplementedError("Not Implemented") 153 | 154 | 155 | 156 | def edges(self): 157 | edges = silearn.nonzero(self.adj) 158 | return edges, self.adj[edges[:, 0]][edges[:, 1]] 159 | 160 | def query_probability(self, es, et): 161 | return self.adj[es][et] 162 | 163 | 164 | 165 | -------------------------------------------------------------------------------- /libcity/model/utils.py: -------------------------------------------------------------------------------- 1 | import scipy.sparse as sp 2 | from scipy.sparse import linalg 3 | import numpy as np 4 | import torch 5 | 6 | 7 | # def build_sparse_matrix(device, lap): 8 | # lap = lap.tocoo() 9 | # indices = np.column_stack((lap.row, lap.col)) 10 | # # this is to ensure row-major ordering to equal torch.sparse.sparse_reorder(L) 11 | # indices = indices[np.lexsort((indices[:, 0], indices[:, 1]))] 12 | # lap = torch.sparse_coo_tensor(indices.T, lap.data, lap.shape, device=device) 13 | # return lap.to(torch.float32) 14 | 15 | 16 | def build_sparse_matrix(device, lap): 17 | """ 18 | 构建稀疏矩阵(tensor) 19 | 20 | Args: 21 | device: 22 | lap: 拉普拉斯 23 | 24 | Returns: 25 | 26 | """ 27 | shape = lap.shape 28 | i = torch.LongTensor(np.vstack((lap.row, lap.col)).astype(int)) 29 | v = torch.FloatTensor(lap.data) 30 | return torch.sparse.FloatTensor(i, v, torch.Size(shape)).to(device) 31 | 32 | 33 | def get_cheb_polynomial(l_tilde, k): 34 | """ 35 | compute a list of chebyshev polynomials from T_0 to T_{K-1} 36 | 37 | Args: 38 | l_tilde(scipy.sparse.coo.coo_matrix): scaled Laplacian, shape (N, N) 39 | k(int): the maximum order of chebyshev polynomials 40 | 41 | Returns: 42 | list(np.ndarray): cheb_polynomials, length: K, from T_0 to T_{K-1} 43 | """ 44 | l_tilde = sp.coo_matrix(l_tilde) 45 | num = l_tilde.shape[0] 46 | cheb_polynomials = [sp.eye(num).tocoo(), l_tilde.copy()] 47 | for i in range(2, k + 1): 48 | cheb_i = (2 * l_tilde).dot(cheb_polynomials[i - 1]) - cheb_polynomials[i - 2] 49 | cheb_polynomials.append(cheb_i.tocoo()) 50 | return cheb_polynomials 51 | 52 | 53 | def get_supports_matrix(adj_mx, filter_type='laplacian', undirected=True): 54 | """ 55 | 选择不同类别的拉普拉斯 56 | 57 | Args: 58 | undirected: 59 | adj_mx: 60 | filter_type: 61 | 62 | Returns: 63 | 64 | """ 65 | supports = [] 66 | if filter_type == "laplacian": 67 | supports.append(calculate_scaled_laplacian(adj_mx, lambda_max=None, undirected=undirected)) 68 | elif filter_type == "random_walk": 69 | supports.append(calculate_random_walk_matrix(adj_mx).T) 70 | elif filter_type == "dual_random_walk": 71 | supports.append(calculate_random_walk_matrix(adj_mx).T) 72 | supports.append(calculate_random_walk_matrix(adj_mx.T).T) 73 | else: 74 | supports.append(calculate_scaled_laplacian(adj_mx)) 75 | return supports 76 | 77 | 78 | def calculate_normalized_laplacian(adj): 79 | """ 80 | L = D^-1/2 (D-A) D^-1/2 = I - D^-1/2 A D^-1/2 81 | 对称归一化的拉普拉斯 82 | 83 | Args: 84 | adj: adj matrix 85 | 86 | Returns: 87 | np.ndarray: L 88 | """ 89 | adj = sp.coo_matrix(adj) 90 | d = np.array(adj.sum(1)) 91 | d_inv_sqrt = np.power(d, -0.5).flatten() 92 | d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0. 93 | d_mat_inv_sqrt = sp.diags(d_inv_sqrt) 94 | normalized_laplacian = sp.eye(adj.shape[0]) - adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo() 95 | return normalized_laplacian 96 | 97 | 98 | def calculate_random_walk_matrix(adj_mx): 99 | """ 100 | L = D^-1 * A 101 | 随机游走拉普拉斯 102 | 103 | Args: 104 | adj_mx: adj matrix 105 | 106 | Returns: 107 | np.ndarray: L 108 | """ 109 | adj_mx = sp.coo_matrix(adj_mx) 110 | d = np.array(adj_mx.sum(1)) 111 | d_inv = np.power(d, -1).flatten() 112 | d_inv[np.isinf(d_inv)] = 0. 113 | d_mat_inv = sp.diags(d_inv) 114 | random_walk_mx = d_mat_inv.dot(adj_mx).tocoo() 115 | return random_walk_mx 116 | 117 | 118 | def calculate_scaled_laplacian(adj_mx, lambda_max=2, undirected=True): 119 | """ 120 | 计算近似后的拉普莱斯矩阵~L 121 | 122 | Args: 123 | adj_mx: 124 | lambda_max: 125 | undirected: 126 | 127 | Returns: 128 | ~L = 2 * L / lambda_max - I 129 | """ 130 | adj_mx = sp.coo_matrix(adj_mx) 131 | if undirected: 132 | bigger = adj_mx > adj_mx.T 133 | smaller = adj_mx < adj_mx.T 134 | notequall = adj_mx != adj_mx.T 135 | adj_mx = adj_mx - adj_mx.multiply(notequall) + adj_mx.multiply(bigger) + adj_mx.T.multiply(smaller) 136 | lap = calculate_normalized_laplacian(adj_mx) 137 | if lambda_max is None: 138 | lambda_max, _ = linalg.eigsh(lap, 1, which='LM') 139 | lambda_max = lambda_max[0] 140 | lap = sp.csr_matrix(lap) 141 | m, _ = lap.shape 142 | identity = sp.identity(m, format='csr', dtype=lap.dtype) 143 | lap = (2 / lambda_max * lap) - identity 144 | return lap.astype(np.float32).tocoo() 145 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/batched_graph.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def combine_batch_graph(edges, w, n_vertices): 5 | r""" 6 | Combine the KNN graphs while avoiding duplicated edges between different batches. :py:mod:`BatchedGraphModuel` 7 | """ 8 | bs = edges.shape[0] 9 | device = edges.device 10 | idx = (torch.arange(bs, device=device, dtype=torch.int64) * n_vertices)\ 11 | .reshape(-1, 1, 1) 12 | return (edges + idx).reshape(-1, 2), w.reshape(-1) 13 | 14 | 15 | class BatchedGraphModule(torch.nn.Module): 16 | r""" 17 | Combine the KNN graphs while avoiding duplicated edges between different batches. 18 | """ 19 | 20 | def __init__(self, num_idx, bs, device): 21 | r""" 22 | Args: 23 | num_idx(int): num_idx should be greater than the maximum node id in graph edges. 24 | bs(int): batch size. 25 | device(torch.device): device the graph stored. 26 | """ 27 | super().__init__() 28 | self.register_buffer("idx", (torch.arange(bs, device=device, dtype=torch.int64) * num_idx)\ 29 | .reshape(-1, 1, 1), persistent=False) 30 | 31 | def combine_batch(self, edges, w): 32 | r""" 33 | Combine the batched KNN graphs. Add different IDs to the edges of different batches so that the node id of the edges in different batches do not overlap. 34 | 35 | Args: 36 | edges: graph edges :obj:`(B, N, 2)`. 37 | w: the weights of graph edges, shape :obj:`(B, N)` 38 | 39 | :rtype: (:class:`Tensor`, :class:`Tensor`) 40 | """ 41 | return (edges + self.idx).reshape(-1, 2), w.reshape(-1) 42 | 43 | def reduce(self, labels): 44 | return 45 | 46 | 47 | class ShiftGraph(torch.nn.Module): 48 | r""" 49 | Build knn graphs over the images with shape :obj:`(B, C, H, W)`. 50 | """ 51 | 52 | def __init__(self, d, metric=None): 53 | r""" 54 | Args: 55 | d(int): Each pixel connect with 2d-1 other pixels. 56 | metric(func): This function controls the metric used for calculating the distance between pixels. Euclidean distances is default. 57 | """ 58 | if metric is None: 59 | metric = lambda f1, f2, x, y: -( 60 | (f1 - f2)**2).sum(dim=1) * (x**2 + y**2)**0.5 61 | super(ShiftGraph, self).__init__() 62 | self.d = d 63 | self.metric = metric 64 | 65 | def forward(self, x: torch.Tensor): 66 | r""" 67 | Return the edges :obj:`(B, N, 2)` and weights :obj:`(B, N)` for batched knn graph. 68 | 69 | Args: 70 | x(torch.tensor): images with shape :obj:`(B, C, H, W)`. 71 | 72 | :rtype: (:class:`Tensor`, :class:`Tensor`) 73 | """ 74 | H, W = x.shape[-2], x.shape[-1] 75 | id0 = torch.arange(W * H, dtype=torch.int64, 76 | device=x.device).reshape(H, -1) 77 | 78 | d = self.d 79 | img_pad = torch.nn.functional.pad(x, (d // 2, d // 2, d // 2, d // 2), 80 | value=-1e10) 81 | imgH, imgW = x.shape[-2], x.shape[-1] 82 | 83 | # neighbor = -10 * torch.ones(imgH, imgW, r * r, dtype=torch.float64, device=img.device) 84 | 85 | w, edges = [], [] 86 | for i in range(d): 87 | for j in range(d): 88 | if i == d // 2 and j == d // 2: 89 | continue 90 | 91 | rr = d // 2 92 | pos_v = (i - rr) * imgW + (j - rr) 93 | imgx = self.metric(img_pad[:, :, i:imgH + i, j:imgW + j], x, 94 | (i - d // 2), (j - d // 2)) 95 | 96 | imgx = imgx[:, 97 | max(d // 2 - i, 0):imgH - max(i - d // 2, 0), 98 | max(d // 2 - j, 0):imgW - max(j - d // 2, 0)] 99 | idx = id0[max(d // 2 - i, 0):imgH - max(i - d // 2, 0), 100 | max(d // 2 - j, 0):imgW - max(j - d // 2, 0)] 101 | idx = idx.unsqueeze(0).repeat((imgx.shape[0], 1, 1)) 102 | # flag = (self.H_idx + i - rr >= 0) & (self.H_idx + i - rr < imgH) & \ 103 | # (self.W_idx + j - rr >= 0) & (self.W_idx + j - rr < imgW) 104 | 105 | w.append(torch.flatten(imgx, start_dim=-2, end_dim=-1)) 106 | pos = torch.flatten(idx, start_dim=-2, end_dim=-1) 107 | edges.append(torch.stack((pos, pos + pos_v), dim=-1)) 108 | 109 | return torch.cat(edges, dim=-2), torch.cat(w, dim=-1) 110 | 111 | 112 | if __name__ == '__main__': 113 | module = ShiftGraph(3) 114 | edges, ew = module(torch.arange(50).reshape(2, 1, 5, 5)) 115 | print(edges.shape) 116 | model = BatchedGraphModule(25, 2, edges.device) 117 | edges, ew = model.combine_batch(edges, ew) 118 | print(edges.shape) -------------------------------------------------------------------------------- /libcity/utils/normalization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch as th 3 | 4 | class Scaler: 5 | """ 6 | 归一化接口 7 | """ 8 | 9 | def transform(self, data, **kw): 10 | """ 11 | 数据归一化接口 12 | 13 | Args: 14 | data(np.ndarray): 归一化前的数据 15 | 16 | Returns: 17 | np.ndarray: 归一化后的数据 18 | """ 19 | raise NotImplementedError("Transform not implemented") 20 | 21 | def inverse_transform(self, data, **kw): 22 | """ 23 | 数据逆归一化接口 24 | 25 | Args: 26 | data(np.ndarray): 归一化后的数据 27 | 28 | Returns: 29 | np.ndarray: 归一化前的数据 30 | """ 31 | raise NotImplementedError("Inverse_transform not implemented") 32 | 33 | 34 | class NoneScaler(Scaler): 35 | """ 36 | 不归一化 37 | """ 38 | 39 | def transform(self, data, **kw): 40 | return data 41 | 42 | def inverse_transform(self, data, **kw): 43 | return data 44 | 45 | 46 | class NormalScaler(Scaler): 47 | """ 48 | 除以最大值归一化 49 | x = x / x.max 50 | """ 51 | 52 | def __init__(self, maxx): 53 | self.max = maxx 54 | 55 | def transform(self, data, **kw): 56 | return data / self.max 57 | 58 | def inverse_transform(self, data, **kw): 59 | return data * self.max 60 | 61 | 62 | class StandardScaler(Scaler): 63 | """ 64 | Z-score归一化 65 | x = (x - x.mean) / x.std 66 | """ 67 | 68 | def __init__(self, mean, std): 69 | self.mean = mean 70 | self.std = std 71 | 72 | def transform(self, data, **kw): 73 | return (data - self.mean) / self.std 74 | 75 | def inverse_transform(self, data, **kw): 76 | return (data * self.std) + self.mean 77 | 78 | class StandardIndependCScaler(Scaler): 79 | """ 80 | Z-score归一化 81 | 每个channel单独进行 82 | """ 83 | def __init__(self, x_train): 84 | self.dim = x_train.shape[-1] 85 | self._channel_mean = [] 86 | self._channel_std = [] 87 | # self.y_channel_dict = [] 88 | for d in range(self.dim): 89 | self._channel_mean.append(x_train[...,d].mean()) 90 | self._channel_std.append(x_train[...,d].std()) 91 | self._channel_mean = np.array(self._channel_mean) 92 | self._channel_std = np.array(self._channel_std) 93 | # self._channel_mean = th.Tensor(self._channel_mean) 94 | # self._channel_std = th.Tensor(self._channel_std) 95 | # self.y_channel_dict.append[{'mean':y_train[...,d].mean(),'std':y_train[...,d].std()}] 96 | 97 | def transform(self, data, **kw): 98 | assert (data.shape[-1]==self.dim), 'Bad channel num for this scalar.' 99 | return (data - self._channel_mean) / self._channel_std 100 | # return (data - self._channel_mean.view([...,self.dim])) / self._channel_std.view([...,self.dim]) 101 | 102 | def inverse_transform(self, data, **kw): 103 | if type(data)==th.Tensor: 104 | _channel_mean = th.from_numpy(self._channel_mean).to(data.device) 105 | _channel_std = th.from_numpy(self._channel_std).to(data.device) 106 | _channel_mean.requires_grad=False 107 | _channel_std.requires_grad=False 108 | else: 109 | _channel_mean = self._channel_mean 110 | _channel_std = self._channel_std 111 | if kw.__contains__('channel_idx') is False: 112 | assert (data.shape[-1]==self.dim), 'Bad channel num for this scalar.' 113 | return (data * _channel_std) + _channel_mean 114 | elif type(kw['channel_idx']) == list: 115 | assert (len(kw['channel_idx'])<=self.dim), 'Bad channel num for this scalar.' 116 | return (data * _channel_std[kw['channel_idx']]) + _channel_mean[kw['channel_idx']] 117 | else: raise TypeError 118 | # return (data * self._channel_std.view([...,self.dim])) + self._channel_mean.view([...,self.dim]) 119 | 120 | 121 | class MinMax01Scaler(Scaler): 122 | """ 123 | MinMax归一化 结果区间[0, 1] 124 | x = (x - min) / (max - min) 125 | """ 126 | 127 | def __init__(self, minn, maxx): 128 | self.min = minn 129 | self.max = maxx 130 | 131 | def transform(self, data, **kw): 132 | return (data - self.min) / (self.max - self.min) 133 | 134 | def inverse_transform(self, data, **kw): 135 | return data * (self.max - self.min) + self.min 136 | 137 | 138 | class MinMax11Scaler(Scaler): 139 | """ 140 | MinMax归一化 结果区间[-1, 1] 141 | x = (x - min) / (max - min) 142 | x = x * 2 - 1 143 | """ 144 | 145 | def __init__(self, minn, maxx): 146 | self.min = minn 147 | self.max = maxx 148 | 149 | def transform(self, data, **kw): 150 | return ((data - self.min) / (self.max - self.min)) * 2. - 1. 151 | 152 | def inverse_transform(self, data, **kw): 153 | return ((data + 1.) / 2.) * (self.max - self.min) + self.min 154 | 155 | 156 | class LogScaler(Scaler): 157 | """ 158 | Log scaler 159 | x = log(x+eps) 160 | """ 161 | 162 | def __init__(self, eps=0.999): 163 | self.eps = eps 164 | 165 | def transform(self, data, **kw): 166 | return np.log(data + self.eps) 167 | 168 | def inverse_transform(self, data, **kw): 169 | return np.exp(data) - self.eps 170 | -------------------------------------------------------------------------------- /libcity/utils/argument_list.py: -------------------------------------------------------------------------------- 1 | """ 2 | store the arguments can be modified by the user 3 | """ 4 | import argparse 5 | 6 | general_arguments = { 7 | "gpu": { 8 | "type": "bool", 9 | "default": None, 10 | "help": "whether use gpu" 11 | }, 12 | "gpu_id": { 13 | "type": "int", 14 | "default": None, 15 | "help": "the gpu id to use" 16 | }, 17 | "train_rate": { 18 | "type": "float", 19 | "default": None, 20 | "help": "the train set rate" 21 | }, 22 | "eval_rate": { 23 | "type": "float", 24 | "default": None, 25 | "help": "the validation set rate" 26 | }, 27 | "batch_size": { 28 | "type": "int", 29 | "default": None, 30 | "help": "the batch size" 31 | }, 32 | "learning_rate": { 33 | "type": "float", 34 | "default": None, 35 | "help": "learning rate" 36 | }, 37 | "max_epoch": { 38 | "type": "int", 39 | "default": None, 40 | "help": "the maximum epoch" 41 | }, 42 | "dataset_class": { 43 | "type": "str", 44 | "default": None, 45 | "help": "the dataset class name" 46 | }, 47 | "executor": { 48 | "type": "str", 49 | "default": None, 50 | "help": "the executor class name" 51 | }, 52 | "evaluator": { 53 | "type": "str", 54 | "default": None, 55 | "help": "the evaluator class name" 56 | }, 57 | } 58 | 59 | hyper_arguments = { 60 | "gpu": { 61 | "type": "bool", 62 | "default": None, 63 | "help": "whether use gpu" 64 | }, 65 | "gpu_id": { 66 | "type": "int", 67 | "default": None, 68 | "help": "the gpu id to use" 69 | }, 70 | "train_rate": { 71 | "type": "float", 72 | "default": None, 73 | "help": "the train set rate" 74 | }, 75 | "eval_rate": { 76 | "type": "float", 77 | "default": None, 78 | "help": "the validation set rate" 79 | }, 80 | "batch_size": { 81 | "type": "int", 82 | "default": None, 83 | "help": "the batch size" 84 | } 85 | } 86 | 87 | 88 | def str2bool(s): 89 | if isinstance(s, bool): 90 | return s 91 | if s.lower() in ('yes', 'true'): 92 | return True 93 | elif s.lower() in ('no', 'false'): 94 | return False 95 | else: 96 | raise argparse.ArgumentTypeError('bool value expected.') 97 | 98 | 99 | def str2float(s): 100 | if isinstance(s, float): 101 | return s 102 | try: 103 | x = float(s) 104 | except ValueError: 105 | raise argparse.ArgumentTypeError('float value expected.') 106 | return x 107 | 108 | 109 | def add_general_args(parser): 110 | for arg in general_arguments: 111 | if general_arguments[arg]['type'] == 'int': 112 | parser.add_argument('--{}'.format(arg), type=int, 113 | default=general_arguments[arg]['default'], help=general_arguments[arg]['help']) 114 | elif general_arguments[arg]['type'] == 'bool': 115 | parser.add_argument('--{}'.format(arg), type=str2bool, 116 | default=general_arguments[arg]['default'], help=general_arguments[arg]['help']) 117 | elif general_arguments[arg]['type'] == 'float': 118 | parser.add_argument('--{}'.format(arg), type=str2float, 119 | default=general_arguments[arg]['default'], help=general_arguments[arg]['help']) 120 | elif general_arguments[arg]['type'] == 'str': 121 | parser.add_argument('--{}'.format(arg), type=str, 122 | default=general_arguments[arg]['default'], help=general_arguments[arg]['help']) 123 | elif general_arguments[arg]['type'] == 'list of int': 124 | parser.add_argument('--{}'.format(arg), nargs='+', type=int, 125 | default=general_arguments[arg]['default'], help=general_arguments[arg]['help']) 126 | 127 | 128 | def add_hyper_args(parser): 129 | for arg in hyper_arguments: 130 | if hyper_arguments[arg]['type'] == 'int': 131 | parser.add_argument('--{}'.format(arg), type=int, 132 | default=hyper_arguments[arg]['default'], help=hyper_arguments[arg]['help']) 133 | elif hyper_arguments[arg]['type'] == 'bool': 134 | parser.add_argument('--{}'.format(arg), type=str2bool, 135 | default=hyper_arguments[arg]['default'], help=hyper_arguments[arg]['help']) 136 | elif hyper_arguments[arg]['type'] == 'float': 137 | parser.add_argument('--{}'.format(arg), type=str2float, 138 | default=hyper_arguments[arg]['default'], help=hyper_arguments[arg]['help']) 139 | elif hyper_arguments[arg]['type'] == 'str': 140 | parser.add_argument('--{}'.format(arg), type=str, 141 | default=hyper_arguments[arg]['default'], help=hyper_arguments[arg]['help']) 142 | elif hyper_arguments[arg]['type'] == 'list of int': 143 | parser.add_argument('--{}'.format(arg), nargs='+', type=int, 144 | default=hyper_arguments[arg]['default'], help=hyper_arguments[arg]['help']) 145 | 146 | -------------------------------------------------------------------------------- /libcity/utils/dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | 数据预处理阶段相关的工具函数 3 | """ 4 | import numpy as np 5 | import time 6 | from datetime import datetime, timedelta 7 | from collections import defaultdict 8 | 9 | 10 | def parse_time(time_in, timezone_offset_in_minute=0): 11 | """ 12 | 将 json 中 time_format 格式的 time 转化为 local datatime 13 | """ 14 | date = datetime.strptime(time_in, '%Y-%m-%dT%H:%M:%SZ') # 这是 UTC 时间 15 | return date + timedelta(minutes=timezone_offset_in_minute) 16 | 17 | 18 | def cal_basetime(start_time, base_zero): 19 | """ 20 | 用于切分轨迹成一个 session， 21 | 思路为：给定一个 start_time 找到一个基准时间 base_time， 22 | 在该 base_time 到 base_time + time_length 区间的点划分到一个 session 内， 23 | 选取 base_time 来做的理由是：这样可以保证同一个小时段总是被 encode 成同一个数 24 | """ 25 | if base_zero: 26 | return start_time - timedelta(hours=start_time.hour, 27 | minutes=start_time.minute, 28 | seconds=start_time.second, 29 | microseconds=start_time.microsecond) 30 | else: 31 | # time length = 12 32 | if start_time.hour < 12: 33 | return start_time - timedelta(hours=start_time.hour, 34 | minutes=start_time.minute, 35 | seconds=start_time.second, 36 | microseconds=start_time.microsecond) 37 | else: 38 | return start_time - timedelta(hours=start_time.hour - 12, 39 | minutes=start_time.minute, 40 | seconds=start_time.second, 41 | microseconds=start_time.microsecond) 42 | 43 | 44 | def cal_timeoff(now_time, base_time): 45 | """ 46 | 计算两个时间之间的差值，返回值以小时为单位 47 | """ 48 | # 先将 now 按小时对齐 49 | delta = now_time - base_time 50 | return delta.days * 24 + delta.seconds / 3600 51 | 52 | 53 | def caculate_time_sim(data): 54 | time_checkin_set = defaultdict(set) 55 | tim_size = data['tim_size'] 56 | data_neural = data['data'] 57 | for uid in data_neural: 58 | uid_sessions = data_neural[uid] 59 | for session in uid_sessions: 60 | for checkin in session: 61 | timid = checkin[1] 62 | locid = checkin[0] 63 | if timid not in time_checkin_set: 64 | time_checkin_set[timid] = set() 65 | time_checkin_set[timid].add(locid) 66 | sim_matrix = np.zeros((tim_size, tim_size)) 67 | for i in range(tim_size): 68 | for j in range(tim_size): 69 | set_i = time_checkin_set[i] 70 | set_j = time_checkin_set[j] 71 | if len(set_i | set_j) != 0: 72 | jaccard_ij = len(set_i & set_j) / len(set_i | set_j) 73 | sim_matrix[i][j] = jaccard_ij 74 | return sim_matrix 75 | 76 | 77 | def parse_coordinate(coordinate): 78 | items = coordinate[1:-1].split(',') 79 | return float(items[0]), float(items[1]) 80 | 81 | 82 | def string2timestamp(strings, offset_frame): 83 | ts = [] 84 | for t in strings: 85 | dtstr = '-'.join([t[:4].decode(), t[4:6].decode(), t[6:8].decode()]) 86 | slot = int(t[8:]) - 1 87 | ts.append(np.datetime64(dtstr, 'm') + slot * offset_frame) 88 | return ts # [numpy.datetime64('2014-01-01T00:00'), ...] 89 | 90 | 91 | def timestamp2array(timestamps, t): 92 | """ 93 | 把时间戳的序列中的每一个时间戳转成特征数组，考虑了星期和小时， 94 | 时间戳: numpy.datetime64('2013-07-01T00:00:00.000000000') 95 | 96 | Args: 97 | timestamps: 时间戳序列 98 | t: 一天有多少个时间步 99 | 100 | Returns: 101 | np.ndarray: 特征数组，shape: (len(timestamps), ext_dim) 102 | """ 103 | vec_wday = [time.strptime( 104 | str(t)[:10], '%Y-%m-%d').tm_wday for t in timestamps] 105 | vec_hour = [time.strptime(str(t)[11:13], '%H').tm_hour for t in timestamps] 106 | vec_minu = [time.strptime(str(t)[14:16], '%M').tm_min for t in timestamps] 107 | ret = [] 108 | for idx, wday in enumerate(vec_wday): 109 | # day 110 | v = [0 for _ in range(7)] 111 | v[wday] = 1 112 | if wday >= 5: # 0是周一, 6是周日 113 | v.append(0) # weekend 114 | else: 115 | v.append(1) # weekday len(v)=8 116 | # hour 117 | v += [0 for _ in range(t)] # len(v)=8+T 118 | hour = vec_hour[idx] 119 | minu = vec_minu[idx] 120 | # 24*60/T 表示一个时间步是多少分钟 121 | # hour * 60 + minu 是从0:0开始到现在是多少分钟，相除计算是第几个时间步 122 | # print(hour, minu, T, (hour * 60 + minu) / (24 * 60 / T)) 123 | v[int((hour * 60 + minu) / (24 * 60 / t))] = 1 124 | # +8是因为v前边有表示星期的8位 125 | if hour >= 18 or hour < 6: 126 | v.append(0) # night 127 | else: 128 | v.append(1) # day 129 | ret.append(v) # len(v)=7+1+T+1=T+9 130 | return np.asarray(ret) 131 | 132 | 133 | def timestamp2vec_origin(timestamps): 134 | """ 135 | 把时间戳的序列中的每一个时间戳转成特征数组，只考虑星期， 136 | 时间戳: numpy.datetime64('2013-07-01T00:00:00.000000000') 137 | 138 | Args: 139 | timestamps: 时间戳序列 140 | 141 | Returns: 142 | np.ndarray: 特征数组，shape: (len(timestamps), 8) 143 | """ 144 | vec = [time.strptime(str(t)[:10], '%Y-%m-%d').tm_wday for t in timestamps] 145 | ret = [] 146 | for i in vec: 147 | v = [0 for _ in range(7)] 148 | v[i] = 1 149 | if i >= 5: 150 | v.append(0) # weekend 151 | else: 152 | v.append(1) # weekday 153 | ret.append(v) 154 | return np.asarray(ret) 155 | -------------------------------------------------------------------------------- /libcity/model/loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from sklearn.metrics import r2_score, explained_variance_score 4 | 5 | 6 | def masked_mae_loss(y_pred, y_true): 7 | mask = (y_true != 0).float() 8 | mask /= mask.mean() 9 | loss = torch.abs(y_pred - y_true) 10 | loss = loss * mask 11 | # trick for nans: 12 | # https://discuss.pytorch.org/t/how-to-set-nan-in-tensor-to-0/3918/3 13 | loss[loss != loss] = 0 14 | return loss.mean() 15 | 16 | 17 | def masked_mae_torch(preds, labels, null_val=np.nan): 18 | labels[torch.abs(labels) < 1e-4] = 0 19 | if np.isnan(null_val): 20 | mask = ~torch.isnan(labels) 21 | else: 22 | mask = labels.ne(null_val) 23 | mask = mask.float() 24 | mask /= torch.mean(mask) 25 | mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask) 26 | loss = torch.abs(torch.sub(preds, labels)) 27 | loss = loss * mask 28 | loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss) 29 | return torch.mean(loss) 30 | 31 | 32 | def log_cosh_loss(preds, labels): 33 | loss = torch.log(torch.cosh(preds - labels)) 34 | return torch.mean(loss) 35 | 36 | 37 | def huber_loss(preds, labels, delta=1.0): 38 | residual = torch.abs(preds - labels) 39 | condition = torch.le(residual, delta) 40 | small_res = 0.5 * torch.square(residual) 41 | large_res = delta * residual - 0.5 * delta * delta 42 | return torch.mean(torch.where(condition, small_res, large_res)) 43 | # lo = torch.nn.SmoothL1Loss() 44 | # return lo(preds, labels) 45 | 46 | 47 | def quantile_loss(preds, labels, delta=0.25): 48 | condition = torch.ge(labels, preds) 49 | large_res = delta * (labels - preds) 50 | small_res = (1 - delta) * (preds - labels) 51 | return torch.mean(torch.where(condition, large_res, small_res)) 52 | 53 | 54 | def masked_mape_torch(preds, labels, null_val=np.nan, eps=0): 55 | labels[torch.abs(labels) < 1e-4] = 0 56 | if np.isnan(null_val) and eps != 0: 57 | loss = torch.abs((preds - labels) / (labels + eps)) 58 | return torch.mean(loss) 59 | if np.isnan(null_val): 60 | mask = ~torch.isnan(labels) 61 | else: 62 | mask = labels.ne(null_val) 63 | mask = mask.float() 64 | mask /= torch.mean(mask) 65 | mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask) 66 | loss = torch.abs((preds - labels) / labels) 67 | loss = loss * mask 68 | loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss) 69 | return torch.mean(loss) 70 | 71 | 72 | def masked_mse_torch(preds, labels, null_val=np.nan): 73 | labels[torch.abs(labels) < 1e-4] = 0 74 | if np.isnan(null_val): 75 | mask = ~torch.isnan(labels) 76 | else: 77 | mask = labels.ne(null_val) 78 | mask = mask.float() 79 | mask /= torch.mean(mask) 80 | mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask) 81 | loss = torch.square(torch.sub(preds, labels)) 82 | loss = loss * mask 83 | loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss) 84 | return torch.mean(loss) 85 | 86 | 87 | def masked_rmse_torch(preds, labels, null_val=np.nan): 88 | labels[torch.abs(labels) < 1e-4] = 0 89 | return torch.sqrt(masked_mse_torch(preds=preds, labels=labels, 90 | null_val=null_val)) 91 | 92 | 93 | def r2_score_torch(preds, labels): 94 | preds = preds.cpu().flatten() 95 | labels = labels.cpu().flatten() 96 | return r2_score(labels, preds) 97 | 98 | 99 | def explained_variance_score_torch(preds, labels): 100 | preds = preds.cpu().flatten() 101 | labels = labels.cpu().flatten() 102 | return explained_variance_score(labels, preds) 103 | 104 | 105 | def masked_rmse_np(preds, labels, null_val=np.nan): 106 | return np.sqrt(masked_mse_np(preds=preds, labels=labels, 107 | null_val=null_val)) 108 | 109 | 110 | def masked_mse_np(preds, labels, null_val=np.nan): 111 | with np.errstate(divide='ignore', invalid='ignore'): 112 | if np.isnan(null_val): 113 | mask = ~np.isnan(labels) 114 | else: 115 | mask = np.not_equal(labels, null_val) 116 | mask = mask.astype('float32') 117 | mask /= np.mean(mask) 118 | rmse = np.square(np.subtract(preds, labels)).astype('float32') 119 | rmse = np.nan_to_num(rmse * mask) 120 | return np.mean(rmse) 121 | 122 | 123 | def masked_mae_np(preds, labels, null_val=np.nan): 124 | with np.errstate(divide='ignore', invalid='ignore'): 125 | if np.isnan(null_val): 126 | mask = ~np.isnan(labels) 127 | else: 128 | mask = np.not_equal(labels, null_val) 129 | mask = mask.astype('float32') 130 | mask /= np.mean(mask) 131 | mae = np.abs(np.subtract(preds, labels)).astype('float32') 132 | mae = np.nan_to_num(mae * mask) 133 | return np.mean(mae) 134 | 135 | 136 | def masked_mape_np(preds, labels, null_val=np.nan): 137 | with np.errstate(divide='ignore', invalid='ignore'): 138 | if np.isnan(null_val): 139 | mask = ~np.isnan(labels) 140 | else: 141 | mask = np.not_equal(labels, null_val) 142 | mask = mask.astype('float32') 143 | mask /= np.mean(mask) 144 | mape = np.abs(np.divide(np.subtract( 145 | preds, labels).astype('float32'), labels)) 146 | mape = np.nan_to_num(mask * mape) 147 | return np.mean(mape) 148 | 149 | 150 | def r2_score_np(preds, labels): 151 | preds = preds.flatten() 152 | labels = labels.flatten() 153 | return r2_score(labels, preds) 154 | 155 | 156 | def explained_variance_score_np(preds, labels): 157 | preds = preds.flatten() 158 | labels = labels.flatten() 159 | return explained_variance_score(labels, preds) 160 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/model/encoding_tree.py: -------------------------------------------------------------------------------- 1 | import networkx 2 | 3 | import silearn 4 | from silearn.graph import Graph 5 | from silearn import * 6 | 7 | 8 | class GraphEncoding: 9 | r""" 10 | The base Graph Encoding model 11 | """ 12 | 13 | def __init__(self, g: Graph): 14 | self.graph = g 15 | 16 | def uncertainty(self, es, et, p): 17 | raise NotImplementedError("Not Implemented") 18 | 19 | def positioning_entropy(self): 20 | dist = self.graph.stationary_dist 21 | return silearn.entropy(dist, dist) 22 | 23 | def entropy_rate(self, reduction="vertex", norm=False): 24 | edges, p = self.graph.edges 25 | es, et = edges[:, 0], edges[:, 1] 26 | nw = self.graph.vertex_weight_es[es] 27 | entropy = silearn.entropy(p, p / nw) 28 | 29 | if norm: 30 | dist = self.graph.stationary_dist[es] 31 | entropy = entropy / self.positioning_entropy() 32 | 33 | if reduction == "none": 34 | return entropy 35 | elif reduction == "vertex": 36 | return silearn.scatter_sum(entropy, et) 37 | elif reduction == "sum": 38 | return entropy.sum() 39 | else: 40 | return entropy 41 | 42 | def structural_entropy(self, reduction="vertex", norm=False): 43 | edges, p = self.graph.edges 44 | es, et = edges[:, 0], edges[:, 1] 45 | # dist = self.graph.stationary_dist[es] 46 | dist = self.graph.stationary_dist[es] 47 | # tot = w.sum() 48 | entropy = p * self.uncertainty(es, et, p) 49 | 50 | if norm: 51 | entropy = entropy / silearn.entropy(dist, dist) 52 | if reduction == "none": 53 | return entropy 54 | elif reduction == "vertex": 55 | return silearn.scatter_sum(entropy, et) 56 | elif reduction == "sum": 57 | return entropy.sum() 58 | else: 59 | return entropy 60 | 61 | def to_networkx(self, create_using=networkx.DiGraph()): 62 | raise NotImplementedError() 63 | 64 | 65 | class OneDim(GraphEncoding): 66 | 67 | def uncertainty(self, es, et, p): 68 | v1 = self.graph.stationary_dist[es] 69 | return uncertainty(v1) 70 | 71 | 72 | class Partitioning(GraphEncoding): 73 | node_id = None # :torch.LongTensor 74 | 75 | def __init__(self, g: Graph, init_parition): 76 | super().__init__(g) 77 | self.node_id = init_parition 78 | 79 | def uncertainty(self, es, et, p): 80 | v1e = self.graph.stationary_dist[es] 81 | id_et = self.node_id[et] 82 | id_es = self.node_id[es] 83 | v2 = scatter_sum(self.graph.stationary_dist, self.node_id) 84 | v2e = v2[id_es] 85 | flag = id_es != id_et 86 | # print(v1e, v2, flag) 87 | return uncertainty(v1e / v2e) + flag * uncertainty(v2e / v2.sum()) 88 | 89 | def structural_entropy(self, reduction="vertex", norm=False): 90 | entropy = super(Partitioning, self).structural_entropy(reduction, norm) 91 | if reduction == "module": 92 | et = self.graph.edges[0][:, 1] 93 | return scatter_sum(entropy, self.node_id[et]) 94 | return entropy 95 | 96 | def compound(self, hyper_partitioning): 97 | self.node_id = hyper_partitioning[self.node_id] 98 | 99 | # def to_graph(self): 100 | # import numpy as np 101 | # import torch 102 | # 103 | # a = np.array([[0, 1.2, 0], [2, 3.1, 0], [0.5, 0, 0]]) 104 | # idx = a.nonzero() # (row, col) 105 | # data = a[idx] 106 | # 107 | # # to torch tensor 108 | # idx_t = torch.LongTensor(np.vstack(idx)) 109 | # data_t = torch.FloatTensor(data) 110 | # coo_a = torch.sparse_coo_tensor(idx_t, data_t, a.shape) 111 | 112 | def to_networkx(self, 113 | create_using=networkx.DiGraph(), 114 | label_name="partition"): 115 | nx_graph = self.graph.to_networkx(create_using=create_using) 116 | label_np = silearn.convert_backend(self.node_id, "numpy") 117 | for i in range(label_np.shape[0]): 118 | nx_graph._node[i][label_name] = label_np[i] 119 | return nx_graph 120 | 121 | 122 | class EncodingTree(GraphEncoding): 123 | parent_id: [] 124 | 125 | def uncertainty(self, es, et, p): 126 | v1 = self.graph.stationary_dist[et] 127 | cur_ids = es 128 | cur_idt = et 129 | ret = 0 130 | for i in range(len(self.parent_id)): 131 | id_es = self.parent_id[i][cur_ids] 132 | id_et = self.parent_id[i][cur_idt] 133 | vp = scatter_sum( 134 | v1, 135 | id_et)[id_et] if i != len(self.parent_id) - 1 else v1.sum() 136 | if i == 0: 137 | ret += uncertainty(v1 / vp) 138 | else: 139 | flag = cur_ids != cur_idt 140 | ret += flag * uncertainty(v1 / vp) 141 | v1 = vp 142 | cur_ids, cur_idt = id_es, id_et 143 | return ret 144 | 145 | def structural_entropy(self, reduction="vertex", norm=False): 146 | entropy = super(EncodingTree, self).structural_entropy(reduction, norm) 147 | if reduction.startswith("level"): 148 | level = int(reduction[5:]) 149 | level = min(-len(self.parent_id), level) 150 | level = max(len(self.parent_id) - 1, level) 151 | et = self.graph.edges[2] 152 | return scatter_sum(entropy, self.parent_id[level][et]) 153 | return entropy 154 | 155 | """ 156 | 2-Dim Enc Tree: Level - -1, 0 157 | 3-Dim Enc Tree: Level - -2, -1, 0, 1 158 | """ 159 | 160 | def as_partition(self, level=-1): 161 | height = len(self.parent_id) 162 | assert -height <= level < height 163 | if level < 0: 164 | level = height + level 165 | if level != 0: 166 | trans = self.parent_id[level] 167 | for i in reversed(range(level)): 168 | trans = trans[self.parent_id[i]] 169 | return trans 170 | else: 171 | return self.parent_id 172 | -------------------------------------------------------------------------------- /libcity/data/utils.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import numpy as np 3 | from torch.utils.data import DataLoader 4 | import copy 5 | 6 | from libcity.data.list_dataset import ListDataset 7 | from libcity.data.batch import Batch, BatchPAD 8 | 9 | 10 | def get_dataset(config): 11 | """ 12 | according the config['dataset_class'] to create the dataset 13 | 14 | Args: 15 | config(ConfigParser): config 16 | 17 | Returns: 18 | AbstractDataset: the loaded dataset 19 | """ 20 | try: 21 | return getattr(importlib.import_module('libcity.data.dataset'), 22 | config['dataset_class'])(config) 23 | except AttributeError: 24 | try: 25 | return getattr(importlib.import_module('libcity.data.dataset.dataset_subclass'), 26 | config['dataset_class'])(config) 27 | except AttributeError: 28 | raise AttributeError('dataset_class is not found') 29 | 30 | 31 | def generate_dataloader(train_data, eval_data, test_data, feature_name, 32 | batch_size, num_workers, shuffle=True, 33 | pad_with_last_sample=False): 34 | """ 35 | create dataloader(train/test/eval) 36 | 37 | Args: 38 | train_data(list of input): 训练数据，data 中每个元素是模型单次的输入，input 是一个 list，里面存放单次输入和 target 39 | eval_data(list of input): 验证数据，data 中每个元素是模型单次的输入，input 是一个 list，里面存放单次输入和 target 40 | test_data(list of input): 测试数据，data 中每个元素是模型单次的输入，input 是一个 list，里面存放单次输入和 target 41 | feature_name(dict): 描述上面 input 每个元素对应的特征名, 应保证len(feature_name) = len(input) 42 | batch_size(int): batch_size 43 | num_workers(int): num_workers 44 | shuffle(bool): shuffle 45 | pad_with_last_sample(bool): 对于若最后一个 batch 不满足 batch_size的情况，是否进行补齐（使用最后一个元素反复填充补齐）。 46 | 47 | Returns: 48 | tuple: tuple contains: 49 | train_dataloader: Dataloader composed of Batch (class) \n 50 | eval_dataloader: Dataloader composed of Batch (class) \n 51 | test_dataloader: Dataloader composed of Batch (class) 52 | """ 53 | if pad_with_last_sample: 54 | num_padding = (batch_size - (len(train_data) % batch_size)) % batch_size 55 | data_padding = np.repeat(train_data[-1:], num_padding, axis=0) 56 | train_data = np.concatenate([train_data, data_padding], axis=0) 57 | num_padding = (batch_size - (len(eval_data) % batch_size)) % batch_size 58 | data_padding = np.repeat(eval_data[-1:], num_padding, axis=0) 59 | eval_data = np.concatenate([eval_data, data_padding], axis=0) 60 | num_padding = (batch_size - (len(test_data) % batch_size)) % batch_size 61 | data_padding = np.repeat(test_data[-1:], num_padding, axis=0) 62 | test_data = np.concatenate([test_data, data_padding], axis=0) 63 | 64 | train_dataset = ListDataset(train_data) 65 | eval_dataset = ListDataset(eval_data) 66 | test_dataset = ListDataset(test_data) 67 | 68 | def collator(indices): 69 | batch = Batch(feature_name) 70 | for item in indices: 71 | batch.append(copy.deepcopy(item)) 72 | return batch 73 | 74 | train_dataloader = DataLoader(dataset=train_dataset, batch_size=batch_size, 75 | num_workers=num_workers, collate_fn=collator, 76 | shuffle=shuffle) 77 | eval_dataloader = DataLoader(dataset=eval_dataset, batch_size=batch_size, 78 | num_workers=num_workers, collate_fn=collator, 79 | shuffle=shuffle) 80 | test_dataloader = DataLoader(dataset=test_dataset, batch_size=batch_size, 81 | num_workers=num_workers, collate_fn=collator, 82 | shuffle=False) 83 | return train_dataloader, eval_dataloader, test_dataloader 84 | 85 | 86 | def generate_dataloader_pad(train_data, eval_data, test_data, feature_name, 87 | batch_size, num_workers, pad_item=None, 88 | pad_max_len=None, shuffle=True): 89 | """ 90 | create dataloader(train/test/eval) 91 | 92 | Args: 93 | train_data(list of input): 训练数据，data 中每个元素是模型单次的输入，input 是一个 list，里面存放单次输入和 target 94 | eval_data(list of input): 验证数据，data 中每个元素是模型单次的输入，input 是一个 list，里面存放单次输入和 target 95 | test_data(list of input): 测试数据，data 中每个元素是模型单次的输入，input 是一个 list，里面存放单次输入和 target 96 | feature_name(dict): 描述上面 input 每个元素对应的特征名, 应保证len(feature_name) = len(input) 97 | batch_size(int): batch_size 98 | num_workers(int): num_workers 99 | pad_item(dict): 用于将不定长的特征补齐到一样的长度，每个特征名作为 key，若某特征名不在该 dict 内则不进行补齐。 100 | pad_max_len(dict): 用于截取不定长的特征，对于过长的特征进行剪切 101 | shuffle(bool): shuffle 102 | 103 | Returns: 104 | tuple: tuple contains: 105 | train_dataloader: Dataloader composed of Batch (class) \n 106 | eval_dataloader: Dataloader composed of Batch (class) \n 107 | test_dataloader: Dataloader composed of Batch (class) 108 | """ 109 | train_dataset = ListDataset(train_data) 110 | eval_dataset = ListDataset(eval_data) 111 | test_dataset = ListDataset(test_data) 112 | 113 | def collator(indices): 114 | batch = BatchPAD(feature_name, pad_item, pad_max_len) 115 | for item in indices: 116 | batch.append(copy.deepcopy(item)) 117 | batch.padding() 118 | return batch 119 | 120 | train_dataloader = DataLoader(dataset=train_dataset, batch_size=batch_size, 121 | num_workers=num_workers, collate_fn=collator, 122 | shuffle=shuffle) 123 | eval_dataloader = DataLoader(dataset=eval_dataset, batch_size=batch_size, 124 | num_workers=num_workers, collate_fn=collator, 125 | shuffle=shuffle) 126 | test_dataloader = DataLoader(dataset=test_dataset, batch_size=batch_size, 127 | num_workers=num_workers, collate_fn=collator, 128 | shuffle=shuffle) 129 | return train_dataloader, eval_dataloader, test_dataloader 130 | -------------------------------------------------------------------------------- /libcity/evaluator/eval_funcs.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | # 均方误差（Mean Square Error） 6 | def mse(loc_pred, loc_true): 7 | assert len(loc_pred) == len(loc_true), 'MSE: 预测数据与真实数据大小不一致' 8 | return np.mean(sum(pow(loc_pred - loc_true, 2))) 9 | 10 | 11 | # 平均绝对误差（Mean Absolute Error） 12 | def mae(loc_pred, loc_true): 13 | assert len(loc_pred) == len(loc_true), 'MAE: 预测数据与真实数据大小不一致' 14 | return np.mean(sum(loc_pred - loc_true)) 15 | 16 | 17 | # 均方根误差（Root Mean Square Error） 18 | def rmse(loc_pred, loc_true): 19 | assert len(loc_pred) == len(loc_true), 'RMSE: 预测数据与真实数据大小不一致' 20 | return np.sqrt(np.mean(sum(pow(loc_pred - loc_true, 2)))) 21 | 22 | 23 | # 平均绝对百分比误差（Mean Absolute Percentage Error） 24 | def mape(loc_pred, loc_true): 25 | assert len(loc_pred) == len(loc_true), 'MAPE: 预测数据与真实数据大小不一致' 26 | assert 0 not in loc_true, "MAPE: 真实数据有0，该公式不适用" 27 | return np.mean(abs(loc_pred - loc_true) / loc_true) 28 | 29 | 30 | # 平均绝对和相对误差（Mean Absolute Relative Error） 31 | def mare(loc_pred, loc_true): 32 | assert len(loc_pred) == len(loc_true), "MARE：预测数据与真实数据大小不一致" 33 | assert np.sum(loc_true) != 0, "MARE：真实位置全为0，该公式不适用" 34 | return np.sum(np.abs(loc_pred - loc_true)) / np.sum(loc_true) 35 | 36 | 37 | # 对称平均绝对百分比误差（Symmetric Mean Absolute Percentage Error） 38 | def smape(loc_pred, loc_true): 39 | assert len(loc_pred) == len(loc_true), 'SMAPE: 预测数据与真实数据大小不一致' 40 | assert 0 in (loc_pred + loc_true), "SMAPE: 预测数据与真实数据有0，该公式不适用" 41 | return 2.0 * np.mean(np.abs(loc_pred - loc_true) / (np.abs(loc_pred) + 42 | np.abs(loc_true))) 43 | 44 | 45 | # 对比真实位置与预测位置获得预测准确率 46 | def acc(loc_pred, loc_true): 47 | assert len(loc_pred) == len(loc_true), "accuracy: 预测数据与真实数据大小不一致" 48 | loc_diff = loc_pred - loc_true 49 | loc_diff[loc_diff != 0] = 1 50 | return loc_diff, np.mean(loc_diff == 0) 51 | 52 | 53 | def top_k(loc_pred, loc_true, topk): 54 | """ 55 | count the hit numbers of loc_true in topK of loc_pred, used to calculate Precision, Recall and F1-score, 56 | calculate the reciprocal rank, used to calcualte MRR, 57 | calculate the sum of DCG@K of the batch, used to calculate NDCG 58 | 59 | Args: 60 | loc_pred: (batch_size * output_dim) 61 | loc_true: (batch_size * 1) 62 | topk: 63 | 64 | Returns: 65 | tuple: tuple contains: 66 | hit (int): the hit numbers \n 67 | rank (float): the sum of the reciprocal rank of input batch \n 68 | dcg (float): dcg 69 | """ 70 | assert topk > 0, "top-k ACC评估方法：k值应不小于1" 71 | loc_pred = torch.FloatTensor(loc_pred) 72 | val, index = torch.topk(loc_pred, topk, 1) 73 | index = index.numpy() 74 | hit = 0 75 | rank = 0.0 76 | dcg = 0.0 77 | for i, p in enumerate(index): 78 | target = loc_true[i] 79 | if target in p: 80 | hit += 1 81 | rank_list = list(p) 82 | rank_index = rank_list.index(target) 83 | # rank_index is start from 0, so need plus 1 84 | rank += 1.0 / (rank_index + 1) 85 | dcg += 1.0 / np.log2(rank_index + 2) 86 | return hit, rank, dcg 87 | 88 | def Precision_torch(preds, labels, topk): 89 | precision = [] 90 | for i in range(preds.shape[0]): 91 | label = labels[i] 92 | pred = preds[i] 93 | accident_grids = label > 0 94 | sorted, _ = torch.sort(pred.flatten(), descending=True) 95 | threshold = sorted[topk - 1] 96 | pred_grids = pred >= threshold 97 | matched = pred_grids & accident_grids 98 | precision.append(torch.sum(matched.flatten()).item() / topk) 99 | return sum(precision) / len(precision) 100 | 101 | def Recall_torch(preds, labels, topk): 102 | recall = [] 103 | for i in range(preds.shape[0]): 104 | label = labels[i] 105 | pred = preds[i] 106 | accident_grids = label > 0 107 | sorted, _ = torch.sort(pred.flatten(), descending=True) 108 | threshold = sorted[topk - 1] 109 | pred_grids = pred >= threshold 110 | matched = pred_grids & accident_grids 111 | if torch.sum(accident_grids).item() != 0: 112 | recall.append(torch.sum(matched.flatten()).item() / torch.sum(accident_grids.flatten()).item()) 113 | return sum(recall) / len(recall) 114 | 115 | def F1_Score_torch(preds, labels, topk): 116 | precision = Precision_torch(preds, labels, topk) 117 | recall = Recall_torch(preds, labels, topk) 118 | return 2 * precision * recall / (precision + recall) 119 | 120 | 121 | 122 | def MAP_torch(preds, labels, topk): 123 | ap = [] 124 | for i in range(preds.shape[0]): 125 | label = labels[i].flatten() 126 | pred = preds[i].flatten() 127 | accident_grids = label > 0 128 | sorted, rank = torch.sort(pred, descending=True) 129 | rank = rank[:topk] 130 | if topk != 0: 131 | threshold = sorted[topk - 1] 132 | else: 133 | threshold = 0 134 | label = label != 0 135 | pred = pred >= threshold 136 | matched = pred & label 137 | match_num = 0 138 | precision_sum = 0 139 | for i in range(rank.shape[0]): 140 | if matched[rank[i]]: 141 | match_num += 1 142 | precision_sum += match_num / (i + 1) 143 | if rank.shape[0] != 0: 144 | ap.append(precision_sum / rank.shape[0]) 145 | return sum(ap) / len(ap) 146 | 147 | 148 | def PCC_torch(preds, labels, topk): 149 | pcc = [] 150 | for i in range(preds.shape[0]): 151 | label = labels[i].flatten() 152 | pred = preds[i].flatten() 153 | sorted, rank = torch.sort(pred, descending=True) 154 | pred = sorted[:topk] 155 | rank = rank[:topk] 156 | sorted_label = torch.zeros(topk) 157 | for i in range(topk): 158 | sorted_label[i] = label[rank[i]] 159 | label = sorted_label 160 | label_average = torch.sum(label) / (label.shape[0]) 161 | pred_average = torch.sum(pred) / (pred.shape[0]) 162 | if torch.sqrt(torch.sum((label - label_average) * (label - label_average))) * torch.sqrt( 163 | torch.sum((pred - pred_average) * (pred - pred_average))) != 0: 164 | pcc.append((torch.sum((label - label_average) * (pred - pred_average)) / ( 165 | torch.sqrt(torch.sum((label - label_average) * (label - label_average))) * torch.sqrt( 166 | torch.sum((pred - pred_average) * (pred - pred_average))))).item()) 167 | return sum(pcc) / len(pcc) 168 | -------------------------------------------------------------------------------- /libcity/config/config_parser.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import torch 4 | 5 | 6 | class ConfigParser(object): 7 | """ 8 | use to parse the user defined parameters and use these to modify the 9 | pipeline's parameter setting. 10 | 值得注意的是，目前各阶段的参数是放置于同一个 dict 中的，因此需要编程时保证命名空间不冲突。 11 | config 优先级：命令行 > config file > default config 12 | """ 13 | 14 | def __init__(self, task, model, dataset, config_file=None, 15 | saved_model=True, train=True, other_args=None, hyper_config_dict=None): 16 | """ 17 | Args: 18 | task, model, dataset (str): 用户在命令行必须指明的三个参数 19 | config_file (str): 配置文件的文件名，将在项目根目录下进行搜索 20 | other_args (dict): 通过命令行传入的其他参数 21 | """ 22 | self.config = {} 23 | self._parse_external_config(task, model, dataset, saved_model, train, other_args, hyper_config_dict) 24 | self._parse_config_file(config_file) 25 | self._load_default_config() 26 | self._init_device() 27 | 28 | def _parse_external_config(self, task, model, dataset, 29 | saved_model=True, train=True, other_args=None, hyper_config_dict=None): 30 | if task is None: 31 | raise ValueError('the parameter task should not be None!') 32 | if model is None: 33 | raise ValueError('the parameter model should not be None!') 34 | if dataset is None: 35 | raise ValueError('the parameter dataset should not be None!') 36 | # 目前暂定这三个参数必须由用户指定 37 | self.config['task'] = task 38 | self.config['model'] = model 39 | self.config['dataset'] = dataset 40 | self.config['saved_model'] = saved_model 41 | self.config['train'] = False if task == 'map_matching' else train 42 | if other_args is not None: 43 | # TODO: 这里可以设计加入参数检查，哪些参数是允许用户通过命令行修改的 44 | for key in other_args: 45 | self.config[key] = other_args[key] 46 | if hyper_config_dict is not None: 47 | # 超参数调整时传入的待调整的参数，优先级低于命令行参数 48 | for key in hyper_config_dict: 49 | self.config[key] = hyper_config_dict[key] 50 | 51 | def _parse_config_file(self, config_file): 52 | if config_file is not None: 53 | # TODO: 对 config file 的格式进行检查 54 | if os.path.exists('./{}.json'.format(config_file)): 55 | with open('./{}.json'.format(config_file), 'r') as f: 56 | x = json.load(f) 57 | for key in x: 58 | if key not in self.config: 59 | self.config[key] = x[key] 60 | else: 61 | raise FileNotFoundError( 62 | 'Config file {}.json is not found. Please ensure \ 63 | the config file is in the root dir and is a JSON \ 64 | file.'.format(config_file)) 65 | 66 | def _load_default_config(self): 67 | # 首先加载 task config 68 | with open('./libcity/config/task_config.json', 'r') as f: 69 | task_config = json.load(f) 70 | if self.config['task'] not in task_config: 71 | raise ValueError( 72 | 'task {} is not supported.'.format(self.config['task'])) 73 | task_config = task_config[self.config['task']] 74 | # check model and dataset 75 | if self.config['model'] not in task_config['allowed_model']: 76 | raise ValueError('task {} do not support model {}'.format( 77 | self.config['task'], self.config['model'])) 78 | model = self.config['model'] 79 | # 加载 dataset、executor、evaluator 的模块 80 | if 'dataset_class' not in self.config: 81 | self.config['dataset_class'] = task_config[model]['dataset_class'] 82 | if self.config['task'] == 'traj_loc_pred' and 'traj_encoder' not in self.config: 83 | self.config['traj_encoder'] = task_config[model]['traj_encoder'] 84 | if self.config['task'] == 'eta' and 'eta_encoder' not in self.config: 85 | self.config['eta_encoder'] = task_config[model]['eta_encoder'] 86 | if 'executor' not in self.config: 87 | self.config['executor'] = task_config[model]['executor'] 88 | if 'evaluator' not in self.config: 89 | self.config['evaluator'] = task_config[model]['evaluator'] 90 | # 对于 LSTM RNN GRU 使用的都是同一个类，只是 RNN 模块不一样而已，这里做一下修改 91 | if self.config['model'].upper() in ['LSTM', 'GRU', 'RNN']: 92 | self.config['rnn_type'] = self.config['model'] 93 | self.config['model'] = 'RNN' 94 | # if self.config['dataset'] not in task_config['allowed_dataset']: 95 | # raise ValueError('task {} do not support dataset {}'.format( 96 | # self.config['task'], self.config['dataset'])) 97 | # 接着加载每个阶段的 default config 98 | default_file_list = [] 99 | # model 100 | default_file_list.append('model/{}/{}.json'.format(self.config['task'], self.config['model'])) 101 | # dataset 102 | default_file_list.append('data/{}.json'.format(self.config['dataset_class'])) 103 | # executor 104 | default_file_list.append('executor/{}.json'.format(self.config['executor'])) 105 | # evaluator 106 | default_file_list.append('evaluator/{}.json'.format(self.config['evaluator'])) 107 | # 加载所有默认配置 108 | for file_name in default_file_list: 109 | with open('./libcity/config/{}'.format(file_name), 'r') as f: 110 | x = json.load(f) 111 | for key in x: 112 | if key not in self.config: 113 | self.config[key] = x[key] 114 | # 加载数据集config.json 115 | with open('./raw_data/{}/config.json'.format(self.config['dataset']), 'r') as f: 116 | x = json.load(f) 117 | for key in x: 118 | if key == 'info': 119 | for ik in x[key]: 120 | if ik not in self.config: 121 | self.config[ik] = x[key][ik] 122 | else: 123 | if key not in self.config: 124 | self.config[key] = x[key] 125 | 126 | def _init_device(self): 127 | use_gpu = self.config.get('gpu', True) 128 | gpu_id = self.config.get('gpu_id', 0) 129 | if use_gpu: 130 | torch.cuda.set_device(gpu_id) 131 | self.config['device'] = torch.device( 132 | "cuda:%d" % gpu_id if torch.cuda.is_available() and use_gpu else "cpu") 133 | 134 | def get(self, key, default=None): 135 | return self.config.get(key, default) 136 | 137 | def __getitem__(self, key): 138 | if key in self.config: 139 | return self.config[key] 140 | else: 141 | raise KeyError('{} is not in the config'.format(key)) 142 | 143 | def __setitem__(self, key, value): 144 | self.config[key] = value 145 | 146 | def __contains__(self, key): 147 | return key in self.config 148 | 149 | # 支持迭代操作 150 | def __iter__(self): 151 | return self.config.__iter__() 152 | -------------------------------------------------------------------------------- /libcity/data/batch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | 5 | class Batch(object): 6 | 7 | def __init__(self, feature_name): 8 | """Summary of class here 9 | 10 | Args: 11 | feature_name (dict): key is the corresponding feature's name, and 12 | the value is the feature's data type 13 | """ 14 | self.data = {} 15 | self.feature_name = feature_name 16 | for key in feature_name: 17 | self.data[key] = [] 18 | 19 | def __getitem__(self, key): 20 | if key in self.data: 21 | return self.data[key] 22 | else: 23 | raise KeyError('{} is not in the batch'.format(key)) 24 | 25 | def __setitem__(self, key, value): 26 | if key in self.data: 27 | self.data[key] = value 28 | else: 29 | raise KeyError('{} is not in the batch'.format(key)) 30 | 31 | def append(self, item): 32 | """ 33 | append a new item into the batch 34 | 35 | Args: 36 | item (list): 一组输入，跟feature_name的顺序一致，feature_name即是这一组输入的名字 37 | """ 38 | if len(item) != len(self.feature_name): 39 | raise KeyError('when append a batch, item is not equal length with feature_name') 40 | for i, key in enumerate(self.feature_name): 41 | self.data[key].append(item[i]) 42 | 43 | def to_tensor(self, device): 44 | """ 45 | 将数据self.data转移到device上 46 | 47 | Args: 48 | device(torch.device): GPU/CPU设备 49 | """ 50 | for key in self.data: 51 | if self.feature_name[key] == 'int': 52 | self.data[key] = torch.LongTensor(np.array(self.data[key])).to(device) 53 | elif self.feature_name[key] == 'float': 54 | self.data[key] = torch.FloatTensor(np.array(self.data[key])).to(device) 55 | else: 56 | raise TypeError( 57 | 'Batch to_tensor, only support int, float but you give {}'.format(self.feature_name[key])) 58 | 59 | def to_ndarray(self): 60 | for key in self.data: 61 | if self.feature_name[key] == 'int': 62 | self.data[key] = np.array(self.data[key]) 63 | elif self.feature_name[key] == 'float': 64 | self.data[key] = np.array(self.data[key]) 65 | else: 66 | raise TypeError( 67 | 'Batch to_ndarray, only support int, float but you give {}'.format(self.feature_name[key])) 68 | 69 | 70 | class BatchPAD(Batch): 71 | 72 | def __init__(self, feature_name, pad_item=None, pad_max_len=None): 73 | """Summary of class here 74 | 75 | Args: 76 | feature_name (dict): key is the corresponding feature's name, and 77 | the value is the feature's data type 78 | pad_item (dict): key is the feature name, and value is the padding 79 | value. We will just padding the feature in pad_item 80 | pad_max_len (dict): key is the feature name, and value is the max 81 | length of padded feature. use this parameter to truncate the 82 | feature. 83 | """ 84 | super().__init__(feature_name=feature_name) 85 | # 默认是根据 batch 中每个特征最长的长度来补齐，如果某个特征的长度超过了 pad_max_len 则进行剪切 86 | self.pad_len = {} 87 | self.origin_len = {} # 用于得知补齐前轨迹的原始长度 88 | self.pad_max_len = pad_max_len if pad_max_len is not None else {} 89 | self.pad_item = pad_item if pad_item is not None else {} 90 | for key in feature_name: 91 | self.data[key] = [] 92 | if key in self.pad_item: 93 | self.pad_len[key] = 0 94 | self.origin_len[key] = [] 95 | 96 | def append(self, item): 97 | """ 98 | append a new item into the batch 99 | 100 | Args: 101 | item (list): 一组输入，跟feature_name的顺序一致，feature_name即是这一组输入的名字 102 | """ 103 | if len(item) != len(self.feature_name): 104 | raise KeyError('when append a batch, item is not equal length with feature_name') 105 | for i, key in enumerate(self.feature_name): 106 | # 需保证 item 每个特征的顺序与初始化时传入的 feature_name 中特征的顺序一致 107 | self.data[key].append(item[i]) 108 | if key in self.pad_item: 109 | self.origin_len[key].append(len(item[i])) 110 | if self.pad_len[key] < len(item[i]): 111 | # 保持 pad_len 是最大的 112 | self.pad_len[key] = len(item[i]) 113 | 114 | def padding(self): 115 | """ 116 | 只提供对一维数组的特征进行补齐 117 | """ 118 | for key in self.pad_item: 119 | # 只对在 pad_item 中的特征进行补齐 120 | if key not in self.data: 121 | raise KeyError('when pad a batch, raise this error!') 122 | max_len = self.pad_len[key] 123 | if key in self.pad_max_len: 124 | max_len = min(self.pad_max_len[key], max_len) 125 | for i in range(len(self.data[key])): 126 | if len(self.data[key][i]) < max_len: 127 | self.data[key][i] += [self.pad_item[key]] * \ 128 | (max_len - len(self.data[key][i])) 129 | else: 130 | # 截取的原则是，抛弃前面的点 131 | # 因为是时间序列嘛 132 | self.data[key][i] = self.data[key][i][-max_len:] 133 | # 对于剪切了的，我们没办法还原，但至少不要使他出错 134 | self.origin_len[key][i] = max_len 135 | 136 | def get_origin_len(self, key): 137 | return self.origin_len[key] 138 | 139 | def to_tensor(self, device): 140 | """ 141 | 将数据self.data转移到device上 142 | 143 | Args: 144 | device(torch.device): GPU/CPU设备 145 | """ 146 | for key in self.data: 147 | if self.feature_name[key] == 'int': 148 | self.data[key] = torch.LongTensor(np.array(self.data[key])).to(device) 149 | elif self.feature_name[key] == 'float': 150 | self.data[key] = torch.FloatTensor(np.array(self.data[key])).to(device) 151 | elif self.feature_name[key] == 'array of int': 152 | for i in range(len(self.data[key])): 153 | for j in range(len(self.data[key][i])): 154 | try: 155 | self.data[key][i][j] = torch.LongTensor(np.array(self.data[key][i][j])).to(device) 156 | except TypeError: 157 | print('device is ', device) 158 | exit() 159 | elif self.feature_name[key] == 'no_pad_int': 160 | for i in range(len(self.data[key])): 161 | self.data[key][i] = torch.LongTensor(np.array(self.data[key][i])).to(device) 162 | elif self.feature_name[key] == 'no_pad_float': 163 | for i in range(len(self.data[key])): 164 | self.data[key][i] = torch.FloatTensor(np.array(self.data[key][i])).to(device) 165 | elif self.feature_name[key] == 'no_tensor': 166 | pass 167 | else: 168 | raise TypeError( 169 | 'Batch to_tensor, only support int, float but you give {}'.format(self.feature_name[key])) 170 | -------------------------------------------------------------------------------- /libcity/evaluator/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | from heapq import nlargest 3 | import pandas as pd 4 | from libcity.model.loss import * 5 | 6 | 7 | def output(method, value, field): 8 | """ 9 | Args: 10 | method: 评估方法 11 | value: 对应评估方法的评估结果值 12 | field: 评估的范围, 对一条轨迹或是整个模型 13 | """ 14 | if method == 'ACC': 15 | if field == 'model': 16 | print('---- 该模型在 {} 评估方法下 avg_acc={:.3f} ----'.format(method, 17 | value)) 18 | else: 19 | print('{} avg_acc={:.3f}'.format(method, value)) 20 | elif method in ['MSE', 'RMSE', 'MAE', 'MAPE', 'MARE', 'SMAPE']: 21 | if field == 'model': 22 | print('---- 该模型在 {} 评估方法下 avg_loss={:.3f} ----'.format(method, 23 | value)) 24 | else: 25 | print('{} avg_loss={:.3f}'.format(method, value)) 26 | else: 27 | if field == 'model': 28 | print('---- 该模型在 {} 评估方法下 avg_acc={:.3f} ----'.format(method, 29 | value)) 30 | else: 31 | print('{} avg_acc={:.3f}'.format(method, value)) 32 | 33 | 34 | def transfer_data(data, model, maxk): 35 | """ 36 | Here we transform specific data types to standard input type 37 | """ 38 | if type(data) == str: 39 | data = json.loads(data) 40 | assert type(data) == dict, "待评估数据的类型/格式不合法" 41 | if model == 'DeepMove': 42 | user_idx = data.keys() 43 | for user_id in user_idx: 44 | trace_idx = data[user_id].keys() 45 | for trace_id in trace_idx: 46 | trace = data[user_id][trace_id] 47 | loc_pred = trace['loc_pred'] 48 | new_loc_pred = [] 49 | for t_list in loc_pred: 50 | new_loc_pred.append(sort_confidence_ids(t_list, maxk)) 51 | data[user_id][trace_id]['loc_pred'] = new_loc_pred 52 | return data 53 | 54 | 55 | def sort_confidence_ids(confidence_list, threshold): 56 | """ 57 | Here we convert the prediction results of the DeepMove model 58 | DeepMove model output: confidence of all locations 59 | Evaluate model input: location ids based on confidence 60 | :param threshold: maxK 61 | :param confidence_list: 62 | :return: ids_list 63 | """ 64 | """sorted_list = sorted(confidence_list, reverse=True) 65 | mark_list = [0 for i in confidence_list] 66 | ids_list = [] 67 | for item in sorted_list: 68 | for i in range(len(confidence_list)): 69 | if confidence_list[i] == item and mark_list[i] == 0: 70 | mark_list[i] = 1 71 | ids_list.append(i) 72 | break 73 | if len(ids_list) == threshold: 74 | break 75 | return ids_list""" 76 | max_score_with_id = nlargest( 77 | threshold, enumerate(confidence_list), lambda x: x[1]) 78 | return list(map(lambda x: x[0], max_score_with_id)) 79 | 80 | 81 | def evaluate_model(y_pred, y_true, metrics, mode='single', path='metrics.csv'): 82 | """ 83 | 交通状态预测评估函数 84 | :param y_pred: (num_samples/batch_size, timeslots, ..., feature_dim) 85 | :param y_true: (num_samples/batch_size, timeslots, ..., feature_dim) 86 | :param metrics: 评估指标 87 | :param mode: 单步or多步平均 88 | :param path: 保存结果 89 | :return: 90 | """ 91 | if y_true.shape != y_pred.shape: 92 | raise ValueError("y_true.shape is not equal to y_pred.shape") 93 | len_timeslots = y_true.shape[1] 94 | if isinstance(y_pred, np.ndarray): 95 | y_pred = torch.FloatTensor(y_pred) 96 | if isinstance(y_true, np.ndarray): 97 | y_true = torch.FloatTensor(y_true) 98 | assert isinstance(y_pred, torch.Tensor) 99 | assert isinstance(y_true, torch.Tensor) 100 | 101 | df = [] 102 | for i in range(1, len_timeslots + 1): 103 | line = {} 104 | for metric in metrics: 105 | if mode.lower() == 'single': 106 | if metric == 'masked_MAE': 107 | line[metric] = masked_mae_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item() 108 | elif metric == 'masked_MSE': 109 | line[metric] = masked_mse_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item() 110 | elif metric == 'masked_RMSE': 111 | line[metric] = masked_rmse_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item() 112 | elif metric == 'masked_MAPE': 113 | line[metric] = masked_mape_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item() 114 | elif metric == 'MAE': 115 | line[metric] = masked_mae_torch(y_pred[:, i - 1], y_true[:, i - 1]).item() 116 | elif metric == 'MSE': 117 | line[metric] = masked_mse_torch(y_pred[:, i - 1], y_true[:, i - 1]).item() 118 | elif metric == 'RMSE': 119 | line[metric] = masked_rmse_torch(y_pred[:, i - 1], y_true[:, i - 1]).item() 120 | elif metric == 'MAPE': 121 | line[metric] = masked_mape_torch(y_pred[:, i - 1], y_true[:, i - 1]).item() 122 | elif metric == 'R2': 123 | line[metric] = r2_score_torch(y_pred[:, i - 1], y_true[:, i - 1]).item() 124 | elif metric == 'EVAR': 125 | line[metric] = explained_variance_score_torch(y_pred[:, i - 1], y_true[:, i - 1]).item() 126 | else: 127 | raise ValueError('Error parameter mode={}, please set `single` or `average`.'.format(mode)) 128 | elif mode.lower() == 'average': 129 | if metric == 'masked_MAE': 130 | line[metric] = masked_mae_torch(y_pred[:, :i], y_true[:, :i], 0).item() 131 | elif metric == 'masked_MSE': 132 | line[metric] = masked_mse_torch(y_pred[:, :i], y_true[:, :i], 0).item() 133 | elif metric == 'masked_RMSE': 134 | line[metric] = masked_rmse_torch(y_pred[:, :i], y_true[:, :i], 0).item() 135 | elif metric == 'masked_MAPE': 136 | line[metric] = masked_mape_torch(y_pred[:, :i], y_true[:, :i], 0).item() 137 | elif metric == 'MAE': 138 | line[metric] = masked_mae_torch(y_pred[:, :i], y_true[:, :i]).item() 139 | elif metric == 'MSE': 140 | line[metric] = masked_mse_torch(y_pred[:, :i], y_true[:, :i]).item() 141 | elif metric == 'RMSE': 142 | line[metric] = masked_rmse_torch(y_pred[:, :i], y_true[:, :i]).item() 143 | elif metric == 'MAPE': 144 | line[metric] = masked_mape_torch(y_pred[:, :i], y_true[:, :i]).item() 145 | elif metric == 'R2': 146 | line[metric] = r2_score_torch(y_pred[:, :i], y_true[:, :i]).item() 147 | elif metric == 'EVAR': 148 | line[metric] = explained_variance_score_torch(y_pred[:, :i], y_true[:, :i]).item() 149 | else: 150 | raise ValueError('Error parameter metric={}!'.format(metric)) 151 | else: 152 | raise ValueError('Error parameter evaluator_mode={}, please set `single` or `average`.'.format(mode)) 153 | df.append(line) 154 | df = pd.DataFrame(df, columns=metrics) 155 | print(df) 156 | df.to_csv(path) 157 | return df 158 | -------------------------------------------------------------------------------- /libcity/executor/scheduler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import math 3 | from logging import getLogger 4 | 5 | 6 | class Scheduler: 7 | 8 | def __init__(self, 9 | optimizer: torch.optim.Optimizer, 10 | param_group_field: str, 11 | noise_range_t=None, 12 | noise_type='normal', 13 | noise_pct=0.67, 14 | noise_std=1.0, 15 | noise_seed=None, 16 | initialize: bool = True): 17 | self.optimizer = optimizer 18 | self.param_group_field = param_group_field 19 | self._initial_param_group_field = f"initial_{param_group_field}" 20 | if initialize: 21 | for i, group in enumerate(self.optimizer.param_groups): 22 | if param_group_field not in group: 23 | raise KeyError(f"{param_group_field} missing from param_groups[{i}]") 24 | group.setdefault(self._initial_param_group_field, group[param_group_field]) 25 | else: 26 | for i, group in enumerate(self.optimizer.param_groups): 27 | if self._initial_param_group_field not in group: 28 | raise KeyError(f"{self._initial_param_group_field} missing from param_groups[{i}]") 29 | self.base_values = [group[self._initial_param_group_field] for group in self.optimizer.param_groups] 30 | self.metric = None 31 | self.noise_range_t = noise_range_t 32 | self.noise_pct = noise_pct 33 | self.noise_type = noise_type 34 | self.noise_std = noise_std 35 | self.noise_seed = noise_seed if noise_seed is not None else 42 36 | self.update_groups(self.base_values) 37 | 38 | def state_dict(self): 39 | return {key: value for key, value in self.__dict__.items() if key != 'optimizer'} 40 | 41 | def load_state_dict(self, state_dict): 42 | self.__dict__.update(state_dict) 43 | 44 | def get_epoch_values(self, epoch): 45 | return None 46 | 47 | def get_update_values(self, num_updates): 48 | return None 49 | 50 | def step(self, epoch, metric=None): 51 | self.metric = metric 52 | values = self.get_epoch_values(epoch) 53 | if values is not None: 54 | values = self._add_noise(values, epoch) 55 | self.update_groups(values) 56 | 57 | def step_update(self, num_updates, metric=None): 58 | self.metric = metric 59 | values = self.get_update_values(num_updates) 60 | if values is not None: 61 | values = self._add_noise(values, num_updates) 62 | self.update_groups(values) 63 | 64 | def update_groups(self, values): 65 | if not isinstance(values, (list, tuple)): 66 | values = [values] * len(self.optimizer.param_groups) 67 | for param_group, value in zip(self.optimizer.param_groups, values): 68 | param_group[self.param_group_field] = value 69 | 70 | def _add_noise(self, lrs, t): 71 | if self.noise_range_t is not None: 72 | if isinstance(self.noise_range_t, (list, tuple)): 73 | apply_noise = self.noise_range_t[0] <= t < self.noise_range_t[1] 74 | else: 75 | apply_noise = t >= self.noise_range_t 76 | if apply_noise: 77 | g = torch.Generator() 78 | g.manual_seed(self.noise_seed + t) 79 | if self.noise_type == 'normal': 80 | while True: 81 | noise = torch.randn(1, generator=g).item() 82 | if abs(noise) < self.noise_pct: 83 | break 84 | else: 85 | noise = 2 * (torch.rand(1, generator=g).item() - 0.5) * self.noise_pct 86 | lrs = [v + v * noise for v in lrs] 87 | return lrs 88 | 89 | 90 | class CosineLRScheduler(Scheduler): 91 | 92 | def __init__(self, 93 | optimizer: torch.optim.Optimizer, 94 | t_initial: int, 95 | t_mul: float = 1., 96 | lr_min: float = 0., 97 | decay_rate: float = 1., 98 | warmup_t=0, 99 | warmup_lr_init=0, 100 | warmup_prefix=False, 101 | cycle_limit=0, 102 | t_in_epochs=True, 103 | noise_range_t=None, 104 | noise_pct=0.67, 105 | noise_std=1.0, 106 | noise_seed=42, 107 | initialize=True) -> None: 108 | super().__init__( 109 | optimizer, param_group_field="lr", 110 | noise_range_t=noise_range_t, noise_pct=noise_pct, noise_std=noise_std, noise_seed=noise_seed, 111 | initialize=initialize) 112 | 113 | assert t_initial > 0 114 | assert lr_min >= 0 115 | self._logger = getLogger() 116 | if t_initial == 1 and t_mul == 1 and decay_rate == 1: 117 | self._logger.warning("Cosine annealing scheduler will have no effect on the learning " 118 | "rate since t_initial = t_mul = eta_mul = 1.") 119 | self.t_initial = t_initial 120 | self.t_mul = t_mul 121 | self.lr_min = lr_min 122 | self.decay_rate = decay_rate 123 | self.cycle_limit = cycle_limit 124 | self.warmup_t = warmup_t 125 | self.warmup_lr_init = warmup_lr_init 126 | self.warmup_prefix = warmup_prefix 127 | self.t_in_epochs = t_in_epochs 128 | if self.warmup_t: 129 | self.warmup_steps = [(v - warmup_lr_init) / self.warmup_t for v in self.base_values] 130 | super().update_groups(self.warmup_lr_init) 131 | else: 132 | self.warmup_steps = [1 for _ in self.base_values] 133 | 134 | def _get_lr(self, t): 135 | if t < self.warmup_t: 136 | lrs = [self.warmup_lr_init + t * s for s in self.warmup_steps] 137 | else: 138 | if self.warmup_prefix: 139 | t = t - self.warmup_t 140 | 141 | if self.t_mul != 1: 142 | i = math.floor(math.log(1 - t / self.t_initial * (1 - self.t_mul), self.t_mul)) 143 | t_i = self.t_mul ** i * self.t_initial 144 | t_curr = t - (1 - self.t_mul ** i) / (1 - self.t_mul) * self.t_initial 145 | else: 146 | i = t // self.t_initial 147 | t_i = self.t_initial 148 | t_curr = t - (self.t_initial * i) 149 | 150 | gamma = self.decay_rate ** i 151 | lr_min = self.lr_min * gamma 152 | lr_max_values = [v * gamma for v in self.base_values] 153 | 154 | if self.cycle_limit == 0 or (self.cycle_limit > 0 and i < self.cycle_limit): 155 | lrs = [ 156 | lr_min + 0.5 * (lr_max - lr_min) * (1 + math.cos(math.pi * t_curr / t_i)) for lr_max in lr_max_values 157 | ] 158 | else: 159 | lrs = [self.lr_min for _ in self.base_values] 160 | 161 | return lrs 162 | 163 | def get_epoch_values(self, epoch: int): 164 | if self.t_in_epochs: 165 | return self._get_lr(epoch) 166 | else: 167 | return None 168 | 169 | def get_update_values(self, num_updates: int): 170 | if not self.t_in_epochs: 171 | return self._get_lr(num_updates) 172 | else: 173 | return None 174 | 175 | def get_cycle_length(self, cycles=0): 176 | if not cycles: 177 | cycles = self.cycle_limit 178 | cycles = max(1, cycles) 179 | if self.t_mul == 1.0: 180 | return self.t_initial * cycles 181 | else: 182 | return int(math.floor(-self.t_initial * (self.t_mul ** cycles - 1) / (1 - self.t_mul))) 183 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/silearn/backends/functional.py: -------------------------------------------------------------------------------- 1 | from inspect import isfunction 2 | 3 | from silearn.graph import Graph 4 | 5 | # (f, backend) --> f 6 | __function_map__ = dict() 7 | 8 | 9 | def get_dat_backend(dat): 10 | if isinstance(dat, torch.Tensor): 11 | return "torch" 12 | 13 | def vertex_reduce(edges, partition, edges_fea, node_fea): 14 | backend = get_dat_backend(edges) 15 | if not __function_map__.__contains__((vertex_reduce, backend)): 16 | raise NotImplementedError(f"vertex_reduce() is not available for backend {backend}") 17 | return __function_map__[vertex_reduce, backend](edges, partition, edges_fea, node_fea) 18 | 19 | 20 | 21 | def scatter_sum(src, idx, clip_length=0): 22 | backend = get_dat_backend(src) 23 | if not __function_map__.__contains__((scatter_sum, backend)): 24 | raise NotImplementedError(f"scatter_sum() is not available for backend {backend}") 25 | return __function_map__[scatter_sum, backend](src, idx, clip_length) 26 | 27 | 28 | def scatter_cnt(idx, clip_length=0): 29 | backend = get_dat_backend(idx) 30 | if not __function_map__.__contains__((scatter_cnt, backend)): 31 | raise NotImplementedError(f"scatter_cnt() is not available for backend {backend}") 32 | return __function_map__[scatter_cnt, backend](idx, clip_length) 33 | 34 | def scatter_max(src, idx): 35 | backend = get_dat_backend(src) 36 | if not __function_map__.__contains__((scatter_max, backend)): 37 | raise NotImplementedError(f"scatter_max() is not available for backend {backend}") 38 | return __function_map__[scatter_max, backend](src, idx) 39 | 40 | 41 | def sumup_duplicates(edges, *weights, operation_ptrs = None): 42 | backend = get_dat_backend(edges) 43 | if not __function_map__.__contains__((sumup_duplicates, backend)): 44 | raise NotImplementedError(f"sumup_duplicates() is not available for backend {backend}") 45 | return __function_map__[sumup_duplicates, backend](edges, *weights, operation_ptrs = operation_ptrs) 46 | 47 | 48 | 49 | def uncertainty(p): # log p 50 | backend = get_dat_backend(p) 51 | if not __function_map__.__contains__((uncertainty, backend)): 52 | raise NotImplementedError(f"uncertainty() is not available for backend {backend}") 53 | return __function_map__[uncertainty, backend](p) 54 | 55 | 56 | 57 | def entropy(p, q): # p log q 58 | backend = get_dat_backend(p) 59 | if not __function_map__.__contains__((entropy, backend)): 60 | raise NotImplementedError(f"entropy() is not available for backend {backend}") 61 | return __function_map__[entropy, backend](p, q) 62 | 63 | def log2(x): 64 | backend = get_dat_backend(x) 65 | if not __function_map__.__contains__((log2, backend)): 66 | raise NotImplementedError(f"log2() is not available for backend {backend}") 67 | return __function_map__[log2, backend](x) 68 | 69 | def log_e(x): 70 | backend = get_dat_backend(x) 71 | if not __function_map__.__contains__((log_e, backend)): 72 | raise NotImplementedError(f"log_e() is not available for backend {backend}") 73 | return __function_map__[log_e, backend](x) 74 | def logical_and(x): 75 | backend = get_dat_backend(x) 76 | if not __function_map__.__contains__((logical_and, backend)): 77 | raise NotImplementedError(f"logical_and() is not available for backend {backend}") 78 | return __function_map__[logical_and, backend](x) 79 | 80 | def logical_or(x): 81 | backend = get_dat_backend(x) 82 | if not __function_map__.__contains__((logical_or, backend)): 83 | raise NotImplementedError(f"logical_or() is not available for backend {backend}") 84 | return __function_map__[logical_or, backend](x) 85 | 86 | 87 | def logical_not(x): 88 | backend = get_dat_backend(x) 89 | if not __function_map__.__contains__((logical_not, backend)): 90 | raise NotImplementedError(f"logical_not() is not available for backend {backend}") 91 | return __function_map__[logical_not, backend](x) 92 | 93 | def nonzero(edges, return_weights = False): 94 | backend = get_dat_backend(edges) 95 | if not __function_map__.__contains__((nonzero, backend)): 96 | raise NotImplementedError(f"nonzero() is not available for backend {backend}") 97 | return __function_map__[nonzero, backend](edges, return_weights) 98 | 99 | def concat(list_of_tensors, dim = -1): 100 | backend = get_dat_backend(list_of_tensors[0]) 101 | if not __function_map__.__contains__((concat, backend)): 102 | raise NotImplementedError(f"concat() is not available for backend {backend}") 103 | return __function_map__[concat, backend](list_of_tensors, dim) 104 | 105 | 106 | def stack(list_of_tensors, dim = -1): 107 | backend = get_dat_backend(list_of_tensors[0]) 108 | if not __function_map__.__contains__((stack, backend)): 109 | raise NotImplementedError(f"stack() is not available for backend {backend}") 110 | return __function_map__[stack, backend](list_of_tensors, dim) 111 | 112 | def clone(t): 113 | backend = get_dat_backend(t) 114 | if not __function_map__.__contains__((clone, backend)): 115 | raise NotImplementedError(f"clone() is not available for backend {backend}") 116 | return __function_map__[clone, backend](t) 117 | 118 | 119 | 120 | # TODO 121 | def convert_backend(p, backend): 122 | """ 123 | @:param 124 | backends ∈ {"torch", "numpy"} 125 | """ 126 | backend_orig = get_dat_backend(p) 127 | if not __function_map__.__contains__((convert_backend, backend_orig)): 128 | raise NotImplementedError(f"convert_backend() is not available for original backend {backend_orig}") 129 | return __function_map__[convert_backend, backend_orig](p, backend) 130 | 131 | 132 | def full_coo_graph(N, device, backend): 133 | """ 134 | Get the edge vectors of full graph. 135 | 136 | @:param 137 | N: num of node. 138 | dev: torch device. 139 | @:returns es, et, w: torch tensor. Same as @get_sparse_conv 140 | """ 141 | if not __function_map__.__contains__((full_coo_graph, backend)): 142 | raise NotImplementedError(f"full_graph() is not available for backend {backend}") 143 | return __function_map__[full_coo_graph, backend](N, device, backend) 144 | 145 | def spatial_knn_graph(feature_map, k, r, metric = None): 146 | """ 147 | Build the Graph from Image by KNN. 148 | 149 | @:param 150 | img: 3 x H x W. 151 | k: k edges per node. 152 | r: horizontal and vertical distance bound for linking pair of pixels. 153 | 154 | @:returns (w, es, et) 155 | w: edge weights representing distances. 156 | es: edge start nodes. 157 | et: edge target nodes. 158 | """ 159 | backend = get_dat_backend(feature_map) 160 | if not __function_map__.__contains__((spatial_knn_graph, backend)): 161 | raise NotImplementedError(f"spatial_knn_graph() is not available for backend {backend}") 162 | return __function_map__[spatial_knn_graph, backend](feature_map, k, r, metric) 163 | 164 | 165 | 166 | 167 | 168 | # def convert_backend(g: Graph, backend): 169 | # if not __function_map__.__contains__((convert_backend, g.backend)): 170 | # raise NotImplementedError(f"convert_backend() is not available for backend {g.backend}") 171 | # return __function_map__[convert_backend, g.backend](g, backend) 172 | 173 | 174 | 175 | # noinspection PyUnresolvedReferences 176 | 177 | __all__ = ["vertex_reduce", 178 | "convert_backend", 179 | "scatter_sum", 180 | "scatter_cnt", 181 | "scatter_max", 182 | "entropy", 183 | "uncertainty", 184 | "logical_and", 185 | "logical_or", 186 | "logical_not", 187 | "log2", 188 | "log_e", 189 | "concat", 190 | "stack", 191 | "clone", 192 | "convert_backend", 193 | "get_dat_backend", 194 | 195 | "sumup_duplicates", 196 | "full_coo_graph", 197 | "spatial_knn_graph"] 198 | 199 | # from .functional import * 200 | 201 | def __include_functions__(lib, name): 202 | for k, v in lib.__dict__.items(): 203 | try: 204 | if isfunction(v) and isfunction(eval(k)): 205 | __function_map__[eval(k), name] = v 206 | except: 207 | pass 208 | 209 | 210 | try: 211 | import torch 212 | import silearn.backends.torch_ops as torch_ops 213 | __include_functions__(torch_ops, "torch") 214 | 215 | import silearn.backends.scipy_ops as scipy_ops 216 | __include_functions__(scipy_ops, "numpy") 217 | finally: 218 | pass 219 | 220 | -------------------------------------------------------------------------------- /libcity/utils/utils.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import logging 3 | import datetime 4 | import os 5 | import sys 6 | import numpy as np 7 | import random 8 | import torch 9 | 10 | 11 | def get_executor(config, model, data_feature): 12 | """ 13 | according the config['executor'] to create the executor 14 | 15 | Args: 16 | config(ConfigParser): config 17 | model(AbstractModel): model 18 | 19 | Returns: 20 | AbstractExecutor: the loaded executor 21 | """ 22 | try: 23 | return getattr(importlib.import_module('libcity.executor'), 24 | config['executor'])(config, model, data_feature) 25 | except AttributeError: 26 | raise AttributeError('executor is not found') 27 | 28 | 29 | def get_model(config, data_feature): 30 | """ 31 | according the config['model'] to create the model 32 | 33 | Args: 34 | config(ConfigParser): config 35 | data_feature(dict): feature of the data 36 | 37 | Returns: 38 | AbstractModel: the loaded model 39 | """ 40 | if config['task'] == 'traj_loc_pred': 41 | try: 42 | return getattr(importlib.import_module('libcity.model.trajectory_loc_prediction'), 43 | config['model'])(config, data_feature) 44 | except AttributeError: 45 | raise AttributeError('model is not found') 46 | elif config['task'] == 'traffic_state_pred': 47 | try: 48 | return getattr(importlib.import_module('libcity.model.traffic_flow_prediction'), 49 | config['model'])(config, data_feature) 50 | except AttributeError: 51 | try: 52 | return getattr(importlib.import_module('libcity.model.traffic_speed_prediction'), 53 | config['model'])(config, data_feature) 54 | except AttributeError: 55 | try: 56 | return getattr(importlib.import_module('libcity.model.traffic_demand_prediction'), 57 | config['model'])(config, data_feature) 58 | except AttributeError: 59 | try: 60 | return getattr(importlib.import_module('libcity.model.traffic_od_prediction'), 61 | config['model'])(config, data_feature) 62 | except AttributeError: 63 | try: 64 | return getattr(importlib.import_module('libcity.model.traffic_accident_prediction'), 65 | config['model'])(config, data_feature) 66 | except AttributeError: 67 | raise AttributeError('model is not found') 68 | elif config['task'] == 'map_matching': 69 | try: 70 | return getattr(importlib.import_module('libcity.model.map_matching'), 71 | config['model'])(config, data_feature) 72 | except AttributeError: 73 | raise AttributeError('model is not found') 74 | elif config['task'] == 'road_representation': 75 | try: 76 | return getattr(importlib.import_module('libcity.model.road_representation'), 77 | config['model'])(config, data_feature) 78 | except AttributeError: 79 | raise AttributeError('model is not found') 80 | elif config['task'] == 'eta': 81 | try: 82 | return getattr(importlib.import_module('libcity.model.eta'), 83 | config['model'])(config, data_feature) 84 | except AttributeError: 85 | raise AttributeError('model is not found') 86 | else: 87 | raise AttributeError('task is not found') 88 | 89 | 90 | def get_evaluator(config): 91 | """ 92 | according the config['evaluator'] to create the evaluator 93 | 94 | Args: 95 | config(ConfigParser): config 96 | 97 | Returns: 98 | AbstractEvaluator: the loaded evaluator 99 | """ 100 | try: 101 | return getattr(importlib.import_module('libcity.evaluator'), 102 | config['evaluator'])(config) 103 | except AttributeError: 104 | raise AttributeError('evaluator is not found') 105 | 106 | 107 | def get_logger(config, name=None): 108 | """ 109 | 获取Logger对象 110 | 111 | Args: 112 | config(ConfigParser): config 113 | name: specified name 114 | 115 | Returns: 116 | Logger: logger 117 | """ 118 | log_dir = './libcity/log' 119 | if not os.path.exists(log_dir): 120 | os.makedirs(log_dir) 121 | log_filename = '{}-{}-{}-{}.log'.format(config['exp_id'], 122 | config['model'], config['dataset'], get_local_time()) 123 | logfilepath = os.path.join(log_dir, log_filename) 124 | 125 | logger = logging.getLogger(name) 126 | 127 | log_level = config.get('log_level', 'INFO') 128 | 129 | if log_level.lower() == 'info': 130 | level = logging.INFO 131 | elif log_level.lower() == 'debug': 132 | level = logging.DEBUG 133 | elif log_level.lower() == 'error': 134 | level = logging.ERROR 135 | elif log_level.lower() == 'warning': 136 | level = logging.WARNING 137 | elif log_level.lower() == 'critical': 138 | level = logging.CRITICAL 139 | else: 140 | level = logging.INFO 141 | 142 | logger.setLevel(level) 143 | 144 | formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') 145 | file_handler = logging.FileHandler(logfilepath) 146 | file_handler.setFormatter(formatter) 147 | 148 | console_formatter = logging.Formatter( 149 | '%(asctime)s - %(levelname)s - %(message)s') 150 | console_handler = logging.StreamHandler(sys.stdout) 151 | console_handler.setFormatter(console_formatter) 152 | 153 | logger.addHandler(file_handler) 154 | logger.addHandler(console_handler) 155 | 156 | logger.info('Log directory: %s', log_dir) 157 | return logger 158 | 159 | 160 | def get_local_time(): 161 | """ 162 | 获取时间 163 | 164 | Return: 165 | datetime: 时间 166 | """ 167 | cur = datetime.datetime.now() 168 | cur = cur.strftime('%b-%d-%Y_%H-%M-%S') 169 | return cur 170 | 171 | 172 | def ensure_dir(dir_path): 173 | """Make sure the directory exists, if it does not exist, create it. 174 | 175 | Args: 176 | dir_path (str): directory path 177 | """ 178 | if not os.path.exists(dir_path): 179 | os.makedirs(dir_path) 180 | 181 | 182 | def trans_naming_rule(origin, origin_rule, target_rule): 183 | """ 184 | 名字转换规则 185 | 186 | Args: 187 | origin (str): 源命名格式下的变量名 188 | origin_rule (str): 源命名格式，枚举类 189 | target_rule (str): 目标命名格式，枚举类 190 | 191 | Return: 192 | target (str): 转换之后的结果 193 | """ 194 | # TODO: 请确保输入是符合 origin_rule，这里目前不做检查 195 | target = '' 196 | if origin_rule == 'upper_camel_case' and target_rule == 'under_score_rule': 197 | for i, c in enumerate(origin): 198 | if i == 0: 199 | target = c.lower() 200 | else: 201 | target += '_' + c.lower() if c.isupper() else c 202 | return target 203 | else: 204 | raise NotImplementedError( 205 | 'trans naming rule only support from upper_camel_case to \ 206 | under_score_rule') 207 | 208 | 209 | def preprocess_data(data, config): 210 | """ 211 | split by input_window and output_window 212 | 213 | Args: 214 | data: shape (T, ...) 215 | 216 | Returns: 217 | np.ndarray: (train_size/test_size, input_window, ...) 218 | (train_size/test_size, output_window, ...) 219 | 220 | """ 221 | train_rate = config.get('train_rate', 0.7) 222 | eval_rate = config.get('eval_rate', 0.1) 223 | 224 | input_window = config.get('input_window', 12) 225 | output_window = config.get('output_window', 3) 226 | 227 | x, y = [], [] 228 | for i in range(len(data) - input_window - output_window): 229 | a = data[i: i + input_window + output_window] # (in+out, ...) 230 | x.append(a[0: input_window]) # (in, ...) 231 | y.append(a[input_window: input_window + output_window]) # (out, ...) 232 | x = np.array(x) # (num_samples, in, ...) 233 | y = np.array(y) # (num_samples, out, ...) 234 | 235 | train_size = int(x.shape[0] * (train_rate + eval_rate)) 236 | trainx = x[:train_size] # (train_size, in, ...) 237 | trainy = y[:train_size] # (train_size, out, ...) 238 | testx = x[train_size:x.shape[0]] # (test_size, in, ...) 239 | testy = y[train_size:x.shape[0]] # (test_size, out, ...) 240 | return trainx, trainy, testx, testy 241 | 242 | 243 | def set_random_seed(seed): 244 | """ 245 | 重置随机数种子 246 | 247 | Args: 248 | seed(int): 种子数 249 | """ 250 | random.seed(seed) 251 | np.random.seed(seed) 252 | torch.manual_seed(seed) 253 | torch.cuda.manual_seed_all(seed) 254 | torch.backends.cudnn.deterministic = True 255 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/pe_layers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch import Tensor 4 | from torch_geometric.utils import to_dense_adj,dense_to_sparse,degree 5 | import numpy as np 6 | import scipy.sparse as sp 7 | import math 8 | 9 | # pos_encoding 10 | 11 | def SinCosPosEncoding(q_len, d_model, normalize=True): 12 | pe = torch.zeros(q_len, d_model) 13 | position = torch.arange(0, q_len).unsqueeze(1) 14 | div_term = torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model)) 15 | pe[:, 0::2] = torch.sin(position * div_term) 16 | pe[:, 1::2] = torch.cos(position * div_term) 17 | if normalize: 18 | pe = pe - pe.mean() 19 | pe = pe / (pe.std() * 10) 20 | return pe 21 | 22 | 23 | def Coord2dPosEncoding(q_len, d_model, exponential=False, normalize=True, eps=1e-3, verbose=False): 24 | x = .5 if exponential else 1 25 | i = 0 26 | for i in range(100): 27 | cpe = 2 * (torch.linspace(0, 1, q_len).reshape(-1, 1) ** x) * (torch.linspace(0, 1, d_model).reshape(1, -1) ** x) - 1 28 | print (f'{i:4.0f} {x:5.3f} {cpe.mean():+6.3f}', verbose) 29 | if abs(cpe.mean()) <= eps: break 30 | elif cpe.mean() > eps: x += .001 31 | else: x -= .001 32 | i += 1 33 | if normalize: 34 | cpe = cpe - cpe.mean() 35 | cpe = cpe / (cpe.std() * 10) 36 | return cpe 37 | 38 | def Coord1dPosEncoding(q_len, exponential=False, normalize=True): 39 | cpe = (2 * (torch.linspace(0, 1, q_len).reshape(-1, 1)**(.5 if exponential else 1)) - 1) 40 | if normalize: 41 | cpe = cpe - cpe.mean() 42 | cpe = cpe / (cpe.std() * 10) 43 | return cpe 44 | 45 | 46 | class Positional_Encoding(nn.Module): 47 | """ 48 | general positional encoding layer 49 | return [len,d_model] 50 | """ 51 | def __init__(self, pe_type, learn_pe, q_len, d_model, device=torch.device('cpu')): 52 | super(Positional_Encoding,self).__init__() 53 | # Positional encoding 54 | self.device = device 55 | self.pe_type = pe_type 56 | if pe_type == None: # random pe , for measuring impact of pe 57 | W_pos = torch.empty((q_len, d_model)) 58 | nn.init.uniform_(W_pos, -0.02, 0.02) 59 | learn_pe = False 60 | elif pe_type == 'zero': # 1 dim random pe 61 | W_pos = torch.empty((q_len, 1)) 62 | nn.init.uniform_(W_pos, -0.02, 0.02) 63 | elif pe_type == 'zeros': # n dim random pe 64 | W_pos = torch.empty((q_len, d_model)) 65 | nn.init.uniform_(W_pos, -0.02, 0.02) 66 | elif pe_type == 'normal' or pe_type == 'gauss': 67 | W_pos = torch.zeros((q_len, 1)) 68 | torch.nn.init.normal_(W_pos, mean=0.0, std=0.1) 69 | elif pe_type == 'uniform': 70 | W_pos = torch.zeros((q_len, 1)) 71 | nn.init.uniform_(W_pos, a=0.0, b=0.1) 72 | elif pe_type == 'lin1d': W_pos = Coord1dPosEncoding(q_len, exponential=False, normalize=True) 73 | elif pe_type == 'exp1d': W_pos = Coord1dPosEncoding(q_len, exponential=True, normalize=True) 74 | elif pe_type == 'lin2d': W_pos = Coord2dPosEncoding(q_len, d_model, exponential=False, normalize=True) 75 | elif pe_type == 'exp2d': W_pos = Coord2dPosEncoding(q_len, d_model, exponential=True, normalize=True) 76 | elif pe_type == 'sincos': W_pos = SinCosPosEncoding(q_len, d_model, normalize=True) 77 | elif self.__class__ is Positional_Encoding: 78 | raise ValueError(f"{pe_type} is not a valid pe (positional encoder. Available types: 'gauss'=='normal', \ 79 | 'zeros', 'zero', uniform', 'lin1d', 'exp1d', 'lin2d', 'exp2d', 'sincos', None.)") 80 | else: W_pos = None 81 | if W_pos is not None: 82 | self.W_pos = nn.Parameter(W_pos, requires_grad=learn_pe).to(self.device) 83 | def forward(self): 84 | return self.W_pos 85 | 86 | 87 | """ 88 | external encoding() 89 | """ 90 | class External_Encoding(nn.Module): 91 | ''' 92 | External encoding 93 | output [batch, _, t_seq, embed_dim] 94 | ''' 95 | def __init__(self, d_model,device): 96 | super().__init__() 97 | self.day_embedding = nn.Embedding(7 ,64) 98 | self.time_embedding = nn.Embedding(24*12, 64) 99 | 100 | def forward(self,x:Tensor): 101 | ''' 102 | Args: 103 | x: [b, #node, #len, 11] 104 | Output: 105 | x: [b, #node, #len, 3] 106 | ext: [b, #node, #len, 64] 107 | ''' 108 | day_info = torch.argmax(x[...,-7:], dim=-1) 109 | time_info = (x[...,-8:-7] * 288).int().squeeze(-1) 110 | x = x[...,:-8] 111 | # day_ebd = self.day_embedding(day_info) 112 | time_ebd = self.time_embedding(time_info) 113 | return x, time_ebd 114 | 115 | 116 | class S_Positional_Encoding(Positional_Encoding): 117 | def __init__(self, pe_type, learn_pe, node_num, d_model, dim_red_rate=0.5, device=torch.device('cpu')): 118 | super(S_Positional_Encoding,self).__init__(pe_type, learn_pe, node_num, d_model, device) 119 | self.pe_type = pe_type 120 | if pe_type == 'laplacian': 121 | self.pe_encoder = LaplacianPE(round(node_num*dim_red_rate),d_model,device = self.device) 122 | elif pe_type == 'centrality': 123 | self.pe_encoder = CentralityPE(node_num,d_model) 124 | else : raise ValueError(f"{pe_type} is not a valid spatial pe (positional encoder. Available types: 'laplacian','centrality','gauss'=='normal', \ 125 | 'zeros', 'zero', uniform', 'lin1d', 'exp1d', 'lin2d', 'exp2d', 'sincos', None.)") 126 | 127 | def forward(self,adj_mx=None): 128 | if self.pe_type == 'laplacian': return self.pe_encoder(adj_mx) 129 | elif self.pe_type == 'centrality': return self.pe_encoder(adj_mx) 130 | else: return self.W_pos.to(self.device) 131 | 132 | 133 | 134 | class LaplacianPE(nn.Module): # from [Dwivedi and Bresson, 2020] code from PDformer 135 | def __init__(self, lape_dim, embed_dim, learn_pe=False,device=torch.device('cpu')): 136 | super().__init__() 137 | self.device = device 138 | self.lape_dim = lape_dim 139 | self.learn_pe = learn_pe 140 | self.embedding_lap_pos_enc = nn.Linear(lape_dim, embed_dim) 141 | 142 | def _calculate_normalized_laplacian(self, adj): 143 | adj = sp.coo_matrix(adj) 144 | d = np.array(adj.sum(1)) 145 | isolated_point_num = np.sum(np.where(d, 0, 1)) 146 | d_inv_sqrt = np.power(d, -0.5).flatten() 147 | d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0. 148 | d_mat_inv_sqrt = sp.diags(d_inv_sqrt) 149 | normalized_laplacian = sp.eye(adj.shape[0]) - adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo() 150 | return normalized_laplacian, isolated_point_num 151 | 152 | def _calculate_random_walk_laplacian(self, adj): 153 | adj = sp.coo_matrix(adj) 154 | d = np.array(adj.sum(1)) 155 | isolated_point_num = np.sum(np.where(d, 0, 1)) 156 | d_inv = np.power(d, -1).flatten() 157 | d_inv[np.isinf(d_inv)] = 0. 158 | d_mat_inv = sp.diags(d_inv) 159 | random_walk_mx = sp.eye(adj.shape[0]) - d_mat_inv.dot(adj).tocoo() 160 | return random_walk_mx, isolated_point_num 161 | 162 | def _cal_lape(self, dense_adj_mx): 163 | L, isolated_point_num = self._calculate_normalized_laplacian(dense_adj_mx) 164 | EigVal, EigVec = np.linalg.eig(L.toarray()) 165 | idx = EigVal.argsort() 166 | EigVal, EigVec = EigVal[idx], np.real(EigVec[:, idx]) 167 | 168 | laplacian_pe:Tensor = torch.from_numpy( 169 | EigVec[:, isolated_point_num + 1: self.lape_dim + isolated_point_num + 1] 170 | ).float().to(self.device) 171 | laplacian_pe.require_grad = self.learn_pe 172 | return laplacian_pe 173 | 174 | def forward(self, adj_mx): 175 | lap_mx = self._cal_lape(adj_mx) 176 | lap_pos_enc = self.embedding_lap_pos_enc(lap_mx) 177 | return lap_pos_enc 178 | 179 | 180 | class WLPE(nn.Module): #from graph-bert 181 | def __init__(self, n_dim, embed_dim): 182 | super().__init__() 183 | raise NotImplementedError 184 | def forward(self,x): 185 | raise NotImplementedError 186 | 187 | 188 | 189 | class CentralityPE(nn.Module): # from Graphormer 190 | """ 191 | for link (unweight) graph 192 | """ 193 | def __init__(self, num_node, embed_dim, device=torch.device('cpu'),): 194 | super().__init__() 195 | self.device = device 196 | self.max_in_degree = num_node+1 197 | self.max_out_degree = num_node+1 198 | self.in_degree_encoder = nn.Embedding(self.max_in_degree, embed_dim, padding_idx=0) 199 | self.out_degree_encoder = nn.Embedding(self.max_out_degree, embed_dim, padding_idx=0) 200 | 201 | def forward(self, dense_adj_mx): 202 | (edge_index,_) = dense_to_sparse(torch.from_numpy(dense_adj_mx)) 203 | outdegree = degree(edge_index[0]).to(self.device) 204 | indegree = degree(edge_index[1]).to(self.device) 205 | cen_pos_en = self.in_degree_encoder(indegree.long())+self.out_degree_encoder(outdegree.long()) 206 | return cen_pos_en 207 | 208 | if __name__ == '__main__': 209 | adj_mx = np.array([[1.,0.,2.,3.],[0.,1.,2.,1.],[1.,1.,1.,2.],[0.,1.,2.,1.]],dtype=np.float16) 210 | print(adj_mx) 211 | pe_encoder = CentralityPE(4,2) 212 | lap_pe = pe_encoder.forward(adj_mx) 213 | print(lap_pe) -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/layers/mask_layers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch import Tensor 4 | import numpy as np 5 | import scipy.sparse as sp 6 | from infomap import Infomap 7 | import networkx as nx 8 | import math 9 | from torch_geometric.utils import to_dense_adj 10 | 11 | class Mask_Bias_Generator(): 12 | """ 13 | mask_heads_share : [True: shape (q,k,h) 14 | False: shape (q,k) or (t/n,q,k)] 15 | 16 | """ 17 | def __init__(self, q_size, v_size): 18 | self.q_size = q_size 19 | self.v_size = v_size 20 | self._bias = None 21 | self._mask = None 22 | 23 | def get(self): 24 | return self._bias or self._mask 25 | 26 | 27 | 28 | 29 | class Graph_Mask_Generator(Mask_Bias_Generator): 30 | """ 31 | mask_heads_share: True, single graph, False, Multi-relation graph 32 | graph 33 | """ 34 | def __init__(self, num_node, graph_data): 35 | super(Graph_Mask_Generator,self).__init__(num_node, num_node) 36 | self.num_node = num_node 37 | if len(graph_data) == len(graph_data[0]) and type(graph_data)==np.ndarray: # dense 38 | dense_adj_mx = graph_data 39 | else: # edge index 40 | dense_adj_mx = to_dense_adj(edge_index = graph_data, max_num_nodes=self.num_node) 41 | assert ( 42 | dense_adj_mx.shape[0]==self.q_size and dense_adj_mx.shape[1]==self.v_size 43 | ), "Wrong adj matrix" 44 | dense_adj_mx = torch.from_numpy(graph_data) 45 | out0 = dense_adj_mx>0 46 | out1 = torch.where(dense_adj_mx==torch.inf, 47 | torch.tensor([False,]).expand(dense_adj_mx.shape), 48 | torch.tensor([True,]).expand(dense_adj_mx.shape)) 49 | self._mask = out0 * out1 50 | 51 | 52 | 53 | 54 | 55 | class Infomap_Multi_Mask_Generator(Mask_Bias_Generator): 56 | # get masks shaped as [q x k x h] 57 | def __init__(self, num_node, graph_data): 58 | super(Infomap_Multi_Mask_Generator,self).__init__(num_node, num_node) 59 | self.im = Infomap(silent=True,num_trials=20) 60 | self.num_node = num_node 61 | if type(graph_data) is (nx.DiGraph or nx.Graph): 62 | self.G = graph_data 63 | else: 64 | self.G = nx.DiGraph(graph_data) # dense_adj_mx 65 | self.im.add_networkx_graph(self.G) 66 | self._gen_mask() 67 | 68 | def _gen_mask(self): 69 | self.im.run() 70 | im = self.im 71 | num_levels = im.num_levels 72 | max_num_module = im.num_leaf_modules 73 | self.num_mask = num_levels-1 74 | masks = list() 75 | for each_level in range(1,num_levels): 76 | itr = im.get_nodes(depth_level=each_level) 77 | # clu_tag = torch.zeros([self.num_node,],dtype=torch.int) 78 | # clu_tags = torch.zeros([max_num_module,self.num_node],dtype=torch.int) 79 | clu_tags = torch.full([max_num_module,self.num_node],-1,dtype=torch.int) 80 | # ind = torch.zeros([self.num_node,],dtype=torch.int) 81 | for each in itr: 82 | # 一个行复制，一个列复制，用两个矩阵where相等为True,overlap部分无法处理(mend) 83 | clu_tags[each.module_id-1][each.node_id] = 1 84 | temp1 = clu_tags.unsqueeze(2).expand([max_num_module,self.num_node,self.num_node]) 85 | temp2 = temp1.transpose(1,2) 86 | out = torch.any((temp1==temp2)*(temp1!=-1),dim=0) 87 | masks.append(out) 88 | masks = torch.stack(masks,dim=2) 89 | self._mask = masks 90 | 91 | 92 | 93 | 94 | 95 | class Infomap_Multilevel_Bias_Generator(Mask_Bias_Generator): 96 | """ 97 | Input: static graph_data (nx.Digraph or dense_adj_mx) 98 | Output: bias , shape=(node_num,node_num), dtype=float 99 | """ 100 | def __init__(self, num_node, graph_data, bias_scale_type=0): 101 | super(Infomap_Multilevel_Bias_Generator,self).__init__(True, num_node, num_node) 102 | self.im = Infomap(silent=True,num_trials=20) 103 | self.bias_scale_type = bias_scale_type 104 | # only for static graph 105 | if type(graph_data) is (nx.DiGraph or nx.Graph): 106 | self.G = graph_data 107 | else: 108 | self.G = nx.DiGraph(graph_data) # dense_adj_mx 109 | self.im.add_networkx_graph(self.G) 110 | self._gen_bias() 111 | 112 | def _gen_bias(self): 113 | self.im.run() 114 | # read tree 115 | bias_type = self.bias_scale_type 116 | im = self.im 117 | num_nodes=im.num_nodes 118 | itr = im.get_tree(depth_level=1, states=True) 119 | path_modcentral_dict=dict() 120 | path_nodeid_dict = dict() 121 | for each in itr: 122 | path_modcentral_dict[each.path] = each.modular_centrality 123 | if each.is_leaf: 124 | path_nodeid_dict[each.path] = each.node_id 125 | 126 | single_layer_att_bias = torch.zeros([num_nodes,num_nodes],dtype=torch.float64) 127 | 128 | nodes = im.get_nodes(depth_level=1, states=True) 129 | for each in nodes: 130 | path = each.path 131 | nd_from = torch.Tensor([path_nodeid_dict[path],]).type(torch.long) 132 | for i in range(len(path),0,-1): # 0 for min attention layer 133 | now_path = path[:i] 134 | common_prefix = now_path[:-1] 135 | ## bias type 136 | if bias_type == 0: 137 | b1 = path_modcentral_dict[now_path] 138 | elif bias_type == 1: 139 | b1 = math.exp(path_modcentral_dict[now_path]) 140 | else: 141 | b1 = 1 142 | 143 | nd_to = [] 144 | for key in path_nodeid_dict.keys(): 145 | if key[:len(common_prefix)]==common_prefix: # key!=now_path 146 | nd_to.append(path_nodeid_dict[key]) 147 | nd_to = torch.Tensor(nd_to).type(torch.long) 148 | single_layer_att_bias[nd_from,nd_to]+=b1 149 | single_layer_att_bias[nd_to,nd_from]+=b1 150 | self._bias = single_layer_att_bias/2 151 | 152 | def get_static_multihead_mask(num_head,mask_generator_list:list,device=torch.device('cpu')): 153 | all_mask = list() 154 | for each_mg in mask_generator_list: 155 | temp_mask:Tensor = each_mg.get() 156 | if len(temp_mask.shape) == 2: 157 | temp_mask = temp_mask.unsqueeze(dim=-1) 158 | assert ( 159 | len(temp_mask.shape) == 3 160 | ), "Unaccpetable static multihead mask" 161 | all_mask.append(temp_mask) 162 | all_mask = torch.cat(all_mask,dim=-1) 163 | assert ( 164 | all_mask.shape[2] < num_head 165 | ), "Not enough multihead num" 166 | all_true_mask = torch.full( 167 | [all_mask.shape[0],all_mask.shape[1],num_head-all_mask.shape[2]],True) 168 | all_mask=torch.cat([all_mask,all_true_mask],dim=-1).contiguous().to(device) 169 | return all_mask 170 | 171 | if __name__ == '__main__': 172 | 173 | 174 | # from torch_geometric_temporal.dataset import METRLADatasetLoader 175 | # loader = METRLADatasetLoader() 176 | # dataset = loader.get_dataset(num_timesteps_in=12, num_timesteps_out=12) 177 | # for snapshot in dataset: 178 | # edge_index = snapshot.edge_index 179 | # edge_weight = snapshot.edge_weight 180 | # num_nodes = snapshot.num_nodes 181 | # break 182 | # dense_adj_mx = to_dense_adj(edge_index=edge_index, edge_attr=edge_weight) 183 | # dense_adj_mx = dense_adj_mx.squeeze().numpy() 184 | # I = Infomap_Multi_Mask_Generator(num_node=num_nodes, graph_data=dense_adj_mx) 185 | 186 | # dense_adj_mx = np.array([[1.,0.,2.,-3.],[0.,1.,2.,-1.],[1.,1.,1.,-2.],[0.,1.,1,-1.]],dtype=np.float16) 187 | # temp1 = torch.from_numpy(dense_adj_mx) 188 | # temp2 = temp1.T 189 | # out = torch.where(temp1==temp2, 190 | # torch.tensor([True,]).expand([4,4]), 191 | # torch.tensor([False,]).expand([4,4])) 192 | # print(out) 193 | 194 | 195 | # dense_adj_mx = np.array([[1.,0.,2.,-3.],[0.,1.,2.,-1.],[1.,1.,1.,-2.],[0.,1.,torch.inf,-1.]],dtype=np.float16) 196 | # # out = dense_adj_mx>0 and dense_adj_mx is not torch.inf 197 | # dense_adj_mx = torch.from_numpy(dense_adj_mx) 198 | # out0 = dense_adj_mx>0 199 | # out1 = torch.where(dense_adj_mx==torch.inf, 200 | # torch.full_like(dense_adj_mx,False,dtype=torch.bool), 201 | # torch.full_like(dense_adj_mx,True,dtype=torch.bool)) 202 | # print(out0 * out1) 203 | 204 | 205 | 206 | itr = [ 207 | [0,1], 208 | [1,1], 209 | [1,2], 210 | [2,2] 211 | ] 212 | max_num_module = 2 213 | num_node = 3 214 | clu_tags = torch.full([max_num_module,num_node],-1,dtype=torch.int) 215 | for each in itr: 216 | # 一个行复制，一个列复制，用两个矩阵where相等为True,overlap部分无法处理(mend) 217 | clu_tags[each[1]-1][each[0]] = 1 218 | temp1 = clu_tags.unsqueeze(2).expand([max_num_module,num_node,num_node]) 219 | temp2 = temp1.transpose(1,2) 220 | out = torch.any((temp1==temp2)*(temp1!=-1),dim=0) 221 | print(out) 222 | 223 | # dense_adj_mx = np.array([[1.,1.,1.,0.],[1.,1.,1.,0],[1.,1.,1.,0.],[0.,0.,0.,1.]],dtype=np.float16) 224 | # mg1 = Infomap_Multi_Mask_Generator(num_node=4,graph_data=dense_adj_mx) 225 | # print(mg1.get(),mg1.get().shape) 226 | # mg2 = Graph_Mask_Generator(num_node=4,graph_data=dense_adj_mx) 227 | # print(mg2.get(),mg2.get().shape) 228 | # mask = get_static_multihead_mask(num_head=8,mask_generator_list=[mg1,mg2]) 229 | # print(mask.shape) -------------------------------------------------------------------------------- /libcity/evaluator/traffic_state_evaluator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import datetime 4 | import pandas as pd 5 | from libcity.utils import ensure_dir 6 | from libcity.model import loss 7 | from logging import getLogger 8 | from libcity.evaluator.abstract_evaluator import AbstractEvaluator 9 | 10 | 11 | class TrafficStateEvaluator(AbstractEvaluator): 12 | 13 | def __init__(self, config): 14 | self.metrics = config.get('metrics', ['MAE']) # 评估指标, 是一个 list 15 | self.allowed_metrics = ['MAE', 'MSE', 'RMSE', 'MAPE', 'masked_MAE', 16 | 'masked_MSE', 'masked_RMSE', 'masked_MAPE', 'R2', 'EVAR'] 17 | self.save_modes = config.get('save_mode', ['csv', 'json']) 18 | self.mode = config.get('evaluator_mode', 'single') # or average 19 | self.config = config 20 | self.len_timeslots = 0 21 | self.result = {} # 每一种指标的结果 22 | self.intermediate_result = {} # 每一种指标每一个batch的结果 23 | self._check_config() 24 | self._logger = getLogger() 25 | 26 | def _check_config(self): 27 | if not isinstance(self.metrics, list): 28 | raise TypeError('Evaluator type is not list') 29 | for metric in self.metrics: 30 | if metric not in self.allowed_metrics: 31 | raise ValueError('the metric {} is not allowed in TrafficStateEvaluator'.format(str(metric))) 32 | 33 | def collect(self, batch): 34 | """ 35 | 收集一 batch 的评估输入 36 | 37 | Args: 38 | batch(dict): 输入数据，字典类型，包含两个Key:(y_true, y_pred): 39 | batch['y_true']: (num_samples/batch_size, timeslots, ..., feature_dim) 40 | batch['y_pred']: (num_samples/batch_size, timeslots, ..., feature_dim) 41 | """ 42 | if not isinstance(batch, dict): 43 | raise TypeError('evaluator.collect input is not a dict of user') 44 | y_true = batch['y_true'] # tensor 45 | y_pred = batch['y_pred'] # tensor 46 | if y_true.shape != y_pred.shape: 47 | raise ValueError("batch['y_true'].shape is not equal to batch['y_pred'].shape") 48 | self.len_timeslots = y_true.shape[1] 49 | for i in range(1, self.len_timeslots+1): 50 | for metric in self.metrics: 51 | if metric+'@'+str(i) not in self.intermediate_result: 52 | self.intermediate_result[metric+'@'+str(i)] = [] 53 | if self.mode.lower() == 'average': # 前i个时间步的平均loss 54 | for i in range(1, self.len_timeslots+1): 55 | for metric in self.metrics: 56 | if metric == 'masked_MAE': 57 | self.intermediate_result[metric + '@' + str(i)].append( 58 | loss.masked_mae_torch(y_pred[:, :i], y_true[:, :i], 0).item()) 59 | elif metric == 'masked_MSE': 60 | self.intermediate_result[metric + '@' + str(i)].append( 61 | loss.masked_mse_torch(y_pred[:, :i], y_true[:, :i], 0).item()) 62 | elif metric == 'masked_RMSE': 63 | self.intermediate_result[metric + '@' + str(i)].append( 64 | loss.masked_rmse_torch(y_pred[:, :i], y_true[:, :i], 0).item()) 65 | elif metric == 'masked_MAPE': 66 | self.intermediate_result[metric + '@' + str(i)].append( 67 | loss.masked_mape_torch(y_pred[:, :i], y_true[:, :i], 0).item()) 68 | elif metric == 'MAE': 69 | self.intermediate_result[metric + '@' + str(i)].append( 70 | loss.masked_mae_torch(y_pred[:, :i], y_true[:, :i]).item()) 71 | elif metric == 'MSE': 72 | self.intermediate_result[metric + '@' + str(i)].append( 73 | loss.masked_mse_torch(y_pred[:, :i], y_true[:, :i]).item()) 74 | elif metric == 'RMSE': 75 | self.intermediate_result[metric + '@' + str(i)].append( 76 | loss.masked_rmse_torch(y_pred[:, :i], y_true[:, :i]).item()) 77 | elif metric == 'MAPE': 78 | self.intermediate_result[metric + '@' + str(i)].append( 79 | loss.masked_mape_torch(y_pred[:, :i], y_true[:, :i]).item()) 80 | elif metric == 'R2': 81 | self.intermediate_result[metric + '@' + str(i)].append( 82 | loss.r2_score_torch(y_pred[:, :i], y_true[:, :i]).item()) 83 | elif metric == 'EVAR': 84 | self.intermediate_result[metric + '@' + str(i)].append( 85 | loss.explained_variance_score_torch(y_pred[:, :i], y_true[:, :i]).item()) 86 | elif self.mode.lower() == 'single': # 第i个时间步的loss 87 | for i in range(1, self.len_timeslots + 1): 88 | for metric in self.metrics: 89 | if metric == 'masked_MAE': 90 | self.intermediate_result[metric + '@' + str(i)].append( 91 | loss.masked_mae_torch(y_pred[:, i-1], y_true[:, i-1], 0).item()) 92 | elif metric == 'masked_MSE': 93 | self.intermediate_result[metric + '@' + str(i)].append( 94 | loss.masked_mse_torch(y_pred[:, i-1], y_true[:, i-1], 0).item()) 95 | elif metric == 'masked_RMSE': 96 | self.intermediate_result[metric + '@' + str(i)].append( 97 | loss.masked_rmse_torch(y_pred[:, i-1], y_true[:, i-1], 0).item()) 98 | elif metric == 'masked_MAPE': 99 | self.intermediate_result[metric + '@' + str(i)].append( 100 | loss.masked_mape_torch(y_pred[:, i-1], y_true[:, i-1], 0).item()) 101 | elif metric == 'MAE': 102 | self.intermediate_result[metric + '@' + str(i)].append( 103 | loss.masked_mae_torch(y_pred[:, i-1], y_true[:, i-1]).item()) 104 | elif metric == 'MSE': 105 | self.intermediate_result[metric + '@' + str(i)].append( 106 | loss.masked_mse_torch(y_pred[:, i-1], y_true[:, i-1]).item()) 107 | elif metric == 'RMSE': 108 | self.intermediate_result[metric + '@' + str(i)].append( 109 | loss.masked_rmse_torch(y_pred[:, i-1], y_true[:, i-1]).item()) 110 | elif metric == 'MAPE': 111 | self.intermediate_result[metric + '@' + str(i)].append( 112 | loss.masked_mape_torch(y_pred[:, i-1], y_true[:, i-1]).item()) 113 | elif metric == 'R2': 114 | self.intermediate_result[metric + '@' + str(i)].append( 115 | loss.r2_score_torch(y_pred[:, i-1], y_true[:, i-1]).item()) 116 | elif metric == 'EVAR': 117 | self.intermediate_result[metric + '@' + str(i)].append( 118 | loss.explained_variance_score_torch(y_pred[:, i-1], y_true[:, i-1]).item()) 119 | else: 120 | raise ValueError('Error parameter evaluator_mode={}, please set `single` or `average`.'.format(self.mode)) 121 | 122 | def evaluate(self): 123 | """ 124 | 返回之前收集到的所有 batch 的评估结果 125 | """ 126 | for i in range(1, self.len_timeslots + 1): 127 | for metric in self.metrics: 128 | self.result[metric+'@'+str(i)] = sum(self.intermediate_result[metric+'@'+str(i)]) / \ 129 | len(self.intermediate_result[metric+'@'+str(i)]) 130 | return self.result 131 | 132 | def save_result(self, save_path, filename=None): 133 | """ 134 | 将评估结果保存到 save_path 文件夹下的 filename 文件中 135 | 136 | Args: 137 | save_path: 保存路径 138 | filename: 保存文件名 139 | """ 140 | self._logger.info('Note that you select the {} mode to evaluate!'.format(self.mode)) 141 | self.evaluate() 142 | ensure_dir(save_path) 143 | if filename is None: # 使用时间戳 144 | filename = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + '_' + \ 145 | self.config['model'] + '_' + self.config['dataset'] 146 | 147 | if 'json' in self.save_modes: 148 | self._logger.info('Evaluate result is ' + json.dumps(self.result)) 149 | with open(os.path.join(save_path, '{}.json'.format(filename)), 'w') as f: 150 | json.dump(self.result, f) 151 | self._logger.info('Evaluate result is saved at ' + 152 | os.path.join(save_path, '{}.json'.format(filename))) 153 | 154 | dataframe = {} 155 | if 'csv' in self.save_modes: 156 | for metric in self.metrics: 157 | dataframe[metric] = [] 158 | for i in range(1, self.len_timeslots + 1): 159 | for metric in self.metrics: 160 | dataframe[metric].append(self.result[metric+'@'+str(i)]) 161 | dataframe = pd.DataFrame(dataframe, index=range(1, self.len_timeslots + 1)) 162 | dataframe.to_csv(os.path.join(save_path, '{}.csv'.format(filename)), index=False) 163 | self._logger.info('Evaluate result is saved at ' + 164 | os.path.join(save_path, '{}.csv'.format(filename))) 165 | self._logger.info("\n" + str(dataframe)) 166 | return dataframe 167 | 168 | def clear(self): 169 | """ 170 | 清除之前收集到的 batch 的评估信息，适用于每次评估开始时进行一次清空，排除之前的评估输入的影响。 171 | """ 172 | self.result = {} 173 | self.intermediate_result = {} 174 | -------------------------------------------------------------------------------- /libcity/executor/hyper_tuning.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from functools import partial 3 | from logging import getLogger 4 | import hyperopt 5 | from hyperopt import hp, fmin, tpe, atpe, rand 6 | from hyperopt.pyll.base import Apply 7 | 8 | 9 | def _recursivefindnodes(root, node_type='switch'): 10 | nodes = [] 11 | if isinstance(root, (list, tuple)): 12 | for node in root: 13 | nodes.extend(_recursivefindnodes(node, node_type)) 14 | elif isinstance(root, dict): 15 | for node in root.values(): 16 | nodes.extend(_recursivefindnodes(node, node_type)) 17 | elif isinstance(root, Apply): 18 | if root.name == node_type: 19 | nodes.append(root) 20 | for node in root.pos_args: 21 | if node.name == node_type: 22 | nodes.append(node) 23 | for _, node in root.named_args: 24 | if node.name == node_type: 25 | nodes.append(node) 26 | return nodes 27 | 28 | 29 | def _parameters(space): 30 | # Analyze the domain instance to find parameters 31 | parameters = {} 32 | if isinstance(space, dict): 33 | space = list(space.values()) 34 | for node in _recursivefindnodes(space, 'switch'): 35 | # Find the name of this parameter 36 | paramnode = node.pos_args[0] 37 | assert paramnode.name == 'hyperopt_param' 38 | paramname = paramnode.pos_args[0].obj 39 | # Find all possible choices for this parameter 40 | values = [literal.obj for literal in node.pos_args[1:]] 41 | parameters[paramname] = np.array(range(len(values))) 42 | return parameters 43 | 44 | 45 | def _spacesize(space): 46 | # Compute the number of possible combinations 47 | params = _parameters(space) 48 | return np.prod([len(values) for values in params.values()]) 49 | 50 | 51 | class ExhaustiveSearchError(Exception): 52 | pass 53 | 54 | 55 | def _validate_space_exhaustive_search(space): 56 | from hyperopt.pyll.base import dfs, as_apply 57 | from hyperopt.pyll.stochastic import implicit_stochastic_symbols 58 | supported_stochastic_symbols = ['randint', 'quniform', 'qloguniform', 59 | 'qnormal', 'qlognormal', 'categorical'] 60 | for node in dfs(as_apply(space)): 61 | if node.name in implicit_stochastic_symbols: 62 | if node.name not in supported_stochastic_symbols: 63 | raise ExhaustiveSearchError('Exhaustive search is only possible \ 64 | with the following stochastic symbols: ' 65 | + ', '.join(supported_stochastic_symbols)) 66 | 67 | 68 | def exhaustive_search(new_ids, domain, trials, seed, nb_max_sucessive_failures=1000): 69 | from hyperopt import pyll 70 | from hyperopt.base import miscs_update_idxs_vals 71 | # Build a hash set for previous trials 72 | hashset = set([hash(frozenset([(key, value[0]) if len(value) > 0 else ((key, None)) 73 | for key, value in trial['misc']['vals'].items()])) for trial in trials.trials]) 74 | 75 | rng = np.random.RandomState(seed) 76 | rval = [] 77 | for _, new_id in enumerate(new_ids): 78 | new_sample = False 79 | nb_sucessive_failures = 0 80 | while not new_sample: 81 | # -- sample new specs, idxs, vals 82 | idxs, vals = pyll.rec_eval( 83 | domain.s_idxs_vals, 84 | memo={ 85 | domain.s_new_ids: [new_id], 86 | domain.s_rng: rng, 87 | }) 88 | new_result = domain.new_result() 89 | new_misc = dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir) 90 | miscs_update_idxs_vals([new_misc], idxs, vals) 91 | 92 | # Compare with previous hashes 93 | h = hash(frozenset([(key, value[0]) if len(value) > 0 else ( 94 | (key, None)) for key, value in vals.items()])) 95 | if h not in hashset: 96 | new_sample = True 97 | else: 98 | # Duplicated sample, ignore 99 | nb_sucessive_failures += 1 100 | 101 | if nb_sucessive_failures > nb_max_sucessive_failures: 102 | # No more samples to produce 103 | return [] 104 | 105 | rval.extend(trials.new_trial_docs([new_id], [None], [new_result], [new_misc])) 106 | return rval 107 | 108 | 109 | class HyperTuning: 110 | """ 111 | 自动调参 112 | 113 | Note: 114 | HyperTuning is based on the hyperopt (https://github.com/hyperopt/hyperopt) 115 | 116 | https://github.com/hyperopt/hyperopt/issues/200 117 | """ 118 | 119 | def __init__(self, objective_function, space=None, params_file=None, algo='grid_search', 120 | max_evals=100, task=None, model_name=None, dataset_name=None, config_file=None, 121 | saved_model=True, train=True, other_args=None): 122 | self.task = task 123 | self.model_name = model_name 124 | self.dataset_name = dataset_name 125 | self.config_file = config_file 126 | self.saved_model = saved_model 127 | self.train = train 128 | self.other_args = other_args 129 | self._logger = getLogger() 130 | 131 | self.best_score = None 132 | self.best_params = None 133 | self.best_test_result = None 134 | self.params2result = {} # 每一种参数组合对应的最小验证集误差等结果 135 | 136 | self.objective_function = objective_function 137 | self.max_evals = max_evals 138 | if space: 139 | self.space = space 140 | elif params_file: 141 | self.space = self._build_space_from_file(params_file) 142 | else: 143 | raise ValueError('at least one of `space` and `params_file` is provided') 144 | 145 | if isinstance(algo, str): 146 | if algo == 'grid_search': 147 | self.algo = partial(exhaustive_search, nb_max_sucessive_failures=1000) 148 | self.max_evals = _spacesize(self.space) 149 | elif algo == 'tpe': 150 | self.algo = tpe.suggest 151 | elif algo == 'atpe': 152 | self.algo = atpe.suggest 153 | elif algo == 'random_search': 154 | self.algo = rand.suggest 155 | else: 156 | raise ValueError('Illegal hyper algorithm type [{}]'.format(algo)) 157 | else: 158 | self.algo = algo 159 | 160 | @staticmethod 161 | def _build_space_from_file(file): 162 | space = {} 163 | with open(file, 'r') as fp: 164 | for line in fp: 165 | para_list = line.strip().split(' ') 166 | if len(para_list) < 3: 167 | continue 168 | para_name, para_type, para_value = para_list[0], para_list[1], "".join(para_list[2:]) 169 | if para_type == 'choice': 170 | para_value = eval(para_value) 171 | space[para_name] = hp.choice(para_name, para_value) 172 | elif para_type == 'uniform': 173 | low, high = para_value.strip().split(',') 174 | space[para_name] = hp.uniform(para_name, float(low), float(high)) 175 | elif para_type == 'quniform': 176 | low, high, q = para_value.strip().split(',') 177 | space[para_name] = hp.quniform(para_name, float(low), float(high), float(q)) 178 | elif para_type == 'loguniform': 179 | low, high = para_value.strip().split(',') 180 | space[para_name] = hp.loguniform(para_name, float(low), float(high)) 181 | else: 182 | raise ValueError('Illegal parameter type [{}]'.format(para_type)) 183 | return space 184 | 185 | @staticmethod 186 | def params2str(params): 187 | # dict to str 188 | params_str = '' 189 | for param_name in params: 190 | params_str += param_name + ':' + str(params[param_name]) + ', ' 191 | return params_str[:-2] 192 | 193 | def save_result(self, filename=None): 194 | with open(filename, 'w') as fp: 195 | fp.write('best params: ' + str(self.best_params) + '\n') 196 | fp.write('best_valid_score: \n') 197 | fp.write(str(self.params2result[self.params2str(self.best_params)]['best_valid_score']) + '\n') 198 | fp.write('best_test_result: \n') 199 | fp.write(str(self.params2result[self.params2str(self.best_params)]['test_result']) + '\n') 200 | fp.write('----------------------------------------------------------------------------\n') 201 | fp.write('All parameters tune and result: \n') 202 | for params in self.params2result: 203 | fp.write(params + '\n') 204 | fp.write('Test result:\n' + str(self.params2result[params]['test_result']) + '\n') 205 | self._logger.info('hyper-tuning result is saved at {}'.format(filename)) 206 | 207 | def fn(self, params): 208 | hyper_config_dict = params.copy() 209 | params_str = self.params2str(params) 210 | self._logger.info('running parameters:') 211 | self._logger.info(str(hyper_config_dict)) 212 | result_dict = self.objective_function( 213 | task=self.task, model_name=self.model_name, dataset_name=self.dataset_name, 214 | config_file=self.config_file, saved_model=self.saved_model, train=self.train, 215 | other_args=self.other_args, hyper_config_dict=hyper_config_dict) 216 | self.params2result[params_str] = result_dict 217 | 218 | score = result_dict['best_valid_score'] 219 | if not self.best_score: 220 | self.best_score = score 221 | self.best_params = params 222 | elif score < self.best_score: 223 | self.best_score = score 224 | self.best_params = params 225 | self._logger.info('current parameters:') 226 | self._logger.info(str(hyper_config_dict)) 227 | self._logger.info('current best valid score: %.4f' % result_dict['best_valid_score']) 228 | self._logger.info('current test result:') 229 | self._logger.info(result_dict['test_result']) 230 | return {'loss': score, 'status': hyperopt.STATUS_OK} 231 | 232 | def start(self): 233 | fmin(self.fn, self.space, algo=self.algo, max_evals=self.max_evals) 234 | -------------------------------------------------------------------------------- /libcity/model/traffic_flow_prediction/ST_transformer.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.nn.init as init 7 | from functools import partial 8 | from logging import getLogger 9 | from libcity.model import loss 10 | from libcity.model.abstract_traffic_state_model import AbstractTrafficStateModel 11 | from libcity.model.traffic_flow_prediction.layers.pe_layers import * 12 | from libcity.model.traffic_flow_prediction.layers.STTransformer_layers import * 13 | from libcity.model.traffic_flow_prediction.layers.patch_layers import * 14 | from libcity.model.traffic_flow_prediction.layers.mask_layers import * 15 | 16 | class STTransformer(AbstractTrafficStateModel): 17 | def __init__(self, config, data_feature): 18 | super().__init__(config, data_feature) 19 | self._scaler = self.data_feature.get('scaler') 20 | self.adj_mx = data_feature.get('adj_mx') 21 | self.feature_dim = self.data_feature.get("feature_dim", 1) 22 | outfeat_dim = config.get('outfeat_dim',None) 23 | self.output_dim = outfeat_dim if outfeat_dim is not None else self.data_feature.get('output_dim', 1) 24 | self.num_nodes = self.data_feature.get("num_nodes", 1) 25 | # self.ext_dim = self.data_feature.get("ext_dim", 0) 26 | # self.num_batches = self.data_feature.get('num_batches', 1) 27 | self.load_external = config.get('load_external', False) 28 | if self.load_external: 29 | self.feature_dim -= 8 30 | self._logger = getLogger() 31 | 32 | self.device = config.get('device', torch.device('cpu')) 33 | self.embed_dim = config.get('embed_dim', 64) 34 | self.skip_conv_flag = config.get('skip_conv_flag', True) 35 | self.residual_conv_flag = config.get('residual_conv_flag', True) 36 | self.skip_dim = config.get('skip_dim', self.embed_dim) 37 | self.num_layers = config.get('num_layers', 3) 38 | self.num_heads = config.get('num_heads', 8) 39 | self.input_window = config.get("input_window", 12) 40 | self.output_window = config.get('output_window', 12) 41 | 42 | self.gconv_hop_num = config.get('gconv_hop_num',3) 43 | self.gconv_alpha = config.get('gconv_alpha',0) 44 | 45 | self.conv_kernels = config.get('conv_kernels',[2,3,6,12]) 46 | self.conv_stride = config.get('conv_stride',1) 47 | self.conv_if_gc = config.get('conv_if_gc',False) 48 | 49 | self.norm_type = config.get('norm_type','BatchNorm') 50 | 51 | self.att_scale = config.get('att_scale',None) 52 | self.att_dropout = config.get('att_dropout',0.1) 53 | self.ffn_dropout = config.get('ffn_dropout',0.1) 54 | self.Spe_type = config.get('Satt_pe_type','laplacian') 55 | self.Spe_learnable = config.get('Spe_learnable',False) 56 | self.Tpe_type = config.get('Tatt_pe_type','sincos') 57 | self.Tpe_learnable = config.get('Tpe_learnable',False) 58 | self.Smask_flag = config.get('Smask_flag',True) 59 | self.block_forward_mode = config.get('block_forward_mode',0) 60 | self.sstore_attn = config.get('sstore_attn',False) 61 | # static parameters 62 | self.activition_fn = nn.ReLU 63 | 64 | if self.skip_conv_flag is False: 65 | self.skip_dim = self.embed_dim 66 | """ 67 | 3/28: 需要skip connection 68 | 需要depatch解码器/ST解码器 69 | 加入multi-mask机制 70 | """ 71 | 72 | self.patchencoder = patching_STconv( 73 | in_channel=self.feature_dim , embed_dim=self.embed_dim, 74 | in_seq_len=self.input_window, 75 | gdep = self.gconv_hop_num, alpha = self.gconv_alpha, 76 | kernel_sizes=self.conv_kernels,stride=self.conv_stride,device=self.device 77 | ) if self.conv_if_gc else patching_conv( 78 | in_channel=self.feature_dim , embed_dim=self.embed_dim, 79 | in_seq_len=self.input_window, kernel_sizes=self.conv_kernels,stride=self.conv_stride 80 | ) 81 | self.hid_seq_len = self.patchencoder.out_seq_len 82 | if self.Smask_flag: 83 | self.infomask = Infomap_Multi_Mask_Generator(self.num_nodes,self.adj_mx) 84 | self.graphmask = Graph_Mask_Generator(self.num_nodes,self.adj_mx) 85 | self.externalPEencoder = External_Encoding(d_model=self.embed_dim, device=self.device) 86 | self.nodePEencoder = S_Positional_Encoding( 87 | pe_type=self.Spe_type, learn_pe=self.Spe_learnable, node_num=self.num_nodes, 88 | d_model=self.embed_dim,device = self.device) 89 | self.tseqPEencoder = Positional_Encoding( 90 | pe_type=self.Tpe_type, learn_pe=self.Tpe_learnable, q_len=self.hid_seq_len, 91 | d_model=self.embed_dim,device = self.device) 92 | self.STencoders = nn.ModuleList( 93 | [STBlock( 94 | seq_len=self.hid_seq_len,node_num=self.num_nodes,embed_dim=self.embed_dim,num_heads=self.num_heads, 95 | forward_mode=self.block_forward_mode,norm=self.norm_type,scale=self.att_scale, 96 | global_nodePE=self.nodePEencoder,global_tseqPE=self.tseqPEencoder,smask_flag=self.Smask_flag,sbias_flag=False, 97 | tmask_flag=False,tbias_flag=False,key_missing_mask_flag=False, 98 | attention_dropout=self.att_dropout,proj_dropout=self.ffn_dropout,activation_fn=self.activition_fn, 99 | pre_norm=False,sstore_attn=self.sstore_attn 100 | ) for _ in range(self.num_layers)] 101 | ) 102 | 103 | if self.skip_conv_flag: 104 | self.skip_convs = nn.ModuleList([ 105 | nn.Conv2d( 106 | in_channels=self.embed_dim, out_channels=self.skip_dim, kernel_size=1, 107 | ) for _ in range(self.num_layers+1) 108 | ]) 109 | 110 | if self.residual_conv_flag: 111 | self.residual_convs = nn.ModuleList([ 112 | nn.Conv2d( 113 | in_channels=self.embed_dim, out_channels=self.embed_dim, kernel_size=1, 114 | ) for _ in range(self.num_layers) 115 | ]) 116 | 117 | self.lineardecoder = depatching_conv(embed_dim=self.skip_dim, unpatch_channel=self.skip_dim//2, out_channel=self.output_dim, 118 | hid_seq_len = self.hid_seq_len, out_seq_len=self.output_window) 119 | 120 | # self.lineardecoder = nn.Sequential( 121 | # # in [b,n,patch_seq_len,embed_dim] 122 | # # out [b,n,out_seq_len,b,n,out_dim] 123 | # nn.Linear(self.skip_dim,self.output_dim), 124 | # Permution(0,1,3,2), 125 | # nn.Linear(self.hid_seq_len,self.output_window), 126 | # Permution(0,1,3,2) 127 | # ) 128 | 129 | self.droput_layer = nn.Dropout(p=self.ffn_dropout) 130 | 131 | def forward(self, batch): 132 | # dense_adj_mx = self.adj_mx 133 | # multimask = self.multimask 134 | # npe = self.nodePEencoder(dense_adj_mx).reshape(1,-1,1,self.embed_dim).contiguous() 135 | # tpe = self.tseqPEencoder().reshape(1,1,-1,self.embed_dim).contiguous() 136 | # x = batch['X'].permute(0,2,1,3).contiguous() 137 | # if self.conv_if_gc: 138 | # x = self.patchencoder(x,dense_adj_mx) 139 | # else: x = self.patchencoder(x) # [b,n,patch_seq_len,embed_dim] 140 | # skips = [] 141 | # for block in self.STencoders: 142 | # x = block(x,dense_adj_mx, npe, tpe, sattn_mask=multimask) # [b,n,patch_seq_len,embed_dim] 143 | # skips.append(x) 144 | # # out = torch.sum(torch.stack(skips)) 145 | # x = self.droput_layer(x) 146 | # out = self.lineardecoder(x).permute(0,2,1,3).contiguous() 147 | # return out 148 | dense_adj_mx = self.adj_mx 149 | if self.Smask_flag: 150 | multimask = get_static_multihead_mask(self.num_heads,[self.infomask,self.graphmask],device=self.device) 151 | else : multimask = None 152 | npe = self.nodePEencoder(dense_adj_mx).reshape(1,-1,1,self.embed_dim).contiguous() 153 | tpe = self.tseqPEencoder().reshape(1,1,-1,self.embed_dim).contiguous() 154 | x = batch['X'].permute(0,2,1,3).contiguous() # btnc -> bntc 155 | if self.load_external: 156 | x, epe = self.externalPEencoder(x) 157 | npe, tpe = npe+epe, tpe+epe 158 | if self.conv_if_gc: 159 | x = self.patchencoder(x,dense_adj_mx) 160 | else: x = self.patchencoder(x) # [b,n,patch_seq_len,embed_dim] 161 | 162 | skip = self.skip_convs[-1](x.permute(0,3,2,1)) if self.skip_conv_flag else x 163 | if self.sstore_attn: 164 | for i,block in enumerate(self.STencoders): 165 | h,attention_score, attention_weight = block(x,dense_adj_mx, npe, tpe, sattn_mask=multimask) # [b,n,patch_seq_len,embed_dim] 166 | skip = skip+self.skip_convs[i](h.permute(0,3,2,1)) if self.skip_conv_flag else skip+h 167 | x = self.residual_convs[i](x.permute(0,3,2,1)).permute(0,3,2,1)+h if self.residual_conv_flag else x+h 168 | if self.training is not True: 169 | import time 170 | t = time.localtime() 171 | torch.save({'attention_score':attention_score, 'attention_weight':attention_weight},"./attn_save/{}_att.pt".format(time.strftime("%d_%H_%M_%S",t))) 172 | 173 | else: 174 | for i,block in enumerate(self.STencoders): 175 | h = block(x,dense_adj_mx, npe, tpe, sattn_mask=multimask) # [b,n,patch_seq_len,embed_dim] 176 | skip = skip+self.skip_convs[i](h.permute(0,3,2,1)) if self.skip_conv_flag else skip+h 177 | x = self.residual_convs[i](x.permute(0,3,2,1)).permute(0,3,2,1)+h if self.residual_conv_flag else x+h 178 | skip = skip.permute(0,3,2,1) if self.skip_conv_flag else skip 179 | # out = torch.sum(torch.stack(skips)) 180 | skip = self.droput_layer(skip) 181 | out = self.lineardecoder(skip).permute(0,2,1,3).contiguous() 182 | return out 183 | 184 | def calculate_loss(self, batch): 185 | y_true = batch['y'] 186 | y_predicted = self.predict(batch) 187 | y_true = self._scaler.inverse_transform(y_true[..., :self.output_dim]) 188 | y_predicted = self._scaler.inverse_transform(y_predicted[..., :self.output_dim]) 189 | return loss.masked_mae_torch(y_predicted, y_true) 190 | 191 | def predict(self, batch): 192 | return self.forward(batch) --------------------------------------------------------------------------------