learning-corridor-selection/evaluation/evaluate_models.py at main · BreenSammy/learning-corridor-selection · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
import os.path as osp

import matplotlib as mpl
import pandas as pd

import torch

from learning_corridor_selection.evaluation.classifier_evaluation import ClassifierEvaluation
from learning_corridor_selection.evaluation.regression_evaluation import RegressionEvaluation
from learning_corridor_selection.evaluation.trajectory_prediction_evaluation import TrajectoryPredictionEvaluation
from learning_corridor_selection.evaluation.ranking_evaluation import RankingEvaluation, RankingResult

def get_config_and_model(folder):
    return osp.join(folder,'config.yaml'), osp.join(folder, 'models', 'best.pt')


root = osp.join(osp.dirname(osp.abspath(__file__)))
data_root = osp.join(root, "..","data")
models_dir = osp.join(root, '..', 'models')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

cls_config_path, cls_model_checkpoint =  get_config_and_model(osp.join(models_dir, 'classification_base'))
reg_config_path, reg_model_checkpoint =  get_config_and_model(osp.join(models_dir, 'regression_base'))
traj_config_path, traj_model_checkpoint =  get_config_and_model(osp.join(models_dir, 'trajectory_base'))
traj_big_config_path, traj_big_model_checkpoint =  get_config_and_model(osp.join(models_dir, 'trajectory_big'))

cls_treshold = 0.301

cls_evaluation = ClassifierEvaluation(
    root,
    name='classification_base',
    config_path=cls_config_path,
    device=device,
    data_root=data_root,
    model_checkpoint=cls_model_checkpoint,
    rerun_inference=False,
    # columnwidth=252,
)

reg_evaluation = RegressionEvaluation(
    root,
    name='regression_base',
    config_path=reg_config_path,
    device=device,
    data_root=data_root,
    model_checkpoint=reg_model_checkpoint,
    rerun_inference=False,
    )

traj_evaluation = TrajectoryPredictionEvaluation(
    root,
    name='trajectory_base',
    config_path=traj_config_path,
    device=device,
    data_root=data_root,
    model_checkpoint=traj_model_checkpoint,
    rerun_inference=False,
)

traj_big_evaluation = TrajectoryPredictionEvaluation(
    root,
    name='trajectory_big',
    config_path=traj_big_config_path,
    device=device,
    data_root=data_root,
    model_checkpoint=traj_big_model_checkpoint,
    rerun_inference=False,
)

record_inference_runtime = False

if record_inference_runtime:
    cls_evaluation.record_inference_time()
    reg_evaluation.record_inference_time()
    traj_evaluation.record_inference_time()
    traj_big_evaluation.record_inference_time()

cls_evaluation.run_evaluation(splits=['val', 'test'], threshold=cls_treshold)
# cls_evaluation.plot_attention_maps('test', samples=20)

# # reg_evaluation.run_evaluation(splits=['val', 'test'])
# reg_evaluation.plot_attention_maps('test', samples=500)

traj_evaluation.run_evaluation(splits=['test'])
traj_big_evaluation.run_evaluation(splits=['test'])
# traj_evaluation.plot_attention_maps('test', samples=None)

ranking_evaluation = RankingEvaluation(
    root,
    name='ranking_base',
    cls_config_path=cls_config_path,
    reg_config_path=reg_config_path,
    cls_model_checkpoint=cls_model_checkpoint,
    reg_model_checkpoint=reg_model_checkpoint,
    device=device,
    data_root=data_root,
    rerun_inference=False,
    threshold=cls_treshold
    )

ranking_evaluation.run_evaluation(splits=['test', 'val'])

# new_file_path = osp.join(osp.dirname(reg_config_path), "config_exiD.yaml")

# ranking_evaluation_exid = RankingEvaluation(
#     root,
#     name='ranking_base_exiD',
#     cls_config_path=cls_config_path,
#     reg_config_path=new_file_path,
#     cls_model_checkpoint=cls_model_checkpoint,
#     reg_model_checkpoint=reg_model_checkpoint,
#     device=device,
#     data_root=data_root,
#     rerun_inference=True,
#     threshold=cls_treshold
#     )

# ranking_evaluation_exid.run_evaluation()


# ranking_evaluation_highd = RankingEvaluation(
#     root,
#     name='ranking_base_highD',
#     cls_config_path=cls_config_path,
#     reg_config_path=osp.join(osp.dirname(reg_config_path), "config_highD.yaml"),
#     cls_model_checkpoint=cls_model_checkpoint,
#     reg_model_checkpoint=reg_model_checkpoint,
#     device=device,
#     data_root=data_root,
#     rerun_inference=True,
#     threshold=cls_treshold
#     )

# ranking_evaluation_highd.run_evaluation()