Skip to content

Nuscene and truckscenes Radar compatibility #1769

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MatteGombia wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Nuscene and truckscenes Radar compatibility #1769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
03f08fe
Nuscene and truckscenes Radar compatibility
MatteGombia Jun 19, 2026
38e8dea
Refactor dataset imports and update __all__ exports
MatteGombia Jun 19, 2026
d51fb7c
Support for my custom dataset
MatteGombia Jun 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions pcdet/datasets/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,37 @@
from .dataset import DatasetTemplate
from .kitti.kitti_dataset import KittiDataset
from .nuscenes.nuscenes_dataset import NuScenesDataset
from .nuscenes_cars.nuscenes_dataset import NuScenesCarsDataset
from .waymo.waymo_dataset import WaymoDataset
from .pandaset.pandaset_dataset import PandasetDataset
from .lyft.lyft_dataset import LyftDataset
from .once.once_dataset import ONCEDataset
from .argo2.argo2_dataset import Argo2Dataset
from .custom.custom_dataset import CustomDataset
from .our_data.custom_dataset import CustomDataset
#from .custom.custom_dataset import CustomDataset
from .truckscenes.truckscenes_dataset import TruckScenesDataset
from .truckscenes_multiradar.truckscenes_dataset import TruckScenesMultiRadarDataset
from .truckscenes_multiradar_cars.truckscenes_dataset import TruckScenesMultiRadarCarsDataset
from .truckscenes_cars.truckscenes_dataset import TruckScenesCarsDataset
from .truckscenes_multiradar_cars_comp.truckscenes_dataset import TruckScenesMultiRadarCarsCompDataset


__all__ = {
'DatasetTemplate': DatasetTemplate,
'KittiDataset': KittiDataset,
'NuScenesDataset': NuScenesDataset,
'NuScenesCarsDataset': NuScenesCarsDataset,
'WaymoDataset': WaymoDataset,
'PandasetDataset': PandasetDataset,
'LyftDataset': LyftDataset,
'ONCEDataset': ONCEDataset,
'CustomDataset': CustomDataset,
'Argo2Dataset': Argo2Dataset
'Argo2Dataset': Argo2Dataset,
'TruckScenesDataset': TruckScenesDataset,
'TruckScenesMultiRadarDataset': TruckScenesMultiRadarDataset,
'TruckScenesMultiRadarCarsDataset': TruckScenesMultiRadarCarsDataset,
'TruckScenesCarsDataset': TruckScenesCarsDataset,
'TruckScenesMultiRadarCarsCompDataset': TruckScenesMultiRadarCarsCompDataset
}


Expand Down
69 changes: 60 additions & 9 deletions pcdet/datasets/nuscenes/nuscenes_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from ..dataset import DatasetTemplate
from pyquaternion import Quaternion
from PIL import Image
from nuscenes.utils.data_classes import RadarPointCloud


class NuScenesDataset(DatasetTemplate):
Expand Down Expand Up @@ -101,20 +102,43 @@ def remove_ego_points(points, center_radius=1.0):
def get_lidar_with_sweeps(self, index, max_sweeps=1):
info = self.infos[index]
lidar_path = self.root_path / info['lidar_path']
points = np.fromfile(str(lidar_path), dtype=np.float32, count=-1).reshape([-1, 5])[:, :4]

pc = RadarPointCloud.from_file(str(lidar_path))
points_raw = pc.points.T # Shape becomes (N, 18)

# Extract [x, y, z, rcs, vx_comp, vy_comp]
# nuScenes radar indices: 0=x, 1=y, 2=z, 5=rcs, 8=vx_comp, 9=vy_comp
points = points_raw[:, [0, 1, 2, 5, 8, 9]]

sweep_points_list = [points]
sweep_times_list = [np.zeros((points.shape[0], 1))]

for k in np.random.choice(len(info['sweeps']), max_sweeps - 1, replace=False):
points_sweep, times_sweep = self.get_sweep(info['sweeps'][k])
sweep_points_list.append(points_sweep)
sweep_times_list.append(times_sweep)
is_valid_sweep = True
if len(info['sweeps']) == 0:
is_valid_sweep = False
else:
sweep = info['sweeps'][k]
sweep_path = self.root_path / sweep['lidar_path']

points = np.concatenate(sweep_points_list, axis=0)
times = np.concatenate(sweep_times_list, axis=0).astype(points.dtype)
sweep_pc = RadarPointCloud.from_file(str(sweep_path))
sweep_points_raw = sweep_pc.points.T
sweep_points = sweep_points_raw[:, [0, 1, 2, 5, 8, 9]]

# If there is no matrix (e.g., first frame of a scene) -> skip the transformation
if sweep['transform_matrix'] is not None:
sweep_points[:, :3] = sweep_points[:, :3] @ sweep['transform_matrix'][:3, :3].T
sweep_points[:, :3] += sweep['transform_matrix'][:3, 3]

sweep_points_list.append(sweep_points)
sweep_times_list.append(sweep['time_lag'] * np.ones((sweep_points.shape[0], 1)))

points = np.concatenate(sweep_points_list, axis=0)
times = np.concatenate(sweep_times_list, axis=0)

# Final shape: (N, 7) -> [x, y, z, rcs, vx_comp, vy_comp, time]
points = np.concatenate((points, times), axis=1)

return points

def crop_image(self, input_dict):
Expand Down Expand Up @@ -222,6 +246,8 @@ def __getitem__(self, index):
info = copy.deepcopy(self.infos[index])
points = self.get_lidar_with_sweeps(index, max_sweeps=self.dataset_cfg.MAX_SWEEPS)

#print(f"DEBUG - PATH : {info['lidar_path']}, POINTS SHAPE: {points.shape}")

input_dict = {
'points': points,
'frame_id': Path(info['lidar_path']).stem,
Expand All @@ -230,7 +256,32 @@ def __getitem__(self, index):

if 'gt_boxes' in info:
if self.dataset_cfg.get('FILTER_MIN_POINTS_IN_GT', False):
mask = (info['num_lidar_pts'] > self.dataset_cfg.FILTER_MIN_POINTS_IN_GT - 1)
import torch
from pcdet.ops.roiaware_pool3d import roiaware_pool3d_utils

gt_boxes = info['gt_boxes']

# Filter out empty gt_boxes
if len(gt_boxes) == 0:
mask = np.zeros(0, dtype=bool)
else:
boxes_tensor = torch.from_numpy(gt_boxes[:, :7]).float().reshape(-1, 7)
points_tensor = torch.from_numpy(points[:, :3]).float().reshape(-1, 3)

try:
point_indices = roiaware_pool3d_utils.points_in_boxes_cpu(boxes_tensor, points_tensor).numpy()
except AssertionError:
point_indices = roiaware_pool3d_utils.points_in_boxes_cpu(points_tensor, boxes_tensor).numpy()

if point_indices.ndim == 2:
point_counts = point_indices.sum(axis=1)
else:
point_counts = np.zeros(gt_boxes.shape[0], dtype=np.int32)
valid_indices = point_indices[point_indices != -1]
unique_boxes, counts = np.unique(valid_indices, return_counts=True)
point_counts[unique_boxes] = counts

mask = point_counts >= self.dataset_cfg.FILTER_MIN_POINTS_IN_GT
else:
mask = None

Expand Down Expand Up @@ -261,8 +312,8 @@ def evaluation(self, det_annos, class_names, **kwargs):
nusc_annos = nuscenes_utils.transform_det_annos_to_nusc_annos(det_annos, nusc)
nusc_annos['meta'] = {
'use_camera': False,
'use_lidar': True,
'use_radar': False,
'use_lidar': False,
'use_radar': True,
'use_map': False,
'use_external': False,
}
Expand Down
10 changes: 5 additions & 5 deletions pcdet/datasets/nuscenes/nuscenes_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ def get_available_scenes(nusc):
scene_token = scene['token']
scene_rec = nusc.get('scene', scene_token)
sample_rec = nusc.get('sample', scene_rec['first_sample_token'])
sd_rec = nusc.get('sample_data', sample_rec['data']['LIDAR_TOP'])
sd_rec = nusc.get('sample_data', sample_rec['data']['RADAR_FRONT'])
has_more_frames = True
scene_not_exist = False
while has_more_frames:
Expand Down Expand Up @@ -314,8 +314,8 @@ def fill_trainval_infos(data_path, nusc, train_scenes, val_scenes, test=False, m
val_nusc_infos = []
progress_bar = tqdm.tqdm(total=len(nusc.sample), desc='create_info', dynamic_ncols=True)

ref_chan = 'LIDAR_TOP' # The radar channel from which we track back n sweeps to aggregate the point cloud.
chan = 'LIDAR_TOP' # The reference channel of the current sample_rec that the point clouds are mapped to.
ref_chan = 'RADAR_FRONT' # The radar channel from which we track back n sweeps to aggregate the point cloud.
chan = 'RADAR_FRONT' # The reference channel of the current sample_rec that the point clouds are mapped to.

for index, sample in enumerate(nusc.sample):
progress_bar.update()
Expand Down Expand Up @@ -440,7 +440,7 @@ def fill_trainval_infos(data_path, nusc, train_scenes, val_scenes, test=False, m
# the filtering gives 0.5~1 map improvement
num_lidar_pts = np.array([anno['num_lidar_pts'] for anno in annotations])
num_radar_pts = np.array([anno['num_radar_pts'] for anno in annotations])
mask = (num_lidar_pts + num_radar_pts > 0)
mask = (num_radar_pts > -1)

locs = np.array([b.center for b in ref_boxes]).reshape(-1, 3)
dims = np.array([b.wlh for b in ref_boxes]).reshape(-1, 3)[:, [1, 0, 2]] # wlh == > dxdydz (lwh)
Expand Down Expand Up @@ -488,7 +488,7 @@ def boxes_lidar_to_nusenes(det_info):

def lidar_nusc_box_to_global(nusc, boxes, sample_token):
s_record = nusc.get('sample', sample_token)
sample_data_token = s_record['data']['LIDAR_TOP']
sample_data_token = s_record['data']['RADAR_FRONT']

sd_record = nusc.get('sample_data', sample_data_token)
cs_record = nusc.get('calibrated_sensor', sd_record['calibrated_sensor_token'])
Expand Down
Loading