# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np
from .builder import DATASETS, build_dataset
@DATASETS.register_module()
class RepeatDataset:
"""A wrapper of repeated dataset.
The length of repeated dataset will be ``times`` larger than the original
dataset. This is useful when the data loading time is long but the dataset
is small. Using RepeatDataset can reduce the data loading time between
epochs.
Args:
dataset (dict): The config of the dataset to be repeated.
times (int): Repeat times.
test_mode (bool): Store True when building test or validation dataset.
Default: False.
"""
def __init__(self, dataset, times, test_mode=False):
dataset['test_mode'] = test_mode
self.dataset = build_dataset(dataset)
self.times = times
self._ori_len = len(self.dataset)
def __getitem__(self, idx):
"""Get data."""
return self.dataset[idx % self._ori_len]
def __len__(self):
"""Length after repetition."""
return self.times * self._ori_len
@DATASETS.register_module()
class ConcatDataset:
"""A wrapper of concatenated dataset.
The length of concatenated dataset will be the sum of lengths of all
datasets. This is useful when you want to train a model with multiple data
sources.
Args:
datasets (list[dict]): The configs of the datasets.
test_mode (bool): Store True when building test or validation dataset.
Default: False.
"""
def __init__(self, datasets, test_mode=False):
for item in datasets:
item['test_mode'] = test_mode
datasets = [build_dataset(cfg) for cfg in datasets]
self.datasets = datasets
self.lens = [len(x) for x in self.datasets]
self.cumsum = np.cumsum(self.lens)
def __getitem__(self, idx):
"""Get data."""
dataset_idx = np.searchsorted(self.cumsum, idx, side='right')
item_idx = idx if dataset_idx == 0 else idx - self.cumsum[dataset_idx]
return self.datasets[dataset_idx][item_idx]
def __len__(self):
"""Length after repetition."""
return sum(self.lens)