mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-10-03 08:06:24 +00:00
[misc] update pre-commit and run all files (#4752)
* [misc] update pre-commit * [misc] run pre-commit * [misc] remove useless configuration files * [misc] ignore cuda for clang-format
This commit is contained in:
@@ -22,9 +22,7 @@ import torch
|
||||
|
||||
|
||||
class BlendableDataset(torch.utils.data.Dataset):
|
||||
|
||||
def __init__(self, datasets, weights):
|
||||
|
||||
self.datasets = datasets
|
||||
num_datasets = len(datasets)
|
||||
assert num_datasets == len(weights)
|
||||
@@ -46,12 +44,16 @@ class BlendableDataset(torch.utils.data.Dataset):
|
||||
self.dataset_sample_index = np.zeros(self.size, dtype=np.int64)
|
||||
|
||||
from . import helpers
|
||||
helpers.build_blending_indices(self.dataset_index,
|
||||
self.dataset_sample_index,
|
||||
weights, num_datasets, self.size,
|
||||
torch.distributed.get_rank() == 0)
|
||||
print('> elapsed time for building blendable dataset indices: '
|
||||
'{:.2f} (sec)'.format(time.time() - start_time))
|
||||
|
||||
helpers.build_blending_indices(
|
||||
self.dataset_index,
|
||||
self.dataset_sample_index,
|
||||
weights,
|
||||
num_datasets,
|
||||
self.size,
|
||||
torch.distributed.get_rank() == 0,
|
||||
)
|
||||
print("> elapsed time for building blendable dataset indices: " "{:.2f} (sec)".format(time.time() - start_time))
|
||||
|
||||
def __len__(self):
|
||||
return self.size
|
||||
|
Reference in New Issue
Block a user