github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-13 05:01:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[test] Hotfix/fix some model test and refactor check util api (#4369)

Browse Source 
* fix llama test

* fix test bug of bert, blip2, bloom, gpt2

* fix llama test

* fix opt test

* fix sam test

* fix sam test

* fix t5 test

* fix vit test

* fix whisper test

* fix whisper test

* polish code

* adjust allclose parameter

* Add mistakenly deleted code

* addjust allclose

* change loss function for some base model
This commit is contained in:
Bin Jia
2023-08-03 14:51:36 +08:00
committed by Hongxin Liu
parent c3ca53cf05
commit 5c6f183192
16 changed files with 135 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/kit/model_zoo/transformers/bert.py
View File

@@ -102,7 +102,7 @@ def data_gen_for_qa():
output_transform_fn = lambda x: x
# define loss funciton
loss_fn_for_bert_model = lambda x: x.pooler_output.mean()
loss_fn_for_bert_model = lambda x: x.pooler_output.sum()
loss_fn = lambda x: x.loss
config = transformers.BertConfig(hidden_size=128,

14
tests/kit/model_zoo/transformers/bloom.py
View File

@@ -55,17 +55,23 @@ def data_gen_for_question_answering():
input_ids = torch.tensor(
[[57647, 1620, 23967, 620, 107373, 34, 91514, 620, 107373, 1620, 267, 35378, 48946, 18161]], dtype=torch.int64)
attention_mask = torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=torch.int64)
return dict(input_ids=input_ids, attention_mask=attention_mask)
start_positions = torch.tensor([1], dtype=torch.int64)
end_positions = torch.tensor([10], dtype=torch.int64)
return dict(input_ids=input_ids,
attention_mask=attention_mask,
start_positions=start_positions,
end_positions=end_positions)
# define output transform function
output_transform_fn = lambda x: x
# define loss function
loss_fn_for_bloom_model = lambda x: x.last_hidden_state.mean()
loss_fn_for_bloom_model = lambda x: torch.nn.functional.mse_loss(x.last_hidden_state,
torch.ones_like(x.last_hidden_state))
loss_fn_for_causal_lm = lambda x: x.loss
loss_fn_for_classification = lambda x: x.logits.mean()
loss_fn_for_question_answering = lambda x: x.end_logits.mean()
loss_fn_for_classification = lambda x: x.loss
loss_fn_for_question_answering = lambda x: x.loss
config = transformers.BloomConfig(n_layer=1,
n_head=4,

20
tests/kit/model_zoo/transformers/gpt.py
View File

@@ -1,3 +1,5 @@
import copy
import torch
import transformers
@@ -44,14 +46,14 @@ def data_gen_for_token_classification():
# token classification data gen
# `labels` is the type not the token id for token classification, 0 or 1
data = data_gen()
data['labels'] = torch.tensor([[0, 0, 0, 0, 0, 0]], dtype=torch.int64)
data['labels'] = torch.tensor([[0, 0, 0, 0, 0, 1]], dtype=torch.int64)
return data
def data_gen_for_sequence_classification():
# sequence classification data gen
data = data_gen()
data['labels'] = torch.tensor([0], dtype=torch.int64)
data['labels'] = torch.tensor([1], dtype=torch.int64)
return data
@@ -59,7 +61,8 @@ def data_gen_for_sequence_classification():
output_transform_fn = lambda x: x
# define loss function
loss_fn_for_gpt2_model = lambda x: x.last_hidden_state.mean()
loss_fn_for_gpt2_model = lambda x: torch.nn.functional.mse_loss(x.last_hidden_state, torch.ones_like(x.last_hidden_state
))
loss_fn = lambda x: x.loss
config = transformers.GPT2Config(n_layer=2,
@@ -69,9 +72,10 @@ config = transformers.GPT2Config(n_layer=2,
embd_pdrop=0,
resid_pdrop=0,
summary_first_dropout=0,
hidden_dropout=0,
problem_type="single_label_classification",
pad_token_id=50256)
hidden_dropout=0)
config_for_token_classification = copy.deepcopy(config)
config_for_token_classification.num_labels = 2
# register the following models
model_zoo.register(name='transformers_gpt',
@@ -99,13 +103,13 @@ model_zoo.register(name='transformers_gpt_for_question_answering',
loss_fn=loss_fn,
model_attribute=ModelAttribute(has_control_flow=True))
model_zoo.register(name='transformers_gpt_for_token_classification',
model_fn=lambda: transformers.GPT2ForTokenClassification(config),
model_fn=lambda: transformers.GPT2ForTokenClassification(config_for_token_classification),
data_gen_fn=data_gen_for_token_classification,
output_transform_fn=output_transform_fn,
loss_fn=loss_fn,
model_attribute=ModelAttribute(has_control_flow=True))
model_zoo.register(name='transformers_gpt_for_sequence_classification',
model_fn=lambda: transformers.GPT2ForSequenceClassification(config),
model_fn=lambda: transformers.GPT2ForSequenceClassification(config_for_token_classification),
data_gen_fn=data_gen_for_sequence_classification,
output_transform_fn=output_transform_fn,
loss_fn=loss_fn,

3
tests/kit/model_zoo/transformers/opt.py
View File

@@ -44,7 +44,8 @@ def data_gen_for_question_answering():
output_transform_fn = lambda x: x
loss_fn_for_opt_model = lambda x: x.last_hidden_state.mean()
loss_fn_for_opt_model = lambda x: torch.nn.functional.mse_loss(x.last_hidden_state, torch.ones_like(x.last_hidden_state)
)
loss_fn_for_lm = lambda x: x.loss
config = transformers.OPTConfig(
hidden_size=128,

4
tests/kit/model_zoo/transformers/whisper.py
View File

@@ -22,7 +22,7 @@ def data_gen():
# input_features = inputs.input_features
# decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
input_features = torch.randn(1, 80, 3000)
input_features = torch.rand(1, 80, 3000)
decoder_input_ids = torch.tensor([[1, 1]]) * 50258
return dict(input_features=input_features, decoder_input_ids=decoder_input_ids)
@@ -53,7 +53,7 @@ def data_gen_for_audio_classification():
output_transform_fn = lambda x: x
# define loss funciton
loss_fn = lambda x: x.last_hidden_state.mean()
loss_fn = lambda x: torch.nn.functional.mse_loss(x.last_hidden_state, torch.ones_like(x.last_hidden_state))
loss_fn_attr = lambda x: x.loss
config = transformers.WhisperConfig(