From e22b82755dd9702a68fd3f8928814c39e282a05a Mon Sep 17 00:00:00 2001
From: Haze188 <haze188@qq.com>
Date: Tue, 4 Jun 2024 13:57:54 +0800
Subject: [PATCH] [CI/tests] simplify some test case to reduce testing time
(#5755)
* [ci/tests] simplify some test case to reduce testing time
* [ci/tests] continue to remove test case to reduce ci time cost
* restore some test config
* [ci/tests] continue to reduce ci time cost
---
.../test_model/test_shard_bert.py | 32 ------------
.../test_model/test_shard_blip2.py | 4 +-
.../test_model/test_shard_bloom.py | 23 +--------
.../test_model/test_shard_falcon.py | 23 +--------
.../test_model/test_shard_gpt2.py | 40 ---------------
.../test_model/test_shard_llama.py | 49 -------------------
.../test_model/test_shard_vit.py | 1 -
7 files changed, 5 insertions(+), 167 deletions(-)
diff --git a/tests/test_shardformer/test_model/test_shard_bert.py b/tests/test_shardformer/test_model/test_shard_bert.py
index 3ec394768..b97de0ef8 100644
--- a/tests/test_shardformer/test_model/test_shard_bert.py
+++ b/tests/test_shardformer/test_model/test_shard_bert.py
@@ -122,20 +122,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 2,
- "pp_size": 1,
- "enable_all_optimization": True,
- "use_lazy_init": True,
- "precision": "fp32",
- },
- {
- "tp_size": 1,
- "pp_size": 2,
- "num_microbatches": 4,
- "use_lazy_init": True,
- "precision": "fp32",
- },
{
"tp_size": 2,
"pp_size": 2,
@@ -145,14 +131,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 4,
- "pp_size": 1,
- "enable_all_optimization": True,
- "use_lazy_init": False,
- "precision": "fp32",
- },
- {"tp_size": 2, "pp_size": 1, "enable_all_optimization": True, "use_lazy_init": False, "precision": "fp32"},
{
"tp_size": 2,
"pp_size": 1,
@@ -162,16 +140,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 1,
- "pp_size": 2,
- "num_microbatches": 2,
- "enable_all_optimization": True,
- "use_lazy_init": True,
- "zero_stage": 1,
- "precision": "fp16",
- "initial_scale": 1,
- },
],
)
def run_bert_test(test_config):
diff --git a/tests/test_shardformer/test_model/test_shard_blip2.py b/tests/test_shardformer/test_model/test_shard_blip2.py
index 712c5c1e1..04a8f57e9 100644
--- a/tests/test_shardformer/test_model/test_shard_blip2.py
+++ b/tests/test_shardformer/test_model/test_shard_blip2.py
@@ -67,8 +67,8 @@ def check_forward_backward(org_model, sharded_model, data_gen_fn, output_transfo
@parameterize("enable_fused_normalization", [True, False])
@parameterize("enable_tensor_parallelism", [True, False])
-@parameterize("enable_flash_attention", [True, False])
-@parameterize("enable_jit_fused", [True, False])
+@parameterize("enable_flash_attention", [True])
+@parameterize("enable_jit_fused", [True])
def run_blip2_test(
enable_fused_normalization,
enable_tensor_parallelism,
diff --git a/tests/test_shardformer/test_model/test_shard_bloom.py b/tests/test_shardformer/test_model/test_shard_bloom.py
index 6ab0369e0..feabc9083 100644
--- a/tests/test_shardformer/test_model/test_shard_bloom.py
+++ b/tests/test_shardformer/test_model/test_shard_bloom.py
@@ -110,17 +110,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp32",
"initial_scale": 1,
},
- {
- "tp_size": 4,
- "pp_size": 1,
- "num_microbatches": 1,
- "enable_sequence_parallelism": True,
- "sequence_parallelism_mode": "split_gather",
- "enable_flash_attention": False,
- "use_lazy_init": True,
- "precision": "fp16",
- "initial_scale": 1,
- },
{
"tp_size": 2,
"pp_size": 2,
@@ -128,6 +117,7 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"enable_all_optimization": True,
"use_lazy_init": True,
"precision": "fp16",
+ "zero_stage": 1,
"initial_scale": 1,
},
{
@@ -138,17 +128,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"use_lazy_init": False,
"precision": "fp32",
},
- {"tp_size": 4, "pp_size": 1, "enable_all_optimization": True, "use_lazy_init": False, "precision": "fp32"},
- {"tp_size": 2, "pp_size": 1, "enable_all_optimization": True, "use_lazy_init": False, "precision": "fp32"},
- {
- "tp_size": 2,
- "pp_size": 1,
- "enable_all_optimization": True,
- "use_lazy_init": True,
- "zero_stage": 2,
- "precision": "fp16",
- "initial_scale": 1,
- },
{
"tp_size": 1,
"pp_size": 2,
diff --git a/tests/test_shardformer/test_model/test_shard_falcon.py b/tests/test_shardformer/test_model/test_shard_falcon.py
index 8074f9d61..3eb82864a 100644
--- a/tests/test_shardformer/test_model/test_shard_falcon.py
+++ b/tests/test_shardformer/test_model/test_shard_falcon.py
@@ -92,21 +92,12 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"tp_size": 2,
"pp_size": 2,
"num_microbatches": 4,
- "enable_all_optimization": True,
- "use_lazy_init": True,
+ "enable_all_optimization": False,
+ "use_lazy_init": False,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 1,
- "pp_size": 2,
- "num_microbatches": 4,
- "enable_all_optimization": False,
- "use_lazy_init": False,
- "precision": "fp32",
- },
{"tp_size": 4, "pp_size": 1, "enable_all_optimization": True, "use_lazy_init": False, "precision": "fp32"},
- {"tp_size": 2, "pp_size": 1, "enable_all_optimization": True, "use_lazy_init": False, "precision": "fp32"},
{
"tp_size": 2,
"pp_size": 1,
@@ -116,16 +107,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 1,
- "pp_size": 2,
- "num_microbatches": 2,
- "enable_all_optimization": True,
- "use_lazy_init": True,
- "zero_stage": 1,
- "precision": "fp16",
- "initial_scale": 1,
- },
],
)
def run_falcon_test(test_config):
diff --git a/tests/test_shardformer/test_model/test_shard_gpt2.py b/tests/test_shardformer/test_model/test_shard_gpt2.py
index 72ea2b089..f9e368c0e 100644
--- a/tests/test_shardformer/test_model/test_shard_gpt2.py
+++ b/tests/test_shardformer/test_model/test_shard_gpt2.py
@@ -162,46 +162,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 1,
- "pp_size": 2,
- "num_microbatches": 4,
- "enable_all_optimization": True,
- "use_lazy_init": True,
- "precision": "fp16",
- "initial_scale": 1,
- },
- {
- "tp_size": 4,
- "pp_size": 1,
- "enable_all_optimization": False,
- "use_lazy_init": False,
- "precision": "fp32",
- },
- {
- "tp_size": 2,
- "pp_size": 1,
- "enable_all_optimization": False,
- "use_lazy_init": False,
- "precision": "fp32",
- },
- {
- "tp_size": 2,
- "pp_size": 2,
- "num_microbatches": 4,
- "enable_all_optimization": False,
- "use_lazy_init": True,
- "precision": "fp32",
- },
- {
- "tp_size": 2,
- "pp_size": 1,
- "enable_all_optimization": True,
- "use_lazy_init": True,
- "zero_stage": 2,
- "precision": "fp16",
- "initial_scale": 1,
- },
{
"tp_size": 1,
"pp_size": 2,
diff --git a/tests/test_shardformer/test_model/test_shard_llama.py b/tests/test_shardformer/test_model/test_shard_llama.py
index c38570f85..1628bf2f3 100644
--- a/tests/test_shardformer/test_model/test_shard_llama.py
+++ b/tests/test_shardformer/test_model/test_shard_llama.py
@@ -145,17 +145,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 4,
- "pp_size": 1,
- "num_microbatches": 1,
- "enable_sequence_parallelism": True,
- "sequence_parallelism_mode": "ring",
- "enable_flash_attention": False,
- "use_lazy_init": True,
- "precision": "fp32",
- "initial_scale": 1,
- },
{
"tp_size": 4,
"pp_size": 1,
@@ -167,17 +156,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 1,
- "pp_size": 1,
- "sp_size": 2,
- "num_microbatches": 1,
- "enable_sequence_parallelism": True,
- "sequence_parallelism_mode": "all_to_all",
- "use_lazy_init": True,
- "precision": "fp16",
- "initial_scale": 1,
- },
{
"tp_size": 1,
"pp_size": 1,
@@ -190,17 +168,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp16",
"initial_scale": 1,
},
- {
- "tp_size": 1,
- "pp_size": 1,
- "num_microbatches": 1,
- "enable_sequence_parallelism": True,
- "sequence_parallelism_mode": "all_to_all",
- "enable_flash_attention": False,
- "use_lazy_init": True,
- "precision": "fp16",
- "initial_scale": 1,
- },
{
"tp_size": 2,
"pp_size": 2,
@@ -221,22 +188,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"enable_gradient_checkpointing": True,
"gradient_checkpoint_config": PipelineGradientCheckpointConfig(num_ckpt_layers_per_stage=[4, 0]),
},
- {
- "tp_size": 4,
- "pp_size": 1,
- "enable_all_optimization": False,
- "use_lazy_init": False,
- "precision": "fp32",
- },
- {
- "tp_size": 1,
- "pp_size": 4,
- "num_microbatches": 4,
- "enable_all_optimization": False,
- "use_lazy_init": False,
- "precision": "fp32",
- },
- {"tp_size": 2, "pp_size": 1, "enable_all_optimization": False, "use_lazy_init": False, "precision": "fp32"},
{
"tp_size": 2,
"pp_size": 1,
diff --git a/tests/test_shardformer/test_model/test_shard_vit.py b/tests/test_shardformer/test_model/test_shard_vit.py
index d33b52b42..99d6f7d2c 100644
--- a/tests/test_shardformer/test_model/test_shard_vit.py
+++ b/tests/test_shardformer/test_model/test_shard_vit.py
@@ -108,7 +108,6 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn,
"precision": "fp32",
},
{"tp_size": 4, "pp_size": 1, "enable_all_optimization": True, "use_lazy_init": False, "precision": "fp32"},
- {"tp_size": 2, "pp_size": 1, "enable_all_optimization": True, "use_lazy_init": False, "precision": "fp32"},
{
"tp_size": 2,
"pp_size": 1,