mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2026-07-15 15:29:48 +00:00
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
This commit is contained in:
@@ -34,8 +34,8 @@ class PreTrainingDataset:
|
||||
self.do_whole_word_mask = do_whole_word_mask
|
||||
self.max_predictions_per_seq = max_predictions_per_seq
|
||||
self.vocab_words = list(tokenizer.vocab.keys())
|
||||
self.rec = re.compile("[\u4E00-\u9FA5]")
|
||||
self.whole_rec = re.compile("##[\u4E00-\u9FA5]")
|
||||
self.rec = re.compile("[\u4e00-\u9fa5]")
|
||||
self.whole_rec = re.compile("##[\u4e00-\u9fa5]")
|
||||
|
||||
self.mlm_p = 0.15
|
||||
self.mlm_mask_p = 0.8
|
||||
|
||||
@@ -75,15 +75,15 @@ auto get_new_segment(
|
||||
return new_segment;
|
||||
}
|
||||
|
||||
bool startsWith(const std::string &s, const std::string &sub) {
|
||||
bool startsWith(const std::string& s, const std::string& sub) {
|
||||
return s.find(sub) == 0 ? true : false;
|
||||
}
|
||||
|
||||
auto create_whole_masked_lm_predictions(
|
||||
std::vector<std::string> &tokens,
|
||||
const std::vector<std::string> &original_tokens,
|
||||
const std::vector<std::string> &vocab_words,
|
||||
std::map<std::string, int> &vocab, const int max_predictions_per_seq,
|
||||
std::vector<std::string>& tokens,
|
||||
const std::vector<std::string>& original_tokens,
|
||||
const std::vector<std::string>& vocab_words,
|
||||
std::map<std::string, int>& vocab, const int max_predictions_per_seq,
|
||||
const double masked_lm_prob) {
|
||||
// for (auto item : vocab) {
|
||||
// std::cout << "key=" << std::string(py::str(item.first)) << ", "
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" PyTorch DeBERTa-v2 model."""
|
||||
"""PyTorch DeBERTa-v2 model."""
|
||||
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
|
||||
@@ -126,11 +126,9 @@ def evaluate_model(
|
||||
avg_loss = "{:.4f}".format(accum_loss.item())
|
||||
accuracy = "{:.4f}".format(accum_correct.item() / total_num.item())
|
||||
if coordinator.is_master():
|
||||
print(
|
||||
f"Evaluation result for epoch {epoch + 1}: \
|
||||
print(f"Evaluation result for epoch {epoch + 1}: \
|
||||
average_loss={avg_loss}, \
|
||||
accuracy={accuracy}."
|
||||
)
|
||||
accuracy={accuracy}.")
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -20,6 +20,7 @@ on a text file or a dataset without using HuggingFace Trainer.
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=text-generation
|
||||
"""
|
||||
|
||||
# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
|
||||
|
||||
import math
|
||||
|
||||
@@ -18,7 +18,7 @@ def cyclic_iter(iter):
|
||||
def build_train_valid_test_data_iterators(
|
||||
train_iters, global_batch_size, eval_interval, eval_iters, dataloader_type="single", **kwargs
|
||||
):
|
||||
(train_dataloader, valid_dataloader, test_dataloader) = (None, None, None)
|
||||
train_dataloader, valid_dataloader, test_dataloader = (None, None, None)
|
||||
|
||||
logger = get_dist_logger()
|
||||
logger.info("> building train, validation, and test datasets ...", ranks=[0])
|
||||
|
||||
@@ -252,7 +252,7 @@ def build_training_sample(
|
||||
|
||||
# Masking.
|
||||
max_predictions_per_seq = masked_lm_prob * max_num_tokens
|
||||
(tokens, masked_positions, masked_labels, _) = create_masked_lm_predictions(
|
||||
tokens, masked_positions, masked_labels, _ = create_masked_lm_predictions(
|
||||
tokens,
|
||||
vocab_id_list,
|
||||
vocab_id_to_token_dict,
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Dataloaders."""
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from colossalai.legacy.context import ParallelMode
|
||||
|
||||
Reference in New Issue
Block a user