github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-08-08 03:24:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

[doc] Fix gradient accumulation doc. (#4349)

Browse Source 
* [doc] fix gradient accumulation doc

* [doc] fix gradient accumulation doc
This commit is contained in:
flybird1111 2023-08-04 17:24:35 +08:00 committed by GitHub
parent 38b792aab2
commit f40b718959
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/source/en/features/gradient_accumulation_with_booster.md
View File

@ -103,10 +103,12 @@ for idx, (img, label) in enumerate(train_dataloader):
with sync_context: with sync_context:
output = model(img) output = model(img)
train_loss = criterion(output, label) train_loss = criterion(output, label)
train_loss = train_loss / GRADIENT_ACCUMULATION
booster.backward(train_loss, optimizer) booster.backward(train_loss, optimizer)
else: else:
output = model(img) output = model(img)
train_loss = criterion(output, label) train_loss = criterion(output, label)
train_loss = train_loss / GRADIENT_ACCUMULATION
booster.backward(train_loss, optimizer) booster.backward(train_loss, optimizer)
optimizer.step() optimizer.step()
optimizer.zero_grad() optimizer.zero_grad()

2
docs/source/zh-Hans/features/gradient_accumulation_with_booster.md
View File

@ -106,10 +106,12 @@ for idx, (img, label) in enumerate(train_dataloader):
with sync_context: with sync_context:
output = model(img) output = model(img)
train_loss = criterion(output, label) train_loss = criterion(output, label)
train_loss = train_loss / GRADIENT_ACCUMULATION
booster.backward(train_loss, optimizer) booster.backward(train_loss, optimizer)
else: else:
output = model(img) output = model(img)
train_loss = criterion(output, label) train_loss = criterion(output, label)
train_loss = train_loss / GRADIENT_ACCUMULATION
booster.backward(train_loss, optimizer) booster.backward(train_loss, optimizer)
optimizer.step() optimizer.step()
optimizer.zero_grad() optimizer.zero_grad()