Add starcoder support.

This commit is contained in:
Adam Treat
2023-07-06 16:34:04 -04:00
parent 397f3ba2d7
commit 6d03b3e500
8 changed files with 1086 additions and 1 deletions

View File

@@ -961,6 +961,11 @@ DLL_EXPORT const char *get_build_variant() {
DLL_EXPORT bool magic_match(std::istream& f) {
uint32_t magic = 0;
f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
gptj_hparams hparams;
f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
return false; // not a gptj.
}
return magic == 0x67676d6c;
}