Download train.py from AxionLab-official/MiniText-v1.0-base: direct link, hf CLI and curl.
- Browser
- Download file 2.18 kB
-
https://huggingface.co/AxionLab-official/MiniText-v1.0-base/resolve/main/train.py
- Command line
-
hf download hf://AxionLab-official/MiniText-v1.0-base/train.py
-
curl -L -o train.py https://huggingface.co/AxionLab-official/MiniText-v1.0-base/resolve/main/train.py
2.18 kB
| import torch | |
| import torch.nn as nn | |
| import os | |
| from model import MiniText | |
| # ----------------------- | |
| # hiperparâmetros | |
| # ----------------------- | |
| SEQ_LEN = 64 | |
| EPOCHS = 12000 | |
| LR = 1e-4 | |
| SAVE_EVERY = 2000 # salva checkpoint a cada X epochs | |
| CHECKPOINT_PATH = "checkpoint.pt" | |
| # ----------------------- | |
| # dataset | |
| # ----------------------- | |
| with open("dataset.txt", "rb") as f: | |
| data = torch.tensor(list(f.read()), dtype=torch.long) | |
| # ----------------------- | |
| # model + optimizer | |
| # ----------------------- | |
| model = MiniText() | |
| optimizer = torch.optim.Adam(model.parameters(), lr=LR) | |
| loss_fn = nn.CrossEntropyLoss() | |
| start_epoch = 0 | |
| # ----------------------- | |
| # load checkpoint (se existir) | |
| # ----------------------- | |
| if os.path.exists(CHECKPOINT_PATH): | |
| print("Checkpoint encontrado, retomando treino...") | |
| checkpoint = torch.load(CHECKPOINT_PATH) | |
| model.load_state_dict(checkpoint["model"]) | |
| optimizer.load_state_dict(checkpoint["optimizer"]) | |
| start_epoch = checkpoint["epoch"] + 1 | |
| else: | |
| print("Nenhum checkpoint encontrado, treino do zero.") | |
| # ----------------------- | |
| # batch sampler | |
| # ----------------------- | |
| def get_batch(): | |
| idx = torch.randint(0, len(data) - SEQ_LEN - 1, (1,)) | |
| x = data[idx:idx + SEQ_LEN].unsqueeze(0) | |
| y = data[idx + 1:idx + SEQ_LEN + 1].unsqueeze(0) | |
| return x, y | |
| # ----------------------- | |
| # training loop | |
| # ----------------------- | |
| for epoch in range(start_epoch, EPOCHS): | |
| x, y = get_batch() | |
| logits, _ = model(x) | |
| loss = loss_fn(logits.view(-1, 256), y.view(-1)) | |
| optimizer.zero_grad() | |
| loss.backward() | |
| optimizer.step() | |
| print(f"Epoch {epoch+1}/{EPOCHS} | Loss: {loss.item():.4f}") | |
| # salvar checkpoint | |
| if (epoch + 1) % SAVE_EVERY == 0: | |
| torch.save({ | |
| "epoch": epoch, | |
| "model": model.state_dict(), | |
| "optimizer": optimizer.state_dict() | |
| }, CHECKPOINT_PATH) | |
| print("Checkpoint salvo.") | |
| # ----------------------- | |
| # salvar modelo final | |
| # ----------------------- | |
| torch.save(model.state_dict(), "minitext.pt") | |
| print("Treino finalizado. Modelo salvo em minitext.pt") | |