Printing the vocabulary of all the lower case letters and the special dot token
# build the vocabulary of characters and mappings to/from integerschars = sorted(list(set(''.join(words))))stoi = {s:i+1 for i,s in enumerate(chars)}stoi['.'] = 0itos = {i:s for s,i in stoi.items()}vocab_size = len(itos)print(itos)print(vocab_size)
Here we are reading the dataset and processing it. In the end of this cell, we are also splitting the dataset into three- Train, Dev and Loss split
# build the datasetblock_size = 3 # context length: how many characters do we take to predict the next one?def build_dataset(words): X, Y = [], [] for w in words: context = [0] * block_size for ch in w + '.': ix = stoi[ch] X.append(context) Y.append(ix) context = context[1:] + [ix] # crop and append X = torch.tensor(X) Y = torch.tensor(Y) print(X.shape, Y.shape) return X, Yimport randomrandom.seed(42)random.shuffle(words)n1 = int(0.8*len(words))n2 = int(0.9*len(words))Xtr, Ytr = build_dataset(words[:n1]) # 80%Xdev, Ydev = build_dataset(words[n1:n2]) # 10%Xte, Yte = build_dataset(words[n2:]) # 10%
Almost the same MLP, but we have cleaned it up to add those hard coded values into variables so we just have to modify them there
# MLP revisitedn_embd = 10 # the dimensionality of the character embedding vectorsn_hidden = 200 # the number of neurons in the hidden layer of the MLPg = torch.Generator().manual_seed(2147483647) # for reproducibilityC = torch.randn((vocab_size, n_embd), generator=g)W1 = torch.randn((n_embd * block_size, n_hidden), generator=g)b1 = torch.randn(n_hidden, generator=g)W2 = torch.randn((n_hidden, vocab_size), generator=g)b2 = torch.randn(vocab_size, generator=g)parameters = [C, W1, b1, W2, b2]print(sum(p.nelement() for p in parameters)) # number of parameters in totalfor p in parameters: p.requires_grad = True
Output
11897
Here we are optimizing the NN. Same as before, just those hard coded numbers (or magic numbers as Andrej sensei calls it) have been replaced with variable names for more readability
# same optimization as last timemax_steps = 200000batch_size = 32lossi = []for i in range(max_steps): # minibatch construct ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g) Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y # forward pass emb = C[Xb] # embed the characters into vectors embcat = emb.view(emb.shape[0], -1) # concatenate the vectors hpreact = embcat @ W1 + b1 # hidden layer pre-activation h = torch.tanh(hpreact) # hidden layer logits = h @ W2 + b2 # output layer loss = F.cross_entropy(logits, Yb) # loss function # backward pass for p in parameters: p.grad = None loss.backward() # update lr = 0.1 if i < 100000 else 0.01 # step learning rate decay for p in parameters: p.data += -lr * p.grad # track stats if i % 10000 == 0: # print every once in a while print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}') lossi.append(loss.log10().item())
Seeing the loss in train and val loss. There is a slight modification to this as to how the splitting is done.
Here the decorator @torch.no_grad() basically tells PyTorch to not maintain the grad value, as it assumes/anticipated that the backpropagation will be calculated after this and we are saying No.
Sampling of the model: Forward pass -> Sampling from the distribution -> Continuing till we get the special token '.'
# sample from the modelg = torch.Generator().manual_seed(2147483647 + 10)for _ in range(20): out = [] context = [0] * block_size # initialize with all ... while True: # forward pass the neural net emb = C[torch.tensor([context])] # (1,block_size,d) h = torch.tanh(emb.view(1, -1) @ W1 + b1) logits = h @ W2 + b2 probs = F.softmax(logits, dim=1) # sample from the distribution ix = torch.multinomial(probs, num_samples=1, generator=g).item() context = context[1:] + [ix] out.append(ix) # if we sample the special '.' token, break if ix == 0: break print(''.join(itos[i] for i in out)) # decode and print the generated word
