Final code

In [1]:

import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt # for making figures
%matplotlib inline

import torch import torch.nn.functional as F import matplotlib.pyplot as plt # for making figures %matplotlib inline

In [ ]:

# download the names.txt file from github
!wget https://raw.githubusercontent.com/karpathy/makemore/master/names.txt

# download the names.txt file from github !wget https://raw.githubusercontent.com/karpathy/makemore/master/names.txt

In [3]:

# read in all the words
words = open('names.txt', 'r').read().splitlines()
print(len(words))
print(max(len(w) for w in words))
print(words[:8])

# read in all the words words = open('names.txt', 'r').read().splitlines() print(len(words)) print(max(len(w) for w in words)) print(words[:8])

32033
15
['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']

In [4]:

# build the vocabulary of characters and mappings to/from integers
chars = sorted(list(set(''.join(words))))
stoi = {s:i+1 for i,s in enumerate(chars)}
stoi['.'] = 0
itos = {i:s for s,i in stoi.items()}
vocab_size = len(itos)
print(itos)
print(vocab_size)

# build the vocabulary of characters and mappings to/from integers chars = sorted(list(set(''.join(words)))) stoi = {s:i+1 for i,s in enumerate(chars)} stoi['.'] = 0 itos = {i:s for s,i in stoi.items()} vocab_size = len(itos) print(itos) print(vocab_size)

{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}
27

In [5]:

# build the dataset
block_size = 3 # context length: how many characters do we take to predict the next one?

def build_dataset(words):
  X, Y = [], []

  for w in words:
    context = [0] * block_size
    for ch in w + '.':
      ix = stoi[ch]
      X.append(context)
      Y.append(ix)
      context = context[1:] + [ix] # crop and append

  X = torch.tensor(X)
  Y = torch.tensor(Y)
  print(X.shape, Y.shape)
  return X, Y

import random
random.seed(42)
random.shuffle(words)
n1 = int(0.8*len(words))
n2 = int(0.9*len(words))

Xtr,  Ytr  = build_dataset(words[:n1])     # 80%
Xdev, Ydev = build_dataset(words[n1:n2])   # 10%
Xte,  Yte  = build_dataset(words[n2:])     # 10%

# build the dataset block_size = 3 # context length: how many characters do we take to predict the next one? def build_dataset(words): X, Y = [], [] for w in words: context = [0] * block_size for ch in w + '.': ix = stoi[ch] X.append(context) Y.append(ix) context = context[1:] + [ix] # crop and append X = torch.tensor(X) Y = torch.tensor(Y) print(X.shape, Y.shape) return X, Y import random random.seed(42) random.shuffle(words) n1 = int(0.8*len(words)) n2 = int(0.9*len(words)) Xtr, Ytr = build_dataset(words[:n1]) # 80% Xdev, Ydev = build_dataset(words[n1:n2]) # 10% Xte, Yte = build_dataset(words[n2:]) # 10%

torch.Size([182625, 3]) torch.Size([182625])
torch.Size([22655, 3]) torch.Size([22655])
torch.Size([22866, 3]) torch.Size([22866])

In [6]:

# utility function we will use later when comparing manual gradients to PyTorch gradients
def cmp(s, dt, t):
  ex = torch.all(dt == t.grad).item()
  app = torch.allclose(dt, t.grad)
  maxdiff = (dt - t.grad).abs().max().item()
  print(f'{s:15s} | exact: {str(ex):5s} | approximate: {str(app):5s} | maxdiff: {maxdiff}')

# utility function we will use later when comparing manual gradients to PyTorch gradients def cmp(s, dt, t): ex = torch.all(dt == t.grad).item() app = torch.allclose(dt, t.grad) maxdiff = (dt - t.grad).abs().max().item() print(f'{s:15s} | exact: {str(ex):5s} | approximate: {str(app):5s} | maxdiff: {maxdiff}')

---

FINAL CODE

1:50:03 to 1:54:25 - Putting all of the codes together to form a Neural Net, but by commenting out the loss.backward() :)

In [7]:

# Exercise 4: putting it all together!
# Train the MLP neural net with your own backward pass

# init
n_embd = 10 # the dimensionality of the character embedding vectors
n_hidden = 200 # the number of neurons in the hidden layer of the MLP

g = torch.Generator().manual_seed(2147483647) # for reproducibility
C  = torch.randn((vocab_size, n_embd),            generator=g)
# Layer 1
W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5)
b1 = torch.randn(n_hidden,                        generator=g) * 0.1
# Layer 2
W2 = torch.randn((n_hidden, vocab_size),          generator=g) * 0.1
b2 = torch.randn(vocab_size,                      generator=g) * 0.1
# BatchNorm parameters
bngain = torch.randn((1, n_hidden))*0.1 + 1.0
bnbias = torch.randn((1, n_hidden))*0.1

parameters = [C, W1, b1, W2, b2, bngain, bnbias]
print(sum(p.nelement() for p in parameters)) # number of parameters in total
for p in parameters:
  p.requires_grad = True

# same optimization as last time
max_steps = 200000
batch_size = 32
n = batch_size # convenience
lossi = []

# use this context manager for efficiency once your backward pass is written (TODO)
with torch.no_grad():

  # kick off optimization
  for i in range(max_steps):

    # minibatch construct
    ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)
    Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y

    # forward pass
    emb = C[Xb] # embed the characters into vectors
    embcat = emb.view(emb.shape[0], -1) # concatenate the vectors
    # Linear layer
    hprebn = embcat @ W1 + b1 # hidden layer pre-activation
    # BatchNorm layer
    # -------------------------------------------------------------
    bnmean = hprebn.mean(0, keepdim=True)
    bnvar = hprebn.var(0, keepdim=True, unbiased=True)
    bnvar_inv = (bnvar + 1e-5)**-0.5
    bnraw = (hprebn - bnmean) * bnvar_inv
    hpreact = bngain * bnraw + bnbias
    # -------------------------------------------------------------
    # Non-linearity
    h = torch.tanh(hpreact) # hidden layer
    logits = h @ W2 + b2 # output layer
    loss = F.cross_entropy(logits, Yb) # loss function

    # backward pass
    for p in parameters:
      p.grad = None
    #loss.backward() # use this for correctness comparisons, delete it later!

    # manual backprop! #swole_doge_meme
    # -----------------
    dlogits = F.softmax(logits, 1)
    dlogits[range(n), Yb] -= 1
    dlogits /= n
    # 2nd layer backprop
    dh = dlogits @ W2.T
    dW2 = h.T @ dlogits
    db2 = dlogits.sum(0)
    # tanh
    dhpreact = (1.0 - h**2) * dh
    # batchnorm backprop
    dbngain = (bnraw * dhpreact).sum(0, keepdim=True)
    dbnbias = dhpreact.sum(0, keepdim=True)
    dhprebn = bngain*bnvar_inv/n * (n*dhpreact - dhpreact.sum(0) - n/(n-1)*bnraw*(dhpreact*bnraw).sum(0))
    # 1st layer
    dembcat = dhprebn @ W1.T
    dW1 = embcat.T @ dhprebn
    db1 = dhprebn.sum(0)
    # embedding
    demb = dembcat.view(emb.shape)
    dC = torch.zeros_like(C)
    for k in range(Xb.shape[0]):
      for j in range(Xb.shape[1]):
        ix = Xb[k,j]
        dC[ix] += demb[k,j]
    grads = [dC, dW1, db1, dW2, db2, dbngain, dbnbias]
    # -----------------

    # update
    lr = 0.1 if i < 100000 else 0.01 # step learning rate decay
    for p, grad in zip(parameters, grads):
      #p.data += -lr * p.grad # old way of cheems doge (using PyTorch grad from .backward())
      p.data += -lr * grad # new way of swole doge TODO: enable

    # track stats
    if i % 10000 == 0: # print every once in a while
      print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')
    lossi.append(loss.log10().item())

  #   if i >= 100: # TODO: delete early breaking when you're ready to train the full net
  #     break

# Exercise 4: putting it all together! # Train the MLP neural net with your own backward pass # init n_embd = 10 # the dimensionality of the character embedding vectors n_hidden = 200 # the number of neurons in the hidden layer of the MLP g = torch.Generator().manual_seed(2147483647) # for reproducibility C = torch.randn((vocab_size, n_embd), generator=g) # Layer 1 W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5) b1 = torch.randn(n_hidden, generator=g) * 0.1 # Layer 2 W2 = torch.randn((n_hidden, vocab_size), generator=g) * 0.1 b2 = torch.randn(vocab_size, generator=g) * 0.1 # BatchNorm parameters bngain = torch.randn((1, n_hidden))*0.1 + 1.0 bnbias = torch.randn((1, n_hidden))*0.1 parameters = [C, W1, b1, W2, b2, bngain, bnbias] print(sum(p.nelement() for p in parameters)) # number of parameters in total for p in parameters: p.requires_grad = True # same optimization as last time max_steps = 200000 batch_size = 32 n = batch_size # convenience lossi = [] # use this context manager for efficiency once your backward pass is written (TODO) with torch.no_grad(): # kick off optimization for i in range(max_steps): # minibatch construct ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g) Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y # forward pass emb = C[Xb] # embed the characters into vectors embcat = emb.view(emb.shape[0], -1) # concatenate the vectors # Linear layer hprebn = embcat @ W1 + b1 # hidden layer pre-activation # BatchNorm layer # ------------------------------------------------------------- bnmean = hprebn.mean(0, keepdim=True) bnvar = hprebn.var(0, keepdim=True, unbiased=True) bnvar_inv = (bnvar + 1e-5)**-0.5 bnraw = (hprebn - bnmean) * bnvar_inv hpreact = bngain * bnraw + bnbias # ------------------------------------------------------------- # Non-linearity h = torch.tanh(hpreact) # hidden layer logits = h @ W2 + b2 # output layer loss = F.cross_entropy(logits, Yb) # loss function # backward pass for p in parameters: p.grad = None #loss.backward() # use this for correctness comparisons, delete it later! # manual backprop! #swole_doge_meme # ----------------- dlogits = F.softmax(logits, 1) dlogits[range(n), Yb] -= 1 dlogits /= n # 2nd layer backprop dh = dlogits @ W2.T dW2 = h.T @ dlogits db2 = dlogits.sum(0) # tanh dhpreact = (1.0 - h**2) * dh # batchnorm backprop dbngain = (bnraw * dhpreact).sum(0, keepdim=True) dbnbias = dhpreact.sum(0, keepdim=True) dhprebn = bngain*bnvar_inv/n * (n*dhpreact - dhpreact.sum(0) - n/(n-1)*bnraw*(dhpreact*bnraw).sum(0)) # 1st layer dembcat = dhprebn @ W1.T dW1 = embcat.T @ dhprebn db1 = dhprebn.sum(0) # embedding demb = dembcat.view(emb.shape) dC = torch.zeros_like(C) for k in range(Xb.shape[0]): for j in range(Xb.shape[1]): ix = Xb[k,j] dC[ix] += demb[k,j] grads = [dC, dW1, db1, dW2, db2, dbngain, dbnbias] # ----------------- # update lr = 0.1 if i < 100000 else 0.01 # step learning rate decay for p, grad in zip(parameters, grads): #p.data += -lr * p.grad # old way of cheems doge (using PyTorch grad from .backward()) p.data += -lr * grad # new way of swole doge TODO: enable # track stats if i % 10000 == 0: # print every once in a while print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}') lossi.append(loss.log10().item()) # if i >= 100: # TODO: delete early breaking when you're ready to train the full net # break

12297
      0/ 200000: 3.8069
  10000/ 200000: 2.1598
  20000/ 200000: 2.4110
  30000/ 200000: 2.4295
  40000/ 200000: 2.0158
  50000/ 200000: 2.4050
  60000/ 200000: 2.3825
  70000/ 200000: 2.0596
  80000/ 200000: 2.3024
  90000/ 200000: 2.2073
 100000/ 200000: 2.0443
 110000/ 200000: 2.2937
 120000/ 200000: 2.0340
 130000/ 200000: 2.4557
 140000/ 200000: 2.2876
 150000/ 200000: 2.2016
 160000/ 200000: 1.9720
 170000/ 200000: 1.8015
 180000/ 200000: 2.0065
 190000/ 200000: 1.9932

In [ ]:

# Looking at this, probably the batch norm layer backward pass was the most complicated one
# Otherwise, the rest of them were pretty straight forward :)

# Looking at this, probably the batch norm layer backward pass was the most complicated one # Otherwise, the rest of them were pretty straight forward :)

In [ ]:

# useful for checking your gradients
# for p,g in zip(parameters, grads):
#   cmp(str(tuple(p.shape)), g, p)

# useful for checking your gradients # for p,g in zip(parameters, grads): # cmp(str(tuple(p.shape)), g, p)

In [8]:

# calibrate the batch norm at the end of training

with torch.no_grad():
  # pass the training set through
  emb = C[Xtr]
  embcat = emb.view(emb.shape[0], -1)
  hpreact = embcat @ W1 + b1
  # measure the mean/std over the entire training set
  bnmean = hpreact.mean(0, keepdim=True)
  bnvar = hpreact.var(0, keepdim=True, unbiased=True)

# calibrate the batch norm at the end of training with torch.no_grad(): # pass the training set through emb = C[Xtr] embcat = emb.view(emb.shape[0], -1) hpreact = embcat @ W1 + b1 # measure the mean/std over the entire training set bnmean = hpreact.mean(0, keepdim=True) bnvar = hpreact.var(0, keepdim=True, unbiased=True)

In [9]:

# evaluate train and val loss

@torch.no_grad() # this decorator disables gradient tracking
def split_loss(split):
  x,y = {
    'train': (Xtr, Ytr),
    'val': (Xdev, Ydev),
    'test': (Xte, Yte),
  }[split]
  emb = C[x] # (N, block_size, n_embd)
  embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)
  hpreact = embcat @ W1 + b1
  hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias
  h = torch.tanh(hpreact) # (N, n_hidden)
  logits = h @ W2 + b2 # (N, vocab_size)
  loss = F.cross_entropy(logits, y)
  print(split, loss.item())

split_loss('train')
split_loss('val')

# evaluate train and val loss @torch.no_grad() # this decorator disables gradient tracking def split_loss(split): x,y = { 'train': (Xtr, Ytr), 'val': (Xdev, Ydev), 'test': (Xte, Yte), }[split] emb = C[x] # (N, block_size, n_embd) embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd) hpreact = embcat @ W1 + b1 hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias h = torch.tanh(hpreact) # (N, n_hidden) logits = h @ W2 + b2 # (N, vocab_size) loss = F.cross_entropy(logits, y) print(split, loss.item()) split_loss('train') split_loss('val')

train 2.0708959102630615
val 2.1080715656280518

In [ ]:

# Okay probably relatively slightly lower but thats cool

# Okay probably relatively slightly lower but thats cool

In [10]:

# sample from the model
g = torch.Generator().manual_seed(2147483647 + 10)

for _ in range(20):
    
    out = []
    context = [0] * block_size # initialize with all ...
    while True:
      # ------------
      # forward pass:
      # Embedding
      emb = C[torch.tensor([context])] # (1,block_size,d)      
      embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)
      hpreact = embcat @ W1 + b1
      hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias
      h = torch.tanh(hpreact) # (N, n_hidden)
      logits = h @ W2 + b2 # (N, vocab_size)
      # ------------
      # Sample
      probs = F.softmax(logits, dim=1)
      ix = torch.multinomial(probs, num_samples=1, generator=g).item()
      context = context[1:] + [ix]
      out.append(ix)
      if ix == 0:
        break
    
    print(''.join(itos[i] for i in out))

# sample from the model g = torch.Generator().manual_seed(2147483647 + 10) for _ in range(20): out = [] context = [0] * block_size # initialize with all ... while True: # ------------ # forward pass: # Embedding emb = C[torch.tensor([context])] # (1,block_size,d) embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd) hpreact = embcat @ W1 + b1 hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias h = torch.tanh(hpreact) # (N, n_hidden) logits = h @ W2 + b2 # (N, vocab_size) # ------------ # Sample probs = F.softmax(logits, dim=1) ix = torch.multinomial(probs, num_samples=1, generator=g).item() context = context[1:] + [ix] out.append(ix) if ix == 0: break print(''.join(itos[i] for i in out))

mora.
mayah.
see.
mad.
ryla.
reisha.
endraegan.
chedielin.
shi.
jen.
eden.
sana.
arleigh.
malaia.
noshubergshira.
sten.
joselle.
jose.
casubenteda.
jamell.

In [ ]:

# I've definetly got some wayyy better names here through most are gibberish xD

# I've definetly got some wayyy better names here through most are gibberish xD

And that marks the end of exploring the basic understanding of the 'intuition' of training NN using (traditional) methods. We will be moving on to more complex ones from here on - like RNN etc. So looking forward to that :)