# build the vocabulary of characters and mappings to/from integerschars = sorted(list(set(''.join(words))))stoi = {s:i+1 for i,s in enumerate(chars)}stoi['.'] = 0itos = {i:s for s,i in stoi.items()}vocab_size = len(itos)print(itos)print(vocab_size)
# build the datasetblock_size = 3 # context length: how many characters do we take to predict the next one?def build_dataset(words): X, Y = [], [] for w in words: context = [0] * block_size for ch in w + '.': ix = stoi[ch] X.append(context) Y.append(ix) context = context[1:] + [ix] # crop and append X = torch.tensor(X) Y = torch.tensor(Y) print(X.shape, Y.shape) return X, Yimport randomrandom.seed(42)random.shuffle(words)n1 = int(0.8*len(words))n2 = int(0.9*len(words))Xtr, Ytr = build_dataset(words[:n1]) # 80%Xdev, Ydev = build_dataset(words[n1:n2]) # 10%Xte, Yte = build_dataset(words[n2:]) # 10%
# utility function we will use later when comparing manual gradients to PyTorch gradientsdef cmp(s, dt, t): ex = torch.all(dt == t.grad).item() app = torch.allclose(dt, t.grad) maxdiff = (dt - t.grad).abs().max().item() print(f'{s:15s} | exact: {str(ex):5s} | approximate: {str(app):5s} | maxdiff: {maxdiff}')
n_embd = 10 # the dimensionality of the character embedding vectorsn_hidden = 64 # the number of neurons in the hidden layer of the MLPg = torch.Generator().manual_seed(2147483647) # for reproducibilityC = torch.randn((vocab_size, n_embd), generator=g)# Layer 1W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5)b1 = torch.randn(n_hidden, generator=g) * 0.1 # using b1 just for fun, it's useless because of BN# Layer 2W2 = torch.randn((n_hidden, vocab_size), generator=g) * 0.1b2 = torch.randn(vocab_size, generator=g) * 0.1# BatchNorm parametersbngain = torch.randn((1, n_hidden))*0.1 + 1.0bnbias = torch.randn((1, n_hidden))*0.1# Note: I am initializating many of these parameters in non-standard ways# because sometimes initializating with e.g. all zeros could mask an incorrect# implementation of the backward pass.parameters = [C, W1, b1, W2, b2, bngain, bnbias]print(sum(p.nelement() for p in parameters)) # number of parameters in totalfor p in parameters: p.requires_grad = True
Output
4137
batch_size = 32n = batch_size # a shorter variable also, for convenience# construct a minibatchix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y
# forward pass, "chunkated" into smaller steps that are possible to backward one at a timeemb = C[Xb] # embed the characters into vectorsembcat = emb.view(emb.shape[0], -1) # concatenate the vectors# Linear layer 1hprebn = embcat @ W1 + b1 # hidden layer pre-activation# BatchNorm layerbnmeani = 1/n*hprebn.sum(0, keepdim=True)bndiff = hprebn - bnmeanibndiff2 = bndiff**2bnvar = 1/(n-1)*(bndiff2).sum(0, keepdim=True) # note: Bessel's correction (dividing by n-1, not n)bnvar_inv = (bnvar + 1e-5)**-0.5bnraw = bndiff * bnvar_invhpreact = bngain * bnraw + bnbias# Non-linearityh = torch.tanh(hpreact) # hidden layer# Linear layer 2logits = h @ W2 + b2 # output layer# cross entropy loss (same as F.cross_entropy(logits, Yb))logit_maxes = logits.max(1, keepdim=True).valuesnorm_logits = logits - logit_maxes # subtract max for numerical stabilitycounts = norm_logits.exp()counts_sum = counts.sum(1, keepdims=True) #DONEcounts_sum_inv = counts_sum**-1 # if I use (1.0 / counts_sum) instead then I can't get backprop to be bit exact... #DONEprobs = counts * counts_sum_inv #DONElogprobs = probs.log() #DONEloss = -logprobs[range(n), Yb].mean() #DONE# PyTorch backward passfor p in parameters: p.grad = Nonefor t in [logprobs, probs, counts, counts_sum, counts_sum_inv, # afaik there is no cleaner way norm_logits, logit_maxes, logits, h, hpreact, bnraw, bnvar_inv, bnvar, bndiff2, bndiff, hprebn, bnmeani, embcat, emb]: t.retain_grad()loss.backward()loss
Output
tensor(3.3221, grad_fn=<NegBackward0>)
EXERCISE 1
13:01 to 19:05cmp('logprobs', dlogprobs, logprobs)
#simple breakdown#now here we know there are 32 examples, for explanation lets assume we only have 3 in total i.e. a,b,c#loss = - (a + b + c) / 3 ==> so we are basically doing the mean calculation here#loss = - (1/3a + 1/3b + 1/3c) ==> same equation#so now, when we take the derivative wrt a#dloss/da = -1/3 ==>where 3 is the number of elements we consider, so we can also say that it is -1/n, therefore#dloss/dn = -1/n
#So based on our calculation abovedlogprobs = torch.zeros_like(logprobs) #same as torch.zeros((32, 27)) as we need the same shape as logprobs. So instead of hardcoding the values we did thisdlogprobs[range(n), Yb] = -1.0/n #as we need to do it for each of the elements in the array#Now, lets checkcmp('logprobs', dlogprobs, logprobs)
dprobs = (1.0/probs) * dlogprobs #we had to take the derivative of logprobs, which was 1/x --> d/dx(log(x)) = 1/x #then we multiplied it with dlogprobs (the one we calculated before this for the chainrule)cmp('probs', dprobs, probs)
20:56 to 26:21cmp('counts_sum_inv', dcounts_sum_inv, counts_sum_inv)
# probs = counts * counts_sum_inv, now here before we do the multiplication, take a look at the matrix dimensions using `.shape`# You would see that `counts` would have 3x3 and `counts_sum_inv` will have 3x1# So before the backpropagation calculation, there is 'broadcasting' happening where the value of b is been replicated/broadcasted multiple time across the matrix# Example# c = a * b# a[3x3] * b[3x1] ----># a[1,1]*b1 + a[1,2]*b1 + a[1,3]*b1# a[2,1]*b2 + a[2,2]*b2 + a[2,3]*b2# a[3,1]*b3 + a[3,2]*b3 + a[2,3]*b3# ====> c[3x3]# The point of this is just to show that there are two things happening internally: The broadcasting and then the backpropagation# (first case) The derivative of c wrt b will be a# So, here just `counts` will remain -> then `dprobs` is multiplied because chain rule.# Finally, in order to make `dcounts_sum_inv` the same dimension as `counts_sum_inv` we sum all of them by 1 and also keepdims as truedcounts_sum_inv = (counts * dprobs).sum(1, keepdims=True) # So this is our final manually calculated equationcmp('counts_sum_inv', dcounts_sum_inv, counts_sum_inv)
# Here we have to calculate the second half of `dcounts` i.e. (Second case) The derivative of c wrt a will be bdcounts = counts_sum_inv * dprobs#but we cant compare it yet as `counts` is later depended on top again as well, which we will check
27:57 to 28:59cmp('counts_sum', dcounts_sum, counts_sum)
# counts_sum_inv = counts_sum**-1# Okay so for this, the derivative of x^-1 is -(x^-2)dcounts_sum = (-counts_sum**-2) * dcounts_sum_inv #Remember for this its the one before the `26:26 to 27:56 first contribution of counts` sectioncmp('counts_sum', dcounts_sum, counts_sum)
# counts_sum = counts.sum(1, keepdims=True)# Now here we know the shape of counts_sum is 32 by 1 and the shape of counts is 32 by 27. So we need to broadcast counts_sum 27 times# We are dirctly using a PyTorch function where it keeps adding numbers from `counts`dcounts += torch.ones_like(counts) * dcounts_sum #Also here we are adding `dcounts` as remember this is the second iteration of it, we had calculated one more value of it at the topcmp('counts', dcounts, counts)
32:27 to 33:13cmp('norm_logits', dnorm_logits, norm_logits)
# counts = norm_logits.exp()# Now here, the derivative of `norm_logits.exp()`, now the derivate of e^x is (famously) e^x, so its just `norm_logits.exp()` itself# so we can also just write it as `counts` directly as it holds that valuednorm_logits = counts * dcountscmp('norm_logits', dnorm_logits, norm_logits)
33:14 to 36:20cmp('logit_maxes', dlogit_maxes, logit_maxes)
# norm_logits = logits - logit_maxes# Now here if you would look at the shape of all these variables, you would notice that there is internal broadcasting happening here (logit_maxes)dlogits = dnorm_logits.clone()dlogit_maxes = (-dnorm_logits).sum(1, keepdim=True) #WILL HAVE TO REWATCH THIS PART AGAIN, DIDN'T COMPLETELY GET ITcmp('logit_maxes', dlogit_maxes, logit_maxes)
# logit_maxes = logits.max(1, keepdim=True).values# Here, this step is similar to that of the first one in `dlogprobs` where we used torch.zeros_like() function# So we are doing another alternative way of doing thatdlogits += F.one_hot(logits.max(1).indices, num_classes=logits.shape[1]) * dlogit_maxes #Just remember the += here as we already have one dlogits abovecmp('logits', dlogits, logits)
# watch the last few minutes, probably from 51 to see how he broke down this based on the matrix sizesdh = dlogits @ W2.TdW2 = h.T @ dlogitsdb2 = dlogits.sum(0)cmp('h', dh, h)cmp('W2', dW2, W2)cmp('b2', db2, b2)
# h = torch.tanh(hpreact) # hidden layerdhpreact = (1.0 - h**2)*dh #we saw that the derivative of tanh is also (1-a^2) where a was the external variable `a`, not the input `z` to tanh i.e. a = tanh(z)cmp('hpreact', dhpreact, hpreact)
59:40 to 1:04:1 cmp('bnvar_inv', dbnvar_inv, bnvar_inv)
# From here we are working on the batch norm layer# the code has been spread out and broken down to different parts (based on the equations on the "bottom right corner box" in the paper for batch norm - See prev lecture) inorder to perform manual backprop more easily
# bnraw = bndiff * bnvar_invdbnvar_inv = (bndiff * dbnraw).sum(0, keepdim=True)dbndiff = bnvar_inv * dbnraw #We will come back to this in 1:12:43 - (1)cmp('bnvar_inv', dbnvar_inv, bnvar_inv)
# bnvar_inv = (bnvar + 1e-5)**-0.5#This is a direct equation of derivative of x^n so the output should be n*x^n-1dbnvar = (-0.5 * ((bnvar + 1e-5) ** (-1.5))) * dbnvar_invcmp('bnvar', dbnvar, bnvar)
1:05:17 to 1:09:01 - Why he implemented the bessel's correction (as there seem to be some problem/issue in the paper. Using Bias during training time and Unbiased during testing). But we prefer to use Unbiased during both training and testing and that is what we went ahead with.
1:09:02 to 1:12:42 cmp('bndiff2', dbndiff2, bndiff2)
1:13:59 to 1:18:35 cmp('bnmeani', dbnmeani, bnmeani) and cmp('hprebn', dhprebn, hprebn)
## Please go thorugh this one again, i didnt completely get it# bnmeani = 1/n*hprebn.sum(0, keepdim=True)# bndiff = hprebn - bnmeanidhprebn = dbndiff.clone() #we are making a copy of itdbnmeani = (-dbndiff).sum(0)dhprebn += (1.0/n)*(torch.ones_like(hprebn) * dbnmeani)cmp('bnmeani', dbnmeani, bnmeani)cmp('hprebn', dhprebn, hprebn)
## Please rewatch this as well# emb = C[Xb]dC = torch.zeros_like(C)for k in range(Xb.shape[0]): for j in range(Xb.shape[1]): ix = Xb[k,j] dC[ix] += demb[k,j]cmp('C', dC, C)
Output
C | exact: True | approximate: True | maxdiff: 0.0
And we are done with the first exercise!!
# Exercise 1: backprop through the whole thing manually,# backpropagating through exactly all of the variables# as they are defined in the forward pass above, one by one# -----------------# YOUR CODE HERE :)# -----------------# cmp('logprobs', dlogprobs, logprobs)# cmp('probs', dprobs, probs)# cmp('counts_sum_inv', dcounts_sum_inv, counts_sum_inv)# cmp('counts_sum', dcounts_sum, counts_sum)# cmp('counts', dcounts, counts)# cmp('norm_logits', dnorm_logits, norm_logits)# cmp('logit_maxes', dlogit_maxes, logit_maxes)# cmp('logits', dlogits, logits)# cmp('h', dh, h)# cmp('W2', dW2, W2)# cmp('b2', db2, b2)# cmp('hpreact', dhpreact, hpreact)# cmp('bngain', dbngain, bngain)# cmp('bnbias', dbnbias, bnbias)# cmp('bnraw', dbnraw, bnraw)# cmp('bnvar_inv', dbnvar_inv, bnvar_inv)# cmp('bnvar', dbnvar, bnvar)# cmp('bndiff2', dbndiff2, bndiff2)# cmp('bndiff', dbndiff, bndiff)# cmp('bnmeani', dbnmeani, bnmeani)# cmp('hprebn', dhprebn, hprebn)# cmp('embcat', dembcat, embcat)# cmp('W1', dW1, W1)# cmp('b1', db1, b1)# cmp('emb', demb, emb)# cmp('C', dC, C)
