Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "590b6432",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "269576"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('Rabindranath.txt','r',encoding='utf-8') as f:\n",
    "    text = f.read()\n",
    "len(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2aa61af1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Rabindranath Tagore\\n- poems -\\n\\n\\n\\n\\nPublication Date:\\n 2012\\nPublisher:\\nPoemhunter.com - The World's Po\""
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d9b5fab1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "84"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars = sorted(list(set(text)))\n",
    "vocab_size = len(chars)\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c74c873d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[42, 51, 52, 59, 64, 54, 68, 51, 64, 51, 70, 58, 1, 44, 51, 57, 65, 68, 55]\n",
      "Rabindranath Tagore\n"
     ]
    }
   ],
   "source": [
    "str2int = {ch:i for i,ch in enumerate(chars)}\n",
    "int2str = {i:ch for i,ch in enumerate(chars)}\n",
    "encode = lambda s:[str2int[c] for c in s]\n",
    "decode = lambda l:''.join([int2str[n] for n in l])\n",
    "\n",
    "e = encode('Rabindranath Tagore')\n",
    "print(e)\n",
    "d = decode(e)\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "488ef29c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([269576])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([42, 51, 52, 59, 64, 54, 68, 51, 64, 51, 70, 58,  1, 44, 51, 57, 65, 68,\n",
       "        55,  0, 10,  1, 66, 65, 55, 63, 69,  1, 10,  0,  0,  0,  0,  0, 40, 71,\n",
       "        52, 62, 59, 53, 51, 70, 59, 65, 64,  1, 28, 51, 70, 55, 22,  0,  1, 14,\n",
       "        12, 13, 14,  0, 40, 71, 52, 62, 59, 69, 58, 55, 68, 22,  0, 40, 65, 55,\n",
       "        63, 58, 71, 64, 70, 55, 68, 11, 53, 65, 63,  1, 10,  1, 44, 58, 55,  1,\n",
       "        47, 65, 68, 62, 54,  6, 69,  1, 40, 65])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "data = torch.tensor(encode(text),dtype=torch.long)\n",
    "print(data.shape)\n",
    "data[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "245832fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = int(0.9*len(data))\n",
    "train_data = data[:n]\n",
    "val_data = data[n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "bd34582a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([42, 51, 52, 59, 64, 54, 68, 51, 64])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block_size = 8\n",
    "train_data[:block_size+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "0ab41431",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "when context is tensor([42]) then target: 51\n",
      "when context is tensor([42, 51]) then target: 52\n",
      "when context is tensor([42, 51, 52]) then target: 59\n",
      "when context is tensor([42, 51, 52, 59]) then target: 64\n",
      "when context is tensor([42, 51, 52, 59, 64]) then target: 54\n",
      "when context is tensor([42, 51, 52, 59, 64, 54]) then target: 68\n",
      "when context is tensor([42, 51, 52, 59, 64, 54, 68]) then target: 51\n",
      "when context is tensor([42, 51, 52, 59, 64, 54, 68, 51]) then target: 64\n"
     ]
    }
   ],
   "source": [
    "x = train_data[:block_size]\n",
    "y = train_data[1:block_size+1]\n",
    "for t in range(block_size):\n",
    "    context = x[:t+1]\n",
    "    target = y[t]\n",
    "    print(f'when context is {context} then target: {target}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "23ba1dad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([4, 8]), torch.Size([4, 8]))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(7)\n",
    "batch_size = 4 # size of parallel batches of block_size (batch_dimension)\n",
    "block_size = 8 # size of chunk of data we process (time_dimension)\n",
    "\n",
    "def get_batch(split):\n",
    "    # to get random chunk of data for each training or validation\n",
    "    data = train_data if split == 'train' else val_data\n",
    "    idx = torch.randint(len(data)-block_size-1,(batch_size,))\n",
    "    x = torch.stack([data[i:block_size+i] for i in idx])\n",
    "    y = torch.stack([data[i+1:block_size+i+1] for i in idx])\n",
    "    return x,y\n",
    "\n",
    "xb,yb = get_batch('train')\n",
    "xb.shape,yb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "cefea44c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{tensor([51]): tensor(72)}\n",
      "{tensor([51, 72]): tensor(55)}\n",
      "{tensor([51, 72, 55]): tensor(1)}\n",
      "{tensor([51, 72, 55,  1]): tensor(70)}\n",
      "{tensor([51, 72, 55,  1, 70]): tensor(58)}\n",
      "{tensor([51, 72, 55,  1, 70, 58]): tensor(55)}\n",
      "{tensor([51, 72, 55,  1, 70, 58, 55]): tensor(0)}\n",
      "{tensor([51, 72, 55,  1, 70, 58, 55,  0]): tensor(69)}\n",
      "{tensor([51]): tensor(64)}\n",
      "{tensor([51, 64]): tensor(54)}\n",
      "{tensor([51, 64, 54]): tensor(69)}\n",
      "{tensor([51, 64, 54, 69]): tensor(1)}\n",
      "{tensor([51, 64, 54, 69,  1]): tensor(73)}\n",
      "{tensor([51, 64, 54, 69,  1, 73]): tensor(58)}\n",
      "{tensor([51, 64, 54, 69,  1, 73, 58]): tensor(65)}\n",
      "{tensor([51, 64, 54, 69,  1, 73, 58, 65]): tensor(1)}\n",
      "{tensor([54]): tensor(65)}\n",
      "{tensor([54, 65]): tensor(73)}\n",
      "{tensor([54, 65, 73]): tensor(64)}\n",
      "{tensor([54, 65, 73, 64]): tensor(1)}\n",
      "{tensor([54, 65, 73, 64,  1]): tensor(71)}\n",
      "{tensor([54, 65, 73, 64,  1, 71]): tensor(66)}\n",
      "{tensor([54, 65, 73, 64,  1, 71, 66]): tensor(65)}\n",
      "{tensor([54, 65, 73, 64,  1, 71, 66, 65]): tensor(64)}\n",
      "{tensor([1]): tensor(31)}\n",
      "{tensor([ 1, 31]): tensor(51)}\n",
      "{tensor([ 1, 31, 51]): tensor(68)}\n",
      "{tensor([ 1, 31, 51, 68]): tensor(54)}\n",
      "{tensor([ 1, 31, 51, 68, 54]): tensor(55)}\n",
      "{tensor([ 1, 31, 51, 68, 54, 55]): tensor(64)}\n",
      "{tensor([ 1, 31, 51, 68, 54, 55, 64]): tensor(55)}\n",
      "{tensor([ 1, 31, 51, 68, 54, 55, 64, 55]): tensor(68)}\n"
     ]
    }
   ],
   "source": [
    "for b in range(batch_size): # (batch_dimension)\n",
    "    for t in range(block_size): # (time_dimension)\n",
    "        context = xb[b,:t+1]\n",
    "        target = yb[b,t]\n",
    "        print({context:target})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5adc5621",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d5831f4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 84])\n",
      "tensor(5.1680, grad_fn=<NllLossBackward0>)\n",
      "torch.Size([1, 101])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n3NyOGn7rE !6v?e8(H7–06…7D19:Hdnz—3vVF4Pb“VErS1–JCivF—4 6b’FOV”qcvo\"vi!ayhVGn`”h\"TU\\nQ“o“roVB1o\\'VyK&NQ'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "torch.manual_seed(7)\n",
    "\n",
    "class BigramModel(nn.Module):\n",
    "    def __init__(self,vocab_size):\n",
    "        super().__init__()\n",
    "        # lookup table for tokens\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size,vocab_size)\n",
    "        \n",
    "    def forward(self,idx,target=None):\n",
    "        # for each index in idx it reutrns token rows\n",
    "        logits = self.token_embedding_table(idx) # return (B,T,C) (Batch,Time,Channel)\n",
    "        if target is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B,T,C = logits.shape\n",
    "            logits = logits.view(B*T,C) # since cross_entropy accept differnt dime of input (B,C,T)\n",
    "            target = target.view(B*T)\n",
    "            loss = F.cross_entropy(logits,target)\n",
    "        return logits,loss\n",
    "\n",
    "    def generate(self,idx,max_tokens):\n",
    "        for _ in range(max_tokens):\n",
    "            logits,loss = self.forward(idx) # getting predictions\n",
    "             # taking only the last idx prediction --> (B,C)\n",
    "            logits = logits[:,-1,:] # (B,C)\n",
    "            # softing for get probabilities\n",
    "            probs = F.softmax(logits,dim=-1) # (B,C)\n",
    "            # get one sample index from given probabilities and add to end of idx\n",
    "            sample_idx = torch.multinomial(probs,num_samples=1)\n",
    "            idx = torch.cat((idx,sample_idx),dim=1) # (B,T+1)\n",
    "        return idx\n",
    "    \n",
    "m = BigramModel(vocab_size)\n",
    "logits, loss = m(xb,yb)\n",
    "print(logits.shape)\n",
    "print(loss)\n",
    "\n",
    "\n",
    "# sample text to test the model\n",
    "input_idx = torch.zeros((1,1), dtype = torch.long)\n",
    "# generated tokens of length 100\n",
    "g_idx = m.generate(input_idx,100)\n",
    "\n",
    "print(g_idx.shape)\n",
    "decode(g_idx[0].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ab88ba0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method Module.parameters of BigramModel(\n",
      "  (token_embedding_table): Embedding(84, 84)\n",
      ")>\n"
     ]
    }
   ],
   "source": [
    "print(m.parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "99a8a4be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pytorch optimizer model AdamW\n",
    "optimizer = torch.optim.AdamW(m.parameters(),lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8322d821",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.034038543701172\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "for _ in range(1000):\n",
    "    xb,yb = get_batch('train')\n",
    "    logits, loss = m(xb,yb)\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "65c86be6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 301])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n6!o C\\'l!e5GDykjugguo ’o\"nmo\"SLD1 Ob 9`U&2VB8rS2p,Q\\'rUX0E3u--–cvO\\'s-kCd,NX9c130PyA\";oXaXbkTaTgi60CU2’rGcCGn#)Yhj6.2`IixbbuVu&;1gh-gEAAS5U’?2b!0!–EzQk\\'D.2-&C1…2#sM;paEO#sF8z#kxTByoh’-dD&FJbnvu12qjXl.Abf?BenUAPHP:RMm‘EHthdJ:MrLryzeU;:”(aTRN3gi.…czbfsaQzmmssu?; K`“QAP)Q#6Tg3u 2q#fN7y G,-c2SBvaIOQgK2B70,'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sample text to test the model\n",
    "input_idx = torch.zeros((1,1), dtype = torch.long)\n",
    "# generated tokens of length 100\n",
    "g_idx = m.generate(input_idx,300)\n",
    "\n",
    "print(g_idx.shape)\n",
    "decode(g_idx[0].tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2529d5e4",
   "metadata": {},
   "source": [
    "# Self attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "19687ddd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a \n",
      " tensor([[1.0000, 0.0000, 0.0000],\n",
      "        [0.5000, 0.5000, 0.0000],\n",
      "        [0.3333, 0.3333, 0.3333]])\n",
      "\n",
      "b \n",
      " tensor([[5., 2.],\n",
      "        [1., 6.],\n",
      "        [3., 7.]])\n",
      "\n",
      "mean of each row\n",
      "tensor([[5., 2.],\n",
      "        [3., 4.],\n",
      "        [3., 5.]])\n"
     ]
    }
   ],
   "source": [
    "# self attention\n",
    "torch.manual_seed(7)\n",
    "a = torch.tril(torch.ones(3,3))\n",
    "a = a/torch.sum(a,1,keepdim=True)\n",
    "b = torch.randint(0,10,(3,2)).float()\n",
    "\n",
    "print('a \\n',a)\n",
    "print()\n",
    "print('b \\n',b)\n",
    "print()\n",
    "c = a @ b\n",
    "\n",
    "print('mean of each row')\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "c45041b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 4, 2])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let \n",
    "torch.manual_seed(7)\n",
    "B,T,C = 4,4,2\n",
    "x = torch.rand(B,T,C)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "ef38a347",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.5349, 0.1988],\n",
       "         [0.5971, 0.4278],\n",
       "         [0.4756, 0.4269],\n",
       "         [0.4085, 0.4776]],\n",
       "\n",
       "        [[0.3653, 0.8513],\n",
       "         [0.6101, 0.7011],\n",
       "         [0.5024, 0.5362],\n",
       "         [0.4880, 0.4920]],\n",
       "\n",
       "        [[0.7204, 0.0731],\n",
       "         [0.8451, 0.0904],\n",
       "         [0.8577, 0.1980],\n",
       "         [0.8326, 0.3222]],\n",
       "\n",
       "        [[0.5209, 0.5932],\n",
       "         [0.7003, 0.6109],\n",
       "         [0.7220, 0.4450],\n",
       "         [0.7555, 0.5018]]])"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1 way\n",
    "xbow = torch.zeros((B,T,C))\n",
    "for b in range(B):\n",
    "    for t in range(T):\n",
    "        xprev = x[b,:t+1]\n",
    "        xbow[b,t] = torch.mean(xprev,dim=0)\n",
    "xbow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "22989081",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2nd way\n",
    "wei = torch.tril(torch.ones(T,T))\n",
    "wei = wei/torch.sum(wei,1,keepdim=True)\n",
    "xbow1 = wei @ x\n",
    "torch.allclose(xbow1,xbow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "e034824e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.5349, 0.1988],\n",
       "         [0.5971, 0.4278],\n",
       "         [0.4756, 0.4269],\n",
       "         [0.4085, 0.4776]],\n",
       "\n",
       "        [[0.3653, 0.8513],\n",
       "         [0.6101, 0.7011],\n",
       "         [0.5024, 0.5362],\n",
       "         [0.4880, 0.4920]],\n",
       "\n",
       "        [[0.7204, 0.0731],\n",
       "         [0.8451, 0.0904],\n",
       "         [0.8577, 0.1980],\n",
       "         [0.8326, 0.3222]],\n",
       "\n",
       "        [[0.5209, 0.5932],\n",
       "         [0.7003, 0.6109],\n",
       "         [0.7220, 0.4450],\n",
       "         [0.7555, 0.5018]]])"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xbow1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "385a582e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.5349, 0.1988],\n",
       "         [0.5971, 0.4278],\n",
       "         [0.4756, 0.4269],\n",
       "         [0.4085, 0.4776]],\n",
       "\n",
       "        [[0.3653, 0.8513],\n",
       "         [0.6101, 0.7011],\n",
       "         [0.5024, 0.5362],\n",
       "         [0.4880, 0.4920]],\n",
       "\n",
       "        [[0.7204, 0.0731],\n",
       "         [0.8451, 0.0904],\n",
       "         [0.8577, 0.1980],\n",
       "         [0.8326, 0.3222]],\n",
       "\n",
       "        [[0.5209, 0.5932],\n",
       "         [0.7003, 0.6109],\n",
       "         [0.7220, 0.4450],\n",
       "         [0.7555, 0.5018]]])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3rd way\n",
    "tril = torch.tril(torch.ones(T,T))\n",
    "wei = torch.zeros(T,T)\n",
    "wei = wei.masked_fill(tril == 0,float('-inf'))\n",
    "wei = F.softmax(wei,dim=-1)\n",
    "xbow2 = wei @ x\n",
    "xbow2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "6d209eaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.allclose(xbow,xbow2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "dab1ed3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#version 4 for self attention\n",
    "torch.manual_seed(7)\n",
    "B,T,C = 4,8,32\n",
    "x = torch.randn(B,T,C)\n",
    "# single head performing self attention\n",
    "head_size = 16\n",
    "key = nn.Linear(C,head_size,bias=False)\n",
    "query = nn.Linear(C,head_size,bias=False)\n",
    "value = nn.Linear(C,head_size,bias=False)\n",
    "k = key(x) # (B,T,16)\n",
    "q = query(x) # (B,T,16)\n",
    "v = value(x)\n",
    "\n",
    "wei = q @ k.transpose(-2,-1) # (B,T,16) @ (B,16,T) --> (B,T,T)\n",
    "\n",
    "tril = torch.tril(torch.ones(T,T))\n",
    "wei = wei.masked_fill(tril == 0, float('-inf'))\n",
    "wei = F.softmax(wei,dim=-1)\n",
    "\n",
    "# out = wei @ x\n",
    "out = wei @ v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "65d7eaa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 8, 16])"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "f87c8a0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.3683, 0.6317, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.3519, 0.5907, 0.0574, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.3486, 0.1974, 0.1348, 0.3193, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0112, 0.1489, 0.1402, 0.0980, 0.6017, 0.0000, 0.0000, 0.0000],\n",
       "        [0.1663, 0.0633, 0.1945, 0.5067, 0.0494, 0.0198, 0.0000, 0.0000],\n",
       "        [0.1435, 0.0190, 0.0247, 0.0396, 0.5494, 0.1052, 0.1185, 0.0000],\n",
       "        [0.0690, 0.3477, 0.0400, 0.0806, 0.2945, 0.0274, 0.0459, 0.0949]],\n",
       "       grad_fn=<SelectBackward0>)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wei[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "2795621f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.5680,  0.9682,  1.0205, -0.7679, -0.4110,  0.0781, -0.5025, -0.4608,\n",
       "         -0.0136,  0.5688, -0.4703, -0.6915, -0.9830,  0.1170, -0.1094, -0.4791],\n",
       "        [-0.6052,  0.1062, -0.3205,  0.6433, -0.0320,  0.0016,  0.4219, -0.2060,\n",
       "         -0.4843,  0.1708, -0.3733, -0.4373, -0.5565,  0.6194,  0.2879,  0.3988],\n",
       "        [-0.5433,  0.0536, -0.3201,  0.6323, -0.0060, -0.0088,  0.3987, -0.2368,\n",
       "         -0.4794,  0.1564, -0.3282, -0.4107, -0.5633,  0.6294,  0.2876,  0.3543],\n",
       "        [ 0.1141,  0.2567, -0.0596, -0.0305,  0.0438,  0.0878, -0.0655, -0.3837,\n",
       "         -0.5834, -0.0459, -0.0787, -0.5164, -0.6511,  0.2166,  0.1639,  0.1029],\n",
       "        [ 0.1636, -0.2151,  0.0176,  0.3056,  0.1306,  0.3668,  0.0538, -0.1521,\n",
       "         -0.1427, -0.2161, -0.1026, -0.1479,  0.0219,  0.3281, -0.0691, -0.0648],\n",
       "        [ 0.2963,  0.1498, -0.1679, -0.1112,  0.1672,  0.1563, -0.1724, -0.4091,\n",
       "         -0.7127, -0.2730,  0.1151, -0.4793, -0.5435,  0.0611,  0.1502,  0.1074],\n",
       "        [ 0.2294, -0.0054,  0.3894, -0.0113, -0.1068,  0.2466, -0.2461, -0.0927,\n",
       "         -0.0144, -0.0484, -0.1492, -0.1550,  0.0406,  0.2135, -0.1675, -0.2386],\n",
       "        [-0.2957, -0.0944, -0.2165,  0.4720,  0.0703,  0.0799,  0.2241, -0.1076,\n",
       "         -0.3233, -0.1255, -0.2164, -0.2301, -0.2003,  0.4977,  0.1253,  0.2182]],\n",
       "       grad_fn=<SelectBackward0>)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "5d7506fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0532, grad_fn=<VarBackward0>)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wei.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "629c247f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1892, grad_fn=<VarBackward0>)"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "0e75249b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.3272, grad_fn=<VarBackward0>)"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "99c4634a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.3398, grad_fn=<VarBackward0>)"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c0f418d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nanogpt",
   "language": "python",
   "name": "nanogpt"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}