{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning from predictions\n",
    "\n",
    "**Lecturer:** Vegard H. Larsen\n",
    "\n",
    "--------\n",
    "**Estimating an n-gram Language Models Model using a neural network with a multi-layer perceptron architecture** \n",
    "\n",
    "This will be based on Andrej Karpathys video [Building makemore Part 2: MLP](https://www.youtube.com/watch?v=TCH_1BHY58I&t=3426s)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = open('data/names.txt', 'r').read().splitlines()\n",
    "words[:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32033"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n"
     ]
    }
   ],
   "source": [
    "# Build the vocabulary of characters and mappings to/from integers\n",
    "\n",
    "# Combine all words into a single string and extract unique characters\n",
    "# 'words' is assumed to be a list of strings (our dataset of words)\n",
    "all_chars = ''.join(words)             # Concatenate all words into one long string\n",
    "unique_chars = set(all_chars)          # Create a set of unique characters\n",
    "chars = sorted(list(unique_chars))     # Sort the unique characters alphabetically\n",
    "\n",
    "# Create a mapping from each character to a unique integer index\n",
    "# Starting indices from 1 to reserve 0 for a special token\n",
    "stoi = {s: i+1 for i, s in enumerate(chars)}  # 'stoi' means 'string to integer'\n",
    "\n",
    "# Explicitly map the period character '.' to index 0 (could represent padding or end token)\n",
    "stoi['.'] = 0\n",
    "\n",
    "# Create a reverse mapping from integer index to character\n",
    "itos = {i: s for s, i in stoi.items()}  # 'itos' means 'integer to string'\n",
    "\n",
    "# Print the integer-to-character mapping to verify the mappings\n",
    "print(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182625, 5]) torch.Size([182625])\n",
      "torch.Size([22655, 5]) torch.Size([22655])\n",
      "torch.Size([22866, 5]) torch.Size([22866])\n"
     ]
    }
   ],
   "source": [
    "# Build the dataset for training the language model\n",
    "\n",
    "block_size = 5  # Context length: number of previous characters used to predict the next one\n",
    "\n",
    "import torch  # Import PyTorch library for tensor operations\n",
    "import random  # Import the random module for shuffling\n",
    "\n",
    "def build_dataset(words):\n",
    "    X, Y = [], []  # Initialize empty lists for inputs (X) and targets (Y)\n",
    "\n",
    "    # Loop over a subset of words for demonstration (currently first 3 words)\n",
    "    #for w in words[0:3]:\n",
    "        # For processing the entire dataset, uncomment the next line and comment out the above line\n",
    "    for w in words:\n",
    "        #print(w)  # Print the current word for reference\n",
    "\n",
    "        # Initialize the context with 'block_size' zeros (padding at the beginning)\n",
    "        context = [0] * block_size\n",
    "\n",
    "        # Loop over each character in the word, plus a special end token '.'\n",
    "        for ch in w + '.':\n",
    "            ix = stoi[ch]  # Convert character to integer index using the mapping\n",
    "\n",
    "            X.append(context)  # Append the current context to the inputs\n",
    "            Y.append(ix)       # Append the index of the next character to the targets\n",
    "\n",
    "            # Print the mapping from context to next character for visualization\n",
    "            #print(''.join(itos[i] for i in context), '--->', itos[ix])\n",
    "\n",
    "            # Update the context by removing the oldest character and adding the current one\n",
    "            context = context[1:] + [ix]\n",
    "\n",
    "    # Convert lists to PyTorch tensors for model input\n",
    "    X = torch.tensor(X)\n",
    "    Y = torch.tensor(Y)\n",
    "    print(X.shape, Y.shape)  # Print the shapes of the tensors for verification\n",
    "\n",
    "    return X, Y  # Return the input and target tensors\n",
    "\n",
    "# Set a seed for reproducibility and shuffle the words\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "\n",
    "# Split the data into training, development, and test sets (80%, 10%, 10%)\n",
    "n1 = int(0.8 * len(words))\n",
    "n2 = int(0.9 * len(words))\n",
    "\n",
    "# Build datasets for training, development, and testing\n",
    "Xtr, Ytr = build_dataset(words[:n1])      # Training set\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])  # Development set\n",
    "Xte, Yte = build_dataset(words[n2:])      # Test set\n",
    "\n",
    "# Uncomment the following line to print the training data tensors\n",
    "# print(Xtr, Ytr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0,  0,  0,  0],\n",
       "        [ 0,  0,  0,  0, 25],\n",
       "        [ 0,  0,  0, 25, 21],\n",
       "        ...,\n",
       "        [ 0,  8, 15, 12,  4],\n",
       "        [ 8, 15, 12,  4,  1],\n",
       "        [15, 12,  4,  1, 14]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What does the training data look like?\n",
    "Xtr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the random number generator with a fixed seed for reproducibility\n",
    "g = torch.Generator()\n",
    "\n",
    "# Create the character embedding matrix 'C'\n",
    "# Shape: (27, 10)\n",
    "# - 27 represents the number of unique characters (assuming 26 letters + special character like '.')\n",
    "# - 10 is the dimensionality of the character embeddings\n",
    "C = torch.randn((27, 10), generator=g)\n",
    "\n",
    "# Initialize weights and biases for the first hidden layer (fully connected layer)\n",
    "# W1: Weight matrix for the first layer\n",
    "# Shape: (block_size * 10, 300)\n",
    "# - block_size * 10 is the input size (concatenated embeddings of 'block_size' characters)\n",
    "# - 300 is the number of neurons in the hidden layer\n",
    "W1 = torch.randn((block_size * 10, 300), generator=g)\n",
    "\n",
    "# b1: Bias vector for the first hidden layer\n",
    "# Shape: (300,)\n",
    "b1 = torch.randn(300, generator=g)\n",
    "\n",
    "# Initialize weights and biases for the output layer\n",
    "# W2: Weight matrix for the output layer\n",
    "# Shape: (300, 27)\n",
    "# - 300 is the size of the hidden layer\n",
    "# - 27 is the number of possible output classes (unique characters)\n",
    "W2 = torch.randn((300, 27), generator=g)\n",
    "\n",
    "# b2: Bias vector for the output layer\n",
    "# Shape: (27,)\n",
    "b2 = torch.randn(27, generator=g)\n",
    "\n",
    "# Collect all parameters into a list for optimization\n",
    "parameters = [C, W1, b1, W2, b2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23697"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of parameters in total \n",
    "sum(p.nelement() for p in parameters) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in parameters:\n",
    "  p.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lre = torch.linspace(-3, 0, 1000)\n",
    "lrs = 10**lre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "lri = []\n",
    "lossi = []\n",
    "stepi = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F  # Import PyTorch's functional module for loss functions\n",
    "\n",
    "# Training loop for the neural network model\n",
    "for i in range(200000):  # Loop over a large number of iterations\n",
    "    \n",
    "    # ---------------------------\n",
    "    # Mini-batch Construction\n",
    "    # ---------------------------\n",
    "    \n",
    "    # Randomly sample a batch of 32 indices from the training data\n",
    "    ix = torch.randint(0, Xtr.shape[0], (32,))\n",
    "    \n",
    "    # ---------------------------\n",
    "    # Forward Pass\n",
    "    # ---------------------------\n",
    "    \n",
    "    # Retrieve embeddings for the batch of input contexts\n",
    "    # Xtr[ix]: selects the batch of input indices (contexts)\n",
    "    # emb shape: (32, block_size, embedding_dim)\n",
    "    emb = C[Xtr[ix]]  # Lookup embeddings for each character in the context\n",
    "    \n",
    "    # Reshape embeddings and compute hidden layer activations\n",
    "    # emb.view(-1, block_size * embedding_dim): flatten embeddings\n",
    "    # W1: weights of the first hidden layer\n",
    "    # b1: biases of the first hidden layer\n",
    "    # h: activations after applying the tanh function\n",
    "    # h shape: (32, hidden_size)\n",
    "    h = torch.tanh(emb.view(-1, block_size * 10) @ W1 + b1)\n",
    "    \n",
    "    # Compute logits for the output layer\n",
    "    # logits shape: (32, vocab_size)\n",
    "    logits = h @ W2 + b2\n",
    "    \n",
    "    # Compute the loss using cross-entropy between logits and true labels\n",
    "    # Ytr[ix]: true labels for the batch\n",
    "    loss = F.cross_entropy(logits, Ytr[ix])\n",
    "    \n",
    "    # Optionally, print the loss to monitor training progress\n",
    "    # print(loss.item())\n",
    "    \n",
    "    # ---------------------------\n",
    "    # Backward Pass\n",
    "    # ---------------------------\n",
    "    \n",
    "    # Zero the gradients before backward pass\n",
    "    for p in parameters:\n",
    "        p.grad = None\n",
    "    \n",
    "    # Compute gradients of the loss with respect to parameters\n",
    "    loss.backward()\n",
    "    \n",
    "    # ---------------------------\n",
    "    # Parameter Update\n",
    "    # ---------------------------\n",
    "    \n",
    "    # Define learning rate schedule\n",
    "    # Use a higher learning rate in the first half of training, then decrease it\n",
    "    lr = 0.1 if i < 100000 else 0.01  # Adjust learning rate after 100,000 iterations\n",
    "    \n",
    "    # Update each parameter using gradient descent\n",
    "    for p in parameters:\n",
    "        p.data += -lr * p.grad  # Subtract the gradient scaled by the learning rate\n",
    "    \n",
    "    # ---------------------------\n",
    "    # Tracking Statistics (Optional)\n",
    "    # ---------------------------\n",
    "    \n",
    "    # Append current iteration to step tracker\n",
    "    stepi.append(i)\n",
    "    \n",
    "    # Append the logarithm (base 10) of the loss to the loss tracker\n",
    "    lossi.append(loss.log10().item())\n",
    "    \n",
    "# Optionally, print the final loss after training\n",
    "# print(loss.item())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x12fbe8110>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAT9ZJREFUeJzt3XdclWXjBvDrsBHhKCIgMsQ9cOIeuVEys6lmqaX2ZmmatvS1UlvY0J9vObJcWWZWrkpTMbe4QFDcCwEZIihDUeb9+wM5nMPZhwPngXN9Px8+ynOecT8c4Lm4p0wIIUBEREQkETaWLgARERGRMoYTIiIikhSGEyIiIpIUhhMiIiKSFIYTIiIikhSGEyIiIpIUhhMiIiKSFIYTIiIikhQ7SxfAEMXFxUhOToarqytkMpmli0NEREQGEEIgJycHPj4+sLExvD6kWoST5ORk+Pn5WboYREREZILExET4+voavH+1CCeurq4ASm7Ozc3NwqUhIiIiQ2RnZ8PPz0/xHDdUtQgnpU05bm5uDCdERETVjLFdMtghloiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJKVaLPxXWTZF3URsUhaGBnmje+N6li4OERERwcprTg5cvo21ETdwPjnb0kUhIiKiR6w6nBAREZH0MJwAEJYuABERESlYdTiRySxdAiIiIirPqsMJERERSQ/DCQAh2LBDREQkFVYdTtiqQ0REJD1WHU6IiIhIehhOiIiISFKsOpzIOFyHiIhIcqw6nBAREZH0MJwA4GAdIiIi6bDqcMJGHSIiIumx6nBCRERE0sNwAkBwdR0iIiLJsO5wwnYdIiIiybHucEJERESSw3ACjtYhIiKSEqsOJzK26xAREUmOVYcTIiIikh6GE4BjdYiIiCTEqsMJl9YhIiKSHqPDycGDBzF8+HD4+PhAJpNh69atBh975MgR2NnZoUOHDsZeloiIiKyE0eHk/v37aN++PZYsWWLUcVlZWRg3bhwGDhxo7CUrHUfrEBERSYedsQeEhoYiNDTU6Au99tprGDNmDGxtbY2qbalMbNUhIiKSnirpc7JmzRpcu3YNc+fONWj/vLw8ZGdnq3xUJk5fT0REJB2VHk6uXLmCWbNmYf369bCzM6yiJiwsDHK5XPHh5+dXKWVjh1giIiLpqdRwUlRUhDFjxmD+/Plo3ry5wcfNnj0bWVlZio/ExMRKLCURERFJidF9ToyRk5ODyMhIREdHY+rUqQCA4uJiCCFgZ2eH3bt3Y8CAAWrHOTo6wtHRsTKLpoIdYomIiKSjUsOJm5sbYmNjVbYtW7YMe/fuxR9//IHAwMDKvLxenL6eiIhIeowOJ/fu3cPVq1cVn8fFxSEmJgbu7u7w9/fH7NmzkZSUhHXr1sHGxgZBQUEqx3t6esLJyUltOxERERFgQjiJjIxE//79FZ/PnDkTADB+/HisXbsWKSkpSEhIMF8JiYiIyKrIhJB+j4vs7GzI5XJkZWXBzc3NbOedtekMfj2ZiHeHtMCU/k3Ndl4iIiIy/flt1WvrEBERkfQwnACoBpVHREREVsOqwwknYSMiIpIeqw4nREREJD0MJ+AkbERERFJi5eGE7TpERERSY+XhhIiIiKSG4QQAW3WIiIikw6rDCUfrEBERSY9VhxMiIiKSHoYTcLQOERGRlFh1OGGrDhERkfRYdTghIiIi6WE4ASA4XoeIiEgyrDqccLQOERGR9Fh1OCEiIiLpYTgBR+sQERFJiVWHExnH6xAREUmOVYcTIiIikh6GE3BtHSIiIimx6nDC0TpERETSY9XhhIiIiKSH4QTgcB0iIiIJsepwwlYdIiIi6bHqcEJERETSw3ACjtYhIiKSEqsOJzIO1yEiIpIcqw4nREREJD0MJ+BgHSIiIilhOCEiIiJJYTghIiIiSWE4ASA4XoeIiEgyrDqccLAOERGR9Fh1OCEiIiLpYTgBR+sQERFJiVWHExlX1yEiIpIco8PJwYMHMXz4cPj4+EAmk2Hr1q0699+8eTMGDx6M+vXrw83NDT169MCuXbtMLa9Z3bybCwA4dj3DwiUhIiKiUkaHk/v376N9+/ZYsmSJQfsfPHgQgwcPxo4dOxAVFYX+/ftj+PDhiI6ONrqw5rb7/C0AwKmETMsWhIiIiBTsjD0gNDQUoaGhBu+/ePFilc8///xzbNu2DX/99Rc6duxo7OWJiIiohqvyPifFxcXIycmBu7t7VV+aiIiIqgGja04qauHChbh//z5GjhypdZ+8vDzk5eUpPs/Ozq6KohEREZEEVGnNyYYNGzBv3jxs3LgRnp6eWvcLCwuDXC5XfPj5+VVhKYmIiMiSqiycbNy4ERMnTsRvv/2GQYMG6dx39uzZyMrKUnwkJiZWUSmJiIjI0qqkWWfDhg2YMGECNmzYgGHDhund39HREY6OjlVQMiIiIpIao8PJvXv3cPXqVcXncXFxiImJgbu7O/z9/TF79mwkJSVh3bp1AEqCybhx4/C///0P3bt3R2pqKgDA2dkZcrncTLdBRERENYXRzTqRkZHo2LGjYhjwzJkz0bFjR3z00UcAgJSUFCQkJCj2X7FiBQoLCzFlyhQ0aNBA8TF9+nQz3QIRERHVJEbXnPTr1w9Cx2I0a9euVfl8//79xl6CiIiIrJhVr61DRERE0sNwQkRERJLCcEJERESSwnBCREREksJwQkRERJLCcEJERESSwnBCREREksJwQkRERJLCcEJERESSwnBCREREksJwQkRERJLCcEJERESSwnBCREREksJwQkRERJLCcEJERESSwnBCREREksJwQkRERJLCcEJERESSwnDySFrOQ0sXgYiIiMBworD9TIqli0BERERgOCEiIiKJYTghIiIiSWE4ISIiIklhOHlECEuXgIiIiACGEyIiIpIYhhMiIiKSFIYTIiIikhSGEyIiIpIUhhMiIiKSFIYTIiIikhSGEyIiIpIUhhMiIiKSFIYTIiIikhSGk0c4QSwREZE0MJwQERGRpDCcEBERkaQwnBAREZGkGB1ODh48iOHDh8PHxwcymQxbt27Ve8yBAwcQHBwMJycnNG7cGN99950pZa1UMksXgIiIiACYEE7u37+P9u3bY8mSJQbtHxcXh8cffxx9+vRBdHQ0/vvf/2LatGnYtGmT0YWtTFEJdy1dBCIiIgJgZ+wBoaGhCA0NNXj/7777Dv7+/li8eDEAoFWrVoiMjMTXX3+NZ5991tjLV5qYhExLF4GIiIhQBX1Ojh49ipCQEJVtQ4YMQWRkJAoKCjQek5eXh+zsbJUPIiIisg6VHk5SU1Ph5eWlss3LywuFhYVIT0/XeExYWBjkcrniw8/Pr7KLSURERBJRJaN1ZDLV7qZCCI3bS82ePRtZWVmKj8TExEovIxEREUmD0X1OjOXt7Y3U1FSVbWlpabCzs0O9evU0HuPo6AhHR8fKLpqK0sBEREREllXpNSc9evRAeHi4yrbdu3ejc+fOsLe3r+zLExERUTVjdDi5d+8eYmJiEBMTA6BkqHBMTAwSEhIAlDTJjBs3TrH/5MmTER8fj5kzZ+LChQtYvXo1Vq1ahXfeecc8d0BEREQ1itHNOpGRkejfv7/i85kzZwIAxo8fj7Vr1yIlJUURVAAgMDAQO3bswIwZM7B06VL4+Pjgm2++kdQwYiIiIpIOo8NJv379dPbPWLt2rdq2vn374tSpU8ZeioiIiKwQ19Z5hN1hiYiIpIHh5JHU7IeWLgIRERGB4USBI4mJiIikgeGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhhIiIiCSF4YSIiIgkheFEyaErty1dBCIiIqvHcKLkfHK2pYtARERk9RhOiIiISFIYToiIiEhSGE6IiIhIUhhOiIiISFIYToiIiEhSGE6IiIhIUhhOiIiISFIYToiIiEhSGE6IiIhIUhhOiIiISFJMCifLli1DYGAgnJycEBwcjEOHDuncf/369Wjfvj1q1aqFBg0a4JVXXkFGRoZJBa5MMpmlS0BERERGh5ONGzfirbfewpw5cxAdHY0+ffogNDQUCQkJGvc/fPgwxo0bh4kTJ+LcuXP4/fffcfLkSUyaNKnChSciIqKax+hwsmjRIkycOBGTJk1Cq1atsHjxYvj5+WH58uUa9z927BgaNWqEadOmITAwEL1798Zrr72GyMjIChfe3ISwdAmIiIjIqHCSn5+PqKgohISEqGwPCQlBRESExmN69uyJmzdvYseOHRBC4NatW/jjjz8wbNgwrdfJy8tDdna2ykdVKCxmOiEiIrI0o8JJeno6ioqK4OXlpbLdy8sLqampGo/p2bMn1q9fj1GjRsHBwQHe3t6oU6cOvv32W63XCQsLg1wuV3z4+fkZU0yTrTkSVyXXISIiIu1M6hArK9dzVAihtq3U+fPnMW3aNHz00UeIiorCzp07ERcXh8mTJ2s9/+zZs5GVlaX4SExMNKWYRku/l18l1yEiIiLt7IzZ2cPDA7a2tmq1JGlpaWq1KaXCwsLQq1cvvPvuuwCAdu3awcXFBX369MGnn36KBg0aqB3j6OgIR0dHY4pGRERENYRRNScODg4IDg5GeHi4yvbw8HD07NlT4zG5ubmwsVG9jK2tLYCSGhciIiIiZUY368ycORMrV67E6tWrceHCBcyYMQMJCQmKZprZs2dj3Lhxiv2HDx+OzZs3Y/ny5bh+/TqOHDmCadOmoWvXrvDx8THfnRAREVGNYFSzDgCMGjUKGRkZ+Pjjj5GSkoKgoCDs2LEDAQEBAICUlBSVOU9efvll5OTkYMmSJXj77bdRp04dDBgwAF988YX57oKIiIhqDJmoBm0r2dnZkMvlyMrKgpubm9nO22jWdrVtNxZoH+JMREREhjP1+c21dYiIiEhSGE7IaPEZ9/HVrovIuJdn6aIQEVENZHSfE6InlxxB1oMCxCZlY92ErpYuDhER1TBWXXMS1NB8/VesSdaDAgBA5I07Fi4JERHVRFYdTno3rW/pIhAREVE5Vh1OxvYIsHQRiIiIqByrDidOdlZ9+0RERJLEpzMRERFJCsNJOTfS71u6CERERFaN4aScfl/vt3QRiIiIrJpVhxOZTGbpIhAREVE5Vh1OiIiISHoYToiIiEhSrDqc6FqQ+dj1DCRnPqjC0uhXDRaQJiIiqjCrDifanLxxB6O/P4aeC/ZauigKS/ddRe8v9iE166Gli1Kl1h+Px96LtyxdDCIiqkIMJxqciJPemjFf7bqEpMwHWLznsqWLUmUupmZjzpazmLA20tJFISKiKmTV4aQyRuuk5TzEnfv5Zj9vqaJi62naScvOs3QRiIjIAuwsXYCaJDe/EF0/+xcAEBf2OIcqExERmcCqa07MLUWpP0hlVXBIqd6E/XMr17Xb93A7h7VHRGR9WHNCkmXNFU/JmQ8wcOEBAMCNBcMsXBoioqrFmhMNvtp1SfH/j7ad1bjP3G1nMXtzbFUViazM2aQsSxeBiMhiGE70WHc0Xm3b/bxC/Hg0HhtOJCAtR/pDe6+m3cP55GyznCstu+ruVwYrrjqpItvPpOBcMoMQEUkLw4kJipU6WwxedBAP8ovU9vnrdLJJ557+azTG/HAMxVo6rRjbz0MIgUGLDuDxbw4h60GBxn3yC4vx3YFrBj2kZvwWY1wBKsCam3WqQuSNO5jyyykM++awpYtCRKSC4cQEyqNwsh4UYFtMkto+b22MUfk88U4upvxyCjGJmTrPvS0mGRHXMnDpVo5iW1S86fOuKGec9HuaO1eujYjDgn8uGvSQOmemGhiyvIupOfp3IiKyAIYTA5y8oTsclAYAXX/oT/3lFLafScFTS48YdE3lGpI/otTDjzlVNHBcv30Pn/x9vkqbfCpLQVGxRZvqDl6+jff/OINcDbVx2mw+dRNL912txFIZb1PUTbz3x2kUFhVbuihEVA1xtI4Bnv/uKG4sGIb8wmLczc1HbUfVL5uu5ofiYoFX1p7E6ZuWb9fX1iSkq6lICIHXfz4FnzrO+Gh4a437PL0sAlkPChB7Mwu/Te5hhpKWsESrzvBvD+Niag7+md4HrRq4Vfn1x60+AQCINKK2bOZvpwEAA1p6ooHcCXVqOVRK2Yzx9u8lZereuB6e6eRr4dIQUXXDmhMDFRYVY/i3h9Ht83/VqsNLH6KanvEnb9zBgcu3zVaOzFzjZp815AGvqxvL6ZtZ2HkuFauPxGndp7Qvi74mq+qg9L3dFmNanyFzSc40vvZm0o+R6PBxOI5ey6iEEpnmbq7mfk5ERLownBio7bzdin4gf59RfXBpm3BNCIHP/7lohquXXeDfi2lmOY+hCoyolhcGnj/xTi5u3s3VvyM7xBol6dEq2lJr4iEiMpZVhxNjnn0PCsr6AKw5ckPltRUHr2k85uCVdJzWUptQXCwgTJxi9bsD17Dy0HWTjtXE1HKY4mFBEfp8uQ+9v9inN/hU1VDiq2n38NG2s0jJelAl17Mm+YXF2BaTpLUzNhGRJuxzYgbxGSW1AOUfpRlafiHnFRZh6OJDaOntiuUvBRt9vQWPamNe6OoPF0fdb6GpsSMp8wE+334B7XzlBh9jSMZRXhTxQUER7G0tn4+fWnoE9/IKcUYC/YJqmv8Lv4z8omIE1KuFA+/2t3RxiKiasPyTwYIstTRMxNUMxKXfxz9nUxXb4jPuY+yq44rPBQQKiooRFX8HBUWaSxqfkYsvd17ELS2jZK7dvoceYf+WnVNb85OGbdM3RGN7bArCzNIsVXp9gRvp9xWf5xVIYyTHvbxCAKqzshraRGXtkjIfYMOJBOQVah5dlP+odqw0wBMRGcKqw4mlZD9U7yQ4bUM0Dl1JV9k2e3Msnl1+FH9E3dR4nhdXHsOy/dfw6rpIja/P3XYOaYYsHKf0HE7JeoDQ/x1CZPxd/cdpP41Gqw7HYczKsgA2YOF+tX2KiwXyC0seaMZOwpZ4Jxd/RN00ePhqXPp9o4a6CiFwKTVHpTlKCIHrt+9pnTTPEgwNVsaUWAjNzZCDFx3A7M2xWLL3KoQQeFhg+BBoIiJtGE4qkbZmgi3RZfOW5BcW49t/r2gcaqwtlJQqHQmh7TqFxaoPXoGSWoIt0TcVI2xupN/H9tgUxT6fbr+ACymGzXvyQMOD6Odj8Xjvj9MaH9YLd19W+TznYaHaPs+vOIoOH+/G/Tz11/Tp8+U+vPP7aayNuKF3320xSej/9X7856cog8//87F4DFl8EG+sP6XYtvJQHAYsPIAPlNZgSsp8gOm/RlfL0UsP8otQWFSMRbsv4dj1klE/xcUCTy2LwKgVx9QCSul8LEeupuON9afQ8sOdSLzDWhIiqhiGk0qk7SG5/1LZ0OJVh+OwMPyyxv0qw/t/nMGMjafx+s8lD+V+X+9XeV3TVPzKyldmlH9YfbD1LH6LvKlxVJG2qn9lUfF3kZtfhONxGWrXSsp8gKGLD2LjyQSd51AeSvuwoAiLwi+rdUxefbhkaPRepXKWr6lJznyAUSuOYte5kua3lY+OCT9/S7HP17tLFon85XhZmaZtiMa2mGQ8tfQIikysUbHE1P3/3RKLVh/txIfbzuKbvVcx+vtjAICU7Ic4nZiJEzfu6JwcrrSZ8pcTut8fZX+fSUbEtXT9OxKRVbHqcGLO3/+XUnNUHnSG+mKn5j4dpq54XFQsMGNjDH46ekPjaJfSWpIILXNhGDtyR7nfjLIcDU1Xmp7T2maVffu301h3THXRxeeXR+Biag7e3xSLtOyHWHMkTuN6QcqXWXHgOr759wpGLD0CIQTOJmUZFJIggDlbYnE87g5eM6J2BYDKIouRemYXNkaRCSO8Eu/kGnxMacDacCLRoP1z842v3VJ2I/0+pv4SjTE/HNe/MxFZFasOJ+Y0ZPFBfLr9gtnOZ+rIkV3nUrElOgkfbjuHOKXOp4DxiwYaYvn+smHUyg/Bn47FGzR8VNt0/ndzC7D9TFlz0yd/n0dyVlmQGbPyOOb/dR7v/3FG7diComKMWHIY7/x+GpdulQWF9ccT8MS3hzFh7Um95QKAO2aYQGzUo9qHisp5WICun+3B1A3REELgyq0cvf1cVh+OQ58v92H+X+fNUobynlkWofi/zISqHm0duYmITAony5YtQ2BgIJycnBAcHIxDhw7p3D8vLw9z5sxBQEAAHB0d0aRJE6xevdqkApN2pxMzseJAWVhILffL35AJ1YzNL7FKI1yUn5XRCZl4ZY3+EKAcOHRZdVh1htqrafcAADvPpapNinc87g5O38xS67Oz7ugNAMCRq/pnUF1x8LrKyCJAc02b8tcr+2GBzkUar9++hyNX07HhRAKi9HQ4Lt98sv1MCjLu52P7mRR8vfsSBv+f9jB85GoGfjuZqBhybkgfHGOUdgLmwoFEVFmMnudk48aNeOutt7Bs2TL06tULK1asQGhoKM6fPw9/f3+Nx4wcORK3bt3CqlWr0LRpU6SlpaGwsGJVwqRuhJ5FBXefU22C0VQtr+uP8VYf7tTYCVYb5eBiiCwTayqm/hKNJvVrKz5XrlFIyzZ88q+SZrCyYzU1GZW6ez8fB6/cVowsAoAnvz2MGzqGzA5YeEDl8z0z+6KpZ20te6tSDppL95UE0NVH4rSud/TepjNw0DOHzLVHAU+f8s1C648n4IOtZ7XsTURUcUaHk0WLFmHixImYNGkSAGDx4sXYtWsXli9fjrCwMLX9d+7ciQMHDuD69etwd3cHADRq1KhipSaT5JWrOWn90S61fXQ1FRgTTEzx/IoI/TtpcfOu5tldlYdEqzxjNTRD5BsxrHjs6uM4m6Q6qklXMNHkRNwdNPWsjdSshygSAg3rOGvc73ZOHhbvuWLUubURQiCvsBj38gpValSUa9x0kcmAeX+eU99uQllMGa5ORNbBqGad/Px8REVFISQkRGV7SEgIIiI0P1j+/PNPdO7cGV9++SUaNmyI5s2b45133sGDB9qnCs/Ly0N2drbKR2Wwsak5i7eYa+r1w1ctN3Li8i3D/pLXZJHSiKdCLQErU6kmpKLvfPlgYqqiYoHuYf+i14K9WjuYdvlsj0nnVp7vZPuZFMzeHIuBCw+g5Yc7VUaMAdA52V75vkqavr4Z941bkBIAvtp1yehjiMg6GFVzkp6ejqKiInh5eals9/LyQmqq5lEb169fx+HDh+Hk5IQtW7YgPT0db7zxBu7cuaO130lYWBjmz59vTNFMIne2r/RrVJUeYXv17nMhRZp9BGISM9HBr06FzmHI3Cy3DZmQTos952+Zfc7Y0lmAS2XcM/4Br4vyzMJTfjml8trP5UZCGUrbekflO19rszU6CV/tuoTvTFi2gYish0kdYsv3zBdCaO2tX1xcDJlMhvXr16Nr1654/PHHsWjRIqxdu1Zr7cns2bORlZWl+EhMNGxoI+l28PJt/TtZwFNLj2BLtO4J5yxt0rpIs0/BnplboLaIpNSYe8bXtzbGICnzAYYvOWzW8xJRzWJUzYmHhwdsbW3VaknS0tLUalNKNWjQAA0bNoRcXraAXKtWrSCEwM2bN9GsWTO1YxwdHeHo6GhM0UiihBDYde6WzlEsADBj4+kqKlGJqpy9NeNeHurVVv9+tmSzhqH3v+Cfi5jYO1DxeUUmhzNofhkiIhhZc+Lg4IDg4GCEh4erbA8PD0fPnj01HtOrVy8kJyfj3r2y/gSXL1+GjY0NfH19TSgyVSe7z9/C5J+j8MOhOP0711DBn+5RmZitOtl3yfiJBbUpPxyciEgbo5t1Zs6ciZUrV2L16tW4cOECZsyYgYSEBEyePBlASZPMuHHjFPuPGTMG9erVwyuvvILz58/j4MGDePfddzFhwgQ4O2senUA1h771gazF7C36Z/zddU5zvy1dTO07YqjyFSUVmTPly53sAEtEhjF6KPGoUaOQkZGBjz/+GCkpKQgKCsKOHTsQEBAAAEhJSUFCQtnaGrVr10Z4eDjefPNNdO7cGfXq1cPIkSPx6aefmu8uSLKU16GxZuXX9tHElBmGq2K+EeXROgt0jOopJaUVmomoepIJYxfrsIDs7GzI5XJkZWXBzc3NrOduNGu7Wc9HVJMEerjgzQFNMfO3yu0TdP3zx2vU0H4iKmHq85tr6xCRVnHp9ys9mAAls90SEZViOCEiizPnoplEVP0xnBAREZGkMJwQERGRpDCcEBERkaQwnBAREZGkMJwQERGRpDCcEBERkaQwnBAREZGkMJwQERGRpDCcEBERkaQwnBAREZGkWH04mdAr0NJFICIiIiVWH06aeta2dBGIiIgqxZ7zt/D9wWuWLobR7CxdACIiIqock9ZFAgA6+ddF50buFi6N4ay+5kRAWLoIREQkEVm5BXh2eQR+OhZv6aKYVVpOnqWLYBSrDydERMo2Rd3Eb5GJ+Cc2BVkPCixdnBorN79QbVty5gO8/nMUTt64Y4ESlVi6/yqi4u/iw61nq+R69/PUvw7lFRULRMXfxcOCIpOvIzP5SMuw+nAiq3ZvGREZ62FBESKupSO/sFjnfnfv5+Pt30/jvT/O4PX1p/DymhNVUj4hBGZsjMHnOy5UyfUsbem+q2j90S7siE1R2T7ztxj8czYVz3931EIlMywsmMvfZ5LRZu4uLN13FQCQ87AAqVkP1fZbtu8qnl0egTfWn6qyslma1YcTIqrehBD4etclbD+TonWfd/84gzE/HMcnf5/Xeg4AyC33l2l0QqbB5Ui/l4fiYtOaiS/dysGW6CR8f/A6Ii1Ya1BYVIzPtp/HvotplXqdr3ZdAgC8v+mMyvbEOw90HvdH1E1Exd8FAERcS0dc+v3KKaAJhBD49t8rOHD5tsHHvP9Hyf2Xfj3aztuN7mH/4na5Jpg1ETcAAHsr8L7Iqtnf4QwnRFQt3c7Jw937+Th0JR1L9l3FlF+0/1X51+lkANDYj+Cf2BS0nbcbEVfTTa5HPX49A50/3YP//BRp0vEFhWWh5jkdtQalQWxT1E2V7XvO38Kwbw7h8q0ck65f6veom/jhUBxeWXtS4+uHrtzGt/9eUYS5qnTyxh288/tpPLs8AhdTszHmh+Po//V+o89TXCyQklUWgnafS8UvxxMqXL7d529hYfhljF9d8dq2s0lZKp/fuZ9f4XNWt4Ydqw8n7BBLVP08LChCl8/2oOMn4Sod/YqLBQqKinE17Z7WYxf8c1Hl89fXn8K9vEKMWXkc3x+8rvO6d+/n46dj8cjMVX1YrDocBwDYc0H1L9vbOXnYEn1Tb1+B9PuGdVaMjL+LJfuu4u3fT6tsn7QuEueSsytc7Z+cqbvmYuyqE1gYfhk7z6aadP6o+DsI09F0pSv0xN0uqyU5n5xt0vWv376HJnN2oEfYXuw8W1LT9p+fovDfLbG4frvke8bUJ0LSXd1fu1JClPQfyX6o2p/p2PUME69cM1l9OGGfEyJpiL2ZpX+nR25ll7XLFxaV9SPp8+U+jF11HIMWHVCrXSj13QHtcz6sfVR9riyvsAif/H0eh67cxuSfo/Dh1rPo9vm/eHHlMVxNK6mp2H3+lsbzPbP8CGZsPI1F4Ze1XvPvM8l4ZY3mmopSpfeo7y/onIeGd+C9kX5f0UQClAS+b/deNejYmwY+iMt7dvlRrFAKgOV/+yoHg4PlmkeU/5C8dlt7+NRlwMIDKM0/X+++jPiMssBT+rXVVovy9a5L+PpR80tF/Hk6Gc8uj8Dwbw+rbB/9/THF/8v3xdHldk4ewv65gBt6mrjKN+sUFwuTmyGrgtWHEzsbhhMiKRi+5LDG7UIIHLpyG5N+jMRfp5Mxd9tZlVE0yr9ekzIf4Nj1kj4bmoKGKX46Go9Vh+MwdtUJHI8rOXdeYTGOXM3Aq+ui1Pb/+Vg83v39NB7kFyn6UIRrCS8A8OVO3Q+8tJyHaD9/N2ZvjlXZnptfiH9iU7D7nHotxvRfozFu9QkIISCEwJc7L+LPR01bpfp9vR/PLo9QPKBbfrhT5fWMe9prc84mqwfJxDu5eJBv3GiSomKBLdE3kaShxmbc6hMqHZiVK1WW7tMcMJMyHyBdR7mVXU27h75f7Tdo36wHBViy7yqW7LuKzNx8/HT0BgYu3K+zn9PGkwloNGs7LqZmI/1eniI4Tv81BgAQn5GrtZbm96ib+ObfK/j23ys4p+FrrWzahmisOHAdTy87AiEEpv5yCvP/OgcAWmvshBB4/JtDGLL4oGQDitVPwjasXQO8V65TFhFJx2s/RSlqJvZcKPn3x6P656CITcrC3fv5qOvioPba7Zw8RCfcxcBWXnrPo6uWQFOHzA8eDUH9XUvNTXGxgI0BfxQ9LChCwp1c/BmTjPv5RdhwIgH9WtRXvN76o11qx9zKzsMPB69jW0xJEAmcvUPl9Sfb+6gdc+XWPQTUc1HbPuHHSGyb0ktj2bbFJON/ozsqPj+fnI3HvzkEbzcnHPvvQLX9U7Ie4P801B7dzy/CjI2n4WBrg8ufhaJ8q86r6yLx44SuKC7W3gAvhIBMJkPWgwL0WrAXAHBjwTBEJ9xFfEYunurYUMuR+uXmFyI16yHkzvaKbVujkzDvr5KO1VN+OYUujQbC080J25VqO0YsOYzTj2oChy4+pNge+cEgg69dWtu2UEetGwBF7dfd3AKV93vu8DZ4aeVxxecylLxPHq4OcLS1xcXUklq/2/fy4OXmZHC5qorVhxNne1tLF4GIHmk0azvGdPPHfx9vhaJH/Ue0NZmU0tU3s+Mn4bixYJja9i6f7QEAzH+yTYXKC8CgOTkKHjXLzN12Fj8ejcfE3oEY2MoT/15I0zq8ecwPx3AqIROBHurBQZfPdPTp+GrXRbw9uIVaOLquoZnkdGImohPuwtXJHuuPx+tch6y0Zig1+yHu5RWitqPqo2X6hhic0PF1yn/09SkfQQ5cvo0t0Tcxd9s5rUGy39f7EZ+RC4/aqiH06WURAAD/erXQyb+u1mvrUhoAV47rrNi265zq92P8nVzY29qoNJGd1tJEGfJ/B1U+zzWypklZTGIm7GxkKNbxAxCpVKb//FRWy3f6oxDF/0/E3UFBUTGe7tgQMgkN6bH6cCKh94KIUNLmb8zoCX2d2pVHZpQ3989zBl9HG0Pm5CitfSmt8Vl1OE7RiVaTq2k5OPVoGLM5h8su3XcNx6/fwR+v91RsO5+SrZjivLzSBzyg3jS1/1Ia+rXwVDsmaO4uRMwagDM3MzG4tTc+2HpWZzAp1WjWdo3bZ2ws6fy7JTpJ4+vxGbkAgPR7mvvjnE7MxNboJDzTyVdvGbRR/vqcuZmp9vqXuy6qbdPEPKNugJt3c/HU0iNmOdebG6IBAPsv3cY3L3TUs3fVYThhOiGq0XqE7bV0EQAAadnqk2tpM2jRQY3bTe0Iqkz5r2kAOjvrKivfvPXympOKWqnyv0Z7LpDG1xwAPtt+AYXFAusMaAo0xP1ytR1v/3ba6Nqtilh56Do+3a5/sr5tMZrDHABcT1f/PvrzdLKkwonVd4glouptzpbKnWZc17BkY3T9/N8Kn0Nf51lDmWta/hNxd3D3fr7ZvkbmotyZt9DADp8r9Awj1ybhTq5RE69VlCHBBCjreKvJ4SvpZipN5ZEJS8ymY6Ts7GzI5XJkZWXBzc3N7OfXVp1IRETWY97w1orOrtZIU/+sijL1+c2aEyIiIsCqg4nUMJwQERGRpDCcEBERkaQwnBAREZGkMJwQERGRYvVuKWA4ISIiIp1rQFU1hhMiIiLSM9dy1TIpnCxbtgyBgYFwcnJCcHAwDh06pP8gAEeOHIGdnR06dOhgymWJiIiokkhp2jOjw8nGjRvx1ltvYc6cOYiOjkafPn0QGhqKhATda2FkZWVh3LhxGDhQfcVKIiIisqy9F9MsXQQFo8PJokWLMHHiREyaNAmtWrXC4sWL4efnh+XLl+s87rXXXsOYMWPQo0cPkwtLRERElaMiqySbm1HhJD8/H1FRUQgJCVHZHhISgoiICC1HAWvWrMG1a9cwd+5cg66Tl5eH7OxslY/K9NfU3pV6fiIiIjKcUeEkPT0dRUVF8PLyUtnu5eWF1NRUjcdcuXIFs2bNwvr162FnZ9giyGFhYZDL5YoPPz8/Y4pptLa+8ko9PxERERnOpA6xsnLrYwsh1LYBQFFREcaMGYP58+ejefPmBp9/9uzZyMrKUnwkJiaaUkwiIiKqhgyrynjEw8MDtra2arUkaWlparUpAJCTk4PIyEhER0dj6tSpAIDi4mIIIWBnZ4fdu3djwIABasc5OjrC0dHRmKIRERFRDWFUzYmDgwOCg4MRHh6usj08PBw9e/ZU29/NzQ2xsbGIiYlRfEyePBktWrRATEwMunXrVrHSExERUY1jVM0JAMycORNjx45F586d0aNHD3z//fdISEjA5MmTAZQ0ySQlJWHdunWwsbFBUFCQyvGenp5wcnJS205EREQEmBBORo0ahYyMDHz88cdISUlBUFAQduzYgYCAAABASkqK3jlPpGjagKb4Zu9VSxeDiIjI6smElKaE0yI7OxtyuRxZWVlwc3OrlGusPHQdn26/UCnnJiIiqg5uLBhm1vOZ+vzm2jqPSD+iERERWQeGEyIiIpIUhpNHhKTWYyQiIrJeDCdEREQkKQwnREREJCkMJ4880c7H0kUgIiIiMJwo+NRxhr2t+vpAREREVLUYTpR4uTlZughERERWj+GEiIiIJIXhhIiIiCSF4YSIiIgkheGEiIiIJIXhRMmwdg0AAM08ayu2ffVcO0sVh4iIyCrZWboAUjJzcHO0962DHo3r4Z3fT+N43B2EtPHG2aQsxCZl4VRCpqWLSEREVOPJhJD+erymLrlcEUIIFBYL2NuWVS41mrW9Sq5NRERkCTcWDDPr+Ux9frNZRwuZTKYSTMr7YFirKiwNERGR9WA4MdGkPo0RHFBX7349m9RDYw+XKigRERFRzcBwUgGrx3fB/0Z30LnPiA4+2DOzL94d0qJqCkVERFTNMZxUgLyWPUZ0aKi2XbnNzsHOBjY2Mrzap3GFruVR26FCxxMREVUXDCeVZPrAZuje2B3D2pasduxgp/qlfr1fE6POt3tGX7OVjYiISMoYTirJjMHN8et/eqiEEn/3Wor/vzekBY7OHmDw+dxdHNDRvw4AoIGcCxQSEVHNxXBiITKZDA3kzmrby9ewAMCznXwBAFve6IWLnwxFxCzDQw1V3KkPB6ODXx1LF4OIyGownJhBc6/a+nfSYt7w1nCwtcFHT7TGq30CEfXBIJXXWzVww/tDyzrTOtnbQiaTmXw9Mp67iwO2Tull6WIQEVkNzhBrBn9O7Y25285hY2SiytT35WnKFC/3CsTYHo1ga6P+4phu/vj86bY6r+3p6oi0nDyjy0zGa+Hliku3cixdDCKiGo/hxAyc7G3x+TNtMaxdA7Q3ofpfUzABSoKHVDjb2+JBQZGli2FRrLAiIqoabNYxE1sbGR5rXh9yZ3ut+xj6bPthXGc806kh/vNYxYYfP9NRfZizKQ6+2x+9mtZTfH5izkCznLc6sNMSHImIqPIwnEjQ4NZeWDSyA2o5mFax5e7igFGd/TCotVeFy9LRvw7869VSzNMypI0XPF2lMVpI39wvLb1dK3wN5YWn2NeHiGqyurW0/3Fd1RhOqlBlP9xWjuuMX//THZFzBuGL59rp3LdxfRe4OekOPyGtvfDTxG4AgG6N6yHqg0FY/mKw2cqrbMe0PvB2s3zoebytt9bXvn6+Heq5mGcyPAcd6zZVNkcNI8KIiJ5o52PpIijwt5QR6jxKlb511YcAW0p9pX4p/Vt6onvjerB51BQRqGNNn71v98Mzj4Yoa7LljZ74flxn1HYsCzD1ajsqzl1qzuOt8NfU3qYWX6FhXWesndClwudRZkoYnP9kEJa/2Enja2185Ij8YBBc9YQ6Qxx6vz9+GNe5wucxxWt9jZsAkIisg5RasRlOjPDH5B54plND/PyoNkEK2vnKcei9/jg6e4Bax9pWDdzw5bO6a1CUbZ/WGy/3bIRTHw5GR3/9ixqWqlfBqfUPvdcfcmd7tPRWX067IvOLmPJz5mBrg9C2DRSfCyFUXpfJZPjttR4q29ZN6GrUNf43ugO83Jww2AzNbnWUqmE9ahvWgbqVGZq7NJnUO9BsNUtEZN0YTozQ1NMVi0Z2QCMTVxk2Zygd0NITAPDJiCD4udfSOKEbAKMegG185Jj3ZBu4G/GAERD6d9LDT2nm3PKa6hiaXd6Ybv6K/3/3UrDO0TUVCZitGqiGqMea1zf42NqOdhrXYzJV0/plX59/Z/bFZ08H6T0msH7lrJLtLXfC0CDtzWJEJG0V/21uPgwn1dTql7vgxoJhsLNg3wWpUZ5jZmiQt0rzS/nOs72beWhs9jI1bHUOKKlpenNAU5WFH81pQq9AfPREa5VtXQLdFf+X17KHey3dwXLluM4aa6jMQSaT4YNhrTErtGWlnJ+IrAefbFVJIu15Lg62att+mmhc00RAvZLajsGtq+4v5eHtfbTO/bLgmbYY3cUfnQPq4t0hJTPqfvFsO7RtKMeSMR2xTUO/GOUmm+6N3dG7qYfaUHBDo8rPk7rhz6m9MHNwcwOPUPee0kzAADCig2rntNpOdpjQOxBPPxoi/kJXf0wf2AwfPtEa+97pBwBooqemydgRXINaeRq1v7ODLZ4LLuvL9MmINkYdT0QEMJxYlYXPt0dzr9r4eZJ6k0afZoY3TQDA7hmP4cScgQj0cIG3mxPa+crNVUyFnyd2U8lz377QEUdna55jZXRXfzg72OKP13tiSv+mAICAei74683eeKKdDxrWccacx1tpvdaGV7vjp4ld1TrRCgPTiZO9Ldr51tHbCXfawKZaX3ujX9lrTvY2CA1qoHG/Bc+2xS+TumH+k23gZG+Lib0DFbVAzb1cseaVLhjYUj1UaNrWsI6zIuxo4mnECKoADc1zw9r5KDoYS2lSQSJSZ+jvu6rAcGJF+jT3wO4ZfY3q7KqNo52tYr4TGxsZtk3ppXfeEWOM7OyL3s081LYrd/pVHklkClensloSmUxWKUO9vdzKHsh7Zj6mmC+mlL+W/jYyDdVsg1uV1Ho42tmiZ1MPjYtEAkD/Fp54qXuA2vYVY8uGgX/6VBD83J2xflI3zB/RRrHidXnKv6x2TOuj8lr5QDrwUS2L8vtSy8EWoW0bIC7scRz/r/VM3kdUHZmjD6G5MJxUIYm06lQKmUyGRSM7VMm1PnkqCCGtvRSrNZvq/0Z1QBsfN3z3kvFzt3R69DDXNS8KoPpwb+rpqhaA+mgIYEpHK/63751+aGtE7VS/FvWxclxnrHm5bHi28rVf6h6AQ+8NQCMPF7g52WPz6z0VrzX2cMGXz7bDptd7qDR9tfZxw5eP5s8Z2dlXLZyUnt/J3hb/TO+DHdP6wMneVvEaJ7EjkrZqX3OybNkyBAYGwsnJCcHBwTh06JDWfTdv3ozBgwejfv36cHNzQ48ePbBr1y6TC0zS9Vjz+rj8aahB+yr3S9CktNNmbQ1zioztHoDvx3XWWnOgTfm/Cpp61sb2aX1MGmGyanwXfPlsO3z5XHujjzVE+ee4rjlrNB8vw6DWXiodZnXNYSCTybDsxU5o7OGCJWM6YWQXPwQHuKvtN7KzH47MGoAvnm2HGYO0969p1cANrX0qp+MtEVWO7o3r6d+pihgdTjZu3Ii33noLc+bMQXR0NPr06YPQ0FAkJCRo3P/gwYMYPHgwduzYgaioKPTv3x/Dhw9HdHR0hQtf3VjiL8fSv1wBmDwdvjGUA0Mn/zpo21DzX/va1qz5+83emB3aEmN7lDRLTBvQDF0a1cWCZ9RXZ7ZkyK/r4oCRXfz0Ni3pK+OoLn4AympiSpnrO6W2ox2OzBqAE/8dqPf77/G2DbD3nX4qoULTX1IN6zhDJpOhXm1Hrj1EZEbamlerSkWbys3J6JIsWrQIEydOxKRJkwAAixcvxq5du7B8+XKEhYWp7b948WKVzz///HNs27YNf/31Fzp27Ghaqclgzg62+H5sMASq7hvPwc4G+YXF+OzptmjVwA2NZm1X20fbczKooRxBSoGmrosDfp/cU/PORpJSlWWpdr51cGLOQI1DgI2tGdKmYZ3Km9H4iXYNsDUmGe1N7BA9qXcgVh6OM3OpzK9zQF1Ext+1dDGohtvyRi+Nvy+rSl8j5myqbEb99svPz0dUVBRCQkJUtoeEhCAiIsKgcxQXFyMnJwfu7upVxqXy8vKQnZ2t8lETfPVcOzjZ26jNVVHZQtp4Y0gb9aaLCb0CAZhv9eJSJ+cMwp6ZfdUmKzM3c0wjLwWerk5q89XIZDI81qw++reoj6n9tY/wqWz6/pL77Om2+Oq5dlj7imFD0csvxvhBFf8sPNPJtO91c075r9xJ2lDGTEZI5rXmlS4mf99UN+WXJ7Eko367p6eno6ioCF5eqnMleHl5ITU11aBzLFy4EPfv38fIkSO17hMWFob58+cbU7RqoaN/XZybP1RtmnlL8a9XC5c+HWr2RejkzvZq84WU90a/pth8KgkvdPXXuZ8uzwX74uDl2wbP0FqVFSduTnbIfliIDn51EH7+lknnsLO1wRoDH/qV5fnOfrCRyRDcSPMILxdHOzzf2a+KS2W6RSM74HRiJq7dvm/UccbO91LekjEdMfWXkqbsecPb4PX1p4w6XkqrxUrF6C5+2HUuFXdzCyr1Ov1beKJ/C09sPpVUqdcJMcNyFjWJSU8l9bkghEH9KTZs2IB58+Zh48aN8PTU/sM+e/ZsZGVlKT4SExNNKaYkSSWYlHK0s63yvjB9m9eHn3stnJs/BPOeNH2SLkc7W6wY2xkvdlMfNmtpJ+YMQsxHg01+qEjlu8TWRoaRXfzQpH7l/eWuaQh66UR6mozo4INBrcz/izxi1gD8MqkbhmqoZdT2M/LJU0FoUt8FIzvr7uA9rG0D9G7qgXa+cgxp441fNMw1pIvc2QE/TexaoUn+NNF0r9WFTCYzaDbiib1Laogrq3nz/aEttfatK6XvjzUAGNZO87xGxmggt/zK7uZiVDjx8PCAra2tWi1JWlqaWm1KeRs3bsTEiRPx22+/YdCgQTr3dXR0hJubm8oHVX8zBzfH2le6YNmjSbmqeur9quxz4mRvizp6ppLXSSrpxMw0vQeH3hug8nmfZh46O9q+2qcx5gzTPqGeMlcD+1lNG9gMPnWc0bOpB74bqzq0vL6OyePGdg/Av2/3Q0A93aOpZDIZfprYFdum9IKNjfHDqmWykokSpw1spnc+oRe6Gl6TtfylTmafHK9X03oamyL7tTBPf4Yuj2rxRnXxw3PBfpg+sJnWfV97rDE+fKI1biwYhsaVtKZUHQP+AJk7XHPzZUWXuujVtGx0zd63+yJi1gAde1cvRj0dHBwcEBwcjPDwcJXt4eHh6NlTe6fFDRs24OWXX8Yvv/yCYcMqZ90Rkj57Wxv0a+EJFwn1CJcqLyNmZq3unDUsp6CLoQ/TVeM7I3b+ELXt/Vqo19oaUiOx4dXuGNjSU2NH5XE9ymrv3LT0hVKe66WRh/bFLjUeq/R/5f49mvpdDW7thfE99NcmjusRAJlMZvTX31TKc+5o085XDmd73eXZ8Gp3nJgzEB386sDWRoYZg5vj9NwQxWKohpjS3/g+RGte0V5+fZOX9X/0Pedtws+1toEMp+eGoEujsr6bjevXrlFzCRn9p+vMmTOxcuVKrF69GhcuXMCMGTOQkJCAyZMnAyhpkhk3bpxi/w0bNmDcuHFYuHAhunfvjtTUVKSmpiIrK8t8d0HVgqV/bjTNmVLZlGehNcSv/+mOXk3rqczmao10/ar3dHOCva3+b6aBWpp+dDUZaVJ6pR5N6mHVy10UtQKtlTp8K7/Phrzn5VcRN6b5L6ihHDEfDcah9/prfNjJZDLMH1G2OnWgh4vGQNctUPecFq/1bYydb/UxS5+0ucNbm+XB+XTHhrCztVHMTl1K7myPjzWs46StqeTdIcYvTtm/hafGVb+95U4oLtZ9bF0XB5yZF4KD7/U36prrJ3XTOou03NkeE3oHom1DOd4fWvMW2zT6t/WoUaOQkZGBjz/+GCkpKQgKCsKOHTsQEFCS1FNSUlTmPFmxYgUKCwsxZcoUTJkyRbF9/PjxWLt2bcXvgMhAzwf74sClNIM70JrDtAHNcD45G08b2Nu/e+N6kpoIydwMnR5bXxOcb13jah6AslDiVO4v8+9e6mTUed7o1wTtfOXoFFDxZSBKDWrlhd+jbmp9vfxzvU4tB9Sp5aAx8Jff1KqBKyJvlA2D3jOzL84mZSlmN9YWGerXdkRLbzc8G+yLDSfKfqc3rOMMJ3sb9GlWH2sjbmg8Vvl9NqbpQlbu2N5NPXD4arri8zAN8x2V8q1bCxGzBuDkjTuY/msMAKCZp6vW/U3xXLAv9l5IQ59mHgio54LzKdno17w+vtp5Se+xbnpCa/nv+YB6tdCrqYdKP8Wujdxx4sYdlXP+9ab6oqbG6BrojhNxd/TvWMVM+lPyjTfewBtvvKHxtfKBY//+/aZcgsjsnOxtsXK8/qploKSj44dbz+KbFyo2F4+8lj02/Kd7hc5hbQa18sL9/EK9+3nUdkT6vTy9+/39Zm8cvpqu6BipzNHOBkM1LLD43tAW+PLRA6d8/xa7R82T5X3xbFt8vuMilozpiKeXGTa1AlCyvpKplQqa1mDSRPm519SztsrQZGNrNI7MGgAhBB4WFGsMJxXq2yWToaNfXRy9ngEA+G5sMI5dy0ArHzfUdrRTC5bl+dRxRge/OsqnMytHO1usUmqe6v+oKckc3dnKNxeWFv3L59ph3OoTeGtQM/wZk1zh65QPfMPb+2Bi70CV+aWkgI3/VGW6Bmqf20ZqxnYPwPPBvnp/GZJxtD24zs0fgpSsB4jPyEW/Fp5YcfCa3nP9PrkH1hyJg5ebE77aVRIknmzvgz9Pq/4CLz+xnyHe6NcUkx9rgswHBXB3Maxj86gu/hjZ2Q8ymQyLR3XAWxtjjLqmLuN7NtK4XdPDV9NSB7oCw/tDW2Dyz8YNbdbVV6WigWDG4OY4uuIogJL+FoPMNMTWpQpmyDbFtIHNEJ1wF4O13GerBm6KGZ5NCSd/v9kbH2w9i5jETIzu4oe7ufkqr8sAjfNgWRoX/qNKd/y/A7Hp9R7oZIbVkKsSg4n5KT8jlfvVuDjaoamnKwa28oKtjQw+cv3DPgM9XPDxiCC81D0A9rYydAt0R8sG5qvGt7GRGRxMSpXWQjxVgYkNd0zro1bL07OJ5gUiS1ef7tG4Hna+1QfrJ3XTO3KovKFBDRD1gfYRlLrCxi+vlgyJnq80JYByzYU+5ZtYZSgZjfNSd3/893HT+lFoC2IfDW+Ndr5yLBpp/vWwnurgo7bN0MkTZw5ujp8mdoO9jr49pd9Xxs4a/dagZghqKMfm13vi8Pv9seDZdmr7NDLy+6WqSDNKUo3i5eZkVaNPyDC6/lob3t4Hl27lIOJqOk7fLOs8v++dfmr7yp3tcXb+EDjY2uC4BNvOjdXaxw2tfVpjlQHT+r/YzR8d/OqgqWdttTDtUdsB6ffyEdLaW2+fgnq1TRtO3LOJh6JPydw/zwEAbGUydAl0x9J9+mu//jeqA36LTETi3VysP56AeU+2gUwmw6dPae9boo9yNlEOVj51nPHnVO39M36e2A0vrTpu0jUn9g5EqwZuOJVwF4v3XMG+d/phyyntfYgMoam57eMRQRi76jgmGzBjcf8W9fHWo8U5bWxkin5ajnZl3yezQluqDEeWEoYTIpIcWxuZYgRC6VojLbxcta7OXPoLt3vjelg3oavRqzhXVzKZTGuTVfiMvriQmo0ejevhk7/Pm34Npf8bWhvQp1l9/DSxq97J++q6OCiWBpg7vI3O2gNDCRM7vdhU4NJ2tjZ4rHl9PNa8LBBoK4VyfyZjBXq44PD7uucy+X5sMH4+noAvnlOvJQFUa18MCTmWwmYdIqoypjw4Vo7rjLYN5Vj6omGdkx97NAOxIXRNsFbd1XVxQM8mHpDJZAZ12Fw6ppPeGWPfMWIYdp9m9eGjY1bWhc+rNq+YI5iUp6vD8IdPtFYZCSOV2t3eTUua8Eqb7IwV0sYb6yZ0VRtuXaq6zITCmhMikrRBrb3M1imy1Mb/dMei8Mv4eIT6vBXWali7BhjWroHaqrjGdnA1NH4+G6x7yn9TKYccXWWf2DsQ43sE4HjcHaTlPEST+rXh5+6MxDsPzFIObTlcXz5fOb4zLqXm6J0Sv6ZjOCEiq9OtcT1sfK2HpYtRZUypsaquTWN+7rXwUnd/1Ha011sbY2drg15Nyzobb32jF07euIs1R+Is1n/Jyd4W7Y3oVGwsS0+GaSg26xBRlanKlaGl7otn26JuLXt8W8G5dMztj8k9MP/JNorp4J/pVFLD0dK74iOhSld31rdQYkV9+lRbgxYFLK9ebUcMDfKGnQEzEFPlYs0JEZEFKM+Lsv54fKVe672hLTF7cyzGGtCPoXMjd3RWWrOlk39dHH6/v9Y+DMb45oWOOHotQ6W2oqbSNhuyqR12zeXNAc3w95kUjOnqb9Fy6MNwQkRkIbpmZx3RwQfbYpLRp1nFH+QvdPVHvxb1TVp4DjBtuQBNajnYaV3ziKqGn3stxM4botIZWIoYToioyjzWrD6u375v0DLz1i7smbYY1MoL/VqYZy2o8osNknYWrtyodFIPJgDDCRFVofeHtkQTz9oYaMTy9tagb3NP/BZ5U2UF4FoOdhjeXn3mUanTNputtWnvW0fj9poefMyF4YSIqoyzg61B/R6szeNtvfHjhK5oZcbp96vaiTkDEZ+Riy6Nqs8aWpVpcGsv/G90B7Txse4hwaZiOCEisjCZTIa+zc3TfGMpnq5OZuk0W1PIZDKM6GD6GkvWjkOJiYiISFIYToiIKoFv3ZIOqA6VMC07Va7K7BfSmc1eBmGzDhFRJfhxQld8vesSpg4wbLE8sg49mtTDTxOtZ3FKUzGcEBFVgib1a2P5S8GWLgZJUJ9m1bt/UVVgfSMRERFJCsMJERERSQrDCRERkZLujetZughWj31OiIiIlEzu1xj1ajvgMfYNsRiGEyIiIiWOdrZ4iTMZWxSbdYiIiEhSGE6IiIhIUhhOiIiISFIYToiIiEhSGE6IiIhIUhhOiIiISFIYToiIiEhSGE6IiIhIUhhOiIiISFIYToiIiEhSGE6IiIhIUhhOiIiISFIYToiIiEhSqsWqxEIIAEB2draFS0JERESGKn1ulz7HDVUtwklOTg4AwM/Pz8IlISIiImPl5ORALpcbvL9MGBtnLKC4uBjJyclwdXWFTCYz23mzs7Ph5+eHxMREuLm5me28UlLT75H3V/3V9Hus6fcH1Px75P2ZTgiBnJwc+Pj4wMbG8J4k1aLmxMbGBr6+vpV2fjc3txr5Daespt8j76/6q+n3WNPvD6j598j7M40xNSal2CGWiIiIJIXhhIiIiCTFqsOJo6Mj5s6dC0dHR0sXpdLU9Hvk/VV/Nf0ea/r9ATX/Hnl/Va9adIglIiIi62HVNSdEREQkPQwnREREJCkMJ0RERCQpDCdEREQkKVYdTpYtW4bAwEA4OTkhODgYhw4dsnSREBYWhi5dusDV1RWenp546qmncOnSJZV9Xn75ZchkMpWP7t27q+yTl5eHN998Ex4eHnBxccGTTz6Jmzdvquxz9+5djB07FnK5HHK5HGPHjkVmZqbKPgkJCRg+fDhcXFzg4eGBadOmIT8/3+T7mzdvnlrZvb29Fa8LITBv3jz4+PjA2dkZ/fr1w7lz56rFvQFAo0aN1O5PJpNhypQpAKrne3fw4EEMHz4cPj4+kMlk2Lp1q8rrUnvPYmNj0bdvXzg7O6Nhw4b4+OOPda7roev+CgoK8P7776Nt27ZwcXGBj48Pxo0bh+TkZJVz9OvXT+19HT16tCTuT989AtL7vjTnewhA48+kTCbDV199pdhHyu+hIc+F6v5zqEZYqV9//VXY29uLH374QZw/f15Mnz5duLi4iPj4eIuWa8iQIWLNmjXi7NmzIiYmRgwbNkz4+/uLe/fuKfYZP368GDp0qEhJSVF8ZGRkqJxn8uTJomHDhiI8PFycOnVK9O/fX7Rv314UFhYq9hk6dKgICgoSERERIiIiQgQFBYknnnhC8XphYaEICgoS/fv3F6dOnRLh4eHCx8dHTJ061eT7mzt3rmjTpo1K2dPS0hSvL1iwQLi6uopNmzaJ2NhYMWrUKNGgQQORnZ0t+XsTQoi0tDSVewsPDxcAxL59+4QQ1fO927Fjh5gzZ47YtGmTACC2bNmi8rqU3rOsrCzh5eUlRo8eLWJjY8WmTZuEq6ur+Prrr026v8zMTDFo0CCxceNGcfHiRXH06FHRrVs3ERwcrHKOvn37ildffVXlfc3MzFTZx1L3p+8ehZDW96W530MhhMp9paSkiNWrVwuZTCauXbum2EfK76Ehz4Xq/nNYntWGk65du4rJkyerbGvZsqWYNWuWhUqkWVpamgAgDhw4oNg2fvx4MWLECK3HZGZmCnt7e/Hrr78qtiUlJQkbGxuxc+dOIYQQ58+fFwDEsWPHFPscPXpUABAXL14UQpT8wNvY2IikpCTFPhs2bBCOjo4iKyvLpPuZO3euaN++vcbXiouLhbe3t1iwYIFi28OHD4VcLhffffed5O9Nk+nTp4smTZqI4uJiIUT1fu+EEGq/+KX2ni1btkzI5XLx8OFDxT5hYWHCx8dH8R4Yc3+anDhxQgBQ+UOmb9++Yvr06VqPkcr9abtHKX1fVsV7OGLECDFgwACVbdXpPSz/XKhpP4dCCGGVzTr5+fmIiopCSEiIyvaQkBBERERYqFSaZWVlAQDc3d1Vtu/fvx+enp5o3rw5Xn31VaSlpSlei4qKQkFBgcr9+fj4ICgoSHF/R48ehVwuR7du3RT7dO/eHXK5XGWfoKAg+Pj4KPYZMmQI8vLyEBUVZfI9XblyBT4+PggMDMTo0aNx/fp1AEBcXBxSU1NVyu3o6Ii+ffsqyiT1e1OWn5+Pn3/+GRMmTFBZsLI6v3flSe09O3r0KPr27asymdSQIUOQnJyMGzdumOWes7KyIJPJUKdOHZXt69evh4eHB9q0aYN33nlHsZp6dbk/qXxfVvZ7eOvWLWzfvh0TJ05Ue626vIflnws18efQKsNJeno6ioqK4OXlpbLdy8sLqampFiqVOiEEZs6cid69eyMoKEixPTQ0FOvXr8fevXuxcOFCnDx5EgMGDEBeXh4AIDU1FQ4ODqhbt67K+ZTvLzU1FZ6enmrX9PT0VNmn/Neobt26cHBwMPnr1K1bN6xbtw67du3CDz/8gNTUVPTs2RMZGRmKc+p6X6R8b+Vt3boVmZmZePnllxXbqvN7p4nU3jNN+5R+bo77fvjwIWbNmoUxY8aoLJD24osvYsOGDdi/fz8+/PBDbNq0Cc8884zidanfn5S+Lyv7Pfzxxx/h6uqq8v4A1ec91PRcqIk/h9ViVeLKovzXLFDyppffZklTp07FmTNncPjwYZXto0aNUvw/KCgInTt3RkBAALZv3672A6es/P1puldT9jFGaGio4v9t27ZFjx490KRJE/z444+KDnimvC9SuLfyVq1ahdDQUJW/MKrze6eLlN4zTWXRdqwxCgoKMHr0aBQXF2PZsmUqr7366quK/wcFBaFZs2bo3LkzTp06hU6dOplcdkP2Mcf9Se37srLeQwBYvXo1XnzxRTg5Oalsry7vobbngrbzVtefQ6usOfHw8ICtra1agktLS1NLe5by5ptv4s8//8S+ffvg6+urc98GDRogICAAV65cAQB4e3sjPz8fd+/eVdlP+f68vb1x69YttXPdvn1bZZ/yX6O7d++ioKDAbF8nFxcXtG3bFleuXFGM2tH1vlSXe4uPj8eePXswadIknftV5/eu9DqAdN4zTfuUNk9U5L4LCgowcuRIxMXFITw8XO+y8p06dYK9vb3K+yrl+yvPkt+XlXmPhw4dwqVLl/T+XALSfA+1PRdq5M+hQT1TaqCuXbuK119/XWVbq1atLN4htri4WEyZMkX4+PiIy5cvG3RMenq6cHR0FD/++KMQoqzj08aNGxX7JCcna+z4dPz4ccU+x44d09jxKTk5WbHPr7/+atZOow8fPhQNGzYU8+fPV3Tq+uKLLxSv5+XlaezUJfV7mzt3rvD29hYFBQU696tu7x20dIiVynu2bNkyUadOHZGXl6fYZ8GCBRXqTJmfny+eeuop0aZNG5WRZbrExsaqdFiUyv1pu8fyLPl9WRnvYanx48erjbTSRkrvob7nQk37ORTCikfrlA4lXrVqlTh//rx46623hIuLi7hx44ZFy/X6668LuVwu9u/frzKkLTc3VwghRE5Ojnj77bdFRESEiIuLE/v27RM9evQQDRs2VBsy5uvrK/bs2SNOnTolBgwYoHHIWLt27cTRo0fF0aNHRdu2bTUOGRs4cKA4deqU2LNnj/D19a3QcNu3335b7N+/X1y/fl0cO3ZMPPHEE8LV1VXxdV+wYIGQy+Vi8+bNIjY2Vrzwwgsah8NJ8d5KFRUVCX9/f/H++++rbK+u711OTo6Ijo4W0dHRAoBYtGiRiI6OVoxWkdJ7lpmZKby8vMQLL7wgYmNjxebNm4Wbm5vOIYy67q+goEA8+eSTwtfXV8TExKj8TJb+4r169aqYP3++OHnypIiLixPbt28XLVu2FB07dpTE/em7R6l9X5r7PSyVlZUlatWqJZYvX652vNTfQ33PBSGq/89heVYbToQQYunSpSIgIEA4ODiITp06qQzXtRQAGj/WrFkjhBAiNzdXhISEiPr16wt7e3vh7+8vxo8fLxISElTO8+DBAzF16lTh7u4unJ2dxRNPPKG2T0ZGhnjxxReFq6urcHV1FS+++KK4e/euyj7x8fFi2LBhwtnZWbi7u4upU6eqDA8zVunYe3t7e+Hj4yOeeeYZce7cOcXrxcXFiloHR0dH8dhjj4nY2NhqcW+ldu3aJQCIS5cuqWyvru/dvn37NH5Pjh8/XgghvffszJkzok+fPsLR0VF4e3uLefPm6fxrTdf9xcXFaf2ZLJ27JiEhQTz22GPC3d1dODg4iCZNmohp06apzRNiqfvTd49S/L4053tYasWKFcLZ2Vlt7hIhpP8e6nsuCFH9fw7Lkz26cSIiIiJJsMoOsURERCRdDCdEREQkKQwnREREJCkMJ0RERCQpDCdEREQkKQwnREREJCkMJ0RERCQpDCdEREQkKQwnREREJCkMJ0RERCQpDCdEREQkKQwnREREJCn/D3YzjM9kJIfjAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(stepi, lossi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.1660, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training loss \n",
    "emb = C[Xtr] # (32, 3, 2)\n",
    "h = torch.tanh(emb.view(-1, block_size*10) @ W1 + b1) # (32, 100)\n",
    "logits = h @ W2 + b2 # (32, 27)\n",
    "loss = F.cross_entropy(logits, Ytr)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.1929, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# validation loss\n",
    "emb = C[Xdev] # (32, 3, 2)\n",
    "h = torch.tanh(emb.view(-1, block_size*10) @ W1 + b1) # (32, 100)\n",
    "logits = h @ W2 + b2 # (32, 27)\n",
    "loss = F.cross_entropy(logits, Ydev)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.1878, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test loss\n",
    "emb = C[Xte] # (32, 3, 2)\n",
    "h = torch.tanh(emb.view(-1, block_size*10) @ W1 + b1) # (32, 100)\n",
    "logits = h @ W2 + b2 # (32, 27)\n",
    "loss = F.cross_entropy(logits, Yte)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "koinly\n",
      "whetora\n",
      "avrawia\n",
      "jalon\n",
      "hedie\n",
      "niveon\n",
      "jevira\n",
      "arvielyana\n",
      "lopie\n",
      "jayzen\n",
      "zozyla\n",
      "zicfinzan\n",
      "raison\n",
      "alvie\n",
      "brylon\n",
      "azaud\n",
      "mathy\n",
      "ami\n",
      "farte\n",
      "kure\n"
     ]
    }
   ],
   "source": [
    "# sample from the model\n",
    "g = torch.Generator()\n",
    "\n",
    "for _ in range(20):\n",
    "    \n",
    "    out = []\n",
    "    context = [0] * block_size # initialize with all ...\n",
    "    while True:\n",
    "      emb = C[torch.tensor([context])] # (1,block_size,d)\n",
    "      h = torch.tanh(emb.view(1, -1) @ W1 + b1)\n",
    "      logits = h @ W2 + b2\n",
    "      probs = F.softmax(logits, dim=1)\n",
    "      ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
    "      context = context[1:] + [ix]\n",
    "      out.append(ix)\n",
    "      if ix == 0:\n",
    "        break\n",
    "    \n",
    "    print(''.join(itos[i] for i in out)[0:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}