From 4dd0a5df4c491bd9bc01a50b82f0c2f6d73f5bf3 Mon Sep 17 00:00:00 2001 From: bushuhui Date: Mon, 21 Feb 2022 11:17:38 +0800 Subject: [PATCH] Improve train of LeNet5/AlexNet/VGG/GoogLeNet/ResNet --- 7_deep_learning/1_CNN/02-LeNet5.ipynb | 123 +++++++++++++-- 7_deep_learning/1_CNN/03-AlexNet.ipynb | 105 +++++++++++-- 7_deep_learning/1_CNN/04-vgg.ipynb | 250 +++++++++++++++++++------------ 7_deep_learning/1_CNN/05-googlenet.ipynb | 168 +++++++++++++-------- 7_deep_learning/1_CNN/06-resnet.ipynb | 133 +++++++++------- 7_deep_learning/1_CNN/utils.py | 16 +- 6 files changed, 554 insertions(+), 241 deletions(-) diff --git a/7_deep_learning/1_CNN/02-LeNet5.ipynb b/7_deep_learning/1_CNN/02-LeNet5.ipynb index 0390477..2b45bf1 100644 --- a/7_deep_learning/1_CNN/02-LeNet5.ipynb +++ b/7_deep_learning/1_CNN/02-LeNet5.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -54,9 +54,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LeNet5(\n", + " (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n", + " (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n", + " (fc1): Linear(in_features=400, out_features=120, bias=True)\n", + " (fc2): Linear(in_features=120, out_features=84, bias=True)\n", + " (fc3): Linear(in_features=84, out_features=10, bias=True)\n", + ")\n" + ] + } + ], "source": [ "net = LeNet5()\n", "print(net)" @@ -64,9 +78,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[ 0.0124, 0.1326, 0.1647, 0.0728, 0.0722, 0.0113, 0.0829, -0.0055,\n", + " 0.1749, -0.0581]], grad_fn=)\n" + ] + } + ], "source": [ "input = torch.randn(1, 1, 32, 32)\n", "out = net(input)\n", @@ -75,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -105,9 +128,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAEICAYAAAA3EMMNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWHUlEQVR4nO3df6xU5Z3H8fdH1DZF02qsSBFFDavFpl6VYlONxVgrbWyQthrJxmDWiH9AVhNjVk2T0uzimi3irqs1xUr9EayS+Iu4TdW1WtdtigKlyo+yorIWuYGiUtBWDfDdP+bc7tw7M8/MvTP3znkun1cyuXPP95w5T6feD895zjnPUURgZparg7rdADOzdjjEzCxrDjEzy5pDzMyy5hAzs6w5xMwsaw4xM8uaQ8zqkvS8pA8lvV+8NnW7TWb1OMQsZX5EHFa8Tu52Y8zqcYiZWdYcYpbyz5J2SvpvSdO73RizeuR7J60eSWcBG4CPgcuAO4CeiHi9qw0zG8AhZi2R9AvgPyLi37vdFrNqPpy0VgWgbjfCbCCHmNWQ9BlJF0r6pKSDJf0tcC7wVLfbZjbQwd1ugJXSIcA/AacA+4DfAxdHhK8Vs9LxmJiZZc2Hk2aWNYeYmWXNIWZmWXOImVnWRvTspCSfRTAbZhHR1vV8M2bMiJ07d7a07urVq5+KiBnt7K9tETHkFzAD2ARsBm5oYf3wyy+/hvfVzt90RHDmmWdGq4BVTf7mJwLPARuB9cA1xfIFwNvA2uL1zaptbqSSKZuAC5u1d8g9MUljgDuBC4CtwMuSVkTEhqF+ppmVQwcvvdoLXBcRayQdDqyW9ExRuy0iFlWvLGkKlXt1TwU+B/ynpL+JiH2NdtDOmNg0YHNEvBERHwMPATPb+DwzK4n9+/e39GomInojYk3xfg+VHtmExCYzgYci4qOIeJNKj2xaah/thNgE4A9Vv2+t1zhJcyWtkrSqjX2Z2QgZ5JBSyyRNAk4HVhaL5kt6RdJSSUcUy1rKlWrthFi9wcOa/1URsSQipkbE1Db2ZWYjaBAhdlRfJ6V4za33eZIOAx4Bro2I3cBdwElAD9AL3Nq3ar3mpNraztnJrVQG7focC2xr4/PMrCQG0cva2ayDIukQKgG2LCIeLT5/e1X9buDJ4tdB50o7PbGXgcmSTpB0KJXBuBVtfJ6ZlUSnDiclCbgH2BgRi6uWj69abRawrni/ArhM0icknQBMBl5K7WPIPbGI2CtpPpXpWcYASyNi/VA/z8zKo4NnJ88GLgdelbS2WHYTMFtSD5VDxS3A1cV+10taTmVW4b3AvNSZSRjhWSx8savZ8Is2L3Y944wz4sUXX2xp3bFjx67u9ni35xMzsxoj2blpl0PMzGo4xMwsaw4xM8vWUC5k7SaHmJnVaOWWorJwiJlZDffEzCxbPpw0s+w5xMwsaw4xM8uaQ8zMshURPjtpZnlzT8zMsuYQM7OsOcTMLGsOMTPLlgf2zSx77omZWdYcYmaWNYeYmWXLN4CbWfYcYmaWNZ+dNLOsuSdmZtnymJiZZc8hZmZZc4iZWdYcYmaWLd87aWbZc0/MSmPMmDHJ+qc//elh3f/8+fMb1j71qU8ltz355JOT9Xnz5iXrixYtalibPXt2ctsPP/wwWb/llluS9R/84AfJetkdMCEmaQuwB9gH7I2IqZ1olJl11wETYoXzImJnBz7HzEriQAsxMxtFchvYP6jN7QN4WtJqSXPrrSBprqRVkla1uS8zGyF9V+03e5VBuyF2dkScAXwDmCfp3IErRMSSiJjq8TKzfHQqxCRNlPScpI2S1ku6plh+pKRnJL1W/DyiapsbJW2WtEnShc320VaIRcS24ucO4DFgWjufZ2bl0MGe2F7guoj4PPBlKp2dKcANwLMRMRl4tvidonYZcCowA/iRpOQp9iGHmKSxkg7vew98HVg31M8zs3JoNcBaCbGI6I2INcX7PcBGYAIwE7ivWO0+4OLi/UzgoYj4KCLeBDbTpHPUzsD+OOAxSX2f82BE/KKNzxu1jjvuuGT90EMPTda/8pWvJOvnnHNOw9pnPvOZ5Lbf+c53kvVu2rp1a7J+++23J+uzZs1qWNuzZ09y29/97nfJ+q9+9atkPXeDGO86asB495KIWFJvRUmTgNOBlcC4iOgt9tUr6ehitQnAb6o221osa2jIIRYRbwCnDXV7MyuvQZyd3NnKeLekw4BHgGsjYnfR+am7ap1lyURtd2DfzEahTp6dlHQIlQBbFhGPFou3Sxpf1McDO4rlW4GJVZsfC2xLfb5DzMz66eSYmCpdrnuAjRGxuKq0AphTvJ8DPFG1/DJJn5B0AjAZeCm1D1/samY1OngN2NnA5cCrktYWy24CbgGWS7oSeAu4pNjveknLgQ1UzmzOi4h9qR04xMysRqdCLCJepP44F8D5DbZZCCxsdR8OMTOrUZar8VvhEOuAnp6eZP2Xv/xlsj7c0+GUVbMzYN/73veS9ffffz9ZX7ZsWcNab29vctv33nsvWd+0aVOynrPc7p10iJlZDffEzCxrDjEzy5pDzMyy5hAzs2x5YN/MsueemJllzSF2gHnrrbeS9XfeeSdZL/N1YitXrkzWd+3alayfd955DWsff/xxctsHHnggWbfh4xAzs2yVaf78VjjEzKyGQ8zMsuazk2aWNffEzCxbHhMzs+w5xMwsaw6xA8y7776brF9//fXJ+kUXXZSs//a3v03Wmz26LGXt2rXJ+gUXXJCsf/DBB8n6qaee2rB2zTXXJLe17nGImVm2fO+kmWXPPTEzy5pDzMyy5hAzs6w5xMwsWx7YN7PsuSdm/Tz++OPJerPnUu7ZsydZP+200xrWrrzyyuS2ixYtStabXQfWzPr16xvW5s6d29Zn2/DJKcQOaraCpKWSdkhaV7XsSEnPSHqt+HnE8DbTzEZS3/2TzV5l0DTEgHuBGQOW3QA8GxGTgWeL381sFGg1wLIJsYh4ARh4X81M4L7i/X3AxZ1tlpl1U04hNtQxsXER0QsQEb2Sjm60oqS5gAc/zDLis5NVImIJsARAUjmi28waKlMvqxWtjInVs13SeIDi547ONcnMui2nw8mhhtgKYE7xfg7wRGeaY2ZlkFOINT2clPQzYDpwlKStwPeBW4Dlkq4E3gIuGc5Gjna7d+9ua/s//elPQ972qquuStYffvjhZD2nsRNrXVkCqhVNQywiZjcond/htphZCXTytiNJS4GLgB0R8YVi2QLgKuCPxWo3RcTPi9qNwJXAPuDvI+KpZvsY6uGkmY1iHTycvJfa60wBbouInuLVF2BTgMuAU4ttfiRpTLMdOMTMrEanQqzBdaaNzAQeioiPIuJNYDMwrdlGDjEzqzGIEDtK0qqqV6vXhM6X9EpxW2PfbYsTgD9UrbO1WJbkG8DNrMYgBvZ3RsTUQX78XcA/AlH8vBX4O0D1mtLswxxiZtbPcF8+ERHb+95Luht4svh1KzCxatVjgW3NPs8hNgosWLCgYe3MM89MbvvVr341Wf/a176WrD/99NPJuuVpOC+dkTS+77ZFYBbQN0POCuBBSYuBzwGTgZeafZ5DzMxqdKon1uA60+mSeqgcKm4Bri72uV7ScmADsBeYFxH7mu3DIWZmNToVYg2uM70nsf5CYOFg9uEQM7N+ynRLUSscYmZWwyFmZllziJlZ1nK6sd8hZmb9eEzMRlzqsWrNptpZs2ZNsn733Xcn688991yyvmrVqoa1O++8M7ltTn9Io01O371DzMxqOMTMLGsOMTPLVicnRRwJDjEzq+GemJllzSFmZllziJlZ1hxiVhqvv/56sn7FFVck6z/96U+T9csvv3zI9bFjxya3vf/++5P13t7eZN2Gxhe7mln2fHbSzLLmnpiZZc0hZmbZ8piYmWXPIWZmWXOImVnWfHbSsvHYY48l66+99lqyvnjx4mT9/PPPb1i7+eabk9sef/zxyfrChemH4rz99tvJutWX25jYQc1WkLRU0g5J66qWLZD0tqS1xeubw9tMMxtJfUHW7FUGTUMMuBeYUWf5bRHRU7x+3tlmmVk35RRiTQ8nI+IFSZNGoC1mVhJlCahWtNITa2S+pFeKw80jGq0kaa6kVZIaT7ZuZqXRNyliK68yGGqI3QWcBPQAvcCtjVaMiCURMTUipg5xX2Y2wkbV4WQ9EbG9772ku4EnO9YiM+u6sgRUK4bUE5M0vurXWcC6RuuaWX5GVU9M0s+A6cBRkrYC3wemS+oBAtgCXD18TbRuWrcu/e/TpZdemqx/61vfalhrNlfZ1Ven/7OaPHlysn7BBRck69ZYWQKqFa2cnZxdZ/E9w9AWMyuBMvWyWuEr9s2sRlnOPLbCIWZmNXLqibVznZiZjVKdGthvcNvikZKekfRa8fOIqtqNkjZL2iTpwlba6hAzs35aDbAWe2v3Unvb4g3AsxExGXi2+B1JU4DLgFOLbX4kaUyzHTjEzKxGp0IsIl4A3h2weCZwX/H+PuDiquUPRcRHEfEmsBmY1mwfHhOztuzatStZf+CBBxrWfvKTnyS3Pfjg9H+e5557brI+ffr0hrXnn38+ue2BbpjHxMZFRG+xn15JRxfLJwC/qVpva7EsySFmZjUGcXbyqAH3RS+JiCVD3K3qLGuapg4xM+tnkNeJ7RzCfdHbJY0vemHjgR3F8q3AxKr1jgW2Nfswj4mZWY1hvu1oBTCneD8HeKJq+WWSPiHpBGAy8FKzD3NPzMxqdGpMrMFti7cAyyVdCbwFXFLsc72k5cAGYC8wLyL2NduHQ8zManQqxBrctghQ9+ELEbEQSD88YQCHmJn10zcpYi4cYmZWI6fbjhxilvTFL34xWf/ud7+brH/pS19qWGt2HVgzGzZsSNZfeOGFtj7/QOYQM7OsOcTMLGsOMTPLlidFNLPs+eykmWXNPTEzy5pDzMyy5TExK5WTTz45WZ8/f36y/u1vfztZP+aYYwbdplbt25e+ba63tzdZz2lcp2wcYmaWtZz+AXCImVk/Ppw0s+w5xMwsaw4xM8uaQ8zMsuYQM7NsjbpJESVNBO4HjgH2U3kk079JOhJ4GJgEbAEujYj3hq+pB65m12LNnt1oBuDm14FNmjRpKE3qiFWrViXrCxemZylesWJFJ5tjVXLqibXytKO9wHUR8Xngy8C84nHjdR9Fbmb5G+anHXVU0xCLiN6IWFO83wNspPJU3kaPIjezzOUUYoMaE5M0CTgdWEnjR5GbWcbKFFCtaDnEJB0GPAJcGxG7pXpPHK+73Vxg7tCaZ2bdMOpCTNIhVAJsWUQ8Wixu9CjyfiJiCbCk+Jx8vhmzA1hOZyebjomp0uW6B9gYEYurSo0eRW5mmRttY2JnA5cDr0paWyy7iQaPIrda48aNS9anTJmSrN9xxx3J+imnnDLoNnXKypUrk/Uf/vCHDWtPPJH+dy+n3sBoUqaAakXTEIuIF4FGA2B1H0VuZnkbVSFmZgceh5iZZS2nQ3mHmJn1M+rGxMzswOMQM7OsOcTMLGsOsVHoyCOPbFj78Y9/nNy2p6cnWT/xxBOH0qSO+PWvf52s33rrrcn6U089laz/5S9/GXSbrPscYmaWrU5PiihpC7AH2AfsjYipnZyPsJX5xMzsADMMtx2dFxE9ETG1+L1j8xE6xMysxgjcO9mx+QgdYmZWYxAhdpSkVVWvetNuBfC0pNVV9X7zEQJDno/QY2Jm1s8ge1k7qw4RGzk7IrYVE6c+I+n37bWwP/fEzKxGJw8nI2Jb8XMH8BgwjWI+QoDUfIStcIiZWY39+/e39GpG0lhJh/e9B74OrKOD8xEeMIeTZ511VrJ+/fXXJ+vTpk1rWJswYcKQ2tQpf/7znxvWbr/99uS2N998c7L+wQcfDKlNlrcOXic2DnismM7+YODBiPiFpJfp0HyEB0yImVlrOnkDeES8AZxWZ/k7dGg+QoeYmdXwFftmljWHmJllzZMimlm2PCmimWXPIWZmWXOIldCsWbPaqrdjw4YNyfqTTz6ZrO/duzdZT835tWvXruS2ZvU4xMwsaw4xM8tWpydFHG4OMTOr4Z6YmWXNIWZmWXOImVm2fLGrmWUvpxBTs8ZKmgjcDxwD7AeWRMS/SVoAXAX8sVj1poj4eZPPyuebMctURKid7Q899ND47Gc/29K627ZtW93C9NTDqpWe2F7guohYU8zQuFrSM0XttohYNHzNM7NuyKkn1jTEiieR9D2VZI+kjUB3pzI1s2GT25jYoObYlzQJOB1YWSyaL+kVSUslHdFgm7l9j3Nqr6lmNlJG4LmTHdNyiEk6DHgEuDYidgN3AScBPVR6anVv4IuIJRExtdvHzWbWupxCrKWzk5IOoRJgyyLiUYCI2F5VvxtI38VsZtnI6bajpj0xVR5Tcg+wMSIWVy0fX7XaLCqPYTKzzLXaC8upJ3Y2cDnwqqS1xbKbgNmSeqg8onwLcPUwtM/MuqAsAdWKVs5OvgjUu+4keU2YmeVrVIWYmR14HGJmljWHmJlly5Mimln23BMzs6w5xMwsaw4xM8tWmS5kbYVDzMxqOMTMLGs+O2lmWXNPzMyylduY2KAmRTSzA0MnZ7GQNEPSJkmbJd3Q6bY6xMysRqdCTNIY4E7gG8AUKrPfTOlkW304aWY1OjiwPw3YHBFvAEh6CJgJbOjUDkY6xHYC/1v1+1HFsjIqa9vK2i5w24aqk207vgOf8RSVNrXikwOen7EkIpZU/T4B+EPV71uBs9psXz8jGmIR0e9hdpJWlXXu/bK2raztArdtqMrWtoiY0cGPqzcXYUfPGnhMzMyG01ZgYtXvxwLbOrkDh5iZDaeXgcmSTpB0KHAZsKKTO+j2wP6S5qt0TVnbVtZ2gds2VGVuW1siYq+k+VTG2cYASyNifSf3oZwuajMzG8iHk2aWNYeYmWWtKyE23LchtEPSFkmvSlo74PqXbrRlqaQdktZVLTtS0jOSXit+HlGiti2Q9Hbx3a2V9M0utW2ipOckbZS0XtI1xfKufneJdpXie8vViI+JFbch/A9wAZXTry8DsyOiY1fwtkPSFmBqRHT9wkhJ5wLvA/dHxBeKZf8CvBsRtxT/ABwREf9QkrYtAN6PiEUj3Z4BbRsPjI+INZIOB1YDFwNX0MXvLtGuSynB95arbvTE/nobQkR8DPTdhmADRMQLwLsDFs8E7ive30flj2DENWhbKUREb0SsKd7vATZSuXK8q99dol3Whm6EWL3bEMr0f2QAT0taLWlutxtTx7iI6IXKHwVwdJfbM9B8Sa8Uh5tdOdStJmkScDqwkhJ9dwPaBSX73nLSjRAb9tsQ2nR2RJxB5a77ecVhk7XmLuAkoAfoBW7tZmMkHQY8AlwbEbu72ZZqddpVqu8tN90IsWG/DaEdEbGt+LkDeIzK4W+ZbC/GVvrGWHZ0uT1/FRHbI2JfROwH7qaL352kQ6gExbKIeLRY3PXvrl67yvS95agbITbstyEMlaSxxYArksYCXwfWpbcacSuAOcX7OcATXWxLP30BUZhFl747SQLuATZGxOKqUle/u0btKsv3lquuXLFfnEL+V/7/NoSFI96IOiSdSKX3BZVbsh7sZtsk/QyYTmValO3A94HHgeXAccBbwCURMeID7A3aNp3KIVEAW4Cr+8agRrht5wD/BbwK9E2MdROV8aeufXeJds2mBN9brnzbkZllzVfsm1nWHGJmljWHmJllzSFmZllziJlZ1hxiZpY1h5iZZe3/APpBMI71CUMRAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "# 显示其中一个数据\n", "import matplotlib.pyplot as plt\n", @@ -119,9 +155,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([64, 1, 32, 32])\n", + "torch.Size([64])\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS4AAAEICAYAAADhtRloAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZkElEQVR4nO3df5BdZZ3n8fcnTSLBiAHyqxMiiRgDWQoSDAFKFzO4cTpsWcDWZgQWyaRwM2wZC635Q3SrZty1LGBnddZxEarVILosrC5RI5WdjIWyaM2YTQItJESwZSA0iYltEgNJTNLhu3/c03rTfZ9zb3ffvvee7s+r6lb3fb7nnPvkdOfbz3nO8zxHEYGZWZFMaHYFzMyGyonLzArHicvMCseJy8wKx4nLzArHicvMCseJy8wKx4nLKpL0PyTtlXRY0ouSPtrsOpn1kwegWiWS/gXQHRHHJV0EPAn864jY3tyambnFZQkRsTMijve/zV4XNrFKZn/gxGVJkr4i6SjwC2AvsKnJVTIDfKloVUhqA64GlgP3RsTJ5tbIzC0uqyIiTkXET4Hzgf/Q7PqYgROX1e4M3MdlLcKJywaRNEPSTZKmSGqT9KfAzcCPml03M3Afl1UgaTrwv4HLKP1xewX4u4j4alMrZpZx4jKzwvGlopkVjhOXmRWOE5eZFY4Tl5kVzhmN/DBJvhNgNsoiQiPZv6OjI3p7e2vadvv27ZsjomMknzccI0pckjqALwFtwNci4p661MrMmqa3t5dt27bVtK2kaaNcnYqGfamYzWG7D1gJLAJulrSoXhUzs+aJiJpezTKSFtcySus1vQQg6VHgeuD5elTMzJrnzTffbHYVco0kcc0BXi173wNcOXAjSWuBtSP4HDNroGa3pmoxksRVqQNw0L82IjqBTnDnvFlRjOXE1QPMLXt/PrBnZNUxs1bQ6olrJOO4tgILJM2XNAm4CdhYn2qZWTON2c75iOiTtA7YTGk4xPqI2Fm3mplZ07R6i2tE47giYhNeh9xsTImIMX1X0czGqDHd4jKzscmJy8wKx4nLzAql2XcMa+HEZWaDuHPezArHLS4zKxRfKppZITlxmVnhOHGZWeE4cZlZoXjKj5kVkltcZlY4TlxmVjhOXGZWOE5cZlYo7pw3s0Jyi8tawhlnpH/U06dPT8Zmz56djC1alH7+79SpUyuWT5gwvMcc5LUADh8+nIx1dXVVLN+xY0dyn1OnTtVcr7HKicvMCseJy8wKxZOszayQnLjMrHB8V9HMCsctLjMrFPdxWa63v/3tydicOXOSsZkzZyZjZ511VsXyM888M7nPvHnzkrFLL700Gbv22muTsVmzZlUszxuWkSdviMK+ffuSsR/84AcVy7/85S8n9+nu7k7Gjh8/noyNJWM6cUl6GXgdOAX0RcTSelTKzJprTCeuzJ9ERG8djmNmLWI8JC4zG0OKMFdxePMv/iiAf5C0XdLaShtIWitpm6RtI/wsM2uQ/g76aq9mGWmL670RsUfSDOCHkn4REU+VbxARnUAngKTWbn+aGdD6l4ojanFFxJ7s637gu8CyelTKzJqrni0uSR2SXpDULemuCvG3S/qBpJ9L2ilpTbVjDrvFJemtwISIeD37/oPAfx7u8caqSZMmJWNXX311MrZq1apk7JprrknGUkMl8oYTTJ48ORnLq39fX18yduTIkSHvk/dZecM52tvbk7HVq1dXLD9x4kRyn3vuuScZ27NnTzI2ltSrxSWpDbgPWAH0AFslbYyI58s2+xjwfER8SNJ04AVJD0dE8oc0kkvFmcB3JfUf539GxN+P4Hhm1gLq3Dm/DOiOiJcAJD0KXA+UJ64A3qZSMpkCHADSf+EYQeLKKnLZcPc3s9Y1hBbXtAE33jqzfu1+c4BXy973AFcOOMZ/BzYCe4C3AR+OiNzM6eEQZjbIEBJXb5WB56p0+AHv/xToAq4FLqR0o+8nEZFcIXKkwyHMbAyqY+d8DzC37P35lFpW5dYAG6KkG/hn4KK8gzpxmdlpak1aNSaurcACSfMlTQJuonRZWG438AEASTOBhcBLeQf1paKZDVKvu4oR0SdpHbAZaAPWR8ROSXdk8QeAzwHfkPQcpUvLT1WbRujENcpSqyQAfPzjH0/GVqxYkYy1tbUlYwcPHqxYvnv37uQ+F1xwQTKWtxpC3qoMzzzzTMXy3/72t8Oqx3ve855kbMaMGclYaojFbbfdltyns7MzGRsvwyHqOeUnIjYBmwaUPVD2/R5Kw6lq5sRlZoO0+sh5Jy4zO02z5yHWwonLzAZx4jKzwnHiMrPCceIa5/IeOf+Wt7wlGctbmz115xDSa6x//vOfT+6Tt/Z9Nhe1oqNHjyZjqTuOx44dS+6zcuXKZGz69OnJWN5dxdTdsbw7onkTwceDIiwk6MRlZoO4xWVmhePEZWaF48RlZoXjxGVmheLOeTMrJLe4xrne3vQk97vvvjsZe/DBB5Oxw4eT66vxq1/9qmL5K6+8ktwnb8hGnry/yqnJzbNnz07uM2/evGQsb138vP9kv//97yuW553fvIng44UTl5kVjhOXmRWKJ1mbWSE5cZlZ4fiuopkVjltcZlYo7uOy3BUUtmzZkoxNnDgxGctbvSB1+//kyZPJfYarvb09Gevo6KhY/sEPppcWv+yy9POF84ZR5A05+d73vlex/Dvf+U5yn0OHDiVj40WrJ66qA3gkrZe0X9KOsrJzJf1Q0i+zr+eMbjXNrJHq+HiyUVHLyMNvAAP/fN4FPBERC4AnsvdmNkYUPnFFxFPAgQHF1wMPZd8/BNxQ32qZWbP0z1Ws5dUsw+3jmhkRewEiYq+k5BKUktYCa4f5OWbWBK3exzXqnfMR0Ql0Akhq7bNhZkDrJ67hza6FfZLaAbKv++tXJTNrtlbv4xpui2sjsBq4J/v6/brVaIzJ6wd444036v55bW1tFcunTZuW3Ofd7353MrZgwYJk7PLLL0/GrrrqqorlCxcuTO6T9/CQnp6eZOzHP/5xMnb//fdXLH/55ZeT+5w6dSoZGy9avcVVNXFJegRYDkyT1AP8NaWE9W1JtwO7gVWjWUkza5wxsZBgRNycCH2gznUxsxZR+BaXmY0/TlxmVjhOXGZWKM2+Y1gLJy4zG8SJy5IkJWN5Ky+84x3vSMZmzZpVsXz+/PnJfa6++upkLG84RN4xp0yZUrH8xIkTyX22bduWjG3cuDEZe+qpp5Kxrq6uZMzSCn9X0czGlyJcKg535LyZjWH1HDkvqUPSC5K6JVVcSUbSckldknZK+r/VjukWl5kNUq8Wl6Q24D5gBdADbJW0MSKeL9tmKvAVoCMiduct2tDPLS4zG6SOLa5lQHdEvBQRJ4BHKS2LVe4WYENE7M4+u+rcZycuMzvNENfjmiZpW9lr4BJWc4BXy973ZGXl3g2cI+lJSdsl3Vatjr5UNLNBhnCp2BsRS3PilW6dDzz4GcB7KE0jnAz8k6SfRcSLqYM6cY2ySZMmJWPz5s1LxlasWJGMXXPNNcnYBRdcULF8zpyBf+Rqi9XbsWPHkrG84RAPP/xwMrZ3794R1ckGq+NdxR5gbtn784E9FbbpjYgjwBFJTwGXAcnE5UtFMxukjn1cW4EFkuZLmgTcRGlZrHLfB/6lpDMknQVcCezKO6hbXGY2SL1aXBHRJ2kdsBloA9ZHxE5Jd2TxByJil6S/B54F3gS+FhE70kd14jKzAeo9ADUiNgGbBpQ9MOD93wB/U+sxnbjMbBBP+TGzwmn1KT9OXKNs6tSpyditt96ajN1yyy3J2Dvf+c4h1yNvQncjf0nz7rKm7ogCLFmyJBmbMCF9jylvrXpLc+Iys0IpwiRrJy4zG8SJy8wKx4nLzArHdxXNrFDcx2VmheTENc6dddZZydgVV1yRjM2dOzcZy1u3PfX4+Lym/8mTJ5OxPBMnThxybPLkycl9brjhhmQs71w98sgjydinP/3piuWp82QlrZ64qk6ylrRe0n5JO8rKPivptWyp1S5J141uNc2skeq5dPNoqGV1iG8AHRXK/zYiFmevTRXiZlZAQ1xIsCmqXipGxFOS5jWgLmbWIgp/qZhjnaRns0vJc1IbSVrbv6zrCD7LzBpoLFwqVnI/cCGwGNgLfCG1YUR0RsTSKsu7mlkLafXENay7ihGxr/97SV8FHq9bjcys6Vr9UnFYiUtSe0T0L/R9I5C7WuF49vrrrydjmzdvTsaOHj2ajPX19SVjzz33XMXyrVu3JvfZsmVLMpbnwx/+cDK2bNmyiuV56+W/613vSsZmzEg/au/9739/MnbxxRdXLN+1K70y8HgfKtHs1lQtqiYuSY8Ayyk9hqgH+GtguaTFlJ7W8TLwF6NXRTNrtMJP+YmImysUf30U6mJmLaLwLS4zG3+cuMysUMZEH5eZjT9OXGZWOK2euNTICkpq7bMxCtra2pKxc889Nxk7++yzk7G82/WpYRRHjhxJ7pMXy3POOckJE8n6X3TRRcl91qxZk4ytWrUqGdu/f38y9uCDD1Ys/9znPpfc59ixY8lYEURE+skoNWhvb4/Vq1fXtO299967vRmDy93iMrPTuI/LzArJicvMCseJy8wKx4nLzAqlfyHBVubEZWaDuMU1zuUNXfjNb36TjPX29g7r8xr5C3fw4MFk7NChQ0PeZ968ecnY0qXpO+5z5sxJxlasWFGx/O67707uY05cZlZATlxmVjhOXGZWKEUYgDqSh2WY2RhVz8eTSeqQ9IKkbkl35Wx3haRTkv5ttWM6cZnZIPV6WIakNuA+YCWwCLhZ0qLEdvcC6fXMy/hSsUW1elO9mlT989bg//Wvf52MHThwIBnLuxs5c+bMiuUTJvhvdp46/v4tA7oj4iUASY8C1wPPD9ju48BjwBW1HNQ/PTM7Ta2trSy5Tet/bmr2WjvgcHOAV8ve92RlfyBpDqWH7jxQax3d4jKzQYbQ4uqtsqxNpSV2Bh78vwGfiohTUm0r8jhxmdkgdbxU7AHmlr0/H9gzYJulwKNZ0poGXCepLyK+lzqoE5eZDVLHuYpbgQWS5gOvATcBt5RvEBHz+7+X9A3g8bykBU5cZjZAPcdxRUSfpHWU7ha2AesjYqekO7J4zf1a5Zy4zGyQet7VjohNwKYBZRUTVkT8eS3HrOVJ1nOBbwKzgDeBzoj4kqRzgf8FzKP0NOs/i4j0DFobVyZOnFixfMaMGcl95s+fn4zlrW9v9dfqw3FqGQ7RB/xlRFwMXAV8LBtAdhfwREQsAJ7I3pvZGFCvAaijpWqLKyL2Anuz71+XtIvSOIzrgeXZZg8BTwKfGpVamlnDjLmFBCXNA5YAW4CZWVIjIvZKSl8DmFmhtPqlYs2JS9IUSkPyPxERh2sdKJaNpB04mtbMWtiYSFySJlJKWg9HxIaseJ+k9qy11Q5UfCpnRHQCndlxWvtsmBnQ+omraue8Sk2rrwO7IuKLZaGNQP/jblcD369/9cysGQrfOQ+8F/gI8JykrqzsM8A9wLcl3Q7sBtLPSB8lkydPTsZSt+MBjh8/PqyYnW7KlCnJ2MKFCyuWv+9970vu86EPfSgZy1tX/uTJk8nY7373u4rlrd6iaKZmJ6Va1HJX8adUnigJ8IH6VsfMWsGYuqtoZuND4VtcZjb+OHGZWaGMiT4uMxt/nLjMrHDcOV8HZ599dsXySy65JLlP3ioE+/dXHCsLwKuvvpqMpR4f39fXl9xnNEyaNCkZa2trq1ieN2Rg2rRpw6rHxRdfnIytWlV5dMyKFSuS+8yePTsZy6v/nj0DF9T8ox/96EcVyxv9MysSXyqaWSE5cZlZ4ThxmVnhOHGZWeE4cZlZoYy5hQTNbHxwi6sOli9fXrH8k5/8ZHKfJUuWJGN5Qx6efvrpZOyxxx6rWJ43vGI0/nLNnTs3GUs9VGLfvn3JfdasWZOM5a2ysXRp+gHG5513XsXy1HANgBMnTiRjeUMeHn/88WTszjvvTMYszYnLzArHicvMCsUDUM2skJy4zKxwfFfRzArHLa46uPXWWyuW561fPmFC+jkgF110UTK2YMGCZOzGG2+sWN7oH3Levy0Vy/sLmjdpO0/eHcJTp05VLD906FByn23btiVj3/rWt5KxDRs2JGM2dO7jMrNCcuIys8Jx4jKzwnHnvJkVivu4zKyQnLjMrHAKn7gkzQW+CcwC3gQ6I+JLkj4L/HvgN9mmn4mITaNRyV27dlUsX7x4cXKfvEe2500czhsaMNxhA42UGoaQN4E5L5Zn06b0j7urq6ti+TPPPJPc58UXX0zGDhw4kIwdPXo0GbPhafXElR4Q9Ed9wF9GxMXAVcDHJC3KYn8bEYuz16gkLTNrvP5+rmqvWkjqkPSCpG5Jd1WI/ztJz2avf5R0WbVjVm1xRcReYG/2/euSdgHp5oyZFVo9FxKU1AbcB6wAeoCtkjZGxPNlm/0z8P6IOChpJdAJXJl33FpaXOWVmAcsAbZkReuyLLleUuWFoMyscOrY4loGdEfESxFxAngUuH7AZ/1jRPQ/++9nwPnVDlpz4pI0BXgM+EREHAbuBy4EFlNqkX0hsd9aSdskpedzmFlLGULimtb//zt7rR1wqDlA+cqdPeRfsd0O/J9q9avprqKkiZSS1sMRsSH7h+0ri38VqLgMZUR0Umr6Iam1e/zMDBhS53xvRKSXwgVVOnzFDaU/oZS40pOQM7XcVRTwdWBXRHyxrLw96/8CuBHYUe1YZtb66jwAtQcoX2v8fGDQOtySLgW+BqyMiN9WO2gtLa73Ah8BnpPUlZV9BrhZ0mJK2fNl4C9qONawpFYGePLJJ5P7zJo1Kxm78MILk7GFCxfWXK9W9Nprr1Us/8lPflL3z8pbu//gwYMVy/OGNRw5cmTEdbL6qGPi2goskDQfeA24CbilfANJ7wA2AB+JiPSYmDK13FX8KZWbex7+YDZG1euuYkT0SVoHbAbagPURsVPSHVn8AeCvgPOAr5Qu8OircvnpkfNmNlg9B6BmYzw3DSh7oOz7jwIfHcoxnbjM7DSeZG1mheTEZWaF48RlZoXjhQTroLu7e0jlAJMnT07Gpk+fnozNmDGj9oq1oNTDKPLOlVk593GZWSE5cZlZ4ThxmVnhOHGZWeE4cZlZodRzIcHR4sRlZoO4xdUkx44dS8Z27949rJjZeOHEZWaF48RlZoXiAahmVkhOXGZWOL6raGaF4xaXmRWK+7jMrJCcuMyscJy4zKxw3DlvZoXiPi4zKyQnLjMrnFZPXBOqbSDpTEn/T9LPJe2U9J+y8nMl/VDSL7Ov54x+dc2sEfovF6u9mqVq4gKOA9dGxGXAYqBD0lXAXcATEbEAeCJ7b2ZjQOETV5S8kb2dmL0CuB54KCt/CLhhNCpoZo3Vv5BgLa9mqaXFhaQ2SV3AfuCHEbEFmBkRewGyr8V+rpeZ/UGrt7hq6pyPiFPAYklTge9KuqTWD5C0Flg7vOqZWTMUvnO+XEQcAp4EOoB9ktoBsq/7E/t0RsTSiFg6sqqaWaO0eourlruK07OWFpImA/8K+AWwEVidbbYa+P4o1dHMGqjWpNXql4rtwEOS2iglum9HxOOS/gn4tqTbgd3AqlGsp5k1UKtfKqqRFZTU2mfDbAyICI1k/wkTJsTEiRNr2vbEiRPbm9EN5JHzZjZIq7e4nLjM7DTN7r+qxZDuKprZ+FDPznlJHZJekNQtadAMG5X8XRZ/VtLl1Y7pxGVmg9QrcWU39e4DVgKLgJslLRqw2UpgQfZaC9xf7bhOXGY2SB2n/CwDuiPipYg4ATxKabpgueuBb2bTC38GTO0fI5rS6D6uXuCV7Ptp2ftmcz1O53qcrmj1uKAOn7U5+7xanClpW9n7zojoLHs/B3i17H0PcOWAY1TaZg6wN/WhDU1cETG9/3tJ21phNL3r4Xq4HqeLiI46Hq7S0IyB15i1bHMaXyqa2WjqAeaWvT8f2DOMbU7jxGVmo2krsEDSfEmTgJsoTRcstxG4Lbu7eBXwu/6VZ1KaOY6rs/omDeF6nM71OJ3rMQIR0SdpHaV+szZgfUTslHRHFn8A2ARcB3QDR4E11Y7b0Ck/Zmb14EtFMyscJy4zK5ymJK5qUwAaWI+XJT0nqWvAWJTR/tz1kvZL2lFW1vCnJiXq8VlJr2XnpEvSdQ2ox1xJP5a0K3uS1J1ZeUPPSU49GnpO/GSt6hrex5VNAXgRWEHpNuhW4OaIeL6hFSnV5WVgaUQ0dIChpGuANyiNFr4kK/svwIGIuCdL5udExKeaUI/PAm9ExH8dzc8eUI92oD0inpb0NmA7pYev/DkNPCc59fgzGnhOJAl4a0S8IWki8FPgTuDf0ODfkVbVjBZXLVMAxrSIeAo4MKC44U9NStSj4SJib0Q8nX3/OrCL0sjphp6TnHo0lJ+sVV0zEldqeH8zBPAPkrZnD/VoplZ6atK6bJb++kZfjkiaBywBmvokqQH1gAafE/nJWrmakbiGPLx/FL03Ii6nNDv9Y9ml03h3P3AhpYf/7gW+0KgPljQFeAz4REQcbtTn1lCPhp+TiDgVEYspjSJfpiE8WWs8aEbiGvLw/tESEXuyr/uB71K6jG2Wmp6aNNoiYl/2n+ZN4Ks06JxkfTmPAQ9HxIasuOHnpFI9mnVOss8+xBCfrDUeNCNx1TIFYNRJemvWAYuktwIfBHbk7zWqWuKpSQOWE7mRBpyTrDP668CuiPhiWaih5yRVj0afE/nJWtUN5VFE9XpRGt7/IvAr4D82qQ7vBH6evXY2sh7AI5QuOU5SaoHeDpwHPAH8Mvt6bpPq8S3gOeBZSv9R2htQj/dR6i54FujKXtc1+pzk1KOh5wS4FHgm+7wdwF9l5Q3/HWnVl6f8mFnheOS8mRWOE5eZFY4Tl5kVjhOXmRWOE5eZFY4Tl5kVjhOXmRXO/weOzhS3T9Z1xAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]])\n" + ] + } + ], "source": [ "import matplotlib.pyplot as plt\n", "\n", @@ -143,11 +213,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0. Train Loss: 0.292838, Train Acc: 0.908382, Valid Loss: 0.075638, Valid Acc: 0.974684, Time 00:00:13\n", + "Epoch 1. Train Loss: 0.077091, Train Acc: 0.976479, Valid Loss: 0.066128, Valid Acc: 0.978936, Time 00:00:14\n", + "Epoch 2. Train Loss: 0.055866, Train Acc: 0.982759, Valid Loss: 0.042326, Valid Acc: 0.986748, Time 00:00:14\n", + "Epoch 3. Train Loss: 0.043993, Train Acc: 0.986257, Valid Loss: 0.042040, Valid Acc: 0.986847, Time 00:00:14\n", + "Epoch 4. Train Loss: 0.035289, Train Acc: 0.988823, Valid Loss: 0.035118, Valid Acc: 0.988430, Time 00:00:14\n", + "Epoch 5. Train Loss: 0.030174, Train Acc: 0.990572, Valid Loss: 0.036890, Valid Acc: 0.988430, Time 00:00:14\n", + "Epoch 6. Train Loss: 0.025604, Train Acc: 0.991571, Valid Loss: 0.028075, Valid Acc: 0.990803, Time 00:00:14\n", + "Epoch 7. Train Loss: 0.021483, Train Acc: 0.993220, Valid Loss: 0.039955, Valid Acc: 0.988133, Time 00:00:14\n", + "Epoch 8. Train Loss: 0.018553, Train Acc: 0.994020, Valid Loss: 0.031569, Valid Acc: 0.990506, Time 00:00:14\n", + "Epoch 9. Train Loss: 0.016860, Train Acc: 0.994420, Valid Loss: 0.028923, Valid Acc: 0.990803, Time 00:00:14\n", + "Epoch 10. Train Loss: 0.014547, Train Acc: 0.995186, Valid Loss: 0.041005, Valid Acc: 0.987737, Time 00:00:14\n", + "Epoch 11. Train Loss: 0.011832, Train Acc: 0.996085, Valid Loss: 0.039684, Valid Acc: 0.989221, Time 00:00:14\n", + "Epoch 12. Train Loss: 0.012104, Train Acc: 0.996019, Valid Loss: 0.033983, Valid Acc: 0.990012, Time 00:00:14\n", + "Epoch 13. Train Loss: 0.009578, Train Acc: 0.996802, Valid Loss: 0.044510, Valid Acc: 0.989419, Time 00:00:14\n", + "Epoch 14. Train Loss: 0.008961, Train Acc: 0.997018, Valid Loss: 0.033376, Valid Acc: 0.991693, Time 00:00:14\n", + "Epoch 15. Train Loss: 0.008937, Train Acc: 0.997002, Valid Loss: 0.054347, Valid Acc: 0.986847, Time 00:00:15\n", + "Epoch 16. Train Loss: 0.009171, Train Acc: 0.996902, Valid Loss: 0.034495, Valid Acc: 0.991594, Time 00:00:16\n", + "Epoch 17. Train Loss: 0.006915, Train Acc: 0.997818, Valid Loss: 0.046391, Valid Acc: 0.989517, Time 00:00:16\n", + "Epoch 18. Train Loss: 0.007419, Train Acc: 0.997651, Valid Loss: 0.044388, Valid Acc: 0.989419, Time 00:00:16\n", + "Epoch 19. Train Loss: 0.006600, Train Acc: 0.998001, Valid Loss: 0.049959, Valid Acc: 0.987935, Time 00:00:16\n" + ] + } + ], "source": [ "net = LeNet5()\n", "optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)\n", @@ -201,7 +298,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.8.12" } }, "nbformat": 4, diff --git a/7_deep_learning/1_CNN/03-AlexNet.ipynb b/7_deep_learning/1_CNN/03-AlexNet.ipynb index efcaa08..dac9292 100644 --- a/7_deep_learning/1_CNN/03-AlexNet.ipynb +++ b/7_deep_learning/1_CNN/03-AlexNet.ipynb @@ -12,19 +12,21 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import torch.nn as nn\n", "import torch\n", + "from torch.autograd import Variable\n", + "\n", "\n", "class AlexNet(nn.Module):\n", " def __init__(self, num_classes=1000, init_weights=False): \n", " super(AlexNet, self).__init__()\n", " self.features = nn.Sequential( \n", - " nn.Conv2d(1, 96, kernel_size=11, stride=4, padding=2), \n", - " nn.ReLU(inplace=True), #inplace 可以载入更大模型\n", + " nn.Conv2d(3, 96, kernel_size=11, stride=4, padding=2), \n", + " nn.ReLU(inplace=True),\n", " nn.MaxPool2d(kernel_size=3, stride=2), \n", "\n", " nn.Conv2d(96, 256, kernel_size=5, padding=2),\n", @@ -74,7 +76,27 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 256, 6, 6])\n" + ] + } + ], + "source": [ + "test_x = Variable(torch.zeros(1, 3, 227, 227))\n", + "net = AlexNet()\n", + "test_y = net.features(test_x)\n", + "print(test_y.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -102,22 +124,74 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0. Train Loss: 1.716735, Train Acc: 0.363671, Valid Loss: 1.407991, Valid Acc: 0.488726, Time 00:01:30\n", + "Epoch 1. Train Loss: 1.358077, Train Acc: 0.511409, Valid Loss: 1.216201, Valid Acc: 0.570609, Time 00:01:46\n", + "Epoch 2. Train Loss: 1.215461, Train Acc: 0.566356, Valid Loss: 1.082299, Valid Acc: 0.616495, Time 00:01:54\n", + "Epoch 3. Train Loss: 1.127000, Train Acc: 0.601602, Valid Loss: 1.065414, Valid Acc: 0.620055, Time 00:01:44\n", + "Epoch 4. Train Loss: 1.050764, Train Acc: 0.631494, Valid Loss: 1.040739, Valid Acc: 0.636867, Time 00:01:44\n", + "Epoch 5. Train Loss: 0.986251, Train Acc: 0.652853, Valid Loss: 0.959156, Valid Acc: 0.660601, Time 00:01:44\n", + "Epoch 6. Train Loss: 0.947220, Train Acc: 0.668278, Valid Loss: 0.942900, Valid Acc: 0.670491, Time 00:01:45\n", + "Epoch 7. Train Loss: 0.909791, Train Acc: 0.682225, Valid Loss: 0.951977, Valid Acc: 0.662777, Time 00:01:45\n", + "Epoch 8. Train Loss: 0.879402, Train Acc: 0.690337, Valid Loss: 0.917556, Valid Acc: 0.678797, Time 00:01:44\n", + "Epoch 9. Train Loss: 0.850324, Train Acc: 0.700148, Valid Loss: 0.930344, Valid Acc: 0.672271, Time 00:01:44\n", + "Epoch 10. Train Loss: 0.824315, Train Acc: 0.710698, Valid Loss: 0.897207, Valid Acc: 0.690170, Time 00:01:44\n", + "Epoch 11. Train Loss: 0.793646, Train Acc: 0.720348, Valid Loss: 0.869203, Valid Acc: 0.705202, Time 00:01:45\n", + "Epoch 12. Train Loss: 0.775788, Train Acc: 0.729779, Valid Loss: 0.845823, Valid Acc: 0.706092, Time 00:01:44\n", + "Epoch 13. Train Loss: 0.748050, Train Acc: 0.738451, Valid Loss: 0.871864, Valid Acc: 0.701444, Time 00:01:47\n", + "Epoch 14. Train Loss: 0.738969, Train Acc: 0.739390, Valid Loss: 0.848204, Valid Acc: 0.715487, Time 00:01:46\n", + "Epoch 15. Train Loss: 0.713504, Train Acc: 0.748382, Valid Loss: 0.843485, Valid Acc: 0.712915, Time 00:01:46\n", + "Epoch 16. Train Loss: 0.701577, Train Acc: 0.756993, Valid Loss: 0.860446, Valid Acc: 0.709059, Time 00:01:46\n", + "Epoch 17. Train Loss: 0.670257, Train Acc: 0.764926, Valid Loss: 0.836197, Valid Acc: 0.714201, Time 00:01:46\n", + "Epoch 18. Train Loss: 0.645218, Train Acc: 0.774776, Valid Loss: 0.857714, Valid Acc: 0.703916, Time 00:01:46\n", + "Epoch 19. Train Loss: 0.630583, Train Acc: 0.779691, Valid Loss: 0.815565, Valid Acc: 0.720134, Time 00:01:46\n" + ] + } + ], "source": [ "net = AlexNet(num_classes=10)\n", "optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)\n", "criterion = nn.CrossEntropyLoss()\n", "\n", - "res = train(net, train_data, test_data, 20, optimizer, criterion, use_cuda=False)" + "res = train(net, train_data, test_data, 20, optimizer, criterion, use_cuda=True)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAydklEQVR4nO3deXyV5Zn/8c91su/7HrKwBghhEQgCKogLogIqVaxaa+swaq3WTjt26nSx/jpj23Gm40KtWlttKU7rBlZAlEVEEAgKSdi3ACFkIYHse+7fH88JxJiEE5KTk+Rc79frvHJynuVceTg833M/y32LMQallFLuy+bqApRSSrmWBoFSSrk5DQKllHJzGgRKKeXmNAiUUsrNebq6gO6KjIw0KSkpri5DKaUGlJ07d54xxkR1NG3ABUFKSgpZWVmuLkMppQYUETne2TQ9NKSUUm5Og0AppdycBoFSSrm5AXeOQCmluquxsZH8/Hzq6upcXYrT+fr6kpiYiJeXl8PLaBAopQa9/Px8goKCSElJQURcXY7TGGMoLS0lPz+f1NRUh5fTQ0NKqUGvrq6OiIiIQR0CACJCREREt1s+GgRKKbcw2EOg1aX8nW4TBAeLKvnl+3upa2x2dSlKKdWvuE0Q5J+t4eVPjvH5ibOuLkUp5WbOnTvH0qVLu73cvHnzOHfuXO8X1I7bBMFlyeGIwLajZa4uRSnlZjoLgubmro9QrFq1itDQUCdVdYHbXDUU4ufFmLhgth0rdXUpSik386Mf/YgjR44wYcIEvLy8CAwMJC4ujl27drF3714WLlzIyZMnqaur49FHH2XJkiXAhS51qqqquOGGG5g5cyZbtmwhISGBFStW4Ofn1yv1uU0QAGSmRrBs23Hqm5rx8fRwdTlKKRd48r097C2o6NV1jokP5mc3j+10+tNPP01ubi67du1i48aN3HjjjeTm5p6/xPPVV18lPDyc2tpapkyZwm233UZERMSX1nHo0CGWL1/Oyy+/zO23385bb73F3Xff3Sv1u82hIYDMoeHUN7WQnV/u6lKUUm5s6tSpX7rO/9lnn2X8+PFMmzaNkydPcujQoa8sk5qayoQJEwC47LLLyMvL67V63KpFMDUlHIBtR0uZYn+ulHIvXX1z7ysBAQHnn2/cuJGPPvqIrVu34u/vz6xZszq8D8DHx+f8cw8PD2pra3utHrdqEYQFeJMWG8S2Y3rCWCnVd4KCgqisrOxwWnl5OWFhYfj7+7N//34+++yzPq7OzVoEAFNTw3lzZz6NzS14ebhVDiqlXCQiIoIZM2aQnp6On58fMTEx56fNnTuXF198kYyMDEaNGsW0adP6vD63C4LM1Ahe33qcnFPlTEoKc3U5Sik38de//rXD1318fFi9enWH01rPA0RGRpKbm3v+9R/84Ae9WpvbfSWemmqdG9iuh4eUUgpwwyCICvJhWFQA247q/QRKKQVuGAQAmUMjyMo7S3OLcXUpSinlcu4ZBKnhVNY39fpNJUopNRC5aRBYd+xpdxNKKeWmQRAb4ktyhD+faQd0SinlnkEA1uGhHXlltOh5AqVUPxMYGAhAQUEBixYt6nCeWbNmkZWV1Svv58ZBEEF5bSMHijq+208ppVwtPj6eN9980+nv475BMPRCv0NKKeVMjz/++JfGI/j5z3/Ok08+yZw5c5g0aRLjxo1jxYoVX1kuLy+P9PR0AGpra1m8eDEZGRnccccdvdrXkNPuLBaRV4GbgGJjTHon88wCfgt4AWeMMVc5q572EsP8SQj1Y9uxMr45I/XiCyilBofVP4LCnN5dZ+w4uOHpTicvXryY733vezz00EMA/O1vf2PNmjU89thjBAcHc+bMGaZNm8b8+fM7HXP4d7/7Hf7+/mRnZ5Odnc2kSZN6rXxndjHxJ+B54PWOJopIKLAUmGuMOSEi0U6spUOZqeF8fLAEY4zbDGytlOp7EydOpLi4mIKCAkpKSggLCyMuLo7HHnuMTZs2YbPZOHXqFEVFRcTGxna4jk2bNvHII48AkJGRQUZGRq/V57QgMMZsEpGULmb5OvC2MeaEff5iZ9XSmcyh4bz9xSmOlFQxPDqor99eKeUKXXxzd6ZFixbx5ptvUlhYyOLFi1m2bBklJSXs3LkTLy8vUlJSOux+ui1nfWF15TmCkUCYiGwUkZ0i8o3OZhSRJSKSJSJZJSUlvVZA6/0EehmpUsrZFi9ezBtvvMGbb77JokWLKC8vJzo6Gi8vLzZs2MDx48e7XP7KK69k2bJlAOTm5pKdnd1rtbkyCDyBy4AbgeuBn4jIyI5mNMa8ZIyZbIyZHBUV1WsFJEf4ExPso+MTKKWcbuzYsVRWVpKQkEBcXBx33XUXWVlZTJ48mWXLlpGWltbl8g8++CBVVVVkZGTw61//mqlTp/Zaba7shjof6wRxNVAtIpuA8cDBvipARJiaGsG2o6V6nkAp5XQ5ORdOUkdGRrJ169YO56uqqgKswetbu5/28/PjjTfecEpdrmwRrACuEBFPEfEHMoF9fV1EZmo4xZX15JXW9PVbK6VUv+DMy0eXA7OASBHJB36GdZkoxpgXjTH7RGQNkA20AK8YY3I7W5+zTGtzP0FqZMBF5lZKqcHHmVcN3enAPL8BfuOsGhwxLCqQyEBvth8rY/HUJFeWopRyInc5/GtM97vNcds7i1tZ5wnC9YSxUoOYr68vpaWll7STHEiMMZSWluLr69ut5dxuzOKOZKZGsCqnkJNlNQwJ93d1OUqpXpaYmEh+fj69efl5f+Xr60tiYmK3ltEg4MI4xtuOlWkQKDUIeXl5kZqqXcl0xu0PDQGMigki1N9LO6BTSrklDQLAZhOmpISzPU/PEyil3I8GgV1majjHS2soLO+6rw+llBpsNAjspg3VcYyVUu5Jg8BudFwwQT6e2gGdUsrtaBDYediEySlh2iJQSrkdDYI2ModGcLSkmuJKPU+glHIfGgRtZNrvJ9hx7KyLK1FKqb6jQdBGekII/t4eenhIKeVWNAja8PKwcVlyGNv0hLFSyo1oELSTmRrOgaJKyqobXF2KUkr1CQ2CdjLt9xNs195IlVJuQoOgnYzEEHw8bRoESim3oUHQjo+nB5OS9H4CpZT70CDoQObQcPaerqC8ttHVpSillNNpEHRgamo4xkCW9kaqlHIDGgQdmJQUhreHTYevVEq5BQ2CDvh6eTB+SIgOVKOUcgsaBJ3ITI0gt6CCqvomV5eilFJOpUHQicyh4TS3GHYe136HlFKDmwZBJy5LDsPTJnp4SCk16GkQdMLf25P0hBA9YayUGvQ0CLqQOTSc7Pxz1DY0u7oUpZRyGg2CLkxLjaCx2fDFCT1PoJQavDQIujA5JQybwGd6eEgpNYg5LQhE5FURKRaR3IvMN0VEmkVkkbNqAaC+Cr5YBsY4vEiQrxdj4/V+AqXU4ObMFsGfgLldzSAiHsCvgA+cWIdl7wpY8RAcWd+txaamhvPFyXPUNep5AqXU4OS0IDDGbAIudkzlu8BbQLGz6jhv3CIIjIUtz3ZrsczUcBqaWth98pxz6lJKKRdz2TkCEUkAbgFedGDeJSKSJSJZJSUll/aGnj4w7UE4uhEKdjm82NTUcETQy0iVUoOWK08W/xZ43Bhz0WMuxpiXjDGTjTGTo6KiLv0dJ98H3kGw5TmHFwn192ZUTJAOVKOUGrRcGQSTgTdEJA9YBCwVkYVOfUffEJj8TdjzDpw97vBi04ZGsPP4WRqbW5xXm1JKuYjLgsAYk2qMSTHGpABvAg8ZY951+htnPggi8NlSxxdJDae2sZns/HInFqaUUq7hzMtHlwNbgVEiki8i3xaRB0TkAWe9p0NCEmDc1+Dz16HGscM9U1LDAXT4SqXUoOTMq4buNMbEGWO8jDGJxpg/GGNeNMZ85eSwMeabxpg3nVXLV0z/LjTWwI4/ODR7ZKAPw6MD2XZUzxMopQYf97yzOGYsDL8Wtr0IjbUOLZKZGk5WXhlNep5AKTXIuGcQAMx4FGrOwO7lDs0+fVgk1Q3NfLTP+bc8KKVUX3LfIEiZCfETYcvz0HLxu4avGxtDWmwQv3hvD9U6aplSahBx3yAQsVoFZUdg//sXnd3Lw8Yvb0mnoLyOZ9cd6oMClVKqb7hvEACMng9hKfDp/zrUGd1lyeHcOXUIr2w+xv7CCufXp5RSfcC9g8DmAZc/DKey4MRWhxZ5fG4aIX5ePPFOLi0tjvdkqpRS/ZV7BwHAhLvAPwI+dawzulB/b348bzQ7j5/lb1knnVycUko5nwaBtz9MXQIHV0PxfocWuW1SApmp4fzn6v2UVtU7uUCllHIuDQKAKf8Enn6w1bHO6ESEX96STk1DE/+xyrHwUEqp/kqDACAgAibeDbv/DypOO7TI8Oggllw5lLc+z2frEe16Qik1cGkQtLr8O2CarbuNHfTw7BEMCffjiXdzqG/SEcyUUgOTBkGr8FQYswCyXoU6xy4N9fP24Bfz0zlaUs3Lm446uUCllHIODYK2pj8C9RXw+WsOLzI7LZp542J5bv1hjpdWO7E4pZRyDg2CthImQcoVsHUpNDU4vNhPbxqLp0346Yo9GAduTFNKqf5Eg6C9GY9CZQHkvuXwIrEhvvzLdaP4+GAJq3IKnVicUkr1Pg2C9oZfA9FjYMuzDnU70eoblyczNj6YJ9/bQ2VdoxMLVEqp3qVB0J6Ida6geC8c/sjhxTw9bPzylnGUVNXzzNqDTixQKaV6lwZBR9Jvg+AEqzO6bpgwJJS7M5N5fWseOTq+sVJqgNAg6IinN0x7EPI+gVM7u7XoD64fRXiAD0+8m0OzdkqnlBoANAg6M+le8AlxuDO6ViF+XvzkptFk55ezbNtxJxWnlFK9R4OgM77BMPk+2LcSyrp3s9j88fHMHB7Jb9YcoLiizkkFKqVU79Ag6ErmAyAesPWFbi0mIjy1MJ365haeen+fk4pTSqneoUHQleA4GH8HfLEMqs90a9HUyAAemjWM93YXsOlgiZMKVEqpntMguJjpj0BTLWx/uduLPnDVMFIjA/jJilzqGrVTOqVU/6RBcDFRo2DkDbD9JWio6daivl4ePLUgneOlNSzdeMRJBSqlVM9oEDhixiNQWwa7lnV70ZkjIlkwIZ4XNx7hSEmVE4pTSqme0SBwRNLlkDgFtj4PzU3dXvyJG0fj42XjJ+/maqd0Sql+x6EgEJEAEbHZn48Ukfki4nWRZV4VkWIRye1k+l0ikm1/bBGR8d0vv4+IwMzH4GyedYiom6KDfPnXuWlsOVLKf6zap2GglOpXHG0RbAJ8RSQBWAfcB/zpIsv8CZjbxfRjwFXGmAzgKaD7e9i+NGqeda5g3S+gtPvH++/OTOIblyfz8ifHeHr1fg0DpVS/4WgQiDGmBrgVeM4YcwswpqsFjDGbgLIupm8xxpy1//oZkOhgLa4hAjf9N3h4w8pHoKWlm4sLT84fyz3Tkvn9pqM8vUbDQCnVPzgcBCJyOXAX8L79Nc9erOPbwOou3nyJiGSJSFZJiQuvyQ+Oh+t/Ccc3w85Xu724iPCLBWO5e1oSv//4KL9ac0DDQCnlco7uzL8H/BvwjjFmj4gMBTb0RgEiMhsrCGZ2No8x5iXsh44mT57s2j3nxLthz9vw4c9gxHUQmtStxUWEX8xPxxh48eMj2AR+eP0oRMRJBSulVNccahEYYz42xsw3xvzKftL4jDHmkZ6+uYhkAK8AC4wxpT1dX58QgZvt3VOvfKRbg9e0stmEpxakc+fUJJZuPMJ/rdWWgVLKdRy9auivIhIsIgHAXuCAiPywJ28sIknA28A9xpiBNZJLaBJc+yQc3QBf/OWSVmGzCb9cmM6dU4fwwoYjPLP2oIaBUsolHD1HMMYYUwEsBFYBScA9XS0gIsuBrcAoEckXkW+LyAMi8oB9lp8CEcBSEdklIlmX9Be4ymXfguSZ8METUFFwSauwwmAci6cM4fkNh/mfDzUMlFJ9z9FzBF72+wYWAs8bYxpFpMs9ljHmzotMvx+438H3739sNpj/LPxuBvzjMbjzDeuwUbdXI/zHLeMwBp5dfxhE+P61I51QsFJKdczRFsHvgTwgANgkIslAhbOKGjAihsGcn8LBNZDz90tejc0m/Oet47h9ciLPrjvEbz8aWEfKlFIDm0MtAmPMs0DbobqO26/2UZn/DHvegdX/CkNnQWD0Ja3GZhOevjWDFgO//egQgvDoNSN6t1allOqAoyeLQ0Tkv1uv5ReRZ7BaB8rmAQtesHomff9ferYqm/Cr2zK4bVIi//PRQZ5dd6iXilRKqc45emjoVaASuN3+qAD+6KyiBpyokTDrR9awlnve7dGqPGzCrxdlcOukBP77w4M8p2GglHIyR08WDzPG3Nbm9ydFZJcT6hm4pj8Ce1fAqh9AyhUQEHHJq/KwCb9ZNB4MPPPhQWw24Tuzh/disUopdYGjLYJaETl/56+IzABqnVPSAOXhCQuXQu05WPN4z1dnE37ztfHcMjGB33xwgBc2HO55jUop1QFHWwQPAK+LSIj997PAvc4paQCLGQtX/gA2/ieMvRXS5vVodR424b++Nh5jDL/54AAi8NAsbRkopXqXo1cN7QbGi0iw/fcKEfkekO3E2gammd+Hfe9Z9xYkTwe/0B6tzsMmPHP7BAzw6zUHKK9t5PHr07DZtG8ipVTv6NYIZcaYCvsdxgDfd0I9A5+nNyx4HqpLYO0TvbJKD5vwzNfGn++19LvLv6CusblX1q2UUj0ZqlK/knYmfiLMeNTqh+jwR72ySk8PG08tSOfH89J4P+c0d72yjbLqhl5Zt1LKvfUkCLRTnK5c9ThEjoSVj0Jd79yELSIsuXIYS++aRO6pcm5d+inHzlT3yrqVUu6ryyAQkUoRqejgUQnE91GNA5OXr3WjWcUp+OjnvbrqeePiWL5kGhV1Tdyy9FN25HU6EJxSSl1Ul0FgjAkyxgR38AgyxvTmCGWD05CpcPl3IOsPcGxTr656UlIY7zw0nXB/b+56eRsrd19aD6hKKdWTQ0PKEbOfgLBUWPldaOjdwzjJEQG8/dB0JgwJ5ZHlX7B042Htxlop1W0aBM7m7W9dRXQ2D9b/v15ffai/N3++fyoLJsTz6zUH+Le3c2hsbun191FKDV56eKcvpMyEKf8Eny2FA6shLgNiMyBuvPW4xB5LW/l4evDbOyaQFO7Pc+sPc+pcLUvvmkSQr1cv/QFKqcFMBtqhhMmTJ5usrIE1mBkAjbWw/SXIz4LCbKuF0Cowtk042H+GpVzSQDd/23GSH7+Tw/DoQF795hTiQ/167U9QSg1cIrLTGDO5w2kaBC5Sew6KcuH0bjidbYVDyQEw9hvFfEIgdpy91WAPh8iRVp9GF7H50Bke/MtO/H08ePWbUxgbH3LRZZRSg5sGwUDRWAvFey8Ew+lsKNoDTfb+/Tz9IGUGDL/GekQM77TVcKCwkvv+uJ3y2kae//okZqf17PCTUmpg0yAYyJqboPSQFQqnsuDIeii190QammQFwrA5kHol+AZ/adGiijq+/doO9hZU8OSCdO6ZluyCP0Ap1R9oEAw2Z/Pg8DrrcexjaKgCmycMmQbDr7bCIWYc2GxU1zfxyPIvWLe/mCVXDuXxuWl4aId1SrkdDYLBrKkBTm6DI+usfo0Kc6zXA6Jh+BwYNofm1Fk8ub6I17ceZ/qwCP7njgnEBPu6tGylVN/SIHAnlYXW4aPD66yftWWAYOInsidwGvfuy8R4B/DM7eOZPUrPGyjlLjQI3FVLMxTssloKR9bBye3UxUzk7tofkFUM/3RFKj+8Pg1vT72vUKnBrqsg0D3AYGbzgMTLYNbj8O21cMef8T2Ty998nuI7k/15+ZNjLHpxC3nag6lSbk2DwJ2Mvhnu+ju28pP8MP9RXl8YyfHSGm56bjMrdp1ydXVKKRfRIHA3Q2fBvSuhvpIrP7mbtV+PIC02iEff2MUP/76bmoYmV1eolOpjGgTuKOEy+NYasHkS8+YtvHEDPHL1cN78PJ+bn9vM3oLeGUhHKTUwOC0IRORVESkWkdxOpouIPCsih0UkW0QmOasW1YGoUfDtDyAgEs+/3Mr3U0+y7P5MKuuaWLj0U17fmqddWivlJpzZIvgTMLeL6TcAI+yPJcDvnFiL6khoEnzrA4gcDsvvYHrNRlY/egUzhkXw0xV7+Oc/7+RcjY6LrNRg57QgMMZsAroaQ3EB8LqxfAaEikics+pRnQiMgm++D4lT4a37idj3F/5w7xT+/cbRbDhQzLz//USHwlRqkHPlOYIE4GSb3/Ptr32FiCwRkSwRySopKemT4tyKbwjc8zaMvB7e/z62zc9w/8xU3npwOl6eNu74/VaeW3eI5hY9VKTUYOTKIOiow5sO9zTGmJeMMZONMZOjoqKcXJab8vKDO/4CGXfA+qdg7b+TkRDCP747k5vHx/PMhwe565XPyMkvd3WlSqle5soRyvKBIW1+TwR0BHZX8vCChS+CbyhsfR5qzxJ087P89o4JzBweyS/e28vNz29mamo4989MZc7oGO3ATqlBwJVBsBJ4WETeADKBcmPMaRfWowBsNrjhV+AfDhv/E+rKkdv+wNcmD2Fueiz/t+Mkf/w0jyV/3klyhD/fmpHKossSCfDRUU+VGqic1teQiCwHZgGRQBHwM8ALwBjzoogI8DzWlUU1wH3GmIt2IqR9DfWhbb+H1f8KKVfAncvBJwiApuYW1u4t4pVPjvL5iXME+3py59Qk7p2eokNjKtVPaadz6tJl/w3eecAaLvOutyAg4kuTPz9xlj9sPsaa3EIA5o2L4/6ZqYwfEuqCYpVSndEgUD1zYA38/V4ISYSJd1tjKceMg6CY87Pkn63htS15vLH9JJX1TUxODuP+K1K5dkysnkdQqh/QIFA9l/cprHwYyo5eeC0gCmLSrWCIHQcx6VQFpfK3zwv545ZjnCyrZUi4H/dNT+X2KUMIvJTzCE0NUFVojbNQew6Spn1lSE6l1MVpEKjeU3sWivZAYa41GlpRDhTvh+Z6a7qHN0SPpiUmnQMmmeUnQnj3dBjGJ5Q7pgzh3ukpDAn3h5YWqDkDlaetnXxFgfWzsvXnaag4bc3Tlpc/jL0FJn0DhmSCaGtDKUdoECjnam6E0sNWMBTmQJE9JKov3PxX5hlDbkMMgdSQ7FVBWEsZNtO+p1OxWhnBcRDU5tH6u4c37Hkbct60xmmOHGUFwvjFEBDZt3+zUgOMBoFyjcoiq8VQmAtFuTQUHaCg3pfcCj+ON4bQEhjLuLQ0pmaMxT9iCARGW/cyXEx9Fex9Fz5/3Rqv2eYFaTdaoTB0tnUJrFLqSzQIVL9S19jMqpzTvLYlj9355QT5eHLbZYncOz2F1MiA7q2seL8VCLuXW+MzhyRZJ7Qn3mWd3O4pY6CqGMqOWIevkqb1znqV6mMaBKrf+uLEWV7bksf7OadpbDZcNTKKb05P4aqRUdi6c7VRUz3sf98KhaMbAIHh11ithFE3dN3SMAZqyqydfekR6zBX6/Oyo9ZhqPMEhs2GCXdZrRAvvW9CDQwaBKrfK66sY/m2kyzbdpziynpSIvy55/IUvjY5kWBfBw4XtXU2D75YBl/8xTr5HBAFE75u9aPUVAelR+07+sP2nf0RqGvTh5J4WF10RwyD8GHWz4hh4B8BB1bDrr9C+UnwCYH0W61QSJysJ65Vv6ZBoAaMhqYW1uwp5LUteew8fhZ/bw9unZTAvZenMCImqHsra26CI+usVsKB1WCa20wUCBkCEUPb7OyHW89Dk8DTu/P1trRA3iewaxnsXQlNtRA50h42i62T20r1MxoEakDKPVXOn7bksXJ3AQ1NLUwfFsFtkxK5bmwMQd1tJVQWwaEPrG/14cMgLAW8fHteZF2FdeL6i2Vw8jMQGwybY4XCqHm98x5K9QINAjWglVU38MaOE/x12wnyz9bi7Wnj6lHRzJ8Qz9Vp0fh6ebi6REvpEauVsPsNqDhl9eI6bpEVCvGT9NCRcikNAjUoGGP4/MQ53ttdwPs5pymprCfA24Nrx8Qwf0I8M4dH4e3ZDy4dbWmGYx9brYT9/7DOS0SNtgJh7C0QOuTi61Cql2kQqEGnucWw7Wgp72UXsCqnkPLaRkL8vLghPZb54+PJHBrRP/o4qj0He96xWgr5O6zX4ibA6JutR9QoV1an3IgGgRrUGppa2Hy4hPd2n2btnkKqG5qJCvLhxnFx3Dw+nklJoUh/OCxTegT2vWc9Ttk/w5EjIe0mKxTiJ+rhI+U0GgTKbdQ2NLPhQDErdxWw/kAxDU0tJIT6cdP4OOaPj2dMXHD/CIWKAuu+h33vQd5m64qm4EQYfZMVDEmXg4cO9qN6jwaBckuVdY2s3VPEe9kFbD50hqYWQ2pkADekxzJvXBxj4/tJKNSUwcE1VigcWW+dU/ALh7R5kHYzDJ2lVx+pHtMgUG6vrLqBNbmFrMo5zdajpTS3GJIj/LkhPY4bx8WRntBPQqG+Cg5/ZJ1kPvgB1FeAdyCMuNZqKQy72hpGtC+VHrFqydsM4alWf07J08Hbv2/rUD2iQaBUG2XVDazdU8iq3EK2HLZaCkPC/ZiXHse8cXFkJIb0j1Boqodjn8C+lXBglb03V4H4CVYrYegsGDKt91sLzY1wYqu18z/4AZQesl4PTba6B29usHqCHZJp1TBstnUC3NZPLuNVHdIgUKoTZ6sb+HBvEatyT58/fJQQ6se8cbHcMC6OiUP6yYnmlmbrqqOjG61H/g5oaQJPX6sjvNZgiM24tB1ydSkc/tA6RHV4PdSXWzv7lJkwci6MuM5qDTTUwIktVg1HNlq9ywL4hUHqlVZrYegsa97+oPas1SX66d1WB4VhKTD86r4PLmOscTYCIh3rYdcJNAiUckB5TSMf7itiVc5pPjlUQmOzIT7ElxvGxTFvXCwTh4R1ryM8Z6qvhONbLgRD8V7r9fM75Fn2HfLQjpc3xhpg6OAa61t//g7AQGCMtdMfOdda3iew6zqqiuHox/Y6Nlg30oG1wx0622otpF5p1eVMrTvawmw4nQ2nd1nPz524MI9/5IWBjvzC7PVdbT1CEnq/ntLD1uG041vg+KfWtglOhBmPwqR7+rzDQg0CpbqpvLaRdfuKWJVTyKaDJTQ0txAb7Muc0dHMGR3N9GGR/eeOZrB2gsc2XQiG1h1yaFKbw0iZX975t84TP/HCt/64CZc+noMxcOaQFQhHNlg7wYZKq9uNuAlWDaFJ4BP05Yd3IPgEW8+76uOp7fucPWZ9yz+dfWHnX118YZ7wYRCXAXHjrVZS3Hjr23j1GWv7HF5nnZivKrTmjxptBcLwqyF5Rvd30i0tULLP2um37vxb6wmMsc6pxE+yrhQ7+ZnVEeLl34HJ3+6zoVc1CJTqgcq6RtbvL2Z1TiGfHCqhuqEZXy8b04dFcnVaNFenRRMf2o+6o279NtoaCsc+sQ71tPIKsL6pj7ze2vkHxTqnjuZGOLXTCoWjGyA/q13Hfx3w8G4TEO0Cw8sXyo5Zh3rqK6z5bZ4QldZmh59hjaPtyM7VGKsl1RoKx7dYQ656+Fg77mFXw/A5ED3mq/d3tDRbdRz/1BrP+8QW6zAUWN/6U2ZYgZI8w+rQsO3yeZ/CJ/9lvadvCEz9Z5j2oNMvAtAgUKqX1Dc1s/1YGev2FbN+fzEnymoASIsNYs5oKxQmDAnrH3c1t2pusg6VnNxu3cmcMhM8ffq+jsZa607r+kr7o8Ia66Ht7/Vtfj8/rcL+e7XVY2zc+Avf9qNG997J8tbzH4fXWzvpkn3W64GxViikXmm1II5vgROfXQijsFT7jn+mFSBhyY6936nP4ZNnrCvEvAJg8n1w+cNO671Wg0ApJzDGcKSkmvX7i1i/v5gdeWdpbjGE+Xsxe1Q0s9OiuXJkFCF+rjk5qHqo/JTVkjm8zvrZ+o0/clSbb/zTITi+Z+9TvA82/481FrfNwxphb8aj1nmWXqRBoFQfKK9tZNPBEjbsL2bDgWLO1jTiYRMmJ4fZWwsxDI++yMlX1T+1NFs77MAYCIxyznuUHYNP/9fql6qlGcZ9DWY+BtFpvbJ6DQKl+lhzi2HXybOs31/Mun3F7C+sBGBkTCDzx8czf3wCSRF6Q5bqQEUBbH0Bsl61DqeNvgmu+BfrpH4PaBAo5WKnztWybl8R7+0uYEeedYhhwpBQ5o+P56aMOKKDtQsJ1U51KWz7HWx7yTrZP2wOXPU4JGVe0upcFgQiMhf4X8ADeMUY83S76SHAX4AkwBP4L2PMH7tapwaBGuhOnavlvd0FrNxVwN7TFdgEpg2NYMGEeOaOjSPEX88pqDbqKmDHK1YrYeoSmPX4Ja3GJUEgIh7AQeBaIB/YAdxpjNnbZp4fAyHGmMdFJAo4AMQaYxo6W68GgRpMDhdXsXJ3ASt3nSKvtAYvD+Gqkdboa9eMjsbfW3sgVXYNNWBaLn6TXye6CgJnfsqmAoeNMUftRbwBLAD2tpnHAEFi3cMfCJQBTU6sSal+ZXh0IN+/diSPXTOCnFPlrNxVwD+yT/PRviL8vT24ZnQMCybEc8WIfjL6mnIdJ3by58wgSABOtvk9H2h/cOt5YCVQAAQBdxhjWtqvSESWAEsAkpKSnFKsUq4kImQkhpKRGMqP541me14ZK3YVsDr3NCt3FxDi58W8cbFMGxrBuIQQUiIC+k93F2rAc2YQdPQpbX8c6npgF3A1MAz4UEQ+McZUfGkhY14CXgLr0FDvl6pU/2GzCdOGRjBtaARPzh/L5sMlrNxVwIpdBSzfbn23CvLxJD0hhIzEEMYlhpCREMqQcL/+0UGeGnCcGQT5QNtRuhOxvvm3dR/wtLFOVBwWkWNAGrDdiXUpNWB4e9q4Oi2Gq9NiaGxu4VBRFTmnzpFzqpyc/HL++GkeDc1WIzrEz8sKBntApCeEkBCq4aAuzplBsAMYISKpwClgMfD1dvOcAOYAn4hIDDAKOOrEmpQasLw8bIyJD2ZMfDB3TLFea2hq4WBRJdn55eScOkd2fjkvbTpKU4vVcA4P8D4fDOMSQpg2LIJgX70qSX2Z04LAGNMkIg8DH2BdPvqqMWaPiDxgn/4i8BTwJxHJwTqU9Lgx5oyzalJqsPH2tJGeYH37t67ChrrGZvYXVpKTf84eEOW8sKGEFgNeHsL0YZFcNzaGa8fEEB2k9y8ovaFMKbdQ29DM7vxzrNtXxAd7ijhRVoMITEoK47oxMVw/NpaUyABXl6mcSO8sVkqdZ4zhQFElH+QWsXZvIXsKrGszRsUEcf3YGK4bG8vY+H4yhrPqNRoESqlOnSyrYe3eItbuKWRHXhktBhJC/bhubAzXjYllSkoYnh56D8NAp0GglHJIaVU96/YVs3ZvIZsOnaGhqYUwfy+uGW2dU0iLDSY62Kd/jc6mHKJBoJTqtur6Jj4+WMLaPYWs219MZd2Fm/5D/b2IDfYlOtiXmCAfYkOs57HBvsQE+xAT7EtkoE//GqDHzbmqiwml1AAW4OPJvHFxzBsXR0NTC5+fOMuJshqKK+oorKijqKKe4oo6DhRWUFJZT0u775Q2gaggn/OBERvsy6TkUK4aGU14gANjE6s+oy0CpVSPNbcYSqvqzwdEYUUdxRV1FFXUUWgPjFPnaqmsa0LE6oJ79qhoZo+KZmx8sHaX0Qf00JBSyuVaWgw5p8rZcKCYDQdKyM4/hzEQGejDrFFRzB4VzcwRkTq0p5NoECil+p0zVfXW0J4HSth0sITyWmtoz8uSw5g9Kpqr06IZGROol7H2Eg0CpVS/1tTcwq6T56zWwv4S9p627m2ID/FlVpp1CGn6sAgCfPS05qXSIFBKDSiF5XVsPFDMhgPFbD50huqGZrw9bKRE+pMUHkByhD/JEf4MCfcnOdyfxDB/Ha/hIjQIlFIDVkNTC1l5ZXx8qIQjxdWcKKvmRFkNdY0Xhi6xCcSF+JEU3iYgIvxJDg8gKcJfzzugl48qpQYwb08b04dHMn145PnXjDGUVNZzvKyGE6U19p/VHC+r4aN9RZyp+vJotyF+XiRH+DNjeCQLJyQwKjaor/+Mfk1bBEqpQaeqvokTpTWcKKvhRFk1x0trOFxcRdbxszS3GNJig1g4MYH54+OJD/Vzdbl9Qg8NKaUUUFJZz/vZBby7q4BdJ88hAlNTwlk4MYF56XGE+A/eQ0gaBEop1U7emWpW7Cpgxa5THD1TjbeHjdlpUSyckMDstOhB15+SBoFSSnXCGOtGt3e/KOC97AJKKusJ8vXkhvRYFk5IIHNoxKDoM0mDQCmlHNDU3MLWo6W8+0UBH+wppKq+iZhgH+aPj2fBhIQBPU6DBoFSSnVTXWMzH+0r4t0vTrHxQAlNLYbEMD+uGhnFrAF4g5sGgVJK9cDZ6gZW5xayfn8xW46cocZ+g9uU1DBmjYxm1qgohkf37+4wNAiUUqqX1Dc1szPvLBsPlrDxQDEHi6oAa1S3K0dGMWtUFDOGRxLYz1oLGgRKKeUkp87VsskeCp8eLqWqvgkvD2FycjizRlmHkfpD53kaBEop1QcamlrYefwsGw8W8/GBEvYXVgIQF+LLVSOjmJgUyvDoIEbEBBLs27f3LGgQKKWUCxSW1/HxwWI2Hihh86EzVNZfGO4zNtiXETGBjIwJYkR0ICNinBsQGgRKKeVizS2GU2drOVhUyaHiKg4VVXKwuJLDxVVf6kCvNSBGRAcxMiaQETGBDI8O6nHHedrpnFJKuZiHTUiK8Ccpwp9rxsScf72lxZB/tpZDxZUcLKriUHElh4qqWL79BLWNzefniwn24Z+uGMr9Vwzt9do0CJRSyoVsbQJizugvB8SpcxdaEAeLKokK8nFKDRoESinVD9lswpBwa2yFtgHhlPdy5spFZK6IHBCRwyLyo07mmSUiu0Rkj4h87Mx6lFJKfZXTWgQi4gG8AFwL5AM7RGSlMWZvm3lCgaXAXGPMCRGJdlY9SimlOubMFsFU4LAx5qgxpgF4A1jQbp6vA28bY04AGGOKnViPUkqpDjgzCBKAk21+z7e/1tZIIExENorIThH5RkcrEpElIpIlIlklJSVOKlcppdyTM4Ogo/up29+04AlcBtwIXA/8RERGfmUhY14yxkw2xkyOiorq/UqVUsqNOfOqoXxgSJvfE4GCDuY5Y4ypBqpFZBMwHjjoxLqUUkq14cwWwQ5ghIikiog3sBhY2W6eFcAVIuIpIv5AJrDPiTUppZRqx2ktAmNMk4g8DHwAeACvGmP2iMgD9ukvGmP2icgaIBtoAV4xxuQ6qyallFJfNeD6GhKREuD4JS4eCZzpxXJ6W3+vD/p/jVpfz2h9PdOf60s2xnR4knXABUFPiEhWZ50u9Qf9vT7o/zVqfT2j9fVMf6+vM069s1gppVT/p0GglFJuzt2C4CVXF3AR/b0+6P81an09o/X1TH+vr0NudY5AKaXUV7lbi0AppVQ7GgRKKeXmBmUQXGwcBLE8a5+eLSKT+rC2ISKyQUT22cdgeLSDeWaJSLl9nIZdIvLTvqrP/v55IpJjf++vDBDt4u03qs122SUiFSLyvXbz9Pn2E5FXRaRYRHLbvBYuIh+KyCH7z7BOlr3ouB1Oqu83IrLf/m/4jr1b+I6W7fLz4MT6fi4ip9r8O87rZFlXbb//a1Nbnojs6mRZp2+/HjPGDKoH1l3MR4ChgDewGxjTbp55wGqsjvGmAdv6sL44YJL9eRBWv0rt65sF/MOF2zAPiOxiusu2Xwf/1oVYN8q4dPsBVwKTgNw2r/0a+JH9+Y+AX3XyN3T5eXVifdcBnvbnv+qoPkc+D06s7+fADxz4DLhk+7Wb/gzwU1dtv54+BmOLwJFxEBYArxvLZ0CoiMT1RXHGmNPGmM/tzyux+lZq3z13f+ey7dfOHOCIMeZS7zTvNcaYTUBZu5cXAK/Zn78GLOxgUUc+r06pzxiz1hjTZP/1M6yOIV2ik+3nCJdtv1YiIsDtwPLeft++MhiDwJFxEByZx+lEJAWYCGzrYPLlIrJbRFaLyNi+rQwDrLWPEbGkg+n9YvthdWTY2X8+V26/VjHGmNNgfQEAOhqBr79sy29htfI6crHPgzM9bD909Wonh9b6w/a7AigyxhzqZLort59DBmMQODIOgiPzOJWIBAJvAd8zxlS0m/w51uGO8cBzwLt9WRswwxgzCbgB+I6IXNluen/Yft7AfODvHUx29fbrjv6wLZ8AmoBlncxysc+Ds/wOGAZMAE5jHX5pz+XbD7iTrlsDrtp+DhuMQeDoOAgXm8dpRMQLKwSWGWPebj/dGFNhjKmyP18FeIlIZF/VZ4wpsP8sBt7Ban635dLtZ3cD8Lkxpqj9BFdvvzaKWg+Z2X92NBSrqz+L9wI3AXcZ+wHt9hz4PDiFMabIGNNsjGkBXu7kfV29/TyBW4H/62weV22/7hiMQeDIOAgrgW/Yr36ZBpS3NuGdzX488Q/APmPMf3cyT6x9PkRkKta/U2kf1RcgIkGtz7FOKLbvGtxl26+NTr+FuXL7tbMSuNf+/F6s8Tfac+Tz6hQiMhd4HJhvjKnpZB5HPg/Oqq/teadbOnlfl20/u2uA/caY/I4munL7dYurz1Y744F1VctBrKsJnrC/9gDwgP25AC/Yp+cAk/uwtplYTddsYJf9Ma9dfQ8De7CugPgMmN6H9Q21v+9uew39avvZ398fa8ce0uY1l24/rFA6DTRifUv9NhABrAMO2X+G2+eNB1Z19Xnto/oOYx1fb/0cvti+vs4+D31U35/tn69srJ17XH/afvbX/9T6uWszb59vv54+tIsJpZRyc4Px0JBSSqlu0CBQSik3p0GglFJuToNAKaXcnAaBUkq5OQ0CpfqQWD2j/sPVdSjVlgaBUkq5OQ0CpTogIneLyHZ7H/K/FxEPEakSkWdE5HMRWSciUfZ5J4jIZ2369Q+zvz5cRD6yd373uYgMs68+UETeFGssgGWtd0Er5SoaBEq1IyKjgTuwOgubADQDdwEBWP0bTQI+Bn5mX+R14HFjTAbWnbCtry8DXjBW53fTse5MBavH2e8BY7DuPJ3h5D9JqS55uroApfqhOcBlwA77l3U/rA7jWrjQudhfgLdFJAQINcZ8bH/9NeDv9v5lEowx7wAYY+oA7Ovbbux909hHtUoBNjv9r1KqExoESn2VAK8ZY/7tSy+K/KTdfF31z9LV4Z76Ns+b0f+HysX00JBSX7UOWCQi0XB+7OFkrP8vi+zzfB3YbIwpB86KyBX21+8BPjbWGBP5IrLQvg4fEfHvyz9CKUfpNxGl2jHG7BWRf8caVcqG1ePkd4BqYKyI7ATKsc4jgNXF9Iv2Hf1R4D776/cAvxeRX9jX8bU+/DOUcpj2PqqUg0SkyhgT6Oo6lOptemhIKaXcnLYIlFLKzWmLQCml3JwGgVJKuTkNAqWUcnMaBEop5eY0CJRSys39f41PtrzHr3caAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxZ0lEQVR4nO3dd3wc9Z3/8ddHzeq9uKi7F2xjyx2MgWBMMRAgYHp3gCNcuMsFklwSLsn9QhLI5UIIDgQCoYbQyVEMwRXbWJJxr7ItWbJsdav3/f7+mJUty5JRm9317uf5eOih3Z3Z2Y/G63nPfGfm+xVjDEoppXyXn7sLUEop5V4aBEop5eM0CJRSysdpECillI/TIFBKKR+nQaCUUj7O1iAQkUUiskdE8kTkkW6mR4nIByKyRUR2iMgddtajlFLqVGLXfQQi4g/sBS4CioBs4AZjzM5O8/wQiDLGPCwiCcAeYKgxpsWWopRSSp3CziOCmUCeMeaAc8P+OnBll3kMECEiAoQDlUCbjTUppZTqIsDGZY8ACjs9LwJmdZnnD8D7QDEQAVxvjHGcbqHx8fEmPT19EMtUSinvl5ubW26MSehump1BIN281rUd6mJgM3ABMBL4VETWGGNqTlqQyFJgKUBqaio5OTmDX61SSnkxESnoaZqdTUNFQEqn58lYe/6d3QG8bSx5wEFgXNcFGWOeMcZkGWOyEhK6DTSllFL9ZGcQZAOjRSRDRIKAJVjNQJ0dAi4EEJEkYCxwwMaalFJKdWFb05Axpk1EHgA+AfyB540xO0TkXuf0ZcDPgRdEZBtWU9LDxphyu2pSSil1KjvPEWCM+RD4sMtryzo9LgYWDvRzWltbKSoqoqmpaaCL8njBwcEkJycTGBjo7lKUUl7C1iBwlaKiIiIiIkhPT8e6EtU7GWOoqKigqKiIjIwMd5ejlPISXtHFRFNTE3FxcV4dAgAiQlxcnE8c+SilXMcrggDw+hDo4Ct/p1LKdbyiaUgppbxRQ0sb+eUN5FfUc7C8ninJ0ZwzOn7QP0eDYBAcO3aMV199lfvvv79P77v00kt59dVXiY6OtqcwpZTHa2ptp7CygYPl1sa+Y6N/sLyekprmk+a9b8FIDQJPdezYMf74xz+eEgTt7e34+/v3+L4PP/ywx2lKKe9y+Fgje4/WcqC8nnznBv9AWT3F1Y107vszNiyIjPgwzhmVQEZ8KOnxYWTEh5EeF0bYEHs22RoEg+CRRx5h//79TJ06lcDAQMLDwxk2bBibN29m586dXHXVVRQWFtLU1MS//uu/snTpUgDS09PJycmhrq6OSy65hHPOOYd169YxYsQI3nvvPUJCQtz8lyml+qu13UF2fiUrdpfy+e5S9pfVH58WERxAZnwYWekxpMclk9GxsY8PIyrE9ZeGe10Q/NcHO9hZXPP1M/bBhOGR/HTxxB6nP/bYY2zfvp3NmzezcuVKLrvsMrZv3378Es/nn3+e2NhYGhsbmTFjBtdccw1xcXEnLWPfvn289tprPPvss1x33XW89dZb3HzzzYP6dyil7FVW28zKPaWs2FPKmr3l1Da3EeTvx6zMWG6clcbUlCjS48KIDQvyqAs/vC4IPMHMmTNPus7/97//Pe+88w4AhYWF7Nu375QgyMjIYOrUqQBMnz6d/Px8V5WrlOonh8Ow7XA1n++2Nv5bi6oBSIocwmWTh3H+uETOGRVvW5POYPHs6vrhdHvurhIWFnb88cqVK/nss89Yv349oaGhLFiwoNv7AIYMGXL8sb+/P42NjS6pVSnVNzVNrazZW87nu0tZtbeU8roWRODslGi+t3AM549LZMKwSI/a4/86XhcE7hAREUFtbW2306qrq4mJiSE0NJTdu3ezYcMGF1enlBqIljYH24ur2XiwkpV7SsnJr6LNYYgKCeS8MQlcMC6R+WMSiA0Lcnep/aZBMAji4uKYN28ekyZNIiQkhKSkpOPTFi1axLJly5g8eTJjx45l9uzZbqxUKfV1apta2XToGNkHK8nOr2Rz4TGa26zxssYNjeCe+ZlcMC6Rs1OiCfD3jntybRuz2C5ZWVmm68A0u3btYvz48W6qyPV87e9Vyk4lNU1k51c6N/xV7D5ag8OAv58waXgkWemxzEiPYXpaLAkRQ75+gR5KRHKNMVndTdMjAqWUzzDGsL+sjo0Hq8jJryS7oJLCSut8XGiQP9NSY3jwwtHMSI9lakq0x5/kHSy+8VcqpXySw2HYU1LL+v0VrD9QQU5+JVUNrQDEhweRlRbL7XMzmJEew/hhkQR6SVNPX2kQKKW8Rscef8eGf8OBSirrWwBIiwvlG+OTmJERy4z0WNLjQs+oK3vspEGglDpjGWM4VNnA+v0VrHNu/Mtqrf55hkcFc/7YROaMjGPOyDhGROud+j3RIFBKnVEOH2u09vj3V7B+fznF1dZ9OQkRQ5iTGcdc54Y/NVb3+HtLg0Ap5dEq61tYt7+cL/LKWbe/goKKBgBiQgOZMzKO+zKtDf/IhHDd8PeTBoEbhIeHU1dXR3FxMQ8++CBvvvnmKfMsWLCAxx9/nKysbq/2UsprNbW2k51fydp95azNK2eHs++wiCEBzMqM49Y56cwdGcfYpAj8/HTDPxg0CNxo+PDh3YaAUr6k3WHYUVzNmn3WXn9OQRUtbQ4C/YVpqTH8+0VjmDc6nskjorzmBi5Po0EwCB5++GHS0tKOj0fw6KOPIiKsXr2aqqoqWltb+cUvfsGVV1550vvy8/O5/PLL2b59O42Njdxxxx3s3LmT8ePHa19Dymt1nODt2PCv219BdaN1See4oRHcOjuNeaPjmZURS2iQbqJcwfvW8kePwNFtg7vMoWfBJY/1OHnJkiV897vfPR4Eb7zxBh9//DEPPfQQkZGRlJeXM3v2bK644ooe2zCffvppQkND2bp1K1u3bmXatGmD+zcoNYiMMbS2G1raHbS0dfppb6e5zUFzW9fXHdQ3t5FbUMXavHKKqqwdnWFRwSyckMQ5o+OZOzL+jL5z90zmfUHgBmeffTalpaUUFxdTVlZGTEwMw4YN46GHHmL16tX4+flx+PBhSkpKGDp0aLfLWL16NQ8++CAAkydPZvLkya78E5TqVlFVg3VZ5v4KsvMrqW5sPb5h70/vNBHBAczJjGPp/EzOGRVPRnyYnuD1AN4XBKfZc7fTtddey5tvvsnRo0dZsmQJr7zyCmVlZeTm5hIYGEh6enq33U93pv8hlLuV1jSx/kAF6/IqWHeg/Hj3C/HhQczKiCMhYghDAvwYEuBHUMePvx9BAf4EdXl9iH+neQL8CA7wJzkmRNv5PZD3BYGbLFmyhHvuuYfy8nJWrVrFG2+8QWJiIoGBgaxYsYKCgoLTvn/+/Pm88sornH/++Wzfvp2tW7e6qHLly6rqW9hw4MTNWHmldQBEBgcwOzOOu+ZlMHdUPKMT9dJMb6ZBMEgmTpxIbW0tI0aMYNiwYdx0000sXryYrKwspk6dyrhx4077/vvuu4877riDyZMnM3XqVGbOnOmiypUvqW1qJTu/0trj31/BrqM1GGN1uDYzI5brspKZkxnPhOGR+OulmT5Du6E+A/na36v6r91h2Fx4jNV7y1i9r4ytRdW0OwxBAX5MT41h7sg45o6KY3JytM92uOYrtBtqpXzI0eomVu8tY9XeMtbmlVPd2IqfwOTkaO5fMJI5I+OYlhpDcKC/u0tVHkKDQKkzXHNbO9kHq1i9r4xVe8rYU2INm5oYMYSFE5I4b2wC54yKJzr0zB1KUdnLa4LAGOMTJ7POtKY8NfiMMeRXNLBqTymr95Wzfn8Fja3tBPn7kZUeww+mjeO8sQmMTYrwif8TauC8IgiCg4OpqKggLi7Oq7/4xhgqKioIDg52dynKxeqb21i3v4JVe0tZvbecQ5VWx2vpcaFcl5XM/DEJzM6M85kRtXxS6W4YEgFRIwZ90V7xrUlOTqaoqIiysjJ3l2K74OBgkpOT3V2GslnHACsrdpexcm8p2QeraGl3EBrkz9yRcdxzbgbzxySQFhfm7lKV3Yo3w5rHYdcHMOMeuOzxQf8IrwiCwMBAMjIy3F2GUgPS0NLGurwKVuwpZeWeMg4fs27mGp0Yzu3z0lkwJoHp6TEMCdCTvD6hYL0VAHmfwZAomP99mH2fLR/lFUGg1JnI2uuvZ6Vzw7/xYGWnvf547j9/JOeNSSA5JtTdpQ5M2V7I/jNsfR3rpoVYCImF0DjrcWic83lsl+fO6QFu7n+oqRrK86DqIMSPhqGTwa4maGPgwApY/QQUrIXQeLjwJzDjbgiOsucz0SBQyqUaWtpYv//EXn9H52ujEsO5bW4aC8YmkuUNe/3tbbDnQ8h+Fg6uBv8gGL8YwhKhoQIaK6GhHMr3QkMltNT2vKyg8BNBETEMopKtdvJI5++oZOt1/8CB1XusAMr3QcU+5+8863d96cnzRo6AsZdYP+nnDk5QORyw9yNY/TgUb4KI4bDoMZh2GwTZvyOgQaCUzY5WN/HpzqMs31nClwesvf6QQH/mjYrn3vNGsmCsF+z1d6g9Cpv+Cjl/gdpiiEqx9mjPvhXCE3p+X1uLMxwqrGA4HhYV0FDl/F0B1YVwaJ21l34SgYih1kY6Ktn6iRxxIigikyEsARqrOm3o91l7+hX7oPIgOFpPLC4kFuLHwJiFEDfaOhKIToMjW6yA2/yqdZQTFA6jLoSxl8LohVZY9YWjHXa8A2uegNKdEJMOi/8Xptzg0iMhr7izWClPYowhr7SO5TtLWL7jKFuKrI1WRnwYF45LZMHYRGZkeMFefwdj4NB62Pgs7HofHG0w8gLrxOaYi8HPhr+zuQ5qDkN1kfVTcxiqD1tB0fG4rcuYHuIPpv3Ec79AiM20NvJxo5y/nRv9r9ugtzZZRzp7PoQ9H0HdURA/SJ3jPFq4FOJG9vz+tharqWzt/0DlAYgfC+f+O0y6Bvzt2T8/3Z3FGgRKDQKHw/BVYRXLd5SwfGcJB8vrAZiSEs3CCUlcPDGJUYkRA/uQhkprj7a9xfppa4H2ZmhrhvZW52PnaydNd87vaLWaUGIyIDYDolMHttfZXAdb/wbZz0HpDqsNe+rNkHUnxI8a2N86UMZY66umyAqFGudPaPyJDX902uBsdB0OOPKVFQh7PoKS7dbr8WNPhEJylhWIrY2w6SX44n+t2oZNgXO/B+MuBz97u/hwWxCIyCLgfwF/4M/GmMe6TP8P4Cbn0wBgPJBgjKnsaZkaBMpTNLe1s25/Bct3lPDpzhLK65oJ8BPmjIxj4cShXDQ+iaFRA7zno2Nve/1TsPv/gAH8f/ULsPbWO4if1WQSm+4Mh0wrIDqCYkgPwVW2x2oW2fya1bY/9Cxr7/+sayFIL2elqgD2fmwdLeSvtdZ5aDxkngcH11jnHFLnWAEw6kL7Tjx34ZYgEBF/YC9wEVAEZAM3GGN29jD/YuAhY8wFp1uuBoFyp5qmVlbsLmX5zhJW7i6lvqWdsCB/FoxLZOGEJBaMTSQqZAAnLTu0t8LO92D9H6D4K6vNevrtkDDWOvHqH2TtzR//HQj+Q7q81mk+P+eeb3251RRRddD6XXnwxOOGipNrCEs4EQqxmRAWDzvehfw11nInXAUz74HkGS7bmJ1xmqqtyz/3fAT7V1ihOf8/IH2ey0txV6dzM4E8Y8wBZxGvA1cC3QYBcAPwmo31KNUvBRX1fL67lM93l7LhQAWt7Yb48CCumDqchROGMmdk3OB14NZ4zDrZ+uWfrKaDuFFw2W+tk4eDcfVIeIL1kzrr1GlNNd0ExEHI/wK2vgGY3p/8VZbgKKvdf9I17q7ktOwMghFAYafnRUA33z4QkVBgEfBAD9OXAksBUlNTB7dKpbpoaXOw8WAlK/aUsmJ3KQec7f2Z8WHcOS+DhROTmJoSM7j99Vflw4Zl8NVL0FJnXZZ42RPWlSg2tx0fFxxptVkPm3LqtNYm6yqg6DR7Tv4qt7IzCLr7X9JTO9Ri4Iuezg0YY54BngGraWhwylPqhKPVTazcY+31f5FXTn1LO0EBfszOjOOWOWmcPzaR9Hgb2r8LN1rNP7s+sNrsJ10Lc+7vfmPsToHBVvOQ8kp2BkERkNLpeTJQ3MO8S9BmIeVC1oAtVXy+u5QVu8vYeaQGgOFRwVx59gguGJvI3FFxhAbZ8F+kvQ12f2CdAC7KhuBomPddq709cvjgf55SX8POIMgGRotIBnAYa2N/Y9eZRCQKOA+42cZalKKqvoXV+8r4fHcpq/aWcayhFX8/YXpaDA8vGsf542zuurmpxmr62bAMqg9Ze9iXPg5Tb9SrbZRb2RYExpg2EXkA+ATr8tHnjTE7RORe5/Rlzlm/CSw3xtTbVYvyTaU1TWzMryT7YCVfHqxkT0ktxkBcWBAXjEvkgnGJnDsqgajQQbjK53Sa62DD07DuSWiuhrR5cMljMGaRtrcrj6A3lCmvYIyhsLKRjfmVbDxYwcaDleRXWH32hwb5My01hpkZscwfk8DkEVH4uWJg9rYWyH0BVv8a6stg7GUw/3swYpr9n61UFzpmsfI6DodhX2mdc8Nv7fUfrWkCIDo0kKy0WG6alcaMjFgmDo907cDsjnbY9ias+G+rI7O0c2DJa5Ayw3U1KNUHGgTqjGCMYfvhGjYcqODLg5XkFFRyrMHqJCwpcggzM+KYmR7DzIw4RieGu2aP/9QiYe8n8M+fWV0uDJ0MN78FI11396hS/aFBoDxaXXMb73x1mJfXFxwflD09LpSFE5KYkR7LrIw4UmJD3D9EacF6+OxRKNwAsSPh2r9Yd9666h4ApQZAg0B5pD1Ha3l5QwFvbyqivqWdicMj+eXVZ3HhuEQSI/vRf095Hmx7A8KTYNhUSJoAgSEDL/TodusIYN8nVodul/8Ozr55YH3jK+ViGgTKY7S0Ofhkx1Fe2lDAxoOVBAX4sXjycG6encrUlOj+7fUf2QJrfmv129P5fkbxt/rtGTbFasIZNsXqByY4snfLrTwIK/4fbPu79Z5v/BfMXOqSQUSUGmwaBMrtDh9r5LUvD/F6diHldc2kxobyw0vH8a3pKcSEBfVvoQXrrME+8j6DIZFw7r/BrHutboCPbrUC4sgWqyOwLZ3uZYzNPNHNQkdAhMWfmF5bAqt/A7l/sfqzP+chmPcghMQMbCUo5UYaBMotHA7D2rxyXtpQwD93lWCAC8clcvPsNOaPTujfyV5jYN+nsPa3VtfNofFw4U9hxl0nj/cak2YNm9ihtsQZDpvhyFY4vMkaNapD5AgrEMITrc7X2lusIQTP+741KpZSZzgNAmW/hkrI+ycUbaQhMoOPq9P5w44gDlQ2ExcWxL3njeSGmamkxPazWcXRDjvfhTX/AyXbrB4yL/mN1Vbfm6aaiCSIuAhGX3TitcYqOLrNeeTgPILY9ylMuBIu+JH2u6O8igaBGnzGQMkO6wTq3uVQtBGMg1YJItS0cDVwiYRQlzKFmHHnEpAmEDqs759zfLi/30HlfmuM2auehrO+NfCTtSExkDHf+un8d7n76iSlbKBBoAZHSwMcXGVdR7/vU6svfaAudiKfRdzAC2Vj2RcwmtsmBXLjsGKSa7cRUrgB1j4BaxyAQNJESJkJKbOt3zHp3W94W+oh90Wry4baYusqoOtesn+4Pw0B5aU0CFT/VRXAvuXWxv/gamt83MAwHJkL2Dry2/w2P5XVxYHEhwdx+0XpvDA7jejQLid/m2uhKAcKv7R+tv4dcp63poUnnQiG1NnWGLu5L1j99jRWWn32X/kHa6B03Ugr1W8aBKr32lutjfXeT6wAKNttvR6bCVl30phxIW+UpfLsusMUVTWSGR/GL6/O5Jtnj+h5BK8hETDyfOsHrPb+0l3WjVmFG+HQBquv/s7GLIJz/q37UbaUUn2mnc6pr2cMbH4Flv+ndRLVLxDS5sKYi2H0xZQNSeHFdfm8tKGA6sZWstJiWDo/k2+MTxqcrh5qj1oBVLYHxl4KQycNfJlK+RjtdE71X0MlfPCgtVeeNs+6Fj9zAQRHsr+sjj+vPsBbmz6ntd3BwglJLJ0/kulpg3xNfcRQ62odpZQtNAhUz/L+Ce/eDw0V1p2zc78Dfv7kFlTyp1U5fLqrhEB/P66Zlsw952aQmRDu7oqVUv2gQaBO1doIn/0XfPk0xI+Fm97ADJ3MpztL+NPqA+QWVBEdGsgD54/i1jnpJEQMcXfFSqkB0CBQJzu6Dd66B8p2wcxvw0X/RX61gx899yVf5FWQHBPCo4sncN2MFHvG81VKuZz+T1YWhwM2PGX1pBkSAze9RWvmBTy75gD/+9k+gvz9+PlVk7hhRgoBrhzkRSllOw0CBdVF8M69kL/Guilr8e/5qsKPHzy5lt1Ha7lk0lAevWIiSf3p/lkp5fE0CHzd9rfgHw9Bextc8SR1E27g8eV7eXF9PkkRwTxzy3QWTtSO1ZTyZhoEvqqpGj78vtVXz4gsuPoZPisJ58f/s5qjNU3cOjuN7108lohgHWBFKW+nQeCLCtbB29+GmsOw4AeUTn2AR/9vDx9u283YpAieumka01K1f32lfIUGgS9pa4FVj8Ha/4HoNBx3fMxrR5J47Hdf0Nzm4D8uHsvS+ZkE6slgpXyKBoGvKFgHH//AGnzl7FvYP/1HPPKPg2Tnb2fuyDj++5tnkREf5u4qlVJuoEHg7fK/gJW/tK4ICkuk9dq/8ocj4/nj018RNiSAx781hWumjejfeMBKKa+gQeCt8tfCysesAAhPgot/SU78lTz8/j72l+3jqqnD+c/LJxAfrncFK+XrNAi8TdcAWPQYZtptLFt3hF89t5nkmBBevHMm541JcHelSikPoUHgLQ6usQKgYC2ED4VFv4Lpt9HuH8yj7+/gpQ0FLJ4ynF9dc5Z2DaGUOoluEc50PQQAgSE0tbbz4Mu5LN9ZwrfnZ/LwonGDMz6AUsqraBCciYyxmn5WPgYFX1gBcMmvYdqtEBgCQFV9C3e9mM1Xhcf46eIJ3DEvw81FK6U8lQbBmcQYa2zglY/BoXUQMQwu+Y0zAE70A1RY2cBtz2+k6Fgjf7xxGpecNcyNRSulPJ0GgSdqbYTaI1BzxPm72Pp9ONcasrGHAADYVlTNHS9k09ru4JW7ZzEjPdZNf4RS6kyhQeBKxlijfXVs2E/5fQRqi61xgbsKCofoVLj0cTj7llMCAGDlnlLuf2UTMaFBvL50FqMSI1zwRymlznQaBK6yfwW8cRs0V3eZIBCeaO3lx6RB6myIHAYRw63fkSOsacGRp13833MKeeTtbYxJiuCFO2Zol9FKqV7TIHCF2hJ4+x6ISILzf3jyhj48Cfz738OnMYYnP8/jt5/u5dzR8fzxpmnaY6hSqk80COzmcMA7S6G5Dm77ABLHD9qi29od/Pi97by2sZCrp43gsasnExSgHcYppfpGg8BuX/wODqyExb8f1BBoaGnjgVe/4vPdpfzL+SP53sKx2l+QUqpfNAjsVLgRPv8FTLzausJnkJTXNXPXC9lsO1zNz6+axC2z0wZt2Uop32NrO4KILBKRPSKSJyKP9DDPAhHZLCI7RGSVnfW4VGMVvHknRCXD4t/BIO2t55fXc83T69hTUsuym6drCCilBsy2IwIR8QeeAi4CioBsEXnfGLOz0zzRwB+BRcaYQyKSaFc9LmUMvP8d65LQO5dDcNSgLHZz4THueiEbhzG8es9sHUVMKTUo7DwimAnkGWMOGGNagNeBK7vMcyPwtjHmEIAxptTGelwn5znY9QFc+FNInj4oi1yXV84Nz2wgdIg/b903V0NAKTVo7AyCEUBhp+dFztc6GwPEiMhKEckVkcFrSHeXo9vh4x/CqG/AnAcGZZG5BZXc/dccUmJDePu+eWQmhA/KcpVSCuw9Wdxdo7jp5vOnAxcCIcB6EdlgjNl70oJElgJLAVJTU20odZC01MObd0BINFy1DPwGnrPbiqq5/flskiKDefnuWSRE6EAySqnBZecRQRGQ0ul5MlDczTwfG2PqjTHlwGpgStcFGWOeMcZkGWOyEhI8eECVj74P5fvg6mcgfOB17jlayy3Pf0lkSCCv3D2LxAi9W1gpNfjsDIJsYLSIZIhIELAEeL/LPO8B54pIgIiEArOAXTbWZJ+tf4evXob534PMBQNe3MHyem7685cE+fvx6j2zGB4dMvAalVKqG7Y1DRlj2kTkAeATwB943hizQ0TudU5fZozZJSIfA1sBB/BnY8x2u2qyTcV++MdDkDIbzuv2Ktk+Kapq4KZnN+AwhtfumU1aXNggFKmUUt0TY7o223u2rKwsk5OT4+4yTmhrgecugqp8uHctRKd87VtOp6SmiW8tW8+xhhZeWzqbicMH59JTpZRvE5FcY0xWd9P0zuKB+uxROLIZrn9lwCFQUdfMTX/+koq6Zl6+e5aGgFLKJTQIBmLPx7DhKZi5FMZfPqBFVTe0cvNzGymqauCFO2Zytt4noJRyka89WSwiYSLi1+m5n/PErm+rKYZ374Oks+Cinw9oUXXNbdz2l43sL63jT7dkMTszbpCKVEqpr9ebq4b+CXTe8IcCn9lTzhnC0Q5v3QNtzfCtv3Q7WlhvNba0c6ezA7knbzyb88Z48OWxSimv1JumoWBjTF3HE2NMnc8fEax+HArWwlVPQ/zofi+mua2db7+cS3Z+Jb+7fioXTxw6iEUqpVTv9OaIoF5EpnU8EZHpQKN9JXm4/C9g1WMw+XqYckO/F9Pa7uA7r37F6r1lPHb1WVw5tWvvG0op5Rq9OSL4LvB3Eem4K3gYcL1tFXmyhkp4626ISYfLnuh319LtDsO/v7GF5TtLeHTxBK6f4cHdZiilvN7XBoExJltExgFjsfoP2m2MabW9Mk9jDLx7P9SXwd2fwZCIfi3G4TD88O1tvL+lmO8vGsvt8zIGuVCllOqb3lw19C9AmDFmuzFmGxAuIvfbX5qH2fku7P0ILvoZDJ/ar0UYY/jZP3byt5xCvnPBKO5fMGpQS1RKqf7ozTmCe4wxxzqeGGOqgHtsq8hTbXwWotNg1r39ersxhl9/socX1uVz1zkZ/NtFYwa5QKWU6p/eBIGfdBoV3TnyWJB9JXmg0l1Q8AVk3dHvrqXf21zM0yv3c8PMVP7zsvE60LxSymP05mTxJ8AbIrIMazyBe4GPbK3K02Q/B/5BcPYt/Xp7dUMrv/i/nUxNieYXV03SEFBKeZTeBMHDWIPC3Id1svgrrCuHfENzHWx5HSZcBWHx/VrEb5bvprK+hRfumIm/n4aAUsqzfG07hzHGAWwADgBZWKOJnZljBvTHtr9DSy3MuLtfb99ceIxXvjzEbXPTmTRCO5FTSnmeHo8IRGQM1mAyNwAVwN8AjDHnu6Y0D2CM1SyUNAlSZvb57W3tDn70zjYSI4boyWGllMc63RHBbqy9/8XGmHOMMU8C7a4py0MUZUPJNphxV79uHntpQwE7imv48eUTiAgOtKFApZQauNMFwTXAUWCFiDwrIhfS/YD03iv7zxAUAWdd1+e3ltQ08cTyvZw7Op7LzvKdUypKqTNPj0FgjHnHGHM9MA5YCTwEJInI0yKy0EX1uU99Bex4B6ZcD0PC+/z2n/9jJy3tDn5+pV4lpJTybL05WVxvjHnFGHM5kAxsBgY+MK+n2/wytLdA1l19fuuafWX8Y+sR7l8wkvR4HW9YKeXZ+nR3lDGm0hjzJ2PMBXYV5BEcDsh5HlLnQtKEPr21qbWdH7+7nfS4UO49b6RNBSql1ODp322y3m7/59Zg9DP6fjSwbNV+8isa+PlVkwgO9B/82pRSapBpEHQn+88QlgDjr+jT2/LL6/njyv1cPnkY547WkcaUUmcGDYKujh2CfZ/AtFshoPddKhlj+PF72xni78ePL+9bc5JSSrmTBkFXuS9YN5JNv71Pb/u/bUdYs6+cf184hqTI/o9hrJRSrqZB0FlbC2z6K4y5GKJ7P2pYbVMrP/tgJ5NGRHLLnHT76lNKKRv0ptM537H7A2sEsj72K/TE8r2U1TXz7K1Z2qmcUuqMo0cEnWU/Zw0+M/LCXr9l++Fq/ro+n5tmpTIlJdq+2pRSyiYaBB2ODz5zZ68Hn2l3GH707nZiw4L4j4vH2VygUkrZQ4Ogw/HBZ27u9Vte23iILYXH+NFl44kK0U7llFJnJg0CODH4zMRv9nrwmbLaZn798W7mZMZx1dQRNheolFL20SCAE4PP9KFfoV9+uIvG1nZ+rkNPKqXOcBoE/Rh8Zv3+Ct7+6jBL52cyKrHvPZMqpZQn0SDo4+AzLW0OfvzedlJiQ3jg/NEuKFAppeyl9xH0cfCZZ9ccIK+0jr/cPoOQIO1UTil15vPtI4Ljg88s6dXgM4WVDTz5+T4WTRzK+eMSXVCgUkrZz7eDoGPwmV50N22M4afv78BPhJ8s1k7llFLew3eDoGPwmbR5kDj+a2f/8mAln+8u5aFvjGF4dIgLClRKKdfw3SDoGHwm685ezf5FXjn+fsKNs3rfGZ1SSp0JfDcI+jj4TE5+FROGRRI2RM+vK6W8i61BICKLRGSPiOSJyCkD3ovIAhGpFpHNzp+f2FnPcX0cfKat3cHmwmNMT4txQXFKKeVatu3eiog/8BRwEVAEZIvI+8aYnV1mXWOMudyuOrrVx8Fndh+tpbG1XYNAKeWV7DwimAnkGWMOGGNagNeBK238vN45PvjMol4PPpOTXwmgQaCU8kp2BsEIoLDT8yLna13NEZEtIvKRiEzsbkEislREckQkp6ysbGBVHR98pvf9CuUeOsawqGC9Wkgp5ZXsDILu+mswXZ5vAtKMMVOAJ4F3u1uQMeYZY0yWMSYrISFhYFX1Y/CZ3PxKPRpQSnktO4OgCEjp9DwZKO48gzGmxhhT53z8IRAoIr3rB7o/+jH4TPGxRoqrmzQIlFJey84gyAZGi0iGiAQBS4D3O88gIkPF2YeziMx01lNhX0V9H3wmt6AKgKy0WLuqUkopt7LtqiFjTJuIPAB8AvgDzxtjdojIvc7py4BrgftEpA1oBJYYY7o2Hw2Ofgw+A1YQhAT6M25YhC1lKaWUu9l6d5SzuefDLq8t6/T4D8Af7KzhuB1v93nwGYBNh6qYkhJFoL/v3nunlPJuvnOb7OQl1p3EvRx8BqChpY0dxTXcd95IGwtTSin38p0gCAiCsZf06S1bCqtpdxg9UayU8mra3nEauQXWjWTTUjUIlFLeS4PgNHILqhidGE5UaKC7S1FKKdtoEPTA4TDkFlRps5BSyutpEPRgf1kdNU1tGgRKKa+nQdCDjhvJNAiUUt5Og6AHOQVVxIYFkREf5u5SlFLKVhoEPdhUUMW01BicPWAopZTX0iDoRkVdMwfK67VZSCnlEzQIurHp0DEAstI1CJRS3k+DoBu5BVUE+gtnjYhydylKKWU7DYJu5BZUMmlEFMGB/u4uRSmlbKdB0EVLm4MtRdVM124llFI+QoOgi+3F1bS0OfT8gFLKZ2gQdLHJeSOZdjSnlPIVGgRd5ORXkRIbQmJksLtLUUopl9Ag6MQYQ+6hKh2fWCnlUzQIOimqaqSstplpeiOZUsqHaBB0kuMciCZLg0Ap5UM0CDrJLagifEgAY5Ii3F2KUkq5jAZBJzn5VZydGo2/n3Y0p5TyHRoETrVNrewpqdWO5pRSPkeDwOmrQ8cwRgeiUUr5Hg0Cp9yCKvwEpqZEu7sUpZRyKQ0Cp02Hqhg7NJKI4EB3l6KUUi6lQQC0OwxfHTrG9LRod5eilFIup0EA7DlaS11zm95RrJTySRoEWOMPgJ4oVkr5Jg0CrBPFiRFDSI4JcXcpSinlchoEQE5BFdPTYhDRG8mUUr7H54OgpKaJoqpGbRZSSvksnw+CjoFoNAiUUr7K54Mgp6CKIQF+TBwe5e5SlFLKLXw+CHILqpiSHE1QgM+vCqWUj/LprV9Tazs7iqt1IBqllE/z6SDYWlRNa7vRgWiUUj7Np4OgY0QyPSJQSvkyW4NARBaJyB4RyRORR04z3wwRaReRa+2sp6tNBVVkxocRGxbkyo9VSimPYlsQiIg/8BRwCTABuEFEJvQw36+AT+yqpTvGGHKdN5IppZQvs/OIYCaQZ4w5YIxpAV4Hruxmvu8AbwGlNtZyigPl9VQ1tGoQKKV8np1BMAIo7PS8yPnacSIyAvgmsMzGOrqV67yRLCtdg0Ap5dvsDILuOu4xXZ7/DnjYGNN+2gWJLBWRHBHJKSsrG5TicvOriAoJJDM+fFCWp5RSZ6oAG5ddBKR0ep4MFHeZJwt43dnZWzxwqYi0GWPe7TyTMeYZ4BmArKysrmHSL7mHqpiWGo2fn3Y0p5TybXYeEWQDo0UkQ0SCgCXA+51nMMZkGGPSjTHpwJvA/V1DwA7HGlrIK60jK10HolFKKduOCIwxbSLyANbVQP7A88aYHSJyr3O6y88LdNh0yDo/MC1Vzw8opZSdTUMYYz4EPuzyWrcBYIy53c5aOsstqMLfT5iaEu2qj1RKKY/lk3cW5+RXMXF4JCFB/u4uRSml3M7ngqC13cGWomN6/4BSSjn5XBDsLK6hqdWhQaCUUk4+FwS5OiKZUkqdxPeC4FAVI6JDGBYV4u5SlFLKI/hUEBhjyM2v0m6nlVKqE58KguLqJo7WNOlANEop1YlPBUFOvjUQjZ4fUEqpE3wqCDYVVBEa5M+4oRHuLkUppTyGTwVBTkEVU1OiCfD3qT9bKaVOy2e2iPXNbew6UqPNQkop1YXPBMGWwmM4jJ4fUEqprnwmCIIC/LhgXCJna4+jSil1Elt7H/UkWemxPH+7jj+glFJd+cwRgVJKqe5pECillI/TIFBKKR+nQaCUUj5Og0AppXycBoFSSvk4DQKllPJxGgRKKeXjxBjj7hr6RETKgIJ+vj0eKB/Ecgabp9cHnl+j1jcwWt/AeHJ9acaYhO4mnHFBMBAikmOMyXJ3HT3x9PrA82vU+gZG6xsYT6+vJ9o0pJRSPk6DQCmlfJyvBcEz7i7ga3h6feD5NWp9A6P1DYyn19ctnzpHoJRS6lS+dkSglFKqC68MAhFZJCJ7RCRPRB7pZrqIyO+d07eKyDQX1pYiIitEZJeI7BCRf+1mngUiUi0im50/P3FVfc7PzxeRbc7PzulmujvX39hO62WziNSIyHe7zOPy9Sciz4tIqYhs7/RarIh8KiL7nL+7HRXp676vNtb3GxHZ7fw3fEdEont472m/DzbW96iIHO7073hpD+911/r7W6fa8kVkcw/vtX39DZgxxqt+AH9gP5AJBAFbgAld5rkU+AgQYDbwpQvrGwZMcz6OAPZ2U98C4B9uXIf5QPxpprtt/XXzb30U6/pot64/YD4wDdje6bVfA484Hz8C/KqHv+G031cb61sIBDgf/6q7+nrzfbCxvkeB7/XiO+CW9ddl+hPAT9y1/gb6441HBDOBPGPMAWNMC/A6cGWXea4E/mosG4BoERnmiuKMMUeMMZucj2uBXcAIV3z2IHLb+uviQmC/Maa/NxgOGmPMaqCyy8tXAi86H78IXNXNW3vzfbWlPmPMcmNMm/PpBiB5sD+3t3pYf73htvXXQUQEuA54bbA/11W8MQhGAIWdnhdx6oa2N/PYTkTSgbOBL7uZPEdEtojIRyIy0bWVYYDlIpIrIku7me4R6w9YQs//+dy5/jokGWOOgLUDACR2M4+nrMs7sY7yuvN13wc7PeBsunq+h6Y1T1h/5wIlxph9PUx35/rrFW8MAunmta6XRvVmHluJSDjwFvBdY0xNl8mbsJo7pgBPAu+6sjZgnjFmGnAJ8C8iMr/LdE9Yf0HAFcDfu5ns7vXXF56wLn8EtAGv9DDL130f7PI0MBKYChzBan7pyu3rD7iB0x8NuGv99Zo3BkERkNLpeTJQ3I95bCMigVgh8Iox5u2u040xNcaYOufjD4FAEYl3VX3GmGLn71LgHazD787cuv6cLgE2GWNKuk5w9/rrpKSjycz5u7Sbedz9XbwNuBy4yTgbtLvqxffBFsaYEmNMuzHGATzbw+e6e/0FAFcDf+tpHnetv77wxiDIBkaLSIZzr3EJ8H6Xed4HbnVe/TIbqO44hLebsz3xOWCXMea3Pcwz1DkfIjIT69+pwkX1hYlIRMdjrBOK27vM5rb110mPe2HuXH9dvA/c5nx8G/BeN/P05vtqCxFZBDwMXGGMaehhnt58H+yqr/N5p2/28LluW39O3wB2G2OKupvozvXXJ+4+W23HD9ZVLXuxrib4kfO1e4F7nY8FeMo5fRuQ5cLazsE6dN0KbHb+XNqlvgeAHVhXQGwA5rqwvkzn525x1uBR68/5+aFYG/aoTq+5df1hhdIRoBVrL/UuIA74J7DP+TvWOe9w4MPTfV9dVF8eVvt6x/dwWdf6evo+uKi+l5zfr61YG/dhnrT+nK+/0PG96zSvy9ffQH/0zmKllPJx3tg0pJRSqg80CJRSysdpECillI/TIFBKKR+nQaCUUj5Og0ApFxKrZ9R/uLsOpTrTIFBKKR+nQaBUN0TkZhHZ6OxD/k8i4i8idSLyhIhsEpF/ikiCc96pIrKhU7/+Mc7XR4nIZ87O7zaJyEjn4sNF5E2xxgJ4peMuaKXcRYNAqS5EZDxwPVZnYVOBduAmIAyrf6NpwCrgp863/BV42BgzGetO2I7XXwGeMlbnd3Ox7kwFq8fZ7wITsO48nWfzn6TUaQW4uwClPNCFwHQg27mzHoLVYZyDE52LvQy8LSJRQLQxZpXz9ReBvzv7lxlhjHkHwBjTBOBc3kbj7JvGOapVOrDW9r9KqR5oECh1KgFeNMb84KQXRX7cZb7T9c9yuuae5k6P29H/h8rNtGlIqVP9E7hWRBLh+NjDaVj/X651znMjsNYYUw1Uici5ztdvAVYZa4yJIhG5yrmMISIS6so/Qqne0j0RpbowxuwUkf/EGlXKD6vHyX8B6oGJIpILVGOdRwCri+llzg39AeAO5+u3AH8SkZ85l/EtF/4ZSvWa9j6qVC+JSJ0xJtzddSg12LRpSCmlfJweESillI/TIwKllPJxGgRKKeXjNAiUUsrHaRAopZSP0yBQSikfp0GglFI+7v8D9+VUMaO8ERcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", @@ -138,6 +212,17 @@ "plt.savefig('fig-res-alexnet-train-validate-acc.pdf')\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# save raw data\n", + "import numpy\n", + "numpy.save('fig-res-alexnet_data.npy', res)" + ] } ], "metadata": { @@ -156,7 +241,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.8.12" } }, "nbformat": 4, diff --git a/7_deep_learning/1_CNN/04-vgg.ipynb b/7_deep_learning/1_CNN/04-vgg.ipynb index 96ee1da..4bb2cf6 100644 --- a/7_deep_learning/1_CNN/04-vgg.ipynb +++ b/7_deep_learning/1_CNN/04-vgg.ipynb @@ -59,27 +59,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "VGG 的一个关键就是使用很多层 3 x 3 的卷积然后再使用一个最大池化层，这个模块被使用了很多次，下面照着这个结构把网络用PyTorch实现出来：" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "ExecuteTime": { - "end_time": "2017-12-22T09:01:51.296457Z", - "start_time": "2017-12-22T09:01:50.883050Z" - }, - "collapsed": true - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import torch\n", - "from torch import nn\n", - "from torch.autograd import Variable\n", - "from torchvision.datasets import CIFAR10\n", - "from torchvision import transforms as tfs" + "VGG 的一个关键就是使用很多层 3 x 3 的卷积然后再使用一个最大池化层，这个模块被使用了很多次，下面照着这个结构把网络用PyTorch实现出来。" ] }, { @@ -96,22 +76,32 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T09:01:51.312500Z", "start_time": "2017-12-22T09:01:51.298777Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ + "import numpy as np\n", + "import torch\n", + "from torch import nn\n", + "from torch.autograd import Variable\n", + "from torchvision.datasets import CIFAR10\n", + "from torchvision import transforms as tfs\n", + "\n", "def VGG_Block(num_convs, in_channels, out_channels):\n", - " net = [nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1), nn.ReLU(True)] # 定义第一层\n", + " # 定义第一层\n", + " net = [nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1), \n", + " nn.BatchNorm2d(out_channels),\n", + " nn.ReLU(True)] \n", "\n", " for i in range(num_convs-1): # 定义后面的很多层\n", " net.append(nn.Conv2d(out_channels, out_channels, \n", " kernel_size=3, padding=1))\n", + " net.append(nn.BatchNorm2d(out_channels))\n", " net.append(nn.ReLU(True))\n", " \n", " net.append(nn.MaxPool2d(2, 2)) # 定义池化层\n", @@ -127,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T08:20:40.819497Z", @@ -141,12 +131,15 @@ "text": [ "Sequential(\n", " (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (1): ReLU(inplace=True)\n", - " (2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (3): ReLU(inplace=True)\n", - " (4): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU(inplace=True)\n", + " (3): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (5): ReLU(inplace=True)\n", - " (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", + " (6): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (8): ReLU(inplace=True)\n", + " (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", ")\n" ] } @@ -158,7 +151,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T07:52:04.632406Z", @@ -192,13 +185,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T09:01:54.497712Z", "start_time": "2017-12-22T09:01:54.489255Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -215,12 +207,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "作为实例，我们定义一个稍微简单一点的 VGG 结构，其中有 8 个卷积层" + "作为示例，我们定义一个稍微简单一点的 VGG 结构，其中有 8 个卷积层" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T09:01:55.149378Z", @@ -235,51 +227,52 @@ "Sequential(\n", " (0): Sequential(\n", " (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (1): ReLU(inplace=True)\n", - " (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (3): ReLU(inplace=True)\n", - " (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU(inplace=True)\n", + " (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (5): ReLU(inplace=True)\n", + " (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", " )\n", " (1): Sequential(\n", " (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (1): ReLU(inplace=True)\n", - " (2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (3): ReLU(inplace=True)\n", - " (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", + " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU(inplace=True)\n", + " (3): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (5): ReLU(inplace=True)\n", + " (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", " )\n", " (2): Sequential(\n", " (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (1): ReLU(inplace=True)\n", - " (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (3): ReLU(inplace=True)\n", - " (4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU(inplace=True)\n", + " (3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (5): ReLU(inplace=True)\n", " (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", " )\n", " (3): Sequential(\n", " (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (1): ReLU(inplace=True)\n", - " (2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (3): ReLU(inplace=True)\n", - " (4): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU(inplace=True)\n", + " (3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (5): ReLU(inplace=True)\n", - " (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", - " )\n", - " (4): Sequential(\n", - " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (1): ReLU(inplace=True)\n", - " (2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (3): ReLU(inplace=True)\n", - " (4): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", - " (5): ReLU(inplace=True)\n", - " (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", + " (6): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (8): ReLU(inplace=True)\n", + " (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", " )\n", ")\n" ] } ], "source": [ - "vgg_net = VGG_Stack((2, 2, 3, 3, 3), ((3, 64), (64, 128), (128, 256), (256, 512), (512, 512)))\n", + "vgg_net_11 = VGG_Stack((2, 2, 3, 3, 3), \n", + " ((3, 64), (64, 128), (128, 256), (256, 512), (512, 512)))\n", + "vgg_net = VGG_Stack((2, 2, 2, 3), \n", + " ((3, 64), (64, 128), (128, 256), (256, 512)))\n", "print(vgg_net)" ] }, @@ -287,12 +280,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "可以看到网络结构中有个 5 个 最大池化，说明图片的大小会减少 5 倍。可以验证一下，输入一张 224 x 224 的图片看看结果是什么" + "可以看到网络结构中有个 5 个 最大池化，说明图片的大小会减少 5 倍。可以验证一下，输入一张 32 x 32 的图片看看结果是什么" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T08:52:44.049650Z", @@ -304,12 +297,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "torch.Size([1, 512, 7, 7])\n" + "torch.Size([1, 512, 2, 2])\n" ] } ], "source": [ - "test_x = Variable(torch.zeros(1, 3, 224, 224))\n", + "test_x = Variable(torch.zeros(1, 3, 32, 32))\n", "test_y = vgg_net(test_x)\n", "print(test_y.shape)" ] @@ -323,24 +316,28 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T09:01:57.323034Z", "start_time": "2017-12-22T09:01:57.306864Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "class VGG_Net(nn.Module):\n", " def __init__(self):\n", " super(VGG_Net, self).__init__()\n", - " self.feature = VGG_Stack((2, 2, 3, 3, 3), ((3, 64), (64, 128), (128, 256), (256, 512), (512, 512)))\n", + " self.feature = VGG_Stack((2, 2, 2, 3), \n", + " ((3, 64), (64, 128), (128, 256), (256, 512)))\n", " self.fc = nn.Sequential(\n", - " nn.Linear(512*7*7, 4096),\n", - " nn.ReLU(True),\n", - " nn.Linear(4096, 10)\n", + " nn.Linear(2*2*512, 1024),\n", + " nn.ReLU(True),\n", + " nn.Dropout(),\n", + " nn.Linear(1024, 1024),\n", + " nn.ReLU(True),\n", + " nn.Dropout(),\n", + " nn.Linear(1024, 10)\n", " )\n", " def forward(self, x):\n", " x = self.feature(x)\n", @@ -358,7 +355,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T09:01:59.921373Z", @@ -372,7 +369,7 @@ "# 使用数据增强\n", "def train_tf(x):\n", " im_aug = tfs.Compose([\n", - " tfs.Resize(224),\n", + " #tfs.Resize(224),\n", " tfs.ToTensor(),\n", " tfs.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n", " ])\n", @@ -381,7 +378,7 @@ "\n", "def test_tf(x):\n", " im_aug = tfs.Compose([\n", - " tfs.Resize(224),\n", + " #tfs.Resize(224),\n", " tfs.ToTensor(),\n", " tfs.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n", " ])\n", @@ -394,59 +391,118 @@ "test_data = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)\n", "\n", "net = VGG_Net()\n", - "optimizer = torch.optim.SGD(net.parameters(), lr=1e-1)\n", + "optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)\n", "criterion = nn.CrossEntropyLoss()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T09:12:46.868967Z", "start_time": "2017-12-22T09:01:59.924086Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0. Train Loss: 1.746658, Train Acc: 0.313979, Valid Loss: 1.441544, Valid Acc: 0.459454, Time 00:00:24\n", + "Epoch 1. Train Loss: 1.293454, Train Acc: 0.532868, Valid Loss: 1.064248, Valid Acc: 0.630934, Time 00:00:27\n", + "Epoch 2. Train Loss: 0.976450, Train Acc: 0.663643, Valid Loss: 0.865004, Valid Acc: 0.693730, Time 00:00:27\n", + "Epoch 3. Train Loss: 0.797400, Train Acc: 0.734835, Valid Loss: 0.752283, Valid Acc: 0.745847, Time 00:00:27\n", + "Epoch 4. Train Loss: 0.669900, Train Acc: 0.782389, Valid Loss: 0.654101, Valid Acc: 0.787085, Time 00:00:27\n", + "Epoch 5. Train Loss: 0.573636, Train Acc: 0.815837, Valid Loss: 0.637741, Valid Acc: 0.797765, Time 00:00:27\n", + "Epoch 6. Train Loss: 0.499975, Train Acc: 0.839174, Valid Loss: 0.571682, Valid Acc: 0.809237, Time 00:00:27\n", + "Epoch 7. Train Loss: 0.428868, Train Acc: 0.862372, Valid Loss: 0.505937, Valid Acc: 0.831982, Time 00:00:27\n", + "Epoch 8. Train Loss: 0.373129, Train Acc: 0.882133, Valid Loss: 0.559180, Valid Acc: 0.821203, Time 00:00:27\n", + "Epoch 9. Train Loss: 0.323578, Train Acc: 0.895920, Valid Loss: 0.582161, Valid Acc: 0.827927, Time 00:00:27\n", + "Epoch 10. Train Loss: 0.274420, Train Acc: 0.913163, Valid Loss: 0.623256, Valid Acc: 0.813390, Time 00:00:27\n", + "Epoch 11. Train Loss: 0.239558, Train Acc: 0.924692, Valid Loss: 0.561333, Valid Acc: 0.840882, Time 00:00:27\n", + "Epoch 12. Train Loss: 0.207196, Train Acc: 0.935042, Valid Loss: 0.537481, Valid Acc: 0.843256, Time 00:00:27\n", + "Epoch 13. Train Loss: 0.180536, Train Acc: 0.943454, Valid Loss: 0.697694, Valid Acc: 0.818631, Time 00:00:27\n", + "Epoch 14. Train Loss: 0.157166, Train Acc: 0.950607, Valid Loss: 0.542898, Valid Acc: 0.857298, Time 00:00:27\n", + "Epoch 15. Train Loss: 0.138539, Train Acc: 0.957201, Valid Loss: 0.602181, Valid Acc: 0.847112, Time 00:00:27\n", + "Epoch 16. Train Loss: 0.130947, Train Acc: 0.960418, Valid Loss: 0.607590, Valid Acc: 0.852453, Time 00:00:27\n", + "Epoch 17. Train Loss: 0.109348, Train Acc: 0.966972, Valid Loss: 0.636679, Valid Acc: 0.848497, Time 00:00:27\n", + "Epoch 18. Train Loss: 0.099000, Train Acc: 0.970009, Valid Loss: 0.640463, Valid Acc: 0.849684, Time 00:00:27\n", + "Epoch 19. Train Loss: 0.083908, Train Acc: 0.974604, Valid Loss: 0.630587, Valid Acc: 0.859771, Time 00:00:27\n" + ] + } + ], "source": [ - "(l_train_loss, l_train_acc, l_valid_loss, l_valid_acc) = train(net, \n", - " train_data, test_data, \n", - " 20, \n", - " optimizer, criterion,\n", - " use_cuda=False)" + "res = train(net, train_data, test_data, 20, optimizer, criterion)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2bElEQVR4nO3deXxU9bn48c+Tfd8TEhJIWBO2ACEiiCKLRUArLojghq1LsXq72qve219bve291u7aWkXrjlIFccUVEVRA9iXIFiBAEiCQkJCQPfn+/jgTGMJMmJDMTJbn/XrNa2bO+Z6ZJ4dhnvmuR4wxKKWUUs35eDsApZRSHZMmCKWUUg5pglBKKeWQJgillFIOaYJQSinlkJ+3A2hPcXFxJi0tzdthKKVUp7Fhw4bjxph4R/u6VIJIS0tj/fr13g5DKaU6DRE54GyfNjEppZRySBOEUkophzRBKKWUcqhL9UEopVRr1NXVkZ+fT3V1tbdDcbugoCBSUlLw9/d3+RhNEEqpbis/P5/w8HDS0tIQEW+H4zbGGIqLi8nPz6dPnz4uH+e2JiYReV5EikQkx8n+X4jIZtstR0QaRCTGti9PRLbZ9umwJKWUW1RXVxMbG9ulkwOAiBAbG9vqmpI7+yBeBKY622mM+YMxZoQxZgTwMLDCGFNiV2SibX+2G2NUSnVzXT05NLmQv9NtCcIYsxIoOW9ByxzgdXfF0pLqugbmr9zL17nHvfH2SinVYXl9FJOIhGDVNBbbbTbAJyKyQUTuOc/x94jIehFZf+zYsVa/f4CvD/NX7uPf6w61+lillGqL0tJSnnrqqVYfN336dEpLS9s/oGa8niCA7wJfN2teGmeMyQKmAfeJyHhnBxtj5htjso0x2fHxDmeLt8jHR5iYnsAXu4qob2hs9fFKKXWhnCWIhoaGFo9bunQpUVFRborqjI6QIGbTrHnJGFNouy8ClgCj3RnA5EEJnKyuZ8OBE+58G6WUOstDDz3E3r17GTFiBBdddBETJ07k5ptvZtiwYQBce+21jBo1iiFDhjB//vzTx6WlpXH8+HHy8vIYNGgQd999N0OGDGHKlClUVVW1W3xeHeYqIpHA5cCtdttCAR9jTLnt8RTgUXfGcemAePx9hc93FnFx31h3vpVSqoN65L3tfFt4sl1fc3DPCH793SFO9z/22GPk5OSwefNmvvjiC6666ipycnJOD0V9/vnniYmJoaqqiosuuogbbriB2Nizv6P27NnD66+/zrPPPsusWbNYvHgxt956q6O3azV3DnN9HVgNpItIvojcKSLzRGSeXbHrgE+MMafstvUAvhKRLcBa4ANjzEfuihMgLNCPMX1jWbazyJ1vo5RSLRo9evRZ8xSeeOIJhg8fzpgxYzh06BB79uw555g+ffowYsQIAEaNGkVeXl67xeO2GoQxZo4LZV7EGg5rv20fMNw9UTk3KSOBR977lgPFp0iNDfX02yulvKylX/qeEhp65rvniy++4LPPPmP16tWEhIQwYcIEh/MYAgMDTz/29fVt1yamjtAH0SFMykgAYNkOrUUopTwjPDyc8vJyh/vKysqIjo4mJCSEnTt3smbNGg9Hp0ttnJYaG0r/hDA+31nE9y91fSq6UkpdqNjYWMaNG8fQoUMJDg6mR48ep/dNnTqVp59+mszMTNLT0xkzZozH49MEYWdyRgLPf72f8uo6woNcX9BKKaUu1GuvveZwe2BgIB9++KHDfU39DHFxceTknFnN6IEHHmjX2LSJyc7kQT2oazB8tUdnVSullCYIO1m9o4gM9tfRTEophSaIs/j5+jAhPZ7lO4tobDTeDkcppbxKE0QzkzISKD5Vy5b8Um+HopRSXqUJopnLB8bj6yM63FUp1e1pgmgmKiSAUanR2g+hlOr2NEE4MDkjgR2HT1JY2n4zEpVSqj2EhYUBUFhYyMyZMx2WmTBhAuvXt/1inJogHJg8yJpV/bnWIpRSHVTPnj1ZtGiRW99DE4QD/eLD6B0ToglCKeV2Dz744FnXhPjNb37DI488wuTJk8nKymLYsGG888475xyXl5fH0KFDAaiqqmL27NlkZmZy0003tdt6TDqT2gERYfKgBF775iBVtQ0EB/h6OySllLt9+BAc2da+r5k4DKY91mKR2bNn85Of/IQf/vCHALzxxht89NFH/PSnPyUiIoLjx48zZswYrrnmGqfXlf7nP/9JSEgIW7duZevWrWRlZbVL+FqDcGJyRg9q6htZtVdnVSul3GfkyJEUFRVRWFjIli1biI6OJikpif/6r/8iMzOTK664goKCAo4ePer0NVauXHn6GhCZmZlkZma2S2xag3BidJ8YQgN8WbaziMmDepz/AKVU53aeX/ruNHPmTBYtWsSRI0eYPXs2CxYs4NixY2zYsAF/f3/S0tIcLvVtz1ntoi20BuFEgJ8P4wfG8/mOIozRWdVKKfeZPXs2CxcuZNGiRcycOZOysjISEhLw9/dn+fLlHDhwoMXjx48fz4IFCwDIyclh69at7RKXJogWTMpI4MjJara382UIlVLK3pAhQygvLyc5OZmkpCRuueUW1q9fT3Z2NgsWLCAjI6PF4++9914qKirIzMzk8ccfZ/To0e0SlzYxtWBCegIi1nDXocmR3g5HKdWFbdt2poM8Li6O1atXOyxXUVEBQFpa2umlvoODg1m4cGG7x6Q1iBbEhwcyPCVKZ1UrpbolTRDnMTkjgS2HSjlWXuPtUJRSyqPcliBE5HkRKRKRHCf7J4hImYhstt1+ZbdvqojsEpFcEXnIXTG6omkE0/JdWotQqivqLoNQLuTvdGcN4kVg6nnKfGmMGWG7PQogIr7AP4BpwGBgjogMdmOcLRqUFE5SZBCf6+quSnU5QUFBFBcXd/kkYYyhuLiYoKCgVh3ntk5qY8xKEUm7gENHA7nGmH0AIrIQmAF8247huUxEmJSRwNubCqipbyDQT2dVK9VVpKSkkJ+fz7Fjx7wditsFBQWRkpLSqmO8PYpprIhsAQqBB4wx24Fk4JBdmXzgYm8E12TyoAQWfHOQb/aVMH5gvDdDUUq1I39/f/r06ePtMDosb3ZSbwRSjTHDgSeBt23bHU0HdFr/E5F7RGS9iKx316+AS/rFEeTvo4v3KaW6Fa8lCGPMSWNMhe3xUsBfROKwagy97IqmYNUwnL3OfGNMtjEmOz7ePb/ug/x9GdcvjmU7j3b5tkqllGritQQhIoliWzxEREbbYikG1gEDRKSPiAQAs4F3vRVnk0mDEjhUUkVuUYW3Q1FKKY9wWx+EiLwOTADiRCQf+DXgD2CMeRqYCdwrIvVAFTDbWD/P60XkfuBjwBd43tY34VWTMqyLCC3bWcSAHuFejkYppdzPnaOY5pxn/9+BvzvZtxRY6o64LlRSZDBDekbw+Y4i5l3ez9vhKKWU2+lMaoAjOVCy/7zFJmcksP5ACaWVtR4ISimlvEsTRO0peO4KWO2wMnOWSYN60Ghgxe6uP2ZaKaU0QQSEQsZ0yHkLGupaLJqZHElcWACf6axqpVQ3oAkCYNgsqCqB3GUtFvPxESamJ7BiVxF1DY0eCk4ppbxDEwRA/8kQHAPb3jhv0cmDEjhZXc+GAyc8EJhSSnmPJggAX38Yej3sXAo15S0WvXRAPP6+orOqlVJdniaIJsNmQX0V7Hi/xWJhgX6M6RvLsh1HPRSYUkp5hyaIJr1GQ1QqbP33eYtOykhg77FT5B0/5YHAlFLKOzRBNBGBzFmwfwWUt1w7mJxhXURIm5mUUl2ZJgh7w2aBaYScxS0W6x0bwoCEME0QSqkuTROEvfiBkDTCtWamQQl8s7+Y8uqW504opVRnpQmiucxZcHgzHNvdYrHJGT2oazB8uee4Z+JSSikP0wTR3NAbQHzOOyciq3cUkcH+LNNZ1UqpLkoTRHPhidDnctj6BrRwcSA/Xx8mpMfzxa4iGhr1IkJKqa5HE4QjmTdB6QE4tLbFYpMyEig+VcuW/FLPxKWUUh6kCcKRQVeDX/B5m5kmDEzA10f4XJuZlFJdkCYIRwLDXVrhNTLEn+zUaJbpcFelVBekCcIZF1d4nTwogR2HT1JYWuWhwJRSyjM0QTjj4gqvk2yzqrUWoZTqajRBOOPiCq/94kPpnxDGovWHPBicUkq5nyaIlriwwquIcNuYVLbkl7HpoF4jQinVdbgtQYjI8yJSJCI5TvbfIiJbbbdVIjLcbl+eiGwTkc0ist5dMZ6Xiyu83jAqhbBAP15efcBDgSmllPu5swbxIjC1hf37gcuNMZnA/wDzm+2faIwZYYzJdlN853fWCq9HnBYLC/Rj5qgU3t9ayLHyGg8GqJRS7uO2BGGMWQmUtLB/lTGmqU1mDZDirljaxMUVXm8fm0pdg+H1tQc9FJhSSrlXR+mDuBP40O65AT4RkQ0ick9LB4rIPSKyXkTWHzt2rP0jO73Ca8ujmfrGhzF+YDwLvjlAXUNj+8ehlFIe5vUEISITsRLEg3abxxljsoBpwH0iMt7Z8caY+caYbGNMdnx8vHuCdHGF17ljUzl6soaPcpw3RymlVGfh1QQhIpnAc8AMY0xx03ZjTKHtvghYAoz2ToQ2Lq7wOiE9gd4xIby8Os8zcSmllBt5LUGISG/gLeA2Y8xuu+2hIhLe9BiYAjgcCeUxLq7w6usj3D42lXV5J9heWObBAJVSqv25c5jr68BqIF1E8kXkThGZJyLzbEV+BcQCTzUbztoD+EpEtgBrgQ+MMR+5K06XubjC643ZvQj29+WlVXmeiUsppdzEz10vbIyZc579dwF3Odi+Dxh+7hFeNuhqeN+2wmvvi50Wiwz257qsZBZvyOfhaYOIDg3wYJBKKdV+vN5J3WkEhkP6tPOu8ArWkNea+kYWrtPlN5RSnZcmiNbIvMmlFV4zEiMY0zeGV9cc0KvNKaU6LU0QrdG0wut5lt4AuOOSNApKq/hsx1EPBKaUUu1PE0RrNK3wumspVJ9ssegVg3rQMzJIO6uVUp2WJojWGjYL6qthp/MVXgH8fH24dWwqq/YWs/uo8+XClVKqo9IE0VqnV3htedIcwOyLehPg56O1CKVUp6QJorVcXOEVICY0gGuG9+StjQWUVbU88kkppToaTRAXwsUVXsHqrK6qa2DRhnwPBKaUUu1HE8SFcHGFV4ChyZGMSo3mldV5NOqQV6VUJ6IJ4kK5uMIrwNxL0sgrrmTFbjcsR66UUm6iCeJCubjCK8DUIYkkhAfyonZWK6U6EU0QF8rFFV4BAvx8uPni3qzYfYz9x095KECllGobTRBtkTnLpRVeAW6+uDf+vqLXilBKdRqaINoi42rwC3Zp6Y2E8CCmD0ti0fp8TtXUeyA4pZRqG00QbREUYa3wun0J1Neet/jcS9Ior6nnrY065FUp1fFpgmirphVe97a8wivAyF5RZKZE8tLqA5jz9FsopZS3aYJoq/6TITQBVvweGlpuOhIRbh+bRm5RBav2FrdYVimlvE0TRFv5+sP0x6FwE3z1l/MWvzoziZjQAB3yqpTq8DRBtIch18GQ661axJFtLRYN8vdlzuheLNtxlEMllR4KUCmlWk8TRHu56k8QHA1L5p23w/rWMamICK+uOeCh4JRSqvU0QbSXkBj47t/gaA6sfLzFokmRwVw5pAcL1x2iqrbBQwEqpVTruC1BiMjzIlIkIjlO9ouIPCEiuSKyVUSy7PZNFZFdtn0PuSvGdpcxHYbfDF/+GQo2tFh07tg0yqrqeGdzgYeCU0qp1nFnDeJFYGoL+6cBA2y3e4B/AoiIL/AP2/7BwBwRGezGONvX1P+DsB6w5F6oq3ZabHSfGDISw3XIq1Kqw3JbgjDGrARKWigyA3jZWNYAUSKSBIwGco0x+4wxtcBCW9nOITgKZjwJx3fB8t85LSYizL0kjR2HT7Iu74Tn4lNKKRd5sw8iGThk9zzfts3ZdodE5B4RWS8i648d6yDLafe/AkbdAauehINrnBa7dkQykcH+eklSpVSH5M0EIQ62mRa2O2SMmW+MyTbGZMfHx7dbcG025bcQ1QvevhdqHa/gGhzgy00X9eKj7Uc4XFbl4QCVUqplLiUIEQkVER/b44Eico2I+LfxvfOBXnbPU4DCFrZ3LoHhMOMfULIPPnvEabHbxqTSaAzPrNjnweCUUur8XK1BrASCRCQZWAZ8D6sTui3eBW63jWYaA5QZYw4D64ABItJHRAKA2baynU+f8TD6B7D2Gdi/0mGRXjEh3HJxb15clceyHUc9HKBSSjnnaoIQY0wlcD3wpDHmOqwRRs4PEHkdWA2ki0i+iNwpIvNEZJ6tyFJgH5ALPAv8EMAYUw/cD3wM7ADeMMZsb+Xf1XFc8RuI6Qfv3Ac15Q6L/PKqwQxOiuDnb26hoFSbmpRSHYO4MsRSRDZhfYH/BbjTGLNdRLYZY4a5O8DWyM7ONuvXr/d2GOc6+A28MBWybrcm0zmQd/wUVz/5FQN6hPHGD8bi76tzGJVS7iciG4wx2Y72ufot9BPgYWCJLTn0BZa3U3xdX++LYez9sOFFyP3MYZG0uFAeu2EYmw6W8vhHOz0bn1JKOeBSgjDGrDDGXGOM+b2ts/q4MeZHbo6ta5n43xCfAe/8B1SVOixydWZPbhuTyrNf7ufTb7U/QinlXa6OYnpNRCJEJBT4FtglIr9wb2hdjH8QXPtPqDgKHz3stNgvrx7E0OQIfv7GZl3tVSnlVa42MQ02xpwErsXqXO4N3OauoLqs5Cy47Gew5TXYudRhkUA/X/5xcxbGwP2vb6K2vtHDQSqllMXVBOFvm/dwLfCOMaaOFiavqRaM/0/oMQze+zFUOl6JJDU2lMdnZrLlUCm/1/4IpZSXuJogngHygFBgpYikAifdFVSX5hcA1/0Tqk7ABz93WmzasCTuuCSNf321n4+3H/FggEopZXG1k/oJY0yyMWa6bXG9A8BEN8fWdSUOgwkPwva3YPsSp8Uenp5BZkokD7y5RfsjlFIe52ondaSI/LlpUTwR+RNWbUJdqHE/hZ5Z8P7PoKLIYZGm/giA+1/bqP0RSimPcrWJ6XmgHJhlu50EXnBXUN2Crx9c97S1kN/7PwUnExZ7xYTwh5nD2ZJfxv8u3eHhIJVS3ZmrCaKfMebXtms07DPGPAL0dWdg3UJ8Okz6Jex8H9Y957TY1KGJfG9cGi+uyuPDbYc9GKBSqjtzNUFUicilTU9EZBygiwa1h7H3WdePWPqAtepro+NmpIenDWJ4ryj+c9FWDhZrf4RSyv1cTRDzgH+ISJ6I5AF/B37gtqi6Ex9fmLPQusDQV3+GN26DmopzigX4+fD3OSMRgfte20hNfYPnY1VKdSuujmLaYowZDmQCmcaYkcAkt0bWnfj6w9V/ham/h11LrYX9yvLPKdYrJoQ/3jicbQVl/O4D7Y9QSrlXq5YMNcactM2oBviZG+LpvkRgzDy4+U04cQDmT4T8c1emnTIkkbsu7cPLqw/wwVbtj1BKuU9b1pR2dGlQ1VYDroA7P4WAEHhhOmxbdE6RB6dlMLJ3FA8u3krecceXM1VKqbZqS4LQpTbcJSED7vockkfB4jth+f+e1Xnt7+vDk3NG4usj3PfaRqrrtD9CKdX+WkwQIlIuIicd3MqBnh6KsXsKjYXb34ERt8KK38Oi70HtmdFLKdEh/HnWcLYXnuS3H3zrxUCVUl1ViwnCGBNujIlwcAs3xvh5Kshuyy8AZvwdvvM/8O078MI0OFl4evfkQT34wfi+vLrmIIs3nNuprZRSbaHXtezoRGDcj2DO61CcC89OgsJNp3c/cGU6Y/rG8ItFW3hj3SEvBqqU6mo0QXQW6dPg+x+Djx88Pw22vw1Y/REv3DGacf3j+M/FW3n+q/3ejVMp1WVoguhMEofC3Z9bq8G+ORdW/AGMITjAl+fmZjN1SCKPvv8tTy7bg3GytpNSSrnKrQlCRKaKyC4RyRWRhxzs/4WIbLbdckSkQURibPvyRGSbbd+5EwK6q7AEmPseZN4Ey38Li++CuioC/Xz5+80juX5kMn/6dDePfbhTk4RSqk3c1tEsIr7AP4DvAPnAOhF51xhzesiNMeYPwB9s5b8L/NQYY3+ZtYnGmOPuirHT8g+C656xFvtb9iicyIPZC/ALT+SPNw4nNNCPZ1buo6Kmnv+ZMRQfH52yopRqPXfWIEYDubbVX2uBhcCMFsrPAV53Yzxdiwhc9nOY9QoUfQtPXwq5y/DxER6dMYR7J/RjwTcH+fmbW6hv0OtIKKVaz50JIhmwH1aTb9t2DhEJAaYCi+02G+ATEdkgIvc4exMRuafpQkbHjh1rh7A7mcHXwF3LICQOXr0ePvl/SEMdD07N4BdXprNkUwE/XKCL+ymlWs+dCcJRu4azRvHvAl83a14aZ4zJAqYB94nIeEcHGmPmG2OyjTHZ8fHxbYu4s+ox2Oq8zv4+rHoCnr8SSvZx38T+PHLNED759ih3vbSeytp6b0eqlOpE3Jkg8oFeds9TgEInZWfTrHnJGFNouy8ClmA1WSlnAkLg6r/ArJehZC88PR62vsHcS9L4w8xMvs49zu3/WktZVZ23I1VKdRLuTBDrgAEi0kdEArCSwLvNC4lIJHA58I7dtlARCW96DEwBctwYa9cxeAbM+9oaEvvW3bDkXm4cFs3fb85iS34pNz+7huKKGm9HqZTqBNyWIIwx9cD9wMfADuANY8x2EZknIvPsil4HfGKMsV+WtAfwlYhsAdYCHxhjPnJXrF1OVC+Y+z5c/iBsXQjPjGd67FHm355NblEFN81fw5Gyam9HqZTq4KQrjZXPzs4269frlImz5H0Fi++GU8fgO4/yTcIs7nx5A9Gh/iy4cwy9Y0O8HaHq7L78E1SVwhWPgI/Ove1sRGSDMSbb0T791+zq0i6Fe7+GAVPg44e5eM29LLylP+XV9dz4zCpyi8q9HaHqzNb9y5qLs+oJWP47b0ej2pkmiO4gJAZmL4Dpf4R9Kxj67jTem95AQyPMemYNOQVl3o5QdUZ7l8PSX8CAKyHrdvjyj7DhJW9HpdqRJojuQgRG320Nhw2Kotf7c/hs+BeE+RnmzF/D6r3F3o5QdSbHdsMbc63Z/Dc8B1f9GfpNhvd/CrmfeTs61U40QXQ3iUPhnuWQdTtRG57ks+jHGBZWyi3PreHPn+zSWdfq/CpL4LVZ1vVK5iyEoAjw9YdZL0HCYCtxHNnm7ShVO9BO6u5s+xJ498cYGlkZOpXlR4MJjEvljmnjSUodAEFRVs1DqSb1tfDKdZC/Du54H3o1m55UVgDPXWE9vusziHS4eILqQFrqpNYE0d2dOADv/RgOroH6qrN2mYBwJKoXRPWGyF7W8NlIu+dhCZpAuhNj4N37YdOrcP2zkDnLcbkjOfD8VIhOhe99aNUwVIfVUoLQy4Z2d9GpcPvb1n/+ymKOHtzNqx9/RdWxPC6LquKSiEr8ywrg4GqobtaZ7RsIkSlW4ug3yVrqIzDcK3+G8oBVT1rJYfwvnCcHsJoxZ70EC26EN26HW960mqC6mvoaOLwVCtZbP7QaaqwaVn31mcdnbau1jmmose7ra85sEx9rMElwNATH2D2Otj2OOftx036/QLf+iVqDUOdoaDQ8tTyXvy7bQ2JEEH+dPYKL0mKg+iSUHYLSQ7b7g9atZB8c2Wp9YMfeB6PvgaBIb/8Zqj3tXAoLb7Zm6s98wbX5DhtfsWocI2+Da57s3LVNY6zPecEGyF9vNbEd2QaNtqVrAsKtL2u/IKtvxjfQ7j4QfAOc7/MLhMZ6qDph3SpPQFWJ1ddTVWIlEWf8Q61kEdUbvrf0gv40bWJSF2TTwRP8eOFm8k9Ucv/E/vxo8gD8fJ18MeRvgJWPw+6PrORw8b0wZp6VNNS5Ghvg0FrYtRT2fGLVvMbeB4OuAR9fb0d3tiPb4F9XQvxAuGOpte6Xqz7/Laz8A0z6pVXz6CyqTtiSwQarhpC/3vqyButLuedISMm2bsnZEJHknjiMgbpKW7KwTxxNj233Pr4w4x8X9BaaINQFq6ip59fvbGfxxnxG9o7ibzeNbHn2deFm6wth5/vWr6qL74Ex90ForMdi7rBqKmDfcuvX+J6PobIYfPwhbZxVKyvZC9F9rEQx4pbWfRG7S/lReHYSmEZr9Ft4YuuONwaW/AC2/rvlfgtvMMb6dV5XaTUR5a87U0Mo3mMrJBCfcXYyiM8A367TOq8JQrXZe1sK+a8l2zAGHp0xhOtGJiMtNRkcybEmTm1/G/xD4KLvwyU/sjq2u5OTh2H3h7DrQ9i3wmp/Doq0JpelT4P+k63njQ1WbeKrv1q/WENiraa6i+72XnKtq4IXr4KiHfD9jyBp+IW9Tn2tda2Sg2vgtiXQ57L2i9EY2L8SCjdCbaX1ZV9XZbudOvO4tulxpd19pZX47IUmnJ0Meo7s8p3smiBUu8g/UcnP/r2FtXklXDO8J/9z7VAig8/T+Vi001qrJ2eR1eaa/T0rUbirSu5txsDR7VZC2PUBFG6ytkenQfpVVlLoPcZ5p60x1oCAr5+wEotfMGTdZtUqotM89VdYcSz6vjUU+qZXYdDVbXu9qhNWM1XFEbjzU2uCXVvUVcO2N2HNP6Fo+5nt/iF2t2CrFtb02D/Yah7yDz53f1gPKylE9urcfSUXQBOEajdOO7DPp3ivlSi2LAQfP+tLb9xPrBFQnV19LRz42pYUPoSyg4BAykVWQkifbn0htvaLp2inNXJo67/BNMDga2Hcj6xfte62/P9gxWNwxW/g0p+2z2ueOGDNkfALsuZIhPdo/WuUH4X1/7LWgKo8Dj2Gwph7rc7zgLBu9+XeHjRBqHbXqg5seyX74au/wObXrOcjbobLfubZX8cXqrHBiv/YDqvZpehb60u8eI81CsUvGPpNtJLCgCsv7AvQkZOF8M3TsP4FqDkJfcbDuB9bS1u44wtx2yJYfCcMvxmufap936Ngo9VsFTfQGnUTEOracYe3WrWFnEXQUAcDp1qJoc94TQptpAlCuYV9B3ZGYjgPTcvg8oHxLfdNNCk9BF//FTa+bH3xDp8Nl/0cYvu5Pe7zamy0hvEW7Tg7GRzfY41nB0CsOSQJgyFhkFVb6HO5ezuWq8usxfDWPAXlh61fz5f8CIZe337zDA6ts77Ak0dZ82PcMc5+14fWkNkBV1qLSDobtdXYALs/tv7evC+t5qGRt8DF8zrG56SL0ASh3OqjnMP8bukODpVUMa5/LA9PG8TQZBfnQZwstNrbN7xgjSgZegNc9gAkZLg36CbGWL9qD62x1Qh2wLFdUFtxpkxEspUEEgZBfNN9uuu/fttbfa3V/r7qSSuBRaRA5o2QOAwShkBs/wsbZVN6yBqxFBACd33u3s7xtc/C0gesjvhpj59dC6ipgM0LrBrDif1Wv8Doe6xmSR023e40QSi3q6lvYMGagzz5+R5OVNZx7Yie/HxKOr1iXPxFXVFkfeGt+5c1umTwNda4+cRh7gm46gRs+TdsfMlKDACh8bZEMNgaypgw2EoEwVHuiaGtGhsh91PrvB1YZfVTgDUpKz7dShY9BkOPIdbj8ETnzTE15dbyGKUHrU5kTyToj/8bVv8dpvwOLrnfeu9vnrEm2NWUQcpoqxlp0DVdalhpR6MJQnlMWVUdT6/Yy/Nf7ccYmHtJKvdN7E9USIBrL3Cq2GpSWDvfam9Pn24liuSstgdnjPVFuvEl+PYdq7mo50jImgsZV0NYfNvfw1vqa6yaT9G31iiqo9utx+WHz5QJjrYlDVviSBhiJUT/YFh4izVh75Y3oP8Vnom5sRHenAs73rOWatm3HBAYci2M+aE1qki5nSYI5XGFpVX85dPdLNqYT3igH/dP6s/tY9MI8ndxlnBVqfVrcs1TUF1qfWmN/0/ofXHrgzl13OoU3/iy1aEcGGFN2MqaC0mZrX+9zqSy5EyyOJ04dlhzBJqEJsCpIuuCUqPv9mx8dVXW6rBF38KoO6ympMgUz8bQzWmCUF6z4/BJfv/RTr7YdYzkqGAeuHIgM4Yn4+Pj4siT6pOw7jmrKaKyGNIug8sftC6l2lJneGMj7F8BG16EnR9Ya+b0GgOj5lrDRTvCLGVvaWyE0gNnJ46Ui6xmHm9oqLcmrPm5WMtU7cprCUJEpgJ/A3yB54wxjzXbPwF4B9hv2/SWMeZRV451RBNEx/V17nH+78Md5BScZEjPCB6eNohLB8S5/gK1p6xhnquegIqj0HssjH/g3KGe5UesFUc3vmx9CQZHw/A5Vm3BUx3fSnUiXkkQIuIL7Aa+A+QD64A5xphv7cpMAB4wxlzd2mMd0QTRsTU2Gt7bWsjjH+2ioLSK8QPjeWhqBoN7tmIpg7pq2PSKtSTFyXxrOOb4X1jLJW94yVos0DRYNY1Rd1h9C/5B7vqTlOr0vHU9iNFArjFmny2IhcAMoMUv+XY4VnVQPj7CjBHJTB2ayCurD/Dk57lc9eSXXD8yhZ9NGUhyVPD5X8Q/yGonz5oLW16DL/8Mr8+29oXGwyX/AVm36zh5pdqBOxNEMnDI7nk+4KiHcayIbAEKsWoT21txLCJyD3APQO/evdshbOVugX6+3HVZX24c1YunVuTywtd5vLelkNvGpvLDCf2IDXNhcpZfgFVDGHGLNQrG19+aeKXt2Eq1GxfWRrhgjnoQm7dnbQRSjTHDgSeBt1txrLXRmPnGmGxjTHZ8fCceptgNRYb48/C0QSx/YALXjuzJC1/vZ/zjy/nLp7spr65z7UV8/a2ZxIO+q8lBqXbmzgSRD9ivxJaCVUs4zRhz0hhTYXu8FPAXkThXjlVdR3JUMI/PHM4nP72cy9Pj+duyPYx/fDnPfbmP6roGb4enVLflzgSxDhggIn1EJACYDbxrX0BEEsW2cI+IjLbFU+zKsarr6Z8QxlO3jOK9+y9laHIkv/1gBxP/+AUL1x6kvqHx/C+glGpXbksQxph64H7gY2AH8IYxZruIzBORebZiM4EcWx/EE8BsY3F4rLtiVR3LsJRIXrnzYl6/ewyJkUE89NY2pvxlJe9vLaSxsevM21Gqo9OJcqpDM8bw6bdH+eMnu9h9tIIhPSP4xZXprq8aq5RqUUvDXN3ZxKRUm4kIU4Yk8uGPx/PnWcMpq6rjjhfWcdP8NWw4UOLt8JTq0rQGoTqV2vpGFq47yBPLcjleUcMVgxL4+ZR0BiV17esGK+UuuhaT6nIqa+t54es8nl6xl4qaeq4cnMjtY1MZ2y9Wm56UagVNEKrLKqusY/6Xe1nwzUFKK+voFx/KrWNSuT4rhcjgdrrKmlJdmCYI1eVV1zXwwdbDvLLmAJsPlRLs78u1I3ty65hUhvR08ep2SnVDmiBUt5JTUMaraw7w9uYCqusayeodxW1jU5k2NMn161Eo1U1oglDdUlllHYs35vPqmgPsO36KmNAAZmX34paLe7t+KVSlujhNEKpbM8awam8xr6w+wKc7jtJoDBPTE7htTCrjB8bj6+rFi5TqgjRBKGVzuKyK19ce4vW1BzlWXkOvmGBuHp3KjdkpxLmyiqxSXYwmCKWaqWto5JPtR3llTR5r9pXg5yNMSI/n+qwUJmUkaF+F6ja8dcEgpTosf18frspM4qrMJHKLynlzQz5vbyrgsx1FRAT58d3hPbk+K4Ws3lE6r0J1W1qDUMqmodGwau9x3tpYwEc5R6iqayAtNoTrs1K4bmSydmyrLkmbmJRqpYqaej7cdpi3Nhawel8xAKP7xHBDVjLThiUREaST8FTXoAlCqTbIP1HJO5sLWbwhn33HTxHo58OUIYlcn5XMZf3j8PPVNS9V56UJQql2YIxhS34Zizfk897WQkor64gPD+TaET25bmQKg3vqgoGq89EEoVQ7q6lvYPnOY7y1MZ/lu4qoazBkJIZzfVYyM0Yk0yMiyNshKuUSTRBKuVHJqVre31rIWxsL2HyoFB+Bcf3juD4rmSuHJBISoIMFVcelCUIpD9l7rIK3NxWwZFMB+SeqCAnwZerQRK4fmcLYfrE6a1t1OJoglPKwxkbDurwSlmwq4IOthymvqScxIogZI3tyQ1YKA3uEeztEpQBNEEp5VXVdA5/tOMpbGwtYsfsYDY2GIT0juD4rhWuG9yQ+XJf4UN6jCUKpDuJ4RQ3vbbH6K7YVlOHrI1w2II7pw5IY1z+O5Khgb4eouhmvJQgRmQr8DfAFnjPGPNZs/y3Ag7anFcC9xpgttn15QDnQANQ7+wPsaYJQnUluUTlvbSzg7U0FFJZVA5AaG8Il/eK4pF8sY/vF6gKCyu28kiBExBfYDXwHyAfWAXOMMd/albkE2GGMOSEi04DfGGMutu3LA7KNMcddfU9NEKozMsaw+2gFX+ceZ9XeYr7ZV0x5TT0A6T3CGdsvlnH94xjdJ0Yvo6ranbcW6xsN5Bpj9tmCWAjMAE4nCGPMKrvya4AUN8ajVIckIqQnhpOeGM73L+1DfUMj2wtP8vXe46zeW8zCdQd5cVUePgLDkiMZ2y+Ocf1jyU6NIThAV51V7uPOBJEMHLJ7ng9c3EL5O4EP7Z4b4BMRMcAzxpj5jg4SkXuAewB69+7dpoCV6gj8fH0Y3iuK4b2i+OGE/tTUN7DpYCmr9hazeu9xnvtyH0+v2Iu/rzCyd7TVHNU3luG9onSZctWu3NnEdCNwpTHmLtvz24DRxpj/cFB2IvAUcKkxpti2racxplBEEoBPgf8wxqxs6T21iUl1B6dq6lmXV8LqvcWs2ltMTmEZxkCArw/De0VyUVoMo/vEMCo1mnBdVFCdh7eamPKBXnbPU4DC5oVEJBN4DpjWlBwAjDGFtvsiEVmC1WTVYoJQqjsIDfRjQnoCE9ITACitrGVd3gnW5ZXwzf4Snlm5j6e+2IuPwOCeEVyUFsPFfWLITovRTm/VKu6sQfhhdVJPBgqwOqlvNsZstyvTG/gcuN2+P0JEQgEfY0y57fGnwKPGmI9aek+tQSgFlbX1bDpYyjf7S1i3v4SNB09QU98IQL/4UEb3sWoYF6XFkBKt17jo7rxSgzDG1IvI/cDHWMNcnzfGbBeRebb9TwO/AmKBp2xX7WoaztoDWGLb5ge8dr7koJSyhAT4Ma5/HOP6xwFQW9/ItoIy1u4vYV1eCe9vPczra63uweSoYC5Ki2ZUajRDkyMZlBSh/RjqNJ0op1Q309Bo2HWknLX7i1mXd4K1eSUcK68BwNdHGJAQxtDkSIYlRzI0OYLBSZE6WqoL05nUSimnjDEUllWzLb+MnIIycgqt++MVtQD4CPSLD7MlDOs2pGcEoYG6Sm1X4K1OaqVUJyAiJEcFkxwVzNShiYCVNI6crCan4CTbCqyE8VXucd7aVGA7BvrGhZ6uaQxOiiA9MZxY7QTvUjRBKKXOISIkRQaTFBnMdwb3OL296GS1LWFYieObfSW8s/nM4MS4sADSE8MZ2COcDNv9wB7hWtvopPRfTSnlsoSIICZHBDF50Jmkcay8hp1HTrLrSDm7jpSz+2g5C9ceoqqu4XSZXjHBpNuSRdOs8b5xYQT46fW8OzJNEEqpNokPDyQ+PJ7LBsSf3tbYaDh0ovJ00th11Lpfvsta7hzAz0foGx/KwB7hDEgIJzU2xHYLJTrEH9soRuVFmiCUUu3Ox0dIjQ0lNTaUKUMST2+vqW9g//FTZxLHkXI2HSzl/a2Hzzo+PMjPShYxoWcljtTYEHqEB+GjV+bzCE0QSimPCfTzJSMxgozEiLO2V9c1cKikkgPFleQVn+Kg7fH2wjI+3n6E+kZj9xo+9I45O2kkRgQRHuRPeJAfYYF+1n2QH4F+Ojy3LTRBKKW8LsjflwE9whng4FKs9Q2NHC6rJq/4FAeKKzlYUknecSuJfJ1bfFZfR3MBfj6EB1rJ4kzy8G+2zZ+4sAD6J4TRPyFM16+yowlCKdWh+fn60CsmhF4xIVw24Ox9xhiOlddQVF5DeXU95dV1VNTUU15df/r+rG3V9RwqqaSi5sz+hsaz54L1iAi0kkW8lTD62RJHfFhgt+sX0QShlOq0RISEiCASIoIu6HhjDNV1jRwuqyK3qILcYxXkFlWwt6iCRRvyOVV7pnYSGex/VuJouiVHBXfZPhFNEEqpbktECA7wpW98GH3jw5hit69psmBuUcVZt2U7j/Lv9WcudRPk70Ov6BCiQwOIDvEnJjSA6BDbzbYtOjSAGNu28CC/TpNQNEEopZQD9pMF7YfwgrXEun3SyD9RxYnKWvKOV7LxYCknTtWe1bFuz9dHiA7xJyrEShpRIf7EhQeSFBFEUlQwPSODSIwMIiky2OtrYGmCUEqpVooKCSA7zbrGhiPGGCpq6jlxqo4TlbWUVNZy4lQtJyrrOHHKel5aWUvJqVoOFFey/sAJSk7VnvM60SH+JEZaSSMpKsiWsM7cJ0YGuXX1XU0QSinVzkTENuzWn96xrl1zo7qugSNl1Rwuq+ZwWRWHy6opLK3iSFk1hWXVbDh4gtLKunOOiw0NoG98KG/Ou6S9/wxNEEop1REE+fuSFhdKWlyo0zJVtQ0Ok4e7VuXWBKGUUp2EfYe6J+hKWUoppRzSBKGUUsohTRBKKaUc0gShlFLKIU0QSimlHNIEoZRSyiFNEEoppRzSBKGUUsohcdcMPG8QkWPAgQs8PA443o7htDeNr200vrbR+NqmI8eXaoyJd7SjSyWIthCR9caYbG/H4YzG1zYaX9tofG3T0eNzRpuYlFJKOaQJQimllEOaIM6Y7+0AzkPjaxuNr200vrbp6PE5pH0QSimlHNIahFJKKYc0QSillHKoWyUIEZkqIrtEJFdEHnKwX0TkCdv+rSKS5eH4eonIchHZISLbReTHDspMEJEyEdlsu/3KwzHmicg223uvd7Dfa+dQRNLtzstmETkpIj9pVsaj509EnheRIhHJsdsWIyKfisge2320k2Nb/Ly6Mb4/iMhO27/fEhGJcnJsi58FN8b3GxEpsPs3nO7kWG+dv3/bxZYnIpudHOv289dmxphucQN8gb1AXyAA2AIMblZmOvAhIMAY4BsPx5gEZNkehwO7HcQ4AXjfi+cxD4hrYb9Xz2Gzf+8jWJOAvHb+gPFAFpBjt+1x4CHb44eA3zuJv8XPqxvjmwL42R7/3lF8rnwW3Bjfb4AHXPj398r5a7b/T8CvvHX+2nrrTjWI0UCuMWafMaYWWAjMaFZmBvCysawBokQkyVMBGmMOG2M22h6XAzuAZE+9fzvx6jm0MxnYa4y50Jn17cIYsxIoabZ5BvCS7fFLwLUODnXl8+qW+Iwxnxhj6m1P1wAp7f2+rnJy/lzhtfPXREQEmAW83t7v6yndKUEkA4fsnudz7pevK2U8QkTSgJHANw52jxWRLSLyoYgM8WxkGOATEdkgIvc42N9RzuFsnP/H9Ob5A+hhjDkM1o8CIMFBmY5yHr+PVSN05HyfBXe639YE9ryTJrqOcP4uA44aY/Y42e/N8+eS7pQgxMG25mN8XSnjdiISBiwGfmKMOdls90asZpPhwJPA2x4Ob5wxJguYBtwnIuOb7ff6ORSRAOAa4E0Hu719/lzVEc7jfwP1wAInRc73WXCXfwL9gBHAYaxmnOa8fv6AObRce/DW+XNZd0oQ+UAvu+cpQOEFlHErEfHHSg4LjDFvNd9vjDlpjKmwPV4K+ItInKfiM8YU2u6LgCVYVXl7Xj+HWP/hNhpjjjbf4e3zZ3O0qdnNdl/koIxXz6OIzAWuBm4xtgbz5lz4LLiFMeaoMabBGNMIPOvkfb19/vyA64F/OyvjrfPXGt0pQawDBohIH9svzNnAu83KvAvcbhuJMwYoa2oK8ARbm+W/gB3GmD87KZNoK4eIjMb6Nyz2UHyhIhLe9BirMzOnWTGvnkMbp7/cvHn+7LwLzLU9ngu846CMK59XtxCRqcCDwDXGmEonZVz5LLgrPvs+reucvK/Xzp/NFcBOY0y+o53ePH+t4u1eck/esEbY7MYa3fDftm3zgHm2xwL8w7Z/G5Dt4fguxaoGbwU2227Tm8V4P7Ada1TGGuASD8bX1/a+W2wxdMRzGIL1hR9pt81r5w8rUR0G6rB+1d4JxALLgD22+xhb2Z7A0pY+rx6KLxer/b7pM/h08/icfRY8FN8rts/WVqwv/aSOdP5s219s+szZlfX4+WvrTZfaUEop5VB3amJSSinVCpoglFJKOaQJQimllEOaIJRSSjmkCUIppZRDmiCU6gDEWmX2fW/HoZQ9TRBKKaUc0gShVCuIyK0ista2hv8zIuIrIhUi8icR2Sgiy0Qk3lZ2hIissbuuQrRte38R+cy2YOBGEelne/kwEVkk1rUYFjTN+FbKWzRBKOUiERkE3IS1yNoIoAG4BQjFWvspC1gB/Np2yMvAg8aYTKyZv03bFwD/MNaCgZdgzcQFa/XenwCDsWbajnPzn6RUi/y8HYBSnchkYBSwzvbjPhhrob1GzizK9irwlohEAlHGmBW27S8Bb9rW30k2xiwBMMZUA9heb62xrd1juwpZGvCV2/8qpZzQBKGU6wR4yRjz8FkbRf5fs3ItrV/TUrNRjd3jBvT/p/IybWJSynXLgJkikgCnry2divX/aKatzM3AV8aYMuCEiFxm234bsMJY1/fIF5Frba8RKCIhnvwjlHKV/kJRykXGmG9F5JdYVwHzwVrB8z7gFDBERDYAZVj9FGAt5f20LQHsA75n234b8IyIPGp7jRs9+Gco5TJdzVWpNhKRCmNMmLfjUKq9aROTUkoph7QGoZRSyiGtQSillHJIE4RSSimHNEEopZRySBOEUkophzRBKKWUcuj/AzJn/CK4gz6WAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEICAYAAABS0fM3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzuklEQVR4nO3deXyU1b348c83+05WEkgICWEHMUBYVFRwqWBV3MWlC1q5bnW5y09v21vbu9rl9rbea6W2F7UVRVxQiqhVr4pVWcJq2CQQIAuB7Ps6c35/PJMwhMkC5MlMMt/36zWvWZ6Tme88DOf7POc85xwxxqCUUsp/BXg7AKWUUt6liUAppfycJgKllPJzmgiUUsrPaSJQSik/p4lAKaX8nG2JQERWiMgJEcnrZruIyNMiki8iu0Rkhl2xKKWU6l6Qje/9AvA/wB+72b4IGOe6zQGedd33KDEx0WRkZPRPhEop5Se2bt1aboxJ8rTNtkRgjNkgIhk9FFkM/NFYI9o2ikisiIwwxhzr6X0zMjLIzc3tz1CVUmrIE5Ej3W3zZh9BKlDo9rzI9ZpSSqkB5M1EIB5e8zjfhYgsE5FcEcktKyuzOSyllPIv3kwERcAot+dpQImngsaY54wxOcaYnKQkj01cSimlzpKdncW9WQs8JCKrsDqJa3rrH+hOW1sbRUVFNDc392uAvigsLIy0tDSCg4O9HYpSaoiwLRGIyCvAfCBRRIqAJ4FgAGPMcmA9cDWQDzQCS8/2s4qKioiOjiYjIwMRTy1OQ4MxhoqKCoqKisjMzPR2OEqpIcLOq4Zu72W7AR7sj89qbm4e8kkAQERISEhA+0mUUv1pyIwsHupJoIO/fE+l1MDxZh+BUkqpLowx1LW0U1nfSkVDC+X1rVTUt1JR30J2eiwXj+v/C2Y0EfSD6upqXn75ZR544IEz+rurr76al19+mdjYWHsCU0p5ncNpqG9pp665jaqGNsobWjor9oqGVsrrXc87X2+l1eH0+F73z8/SROCrqqur+e1vf3taInA4HAQGBnb7d+vXr7c7NKXUOTLGUNXYRkl1E1WNrdQ3t1PX3E6dq3LveF7f0k5tc1vn445tDa2Obt87NCiAxKhQEqNCGB4dxqSUGBJczxOiQkiIDCUhKoTEqFDiIkIICbKnNV8TQT944oknOHjwINnZ2QQHBxMVFcWIESPYsWMHe/bs4frrr6ewsJDm5mYeeeQRli1bBpycLqO+vp5FixYxb948vvjiC1JTU3n77bcJDw/38jdTauhzOg3l9S0UVjVRXN1EcVUTRVWNbo+baGrrvjKPDAkkKiyI6LBgokKDiA4LYmRsGNGhwa7Xgzpfj++o2F33ESGBPtHvN+QSwU//vJs9JbX9+p6TR8bw5LVTut3+1FNPkZeXx44dO/jkk0/45je/SV5eXuclnitWrCA+Pp6mpiZmzZrFTTfdREJCwinvceDAAV555RV+//vfc+utt/LGG29w11139ev3UMqftDuc1lF6k3WkXtvcRmlNM0VVVgVfXG1V+CXVzac1xcRFBJMaF05WUhSXjE8iNTac1Lhw4iND3Cp2q+IPDPB+RX6uhlwi8AWzZ88+5Tr/p59+mjVr1gBQWFjIgQMHTksEmZmZZGdnAzBz5kwOHz48UOEq5dOaWh0UVzdSXN1MdWMrtc3t1DZZTTAdTTF1zW2dr3W83thDk0xSdCipseFMTR3GVVNTSIsNJy0ugtS4cFJjw4kM9a+qcch9256O3AdKZGRk5+NPPvmEDz/8kC+//JKIiAjmz5/vcQR0aGho5+PAwECampoGJFalvK2+pd1qinE1w3QcqXc8r2ho9fh3wYFCTFgwMeHBRLuaYJJjwogOCyImLJjoMOt19+0pMWGMjA0nLLj7vjt/NOQSgTdER0dTV1fncVtNTQ1xcXFERESwb98+Nm7cOMDRKeVdDS3tFFY1crSikSJX5d7RBl9U1URNU9sp5UOCAkhzNcV8Y2SMdaTueh4XEUJMuFXRhwYF+ET7+lCgiaAfJCQkcNFFFzF16lTCw8NJTk7u3LZw4UKWL1/OtGnTmDBhAnPnzvVipEr1P6fTcLyumaMVjRytPPVWWNlIef2pR/ThwYGkxYWTFhfO9PTYzoo+Lc6q7BMjQwkYAu3ug4lYMz0MHjk5OabrwjR79+5l0qRJXopo4Pnb91Xe19zm4EhFI0cqGjor+KOVjRyptI7yW9tPdrYGCIyMDSc9PoL0+AhGxUcwOiGCUXHW47iIYD2S9wIR2WqMyfG0Tc8IlFKdqhtbyT9Rz8Gyeg6WNXQ+LqxsxOl2zBgdGkR6QgQTkqO5clIyo1yV/uiECEbGhhMcOGRmr/ELmgiU8jNOp6G4uon8snoOdlT6Jxo4WFZ/SsdsSFAAYxIjmZo6jMXZqWQlRZKREEl6fASxelQ/pGgiUGoIq2xoZXdJDbtLatldUkv+iXoOldXT4taUEx8ZQlZSJFdOTmbs8Ciykqxbalz4kLhGXvVOE4FSQ4AxhtLaZnYX15LXUfEX11BSc/JS5dTYcMYnRzFvbIJV2bsq/fjIEC9GrnyBJgKlBhmn03C0srGzws8rrmFPSW1ns44IjEmMZFZmPFNGxjB15DAmj4whNkIrfOWZJgKlfJgxhsLKJrYXVrGzsIa8EqvSr29pB6xBVeOTo7l80nCmpg5jysgYJqbE+N3IWHVu9NfiBVFRUdTX11NSUsLDDz/M66+/flqZ+fPn88tf/pKcHI9Xe6khqqapjZ2F1exw3XYWVnce6YcFBzB5RAw3zkhlysgYpowcxrjkKEKDdJSsOjeaCLxo5MiRHpOA8g9tDif7S+vYXljNjqPV7Cis4mBZA2A172QlRXHZxOFkp8eSPSqWCcnRBOllmcoGmgj6weOPP87o0aM71yP4yU9+goiwYcMGqqqqaGtr41//9V9ZvHjxKX93+PBhrrnmGvLy8mhqamLp0qXs2bOHSZMm6VxDQ1BJdRPbjla5Kv1q8kpqaG6zrt5JjAohe1QsN0xPJXtUHNNGDSMmLNjLESt/MfQSwbtPQOlX/fueKefBoqe63bxkyRIeffTRzkSwevVq3nvvPR577DFiYmIoLy9n7ty5XHfddd1ee/3ss88SERHBrl272LVrFzNmzOjf76C84sDxOt7LK+W93aXsdk2PHhoUwNTUYdw5ZzTZo6yj/bS4cL0uX3mNrYlARBYCvwECgT8YY57qsj0OWAFkAc3A3caYPDtjssP06dM5ceIEJSUllJWVERcXx4gRI3jsscfYsGEDAQEBFBcXc/z4cVJSUjy+x4YNG3j44YcBmDZtGtOmTRvIr6D6iTGG3SW1vJt3jPfySjubemaOjuMHV09k7pgEJqbE2LbSlFJnw7ZEICKBwDPAlUARsEVE1hpj9rgV+wGwwxhzg4hMdJW//Jw+uIcjdzvdfPPNvP7665SWlrJkyRJWrlxJWVkZW7duJTg4mIyMDI/TT7vTI8LByek0bDta1XnkX1TVRGCAMCcznu9emME3pqSQHBPm7TCV6padZwSzgXxjzCEAEVkFLAbcE8Fk4D8AjDH7RCRDRJKNMcdtjMsWS5Ys4d5776W8vJxPP/2U1atXM3z4cIKDg/n44485cuRIj39/ySWXsHLlShYsWEBeXh67du0aoMjV2WhzONlcUMm7ecd4f/dxyupaCAkMYN64RB6+bBxXTE7WgVpq0LAzEaQChW7Pi4A5XcrsBG4E/iois4HRQBow6BLBlClTqKurIzU1lREjRnDnnXdy7bXXkpOTQ3Z2NhMnTuzx7++//36WLl3KtGnTyM7OZvbs2QMUueqr5jYHn+eX825eKR/uPU51YxvhwYHMn5DEwqkpXDZxONHawasGITsTgad2jq5zXj8F/EZEdgBfAduB9tPeSGQZsAwgPT29f6PsR199dbKTOjExkS+//NJjufr6esBavD4vz+oSCQ8PZ9WqVfYHqfrM6TTsOVbLlwcr+OJgOZsLKmlodRAdFsQVk5K5akoKl45PIjxEr+NXg5udiaAIGOX2PA0ocS9gjKkFlgKI1UBe4LrRpdxzwHNgrUdgU7zKzxljyD9Rzxeuin/jocrO1bOykiK5YUYqV0xK5sKsRO3sVUOKnYlgCzBORDKBYmAJcId7ARGJBRqNMa3A94ANruSglO06pm/44mC5q/KvoLy+BbAmaLtqilXpX5CVoJ29akizLREYY9pF5CHgfazLR1cYY3aLyH2u7cuBScAfRcSB1Yl8zzl8nl9cdTPYVpTzNaU1zXx5qJwv8q2Kv7jaGriXFB3KRWMTuDArgQuzEhkVH+HlSJUaOLaOIzDGrAfWd3ltudvjL4Fx5/o5YWFhVFRUkJCQMKSTgTGGiooKwsL06PRMFFY28ua2YtbuLO68rj82IpgLxiRw36VjuCArkaykyCH921GqJ0NiZHFaWhpFRUWUlZV5OxTbhYWFkZaW5u0wfF5dcxvrvzrGG9uK2VxQiQjMzUzg9tnpXJCVwKSUGF0gXSmXIZEIgoODyczM9HYYysscTsNnB8p4c1sx7+8upaXdyZjESP7hqglcPz2V1Nhwb4eolE8aEolA+bf9pXW8sa2It7YXc6KuhWHhwdyaM4obZ6SSPSpWm3yU6oUmAjUolde3sHZHCW9uLyKvuJagAGH+hOHcPDOVBROH6xz9Sp0BTQRq0Ghpd/DR3hO8ua2IT/aX0e40nJc6jCevncx1548kISrU2yEqNShpIlA+r6iqkT99eYRVWwqpaWojOSaUey7O5MbpaUxIifZ2eErZzxioLYGAQIj2PIPxudBEoHySMYbNBZW88MVh3t9diohw1ZRkbpuVzryxiQTqFT+Di9MJX78HwWGQcj5EJng7It/laIeKA9a6KqW74Ngu63FTJcz7W7jiyX7/SE0Eyqc0tzn4884SXvjiMLtLahkWHsyyS7L41gWj9aqfwaquFNbcB4c+Pvla9EgYMQ1Spp28j0231uj0Jy31cHy3VeGXfmXdTuyBdteU9YGhMHwSTPwmjDgfRl9kSxiaCJRPOFHbzEsbj7By01EqGloZNzyKf7/hPG6YnqqTug1mX/8F3rofWhvg6l9C4riTR7ilu+DAX8BYy3USNsxKCO7JIXE8BPahmmptsBJOXSnUu+7rjkHdceu+/rh1CwqD8HgIj4MI9/v4bu7jIPAsZpR1OsDR6rq1W/ftzVBx0FXpu/ZBxUE65+IMi7W+96zvufbDedb+OpvPP0OaCJRX7Sys5vnPC3jnq2O0Ow2XTRjO0osyuWjs0B4lPuS1NcOHP4FNz8LwKXDzChjumop9zPyT5Vob4cReKN3pShC7IPd/Tz0iTp5sVYzJU8HZdmoFX1dqVfAtHqYoCwy12tOjU6yj6sxLrQq5qRIaq6DyEDRWWs8drd1/l5BoiIizkoMIONpct1Zwtp9e4TtaOX2i5S5iR1sV/Xm3WvcjpkFMqtfOiGSwzV2Tk5NjcnNzvR2GOgdtDifv5ZXy/OcFbDtaTVRoELfkpPGdCzLISIz0dnhDR3srtDVYR8utjeBogaRJfTvCPhdl++H1e+D4VzDnPrjip1bfQF852qEi39U+vvPk2UNTlbU9KMyq3KNSTlb0np6HxfatYjXG2kdNlScTQ2Ol9Xkdz5uqTn5+QLB1lB4Y4rp3Pe7L63GjrYQWHnume/WcichWY0yOp216RqAGTGVDK69sPsqfvjxCaW0zGQkRPHntZG6emaYLuvSmsRJ2vmId/bY2WhWXeyXv6bHztKU9YFg6XPAgzPgWhPRz0jUGtr0I7z4BIRFw+6swYeGZv09gkHX2MHwiTLv15HvXlUJwuNWE1J9HziIQGmXdYn13vRM7aSJQtttfWsfznxewZnsxLe1O5o1N5N9umMqCCcN1vp/eNNfAxmfhy2es5o/AUKuSDYmC4IiTj6NSXI8jITjSw+Moqzlj24vw3uPw6VMw616YvQyiks49zsZK+PMjsHet1fRzw+/69zJHEYgZ0X/vp06hiUDZwuk0fLz/BCs+L+Dz/ArCggO4cUYaSy/KYHyyXvvfq5Z62PwcfP4baK6GidfAgh9A8pRze9/s2+HoJvjiadjwC+s++w644CFIyDq79zz8Obx5r3W2cuU/wwXfhwBduGcw0USg+lV9Szuv5xbywheHOVzRyIhhYfy/hRO4fVY6cWe7mHtTFWx/CfashWFpMDIbRk63LqcLG9av8XtdWxPkroDPfgWN5TDuKisBjMzuv89InwPpK6H8AHzx39a+zX0eJl8HFz4CaTP79j6Odvj0Z/DZLyEuA+75AFJn9F+casBoZ7HqF4WVjbzwxWFWbymkrqWd6emx3H1RJgunphAceJZHh6V51lHxrtXQ3gQjsq2Ou+qjJ8vEZ1lJoSM5pEyDsJhz/0JOh3VVSvVR61Z1xDriTTkPshZYFV9/am+BbX+Ez/7T+twx82HBD2HU7P79HE/qSmHT72DL/0JLDYyeBxc9AuOu7L4tvuqIdRZQuAnOvwOu/jmE6pmeL+ups1gTgTprxhg2FVTy/OcFfLDnOAEiXH3eCJZelMH09Lize1NHO+xbZyWAI59DUDhMu8Vqy045zyrTUAHHtkPJDijZbl1ZUlPoegOBhLGnJ4fQqFM/x+m0rjevOnKysq8+cvK+puj0ztbQmJOXKcaPgTELIOsyyLz47M9MHG1WJ/CnP7e+Q/qFcNkPIWPe2b3fuWips5LRl89AbTEMnwwXPgxTb4Igt7O5vDfgz48BBq75Lzjv5oGPVZ0xTQSqX3WM/n3+88PsOVZLXEQwd8xJ51tzM0gZdparpzWUw9YXrGaR2mLr6o1Z98L0u6yBPX35+47EULIdju2w3gcAsQYmJU+2mpmqj1oVfddrx6OSrc/tvI0+eT8sDYJCreaUQx/Dwf+Dw3+F1nqQQEjLOZkYUmf2fomm0wFfvQ6f/AdUFVh/s+CH1t97e/yEo82q7D9/Gk7stkYBX/AAnHcLfPQvsOMlSJsFN/2h/8+MlG00Eah+caKumZUbj7Jy0xHK61sZnxzF3Rdlcv30VMKCz3L0b/E26+g/7w2rYh4zH2b/DYy/yppg61zUn3A7a9hhDVyKiHer4N0r+1HWpYlnor0VirZYSeHQx9Z3wVhnDpmXWN8l6zLr7KGjcnc6Ye/b8PF/QPl+6yxnwY+s7+vtBNCVMZD/EXzxGyjY4HpR4OK/g/lPDMiIV9V/NBGoc9Lc5uC3nxxk+ScHaXU4uXzicO6el8mFWWc5+re9Ffa8ZbVLF+dalzaefzvMvheSJvR7/AOmsdKqMA/+Hxz8GGpcfRmx6VZCSJ5qnfUcz4OkiVYn8MRrB8cVNsXbrGQ9YZF3mq3UOdNEoM7aF/nl/PCtPArKG1icPZJHrxhP5tmO/q09Bluft65QaThhdfTOXmZdvtgfHby+xBhrCoOOpFCwAVrrrLOD+T+AqTee+xmPUmfAayOLRWQh8BsgEPiDMeapLtuHAS8B6a5YfmmMed7OmFTfVNS38G/r9/LmtmJGJ0Tw0j1zmDcuse9v4GiHsn3WEX9RLhRvtZ4bA+O+AXOWwZjLBsfR8NkQsa7LT8iyznQcbda0CQnj7J/iQakzZNsvUkQCgWeAK4EiYIuIrDXG7HEr9iCwxxhzrYgkAftFZKUxpocZoJSdjDG8trWIf1+/l4aWdr5/2VgeXDC29z6AmuJTK/2SHdZUB2DN4JiaA5Ovt64Aih9j99fwPYHB1sRnSvkgOw9NZgP5xphDACKyClgMuCcCA0SL1dAcBVQCHiZIUQMh/0QdP1iTx+aCSmZlxPHvN5zHOE+jgFvqrA7Yjkq/eKt17TtYE2ulTLOu9knLsa6Gce8sVUr5HDsTQSpQ6Pa8CJjTpcz/AGuBEiAauM2YjsnJTxKRZcAygPR0/5wUyk7NbQ5++3E+z356kIiQIH5203ncMnOUNQ9QW7O1cEbHdfvFW62rbzqm2Y0fAxkXuyr9HEiZal1mqZQaNOxMBJ4OAbv2TF8F7AAuA7KAD0TkM2PMKZOLG2OeA54Dq7O4/0P1X5/nl/MjV2fwLecn8sMcJ7FV78Ofd0DJTijbe3JgVXi8dYQ/ebFV6afO6Ns1/kopn2ZnIigCRrk9T8M68ne3FHjKWJcu5YtIATAR2GxjXAqoqKrmT2+vp/zAZv4+/CiXjigh6sAB2O9W6Y/MtqYZ6BilO2yUNvEoNQTZmQi2AONEJBMoBpYAd3QpcxS4HPhMRJKBCcAhG2PyX23NsPNlTFEuNQe3MKwun0dxQjCYkAQkPhumXm3N56OVvlJ+xbZEYIxpF5GHgPexLh9dYYzZLSL3ubYvB/4FeEFEvsJqSnrcGFNuV0x+q7IAVn8bSndRFzCMnW2jqYi5lQsuvpwREy9AhqVppa+UH7P1gmZjzHpgfZfXlrs9LgG+YWcMfm//e7BmGa0Ow/fb/46NQXP4wXWTTnYGK6X8no5sGaoc7fDxv8Fff0VZ1ARurLmP9KzJfLRkOolRelWPUuokTQRDUf0JeP1uOPwZW+Kv5a6Sm1iUncHPbz6fkKAhOpJXKXXWNBEMNUc3wmvfxTRV8ULSP/DTwuncd2kW/++qCdoUpJTySBPBUGEMbPwtfPBjHDFp/H30L3irKIGfXjeF71yY4e3olFI+TBPBUNBcC2sfgj1v0zhmIbcd/xb7awJ59s5sFk4d4e3olFI+ThPBYHd8D6z+FlQWcGz2D7h263TanPDy93LIydBRv0qp3mkiGMx2vgrrHoWQKLZf9iJ3fhBMXEQQq+6exdjhupC4Uqpv9BKSwai9BdY9BmuWwcjp/PmCV7nl3UAyEiJZ88CFmgSUUmdEzwgGm6oj8Np3oGQ75sKHeSbgDn657hDzxiby7F0ziA7TdWSVUmdGE8FgcuADePNecDpw3PInfrQ/k1c2H+KG6an87KZpOkZAKXVWtOYYLLb8AVbeAjGpNC/9iL/JHcErm49y//wsfnWrDhRTSp09PSMYDDY+C+89AeMXUrFoOfe8vIedRdX88+IpfPuCDG9Hp5Qa5DQR+Lq//ho+fBImXceRBU/z3f/dSUl1E8/eOZOFU1O8HZ1SagjQRODLPv25NXHc1Js4dPGvuPW5LbQ5DCu/N0fHCCil+o0mAl9kjJUANvwCzr+d5quf5oFnN+JwGt64/wK9PFQp1a80EfgaY+CDH8MXT8OMb8M1v+Ena/LYV1rH80t1oJhSqv9pIvAlxsB7/wibnoVZ34NFv2DNzhJWbSnk/vlZLJgw3NsRKqWGIE0EvsLphPV/B7krYO6DcNW/kV/WwA/X5DErI46/u3K8tyNUSg1Rmgh8gdMBf34Ytr8E8x6Dy5+kqc3Jgyu3ERYcyH/fPoOgQB0noJSyhyYCb3O0w9sPwK5X4dInYP4TIMKTa/PYf7yOF++eTcqwMG9HqZQawjQReJOjzZoyYvcauOyf4JK/B+CNrUWszi3iwQVZXDo+yctBKqWGOlvbG0RkoYjsF5F8EXnCw/Z/EJEdrlueiDhExD8ukG9vhde+ayWBb/xrZxI4cLyOH72Vx5zMeB67QvsFlFL2sy0RiEgg8AywCJgM3C4ik93LGGN+YYzJNsZkA/8IfGqMqbQrJp/R1gyv3gX71sGin8OF3wegsbWdB1ZuIyIkkKdvn679AkqpAWFnTTMbyDfGHDLGtAKrgMU9lL8deMXGeHxDayOsuh0OvA/X/Brm/E3nph+/vZv8snp+vSSb5BjtF1BKDQw7E0EqUOj2vMj12mlEJAJYCLzRzfZlIpIrIrllZWX9HuiAaW2Al2+Fgx/D4mcgZ2nnptdyC3l9axHfXzCWi8dpv4BSauDYmQjEw2umm7LXAp931yxkjHnOGJNjjMlJShqklWRLPbx0Exz5HG58Dqbf1bnp6+N1/NPbecwdE88j2i+glBpgdl41VASMcnueBpR0U3YJQ7lZyOmA1++Gws1w8wqYckPnpo5+gajQIJ5eMp3AAE/5Uyml7GPnGcEWYJyIZIpICFZlv7ZrIREZBlwKvG1jLN71/g+sPoGrf35KEjDG8KO38jhYVs9vlkxnuPYLKKW8wLYzAmNMu4g8BLwPBAIrjDG7ReQ+1/blrqI3AH8xxjTYFYtXbXoONi23po2Y9b1TNr22tYg3txXzyOXjuGhsopcCVEr5OzGmu2Z735STk2Nyc3O9HUbffP0XeOU2GL8QbnsJAgI7N+0vrWPxM39lRnocf7pnjjYJKaVsJSJbjTE5nrbphep2Kc2D15dC8lS48fenJIGGlnYeWLmVqNBgfr0kW5OAUsqrNBHYoa4UXr4NQqPhjlchNKpzU0e/QEF5A08vyWZ4tPYLKKW8S+ca6m+tjfDKEmiqhLvfg5iRp2xenVvImu3FPHbFeC7UfgGllA/QRNCfnE5YswxKdsCSl2HE+ads3nuslh+/vZt5YxN56LKx3olRKaW60ETQnz76Kez9M1z1HzDx6lM2NbS08+DKbcSEB/Nft2m/gFLKd/TaRyAikSIS4PY8wDUlhHK39UX4/NeQcw/Mvf+0za9sPsqh8gZ+c1s2SdGhAx+fUkp1oy+dxR8B7hV/BPChPeEMUoc+gXf+FrIus2YTlVOP9p1Ow8pNR5k5Ok77BZRSPqcviSDMGFPf8cT1WM8IOpTth1e/DQnj4JYXIPD01rYvD1VQUN7AXXPTBz4+pZTqRV8SQYOIzOh4IiIzgSb7QhpEGsqt2USDQuDO1RA2zGOxlzYeIS4imEVTRwxwgEop1bu+dBY/CrwmIh0Txo0AbrMtosGirRlW3WGNGfjuOxDr+Wj/eG0zf9lznHvmZRIWHOixjFJKeVOvicAYs0VEJgITsKaW3meMabM9Ml9mDLz9IBRuspqD0jyO2gZg1eZCHE7DHbO1WUgp5Zv6ctXQg0CkMSbPGPMVECUiD9gfmg/75CnIex0u//Eps4l21e5w8srmo1w8LpGMxMgBDFAppfquL30E9xpjqjueGGOqgHtti8jX7VoNnz4F2XfBvL/tsej/7TtBaW0zd80dPUDBKaXUmetLIggQOXk9pGtR+hD7QvJhR76wmoQyLoZr/uu0y0S7emnTUVJiwrh84vABClAppc5cXxLB+8BqEblcRC7DWknsXXvD8kE1xbDqTqtT+NY/WlcK9eBIRQMbvi5jyexRBAXq3H5KKd/Vl6uGHgeWAfdjdRZvx7pyyL/sfNmaSO6ev0BEfK/FX950lMAAYcks7SRWSvm2Xg9VjTFOYCNwCMgBLgf22hyX79n3DqTNgsRxvRZtbnOwOreQKyclkzJMp5lWSvm2bs8IRGQ81jrDtwMVwKsAxpgFAxOaD6kpgpLtcMVP+lT8vbxSqhrbtJNYKTUo9NQ0tA/4DLjWGJMPICKPDUhUvma/q0tk4jV9Kv7SxiNkJERwYVaCjUEppVT/6Klp6CagFPhYRH4vIpdj9RH4n33rIHF8n5qF9pXWknukijvnjCZAp5pWSg0C3SYCY8waY8xtwETgE+AxIFlEnhWRbwxQfN7XVAWH/woTru69LNbZQEhQADfPTLM5MKWU6h996SxuMMasNMZcA6QBO4An+vLmIrJQRPaLSL6IePwbEZkvIjtEZLeIfHomwQ+IAx+As71PzUL1Le2s2VbMNdNGEBfpn0MtlFKDzxmtUGaMqQR+57r1yDXw7BngSqAI2CIia40xe9zKxAK/BRYaY46KiO+NvNq3DqKSIXVmr0Xf3lFMQ6tDO4mVUoOKnSOdZgP5xphDxphWYBWwuEuZO4A3jTFHAYwxJ2yM58y1NcOBD61moYCed5Uxhpc2HmXSiBimj4odmPiUUqof2JkIUoFCt+dFrtfcjQfiROQTEdkqIt/29EYiskxEckUkt6yszKZwPSj4FNoa+tQstO1oNXuP1XLX3HSkl6knlFLKl9iZCDzVhqbL8yBgJvBN4Crgn1zjF079I2OeM8bkGGNykpKS+j/S7uxbByHRkHlxr0VXbjxCVGgQ12d3zXVKKeXbzqiP4AwVAaPcnqcBJR7KlBtjGrBWQtsAnA98bWNcfeN0WOMHxl0JQT0vNl/V0Mq6r45xW84oIkPt3KVKKdX/7Dwj2AKME5FMEQnBGqW8tkuZt4GLRSRIRCKAOfjK9BVFudBQBhO/2WvR17cW0dru5E5dk1gpNQjZdvhqjGkXkYewZi8NBFYYY3aLyH2u7cuNMXtF5D1gF+AE/mCMybMrpjOybx0EBFtnBD1wOg0rNx0hZ3QcE1NiBig4pZTqP7a2Yxhj1gPru7y2vMvzXwC/sDOOM2aMlQgyL+l2QfoOnx8s53BFI49ecVrXhlJKDQo6Ub4nZfuh8lCfmoVe2niE+MgQFp2XMgCBKaVU/9NE4Mm+ddZ9L9NKlNY08+HeE9ySk0ZoUOAABKaUUv1PE4En+96B1ByI6Xn9nVVbjuJwGu6crSOJlVKDlyaCrmqKoWRbr81C7Q4nqzYXcsn4JNITIgYoOKWU6n+aCLra7+rb7mU08Yd7T1Ba28xdc/SSUaXU4KaJoKt970DCWEjq+SqglZuOMGJYGJdN9L158pRS6kxoInDXVA2HP+u1WaigvIHPDpRz++x0ggJ1FyqlBjetxdz1ce2BVzYfJTBAWDJrVI/llFJqMNBE4G7fOogcbl0x1I3mNgev5RbyjcnJDI8JG8DglFLKHpoIOrS3QP6HMLHntQfWf3WMqsY2XXxGKTVkaCLoULABWut7bRZ6aeMRxiRGcmFWwgAFppRS9tJE0GHfOgiJsuYX6saeklq2Ha3mjjm6+IxSaujQRADgdMK+9b2uPbBy0xFCgwK4eWbaAAanlFL20kQAUJwLDSd6bBaqb2nnre3FXDNtJLERIQMYnFJK2UsTAfRp7YEP9xynodXBHXP0klGl1NCiicAY2LvOWpe4h7UHNh6qIDosiOxRcQMYnFJK2U8TQfnXUHmw19HEmwsqmZ0RT2CAdhIrpYYWTQR9WHvgRG0zh8obmJ0ZP0BBKaXUwNFEsO8dSJ0JMSO7LbL5cCUAc8bo2AGl1NDj34mgtgSKt/a6EtmmQ5VEhAQydaQuTq+UGnr8OxH0ce2BTQUVzBwdpzONKqWGJFtrNhFZKCL7RSRfRJ7wsH2+iNSIyA7X7cd2xnOafeshPguSJnRbpLKhla+P1zNXm4WUUkNUkF1vLCKBwDPAlUARsEVE1hpj9nQp+pkxpudDcjs011jzC829H3qYLmJzgat/QDuKlVJDlJ1nBLOBfGPMIWNMK7AKWGzj552ZAx+As63XZqHNBZWEBgVwXlr3YwyUUmowszMRpAKFbs+LXK91dYGI7BSRd0Vkiqc3EpFlIpIrIrllZWX9E92+d6y1B9K6X3sArP6BGelxhAYF9s/nKqWUj7EzEXhqbzFdnm8DRhtjzgf+G3jL0xsZY54zxuQYY3KSkpLOPbL2FuuMYMIiCOi+gq9pamPPsVodP6CUGtLsTARFgPvEPGlAiXsBY0ytMabe9Xg9ECwiiTbGZCn4DFrrem0W2nqkEmNgzhhNBEqpocvORLAFGCcimSISAiwB1roXEJEUcU3sLyKzXfFU2BiTpQ9rD4A1fiA4UJiRrvMLKaWGLtuuGjLGtIvIQ8D7QCCwwhizW0Tuc21fDtwM3C8i7UATsMQY07X5qH85ndb4gbFXQHDPaw5vKqjk/LRYwoK1f0ApNXTZlgigs7lnfZfXlrs9/h/gf+yM4TTFW6H+eK/NQg0t7XxVXMN9l44ZoMCUUso7/G+o7L51EBDU49oDAFuPVOFwGmZn6kAypdTQ5oeJ4B3IuBjCY3sstrmgksAAYeZo7R9QSg1t/pUIyr6GigO9rj0A1viBqanDiAq1tfVMKaW8zr8Swf53rPteZhttbnOws7BGp5VQSvkF/0oE+96BkdNhmKcBzidtP1pNq8OpiUAp5Rf8JxHUlULRlj43C4lAToYmAqXU0Oc/iSD/Q+u+l8tGweoonpQSw7DwYJuDUkop7/OfntDsO2HE+ZA0scdire1Oth2t4vbZ6QMUmFJKeZf/JAIRSDmv12K7iqppbtP+AaWU//CfpqE+2uRaiEYHkiml/IUmgi42FVQyPjmK+MgQb4eilFIDQhOBm3aHk62HK3X9AaWUX9FE4GZ3SS0NrQ7maLOQUsqPaCJws6nAWgpBO4qVUv5EE4GbzQWVZCZGMjym53UKlFJqKNFE4OJwGjYXVOrZgFLK72gicNlfWkdtc7t2FCul/I4mApfO/oEx2lGslPIvmghcNh2qJDU2nNTYcG+HopRSA0oTAWCMYfPhSuaM0WYhpZT/0UQA5J+op7Khlbk6fkAp5YdsTQQislBE9otIvog80UO5WSLiEJGb7YynOyfnF9IzAqWU/7EtEYhIIPAMsAiYDNwuIpO7Kfcz4H27YunNpoJKkmNCGZ0Q4a0QlFLKa+w8I5gN5BtjDhljWoFVwGIP5b4PvAGcsDGWbhlj2HSogtmZCYiIN0JQSimvsjMRpAKFbs+LXK91EpFU4AZgeU9vJCLLRCRXRHLLysr6NcgjFY2cqGvRgWRKKb9lZyLwdHhtujz/NfC4McbR0xsZY54zxuQYY3KSkpL6Kz7g5PiBuXrFkFLKT9m5QlkRMMrteRpQ0qVMDrDK1SSTCFwtIu3GmLdsjOsUmwoqSYgMISspaqA+UimlfIqdiWALME5EMoFiYAlwh3sBY0xmx2MReQFYN5BJAKyBZLMz47V/QCnlt2xrGjLGtAMPYV0NtBdYbYzZLSL3ich9dn3umSiqaqS4ukkvG1VK+TVbF683xqwH1nd5zWPHsDHmu3bG4slm1/gBXYhGKeXP/Hpk8aZDlcSEBTExJdrboSillNf4dSLY7FqfOCBA+weUUv7LbxPBidpmCsobtFlIKeX3/DYRbOzoH9DxA0opP+e3iWBzQQVRoUFMHhHj7VCUUsqr/DYRbDpUyczRcQQF+u0uUEopwE8TQUV9CwdO1Ov4AaWUwk8TwZbDVv+Azi+klFJ+mgg2HqokLDiA81JjvR2KUkp5nV8mgs0FlcxIjyMkyC+/vlJKncLvasKaxjb2ltbq+AGllHLxu0Sw5XAlxuj6xEop1cHvEsHmw5WEBAYwPT3W26EopZRP8LtEsOlQBdmjYgkLDvR2KEop5RP8KhHUt7STV1KrzUJKKeXGrxLB1iNVOJxG5xdSSik3fpUINh2qIDBAmJEe5+1QlFLKZ/hVIthcUMl5qcOIDLV1YTallBpU/CYRNLU62FlUrc1CSinVhd8kgu2FVbQ5DHO0o1gppU7hN4kgJDCABROSyMnQRKCUUu5sTQQislBE9otIvog84WH7YhHZJSI7RCRXRObZFUtORjzPL51NTFiwXR+hlFKDkm29piISCDwDXAkUAVtEZK0xZo9bsY+AtcYYIyLTgNXARLtiUkopdTo7zwhmA/nGmEPGmFZgFbDYvYAxpt4YY1xPIwGDUkqpAWVnIkgFCt2eF7leO4WI3CAi+4B3gLs9vZGILHM1HeWWlZXZEqxSSvkrOxOBeHjttCN+Y8waY8xE4HrgXzy9kTHmOWNMjjEmJykpqX+jVEopP2dnIigCRrk9TwNKuitsjNkAZIlIoo0xKaWU6sLORLAFGCcimSISAiwB1roXEJGxIiKuxzOAEKDCxpiUUkp1YdtVQ8aYdhF5CHgfCARWGGN2i8h9ru3LgZuAb4tIG9AE3ObWeayUUmoAyGCrd3Nyckxubq63w1BKqUFFRLYaY3I8bhtsiUBEyoAjZ/nniUB5P4bT33w9PvD9GDW+c6PxnRtfjm+0Mcbj1TaDLhGcCxHJ7S4j+gJfjw98P0aN79xofOfG1+Prjt/MNaSUUsozTQRKKeXn/C0RPOftAHrh6/GB78eo8Z0bje/c+Hp8HvlVH4FSSqnT+dsZgVJKqS6GZCLowzoIIiJPu7bvco1qHqjYRonIxyKyV0R2i8gjHsrMF5Ea1zoNO0TkxwMVn+vzD4vIVx3rRHjY7s39N8Ftv+wQkVoRebRLmQHffyKyQkROiEie22vxIvKBiBxw3cd187c9/l5tjO8XIrLP9W+4RkRiu/nbHn8PNsb3ExEpdvt3vLqbv/XW/nvVLbbDIrKjm7+1ff+dM2PMkLphjWI+CIzBmrJiJzC5S5mrgXexJsabC2wawPhGADNcj6OBrz3ENx9Y58V9eBhI7GG71/afh3/rUqzro726/4BLgBlAnttrPweecD1+AvhZN9+hx9+rjfF9AwhyPf6Zp/j68nuwMb6fAH/fh9+AV/Zfl+3/CfzYW/vvXG9D8Yyg13UQXM//aCwbgVgRGTEQwRljjhljtrke1wF78TA9t4/z2v7r4nLgoDHmbAcY9htjTZpY2eXlxcCLrscvYs2w21Vffq+2xGeM+Ysxpt31dCPWxJBe0c3+6wuv7b8OrvnSbgVe6e/PHShDMRH0ZR2EPq2VYDcRyQCmA5s8bL5ARHaKyLsiMmVgI8MAfxGRrSKyzMN2n9h/WBMZdvefz5v7r0OyMeYYWAcAwHAPZXxlX96NdZbnSW+/Bzs95Gq6WtFN05ov7L+LgePGmAPdbPfm/uuToZgI+rIOQp/WSrCTiEQBbwCPGmNqu2zehtXccT7w38BbAxkbcJExZgawCHhQRC7pst0X9l8IcB3wmofN3t5/Z8IX9uUPgXZgZTdFevs92OVZIAvIBo5hNb905fX9B9xOz2cD3tp/fTYUE0Ff1kE4o7US+puIBGMlgZXGmDe7bjfG1Bpj6l2P1wPBMoDrNBhjSlz3J4A1WKff7ry6/1wWAduMMce7bvD2/nNzvKPJzHV/wkMZb/8WvwNcA9xpXA3aXfXh92ALY8xxY4zDGOMEft/N53p7/wUBNwKvdlfGW/vvTAzFRNDrOgiu5992Xf0yF6jpOIW3m6s98X+BvcaYX3VTJsVVDhGZjfXvNCDrNIhIpIhEdzzG6lDM61LMa/vPTbdHYd7cf12sBb7jevwd4G0PZfrye7WFiCwEHgeuM8Y0dlOmL78Hu+Jz73e6oZvP9dr+c7kC2GeMKfK00Zv774x4u7fajhvWVS1fY11N8EPXa/cB97keC/CMa/tXQM4AxjYP69R1F7DDdbu6S3wPAbuxroDYCFw4gPGNcX3uTlcMPrX/XJ8fgVWxD3N7zav7DyspHQPasI5S7wESgI+AA677eFfZkcD6nn6vAxRfPlb7esfvcHnX+Lr7PQxQfH9y/b52YVXuI3xp/7lef6Hjd+dWdsD337nedGSxUkr5uaHYNKSUUuoMaCJQSik/p4lAKaX8nCYCpZTyc5oIlFLKz2kiUGoAiTUz6jpvx6GUO00ESinl5zQRKOWBiNwlIptdc8j/TkQCRaReRP5TRLaJyEcikuQqmy0iG93m9Y9zvT5WRD50TX63TUSyXG8fJSKvi7UWwMqOUdBKeYsmAqW6EJFJwG1Yk4VlAw7gTiASa36jGcCnwJOuP/kj8LgxZhrWSNiO11cCzxhr8rsLsUamgjXj7KPAZKyRpxfZ/JWU6lGQtwNQygddDswEtrgO1sOxJoxzcnJysZeAN0VkGBBrjPnU9fqLwGuu+WVSjTFrAIwxzQCu99tsXHPTuFa1ygD+avu3UqobmgiUOp0ALxpj/vGUF0X+qUu5nuZn6am5p8XtsQP9f6i8TJuGlDrdR8DNIjIcOtceHo31/+VmV5k7gL8aY2qAKhG52PX6t4BPjbXGRJGIXO96j1ARiRjIL6FUX+mRiFJdGGP2iMiPsFaVCsCacfJBoAGYIiJbgRqsfgSwpphe7qroDwFLXa9/C/idiPyz6z1uGcCvoVSf6eyjSvWRiNQbY6K8HYdS/U2bhpRSys/pGYFSSvk5PSNQSik/p4lAKaX8nCYCpZTyc5oIlFLKz2kiUEopP6eJQCml/Nz/B/rqYusd4O+SAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", - "plt.plot(l_train_loss, label='train')\n", - "plt.plot(l_valid_loss, label='valid')\n", + "plt.plot(res[0], label='train')\n", + "plt.plot(res[2], label='valid')\n", "plt.xlabel('epoch')\n", + "plt.ylabel('Loss')\n", "plt.legend(loc='best')\n", "plt.savefig('fig-res-vgg-train-validate-loss.pdf')\n", "plt.show()\n", "\n", - "plt.plot(l_train_acc, label='train')\n", - "plt.plot(l_valid_acc, label='valid')\n", + "plt.plot(res[1], label='train')\n", + "plt.plot(res[3], label='valid')\n", "plt.xlabel('epoch')\n", + "plt.ylabel('Acc')\n", "plt.legend(loc='best')\n", "plt.savefig('fig-res-vgg-train-validate-acc.pdf')\n", "plt.show()" ] }, { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# save raw data\n", + "import numpy\n", + "numpy.save('fig-res-vgg_data.npy', res)" + ] + }, + { "cell_type": "markdown", "metadata": {}, "source": [ - "可以看到，跑完 20 次，VGG 能在 CIFAR10 上取得 76% 左右的测试准确率" + "可以看到，跑完 20 次，VGG 能在 CIFAR10 上取得 86% 左右的测试准确率" ] }, { @@ -475,7 +531,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.8.12" } }, "nbformat": 4, diff --git a/7_deep_learning/1_CNN/05-googlenet.ipynb b/7_deep_learning/1_CNN/05-googlenet.ipynb index 4e8f873..c39fb9e 100644 --- a/7_deep_learning/1_CNN/05-googlenet.ipynb +++ b/7_deep_learning/1_CNN/05-googlenet.ipynb @@ -37,35 +37,31 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:51:05.427292Z", "start_time": "2017-12-22T12:51:04.924747Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ - "import sys\n", - "sys.path.append('..')\n", - " \n", "import numpy as np\n", "import torch\n", "from torch import nn\n", "from torch.autograd import Variable\n", - "from torchvision.datasets import CIFAR10" + "from torchvision.datasets import CIFAR10\n", + "from torchvision import transforms as tfs" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:51:08.890890Z", "start_time": "2017-12-22T12:51:08.876313Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -81,19 +77,18 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:51:09.671474Z", "start_time": "2017-12-22T12:51:09.587337Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "class Inception(nn.Module):\n", " def __init__(self, in_channel, out1_1, out2_1, out2_3, out3_1, out3_5, out4_1):\n", - " super(inception, self).__init__()\n", + " super(Inception, self).__init__()\n", " # 第一条线路\n", " self.branch1x1 = Conv_ReLU(in_channel, out1_1, 1)\n", " \n", @@ -126,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:51:10.948630Z", @@ -167,19 +162,18 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:51:13.149380Z", "start_time": "2017-12-22T12:51:12.934110Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ - "class GoogleNet(nn.Module):\n", + "class GoogLeNet(nn.Module):\n", " def __init__(self, in_channel, num_classes, verbose=False):\n", - " super(GoogleNet, self).__init__()\n", + " super(GoogLeNet, self).__init__()\n", " self.verbose = verbose\n", " \n", " self.block1 = nn.Sequential(\n", @@ -239,7 +233,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:51:13.614936Z", @@ -261,7 +255,7 @@ } ], "source": [ - "test_net = GoogleNet(3, 10, True)\n", + "test_net = GoogLeNet(3, 10, True)\n", "test_x = Variable(torch.zeros(1, 3, 96, 96))\n", "test_y = test_net(test_x)\n", "print('output: {}'.format(test_y.shape))" @@ -276,75 +270,131 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:51:16.387778Z", "start_time": "2017-12-22T12:51:15.121350Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "from utils import train\n", "\n", "def data_tf(x):\n", - " x = x.resize((96, 96), 2) # 将图片放大到 96 x 96\n", - " x = np.array(x, dtype='float32') / 255\n", - " x = (x - 0.5) / 0.5 # 标准化，这个技巧之后会讲到\n", - " x = x.transpose((2, 0, 1)) # 将 channel 放到第一维，只是 pytorch 要求的输入方式\n", - " x = torch.from_numpy(x)\n", + " im_aug = tfs.Compose([\n", + " tfs.Resize(96),\n", + " tfs.ToTensor(),\n", + " tfs.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n", + " ])\n", + " x = im_aug(x)\n", " return x\n", " \n", - "train_set = CIFAR10('../../data', train=True, transform=data_tf)\n", + "train_set = CIFAR10('../../data', train=True, transform=data_tf)\n", "train_data = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)\n", - "test_set = CIFAR10('../../data', train=False, transform=data_tf)\n", - "test_data = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)\n", + "test_set = CIFAR10('../../data', train=False, transform=data_tf)\n", + "test_data = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)\n", "\n", - "net = GoogleNet(3, 10)\n", - "optimizer = torch.optim.SGD(net.parameters(), lr=0.01)\n", + "net = GoogLeNet(3, 10)\n", + "optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)\n", "criterion = nn.CrossEntropyLoss()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:17:25.310685Z", "start_time": "2017-12-22T12:51:16.389607Z" - } + }, + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 0. Train Loss: 1.504840, Train Acc: 0.452605, Valid Loss: 1.372426, Valid Acc: 0.514339, Time 00:01:25\n", - "Epoch 1. Train Loss: 1.046663, Train Acc: 0.630734, Valid Loss: 1.147823, Valid Acc: 0.606309, Time 00:01:02\n", - "Epoch 2. Train Loss: 0.833869, Train Acc: 0.710618, Valid Loss: 1.017181, Valid Acc: 0.644284, Time 00:00:54\n", - "Epoch 3. Train Loss: 0.688739, Train Acc: 0.760670, Valid Loss: 0.847099, Valid Acc: 0.712520, Time 00:00:58\n", - "Epoch 4. Train Loss: 0.576516, Train Acc: 0.801111, Valid Loss: 0.850494, Valid Acc: 0.706487, Time 00:01:01\n", - "Epoch 5. Train Loss: 0.483854, Train Acc: 0.832241, Valid Loss: 0.802392, Valid Acc: 0.726958, Time 00:01:08\n", - "Epoch 6. Train Loss: 0.410416, Train Acc: 0.857657, Valid Loss: 0.865246, Valid Acc: 0.721618, Time 00:01:23\n", - "Epoch 7. Train Loss: 0.346010, Train Acc: 0.881813, Valid Loss: 0.850472, Valid Acc: 0.729430, Time 00:01:28\n", - "Epoch 8. Train Loss: 0.289854, Train Acc: 0.900815, Valid Loss: 1.313582, Valid Acc: 0.650712, Time 00:01:22\n", - "Epoch 9. Train Loss: 0.239552, Train Acc: 0.918378, Valid Loss: 0.970173, Valid Acc: 0.726661, Time 00:01:30\n", - "Epoch 10. Train Loss: 0.212439, Train Acc: 0.927270, Valid Loss: 1.188284, Valid Acc: 0.665843, Time 00:01:29\n", - "Epoch 11. Train Loss: 0.175206, Train Acc: 0.939758, Valid Loss: 0.736437, Valid Acc: 0.790051, Time 00:01:29\n", - "Epoch 12. Train Loss: 0.140491, Train Acc: 0.952366, Valid Loss: 0.878171, Valid Acc: 0.764241, Time 00:01:14\n", - "Epoch 13. Train Loss: 0.127249, Train Acc: 0.956981, Valid Loss: 1.159881, Valid Acc: 0.731309, Time 00:01:00\n", - "Epoch 14. Train Loss: 0.108748, Train Acc: 0.962836, Valid Loss: 1.234320, Valid Acc: 0.716377, Time 00:01:23\n", - "Epoch 15. Train Loss: 0.091655, Train Acc: 0.969030, Valid Loss: 0.822575, Valid Acc: 0.790348, Time 00:01:28\n", - "Epoch 16. Train Loss: 0.086218, Train Acc: 0.970309, Valid Loss: 0.943607, Valid Acc: 0.767306, Time 00:01:24\n", - "Epoch 17. Train Loss: 0.069979, Train Acc: 0.976822, Valid Loss: 1.038973, Valid Acc: 0.755340, Time 00:01:22\n", - "Epoch 18. Train Loss: 0.066750, Train Acc: 0.977322, Valid Loss: 0.838827, Valid Acc: 0.801226, Time 00:01:23\n", - "Epoch 19. Train Loss: 0.052757, Train Acc: 0.982577, Valid Loss: 0.876127, Valid Acc: 0.796479, Time 00:01:25\n" + "[ 0] Train:(L=1.329815, Acc=0.523318), Valid:(L=1.289094, Acc=0.566555), Time 00:01:15\n", + "[ 1] Train:(L=0.868416, Acc=0.699808), Valid:(L=0.834760, Acc=0.715190), Time 00:01:15\n", + "[ 2] Train:(L=0.661615, Acc=0.772998), Valid:(L=0.681946, Acc=0.765131), Time 00:01:15\n", + "[ 3] Train:(L=0.538752, Acc=0.817315), Valid:(L=0.604022, Acc=0.794699), Time 00:01:15\n", + "[ 4] Train:(L=0.443314, Acc=0.850264), Valid:(L=0.628162, Acc=0.788370), Time 00:01:15\n", + "[ 5] Train:(L=0.377100, Acc=0.872462), Valid:(L=0.527649, Acc=0.825752), Time 00:01:14\n", + "[ 6] Train:(L=0.310084, Acc=0.894981), Valid:(L=0.520545, Acc=0.833267), Time 00:01:15\n", + "[ 7] Train:(L=0.263667, Acc=0.908628), Valid:(L=0.530805, Acc=0.839399), Time 00:01:14\n", + "[ 8] Train:(L=0.214284, Acc=0.925831), Valid:(L=0.492261, Acc=0.850672), Time 00:01:14\n", + "[ 9] Train:(L=0.178758, Acc=0.938679), Valid:(L=0.543371, Acc=0.843948), Time 00:01:14\n", + "[10] Train:(L=0.154360, Acc=0.945213), Valid:(L=0.560078, Acc=0.839794), Time 00:01:14\n", + "[11] Train:(L=0.127252, Acc=0.957121), Valid:(L=0.607742, Acc=0.833267), Time 00:01:14\n", + "[12] Train:(L=0.122219, Acc=0.957980), Valid:(L=0.579313, Acc=0.842959), Time 00:01:14\n", + "[13] Train:(L=0.100576, Acc=0.964734), Valid:(L=0.551588, Acc=0.856507), Time 00:01:14\n", + "[14] Train:(L=0.085722, Acc=0.969969), Valid:(L=0.571536, Acc=0.851266), Time 00:01:14\n", + "[15] Train:(L=0.078888, Acc=0.972746), Valid:(L=0.649491, Acc=0.847409), Time 00:01:14\n", + "[16] Train:(L=0.079078, Acc=0.973026), Valid:(L=0.681464, Acc=0.840487), Time 00:01:14\n", + "[17] Train:(L=0.069273, Acc=0.976582), Valid:(L=0.615183, Acc=0.848991), Time 00:01:15\n", + "[18] Train:(L=0.062320, Acc=0.978780), Valid:(L=0.618147, Acc=0.858584), Time 00:01:16\n", + "[19] Train:(L=0.060656, Acc=0.979220), Valid:(L=0.613905, Acc=0.857002), Time 00:01:17\n" ] } ], "source": [ - "train(net, train_data, test_data, 20, optimizer, criterion)" + "res = train(net, train_data, test_data, 20, optimizer, criterion)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAyBUlEQVR4nO3deXxU9bn48c+TfV/IAgkJhE32hCUEBBEsVkEFrEXFulcvUrXV2v6q7b231fZ20VZt0SpCi6LiCirWulRQRARkBwFZQ0hCWJKQBLKRZb6/P84EQkhCgJycZOZ5v17zmplzzpx5chjOc77rEWMMSimlvJeP0wEopZRyliYCpZTycpoIlFLKy2kiUEopL6eJQCmlvJyf0wGcq9jYWJOSkuJ0GEop1aGsX7++wBgT19i6DpcIUlJSWLdundNhKKVUhyIi+5tap1VDSinl5TQRKKWUl9NEoJRSXq7DtREopdS5qq6uJjc3l8rKSqdDsV1QUBBJSUn4+/u3+DOaCJRSHi83N5fw8HBSUlIQEafDsY0xhsLCQnJzc+nRo0eLP6dVQ0opj1dZWUlMTIxHJwEAESEmJuacSz6aCJRSXsHTk0Cd8/k7vSYR7Dx0nN//ezvlVTVOh6KUUu2K1ySC3KJy5n65jy25JU6HopTyMsXFxTz33HPn/LmrrrqK4uLi1g+oAa9JBEO7RQOwIbvI4UiUUt6mqURQW1vb7Oc+/PBDoqKibIrqFK/pNdQpNIAesaFs2F/sdChKKS/zyCOPsHfvXoYMGYK/vz9hYWEkJCSwadMmtm/fzrXXXktOTg6VlZU88MADzJgxAzg1pU5paSmTJk3ikksuYeXKlXTt2pXFixcTHBzcKvF5TSIAGNotii925mOM8ZqGI6XU6R771za25x1r1X0OSIzgN5MHNrn+T3/6E1u3bmXTpk0sW7aMq6++mq1bt57s4jlv3jw6depERUUFI0aM4Pvf/z4xMTGn7WP37t28/vrrzJ07lxtuuIFFixZxyy23tEr8XlM1BDCsWzSFZVVkHy13OhSllBfLyMg4rZ//rFmzSEtLY9SoUeTk5LB79+4zPtOjRw+GDBkCwPDhw8nKymq1eLynRFBbzejgbAQXG7KL6B4T6nRESikHNHfl3lZCQ0+df5YtW8aSJUtYtWoVISEhjB8/vtFxAIGBgSdf+/r6UlFR0WrxeE+JYMtb9Hz3GgYHHNJ2AqVUmwoPD+f48eONrispKSE6OpqQkBB27NjB6tWr2zg6byoRJI8EYEqnHN7N7udwMEopbxITE8OYMWMYNGgQwcHBdO7c+eS6iRMnMnv2bFJTU+nbty+jRo1q8/i8JxHE9ILgTozy38sfD4yivKqGkADv+fOVUs567bXXGl0eGBjIRx991Oi6unaA2NhYtm7denL5z3/+81aNzXuqhkQgeSQ9KrdT6zJsztGBZUopBd6UCACSMwg9nkk0x3RgmVJKuXlZIrDaCSZF57JRE4FSSgHelggSh4KPHxNCs9iQXYwxxumIlFLKcd6VCAJCoEsqg1w7OFpWxf5CHVimlFLelQgAkkcSd2wrftSwfr9WDymllBcmggx8aioZHnhAG4yVUu1SWFgYAHl5eUybNq3RbcaPH8+6deta5fu8MBFYDcZXR2WzIbvY2ViUUqoZiYmJLFy40PbvsS0RiMg8ETkiIlubWH+ziGxxP1aKSJpdsZwmsitEJJHht4edh45RekLvWKaUstfDDz982v0IHn30UR577DEmTJjAsGHDGDx4MIsXLz7jc1lZWQwaNAiAiooKpk+fTmpqKjfeeGOrzjVk59Dal4BngZebWL8PGGeMKRKRScAcYKSN8ZySnEHKvlW4DGzJKWZ079g2+VqlVDvw0SNw6JvW3WeXwTDpT02unj59Og8++CD33nsvAG+99RYff/wxP/3pT4mIiKCgoIBRo0YxZcqUJqfIf/755wkJCWHLli1s2bKFYcOGtVr4tpUIjDHLgaPNrF9pjKmrpF8NJNkVyxmSRxJUfpAECrWdQCllu6FDh3LkyBHy8vLYvHkz0dHRJCQk8Ktf/YrU1FQuv/xyDhw4wOHDh5vcx/Lly0/efyA1NZXU1NRWi6+9TLZzF9D4ZBuAiMwAZgB069btwr8tOQOASVHZbMjuf+H7U0p1HM1cudtp2rRpLFy4kEOHDjF9+nQWLFhAfn4+69evx9/fn5SUlEann67PrhtqOd5YLCKXYSWCh5vaxhgzxxiTboxJj4uLu/Av7TIY/IK5LHQfG7OLdGCZUsp206dP54033mDhwoVMmzaNkpIS4uPj8ff35/PPP2f//v3Nfv7SSy9lwYIFAGzdupUtW7a0WmyOJgIRSQX+AUw1xhS22Rf7+kPX4Qyo2UFReTX7Csra7KuVUt5p4MCBHD9+nK5du5KQkMDNN9/MunXrSE9PZ8GCBfTr1/z0+D/60Y8oLS0lNTWVJ554goyMjFaLzbGqIRHpBrwD3GqM2dXmASRn0ClnFkGcYEN2MT3jwto8BKWUd/nmm1ON1LGxsaxatarR7UpLSwHr5vV1008HBwfzxhtv2BKXnd1HXwdWAX1FJFdE7hKRmSIy073Jr4EY4DkR2SQirTMyoqWSRyKuGkYF7dcGY6WUV7OtRGCMueks6+8G7rbr+88qaQQAkyKzeVGnmlBKeTHHG4sdExoDMX1I99nNrsPHdWCZUh7OWzqFnM/f6b2JACB5JMnlW3EZw+acYqejUUrZJCgoiMLCQo9PBsYYCgsLCQoKOqfPtZdxBM5IHkHApldJkUNs2F/EGB1hrJRHSkpKIjc3l/z8fKdDsV1QUBBJSec2PtfLE4E1o8VVkdnaYKyUB/P396dHjx5Oh9FueXfVUGxfCIxkXPA+NuboHcuUUt7JuxOBjw8kj6BvzXaKy6vJ1IFlSikv5N2JACB5JJHH9xJBmd6xTCnllTQRJGcgGEYHWfMOKaWUt9FE0HU4iA9XRmSzYX+x09EopVSb00QQGA6dBzLcZxe7jhznWGW10xEppVSb0kQAkDySxNJtiHHpwDKllNfRRACQPBK/mjL6+eRo9ZBSyutoIoBTdyyL0IFlSinvo4kAIKo7hHXmkqC9bMwuwuXSgWVKKe+hiQBABJIz6FO1nWOVNWQWlDodkVJKtRlNBHWSRxJWnkscxdpOoJTyKpoI6rgnoLskKFPbCZRSXkUTQZ2ENPAN4IoIvXWlUsq7aCKo4xcIiUNJYye7j5TqwDKllNfQRFBfcgZdSnfgb6rZlF3sdDRKKdUmNBHUlzwSH1cVA32ytHpIKeU1NBHUl1R/YFmxs7EopVQbsS0RiMg8ETkiIlubWC8iMktE9ojIFhEZZlcsLRbeGaJTuDhQB5YppbyHnSWCl4CJzayfBPRxP2YAz9sYS8slj6R35VaOV1azN18HlimlPJ9ticAYsxw42swmU4GXjWU1ECUiCXbF02LJGQSfKCBJCvSOZUopr+BkG0FXIKfe+1z3Mme5B5aNDdqrDcZKKa/gZCKQRpY1WikvIjNEZJ2IrMvPz7c3qvgBEBDG5WH7tcFYKeUVnEwEuUByvfdJQF5jGxpj5hhj0o0x6XFxcfZG5eMLSekMNjvYc6SUknIdWKaU8mxOJoL3gdvcvYdGASXGmIMOxnNKUgZxZbsJoZKNOVo9pJTybHZ2H30dWAX0FZFcEblLRGaKyEz3Jh8CmcAeYC5wr12xnLPkkYhxMdRnr1YPKaU8np9dOzbG3HSW9Qa4z67vvyBJ6QBcEbGfJdpgrJTycDqyuDHBURDXn1H+e9iUXawDy5RSHk0TQVOSM+hRuZ3SE1XsPqIDy5RSnksTQVOSRxJQfYxekqfjCZRSHk0TQVPcA8suDcpkg44wVkp5ME0ETYnpBcGd+E7oPi0RKKU8miaCpohA8kgG1u5gb34ZxeVVTkeklFK20ETQnOQMoiv2E8VxNuYUOx2NUkrZQhNBc9ztBMN997BR2wmUUh5KE0FzEoeCjx9XhOsEdEopz6WJoDkBIdAllRF+u9mUU0ytDixTSnkgTQRnkzySbhU7qDxRye4jx52ORimlWp0mgrNJzsCvtoJ+kq13LFNKeSRNBGdz2sCyYmdjUUopG2giOJvIrhCRxLiQfWzUgWVKKQ+kiaAlkjMYUPMtmQVlHC3TgWVKKc+iiaAlkkcSfuIQXSjkjbXZTkejlFKtShNBSyRnAHBr0mHmLs+k7ESNwwEppVTr0UTQEl0Gg18w13fOo6i8mpdX7Xc6IqWUajWaCFrC1x+6Die+eDPjLopj7pdaKlBKeQ5NBC2VnAEHN/Pg+GSOllXx6motFSilPIMmgpbqMRZcNQw9tIixfWKZszyT8iotFSilOj5NBC3V8zK4aBJ89jseHuFDoZYKlFIeQhNBS4nA5L+BfzCD1vySS3tHa6lAKeURbE0EIjJRRHaKyB4ReaSR9ZEi8i8R2Swi20TkTjvjuWDhneGqv0DuWv7QeRkFpVUsWK3jCpRSHZttiUBEfIG/A5OAAcBNIjKgwWb3AduNMWnAeOBJEQmwK6ZWMej70H8ySZue5obupbywfC8VVbVOR6WUUufNzhJBBrDHGJNpjKkC3gCmNtjGAOEiIkAYcBRo33UtInD10xAYzq9rnqWotIIFX2tbgVKq47IzEXQFcuq9z3Uvq+9ZoD+QB3wDPGCMcTXckYjMEJF1IrIuPz/frnhbLiwOrn6SsMIt/CF+KbO/yNRSgVKqw7IzEUgjyxre4utKYBOQCAwBnhWRiDM+ZMwcY0y6MSY9Li6uteM8PwO/BwOv4/rSBcSW7ea1NdpWoJTqmOxMBLlAcr33SVhX/vXdCbxjLHuAfUA/G2NqXVf9BZ/gKGaHzmXusp1UVmupQCnV8diZCNYCfUSkh7sBeDrwfoNtsoEJACLSGegLZNoYU+sKjYHJfyOlZi83VrzFa19rqUAp1fHYlgiMMTXA/cAnwLfAW8aYbSIyU0Rmujf7HTBaRL4BlgIPG2MK7IrJFv2uhtQb+bH/e3y+7FMtFSilOhwxpmG1ffuWnp5u1q1b53QYp6soompWBnvLAln73UXcNrav0xEppdRpRGS9MSa9sXU6srg1BEcT8L1n6e+TQ83nj2upQCnVoWgiaC0XXcnhXtO4rfZdliz5yOlolFKqxVqUCEQkVER83K8vEpEpIuJvb2gdT/y0v1Ds24mBXz9MZUWZ0+EopVSLtLREsBwIEpGuWI26dwIv2RVURyXB0Rwe/2d6kMueN3/ldDhKKdUiLU0EYowpB64DnjHGfA9r/iDVwICx3+M/wZPon/UyVVmrnA5HKaXOqsWJQEQuBm4G/u1e5mdPSB2biBA++U8cNJ2oeOseqCp3OiSllGpWSxPBg8AvgXfdYwF6Ap/bFlUHN6p/d+Z2+hmR5fupWfpbp8NRSqlmtSgRGGO+MMZMMcY87m40LjDG/MTm2DosEWHCpOuZX/NdfL+eDftXOh2SUko1qaW9hl4TkQgRCQW2AztF5P/ZG1rHNrZPLJ90mUke8Zj37oUq7UWklGqfWlo1NMAYcwy4FvgQ6AbcaldQnkBEuOeKNH56YgYUZcGSR50OSSmlGtXSRODvHjdwLbDYGFPNmVNKqwYu7RNLVdLFvOV7DayZA/uWOx2SUkqdoaWJ4AUgCwgFlotId+CYXUF5ChHhgcv78Juy6zgW0h0W3wfHDzsdllLexRg49A2ses56Vmc470nnRMTPPcNom2qXk841wxjDtc+tpHPJFl4wv0UCwmDaP6HHpU6HppTnqq2B7FWw49/Wo8Q9Rbz4wpgHYNzD4B/kbIxt7IInnRORSBF5qu52kSLyJFbpQJ2FiPDghD7851g3Phq1AIKj4eWp8MWfwXXGXTmVUuerqgy2vw/vzoS/9Ib518C6eRDfHybPgvvXQdpNsOIpmD0Gsr5yOuJ2o0UlAhFZBGwF5rsX3QqkGWOuszG2RnW0EgFYpYKpf/+Ko2VVfPbjdAI++hl88xb0mgDXzYHQWKdDVKpjKs2HXR/Bjg8h83OoqYSgKLhoIvS7yvo/Fhh2+mf2fgb/egCKsyH9h3D5YxB0xh1yPU5zJYKWJoJNxpghZ1vWFjpiIgBYtvMId7y4lptHduP/pg5ENr4MH/4CQmLg+heh2yinQ1SqYyjce6rKJ+drwEBksnWTqL5XQffR4HuWOTGryuDzP8Dq5yCsC1zzFPSd1CbhO6W5RNDSaSIqROQSY8wK9w7HABWtFaA3GN83npnjejH7i730iQ/jjjF3QOIwePt2ePEquPxRGP1jEHE6VKXan+pKWDkLti6C/B3Wsi6Drbr+fldbr8/l/05AKFz5exh4Hbz/Y3h9Ogz8Hkx6AsLi7fkb2rGWlgjSgJeBSPeiIuB2Y8wWG2NrVEctEQC4XIZ7Xl3P0m8PM++OEYzvGw+VJdYPcftiuGgSXPschHRyOlSl2o+j+6wLpoObIWXsqSv/6O6ts/+aKvjqb7D8CXeC+COkTfe4i7ILrhqqt6MIAGPMMRF50Bjz19YJseU6ciIAKDtRw7TZq8g9Ws47946mT+dwq3vbmjnwyX9DeAJc/xIkDXc6VKWc9+0H8N69IMC1z1tJwC75O62Lspyvodd34Jq/tl6yaQda7VaVxphj7hHGAA9dcGReKDTQj3/cnk6gvy93zV/H0bIq68pj5D3ww0+sjeZdCV+/YCWI1lJdYY1w7mD3qFZeqrbaujB682aI6Qn3LLc3CQDE9YU7P4ZJf4acNfDcxbD6eXB5/q1nL2QcQY4xJrmV4zmrjl4iqLMhu4jpc1YzJDmKV+8aSYCfOyeXH4X3fgS7PoYBU2HKMxAU2fzOGlNWCDmrrb7U2ashbxO4qmH0T+C7v/W4Yq/yIMU5sPBOyF0LGTPgiv8Dv8C2j+GDn8KeT6FrOkx91uqG2oG1WtVQg51mG2O6XVBk58FTEgHA4k0HeOCNTdyQnsTj309F6k7OLhesegaWPGYVTa+fDwmpTe/IGOtqP3vVqRN/wS5rnW8AdB1u9Uo6fhg2vwaj7oUr/6DJQLU/uz+Fd/7LGhA2ZRYMavMe6qcYA98shI8fhspjMPYhqztq7QmocT/qvz75vsrqxnradlXWs68/+IdYj4AQ8A8G/1DrOcD9fNr6Bq/P1huqGefda0hEjtP4nEICBLfgiycCfwN8gX8YY/7UyDbjgb8C/ljTW4872349xdQhXdl7pJRZn+2hT3w4/3VpT2uFj481+jEpw7oy+sflcNUTMOx26+RdWwOHt1on/LoTf+kh67NBkZA8yho40+1iSBx6agSlMdb61c9ZP8xJf7a+S3UMxnhu8q6tgWV/gC+fhM6DrIuf2N7OxiQCqddDr8vg41/CF49bjxZ91gf8gqwLMb8g8AuwXtdWW9W01eXuGYnP8UJ8zANWib6VnXeJ4Kw7FvEFdgHfBXKBtcBNxpjt9baJAlYCE40x2SISb4w50tx+PalEAFZPovtf38BHWw/xj9vSmdC/8+kblBVYV0h7P7MGydScsIrMVaXW+shu1tV+t1HWiT+uX/Mnd2Pg019bXfGG3W41iGkyaN8K9lglxM1vQHSKdVXa6zuQMsa6guzojh+ChXfB/hUw7DarC2d7/LvyNlpVt36B4BtoPdc9fBu89m1Bz3xjrP/P1eXuxFB+6vXJ9xVQXXbqdVK6lZjOgy1VQy340ouBR40xV7rf/xLAGPPHetvcCyQaY/6npfv1tEQAUFFVyw0vrCIzv5RF946mX5cGoxxdLutKacVT0KlXvRP/KIhMOvcvNAY++z/48i8w5GarHcLHt3X+GNV6ctfDV3+Fb/9lXU0Ous46ae5faVU7+AZag6d6uxND/ICOV2LI/AIW3WVdHV/9FAy5yemIPJZTiWAa1pX+3e73twIjjTH319vmr1hVQgOBcOBvxpiXG9nXDGAGQLdu3Ybv37/flpiddKikkql/X4Gfjw+L7x9DbFgjjWOtXTWw7HGrOD74erh2dsuuYpS9jIE9S6x+7VlfWlV5I+6GkTNPDXSqKofslbDnM9i79NQAq/AEKyH0+g70vAxCY5z7O87G5bIuRD7/A8ReBDfM7/CNse1da4wsPq/vbWRZw6zjBwwHJmC1OawSkdXGmF2nfciYOcAcsEoENsTquC6RQcy9LZ0bXljFPa+sZ8HdIwnyb3CV3tpXe+Mftk7+S38Lrhq4bu4FNUapC1BbDdvetRLA4a0QnghX/B6G3w6B4advGxACvS+3HgAluVbV4d7PrGkXNi0ABBKHnKpGSs5oP/+29as7B98A1zx95nxAqk3ZmQhygfrdS5OAvEa2KTDGlAFlIrIcSMNqW/A6qUlRPHXDEO5dsIFfvfMNT96QdqonkV3G/syqdvjP/1gno2kvWg1bqm1UlcGGV2DV362pkuP6wdTnrFJaS/8dIpOsuvVht1l93vM2wp6l1ol2xdPWlXdAuDX1ea/LrNJCTC9nqpH2r4KFP4TyQqt9avgdHa86ywPZWTXkh3VCnwAcwGos/oExZlu9bfoDzwJXAgHAGmC6MWZrU/v1xDaChp5ZupsnP93FLyb25d7xbdRz4usX4KNfWA3SN7zc9v22vU1ZgTWafM0cqCiyenpd8iD0ubJ1G+8riq074+1dalUl1c3LH5kMPce7q5HG2zutSU0V5H8LOz+CL56AqG5WVVBCmn3fqc7gSNWQMaZGRO4HPsHqPjrPGLNNRGa61882xnwrIh8DWwAXVhfTJpOAt7j/O73ZfaSUJz7eSc/YMCYO6mL/l468B3z84N8PwRs/gBtfbZ89Nzq6oixY+SxsfBVqKqw5c8Y8YN/ss8FRMGCK9TAGjmZaJYXMZdb8VhtfAcQ6KdeVFrqNOv8LgapyOLwNDm6y5gY6tAUOb7cGMwL0n2INzjqfQZLKNraVCOziDSUCgMrqWqbPWc3OQ8d5e+bFDOraRv9xNrxizbfS41K46Q2rPlqdH5fLugLP32U16OauterwxQdSb7Rmm43v51x8tTVWNVLm57D3c8hdY7UV+QVbvZHqEkPngY1X31QUW7d+rDvhH9xsDWQ07hsuBXeyEkxCmjUgMmEIdOqpVUEOcaTXkF28JREAHDleybXPfoUBFt83hviINrq13uY3rGkuuo2GH7ypDXlnU1sDRfusk33+TvdjBxTstq7664TGQ+oN1sjuyK7OxduUE8etu3bVJYaCndby0Hir+ihljFWlVXfSL8o69dnwxHonfPfJP6KrnvTbEU0EHdj2vGNMm72SPp3DeXPGqDN7Etnlm4XwzgxIGgE3v+0Vd3A6q+pKKNxjnSDz6z0K95yq+gCISLImMKt7xLqfO9r04iUHrCqkzM+t57J8a3l0Sr0r/TTokgZhcQ4GqlpCE0EH98m2Q8x8dT3XpCYya/oQ+3sS1dn2Liy62yrS37LIqm/2RscOWgO71r9kzSEDgFgnxLh+DU76F53Z3dMTuFxW+0JorPf+Djo4p8YRqFZy5cAu/OLKfjz+8Q6C/Hz443WD8fNtg2khBn4PfPzh7Tvg5alw67vOXdUaY82P1Ja9mY7lWd0v18+36s5Tb7RG8cb1hZje3tWY7uPj/Nw/yjaaCDqImeN6Ulldy9+W7qakoppZNw1tm2qi/tfA9AXw5i3w0tXQfzIER1s3CA+Odj+iTi071zEIxlh3aTt+yJo473i9R8P3NRVWu8WQH8DAa+278i45YCWADfOths+0m6zxFp162PN9SjlMq4Y6mHkr9vHbD7YzulcMc25LJyywjXL5niWw+H7rhNzcjIn+oQ0SRNSpJBEYbg0kaniiP1ndUk9AOIR3tqZNCOsM4V2s0sD296FwtzUlb/8pVlJIGds6fe9LcuHLp6wulcZlzcM09iGrCkipDk7bCDzMovW5/GLRFgYlRvDSnRlEh7bhSGBXrXUFX1lsDYSqKLK6EdY9N7rc/ag94T7Bdzn1CHOf7E8uc5/4m+qpZAzkrrOmUdj6DpwosQZHpd1k3Wc2pte5/03FOdaEfhtesd4PvRkuecijblOolCYCD/Tp9sPc99oGunUK4ZW7MkiI7AD11bXVrTvfTXWFe26d16xBUhhrKu4hP4AB1569p1NxtjWr68YF1vtht1oJIKrNb7ynlO00EXioVXsL+a+X1xEZ7M+rd4+kR2yo0yE5p+QAbHnTSgqFu61BUQPqqo4uPb3qqCjLSgCbXrMGdw27DS756flN6a1UB6GJwIN9k1vC7S+uwUdg/g8zGJjo5UP3jYED662qo28WWVVHEUnWPPe9JljLN7/uTgC3uxNAOxzcpVQr00Tg4fYcKeW2f37N8coa/nnHCDJ6dLCBS3aproCdH56qOjIu62Yuw++wJniLSHQ6QqXajCYCL3CguIJb//k1B4oqeP6WYXynX+ezf8ibHMuzZuHsMQ4iEpyORqk211wi0JvVeoiuUcG8fc/F9OkcxoyX17N40wGnQ2pfIhKtXkWaBJQ6gyYCDxITFsjr/zWK4d2jefDNTbyyKsvpkJRSHYAmAg8THuTP/B9mMKFfPP+7eBvPLN1NR6v+U0q1LU0EHijI35fnbxnOdUO78uSnu/jdB9/icmkyUEo1Tuca8lD+vj785fo0IoL9mffVPkoqqnn8+200WZ1SqkPRRODBfHyE30weQHRIAE8v2cWxymqevnFI281PpJTqEPTy0MOJCA9c3ofHpgxkybeHueKpL/h85xGnw1JKtSOaCLzE7aNTWDhzNCGBftz54loefGMjhaUnnA5LKdUOaCLwIsO7R/Pvn1zCAxP68O9vDvLdp5fz3sYD2qtIKS+nicDLBPr58tPvXsQHPx5Lt04hPPjmJu58aS0HiivO/mGllEeyNRGIyEQR2Skie0TkkWa2GyEitSIyzc541Cl9u4Sz6Eej+c3kAazZd5QrnvqC+SuztJupUl7ItkQgIr7A34FJwADgJhEZ0MR2jwOf2BWLapyvj3DnmB588uClDE/pxG/e38a02SvZffi406EppdqQnSWCDGCPMSbTGFMFvAFMbWS7HwOLAO3K4pDkTiHMv3MET92QRmZBGVfPWsHfluymqsbldGhKqTZgZyLoCuTUe5/rXnaSiHQFvgfMbm5HIjJDRNaJyLr8/PxWD1RZ3UyvG5bEkofGMXFQF55esovJz6xgY3aR06EppWxmZyKQRpY1rID+K/CwMaa2uR0ZY+YYY9KNMelxcXGtFZ9qRGxYILNuGso/b0/nWGU11z2/ksf+tY2yEzVOh6aUsomdQ0xzgfo3f00C8hpskw68ISIAscBVIlJjjHnPxrhUC0zo35mMHp144uOdvPhVFv/Zdpg/XDeYcRdpIlbK09hZIlgL9BGRHiISAEwH3q+/gTGmhzEmxRiTAiwE7tUk0H6EB/nzu2sH8fbMiwn09+H2eWv42VubKS6vcjo0pVQrsi0RGGNqgPuxegN9C7xljNkmIjNFZKZd36ta34iUTnz4k7Hcf1lv3tt0gMufWs7HWw86HZZSqpXorSrVOdmWV8IvFm5hW94xJg3qwmNTBxIfHuR0WEqps9BbVapWMzAxkvfuG8MvJvZl6Y4jfPep5Sxan6vTVCjVgWkiUOfM39eHe8f35sOfjKVPfBg/e3szd7yo01Qo1VFpIlDnrXd8GG/dczGPTh7A2ixrmopXVuk0FUp1NJoI1AXx8RHucE9TMax7NP+7eBvT565mX0GZ06EppVpIE4FqFcmdQnj5hxk8MS2VHQePMfGvy3nhi73U1Oo0FUq1d5oIVKsREW5IT2bJQ+MYd1Ecf/xoB9c9v5JvDx5zOjSlVDM0EahWFx8RxAu3DufZHwzlQFEFk59ZwVOf7uJETbMziSilHKKJQNlCRLgmNZFPHxrH5LREZi3dzeRnVvDx1oPUamOyUu2KJgJlq06hATx94xDm3ZFOZbWLma9u4DtPLuPlVVmUV+lEdkq1BzqyWLWZWpfhk22HmPtlJhuzi4kK8eeWkd25bXR3HZ2slM2aG1msiUA5Yv3+o8xZnsl/th/G38eHa4cmcvfYnlzUOdzp0JTySJoIVLu1r6CMeSv28fb6HCqrXYzvG8eMsT25uFcM7unJlVKtQBOBaveOllWxYPV+5q/KoqC0igEJEcy4tCdXpybg76tNWUpdKE0EqsOorK5l8aYDzP1yH3uOlJIQGcSdY1KYntGNiCB/p8NTqsPSRKA6HJfLsGzXEeYu38eqzELCAv2YPiKZu8b2ICEy2OnwlOpwNBGoDm3rgRLmfpnJB1sO4iMwbXgSM8f1ontMqNOhKdVhaCJQHiHnaDkvLN/LW+tyqal1MSUtkXsv6609jZRqAU0EyqMcOVbJ3C8zWfB1NuVVtVw5sDP3X9aHwUmRToemVLuliUB5pKKyKl5cmcVLX+3jWGUNl14Ux33jezGyZ4zToSnV7mgiUB7teGU1r67O5p8rMikorWJESjT3XdabcRfF6VgEpdw0ESivUFFVy5trs3lheSYHSyoZ3DWS+y7rxRUDuuDjowlBeTdNBMqrVNW4eHdjLs8v20tWYTl94sO497JeTE5NxE8Hpykv1VwisPV/hYhMFJGdIrJHRB5pZP3NIrLF/VgpIml2xqO8Q4CfDzeO6MaSh8bxt+lD8BHhp29u5rInlzFr6W69jaZSDdhWIhARX2AX8F0gF1gL3GSM2V5vm9HAt8aYIhGZBDxqjBnZ3H61RKDOlctlWLrjCHO/zGTNvqMADO4ayZS0RK5OTSAxSgeoKc/nSNWQiFyMdWK/0v3+lwDGmD82sX00sNUY07W5/WoiUBcir7iCf285yL+25LEltwSAESnRTElLZNLgBGLDAh2OUCl7OJUIpgETjTF3u9/fCow0xtzfxPY/B/rVbd9g3QxgBkC3bt2G79+/35aYlXfZV1DGB5vzeH9zHruPlOLrI4zuFcPktESuHNiFyGCd20h5DqcSwfXAlQ0SQYYx5seNbHsZ8BxwiTGmsLn9aolAtTZjDDsPH+df7qSQc7SCAF8fxvWNY0paIhP6xxMS4Od0mEpdkOYSgZ2/7lwgud77JCCv4UYikgr8A5h0tiSglB1EhH5dIujXJYKfX9GXzbklvL8pjw+25PHp9sME+/ty+YDOTE5NYHTvWMICNSkoz2JnicAPq7F4AnAAq7H4B8aYbfW26QZ8BtxmjFnZkv1qiUC1lVqXYW3WUd7fnMdH3xykqLwaH4EBiRGMSOlERkon0lM6EReu7Qqq/XNsHIGIXAX8FfAF5hljfi8iMwGMMbNF5B/A94G6Sv+apgKto4lAOaG61sWafUf5OrOQNVlH2ZRTTGW1C4AesaGMSIlmREonRqR0ontMiI5oVu2ODihTqpVV1bjYmlfC2n1HWZt1lLVZRZRUVAMQHx7oTgrRpKd0on9CBL46slk5TBOBUjZzuQx78ktZU5cY9h0lr6QSgPBAP4Z1j2ZESjQX94olLSlSRzirNqeJQCkHHCiuYO2+o6xxJ4bdR0oBKzGM7NmJ0b1iuaRPLH3iw7QqSdnOqV5DSnm1rlHBdB3alWuHWmMkj5ZVsWpvISv2FLBybwFLvj0CQFx4IGN6xTC6dyxjesfSVUc6qzamJQKlHJJztJyVewtYsaeQlXsKKCyrAqzG5zG9YxjTK5aLe8UQFRLgcKTKE2jVkFLtnMtlDWr7ak8BK/cWsjqzkPKqWkRgUGIko3vHMLpXLAmRQYQE+BIW6EdIgB8BftrWoFpGE4FSHUx1rYvNOcVWNdKeQjZkF1HjOvP/aoCvDyGBvoQG+FnJIbAuSfgSGuhHaICf+9mXsCA/LuoczuCkSCKCdPoMb6NtBEp1MP6+PqS7B6w9eDmUnahhU04xReVVlJ2ooexErfVcVfdcQ9mJGsqraik9UcORYycodS8vP1FLVa3rtP33igslLTmKIclRpCVF0S8hnEA/X4f+WuU0TQRKdQChgX6M6R173p+vqnFRUlHNtwePsTmnmM25JSzfVcA7Gw4AVsmif0I4ae7EkJYcRc/YUL2zm5fQqiGlvJQxhoMllWzOKWZTbjGbc4r5JreEsqpawOrmmpoceTIxpCVF0TkiULu6dlBaNaSUOoOIkBgVTGJUMJMGJwDW/Ep780vdpYZiNueUMGd55sn2ifBAP1JiQ0mJDaVHTEi916FEh2rvpo5KSwRKqWZVVtey/eAxtuQUk1lQxr6CMrIKyzhQVEH99uvIYP/TE0TMqSQRGaKN007TEoFS6rwF+fsyrFs0w7pFn7a8qsZF9tFy9heeSg5ZBeWszSpi8eY86l9jRodYSSI2LJDIYP9GHxEN3mvX2LajiUApdV4C/HzoHR9G7/iwM9ZVVteSc7T8ZILYV2AljJyj5WyrqKakovpkW0RTgv19z0gUfj6CwcowdYmmLt+cSjyNrwer1BIfHkhcvUd8eBBx4YFEBPl5bfuHJgKlVKsL8velT+dw+nQOb3Kb6loXx9xJodj9XPe+pNz9XO9xoLgCl7suquH5uu4ELiffN3hGcBnDzkPHyS89QVXN6d1pAQL9fOolh9OTRFxYIPERgUSHBBAR7E94oJ9H9ajSRKCUcoS/rw8xYYHEhLXtjX2MMRyrrCH/eCVHjp8gv96j7n1WQTlr9h2lqLy60X34CGdUZdU9okIaLguwnkP8CfTzwd/HB19fwc9H8PWxnp0uiWgiUEp5FRE5eZLuHd90iQWsdpDCshMcOWYliKLyqjNKKsXu0suBooqTJZvaRkaBN8dHwM/X57Tk4Ovjg79v/ffCTRnduHtszwv58xuliUAppZoQ4OdDQmQwCZEtnxHWGENZVS3F9ZOGO1lU1bqoqTXUugw1LkOty0V1g/fWs3EvP/W+xmWItan0pIlAKaVakYgQFmjN/ZQUffbt2wPtn6WUUl5OE4FSSnk5TQRKKeXlNBEopZSXszURiMhEEdkpIntE5JFG1ouIzHKv3yIiw+yMRyml1JlsSwQi4gv8HZgEDABuEpEBDTabBPRxP2YAz9sVj1JKqcbZWSLIAPYYYzKNMVXAG8DUBttMBV42ltVAlIgk2BiTUkqpBuxMBF2BnHrvc93LznUbRGSGiKwTkXX5+fmtHqhSSnkzOweUNTZ5RsNx1y3ZBmPMHGAOgIjki8j+84wpFig4z8+2hfYeH7T/GDW+C6PxXZj2HF/3plbYmQhygeR675OAvPPY5jTGmLjzDUhE1jV1Y4b2oL3HB+0/Ro3vwmh8F6a9x9cUO6uG1gJ9RKSHiAQA04H3G2zzPnCbu/fQKKDEGHPQxpiUUko1YFuJwBhTIyL3A58AvsA8Y8w2EZnpXj8b+BC4CtgDlAN32hWPUkqpxtk66Zwx5kOsk339ZbPrvTbAfXbG0MCcNvyu89He44P2H6PGd2E0vgvT3uNrVIe7eb1SSqnWpVNMKKWUl9NEoJRSXs4jE0F7nuNIRJJF5HMR+VZEtonIA41sM15ESkRkk/vx67aKz/39WSLyjfu71zWy3snj17fecdkkIsdE5MEG27T58ROReSJyRES21lvWSUQ+FZHd7udGb1Nytt+rjfH9WUR2uP8N3xWRqCY+2+zvwcb4HhWRA/X+Ha9q4rNOHb8368WWJSKbmvis7cfvghljPOqB1UNpL9ATCAA2AwMabHMV8BHWgLZRwNdtGF8CMMz9OhzY1Uh844EPHDyGWUBsM+sdO36N/FsfAro7ffyAS4FhwNZ6y54AHnG/fgR4vIm/odnfq43xXQH4uV8/3lh8Lfk92Bjfo8DPW/AbcOT4NVj/JPBrp47fhT48sUTQruc4MsYcNMZscL8+DnxLI9NqtHPtZY6oCcBeY8z5jjRvNcaY5cDRBounAvPdr+cD1zby0Zb8Xm2JzxjzH2NMjfvtaqwBnY5o4vi1hGPHr46ICHAD8Hprf29b8cRE0GpzHNlNRFKAocDXjay+WEQ2i8hHIjKwbSPDAP8RkfUiMqOR9e3i+GENUmzqP5+Tx69OZ+MeIOl+jm9km/ZyLH+IVcprzNl+D3a63111Na+JqrX2cPzGAoeNMbubWO/k8WsRT0wErTbHkZ1EJAxYBDxojDnWYPUGrOqONOAZ4L22jA0YY4wZhjVN+H0icmmD9e3h+AUAU4C3G1nt9PE7F+3hWP43UAMsaGKTs/0e7PI80AsYAhzEqn5pyPHjB9xE86UBp45fi3liIrBljqPWJCL+WElggTHmnYbrjTHHjDGl7tcfAv4iEttW8Rlj8tzPR4B3sYrf9Tl6/NwmARuMMYcbrnD6+NVzuK7KzP18pJFtnP4t3g5cA9xs3BXaDbXg92ALY8xhY0ytMcYFzG3ie50+fn7AdcCbTW3j1PE7F56YCNr1HEfu+sR/At8aY55qYpsu7u0QkQysf6fCNoovVETC615jNShubbBZe5gjqsmrMCePXwPvA7e7X98OLG5km5b8Xm0hIhOBh4EpxpjyJrZpye/Brvjqtzt9r4nvdez4uV0O7DDG5Da20snjd06cbq2244HVq2UXVm+C/3YvmwnMdL8WrLun7QW+AdLbMLZLsIquW4BN7sdVDeK7H9iG1QNiNTC6DePr6f7eze4Y2tXxc39/CNaJPbLeMkePH1ZSOghUY12l3gXEAEuB3e7nTu5tE4EPm/u9tlF8e7Dq1+t+h7MbxtfU76GN4nvF/fvagnVyT2hPx8+9/KW63129bdv8+F3oQ6eYUEopL+eJVUNKKaXOgSYCpZTycpoIlFLKy2kiUEopL6eJQCmlvJwmAqXakFgzo37gdBxK1aeJQCmlvJwmAqUaISK3iMga9xzyL4iIr4iUisiTIrJBRJaKSJx72yEisrrevP7R7uW9RWSJe/K7DSLSy737MBFZKNa9ABbUjYJWyimaCJRqQET6AzdiTRY2BKgFbgZCseY3GgZ8AfzG/ZGXgYeNMalYI2Hrli8A/m6sye9GY41MBWvG2QeBAVgjT8fY/Ccp1Sw/pwNQqh2aAAwH1rov1oOxJoxzcWpysVeBd0QkEogyxnzhXj4feNs9v0xXY8y7AMaYSgD3/tYY99w07rtapQArbP+rlGqCJgKlziTAfGPML09bKPK/DbZrbn6W5qp7TtR7XYv+P1QO06ohpc60FJgmIvFw8t7D3bH+v0xzb/MDYIUxpgQoEpGx7uW3Al8Y6x4TuSJyrXsfgSIS0pZ/hFItpVciSjVgjNkuIv+DdVcpH6wZJ+8DyoCBIrIeKMFqRwBriunZ7hN9JnCne/mtwAsi8lv3Pq5vwz9DqRbT2UeVaiERKTXGhDkdh1KtTauGlFLKy2mJQCmlvJyWCJRSystpIlBKKS+niUAppbycJgKllPJymgiUUsrL/X9TM38CkKoylAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEJCAYAAACZjSCSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvDklEQVR4nO3deZxcVZnw8d/T+55Ob+kt6U5C9j00IZAAUUQTEAKKGkDF6IggDMI77wzM6KijjqOvziIjEkUZZNgVkIgRECSBBAJZSHc6G9k66eo9nfS+VVed9497O13pdCdN0requu7z/XzqU3erqqduV9/n3nPOPUeMMSillHKvqFAHoJRSKrQ0ESillMtpIlBKKZfTRKCUUi6niUAppVxOE4FSSrmcY4lARB4RkXoRKR9ivYjIAyJyQETKRGShU7EopZQampNXBI8Cy8+wfgUwxX7cBjzkYCxKKaWGEOPUGxtj3hSR4jNsshJ4zFh3tG0WkXQRyTPG1JzpfbOyskxx8ZneViml1EDbtm07ZozJHmydY4lgGAqAyoB5j73stEQgIrdhXTUwYcIEtm7dGpQAlVIqUojIkaHWhbKyWAZZNmh/F8aYXxljSowxJdnZgyY0pZRS5yiUicADjA+YLwSqQxSLUkq5VigTwVrgi3brocVA89nqB5RSSo08x+oIROQpYBmQJSIe4DtALIAxZg2wDrgaOAB0AKvP9bO8Xi8ej4eurq7zDTvsJSQkUFhYSGxsbKhDUUpFCCdbDd10lvUGuHMkPsvj8ZCamkpxcTEig1U9RAZjDI2NjXg8HiZOnBjqcJRSESIi7izu6uoiMzMzopMAgIiQmZnpiisfpVTwREQiACI+CfRxy/dUSgVPKO8jUEqpiOTzG9q6emnp8tLS5aW1q5fWrl68Pj8+v8FvDD6/OTnd6zf47XmfwZq2t+mb9vsNJcUZXD515JvQayIYAU1NTTz55JN8/etf/1Cvu/rqq3nyySdJT093JjCl1DkzxtDW3Ut9azf1Ld00tHXT3OmlpbPvwG499x/o7flOL+09PkdiumPZZE0E4aqpqYlf/OIXpyUCn89HdHT0kK9bt26d06EppQbw+Q2N7f0H94aWbupbu2ho7bYO+q3d9nQXXV7/oO8RGy2kJcSSmhBDqv2cnZVyynxaov0csCwuJopoEaKihGgRoqNOnY62p6OisNZJ4DLnioU1EYyA+++/n4MHDzJ//nxiY2NJSUkhLy+PHTt2sHv3bq6//noqKyvp6uriG9/4BrfddhsAxcXFbN26lba2NlasWMHSpUt5++23KSgo4MUXXyQxMTHE30yp4PD6/DR1eGnq6OFEh5cTHT2caLemrWXWdK/Pj9+A3xiM/Ww9rDP4wHWB8367uOV4Rw+Nbd34B+nDIC0hhpy0BLJT4lkwIZ3slHhy0uLJSU0gOzWe7NR40pNiSUuIJT4mKqLq6yIuEfzLH3exu7plRN9zZn4a37l21pDrf/SjH1FeXs6OHTtYv34911xzDeXl5SebeD7yyCNkZGTQ2dnJRRddxKc//WkyMzNPeY/9+/fz1FNP8fDDD/PZz36W5557js9//vMj+j2UCrbWLi9HGjs4fKydo8c7ONbWTVOHl+PtPacc9Fu7eod8j7iYKMYmxZKeGEd8rHUAjhKIsp9PzkdFIfbyobZZkJROdmo8OfaBPTs14eR0QuzQV++RLuISQThYtGjRKe38H3jgAV544QUAKisr2b9//2mJYOLEicyfPx+ACy+8kIqKimCFq9R5ae70cqSxnYrGDiqOtVPR2M4Re7qxveeUbVPiY0hPiiUjOY70pDiKs5IZmxRnPZJjSU+KY2xSrD1vTSfGRkfU2Xc4irhEcKYz92BJTk4+Ob1+/Xpee+013nnnHZKSkli2bNmg9wHEx8efnI6OjqazszMosSo1HJ09Pj6oa6WisZ3Dx+wDfWM7FcfaOdHhPWXbvDEJFGUm8fFZ4yjKTKY4M4mizGSKMpNIiou4Q05E0L/KCEhNTaW1tXXQdc3NzYwdO5akpCT27t3L5s2bgxydUsNnjKGmuYs9NS32o5U9tS1UHGs/Wa4uAvljEinKTGLFnLyTB/pi+2Dv5iKW0UoTwQjIzMxkyZIlzJ49m8TERMaNG3dy3fLly1mzZg1z585l2rRpLF68OISRKtWvy+tjf10be2pa2G0f+PfWttLc2X+GPz4jkRm5aVw7N58ZealMzk5hfIYe7CONWF3+jB4lJSVm4MA0e/bsYcaMGSGKKPjc9n3V6Yx9E5LX58fba+jx+en19097Ax49vdZ2nV4fBxvarLP8mhYONbSdPMtPjI1mWm4qM/LSmJlnPU/LTSU1QTs3jBQiss0YUzLYOr0iUCqM9fT6eedQIy+X17B+XwMtnV68Putgf64K0hOZkZfKitm5zMhLY0ZeGkUZSY62U1fhTROBUmGmy+tjwwcNvFJey2t76mjp6iU5LporpmWTNyaR2Ogo4qKF2OgoYmOiTp23l50yHx1FXIwQFx3NhMwkxiTqWb46lSYCpcJAa5eXN/Y18HJ5DW/sbaDT62NMYiwfn5XL8lm5LJ2SpeXyyjGaCJQKkRPtPfxlTx2vlNfy1v5j9Pj8ZKXE86mFBayYncfFkzKIjY6YDoJVGNNEoFQQ1bd08cquWl7eVcvmQ8fx+Q0F6Yl84ZIils/OZeGEsURrWb0KMk0ESjmktcvLB3Wt7KlpZV9tKzurmin1NGEMTMpK5muXT2LF7DxmF6TpnbMqpDQRhEBKSgptbW1UV1dz99138/vf//60bZYtW8ZPf/pTSkoGbe2lwojX5+fwsXb21rayr7aFfbXWwb+qqf/u8JT4GKbnpnLvx6ayfHYuU3JS9OCvwoYmghDKz88fNAmo8GSMoa6lm721LfZBv5W9ta0crG872ZwzOkqYnJ3MwqKx3HzxBKbnpjItN5WC9EQ98KuwpYlgBNx3330UFRWdHI/gu9/9LiLCm2++yYkTJ/B6vfzgBz9g5cqVp7yuoqKCT37yk5SXl9PZ2cnq1avZvXs3M2bM0L6GwoTfb3j38HHWllbz6q7aUzpRy01LYFpuKpdPyWJ6XirTxqUxOSeZ+Bht3aNGl8hLBH++H2p3jux75s6BFT8acvWqVau45557TiaCZ599lpdffpl7772XtLQ0jh07xuLFi7nuuuuGPCt86KGHSEpKoqysjLKyMhYuXDiy30ENmzGGMk8za0ureamsmrqWbhJjo/nYzHGUFI1lWm4q03NTSU+KC3WoSo2IyEsEIbBgwQLq6+uprq6moaGBsWPHkpeXx7333subb75JVFQUVVVV1NXVkZubO+h7vPnmm9x9990AzJ07l7lz5wbzKyhgf10ra0ur+WNpNRWNHcRGC1dMzeGb1+TzsRk52nOmiliR98s+w5m7k2688UZ+//vfU1tby6pVq3jiiSdoaGhg27ZtxMbGUlxcPGj304G0DDn4Ko938MeyatbuqGZvbStRApdMzuSOZZNZPiuPMUl6F66KfJGXCEJk1apVfPWrX+XYsWNs2LCBZ599lpycHGJjY3njjTc4cuTIGV9/+eWX88QTT/CRj3yE8vJyysrKghS5+zS0dvOnsmrWllaz/WgTAAsmpPOda2dyzdw8clITQhugUkGmiWCEzJo1i9bWVgoKCsjLy+OWW27h2muvpaSkhPnz5zN9+vQzvv6OO+5g9erVzJ07l/nz57No0aIgRe4OXV4ffyyt5sUd1bx98Bh+A9NzU/n7T0zjunn5jM9ICnWISoWMdkM9Crnt+56P5k4vj28+wv9sOsyxth4mZCRx3bx8rpufz9RxqaEOT6mg0W6olevUNnfxm42HePLdo7T3+Lh8aja3XzGJSyZlal2MUgNoIlAR5UB9G7968yAvvF+Fz2/45Nx8vnbFJGbljwl1aEqFrYhJBMYYV5zpjbaivGDZfvQEa9Yf5C976oiLjuKmRRP46mWTtOxfqWGIiESQkJBAY2MjmZmRfdlvjKGxsZGEBG3VAtb+WP9BA2vWH+Tdw8cZkxjLXR+5gFsvLSYrJT7U4Sk1akREIigsLMTj8dDQ0BDqUByXkJBAYWFhqMMIqV6fnz/trGHNhkPsqWkhb0wC37pmBjctmkByfET8pJUKqoj4r4mNjWXixImhDkM5rLPHx7NbK3n4rUN4TnQyJSeFn35mHtfNyycuRgdwUepcRUQiUJHLGMOu6hae2+7hD+9XcaLDy4VFY/nOtbO4cnqODriu1AjQRKDCUn1LF3/YUcVz26rYV9dKXHQUH5uZw+olE7moOCPU4SkVURxNBCKyHPgZEA382hjzowHrxwKPAJOBLuDLxphyJ2NS4avL6+OVXbU8v72Kt/Y34DdW1w8/uH42187N135/lHKIY4lARKKBB4GrAA+wRUTWGmN2B2z2T8AOY8wNIjLd3v5Kp2JS4ccYw5aKEzy/3cOfympo7e6lID2ROz9yATcsKGBSdkqoQ1Qq4jl5RbAIOGCMOQQgIk8DK4HARDAT+DcAY8xeESkWkXHGmDoH41Jh4GhjB8+/7+H57VUcPd5Bclw0K+bk8emFhVw8MUPL/pUKIicTQQFQGTDvAS4esE0p8Clgo4gsAoqAQuCURCAitwG3AUyYMMGpeJXDWrq8rCur4fntVbxXcRwRWDI5i3uvmsInZuVqf/9KhYiT/3mDndINvC32R8DPRGQHsBN4H+g97UXG/Ar4FVidzo1smMpp3b0+Hn7zEA++cZBOr4/J2cn8w/Jp3LCggLwxiaEOTynXczIReIDxAfOFQHXgBsaYFmA1gFi3BB+2HypCbNx/jG+/WM6hY+1cPSeXr10+mbmFYyL6DnClRhsnE8EWYIqITASqgFXAzYEbiEg60GGM6QH+BnjTTg5qlKtr6eL7L+3mpbIaijOT+O2XF3HF1OxQh6WUGoRjicAY0ysidwGvYDUffcQYs0tEbrfXrwFmAI+JiA+rEvkrTsWjgqPX5+exd47wH3/5gB6fn3s/NpWvXTGJhNjoUIemlBqCo7Vzxph1wLoBy9YETL8DTHEyBhU8246c4Ft/KGdPTQtXTM3meytnUZSZHOqwlFJnoc001Hk70d7Dj1/ey9NbKskbk8Cazy/kE7NytR5AqVFCE4E6Z36/4XfbKvnRn/fS2tXL1y6fxN1XTtEeQJUaZfQ/Vp2T3dUtfOsPO9l+tIlFxRl8//rZTMvVMYCVGo00EagPpbXLy3/+ZT+Pvn2YsUlx/Ptn5vGphQVaDKTUKKaJQA2LMYaXymr4/ku7aWjr5paLJ/D3H5+uHcEpFQE0EaizOlDfyrdf3MXbBxuZUzCGh79Ywrzx6aEOSyk1QjQRqCG1d/fywF/385u3DpMUF833V87i5ouLiNYO4ZSKKJoI1GmMMazbWcv3X9pNbUsXny0p5L7l08nUAeGVikiaCNQpDtS38d21u9h44Bgz89J48JaFXFg0NtRhKaUcpIlAAVYx0H//9QC/2XiIxNhovrdyFrdoMZBSrqCJwOWMMfy53CoGqmnu4jMXFnLfiulkaTGQUiPHGDh+CHq7IS4JYpOt55hEiIoKdXSaCNzsYINVDPTWfqsY6Oc3L+DCIh0YXg3C1wue9yAuGTImQbzePHhWTUfh0AY4/Kb1aKsdfLvYJOsRmCCGWjbxcrhg5Efz1UTgQh09VjHQr986REJsNP9y3SxuuXgCMdGhPzNRYcQYqC2D0mdg57PQ3tC/LjkbMiZbSSFjEmRO6p9OGBO6mEOprQEO9x34N8CJCmt5crZ1AC++DBLToacDvB3Q0z7guQO87eDttKbbak/fNipaE4E6P8YYXraLgaqbu7jxQqs1UHaqFgOpAC011oG/9Gmo3w1RsTBtOcy+EUSsIo7Gg3D8MBxaD6VPnvr6pMwBSWIyZEy0phMjqOFBVzNUbOo/8Nfbw7HHp0HxUrj4DisB5Myw9ttIMM4M0KiJwCXqW7v4u2dLeWv/MabnpvLATQsoKXZBMZAx1j/ovnVw5G1IzYfc2TBuFoybDUku2AfD0dMOe/8EpU9ZB3fjh8JFcM1/wKwbzryfetqts9+TCeKQ9ah4C8qePnXbuBRIzYXUPEjLt6fzIS3Pek7NtR7RYXjHurcTjm7uP/BXv2/tp5hEmLAY5nwGJl0BufMg2qFDq0NduYhxKMM4paSkxGzdujXUYYwqBxvauPWR92hs6+Eflk/jC4uLIrsYyOeFI5tg35+tBNB01FqeM9Mq3ggs4kgr6E8K42ZB7hzrbNapf+Rw4vfDkY3Wmf/uF6GnDdInwNxVMG+VdSZ/vryd1pVDX3JoqYbWamitta48WmvA7x3wIrGKUwKTQ1q+lTyyp1l/x/iU84/tbNrqofJd6+Bf+S5U77BijYqBghLrbH/SFVB4EcSE/1W1iGwzxpQMts4Fv3Z32370BF95dAtRIjx92+LI7RqiswkOvGYd/Pf/BbqbISYBJi2Dy/4Opi63DigArXVQV24/dkFtORz8K/h7rfUxCZA93b5ymN2fJCLl6qHhA+vMv+xZaPFYRRmzP2UlgAmXjGwrlthEGDfTegzG74eORishtNbYiaLWShYtNdDssSqpOxoDXiRWksqdYz/mWs8p4879jNnvh2P7+g/6RzfDCXv49Oh4yF8Al3zdKuefsDjiKsv1iiCC/WV3HX/71HbGpSXw2JcXRd5oYSeO9J/1H9lkHciTsqzy7GlXW0kgbpjfubcbjn1gJYW+JFFbDh3H+rdJzYexRdZVxJgCSCu0zlT7ppOzHLt0Py/GQOMBOPA6lD0D1dtB7ErHeausfRWbGOooz6y3G1qqoH4v1O60KrFrd0LTkf5tkrNPTw6ZF1gVrAP1dEDVNuugX/kuVL4HXU3WuqQsGH8xTLgYxi+G/Pmj4oz/bM50RaCJIEI9+e5RvvWHncwuGMMjX7ooMu4L8Puh5n3r4L93HdTvspZnTYNpK6wDWmHJ4P/456q1Dup2WlcOdbutM9QWj3Xm6us5ddvoOCsxpBXayaGg/zmtAMYUBu+qoqXabrq4wXpurbaW586BeTdZFb+p44ITi5M6m+yrup39CaJhb//fJsa+IsmdA9kzrMRxdLO1Xd8VYNa0/oP+hMVWpXY4JvTzpInARYwx/Odr+3ng9f0sm5bNgzcvHJ0jhnU1Q8M+65+6fq/1XLsT2utBomDCpfbBf8XIlGV/WH6/dbXQUgXNVfaznSD6lrVW9x9s+iRnW2Xc42bZzzOtA1Rc0vnF03kCKjZaB/1D66Fxv7U8MaO/LHviFaHZV8HW22Nf3e089eqhq8kq9stf2H/gH78ocor8zkITgUv0+vx884VyntlayWdLCvnXG+YQG+6Vwp1N9gF/j/Vcbz/3ncGC9c+bNdU6cE7+KEy5anT88/p9VsV0c5V1FdFUaX3Put3W9+zttDcU6yx03EzImWWXqc+GscVDX914O+HoO/1n/TWlVguW2CQoutQqFpt4hfU+YXDnasgZA211VmKMiQt1NCGhlcUu0NHTy51PbOeNfQ3c/dELuPeqqeE1alhPu1XmXr/71AN/a03/NjGJVquQiZfbrUNmWM/pRSNb3BMsUdH9zSG58NR1fp/V5LJul7VP6sqtBLHnJcA+OYtJhJzp/ckhY7J1Znt4g1Wu7euxWrAUXgSX/4N11l9Q4toD3RmJ9DcWUKfRK4IIcKytm688uoWdVc384Po53HzxhNAG5O20DvrV7/c/ju2zzljBOmvNnma1zOl75EyHMRP07LWnwyoGO5kgdlmPwErrcXP6i3qKLg1OU0o16ukVQQQ70tjOrY+8R21LF7/8QglXzQxyBWBvj1Vp23fAr3rfOoAZn7U+Odsqk5250mp9kTMTxozXA/5Q4pKgYKH1CNRWb7X8yZpqtU5SagRpIhjFyjxNrP6fLfiN4Ym/Wez8uAE+r3W2GnimX7erv4VG4lirvfXUe6zn/AVWa5lwKqIarVJyrIdSDtBEMEq9sa+eO5/YTkZyHL/98iImZztcPPDBq/DcV6C7xZqPT7PO8Bff0X/QTy/Sg75So5AmglHod1sruf/5nUwbl8qjqy8iJy3B2Q/c/xo8c4tVln/p3VaxxdiJWryjVITQRDCKGGN48I0D/PTVD1h6QRYPfX4hqQkOd8514HV4+marcveLL46OZptKqQ9FE8Eo4fMbvrO2nMc3H+WGBQX8+NNziYtx+Iz84BtWEsiaCl9cq0lAqQiliWCU+Ld1e3h881G+dsUk7vvEdKKcHkv40AZ46iar7bpeCSgV0TQRjALPbDnKrzce5kuXFvOPK2Y4/4GH34InP2fd2XrrWkjOdP4zlVIho7V9Ye7dQ4186w/lXDYli29dE4QkULEJnvys1S/9rWu1zbpSLqCJIIwdbezg9se3MT4jiZ/fvND5wWSOvANPfMbqJfPWP2q7daVcQhNBmGrt8vKV327Bb+CRWy9iTKLDrYOOvgtP3GiNCnXrHyOji2Kl1LA4mghEZLmI7BORAyJy/yDrx4jIH0WkVER2ichqJ+MZLXx+w98+9T6Hj7Xz0OcXUpzl8IAylVvg8U9bIzzd+pJ2zqWUyziWCEQkGngQWAHMBG4SkYHj1d0J7DbGzAOWAf8uIq7vOvGH6/awfl8D/7JyFpdOdriM3rMNHv+UVRfwpZesKwKllKs4eUWwCDhgjDlkjOkBngZWDtjGAKli9ZecAhwHBozk4S7PbDnKb+wWQrdcXOTsh1Vth/+9wWoa+qWXrNG1lFKu42QiKAAqA+Y99rJAPwdmANXATuAbxvT1VdxPRG4Tka0isrWhocGpeENus91C6PKp2c63EKreAf97PSSOsYqDxhQ6+3lKqbDlZCIY7I6ngYMffALYAeQD84Gfi0jaaS8y5lfGmBJjTEl2dvZIxxkWjjZ2cMfj25iQkcR/37Tg9BZCHcdh8xoof94autHnPfcPqymDx1ZaHcfd+hKkjz+/4JVSo5qTN5R5gMAjTCHWmX+g1cCPjDU6zgEROQxMB95zMK6w02K3EDLAbwZrIVRbbnX6dqKif1lULGRNsUbxyplh9fOfMwPSi8/cGVxtuZUE4lKs4qCxDhc/KaXCnpOJYAswRUQmAlXAKuDmAdscBa4E3hKRccA04JCDMYUdn99wt91C6LGvLDq9hVD58/DindbZ+5fWQXyqPaC7Pe6tZwuUP9e/fd9wj32Joe85Ld/a/rHrIDYRvvRH685hpZTrOZYIjDG9InIX8AoQDTxijNklIrfb69cA3wceFZGdWEVJ9xljjg35phGor4XQv94w+9QWQn4fvP4vsOlnMP5i+Oxj/c068+ae+ibdrfbA77utYqP63XDwr1D6ZP828WnWAN7xKdZ9AhmTnP9ySqlRwdG+howx64B1A5atCZiuBj7uZAzh7On3hmgh1HHcGgTm4F+h5Muw/MdnHpA8PhUKS6xHoI7j1lVA39VD53H4yLcgc7IzX0gpNSppp3MhMmQLobpdVtfPLdVw7QNw4a3n/iFJGVC8xHoopdQQNBGEwJHGdu54fBtFmUn8/OaAFkID6wPGXxTaQJVSrqCJIMisFkJbT7YQSkuItesDvgeb/uv0+gCllHKYJoIg6vX5+dsn36cisIVQx3F47m/g4OvDqw9QSqkRpokgiH64bi8bPmjghzfMsVoIjWR9gFJKnSNNBEHy+p46Htl0mNVLirn54glaH6CUChuaCILA7zf89NUPKM5M4p+WT4W/fEfrA5RSYUMTQRC8uruWPTUtPLCyiNinP6f1AUqpsKKJwGF+v+G/XtvPJRntXPveLVofoJQKO2dNBCKSDHT2dQ8tIlFAgjGmw+ngIsHLu2o5UHuC9/J+jnQchy/9CcYvCnVYSil10nC6oX4dSAqYTwJecyacyOL3G3722n5+kPYHMk6UwnUPaBJQSoWd4SSCBGNMW9+MPZ10hu2VbV15DbkNG1nV87xVJzDrhlCHpJRSpxlOImgXkYV9MyJyIdDpXEiRwec3PP7qZn4W/xBm3Cz4xA9DHZJSSg1qOJXF9wC/E5G+QWXygM85FlGEeKm0km80/4SUOC9y46PWGABKKRWGzpoIjDFbRGQ61qAxAuw1xpzHOImRz+c3nPjzD1kZvRv/J38B2VNDHZJSSg3prEVDInInkGyMKTfG7ARSROTrzoc2er39+ot8oftpPBNWErXgllCHo5RSZzScOoKvGmOa+maMMSeArzoW0SjX21LPjLfvpTo6n/ybHwx1OEopdVbDSQRRIiJ9MyISDejtsIPx+zn2+JdJ9bdy9KMPEpWQGuqIlFLqrIaTCF4BnhWRK0Xko8BTwJ+dDWt08r393+TWv8XDyV/lkkuXhTocpZQaluG0GroPuA24A6uy+H2slkMqUOUW5PXvsc63iKlX301UlJz9NUopFQbOekVgdy2xGTgElABXAnscjmt06WzCPPdl6sjgt1n/h6tmaW+iSqnRY8grAhGZCqwCbgIagWcAjDEfCU5oo4QxsPZvMc3V3NH1be66aiEBVSpKKRX2zlQ0tBd4C7jWGHMAQETuDUpUo8mWX8OetTwUcyv+ggu5ckZOqCNSSqkP5UxFQ58GaoE3RORhEbkSq45A9akpg1e+SXX2Un7adhX3fmyqXg0opUadIROBMeYFY8zngOnAeuBeYJyIPCQiHw9SfOGruw1+vxqTlMFXW/6GeeMzWDYtO9RRKaXUhzacyuJ2Y8wTxphPAoXADuB+pwMLe3/6Ozh+iNdm/Cu7muO452NT9GpAKTUqDec+gpOMMceNMb80xnzUqYBGhR1PQtnT9F7293ynNJ0FE9K5YqpeDSilRqcPlQgU0LDPuhoovoynE1dR3dyldQNKqVFNE8GH4e2E362G2CS6rlvDz984TEnRWC6bkhXqyJRS6pxpIvgwXv5HqN8FN/ySZ/b2UtvSxb1X6dWAUmp000QwXEfehm3/A0u+QVfxR/jF+gMsKs7g0smZoY5MKaXOiyaC4frgFYiKhSvu46n3jlLX0s09V2lLIaXU6KeJYLiObIKCC+mSBH6x/iAXT8zg0slaN6CUGv00EQxHdxtUbYfiJTy++QgNrd3ce5UOP6mUigyOJgIRWS4i+0TkgIicdhOaiPy9iOywH+Ui4hORDCdjOieVm8H46C68lDUbDnHp5EwWT9K6AaVUZHAsEdgjmT0IrABmAjeJyMzAbYwxPzHGzDfGzAf+EdhgjDnuVEznrGITRMXwZHUex9r0akApFVmcvCJYBBwwxhwyxvQATwMrz7D9TVijn4Wfio348hbw843VLL0gi4uKw++iRSmlzpWTiaAAqAyY99jLTiMiScBy4Lkh1t8mIltFZGtDQ8OIB3pGPe1QvZ2DyfNpbO/hro9eENzPV0ophzmZCAZrV2mG2PZaYNNQxULGmF8ZY0qMMSXZ2UHu06fyXfD38p5/BgmxUZQUjQ3u5yullMOGM2bxufIA4wPmC4HqIbZdRRgXCyHRrGsqYk5BEjHR2tBKKRVZnDyqbQGmiMhEEYnDOtivHbiRiIwBrgBedDCWc1exCX/+QrbVeplbmB7qaJRSasQ5lgiMMb3AXcArWIPdP2uM2SUit4vI7QGb3gC8aoxpdyqWc9bTDlXbaMwqobvXz7zx6aGOSCmlRpyTRUMYY9YB6wYsWzNg/lHgUSfjOGeV74Hfy87YOQDMKxwT4oCUUmrkaYH3mRzZBBLNG+2TSU+KZUJGUqgjUkqpEaeJ4EwqNkL+fLbU9DCvMF07mFNKRSRNBEPp6QDPVrzjL+WDulYtFlJKRSxNBEPxbAG/l0PJC/AbtKJYKRWxNBEMpWIjSBTveKcAaNNRpVTE0kQwlIqNkDefrbW9FKQnkp0aH+qIlFLKEZoIBuPthKqtULyEUk8T88Zr/YBSKnJpIhiMZwv4emjNXUzl8U4tFlJKRTRNBIOx6wfelxkAzNNEoJSKYJoIBlOxCXLnsr3OhwjM0aajSqkIpolgIG+XVTRUvJQyTzMXZKeQEu9oTxxKKRVSmggGqtoKvm5M8VJKK5v0/gGlVMTTRDBQxUZAqB5jjUimdxQrpSKdJoKBKjZC3lx21FuzekWglIp0mggC9dUPFC2lzNNEXHQU03PTQh2VUko5ShNBoKpt0NsFxUvZUdnEjPw04mJ0FymlIpse5QLZ9QO+8Zews6qZ+Vo/oJRyAU0EgY5shNzZHGyLoaPHp3cUK6VcQRNBn95ua2jK4svYUdkEaEWxUsodNBH0CagfKPM0kRofw6Ss5FBHpZRSjtNE0KdiEyAw4RJKK5uZUziGqCgdmlIpFfk0EfSpeAvGzaYrdgx7alq0fkAp5RqaCAB6e+z6gSXsqWmh12+Yr2MQKKVcQhMBQPV26O0Eu38h0IpipZR7aCIAq1gIoGgJZZ5mslPjyU1LCG1MSikVJJoIwLqRLGcWJGWww9PEvMJ0RLSiWCnlDpoITtYPLKW508uhhnbtcVQp5SqaCKrfB28HFC+lvKoZ0PoBpZS7aCI4stF6Llpy8o7iuXpFoJRyEU0EFRshZyYkZ1LmaaI4M4n0pLhQR6WUUkHj7kTg88LRd6F4KQCllc1aLKSUch13J4LqHeBth6Il1LV0UdvSpXcUK6Vcx92JIOD+gb4byfSOYqWU27g8EWyE7OmQkk2pp4noKGFWviYCpZS7uDcR+LxQ2V8/UOZpZtq4VBJio0McmFJKBZejiUBElovIPhE5ICL3D7HNMhHZISK7RGSDk/GcoqYUetqgeCnGGEorm7SiWCnlSjFOvbGIRAMPAlcBHmCLiKw1xuwO2CYd+AWw3BhzVERynIrnNAH1AxWNHbR09eodxUopV3LyimARcMAYc8gY0wM8DawcsM3NwPPGmKMAxph6B+M5VcUmyJoGKTna46hSytWcTAQFQGXAvMdeFmgqMFZE1ovINhH54mBvJCK3ichWEdna0NBw/pH5euHoOyfrB3ZUNpEYG82UnJTzf2+llBplnEwEg3XfaQbMxwAXAtcAnwD+WUSmnvYiY35ljCkxxpRkZ2eff2Qn6weWAFDmaWJ2QRox0e6tO1dKuZeTRz4PMD5gvhCoHmSbl40x7caYY8CbwDwHY7Kc7F9oKV6fn13VLczTG8mUUi7lZCLYAkwRkYkiEgesAtYO2OZF4DIRiRGRJOBiYI+DMVkqNkLWVEgdx77aVrp7/czV+gGllEs51mrIGNMrIncBrwDRwCPGmF0icru9fo0xZo+IvAyUAX7g18aYcqdiAqz6gSPvwJwbASj1NAEwX68IlFIu5VgiADDGrAPWDVi2ZsD8T4CfOBnHKWrLoKe1/0ayymbGJsUyPiMxaCEopVQ4cV/taIVdP9DX46inibk6NKVSysXclwiObILMCyA1l46eXj6oa9X7B5RSruauROD3wZG3T14NlFe14DfoHcVKKVdzVyKoLYPuFijqG4imCUDHIFBKuZq7EkHFJuvZvpGs1NNEQXoi2anxIQxKKaVCy2WJYCNkTIa0fMBKBPN0IBqllMu5JxGcrB+wrgYa27qpPN6pdxQrpVzPPYmgrhy6m6H4MgDKqpoBrR9QSin3JIITRyAuBYrs+oHKJkRgjrYYUkq5nKN3FoeVmdfBtKsh2vrKZZ5mLshOISXePbtAKaUG454rAjiZBHRoSqWU6ueuRGDznOiksb1HE4FSSuHSRFDmsSqK9Y5ipZRyaSIo9TQRFx3F9Ny0UIeilFIh585EUNnEjPw04mJc+fWVUuoUrjsS+vyGnVXNzNdiIaWUAlyYCA42tNHR49MbyZRSyua6RLDD7nFUWwwppZTFdYmgtLKJ1PgYJmUlhzoUpZQKC65LBGWeZuYUjiEqSoemVEopcFki6PL62FPTosVCSikVwFWJYE9NC71+ozeSKaVUAFclglKtKFZKqdO4KxF4mslJjSc3LSHUoSilVNhwWSJoYm5hOiJaUayUUn1ckwiaO70camhnvo5RrJRSp3BNIijXoSmVUmpQrkkE8TFRXDk9h7naYkgppU7hmnEaS4oz+M2XMkIdhlJKhR3XXBEopZQanCYCpZRyOU0ESinlcpoIlFLK5TQRKKWUy2kiUEopl9NEoJRSLqeJQCmlXE6MMaGO4UMRkQbgyDm+PAs4NoLhjLRwjw/CP0aN7/xofOcnnOMrMsZkD7Zi1CWC8yEiW40xJaGOYyjhHh+Ef4wa3/nR+M5PuMc3FC0aUkopl9NEoJRSLue2RPCrUAdwFuEeH4R/jBrf+dH4zk+4xzcoV9URKKWUOp3brgiUUkoNoIlAKaVcLiITgYgsF5F9InJARO4fZL2IyAP2+jIRWRjE2MaLyBsiskdEdonINwbZZpmINIvIDvvx7WDFZ39+hYjstD976yDrQ7n/pgXslx0i0iIi9wzYJuj7T0QeEZF6ESkPWJYhIn8Rkf3289ghXnvG36uD8f1ERPbaf8MXRCR9iNee8ffgYHzfFZGqgL/j1UO8NlT775mA2CpEZMcQr3V8/503Y0xEPYBo4CAwCYgDSoGZA7a5GvgzIMBi4N0gxpcHLLSnU4EPBolvGfBSCPdhBZB1hvUh23+D/K1rsW6UCen+Ay4HFgLlAcv+H3C/PX0/8OMhvsMZf68OxvdxIMae/vFg8Q3n9+BgfN8F/u8wfgMh2X8D1v878O1Q7b/zfUTiFcEi4IAx5pAxpgd4Glg5YJuVwGPGshlIF5G8YARnjKkxxmy3p1uBPUBBMD57BIVs/w1wJXDQGHOud5qPGGPMm8DxAYtXAr+1p38LXD/IS4fze3UkPmPMq8aYXnt2M1A40p87XEPsv+EI2f7rIyICfBZ4aqQ/N1giMREUAJUB8x5OP9AOZxvHiUgxsAB4d5DVl4hIqYj8WURmBTcyDPCqiGwTkdsGWR8W+w9YxdD/fKHcf33GGWNqwDoBAHIG2SZc9uWXsa7yBnO234OT7rKLrh4ZomgtHPbfZUCdMWb/EOtDuf+GJRITgQyybGAb2eFs4ygRSQGeA+4xxrQMWL0dq7hjHvDfwB+CGRuwxBizEFgB3Ckilw9YHw77Lw64DvjdIKtDvf8+jHDYl98EeoEnhtjkbL8HpzwETAbmAzVYxS8DhXz/ATdx5quBUO2/YYvEROABxgfMFwLV57CNY0QkFisJPGGMeX7gemNMizGmzZ5eB8SKSFaw4jPGVNvP9cALWJffgUK6/2wrgO3GmLqBK0K9/wLU9RWZ2c/1g2wT6t/ircAngVuMXaA90DB+D44wxtQZY3zGGD/w8BCfG+r9FwN8CnhmqG1Ctf8+jEhMBFuAKSIy0T5rXAWsHbDNWuCLduuXxUBz3yW80+zyxN8Ae4wx/zHENrn2dojIIqy/U2OQ4ksWkdS+aawKxfIBm4Vs/wUY8iwslPtvgLXArfb0rcCLg2wznN+rI0RkOXAfcJ0xpmOIbYbze3AqvsB6pxuG+NyQ7T/bx4C9xhjPYCtDuf8+lFDXVjvxwGrV8gFWa4Jv2stuB263pwV40F6/EygJYmxLsS5dy4Ad9uPqAfHdBezCagGxGbg0iPFNsj+31I4hrPaf/flJWAf2MQHLQrr/sJJSDeDFOkv9CpAJvA7st58z7G3zgXVn+r0GKb4DWOXrfb/DNQPjG+r3EKT4/tf+fZVhHdzzwmn/2csf7fvdBWwb9P13vg/tYkIppVwuEouGlFJKfQiaCJRSyuU0ESillMtpIlBKKZfTRKCUUi6niUCpIBKrZ9SXQh2HUoE0ESillMtpIlBqECLyeRF5z+5D/pciEi0ibSLy7yKyXUReF5Fse9v5IrI5oF//sfbyC0TkNbvzu+0iMtl++xQR+b1YYwE80XcXtFKhoolAqQFEZAbwOazOwuYDPuAWIBmrf6OFwAbgO/ZLHgPuM8bMxboTtm/5E8CDxur87lKsO1PB6nH2HmAm1p2nSxz+SkqdUUyoA1AqDF0JXAhssU/WE7E6jPPT37nY48DzIjIGSDfGbLCX/xb4nd2/TIEx5gUAY0wXgP1+7xm7bxp7VKtiYKPj30qpIWgiUOp0AvzWGPOPpywU+ecB252pf5YzFfd0B0z70P9DFWJaNKTU6V4HbhSRHDg59nAR1v/LjfY2NwMbjTHNwAkRucxe/gVgg7HGmPCIyPX2e8SLSFIwv4RSw6VnIkoNYIzZLSLfwhpVKgqrx8k7gXZglohsA5qx6hHA6mJ6jX2gPwSstpd/AfiliHzPfo/PBPFrKDVs2vuoUsMkIm3GmJRQx6HUSNOiIaWUcjm9IlBKKZfTKwKllHI5TQRKKeVymgiUUsrlNBEopZTLaSJQSimX+/8v0/9+M9QeWgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "plt.plot(res[0], label='train')\n", + "plt.plot(res[2], label='valid')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('Loss')\n", + "plt.legend(loc='best')\n", + "plt.savefig('fig-res-googlenet-train-validate-loss.pdf')\n", + "plt.show()\n", + "\n", + "plt.plot(res[1], label='train')\n", + "plt.plot(res[3], label='valid')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('Acc')\n", + "plt.legend(loc='best')\n", + "plt.savefig('fig-res-googlenet-train-validate-acc.pdf')\n", + "plt.show()\n", + "\n", + "# save raw data\n", + "import numpy\n", + "numpy.save('fig-res-googlenet_data.npy', res)" ] }, { @@ -392,7 +442,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.8.12" } }, "nbformat": 4, diff --git a/7_deep_learning/1_CNN/06-resnet.ipynb b/7_deep_learning/1_CNN/06-resnet.ipynb index df09e12..00160a2 100644 --- a/7_deep_learning/1_CNN/06-resnet.ipynb +++ b/7_deep_learning/1_CNN/06-resnet.ipynb @@ -42,36 +42,32 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:56:06.772059Z", "start_time": "2017-12-22T12:56:06.766027Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ - "import sys\n", - "sys.path.append('..')\n", - "\n", "import numpy as np\n", "import torch\n", "from torch import nn\n", "import torch.nn.functional as F\n", "from torch.autograd import Variable\n", - "from torchvision.datasets import CIFAR10" + "from torchvision.datasets import CIFAR10\n", + "from torchvision import transforms as tfs" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T12:47:49.222432Z", "start_time": "2017-12-22T12:47:49.217940Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -82,19 +78,18 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:14:02.429145Z", "start_time": "2017-12-22T13:14:02.383322Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "class Residual_Block(nn.Module):\n", " def __init__(self, in_channel, out_channel, same_shape=True):\n", - " super(residual_block, self).__init__()\n", + " super(Residual_Block, self).__init__()\n", " self.same_shape = same_shape\n", " stride=1 if self.same_shape else 2\n", " \n", @@ -127,7 +122,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:14:05.793185Z", @@ -155,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:14:11.929120Z", @@ -201,13 +196,12 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:27:46.099404Z", "start_time": "2017-12-22T13:27:45.986235Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -272,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:28:00.597030Z", @@ -302,39 +296,39 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:29:01.484172Z", "start_time": "2017-12-22T13:29:00.095952Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "from utils import train\n", "\n", "def data_tf(x):\n", - " x = x.resize((96, 96), 2) # 将图片放大到 96 x 96\n", - " x = np.array(x, dtype='float32') / 255\n", - " x = (x - 0.5) / 0.5 # 标准化，这个技巧之后会讲到\n", - " x = x.transpose((2, 0, 1)) # 将 channel 放到第一维，只是 pytorch 要求的输入方式\n", - " x = torch.from_numpy(x)\n", + " im_aug = tfs.Compose([\n", + " tfs.Resize(96),\n", + " tfs.ToTensor(),\n", + " tfs.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n", + " ])\n", + " x = im_aug(x)\n", " return x\n", " \n", - "train_set = CIFAR10('../../data', train=True, transform=data_tf)\n", + "train_set = CIFAR10('../../data', train=True, transform=data_tf)\n", "train_data = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)\n", - "test_set = CIFAR10('../../data', train=False, transform=data_tf)\n", - "test_data = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)\n", + "test_set = CIFAR10('../../data', train=False, transform=data_tf)\n", + "test_data = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)\n", "\n", "net = ResNet(3, 10)\n", - "optimizer = torch.optim.SGD(net.parameters(), lr=0.01)\n", + "optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)\n", "criterion = nn.CrossEntropyLoss()" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2017-12-22T13:45:00.783186Z", @@ -346,31 +340,60 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 0. Train Loss: 1.437317, Train Acc: 0.476662, Valid Loss: 1.928288, Valid Acc: 0.384691, Time 00:00:44\n", - "Epoch 1. Train Loss: 0.992832, Train Acc: 0.648198, Valid Loss: 1.009847, Valid Acc: 0.642405, Time 00:00:48\n", - "Epoch 2. Train Loss: 0.767309, Train Acc: 0.732617, Valid Loss: 1.827319, Valid Acc: 0.430380, Time 00:00:47\n", - "Epoch 3. Train Loss: 0.606737, Train Acc: 0.788043, Valid Loss: 1.304808, Valid Acc: 0.585245, Time 00:00:46\n", - "Epoch 4. Train Loss: 0.484436, Train Acc: 0.834499, Valid Loss: 1.335749, Valid Acc: 0.617089, Time 00:00:47\n", - "Epoch 5. Train Loss: 0.374320, Train Acc: 0.872922, Valid Loss: 0.878519, Valid Acc: 0.724288, Time 00:00:47\n", - "Epoch 6. Train Loss: 0.280981, Train Acc: 0.904212, Valid Loss: 0.931616, Valid Acc: 0.716871, Time 00:00:48\n", - "Epoch 7. Train Loss: 0.210800, Train Acc: 0.929747, Valid Loss: 1.448870, Valid Acc: 0.638548, Time 00:00:48\n", - "Epoch 8. Train Loss: 0.147873, Train Acc: 0.951427, Valid Loss: 1.356992, Valid Acc: 0.657536, Time 00:00:47\n", - "Epoch 9. Train Loss: 0.112824, Train Acc: 0.963895, Valid Loss: 1.630560, Valid Acc: 0.627769, Time 00:00:47\n", - "Epoch 10. Train Loss: 0.082685, Train Acc: 0.973905, Valid Loss: 0.982882, Valid Acc: 0.744264, Time 00:00:44\n", - "Epoch 11. Train Loss: 0.065325, Train Acc: 0.979680, Valid Loss: 0.911631, Valid Acc: 0.767009, Time 00:00:47\n", - "Epoch 12. Train Loss: 0.041401, Train Acc: 0.987952, Valid Loss: 1.167992, Valid Acc: 0.729826, Time 00:00:48\n", - "Epoch 13. Train Loss: 0.037516, Train Acc: 0.989011, Valid Loss: 1.081807, Valid Acc: 0.746737, Time 00:00:47\n", - "Epoch 14. Train Loss: 0.030674, Train Acc: 0.991468, Valid Loss: 0.935292, Valid Acc: 0.774031, Time 00:00:45\n", - "Epoch 15. Train Loss: 0.021743, Train Acc: 0.994565, Valid Loss: 0.879348, Valid Acc: 0.790150, Time 00:00:47\n", - "Epoch 16. Train Loss: 0.014642, Train Acc: 0.996463, Valid Loss: 1.328587, Valid Acc: 0.724387, Time 00:00:47\n", - "Epoch 17. Train Loss: 0.011072, Train Acc: 0.997363, Valid Loss: 0.909065, Valid Acc: 0.792919, Time 00:00:47\n", - "Epoch 18. Train Loss: 0.006870, Train Acc: 0.998561, Valid Loss: 0.923746, Valid Acc: 0.794403, Time 00:00:46\n", - "Epoch 19. Train Loss: 0.004240, Train Acc: 0.999500, Valid Loss: 0.877908, Valid Acc: 0.802314, Time 00:00:46\n" + "[ 0] Train:(L=1.506980, Acc=0.449868), Valid:(L=1.119623, Acc=0.598596), T: 00:00:48\n", + "[ 1] Train:(L=1.022635, Acc=0.641504), Valid:(L=0.942414, Acc=0.669600), T: 00:00:47\n", + "[ 2] Train:(L=0.806174, Acc=0.717551), Valid:(L=0.921687, Acc=0.682061), T: 00:00:47\n", + "[ 3] Train:(L=0.638939, Acc=0.775555), Valid:(L=0.802450, Acc=0.729727), T: 00:00:47\n", + "[ 4] Train:(L=0.497571, Acc=0.826606), Valid:(L=0.658700, Acc=0.775316), T: 00:00:47\n", + "[ 5] Train:(L=0.364864, Acc=0.872442), Valid:(L=0.717290, Acc=0.768888), T: 00:00:47\n", + "[ 6] Train:(L=0.263076, Acc=0.907888), Valid:(L=0.832575, Acc=0.750000), T: 00:00:47\n", + "[ 7] Train:(L=0.181254, Acc=0.935782), Valid:(L=0.818366, Acc=0.764933), T: 00:00:47\n", + "[ 8] Train:(L=0.124111, Acc=0.957820), Valid:(L=0.883527, Acc=0.778184), T: 00:00:47\n", + "[ 9] Train:(L=0.108587, Acc=0.961657), Valid:(L=0.899127, Acc=0.780756), T: 00:00:47\n", + "[10] Train:(L=0.091386, Acc=0.968670), Valid:(L=0.975022, Acc=0.781448), T: 00:00:47\n", + "[11] Train:(L=0.079259, Acc=0.972287), Valid:(L=1.061239, Acc=0.770075), T: 00:00:47\n", + "[12] Train:(L=0.067858, Acc=0.976123), Valid:(L=1.025909, Acc=0.782140), T: 00:00:47\n", + "[13] Train:(L=0.064745, Acc=0.977701), Valid:(L=0.987410, Acc=0.789062), T: 00:00:47\n", + "[14] Train:(L=0.056921, Acc=0.979779), Valid:(L=1.165746, Acc=0.773438), T: 00:00:47\n", + "[15] Train:(L=0.058128, Acc=0.980039), Valid:(L=1.057119, Acc=0.782437), T: 00:00:47\n", + "[16] Train:(L=0.050794, Acc=0.982257), Valid:(L=1.098127, Acc=0.779074), T: 00:00:47\n", + "[17] Train:(L=0.046720, Acc=0.984415), Valid:(L=1.066124, Acc=0.787184), T: 00:00:47\n", + "[18] Train:(L=0.044737, Acc=0.984375), Valid:(L=1.053032, Acc=0.792029), T: 00:00:47\n" ] } ], "source": [ - "train(net, train_data, test_data, 20, optimizer, criterion)" + "res = train(net, train_data, test_data, 20, optimizer, criterion)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "plt.plot(res[0], label='train')\n", + "plt.plot(res[2], label='valid')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('Loss')\n", + "plt.legend(loc='best')\n", + "plt.savefig('fig-res-resnet-train-validate-loss.pdf')\n", + "plt.show()\n", + "\n", + "plt.plot(res[1], label='train')\n", + "plt.plot(res[3], label='valid')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('Acc')\n", + "plt.legend(loc='best')\n", + "plt.savefig('fig-res-resnet-train-validate-acc.pdf')\n", + "plt.show()\n", + "\n", + "# save raw data\n", + "import numpy\n", + "numpy.save('fig-res-resnet_data.npy', res)" ] }, { @@ -418,7 +441,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.4" + "version": "3.8.12" } }, "nbformat": 4, diff --git a/7_deep_learning/1_CNN/utils.py b/7_deep_learning/1_CNN/utils.py index ca22ef4..e1c6b24 100644 --- a/7_deep_learning/1_CNN/utils.py +++ b/7_deep_learning/1_CNN/utils.py @@ -47,10 +47,7 @@ def train(net, train_data, valid_data, num_epochs, optimizer, criterion, use_cud train_loss += loss.item() train_acc += get_acc(output, label) - cur_time = datetime.now() - h, remainder = divmod((cur_time - prev_time).seconds, 3600) - m, s = divmod(remainder, 60) - time_str = "Time %02d:%02d:%02d" % (h, m, s) + if valid_data is not None: valid_loss = 0 valid_acc = 0 @@ -67,7 +64,7 @@ def train(net, train_data, valid_data, num_epochs, optimizer, criterion, use_cud valid_loss += loss.item() valid_acc += get_acc(output, label) epoch_str = ( - "Epoch %d. Train Loss: %f, Train Acc: %f, Valid Loss: %f, Valid Acc: %f, " + "[%2d] Train:(L=%f, Acc=%f), Valid:(L=%f, Acc=%f), " % (epoch, train_loss / len(train_data), train_acc / len(train_data), valid_loss / len(valid_data), valid_acc / len(valid_data))) @@ -75,13 +72,18 @@ def train(net, train_data, valid_data, num_epochs, optimizer, criterion, use_cud l_valid_acc.append(valid_acc / len(valid_data)) l_valid_loss.append(valid_loss / len(valid_data)) else: - epoch_str = ("Epoch %d. Train Loss: %f, Train Acc: %f, " % + epoch_str = ("[%2d] Train:(L=%f, Acc=%f), " % (epoch, train_loss / len(train_data), train_acc / len(train_data))) l_train_acc.append(train_acc / len(train_data)) l_train_loss.append(train_loss / len(train_data)) - + + cur_time = datetime.now() + h, remainder = divmod((cur_time - prev_time).seconds, 3600) + m, s = divmod(remainder, 60) + time_str = "T: %02d:%02d:%02d" % (h, m, s) + prev_time = cur_time print(epoch_str + time_str)