{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AlexNet\n",
    "\n",
    "\n",
    "第一个典型的卷积神经网络是 LeNet5 ，但是第一个开启深度学习的网络却是 AlexNet，这个网络在2012年的ImageNet竞赛中取得冠军。这网络提出了深度学习常用的技术：ReLU和Dropout。AlexNet网络结构在整体上类似于LeNet，都是先卷积然后在全连接，但在细节上有很大不同，AlexNet更为复杂，Alexnet模型由5个卷积层和3个池化Pooling层，其中还有3个全连接层构成，共有$6 \\times 10^7$个参数和65000个神经元，最终的输出层是1000通道的Softmax。AlexNet 跟 LeNet 结构类似，但使⽤了更多的卷积层和更⼤的参数空间来拟合⼤规模数据集 ImageNet，它是浅层神经⽹络和深度神经⽹络的分界线。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch\n",
    "\n",
    "class AlexNet(nn.Module):\n",
    "    def __init__(self, num_classes=1000, init_weights=False):   \n",
    "        super(AlexNet, self).__init__()\n",
    "        self.features = nn.Sequential( \n",
    "            nn.Conv2d(1, 96, kernel_size=11, stride=4, padding=2),  \n",
    "            nn.ReLU(inplace=True), #inplace 可以载入更大模型\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),       \n",
    "\n",
    "            nn.Conv2d(96, 256, kernel_size=5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "\n",
    "            nn.Conv2d(256, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "\n",
    "            nn.Conv2d(384, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "\n",
    "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "        )\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Linear(256*6*6, 4096),  #全链接\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(4096, num_classes),\n",
    "        )\n",
    "        if init_weights:\n",
    "            self._initialize_weights()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = torch.flatten(x, start_dim=1) #展平或者view()\n",
    "        x = self.classifier(x)\n",
    "        return x\n",
    "\n",
    "    def _initialize_weights(self):\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                #何教授方法\n",
    "                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') \n",
    "                if m.bias is not None:\n",
    "                    nn.init.constant_(m.bias, 0)\n",
    "            elif isinstance(m, nn.Linear):\n",
    "                #正态分布赋值\n",
    "                nn.init.normal_(m.weight, 0, 0.01) \n",
    "                nn.init.constant_(m.bias, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.datasets import CIFAR10\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms as tfs\n",
    "from utils import train\n",
    "\n",
    "\n",
    "# 数据转换\n",
    "def data_tf(x):\n",
    "    im_aug = tfs.Compose([\n",
    "        tfs.Resize(227),\n",
    "        tfs.ToTensor(),\n",
    "        tfs.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "    ])\n",
    "    x = im_aug(x)\n",
    "    return x\n",
    "     \n",
    "train_set  = CIFAR10('../../data', train=True, transform=data_tf)\n",
    "train_data = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)\n",
    "test_set   = CIFAR10('../../data', train=False, transform=data_tf)\n",
    "test_data  = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = AlexNet(num_classes=10)\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "res = train(net, train_data, test_data, 20, optimizer, criterion, use_cuda=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.plot(res[0], label='train')\n",
    "plt.plot(res[2], label='valid')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend(loc='best')\n",
    "plt.savefig('fig-res-alexnet-train-validate-loss.pdf')\n",
    "plt.show()\n",
    "\n",
    "plt.plot(res[1], label='train')\n",
    "plt.plot(res[3], label='valid')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('Acc')\n",
    "plt.legend(loc='best')\n",
    "plt.savefig('fig-res-alexnet-train-validate-acc.pdf')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}