OpenI
/
machinelearning_notebook

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 链接主成分分析和逻辑回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "主成分分析所做的是无监督下的降维工作，而逻辑回归是预测的工作。\n",
    "\n",
    "我们用GridSearchCV设置PCA的维数。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAADFCAYAAAC7ICzVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deXhV1bXAf+ve3MwTIQMkhERmwQEVAUFKWq3V1mqxiqBYtSLSV62177VO7+HUOrT11Q4+raKg8opTtaC2FESw+hgUBEGGAAIhJEAGhszJTbLeH+ckJJDpXBNu7r37933nu+fss9fZa99h3T2svbaoKgaDITRx+VsBg8HgP4wBMBhCGGMADIYQxhgAgyGEMQbAYAhhjAEwGEKYMH8r8FVJTk7W7Oxsf6thMPRq1q9fX6KqKSemB7wByM7OZt26df5Ww2Do1YhIXlvppgtgMAQBL368hxc/3uNYzhgAgyEIWPVlCau+LHEsF/BdAIPBAHNvPN8nuZBoAVTV1WPWPBgMJxP0LYBZL6/jg+1FLPnpJIakxnWYt6ysjKKiIrxe7ynSzmDoHg5XN7CvOozpE0c4kgt6AxAe5qK+UVmz+3CHBqCsrIxDhw6RkZFBVFQUInIKtTQYfEdV2X3wMBQWUlZWRnx8fJdlg74LMG5QXwDW7C7tMF9RUREZGRlER0ebH78hoBARBvfvyzmnD6aoqMiRbNAbgAsGJQGwds/hDscBvF4vUVFRp0otg6HbiYqKctx9DXoDMDglluTYcIrLa9ldUtlhXvPPbwhUisprKK6odSwX9AZARBh3mtUNWLv7sJ+1MQQ6OTk5zJ07199qnERNXQM1dQ2O5YLeAACMa+4GdDwOYAgeVq5cyYABA/ytxiljYN8YBvaNcSwXEgZgfIuBQOMPYDAcJyQMwNDUWJJiwjlUVkteaZW/1fGJ7Oxsfvvb33LWWWeRkJDAtddeS01NTadyixYtYvTo0cTHxzN48GCWLFkCQGFhIVdccQVJSUkMGTKE559/vlnmwQcf5JprrmHGjBnExcVx5plnsmPHDh577DFSU1PJzMxk6dKlzflzcnK49957GTt2LPHx8Vx55ZUcPny8u7V48WJGjRpFYmIiOTk5bNu2rcv1evfddxk9ejSJiYlMmDCBTZs2dSpbWVnJZZddRmFhIbGxscTGxlJYWMgnn3zCmDFjiI+PJy0tjZ/97GeO3zeAvLw8Jk6cSFxcHJdccgklJcddcDuq6xNPPEFGRgZxcXEMHz6c5cuXA9DY2Mjjjz/O4MGD6du3L1OnTm1+//bu3YuI8NJLLzFw4ECSk5P51a9+dZK+h8pqOFTW+ffhJFQ1oI/zzjtPu8JtL6/TrLvf1YVr89q8v3Xr1i49x19kZWXp+eefrwUFBVpaWqojRozQZ555pkOZtWvXanx8vC5dulQbGhp0//79um3bNlVVnTRpkv7oRz/S6upq3bBhgyYnJ+vy5ctVVfWBBx7QiIgIXbJkiXq9Xr3hhhs0Oztbf/nLX2pdXZ0+99xzmp2d3VzO5MmTNT09XTdv3qwVFRV61VVX6fXXX6+qqrm5uRodHa1Lly7Vuro6feKJJ3Tw4MFaW1vbab0+++wzTUlJ0TVr1mh9fb3Onz9fs7KytKamplPZFStWaEZGRqv3Y/z48fryyy+rqmp5ebmuXr3a8fs2efJkHTRokObm5mpVVZVOnjxZ77777k7run37dh0wYIAWFBSoquqePXt0165dqqr61FNP6bhx4zQ/P19ramp01qxZOm3atOZ8gM6cOVOrqqp048aNGh4eftL3Na+0UvNKK9v9HgPrtI3fj99/wF/16KoBmPfxbs26+1396asb2rzf1hs39dlV+vqn+1RVta6+Qac+u0rf+ixfVVWraut16rOrdPFG6wM9Vl2nU59dpf/YXKiqqqUVtTr12VW6bMtBVVU9VFatU59dpSu2H1JV1YIjVV3Su4msrCx95ZVXmq9//vOf62233dahzKxZs/SnP/3pSen79u1Tl8ulZWVlzWn33HOP3njjjapqGYCLL764+d7ixYs1JiZG6+vrVVW1rKxMAT1y5Iiqaqsfgarqli1b1OPxaH19vT788MN6zTXXNN9raGjQ9PR0XbFiRaf1mj17tv7nf/5nK92HDRumK1eu7FS2LQMwadIknTNnjhYXF3f0trX7vjXV9ZFHHmm+fvrpp/Vb3/qWqmqHdd25c6empKTosmXLtK6urtUzR4wYoe+//37zdWFhoYaFhanX6202APn5+c33zz//fF24cGGb+jk1ACHRBYDWDkEaoOMA/fr1az6Pjo6moqKiw/z5+fkMHjz4pPTCwkKSkpKIizvuGZmVlUVBQUHzdVpaWvN5VFQUycnJuN3u5mugVfmZmZmtnuX1eikpKaGwsJCsrKzmey6Xi8zMzFZltVevvLw8nnzySRITE5uP/Px8CgsLfXpPXnjhBXbs2MGIESM4//zzeffdd9vM19771lmZHdV1yJAhPPXUUzz44IOkpqYybdq05nrk5eUxZcqU5jqefvrpuN1uDh065FM9nRD0rsBNDE+LIzHaw4FjNew/Uk1mUnSnMq/ddkHzucftanUdFe5udR0f6Wl1nRQT3uo6NS6y1XV6Ys87HWVmZvLll1+elJ6ens7hw4cpLy9vNgL79u0jIyPD57Ly8/Obz/ft24fH4yE5OZn09HQ2b97cfE9Vyc/P71JZmZmZ3H///dx///2O9WnLp2Po0KEsXLiQxsZG3nrrLa6++mpKS0uJiWk9et7e+9YZndX1uuuu47rrrqOsrIzbbruNu+++m1deeYXMzExefPFFJk6ceNIz9+7d26WyDx7zof9PiAwCArhcwlkDEgHYeqDMz9qcGm655RbmzZvH8uXLaWxspKCggO3bt5OZmcmECRO49957qampYdOmTbzwwgvMmDHD57IWLFjA1q1bqaqqYs6cOVx99dW43W6mTp3Ke++9x/Lly/F6vTz55JNEREQwYcKETp9566238uyzz7J27VpUlcrKSt577z3Ky8s7lU1LS6O0tJRjx4610rG4uBiXy0ViovVdcLlO/gm09751Rkd1zc3N5YMPPqC2tpbIyEiioqKay549ezb3338/eXlW0J7i4mIWLVrUaXkt8TY04m1odCQDIWQAAIanxQKQe7DzL1AwMHbsWObNm8ddd91FQkICkydPbv6SLVy4kL1795Kens6UKVN46KGHuPjii30u64YbbuCmm26iX79+1NTU8Ic//AGA4cOHs2DBAu644w6Sk5N55513eOeddwgPD+/0mWPGjOH555/n9ttvp0+fPgwZMoT58+d3SZ8RI0Ywffp0Bg0aRGJiIoWFhSxZsoRRo0YRGxvLnXfeyauvvtqm+3dH71tHdFTX2tpa7rnnHpKTk+nXrx9FRUU89thjANx5551cccUVXHLJJcTFxTF+/HjWrl3bpXo2kZkU3aVW7YlIoPaHmxgzZox2NSbgG+vy+fmbm/jOWf15+rpzW93btm0bp59+ek+oGPTk5OQwY8YMZs6c6W9VQp72vscisl5Vx5yYHjJjAAAj+lnLJHeESAvAEDocOFbtk1xIdQGGpMYiAntKKqmtd+433Rt59NFHm51dWh6XXXaZv1UznEIaGpWGRuet+ZBqAUSFu8lKimZvaRV7SiqbWwSBzH333cd9993nVx1Wrlzp1/INMKCP1f/fdtCZXI+2AETkRREpEpEvWqQ9KCIFIrLRPr7d4t69IrJLRHJF5Fs9odPwfta0V6gMBBoMHdFtBkBE+rSRPB+4tI3036nqaPv4uy0/EpgGjLJl/kdE3N2lXxPD09o3AIE+IGoIXQ4crabwiPN1Lt3ZAlh+YoKq/gvo6iL8K4FXVbVWVfcAu4Cx3agfAMPsFsCOQ60NgMfjobrat4EUg8HfNCrU1dbi8XgcyXWnAXASTud2EdlkdxGaWg4ZQH6LPPvttG6luQVwggFITU2loKCAqqoq0xIwBBSqSp8IxVteQmpqqiPZ7hwE7Oqv5hngETv/I8CTwA+dFCQis4BZAAMHDnQiSnZyDOFuF/mHq6morSc2wnoLmiKpFhYWmrDghoDD4/GQlpbmKCIw+GEWQFWbVziIyPNA04qMAiCzRdYBdlpbz3gOeA4sRyAn5XvcLgalxLD9YDk7D5VzzsDjQxfx8fGO30CDoTfw0DtbYFs+D3x3lCO5U94FEJH+LS6nAE0zBIuBaSISISKnAUOBT7pRv2aGtzMOYDCEGo5aACJyITBUVeeJSAoQaw/YAVzURv6FQA6QLCL7gQeAHBEZjdUF2AvcBqCqW0TkdWArUA/8WFV7xFtnWPNMQPcsqTQY/I3Tf/4mumwAROQBYAwwHJgHeIAFwEQAVT1ptF9Vp7fxqBfaK0NVfwWcHO+omxlhWgAGA+CsCzAFuAKoBFDVQqDjzfZ6KU0tgO3GGcgQJPzX377gv/72RecZT8CJAaizQwspgIg4j0HcS8hIjCIm3E1JRS1Hq+r8rY7B8JWJ9LiI9Dgf0nMyBvC6iPwZSBSRW7Gm7p7vRKZX4nIJ/ROj2FVUwaGyWhKjO1+bbjD0Zu7/zkif5LpsMlT1t8CbwF+xxgHmqOoffSq1F5Aca/3oi8udb6dkMAQLTgYBTwM+UtVl9nWUiGSr6t6eUq4nSYmLBKC4wrdYagZDb+Let6z9Eh676ixHck46DW8ALYOONdhpAUlKbARgWgCG4CAxOtynrqyTMYAwVW0eMVPVOhEJ2M5zSpxlAEoqzCCgIfC5+9IRPsk5aQEUi8gVTRciciVQ0kH+Xk2TATAtAEMo46QFMBv4XxH5E5bbbz7wgx7R6hRgDIAhmPiPNz4H4LfXnO1IrssGQFW/BMaLSKx9HdB+tGYWwBBMpCdE+iTnZBYgAvg+kA2ENe28oqoP+1Syn2luAVQYA2AIfH52yXCf5Jx0ARYBx4D1QMD/avrGROASOFJVh7ehEY87pAIkGwyAMwMwQFXbiu8XkLhdQlJMBCUVtRyurCMt3rcmlMHQG/jpqxsAeGraOY7knPztrRKRMx09vZdjBgINwcKglFgGpcQ6lnPSArgQuElE9mB1AQRQVXXmetSLSImLYNsBYwAMgc9PLhrqk5wTAxB0W82YmQBDqONkGjAPQERSgaDoMJuZAEOwcPtfPgPgTydsetsZTqYBr8CK4JsOFAFZwDasjTwCErMewBAsjEz3LZitky7AI8B44H1VPUdEvg7M8KnUXoJpARiChX/LGeKTnJNZAK+qlgIuEXGp6gqsGIEBi5kFMIQ6TloAR2034H9hrQkowo4PGKikNq0INAbAEODMfmU9AM/ecJ4jOScG4EqgBrgLuB5IAALSDbiJZDMGYAgSzs1K9EnOySxAy3/7l3wqrZeREOXB4xbKa+up8TYQ6en2zYgNhlPCrK8N9kmu0zEAEfnYfi0XkbIWR7mIlPlUai9BRMxMgCGk6dQAqOqF9mucqsa3OOJUNeA30jMzAYZgYOZLnzLzpU8dy3WpCyAibmCLqvoWd6gXY2YCDMHAhMHJPsl1yQCoaoOI5IrIQFXd51NJvRRjAAzBwA8vPM0nOSezAH2ALSLyCS2m/1T1ivZFej9mJsAQyjgxAP/VY1r4ETMGYAgGbnzxEwBe+uFYR3JOpgE/dKZSYNA0C2CcgQyBzMWnp/ok12VXYBEZLyKfikiFiNSJSENn04Ai8qKIFInIFy3SkkRkmYjstF/72OkiIn8QkV0isklEnC1r8hHTAjAEAzdckM0NF2Q7lnOyFuBPwHRgJxAFzASe7kRmPnBiGLF7gOWqOhRYbl+DFW9gqH3MAp5xoJvPmEFAQyjjKBKmqu4C3KraoKrzOPnHfWL+fwGHT0i+kuOehC8B32uR/rJarMHahbi/E/18oaUBsHY/NxgCj+vnruH6uWscyzkZBKyytwLbKCK/Bg7g0IDYpKnqAfv8IJBmn2dgbTbSxH477QA9SHR4GHGRYZTX1HOkyktSTMDudmYIYS4/K90nOSc/4Bvs/LdjTQNmYu0T4DNq/eU6/tsVkVkisk5E1hUXF38VFQDISIwCoOBI9Vd+lsHgD6aPHcj0sQMdyzkxAOdh/WbLVPUhVf2Z3SVwyqGmpr39WmSnF2AZlSYG2GknoarPqeoYVR2TkpLigwqtaTYAR40BMIQWTgzAd4EdIvKKiFwuIk66Dy1ZDNxon9+IteFIU/oP7NmA8cCxFl2FHiWjjzEAhsDm2j+v5to/r3Ys58QP4GYR8WCN1k8HnhaRZao6sz0ZEVkI5ADJIrIfeAB4HHhdRG4B8oCpdva/A98GdgFVwM2Oa+Mj6aYLYAhwrj5vgE9yjv7FVdUrIv/A6rdHYY3gt2sAVHV6O7cuaiOvAj92ok930dQFKDQtAEOAcs2YzM4ztYETR6DLRGQ+lh/A94G5QD+fSu1lmC6AIdDxNjTibWh0LOekBfAD4DXgNlUNKq+ZAWYQ0BDgzJi7FoDXbrvAkZyTMYD2mvMAiMhqVXVWei8hOTaCcLeLw5V1VNXVEx3u6/imweAfpo31rQvQnd/0gN0tyOUS+idGkldaReHRGoakOt9k0WDwJ1PO8W0Q0BdPvvYIaD9a4wtgCGSq6xqormtwLNedBiCgMVOBhkDmpnmfcNO8TxzLdWcXQLrxWaccMxVoCGRmjM/ySa47DcAN3fisU46ZCjQEMt8927fFQJ0aABEpp4P+fVNocFX9or08gYBZEGQIZMpqvADER3ocyXVqAFQ1DkBEHsFamvsKVnP/eqDH1+ufKswgoCGQufWldUAP+gEAV6jq2S2unxGRz4E5jkrspfRPtGYxD5bVUN/QSJjbjI8aAoebJ2b7JOfkW14pIteLiFtEXCJyPQG+O3BLIsLcpMZF0NCoHDLhwQwBxqVn9OfSM5w3yJ0YgOuwVu4dso9r7LSgwUwFGgKVw5V1HK6scyznxBV4L1bcvqAlo08UG/OPmqlAQ8DxowXrgR4cAxCRYViRetNU9QwROQtrXOCXjkrsxZhFQYZA5dZJg3ySc9IFeB64F/ACqOomYJpPpfZSmroA+00XwBBgXDwyjYtHpnWe8QScGIBoVT3R17DecYm9GDMVaAhUisprKCqvcSznZBqwREQGYzsFicjV9HDI7lNNszfgkSo/a2IwOOOOv2wAetYP4MfAc8AIESkA9gAzHJXWy8nqG014mIvdJZUcOFZN/4Qof6tkMHSJH+UM9kmuy10AVd2tqhcDKcAIVb3QnhkIGqLDw7j49FRU4W8bCv2tjsHQZXKGp5Iz3PkGoU5iAkaIyHXAncBdIjJHRILCC7AlTYEV3t6w32wVZggYCo9W+zR97WQQcBGWH0A9lgdg0xFUTB6WQp9oDzsOVbClsMPNjw2GXsNdr23krtc2OpZzMgYwQFU73Aw0GAgPc/Hds9N5eXUeb28o4IyMBH+rZDB0yh3fGOqTnJMWwCoROdOnUgKMq861ugGLNhZS70OoZYPhVHPh0GQuHJrsWM6JAbgQWC8iuSKySUQ2i8gmxyUGAGcPSGBQcgwlFbV8tKvE3+oYDJ2yr7SKfaXOp6+dGIDLgKHAJVj7BF5uvwYdIsKUczIA+Ov6/X7WxmDonJ+/+Tk/f/Nzx3KdGgARibdPy9s5gpIp52bgdgn/+OIgXxZX+Fsdg6FD7vrmMO765jDHcl1pAfzFfl0PrLNf17e4DkoG9Ilm6phMGhqV/162w9/qGAwdMn5QX8YP6utYrlMDoKqX26+nqeog+7Xp8G0JUoDwk4uGEB7m4r1NB/ii4Ji/1TEY2uXL4gqfWqqO4l6JSB8RGSsiX2s6HJd4/Fl77YHEjSKyzk5LEpFlIrLTfu3j6/O7g/4JUfzADrf85NJcf6piMHTIfW9t5r63NjuWc+IJOBP4F/BP4CH79UHHJbbm66o6WlXH2Nf3AMtVdSiw3L72K//29SHEhLtZkVvMp3sP+1sdg6FNfnHpcH5x6XDHck5aAHcC5wN5qvp14BzgqOMSO+ZK4CX7/CXge938fMckxYQz0w62YFoBht7KeVlJnJeV5FjOiQGoUdUasNYFqOp2wLnJOY4CS0VkvYjMstPSVLVpifFBwHmEgx7glkmnERcZxprdh1mfZ1oBht5H7sFycg86n5RzYgD2i0gi8DdgmYgsAvIcl3icC1X1XCz/gh+fOJ6g1kqcNlfjiMgsEVknIuuKi4u/ggpdIz7Sw00TsgH40we7erw8g8EpcxZ9wZxFzvfmEV9WvInIZCABWKKqzkORnvy8B4EK4FYgR1UPiEh/YKWqdtjKGDNmjK5b1/OzkYcr65j4+AdUext4944LzRoBQ6/i83yrN352ZmKb90VkfYuxtma64giUdOIBbAY+BmJ9UVZEYkSkacehGCzvwi+AxcCNdrYbsVYg9gqSYsKZMX4gAE+vMK0AQ+/i7MzEdn/8HdGV1YDrsZribe3+q4AvvgBpwNsi0qTDX1R1iYh8CrwuIrdgdS+m+vDsHuPWSYN4aXUeS7YcZFdROUNS4/ytksEAwJZCy09lVLqzlmlX9gY8zTeVOnzmbuDsNtJLgYu6u7zuIjU+kqljBrBgzT7+441NvHzLWMebMRoMPcHD72wFejYmICJyFdaqQAU+UtW/OSotCPjJN4ayYnsxG/OPMmPuWl754TgSoo0RMPiXOd8d6ZOcE0eg/wFmY/X/vwBmi8jTPpUawKTGR/LabeMZmBTNpv3HmP78Gp+2ZDIYupNR6QmOm//gbBrwG8C3VHWeqs4Dvm2nhRwD+kTz2m3jOS05hq0Hyvj+M6vYWxJ00dEMAcTn+UebZwKc4MQA7AIGtrjOtNNCkv4JUbw2azwj+8ezp6SSq55Zxfq8I/5WyxCiPPr3bTz6922O5ZwYgDhgm4isFJEVwFYgXkQWi8hixyUHAanxkbw++wImD0vhcGUd1z2/xhgBg194+MozePjKMxzLddkRyHb+aRdV/dBx6d3AqXIE6oj6hkZ+8ddNvPVZAZOGJvPKLeP8qo/BcCLtOQI5mQUoVtWtJzw0R1VXflXlAp0wt4sHLh/FPzYf5KOdJeQeLGd4P+MjYDh1NK1RcbogyEkX4HUR+YVYRInIH4HHHJUWxCREe5g6xoom/MLHu/2sjSHU+PWSXH69xPlqVScGYBzWIOAq4FOgEJjouMQg5uaJpyFibStWXF7rb3UMIcSjV53Jo1c5j9rvxAB4gWogCogE9qiqCZrfguzkGL55ehp1DY28snqvv9UxhBCDU2IZnOJ8aY4TA/AplgEYA0wCpovIG45LDHKagoe8sibPtAIMp4w1u0tZs7vUsZwTA3ArsBO4zw7acQfgPBB5kHN+dh/OHpDAkSov4x59n2v/vJq/rN1HY6PZaNTQc/xu2Q5+50P0aicG4GZgPDDdvi7HCuFlaIGI8N/XjmbysBTcLmHtnsPc9/ZmnvvIDAwaeo7fXH02v7n6pPV1neJkGnCcqp4rIhsAVPWIiJhVMG0wOCWWl344lrIaL29/VsADi7fw5NJcJg1N9slf22DojIF9o32SczQIKCJu7DBdIpJCOyG7DBbxkR5unJDNjPED8TYod722kRpvg7/VMgQhH+8s4eOdzvexdGIA/gC8DaSKyK+wIgI96rjEEOT+b49kUHIMOw5V8Mi7WzlwrBpfQrEZDO3xxw928scPdjqWcxQTUERGYAXsEKz4/c5XH3QzvcEVuCt8nn+U7z+zinp7MDDK42bikL789pqzSYwO97N2hkCn8Gg1AOmJUW3eb88V2KegoL2JQDEAAO9tOsD8VXvYXVxJqR1DYGT/eBbMHEdSjDEChp7DGIBeRv7hKm588RN2l1QyPC2OBTPHkRIX4W+1DAHKytwiAHKGp7Z5vzsWAxm6kcykaF6dNZ7r564l91A5Ob9Zwen94xnRP45zB/Zh8rAU+sYag2DoGs+s/BJo3wC0h2kB+JmSilpmvbyOz/a1juYiAmcNSGTC4L6cN7AP5wxMNAbB0C5F5TUApMZFtnnfdAF6OSUVtWw7UMaWwjL+b1cJa3cfpq6h9VKL75zZn0ennGmCkBocYwxAgFFVV8/qL0tZl3eEz/KOsCH/KHX1jWQkRvH7aaMZk+18I0hD8PL+1kMAXDyy7e00zRhAgBEdHsZFp6dx0enWB5pXWslPFm7g8/3HuPa5NVw5Op3vjc5gwuC+hLmduHMYgpHnbVfz9gxAe5gWQABRV9/Ik8ty+fOHx9cV9I0JJ6NPFDHhYSRGexh7WhLfGJFKVt8YP2pqONU0haZvbzrZdAGCiL0llfxtYwGLNhayp51w5AOTohnQJ4qUuAjS4iPJ6htNdt8YMvtEkxwXTnS4afyFEsYABCGqSl5pFUervVTW1lN4tJoPdxTz4Y5iymvqO5SN8rjtACapXDKqH6PS47H3ajQEIEu+OADApWf0b/O+MQAhhLehkV1FFRSX11JcXsuBY9XsLa0ir7SSgiPVlFTWUVffeoYhyuMmIcpDQpSHAX2iGJUez8j0BAYmRZMUE06fGA8RYW4/1cjQGdf+eTXQ/t6AxgAYmlFVKmrr+WzfUf655SDLth7qUvSiJsNwRnoCI/rHM6JfHBmJUbhcpuXgb8pqvADtblZrDIChXZoMQllNPUer6thdXMmWwjK2Hijj0LEaSivrOFJVR0MbUY2iw90kxYSTEOUhPtJDbGQYsRFhRIe7iQhz4wkTojxu4iM9xEd5iI2w7keFu0mM9pCeEEVUuGlZ9DQBMw0oIpcCvwfcwFxVfdzPKgU9IkJcpIe4SA8ZiVGMSk/gu2ent8pT39DIl8WVbCk8xpbCMnIPlrP9YDklFbVU1VWz/0i1z+UnxYTTLz6S5LgIkmPDiYsIw+1y4XEL4WEuIj1uIsJczUYkMdpDcmwEafERxEaEmbEL4J3PCwFO+tw6o1cZADvgyNPAN4H9wKcisvjEDUkMp54wt4vh/eIY3i+Oq849nn6s2svRqjrKqus5Vu2loraeytp6KuvqqatvpLa+kRpvA+U19ZRVeymrqafaW09lbQOHK+s4cKyaw5V11jTWAed6RXpcRHnceNwuPG4XER4XkWFuou0WRmK01TqJCHMRHuYiIsxNpMcyKtHhTS2TMOIjPSREe+y8gdciWSpjHfwAAAegSURBVLAmDwhwAwCMBXap6m4AEXkVK+6gMQC9lKaBQ19pbFSKK2o5VFZDSYU1aFlV10BDo+JtUOrqG6mpb2g2IkerLINTXFFLUVkt1d4GarzdG50+PMxFTLibGLu7Em93b2Ii3ER53ER63IS5BLdLEBFaNkAEcNlpIoJgresQjudr2V7pjsbLOQP7MP/msT7J9jYDkAHkt7jej7UhiSFIcbmEtPhI0uLbXsTSEapKVV0DtfWNeBsa7RaHZRAqa+s5Wu3lSGUdZTVe6uobW7VIaryNVNbVU15jtVzKaryUVXs5WnU875Eqbw/UuPuZ9bVBTByS7JNsbzMAXUJEZgGzAAYOHNhJbkOwIiLERIQR042LJFWV2nrLgFTWNlBe66Wsup6yGi9VdfVU1zVS7W2gobGRRqXNgdHGRqVRQbFeUW0OntlyzF27KaTmmOwk3t6wH4Ap5wxwJNvbDEABkNnieoCd1gpVfQ54DqxZgFOjmiEUEBEi7WZ+X+cb7fiNJj+AQDcAnwJDReQ0rB/+NOA6/6pkMPR+Fsz0rafcqwyAqtaLyO3AP7GmAV9U1S1+Vstg6PV4fFwR2qsMAICq/h34u7/1MBgCiTfWWWPn14zJ7CRna8xCcoMhCHhz/X7eXL/fsVzAuwKLSDGQ10m2ZMD5timBR6jUE0Knrt1VzyxVTTkxMeANQFcQkXVt+UEHG6FSTwiduvZ0PU0XwGAIYYwBMBhCmFAxAM/5W4FTRKjUE0Knrj1az5AYAzAYDG0TKi0Ag8HQBkFvAETkUhHJFZFdInKPv/XpLkQkU0RWiMhWEdkiInfa6UkiskxEdtqvffyta3cgIm4R2SAi79rXp4nIWvtzfU1EAn57ZRFJFJE3RWS7iGwTkQt6+vMMagPQIsDIZcBIYLqIjPSvVt1GPfDvqjoSGA/82K7bPcByVR0KLLevg4E7gW0trp8AfqeqQ4AjwC1+0ap7+T2wRFVHAGdj1bdnP09VDdoDuAD4Z4vre4F7/a1XD9V1EVYkpVygv53WH8j1t27dULcB9pf/G8C7WDE1SoCwtj7nQDyABGAP9rhci/Qe/TyDugVA2wFGMvykS48hItnAOcBaIE1Vm4JrHQSc7RXVO3kK+AXQFPqnL3BUVZs2PwiGz/U0oBiYZ3d15opIDD38eQa7AQh6RCQW+CvwU1Uta3lPrb+NgJ7mEZHLgSJVXe9vXXqYMOBc4BlVPQeo5ITmfk98nsFuALoUYCRQEREP1o//f1X1LTv5kIj0t+/3B4r8pV83MRG4QkT2Aq9idQN+DySKSNNq1mD4XPcD+1V1rX39JpZB6NHPM9gNQHOAEXuUeBqw2M86dQtixcJ+Adimqv/d4tZi4Eb7/EassYGARVXvVdUBqpqN9fl9oKrXAyuAq+1swVDPg0C+iAy3ky7CCobbo59n0DsCici3sfqQTQFGfuVnlboFEbkQ+AjYzPG+8X1Y4wCvAwOxVklOVdXDflGymxGRHOA/VPVyERmE1SJIAjYAM1S18+2NejEiMhqYC4QDu4Gbsf6ke+zzDHoDYDAY2ifYuwAGg6EDjAEwGEIYYwAMhhDGGACDIYQxBsBgCGGMATAENCKSIyIT/K1HoGIMgCHQyQGMAfARYwCCABHJttePP2/HBlgqIlHt5B0iIu+LyOci8pmIDBaL34jIFyKyWUSutfPmiMiHIrJIRHaLyOMicr2IfGLnG2znmy8iz4rIOhHZYfvvIyKRIjLPzrtBRL5up98kIm+JyBJ7nfuvW+h3iYistnV7w17rgIjsFZGH7PTNIjLCXgQ1G7hLRDaKyCQRucaux+ci8q+efN+DAn8vgzTHVz+AbKz4AKPt69exPOPayrsWmGKfRwLRwPeBZVjekmnAPqylpznAUfs8Asvf/iFb9k7gKft8PrAE6w9lKJZfeyTw71jelwAj7OdGAjdhebol2Nd5WGs2koF/ATG2zN3AHPt8L3CHff5vwFz7/EEs78Cm+m0GMuzzRH9/Nr39MC2A4GGPqm60z9djGYVWiEgc1o/jbQBVrVHVKuBCYKGqNqjqIeBD4Hxb7FNVPaCWm+2XwFI7ffMJZbyuqo2quhPrxz3Cfu4Cu6ztWD/0YXb+5ap6TFVrsHzes7ACm4wE/k9ENmL5vme1KKNpwVOb9bP5P2C+iNyKZdAMHdDr9gY0+ExLP/gGoM0uwFd8bmOL60Zaf39O9CnvzMf8RH3DsAJ9LFPV6Z3INOU/CVWdLSLjgO8A60XkPFUt7USXkMW0AEIIVS0H9ovI9wBEJEJEorEWFV1rx91LAb4GfOLw8deIiMseFxiEFcnmI+B6u6xhWAtacjt4xhpgoogMsWVibLmOKAfimi5EZLCqrlXVOVgBNpztlhliGAMQetwA/ERENgGrgH7A28Am4HPgA+AXai1PdcI+LKPxD2C23bT/H8AlIpuB14CbtIMVe6pajDU+sNDWbzVWV6Ij3gGmNA0CAr+xBwm/sOv3ucN6hBRmNaDhKyMi84F3VfVNf+ticIZpARgMIYxpAQQpIvI0VjitlvxeVef5Qx9D78QYAIMhhDFdAIMhhDEGwGAIYYwBMBhCGGMADIYQxhgAgyGEMQbAYAhh/h/ha7LnAG3h7QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# % matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import linear_model, decomposition, datasets\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "logistic = linear_model.LogisticRegression()\n",
    "\n",
    "pca = decomposition.PCA()\n",
    "pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "X_digits = digits.data\n",
    "y_digits = digits.target\n",
    "\n",
    "# Plot the PCA spectrum\n",
    "pca.fit(X_digits)\n",
    "\n",
    "plt.figure(1, figsize=(4, 3))\n",
    "plt.clf()\n",
    "plt.axes([.2, .2, .7, .7])\n",
    "plt.plot(pca.explained_variance_, linewidth=2)\n",
    "plt.axis('tight')\n",
    "plt.xlabel('n_components')\n",
    "plt.ylabel('explained_variance_')\n",
    "\n",
    "# Prediction\n",
    "n_components = [20, 40, 64]\n",
    "Cs = np.logspace(-4, 4, 3)\n",
    "\n",
    "# Parameters of pipelines can be set using ‘__’ separated parameter names:\n",
    "estimator = GridSearchCV(pipe,\n",
    "                         dict(pca__n_components=n_components,\n",
    "                              logistic__C=Cs))\n",
    "estimator.fit(X_digits, y_digits)\n",
    "\n",
    "plt.axvline(estimator.best_estimator_.named_steps['pca'].n_components,\n",
    "            linestyle=':', label='n_components chosen')\n",
    "plt.legend(prop=dict(size=12))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1797, 64)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:143: FutureWarning: The sklearn.linear_model.logistic module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.linear_model. Anything that cannot be imported from sklearn.linear_model is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAL1UlEQVR4nO3df6hX9R3H8ddrptVS0laL0MiMIUSw/IEsitg0w1a4f5YoFCw29I8tkg3K9s/ov/6K9scIxGpBZqQljNhaSkYMtprXbJnaKDFSKgsNsz+U7L0/vsdhznXPvZ3P537v9/18wBe/997vPe/3vdfX95zz/Z5z3o4IARhs3xrrBgCUR9CBBAg6kABBBxIg6EACBB1IoC+CbnuJ7bdtv2N7TeFaj9k+ZHtXyTqn1bvc9jbbu22/ZfuewvXOs/2a7Teaeg+UrNfUnGD7ddvPl67V1Ntv+03bO21vL1xrqu1Ntvfa3mP7uoK1Zjc/06nbUdurO1l4RIzpTdIESe9KmiVpkqQ3JF1dsN6NkuZK2lXp57tM0tzm/hRJ/y7881nS5Ob+REmvSvpB4Z/x15KekvR8pd/pfkkXV6r1hKRfNPcnSZpaqe4ESR9KuqKL5fXDGn2BpHciYl9EnJD0tKSflCoWEa9IOlxq+Wep90FE7GjufyZpj6TpBetFRBxrPpzY3IodFWV7hqRbJa0rVWOs2L5QvRXDo5IUESci4tNK5RdJejci3utiYf0Q9OmS3j/t4wMqGISxZHumpDnqrWVL1plge6ekQ5K2RETJeg9LulfSlwVrnCkkvWh7yPbKgnWulPSxpMebXZN1ti8oWO90yyVt6Gph/RD0FGxPlvSspNURcbRkrYg4GRHXSpohaYHta0rUsX2bpEMRMVRi+V/jhoiYK+kWSb+0fWOhOueot5v3SETMkfS5pKKvIUmS7UmSlkra2NUy+yHoByVdftrHM5rPDQzbE9UL+fqIeK5W3WYzc5ukJYVKXC9pqe396u1yLbT9ZKFa/xURB5t/D0narN7uXwkHJB04bYtok3rBL+0WSTsi4qOuFtgPQf+npO/ZvrJ5Jlsu6U9j3FNnbFu9fbw9EfFQhXqX2J7a3D9f0mJJe0vUioj7I2JGRMxU7+/2UkTcUaLWKbYvsD3l1H1JN0sq8g5KRHwo6X3bs5tPLZK0u0StM6xQh5vtUm/TZExFxBe2fyXpr+q90vhYRLxVqp7tDZJ+KOli2wck/S4iHi1VT7213p2S3mz2myXptxHx50L1LpP0hO0J6j2RPxMRVd72quRSSZt7z586R9JTEfFCwXp3S1rfrIT2SbqrYK1TT16LJa3qdLnNS/kABlg/bLoDKIygAwkQdCABgg4kQNCBBPoq6IUPZxyzWtSj3ljX66ugS6r5y6z6h6Me9cayXr8FHUABRQ6YsT3QR+FMmzZtxN9z/PhxnXvuuaOqN336yE/mO3z4sC666KJR1Tt6dOTn3Bw7dkyTJ08eVb2DB0d+akNEqDk6bsROnjw5qu8bLyLif34xY34I7Hh00003Va334IMPVq23devWqvXWrCl+QthXHDlypGq9fsCmO5AAQQcSIOhAAgQdSICgAwkQdCABgg4kQNCBBFoFvebIJADdGzbozUUG/6DeJWivlrTC9tWlGwPQnTZr9KojkwB0r03Q04xMAgZVZye1NCfK1z5nF0ALbYLeamRSRKyVtFYa/NNUgfGmzab7QI9MAjIYdo1ee2QSgO612kdv5oSVmhUGoDCOjAMSIOhAAgQdSICgAwkQdCABgg4kQNCBBAg6kACTWkah9uSUWbNmVa03mpFT38Thw4er1lu2bFnVehs3bqxa72xYowMJEHQgAYIOJEDQgQQIOpAAQQcSIOhAAgQdSICgAwkQdCCBNiOZHrN9yPauGg0B6F6bNfofJS0p3AeAgoYNekS8IqnuWQcAOsU+OpAAs9eABDoLOrPXgP7FpjuQQJu31zZI+ruk2bYP2P55+bYAdKnNkMUVNRoBUA6b7kACBB1IgKADCRB0IAGCDiRA0IEECDqQAEEHEhiI2Wvz5s2rWq/2LLSrrrqqar19+/ZVrbdly5aq9Wr/f2H2GoAqCDqQAEEHEiDoQAIEHUiAoAMJEHQgAYIOJEDQgQQIOpBAm4tDXm57m+3dtt+yfU+NxgB0p82x7l9I+k1E7LA9RdKQ7S0RsbtwbwA60mb22gcRsaO5/5mkPZKml24MQHdGtI9ue6akOZJeLdEMgDJan6Zqe7KkZyWtjoijZ/k6s9eAPtUq6LYnqhfy9RHx3Nkew+w1oH+1edXdkh6VtCciHirfEoCutdlHv17SnZIW2t7Z3H5cuC8AHWoze+1vklyhFwCFcGQckABBBxIg6EACBB1IgKADCRB0IAGCDiRA0IEEBmL22rRp06rWGxoaqlqv9iy02mr/PjNijQ4kQNCBBAg6kABBBxIg6EACBB1IgKADCRB0IAGCDiRA0IEE2lwF9jzbr9l+o5m99kCNxgB0p82x7sclLYyIY8313f9m+y8R8Y/CvQHoSJurwIakY82HE5sbAxqAcaTVPrrtCbZ3SjokaUtEMHsNGEdaBT0iTkbEtZJmSFpg+5ozH2N7pe3ttrd33SSAb2ZEr7pHxKeStklacpavrY2I+RExv6vmAHSjzavul9ie2tw/X9JiSXtLNwagO21edb9M0hO2J6j3xPBMRDxfti0AXWrzqvu/JM2p0AuAQjgyDkiAoAMJEHQgAYIOJEDQgQQIOpAAQQcSIOhAAsxeG4WtW7dWrTfoav/9jhw5UrVeP2CNDiRA0IEECDqQAEEHEiDoQAIEHUiAoAMJEHQgAYIOJEDQgQRaB70Z4vC6bS4MCYwzI1mj3yNpT6lGAJTTdiTTDEm3SlpXth0AJbRdoz8s6V5JXxbsBUAhbSa13CbpUEQMDfM4Zq8BfarNGv16SUtt75f0tKSFtp8880HMXgP617BBj4j7I2JGRMyUtFzSSxFxR/HOAHSG99GBBEZ0KamIeFnSy0U6AVAMa3QgAYIOJEDQgQQIOpAAQQcSIOhAAgQdSICgAwkMxOy12rO05s2bV7VebbVnodX+fW7cuLFqvX7AGh1IgKADCRB0IAGCDiRA0IEECDqQAEEHEiDoQAIEHUiAoAMJtDoEtrnU82eSTkr6gks6A+PLSI51/1FEfFKsEwDFsOkOJNA26CHpRdtDtleWbAhA99puut8QEQdtf1fSFtt7I+KV0x/QPAHwJAD0oVZr9Ig42Px7SNJmSQvO8hhmrwF9qs001QtsTzl1X9LNknaVbgxAd9psul8qabPtU49/KiJeKNoVgE4NG/SI2Cfp+xV6AVAIb68BCRB0IAGCDiRA0IEECDqQAEEHEiDoQAIEHUjAEdH9Qu3uF/o1Zs2aVbOctm/fXrXeqlWrqta7/fbbq9ar/febP3+wT8eICJ/5OdboQAIEHUiAoAMJEHQgAYIOJEDQgQQIOpAAQQcSIOhAAgQdSKBV0G1Ptb3J9l7be2xfV7oxAN1pO8Dh95JeiIif2p4k6dsFewLQsWGDbvtCSTdK+pkkRcQJSSfKtgWgS2023a+U9LGkx22/bntdM8jhK2yvtL3ddt1TuwAMq03Qz5E0V9IjETFH0ueS1pz5IEYyAf2rTdAPSDoQEa82H29SL/gAxolhgx4RH0p63/bs5lOLJO0u2hWATrV91f1uSeubV9z3SbqrXEsAutYq6BGxUxL73sA4xZFxQAIEHUiAoAMJEHQgAYIOJEDQgQQIOpAAQQcSGIjZa7WtXLmyar377ruvar2hoaGq9ZYtW1a13qBj9hqQFEEHEiDoQAIEHUiAoAMJEHQgAYIOJEDQgQQIOpDAsEG3Pdv2ztNuR22vrtEcgG4Me824iHhb0rWSZHuCpIOSNhfuC0CHRrrpvkjSuxHxXolmAJQx0qAvl7ShRCMAymkd9Oaa7kslbfw/X2f2GtCn2g5wkKRbJO2IiI/O9sWIWCtprTT4p6kC481INt1XiM12YFxqFfRmTPJiSc+VbQdACW1HMn0u6TuFewFQCEfGAQkQdCABgg4kQNCBBAg6kABBBxIg6EACBB1IgKADCZSavfaxpNGcs36xpE86bqcfalGPerXqXRERl5z5ySJBHy3b2yNi/qDVoh71xroem+5AAgQdSKDfgr52QGtRj3pjWq+v9tEBlNFva3QABRB0IAGCDiRA0IEECDqQwH8An6mM7cqa+WgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compare the performance\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.linear_model.logistic import LogisticRegression\n",
    "from sklearn import decomposition\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# load digital data\n",
    "digits, dig_label = load_digits(return_X_y=True)\n",
    "print(digits.shape)\n",
    "\n",
    "# draw one digital\n",
    "plt.gray() \n",
    "plt.matshow(digits[0].reshape([8, 8])) \n",
    "plt.show() \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy train = 1.000000, accuracy_test = 0.905556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mao/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# calculate train/test data number\n",
    "N = len(digits)\n",
    "N_train = int(N*0.8)\n",
    "N_test = N - N_train\n",
    "\n",
    "# split train/test data\n",
    "x_train = digits[:N_train, :]\n",
    "y_train = dig_label[:N_train]\n",
    "x_test  = digits[N_train:, :]\n",
    "y_test  = dig_label[N_train:]\n",
    "\n",
    "# do logistic regression\n",
    "lr=LogisticRegression()\n",
    "lr.fit(x_train,y_train)\n",
    "\n",
    "pred_train = lr.predict(x_train)\n",
    "pred_test  = lr.predict(x_test)\n",
    "\n",
    "# calculate train/test accuracy\n",
    "acc_train = accuracy_score(y_train, pred_train)\n",
    "acc_test = accuracy_score(y_test, pred_test)\n",
    "print(\"accuracy train = %f, accuracy_test = %f\" % (acc_train, acc_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy train = 0.987474, accuracy_test = 0.894444\n"
     ]
    }
   ],
   "source": [
    "# do PCA with 'n_components=40'\n",
    "pca = decomposition.PCA(n_components=40)\n",
    "pca.fit(x_train)\n",
    "\n",
    "x_train_pca = pca.transform(x_train)\n",
    "x_test_pca = pca.transform(x_test)\n",
    "\n",
    "# do logistic regression\n",
    "lr=LogisticRegression()\n",
    "lr.fit(x_train_pca,y_train)\n",
    "\n",
    "pred_train = lr.predict(x_train_pca)\n",
    "pred_test  = lr.predict(x_test_pca)\n",
    "\n",
    "# calculate train/test accuracy\n",
    "acc_train = accuracy_score(y_train, pred_train)\n",
    "acc_test = accuracy_score(y_test, pred_test)\n",
    "print(\"accuracy train = %f, accuracy_test = %f\" % (acc_train, acc_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy train = 0.148921, accuracy_test = 0.102778\n"
     ]
    }
   ],
   "source": [
    "# do kernel PCA\n",
    "#   Ref: http://scikit-learn.org/stable/auto_examples/decomposition/plot_kernel_pca.html\n",
    "\n",
    "from sklearn.decomposition import PCA, KernelPCA\n",
    "\n",
    "kpca = KernelPCA(n_components=45, kernel=\"rbf\", fit_inverse_transform=True, gamma=10)\n",
    "kpca.fit(x_train)\n",
    "\n",
    "x_train_pca = kpca.transform(x_train)\n",
    "x_test_pca = kpca.transform(x_test)\n",
    "\n",
    "# do logistic regression\n",
    "lr=LogisticRegression()\n",
    "lr.fit(x_train_pca,y_train)\n",
    "\n",
    "pred_train = lr.predict(x_train_pca)\n",
    "pred_test  = lr.predict(x_test_pca)\n",
    "\n",
    "# calculate train/test accuracy\n",
    "acc_train = accuracy_score(y_train, pred_train)\n",
    "acc_test = accuracy_score(y_test, pred_test)\n",
    "print(\"accuracy train = %f, accuracy_test = %f\" % (acc_train, acc_test))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## References\n",
    "* [Pipelining: chaining a PCA and a logistic regression](http://scikit-learn.org/stable/auto_examples/plot_digits_pipe.html)\n",
    "* [PCA进行无监督降维](https://ljalphabeta.gitbooks.io/python-/content/pca.html)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}