graphkit-learn/lang/fr/notebooks/tests/test_modelselection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--- This is a regression problem ---\n",
      "\n",
      "\n",
      "1. Loading dataset from file...\n",
      "\n",
      "2. Calculating gram matrices. This could take a while...\n",
      "\n",
      " None edge weight specified. Set all weight to 1.\n",
      "\n",
      "getting sp graphs: 183it [00:00, 2868.18it/s]\n",
      "calculating kernels: 16836it [00:03, 4962.86it/s]\n",
      "\n",
      " --- shortest path kernel matrix of size 183 built in 3.5163042545318604 seconds ---\n",
      "\n",
      "the gram matrix with parameters {'node_kernels': {'symb': <function deltakernel at 0x7ff63c02c158>, 'nsymb': <function gaussiankernel at 0x7ff642e968c8>, 'mix': functools.partial(<function kernelproduct at 0x7ff60b9d21e0>, <function deltakernel at 0x7ff63c02c158>, <function gaussiankernel at 0x7ff642e968c8>)}, 'n_jobs': 1} is: \n",
      "\n",
      "\n",
      "\n",
      "1 gram matrices are calculated, 0 of which are ignored.\n",
      "\n",
      "3. Fitting and predicting using nested cross validation. This could really take a while...\n",
      "cross validation: 2it [00:00,  4.47it/s]\n",
      "\n",
      "4. Getting final performance...\n",
      "best_params_out:  [{'node_kernels': {'symb': <function deltakernel at 0x7ff63c02c158>, 'nsymb': <function gaussiankernel at 0x7ff642e968c8>, 'mix': functools.partial(<function kernelproduct at 0x7ff60b9d21e0>, <function deltakernel at 0x7ff63c02c158>, <function gaussiankernel at 0x7ff642e968c8>)}, 'n_jobs': 1}]\n",
      "best_params_in:  [{'alpha': 0.0001}]\n",
      "\n",
      "best_val_perf:  9.922073568477266\n",
      "best_val_std:  0.3829108688812842\n",
      "final_performance:  [8.039190309451554]\n",
      "final_confidence:  [2.8576078550320037]\n",
      "train_performance: [6.285008316076738]\n",
      "train_std:  [0.23613211181729038]\n",
      "\n",
      "time to calculate gram matrix with different hyper-params: 3.52±nans\n",
      "time to calculate best gram matrix: 3.52±nans\n",
      "total training time with all hyper-param choices: 4.34s\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:140: RuntimeWarning: Degrees of freedom <= 0 for slice\n",
      "  keepdims=keepdims)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:132: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import sys\n",
    "import functools\n",
    "sys.path.insert(0, \"../../\")\n",
    "from gklearn.utils.model_selection_precomputed import model_selection_for_precomputed_kernel\n",
    "from gklearn.kernels.spKernel import spkernel\n",
    "from gklearn.utils.kernels import deltakernel, gaussiankernel, kernelproduct\n",
    "\n",
    "datafile = '../../datasets/acyclic/dataset_bps.ds'\n",
    "estimator = spkernel\n",
    "# hyper-parameters\n",
    "mixkernel = functools.partial(kernelproduct, deltakernel, gaussiankernel)\n",
    "param_grid_precomputed = {'node_kernels': [{'symb': deltakernel, 'nsymb': gaussiankernel, 'mix': mixkernel}]}\n",
    "param_grid = {\"alpha\": np.logspace(-5, 5, num = 21, base = 10)}\n",
    "\n",
    "model_selection_for_precomputed_kernel(datafile, estimator, param_grid_precomputed, param_grid, \n",
    "                                       'regression', NUM_TRIALS=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'o1': 1, 'o2': 2}, {'o1': 3, 'o2': 4}]\n",
      "[{'i2': 6, 'i1': 5}, {'i2': 8, 'i1': 7}, {'i2': 10, 'i1': 9}]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[({'o1': 1, 'o2': 2}, {'o1': 3, 'o2': 4}),\n",
       " ({'o1': 3, 'o2': 4}, {'o1': 1, 'o2': 2})]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = [ {'o1':1,'o2':2},{'o1':3,'o2':4}]\n",
    "print(x)\n",
    "y = [ {'i1':5,'i2':6},{'i1':7,'i2':8},{'i1':9,'i2':10}]\n",
    "print(y)\n",
    "from itertools import permutations\n",
    "[item for item in permutations(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n",
      "\n",
      "0\n",
      "{'mean_fit_time': array([0.00053245, 0.00036639, 0.00027829, 0.00026679, 0.00036031,\n",
      "       0.00035846]), 'std_fit_time': array([2.35126922e-04, 6.76477532e-05, 1.11031151e-06, 1.48806431e-05,\n",
      "       7.18533189e-05, 6.76927198e-05]), 'mean_score_time': array([0.00021952, 0.00019908, 0.00016928, 0.00015849, 0.00017059,\n",
      "       0.00019705]), 'std_score_time': array([3.96984386e-05, 4.61193611e-05, 7.34855414e-07, 2.82289511e-06,\n",
      "       1.66522193e-05, 2.63754426e-05]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.92857143]), 'split1_test_score': array([0.89285714, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.96428571]), 'split2_test_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.82142857, 0.92857143, 0.92857143, 0.92857143, 0.92857143,\n",
      "       0.92857143]), 'mean_test_score': array([0.88392857, 0.95535714, 0.95535714, 0.97321429, 0.97321429,\n",
      "       0.95535714]), 'std_test_score': array([0.05282214, 0.02961272, 0.02961272, 0.02961272, 0.02961272,\n",
      "       0.02961272]), 'rank_test_score': array([6, 3, 3, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.91666667, 0.97619048, 0.97619048, 0.97619048, 0.97619048,\n",
      "       0.98809524]), 'split1_train_score': array([0.91666667, 0.97619048, 0.96428571, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split2_train_score': array([0.9047619 , 0.97619048, 0.97619048, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split3_train_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.98809524]), 'mean_train_score': array([0.92559524, 0.98214286, 0.97916667, 0.98214286, 0.98214286,\n",
      "       0.98214286]), 'std_train_score': array([0.02286055, 0.01030983, 0.01297291, 0.01330993, 0.01330993,\n",
      "       0.01030983])}\n",
      "\n",
      "1\n",
      "{'mean_fit_time': array([0.00036663, 0.0002979 , 0.00025547, 0.0002926 , 0.00026369,\n",
      "       0.0002656 ]), 'std_fit_time': array([2.12649960e-05, 2.13949957e-05, 4.63078609e-06, 4.18155790e-05,\n",
      "       1.60335387e-05, 7.18625396e-06]), 'mean_score_time': array([0.00018156, 0.00017273, 0.00015974, 0.00017357, 0.00015533,\n",
      "       0.00015408]), 'std_score_time': array([2.59263076e-06, 1.78372315e-05, 8.02872574e-06, 2.67129904e-05,\n",
      "       1.22153172e-06, 7.79431869e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.89285714, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.92857143]), 'split1_test_score': array([0.82142857, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.96428571]), 'split3_test_score': array([0.89285714, 0.92857143, 0.89285714, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'mean_test_score': array([0.88392857, 0.95535714, 0.94642857, 0.98214286, 0.98214286,\n",
      "       0.96428571]), 'std_test_score': array([0.03891874, 0.02961272, 0.03992979, 0.01785714, 0.01785714,\n",
      "       0.02525381]), 'rank_test_score': array([6, 4, 5, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.92857143, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split1_train_score': array([0.91666667, 0.96428571, 0.96428571, 0.97619048, 0.97619048,\n",
      "       0.97619048]), 'split2_train_score': array([0.92857143, 0.95238095, 0.95238095, 0.97619048, 0.98809524,\n",
      "       0.97619048]), 'split3_train_score': array([0.91666667, 0.95238095, 0.95238095, 1.        , 1.        ,\n",
      "       0.98809524]), 'mean_train_score': array([0.92261905, 0.96130952, 0.96130952, 0.98511905, 0.98809524,\n",
      "       0.98214286]), 'std_train_score': array([0.00595238, 0.00987091, 0.00987091, 0.00987091, 0.00841794,\n",
      "       0.00595238])}\n",
      "\n",
      "2\n",
      "{'mean_fit_time': array([0.00041491, 0.00027972, 0.00026184, 0.00033522, 0.0002436 ,\n",
      "       0.00027043]), 'std_fit_time': array([3.45070743e-05, 1.29256902e-05, 1.01382227e-05, 9.88078909e-05,\n",
      "       8.90705614e-06, 2.57328515e-05]), 'mean_score_time': array([0.00019699, 0.00015837, 0.00018364, 0.00016433, 0.00014621,\n",
      "       0.00014848]), 'std_score_time': array([1.55497312e-05, 1.34209755e-06, 4.97970678e-05, 1.56898401e-05,\n",
      "       9.88431212e-07, 8.49235466e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split1_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.85714286, 0.89285714, 0.89285714, 0.89285714, 0.92857143,\n",
      "       0.92857143]), 'mean_test_score': array([0.90178571, 0.95535714, 0.95535714, 0.96428571, 0.97321429,\n",
      "       0.97321429]), 'std_test_score': array([0.04639422, 0.04639422, 0.04639422, 0.04374089, 0.02961272,\n",
      "       0.02961272]), 'rank_test_score': array([6, 4, 4, 3, 1, 1], dtype=int32), 'split0_train_score': array([0.94047619, 0.96428571, 0.96428571, 0.96428571, 0.96428571,\n",
      "       0.97619048]), 'split1_train_score': array([0.91666667, 0.97619048, 0.96428571, 0.97619048, 0.97619048,\n",
      "       0.98809524]), 'split2_train_score': array([0.89285714, 0.95238095, 0.95238095, 0.97619048, 0.97619048,\n",
      "       0.97619048]), 'split3_train_score': array([0.92857143, 0.98809524, 0.98809524, 1.        , 0.98809524,\n",
      "       1.        ]), 'mean_train_score': array([0.91964286, 0.9702381 , 0.9672619 , 0.97916667, 0.97619048,\n",
      "       0.98511905]), 'std_train_score': array([0.01760738, 0.01330993, 0.01297291, 0.01297291, 0.00841794,\n",
      "       0.00987091])}\n",
      "\n",
      "3\n",
      "{'mean_fit_time': array([0.00033492, 0.0002749 , 0.00025362, 0.00025046, 0.00024259,\n",
      "       0.00024784]), 'std_fit_time': array([1.22110993e-05, 8.60786829e-06, 1.90268632e-06, 8.94069672e-06,\n",
      "       1.44803716e-05, 9.40847565e-06]), 'mean_score_time': array([0.0001812 , 0.00015831, 0.0001632 , 0.00018615, 0.00014585,\n",
      "       0.00014651]), 'std_score_time': array([1.21605162e-05, 1.35919502e-06, 1.42478791e-05, 6.11994010e-05,\n",
      "       1.97686242e-07, 6.84805232e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 1.        ,\n",
      "       0.96428571]), 'split1_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.89285714]), 'split2_test_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.92857143, 0.92857143, 0.92857143, 0.89285714, 0.89285714,\n",
      "       0.85714286]), 'mean_test_score': array([0.90178571, 0.94642857, 0.94642857, 0.95535714, 0.96428571,\n",
      "       0.92857143]), 'std_test_score': array([0.04639422, 0.03092948, 0.03092948, 0.03891874, 0.04374089,\n",
      "       0.05646924]), 'rank_test_score': array([6, 3, 3, 2, 1, 5], dtype=int32), 'split0_train_score': array([0.92857143, 0.94047619, 0.94047619, 0.98809524, 0.98809524,\n",
      "       0.97619048]), 'split1_train_score': array([0.95238095, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_train_score': array([0.92857143, 0.95238095, 0.95238095, 0.97619048, 0.96428571,\n",
      "       0.97619048]), 'split3_train_score': array([0.91666667, 0.96428571, 0.97619048, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'mean_train_score': array([0.93154762, 0.95535714, 0.95833333, 0.98809524, 0.98511905,\n",
      "       0.98809524]), 'std_train_score': array([0.01297291, 0.00987091, 0.01330993, 0.00841794, 0.01297291,\n",
      "       0.01190476])}\n",
      "\n",
      "4\n",
      "{'mean_fit_time': array([0.00034726, 0.00027776, 0.0002659 , 0.00026214, 0.00024235,\n",
      "       0.00024557]), 'std_fit_time': array([7.59672831e-06, 6.21948121e-06, 8.52721472e-06, 5.29509361e-06,\n",
      "       5.88971106e-06, 2.99212942e-06]), 'mean_score_time': array([0.00018328, 0.00016528, 0.00016391, 0.00015694, 0.00015819,\n",
      "       0.00014925]), 'std_score_time': array([5.45341785e-06, 1.78714566e-06, 1.12461819e-06, 1.38380370e-06,\n",
      "       4.00550309e-06, 8.92080638e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([1.        , 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.96428571]), 'split1_test_score': array([0.89285714, 0.92857143, 0.92857143, 0.92857143, 0.96428571,\n",
      "       0.96428571]), 'split2_test_score': array([0.92857143, 0.92857143, 0.96428571, 0.96428571, 1.        ,\n",
      "       0.96428571]), 'split3_test_score': array([0.57142857, 0.92857143, 0.92857143, 0.92857143, 0.89285714,\n",
      "       0.89285714]), 'mean_test_score': array([0.84821429, 0.94642857, 0.95535714, 0.95535714, 0.96428571,\n",
      "       0.94642857]), 'std_test_score': array([0.16439243, 0.03092948, 0.02961272, 0.02961272, 0.04374089,\n",
      "       0.03092948]), 'rank_test_score': array([6, 4, 2, 2, 1, 4], dtype=int32), 'split0_train_score': array([0.89285714, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split1_train_score': array([0.94047619, 0.97619048, 0.97619048, 1.        , 0.98809524,\n",
      "       1.        ]), 'split2_train_score': array([0.94047619, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split3_train_score': array([0.91666667, 0.96428571, 0.96428571, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.92261905, 0.97321429, 0.97321429, 0.99107143, 0.98809524,\n",
      "       0.99107143]), 'std_train_score': array([0.01974181, 0.00515491, 0.00515491, 0.00515491, 0.        ,\n",
      "       0.00515491])}\n",
      "\n",
      "5\n",
      "{'mean_fit_time': array([0.00037438, 0.00028503, 0.0002768 , 0.00026542, 0.00026023,\n",
      "       0.00027144]), 'std_fit_time': array([4.16169772e-05, 7.12169506e-06, 1.29175793e-05, 1.24644985e-05,\n",
      "       1.07069587e-05, 4.76091517e-06]), 'mean_score_time': array([0.00018823, 0.00016671, 0.00016403, 0.00015712, 0.0001545 ,\n",
      "       0.00015604]), 'std_score_time': array([2.45756246e-06, 1.61153148e-06, 2.50623179e-06, 6.95103633e-07,\n",
      "       2.53442426e-06, 2.43432258e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.78571429, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split1_test_score': array([0.82142857, 0.89285714, 0.89285714, 0.92857143, 0.92857143,\n",
      "       0.89285714]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.92857143, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.89285714]), 'mean_test_score': array([0.86607143, 0.95535714, 0.95535714, 0.98214286, 0.98214286,\n",
      "       0.94642857]), 'std_test_score': array([0.06376275, 0.03891874, 0.03891874, 0.03092948, 0.03092948,\n",
      "       0.05357143]), 'rank_test_score': array([6, 3, 3, 1, 1, 5], dtype=int32), 'split0_train_score': array([0.80952381, 0.96428571, 0.96428571, 0.96428571, 0.96428571,\n",
      "       0.98809524]), 'split1_train_score': array([0.91666667, 0.98809524, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_train_score': array([0.92857143, 0.96428571, 0.96428571, 0.98809524, 0.97619048,\n",
      "       0.98809524]), 'split3_train_score': array([0.89285714, 0.96428571, 0.96428571, 0.97619048, 0.97619048,\n",
      "       0.97619048]), 'mean_train_score': array([0.88690476, 0.9702381 , 0.9702381 , 0.98214286, 0.97916667,\n",
      "       0.98809524]), 'std_train_score': array([0.04648958, 0.01030983, 0.01030983, 0.01330993, 0.01297291,\n",
      "       0.00841794])}\n",
      "\n",
      "6\n",
      "{'mean_fit_time': array([0.00034338, 0.0002805 , 0.00026369, 0.00026369, 0.00024992,\n",
      "       0.0002557 ]), 'std_fit_time': array([3.23131947e-06, 3.28853469e-06, 5.84003864e-07, 1.35710234e-05,\n",
      "       1.34557403e-05, 9.11074105e-06]), 'mean_score_time': array([0.00018913, 0.00016457, 0.00017023, 0.00016582, 0.00015312,\n",
      "       0.0001536 ]), 'std_score_time': array([1.62092856e-05, 1.56682008e-06, 1.34922589e-05, 1.07849824e-05,\n",
      "       7.22667912e-07, 3.09714819e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.85714286, 0.96428571, 0.96428571, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split1_test_score': array([0.96428571, 1.        , 0.96428571, 1.        , 1.        ,\n",
      "       0.96428571]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.96428571]), 'mean_test_score': array([0.91964286, 0.99107143, 0.98214286, 0.99107143, 0.99107143,\n",
      "       0.97321429]), 'std_test_score': array([0.03891874, 0.01546474, 0.01785714, 0.01546474, 0.01546474,\n",
      "       0.01546474]), 'rank_test_score': array([6, 1, 4, 1, 1, 5], dtype=int32), 'split0_train_score': array([0.89285714, 0.98809524, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'split1_train_score': array([0.96428571, 0.98809524, 0.98809524, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split2_train_score': array([0.92857143, 0.98809524, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split3_train_score': array([0.95238095, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.93452381, 0.98511905, 0.98214286, 0.99107143, 0.99107143,\n",
      "       0.99107143]), 'std_train_score': array([0.02727724, 0.00515491, 0.00595238, 0.00515491, 0.00515491,\n",
      "       0.00515491])}\n",
      "\n",
      "7\n",
      "{'mean_fit_time': array([0.00034738, 0.00028139, 0.00030065, 0.00026196, 0.00025433,\n",
      "       0.00026512]), 'std_fit_time': array([2.23039275e-05, 5.79454027e-06, 6.06874812e-05, 6.98443351e-06,\n",
      "       1.43758446e-05, 1.10845410e-05]), 'mean_score_time': array([0.00017852, 0.00016618, 0.0001632 , 0.00016266, 0.00015312,\n",
      "       0.00015479]), 'std_score_time': array([2.90721617e-06, 1.46971083e-06, 2.00541065e-06, 1.30569549e-05,\n",
      "       1.66359970e-06, 2.50268540e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.96428571, 1.        , 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split1_test_score': array([0.75, 1.  , 1.  , 1.  , 1.  , 1.  ]), 'split2_test_score': array([0.67857143, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.96428571]), 'split3_test_score': array([0.96428571, 0.92857143, 0.92857143, 1.        , 1.        ,\n",
      "       0.96428571]), 'mean_test_score': array([0.83928571, 0.97321429, 0.96428571, 1.        , 1.        ,\n",
      "       0.98214286]), 'std_test_score': array([0.12752551, 0.02961272, 0.02525381, 0.        , 0.        ,\n",
      "       0.01785714]), 'rank_test_score': array([6, 4, 5, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.94047619, 0.96428571, 0.97619048, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split1_train_score': array([0.83333333, 0.97619048, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_train_score': array([0.80952381, 0.98809524, 0.98809524, 0.98809524, 1.        ,\n",
      "       0.98809524]), 'split3_train_score': array([0.89285714, 0.98809524, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'mean_train_score': array([0.86904762, 0.97916667, 0.98511905, 0.99404762, 0.99702381,\n",
      "       0.99702381]), 'std_train_score': array([0.05120432, 0.00987091, 0.00515491, 0.00595238, 0.00515491,\n",
      "       0.00515491])}\n",
      "\n",
      "8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([0.00067562, 0.00042343, 0.00038534, 0.00038195, 0.00039953,\n",
      "       0.00027204]), 'std_fit_time': array([1.49889683e-04, 3.03546273e-05, 1.74643643e-05, 1.06530693e-05,\n",
      "       7.33150457e-05, 1.69947466e-05]), 'mean_score_time': array([0.00028437, 0.00026864, 0.00024623, 0.00024372, 0.0002315 ,\n",
      "       0.00015861]), 'std_score_time': array([5.55748990e-05, 2.98201389e-05, 1.44350341e-05, 9.34615275e-06,\n",
      "       6.64798903e-06, 2.83294550e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.89285714, 1.        , 1.        , 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split1_test_score': array([0.89285714, 1.        , 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_test_score': array([0.89285714, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.78571429, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'mean_test_score': array([0.86607143, 1.        , 0.99107143, 0.99107143, 0.99107143,\n",
      "       0.99107143]), 'std_test_score': array([0.04639422, 0.        , 0.01546474, 0.01546474, 0.01546474,\n",
      "       0.01546474]), 'rank_test_score': array([6, 1, 2, 2, 2, 2], dtype=int32), 'split0_train_score': array([0.91666667, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split1_train_score': array([0.91666667, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split2_train_score': array([0.95238095, 0.98809524, 0.98809524, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split3_train_score': array([0.91666667, 1.        , 1.        , 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.92559524, 0.99107143, 0.99107143, 0.99107143, 0.99107143,\n",
      "       0.99404762]), 'std_train_score': array([0.01546474, 0.00987091, 0.00987091, 0.00515491, 0.00515491,\n",
      "       0.00595238])}\n",
      "\n",
      "9\n",
      "{'mean_fit_time': array([0.00042182, 0.00032252, 0.00031126, 0.00025773, 0.0002439 ,\n",
      "       0.00026196]), 'std_fit_time': array([8.71577873e-05, 3.58894707e-05, 1.64262442e-05, 3.96449307e-06,\n",
      "       4.98117981e-06, 1.21702993e-05]), 'mean_score_time': array([0.00018883, 0.0001893 , 0.00019503, 0.00015533, 0.00015396,\n",
      "       0.00015324]), 'std_score_time': array([4.33436846e-06, 1.25877333e-05, 2.04790318e-05, 5.46285593e-07,\n",
      "       7.79431869e-07, 3.12398855e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.92857143, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split1_test_score': array([0.92857143, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.92857143]), 'split2_test_score': array([1., 1., 1., 1., 1., 1.]), 'split3_test_score': array([0.89285714, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'mean_test_score': array([0.9375    , 0.97321429, 0.97321429, 0.99107143, 0.99107143,\n",
      "       0.97321429]), 'std_test_score': array([0.03891874, 0.02961272, 0.02961272, 0.01546474, 0.01546474,\n",
      "       0.02961272]), 'rank_test_score': array([6, 3, 3, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.9047619 , 0.98809524, 0.98809524, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split1_train_score': array([0.9047619 , 0.96428571, 0.94047619, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split2_train_score': array([0.92857143, 0.96428571, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.97619048]), 'split3_train_score': array([0.91666667, 0.95238095, 0.95238095, 0.97619048, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.91369048, 0.9672619 , 0.96428571, 0.98511905, 0.98809524,\n",
      "       0.99107143]), 'std_train_score': array([0.00987091, 0.01297291, 0.01882308, 0.00515491, 0.        ,\n",
      "       0.00987091])}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, KFold, train_test_split\n",
    "import numpy as np\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "# Number of random trials\n",
    "NUM_TRIALS = 10\n",
    "\n",
    "# Load the dataset\n",
    "iris = load_iris()\n",
    "X_iris = iris.data\n",
    "y_iris = iris.target\n",
    "\n",
    "# Set up possible values of parameters to optimize over\n",
    "p_grid = {\"C\": [1, 10, 100],\n",
    "          \"gamma\": [.01, .1]}\n",
    "\n",
    "# We will use a Support Vector Classifier with \"rbf\" kernel\n",
    "svm = SVC(kernel=\"rbf\")\n",
    "\n",
    "# Arrays to store scores\n",
    "val_scores =  np.zeros((NUM_TRIALS,len(p_grid['C'])))\n",
    "test_scores = np.zeros((NUM_TRIALS,len(p_grid['C'])))\n",
    "\n",
    "best_params = []\n",
    "# Loop for each trial\n",
    "for i in range(NUM_TRIALS): #Test set level\n",
    "    print()\n",
    "    print(i)\n",
    "    X_app,X_test,y_app,y_test = train_test_split(X_iris,y_iris)\n",
    "    inner_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "    # Non_nested parameter search and scoring\n",
    "    clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv)\n",
    "    clf.fit(X_app, y_app)\n",
    "    print(clf.cv_results_)\n",
    "    best_params.append(clf.best_params_)\n",
    "    val_scores[i] = clf.best_score_\n",
    "    test_scores[i] = clf.score(X_test,y_test)\n",
    "\n",
    "final_performance = np.mean(test_scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_split.py:1943: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.datasets.samples_generator import make_classification\n",
    "from sklearn.svm import LinearSVC, SVC\n",
    "from sklearn.metrics import f1_score, precision_score\n",
    "\n",
    "X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)\n",
    "# compute the training kernel matrix corresponding to the linear kernel\n",
    "K_train = np.dot(X_[:180], X_[:180].T)\n",
    "y_train = y_[:180]\n",
    "clf = SVC(kernel='precomputed')\n",
    "cv = GridSearchCV(clf, {'C': [0.1, 1.0]})\n",
    "cv.fit(K_train, y_train)\n",
    "# compute the test kernel matrix\n",
    "K_test = np.dot(X_[180:], X_[:180].T)\n",
    "y_test = y_[180:]\n",
    "y_pred = cv.predict(K_test)\n",
    "np.mean(y_pred == y_test) >= 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 3] [0 1]\n",
      "[0 1] [2 3]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "X = [\"a\", \"b\", \"c\", \"d\"]\n",
    "kf = KFold(n_splits=2)\n",
    "for train, test in kf.split(X):\n",
    "    print(\"%s %s\" % (train, test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9733333333333334"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import datasets\n",
    "from sklearn import svm\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "iris.data.shape, iris.target.shape\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    iris.data, iris.target, test_size=0.4, random_state=0)\n",
    "\n",
    "X_train.shape, y_train.shape\n",
    "\n",
    "X_test.shape, y_test.shape\n",
    "\n",
    "\n",
    "clf = svm.SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)\n",
    "\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, cross_validate\n",
    "clf = svm.SVC(kernel='linear', C=1)\n",
    "scores = cross_validate(clf, iris.data, iris.target, cv=5, return_train_score=True)\n",
    "scores  \n",
    "# print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "\n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import accuracy_score\n",
    "predicted = cross_val_predict(clf, iris.data, iris.target, cv=10)\n",
    "accuracy_score(iris.target, predicted) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=0, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.085) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.079) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.079) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.1}\n",
      "0.960 (+/-0.060) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.038) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=1, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.101) for {'C': 1, 'gamma': 0.01}\n",
      "0.947 (+/-0.064) for {'C': 1, 'gamma': 0.1}\n",
      "0.940 (+/-0.086) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.1}\n",
      "0.960 (+/-0.027) for {'C': 100, 'gamma': 0.01}\n",
      "0.933 (+/-0.080) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=2, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.039) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.066) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.070) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=3, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.117) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.046) for {'C': 10, 'gamma': 0.1}\n",
      "0.960 (+/-0.046) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.059) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=4, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.042) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.037) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=5, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.066) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.023) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.023) for {'C': 100, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=6, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.084) for {'C': 1, 'gamma': 0.01}\n",
      "0.947 (+/-0.064) for {'C': 1, 'gamma': 0.1}\n",
      "0.953 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.001) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.039) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=7, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.124) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.087) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.058) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.058) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.933 (+/-0.096) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=8, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.078) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.026) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.084) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=9, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.129) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.045) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.028) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=10, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.101) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.044) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=11, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.083) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.026) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.043) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.037) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=12, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.069) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.024) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=13, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.091) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 1, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.053) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.065) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=14, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.086) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=15, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.953 (+/-0.044) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.038) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.059) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=16, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.900 (+/-0.088) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.024) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.057) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=17, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.084) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.043) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=18, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.059) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.054) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.054) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.045) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.070) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=19, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.907 (+/-0.057) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.057) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.043) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.023) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=20, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.893 (+/-0.107) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.038) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.023) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.080) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=21, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.038) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 1, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.987 (+/-0.046) for {'C': 100, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=22, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.078) for {'C': 1, 'gamma': 0.01}\n",
      "0.940 (+/-0.044) for {'C': 1, 'gamma': 0.1}\n",
      "0.953 (+/-0.043) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.953 (+/-0.043) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.045) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=23, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.057) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.091) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=24, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.907 (+/-0.048) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.070) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.066) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=25, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.069) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.043) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.068) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=26, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.907 (+/-0.091) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=27, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.001) for {'C': 1, 'gamma': 0.01}\n",
      "0.980 (+/-0.044) for {'C': 1, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.023) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.967 (+/-0.044) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=28, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.933 (+/-0.060) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.089) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=29, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.022) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.001) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.043) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average difference of 0.007742 with std. dev. of 0.007688.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAEiCAYAAAB6PJwbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzsnXlYVcUbx7+vuCDijppbobmiIi5Y7pZbmru5oKVmZmppWbn3azWzMi2X0jJ3TdMWyxZTyzUtwRQRUdFQUXLBBRFB4L6/P95z4HK5l7vCxct8nuc8cOfMmeWcOeedeeedd4iZoVAoFAqFJ1HI3QVQKBQKhcLVKOGmUCgUCo9DCTeFQqFQeBxKuCkUCoXC41DCTaFQKBQehxJuCoVCofA47BZuRPQmETERbTVzbhMR7XRJyWwvT4xWnhCTcF8tfISL8xvoyjRzq5wOlCOUiFZYibNTK+s0M+euEtGbLi5TFyJ6ycVp2lROIipFRG8TUSQR3SGiW0S0h4hGEZGXK8vkCohoARHFE1ERC+dfJaJ0IqpiY3qFtWc9xihsLxGtt3JdGe26J+0s/2AiGmYm3Gqe9wpEVEu7N4/ZEHcNER1wMr81Wn5MRGla299DRJOJqJQD6Xlr3/9AZ8rlDEQ0hoh62RLXmZFbFyIKduJ6VzOdiCgP8hkIYEQe5JOfmUhEPnmQTxcALhVutkBEFQEcADAKwDIA3QEMALAfwFwAPfK6TDbwFYBykHtmjsEAdjHzRSfyGA3gNSeuz4nBALIJt1zOMz/zBoBnXJDOMQAtAbQFMBzAbgBTAPxDRNXtTMtbK5fbhBuAMQByVbhdA3AUwAwHr3c1OwE0BNDbzeUoCOwHUAry0fFUPgNQFkAwM89h5j+Y+VdmngxpZ2fMXUSCd14W1Ij9AM5ChEQWiKgWgGYQAegwzBzJzNHOpHEv5OlOiKg4ADDzaWY+5oIkE5n5ADPvZ+afmHkGgCYASkA6bh6Lo8KNAbwLoBcRNcopIhEFEdEOIkoioutEtJaIKhmd99eGzQOJaAkR3SSiWCJ6i4hsLV8YgF9hg7Alot6aCi6ZiP4jog+MVTlEVI2Iviaiy5o66jQRvaOdWwGgP4D2RsP9N21NW4vTn4hOamnvBlDPlgoS0WwiOkpEidr9WUtE95nEiSGiOUQ0UYtznYjWE1EZk3gNiWifVs7jtg7zNS4CWA7gVSIqZqXMbYlol/bs44noCyIqaXS+DBEtJaKLWlnOEdEX2rk3AbwC4AGje73C1rS1OO2I6IiWdhgRtbJWOSLyB9AXwCxmvmB6npnPMfNRvYyaqqcNER0EkAwZ4YGIahDR90SUQKLS/FETMsZ5PUOZas+rWn0aGJ2fRkTRWvkvEdGvps/cqFwMYD2A3mYE7GAAqQC+0dItSUSLiOiEdv/+JaKFpvfPzL3JpiLU3ttTWh12Aqhj5rqntfZ2TTt2EFFTo/NrIB3TjkbP+rUc8uxERH8bvWcLyUiToJ1nrY18Q0S3iegMET2XU/20a72I6B0iOk9EKUQUQUSDTeKsIaIDRNRVO59Iou6rby19M/nFEtH7RPQGEV2ADByyqSWJqCwRLSOiOK3eZ4losb35AdKGAcwE0Elvk9baBBEVBnBdS2K10XOqpp3/kLJ+n1aTaECM69qXiA5pz+O6dg/bGp33IqIZJN/cFK0sTxmd3wugMYBnjPK3rP5mZrsOAG8CuAoRjCcArDc6twnATqPfFQDcgPQq+wB4EkAsgHAARbU4/hBhGQPgIwCdAczWwgbaUJ4YAHMAtNaueUwL99V+jzCKOxBAOoBPIeqbsVr55hjF+d2ovB0AjATwgXbuQe38IQAPa0c1O9JuCiANwEYA3QBMgowCspTTQj2XAQgB0B7AE1oZIwEUMrkX5wBsgajSRgNIBPCpUZziAC4AOAKgH4ChWhkuA1hhpQw7tWfsD/lYjjE6dxXAm0a/WwNIAbBBK8tTWr6bTOoUBWCQVq8nAXyunasGYC2AOKN7/aAdaVcBcBvAHxA14mgA/wJIMi6nmTo+pT2P2ja+C0kATgN4DsAjAOoCKKbd0xNa3foDiNDKWE67tp12D6dp7awXgPcAtNLODwNwC8A47d70A7BQvwcWytNYK3t/k/AIAD8a/b4P0k77a2k/pZX1J6M4hbW0jJ/xXmR931tA2vwGAI9B1F3/atc9aXKfngXQEdLu12rP5gGj92oXgL+NnnVVC3kGavftB+3ZjwWQAGCLUZxOWhlOApgO+aas1MKaWnmm7wO4q13XFcCX2nUDjOKsAXAJwD+Q9743gGgAR6ykXQtG3ygtLBbSxn8D0BNAX6M8DhjFWwV53wci811ZbCW/LGmYnKtr/JxsbBP6fX3D6Dnp3/EVyPw+DQDwF0S7V8gov7va/X1Ee3avA+hllP4SSJt/VcvrQwAGZH7TA7Rnutkofz+L9bf2Alt4oa9q/4+ANO462m9T4TYb8oEvZRT2kHaDQrTf/trvVSb5HIZRo86hPDHQBAjk47tH+z+LcANAELXNcpPrRwK4A6C89jsRQM8c8stSRzvT/lproGQUZwZsEG4m6XoBqKpd187kXpwGUNgo7GMA/xn9Hgf5OFQzCtM7BjYJN6PGfEbPC9mF2x4Af5hc/6iWT0PtdwSA8TnkNwdAjJlwW9L+AEA8AB+jOEO1OG/mkOdULU4xG98FBtDbJHwMpBNT0yisGuTlnqb9fhVAWA5pLwTwja1twui6SAAbjX431Mo4JIdrCkM+SoxMoWKLcPsW0lE1bs9vwES4meRVSEs7GsB0o/DvAWw3E980z02QDpFxp26Ilmew9lv/CL9uFKcoZFQ0M4f74Ad5X2eYhP8G4JjR7zWQd8j4+T6h5Vkrh/QtCbdYaELCJA9j4RYFYKydbSEn4VZCK8srdrSJMjk9W6NrvQA8oMXVO2uDAVzK4Rpd2A41CV8HYL/R78MAltpSf2eXAqyBjBSyWc9ptADwGzMn6AHM/BfkI9zGJO5vJr8jIR8EABnWW/phyVptJoA2RNTezLk6AO4H8LVxWpCRmDfkIwDIzXuPiEYQ0f0W8nE07RYAfmDtKWl8a0sGRNSNiP4kopuQD2esUd7G/MHMaUa/IwFUpEz1aAvIR1W/Hsy8DzJys4f3IA14qJmy+kAmsU3vx17IR6GZFvUwgElENI6IsqmzzGFH2i0AbGPmJKPLv7Ojfmw9Ska8X0zCWgA4xMwZc3Pa/d6HzHZ/GEATIppHoj4tapLGYQDdSdTzLXJo86Z8BeBxIvLVfg+CjC43G0ciouFEdJiIEiH3bad2qraN+QA2tmciakCior0E6QynQkZrNj1zM3l+y8wGo7BNkB6+xW8KM9+FCNRqsEwg5H3daBK+AUAAEZUzCjtt/Hwh7xmspG+J7Vr5cuIwgClENJaI7HlGlshmfOdMmyCiHkS03+j7FKOd0p9xOAA/IlpORJ0pu0FaJy3PzSbv9Q7Ie2K3rHJKuGkf0Q8APElED5iJUhkyfDflEsSyy5gbJr/vQhqaPg+SanSctlCe7ZDhsLm5Nz/t788maf2rheuWQ4MAhAKYB+Cs9rA7msvPgbTvQ3YhYlWokFil/gARaE9BPu4Pa6dN51fM3UeCqMoslcGmchjDzCcgH5WpZhpeWUjv7VNkvR8pAIog8368AOmxvw7gBMncTTaDCAfTzlZPTdAlWklfn2eztWNz3cyHyWq719rq0xD15E4AV7U5jxJa3GUQ1dhASJu+REQzbRByX0FUz/o86iCISvK2HoGIBkBG3nshKqSHtL9A9vaUE5VgpT0TUWmIkKkCYCLEai8YMmp3xPgm273V7v912PFNySFtmKZv9LuslbRhJX1LmGsrpowF8CNEW3CSZN5+QM6X5EhV47ydaRNE9DDkPY6BqEtbQrRBGdcycyRkqqc2pDN4VZtX1L+dfpD39xayvtdLId+uLPN3tlDY3gvMsAxiqjvFzLk4mC9UJYgRiK1chLwQOik5xH0XIghamIRf0/6OhujKTfkXAFiMCEZoH+wWkMb0AxHdz8zxFvK0KW0A/yH7/bDlofUFcAXAIL2XbKEzYQv/wbwRi92NB3KvD0NUMsbcgKb+gwh8Uy4CADPfADABwASStTOTAawlonDtZTCHTWnDzL3Weou+2a7Iym4t/a6Qnr41zI3w4gA0MBNeCZltBcy8EsBKIqoAmVObB3m5p2ojk3kA5pGYbA+F3O9YABYNCZg5mohCAQwmohOQj8kkk2gDAOxj5hf0ALJiTGKBS7DenltDBFt7NrJ6JBMjJzvI9k3RtBJlYXRvnUgbWvo3jcJ1A7jryB2sagmY+TqA8UQ0ATLCnALgK+1dOeFAnvqSkf3aX2faRD/IvRti9H160EwdfgTwo9bh6Qlp3x9DBOI1SAehDczfD0vfXos47aGEmVMgcyMjkdnz0fkLQFfKaiEXDJln22tHHneZOdToOJpD3B8hxhKma2NOQHrl/iZp6Ue8SToGZj4A4C0APhAVHGC+92dr2gchFqbGKoF+NtyC4gBSTdQ/2dSBNnIQQDPdygkAiKg1HBBuzBwO6U1Oh5GaQxslHABQ18L9yLbWSktrEqRN6sI32722I+2DAEzVH31tqNNZiPpyOhGZtmcQUXWyYiEMaffNiKiG0XVVAbSCmXbPzFeYeQlkLjHAzPnzzDwbImyznTfDVxDhPAbSGTBVmxZH9g6iI+3JlvZcXPubkR8RtUN29Z21UZXOXwD6mWgLnoC0G5u/KRYIh5HFqxEDAUQys7PC02lYOAIRbl6QuSq70KZbZkDU9roWzJY2YWl0WhzAXVu/T8x8k5nXQAYhenv+HTIv6mvhvU41KoNNo2NXjNwAsXKZDnl5dxmFz4UMp7cS0fuQXvNsiBXNNy7K2xyzIHryDJjZQESvQMxYS0Fe+LsAakKGy09AhsVbIZZJJyHD4Vcgo4DjWlJREHPrPpBe9EVmvmgtbU0l9j7k5fyaiL6EzMXZslBzG4CXiOhjiDBpBentOMJyiOD/icTcvjiAdyAGIY7wLqROpkwGsIOIDBD15S2Iqu9xyIT9Sc209zuIioohFnW3IVZzgNzrSiTeWyIghkwxtqQN6RE+D2ALEc2FjB6mQQwGrDEW0o5DtWvDIG2hvZbmMEgbtsQKyMfnFyJ6HTLP9AbkHi8BACJ6C6JG26mFN9HSn6qdXwLpzR6AjCIegYzCzGlITNkAsTR7BmLkZKo23QbgYyKaqtWth5a3vbwP4E/ICGIFZEQxwiTOn5A5v6VENAfynN5A5ghbJwoyx9gb0lG8wMxxyM47Wpm/1e7RA5Bvyk/MfNCBOmTAzFeJaD6AN7S2dQgi6LpABJzbIKL9EIO0Y5CO5GhIu7dWZ19NbUiQ0W0rZHZ6jL89VtsEMycR0XkAg4joOEQYHtGufUF7V36CjL6yCDciGgegOeT7GgcRyv0g1qhg5mMky4A2EtEHWhmKQzQgNZlZX8YRBeARIuoCeT/OWOx0WLM4MT1gZC1pEj4d8oHaaRLeBCKVk7Qbug5AJaPz/tp1PUyuWwEg1IbyxMDI3F4LKwQRRtmsECGmyHsgH9EEiFptJkTQFwPwBWQklgT56GwB0Mjoej/IB/kaTCzvckrbKM4ASA88GdLTDDZXTjP1nAzgvJb2dsiHjgG8YOVejNDi+RqFBUI+OilaXftA5hlXWCnDThiZ2xuF/2Z6L7TwhyDrDxO0ckdCOjyltfMfQoTELa1t/AGgrdH13hBhfBkm1pzW0tbidID0xlO0Z9EaJladOdS1NORDGqU9q1sQleVIAF45vQvauZqQeYhbkHm+LTBaXgD5eOyAqJuTtecwFZrlofbc9mntLEmrxzN2vKc7tXvWycy5whCV0GXt/m2EfPQyLPlgg7WkFjYYMgeerN0f3RraeClAd+35JEM+ho+ZpgXRHHwPUf0xgNdyyLMz5KOeotVhIYASRud1a8l6JtdlS8vCvXkH0nG9C+lUhZjEyWaFCDOWkGbStmQtOdtMXFNrybmQdyVRu0e/A2htpS5rtPwY0sG6prWpyTCyYLe1TRh9445qz5KRuRRqGqRTchvyPdCtH8do51tDphHitOf2L8QorahR2oUAvKy1lRTIu7HTpC3Vgrw3N03bmemhv0gKhUKhUHgMalcAhUKhUHgcSrgpFAqFwuNQwk2hUCgUHocSbgqFQqHwOJRwUygUCoXHoYSbQqFQKDwOJdwUCoVC4XEo4aZQKBQKj0MJN4VCoVB4HEq4KRQKhcLjUMJNoVAoFB6HEm4KhUKh8DiUcFMoFAqFx6GEm0KhUCg8DldtVprv8PPzY39/f3cXQ6FQKO4pwsLCrjJzBXeXw1k8Vrj5+/sjNDTU3cVQKBSKewoiOuvuMrgCpZZUKBQKhcehhJtCoVAoPA4l3BQKhVuZ+9I5vPzEOXcXQ+FheOycm0KhuAe4eBFrFl5HLFfFXHeXReFRqJGbQqFwDwYD0kc8g8j0urhi8MPN8wnuLpHCg1DCTaFQuIdFi3B622mkwBsAcHr7v24ukMKTUMJNoVDkPZGRwOTJONZseEZQ9J+X3VgghaehhJtCochb7t4Fhg4FSpZERMcXQcQAgOijd9xcMIUnoYSbQqHIW15/HTh8GFi6FBExvqhZk1ClyBVEn1X2bQrXoYSbQqHIO3bvBj74AHj2WaBXL0REAA0bArX8riP6ahmA2d0lVHgISrgpFIq84eZN4KmngAcfBObORUoKcPKkJtweSEN0mj8QF+fuUio8BCXcFApF3jB+PHDhArBmDeDri5MngbQ0Tbg19EYcquD2wUh3l1LhISjhplAocp+vvwZWrwb+9z/goYcAABERcqphQ6BWS3FCf3pXrLtKqPAw1AyuQqHIXWJjgTFjRKjNmJERHBEBFC4M1KkDpKaWBABEH0pAoLvKqfAo1MhNoVDkHgYDMGKEmP+vXi3STCMiAqhbFyhaVKbhACD6lDIoUbiGPBVuRPQYEZ0gomgimmrm/ANEtIOIwoloJxFVMzr3AREdI6LjRDSfiCgvy65QKBxg/nxgxw5g3jygdu0sp3RLSQAoVQqo6HML0f/5ykScQuEkeSbciMgLwCIA3QAEAAghogCTaHMArGLmQABvA3hPu7YVgNYAAgE0BBAMoH0eFV2hUDhCRAQwdSrQqxcwalSWU7dvA2fOZAo3AHiwyh1EG2oAp0/ncUEVnkhezrm1ABDNzGcAgIjWA+gNwNg8KgDAy9r/fwD4XvufAXgDKAqAABQBcCkPyuwZHD0KREcDffu6uySuIS0NWLwYCAkBypfP+/z/+QfYsMG2uD4+wPPPu6ecucHBg8A339gWd/NmoHRp4IsvABNFS6T21hsLt1p1vbAzuhZw9G/RV+Y39uwB7twBunRxWZKrXjuJtb+UA/z8XJYm4i6i3n038MnW+tnue0EiL4VbVQDnjX7HAnjIJM4RAP0AfAKgL4CSRFSemfcT0R8A4iDCbSEzHzfNgIhGAxgNAPfff7/ra3AvEh8PdO0q64e2bwc6dnR3iZxn1izgjTeAS5eAd97J27z/+08+btevZ5k/skhKCnDgAPDTT/f+hyY2VtpSQoJtdff1FbP/ihWznTK2lNSp1aQUVv9UHncOrUTxJ1xUZlcREwP06CH1jouTiUJnYcasD7xwLdWABwNvAz4lnE8zORk4FovbV64AXO/eb3POwMx5cgB4AsBSo99PQYSUcZwqAL4F8A9EwMUCKAOgFoCfAPhqx34AbXPKr1mzZlzgMRiY+/dnLlKE2d+fuWpV5vh4d5fKOQ4cYPbyYi5UiLlmTaljXmEwMHfrxuztzRwZads1CxcyA8yLFuVu2XKb9HTmjh2ZS5RgPnnS6eRefpm5eHHmtLTMsLVr5VYd6zje6fRdSloac9u20uYA5i1bXJLsld8OMcD8Hk1lbtCA+c4d5xJMTWVu2ZK5dGnmc+ccTgZAKOeRXMjNIy8NSi4AqG70u5oWlgEzX2TmfszcBMAMLewGZBR3gJkTmTkRwC8AWuZNse9hVq0SFdLMmcCmTTLSGTv23nVxlJgIPPkkUKUK8NFHMmlz8GDe5b94MfDLL8CHHwL169t2zbhxwGOPAa++CkRF5W75cpMcDEMcISICCAgAvLwyw2rVkr/RkSlOp+9S5swRleSSJUDZssBXX7kk2T8XHgIAtP5fR+DYMWD6dOcSnD0b2L8f+OwzoHp16/E9nbySohAV6BkANSBzZ0cANDCJ4wegkPb/uwDe1v4fBGC7lkYRADsA9MwpvwI/cjtzhrlkSeZ27TK7x+++Kz3P1avdWzZHGT2amYj5jz+Yb9xgLlaM+cUX8ybvqCgZanTtav9o8eJF5vLlmZs1Y05JyZ3y5SZHj8q97tXLZSPlKlWYhw/PGhYfL83zI7zMnJjoknyc5tAh0XwMGCB1Hz1aRq/Oli8tjSeXWMhF6C4nJTHz889L5bdtcyy9v/8WjUZIiHPlYs8ZueVtZkB3ACcBnAYwQwt7G0Av7f8nAJzS4iwFUEwL9wKwBMBxiAHKXGt5FWjhlpbG3Lo1c6lSzDEx1sPvBX74QZrrpEmZYf36MVeqlFW3lRvcvSuCqXx55gsXHEvj22+l/DNmuLZsuU1yMnNgIHPFisyXLrkkSV2Iffhh9nNlfVN4LBbJx9rdJCUx168vklhX5//xhxT+q6+cS3vHDm6NPfxw7Svy+/Zt5nr1HJs6SExkrlOHuVo15mvXnCsXK+GW748CLdxyGqGZG9Hld/77j7lCBebGjeVjq7NpEzvV27WVGTMkn2++cS6dkSNl3mbvXteUKy+YNIldOc/EzLx7tyT5yy/ZzwU3usOdsZV52TKX5ecw48dLQX/7LTMsLU0EUM+eTiWd/PQYLoY7/MqEu5mBoaHMhQszDxxo3wh5zBjRaPz+u1Nl0lHCLZ8fBVa46S/IoEGWX5AVK+TRv/9+3pbNEQwG5h49RC0WEZH1XFKSjEJHjsy9/PfuFYH09NPOp5WQIEYwNWow37zpfHq5zR9/yEdzzBiXJvvpp9L8zp/Pfi5kcDrXoDPMEye6NE+7+fVXKaQ5tfcrr4iq0lHjrORk3ufbhQEZ0GfB3qmDLVsk/iuvOFYWMyjhls+PAincbt9mrltXepY5qSeMrSgPHcq78jnC4sXSTD/+2Pz54cPFOsxZSzNz3LwpgqhGDRFMrmDfPhGWI0a4Jr3c4vp15urVmWvXdvn817hx8sjM9b3+9z/mQkjjlEcfc2mednH1KnPlymLBmJSU/XxYmLTJzz93LP3vv+cP8CoDopTIgj1TB5cuibq4UaOsGg0nUcItnx8FUrjpk9Lbt1uPq7/AAQHmX+D8wIkTzD4+zJ07iym6ObZuZfNdYBfw9NO5o0Z87TUp86ZNrk3XlQwZIgYKuTD31a6dfL/NsXKl3JoT5Vu6PF+bMO74HT5sOU6dOswdOjiWx8CB3LvoT1yrlgXNij510L695akDg0EMfIoWZQ4Pd6wcFlDCLZ8fBU64/fyzPE571Dm6YJgwIffK5Sh37zIHBzOXLcscG2s5Xmqq9F4HDHBt/t98w7lmAHL3LnPz5szlyjluoJKbrFsndX/7bZcnbTBItZ97zvz5ffsk65/QjfnyZZfnbxVdZf/BBznHe/NNUdnm1DbNkZDABu/iXKF4QjZrUbPlsDR18Pnncn7uXPvytwEl3PL5UaCE25UrzPfdx9ywof3quQkTpBls3Zo7ZXOU11+Xcm3caD3u+PGysNpV81jGpvt371qP7wj60oIuXSyPSt3BuXOiM3z4Yek4uJiLF+WxLlhg/vylS3L+E4xn3rHD5fnniC0jJp0TJ6SgH31kXx6rV/MJ1Lau1TQeQf7zT9Zzp07JcoSOHXOl7Sjhls+PAiPcDAbmPn1EPXHkiP3XJyWJarJyZVFV5gf+/FPUgcOG2R4fEJ2WsxgMspateHHm48edTy8nPvtMyj1/fu7mYyvp6cyPPCIfzujoXMnit9+kyn/8Yf68wcBc0jedx+MTy/OsuYE+11W6NPPZs7Zd07y5dIDsoVs3XlbuFfHEcsxKXHNTB6mpzA89xFymjHmLHBeghFs+PwqMcPvyS3mMc+Y4noa+ULV//7x1Z2WOhATmBx8Ud2G2jsQMBonftavz+S9YwHnmLstgYH78cRl1Wv3S5QEffSR1X7o017KYO1eyyEnj2KSJgbsV2cb8zDO5Vo5s6FaKa9bYfo1+v6KibIt/+TKzlxc/E/g3ly1r46BLnzrQrTbffFN+r19vezntpEALNwCVdE8i+fUoEMItOprZ11d6286qJ2bPluawYoVryuYozzwjcxm7d9t33fTpYgDhzELjyEgRNN265Z2Q19fwBQW513vJkSMy+u/TJ1frPnKkrLvPiQEDmGsXPycjlLxAXz4zeLB9dY+Nlbb6xhu2xdfWQNSrcYcff9yO8ulTB7NmSRt/8kk7LrYfTxFuJHWxDhEV0VxijQVQHEAdZj5DRO8DOMvMn9qUUB7RvHlzDg0NdXcxBINBfPP17+86n29paUC7drJ3yNGjzqebng48+qhs5zJypHu8iSckAMuWAdOmied/ezh2TFzML1woW8zYy927wMMPA+fPy/287z6LUf/+Gzh3DnjCVZ7rf/gB6N0b6N4dqFMnx6inrvuh5w+jsLzzV2hZ5ayLCgDg55+Bmzel7hUqWIw2caK4KF23zrFsHnoIKFlSNqiwxPTpwIfvp+OOdzkUvnUdKGS/C9y9e4Hvv7ceD6mp8F6zFJO956NUxJ/iO9IeHn0UuHBB/IZae2fatkX8FQP8TuzDrFnSzG3izh2geXN51++/HwgPl62EcgkiCmPm5rmWQV5hqxQEMBPi/qo3gNsAamrh/QH87W4pbXrkp5Hb3e27eCSW8tHAIa6bpH/nHenNrVvnmvSYZV1N/fqyxsZdR7dujo9gGjVibtXKsWunTZP7+d13VqM2ayYd9p07HcvKLFOnWr036SVLcxuvfQwwDy+yxrX3vXLlrJ44zPDLL3KLLC3AtkZ6ukznWXMHqmvaT6MG8+nT9mfEMujz8pL8cjp8CieLccdEG3d5MOWLL6SwoaE5xzt7lhngH57cwADzrl125vPPP+Khx16NhgPAQ0Zu9gi30wDaa//fMhI+dOpjAAAgAElEQVRudQHccHdFTI/8JNzC+r3DAPNQrGZ+6y3nE9SdpA4Z4nxansR770mT/vdf+67bvVuklQ1zPLqRnJeXrHF2gSs/m9GnA/39RR7lxrp1S9y8KfW9/3522AL9zBm59osvco63a5fE24rOzN9/b3c+iYmiZZw2zUrEn39mA8AVfRL4qafszka4dk3mq19+Oed477/PDPCU565zkSL5d2kps+cIN3vG+1UAmNODFEbebnp6b3H3Lo7+ehEA8J3XE7j11lzRaznK7duZ274sWuSiQnoIgwfL3/Xrbb/m5k3gqaeAGjVkOxcrfPWVaJ+++Ub2rBwzJm92EPr3X2DqVNk9Z8kS0eD+8kvu56szebJo377+GggKkr/2Ym6DUnNkbH2DWqImtZO//xatfdu2OUS6cgUYORLUqBHadPbB3r12ZyOULQt06yZtLj3dcrx164CHH8a+Y2XQtClQvLiD+Slsxh7hdgxAOzPhAwGEuaY4HsjWrQhPehAAkJTujW/LjBThdPu2Y+m9+ipw6hSwciVQpowLC+oB+PsDrVrZNyE0YYLMs61ZI5NBOcAsSbdvL1Nkb78tH/nVq50rtjWYgWeflamnJUtkmqdiRZdtK2aV33+XfCdOlDmzQYNkc/Gzdk756cItICDneJUry8c/ulQzh4Tbnj3SAWnVykIEZmD0aODaNWDNGrTt4IV//xXh7RBDhgAXL0rG5oiMBI4cQcqAJ3HwINC6tYP5KOzD1iEegJ4AbkI2EU0CMAXAcgApADq5ewhqeuQbtWRICHcq8gc3a5rONWsyd2x6TVRgllw05ITuJPXVV11fTk9B3/n66FHrcTdulLj/+59NSZu6FExLE1dSvr65tiyMmTOdUSxenBn2/POyFM9VLi8tkZgorjVr1RLXpcxSV0dWnwwZwvzAA7bFbdSIuWelv2QO2E46dZLpKYuYLJ85eJCd28Xm9m2ZwHv2WfPnX3uNuVAh/vPHqy7ZXCK3gYeoJe2LDHQFsAtAoibg9gLo4u5KmDvyhXC7dYvZx4crFr/JI0eK0w0i5vNjZsqt//FH29PSnaQGBrrUSarHcemSTIhNn55zvNhY8QMVHGyzFxJzzuDPns1Vhx587pw4zTBd7bFnD9u9LMsRXnxR8jG1Y2jWTG6dPQQGss0m8H37Mtcvrz1LO9p7aqrImRdesBAhOloiGN1Q/Zrnn7c5m+w8+aS4ijM1hjIYZCeIzp35ww/ZvLPkfEaBEm6QObXuAMq7u8C2HvlCuK1dy/+hYoZT+1On5I7PnpkqXUtbN4DUnaQWK+ZyJ6keyWOPidWFpTVL6enijNnHRyxEbCAtTfasNLeN11dfyXO1dbmTrRgMzN27SzFNjQbT08XAo0cP1+ZpzN690hkzJyg0+wg+c8a2tO7elWV0kyfbFn/SJOaihdM4DYUsOzA2gz4KM7vGOTWVuWVL6Y2cO5flVKdOInwd5qefJOMffsgafuCAhC9fzn36iH+C/E6BEm5SXyQD8Hd3gW098oVw69GDt/kNZiDTTV7LluJNxxB+VIRVz57WF47q5sa54CTVI9Fdy//5p/nzn3zC2fR8VrC2AfNTT4nHsH377C+uJVatkjw/+cT8+VdfdW5bsZxIShLH9w88IAoIU/79l3P062tKZKTEX7XKtvhLlkj8s6hu+95mnOkBxaw/6hyWz7z1lgjy69dtziord++KP9LBg7OGv/gic7FibLh+gytUsN2jnDspiMLtr/w4t2bpcLtwu3qVuXBh/qjDD1ncDenuBMPCmHnePLbqQTWXnaR6JDdviqcRc0OOiAjpVPToYZc3itGj5TFY2trMeOs3V/hvjosTLVfr1pYfuz4HaM203hEmT5a0c9rkvEUL210rfv21pGfr9oE7dkj8HV6dbR/uMXO/fqIFzIaV5TO//y75/fSTzVllZ+xYmQjVewNpaeKOpV8/PnlS0l+yxIn084iCKNy6ATgCoA+A6gDKGR/urojp4Xbhpm2yObznVa5cOTM4Pl7UMy+9xPLV6tRJ9E4nT2ZPIw+cpHosAwaI2td4Iiw5WdxcVahg18RHSooIGmvLCvV9SJ3tnRsMMufk7Z2z20KDQfYSffRR5/Iz5e+/pR6W7CN05syRL8ipU9bTfP11SdPW9V3ammdeUvUtWdhvAwYDmx8dJSbKMLR6dYtDs9u3ZW3c1Km2lc8sphOh27bJ702bePly+dd0M/n8SEEUbgajI93oMABId3dFTA+3C7f27Znr1eMmTQzZ/Pn262f03Y2NlS9nixbZDRvywEmqx/Ltt3LvjLfy0YcjpvMiVvjhB7lsyxbrcd94g52zvGPmDRskDWtbijFnGildvOh4fsYkJ8sG1FWrMt+4kXNcXQDNmmU93X79RL7YSnq6DLAn1f+RuVo1m67RF9hnG8mOGSM36fffc7z+oYeY27SxvYzZ0CdCdauZp58Wa6CkJB41Svqo94LypSAKt/Y5He6uiOnhVuF27hwzEae+8Q4XK5bdcv+77zirCkTX2RhbJBw4kCdOUj2WO3fEcEDfEXLnTvnAjR5td1IhITKdYotRZWqqWE7as3OKMZcvy+gjONg260t9LsvSvJy96Nvo2SLImWUOOSjIerw6dUTA2UP9+sx9G0RJgWxwBbN0qUTNMtq1Y/nMq6+KVsUpzy+TJ8sQMDY2S/urX1+Mg+4FCpxwu9cOtwo3zeY38pcYs5PoKSlihT5okFHgU0+JMDtwQHT2tWqJryNr3WeFZUaOlJ7zf//JvaxVy7x1RA5oqznsWpZ4+rSsfWvXzvqel6aEhIiRiC3L9HQaNxaB6iz//CPfZXtcUenTxjkZnSYliUry9dftK0/PnsyN/G+y2bUIZhg+XDoGGVOpdi6f+f57yWrPHvvKmYXDhyWRrl1Z1xzEx8u/777rRLp5SIEUbpCtbt4GsAnARgBvAqjk7kqYO9wq3Jo0YQ4O5vXr5Q6bs2QeN07mVDJk140bYppWq5ZMGrjQM296uizvuXr13lCLuIzt2+UB1KmT2XGwk7Vr2SFHtytWsM0qOx394/r22/blpbvUtNUs3xx370qzrVTJPuvL8+cl73fesRzn0CGJ8/XX9pVp4kTm4t7pbABkuxgrPPigzFUys0PLZ65csf+ZZcNgEHNoIGPu4ccf5adLHW3nIgVOuAFoDXGYHA1gtXZEA0gA0NLdFTE93Cbcjh+X2zpvHk+fLj1hc51GffnLl18aBe7aJUINsMtCLCcMBubevSVJQHrQ5csz160rDvR79ZKpgUmTZEu3lSvzt1NXu0hL45Wlx/NkzGbDG286lESPHjLlY2+nwGBgHjhQnn/37jIamjiReeZMsTXauFGWF4SHy3zZpUvimL9xY5vXlGegm+W/95591xmj79X57bf2X9u6tXgUsYS+pCHSTsf7ixbJdRdK1hVLxBy4eFHifvSRFqC7dbFz+Uz9+jbbr1hGX3KgWetOnSrtQPfwkt8piMJtP4DPYbRJKcQ35ecA/nR3RUwPtwk3oxn+nj2ZGzY0H023dGvf3uTEhx/KF9VFXkh0T0MTJshC8v/9T74TAweKlV3jxmI84O2dKQAHDnTPhtwvvST3w1UfgYMHmQsXShMjg8V26gc5YzUHT5rkWP7Xrsmyp2bNZFDu65t5j80dXl62m8qb8vDDVlxO5YC+T+nAgY5dP39+zsJr8mRJ316hrW9CvavhOKuWHroRzt9/s1ge+/g4tHxm9GiZKrNXnZyFc+dkIlLTLbdtK/Zi9woFUbjdAVDXTHg9AHfcXRHTwy3CzWAQtaJmm/3AAzKHYom335YnEBOTO8U5e1a2RunQwbZ3/PbtzDLlxtqpnNCNAQDmUaOcT0+3/q5aVT4uvr72bw2mreZwWOCYIzlZFhgfOSLGexs3ytrHmTOZf/3V8XT1den2jo4SE2W0ct99mWsx7eXCBenPvfmm+fPduzvm/eP0aanTlx1WisTJocc1frysQ0y949zymdWr2eJUgiOkpEjHceJE16SXFxRE4fYfgMfMhHcDEOfuipgebhFuuu+fpUv5xg22qirS97eaOdP1RTEYxMNUiRL2zcWkp0uHt3hx+z+UjhIWJlMjHTsyT5ki98QOpxRmGT060/pbF/KtW9vXI2/XjrlePfeMYu0lLs4xo41Ro+Q+bd/uXP7t2slUkznuv9+xrQdTU7V92bqESqMwcZllTFCQtB9nl8/oKt6FCx26PBv797O+1O2eoSAKt48BXAAwFEAN7XhSC5vr7oqYHm4RbhMniv7l2rWM9ZzWPB60bSvzX67+gOqeUD77zP5rL14Uq7PAwNzfEPPaNXEDWa2ajBxSU+WelCgh05eOoC+1MJ621Od9Zs+2LQ1tNYfdxh3u5NFHRdVta1vSDZ6sbuppA/r8mOki5ZuasaOjRhq1ajEP6HA5x5fpxg0R7G+OOu/08hmDQdpiFktmJ9AXusfFuSa9vKAgCreiAD7RtrjRF3AnA5gHoKi7K2J65LlwS0sTi4A+fZg582W3phnR3Ub+9ZfrinLmjAiHTp0cF5o//yzlcspTuhXS02V6sUgR6eHqxMYy+/nJfKW9828XLojBTNOmWR20GwzM/ftLXraonHQP7rZ438gv6G0pLMx63DNnZDT78MP2z4WZw9LI8c8/2ZF18xk89hhzk8DUHHsmv/wip7dXecoly2cGDxYn2a7ocPbta8EdWD6mwAm3jAsAHwCNtMPH3RWwdOS5cNOd023YwMziFKFMGesvyPXropKzuEWHnaSni1FGqVKOLSI25uWXpUrffeeSomVDt9BbsCD7uV9/lXPPPGN7erqz/+LFzY/6rlwRU/dGjazb62irOe4p4uNFeFtbr3z3rkxLlS4tajhX8cgj2bUQutGio8sUXnhBlioaqlS1OCKbPp25MKVxIkq4xN5e75g6s7SCWe5DxYr2rRvMDxQ44QbgPgDVzIRXy49r3fJcuI0aJVYL2lCjVSuZh7CFgQNltGG6FZQj6JZrWZYYOEhKilj6lS2b43SHQ2zbJj39kBDLHYDp09nsInhL6AuKc3L2rzusyMkC0mg1xz1Hjx7iASonA6KpU9mhdWfW0FXhR45khk2YIFoER9dXfvyxpHnpkUEWzUHbNbjCLXDAZctnwsMlz5UrnUtH3+LKjs0n8gUFUbhtB/CsmfBnAPzm7oqYHnkq3JKTZZimddHS06W3aetoTF/kuXmzc8U4eVJGLd27u24O79Qpkdlt27puM87z50XtGBCQs8OQ1FTpIPj4WDdu0c3Ze/WyXvfnnpP5NEsLs/XVHGa3TcnnrFnDOXrZ2LZN6mbNKbIjXL4sHZYZMzLDHn3UOTN4vTOyb8hCs+sJks/+x8Vwh1+usMply2fS0+V1dvYe6Qv57fE2kx8oiMLthoWlAHUAXHN3RUyPPBVuumuJn39m5kwryJx2sjHm7l0x4Ojf3/EipKWJNWCZMq7/KOvm0ZZMve0hJUXmeXx9bTMYuXBB7k2DBpbn35KS5Lyt5uy3bok3C3//7NvTmKzmuOe4dUs6OOPGZT936ZLco/r1c29BcadOWY1aKlYUL2iOEqW5llw5eq/8c+xY5kmDgfe1elVU5584qYM34fHHxVLWGZ599t5xlmyMpwi3QrCdwgCKmQn3thBecPjqK8DPD+jUCQBw9KgEN2pk2+VFigAhIcCPPwLXrztWhI8/BvbtAxYsAKpUcSwNSzz5JDBsGPD228Du3c6l9eqrwIEDwLJlQL161uNXqQKsWQNERgLjx5uPM2UKcOwYsHw5UKGC9TR9fYHVq4Fz54CJE7OeCwsDoqOBIUOsp5Mf8fUFevQANm4E0tIyww0GYPhwaV8bNgA+PrmT/8CBwKlTwOHDwOXLcjRs6Hh6/v5AoUJANNeUAP3lAoDPP8eeP+UT1jrkfsczMUPbtkBUFHDliuNp7NsHtGwp5Ve4AVulIIAdAD4zE74EwE53S2nTI89GbgkJ2brKuvcde3z0hmpLeRzZzPD4cTFK6d0799ZkJSTIiKZaNfHc4Qjr1kkdHVnQOmMGm50H+eknCX/xRfvT1Of0vv8+M8xoNcc9i7ndfnST9EWLcjfvK1fEGn/q1Ewbq99+cy5Nf3/mIYPSJGFd53niBLOPDz/ut5/r1XN9o9+rDRQdNabSnSXnxhrW3AYeMnKzR7g9DCAJwD4A72jHPojnklburojpkWfCTdfZ7d2bETRggKi97MFgEHVR69b2XZeaKnMa5cvbtf+mQ4SGijWeI0L02DGZO2vd2jHT89RUsQL18cnUTOlO3xs2dGw9XkqKWEVWqCBp6as5eve2P638xJ07Yi379NPy++BBeW59++bNgvSuXcX8Xfea4uxec506afN29erJpOrdu8zBwZxephyXKZ2eK/OHycnSYXz5Zceu1+cK//jDpcXKEwqccJM6IxDAGgDHtGM1gMZ2XP8YgBMQh8tTzZx/QBshhgPYCSPrTAD3A/gNwHEAkQD8c8orz4Rb9+6ytsZIsV63rpF3cjuYNUueSHS07dfoHuHzaj/TuXPZbg8OCQnyXapY0bn5QH3+LSBA3Eb16GGX03ezRERkjnp37GDj1Rz3NMOHi6n/5cvS0ape3T5v/86g+zNt1ky2dnJWoI4ZI+nwgAHMNWpkbDoXPmerS6waLdG2rePLQaZNu7ecJRtT4IQbgAAYGZQA6KIJumkAvGy43gvAaQA1IQvCjwAIMImzEcBw7f9HAaw2OrcTQGftf19YWWOXJ8Lt8mVRlUyZkhHk6N5VzLIuLScffaYcPSoqtAED7M/LUQwGkefFionPxYQEGTGeOSOC4uBBsUL85Rfmb74R670ePeSeuKIXu3Wr3CN9VxFXbNL50UeSVs2aWVZz3NPoC5vr1ZN7b8N2aC4jPl4+7IDty2FyQlepxk/9QP4pVIh52DCXrUezxPTp8nonJtp/bbt29946SR1PEW6F7ZieWwZxwXWCiKoD+A7ALgDPAyilCbmcaAEgmpnPAAARrQfQWxuF6QQAeFn7/w8A32txAwAUZuZtAMDMiXaU2z5u3QJeesm2uOfOAenpWawPIiNl8j4w0P6s778f6NABeOstYM4cmfT38QGKF8/83/j3wYNAmTLAp5/an5ejEAErVgCNGwNNm9p+3Zw5Ujdn6dIFmDEDmDkT6NrVspGJPbz0khjz7NwpxjO5ZWyRl3TsKDZOUVHSntq2zbu8y5UDOncGfvnFOWMSnVq15O/pcsEoB8iLsmAB9o4BqlYVo5PcoG1bYNYsMYDq2NH26+7eBf7+GxgzJnfKpbANe4RbPQCHtP+fAPA3M3cnokcALId14VYVwHmj37EAHjKJcwRAP4ibr74AShJRechygxtE9C3Ep+V2iFoz3fhiIhoNYDQA3H+/g9ZTd+8Cv/1me/w+fbKYRYaHy19HhBsg1o5ffQUkJclx5w4ApKJbt1hUrJgMIHOTlKeflg/JlSvOWXU5wq+/SvmIsh6FCpkP8/ICjh93Td4hIUCvXiLko6Jck+b8+XIPy5VzXTkBwNvbG9WqVUORIkVcl6gNFCkilqlhYdIZyGsGDXK9cIsu3RTBDz0EzJ0LLlkKe/YAbdpIG8sNWraUtPfutU+4rV8PJCfnbYdCkR17hJsXgLva/x0B/Kz9fxqyQ7creBXAQiIaAWA3xClzOqScbQE0AXAOwAYAIwB8aXwxM38O2V8OzZs3Z4dKUL48cP689XgWCA+Xnn/Nmo5d36CBjEqM+fffWJQsWRLly/uDcutNVrgcZkZ8fDxiY2NRo0aNPM9/ypQ8zzKDJ56Q0UufPs6npb9L0ZdKyTAKwLmzQGxs7gqQ0qVFQ7Fnj+3XREUB48ZJuXr1yr2yKaxjzwqMCABjiagtRLj9qoVXBXDVhusvAKhu9LuaFpYBM19k5n7M3ATADC3sBmSUd5iZzzBzGkRdaYdSLO8IDxcB5eXlujSTk5NRvnx5JdjuMYgI5cuXR3JysruLkueUKAEsWgRUrux8WsWLA9WqyfpDHV3gtGnjfPo50aYNsH8/kJpqPW5SEjBggJT3q6+AwvYMHRQuxx7hNgXAsxDDjq+YWV9N2QvA3zZcfxBAbSKqQURFAQwG8INxBCLyIyK9TNMg83z6tWWISF+i+yiyztXlC5hFuDmqkswJJdjuTdRzcw21amUXbqVLu0btmRNt24rQOnzYetwXXxRnAmvWyFygwr3YLNyYeTeACgD8mHmk0aklAMbacH0agBcAbIWY83/NzMeI6G0i0gfwHSAGKychqs53tWvTISrLHUR0FAAB+MLWsucVly4BV6/mjnBzN0SEV155JeP3nDlz8Oabb7ok7TfffBM+Pj64fPlyRpivr69Daa1YsQIXL16065qYmBg0tPCVPHnyJLp3747atWujadOmGDhwIM6ePYvy5csjISEhS9w+ffpgw4YNDpVbkTOmwm3vXqB1a9dqSMyhjwytqSbXrAGWLgWmTxdDJ4X7scsxDDOnM/N1k7AYZr5s6RqTuD8zcx1mfpCZdcH1OjP/oP2/iZlra3FGMXOK0bXbmDmQmRsx8whmvmspH3fhrDFJfqZYsWL49ttvcfWqLRpo+/Hz88NHH33kdDqOCDdLJCcn4/HHH8fYsWNx6tQpHDp0COPGjcOtW7fQtWtXfPfddxlxb968ib1796Jnz54uyVuRlVq1xJVXQgIQHy9WybmtkgTE/VvNmiJMLXH8OPDcc0C7doCL+nsKF6C8nrkQXbjZ6lPyXqJw4cIYPXo05s2bl+1cTEwMHn30UQQGBqJjx444d+4cAGDEiBGYMGECWrVqhZo1a2LTpk0W0x85ciQ2bNiAa9euZTu3Zs0atGjRAkFBQXjuueeQnp6O9PR0jBgxAg0bNkSjRo0wb948bNq0CaGhoRg6dCiCgoJw584dhIWFoX379mjWrBm6du2KuLg4AEBYWBgaN26Mxo0bY9GiRWbLtG7dOrRs2TKLwOrQoQMaNmyIkJAQrF+/PiP8u+++Q9euXeHjCesI8iEZywFOZwqavLJGbNtW8mQzJmpJSeJPs0QJNc+W31CPwoWEh0tPr3z5XMzkpZdsmwCwh6Ag8bxsheeffx6BgYGYPHlylvDx48dj+PDhGD58OJYtW4YJEybg+++/BwDExcVh7969iIqKQq9evfDEE0+YTdvX1xcjR47EJ598grfeeisj/Pjx49iwYQP27duHIkWKYNy4cVi7di0aNGiACxcuICIiAgBw48YNlClTBgsXLsScOXPQvHlzpKamYvz48di8eTMqVKiADRs2YMaMGVi2bBmefvppLFy4EO3atcOkSZPMlikiIgLNmjUze65r164YNWoU4uPjUb58eaxfvx4vvPCC1XuocIyM5QDRYoVZrBgQHJw3ebdtC6xcCZw4kd3Z94QJMs/266+ud1iucA41cnMhR496pkpSp1SpUhg2bBjmz5+fJXz//v0Yoi1kf+qpp7DXSIfTp08fFCpUCAEBAbh06VKO6U+YMAErV67ErVu3MsJ27NiBsLAwBAcHIygoCDt27MCZM2dQs2ZNnDlzBuPHj8evv/6KUqVKZUvvxIkTiIiIQOfOnREUFISZM2ciNjYWN27cwI0bN9CuXbuMMttL0aJF0atXL2zatAlXr17FP//8g65qsiXXePBB+RsdLfNfwcEi4PICXf1pqppcvRr48ktZR9ilS96URWE7auTmIlJTZR4g1xu5DSOs3OSll15C06ZN8fTTT9sUv5jRF4g1vc6MGTPw008/AQAOG41Cy5QpgyFDhmRREzIzhg8fjvfeey9b2keOHMHWrVuxePFifP3111i2bFmW88yMBg0aYP/+/VnCb9y4YVPZGzRogF27dlk8HxISgnfeeQfMjN69e+f5Qu2ChK8vcN99oh0JCwMsDLZzhTp1ZCulPXuAUaMk7Phx8UDSvj3wxht5VxaF7aiRm4s4eVKcm3jyyA0AypUrh4EDB+LLLzPXz7dq1Spj/mnt2rVoa2Uy5N1338Xhw4ezCDadl19+GUuWLEGathlZx44dsWnTpgxLymvXruHs2bO4evUqDAYD+vfvj5kzZ+LQIXGeU7JkyYyRX926dXHlypUM4Zaamopjx46hTJkyKFOmTMYIc+3atWbLOWTIEPz5558ZghgAdu/enaEK7dChA06dOoVFixYhJCTEyp1TOEutWsAPP8g+dXlhTKJDJPnpIzd9PVuJEsC6dWqeLb+ihJuL8GRjElNeeeWVLFaTCxYswPLlyxEYGIjVq1fjk08+cThtPz8/9O3bFykpYigbEBCAmTNnokuXLggMDETnzp0RFxeHCxcuoEOHDggKCsKTTz6ZMbIbMWIExowZg6CgIKSnp2PTpk2YMmUKGjdujKCgIPz5558AgOXLl+P5559HUFBQxojSlOLFi2PLli1YsGABateujYCAAHz66aeooO2IWqhQITzxxBOIj49H+/btHa6zwjZq1cp0+daqVd7m3bYtcOYMcPEi8MILoqVZu1bNs+VnyNKLfa/TvHlzDg0NzbP8pk8HPvwQuH0bKFrUtWkfP34c9evXd22iijxDPT/X8O67wGuviUssV9tUWSM0VOb5evcGNm8G/vc/2ZneEyGiMGZu7u5yOIsaubmI8HCgfn3XCzaFQiHoFpN5qZLUCQoSNeTmzbK7hZpny/8o4eYicsvtlkKhEHQnMvZ46HcVhQvLIu2KFWWeLbc9oyicR02FuoDr12UjgYIw36ZQuIsGDWRNmbs0vCtXitGYK5xBK3IfJdxcwFHNhbQauSkUuUtAgPvyrlDBehxF/kGpJV2AEm4KhUKRv1DCzQWEh8sOzsosWKFQKPIHSri5gPBwmW/z5K27XLXlTUxMDNatW2f3dSNGjLDoeHnOnDmoV68egoKCEBwcjFWrVuGtt97CtGnTssQ7fPiwMslXKAoISrg5icHg+T4lAddteeOocLPE4sWLsW3bNvz99984fPgwdgamtagAACAASURBVOzYAWZGSEhItr3V1q9frzyJKBQFBCXcnCQmRhZue7pwy2nLmytXrqB///4IDg5GcHAw9u3bBwDYtWsXgoKCEBQUhCZNmuDWrVuYOnUq9uzZg6CgIMybNw/p6emYNGkSgoODERgYiCVLlgAQv5AvvPAC6tati06dOmXZyNSYWbNm4bPPPstwnFyqVCkMHz4cderUQdmyZfHXX39lxP3666+VcFMoCgjKWtJJ8nqDUjfueGNxy5sXX3wREydORJs2bXDu3Dl07doVx48fx5w5c7Bo0SK0bt0aiYmJ8Pb2xuzZszFnzhxs2bIFAPD555+jdOnSOHjwIFJSUtC6dWt06dIF//zzD06cOIHIyEhcunQJAQEBGDlyZJZ8ExIScOvWLdSsWdNsefU91x566CEcOHAA5cqVQ+3atR27SQqF4p5CCTcnCQ+XubYGDdxdktzHeMub4sWLZ4Rv374dkZGRGb8TEhKQmJiI1q1b4+WXX8bQoUPRr18/VKtWLVuav/32G8LDwzPm027evIlTp05h9+7dCAkJgZeXF6pUqYJHH33U7vIOGjQIrVq1wkcffaRUkgpFAUMJNycJD5e9pkqUyJv83LzjjdktbwwGAw4cOABvb+8scadOnYrHH38cP//8M1q3bo2tW7dmS4+ZsWDBgmx7of38889Wy1KqVCn4+vpm7O9mSvXq1VGjRg3s2rUL33zzTbatbxQKheei5tycpKC53TK35U2XLl2wYMGCjN/6VjanT59Go0aNMGXKFAQHByMqKirLljSA7Gj92WefITU1FQBw8uRJ3L59G+3atcOGDRuQnp6OuLg4/PHHH2bLM23aNDz//PNISEgAACQmJmLVqlUZ50NCQjBx4kTUrFnT7MhRoVB4Jkq4OUFSkuwMXJCEG5B9y5v58+cjNDQUgYGBCAgIwOLFiwEAH3/8MRo2bIjAwEAUKVIE3bp1Q2BgILy8vNC4cWPMmzcPo0aNQkBAAJo2bYqGDRviueeeQ1paGvr27ZuxzcywYcPQsmVLs2UZO3YsHnnkEQQHB6Nhw4Zo27YtChXKbNYDBgzAsWPHlEpSoShgqC1vnODgQaBFC+Cbb4B+/XIvH7Vlyr2Nen6KewlP2fJGzbmZwCym/bagy86CNnJTKBSK/I4SbiZcvSrbWtiKry9gwRJdoVAoFG5CCTcTSpSQHbVtJTAQKKRmLhUKhSJfUaDm3FJTUxEbG4vk5GQ3lcoxUlNTUaRIEXcXQ+Eg9+rz8/b2RrVq1e7JsiscR8253YPExsaiZMmS8Pf3B91DXo5TUlJQqFAhFC5c+J4qd0GHmZGWlgaDwYBixYq5uzh2wcyIj49HbGwsatSo4e7iKBR2U6CEW3Jy8j0n2ACgSJEiSE1NvedGnAqgUKFC9+TIh4hQvnx5XLlyxd1FUSgcwmOFW1hY2FUiOmsctm3btkbp6elptlyfnp5e2MvLy6a49wKeVh/A8+qUH+vz33//FQ4ICDjqRBJ+AJzbSiJ/4Wn1AbLX6QF3FcSVeKxwY+Zsm8IfOXIkpmHDhjY1zIiIiPoNGzY87vqSuQdPqw/geXXKj/VJT0/3c2b+hYhCPWH+RsfT6gN4Zp0ADxZutuA/9admOcc4a+V8VmJmPx5mLQ4RNRs1atSlL774IhYAXn/99UqJiYlec+fOvWhPXuZ4+eWXq3z22WeVoqOjj1atWjUNAHx8fJokJSX9Y29a8+fPL9+rV68Ef3//VFuvOXHiRNEePXrUPnXq1DHTc+Hh4cXGjx9fPSYmxrtEiRLp/v7+KR9//HFsixYtAs6cORNerlw5gx63U6dODw4aNOjas88+e92WfD2l3i+88EKxoUOHljWu961btwoNHTr0gaioqOLMTKVKlUr7/fffT5UuXdpgmpdCochEGbHnMUWLFuWff/65bFxcXK50LMqUKZM2c+bMSs6ms2bNGr9z5865ZLIoKSmJevbsWfu55567cvbs2YjIyMjj48aNu3Lz5s1Cbdu2vbl27dqyetz4+HivsLAw38GDB9+0Jw9PqPfhw4e9TOs9a9asihUrVkw9efJk5KlTp44tW7YspmjRok6ZOOt+PBUKT0YJtzzGy8uLhw0bdmXWrFnZPsQnTpwo+vDDD9epU6dOQMuWLeucOnWqKAD079/ff8SIEdWbNGlSr1q1ao2WL19eNnvKQkhISPwPP/xQ7tKlS17G4X5+flc+/fTTco0aNapfr169gCFDhjyQlpaGtLQ09O/f37927doN6tSpE/DWW29VXL58edmIiAifYcOG1axXr15AYmIi7dmzxyc4OLhugwYN6rdp06b22bNniwDAnj17fOrWrRtQt27dgLlz55pd/v7555+Xa9q0aeKQIUMyPtw9evS4FRwcnBwSEnJt48aN5fTwtWvXlmnbtm1CyZIlrY5M/Pz8MqwdLNUbAO6Verdp0+a2ab3j4uKKVK1aNUMaNW7cOKV48eIMAAsXLixfp06dgLp16wb06dOnBpBzGxoyZMj9gYGB9caOHVstISGh0IABA/wbNWpUv379+gFr1qwpY+1+O8jnuZSuu/C0+gCeWScl3NzBpEmTLn/77bfl4uPjs3yIx44de//QoUPjT548GTlo0KD4sWPHVtfPXbp0qUhoaGjU5s2bT73xxhtVLaXt6+ubHhIScnX27NlZhOfFixcTN23aVC40NDQqKioqslChQrx48eLy+/fv94mLiyty6tSpYydPnox8/vnn459++unrDRs2TFq1atWZqKioyCJFimDChAn3b968+fSxY8eODx8+/Oqrr75aFQCeeeYZ/48//vjciRMnIs2XCIiIiCjetGnTJHPn+vXrl3Ds2DGf//77zwsANm7cWC4kJOSaLffxvvvuy5g/tVTvQ4cOed8r9R46dGicabzRo0dfXbBgwX1BQUH1JkyYUOXo0aPFACA0NNR7zpw5lXft2nXyxIkTkUuWLDkH5NyG4uLiih46dChq6dKlsdOnT6/8yCOPJBw9evT4nj17Trz22mvVEhISXP49YGaP+nB6Wn0Az6wToISbWyhXrpxhwIAB8bNnz87S4//nn39KjB49+hoAjB079lpYWJivfq5Xr143vLy80KxZs+T4+Pgc1WZTp069/PXXX5e/fv16xvP99ddfS0ZERPg0bty4fr169QL27t1b6syZM8Xq1auXcv78+WLDhw+vvmnTplJly5ZNN00vPDy82KlTp4o/+uijderVqxfw4YcfVr548WKRq1evet26dcurW7duiQAwcuTIeHvvhbe3N3fu3PnG6tWry8bFxRWOjIz06devX4K96XhqvVu1anXn33//PTpx4sT/rl27VrhVq1b1Dx065L1169ZSPXv2vF65cuU0AKhUqVI6kHMb6tev3/XChUUbvnPnzlLz5s2rXK9evYA2bdrUTUlJoejo6KL21kOhyK8UaIMSdzJt2rRLTZs2DRg8eLBN1pve3t4Z8yy6V5nx48dX3bZtW2kAiIqKyhhB+Pn5pfft2/fahx9+WNHoGhowYED8okWLLpimHREREfndd9+VWrx4cYUNGzaU27hxY4zxeWamWrVq3Tl8+HCUcfjVq1ezqQDN0aBBg+Tdu3f7Wjo/ZMiQa++++25lZqYuXbrcKFasWLY5pSeeeMI/IiLCp1KlSnd37doVbS4dT6w3AJQuXdowfPjwG8OHD78xbNgwbN68ubQj826+vr4ZKk9mxqZNm6IbN26cYm86CsW9gBq5uYlKlSql9+zZ8/q6dev89LAmTZrcXrp0aVkAWLJkSbnmzZsn5pTGggULLkRFRUUaCzadGTNmXFq5cmWF9PR0OnLkSKOAgIAKP/74Y6Vdu3bVB4BLly55nTx5smhcXFzh9PR0jBgx4sZ777134ejRoz6AqPlu3rzpBQCBgYHJ165dK7x9+/YSAJCSkkKhoaHefn5+6SVLlkzfunWrLwCsWLGinGk5AODZZ5+NDwsL812/fn1pPeyXX37xPXjwoDcAPP7447diYmK8ly5dWmHIkCHZVJKnT5/2nzFjRumNGzeSLthSU1O9jh8/Xjs8PLxhUlJSOYPBUMi03gDw2GOPJWzZsqXshQsXCuenep8+fdq/atWqNWNiYkrq9T5//nyVw4cPB0ZERAREREQEXLt2rfRvv/1W4sqVK14AkJycTCdPnvT29/e/27Vr14Qff/yxrK7W1OcabW1DjzzySMJHH31UyWAQebdv377i5uLZChFVJ6I/iCiSiI4R0YtaeDki2kZEp7S/FueL8xM51OdNIrpARIe1o7u7y2orRORNRH8T0RGtTm9p4TWI6C8iiiaiDUTkESP4Aj1yMzXdZ2YcPXq0Ye3atU8WK1YsNTIysn6NGjXOlChRIldcg8yYMeO/lStXZqzHW7x48blhw4b5f/LJJ/eVL18+bdWqVTGOpl25cuW0bt26Xf/yyy8rAUCPHj1OXL16teRzzz1X2WAwBBQpUoTnz59/zsfHx/DMM8/4GwwGAoC33347FgCGDRt2dfz48Q9MmjTJEBoaenz9+vWnJ0yYcP+tW7e80tPTaezYsZeaN2+e/OWXX8aMGjXKn4jQoUMHs+pEX19f3rx5c/SECROqT5kypXrhwoW5fv36dz777LNzAODl5YXHH3/8+pYtW8p27979lun1fn5+VytWrHg5JiYmww/UxYsXK5csWfJWtWrVTnl5/b+9e4+zqrzvPf75zg04cvEGGAUK8TYlCCpzNFZNbURj9BiTFo2k5mBjDj2JnCQvc9pgYry9Yl+Yai6vmmpMJTXRE7RqKqkmRgsxMU1RICIgQVBRARWIXAUGhvmdP9aaujPumVmzZ2/2Zb7v12terP3stfb6Pa5x/+Z51rOep/641tbWIcDrnes9efLkPddcc836s88++7j29nYqpd5Dhw7dPGLEiI1Tpkw57vHHH284//zzd2zYsGHI8OHD3zzqqKPe7PiMF1544bCZM2f+EUB7e7umTJmybfr06Vvq6ur44he/+PqZZ57ZXFdXFxMmTNj14IMPrs36OzR79uwNM2bMGNPc3Dy+vb1do0ePbl2wYEHeFnFGbcAXI2KJpCHAYkmPA5cD/x4RsyXNAmYBX+rDeQ6UruoD8M2IuKWMsRWqFfhgROyU1Ag8JemnwFUkdZor6Q7gCuD2cgZaDDU7cXI+S5cuXTtp0qQuuwG3b99+0IYNG45sbm5eDbBu3bojAEaNGvXGgYqxFNKW28rGxsaKmv2iN/bs2dO0evXqY0844YQVAM8999yE448/ftWAAQP2tba2Nq5ater4iRMnLi93nL3RuU6vvfbakXV1dftzk1u5LV269PBJkyaN7e1xkh4Gbkt/zoqI1yW9B/hFRBxf5DBLLqc+pwM7qzS5/RdJ/w14CvgM8AhwRES0SToNuD4iPlTWAIvA3ZI59u7d29TY2Li343VTU9Peffv21UQTfdWqVccuX778j994443De9678rW1tTUMGDBgH0BTU9O+tra2muiF2Lx584hly5aNf/HFF8fu27cv0729SiNpLHASsBAYGREdo0DfAPr8LOKB1qk+ADMlPSdpTrV0s3aQVC/pWWAj8DjwIrA1Ijr+8F0HdDkau5o4ufUDzc3Nv5swYcLK4447bvXmzZtHbNu2rctBDtWo2ibC7srIkSM3Tpw4cdmECROeb2xs3Pfqq6+O7vmoyiJpMPAg8IWI+IPu2ki6iaqqqyhPfW4HjgZOBF4Hbi1jeL0WEfsj4kRgFHAK0FzmkEqmvyW39o57LPl0bql1bslVq5wWTtuwYcO27ty586Byx9RXDQ0Nba2trY0Ara2tjQ0NDVXb5dqhqampTRKSGDFixKZdu3aV9Tql/69knuYrvY/zIHBvRDyUFr+ZdkeS/rux6IGWSL76RMSbaYJoB75HkiCqTkRsBRYApwEHS+ro+RgFvGtkcTXqb8lt+aZNm4Z1leAGDx78dmtr68Ddu3c3tbe3a+vWrYcecsghWw90kMW0f//+ura2trqO7R07dgwdNGjQ7nLH1VdDhw7dumnTpsMANm3adNiwYcOq+jpBkqQ7tt96662DBw4cWLbr1N7erk2bNg0DMt3HVNJ8vgtYGRHfyHlrHjA93Z4OPFzUQEukq/p0JOrUx8j436cSSBou6eB0exBwDrCSJMlNTXermmvUk8wDSiSNBD5J0iT/akRslnQ6sCEiXi5hjEWzePHiEQ0NDf8ETKCLxL5nz55B27dvPxRg0KBBO4cMGdKrOQ4rTVtbW8OWLVv+67mvgQMHvl1tddqyZcvwvXv3Dmhvb6+vq6vbP3jw4K2DBg3atWXLluEdy8Qccsghm+rq6qpmMuF8ddq7d+/Atra2JoD6+vq2YcOG/b6+vv5dD5cfIO3A8ra2tk9Pnjy5x9aWpDOAXwHLeKe192WS+1T3A2OAV4BLIiLTDDTl1E19ppF0SQawFvjrnHuKFU3SROBuoJ7k++/+iLhR0nuBucChwG+ByyKi6p9/zJTcJE0G/h14GXgf0BwRL0m6HjguIj5R0ijNzMx6IWu35C3AtyPiJJJnJTo8RjI01szMrGJkTW6TSZqznb1OFQ7tNTOz2pY1ue0G8j3P0UwVjX4yM7P+IeuDrw8D10m6OH0d6YONN5MMla04hx9+eIwdO7bcYZiZVZXFixdvjojhPe9Z2bIOKBkKPApMBA7inZkGfg2cHxFvlzLIQrS0tMSiRYvKHYaZWVWRtDgiWsodR19larmlT+afIemDwMkk3ZlLIuKJUgZnZmZWiF7NxxcR84H5JYrFzMysKDIlN0nfB5ZHxK2dyq8CxkfEp0sRnJnZgTZ21iPdvr929gUHKBLri6yjJT9M/hbbfKBqFuszM7P+IWtyOxjIt6Lv2yRTtpiZmVWMrMntBfK30C4A+rJ6r5mZWdFlHVByK3CHpBG80z15NvAF4MpSBGZmZlaorI8C3C1pIHANcHVavB64KiK+X6rgzMzMCpH5UYCI+C7wXUnD09ebShaVmZlZH/TqOTdwUjMzs8qX9Tm3Q4GbSO6zjaDTQJSIGFr80MzMzAqTteV2F3AScCewgWQVWjMzs4qUNbmdDZwTEQtLGYyZmVkxZH3ObSP5H+I2MzOrOFmT21eAGyUNLmUwZmZmxZC1W/IaYCywUdIrwL7cNyNiYpHjMjMzK1jW5PZASaMwMzMroqwzlNxQjJNJOg/4NlAP/FNEzO70/gDgB8Bk4PfAxyNiraRzgNlAE7AX+Jt0bTkzM7N3yXrPDUkDJU2V9CVJB6dlR6fPwGU5vh74DsnyOeOBaZLGd9rtCmBLRBwDfBO4OS3fDFwYEScA04EfZo3bzMz6n6wPcR8DPAEMJln+5l+ArcBn0tdZFis9BVgTES+lnzkXuAh4Pmefi4Dr0+0HgNskKSJ+m7PPCmCQpAER0ZolfjMrLS/waZUm6z23bwE/J0lmW3PK5wFZJ04+Cngt5/U64NSu9omINknbgMNIWm4d/gJYki+xSZoBzAAYM2ZMxrCsmvhL1MyyyJrc/gR4f0Tsl5Rb/ipwZNGj6oKk95F0VZ6b7/2IuJNkFhVaWlo8i4qZWT+V+Z4b0JinbAywLePx64HROa9HpWV595HUAAwjGViCpFHAj4H/GREvZg/bzMz6m6zJ7efAVTmvQ9JQ4Aag+36idzwDHCtpnKQm4FKSbs1c80gGjABMBeZHRKQDWB4BZkXErzOez8zM+qmsye0q4AxJq4CBwH3AWuAIYFaWD4iINmAm8BiwErg/IlZIulHSR9Ld7gIOk7QmPWfHZ88EjgGulfRs+jMiY+xmZtbPZH3ObYOkE4FpwMkkSfFO4N6I2J31ZBHxKPBop7Jrc7b3ABfnOe5rwNeynsfMzPq3HpObpEbgHuDLETEHmFPyqMzMzPqgx27JiNhHMjrRow/NzKwqZL3n9hDw56UMxMzMrFiyPuf2KnCNpDOBRcDbuW9GxDeKHZiZmVmhsia3y4EtwMT0J1cATm5mZlYxso6WHFfqQMzMzIqlNzOUACBppKReH2dmZnagZEpSkholfV3SDpIpssam5TdL+mwJ4zMzM+u1rC2w64ALgcuA3Nn4nya5H2dmZlYxsg4omQZ8KiKelNSeU74cOK74YZmZmRUua8vtSOCVPOUNZE+QZmZmB0TW5LYC+ECe8kuAxcULx8zMrO+ytrpuAO6RNBqoBy6W1Ax8AqjJpY+zrvhczpWhy3XuWloNuxR18XUxK7+sz7n9RNIlwJeBdpIBJkuACyPiiRLGZ0VQqV+2pTy3mfVvXSY3SdcCt0TELkljgJ9HxGMHLjQzM7PCdHfP7VpgcLr9MjC89OGYmZn1XXfdkuuBqZIeAQSMkjQw344R8WopgjMzMytEd8ntJuA24B9IJkd+Js8+St+rL35oZmZmhemyWzIi7gRGAJNJktiHgVM6/fz39N9MJJ0naZWkNZJm5Xl/gKT70vcXShqblh8maYGknZJuy149MzPrj7IMKHlW0l8Bv4qI3YWeSFI98B3gHGAd8IykeRHxfM5uVwBbIuIYSZcCNwMfB/YAXwUmpD9mZmZdyjqgZA4wpI/nOgVYExEvRcReYC5wUad9LgLuTrcfAM6WpIh4OyKeIklyZmZm3TqQA0qOAl7Leb0OOLWrfSKiTdI24DBgc4bPR9IMYAbAmDFjshxiZmY1qKYGlKT3Ce8EaGlpiTKHY2ZmZdJlcouIOyXdT7J22xLgPOD3fTjXemB0zutRaVm+fdZJagCG9fGcZmbWD3U7/VZEbAU6BpQ8GRGt3e3fg2eAYyWNI0lil5LMTZlrHjAd+A0wFZgfEW6BmZlZr2SdW/Lunvfq8TPaJM0EHiPpxpwTESsk3Qgsioh5wF3ADyWtAd4iSYAASFoLDAWaJH0UOLfTSEszMzOg+0cBtgPvjYjNknaQ3FvLKyKGZjlZRDwKPNqp7Nqc7T3AxV0cOzbLOczMzLpruf0fYEfOtrsHzcysKnQ3oOTunO1/PiDRmJmZFUHWxUrNSqo/LrTZHxe6rSX+b1jZurvn1k7GrsiIqIjn3GqBv/AqU39deNW/E1atumu5XcI7yW0kcCPwY5Jh+gCnAR8lWZXbzKxo+usfE1Y83d1ze6BjW9I84OqI+F7OLnMkPU2S4P6xdCGamZn1TncTJ+f6ILAgT/kC4KyiRWNmZlYEWZPbZpIZQzqbCmwqXjhmZmZ9l3W05LXA9yX9Ge/cc3s/MIVkDTYzM7OKkXX6rR9IWgV8DvhIWrwSOD0iFpYqODMzs0Jkfs4tTWJ/WcJYzMzMiiLrPTczM7Oq4eRmZmY1x8nNzMxqjpObmZnVnF4nN0kjJTkpmplZxcqUpCQ1Svp6umjpemBsWn6zpM+WMD4zM7Ney9oCuw64ELgMaM0pfxq4vMgxmZmZ9UnW5DYN+N8R8TDQnlO+HDgu68kknSdplaQ1kmbleX+ApPvS9xdKGpvz3tVp+SpJH8p6TjMz63+yJrcjgVfylDeQ8UFwSfXAd4APA+OBaZLGd9rtCmBLRBwDfBO4OT12PHAp8D7gPOAf088zMzN7l6wzlKwAPgCs7VR+CbA442ecAqyJiJcAJM0FLgKez9nnIuD6dPsB4DZJSsvnRkQr8LKkNenn/YYq4UUfzcwOHEX0vNi2pAuBe4CvA18BbgCagU8AF0TEExk+YypwXkR8On39SeDUiJiZs8/ydJ916esXgVNJEt5/RsQ9afldwE9z15xLy2cAMwDGjBkz+ZVX8jU2zayW1NIfjpVQF0mLI6Kl5CcqsUzdkhHxE5JW2rkk99yuA44FLsyS2A6UiLgzIloiomX48OHlDsfMzMqkx25JSQ0kSW1hRPxpH861Hhid83pUWpZvn3XpeYcBv894rJmZGZCh5RYRbcBDwJA+nusZ4FhJ4yQ1kQwQmddpn3nA9HR7KjA/kn7TecCl6WjKcSStxqf7GI+ZmdWorANKlgLH8O4BJZlFRJukmcBjQD0wJyJWSLoRWBQR84C7gB+mA0beIkmApPvdTzL4pA24MiL2FxqLmZnVtqzJ7XrgVknXkYyOfDv3zYh4K8uHRMSjwKOdyq7N2d4DXNzFsTcBN2WM18zM+rGsya1jCM9DQO7wSqWv/cyZmZlVjKzJ7c9KGoWZmVkRZUpuEfFkqQMxMzMrlqwtNySNBK4kmTorSGYtuT0i3ixRbGZmZgXJuuTN6cAakhlJdgN7SFYIWC3ptNKFZ2Zm1ntZW263AD8iWRmgHSBdsPQO4FbgT0oTnpmZWe9lTW4nApd3JDaAiGiX9A3gtyWJzMzMrEBZl7zZBozLUz4O2Fq8cMzMzPoua8ttLnCXpL8F/iMtO51kvbUflSIwMzOzQmVNbn9L8sD2nJxj9gG3A+9aUdvMzKycsj7nthf4vKSrgaPT4hcjYlfJIjMzMytQpuQm6QigIV1EdFlO+Shgn591MzOzSpJ1Je4ngPsi4nudyq8APh4R55YovoK1tLTEokWLyh2GmVlV6VcrcQMtwC/zlP8qfc/MzKxiZE1uDcCAPOUDuyg3MzMrm6zJbSHwmTzlV5KssG1mZlYxsj4K8BVgvqSJwPy07IPAScCUUgRmZmZWqEwDSgAkTQL+hiShQTLt1t9HxNISxdYnkjYBrxTxIw8HNhfx88rJdalMrktl6m91+aOIGH4ggimlzMmtv5O0qBZGEIHrUqlcl8rkulSnrEvejJd0fM7rcyTdI+lqSfWlC8/MzKz3sg4omUPaHSlpNPAwcCjJgJKvlSY0MzOzwmRNbs3AknR7KrAwIs4HPglMK0VgFejOcgdQRK5LZXJdKpPrUoWyzlCyAzghItZK+jfgyYj4e0ljgFURMajUgZqZmWWVteW2HPiMpDOBs4GfpeVHUTujiMzMrEZkTW5fAv4X8AvgRxHRMXnyR4CnSxCXmZlZwTIlt4j4JTAcODwiPpXz1nfJP3NJzZB0nqRVktZIquq16yStlbRM0rOSqm5W7506dQAABg5JREFUaUlzJG2UtDyn7FBJj0tanf57SDljzKqLulwvaX16fZ6VdH45Y8xC0mhJCyQ9L2mFpM+n5VV3XbqpS9VdFwBJAyU9LWlpWp8b0vJxkham32n3SWoqd6yl4OfcupE+5vACcA6wjmSqsWkR8XxZAyuQpLVAS0RUZVeypA8AO4EfRMSEtOzrwFsRMTv94+OQiPhSOePMoou6XA/sjIhbyhlbb0h6D/CeiFgiaQiwGPgocDlVdl26qcslVNl1AZAk4KCI2CmpEXgK+DxwFfBQRMyVdAewNCJuL2espZC1W7K/OgVYExEvpQu2zgUuKnNM/Vbag/BWp+KLgLvT7btJvowqXhd1qToR8XpELEm3dwArSe7FV9116aYuVSkSO9OXjelPkEyd+EBaXhXXphBObt07Cngt5/U6qviXneQX++eSFkuaUe5gimRkRLyebr8BjCxnMEUwU9JzabdlxXfl5ZI0luR52IVU+XXpVBeo0usiqV7Ss8BG4HHgRWBrRLSlu1T7d1qXnNz6lzMi4mTgw8CVaddYzYikj72a+9lvB44GTgReB24tbzjZSRoMPAh8ISK2575XbdclT12q9rpExP6IOBEYRdIT1VzmkA4YJ7furQdG57welZZVpYhYn/67EfgxyS97tXszvVfScc9kY5njKVhEvJl+GbUD36NKrk96P+dB4N6IeCgtrsrrkq8u1XpdckXEVmABcBpwsKSOFWGq+jutO05u3XsGODYdXdQEXArMK3NMBZF0UHqTHEkHAeeSPL9Y7eYB09Pt6SRTw1WljmSQ+hhVcH3SQQt3ASsj4hs5b1XddemqLtV4XQAkDZd0cLo9iGRg3EqSJDc13a0qrk0hPFqyB+mw328B9cCciLipzCEVRNJ7SVprkKzj9/+qrS6SfgScRbJsx5vAdcC/AvcDY0iWOLokIip+oEYXdTmLpOsrgLXAX+fct6pIks4AfgUsA9rT4i+T3KuqquvSTV2mUWXXBUDJ+pt3k3x31QH3R8SN6XfBXJL5gX8LXBYRreWLtDSc3MzMrOa4W9LMzGqOk5uZmdUcJzczM6s5Tm5mZlZznNzMzKzmOLlZTZG0s+e9/mD/s9IFeJH0kZ5WfpB0o6Qp3X1OIdIVGw7vVLYwnYX+VUmbcmalH5vn+O9LOr6Hczwl6cRu3r9A0lfT56N+UmhdzCpBQ8+7mPUPETGPHh7Sj4hrD1A4RMSpAJIuJ1nNYWa+/STVR8RfFeGUZwI/BT5AMoO8WdVyy81qUtqS+oWkByT9TtK96QwUHWv0/U7SEuDPc465XNJtkoZJekVSXVp+kKTXJDVK+mdJU3v4nOsl/d+c18s7WluS/jWduHpFoZNXS2qQtFXStyQ9B5yS2yqTdKekRek5ekzGkv4ynVz3s8A/AHcAn5b04+6PNKtcTm5Wy04CvgCMB94LnC5pIMn8gBcCk4EjOh8UEduAZ4E/TYv+B/BYROzr2CfL53ThUxExGWgBPifpsALqBTAM+GVETIyI33R6b1ZEtACTgHMkje/ugyLiXpI6LIuIicAKYFJEfKzA2MzKzsnNatnTEbEunfD2WWAsyazoL0fE6nS2+nu6OPY+4OPp9qXp61xZP6ezz0laCvwnyaTcx2auzR/ayzvTqXU2LW1NLgH+mCS596QZWJ1uD4yIXQXGZVYRfM/NalnufHn76d3v+zzg7yQdStKqmd+LY9v4wz8cB0LSVQpMAU6LiF2SftHxXgF2R5658yQdS7La8ikRsVXSPT2dI+2SPASol7QSGNnRTRkR/1FgfGZl5Zab9Te/A8ZKOjp9PS3fTukKxs8A3wb+LSL29+Jz1gInA0g6GRiXlg8DtqSJrRl4fx/rks9QYAewPZ3N/kM9HZCu9/Uz4AKStcpmRcSJTmxWzZzcrF+JiD3ADOCRtOuuu3XG7gMu491dkj19zoPAoZJWADOBF9LynwENaetoNknXZLEtAZ4nSb4/AH6d8bhJJLPhnwk8WYK4zA4orwpgZmY1xy03MzOrOU5uZmZWc5zczMys5ji5mZlZzXFyMzOzmuPkZmZmNcfJzczMas7/B4Bh3iAZFPh0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, KFold\n",
    "import numpy as np\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "# Number of random trials\n",
    "NUM_TRIALS = 30\n",
    "\n",
    "# Load the dataset\n",
    "iris = load_iris()\n",
    "X_iris = iris.data\n",
    "y_iris = iris.target\n",
    "\n",
    "# Set up possible values of parameters to optimize over\n",
    "p_grid = {\"C\": [1, 10, 100],\n",
    "          \"gamma\": [.01, .1]}\n",
    "\n",
    "# We will use a Support Vector Classifier with \"rbf\" kernel\n",
    "svm = SVC(kernel=\"rbf\")\n",
    "\n",
    "# Arrays to store scores\n",
    "non_nested_scores = np.zeros(NUM_TRIALS)\n",
    "nested_scores = np.zeros(NUM_TRIALS)\n",
    "\n",
    "# Loop for each trial\n",
    "for i in range(NUM_TRIALS):\n",
    "\n",
    "    # Choose cross-validation techniques for the inner and outer loops,\n",
    "    # independently of the dataset.\n",
    "    # E.g \"LabelKFold\", \"LeaveOneOut\", \"LeaveOneLabelOut\", etc.\n",
    "    inner_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "    outer_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "\n",
    "    # Non_nested parameter search and scoring\n",
    "    clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv)\n",
    "    clf.fit(X_iris, y_iris)\n",
    "    print('clf.params: ', clf.get_params())\n",
    "    print()\n",
    "    non_nested_scores[i] = clf.best_score_\n",
    "    \n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    print()\n",
    "    print(\"Grid scores on development set:\")\n",
    "    print()\n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds = clf.cv_results_['std_test_score']\n",
    "    for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "        print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "              % (mean, std * 2, params))\n",
    "    print()\n",
    "\n",
    "    # Nested CV with parameter optimization\n",
    "    nested_score = cross_val_score(clf, X=X_iris, y=y_iris, cv=outer_cv)\n",
    "    nested_scores[i] = nested_score.mean()\n",
    "\n",
    "score_difference = non_nested_scores - nested_scores\n",
    "\n",
    "print(\"Average difference of {0:6f} with std. dev. of {1:6f}.\"\n",
    "      .format(score_difference.mean(), score_difference.std()))\n",
    "\n",
    "# Plot scores on each trial for nested and non-nested CV\n",
    "plt.figure()\n",
    "plt.subplot(211)\n",
    "non_nested_scores_line, = plt.plot(non_nested_scores, color='r')\n",
    "nested_line, = plt.plot(nested_scores, color='b')\n",
    "plt.ylabel(\"score\", fontsize=\"14\")\n",
    "plt.legend([non_nested_scores_line, nested_line],\n",
    "           [\"Non-Nested CV\", \"Nested CV\"],\n",
    "           bbox_to_anchor=(0, .4, .5, 0))\n",
    "plt.title(\"Non-Nested and Nested Cross Validation on Iris Dataset\",\n",
    "          x=.5, y=1.1, fontsize=\"15\")\n",
    "\n",
    "# Plot bar chart of the difference.\n",
    "plt.subplot(212)\n",
    "difference_plot = plt.bar(range(NUM_TRIALS), score_difference)\n",
    "plt.xlabel(\"Individual Trial #\")\n",
    "plt.legend([difference_plot],\n",
    "           [\"Non-Nested CV - Nested CV Score\"],\n",
    "           bbox_to_anchor=(0, 1, .8, 0))\n",
    "plt.ylabel(\"score difference\", fontsize=\"14\")\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}