{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--- This is a regression problem ---\n",
      "\n",
      "\n",
      "1. Loading dataset from file...\n",
      "\n",
      "2. Calculating gram matrices. This could take a while...\n",
      "\n",
      " None edge weight specified. Set all weight to 1.\n",
      "\n",
      "getting sp graphs: 183it [00:00, 2868.18it/s]\n",
      "calculating kernels: 16836it [00:03, 4962.86it/s]\n",
      "\n",
      " --- shortest path kernel matrix of size 183 built in 3.5163042545318604 seconds ---\n",
      "\n",
      "the gram matrix with parameters {'node_kernels': {'symb': <function deltakernel at 0x7ff63c02c158>, 'nsymb': <function gaussiankernel at 0x7ff642e968c8>, 'mix': functools.partial(<function kernelproduct at 0x7ff60b9d21e0>, <function deltakernel at 0x7ff63c02c158>, <function gaussiankernel at 0x7ff642e968c8>)}, 'n_jobs': 1} is: \n",
      "\n",
      "\n",
      "\n",
      "1 gram matrices are calculated, 0 of which are ignored.\n",
      "\n",
      "3. Fitting and predicting using nested cross validation. This could really take a while...\n",
      "cross validation: 2it [00:00,  4.47it/s]\n",
      "\n",
      "4. Getting final performance...\n",
      "best_params_out:  [{'node_kernels': {'symb': <function deltakernel at 0x7ff63c02c158>, 'nsymb': <function gaussiankernel at 0x7ff642e968c8>, 'mix': functools.partial(<function kernelproduct at 0x7ff60b9d21e0>, <function deltakernel at 0x7ff63c02c158>, <function gaussiankernel at 0x7ff642e968c8>)}, 'n_jobs': 1}]\n",
      "best_params_in:  [{'alpha': 0.0001}]\n",
      "\n",
      "best_val_perf:  9.922073568477266\n",
      "best_val_std:  0.3829108688812842\n",
      "final_performance:  [8.039190309451554]\n",
      "final_confidence:  [2.8576078550320037]\n",
      "train_performance: [6.285008316076738]\n",
      "train_std:  [0.23613211181729038]\n",
      "\n",
      "time to calculate gram matrix with different hyper-params: 3.52±nans\n",
      "time to calculate best gram matrix: 3.52±nans\n",
      "total training time with all hyper-param choices: 4.34s\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:140: RuntimeWarning: Degrees of freedom <= 0 for slice\n",
      "  keepdims=keepdims)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:132: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import sys\n",
    "import functools\n",
    "sys.path.insert(0, \"../../\")\n",
    "from gklearn.utils.model_selection_precomputed import model_selection_for_precomputed_kernel\n",
    "from gklearn.kernels.spKernel import spkernel\n",
    "from gklearn.utils.kernels import deltakernel, gaussiankernel, kernelproduct\n",
    "\n",
    "datafile = '../../datasets/acyclic/dataset_bps.ds'\n",
    "estimator = spkernel\n",
    "# hyper-parameters\n",
    "mixkernel = functools.partial(kernelproduct, deltakernel, gaussiankernel)\n",
    "param_grid_precomputed = {'node_kernels': [{'symb': deltakernel, 'nsymb': gaussiankernel, 'mix': mixkernel}]}\n",
    "param_grid = {\"alpha\": np.logspace(-5, 5, num = 21, base = 10)}\n",
    "\n",
    "model_selection_for_precomputed_kernel(datafile, estimator, param_grid_precomputed, param_grid, \n",
    "                                       'regression', NUM_TRIALS=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'o1': 1, 'o2': 2}, {'o1': 3, 'o2': 4}]\n",
      "[{'i2': 6, 'i1': 5}, {'i2': 8, 'i1': 7}, {'i2': 10, 'i1': 9}]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[({'o1': 1, 'o2': 2}, {'o1': 3, 'o2': 4}),\n",
       " ({'o1': 3, 'o2': 4}, {'o1': 1, 'o2': 2})]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = [ {'o1':1,'o2':2},{'o1':3,'o2':4}]\n",
    "print(x)\n",
    "y = [ {'i1':5,'i2':6},{'i1':7,'i2':8},{'i1':9,'i2':10}]\n",
    "print(y)\n",
    "from itertools import permutations\n",
    "[item for item in permutations(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n",
      "\n",
      "0\n",
      "{'mean_fit_time': array([0.00053245, 0.00036639, 0.00027829, 0.00026679, 0.00036031,\n",
      "       0.00035846]), 'std_fit_time': array([2.35126922e-04, 6.76477532e-05, 1.11031151e-06, 1.48806431e-05,\n",
      "       7.18533189e-05, 6.76927198e-05]), 'mean_score_time': array([0.00021952, 0.00019908, 0.00016928, 0.00015849, 0.00017059,\n",
      "       0.00019705]), 'std_score_time': array([3.96984386e-05, 4.61193611e-05, 7.34855414e-07, 2.82289511e-06,\n",
      "       1.66522193e-05, 2.63754426e-05]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.92857143]), 'split1_test_score': array([0.89285714, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.96428571]), 'split2_test_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.82142857, 0.92857143, 0.92857143, 0.92857143, 0.92857143,\n",
      "       0.92857143]), 'mean_test_score': array([0.88392857, 0.95535714, 0.95535714, 0.97321429, 0.97321429,\n",
      "       0.95535714]), 'std_test_score': array([0.05282214, 0.02961272, 0.02961272, 0.02961272, 0.02961272,\n",
      "       0.02961272]), 'rank_test_score': array([6, 3, 3, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.91666667, 0.97619048, 0.97619048, 0.97619048, 0.97619048,\n",
      "       0.98809524]), 'split1_train_score': array([0.91666667, 0.97619048, 0.96428571, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split2_train_score': array([0.9047619 , 0.97619048, 0.97619048, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split3_train_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.98809524]), 'mean_train_score': array([0.92559524, 0.98214286, 0.97916667, 0.98214286, 0.98214286,\n",
      "       0.98214286]), 'std_train_score': array([0.02286055, 0.01030983, 0.01297291, 0.01330993, 0.01330993,\n",
      "       0.01030983])}\n",
      "\n",
      "1\n",
      "{'mean_fit_time': array([0.00036663, 0.0002979 , 0.00025547, 0.0002926 , 0.00026369,\n",
      "       0.0002656 ]), 'std_fit_time': array([2.12649960e-05, 2.13949957e-05, 4.63078609e-06, 4.18155790e-05,\n",
      "       1.60335387e-05, 7.18625396e-06]), 'mean_score_time': array([0.00018156, 0.00017273, 0.00015974, 0.00017357, 0.00015533,\n",
      "       0.00015408]), 'std_score_time': array([2.59263076e-06, 1.78372315e-05, 8.02872574e-06, 2.67129904e-05,\n",
      "       1.22153172e-06, 7.79431869e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.89285714, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.92857143]), 'split1_test_score': array([0.82142857, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.96428571]), 'split3_test_score': array([0.89285714, 0.92857143, 0.89285714, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'mean_test_score': array([0.88392857, 0.95535714, 0.94642857, 0.98214286, 0.98214286,\n",
      "       0.96428571]), 'std_test_score': array([0.03891874, 0.02961272, 0.03992979, 0.01785714, 0.01785714,\n",
      "       0.02525381]), 'rank_test_score': array([6, 4, 5, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.92857143, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split1_train_score': array([0.91666667, 0.96428571, 0.96428571, 0.97619048, 0.97619048,\n",
      "       0.97619048]), 'split2_train_score': array([0.92857143, 0.95238095, 0.95238095, 0.97619048, 0.98809524,\n",
      "       0.97619048]), 'split3_train_score': array([0.91666667, 0.95238095, 0.95238095, 1.        , 1.        ,\n",
      "       0.98809524]), 'mean_train_score': array([0.92261905, 0.96130952, 0.96130952, 0.98511905, 0.98809524,\n",
      "       0.98214286]), 'std_train_score': array([0.00595238, 0.00987091, 0.00987091, 0.00987091, 0.00841794,\n",
      "       0.00595238])}\n",
      "\n",
      "2\n",
      "{'mean_fit_time': array([0.00041491, 0.00027972, 0.00026184, 0.00033522, 0.0002436 ,\n",
      "       0.00027043]), 'std_fit_time': array([3.45070743e-05, 1.29256902e-05, 1.01382227e-05, 9.88078909e-05,\n",
      "       8.90705614e-06, 2.57328515e-05]), 'mean_score_time': array([0.00019699, 0.00015837, 0.00018364, 0.00016433, 0.00014621,\n",
      "       0.00014848]), 'std_score_time': array([1.55497312e-05, 1.34209755e-06, 4.97970678e-05, 1.56898401e-05,\n",
      "       9.88431212e-07, 8.49235466e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split1_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.85714286, 0.89285714, 0.89285714, 0.89285714, 0.92857143,\n",
      "       0.92857143]), 'mean_test_score': array([0.90178571, 0.95535714, 0.95535714, 0.96428571, 0.97321429,\n",
      "       0.97321429]), 'std_test_score': array([0.04639422, 0.04639422, 0.04639422, 0.04374089, 0.02961272,\n",
      "       0.02961272]), 'rank_test_score': array([6, 4, 4, 3, 1, 1], dtype=int32), 'split0_train_score': array([0.94047619, 0.96428571, 0.96428571, 0.96428571, 0.96428571,\n",
      "       0.97619048]), 'split1_train_score': array([0.91666667, 0.97619048, 0.96428571, 0.97619048, 0.97619048,\n",
      "       0.98809524]), 'split2_train_score': array([0.89285714, 0.95238095, 0.95238095, 0.97619048, 0.97619048,\n",
      "       0.97619048]), 'split3_train_score': array([0.92857143, 0.98809524, 0.98809524, 1.        , 0.98809524,\n",
      "       1.        ]), 'mean_train_score': array([0.91964286, 0.9702381 , 0.9672619 , 0.97916667, 0.97619048,\n",
      "       0.98511905]), 'std_train_score': array([0.01760738, 0.01330993, 0.01297291, 0.01297291, 0.00841794,\n",
      "       0.00987091])}\n",
      "\n",
      "3\n",
      "{'mean_fit_time': array([0.00033492, 0.0002749 , 0.00025362, 0.00025046, 0.00024259,\n",
      "       0.00024784]), 'std_fit_time': array([1.22110993e-05, 8.60786829e-06, 1.90268632e-06, 8.94069672e-06,\n",
      "       1.44803716e-05, 9.40847565e-06]), 'mean_score_time': array([0.0001812 , 0.00015831, 0.0001632 , 0.00018615, 0.00014585,\n",
      "       0.00014651]), 'std_score_time': array([1.21605162e-05, 1.35919502e-06, 1.42478791e-05, 6.11994010e-05,\n",
      "       1.97686242e-07, 6.84805232e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 1.        ,\n",
      "       0.96428571]), 'split1_test_score': array([0.85714286, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.89285714]), 'split2_test_score': array([0.96428571, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.92857143, 0.92857143, 0.92857143, 0.89285714, 0.89285714,\n",
      "       0.85714286]), 'mean_test_score': array([0.90178571, 0.94642857, 0.94642857, 0.95535714, 0.96428571,\n",
      "       0.92857143]), 'std_test_score': array([0.04639422, 0.03092948, 0.03092948, 0.03891874, 0.04374089,\n",
      "       0.05646924]), 'rank_test_score': array([6, 3, 3, 2, 1, 5], dtype=int32), 'split0_train_score': array([0.92857143, 0.94047619, 0.94047619, 0.98809524, 0.98809524,\n",
      "       0.97619048]), 'split1_train_score': array([0.95238095, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_train_score': array([0.92857143, 0.95238095, 0.95238095, 0.97619048, 0.96428571,\n",
      "       0.97619048]), 'split3_train_score': array([0.91666667, 0.96428571, 0.97619048, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'mean_train_score': array([0.93154762, 0.95535714, 0.95833333, 0.98809524, 0.98511905,\n",
      "       0.98809524]), 'std_train_score': array([0.01297291, 0.00987091, 0.01330993, 0.00841794, 0.01297291,\n",
      "       0.01190476])}\n",
      "\n",
      "4\n",
      "{'mean_fit_time': array([0.00034726, 0.00027776, 0.0002659 , 0.00026214, 0.00024235,\n",
      "       0.00024557]), 'std_fit_time': array([7.59672831e-06, 6.21948121e-06, 8.52721472e-06, 5.29509361e-06,\n",
      "       5.88971106e-06, 2.99212942e-06]), 'mean_score_time': array([0.00018328, 0.00016528, 0.00016391, 0.00015694, 0.00015819,\n",
      "       0.00014925]), 'std_score_time': array([5.45341785e-06, 1.78714566e-06, 1.12461819e-06, 1.38380370e-06,\n",
      "       4.00550309e-06, 8.92080638e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([1.        , 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.96428571]), 'split1_test_score': array([0.89285714, 0.92857143, 0.92857143, 0.92857143, 0.96428571,\n",
      "       0.96428571]), 'split2_test_score': array([0.92857143, 0.92857143, 0.96428571, 0.96428571, 1.        ,\n",
      "       0.96428571]), 'split3_test_score': array([0.57142857, 0.92857143, 0.92857143, 0.92857143, 0.89285714,\n",
      "       0.89285714]), 'mean_test_score': array([0.84821429, 0.94642857, 0.95535714, 0.95535714, 0.96428571,\n",
      "       0.94642857]), 'std_test_score': array([0.16439243, 0.03092948, 0.02961272, 0.02961272, 0.04374089,\n",
      "       0.03092948]), 'rank_test_score': array([6, 4, 2, 2, 1, 4], dtype=int32), 'split0_train_score': array([0.89285714, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split1_train_score': array([0.94047619, 0.97619048, 0.97619048, 1.        , 0.98809524,\n",
      "       1.        ]), 'split2_train_score': array([0.94047619, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split3_train_score': array([0.91666667, 0.96428571, 0.96428571, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.92261905, 0.97321429, 0.97321429, 0.99107143, 0.98809524,\n",
      "       0.99107143]), 'std_train_score': array([0.01974181, 0.00515491, 0.00515491, 0.00515491, 0.        ,\n",
      "       0.00515491])}\n",
      "\n",
      "5\n",
      "{'mean_fit_time': array([0.00037438, 0.00028503, 0.0002768 , 0.00026542, 0.00026023,\n",
      "       0.00027144]), 'std_fit_time': array([4.16169772e-05, 7.12169506e-06, 1.29175793e-05, 1.24644985e-05,\n",
      "       1.07069587e-05, 4.76091517e-06]), 'mean_score_time': array([0.00018823, 0.00016671, 0.00016403, 0.00015712, 0.0001545 ,\n",
      "       0.00015604]), 'std_score_time': array([2.45756246e-06, 1.61153148e-06, 2.50623179e-06, 6.95103633e-07,\n",
      "       2.53442426e-06, 2.43432258e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.78571429, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split1_test_score': array([0.82142857, 0.89285714, 0.89285714, 0.92857143, 0.92857143,\n",
      "       0.89285714]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.92857143, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.89285714]), 'mean_test_score': array([0.86607143, 0.95535714, 0.95535714, 0.98214286, 0.98214286,\n",
      "       0.94642857]), 'std_test_score': array([0.06376275, 0.03891874, 0.03891874, 0.03092948, 0.03092948,\n",
      "       0.05357143]), 'rank_test_score': array([6, 3, 3, 1, 1, 5], dtype=int32), 'split0_train_score': array([0.80952381, 0.96428571, 0.96428571, 0.96428571, 0.96428571,\n",
      "       0.98809524]), 'split1_train_score': array([0.91666667, 0.98809524, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_train_score': array([0.92857143, 0.96428571, 0.96428571, 0.98809524, 0.97619048,\n",
      "       0.98809524]), 'split3_train_score': array([0.89285714, 0.96428571, 0.96428571, 0.97619048, 0.97619048,\n",
      "       0.97619048]), 'mean_train_score': array([0.88690476, 0.9702381 , 0.9702381 , 0.98214286, 0.97916667,\n",
      "       0.98809524]), 'std_train_score': array([0.04648958, 0.01030983, 0.01030983, 0.01330993, 0.01297291,\n",
      "       0.00841794])}\n",
      "\n",
      "6\n",
      "{'mean_fit_time': array([0.00034338, 0.0002805 , 0.00026369, 0.00026369, 0.00024992,\n",
      "       0.0002557 ]), 'std_fit_time': array([3.23131947e-06, 3.28853469e-06, 5.84003864e-07, 1.35710234e-05,\n",
      "       1.34557403e-05, 9.11074105e-06]), 'mean_score_time': array([0.00018913, 0.00016457, 0.00017023, 0.00016582, 0.00015312,\n",
      "       0.0001536 ]), 'std_score_time': array([1.62092856e-05, 1.56682008e-06, 1.34922589e-05, 1.07849824e-05,\n",
      "       7.22667912e-07, 3.09714819e-07]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.85714286, 0.96428571, 0.96428571, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split1_test_score': array([0.96428571, 1.        , 0.96428571, 1.        , 1.        ,\n",
      "       0.96428571]), 'split2_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.92857143, 1.        , 1.        , 1.        , 1.        ,\n",
      "       0.96428571]), 'mean_test_score': array([0.91964286, 0.99107143, 0.98214286, 0.99107143, 0.99107143,\n",
      "       0.97321429]), 'std_test_score': array([0.03891874, 0.01546474, 0.01785714, 0.01546474, 0.01546474,\n",
      "       0.01546474]), 'rank_test_score': array([6, 1, 4, 1, 1, 5], dtype=int32), 'split0_train_score': array([0.89285714, 0.98809524, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'split1_train_score': array([0.96428571, 0.98809524, 0.98809524, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split2_train_score': array([0.92857143, 0.98809524, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split3_train_score': array([0.95238095, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.93452381, 0.98511905, 0.98214286, 0.99107143, 0.99107143,\n",
      "       0.99107143]), 'std_train_score': array([0.02727724, 0.00515491, 0.00595238, 0.00515491, 0.00515491,\n",
      "       0.00515491])}\n",
      "\n",
      "7\n",
      "{'mean_fit_time': array([0.00034738, 0.00028139, 0.00030065, 0.00026196, 0.00025433,\n",
      "       0.00026512]), 'std_fit_time': array([2.23039275e-05, 5.79454027e-06, 6.06874812e-05, 6.98443351e-06,\n",
      "       1.43758446e-05, 1.10845410e-05]), 'mean_score_time': array([0.00017852, 0.00016618, 0.0001632 , 0.00016266, 0.00015312,\n",
      "       0.00015479]), 'std_score_time': array([2.90721617e-06, 1.46971083e-06, 2.00541065e-06, 1.30569549e-05,\n",
      "       1.66359970e-06, 2.50268540e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.96428571, 1.        , 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split1_test_score': array([0.75, 1.  , 1.  , 1.  , 1.  , 1.  ]), 'split2_test_score': array([0.67857143, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.96428571]), 'split3_test_score': array([0.96428571, 0.92857143, 0.92857143, 1.        , 1.        ,\n",
      "       0.96428571]), 'mean_test_score': array([0.83928571, 0.97321429, 0.96428571, 1.        , 1.        ,\n",
      "       0.98214286]), 'std_test_score': array([0.12752551, 0.02961272, 0.02525381, 0.        , 0.        ,\n",
      "       0.01785714]), 'rank_test_score': array([6, 4, 5, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.94047619, 0.96428571, 0.97619048, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split1_train_score': array([0.83333333, 0.97619048, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_train_score': array([0.80952381, 0.98809524, 0.98809524, 0.98809524, 1.        ,\n",
      "       0.98809524]), 'split3_train_score': array([0.89285714, 0.98809524, 0.98809524, 1.        , 1.        ,\n",
      "       1.        ]), 'mean_train_score': array([0.86904762, 0.97916667, 0.98511905, 0.99404762, 0.99702381,\n",
      "       0.99702381]), 'std_train_score': array([0.05120432, 0.00987091, 0.00515491, 0.00595238, 0.00515491,\n",
      "       0.00515491])}\n",
      "\n",
      "8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([0.00067562, 0.00042343, 0.00038534, 0.00038195, 0.00039953,\n",
      "       0.00027204]), 'std_fit_time': array([1.49889683e-04, 3.03546273e-05, 1.74643643e-05, 1.06530693e-05,\n",
      "       7.33150457e-05, 1.69947466e-05]), 'mean_score_time': array([0.00028437, 0.00026864, 0.00024623, 0.00024372, 0.0002315 ,\n",
      "       0.00015861]), 'std_score_time': array([5.55748990e-05, 2.98201389e-05, 1.44350341e-05, 9.34615275e-06,\n",
      "       6.64798903e-06, 2.83294550e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.89285714, 1.        , 1.        , 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split1_test_score': array([0.89285714, 1.        , 0.96428571, 1.        , 1.        ,\n",
      "       1.        ]), 'split2_test_score': array([0.89285714, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split3_test_score': array([0.78571429, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'mean_test_score': array([0.86607143, 1.        , 0.99107143, 0.99107143, 0.99107143,\n",
      "       0.99107143]), 'std_test_score': array([0.04639422, 0.        , 0.01546474, 0.01546474, 0.01546474,\n",
      "       0.01546474]), 'rank_test_score': array([6, 1, 2, 2, 2, 2], dtype=int32), 'split0_train_score': array([0.91666667, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'split1_train_score': array([0.91666667, 0.97619048, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'split2_train_score': array([0.95238095, 0.98809524, 0.98809524, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split3_train_score': array([0.91666667, 1.        , 1.        , 0.98809524, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.92559524, 0.99107143, 0.99107143, 0.99107143, 0.99107143,\n",
      "       0.99404762]), 'std_train_score': array([0.01546474, 0.00987091, 0.00987091, 0.00515491, 0.00515491,\n",
      "       0.00595238])}\n",
      "\n",
      "9\n",
      "{'mean_fit_time': array([0.00042182, 0.00032252, 0.00031126, 0.00025773, 0.0002439 ,\n",
      "       0.00026196]), 'std_fit_time': array([8.71577873e-05, 3.58894707e-05, 1.64262442e-05, 3.96449307e-06,\n",
      "       4.98117981e-06, 1.21702993e-05]), 'mean_score_time': array([0.00018883, 0.0001893 , 0.00019503, 0.00015533, 0.00015396,\n",
      "       0.00015324]), 'std_score_time': array([4.33436846e-06, 1.25877333e-05, 2.04790318e-05, 5.46285593e-07,\n",
      "       7.79431869e-07, 3.12398855e-06]), 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_gamma': masked_array(data=[0.01, 0.1, 0.01, 0.1, 0.01, 0.1],\n",
      "             mask=[False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 1, 'gamma': 0.01}, {'C': 1, 'gamma': 0.1}, {'C': 10, 'gamma': 0.01}, {'C': 10, 'gamma': 0.1}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.1}], 'split0_test_score': array([0.92857143, 0.92857143, 0.92857143, 0.96428571, 0.96428571,\n",
      "       0.96428571]), 'split1_test_score': array([0.92857143, 0.96428571, 0.96428571, 1.        , 1.        ,\n",
      "       0.92857143]), 'split2_test_score': array([1., 1., 1., 1., 1., 1.]), 'split3_test_score': array([0.89285714, 1.        , 1.        , 1.        , 1.        ,\n",
      "       1.        ]), 'mean_test_score': array([0.9375    , 0.97321429, 0.97321429, 0.99107143, 0.99107143,\n",
      "       0.97321429]), 'std_test_score': array([0.03891874, 0.02961272, 0.02961272, 0.01546474, 0.01546474,\n",
      "       0.02961272]), 'rank_test_score': array([6, 3, 3, 1, 1, 3], dtype=int32), 'split0_train_score': array([0.9047619 , 0.98809524, 0.98809524, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split1_train_score': array([0.9047619 , 0.96428571, 0.94047619, 0.98809524, 0.98809524,\n",
      "       1.        ]), 'split2_train_score': array([0.92857143, 0.96428571, 0.97619048, 0.98809524, 0.98809524,\n",
      "       0.97619048]), 'split3_train_score': array([0.91666667, 0.95238095, 0.95238095, 0.97619048, 0.98809524,\n",
      "       0.98809524]), 'mean_train_score': array([0.91369048, 0.9672619 , 0.96428571, 0.98511905, 0.98809524,\n",
      "       0.99107143]), 'std_train_score': array([0.00987091, 0.01297291, 0.01882308, 0.00515491, 0.        ,\n",
      "       0.00987091])}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, KFold, train_test_split\n",
    "import numpy as np\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "# Number of random trials\n",
    "NUM_TRIALS = 10\n",
    "\n",
    "# Load the dataset\n",
    "iris = load_iris()\n",
    "X_iris = iris.data\n",
    "y_iris = iris.target\n",
    "\n",
    "# Set up possible values of parameters to optimize over\n",
    "p_grid = {\"C\": [1, 10, 100],\n",
    "          \"gamma\": [.01, .1]}\n",
    "\n",
    "# We will use a Support Vector Classifier with \"rbf\" kernel\n",
    "svm = SVC(kernel=\"rbf\")\n",
    "\n",
    "# Arrays to store scores\n",
    "val_scores =  np.zeros((NUM_TRIALS,len(p_grid['C'])))\n",
    "test_scores = np.zeros((NUM_TRIALS,len(p_grid['C'])))\n",
    "\n",
    "best_params = []\n",
    "# Loop for each trial\n",
    "for i in range(NUM_TRIALS): #Test set level\n",
    "    print()\n",
    "    print(i)\n",
    "    X_app,X_test,y_app,y_test = train_test_split(X_iris,y_iris)\n",
    "    inner_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "    # Non_nested parameter search and scoring\n",
    "    clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv)\n",
    "    clf.fit(X_app, y_app)\n",
    "    print(clf.cv_results_)\n",
    "    best_params.append(clf.best_params_)\n",
    "    val_scores[i] = clf.best_score_\n",
    "    test_scores[i] = clf.score(X_test,y_test)\n",
    "\n",
    "final_performance = np.mean(test_scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_split.py:1943: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.datasets.samples_generator import make_classification\n",
    "from sklearn.svm import LinearSVC, SVC\n",
    "from sklearn.metrics import f1_score, precision_score\n",
    "\n",
    "X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)\n",
    "# compute the training kernel matrix corresponding to the linear kernel\n",
    "K_train = np.dot(X_[:180], X_[:180].T)\n",
    "y_train = y_[:180]\n",
    "clf = SVC(kernel='precomputed')\n",
    "cv = GridSearchCV(clf, {'C': [0.1, 1.0]})\n",
    "cv.fit(K_train, y_train)\n",
    "# compute the test kernel matrix\n",
    "K_test = np.dot(X_[180:], X_[:180].T)\n",
    "y_test = y_[180:]\n",
    "y_pred = cv.predict(K_test)\n",
    "np.mean(y_pred == y_test) >= 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 3] [0 1]\n",
      "[0 1] [2 3]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "X = [\"a\", \"b\", \"c\", \"d\"]\n",
    "kf = KFold(n_splits=2)\n",
    "for train, test in kf.split(X):\n",
    "    print(\"%s %s\" % (train, test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9733333333333334"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import datasets\n",
    "from sklearn import svm\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "iris.data.shape, iris.target.shape\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    iris.data, iris.target, test_size=0.4, random_state=0)\n",
    "\n",
    "X_train.shape, y_train.shape\n",
    "\n",
    "X_test.shape, y_test.shape\n",
    "\n",
    "\n",
    "clf = svm.SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)\n",
    "\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, cross_validate\n",
    "clf = svm.SVC(kernel='linear', C=1)\n",
    "scores = cross_validate(clf, iris.data, iris.target, cv=5, return_train_score=True)\n",
    "scores  \n",
    "# print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "\n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import accuracy_score\n",
    "predicted = cross_val_predict(clf, iris.data, iris.target, cv=10)\n",
    "accuracy_score(iris.target, predicted) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=0, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.085) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.079) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.079) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.1}\n",
      "0.960 (+/-0.060) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.038) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=1, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.101) for {'C': 1, 'gamma': 0.01}\n",
      "0.947 (+/-0.064) for {'C': 1, 'gamma': 0.1}\n",
      "0.940 (+/-0.086) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.1}\n",
      "0.960 (+/-0.027) for {'C': 100, 'gamma': 0.01}\n",
      "0.933 (+/-0.080) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=2, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.039) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.066) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.070) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=3, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.117) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.046) for {'C': 10, 'gamma': 0.1}\n",
      "0.960 (+/-0.046) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.059) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=4, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.042) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.037) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=5, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.066) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.023) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.023) for {'C': 100, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=6, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.084) for {'C': 1, 'gamma': 0.01}\n",
      "0.947 (+/-0.064) for {'C': 1, 'gamma': 0.1}\n",
      "0.953 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.001) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.039) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=7, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.124) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.087) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.058) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.058) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.933 (+/-0.096) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=8, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.078) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.026) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.084) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=9, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.129) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.045) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.028) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=10, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.101) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.044) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=11, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.083) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.026) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.043) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.037) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=12, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.069) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.024) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=13, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.091) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 1, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.053) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.065) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=14, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.086) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=15, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.953 (+/-0.044) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.038) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.059) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=16, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.900 (+/-0.088) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.024) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.027) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.057) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=17, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.920 (+/-0.084) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.940 (+/-0.043) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=18, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.927 (+/-0.059) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.054) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.054) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.045) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.038) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.070) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=19, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.907 (+/-0.057) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.057) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.043) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.023) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=20, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.893 (+/-0.107) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.038) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.023) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.080) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=21, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.038) for {'C': 1, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 1, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.987 (+/-0.046) for {'C': 100, 'gamma': 0.01}\n",
      "0.973 (+/-0.065) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=22, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.078) for {'C': 1, 'gamma': 0.01}\n",
      "0.940 (+/-0.044) for {'C': 1, 'gamma': 0.1}\n",
      "0.953 (+/-0.043) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.059) for {'C': 10, 'gamma': 0.1}\n",
      "0.953 (+/-0.043) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.045) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=23, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.057) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.027) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.091) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=24, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.907 (+/-0.048) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.070) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.973 (+/-0.037) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.947 (+/-0.066) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=25, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.913 (+/-0.069) for {'C': 1, 'gamma': 0.01}\n",
      "0.953 (+/-0.043) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.068) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=26, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.907 (+/-0.091) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.059) for {'C': 1, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.1}\n",
      "0.967 (+/-0.059) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=27, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.947 (+/-0.001) for {'C': 1, 'gamma': 0.01}\n",
      "0.980 (+/-0.044) for {'C': 1, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.023) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.967 (+/-0.044) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf.params:  {'cv': KFold(n_splits=4, random_state=28, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 100, 'gamma': 0.01}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.933 (+/-0.060) for {'C': 1, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 1, 'gamma': 0.1}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.01}\n",
      "0.960 (+/-0.060) for {'C': 10, 'gamma': 0.1}\n",
      "0.973 (+/-0.037) for {'C': 100, 'gamma': 0.01}\n",
      "0.960 (+/-0.089) for {'C': 100, 'gamma': 0.1}\n",
      "\n",
      "clf.params:  {'cv': KFold(n_splits=4, random_state=29, shuffle=True), 'error_score': 'raise-deprecating', 'estimator__C': 1.0, 'estimator__cache_size': 200, 'estimator__class_weight': None, 'estimator__coef0': 0.0, 'estimator__decision_function_shape': 'ovr', 'estimator__degree': 3, 'estimator__gamma': 'auto_deprecated', 'estimator__kernel': 'rbf', 'estimator__max_iter': -1, 'estimator__probability': False, 'estimator__random_state': None, 'estimator__shrinking': True, 'estimator__tol': 0.001, 'estimator__verbose': False, 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False), 'fit_params': None, 'iid': 'warn', 'n_jobs': None, 'param_grid': {'C': [1, 10, 100], 'gamma': [0.01, 0.1]}, 'pre_dispatch': '2*n_jobs', 'refit': True, 'return_train_score': 'warn', 'scoring': None, 'verbose': 0}\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 10, 'gamma': 0.1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.940 (+/-0.022) for {'C': 1, 'gamma': 0.01}\n",
      "0.967 (+/-0.022) for {'C': 1, 'gamma': 0.1}\n",
      "0.973 (+/-0.001) for {'C': 10, 'gamma': 0.01}\n",
      "0.980 (+/-0.044) for {'C': 10, 'gamma': 0.1}\n",
      "0.980 (+/-0.044) for {'C': 100, 'gamma': 0.01}\n",
      "0.953 (+/-0.043) for {'C': 100, 'gamma': 0.1}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average difference of 0.007742 with std. dev. of 0.007688.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 35100 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, KFold\n",
    "import numpy as np\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "# Number of random trials\n",
    "NUM_TRIALS = 30\n",
    "\n",
    "# Load the dataset\n",
    "iris = load_iris()\n",
    "X_iris = iris.data\n",
    "y_iris = iris.target\n",
    "\n",
    "# Set up possible values of parameters to optimize over\n",
    "p_grid = {\"C\": [1, 10, 100],\n",
    "          \"gamma\": [.01, .1]}\n",
    "\n",
    "# We will use a Support Vector Classifier with \"rbf\" kernel\n",
    "svm = SVC(kernel=\"rbf\")\n",
    "\n",
    "# Arrays to store scores\n",
    "non_nested_scores = np.zeros(NUM_TRIALS)\n",
    "nested_scores = np.zeros(NUM_TRIALS)\n",
    "\n",
    "# Loop for each trial\n",
    "for i in range(NUM_TRIALS):\n",
    "\n",
    "    # Choose cross-validation techniques for the inner and outer loops,\n",
    "    # independently of the dataset.\n",
    "    # E.g \"LabelKFold\", \"LeaveOneOut\", \"LeaveOneLabelOut\", etc.\n",
    "    inner_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "    outer_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "\n",
    "    # Non_nested parameter search and scoring\n",
    "    clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv)\n",
    "    clf.fit(X_iris, y_iris)\n",
    "    print('clf.params: ', clf.get_params())\n",
    "    print()\n",
    "    non_nested_scores[i] = clf.best_score_\n",
    "    \n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    print()\n",
    "    print(\"Grid scores on development set:\")\n",
    "    print()\n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds = clf.cv_results_['std_test_score']\n",
    "    for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "        print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "              % (mean, std * 2, params))\n",
    "    print()\n",
    "\n",
    "    # Nested CV with parameter optimization\n",
    "    nested_score = cross_val_score(clf, X=X_iris, y=y_iris, cv=outer_cv)\n",
    "    nested_scores[i] = nested_score.mean()\n",
    "\n",
    "score_difference = non_nested_scores - nested_scores\n",
    "\n",
    "print(\"Average difference of {0:6f} with std. dev. of {1:6f}.\"\n",
    "      .format(score_difference.mean(), score_difference.std()))\n",
    "\n",
    "# Plot scores on each trial for nested and non-nested CV\n",
    "plt.figure()\n",
    "plt.subplot(211)\n",
    "non_nested_scores_line, = plt.plot(non_nested_scores, color='r')\n",
    "nested_line, = plt.plot(nested_scores, color='b')\n",
    "plt.ylabel(\"score\", fontsize=\"14\")\n",
    "plt.legend([non_nested_scores_line, nested_line],\n",
    "           [\"Non-Nested CV\", \"Nested CV\"],\n",
    "           bbox_to_anchor=(0, .4, .5, 0))\n",
    "plt.title(\"Non-Nested and Nested Cross Validation on Iris Dataset\",\n",
    "          x=.5, y=1.1, fontsize=\"15\")\n",
    "\n",
    "# Plot bar chart of the difference.\n",
    "plt.subplot(212)\n",
    "difference_plot = plt.bar(range(NUM_TRIALS), score_difference)\n",
    "plt.xlabel(\"Individual Trial #\")\n",
    "plt.legend([difference_plot],\n",
    "           [\"Non-Nested CV - Nested CV Score\"],\n",
    "           bbox_to_anchor=(0, 1, .8, 0))\n",
    "plt.ylabel(\"score difference\", fontsize=\"14\")\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
