{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--- This is a regression problem ---\n",
      "\n",
      "1. Loading dataset from file...\n",
      "\n",
      "2. Calculating gram matrices. This could take a while...\n",
      "--- shortest path kernel matrix of size 183 built in 12.671883821487427 seconds ---\n",
      "\n",
      "gram matrix with parameters {} is: \n",
      "[[ 3.  1.  3. ...  1.  1.  1.]\n",
      " [ 1.  6.  1. ...  0.  0.  3.]\n",
      " [ 3.  1.  3. ...  1.  1.  1.]\n",
      " ...\n",
      " [ 1.  0.  1. ... 55. 21.  7.]\n",
      " [ 1.  0.  1. ... 21. 55.  7.]\n",
      " [ 1.  3.  1. ...  7.  7. 55.]]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 60964 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbc003bc7f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "3. Fitting and predicting using nested cross validation. This could really take a while...\n",
      "calculate performance:   0%|          | 1/630 [00:00<01:12,  8.66it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ljia/.local/lib/python3.5/site-packages/sklearn/linear_model/ridge.py:154: UserWarning: Singular matrix in solving dual problem. Using least-squares solution instead.\n",
      "  warnings.warn(\"Singular matrix in solving dual problem. Using \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                        \n",
      "4. Getting final performances...\n",
      "\n",
      "best_params_out:  [{}]\n",
      "best_params_in:  [{'alpha': 35.481338923357534}]\n",
      "best_val_perf:  36.08752278925261\n",
      "best_val_std:  0.9660235949265693\n",
      "final_performance:  36.45033825376652\n",
      "final_confidence:  6.625276016213645\n",
      "train_performance:  28.645168242623424\n",
      "train_std:  0.59327479451686\n",
      "time to calculate gram matrix:  12.671883821487427 s\n",
      "\n",
      "params                 train_perf      valid_perf       test_perf          gram_matrix_time\n",
      "---------------------  --------------  ---------------  ---------------  ------------------\n",
      "{'alpha': '1.00e-01'}  463.68±735.85   1255.03±1875.15  1552.81±4500.30               12.67\n",
      "{'alpha': '1.41e-01'}  371.29±454.77   910.40±924.50    893.80±1213.60                12.67\n",
      "{'alpha': '2.00e-01'}  522.21±1406.90  1587.46±4629.03  1301.10±3657.93               12.67\n",
      "{'alpha': '2.82e-01'}  248.98±245.92   721.57±817.51    569.68±566.78                 12.67\n",
      "{'alpha': '3.98e-01'}  350.52±614.63   848.29±1379.89   830.27±1370.70                12.67\n",
      "{'alpha': '5.62e-01'}  167.98±215.81   434.68±466.73    566.30±1462.66                12.67\n",
      "{'alpha': '7.94e-01'}  157.92±268.28   432.36±664.00    452.75±857.58                 12.67\n",
      "{'alpha': '1.12e+00'}  134.98±151.51   438.74±741.59    386.57±589.84                 12.67\n",
      "{'alpha': '1.58e+00'}  97.98±149.69    234.55±335.91    237.38±304.30                 12.67\n",
      "{'alpha': '2.24e+00'}  55.08±52.90     152.16±218.04    142.67±180.16                 12.67\n",
      "{'alpha': '3.16e+00'}  57.00±44.47     170.95±206.48    143.07±176.76                 12.67\n",
      "{'alpha': '4.47e+00'}  41.42±43.03     98.89±127.39     95.73±185.90                  12.67\n",
      "{'alpha': '6.31e+00'}  29.24±1.43      48.94±3.64       48.05±9.90                    12.67\n",
      "{'alpha': '8.91e+00'}  28.33±0.95      43.38±1.83       42.94±6.50                    12.67\n",
      "{'alpha': '1.26e+01'}  27.92±0.76      40.24±1.40       40.08±5.78                    12.67\n",
      "{'alpha': '1.78e+01'}  27.82±0.66      38.14±1.18       38.19±5.84                    12.67\n",
      "{'alpha': '2.51e+01'}  28.04±0.61      36.78±1.05       37.00±6.19                    12.67\n",
      "{'alpha': '3.55e+01'}  28.65±0.59      36.09±0.97       36.45±6.63                    12.67\n",
      "{'alpha': '5.01e+01'}  32.00±10.86     37.44±6.49       38.66±11.46                   12.67\n",
      "{'alpha': '7.08e+01'}  49.88±23.28     50.68±16.29      52.34±21.79                   12.67\n",
      "{'alpha': '1.00e+02'}  35.36±1.02      39.45±1.20       40.39±8.79                    12.67\n",
      "calculate performance: 100%|██████████| 630/630 [01:00<00:00, 16.40it/s]"
     ]
    }
   ],
   "source": [
    "%load_ext line_profiler\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sys\n",
    "sys.path.insert(0, \"../\")\n",
    "from pygraph.utils.model_selection_precomputed import model_selection_for_precomputed_kernel\n",
    "from pygraph.kernels.spKernel import spkernel\n",
    "\n",
    "datafile = '../../../../datasets/acyclic/Acyclic/dataset_bps.ds'\n",
    "estimator = spkernel\n",
    "param_grid_precomputed = {}\n",
    "param_grid = {'alpha': np.logspace(-1, 2, num = 21, base = 10)}\n",
    "\n",
    "model_selection_for_precomputed_kernel(datafile, estimator, param_grid_precomputed, param_grid, \n",
    "                                       'regression', NUM_TRIALS=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " --- This is a regression problem ---\n",
      "\n",
      "\n",
      " Loading dataset from file...\n",
      "\n",
      " Calculating kernel matrix, this could take a while...\n",
      "--- shortest path kernel matrix of size 185 built in 13.3865065574646 seconds ---\n",
      "[[ 3.  1.  3. ...  1.  1.  1.]\n",
      " [ 1.  6.  1. ...  0.  0.  3.]\n",
      " [ 3.  1.  3. ...  1.  1.  1.]\n",
      " ...\n",
      " [ 1.  0.  1. ... 55. 21.  7.]\n",
      " [ 1.  0.  1. ... 21. 55.  7.]\n",
      " [ 1.  3.  1. ...  7.  7. 55.]]\n",
      "\n",
      " Starting calculate accuracy/rmse...\n",
      "calculate performance:  94%|█████████▎| 936/1000 [00:01<00:00, 757.54it/s]\n",
      " Mean performance on train set: 28.360361\n",
      "With standard deviation: 1.357183\n",
      "\n",
      " Mean performance on test set: 35.191954\n",
      "With standard deviation: 4.495767\n",
      "calculate performance: 100%|██████████| 1000/1000 [00:01<00:00, 771.22it/s]\n",
      "\n",
      "\n",
      "  rmse_test    std_test    rmse_train    std_train    k_time\n",
      "-----------  ----------  ------------  -----------  --------\n",
      "     35.192     4.49577       28.3604      1.35718   13.3865\n"
     ]
    }
   ],
   "source": [
    "%load_ext line_profiler\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, \"../\")\n",
    "from pygraph.utils.utils import kernel_train_test\n",
    "from pygraph.kernels.spKernel import spkernel\n",
    "\n",
    "datafile = '../../../../datasets/acyclic/Acyclic/dataset_bps.ds'\n",
    "kernel_file_path = 'kernelmatrices_path_acyclic/'\n",
    "\n",
    "kernel_para = dict(edge_weight = 'atom')\n",
    "\n",
    "kernel_train_test(datafile, kernel_file_path, spkernel, kernel_para, normalize = False)\n",
    "\n",
    "# %lprun -f spkernel \\\n",
    "#     kernel_train_test(datafile, kernel_file_path, spkernel, kernel_para, normalize = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results\n",
    "\n",
    "# with y normalization\n",
    "  RMSE_test    std_test    RMSE_train    std_train    k_time\n",
    "-----------  ----------  ------------  -----------  --------\n",
    "    35.6337     5.23183       32.3805      3.92531   14.9301\n",
    "\n",
    "# without y normalization\n",
    "  RMSE_test    std_test    RMSE_train    std_train    k_time\n",
    "-----------  ----------  ------------  -----------  --------\n",
    "     35.192     4.49577       28.3604      1.35718   14.5768"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "- This script take as input a kernel matrix\n",
      "and returns the classification or regression performance\n",
      "- The kernel matrix can be calculated using any of the graph kernels approaches\n",
      "- The criteria used for prediction are SVM for classification and kernel Ridge regression for regression\n",
      "- For predition we divide the data in training, validation and test. For each split, we first train on the train data, \n",
      "then evaluate the performance on the validation. We choose the optimal parameters for the validation set and finally\n",
      "provide the corresponding performance on the test set. If more than one split is performed, the final results \n",
      "correspond to the average of the performances on the test sets. \n",
      "\n",
      "@references\n",
      "    https://github.com/eghisu/GraphKernels/blob/master/GraphKernelsCollection/python_scripts/compute_perf_gk.py\n",
      "\n",
      "\n",
      " Loading dataset from file...\n",
      "[ -23.7   14.    37.3  109.7   10.8   39.    42.    66.6  135.   148.5\n",
      "   40.    34.6   32.    63.    53.5   67.    64.4   84.7   95.5   92.\n",
      "   84.4  154.   156.   166.   183.    70.3   63.6   52.5   59.    59.5\n",
      "   55.2   88.    83.   104.5  102.    92.   107.4  123.2  112.5  118.5\n",
      "  101.5  173.7  165.5  181.    99.5   92.3   90.1   80.2   82.    91.2\n",
      "   91.5   81.2   93.    69.    86.3   82.   103.   103.5   96.   112.   104.\n",
      "  132.5  123.5  120.3  145.   144.2  142.8  132.   134.2  137.   139.\n",
      "  133.6  120.4  120.   137.   195.8  177.2  181.   185.9  175.7  186.   211.\n",
      "  125.   118.   117.1  107.   102.5  112.    97.4   91.5   87.6  106.5\n",
      "  101.    99.3   90.   137.   114.   126.   124.   140.5  157.5  146.   145.\n",
      "  141.   171.   166.   155.   145.   159.   138.   142.   159.   163.5\n",
      "  229.5  142.   125.   132.   130.5  125.   122.   121.   122.2  112.   106.\n",
      "  114.5  151.   128.5  109.5  126.   147.   158.   147.   165.   188.9\n",
      "  170.   178.   148.5  165.   177.   167.   195.   226.   215.   201.   205.\n",
      "  151.5  165.5  157.   139.   163.   153.5  139.   162.   173.   159.5\n",
      "  159.5  155.5  141.   126.   164.   163.   166.5  146.   165.   159.   195.\n",
      "  218.   250.   235.   186.5  156.5  162.   162.   170.2  173.2  186.8\n",
      "  173.   187.   174.   188.5  199.   228.   215.   216.   240. ]\n",
      "\n",
      " Loading the matrix from file...\n",
      "[[  3.   1.   3. ...,   1.   1.   1.]\n",
      " [  1.   6.   1. ...,   0.   0.   3.]\n",
      " [  3.   1.   3. ...,   1.   1.   1.]\n",
      " ..., \n",
      " [  1.   0.   1. ...,  55.  21.   7.]\n",
      " [  1.   0.   1. ...,  21.  55.   7.]\n",
      " [  1.   3.   1. ...,   7.   7.  55.]]\n",
      "\n",
      " --- This is a regression problem ---\n",
      "\n",
      " Starting split 10...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 12 with parameter alpha = 100.000000\n",
      "The best performance on the validation set is: 40.422382\n",
      "The corresponding performance on test set is: 47.424532\n",
      "\n",
      " Starting split 11...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 12 with parameter alpha = 100.000000\n",
      "The best performance on the validation set is: 33.084913\n",
      "The corresponding performance on test set is: 35.493699\n",
      "\n",
      " Starting split 12...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 12 with parameter alpha = 100.000000\n",
      "The best performance on the validation set is: 31.306710\n",
      "The corresponding performance on test set is: 33.173366\n",
      "\n",
      " Starting split 13...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 12 with parameter alpha = 100.000000\n",
      "The best performance on the validation set is: 43.500424\n",
      "The corresponding performance on test set is: 32.633129\n",
      "\n",
      " Starting split 14...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 10 with parameter alpha = 1.000000\n",
      "The best performance on the validation set is: 53.561752\n",
      "The corresponding performance on test set is: 42.883548\n",
      "\n",
      " Starting split 15...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 12 with parameter alpha = 100.000000\n",
      "The best performance on the validation set is: 40.444773\n",
      "The corresponding performance on test set is: 32.713040\n",
      "\n",
      " Starting split 16...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 11 with parameter alpha = 10.000000\n",
      "The best performance on the validation set is: 37.046818\n",
      "The corresponding performance on test set is: 37.337851\n",
      "\n",
      " Starting split 17...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 12 with parameter alpha = 100.000000\n",
      "The best performance on the validation set is: 39.907628\n",
      "The corresponding performance on test set is: 38.889064\n",
      "\n",
      " Starting split 18...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 12 with parameter alpha = 100.000000\n",
      "The best performance on the validation set is: 29.879950\n",
      "The corresponding performance on test set is: 27.652558\n",
      "\n",
      " Starting split 19...\n",
      "\n",
      " Normalizing output y...\n",
      "The best performance is for trial 11 with parameter alpha = 10.000000\n",
      "The best performance on the validation set is: 44.911892\n",
      "The corresponding performance on test set is: 35.804454\n",
      "\n",
      " Mean performance on val set: 39.406724\n",
      "With standard deviation: 6.720820\n",
      "\n",
      " Mean performance on test set: 36.400524\n",
      "With standard deviation: 5.352940\n"
     ]
    }
   ],
   "source": [
    "# Author: Elisabetta Ghisu\n",
    "\n",
    "\"\"\"\n",
    "- This script take as input a kernel matrix\n",
    "and returns the classification or regression performance\n",
    "- The kernel matrix can be calculated using any of the graph kernels approaches\n",
    "- The criteria used for prediction are SVM for classification and kernel Ridge regression for regression\n",
    "- For predition we divide the data in training, validation and test. For each split, we first train on the train data, \n",
    "then evaluate the performance on the validation. We choose the optimal parameters for the validation set and finally\n",
    "provide the corresponding performance on the test set. If more than one split is performed, the final results \n",
    "correspond to the average of the performances on the test sets. \n",
    "\n",
    "@references\n",
    "    https://github.com/eghisu/GraphKernels/blob/master/GraphKernelsCollection/python_scripts/compute_perf_gk.py\n",
    "\"\"\"\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "import sys\n",
    "import pathlib\n",
    "sys.path.insert(0, \"../\")\n",
    "from tabulate import tabulate\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.kernel_ridge import KernelRidge # 0.17\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error\n",
    "from sklearn import svm\n",
    "\n",
    "from pygraph.kernels.spkernel import spkernel\n",
    "from pygraph.utils.graphfiles import loadDataset\n",
    "\n",
    "print('\\n Loading dataset from file...')\n",
    "dataset, y = loadDataset(\"../../../../datasets/acyclic/Acyclic/dataset_bps.ds\")\n",
    "y = np.array(y)\n",
    "print(y)\n",
    "\n",
    "kernel_file_path = 'kernelmatrix.ds'\n",
    "path = pathlib.Path(kernel_file_path)\n",
    "if path.is_file():\n",
    "    print('\\n Loading the matrix from file...')\n",
    "    Kmatrix = np.loadtxt(kernel_file_path)\n",
    "    print(Kmatrix)\n",
    "else:\n",
    "    print('\\n Calculating kernel matrix, this could take a while...')\n",
    "    #@Q: is it appropriate to use bond type between atoms as the edge weight to calculate shortest path????????\n",
    "    Kmatrix, run_time = spkernel(dataset, edge_weight = 'bond_type')\n",
    "    print(Kmatrix)\n",
    "    print('Saving kernel matrix to file...')\n",
    "    np.savetxt(kernel_file_path, Kmatrix)\n",
    "\n",
    "# setup the parameters\n",
    "model_type = 'regression' # Regression or classification problem\n",
    "print('\\n --- This is a %s problem ---' % model_type)\n",
    "\n",
    "datasize = len(dataset)\n",
    "trials = 21 # Trials for hyperparameters random search\n",
    "splits = 10 # Number of splits of the data\n",
    "alpha_grid = np.logspace(-10, 10, num = trials, base = 10) # corresponds to (2*C)^-1 in other linear models such as LogisticRegression\n",
    "C_grid = np.logspace(-10, 10, num = trials, base = 10)\n",
    "random.seed(20) # Set the seed for uniform parameter distribution\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "-  Here starts the main program\n",
    "-  First we permute the data, then for each split we evaluate corresponding performances\n",
    "-  In the end, the performances are averaged over the test sets\n",
    "\"\"\"\n",
    "\n",
    "# Initialize the performance of the best parameter trial on validation with the corresponding performance on test\n",
    "val_split = []\n",
    "test_split = []\n",
    "\n",
    "# For each split of the data\n",
    "for j in range(10, 10 + splits):\n",
    "    print('\\n Starting split %d...' % j)\n",
    "\n",
    "    # Set the random set for data permutation\n",
    "    random_state = int(j)\n",
    "    np.random.seed(random_state)\n",
    "    idx_perm = np.random.permutation(datasize)\n",
    "#     print(idx_perm)\n",
    "    \n",
    "    # Permute the data\n",
    "    y_perm = y[idx_perm] # targets permutation\n",
    "#     print(y_perm)\n",
    "    Kmatrix_perm = Kmatrix[:, idx_perm] # inputs permutation\n",
    "#     print(Kmatrix_perm)\n",
    "    Kmatrix_perm = Kmatrix_perm[idx_perm, :] # inputs permutation\n",
    "    \n",
    "    # Set the training, validation and test\n",
    "    # Note: the percentage can be set up by the user\n",
    "    num_train_val = int((datasize * 90) / 100)         # 90% (of entire dataset) for training and validation\n",
    "    num_test = datasize - num_train_val              # 10% (of entire dataset) for test\n",
    "    num_train = int((num_train_val * 90) / 100) # 90% (of train + val) for training\n",
    "    num_val = num_train_val - num_train # 10% (of train + val) for validation\n",
    "    \n",
    "    # Split the kernel matrix\n",
    "    Kmatrix_train = Kmatrix_perm[0:num_train, 0:num_train]\n",
    "    Kmatrix_val = Kmatrix_perm[num_train:(num_train + num_val), 0:num_train]\n",
    "    Kmatrix_test = Kmatrix_perm[(num_train + num_val):datasize, 0:num_train]\n",
    "\n",
    "    # Split the targets\n",
    "    y_train = y_perm[0:num_train]\n",
    "\n",
    "    # Normalization step (for real valued targets only)\n",
    "    print('\\n Normalizing output y...')\n",
    "    if model_type == 'regression':\n",
    "        y_train_mean = np.mean(y_train)\n",
    "        y_train_std = np.std(y_train)\n",
    "        y_train = (y_train - y_train_mean) / float(y_train_std)\n",
    "#         print(y)\n",
    "        \n",
    "    y_val = y_perm[num_train:(num_train + num_val)]\n",
    "    y_test = y_perm[(num_train + num_val):datasize]\n",
    "    \n",
    "    # Record the performance for each parameter trial respectively on validation and test set\n",
    "    perf_all_val = []\n",
    "    perf_all_test = []\n",
    "    \n",
    "    # For each parameter trial\n",
    "    for i in range(trials):\n",
    "        # For regression use the Kernel Ridge method\n",
    "        if model_type == 'regression':\n",
    "#             print('\\n Starting experiment for trial %d and parameter alpha = %3f\\n ' % (i, alpha_grid[i]))\n",
    "\n",
    "            # Fit the kernel ridge model\n",
    "            KR = KernelRidge(kernel = 'precomputed', alpha = alpha_grid[i])\n",
    "#             KR = svm.SVR(kernel = 'precomputed', C = C_grid[i])\n",
    "            KR.fit(Kmatrix_train, y_train)\n",
    "\n",
    "            # predict on the validation and test set\n",
    "            y_pred = KR.predict(Kmatrix_val)\n",
    "            y_pred_test = KR.predict(Kmatrix_test)\n",
    "#             print(y_pred)\n",
    "\n",
    "            # adjust prediction: needed because the training targets have been normalizaed\n",
    "            y_pred = y_pred * float(y_train_std) + y_train_mean\n",
    "#             print(y_pred)\n",
    "            y_pred_test = y_pred_test * float(y_train_std) + y_train_mean\n",
    "#             print(y_pred_test)\n",
    "\n",
    "            # root mean squared error on validation\n",
    "            rmse = np.sqrt(mean_squared_error(y_val, y_pred))\n",
    "            perf_all_val.append(rmse)\n",
    "\n",
    "            # root mean squared error in test \n",
    "            rmse_test = np.sqrt(mean_squared_error(y_test, y_pred_test))\n",
    "            perf_all_test.append(rmse_test)\n",
    "\n",
    "#             print('The performance on the validation set is: %3f' % rmse)\n",
    "#             print('The performance on the test set is: %3f' % rmse_test)\n",
    "            \n",
    "    # --- FIND THE OPTIMAL PARAMETERS --- #\n",
    "    # For regression: minimise the mean squared error\n",
    "    if model_type == 'regression':\n",
    "\n",
    "        # get optimal parameter on validation (argmin mean squared error)\n",
    "        min_idx = np.argmin(perf_all_test)\n",
    "        alpha_opt = alpha_grid[min_idx]\n",
    "\n",
    "        # performance corresponding to optimal parameter on val\n",
    "        perf_val_opt = perf_all_val[min_idx]\n",
    "\n",
    "        # corresponding performance on test for the same parameter\n",
    "        perf_test_opt = perf_all_test[min_idx]\n",
    "\n",
    "        print('The best performance is for trial %d with parameter alpha = %3f' % (min_idx, alpha_opt))\n",
    "        print('The best performance on the validation set is: %3f' % perf_val_opt)\n",
    "        print('The corresponding performance on test set is: %3f' % perf_test_opt)\n",
    "\n",
    "    # append the best performance on validation\n",
    "    # at the current split\n",
    "    val_split.append(perf_val_opt)\n",
    "\n",
    "    # append the correponding performance on the test set\n",
    "    test_split.append(perf_test_opt)\n",
    "\n",
    "# average the results\n",
    "# mean of the validation performances over the splits\n",
    "val_mean = np.mean(np.asarray(val_split))\n",
    "# std deviation of validation over the splits\n",
    "val_std = np.std(np.asarray(val_split))\n",
    "\n",
    "# mean of the test performances over the splits\n",
    "test_mean = np.mean(np.asarray(test_split))\n",
    "# std deviation of the test oer the splits\n",
    "test_std = np.std(np.asarray(test_split))\n",
    "\n",
    "print('\\n Mean performance on val set: %3f' % val_mean)\n",
    "print('With standard deviation: %3f' % val_std)\n",
    "print('\\n Mean performance on test set: %3f' % test_mean)\n",
    "print('With standard deviation: %3f' % test_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
