tods/datasets/data-supply/documentation/code/consolidated-new-metrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hamming loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from io import StringIO\n",
    "from sklearn.metrics import hamming_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hammingLoss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Computes the hamming loss. Used for multiclass and multilabel classification.\n",
    "    \"\"\"\n",
    "    from sklearn import preprocessing\n",
    "    from sklearn.metrics import hamming_loss\n",
    "    import numpy as np\n",
    "    from itertools import chain\n",
    "\n",
    "    def to_2d_array(df):\n",
    "        MULTI_CLASS_CONDITION=True\n",
    "        _dict = {}\n",
    "        for [index,value] in df.as_matrix():\n",
    "            if index in _dict.keys():\n",
    "                _dict[index].append(value)\n",
    "                MULTI_CLASS_CONDITION = False\n",
    "            else:\n",
    "                _dict[index]=[value]\n",
    "        return list(_dict.keys()), list(_dict.values()), MULTI_CLASS_CONDITION\n",
    "    \n",
    "    (y_true_keys, y_true_mat, b_multiclass) = to_2d_array(y_true)\n",
    "    (y_pred_keys, y_pred_mat, b_multiclass) = to_2d_array(y_pred)\n",
    "    \n",
    "    assert y_true_keys==y_pred_keys\n",
    "    \n",
    "    if b_multiclass:\n",
    "#         print('this is a multiclass case')\n",
    "        y_true_label_encoded = np.array(y_true_mat).ravel()\n",
    "        y_pred_label_encoded = np.array(y_pred_mat).ravel()\n",
    "    else: # MULTI_LABEL_CONDITION\n",
    "#         print('this is a multilabel case')\n",
    "        y_true_classes=(set(list(chain.from_iterable(y_true_mat))))\n",
    "        y_pred_classes=(set(list(chain.from_iterable(y_pred_mat))))\n",
    "        all_classes = list(y_true_classes.union(y_pred_classes))\n",
    "        lb = preprocessing.MultiLabelBinarizer(classes=all_classes)\n",
    "        y_true_label_encoded = lb.fit_transform(y_true_mat)\n",
    "        y_pred_label_encoded = lb.transform(y_pred_mat)\n",
    "    return hamming_loss(y_true_label_encoded, y_pred_label_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.26666666666666666"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Testcase 1: MultiLabel, typical\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,class_label\n",
    "3,happy-pleased\n",
    "3,relaxing-calm\n",
    "7,amazed-suprised\n",
    "7,happy-pleased\n",
    "13,quiet-still\n",
    "13,sad-lonely\n",
    "\"\"\"))\n",
    "\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,class_label\n",
    "3,happy-pleased\n",
    "3,sad-lonely\n",
    "7,amazed-suprised\n",
    "7,happy-pleased\n",
    "13,quiet-still\n",
    "13,happy-pleased\n",
    "\"\"\"))\n",
    "\n",
    "hammingLoss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Testcase 2: MultiLabel, Zero loss\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,class_label\n",
    "3,happy-pleased\n",
    "3,relaxing-calm\n",
    "7,amazed-suprised\n",
    "7,happy-pleased\n",
    "13,quiet-still\n",
    "13,sad-lonely\n",
    "\"\"\"))\n",
    "\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,class_label\n",
    "3,happy-pleased\n",
    "3,relaxing-calm\n",
    "7,amazed-suprised\n",
    "7,happy-pleased\n",
    "13,quiet-still\n",
    "13,sad-lonely\n",
    "\"\"\"))\n",
    "\n",
    "hammingLoss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Testcase 3: MultiLabel, Complete loss\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,class_label\n",
    "3,happy-pleased\n",
    "3,relaxing-calm\n",
    "7,amazed-suprised\n",
    "7,happy-pleased\n",
    "13,quiet-still\n",
    "13,sad-lonely\n",
    "\"\"\"))\n",
    "\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,class_label\n",
    "3,ecstatic\n",
    "3,sad-lonely\n",
    "3,quiet-still\n",
    "3,amazed-suprised\n",
    "7,ecstatic\n",
    "7,sad-lonely\n",
    "7,relaxing-calm\n",
    "7,quiet-still\n",
    "13,ecstatic\n",
    "13,happy-pleased\n",
    "13,relaxing-calm\n",
    "13,amazed-suprised\n",
    "\"\"\"))\n",
    "\n",
    "hammingLoss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.2"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Testcase 4: Multiclass, Typical\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,species\n",
    "1,versicolor\n",
    "2,versicolor\n",
    "16,virginica\n",
    "17,setosa\n",
    "22,versicolor\n",
    "26,versicolor\n",
    "30,versicolor\n",
    "31,virginica\n",
    "33,versicolor\n",
    "37,virginica\n",
    "\"\"\"))\n",
    "\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,species\n",
    "1,setosa\n",
    "2,versicolor\n",
    "16,virginica\n",
    "17,setosa\n",
    "22,versicolor\n",
    "26,virginica\n",
    "30,versicolor\n",
    "31,virginica\n",
    "33,versicolor\n",
    "37,virginica\n",
    "\"\"\"))\n",
    "\n",
    "hammingLoss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Testcase 5: Multiclass, Zero loss\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,species\n",
    "1,versicolor\n",
    "2,versicolor\n",
    "16,virginica\n",
    "17,setosa\n",
    "22,versicolor\n",
    "26,versicolor\n",
    "30,versicolor\n",
    "31,virginica\n",
    "33,versicolor\n",
    "37,virginica\n",
    "\"\"\"))\n",
    "\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,species\n",
    "1,versicolor\n",
    "2,versicolor\n",
    "16,virginica\n",
    "17,setosa\n",
    "22,versicolor\n",
    "26,versicolor\n",
    "30,versicolor\n",
    "31,virginica\n",
    "33,versicolor\n",
    "37,virginica\n",
    "\"\"\"))\n",
    "\n",
    "hammingLoss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Testcase 6: Multiclass, Complete loss\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,species\n",
    "1,versicolor\n",
    "2,versicolor\n",
    "16,versicolor\n",
    "17,virginica\n",
    "22,versicolor\n",
    "26,versicolor\n",
    "30,versicolor\n",
    "31,virginica\n",
    "33,versicolor\n",
    "37,virginica\n",
    "\"\"\"))\n",
    "\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,species\n",
    "1,setosa\n",
    "2,setosa\n",
    "16,setosa\n",
    "17,setosa\n",
    "22,setosa\n",
    "26,setosa\n",
    "30,setosa\n",
    "31,setosa\n",
    "33,setosa\n",
    "37,setosa\n",
    "\"\"\"))\n",
    "\n",
    "hammingLoss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rootMeanSquaredError(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Computes the root mean squared error, for both univariate and multivariate case\n",
    "    \"\"\"\n",
    "    import numpy as np\n",
    "    from sklearn.metrics import mean_squared_error\n",
    "    from math import sqrt\n",
    "    \n",
    "    rmse = None\n",
    "    \n",
    "    # perform some checks\n",
    "    assert 'd3mIndex' in y_true.columns\n",
    "    assert 'd3mIndex' in y_pred.columns\n",
    "    assert y_true.shape == y_pred.shape\n",
    "    \n",
    "    # preprocessing\n",
    "    y_true.set_index('d3mIndex', inplace=True)\n",
    "    y_pred.set_index('d3mIndex', inplace=True)\n",
    "    \n",
    "    # determine the dimension\n",
    "    y_true_dim=y_true.shape[1]\n",
    "         \n",
    "    # univariate case\n",
    "    if y_true_dim == 1: \n",
    "        y_true_array = y_true.as_matrix().ravel()\n",
    "        y_pred_array = y_pred.as_matrix().ravel()\n",
    "        mse = mean_squared_error(y_true, y_pred)\n",
    "        rmse = sqrt(mse)\n",
    "    \n",
    "    # multivariate case\n",
    "    elif y_true_dim > 1:\n",
    "        y_true_array = y_true.as_matrix()\n",
    "        y_pred_array = y_pred.as_matrix()\n",
    "        mse = mean_squared_error(y_true_array, y_pred_array, multioutput='uniform_average')\n",
    "        rmse = sqrt(mse)\n",
    "    \n",
    "    return rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:25: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:26: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8381527307120105"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test case 1\n",
    "# y_true_uni=[3, -1., 2, 7]\n",
    "# y_pred_uni=[2.1, 0.0, 2, 8]\n",
    "# expected rmse = 0.8381527307120105\n",
    "\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,value\n",
    "1,3\n",
    "2,-1.0\n",
    "16,2\n",
    "17,7\n",
    "\"\"\"))\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,value\n",
    "1,2.1\n",
    "2,0.0\n",
    "16,2\n",
    "17,8\n",
    "\"\"\"))\n",
    "rootMeanSquaredError(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:32: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "/home/svattam/miniconda3/envs/automl/lib/python3.6/site-packages/ipykernel_launcher.py:33: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8416254115301732"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test case 2\n",
    "# y_true_multi=[[0.5, 1],[-1, 1],[7, -6]]\n",
    "# y_pred_multi=[[0, 2],[-1, 2],[8, -5]]\n",
    "# expected rmse = 0.8416254115301732\n",
    "\n",
    "y_true = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,value1, value2\n",
    "1,0.5,1\n",
    "2,-1,1\n",
    "16,7,-6\n",
    "\"\"\"))\n",
    "y_pred = pd.read_csv(StringIO(\"\"\"\n",
    "d3mIndex,value1,value2\n",
    "1,0,2\n",
    "2,-1,2\n",
    "16,8,-5\n",
    "\"\"\"))\n",
    "rootMeanSquaredError(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Object detection average precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TEST CASE 1 --- AP:  0.6666666666666666\n",
      "TEST CASE 2 --- AP:  0.125\n",
      "TEST CASE 3 --- AP:  0.4444444444444444\n",
      "TEST CASE 4 --- AP:  0.4444444444444444\n"
     ]
    }
   ],
   "source": [
    "def group_gt_boxes_by_image_name(gt_boxes):\n",
    "    gt_dict: typing.Dict = {}\n",
    "\n",
    "    for box in gt_boxes:\n",
    "        image_name = box[0]\n",
    "        bounding_polygon = box[1:]\n",
    "        bbox = convert_bouding_polygon_to_box_coords(bounding_polygon)\n",
    "\n",
    "        if image_name not in gt_dict.keys():\n",
    "            gt_dict[image_name] = []\n",
    "\n",
    "        gt_dict[image_name].append({'bbox': bbox})\n",
    "\n",
    "    return gt_dict\n",
    "\n",
    "\n",
    "def convert_bouding_polygon_to_box_coords(bounding_polygon):\n",
    "    # box_coords = [x_min, y_min, x_max, y_max]\n",
    "    box_coords = [bounding_polygon[0], bounding_polygon[1],\n",
    "                  bounding_polygon[4], bounding_polygon[5]]\n",
    "    return box_coords\n",
    "\n",
    "\n",
    "def voc_ap(rec, prec):\n",
    "    import numpy\n",
    "\n",
    "    # First append sentinel values at the end.\n",
    "    mrec = numpy.concatenate(([0.], rec, [1.]))\n",
    "    mpre = numpy.concatenate(([0.], prec, [0.]))\n",
    "\n",
    "    # Compute the precision envelope.\n",
    "    for i in range(mpre.size - 1, 0, -1):\n",
    "        mpre[i - 1] = numpy.maximum(mpre[i - 1], mpre[i])\n",
    "\n",
    "    # To calculate area under PR curve, look for points\n",
    "    # where X axis (recall) changes value.\n",
    "    i = numpy.where(mrec[1:] != mrec[:-1])[0]\n",
    "\n",
    "    # And sum (\\Delta recall) * prec.\n",
    "    ap = numpy.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n",
    "\n",
    "    return float(ap)\n",
    "\n",
    "\n",
    "def object_detection_average_precision(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    This function takes a list of ground truth bounding polygons (rectangles in this case)\n",
    "    and a list of detected bounding polygons (also rectangles) for a given class and\n",
    "    computes the average precision of the detections with respect to the ground truth polygons.\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true: list\n",
    "     List of ground truth polygons. Each polygon is represented as a list of\n",
    "     vertices, starting in the upper-left corner going counter-clockwise.\n",
    "     Since in this case, the polygons are rectangles, they will have the\n",
    "     following format:\n",
    "        [image_name, x_min, y_min, x_min, y_max, x_max, y_max, x_max, y_min].\n",
    "    y_pred: list\n",
    "     List of bounding box polygons with their corresponding confidence scores. Each\n",
    "     polygon is represented as a list of vertices, starting in the upper-left corner\n",
    "     going counter-clockwise. Since in this case, the polygons are rectangles, they\n",
    "     will have the following format:\n",
    "        [image_name, x_min, y_min, x_min, y_max, x_max, y_max, x_max, y_min, confidence_score].\n",
    "    Returns:\n",
    "    --------\n",
    "    ap: float\n",
    "     Average precision between detected polygons (rectangles) and the ground truth polylgons (rectangles).\n",
    "     (it is also the area under the precision-recall curve).\n",
    "    Example 1:\n",
    "    >> predictions_list_1 = [['img_00001.png', 110, 110, 110, 210, 210, 210, 210, 110, 0.6],\n",
    "                             ['img_00002.png', 5, 10, 5, 20, 20, 20, 20, 10, 0.9],\n",
    "                             ['img_00002.png', 120, 130, 120, 200, 200, 200, 200, 130, 0.6]]\n",
    "    >> ground_truth_list_1 = [['img_00001.png', 100, 100, 100, 200, 200, 200, 200, 100],\n",
    "                              ['img_00002.png', 10, 10, 10, 20, 20, 20, 20, 10],\n",
    "                              ['img_00002.png', 70, 80, 70, 150, 140, 150, 140, 80]]\n",
    "    >> ap_1 = object_detection_average_precision(ground_truth_list_1, predictions_list_1)\n",
    "    >> print(ap_1)\n",
    "    0.667\n",
    "    Example 2:\n",
    "    >> predictions_list_2 = [['img_00285.png', 330, 463, 330, 505, 387, 505, 387, 463, 0.0739],\n",
    "                             ['img_00285.png', 420, 433, 420, 498, 451, 498, 451, 433, 0.0910],\n",
    "                             ['img_00285.png', 328, 465, 328, 540, 403, 540, 403, 465, 0.1008],\n",
    "                             ['img_00285.png', 480, 477, 480, 522, 508, 522, 508, 477, 0.1012],\n",
    "                             ['img_00285.png', 357, 460, 357, 537, 417, 537, 417, 460, 0.1058],\n",
    "                             ['img_00285.png', 356, 456, 356, 521, 391, 521, 391, 456, 0.0843],\n",
    "                             ['img_00225.png', 345, 460, 345, 547, 415, 547, 415, 460, 0.0539],\n",
    "                             ['img_00225.png', 381, 362, 381, 513, 455, 513, 455, 362, 0.0542],\n",
    "                             ['img_00225.png', 382, 366, 382, 422, 416, 422, 416, 366, 0.0559],\n",
    "                             ['img_00225.png', 730, 463, 730, 583, 763, 583, 763, 463, 0.0588]]\n",
    "    >> ground_truth_list_2 = [['img_00285.png', 480, 457, 480, 529, 515, 529, 515, 457],\n",
    "                              ['img_00285.png', 480, 457, 480, 529, 515, 529, 515, 457],\n",
    "                              ['img_00225.png', 522, 540, 522, 660, 576, 660, 576, 540],\n",
    "                              ['img_00225.png', 739, 460, 739, 545, 768, 545, 768, 460]]\n",
    "    >> ap_2 = object_detection_average_precision(ground_truth_list_2, predictions_list_2)\n",
    "    >> print(ap_2)\n",
    "    0.125\n",
    "    Example 3:\n",
    "    >> predictions_list_3 = [['img_00001.png', 110, 110, 110, 210, 210, 210, 210, 110, 0.6],\n",
    "                             ['img_00002.png', 120, 130, 120, 200, 200, 200, 200, 130, 0.6],\n",
    "                             ['img_00002.png', 5, 8, 5, 16, 15, 16, 15, 8, 0.9],\n",
    "                             ['img_00002.png', 11, 12, 11, 18, 21, 18, 21, 12, 0.9]]\n",
    "    >> ground_truth_list_3 = [['img_00001.png', 100, 100, 100, 200, 200, 200, 200, 100],\n",
    "                              ['img_00002.png', 10, 10, 10, 20, 20, 20, 20, 10],\n",
    "                              ['img_00002.png', 70, 80, 70, 150, 140, 150, 140, 80]]\n",
    "    >> ap_3 = object_detection_average_precision(ground_truth_list_3, predictions_list_3)\n",
    "    >> print(ap_3)\n",
    "    0.444\n",
    "    Example 4:\n",
    "    (Same as example 3 except the last two box predictions in img_00002.png are switched)\n",
    "    >> predictions_list_4 = [['img_00001.png', 110, 110, 110, 210, 210, 210, 210, 110, 0.6],\n",
    "                             ['img_00002.png', 120, 130, 120, 200, 200, 200, 200, 130, 0.6],\n",
    "                             ['img_00002.png', 11, 12, 11, 18, 21, 18, 21, 12, 0.9],\n",
    "                             ['img_00002.png', 5, 8, 5, 16, 15, 16, 15, 8, 0.9]]\n",
    "    >> ground_truth_list_4 = [['img_00001.png', 100, 100, 100, 200, 200, 200, 200, 100],\n",
    "                              ['img_00002.png', 10, 10, 10, 20, 20, 20, 20, 10],\n",
    "                              ['img_00002.png', 70, 80, 70, 150, 140, 150, 140, 80]]\n",
    "    >> ap_4 = object_detection_average_precision(ground_truth_list_4, predictions_list_4)\n",
    "    >> print(ap_4)\n",
    "    0.444\n",
    "    \"\"\"\n",
    "\n",
    "    \"\"\"\n",
    "    This function is different from others because ``y_true`` and ``y_pred`` are not vectors but arrays.\n",
    "    \"\"\"\n",
    "    import numpy\n",
    "    ovthresh = 0.5\n",
    "\n",
    "    # y_true = typing.cast(Truth, unvectorize(y_true))\n",
    "    # y_pred = typing.cast(Predictions, unvectorize(y_pred))\n",
    "\n",
    "    # Load ground truth.\n",
    "    gt_dict = group_gt_boxes_by_image_name(y_true)\n",
    "\n",
    "    # Extract gt objects for this class.\n",
    "    recs = {}\n",
    "    npos = 0\n",
    "\n",
    "    imagenames = sorted(gt_dict.keys())\n",
    "    for imagename in imagenames:\n",
    "        Rlist = [obj for obj in gt_dict[imagename]]\n",
    "        bbox = numpy.array([x['bbox'] for x in Rlist])\n",
    "        det = [False] * len(Rlist)\n",
    "        npos = npos + len(Rlist)\n",
    "        recs[imagename] = {'bbox': bbox, 'det': det}\n",
    "\n",
    "    # Load detections.\n",
    "    det_length = len(y_pred[0])\n",
    "\n",
    "    # Check that all boxes are the same size.\n",
    "    for det in y_pred:\n",
    "        assert len(det) == det_length, 'Not all boxes have the same dimensions.'\n",
    "\n",
    "    image_ids = [x[0] for x in y_pred]\n",
    "    BP = numpy.array([[float(z) for z in x[1:-1]] for x in y_pred])\n",
    "    BB = numpy.array([convert_bouding_polygon_to_box_coords(x) for x in BP])\n",
    "\n",
    "    confidence = numpy.array([float(x[-1]) for x in y_pred])\n",
    "    boxes_w_confidences_list = numpy.hstack((BB, -1 * confidence[:, None]))\n",
    "    boxes_w_confidences = numpy.empty((boxes_w_confidences_list.shape[0],),\n",
    "                                     dtype=[('x_min', float), ('y_min', float),\n",
    "                                            ('x_max', float), ('y_max', float),\n",
    "                                            ('confidence', float)])\n",
    "    boxes_w_confidences[:] = [tuple(i) for i in boxes_w_confidences_list]\n",
    "\n",
    "    # Sort by confidence.\n",
    "    #sorted_ind = numpy.argsort(-confidence)\n",
    "    sorted_ind = numpy.argsort(\n",
    "        boxes_w_confidences, kind='mergesort',\n",
    "        order=('confidence', 'x_min', 'y_min'))\n",
    "    BB = BB[sorted_ind, :]\n",
    "    image_ids = [image_ids[x] for x in sorted_ind]\n",
    "\n",
    "    # Go down y_pred and mark TPs and FPs.\n",
    "    nd = len(image_ids)\n",
    "    tp = numpy.zeros(nd)\n",
    "    fp = numpy.zeros(nd)\n",
    "    for d in range(nd):\n",
    "        R = recs[image_ids[d]]\n",
    "        bb = BB[d, :].astype(float)\n",
    "        ovmax = -numpy.inf\n",
    "        BBGT = R['bbox'].astype(float)\n",
    "\n",
    "        if BBGT.size > 0:\n",
    "            # Compute overlaps.\n",
    "            # Intersection.\n",
    "            ixmin = numpy.maximum(BBGT[:, 0], bb[0])\n",
    "            iymin = numpy.maximum(BBGT[:, 1], bb[1])\n",
    "            ixmax = numpy.minimum(BBGT[:, 2], bb[2])\n",
    "            iymax = numpy.minimum(BBGT[:, 3], bb[3])\n",
    "            iw = numpy.maximum(ixmax - ixmin + 1., 0.)\n",
    "            ih = numpy.maximum(iymax - iymin + 1., 0.)\n",
    "            inters = iw * ih\n",
    "\n",
    "            # Union.\n",
    "            uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +\n",
    "                   (BBGT[:, 2] - BBGT[:, 0] + 1.) *\n",
    "                   (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)\n",
    "\n",
    "            overlaps = inters / uni\n",
    "            ovmax = numpy.max(overlaps)\n",
    "            jmax = numpy.argmax(overlaps)\n",
    "\n",
    "        if ovmax > ovthresh:\n",
    "            if not R['det'][jmax]:\n",
    "                tp[d] = 1.\n",
    "                R['det'][jmax] = 1\n",
    "            else:\n",
    "                fp[d] = 1.\n",
    "        else:\n",
    "            fp[d] = 1.\n",
    "\n",
    "    # Compute precision recall.\n",
    "    fp = numpy.cumsum(fp)\n",
    "    tp = numpy.cumsum(tp)\n",
    "    rec = tp / float(npos)\n",
    "    # Avoid divide by zero in case the first detection matches a difficult ground truth.\n",
    "    prec = tp / numpy.maximum(tp + fp, numpy.finfo(numpy.float64).eps)\n",
    "    ap = voc_ap(rec, prec)\n",
    "\n",
    "    return ap\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    predictions_list_1 = [\n",
    "        ['img_00001.png', 110, 110, 110, 210, 210, 210, 210, 110, 0.6],\n",
    "        ['img_00002.png', 5, 10, 5, 20, 20, 20, 20, 10, 0.9],\n",
    "        ['img_00002.png', 120, 130, 120, 200, 200, 200, 200, 130, 0.6]\n",
    "    ]\n",
    "    ground_truth_list_1 = [\n",
    "        ['img_00001.png', 100, 100, 100, 200, 200, 200, 200, 100],\n",
    "        ['img_00002.png', 10, 10, 10, 20, 20, 20, 20, 10],\n",
    "        ['img_00002.png', 70, 80, 70, 150, 140, 150, 140, 80]\n",
    "    ]\n",
    "    ap_1 = object_detection_average_precision(\n",
    "        ground_truth_list_1, predictions_list_1)\n",
    "    print('TEST CASE 1 --- AP: ', ap_1)\n",
    "\n",
    "    predictions_list_2 = [\n",
    "        ['img_00285.png', 330, 463, 330, 505, 387, 505, 387, 463, 0.0739],\n",
    "        ['img_00285.png', 420, 433, 420, 498, 451, 498, 451, 433, 0.0910],\n",
    "        ['img_00285.png', 328, 465, 328, 540, 403, 540, 403, 465, 0.1008],\n",
    "        ['img_00285.png', 480, 477, 480, 522, 508, 522, 508, 477, 0.1012],\n",
    "        ['img_00285.png', 357, 460, 357, 537, 417, 537, 417, 460, 0.1058],\n",
    "        ['img_00285.png', 356, 456, 356, 521, 391, 521, 391, 456, 0.0843],\n",
    "        ['img_00225.png', 345, 460, 345, 547, 415, 547, 415, 460, 0.0539],\n",
    "        ['img_00225.png', 381, 362, 381, 513, 455, 513, 455, 362, 0.0542],\n",
    "        ['img_00225.png', 382, 366, 382, 422, 416, 422, 416, 366, 0.0559],\n",
    "        ['img_00225.png', 730, 463, 730, 583, 763, 583, 763, 463, 0.0588],\n",
    "    ]\n",
    "    ground_truth_list_2 = [\n",
    "        ['img_00285.png', 480, 457, 480, 529, 515, 529, 515, 457],\n",
    "        ['img_00285.png', 480, 457, 480, 529, 515, 529, 515, 457],\n",
    "        ['img_00225.png', 522, 540, 522, 660, 576, 660, 576, 540],\n",
    "        ['img_00225.png', 739, 460, 739, 545, 768, 545, 768, 460],\n",
    "    ]\n",
    "    ap_2 = object_detection_average_precision(\n",
    "        ground_truth_list_2, predictions_list_2)\n",
    "    print('TEST CASE 2 --- AP: ', ap_2)\n",
    "\n",
    "    predictions_list_3 = [\n",
    "        ['img_00001.png', 110, 110, 110, 210, 210, 210, 210, 110, 0.6],\n",
    "        ['img_00002.png', 120, 130, 120, 200, 200, 200, 200, 130, 0.6],\n",
    "        ['img_00002.png', 5, 8, 5, 16, 15, 16, 15, 8, 0.9],\n",
    "        ['img_00002.png', 11, 12, 11, 18, 21, 18, 21, 12, 0.9]\n",
    "    ]\n",
    "    ground_truth_list_3 = [\n",
    "        ['img_00001.png', 100, 100, 100, 200, 200, 200, 200, 100],\n",
    "        ['img_00002.png', 10, 10, 10, 20, 20, 20, 20, 10],\n",
    "        ['img_00002.png', 70, 80, 70, 150, 140, 150, 140, 80]\n",
    "    ]\n",
    "    ap_3 = object_detection_average_precision(\n",
    "        ground_truth_list_3, predictions_list_3)\n",
    "    print('TEST CASE 3 --- AP: ', ap_3)\n",
    "\n",
    "    predictions_list_4 = [\n",
    "        ['img_00001.png', 110, 110, 110, 210, 210, 210, 210, 110, 0.6],\n",
    "        ['img_00002.png', 120, 130, 120, 200, 200, 200, 200, 130, 0.6],\n",
    "        ['img_00002.png', 11, 12, 11, 18, 21, 18, 21, 12, 0.9],\n",
    "        ['img_00002.png', 5, 8, 5, 16, 15, 16, 15, 8, 0.9]\n",
    "    ]\n",
    "    ground_truth_list_4 = [\n",
    "        ['img_00001.png', 100, 100, 100, 200, 200, 200, 200, 100],\n",
    "        ['img_00002.png', 10, 10, 10, 20, 20, 20, 20, 10],\n",
    "        ['img_00002.png', 70, 80, 70, 150, 140, 150, 140, 80]\n",
    "    ]\n",
    "    ap_4 = object_detection_average_precision(\n",
    "        ground_truth_list_4, predictions_list_4)\n",
    "    print('TEST CASE 4 --- AP: ', ap_4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}