{"cells":[{"metadata":{"_uuid":"db055fe1dc0a0d50dbdfa5b843cd0aaaaa403a36"},"cell_type":"markdown","source":"# Analysis of train and test predictions"},{"metadata":{"trusted":true,"_uuid":"ddd26c5b2156278e09a68037465699b35942f8bc"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff9c45533179787105c9a1b6d58fd9a98745f852"},"cell_type":"code","source":"import time\nimport warnings\nwarnings.simplefilter(action = 'ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70329d4af5eab2e36be8801f4d9fdefec273a45b"},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, precision_score, recall_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9301745805b9a975e82b4edda3190c587fe480df"},"cell_type":"code","source":"from matplotlib import pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"268163c768a2828ce18ab3166a404eb09e2e64b7"},"cell_type":"markdown","source":"## Functions for analysis"},{"metadata":{"trusted":true,"_uuid":"72d6e0cad7f0a8c0963fe77dff14e80d2c88df95"},"cell_type":"code","source":"# transform prediction from string to table\ndef transform_prediction_to_table(df):\n    prediction = pd.DataFrame(index = ['patientId', 'confidence', 'x', 'y', 'width', 'height'])\n\n    for _, row in df.iterrows():\n        # all except 'NAN'\n        if len(str(row['PredictionString'])) > 3:\n            row_array = row['PredictionString'].strip().split(' ')\n            for i in range(int(len(row_array) / 5)):\n                prediction[prediction.shape[1]] = [row['patientId'], row_array[i * 5]] + \\\n                                                  [b for b in row_array[i * 5 + 1 : i * 5 + 5]]\n        else:\n            prediction[prediction.shape[1]] = [row['patientId'], -1, -1, -1, -1, -1]\n\n    prediction = prediction.T\n    prediction['confidence'] = prediction['confidence'].astype(float)\n    prediction['x'] = prediction['x'].astype(int)\n    prediction['y'] = prediction['y'].astype(int)\n    prediction['width'] = prediction['width'].astype(int)\n    prediction['height'] = prediction['height'].astype(int)\n    \n    prediction.replace(-1, np.nan, inplace = True)\n    return prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"908f4c9fc8da8eab21c35b47ea78d955fac7413d"},"cell_type":"code","source":"# helper function to calculate IoU\n# based on kernel https://www.kaggle.com/chenyc15/mean-average-precision-metric\ndef iou(box1, box2):\n    x11, y11, w1, h1 = list(map(int, box1))\n    x21, y21, w2, h2 = list(map(int, box2))\n    assert w1 * h1 > 0\n    assert w2 * h2 > 0\n    x12, y12 = x11 + w1, y11 + h1\n    x22, y22 = x21 + w2, y21 + h2\n\n    area1, area2 = w1 * h1, w2 * h2\n    xi1, yi1, xi2, yi2 = max([x11, x21]), max([y11, y21]), min([x12, x22]), min([y12, y22])\n    \n    if xi2 <= xi1 or yi2 <= yi1:\n        return 0\n    else:\n        intersect = (xi2 - xi1) * (yi2 - yi1)\n        union = area1 + area2 - intersect\n        return intersect / union","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31f7c3f75bbfb958db4957963f8319bd9dffd190"},"cell_type":"code","source":"# calculate a single IoU metric, based on masks, for two table's rows with boxes, associated with one patientId\ndef mask_iou(boxes_true, boxes_pred):\n    mask_true = np.zeros((1024, 1024))\n    mask_pred = np.zeros((1024, 1024))\n    \n    for _, box in boxes_true.iterrows():\n        x1 = int(box['x'])\n        y1 = int(box['y'])\n        x2 = x1 + int(box['width'])\n        y2 = y1 + int(box['height'])\n        mask_true[y1 : y2, x1 : x2] = 1\n\n    for _, box in boxes_pred.iterrows():\n        x1 = int(box['x'])\n        y1 = int(box['y'])\n        x2 = x1 + int(box['width'])\n        y2 = y1 + int(box['height'])\n        mask_pred[y1 : y2, x1 : x2] = 1\n\n    mask_i = mask_true * mask_pred\n    mask_u = mask_true + mask_pred - mask_i\n    \n    return float(sum(sum(mask_i))) / sum(sum(mask_u))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c29beabf040550e85c8b9535e8222efc8856781e"},"cell_type":"code","source":"# calculate Mean Average Precision IoU metric for two table's rows with boxes, associated with one patientId\n# based on kernel https://www.kaggle.com/chenyc15/mean-average-precision-metric\ndef map_iou(boxes_true, boxes_pred, thresholds = [.4, .45, .5, .55, .6, .65, .7, .75]):\n    \n    # According to the introduction, images with no ground truth bboxes will not be \n    # included in the map score unless there is a false positive detection (?)\n        \n    # return None if both are empty, don't count the image in final evaluation (?)\n    if (boxes_true.shape[0] == 0) and (boxes_pred.shape[0] == 0):\n        return None\n    \n    # [x, y, w, h] for boxes_true\n    # [confidence, x, y, w, h] for boxes_pred\n    assert (boxes_true.shape[1] == 4 and boxes_pred.shape[1] == 5), 'Boxes shape error'\n    \n    # sort boxes_pred by scores in decreasing order\n    boxes_pred = boxes_pred.sort_values('confidence', ascending = False)\n    \n    map_total = 0\n    \n    # loop over thresholds\n    for t in thresholds:\n        matched_bt = set()\n        tp, fn = 0, 0\n        for i, bt in boxes_true.iterrows():\n            matched = False\n            for j, bp in boxes_pred.iterrows():\n                miou = iou(bt, bp[1:])\n                if miou >= t and not matched and j not in matched_bt:\n                    matched = True\n                    tp += 1 # bt is matched for the first time, count as TP\n                    matched_bt.add(j)\n            if not matched:\n                fn += 1 # bt has no match, count as FN\n                \n        fp = boxes_pred.shape[0] - len(matched_bt) # FP is the bp that not matched to any bt\n        m = tp / (tp + fn + fp)\n        map_total += m\n    \n    return map_total / len(thresholds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be74fe8d66c409b0ceb40b926f32142952668c92"},"cell_type":"code","source":"# calculate different metrics and aggregated values for one prediction\ndef get_table_for_one_prediction(df_pred, labels = None):\n    # get values for one patient's boxes\n    def get_evals(boxes):\n        if boxes.shape[0] > 0:\n            area_sum = 0\n            area_min = 1e7\n            area_max = 0\n            area_cnt = 0\n            for j, row in boxes.iterrows():\n                area = row['width'] * row['height']\n                area_sum += area\n                area_cnt += 1\n                if area < area_min:\n                    area_min = area\n                if area > area_max:\n                    area_max = area\n            return [int(area_cnt > 0), area_cnt, area_sum, area_min, area_max, float(area_sum) / area_cnt]\n        else:\n            return [0] * 6\n    \n    if labels is None:\n        # create table for test prediction\n        res = pd.DataFrame(index = ['pred_target', 'pred_cnt', 'pred_area_sum', 'pred_area_min', \n                                    'pred_area_max', 'pred_area_mean'])\n    \n        for patientId in df_pred['patientId'].unique():\n            pred_boxes = df_pred[df_pred['patientId'] == patientId][['confidence', 'x', 'y', 'width', 'height']].dropna()\n            res[patientId] = get_evals(pred_boxes)\n    else:\n        # create table for train prediction\n        res = pd.DataFrame(index = ['mask_iou', 'map_iou', \n                'true_target', 'true_cnt', 'true_area_sum', 'true_area_min', 'true_area_max', 'true_area_mean',\n                'pred_target', 'pred_cnt', 'pred_area_sum', 'pred_area_min', 'pred_area_max', 'pred_area_mean'])\n    \n        for patientId in labels['patientId'].unique():\n            pred_boxes = df_pred[df_pred['patientId'] == patientId][['confidence', 'x', 'y', 'width', 'height']].dropna()\n            true_boxes = labels[labels['patientId'] == patientId][['x', 'y', 'width', 'height']].dropna()\n            res[patientId] = [mask_iou(true_boxes, pred_boxes), map_iou(true_boxes, pred_boxes)] + \\\n                             get_evals(true_boxes) + get_evals(pred_boxes)\n    \n    return res.T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e70524a1f3bc8c81902361e3826a742dd399a5f1"},"cell_type":"code","source":"# calculate different metrics and aggregated values for all predictions\ndef get_table_for_all_predictions(prediction_files, labels = None):\n    if labels is None:\n        res = pd.DataFrame(index = ['LB_score_for_test', \n                              'max_cnt', 'mean_area_sum', 'mean_area_min', 'mean_area_max', 'mean_area_mean'])\n    else:\n        res = pd.DataFrame(index = ['LB_score_for_test', 'mean_map_iou', 'mean_mask_iou', \n                              'accuracy', 'precision_0', 'precision_1', 'recall_0', 'recall_1',\n                              'max_cnt', 'mean_area_sum', 'mean_area_min', 'mean_area_max', 'mean_area_mean'])\n\n    for key in prediction_files.keys():\n        print(key, time.ctime())\n        prediction_string = pd.read_csv(PREDICTIONS_FOLDER + key)\n        prediction = transform_prediction_to_table(prediction_string)\n        eval_table = get_table_for_one_prediction(prediction, labels)\n        \n        if labels is None:\n            # create table for test prediction\n            res[key] = [prediction_files[key], \n                 eval_table['pred_cnt'].max(), eval_table['pred_area_sum'].mean(), \n                 eval_table['pred_area_min'].mean(), eval_table['pred_area_max'].mean(), \n                 eval_table['pred_area_mean'].mean()]\n        else:\n            # create table for train prediction\n            res[key] = [prediction_files[key], \n                 eval_table['map_iou'].mean(), eval_table['mask_iou'].mean(), \n                 accuracy_score(eval_table['true_target'], eval_table['pred_target'])] + \\\n                 list(precision_score(eval_table['true_target'], eval_table['pred_target'], average = None)) + \\\n                 list(recall_score(eval_table['true_target'], eval_table['pred_target'], average = None)) + \\\n                [eval_table['pred_cnt'].max(), eval_table['pred_area_sum'].mean(), \n                 eval_table['pred_area_min'].mean(), eval_table['pred_area_max'].mean(), \n                 eval_table['pred_area_mean'].mean()]\n            \n    return res.T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e7ce81ca266b8fc3b68925af9b0a439b0f489a6"},"cell_type":"code","source":"PREDICTIONS_FOLDER = '../input/rsna-predictions/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ebee1ebaf6b7fdd679b5736a8f8324427006fcf"},"cell_type":"markdown","source":"## Analysis predictions of train data"},{"metadata":{"trusted":true,"_uuid":"6f06b6d43a7575605e6231c578831b3510da912e"},"cell_type":"code","source":"labels = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_1_train_labels.csv')\nlabels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c187cfe774bd3a3c7f14fc6329d4a92f19924c15"},"cell_type":"code","source":"labels.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a2425b43308289dae72434f487749c2ca8fd4c8"},"cell_type":"code","source":"# path_to_file: LB_score for test data, associated with this train data\ntrain_prediction_files = {\n    '1_prediction_train.csv': .093,\n    '2_prediction_train.csv': .106,\n    '3_prediction_train.csv': .113,\n    '4_prediction_train.csv': .119,\n    '5_prediction_train.csv': .124,\n    '6_prediction_train.csv': .129\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34c7896cf05b57d8060c209f0d2b4662f8614a76"},"cell_type":"markdown","source":"### Calculate different evaluations for one prediction of train data"},{"metadata":{"trusted":true,"_uuid":"7cc3e527cf5377fbaf2a91ff4a4c7ea9c7f37c09"},"cell_type":"code","source":"train_current_prediction = '6_prediction_train.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45c49016f70d1f0fa1ea1c2b149a590069bdc146"},"cell_type":"code","source":"prediction_string = pd.read_csv(PREDICTIONS_FOLDER + train_current_prediction)\nprediction_string.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34a5c9a7a890b9598248c522ced9426f2c3c88de"},"cell_type":"code","source":"prediction_string.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8a4d9878e2cfce616dfbe91ddfdf113861575d0"},"cell_type":"code","source":"prediction = transform_prediction_to_table(prediction_string)\nprediction.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3479f66704a6bfe9e2e354fd94dbc4014b1ecc4"},"cell_type":"code","source":"prediction.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04e8a620f94cef76c9da347ce481a1dbd54fcb43"},"cell_type":"code","source":"prediction['confidence'].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebecdd4412f8809666099c1e151218ede9f6d39b"},"cell_type":"code","source":"train_current_table = get_table_for_one_prediction(prediction, labels)\ntrain_current_table.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f79a00558a33111152f2b986d0128dc2295f0bc7"},"cell_type":"code","source":"train_current_table.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a9062ec90915cefcb0e8aef9883f22f26be7841"},"cell_type":"code","source":"plt.figure(figsize = (25, 5))\nplt.subplot(121)\nsns.boxplot(x = 'mask_iou', data = train_current_table);\nplt.subplot(122)\nsns.boxplot(x = 'map_iou', data = train_current_table);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62e1c45c0b16c8cb091d35272b75f64aa5c6329a"},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, sharey = True, figsize = (25, 5))\nsns.boxplot(x = 'true_target', y = 'pred_cnt', data = train_current_table, ax = axes[0]);\nsns.boxplot(y = 'true_cnt', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[1]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e7f03c961e865615b3f1b913163e0127a9335d7"},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, sharey = True, figsize = (25, 5))\nsns.boxplot(x = 'true_target', y = 'pred_area_sum', data = train_current_table, ax = axes[0]);\nsns.boxplot(y = 'true_area_sum', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[1]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"142a26930144530829d3b61852b4f8f2ff471cf6"},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, sharey = True, figsize = (25, 5))\nsns.boxplot(x = 'true_target', y = 'pred_area_min', data = train_current_table, ax = axes[0]);\nsns.boxplot(y = 'true_area_min', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[1]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"058ee3eac98df714f5f1d463e4f67ac211d00c5f"},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, sharey = True, figsize = (25, 5))\nsns.boxplot(x = 'true_target', y = 'pred_area_max', data = train_current_table, ax = axes[0]);\nsns.boxplot(y = 'true_area_max', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[1]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c586edb762bfacf6cd3adaa4bd5fb18ed1446655"},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, sharey = True, figsize = (25, 5))\nsns.boxplot(x = 'true_target', y = 'pred_area_mean', data = train_current_table, ax = axes[0]);\nsns.boxplot(y = 'true_area_mean', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[1]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"244e0814f6b7e4e18742ccedacf30334743726f3"},"cell_type":"markdown","source":"### Compare different predictions of train data"},{"metadata":{"trusted":true,"_uuid":"642b42f2287137cca72faad6626e31b646886fe1"},"cell_type":"code","source":"all_train_predictions = get_table_for_all_predictions(train_prediction_files, labels)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"c00e9e4863cd779d3d82d7ff74bd2c308b49c98b"},"cell_type":"code","source":"all_train_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1f5562aab167788639749fe713895bc55e5786b"},"cell_type":"code","source":"all_train_predictions.to_csv('all_train_predictions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c25fe3fe0289a49460a9d972cc0ebb9a6487771"},"cell_type":"code","source":"all_train_predictions[['LB_score_for_test', 'mean_map_iou', 'mean_mask_iou']].plot(figsize = (10, 5));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"442dd64912119446a25d2d3e36dda6bbb1d28d91"},"cell_type":"code","source":"all_train_predictions.corr()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6dfc3fc5d34a38ca9f0e581eb3aee493b2cd6d72"},"cell_type":"markdown","source":"## Analysis predictions of test data"},{"metadata":{"trusted":true,"_uuid":"0bf789f304b74b7d7180f9c6c66cbceb53b485db"},"cell_type":"code","source":"test_prediction_files = {\n    '1_prediction_test.csv': .093,\n    '2_prediction_test.csv': .106,\n    '3_prediction_test.csv': .113,\n    '4_prediction_test.csv': .119,\n    '5_prediction_test.csv': .124,\n    '6_prediction_test.csv': .129\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"698b94b48f3e4c1ae87e45a6549fc3af7b7ac43a"},"cell_type":"markdown","source":"### Calculate different evaluations for one prediction of test data"},{"metadata":{"trusted":true,"_uuid":"0a972caf5156377466ba90cc736ea40c26495b8d"},"cell_type":"code","source":"test_current_prediction = '6_prediction_test.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e265741c29e0e757c759ad19e796e330a29074c"},"cell_type":"code","source":"prediction_string = pd.read_csv(PREDICTIONS_FOLDER + test_current_prediction)\nprediction_string.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49a68fdb6ff50c990afa0e2dd516d099de7c3e82"},"cell_type":"code","source":"prediction_string.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13952e0790cd34ab3ee9c2f10491f7998adcd0d9"},"cell_type":"code","source":"prediction = transform_prediction_to_table(prediction_string)\nprediction.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85113850d81ccabc8ca42fdf1d50ee845681353e"},"cell_type":"code","source":"prediction.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cfd97873ef22f1592b0417a33f3b824425dbe84"},"cell_type":"code","source":"prediction['confidence'].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cd4e0a282f452170768a1fcd21ea9fd589462a3"},"cell_type":"code","source":"test_current_table = get_table_for_one_prediction(prediction)\ntest_current_table.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd006b2eaff2ed58dd84d78ad2d8df8337ab6712"},"cell_type":"code","source":"_, axes = plt.subplots(1, 3, sharey = True, figsize = (25, 5))\nsns.boxplot(y = 'pred_cnt', data = test_current_table, ax = axes[0]);\nsns.boxplot(y = 'pred_cnt', data = train_current_table, ax = axes[1]);\nsns.boxplot(y = 'true_cnt', data = train_current_table, ax = axes[2]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b96e29114e37240b397cffd7b37bfe24f930744a"},"cell_type":"code","source":"_, axes = plt.subplots(1, 3, sharey = True, figsize = (25, 5))\nsns.boxplot(y = 'pred_area_sum', data = test_current_table[test_current_table['pred_target'] == 1], ax = axes[0]);\nsns.boxplot(y = 'pred_area_sum', data = train_current_table[train_current_table['pred_target'] == 1], ax = axes[1]);\nsns.boxplot(y = 'true_area_sum', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[2]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8eb38e8c60f082e486ea42a948089503a844ec9"},"cell_type":"code","source":"_, axes = plt.subplots(1, 3, sharey = True, figsize = (25, 5))\nsns.boxplot(y = 'pred_area_min', data = test_current_table[test_current_table['pred_target'] == 1], ax = axes[0]);\nsns.boxplot(y = 'pred_area_min', data = train_current_table[train_current_table['pred_target'] == 1], ax = axes[1]);\nsns.boxplot(y = 'true_area_min', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[2]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15407d32fdf2faccaa574f1d4badd3903003f8fb"},"cell_type":"code","source":"_, axes = plt.subplots(1, 3, sharey = True, figsize = (25, 5))\nsns.boxplot(y = 'pred_area_max', data = test_current_table[test_current_table['pred_target'] == 1], ax = axes[0]);\nsns.boxplot(y = 'pred_area_max', data = train_current_table[train_current_table['pred_target'] == 1], ax = axes[1]);\nsns.boxplot(y = 'true_area_max', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[2]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f203195714f634a10fc7ea92db3d6390831f2560"},"cell_type":"code","source":"_, axes = plt.subplots(1, 3, sharey = True, figsize = (25, 5))\nsns.boxplot(y = 'pred_area_mean', data = test_current_table[test_current_table['pred_target'] == 1], ax = axes[0]);\nsns.boxplot(y = 'pred_area_mean', data = train_current_table[train_current_table['pred_target'] == 1], ax = axes[1]);\nsns.boxplot(y = 'true_area_mean', data = train_current_table[train_current_table['true_target'] == 1], ax = axes[2]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f2cb8a61479540101cbdafbe13c624194ead4255"},"cell_type":"markdown","source":"### Compare different predictions of test data"},{"metadata":{"trusted":true,"_uuid":"3af2dac49bfe30543948f4d1f2e0b28aec3b91eb"},"cell_type":"code","source":"all_test_predictions = get_table_for_all_predictions(test_prediction_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71da721154df44696bdc1f0b8b6a3d3513bf4a29"},"cell_type":"code","source":"all_test_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac6087c80799a6bca50c7abe88ae3b5306b37807"},"cell_type":"code","source":"all_test_predictions.to_csv('all_test_predictions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcd7fb60023cbd732f1c732193657aca38ff33de"},"cell_type":"code","source":"all_test_predictions.corr()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}