{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Analysis of image denoising in the classification","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom skimage import io\nimport cv2\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Import one of the best denoising algorithm for the natural noise\nlink to discussion: https://www.kaggle.com/c/alaska2-image-steganalysis/discussion/161986","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# !git clone https://github.com/yzhouas/PD-Denoising-pytorch.git\n# !mv PD-Denoising-pytorch denoise\n!mkdir data\n!mkdir data/img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nfrom glob import glob\nFOLDERS = ['Cover', 'JMiPOD', 'JUNIWARD', 'UERD']\nSRC_FOLDER = '../input/alaska2-image-steganalysis'\nFILTERED_FOLDER = '../input/alaska2filtered-images/filtered'\nFILTERED_POSTFIX = '_pss2_k0.0.png'\nJPEG_POSTFIX = '.jpg'\ndataset = []\nfor path in glob(f'{FILTERED_FOLDER}/*{FILTERED_POSTFIX}'):\n    new_img = path.split('/')[-1].replace(FILTERED_POSTFIX, '').split('_')[-1]\n    if new_img not in dataset:\n        dataset.append(new_img)\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prefiltered images are in dataset. Please use version 4 to generate the images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#denoise/test.py - path to test.py \n#real - is real noise\n#color - our images is color\n#test_data img - NB data folder is hardcoded\n\n!python denoise/test.py \\\n --scale 1\\\n --ps 2 --ps_scale 2\\\n --real 1\\\n --k 0\\\n --mode MC\\\n --color 1\\\n --output_map 0\\\n --zeroout 0 --keep_ind 0\\\n --num_of_layers 20\\\n --delog denoise/logs/logs_color_MC_AWGN_RVIN\\\n --cond 1 --refine 0 --refine_opt 1\\\n --test_data img \\\n --out_dir filtered","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_img(path):\n    \"\"\" Reads image. Image is a float version of uint8 format 0..255\"\"\"\n    image = cv2.imread(path, cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    return image\n    \ndef calc_diff_features(src_path, filterd_path):\n    \"\"\" Calculate the difference between images. \"\"\"\n    img = read_img(src_path)\n    filtered_img = read_img(filterd_path)\n    diff_img = (filtered_img - img) / filtered_img\n    hist = np.histogram(diff_img, bins=24, range=(0.8, 1.3))\n    \n    return hist\n    \ndef plot_diff(path1, path2):\n    \"\"\" Plots the difference between images. \"\"\"\n    image = read_img(path1)\n    base_image = read_img(path2)\n    abs_diff = np.abs(image - base_image)\n    print(f'{path1.split(\"/\")[-1]} and {path2.split(\"/\")[-1]}')\n    print(f'MAE: {np.mean(abs_diff)}')\n    \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n    ax.set_axis_off()\n    ax.imshow(0.5 + (image - base_image) / np.max(abs_diff));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datatable = []\nlabels = []\nfor img_id in dataset:\n    for (label, folder) in enumerate(FOLDERS):\n        p = calc_diff_features(f'{SRC_FOLDER}/{folder}/{img_id}{JPEG_POSTFIX}', f'{FILTERED_FOLDER}/{folder}_{img_id}{FILTERED_POSTFIX}')\n        datatable.append(p[0])\n        labels.append(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datatable = np.array(datatable)\nlabels = np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datatable.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = datatable[0:700]\ntest_X = datatable[700:900]\ntrain_Y = labels[0:700]\ntest_Y = labels[700:900]\ntrain_X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\n\ntpr_thresholds = [0.0, 0.4, 1.0]\nweights = [2, 1]\nareas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])\nnormalization = np.dot(areas, weights)\nprint(normalization)\n    \ndef alaska_weighted_auc(y_true, y_valid):\n    \"\"\"\n    https://www.kaggle.com/anokas/weighted-auc-metric-updated\n    \"\"\"\n    \n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid)\n\n    competition_metric = 0\n    for idx, weight in enumerate(weights):\n        y_min = tpr_thresholds[idx]\n        y_max = tpr_thresholds[idx + 1]\n        mask = (y_min < tpr) & (tpr < y_max)\n        # pdb.set_trace()\n\n        x_padding = np.linspace(fpr[mask][-1], 1, 100)\n\n        x = np.concatenate([fpr[mask], x_padding])\n        y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])\n        y = y - y_min  # normalize such that curve starts at y=0\n        score = metrics.auc(x, y)\n        submetric = score * weight\n        best_subscore = (y_max - y_min) * weight\n        competition_metric += submetric\n\n    return competition_metric / normalization","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## test binary problem","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"gbm1 = xgb.XGBClassifier(max_depth=5, n_estimators=750, learning_rate=0.05, random_state=0).fit(train_X, train_Y > 0)\npredictions = gbm1.predict_proba(test_X)\nprint(f'ROC-AUC: {alaska_weighted_auc(test_Y > 0, predictions[:,1])}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## test 4-classes problem\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"gbm2 = xgb.XGBClassifier(max_depth=5, n_estimators=750, learning_rate=0.05, random_state=0).fit(train_X, train_Y)\npredictions = gbm2.predict_proba(test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set the encrypted class value to the greatest one\npredictions = np.transpose(np.vstack((predictions[:, 0], np.max(predictions[:, 1:3], axis = 1))))\nprint(f'ROC-AUC: {alaska_weighted_auc(test_Y > 0, predictions[:,1])}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalize\npredictions = predictions / np.reshape(np.sum(predictions, axis = 1),(predictions.shape[0],1))\nprint(f'ROC-AUC: {alaska_weighted_auc(test_Y > 0, predictions[:,1])}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Upvote please if you find this idea interesting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}