{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Extracts Meta-Features from JPG Images and Dicom Images\n\nThis notebook extracts some features for all train and test images and stores them in CSV files for later analysis:\n\n* Dicom metadata (Study Time, SOP Instance UID, ...)\n* JPG ImageSizes in Bytes, width in pixel, height in pixels\n* mean pixel value\n* mean pixel value for red, green, blue channel\n* histograms for red/green/blue channel with 32 bins\n\nI use dask for fast parallel processing. To reduce communication overhead to the workers, the workload is batched.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport os\nimport pandas as pd \nimport numpy as np\n\nimport PIL\nfrom PIL import Image\n\n!pip install pydicom\nimport pydicom\n\nfrom dask.distributed import Client, progress\nfrom dask.distributed import fire_and_forget\nfrom dask.distributed import as_completed\n\nfrom tlz import partition_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"client = Client(threads_per_worker=8, n_workers=1)\nclient","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASEPATH = '../input'\nCOMPPATH = os.path.join(BASEPATH, 'siim-isic-melanoma-classification')\ndf_train = pd.read_csv(os.path.join(COMPPATH, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(COMPPATH, 'test.csv'))\ndf_sub = pd.read_csv(os.path.join(COMPPATH, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df_train_features = pd.DataFrame(df_train['image_name']).set_index('image_name')\n#df_test_features = pd.DataFrame(df_test['image_name']).set_index('image_name')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def map_parallel_chunked(lst, func):\n    CHUNKSIZE = 16\n    \n    f_many = lambda chunk: [func(x) for x in chunk]\n    chunks = list(partition_all(CHUNKSIZE, lst))\n    futures = client.map(f_many, chunks)\n    for _ in tqdm(as_completed(futures), total=len(futures), unit='chunks', smoothing=0.2): None\n\n    lst_result = []    \n    for chunk in futures:\n        for res in chunk.result():\n            lst_result.append(res)\n            \n    return lst_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_features_from_jpg(typ, image_name):\n    f = os.path.join(COMPPATH, f'jpeg/{typ}/{image_name}.jpg')\n    img = Image.open(f)\n    \n    features = dict()    \n    features['image_size_bytes'] = os.path.getsize(f)    \n    features['image_size_pixels_x'] = img.size[0]\n    features['image_size_pixels_y'] = img.size[1]\n    \n    if img.width < img.height:\n        x = 256\n        y = int(img.height * (256/img.width))\n    else:\n        x = int(img.width * (256/img.height))\n        y = 256        \n        \n    img = img.resize((x,y), resample=PIL.Image.NEAREST)\n    \n    img = np.array(img)\n    \n    features['image_mean'] = np.mean(img)\n    \n    channel_means = np.mean(img, axis=(0,1))\n    features['image_mean_r'] = channel_means[0]\n    features['image_mean_g'] = channel_means[1]\n    features['image_mean_b'] = channel_means[2]\n    \n    bins = np.array(list(range(33)))*8\n    features['image_hist_r'] = np.histogram(img[:,:,0].flatten(), bins=bins)[0]\n    features['image_hist_g'] = np.histogram(img[:,:,1].flatten(), bins=bins)[0]\n    features['image_hist_b'] = np.histogram(img[:,:,2].flatten(), bins=bins)[0]\n    \n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataframe_from_jpg_features(features):\n\n    df = pd.DataFrame(features)\n\n    df_r = pd.DataFrame(df[\"image_hist_r\"].to_list(), columns=[f'image_hist_r_{i}' for i in range(32)])\n    df_g = pd.DataFrame(df[\"image_hist_g\"].to_list(), columns=[f'image_hist_g_{i}' for i in range(32)])\n    df_b = pd.DataFrame(df[\"image_hist_b\"].to_list(), columns=[f'image_hist_b_{i}' for i in range(32)])\n\n    df = df.join([df_r, df_g, df_b])\n\n    del df['image_hist_r']\n    del df['image_hist_g']\n    del df['image_hist_b']\n    \n    return  df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feat = map_parallel_chunked(df_train[\"image_name\"].values, lambda x: get_features_from_jpg('train', x))\ndf_train_features_jpg = get_dataframe_from_jpg_features(feat)\ndf_train_features_jpg[\"image_name\"] = df_train[\"image_name\"]\n\nfeat = map_parallel_chunked(df_test[\"image_name\"].values, lambda x: get_features_from_jpg('test', x))\ndf_test_features_jpg = get_dataframe_from_jpg_features(feat)\ndf_test_features_jpg[\"image_name\"] = df_test[\"image_name\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_features_jpg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_features_from_dicom(typ, image_name):\n    f = os.path.join(COMPPATH, f'{typ}/{image_name}.dcm')\n    ds = pydicom.read_file(f)\n    features = dict()\n    for elem in ds:\n        desc = pydicom.datadict.dictionary_description(elem.tag)\n        if not desc == 'Pixel Data':\n            features[desc] = str(elem.value)            \n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feat = map_parallel_chunked(df_train[\"image_name\"].values, lambda x: get_features_from_dicom('train', x))\ndf_train_features_dicom = pd.DataFrame(feat)\ndf_train_features_dicom[\"image_name\"] = df_train[\"image_name\"]\n\nfeat = map_parallel_chunked(df_test[\"image_name\"].values, lambda x: get_features_from_dicom('test', x))\ndf_test_features_dicom = pd.DataFrame(feat)\ndf_test_features_dicom[\"image_name\"] = df_test[\"image_name\"]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_features_dicom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_features_jpg.to_csv(\"train_features_jpg.csv\", index=False)\ndf_test_features_jpg.to_csv(\"test_features_jpg.csv\", index=False)\n\ndf_train_features_dicom.to_csv(\"train_features_dicom.csv\", index=False)\ndf_test_features_dicom.to_csv(\"test_features_dicom.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"You can split the \"SOP Instance UID\" into the individual parts with following snippet.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_splitted = df_train_features_dicom[\"SOP Instance UID\"].apply(lambda x: pd.Series(x.split('.')))\ndf_splitted","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using DataFrame.describe() we can see that most parts of the ID are constant, but not all.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_splitted.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Column 9, 10 and 11 are not always constant. Do they contain useful information?\n\nPart 9 - could be\nPart 10 - could be\nPart 11 - dont think so. Every entry is unique like the 'image_name'\n\n\nPlease upvote if you like this notebook. Thank you :-)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}