{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on 10th March, by Marília Prata, mpwolke","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objects as go\n\nimport plotly.offline as py\nimport plotly.express as px\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=Warning)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-10T21:10:48.011880Z","iopub.execute_input":"2023-03-10T21:10:48.012817Z","iopub.status.idle":"2023-03-10T21:11:12.917105Z","shell.execute_reply.started":"2023-03-10T21:10:48.012770Z","shell.execute_reply":"2023-03-10T21:11:12.915902Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/spr-x-ray-gender/train_gender.csv', delimiter=',', encoding='ISO-8859-2')\npd.set_option('display.max_columns', None)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:16:57.243571Z","iopub.execute_input":"2023-03-10T21:16:57.244470Z","iopub.status.idle":"2023-03-10T21:16:57.290787Z","shell.execute_reply.started":"2023-03-10T21:16:57.244425Z","shell.execute_reply":"2023-03-10T21:16:57.289639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#No missing values","metadata":{}},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:17:55.006361Z","iopub.execute_input":"2023-03-10T21:17:55.006833Z","iopub.status.idle":"2023-03-10T21:17:55.016756Z","shell.execute_reply.started":"2023-03-10T21:17:55.006795Z","shell.execute_reply":"2023-03-10T21:17:55.015414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Gender - 0 for female and 1 for male","metadata":{}},{"cell_type":"code","source":"#Code by Lucas Abrahão https://www.kaggle.com/lucasabrahao/trabalho-manufatura-an-lise-de-dados-no-brasil\n\ntrain[\"gender\"].value_counts().plot.barh(color=['blue', '#f5005a'], title='Participants Gender');","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:21:29.397897Z","iopub.execute_input":"2023-03-10T21:21:29.399112Z","iopub.status.idle":"2023-03-10T21:21:29.625722Z","shell.execute_reply.started":"2023-03-10T21:21:29.399052Z","shell.execute_reply":"2023-03-10T21:21:29.624567Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport random\nimport matplotlib.pylab as plt\nfrom glob import glob\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:43:22.551598Z","iopub.execute_input":"2023-03-10T21:43:22.551968Z","iopub.status.idle":"2023-03-10T21:43:22.681902Z","shell.execute_reply.started":"2023-03-10T21:43:22.551938Z","shell.execute_reply":"2023-03-10T21:43:22.680738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ../input/\nPATH = os.path.abspath(os.path.join('..', 'input/spr-x-ray-gender/'))\n\n# ../input/sample/images/\nSOURCE_IMAGES = os.path.join(PATH, \"kaggle\", \"kaggle\")\n\n# ../input/sample/images/*.png\nimages = glob(os.path.join(SOURCE_IMAGES, \"**/*.png\"))\n\n# Load labels\nlabels = pd.read_csv('../input/spr-x-ray-gender/train_gender.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:47:32.157270Z","iopub.execute_input":"2023-03-10T21:47:32.157722Z","iopub.status.idle":"2023-03-10T21:47:32.385323Z","shell.execute_reply.started":"2023-03-10T21:47:32.157681Z","shell.execute_reply":"2023-03-10T21:47:32.384258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First five images paths\nimages[0:5]","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:48:02.009254Z","iopub.execute_input":"2023-03-10T21:48:02.009687Z","iopub.status.idle":"2023-03-10T21:48:02.017377Z","shell.execute_reply.started":"2023-03-10T21:48:02.009647Z","shell.execute_reply":"2023-03-10T21:48:02.016057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Show three random images","metadata":{}},{"cell_type":"code","source":"r = random.sample(images, 3)\nr\n\n# Matplotlib black magic\nplt.figure(figsize=(16,16))\nplt.subplot(131)\nplt.imshow(cv2.imread(r[0]))\n\nplt.subplot(132)\nplt.imshow(cv2.imread(r[1]))\n\nplt.subplot(133)\nplt.imshow(cv2.imread(r[2])); ","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:48:27.048323Z","iopub.execute_input":"2023-03-10T21:48:27.048780Z","iopub.status.idle":"2023-03-10T21:48:28.512652Z","shell.execute_reply.started":"2023-03-10T21:48:27.048744Z","shell.execute_reply":"2023-03-10T21:48:28.511277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Let's try another way since we don't have labels","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import optim,nn\nimport torch.nn.functional as F\nfrom torchvision import transforms as T,models\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torchvision.utils import make_grid","metadata":{"execution":{"iopub.status.busy":"2023-03-10T21:56:56.477455Z","iopub.execute_input":"2023-03-10T21:56:56.477889Z","iopub.status.idle":"2023-03-10T21:56:58.905328Z","shell.execute_reply.started":"2023-03-10T21:56:56.477854Z","shell.execute_reply":"2023-03-10T21:56:58.904251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# install fastkaggle if not available\ntry: import fastkaggle\nexcept ModuleNotFoundError:\n    !pip install -Uq fastkaggle\nfrom fastkaggle import *","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:07:49.857012Z","iopub.execute_input":"2023-03-10T22:07:49.858167Z","iopub.status.idle":"2023-03-10T22:08:02.164918Z","shell.execute_reply.started":"2023-03-10T22:07:49.858122Z","shell.execute_reply":"2023-03-10T22:08:02.163793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Set up","metadata":{}},{"cell_type":"code","source":"comp = 'spr-x-ray-gender/kaggle/kaggle'\npath = setup_comp(comp, install='fastai \"timm>=0.6.2.dev0\"')\ndisplay(path)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:09:04.892747Z","iopub.execute_input":"2023-03-10T22:09:04.893855Z","iopub.status.idle":"2023-03-10T22:09:17.407364Z","shell.execute_reply.started":"2023-03-10T22:09:04.893813Z","shell.execute_reply":"2023-03-10T22:09:17.406064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nset_seed(42)\npath.ls()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:09:33.814913Z","iopub.execute_input":"2023-03-10T22:09:33.815399Z","iopub.status.idle":"2023-03-10T22:09:35.056842Z","shell.execute_reply.started":"2023-03-10T22:09:33.815357Z","shell.execute_reply":"2023-03-10T22:09:35.055479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Show the data","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/code/stpeteishii/x-ray-body-image-classify-fasiai\n\ntrn_path = path/'train'\nfiles = get_image_files(trn_path)\ntst_path = path/'test'","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:09:46.420059Z","iopub.execute_input":"2023-03-10T22:09:46.420483Z","iopub.status.idle":"2023-03-10T22:09:48.621390Z","shell.execute_reply.started":"2023-03-10T22:09:46.420441Z","shell.execute_reply":"2023-03-10T22:09:48.620076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/stpeteishii/x-ray-body-image-classify-fasiai\n\nimg = PILImage.create(files[0])\nprint(img.size)\nimg.to_thumb(128)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:10:00.798583Z","iopub.execute_input":"2023-03-10T22:10:00.799023Z","iopub.status.idle":"2023-03-10T22:10:00.877529Z","shell.execute_reply.started":"2023-03-10T22:10:00.798982Z","shell.execute_reply":"2023-03-10T22:10:00.876599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastcore.parallel import *\ndef f(o): return PILImage.create(o).size\nsizes = parallel(f, files, n_workers=8)\npd.Series(sizes).value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:11:55.064066Z","iopub.execute_input":"2023-03-10T22:11:55.064649Z","iopub.status.idle":"2023-03-10T22:15:13.667816Z","shell.execute_reply.started":"2023-03-10T22:11:55.064598Z","shell.execute_reply":"2023-03-10T22:15:13.664975Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nimport timm\nfrom pathlib import Path\nfrom fastai.vision.all import *\nfrom fastai.vision.all import vision_learner, get_image_files","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:21:25.153818Z","iopub.execute_input":"2023-03-10T22:21:25.154528Z","iopub.status.idle":"2023-03-10T22:21:26.061576Z","shell.execute_reply.started":"2023-03-10T22:21:25.154451Z","shell.execute_reply":"2023-03-10T22:21:26.059915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_aug(): \n    return A.Compose([\n    A.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5\n            ),\n    A.CoarseDropout(p=0.5),\n    A.RandomBrightnessContrast(p=1),    \n    A.RandomGamma(p=1),    \n    A.CLAHE(p=1), \n])","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:20:33.664325Z","iopub.execute_input":"2023-03-10T22:20:33.664905Z","iopub.status.idle":"2023-03-10T22:20:33.675209Z","shell.execute_reply.started":"2023-03-10T22:20:33.664853Z","shell.execute_reply":"2023-03-10T22:20:33.673332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AlbumentationsTransform(Transform):\n    def __init__(self, aug): self.aug = aug\n    def encodes(self, img: PILImage):\n        aug_img = self.aug(image=np.array(img))['image']#Do Not replace that image\n        return PILImage.create(aug_img)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:23:21.267259Z","iopub.execute_input":"2023-03-10T22:23:21.267759Z","iopub.status.idle":"2023-03-10T22:23:21.280232Z","shell.execute_reply.started":"2023-03-10T22:23:21.267718Z","shell.execute_reply":"2023-03-10T22:23:21.279178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_tfms = [Resize(224), AlbumentationsTransform(get_train_aug())]","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:24:12.707470Z","iopub.execute_input":"2023-03-10T22:24:12.708042Z","iopub.status.idle":"2023-03-10T22:24:12.714981Z","shell.execute_reply.started":"2023-03-10T22:24:12.707990Z","shell.execute_reply":"2023-03-10T22:24:12.713916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by https://docs.fast.ai/tutorial.albumentations.html\n\ndef is_male(x): return x[0].isupper()\ndls = ImageDataLoaders.from_name_func(\n    path, get_image_files(path), valid_pct=0.2, seed=42,\n    label_func=is_male, item_tfms=item_tfms)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:24:20.027224Z","iopub.execute_input":"2023-03-10T22:24:20.027680Z","iopub.status.idle":"2023-03-10T22:24:25.837266Z","shell.execute_reply.started":"2023-03-10T22:24:20.027627Z","shell.execute_reply":"2023-03-10T22:24:25.836206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I don't know if the X-Ray below belongs to females since I'm not trusting on my script","metadata":{}},{"cell_type":"code","source":"dls.train.show_batch(max_n=4)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T22:25:08.028868Z","iopub.execute_input":"2023-03-10T22:25:08.030082Z","iopub.status.idle":"2023-03-10T22:25:15.137294Z","shell.execute_reply.started":"2023-03-10T22:25:08.030036Z","shell.execute_reply":"2023-03-10T22:25:15.136310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That's all for now sice I wasn't able to adapt other codes that I found","metadata":{}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nChris Crawford https://www.kaggle.com/code/crawford/resize-and-save-images-as-numpy-arrays-128x128\n\nStpete Ishii https://www.kaggle.com/code/stpeteishii/x-ray-body-image-classify-fasiai\n\nFastai https://docs.fast.ai/tutorial.albumentations.html","metadata":{}}]}