{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## SPR X-Ray Gender Prediction Challenge\n### A baseline solution using Vision Transformers","metadata":{}},{"cell_type":"markdown","source":"**Challenge**: to predict the patient's gender through through tens of thousands of chest X-rays.","metadata":{}},{"cell_type":"markdown","source":"### Vision Transformers Overview\nKolesnikov et al. from Google Research proposed a novel approach to image classification using Vision Transformers, as described in their paper titled [\"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale.\"](research.google/pubs/pub50650/). The authors evaluated the performance of pure [transformers](https://research.google/pubs/pub46201/) on a benchmark task and compared their results to state-of-the-art convolutional networks. Their findings suggest that Vision Transformers are a promising alternative to convolutional networks, with the potential to achieve comparable or superior performance while requiring less computational resources.\n\nThe authors have made their work available online on Github and the HuggingFace Hub. This notebook implements their proposed model architecture for addressing the given challenge, by fine-tuning the it on the provided training dataset.\n\n##### Performance\nWe found a ROC AUC over 90% in both validation and leaderboard datasets.","metadata":{}},{"cell_type":"code","source":"!pip install transformers datasets evaluate ipyplot","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:01:08.781534Z","iopub.execute_input":"2023-03-07T09:01:08.782507Z","iopub.status.idle":"2023-03-07T09:01:08.787467Z","shell.execute_reply.started":"2023-03-07T09:01:08.782464Z","shell.execute_reply":"2023-03-07T09:01:08.786367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport sklearn.metrics as m\n\nfrom transformers import (AutoImageProcessor,\n                          DefaultDataCollator,\n                          AutoModelForImageClassification,\n                          TrainingArguments, \n                          Trainer, pipeline)\n\nfrom torchvision.transforms import (RandomResizedCrop, \n                                    Compose, \n                                    Normalize, \n                                    ToTensor)\n\nimport torch, transformers, datasets, evaluate, ipyplot, warnings, os\n\nTRAIN_IMG_PATH = '/kaggle/input/spr-x-ray-gender/kaggle/kaggle/train/'\nTEST_IMG_PATH = '/kaggle/input/spr-x-ray-gender/kaggle/kaggle/test/'\nTRAIN_CSV_PATH = '/kaggle/input/spr-x-ray-gender/train_gender.csv'\nTEST_CSV_PATH = '/kaggle/input/spr-x-ray-gender/sample_submission_gender.csv'\nDEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n\nfrom transformers.utils import logging\nlogging.set_verbosity_error()\nwarnings.filterwarnings('ignore')\nos.environ[\"WANDB_DISABLED\"] = \"true\"","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:01:09.910096Z","iopub.execute_input":"2023-03-07T09:01:09.912138Z","iopub.status.idle":"2023-03-07T09:01:23.552965Z","shell.execute_reply.started":"2023-03-07T09:01:09.912091Z","shell.execute_reply":"2023-03-07T09:01:23.551756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = datasets.Dataset.from_csv(TRAIN_CSV_PATH)\nds = ds.train_test_split(0.2)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-07T09:01:23.555406Z","iopub.execute_input":"2023-03-07T09:01:23.556550Z","iopub.status.idle":"2023-03-07T09:01:23.799831Z","shell.execute_reply.started":"2023-03-07T09:01:23.556508Z","shell.execute_reply":"2023-03-07T09:01:23.798716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2label = {\n    0: 'Feminino',\n    1: 'Masculino'\n}\nlabel2id = {v: k for k, v in id2label.items()}\nlabel2id","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:01:23.801507Z","iopub.execute_input":"2023-03-07T09:01:23.802209Z","iopub.status.idle":"2023-03-07T09:01:23.810565Z","shell.execute_reply.started":"2023-03-07T09:01:23.802168Z","shell.execute_reply":"2023-03-07T09:01:23.809490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipyplot.plot_class_tabs([TRAIN_IMG_PATH + str(i).zfill(6) + \".png\" for i in ds['train']['imageId'][:20]], \n                        [id2label[i] for i in ds['train']['gender'][:20]], \n                        max_imgs_per_tab=9,\n                        img_width=224, \n                        force_b64=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:01:23.813495Z","iopub.execute_input":"2023-03-07T09:01:23.814130Z","iopub.status.idle":"2023-03-07T09:01:29.375643Z","shell.execute_reply.started":"2023-03-07T09:01:23.814102Z","shell.execute_reply":"2023-03-07T09:01:29.374207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = \"google/vit-base-patch16-224\" # Google Vision Transformer Base Path on 🤗\nimage_processor = AutoImageProcessor.from_pretrained(checkpoint)\n\nnormalize = Normalize(mean=image_processor.image_mean, std=image_processor.image_std)\nsize = (\n    image_processor.size[\"shortest_edge\"]\n    if \"shortest_edge\" in image_processor.size\n    else (image_processor.size[\"height\"], image_processor.size[\"width\"])\n)\ncomposed_transforms = Compose([RandomResizedCrop(size), ToTensor(), normalize])","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:01:29.377659Z","iopub.execute_input":"2023-03-07T09:01:29.385188Z","iopub.status.idle":"2023-03-07T09:01:30.724497Z","shell.execute_reply.started":"2023-03-07T09:01:29.385130Z","shell.execute_reply":"2023-03-07T09:01:30.723532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(examples):\n    images = [Image.open(TRAIN_IMG_PATH + str(i).zfill(6) + \".png\") for i in examples['imageId']]\n    examples[\"pixel_values\"] = [composed_transforms(img.convert(\"RGB\")) for img in images]\n    examples['label'] = examples['gender']\n    return examples\nds = ds.map(preprocess, batched=True, batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:01:30.726296Z","iopub.execute_input":"2023-03-07T09:01:30.726681Z","iopub.status.idle":"2023-03-07T09:11:43.245635Z","shell.execute_reply.started":"2023-03-07T09:01:30.726643Z","shell.execute_reply":"2023-03-07T09:11:43.244636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = DefaultDataCollator()","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:11:43.247769Z","iopub.execute_input":"2023-03-07T09:11:43.248520Z","iopub.status.idle":"2023-03-07T09:11:43.253452Z","shell.execute_reply.started":"2023-03-07T09:11:43.248480Z","shell.execute_reply":"2023-03-07T09:11:43.252381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = evaluate.load(\"accuracy\")\nrocauc = evaluate.load('roc_auc')\n\ndef compute_metrics(eval_pred):\n    predictions, labels = eval_pred\n    predictions = np.argmax(predictions, axis=1)\n    acc=accuracy.compute(references=labels, predictions=predictions)\n    roc_auc = rocauc.compute(references=labels, prediction_scores=predictions)\n    print(m.classification_report(labels, predictions)) \n    print(f'ROC AUC: {m.roc_auc_score(labels, predictions)}')\n    return {'eval_acc': acc['accuracy'], 'eval_roc_auc': roc_auc['roc_auc']}","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:11:43.255225Z","iopub.execute_input":"2023-03-07T09:11:43.255984Z","iopub.status.idle":"2023-03-07T09:11:45.907151Z","shell.execute_reply.started":"2023-03-07T09:11:43.255947Z","shell.execute_reply":"2023-03-07T09:11:45.906254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AutoModelForImageClassification.from_pretrained(\n    checkpoint,\n    num_labels=len(id2label),\n    id2label=id2label,\n    label2id=label2id,\n    ignore_mismatched_sizes=True\n)\n\ntraining_args = TrainingArguments(\n    output_dir=\"hfk_base_vit\",\n    remove_unused_columns=True,\n    evaluation_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    learning_rate=2e-04,\n    gradient_accumulation_steps=1,\n    per_device_train_batch_size=32,\n    per_device_eval_batch_size=32,\n    num_train_epochs=2,\n    logging_steps=10,\n    metric_for_best_model=\"roc_auc\",\n    load_best_model_at_end=True,\n    log_level='critical'\n)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    data_collator=data_collator,\n    train_dataset=ds[\"train\"],\n    eval_dataset=ds[\"test\"],\n    tokenizer=image_processor,\n    compute_metrics=compute_metrics,\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:11:45.908664Z","iopub.execute_input":"2023-03-07T09:11:45.909067Z","iopub.status.idle":"2023-03-07T09:11:53.838184Z","shell.execute_reply.started":"2023-03-07T09:11:45.909028Z","shell.execute_reply":"2023-03-07T09:11:53.837141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:11:53.841946Z","iopub.execute_input":"2023-03-07T09:11:53.842268Z","iopub.status.idle":"2023-03-07T09:40:46.650807Z","shell.execute_reply.started":"2023-03-07T09:11:53.842240Z","shell.execute_reply":"2023-03-07T09:40:46.649592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_path='/kaggle/working/hfk_base_vit/checkpoint-final'\ntrainer.save_model(checkpoint_path)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:40:46.652690Z","iopub.execute_input":"2023-03-07T09:40:46.654061Z","iopub.status.idle":"2023-03-07T09:40:47.501464Z","shell.execute_reply.started":"2023-03-07T09:40:46.654016Z","shell.execute_reply":"2023-03-07T09:40:47.499726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = pipeline(\"image-classification\", model=checkpoint_path)\ntest_ds = datasets.Dataset.from_csv(TEST_CSV_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:40:47.507847Z","iopub.execute_input":"2023-03-07T09:40:47.510644Z","iopub.status.idle":"2023-03-07T09:40:50.048344Z","shell.execute_reply.started":"2023-03-07T09:40:47.510601Z","shell.execute_reply":"2023-03-07T09:40:50.047164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_pipe(examples):\n    images = [Image.open(TEST_IMG_PATH + str(i).zfill(6) + \".png\") for i in examples['imageId']]\n    preds = classifier(images)\n    examples['gender'] = [label2id[i[0]['label']] for i in preds]\n    return examples\n\ntest_ds = test_ds.map(inference_pipe, \n                      batched=True, \n                      batch_size = 64)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:42:32.450276Z","iopub.execute_input":"2023-03-07T09:42:32.451242Z","iopub.status.idle":"2023-03-07T11:09:01.963981Z","shell.execute_reply.started":"2023-03-07T09:42:32.451203Z","shell.execute_reply":"2023-03-07T11:09:01.962906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds.select([i for i in range(5)]).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-03-07T11:14:04.445254Z","iopub.execute_input":"2023-03-07T11:14:04.445623Z","iopub.status.idle":"2023-03-07T11:14:04.502615Z","shell.execute_reply.started":"2023-03-07T11:14:04.445589Z","shell.execute_reply":"2023-03-07T11:14:04.501696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T11:15:20.286954Z","iopub.execute_input":"2023-03-07T11:15:20.288159Z","iopub.status.idle":"2023-03-07T11:15:20.336253Z","shell.execute_reply.started":"2023-03-07T11:15:20.288111Z","shell.execute_reply":"2023-03-07T11:15:20.335241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}