{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-15T04:19:30.151133Z","iopub.execute_input":"2022-05-15T04:19:30.151438Z","iopub.status.idle":"2022-05-15T04:19:30.197531Z","shell.execute_reply.started":"2022-05-15T04:19:30.151407Z","shell.execute_reply":"2022-05-15T04:19:30.196773Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/train\")\ntest_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/test\")\nval_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/val\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:21:47.589921Z","iopub.execute_input":"2022-05-15T04:21:47.590222Z","iopub.status.idle":"2022-05-15T04:21:47.600190Z","shell.execute_reply.started":"2022-05-15T04:21:47.590195Z","shell.execute_reply":"2022-05-15T04:21:47.599424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install cloud-tpu-client==0.10 torch==1.11.0 https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-1.11-cp37-cp37m-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:21:55.673731Z","iopub.execute_input":"2022-05-15T04:21:55.674149Z","iopub.status.idle":"2022-05-15T04:23:11.042583Z","shell.execute_reply.started":"2022-05-15T04:21:55.674120Z","shell.execute_reply":"2022-05-15T04:23:11.041413Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom PIL import Image\nimport tensorflow as tf\nimport io\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset , DataLoader\nfrom torchvision import transforms , models\nfrom torch.optim import SGD\n\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:32:40.469258Z","iopub.execute_input":"2022-05-15T04:32:40.469649Z","iopub.status.idle":"2022-05-15T04:32:40.477418Z","shell.execute_reply.started":"2022-05-15T04:32:40.469595Z","shell.execute_reply":"2022-05-15T04:32:40.476560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\ntest_feature_description = {\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:24:09.664761Z","iopub.execute_input":"2022-05-15T04:24:09.665032Z","iopub.status.idle":"2022-05-15T04:24:09.671213Z","shell.execute_reply.started":"2022-05-15T04:24:09.664998Z","shell.execute_reply":"2022-05-15T04:24:09.670596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _parse_image_function(example_proto):\n return tf.io.parse_single_example(example_proto, train_feature_description)\ndef second_parse_image_function(example_proto):\n return tf.io.parse_single_example(example_proto, test_feature_description)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:24:09.672490Z","iopub.execute_input":"2022-05-15T04:24:09.673174Z","iopub.status.idle":"2022-05-15T04:24:09.685507Z","shell.execute_reply.started":"2022-05-15T04:24:09.673142Z","shell.execute_reply":"2022-05-15T04:24:09.684770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_lis = []\nval_lis = []\ntest_lis = []\n\nfor i in train_imgs:\n  train_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/train/\"+i))\n\nfor i in val_imgs:\n  val_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/val/\"+i))\n\nfor i in test_imgs:\n  test_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/test/\"+i))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:24:09.687428Z","iopub.execute_input":"2022-05-15T04:24:09.687936Z","iopub.status.idle":"2022-05-15T04:24:10.052036Z","shell.execute_reply.started":"2022-05-15T04:24:09.687897Z","shell.execute_reply":"2022-05-15T04:24:10.051067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = []\ntrain_classes = []\ntrain_images = []\n\nval_ids = []\nval_classes = []\nval_images = []\n\ntest_ids = []\ntest_images = []\n\nfor tl in train_lis:\n  temp = tl.map(_parse_image_function)\n  for t in temp:\n    train_classes.append(t[\"class\"].numpy())\n    train_ids.append(str(t[\"id\"].numpy())[2:-1])\n    train_images.append(t[\"image\"].numpy())\n    \nfor tl in val_lis:\n  temp = tl.map(_parse_image_function)\n  for t in temp:\n    val_classes.append(t[\"class\"].numpy())\n    val_ids.append(str(t[\"id\"].numpy())[2:-1])\n    val_images.append(t[\"image\"].numpy())\n\nfor tl in test_lis:\n  temp = tl.map(second_parse_image_function)\n  for t in temp:\n    test_ids.append(str(t[\"id\"].numpy())[2:-1])\n    test_images.append(t[\"image\"].numpy())","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:24:27.953683Z","iopub.execute_input":"2022-05-15T04:24:27.953982Z","iopub.status.idle":"2022-05-15T04:24:44.523386Z","shell.execute_reply.started":"2022-05-15T04:24:27.953954Z","shell.execute_reply":"2022-05-15T04:24:44.522386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustDat(Dataset):\n    def __init__(self , images , classes , ids , transform , mode):\n        self.images = images\n        self.classes = classes\n        self.ids = ids\n        self.transform = transform\n        self.mode = mode\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self , idx):\n        img = Image.open(io.BytesIO(self.images[idx]))\n        img = self.transform(img)\n        if self.mode == \"test\":\n            idd = self.ids[idx]\n            return (img , idd)\n        else:\n            label = self.classes[idx]\n            return (img , label)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:24:53.300109Z","iopub.execute_input":"2022-05-15T04:24:53.300680Z","iopub.status.idle":"2022-05-15T04:24:53.309401Z","shell.execute_reply.started":"2022-05-15T04:24:53.300611Z","shell.execute_reply":"2022-05-15T04:24:53.308455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((120 , 120)) , \n    transforms.ToTensor() , \n    transforms.Normalize((0 , 0 , 0) , (1 , 1 , 1))\n])","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:25:08.779158Z","iopub.execute_input":"2022-05-15T04:25:08.779494Z","iopub.status.idle":"2022-05-15T04:25:08.785273Z","shell.execute_reply.started":"2022-05-15T04:25:08.779465Z","shell.execute_reply":"2022-05-15T04:25:08.784385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cust = CustDat(train_images , train_classes , None , transform , \"train\")\nval_cust = CustDat(val_images , val_classes , None , transform , \"val\")\ntest_cust = CustDat(test_images , None , test_ids , transform , \"test\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:25:15.713881Z","iopub.execute_input":"2022-05-15T04:25:15.714248Z","iopub.status.idle":"2022-05-15T04:25:15.719800Z","shell.execute_reply.started":"2022-05-15T04:25:15.714215Z","shell.execute_reply":"2022-05-15T04:25:15.718788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_model = models.resnet18(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:33:15.949829Z","iopub.execute_input":"2022-05-15T04:33:15.950318Z","iopub.status.idle":"2022-05-15T04:33:16.274553Z","shell.execute_reply.started":"2022-05-15T04:33:15.950268Z","shell.execute_reply":"2022-05-15T04:33:16.273660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tot_fr = temp_model.fc.in_features","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:33:40.737918Z","iopub.execute_input":"2022-05-15T04:33:40.738233Z","iopub.status.idle":"2022-05-15T04:33:40.742664Z","shell.execute_reply.started":"2022-05-15T04:33:40.738198Z","shell.execute_reply":"2022-05-15T04:33:40.741778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_model.fc = nn.Linear(tot_fr , len(train_classes))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:33:53.714030Z","iopub.execute_input":"2022-05-15T04:33:53.714558Z","iopub.status.idle":"2022-05-15T04:33:53.778949Z","shell.execute_reply.started":"2022-05-15T04:33:53.714499Z","shell.execute_reply":"2022-05-15T04:33:53.777484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SERIAL_EXEC = xmp.MpSerialExecutor()\nWRAPPED_MODEL = xmp.MpModelWrapper(temp_model)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:34:02.853712Z","iopub.execute_input":"2022-05-15T04:34:02.854365Z","iopub.status.idle":"2022-05-15T04:34:02.860026Z","shell.execute_reply.started":"2022-05-15T04:34:02.854312Z","shell.execute_reply":"2022-05-15T04:34:02.858873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(rank):\n  train_sampler = torch.utils.data.distributed.DistributedSampler(\n      train_cust , \n      num_replicas = 8 , \n      rank = xm.get_ordinal() , \n      shuffle = True , \n      seed = 0\n  )\n  val_sampler = torch.utils.data.distributed.DistributedSampler(\n      val_cust , \n      num_replicas = 8 , \n      rank = xm.get_ordinal() , \n      shuffle = True , \n      seed = 0\n  )\n  train_loader = DataLoader(\n      train_cust , \n      batch_size = 16 , \n      sampler = train_sampler , \n      num_workers = 1 , \n      drop_last = False\n  )\n  val_loader = DataLoader(\n      val_cust , \n      batch_size = 16 , \n      sampler = val_sampler , \n      num_workers = 1 , \n      drop_last = False\n  )\n  test_loader = DataLoader(\n      test_cust , \n      batch_size = 16 ,\n      num_workers = 1 , \n      drop_last = False\n  )\n  device = xm.xla_device()\n  model = WRAPPED_MODEL.to(device)\n  lr = 0.01 * xm.xrt_world_size()\n  optimizer = SGD(model.parameters() , lr = lr)\n  loss_fn = nn.CrossEntropyLoss()\n\n  num_epochs = 10\n\n  for epoch in range(num_epochs):\n\n    #training\n    para_loader = pl.ParallelLoader(train_loader , [device])\n    train_loss = []\n    train_corr = 0\n    train_sam = 0\n    model.train()\n    for x , (data , label) in enumerate(para_loader.per_device_loader(device)):\n      optimizer.zero_grad()\n      output = model(data)\n      loss = loss_fn(output , label)\n      #accuracy\n      _ , pred = torch.max(output , 1)\n      train_corr += (pred == label).sum()\n      train_sam += label.shape[0]\n      loss.backward()\n      train_loss.append(loss.item())\n      xm.optimizer_step(optimizer)\n      \n    #evaluation\n    model.eval()\n    val_loss = []\n    val_corr = 0\n    val_sam = 0\n    with torch.no_grad():\n      para_loader = pl.ParallelLoader(val_loader , [device])\n      for x , (data , label) in enumerate(para_loader.per_device_loader(device)):\n        output = model(data)\n        loss = loss_fn(output , label)\n        #accuracy\n        _ , pred = torch.max(output , 1)\n        val_corr += (pred == label).sum()\n        val_sam += label.shape[0]\n        val_loss.append(loss.item())\n\n    t_ac = 100.0 * train_corr / train_sam\n    v_ac = 100.0 * val_corr / val_sam\n    t_lo = torch.sum(torch.Tensor(train_loss))\n    v_lo = torch.sum(torch.Tensor(val_loss))\n    \n    print(\"epoch is \",epoch,\" train accu \",t_ac,\" train loss \",t_lo,\" val accu \",v_ac,\" val loss \",v_lo)\n  \n  if xm.is_master_ordinal():\n    dic = {}\n    model.eval()\n    with torch.no_grad():\n      para_loader = pl.ParallelLoader(test_loader , [device])\n      for x , (data , ids) in enumerate(para_loader.per_device_loader(device)):\n        output = model(data)\n        _ , pred = torch.max(output , 1)\n        for i in range(pred.shape[0]):\n          dic[ids[i]] = int(pred[i].cpu().detach().numpy())\n      df = pd.DataFrame(dic.items())\n      df.to_csv(\"fin_sub.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:34:19.057583Z","iopub.execute_input":"2022-05-15T04:34:19.058800Z","iopub.status.idle":"2022-05-15T04:34:19.084864Z","shell.execute_reply.started":"2022-05-15T04:34:19.058751Z","shell.execute_reply":"2022-05-15T04:34:19.083842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xmp.spawn(run , nprocs = 8 , start_method = \"fork\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:34:28.788959Z","iopub.execute_input":"2022-05-15T04:34:28.789265Z","iopub.status.idle":"2022-05-15T04:42:21.422839Z","shell.execute_reply.started":"2022-05-15T04:34:28.789236Z","shell.execute_reply":"2022-05-15T04:42:21.420995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:42:21.425851Z","iopub.execute_input":"2022-05-15T04:42:21.426197Z","iopub.status.idle":"2022-05-15T04:42:21.437863Z","shell.execute_reply.started":"2022-05-15T04:42:21.426155Z","shell.execute_reply":"2022-05-15T04:42:21.437155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dff = pd.read_csv('fin_sub.csv')\ndff.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:42:49.674994Z","iopub.execute_input":"2022-05-15T04:42:49.675318Z","iopub.status.idle":"2022-05-15T04:42:49.733863Z","shell.execute_reply.started":"2022-05-15T04:42:49.675289Z","shell.execute_reply":"2022-05-15T04:42:49.732878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin = pd.DataFrame({\"id\":dff[\"0\"].values , \"label\":dff[\"1\"].values})","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:43:03.541048Z","iopub.execute_input":"2022-05-15T04:43:03.541618Z","iopub.status.idle":"2022-05-15T04:43:03.552385Z","shell.execute_reply.started":"2022-05-15T04:43:03.541568Z","shell.execute_reply":"2022-05-15T04:43:03.551256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin.to_csv('submission.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:43:10.977564Z","iopub.execute_input":"2022-05-15T04:43:10.977890Z","iopub.status.idle":"2022-05-15T04:43:11.002819Z","shell.execute_reply.started":"2022-05-15T04:43:10.977860Z","shell.execute_reply":"2022-05-15T04:43:11.001756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:43:18.549815Z","iopub.execute_input":"2022-05-15T04:43:18.550685Z","iopub.status.idle":"2022-05-15T04:43:18.556587Z","shell.execute_reply.started":"2022-05-15T04:43:18.550615Z","shell.execute_reply":"2022-05-15T04:43:18.555729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove(\"fin_sub.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:43:32.053410Z","iopub.execute_input":"2022-05-15T04:43:32.054037Z","iopub.status.idle":"2022-05-15T04:43:32.058947Z","shell.execute_reply.started":"2022-05-15T04:43:32.053986Z","shell.execute_reply":"2022-05-15T04:43:32.058103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T04:43:44.689542Z","iopub.execute_input":"2022-05-15T04:43:44.689899Z","iopub.status.idle":"2022-05-15T04:43:44.700438Z","shell.execute_reply.started":"2022-05-15T04:43:44.689869Z","shell.execute_reply":"2022-05-15T04:43:44.699845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}