{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-28T08:58:26.319114Z","iopub.execute_input":"2022-11-28T08:58:26.319945Z","iopub.status.idle":"2022-11-28T08:58:26.361607Z","shell.execute_reply.started":"2022-11-28T08:58:26.319903Z","shell.execute_reply":"2022-11-28T08:58:26.360664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/train\")\ntest_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/test\")\nval_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/val\")\nprint(len(train_imgs),len(test_imgs),len(val_imgs))","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:26.363246Z","iopub.execute_input":"2022-11-28T08:58:26.363479Z","iopub.status.idle":"2022-11-28T08:58:26.373290Z","shell.execute_reply.started":"2022-11-28T08:58:26.363451Z","shell.execute_reply":"2022-11-28T08:58:26.372278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:26.375897Z","iopub.execute_input":"2022-11-28T08:58:26.376648Z","iopub.status.idle":"2022-11-28T08:58:27.499969Z","shell.execute_reply.started":"2022-11-28T08:58:26.376610Z","shell.execute_reply":"2022-11-28T08:58:27.498840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install cloud-tpu-client==0.10 torch==1.11.0 https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-1.11-cp37-cp37m-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:27.501693Z","iopub.execute_input":"2022-11-28T08:58:27.501986Z","iopub.status.idle":"2022-11-28T08:58:37.175318Z","shell.execute_reply.started":"2022-11-28T08:58:27.501950Z","shell.execute_reply":"2022-11-28T08:58:37.174327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom PIL import Image\nimport tensorflow as tf\nimport io\n\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset , DataLoader\nfrom torchvision import transforms , models\nfrom torch.optim import SGD\n\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:37.177888Z","iopub.execute_input":"2022-11-28T08:58:37.178171Z","iopub.status.idle":"2022-11-28T08:58:37.185537Z","shell.execute_reply.started":"2022-11-28T08:58:37.178124Z","shell.execute_reply":"2022-11-28T08:58:37.184910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\ntest_feature_description = {\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:37.186380Z","iopub.execute_input":"2022-11-28T08:58:37.186596Z","iopub.status.idle":"2022-11-28T08:58:37.232866Z","shell.execute_reply.started":"2022-11-28T08:58:37.186563Z","shell.execute_reply":"2022-11-28T08:58:37.231703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _parse_image_function(example_proto):\n return tf.io.parse_single_example(example_proto, train_feature_description)\ndef second_parse_image_function(example_proto):\n return tf.io.parse_single_example(example_proto, test_feature_description)","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:37.234272Z","iopub.execute_input":"2022-11-28T08:58:37.234526Z","iopub.status.idle":"2022-11-28T08:58:37.245799Z","shell.execute_reply.started":"2022-11-28T08:58:37.234498Z","shell.execute_reply":"2022-11-28T08:58:37.244948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_lis = []\nval_lis = []\ntest_lis = []\n\nfor i in train_imgs:\n  train_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/train/\"+i))\n\nfor i in val_imgs:\n  val_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/val/\"+i))\n\nfor i in test_imgs:\n  test_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/test/\"+i))","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:37.246961Z","iopub.execute_input":"2022-11-28T08:58:37.247699Z","iopub.status.idle":"2022-11-28T08:58:37.511542Z","shell.execute_reply.started":"2022-11-28T08:58:37.247668Z","shell.execute_reply":"2022-11-28T08:58:37.510664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = []\ntrain_classes = []\ntrain_images = []\n\nval_ids = []\nval_classes = []\nval_images = []\n\ntest_ids = []\ntest_images = []\n\nfor tl in train_lis:\n  temp = tl.map(_parse_image_function)\n  for t in temp:\n    train_classes.append(t[\"class\"].numpy())\n    train_ids.append(str(t[\"id\"].numpy())[2:-1])\n    train_images.append(t[\"image\"].numpy())\n    \nfor tl in val_lis:\n  temp = tl.map(_parse_image_function)\n  for t in temp:\n    val_classes.append(t[\"class\"].numpy())\n    val_ids.append(str(t[\"id\"].numpy())[2:-1])\n    val_images.append(t[\"image\"].numpy())\n\nfor tl in test_lis:\n  temp = tl.map(second_parse_image_function)\n  for t in temp:\n    test_ids.append(str(t[\"id\"].numpy())[2:-1])\n    test_images.append(t[\"image\"].numpy())","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:37.512781Z","iopub.execute_input":"2022-11-28T08:58:37.513020Z","iopub.status.idle":"2022-11-28T08:58:46.777996Z","shell.execute_reply.started":"2022-11-28T08:58:37.512993Z","shell.execute_reply":"2022-11-28T08:58:46.776901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustDat(Dataset):\n    def __init__(self , images , classes , ids , transform , mode):\n        self.images = images\n        self.classes = classes\n        self.ids = ids\n        self.transform = transform\n        self.mode = mode\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self , idx):\n        img = Image.open(io.BytesIO(self.images[idx]))\n        img = self.transform(img)\n        if self.mode == \"test\":\n            idd = self.ids[idx]\n            return (img , idd)\n        else:\n            label = self.classes[idx]\n            return (img , label)","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:46.779565Z","iopub.execute_input":"2022-11-28T08:58:46.779909Z","iopub.status.idle":"2022-11-28T08:58:46.788066Z","shell.execute_reply.started":"2022-11-28T08:58:46.779863Z","shell.execute_reply":"2022-11-28T08:58:46.787015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((120 , 120)) , \n    transforms.ToTensor() , \n    transforms.Normalize((0 , 0 , 0) , (1 , 1 , 1))\n])","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:46.789505Z","iopub.execute_input":"2022-11-28T08:58:46.789800Z","iopub.status.idle":"2022-11-28T08:58:46.801692Z","shell.execute_reply.started":"2022-11-28T08:58:46.789763Z","shell.execute_reply":"2022-11-28T08:58:46.800691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cust = CustDat(train_images , train_classes , None , transform , \"train\")\nval_cust = CustDat(val_images , val_classes , None , transform , \"val\")\ntest_cust = CustDat(test_images , None , test_ids , transform , \"test\")","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:46.803402Z","iopub.execute_input":"2022-11-28T08:58:46.803714Z","iopub.status.idle":"2022-11-28T08:58:46.859205Z","shell.execute_reply.started":"2022-11-28T08:58:46.803675Z","shell.execute_reply":"2022-11-28T08:58:46.858391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BasicBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, padding=1)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(in_channels=out_channels, out_channels=out_channels, kernel_size=1)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n        self.downsample = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1)\n\n    def forward(self, x: Tensor) -> Tensor:\n        identity = self.downsample(x)\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass ResNet18(nn.Module):\n    def __init__(self,\n                 image_size: int,\n                 in_channels: int,\n                 num_classes: int,\n                 base_width=8,\n                 ):\n        super().__init__()\n        self.image_size = image_size\n        self.embed = nn.Conv2d(in_channels=in_channels, out_channels=base_width * 1, kernel_size=1)\n        self.bn1 = nn.BatchNorm2d(base_width * 1)\n        self.relu = nn.ReLU(inplace=True)\n        self.block1 = BasicBlock(in_channels=base_width * 1, out_channels=base_width * 1)\n        self.block2 = BasicBlock(in_channels=base_width * 1, out_channels=base_width * 2)\n        self.block3 = BasicBlock(in_channels=base_width * 2, out_channels=base_width * 4)\n        self.block4 = BasicBlock(in_channels=base_width * 4, out_channels=base_width * 8)\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(base_width * 8, num_classes)\n\n    def forward(self, x: Tensor) -> Tensor:\n        x = self.embed(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:46.862430Z","iopub.execute_input":"2022-11-28T08:58:46.862786Z","iopub.status.idle":"2022-11-28T08:58:46.878767Z","shell.execute_reply.started":"2022-11-28T08:58:46.862748Z","shell.execute_reply":"2022-11-28T08:58:46.877791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_model = models.resnet18(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:46.883479Z","iopub.execute_input":"2022-11-28T08:58:46.883763Z","iopub.status.idle":"2022-11-28T08:58:49.930844Z","shell.execute_reply.started":"2022-11-28T08:58:46.883737Z","shell.execute_reply":"2022-11-28T08:58:49.930207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tot_fr = temp_model.fc.in_features","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:49.932290Z","iopub.execute_input":"2022-11-28T08:58:49.933116Z","iopub.status.idle":"2022-11-28T08:58:49.937138Z","shell.execute_reply.started":"2022-11-28T08:58:49.933084Z","shell.execute_reply":"2022-11-28T08:58:49.936217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_model.fc = nn.Linear(tot_fr , len(train_classes))","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:49.938429Z","iopub.execute_input":"2022-11-28T08:58:49.938659Z","iopub.status.idle":"2022-11-28T08:58:50.002544Z","shell.execute_reply.started":"2022-11-28T08:58:49.938632Z","shell.execute_reply":"2022-11-28T08:58:50.001873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_model = ResNet18(192, 3, len(train_classes))","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:50.003945Z","iopub.execute_input":"2022-11-28T08:58:50.004375Z","iopub.status.idle":"2022-11-28T08:58:50.021082Z","shell.execute_reply.started":"2022-11-28T08:58:50.004344Z","shell.execute_reply":"2022-11-28T08:58:50.020336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SERIAL_EXEC = xmp.MpSerialExecutor()\nWRAPPED_MODEL = xmp.MpModelWrapper(temp_model)","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:50.022720Z","iopub.execute_input":"2022-11-28T08:58:50.023224Z","iopub.status.idle":"2022-11-28T08:58:50.027740Z","shell.execute_reply.started":"2022-11-28T08:58:50.023190Z","shell.execute_reply":"2022-11-28T08:58:50.026922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(rank):\n  train_sampler = torch.utils.data.distributed.DistributedSampler(\n      train_cust , \n      num_replicas = 8 , \n      rank = xm.get_ordinal() , \n      shuffle = True , \n      seed = 0\n  )\n  val_sampler = torch.utils.data.distributed.DistributedSampler(\n      val_cust , \n      num_replicas = 8 , \n      rank = xm.get_ordinal() , \n      shuffle = True , \n      seed = 0\n  )\n  train_loader = DataLoader(\n      train_cust , \n      batch_size = 16 , \n      sampler = train_sampler , \n      num_workers = 1 , \n      drop_last = False\n  )\n  val_loader = DataLoader(\n      val_cust , \n      batch_size = 16 , \n      sampler = val_sampler , \n      num_workers = 1 , \n      drop_last = False\n  )\n  test_loader = DataLoader(\n      test_cust , \n      batch_size = 16 ,\n      num_workers = 1 , \n      drop_last = False\n  )\n  device = xm.xla_device()\n  model = WRAPPED_MODEL.to(device)\n  lr = 0.01 * xm.xrt_world_size()\n  optimizer = SGD(model.parameters() , lr = lr)\n  loss_fn = nn.CrossEntropyLoss()\n\n  num_epochs = 10\n\n  for epoch in range(num_epochs):\n\n    #training\n    para_loader = pl.ParallelLoader(train_loader , [device])\n    train_loss = []\n    train_corr = 0\n    train_sam = 0\n    model.train()\n    for x , (data , label) in enumerate(para_loader.per_device_loader(device)):\n      optimizer.zero_grad()\n      output = model(data)\n      loss = loss_fn(output , label)\n      #accuracy\n      _ , pred = torch.max(output , 1)\n      train_corr += (pred == label).sum()\n      train_sam += label.shape[0]\n      loss.backward()\n      train_loss.append(loss.item())\n      xm.optimizer_step(optimizer)\n      \n    #evaluation\n    model.eval()\n    val_loss = []\n    val_corr = 0\n    val_sam = 0\n    with torch.no_grad():\n      para_loader = pl.ParallelLoader(val_loader , [device])\n      for x , (data , label) in enumerate(para_loader.per_device_loader(device)):\n        output = model(data)\n        loss = loss_fn(output , label)\n        #accuracy\n        _ , pred = torch.max(output , 1)\n        val_corr += (pred == label).sum()\n        val_sam += label.shape[0]\n        val_loss.append(loss.item())\n\n    t_ac = 100.0 * train_corr / train_sam\n    v_ac = 100.0 * val_corr / val_sam\n    t_lo = torch.sum(torch.Tensor(train_loss))\n    v_lo = torch.sum(torch.Tensor(val_loss))\n    \n    print(\"epoch is \",epoch,\" train accu \",t_ac,\" train loss \",t_lo,\" val accu \",v_ac,\" val loss \",v_lo)\n  \n  if xm.is_master_ordinal():\n    dic = {}\n    model.eval()\n    with torch.no_grad():\n      para_loader = pl.ParallelLoader(test_loader , [device])\n      for x , (data , ids) in enumerate(para_loader.per_device_loader(device)):\n        output = model(data)\n        _ , pred = torch.max(output , 1)\n        for i in range(pred.shape[0]):\n          dic[ids[i]] = int(pred[i].cpu().detach().numpy())\n      df = pd.DataFrame(dic.items())\n      df.to_csv(\"fin_sub.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:50.029122Z","iopub.execute_input":"2022-11-28T08:58:50.029456Z","iopub.status.idle":"2022-11-28T08:58:50.047852Z","shell.execute_reply.started":"2022-11-28T08:58:50.029417Z","shell.execute_reply":"2022-11-28T08:58:50.047208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xmp.spawn(run , nprocs = 8 , start_method = \"fork\")","metadata":{"execution":{"iopub.status.busy":"2022-11-28T08:58:50.048852Z","iopub.execute_input":"2022-11-28T08:58:50.049521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dff = pd.read_csv('fin_sub.csv')\ndff.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin = pd.DataFrame({\"id\":dff[\"0\"].values , \"label\":dff[\"1\"].values})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin.to_csv('submission.csv' , index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove(\"fin_sub.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}