{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-15T02:30:02.191105Z","iopub.execute_input":"2022-05-15T02:30:02.191689Z","iopub.status.idle":"2022-05-15T02:30:02.260676Z","shell.execute_reply.started":"2022-05-15T02:30:02.191578Z","shell.execute_reply":"2022-05-15T02:30:02.259693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/train\")\ntest_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/test\")\nval_imgs = os.listdir(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/val\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:02.262451Z","iopub.execute_input":"2022-05-15T02:30:02.262757Z","iopub.status.idle":"2022-05-15T02:30:02.269317Z","shell.execute_reply.started":"2022-05-15T02:30:02.262723Z","shell.execute_reply":"2022-05-15T02:30:02.268380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install cloud-tpu-client==0.10 torch==1.11.0 https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-1.11-cp37-cp37m-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:02.271345Z","iopub.execute_input":"2022-05-15T02:30:02.271625Z","iopub.status.idle":"2022-05-15T02:30:20.390123Z","shell.execute_reply.started":"2022-05-15T02:30:02.271595Z","shell.execute_reply":"2022-05-15T02:30:20.389059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom PIL import Image\nimport tensorflow as tf\nimport io\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset , DataLoader\nfrom torchvision import transforms\nfrom torch.optim import SGD\n\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:20.392216Z","iopub.execute_input":"2022-05-15T02:30:20.392608Z","iopub.status.idle":"2022-05-15T02:30:24.767710Z","shell.execute_reply.started":"2022-05-15T02:30:20.392557Z","shell.execute_reply":"2022-05-15T02:30:24.766346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\ntest_feature_description = {\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:24.772039Z","iopub.execute_input":"2022-05-15T02:30:24.772455Z","iopub.status.idle":"2022-05-15T02:30:24.779583Z","shell.execute_reply.started":"2022-05-15T02:30:24.772405Z","shell.execute_reply":"2022-05-15T02:30:24.778649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _parse_image_function(example_proto):\n return tf.io.parse_single_example(example_proto, train_feature_description)\ndef second_parse_image_function(example_proto):\n return tf.io.parse_single_example(example_proto, test_feature_description)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:24.785408Z","iopub.execute_input":"2022-05-15T02:30:24.785895Z","iopub.status.idle":"2022-05-15T02:30:24.791227Z","shell.execute_reply.started":"2022-05-15T02:30:24.785863Z","shell.execute_reply":"2022-05-15T02:30:24.790525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_lis = []\nval_lis = []\ntest_lis = []\n\nfor i in train_imgs:\n  train_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/train/\"+i))\n\nfor i in val_imgs:\n  val_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/val/\"+i))\n\nfor i in test_imgs:\n  test_lis.append(tf.data.TFRecordDataset(\"/kaggle/input/tpu-getting-started/tfrecords-jpeg-192x192/test/\"+i))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:24.792531Z","iopub.execute_input":"2022-05-15T02:30:24.793044Z","iopub.status.idle":"2022-05-15T02:30:25.150504Z","shell.execute_reply.started":"2022-05-15T02:30:24.793009Z","shell.execute_reply":"2022-05-15T02:30:25.149444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = []\ntrain_classes = []\ntrain_images = []\n\nval_ids = []\nval_classes = []\nval_images = []\n\ntest_ids = []\ntest_images = []\n\nfor tl in train_lis:\n  temp = tl.map(_parse_image_function)\n  for t in temp:\n    train_classes.append(t[\"class\"].numpy())\n    train_ids.append(str(t[\"id\"].numpy())[2:-1])\n    train_images.append(t[\"image\"].numpy())\n\nfor tl in val_lis:\n  temp = tl.map(_parse_image_function)\n  for t in temp:\n    val_classes.append(t[\"class\"].numpy())\n    val_ids.append(str(t[\"id\"].numpy())[2:-1])\n    val_images.append(t[\"image\"].numpy())\n\nfor tl in test_lis:\n  temp = tl.map(second_parse_image_function)\n  for t in temp:\n    test_ids.append(str(t[\"id\"].numpy())[2:-1])\n    test_images.append(t[\"image\"].numpy())","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:25.152037Z","iopub.execute_input":"2022-05-15T02:30:25.152375Z","iopub.status.idle":"2022-05-15T02:30:35.948982Z","shell.execute_reply.started":"2022-05-15T02:30:25.152342Z","shell.execute_reply":"2022-05-15T02:30:35.947966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustDat(Dataset):\n    def __init__(self , images , classes , ids , transform , mode):\n        self.images = images\n        self.classes = classes\n        self.ids = ids\n        self.transform = transform\n        self.mode = mode\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self , idx):\n        img = Image.open(io.BytesIO(self.images[idx]))\n        img = self.transform(img)\n        if self.mode == \"test\":\n            idd = self.ids[idx]\n            return (img , idd)\n        else:\n            label = self.classes[idx]\n            return (img , label)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:35.950451Z","iopub.execute_input":"2022-05-15T02:30:35.950675Z","iopub.status.idle":"2022-05-15T02:30:35.959115Z","shell.execute_reply.started":"2022-05-15T02:30:35.950648Z","shell.execute_reply":"2022-05-15T02:30:35.958209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((120 , 120)) , \n    transforms.ToTensor() , \n    transforms.Normalize((0 , 0 , 0) , (1 , 1 , 1))\n])","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:35.960147Z","iopub.execute_input":"2022-05-15T02:30:35.960914Z","iopub.status.idle":"2022-05-15T02:30:35.973900Z","shell.execute_reply.started":"2022-05-15T02:30:35.960878Z","shell.execute_reply":"2022-05-15T02:30:35.972718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cust = CustDat(train_images , train_classes , None , transform , \"train\")\nval_cust = CustDat(val_images , val_classes , None , transform , \"val\")\ntest_cust = CustDat(test_images , None , test_ids , transform , \"test\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:35.975483Z","iopub.execute_input":"2022-05-15T02:30:35.975761Z","iopub.status.idle":"2022-05-15T02:30:35.987907Z","shell.execute_reply.started":"2022-05-15T02:30:35.975719Z","shell.execute_reply":"2022-05-15T02:30:35.986560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net(nn.Module):\n  def __init__(self , n_classes):\n    super(Net , self).__init__()\n    self.conv1 = nn.Conv2d(3 , 16 , 3 , padding = 1 , stride = 1)\n    self.conv2 = nn.Conv2d(16 , 32 , 3 , padding = 1 , stride = 1)\n    self.fc1 = nn.Linear(32*20*20 , 1028)\n    self.fc2 = nn.Linear(1028 , n_classes)\n\n  def forward(self , x):\n    x = self.conv1(x)\n    x = F.relu(x)\n    x = F.max_pool2d(x , 3)\n    x = self.conv2(x)\n    x = F.relu(x)\n    x = F.max_pool2d(x , 2)\n    x = torch.flatten(x , 1)\n    x = self.fc1(x)\n    x = F.relu(x)\n    x = self.fc2(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:35.992372Z","iopub.execute_input":"2022-05-15T02:30:35.992779Z","iopub.status.idle":"2022-05-15T02:30:36.003334Z","shell.execute_reply.started":"2022-05-15T02:30:35.992731Z","shell.execute_reply":"2022-05-15T02:30:36.002414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SERIAL_EXEC = xmp.MpSerialExecutor()\nWRAPPED_MODEL = xmp.MpModelWrapper(Net(len(train_classes)))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:36.004828Z","iopub.execute_input":"2022-05-15T02:30:36.005401Z","iopub.status.idle":"2022-05-15T02:30:36.367458Z","shell.execute_reply.started":"2022-05-15T02:30:36.005354Z","shell.execute_reply":"2022-05-15T02:30:36.366312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(rank):\n  train_sampler = torch.utils.data.distributed.DistributedSampler(\n      train_cust , \n      num_replicas = 8 , \n      rank = xm.get_ordinal() , \n      shuffle = True , \n      seed = 0\n  )\n  val_sampler = torch.utils.data.distributed.DistributedSampler(\n      val_cust , \n      num_replicas = 8 , \n      rank = xm.get_ordinal() , \n      shuffle = True , \n      seed = 0\n  )\n  train_loader = DataLoader(\n      train_cust , \n      batch_size = 16 , \n      sampler = train_sampler , \n      num_workers = 1 , \n      drop_last = False\n  )\n  val_loader = DataLoader(\n      val_cust , \n      batch_size = 16 , \n      sampler = val_sampler , \n      num_workers = 1 , \n      drop_last = False\n  )\n  test_loader = DataLoader(\n      test_cust , \n      batch_size = 16 ,\n      num_workers = 1 , \n      drop_last = False\n  )\n  device = xm.xla_device()\n  model = WRAPPED_MODEL.to(device)\n  lr = 0.01 * xm.xrt_world_size()\n  optimizer = SGD(model.parameters() , lr = lr)\n  loss_fn = nn.CrossEntropyLoss()\n\n  num_epochs = 10\n\n  for epoch in range(num_epochs):\n\n    #training\n    para_loader = pl.ParallelLoader(train_loader , [device])\n    train_loss = []\n    train_corr = 0\n    train_sam = 0\n    model.train()\n    for x , (data , label) in enumerate(para_loader.per_device_loader(device)):\n      optimizer.zero_grad()\n      output = model(data)\n      loss = loss_fn(output , label)\n      #accuracy\n      _ , pred = torch.max(output , 1)\n      train_corr += (pred == label).sum()\n      train_sam += label.shape[0]\n      loss.backward()\n      train_loss.append(loss.item())\n      xm.optimizer_step(optimizer)\n      \n    #evaluation\n    model.eval()\n    val_loss = []\n    val_corr = 0\n    val_sam = 0\n    with torch.no_grad():\n      para_loader = pl.ParallelLoader(val_loader , [device])\n      for x , (data , label) in enumerate(para_loader.per_device_loader(device)):\n        output = model(data)\n        loss = loss_fn(output , label)\n        #accuracy\n        _ , pred = torch.max(output , 1)\n        val_corr += (pred == label).sum()\n        val_sam += label.shape[0]\n        val_loss.append(loss.item())\n\n    t_ac = 100.0 * train_corr / train_sam\n    v_ac = 100.0 * val_corr / val_sam\n    t_lo = torch.sum(torch.Tensor(train_loss))\n    v_lo = torch.sum(torch.Tensor(val_loss))\n    \n    print(\"epoch is \",epoch,\" train accu \",t_ac,\" train loss \",t_lo,\" val accu \",v_ac,\" val loss \",v_lo)\n  \n  if xm.is_master_ordinal():\n    dic = {}\n    model.eval()\n    with torch.no_grad():\n      para_loader = pl.ParallelLoader(test_loader , [device])\n      for x , (data , ids) in enumerate(para_loader.per_device_loader(device)):\n        output = model(data)\n        _ , pred = torch.max(output , 1)\n        for i in range(pred.shape[0]):\n          dic[ids[i]] = int(pred[i].cpu().detach().numpy())\n      df = pd.DataFrame(dic.items())\n      df.to_csv(\"fin_sub.csv\" , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:36.368821Z","iopub.execute_input":"2022-05-15T02:30:36.369081Z","iopub.status.idle":"2022-05-15T02:30:36.396666Z","shell.execute_reply.started":"2022-05-15T02:30:36.369053Z","shell.execute_reply":"2022-05-15T02:30:36.395917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xmp.spawn(run , nprocs = 8 , start_method = \"fork\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:30:36.397839Z","iopub.execute_input":"2022-05-15T02:30:36.398609Z","iopub.status.idle":"2022-05-15T02:35:03.112695Z","shell.execute_reply.started":"2022-05-15T02:30:36.398570Z","shell.execute_reply":"2022-05-15T02:35:03.111366Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:35:03.115468Z","iopub.execute_input":"2022-05-15T02:35:03.115782Z","iopub.status.idle":"2022-05-15T02:35:03.126643Z","shell.execute_reply.started":"2022-05-15T02:35:03.115747Z","shell.execute_reply":"2022-05-15T02:35:03.125781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dff = pd.read_csv('fin_sub.csv')\ndff.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:35:03.127826Z","iopub.execute_input":"2022-05-15T02:35:03.128625Z","iopub.status.idle":"2022-05-15T02:35:03.158675Z","shell.execute_reply.started":"2022-05-15T02:35:03.128587Z","shell.execute_reply":"2022-05-15T02:35:03.157703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin = pd.DataFrame({\"id\":dff[\"0\"].values , \"label\":dff[\"1\"].values})","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:35:03.160412Z","iopub.execute_input":"2022-05-15T02:35:03.160670Z","iopub.status.idle":"2022-05-15T02:35:03.166612Z","shell.execute_reply.started":"2022-05-15T02:35:03.160639Z","shell.execute_reply":"2022-05-15T02:35:03.165931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fin.to_csv('submission.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:35:03.167897Z","iopub.execute_input":"2022-05-15T02:35:03.169052Z","iopub.status.idle":"2022-05-15T02:35:03.199769Z","shell.execute_reply.started":"2022-05-15T02:35:03.169010Z","shell.execute_reply":"2022-05-15T02:35:03.199029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:35:03.201174Z","iopub.execute_input":"2022-05-15T02:35:03.201916Z","iopub.status.idle":"2022-05-15T02:35:03.207540Z","shell.execute_reply.started":"2022-05-15T02:35:03.201876Z","shell.execute_reply":"2022-05-15T02:35:03.206689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:35:03.209014Z","iopub.execute_input":"2022-05-15T02:35:03.209282Z","iopub.status.idle":"2022-05-15T02:35:03.222464Z","shell.execute_reply.started":"2022-05-15T02:35:03.209252Z","shell.execute_reply":"2022-05-15T02:35:03.221806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove(\"fin_sub.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:37:14.203157Z","iopub.execute_input":"2022-05-15T02:37:14.203438Z","iopub.status.idle":"2022-05-15T02:37:14.208177Z","shell.execute_reply.started":"2022-05-15T02:37:14.203410Z","shell.execute_reply":"2022-05-15T02:37:14.207004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T02:37:20.800333Z","iopub.execute_input":"2022-05-15T02:37:20.802375Z","iopub.status.idle":"2022-05-15T02:37:20.813919Z","shell.execute_reply.started":"2022-05-15T02:37:20.802225Z","shell.execute_reply":"2022-05-15T02:37:20.812228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}