{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
Digit Recognition with PyTorch CNN
","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sns\n\nfrom sklearn.manifold import TSNE\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nimport shap\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:04.126369Z","iopub.execute_input":"2022-08-06T06:38:04.127118Z","iopub.status.idle":"2022-08-06T06:38:04.139736Z","shell.execute_reply.started":"2022-08-06T06:38:04.127065Z","shell.execute_reply":"2022-08-06T06:38:04.138899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLOR = 'black'\nmpl.rcParams.update({'text.color' : COLOR,\n 'axes.labelcolor' : COLOR,\n 'xtick.color' : COLOR,\n 'ytick.color' : COLOR,\n 'axes.labelsize' : 18,\n 'axes.titlesize' : 18,\n 'xtick.labelsize' : 16,\n 'ytick.labelsize' : 16,\n 'axes.grid' : True,\n 'grid.color' : 'k',\n 'grid.alpha' : 0.4,\n 'grid.linestyle' : ':',\n 'grid.linewidth' : 0.5})\n\npalette = [\"#5f0f40\",\"#9a031e\",\"#fb8b24\",\"#e36414\",\"#0f4c5c\"]\nsns.palplot(sns.color_palette(palette))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:06.345525Z","iopub.execute_input":"2022-08-06T06:38:06.345917Z","iopub.status.idle":"2022-08-06T06:38:06.442440Z","shell.execute_reply.started":"2022-08-06T06:38:06.345884Z","shell.execute_reply":"2022-08-06T06:38:06.440994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****\n# | Table of Contents\n\n

1 -- Problem Definition

\n\n

2 -- Dimensionality reduction with t-SNE

\n\n

2.1 -- MNIST Dimensionality Reduction [2D]

\n\n

2.2 -- MNIST Dimensionality Reduction [3D]

\n\n

3 -- CNN Model with PyTorch

\n\n

3.1 -- Data Preparation

\n\n

3.2 -- CNN Model

\n\n

3.3 -- Helper Function Definition

\n\n

3.4 -- Training the CNN Model

\n\n

3.5 -- Shap Deep Explainer

\n\n

3.6 -- Submission

\n\n

4 -- Final Thoughts

\n\n
","metadata":{}},{"cell_type":"markdown","source":"

\n\n****\n\n# 1 | Problem Definition\n","metadata":{}},{"cell_type":"markdown","source":"The goal is to correctly identify digits from a dataset of tens of thousands of handwritten images. ","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('../input/digit-recognizer/test.csv')\n\ndf_train.sample(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:07.569690Z","iopub.execute_input":"2022-08-06T06:38:07.570717Z","iopub.status.idle":"2022-08-06T06:38:11.967979Z","shell.execute_reply.started":"2022-08-06T06:38:07.570676Z","shell.execute_reply":"2022-08-06T06:38:11.967038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = df_train.drop(\"label\", axis=1)\ny_train = df_train[\"label\"]\n\nprint(\"Shape of training set: {}\".format(X_train.shape))\nprint(\"Shape of test set: {}\".format(y_train.shape))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:11.969820Z","iopub.execute_input":"2022-08-06T06:38:11.970478Z","iopub.status.idle":"2022-08-06T06:38:12.059335Z","shell.execute_reply.started":"2022-08-06T06:38:11.970442Z","shell.execute_reply":"2022-08-06T06:38:12.058113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices = [22, 54, 44, 55]\n_, ax = plt.subplots(1, len(indices), figsize = (14,8))\nfor i, v in enumerate(indices):\n title = f\"Label: {y_train.iloc[v]}\"\n ax[i].imshow(X_train.values.reshape(-1,1,28,28)[v, 0] )\n ax[i].set_title(title)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:12.061498Z","iopub.execute_input":"2022-08-06T06:38:12.062352Z","iopub.status.idle":"2022-08-06T06:38:12.455446Z","shell.execute_reply.started":"2022-08-06T06:38:12.062311Z","shell.execute_reply":"2022-08-06T06:38:12.454497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n****\n\n# 2 | Dimensionality reduction with t-SNE \n","metadata":{}},{"cell_type":"markdown","source":"[T-distributed Stochastic Neighbour Embedding Documentation](https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html)\n\n* n_components [default: 2]: Dimension of the embedded space.\n* verbose [default: 0] : Verbosity level.\n* perplexity [default: 30]: The perplexity is related to the number of nearest neighbors that are used in other manifold learning algorithms. Consider selecting a value between 5 and 50.\n* n_iter [default: 1000]: Maximum number of iterations for the optimization. Should be at least 250.","metadata":{}},{"cell_type":"code","source":"X_subset = X_train[0:10000]\ny_subset = y_train[0:10000]\n\nprint(np.unique(y_subset))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:12.458494Z","iopub.execute_input":"2022-08-06T06:38:12.459133Z","iopub.status.idle":"2022-08-06T06:38:12.467233Z","shell.execute_reply.started":"2022-08-06T06:38:12.459091Z","shell.execute_reply":"2022-08-06T06:38:12.465885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=50)\npca_result = pca.fit_transform(X_subset)\ntsne = TSNE(\n random_state = 0xBEEF, \n n_components = 2,\n verbose = 0, \n perplexity = 40, \n n_iter = 300\n).fit_transform(pca_result)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:12.468976Z","iopub.execute_input":"2022-08-06T06:38:12.469676Z","iopub.status.idle":"2022-08-06T06:38:51.138816Z","shell.execute_reply.started":"2022-08-06T06:38:12.469636Z","shell.execute_reply":"2022-08-06T06:38:51.137824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n## 2.1 | MNIST Dimensionality Reduction [2D]","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(figsize = (17,16))\n\nplt.scatter(\n x = tsne[:, 0], \n y = tsne[:, 1], \n s = 7, \n c = y_subset, \n cmap='Spectral'\n)\n\n\nplt.gca().set_aspect('equal', 'datalim')\nplt.colorbar(boundaries = np.arange(11)- 0.5 ).set_ticks(np.arange(10))\nplt.title('MNIST Dimensionality Reduction with t-SNE', fontsize=24)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:51.140459Z","iopub.execute_input":"2022-08-06T06:38:51.140858Z","iopub.status.idle":"2022-08-06T06:38:51.642150Z","shell.execute_reply.started":"2022-08-06T06:38:51.140804Z","shell.execute_reply":"2022-08-06T06:38:51.641194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encoding all the images for inclusion in a dataframe.\n\nfrom io import BytesIO\nfrom PIL import Image\nimport base64\n\n\ndef embeddable_image(data):\n img_data = 255 - 15 * data.astype(np.uint8)\n image = Image.fromarray(img_data, mode='L').resize((28,28), Image.BICUBIC)\n buffer = BytesIO()\n image.save(buffer, format='png')\n for_encoding = buffer.getvalue()\n return 'data:image/png;base64,' + base64.b64encode(for_encoding).decode()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:51.643104Z","iopub.execute_input":"2022-08-06T06:38:51.643403Z","iopub.status.idle":"2022-08-06T06:38:51.653195Z","shell.execute_reply.started":"2022-08-06T06:38:51.643376Z","shell.execute_reply":"2022-08-06T06:38:51.651622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bokeh.plotting import figure, show, output_notebook\nfrom bokeh.models import HoverTool, ColumnDataSource, CategoricalColorMapper\nfrom bokeh.palettes import Spectral10, viridis\n\noutput_notebook()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:51.655554Z","iopub.execute_input":"2022-08-06T06:38:51.656204Z","iopub.status.idle":"2022-08-06T06:38:52.004421Z","shell.execute_reply.started":"2022-08-06T06:38:51.656171Z","shell.execute_reply":"2022-08-06T06:38:52.003486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_subset_reshape = X_subset.values.reshape(-1,28,28)\n\ndigits_df = pd.DataFrame(tsne, columns=('x', 'y'))\ndigits_df['digit'] = [str(x) for x in y_subset]\ndigits_df['image'] = list(map(embeddable_image, x_subset_reshape))\n\n\ndatasource = ColumnDataSource(digits_df)\ncolor_mapping = CategoricalColorMapper(factors=[str(x) for x in range(10)], palette=Spectral10)\n\n\nhover = HoverTool(tooltips=\"\"\"\n
\n
\n \n
\n
\n Digit:\n @digit\n
\n
\n\"\"\")\n\nplot_options = {\n \"title\" : 't-SNE projection of the MNIST dataset',\n \"plot_width\" : 1000,\n \"plot_height\" : 1000,\n \"tools\" : [hover, 'pan, wheel_zoom, box_zoom, reset, save']\n}\n\npltf = figure(**plot_options)\n\npltf.circle(\n 'x',\n 'y',\n source = datasource,\n color=dict(field='digit', transform=color_mapping),\n line_alpha=0.6,\n fill_alpha=0.6,\n size=4\n)\n\npltf.xaxis.axis_label = \"X1\"\npltf.yaxis.axis_label = \"X2\"\n\nshow(pltf)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:52.006060Z","iopub.execute_input":"2022-08-06T06:38:52.006728Z","iopub.status.idle":"2022-08-06T06:38:53.818061Z","shell.execute_reply.started":"2022-08-06T06:38:52.006690Z","shell.execute_reply":"2022-08-06T06:38:53.816651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n## 2.2 | MNIST Dimensionality Reduction [3D]","metadata":{}},{"cell_type":"code","source":"pca_tsne2 = TSNE(random_state = 0xBEEF, n_components=3, verbose=0, perplexity=40, n_iter=300).fit_transform(pca_result)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:38:53.823012Z","iopub.execute_input":"2022-08-06T06:38:53.823774Z","iopub.status.idle":"2022-08-06T06:41:13.986170Z","shell.execute_reply.started":"2022-08-06T06:38:53.823720Z","shell.execute_reply":"2022-08-06T06:41:13.985182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\n\npltfig = go.Figure(data=[go.Scatter3d(\n x = pca_tsne2[:, 0],\n y = pca_tsne2[:, 1],\n z = pca_tsne2[:, 2],\n text = y_subset,\n mode = 'markers',\n marker=dict(\n size=3,\n color = y_subset, \n colorscale='Spectral', \n opacity=0.8\n ),\n hovertemplate = \"Digit: %{text}\",\n \n)])\n\npltfig.update_layout({\n \"title\" : {\n \"text\" : \"MNIST Dimensionality Reduction with t-SNE\", \n \"font\" : {\n \"family\" : \"Courier New, monospace\",\n \"size\" : 28,\n \"color\" : \"#e36414\"\n },\n 'y':0.9, \n 'x':0.5, \n \"xanchor\" : \"center\", \n \"yanchor\" : \"top\"\n }, \n \"autosize\" : False, \n \"width\" : 1000, \n \"height\" : 1000,\n})\n\npltfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:41:13.987780Z","iopub.execute_input":"2022-08-06T06:41:13.988184Z","iopub.status.idle":"2022-08-06T06:41:14.192109Z","shell.execute_reply.started":"2022-08-06T06:41:13.988140Z","shell.execute_reply":"2022-08-06T06:41:14.189917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n****\n\n# 3 | CNN Model with PyTorch","metadata":{}},{"cell_type":"markdown","source":"

\n\n## 3.1 | Data Preparation","metadata":{}},{"cell_type":"code","source":"X_train = df_train.drop(\"label\", axis=1)\ny_train = df_train[\"label\"]\n\nprint(\"\\nBefore Reshape\")\nprint(f\"\\tShape of training set: {X_train.shape}\")\nprint(f\"\\tShape of test set: {y_train.shape}\")\n\nX_train = X_train.values.reshape(-1,1,28,28)\nX_test = df_test.values.reshape(-1,1,28,28)\n\nprint(\"\\nAfter Reshape\")\nprint(f\"\\tShape of training set: {X_train.shape}\")\nprint(f\"\\tShape of test set: {X_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:41:14.193602Z","iopub.execute_input":"2022-08-06T06:41:14.194580Z","iopub.status.idle":"2022-08-06T06:41:14.290530Z","shell.execute_reply.started":"2022-08-06T06:41:14.194533Z","shell.execute_reply":"2022-08-06T06:41:14.289412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_tensor = torch.tensor(X_train)/255.0\nX_test_tensor = torch.tensor(X_test)/255.0\ny_train_tensor = torch.tensor(y_train.values)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T06:41:14.293444Z","iopub.execute_input":"2022-08-06T06:41:14.293781Z","iopub.status.idle":"2022-08-06T06:41:14.929723Z","shell.execute_reply.started":"2022-08-06T06:41:14.293752Z","shell.execute_reply":"2022-08-06T06:41:14.928760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n## 3.2 | CNN Model","metadata":{}},{"cell_type":"markdown","source":"* [Conv2d](https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html)\n* [ReLU](https://pytorch.org/docs/stable/generated/torch.nn.ReLU.html)\n* [MaxPool2d](https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html)\n* [AdaptiveAvgPool2d](https://pytorch.org/docs/stable/generated/torch.nn.AdaptiveAvgPool2d.html)\n* [Linear](https://pytorch.org/docs/stable/generated/torch.nn.Linear.html)\n* [Dropout](https://pytorch.org/docs/stable/generated/torch.nn.Dropout.html)\n* [LogSoftmax](https://pytorch.org/docs/stable/generated/torch.nn.LogSoftmax.html)","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n def __init__(self, output_size = 10, dropout=0.5):\n super(CNN, self).__init__()\n self.conv1 = nn.Sequential(\n # dimension: (1 X 28 X 28)\n nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 3, stride = 1, padding=1),\n # dimension: (32 X 28 X 28)\n nn.ReLU(inplace=True),\n )\n \n self.block1 = nn.Sequential(\n # dimension: (32 X 28 X 28)\n nn.Conv2d(in_channels = 32, out_channels = 32, kernel_size = 3, stride = 1, padding=1),\n # dimension: (32 X 28 X 28)\n nn.ReLU(inplace=True),\n )\n \n self.block2 = nn.Sequential(\n # dimension: (32 X 28 X 28)\n nn.Conv2d(in_channels = 32, out_channels = 32, kernel_size = 3, stride = 1, padding=1),\n # dimension: (32 X 28 X 28)\n nn.ReLU(inplace=True),\n # dimension: (32 X 28 X 28)\n nn.MaxPool2d(kernel_size=2, stride=2),\n )\n \n self.block3 = nn.Sequential(\n # dimension: (32 X 14 X 14)\n nn.Conv2d(in_channels = 32, out_channels = 32, kernel_size = 3, stride = 1, padding=1),\n # dimension: (32 X 14 X 14)\n nn.ReLU(inplace=True),\n )\n \n self.block4 = nn.Sequential(\n # dimension: (32 X 14 X 14)\n nn.Conv2d(in_channels = 32, out_channels = 32, kernel_size = 3, stride = 1, padding=1),\n # dimension: (32 X 14 X 14)\n nn.ReLU(inplace=True),\n )\n \n self.conv2 = nn.Sequential(\n # dimension: (32 X 14 X 14)\n nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size=3, stride=1, padding=1),\n nn.ReLU(inplace=True),\n # dimension: (64 X 14 X 14)\n nn.MaxPool2d(kernel_size=2, stride=2),\n # dimension: (64 X 7 X 7)\n \n nn.Conv2d(in_channels = 64, out_channels = 128, kernel_size=3, stride=1, padding=1),\n nn.ReLU(inplace=True),\n # dimension: (128 X 7 X 7)\n \n nn.Conv2d(in_channels = 128, out_channels = 128, kernel_size=3, stride=1, padding=1),\n nn.ReLU(inplace=True),\n # dimension: (128 X 7 X 7)\n \n nn.Conv2d(in_channels = 128, out_channels = 128, kernel_size=3, stride=1, padding=1),\n nn.ReLU(inplace=True),\n )\n \n self.avg_pool = nn.AdaptiveAvgPool2d(output_size=(6, 6))\n \n self.fc = nn.Sequential(\n nn.Linear(in_features = 128 * 6 * 6, out_features = 512),\n nn.ReLU(inplace=True),\n nn.Dropout(p = dropout),\n \n nn.Linear(in_features = 512, out_features = output_size),\n nn.LogSoftmax(dim=1)\n )\n \n def forward(self, x):\n x1 = self.conv1(x)\n x2 = self.block1(x1)\n x3 = self.block2(x1 + x2)\n x4 = self.block3(x3)\n x5 = self.block4(x3 + x4)\n x6 = self.conv2(x5)\n x7 = self.avg_pool(x6)\n x8 = torch.flatten(x7, 1)\n x9 = self.fc(x8)\n return x9","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:04.281899Z","iopub.execute_input":"2022-08-06T07:29:04.282401Z","iopub.status.idle":"2022-08-06T07:29:04.300333Z","shell.execute_reply.started":"2022-08-06T07:29:04.282368Z","shell.execute_reply":"2022-08-06T07:29:04.299186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n## 3.3 | Helper Function Definition","metadata":{}},{"cell_type":"code","source":"def validation(model, loader, criterion, device=\"cpu\"):\n model.eval()\n loss = 0\n acc = 0\n \n with torch.no_grad():\n for X, y in loader:\n X, y = X.to(device), y.to(device)\n output = model.forward(X)\n loss += criterion(output, y).item()\n probs = torch.exp(output)\n equality = (y.data == probs.max(dim=1)[1])\n acc += equality.type(torch.FloatTensor).mean()\n \n res_loss = loss/len(loader)\n res_acc = (acc.item())/len(loader)\n return res_loss, res_acc","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:04.657392Z","iopub.execute_input":"2022-08-06T07:29:04.658646Z","iopub.status.idle":"2022-08-06T07:29:04.667048Z","shell.execute_reply.started":"2022-08-06T07:29:04.658578Z","shell.execute_reply":"2022-08-06T07:29:04.665953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, trainloader, validloader, \n criterion, optimizer, scheduler, \n epochs = 10, print_every = 5, \n device=\"cpu\", verbose = False):\n \n model.to(device)\n best_acc = 0\n best_epoch = 0\n \n memory = {}\n \n for e in range(epochs):\n model.train() \n for X, y in trainloader:\n X, y = X.to(device), y.to(device)\n optimizer.zero_grad()\n loss = criterion(model.forward(X), y)\n loss.backward()\n optimizer.step()\n scheduler.step()\n\n model.eval()\n train_loss, train_accuracy = validation(model, trainloader, criterion, device)\n valid_loss, valid_accuracy = validation(model, validloader, criterion, device)\n \n memory[e] = {\n \"train_loss\" : train_loss, \n \"valid_loss\" : valid_loss, \n \"valid_accuracy\" : valid_accuracy\n }\n \n if valid_accuracy > best_acc:\n best_acc = valid_accuracy\n best_epoch = e\n torch.save(model.state_dict(), \"model.pt\")\n \n if (e + 1) % print_every == 0 and verbose:\n to_print = f\"\\u001b[1mEpoch\\u001b[0m: {str(e+1)} of {str(epochs)}\\\n \\n\\tTrain Loss: {train_loss:.4f}\\\n \\n\\tValid Loss: {valid_loss:.4f}\\\n \\n\\tValid Accuracy: {valid_accuracy:.3f}\"\n print(to_print)\n \n to_print = \"\\n\\t\\u001b[32mTraining completed\\u001b[0m\\n\"\n to_print += f\"\\tBest Valid Acc is: \\u001b[1m{best_acc:.4f}\\u001b[0m after \\u001b[1m{best_epoch+1}\\u001b[0m epochs\\n\"\n print(to_print)\n \n model.load_state_dict(torch.load(\"model.pt\"))\n \n return model, memory","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:05.195375Z","iopub.execute_input":"2022-08-06T07:29:05.195719Z","iopub.status.idle":"2022-08-06T07:29:05.206901Z","shell.execute_reply.started":"2022-08-06T07:29:05.195689Z","shell.execute_reply":"2022-08-06T07:29:05.205775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction(model, loader, device=\"cpu\"):\n model.to(device)\n model.eval()\n preds_all = torch.LongTensor()\n \n with torch.no_grad():\n for X in loader:\n X = X.to(device) \n probs = torch.exp(model.forward(X))\n pred = probs.to('cpu').max(dim=1)[1]\n preds_all = torch.cat((preds_all, pred), dim=0)\n return preds_all","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:05.480908Z","iopub.execute_input":"2022-08-06T07:29:05.481893Z","iopub.status.idle":"2022-08-06T07:29:05.488649Z","shell.execute_reply.started":"2022-08-06T07:29:05.481829Z","shell.execute_reply":"2022-08-06T07:29:05.487665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_state_memory(state_memory):\n for (i, fold) in state_memory.items():\n fig, (ax1, ax2, ax3) = plt.subplots(ncols = 3, figsize = (6*3,4))\n train_loss = []\n valid_loss = []\n valid_accuracy = []\n\n plt.tight_layout(pad=5)\n for (j, v) in fold.items():\n train_loss.append(v[\"train_loss\"])\n valid_loss.append(v[\"valid_loss\"])\n valid_accuracy.append(v[\"valid_accuracy\"])\n\n ax1.set_title(f\"train_loss fold : {i}\")\n ax2.set_title(f\"valid_loss fold : {i}\")\n ax3.set_title(f\"valid_accuracy fold : {i}\")\n\n ax1.plot(train_loss)\n ax2.plot(valid_loss)\n ax3.plot(valid_accuracy)\n \n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:05.903867Z","iopub.execute_input":"2022-08-06T07:29:05.904732Z","iopub.status.idle":"2022-08-06T07:29:05.915749Z","shell.execute_reply.started":"2022-08-06T07:29:05.904698Z","shell.execute_reply":"2022-08-06T07:29:05.910906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n## 3.4 | Training the CNN Model","metadata":{}},{"cell_type":"markdown","source":"* [CrossEntropyLoss]()\n* [Adam]()\n* [OneCycleLR]()","metadata":{}},{"cell_type":"code","source":"model = CNN(output_size=10, dropout=0.5)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:06.893824Z","iopub.execute_input":"2022-08-06T07:29:06.894209Z","iopub.status.idle":"2022-08-06T07:29:06.926469Z","shell.execute_reply.started":"2022-08-06T07:29:06.894179Z","shell.execute_reply":"2022-08-06T07:29:06.925392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LEARNING_RATE = 0.001\nN_EPOCHS = 10\nKFOLD_N_SPLIT = 10\nTRAIN_BATCH_SIZE = 32\nVALID_BATCH_SIZE = 16\nSTEPS_PER_EPOCH = np.int(np.ceil(len(X_train_tensor)*(1-1/KFOLD_N_SPLIT)/TRAIN_BATCH_SIZE))\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr = MAX_LEARNING_RATE)\nscheduler = optim.lr_scheduler.OneCycleLR(optimizer,\n max_lr = MAX_LEARNING_RATE,\n epochs = N_EPOCHS * KFOLD_N_SPLIT,\n steps_per_epoch = STEPS_PER_EPOCH,\n pct_start = 0.2,\n anneal_strategy = \"cos\")\n\nif torch.cuda.is_available():\n device = \"cuda\"\nelse:\n device = \"cpu\"\n \nprint(f\"GPU is {'enabled' if device == 'cuda' else 'disabled'}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:07.142640Z","iopub.execute_input":"2022-08-06T07:29:07.143209Z","iopub.status.idle":"2022-08-06T07:29:07.153633Z","shell.execute_reply.started":"2022-08-06T07:29:07.143173Z","shell.execute_reply":"2022-08-06T07:29:07.150941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedKFold(n_splits=KFOLD_N_SPLIT, shuffle=True, random_state = 0xBEEF)\n\nstate_memory = {}\n\nfor fold, (train_ind, val_ind) in enumerate(cv.split(X_train_tensor, y_train_tensor)):\n train_X_cv, val_X_cv = X_train_tensor[train_ind], X_train_tensor[val_ind]\n train_y_cv, val_y_cv = y_train_tensor[train_ind], y_train_tensor[val_ind]\n \n train_tensor = TensorDataset(train_X_cv, train_y_cv)\n valid_tensor = TensorDataset(val_X_cv, val_y_cv)\n \n train_loader = DataLoader(train_tensor, batch_size=TRAIN_BATCH_SIZE, shuffle=True)\n valid_loader = DataLoader(valid_tensor, batch_size=VALID_BATCH_SIZE)\n \n if fold % 3 == 0: print(f\"\\n--- Fold: {fold+1} of {cv.n_splits} ---\\n\")\n _, memory = train_model(\n model = model, \n trainloader = train_loader,\n validloader = valid_loader, \n criterion = criterion,\n optimizer = optimizer, \n scheduler = scheduler,\n epochs = N_EPOCHS, \n print_every = 5, \n device = device,\n verbose = fold % 3 == 0\n )\n state_memory[fold+1] = memory","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:07.901514Z","iopub.execute_input":"2022-08-06T07:29:07.901881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n## 3.5 | Shap Deep Explainer","metadata":{}},{"cell_type":"code","source":"%%capture\ntest_loader = DataLoader(TensorDataset(X_train_tensor.cpu(), y_train_tensor.cpu()), batch_size=128, shuffle=False)\n\nbatch = next(iter(test_loader))\nimages, _ = batch\n\nbackground = images[:100]\ntest_images= images[100:110]\n\ne = shap.DeepExplainer(model.cpu(), images)\nshap_values = e.shap_values(test_images)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:03:32.267988Z","iopub.execute_input":"2022-08-04T17:03:32.268595Z","iopub.status.idle":"2022-08-04T17:03:43.434961Z","shell.execute_reply.started":"2022-08-04T17:03:32.268555Z","shell.execute_reply":"2022-08-04T17:03:43.433942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap_numpy = [np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values]\ntest_numpy = np.swapaxes(np.swapaxes(test_images.numpy(), 1, -1), 1, 2)\n\nshap.image_plot(shap_numpy, -test_numpy)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:03:43.436637Z","iopub.execute_input":"2022-08-04T17:03:43.437044Z","iopub.status.idle":"2022-08-04T17:03:47.151492Z","shell.execute_reply.started":"2022-08-04T17:03:43.437008Z","shell.execute_reply":"2022-08-04T17:03:47.150453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n## 3.6 | Submission","metadata":{}},{"cell_type":"code","source":"y_pred = prediction(model, DataLoader(X_test_tensor, batch_size=16))\n\ndf_test = pd.read_csv(\"../input/digit-recognizer/test.csv\")\ndf_result = pd.DataFrame({'ImageId': df_test.index, 'Label': y_pred})\ndf_result[\"ImageId\"] += 1\ndf_result.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:03:47.153061Z","iopub.execute_input":"2022-08-04T17:03:47.153416Z","iopub.status.idle":"2022-08-04T17:03:52.858613Z","shell.execute_reply.started":"2022-08-04T17:03:47.153381Z","shell.execute_reply":"2022-08-04T17:03:52.857715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices = [22, 54, 44, 55]\n_, ax = plt.subplots(1, len(indices), figsize = (14,8))\nfor i, v in enumerate(indices):\n title = f\"Label: {int(y_pred[v])}\"\n ax[i].imshow(X_test_tensor[v, 0] )\n ax[i].set_title(title)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:04:36.932767Z","iopub.execute_input":"2022-08-04T17:04:36.933761Z","iopub.status.idle":"2022-08-04T17:04:37.316712Z","shell.execute_reply.started":"2022-08-04T17:04:36.933707Z","shell.execute_reply":"2022-08-04T17:04:37.315762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:03:53.280509Z","iopub.execute_input":"2022-08-04T17:03:53.280996Z","iopub.status.idle":"2022-08-04T17:03:53.311865Z","shell.execute_reply.started":"2022-08-04T17:03:53.280957Z","shell.execute_reply":"2022-08-04T17:03:53.310998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n\n********\n\n# 4 | Final Thoughts","metadata":{}},{"cell_type":"code","source":"show_state_memory(state_memory)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:06:11.002946Z","iopub.execute_input":"2022-08-04T17:06:11.003493Z","iopub.status.idle":"2022-08-04T17:06:15.5589Z","shell.execute_reply.started":"2022-08-04T17:06:11.003453Z","shell.execute_reply":"2022-08-04T17:06:15.557866Z"},"trusted":true},"execution_count":null,"outputs":[]}]}