{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport os\nimport random\nimport json\nimport glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-15T10:30:01.079306Z","iopub.execute_input":"2023-03-15T10:30:01.080970Z","iopub.status.idle":"2023-03-15T10:30:11.706217Z","shell.execute_reply.started":"2023-03-15T10:30:01.080930Z","shell.execute_reply":"2023-03-15T10:30:11.705002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# setting up paths and commmon items","metadata":{}},{"cell_type":"code","source":"input_dir = \"/kaggle/input/asl-signs/train_landmark_files\"\ntrain_df = pd.read_csv(\"/kaggle/input/asl-signs/train.csv\")\nparticipants = os.listdir(\"/kaggle/input/asl-signs/train_landmark_files\")\nprint(f\"the number of participants:\",len(participants))\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:11.708720Z","iopub.execute_input":"2023-03-15T10:30:11.709572Z","iopub.status.idle":"2023-03-15T10:30:11.919272Z","shell.execute_reply.started":"2023-03-15T10:30:11.709525Z","shell.execute_reply":"2023-03-15T10:30:11.918205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining class->index and index->class","metadata":{}},{"cell_type":"code","source":"def read_json(path):\n with open(path, \"r\") as file:\n json_data = json.load(file)\n return json_data\n\nsign_languages = read_json(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\")\nclass2idx = {}\nidx2class = {}\n\nfor values in sign_languages.items():\n class2idx.update({values[0]:values[1]})\n idx2class.update({values[1]:values[0]})","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:11.921017Z","iopub.execute_input":"2023-03-15T10:30:11.921789Z","iopub.status.idle":"2023-03-15T10:30:11.932064Z","shell.execute_reply.started":"2023-03-15T10:30:11.921718Z","shell.execute_reply":"2023-03-15T10:30:11.930987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Praquet files is generally used in the Hadoop system env for storing ","metadata":{}},{"cell_type":"code","source":"sample_path =\"/kaggle/input/asl-signs/\"+ train_df.path[1]\n\nsample = pd.read_parquet(sample_path)\nsample\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:11.936447Z","iopub.execute_input":"2023-03-15T10:30:11.937258Z","iopub.status.idle":"2023-03-15T10:30:12.034618Z","shell.execute_reply.started":"2023-03-15T10:30:11.937209Z","shell.execute_reply":"2023-03-15T10:30:12.033181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Key takeways\n\n### **Number of frames and number of rows in the praquel files**: is actually varibale across all files so to say\n\n### X,Y,X coordinates of the \"Type\" \n","metadata":{}},{"cell_type":"code","source":"print(f\"# null values: {sample.isnull().sum().sum()}\")\nprint(f\"# rows: {len(sample)}, # rows without duplicates: \"\n f\"{len(sample.drop_duplicates())}\")\nsample","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.036397Z","iopub.execute_input":"2023-03-15T10:30:12.037228Z","iopub.status.idle":"2023-03-15T10:30:12.064981Z","shell.execute_reply.started":"2023-03-15T10:30:12.037183Z","shell.execute_reply":"2023-03-15T10:30:12.063878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A Frame can have many different types of classes for eg","metadata":{}},{"cell_type":"code","source":"for frames in sample.frame.unique():\n classes =set(sample.loc[sample[\"frame\"]==frames,\"type\"].tolist()) # locate the frame-> take all the classes-> make set\n print(f\" Frame :{frames} ,classes :{classes}\")\n ","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.066643Z","iopub.execute_input":"2023-03-15T10:30:12.067414Z","iopub.status.idle":"2023-03-15T10:30:12.080374Z","shell.execute_reply.started":"2023-03-15T10:30:12.067374Z","shell.execute_reply":"2023-03-15T10:30:12.079247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_id = 29\nlandmark_type = \"face\"\nparquet_df_sorted_filtered = sample[\n (sample.frame == frame_id) &\n (sample.type == landmark_type)\n ].sort_values(['landmark_index'])\n\nx = list(parquet_df_sorted_filtered.x)\ny = list(parquet_df_sorted_filtered.y)\nplt.scatter(x, y, color='blue')\nplt.title(\"Face\")","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.081894Z","iopub.execute_input":"2023-03-15T10:30:12.082511Z","iopub.status.idle":"2023-03-15T10:30:12.346955Z","shell.execute_reply.started":"2023-03-15T10:30:12.082468Z","shell.execute_reply":"2023-03-15T10:30:12.345693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_id = 29\nlandmark_type = \"pose\"\nparquet_df_sorted_filtered = sample[\n (sample.frame == frame_id) &\n (sample.type == landmark_type)\n ].sort_values(['landmark_index'])\n\nx = list(parquet_df_sorted_filtered.x)\ny = list(parquet_df_sorted_filtered.y)\nplt.scatter(x, y, color='red')\n\nplt.gca().invert_yaxis() # invert the axis because matplotlib starts y axis from the bottom\nplt.gca().invert_xaxis() \nplt.title(\"Pose\")","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.349836Z","iopub.execute_input":"2023-03-15T10:30:12.350203Z","iopub.status.idle":"2023-03-15T10:30:12.586495Z","shell.execute_reply.started":"2023-03-15T10:30:12.350172Z","shell.execute_reply":"2023-03-15T10:30:12.585526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting it all together","metadata":{}},{"cell_type":"code","source":"frame_id = 29\nlandmark_type = \"face\"\nparquet_df_sorted_filtered = sample[\n (sample.frame == frame_id) &\n (sample.type == landmark_type)\n ].sort_values(['landmark_index'])\n\nx = list(parquet_df_sorted_filtered.x)\ny = list(parquet_df_sorted_filtered.y)\nplt.scatter(x, y, color='blue')\n\nframe_id = 29\nlandmark_type = \"pose\"\nparquet_df_sorted_filtered = sample[\n (sample.frame == frame_id) &\n (sample.type == landmark_type)\n ].sort_values(['landmark_index'])\n\nx = list(parquet_df_sorted_filtered.x)\ny = list(parquet_df_sorted_filtered.y)\nplt.scatter(x, y, color='red')\n\nframe_id = 29\nlandmark_type = \"left_hand\"\nparquet_df_sorted_filtered = sample[\n (sample.frame == frame_id) &\n (sample.type == landmark_type)\n ].sort_values(['landmark_index'])\n\nx = list(parquet_df_sorted_filtered.x)\ny = list(parquet_df_sorted_filtered.y)\nplt.scatter(x, y, color='green')\n\nframe_id = 29\nlandmark_type = \"right_hand\"\nparquet_df_sorted_filtered = sample[\n (sample.frame == frame_id) &\n (sample.type == landmark_type)\n ].sort_values(['landmark_index'])\n\nx = list(parquet_df_sorted_filtered.x)\ny = list(parquet_df_sorted_filtered.y)\nplt.scatter(x, y, color='black')\nplt.gca().invert_yaxis() # invert the axis because matplotlib starts y axis from the bottom\nplt.gca().invert_xaxis() ","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.588138Z","iopub.execute_input":"2023-03-15T10:30:12.588498Z","iopub.status.idle":"2023-03-15T10:30:12.866791Z","shell.execute_reply.started":"2023-03-15T10:30:12.588460Z","shell.execute_reply":"2023-03-15T10:30:12.865729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the data much further","metadata":{}},{"cell_type":"markdown","source":"# Visualizing the data for right hand","metadata":{}},{"cell_type":"code","source":"right_hand_sample = sample.loc[sample[\"type\"]==\"right_hand\"]","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.871263Z","iopub.execute_input":"2023-03-15T10:30:12.871558Z","iopub.status.idle":"2023-03-15T10:30:12.879413Z","shell.execute_reply.started":"2023-03-15T10:30:12.871525Z","shell.execute_reply":"2023-03-15T10:30:12.878445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"edges = [\n (0, 1),\n (1, 2),\n (2, 3),\n (3, 4),\n (0, 5),\n (0, 17),\n (5, 6),\n (6, 7),\n (7, 8),\n (5, 9),\n (9, 10),\n (10, 11),\n (11, 12),\n (9, 13),\n (13, 14),\n (14, 15),\n (15, 16),\n (13, 17),\n (17, 18),\n (18, 19),\n (19, 20),\n]","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.880857Z","iopub.execute_input":"2023-03-15T10:30:12.881372Z","iopub.status.idle":"2023-03-15T10:30:12.890958Z","shell.execute_reply.started":"2023-03-15T10:30:12.881334Z","shell.execute_reply":"2023-03-15T10:30:12.889920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_frame(df, frame_id, ax):\n df = df[df.frame == frame_id].sort_values([\"landmark_index\"])\n ax.scatter(df.x, df.y, color=\"dodgerblue\")\n for i in range(len(x)):\n ax.text(x[i], y[i], str(i))\n\n for edge in edges:\n ax.plot([x[edge[0]], x[edge[1]]], [y[edge[0]], y[edge[1]]], color=\"salmon\")\n ax.set_title(f\"Frame no. {frame_id}\")\n ax.axis(False)\n \ndef plot_frame_seq(df, frame_id_range, n_frames):\n frames = np.linspace(\n frame_id_range[0], frame_id_range[1], n_frames, dtype=int, endpoint=True\n )\n fig, ax = plt.subplots(n_frames, 1, figsize=(5, 25))\n for i in range(n_frames):\n plot_frame(df, frames[i], ax[i])\n\n plt.show()\n \n \nplot_frame_seq(right_hand_sample, (20, 40), 6) # take 1 frame out of 4","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:12.892265Z","iopub.execute_input":"2023-03-15T10:30:12.892570Z","iopub.status.idle":"2023-03-15T10:30:13.559376Z","shell.execute_reply.started":"2023-03-15T10:30:12.892526Z","shell.execute_reply":"2023-03-15T10:30:13.558284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding the sign !","metadata":{}},{"cell_type":"code","source":"# Let's take the example of the user 16069\n\ntrain_16069 = train_df.loc[train_df[\"participant_id\"] == 16069]\nprint(f\"unique signs done by this participant: {train_16069.sign.unique()[:10]}\")\ntrain_16069\n\n# as we can see , that the participant has done many signs including [\"cloud\",\"another\",\"green\",\"bug\",\"noose\"]","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.560550Z","iopub.execute_input":"2023-03-15T10:30:13.561428Z","iopub.status.idle":"2023-03-15T10:30:13.582217Z","shell.execute_reply.started":"2023-03-15T10:30:13.561388Z","shell.execute_reply":"2023-03-15T10:30:13.581144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"# unique sequence ids: {len(np.unique(train_df.sequence_id.values))}\\n\"\n f\"# rows train: {len(train_df)}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.583797Z","iopub.execute_input":"2023-03-15T10:30:13.584184Z","iopub.status.idle":"2023-03-15T10:30:13.594308Z","shell.execute_reply.started":"2023-03-15T10:30:13.584146Z","shell.execute_reply":"2023-03-15T10:30:13.593094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TO Understand , How the model will go through the data is \n\n## 1. In every praquel file , there is a **left hand** , **right hand** , **face** moving thorugh space(X,Y,Z) and time( Frames)\n\n## 2. we need to design a model , which takes the sequences of frames 1-N take the x,y,z axis and give us the sign","metadata":{}},{"cell_type":"code","source":"sample.head()\n\nprint(\"len of sample is :\", len(sample))\nprint(\"the number of frames are :\",len(sample[\"frame\"].unique()))\nprint(\"so on an average the number of samples (x,y,z,landmark) for a given frame is :\", (5973/11))","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.596339Z","iopub.execute_input":"2023-03-15T10:30:13.596821Z","iopub.status.idle":"2023-03-15T10:30:13.604120Z","shell.execute_reply.started":"2023-03-15T10:30:13.596784Z","shell.execute_reply":"2023-03-15T10:30:13.602801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543 # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.605831Z","iopub.execute_input":"2023-03-15T10:30:13.606233Z","iopub.status.idle":"2023-03-15T10:30:13.613595Z","shell.execute_reply.started":"2023-03-15T10:30:13.606196Z","shell.execute_reply":"2023-03-15T10:30:13.612322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's just continue with the sample that we have chosen and explore more\n\n## We will talk about single frame , single batch\n\n1. till rows [0-468] --face values\n2. from rows [468-489] --left hand\n3. From rows [489-522] -- pose values\n4. rows [522:] -- right hand\n","metadata":{}},{"cell_type":"code","source":"sample_single_frame = sample[sample[\"frame\"]==29]","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.615262Z","iopub.execute_input":"2023-03-15T10:30:13.615717Z","iopub.status.idle":"2023-03-15T10:30:13.626256Z","shell.execute_reply.started":"2023-03-15T10:30:13.615679Z","shell.execute_reply":"2023-03-15T10:30:13.625171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding the Preprocessing\n\n1. We will take for example , a single batch , and a single frame !\n\n2. what we are doing is we are taking face values (468,3) -> (468 x 3) | with Batch (Batch_size,468,3) ->(Batch_size,468 x 3)\n\n3. Similarly for other things ","metadata":{}},{"cell_type":"code","source":"sample_single_frame.isna().sum()\n# sample_single_frame=\n# I would say the besrt way to fill up the nan values would be the avg of top and bottom values -->>\n","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.627899Z","iopub.execute_input":"2023-03-15T10:30:13.628281Z","iopub.status.idle":"2023-03-15T10:30:13.639442Z","shell.execute_reply.started":"2023-03-15T10:30:13.628246Z","shell.execute_reply":"2023-03-15T10:30:13.638258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For a single samle","metadata":{}},{"cell_type":"code","source":"# For a single batch\n# for a single frame !!\n# x = torch.tensor(sample.copy(deep=True)[[\"x\",\"y\",\"z\"]])\nx = sample_single_frame.copy(deep=True)\n\n\n# x[:469]\n\nx = torch.tensor(np.array(x[[\"x\",\"y\",\"z\"]]))\nprint(len(x[522:,:]))\n\nprint(f\"Face shape is: {x[:468,:].shape}\")\nface_x = x[:468,:].contiguous().view(-1, 468*3)\nlefth_x = x[468:489,:].contiguous().view(-1, 21*3)\npose_x = x[489:522,:].contiguous().view(-1, 33*3)\nrighth_x = x[522:,:].contiguous().view( -1,21*3)\n\nprint(\"face_x after transformation:\",face_x.shape)\n\nprint(f\"length of lefth_x before removing items is : {lefth_x.shape}\")\nprint(f\"lenght of righth_x before removing items is :{righth_x.shape}\\n\\n\")\n\n\nprint(f\"how many nan values are there\",~torch.any(torch.isnan(lefth_x), dim=0))\nprint(\" how many nan values are there \",~torch.any(torch.isnan(righth_x), dim=0))\n\n\nlefth_x = lefth_x[~torch.any(torch.isnan(lefth_x), dim=1),:]\nrighth_x = righth_x[~torch.any(torch.isnan(righth_x), dim=1),:]\n\nprint(f\"\\n\\nlength of lefth_x after removing items is : {lefth_x.shape}\")\nprint(f\"lenght of righth_x after removing items is :{righth_x.shape}\")\n\nx1m = torch.mean(face_x, 0) # mean along the 0th axis\nx2m = torch.mean(lefth_x, 0)\nx3m = torch.mean(pose_x, 0)\nx4m = torch.mean(righth_x, 0)\n\nx1s = torch.std(face_x, 0) # std along the 0th axix\nx2s = torch.std(lefth_x, 0)\nx3s = torch.std(pose_x, 0)\nx4s = torch.std(righth_x, 0)\n\n\nxfeat = torch.cat([x1m,x2m,x3m,x4m, x1s,x2s,x3s,x4s], axis=0) # concatenate along the 0th axis\nxfeat = torch.where(torch.isnan(xfeat), torch.tensor(0.0, dtype=torch.float32), xfeat) # replace nan ->0\n\nprint(\"the final x features shape is :\",xfeat.shape) # it would actually be (frames,3258) for multi frames","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.640705Z","iopub.execute_input":"2023-03-15T10:30:13.641445Z","iopub.status.idle":"2023-03-15T10:30:13.732001Z","shell.execute_reply.started":"2023-03-15T10:30:13.641410Z","shell.execute_reply":"2023-03-15T10:30:13.730054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch \nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.733693Z","iopub.execute_input":"2023-03-15T10:30:13.734079Z","iopub.status.idle":"2023-03-15T10:30:13.739346Z","shell.execute_reply.started":"2023-03-15T10:30:13.734040Z","shell.execute_reply":"2023-03-15T10:30:13.738152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureGeneration(nn.Module):\n \n def __init__(self):\n super().__init__()\n \n \n def forward(self,x):\n \n face_x = x[:,:468,:].contiguous().view(-1, 468*3)\n lefth_x = x[:,468:489,:].contiguous().view(-1, 21*3)\n pose_x = x[:,489:522,:].contiguous().view(-1, 33*3)\n righth_x = x[:,522:,:].contiguous().view(-1, 21*3)\n \n lefth_x = lefth_x[~torch.any(torch.isnan(lefth_x), dim=1),:]\n righth_x = righth_x[~torch.any(torch.isnan(righth_x), dim=1),:]\n \n x1m = torch.mean(face_x, 0)\n x2m = torch.mean(lefth_x, 0)\n x3m = torch.mean(pose_x, 0)\n x4m = torch.mean(righth_x, 0)\n \n x1s = torch.std(face_x, 0)\n x2s = torch.std(lefth_x, 0)\n x3s = torch.std(pose_x, 0)\n x4s = torch.std(righth_x, 0)\n \n xfeat = torch.cat([x1m,x2m,x3m,x4m, x1s,x2s,x3s,x4s], axis=0)\n xfeat = torch.where(torch.isnan(xfeat), torch.tensor(0.0, dtype=torch.float32), xfeat)\n \n return xfeat\nfeature_converter = FeatureGeneration()","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.741394Z","iopub.execute_input":"2023-03-15T10:30:13.741803Z","iopub.status.idle":"2023-03-15T10:30:13.754051Z","shell.execute_reply.started":"2023-03-15T10:30:13.741710Z","shell.execute_reply":"2023-03-15T10:30:13.752884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.755679Z","iopub.execute_input":"2023-03-15T10:30:13.756261Z","iopub.status.idle":"2023-03-15T10:30:13.767286Z","shell.execute_reply.started":"2023-03-15T10:30:13.756218Z","shell.execute_reply":"2023-03-15T10:30:13.766303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datax = np.load(\"/kaggle/input/gislr-feature-data/feature_data.npy\")\n# datay = np.load(\"/kaggle/input/gislr-feature-data/feature_labels.npy\")\n","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.768970Z","iopub.execute_input":"2023-03-15T10:30:13.769328Z","iopub.status.idle":"2023-03-15T10:30:13.776604Z","shell.execute_reply.started":"2023-03-15T10:30:13.769288Z","shell.execute_reply":"2023-03-15T10:30:13.775536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ASLModel(nn.Module):\n def __init__(self, p):\n super(ASLModel, self).__init__()\n self.dropout = nn.Dropout(p)\n self.layer0 = nn.Linear(3258, 1024)\n self.layer1 = nn.Linear(1024, 512)\n self.layer2 = nn.Linear(512, 250)\n \n def forward(self, x):\n x = self.layer0(x)\n x = self.dropout(x)\n x = self.layer1(x)\n x = self.layer2(x)\n return x","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.778052Z","iopub.execute_input":"2023-03-15T10:30:13.778891Z","iopub.status.idle":"2023-03-15T10:30:13.786992Z","shell.execute_reply.started":"2023-03-15T10:30:13.778855Z","shell.execute_reply":"2023-03-15T10:30:13.785789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_and_save_data():\n np_features = np.zeros((data_lenght_experiment, ROWS_PER_FRAME, 3))\n np_labels = np.zeros(data_lenght_experiment)\n\n print(f\"Total data to processe : {data_lenght_experiment}\")\n for index, row in tqdm(train_df.iterrows()):\n if index > data_lenght_experiment - 1:\n break\n\n data = load_relevant_data_subset(row.path)\n feature, label = convert_row(row)\n np_features[index, :, :] = feature\n np_labels[index] = label\n\n np.save(\"features.npy\", np_features)\n np.save(\"labels.npy\", np_labels)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.788416Z","iopub.execute_input":"2023-03-15T10:30:13.788929Z","iopub.status.idle":"2023-03-15T10:30:13.797691Z","shell.execute_reply.started":"2023-03-15T10:30:13.788892Z","shell.execute_reply":"2023-03-15T10:30:13.796785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543 # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.799313Z","iopub.execute_input":"2023-03-15T10:30:13.799815Z","iopub.status.idle":"2023-03-15T10:30:13.810562Z","shell.execute_reply.started":"2023-03-15T10:30:13.799779Z","shell.execute_reply":"2023-03-15T10:30:13.809863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_to_experiment = load_relevant_data_subset(\"/kaggle/input/asl-signs/train_landmark_files/16069/100015657.parquet\")\ndata_to_experiment.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:30:13.812093Z","iopub.execute_input":"2023-03-15T10:30:13.812761Z","iopub.status.idle":"2023-03-15T10:30:13.850678Z","shell.execute_reply.started":"2023-03-15T10:30:13.812705Z","shell.execute_reply":"2023-03-15T10:30:13.849524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import multiprocessing as mp\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\ndef convert_row(row):\n x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row[1].path))\n x = feature_converter(torch.tensor(x)).cpu().numpy()\n return x, row[1].label\n\ndef convert_and_save_data():\n df = pd.read_csv(TRAIN_FILE)\n df['label'] = df['sign'].map(class2idx)\n npdata = np.zeros((df.shape[0], 3258))\n nplabels = np.zeros(df.shape[0])\n with mp.Pool() as pool:\n results = pool.imap(convert_row, df.iterrows(), chunksize=1000)\n for i, (x,y) in tqdm(enumerate(results), total=df.shape[0]):\n npdata[i,:] = x\n nplabels[i] = y\n \n np.save(\"feature_data.npy\", npdata)\n np.save(\"feature_labels.npy\", nplabels)\n \nconvert_and_save_data()","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:32:05.833948Z","iopub.execute_input":"2023-03-15T10:32:05.834417Z","iopub.status.idle":"2023-03-15T10:46:50.104959Z","shell.execute_reply.started":"2023-03-15T10:32:05.834371Z","shell.execute_reply":"2023-03-15T10:46:50.103774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_data_x = np.load(\"/kaggle/working/feature_data.npy\")\nfinal_data_y = np.load(\"/kaggle/working/feature_labels.npy\")\n\nfinal_data_x.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:47:00.184798Z","iopub.execute_input":"2023-03-15T10:47:00.185187Z","iopub.status.idle":"2023-03-15T10:47:03.764143Z","shell.execute_reply.started":"2023-03-15T10:47:00.185146Z","shell.execute_reply":"2023-03-15T10:47:03.763148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ASLModel(nn.Module):\n def __init__(self, p):\n super(ASLModel, self).__init__()\n self.dropout = nn.Dropout(p)\n self.layer0 = nn.Linear(3258, 1024)\n self.layer1 = nn.Linear(1024, 512)\n self.layer2 = nn.Linear(512, 250)\n \n def forward(self, x):\n x = self.layer0(x)\n x = self.dropout(x)\n x = self.layer1(x)\n x = self.layer2(x)\n return x","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:47:04.624048Z","iopub.execute_input":"2023-03-15T10:47:04.625007Z","iopub.status.idle":"2023-03-15T10:47:04.634328Z","shell.execute_reply.started":"2023-03-15T10:47:04.624950Z","shell.execute_reply":"2023-03-15T10:47:04.632393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ASLData(Dataset):\n def __init__(self, datax, datay):\n self.datax = datax\n self.datay = datay\n \n def __getitem__(self, index):\n return self.datax[index,:], self.datay[index]\n \n def __len__(self):\n return len(self.datay)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:47:05.799184Z","iopub.execute_input":"2023-03-15T10:47:05.800208Z","iopub.status.idle":"2023-03-15T10:47:05.807266Z","shell.execute_reply.started":"2023-03-15T10:47:05.800148Z","shell.execute_reply":"2023-03-15T10:47:05.805870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 50\nBATCH_SIZE = 64\n\ntrainx, testx, trainy, testy = train_test_split(final_data_x, final_data_y, test_size=0.15, random_state=42)\n\ntrain_data = ASLData(trainx, trainy)\nvalid_data = ASLData(testx, testy)\n\ntrain_loader = DataLoader(train_data, batch_size=BATCH_SIZE, num_workers=2, shuffle=True,pin_memory=True)\nval_loader = DataLoader(valid_data, batch_size=BATCH_SIZE, num_workers=2, shuffle=False,pin_memory=True)\n\nmodel = ASLModel(0.2).cuda()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.005)\ncriterion = nn.CrossEntropyLoss()\nsched = torch.optim.lr_scheduler.StepLR(optimizer, step_size=300, gamma=0.95)\n\nfor i in range(EPOCHS):\n model.train()\n \n train_loss_sum = 0.\n train_correct = 0\n train_total = 0\n train_bar = train_loader\n for x,y in train_bar:\n x = torch.Tensor(x).float().cuda()\n y = torch.Tensor(y).long().cuda() \n y_pred = model(x)\n# optimizer.zero_grad()\n optimizer.zero_grad(set_to_none=True)\n loss = criterion(y_pred, y)\n loss.backward()\n optimizer.step()\n \n \n train_loss_sum += loss.item()\n train_correct += np.sum((np.argmax(y_pred.detach().cpu().numpy(), axis=1) == y.cpu().numpy()))\n train_total += 1\n sched.step()\n \n val_loss_sum = 0.\n val_correct = 0\n val_total = 0\n model.eval()\n for x,y in val_loader:\n x = torch.Tensor(x).float().cuda()\n y = torch.Tensor(y).long().cuda()\n \n with torch.no_grad():\n y_pred = model(x)\n loss = criterion(y_pred, y)\n val_loss_sum += loss.item()\n val_correct += np.sum((np.argmax(y_pred.cpu().numpy(), axis=1) == y.cpu().numpy()))\n val_total += 1\n \n print(f\"Epoch:{i} > Train Loss: {(train_loss_sum/train_total):.04f}, Train Acc: {train_correct/len(train_data):0.04f}\")\n print(f\"Epoch:{i} > Val Loss: {(val_loss_sum/val_total):.04f}, Val Acc: {val_correct/len(valid_data):0.04f}\")\n print(\"=\"*50)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T10:53:17.768359Z","iopub.execute_input":"2023-03-15T10:53:17.768819Z","iopub.status.idle":"2023-03-15T11:02:42.927106Z","shell.execute_reply.started":"2023-03-15T10:53:17.768764Z","shell.execute_reply":"2023-03-15T11:02:42.925143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\ngc.collect() # garbage collector","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:03:04.635135Z","iopub.execute_input":"2023-03-15T11:03:04.636175Z","iopub.status.idle":"2023-03-15T11:03:05.025998Z","shell.execute_reply.started":"2023-03-15T11:03:04.636131Z","shell.execute_reply":"2023-03-15T11:03:05.024790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now we will convert this model to tensorflow model\n\nI am not that good with the conversion , SO , I'll just gatther it from outside sources","metadata":{}},{"cell_type":"code","source":"!pip install onnx-tf\n!pip install tflite-runtime","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:03:08.914712Z","iopub.execute_input":"2023-03-15T11:03:08.915806Z","iopub.status.idle":"2023-03-15T11:03:32.240996Z","shell.execute_reply.started":"2023-03-15T11:03:08.915751Z","shell.execute_reply":"2023-03-15T11:03:32.239586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_input = torch.rand((50, 543, 3))\nonnx_feat_gen_path = 'feature_gen.onnx'\n\nfeature_converter.eval()\n\ntorch.onnx.export(\n feature_converter, # PyTorch Model\n sample_input, # Input tensor\n onnx_feat_gen_path, # Output file (eg. 'output_model.onnx')\n opset_version=12, # Operator support version\n input_names=['input'], # Input tensor name (arbitary)\n output_names=['output'], # Output tensor name (arbitary)\n dynamic_axes={\n 'input' : {0: 'input'}\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:03:32.243853Z","iopub.execute_input":"2023-03-15T11:03:32.244582Z","iopub.status.idle":"2023-03-15T11:03:32.590972Z","shell.execute_reply.started":"2023-03-15T11:03:32.244532Z","shell.execute_reply":"2023-03-15T11:03:32.589949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_input = torch.rand((1, 3258)).cuda()\nonnx_model_path = 'asl_model.onnx'\n\nmodel.eval()\n\ntorch.onnx.export(\n model, # PyTorch Model\n sample_input, # Input tensor\n onnx_model_path, # Output file (eg. 'output_model.onnx')\n opset_version=12, # Operator support version\n input_names=['input'], # Input tensor name (arbitary)\n output_names=['output'], # Output tensor name (arbitary)\n dynamic_axes={\n 'input' : {0: 'input'}\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:03:32.592387Z","iopub.execute_input":"2023-03-15T11:03:32.593094Z","iopub.status.idle":"2023-03-15T11:03:32.731027Z","shell.execute_reply.started":"2023-03-15T11:03:32.593051Z","shell.execute_reply":"2023-03-15T11:03:32.729917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import onnx\nfrom onnx_tf.backend import prepare\n\n\ntf_feat_gen_path = '/kaggle/working/tf_feat_gen'\nonnx_feat_gen = onnx.load(onnx_feat_gen_path)\ntf_rep = prepare(onnx_feat_gen)\ntf_rep.export_graph(tf_feat_gen_path)\n\n\ntf_model_path = '/kaggle/working/tf_model'\nonnx_model = onnx.load(onnx_model_path)\ntf_rep = prepare(onnx_model)\ntf_rep.export_graph(tf_model_path)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:03:32.733804Z","iopub.execute_input":"2023-03-15T11:03:32.734214Z","iopub.status.idle":"2023-03-15T11:03:43.721771Z","shell.execute_reply.started":"2023-03-15T11:03:32.734172Z","shell.execute_reply":"2023-03-15T11:03:43.720531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Running the Inference mode in tensorflow ( as trainable =False)","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\nclass ASLInferModel(tf.Module):\n def __init__(self):\n super(ASLInferModel, self).__init__()\n self.feature_gen = tf.saved_model.load(tf_feat_gen_path)\n self.model = tf.saved_model.load(tf_model_path)\n self.feature_gen.trainable = False\n self.model.trainable = False\n \n @tf.function(input_signature=[\n tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')\n ])\n def call(self, input):\n output_tensors = {}\n features = self.feature_gen(**{'input': input})['output']\n output_tensors['outputs'] = self.model(**{'input': tf.expand_dims(features, 0)})['output'][0,:]\n return output_tensors\n \n \nmytfmodel = ASLInferModel()\ntf.saved_model.save(mytfmodel, '/kaggle/working/tf_infer_model', signatures={'serving_default': mytfmodel.call})","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:03:43.723826Z","iopub.execute_input":"2023-03-15T11:03:43.724186Z","iopub.status.idle":"2023-03-15T11:03:44.688649Z","shell.execute_reply.started":"2023-03-15T11:03:43.724154Z","shell.execute_reply":"2023-03-15T11:03:44.687508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the model\n\ntf_infer_model_path = '/kaggle/working/tf_infer_model'\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\ntflite_model = converter.convert()\n\ntflite_model_path = 'model.tflite'\n\n# Save the model\nwith open(tflite_model_path, 'wb') as f:\n f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:03:44.690778Z","iopub.execute_input":"2023-03-15T11:03:44.691962Z","iopub.status.idle":"2023-03-15T11:03:46.605266Z","shell.execute_reply.started":"2023-03-15T11:03:44.691920Z","shell.execute_reply":"2023-03-15T11:03:46.604140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543 # number of landmarks per frame\npq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n\nimport tflite_runtime.interpreter as tflite\ninterpreter = tflite.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n# raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(pq_path))\nsign = np.argmax(output[\"outputs\"])\n\nprint(sign, output[\"outputs\"].shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:04:51.709804Z","iopub.execute_input":"2023-03-15T11:04:51.710945Z","iopub.status.idle":"2023-03-15T11:04:51.864516Z","shell.execute_reply.started":"2023-03-15T11:04:51.710885Z","shell.execute_reply":"2023-03-15T11:04:51.862600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-03-15T11:04:59.569086Z","iopub.execute_input":"2023-03-15T11:04:59.569580Z","iopub.status.idle":"2023-03-15T11:05:01.501329Z","shell.execute_reply.started":"2023-03-15T11:04:59.569533Z","shell.execute_reply":"2023-03-15T11:05:01.500033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}