{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install\n!pip install -q itables 2> /dev/null\n#!pip install -q numpy==1.20.0\n#!pip install -q flatbuffers 2> /dev/null\n#!pip install -q mediapipe 2> /dev/null","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-22T16:51:21.309199Z","iopub.execute_input":"2023-04-22T16:51:21.309737Z","iopub.status.idle":"2023-04-22T16:51:35.627080Z","shell.execute_reply.started":"2023-04-22T16:51:21.309693Z","shell.execute_reply":"2023-04-22T16:51:35.625564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utils\nimport json\nimport os \nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport time\nimport sys\n\nimport numpy as np\n#np.set_printoptions(threshold=sys.maxsize)\nimport pandas as pd\n##import mediapipe as mp\n##from mediapipe.framework.formats import landmark_pb2\n# Iteractive Tables\nfrom itables import init_notebook_mode\ninit_notebook_mode(all_interactive=True, connected=True)\n# Tensorflow\nimport tensorflow as tf\nfrom tensorflow.keras import layers, Sequential, losses","metadata":{"execution":{"iopub.status.busy":"2023-04-22T16:51:35.630301Z","iopub.execute_input":"2023-04-22T16:51:35.631332Z","iopub.status.idle":"2023-04-22T16:51:45.368765Z","shell.execute_reply.started":"2023-04-22T16:51:35.631265Z","shell.execute_reply":"2023-04-22T16:51:45.367351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Config\nclass cfg:\n    EDA_phase = False\n    Preprocess_phase = False\n    Training_phase = False\n    BASE_DIR = '/kaggle/input/asl-signs/'\n    WORK_DIR = '/kaggle/working/'\n    label_map = json.load(open(f'{BASE_DIR}/sign_to_prediction_index_map.json', \"r\"))\n\ntrain_csv = pd.read_csv(f'{cfg.BASE_DIR}/train.csv')\ntrain_csv['label'] = train_csv.sign.map(cfg.label_map)\n#extended_train_csv = pd.read_csv('/kaggle/input/gislr-extended-train-dataframe/extended_train.csv')\n#extended_train_csv['label'] = extended_train_csv.sign.map(cfg.label_map)\n#new_train_csv = pd.read_csv('/kaggle/input/islr-my-preprocess/new_train.csv')\n#shuffled_seq_len_list = pd.read_csv('/kaggle/input/islr-my-preprocess/shuffled_seq_len_list.csv').values.reshape(-1,)\n\n# Index to Use\nclass indx:\n    lhand = np.arange(468,489,2).tolist()\n    rhand = np.arange(522,543,2).tolist()\n    #pose = []\n    face = [0,13,14,17,37,39,40,61,78,80,81,82,84,87,88,91,95,146,178,181,185,191,267,269,270,291,308,310,311,312,314,317,318,321,324,375,402,405,409,415]","metadata":{"execution":{"iopub.status.busy":"2023-04-22T16:53:35.385269Z","iopub.execute_input":"2023-04-22T16:53:35.385763Z","iopub.status.idle":"2023-04-22T16:53:35.543921Z","shell.execute_reply.started":"2023-04-22T16:53:35.385717Z","shell.execute_reply":"2023-04-22T16:53:35.542181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utility Classes and Func\n## load desiered parq\nclass load_parq:\n    def tolist(path):\n        parq_list = []\n        parq = pd.read_parquet(path)\n        for fr in tqdm(parq.frame.unique()):\n            parq_list.append(parq[parq.frame==fr])\n        return parq_list\n    \n    def toarray(path):\n        parq = pd.read_parquet(path)\n        parq_3dim_array = parq[['x','y','z']].values.reshape(parq.frame.nunique(), int(parq.shape[0]/parq.frame.nunique()), 3)\n        return parq_3dim_array\n    \n    def todf(path):\n        parq = pd.read_parquet(path)\n        return parq\n\n\n#check preprocess\ndef check_dum(dum):\n    shape = dum.shape\n    MV = tf.math.reduce_all(tf.math.is_nan(dum[:,:22,0]),axis=1)\n    print('-'*60)\n    print(f'shape : {shape}')\n    print('-'*60)\n    print(f'is both hands na? : {MV}')\n    #is both hand non na?\n    \n# Clear output folder\ndef remove_folder_contents(folder):\n    for the_file in os.listdir(folder):\n        file_path = os.path.join(folder, the_file)\n        try:\n            if os.path.isfile(file_path):\n                os.unlink(file_path)\n            elif os.path.isdir(file_path):\n                remove_folder_contents(file_path)\n                os.rmdir(file_path)\n        except Exception as e:\n            print(e)","metadata":{"execution":{"iopub.status.busy":"2023-04-22T16:53:37.686946Z","iopub.execute_input":"2023-04-22T16:53:37.687412Z","iopub.status.idle":"2023-04-22T16:53:37.701177Z","shell.execute_reply.started":"2023-04-22T16:53:37.687373Z","shell.execute_reply":"2023-04-22T16:53:37.699776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reduce the number of landmarks and impute missing values\nclass Reduce_and_Impute(tf.keras.layers.Layer):\n    def __init__(self):\n        super(Reduce_and_Impute, self).__init__()\n        self.lhand = tf.range(468,489,2)#.numpy().tolist()\n        self.rhand = tf.range(522,543,2)#.numpy().tolist()\n        #self.pose = tf.range()\n        self.face = tf.constant([0,13,14,17,37,39,40,61,78,80,81,82,84,87,88,91,95,146,178,181,185,191,267,269,270,291,308,310,311,312,314,317,318,321,324,375,402,405,409,415])\n    \n    #def build(self, input_shape):\n        #self.batch_size = input_shape[0]\n        #self.seq_len = input_shape[1]\n        #self.in_shape = input_shape\n        #super(Preprocess_Layer, self).build(input_shape)  # Be sure to call this at the end\n    \n    #@tf.function(input_signature=[tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='x')])\n    def call(self, x):\n        # convert to tensor\n        x = tf.convert_to_tensor(x, dtype=tf.float32)\n        x = tf.cast(x, dtype=tf.float32)\n        # expand_dim 3=>4 if needed\n        #if tf.rank(x)==3:\n        #    x = tf.expand_dims(x, axis=0)\n        # reduce landmarks\n        lh = tf.gather(x, self.lhand, axis=1)\n        rh = tf.gather(x, self.rhand, axis=1)\n        #p = tf.gather(x, self.pose, axis=1)\n        f = tf.gather(x, self.face, axis=1)\n        x = tf.concat([lh, rh, f], axis=1)\n        #in_shape = tf.shape(x)\n        #in_shape = x.shape\n        #in_shape = self.in_shape\n        \n        # take avg of previous and next frame where we have MV \n        \n        padding = tf.constant([[1,1],[0,0],[0,0]])\n        padded = tf.pad(x, paddings=padding, constant_values=float('nan'))\n        shift_forth = padded[2:,:,:]\n        shift_back = padded[:-2,:,:]\n        x = tf.where(tf.math.is_nan(x), (shift_forth+shift_back)/2, x)\n        \n        #nantns = tf.constant(float('nan'), shape=(in_shape[0], 1, 62, 3))\n        #shift_forth = tf.concat([tf.gather(x, list(range(1,in_shape[1])), axis=1), nantns], axis=1)\n        #shift_back = tf.concat([nantns, tf.gather(x, list(range(0,in_shape[1]-1)), axis=1)], axis=1)\n        #x = tf.where(tf.math.is_nan(x), tf.reshape(((shift_forth+shift_back)/2), (in_shape[0], in_shape[1], 62, in_shape[3])), x)\n        \n        #x = x.numpy() ###FIX_THIS###\n        #shift_forth = np.insert(np.delete(x, 0, axis=1), x.shape[0]-1, float('nan'), axis=1)\n        #shift_back = np.insert(np.delete(x, x.shape[0]-1, axis=1), 0, float('nan'), axis=1)\n        #x[np.isnan(x)] = ((shift_forth+shift_back)/2).reshape(x.shape)[np.isnan(x)]\n        #x = tf.convert_to_tensor(x)\n        #x = tf.cast(x, dtype=tf.float32)\n        \n        return x\n\n# Extracts longest streaks of frames that at least one hand is present\nclass Filter_Frames(tf.keras.layers.Layer):\n    def __init__(self):\n        super(Filter_Frames, self).__init__()\n    \n    #@tf.function(input_signature=[tf.TensorSpec(shape=[None, 62, 3], dtype=tf.float32, name='x')])\n    def call(self, x):\n        isnan = tf.math.reduce_all(tf.math.is_nan(x[:,:22,0]),axis=1)\n        isnan = tf.concat([[True],isnan,[True]], axis=0)\n        #print(isnan)\n        i = tf.cast(tf.transpose(tf.where(isnan==True))[0], dtype=tf.float32)\n        #print(i)\n        fi = tf.concat([i[1:],tf.constant([float('nan')])], axis=0)\n        #print(fi)\n        ind = tf.argmax(fi-i)\n        #print(i[ind])\n        #print(fi[ind]-1)\n        #print(x[tf.cast(i[ind],dtype=tf.int32):tf.cast(fi[ind]-1,dtype=tf.int32)].shape)\n        x = x[tf.cast(i[ind],dtype=tf.int32):tf.cast(fi[ind]-1,dtype=tf.int32)]\n        return x\n\n# Replace missing hand NaN values with zero\nclass Nan2zero(tf.keras.layers.Layer):\n    def __init__(self):\n        super(Nan2zero, self).__init__()\n        \n    #@tf.function(input_signature=[tf.TensorSpec(shape=[None, 62, 3], dtype=tf.float32, name='x')])\n    def call(self, x):\n        return tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-04-22T16:53:38.517895Z","iopub.execute_input":"2023-04-22T16:53:38.518391Z","iopub.status.idle":"2023-04-22T16:53:38.540683Z","shell.execute_reply.started":"2023-04-22T16:53:38.518348Z","shell.execute_reply":"2023-04-22T16:53:38.539004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Preprocess_model = Sequential([\n    Reduce_and_Impute(),\n    Filter_Frames(),\n    Nan2zero()\n])\nPreprocess_model.build((None,543,3))\n#------------------------------------------------------------------------------------------------\na = Preprocess_model.predict(load_parq.toarray('/kaggle/input/asl-signs/train_landmark_files/22343/4146590997.parquet'))\n#------------------------------------------------------------------------------------------------\nb = Preprocess_model(load_parq.toarray('/kaggle/input/asl-signs/train_landmark_files/22343/4146590997.parquet'))\n#------------------------------------------------------------------------------------------------\ndum = load_parq.toarray('/kaggle/input/asl-signs/train_landmark_files/22343/4146590997.parquet')\nl = Reduce_and_Impute()\nduml1 = l(x=dum)\nl2 = Filter_Frames()\nduml2 = l2(duml1)\nl3 = Nan2zero()\nc = l3(duml2)\n#------------------------------------------------------------------------------------------------","metadata":{"execution":{"iopub.status.busy":"2023-04-22T16:57:40.561789Z","iopub.execute_input":"2023-04-22T16:57:40.562302Z","iopub.status.idle":"2023-04-22T16:57:41.432800Z","shell.execute_reply.started":"2023-04-22T16:57:40.562253Z","shell.execute_reply":"2023-04-22T16:57:41.431305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'shape using predict:{a.shape}')\nprint(f'shape using model(x):{b.shape}')\nprint(f'shape using layers:{c.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-04-22T16:57:42.655418Z","iopub.execute_input":"2023-04-22T16:57:42.655905Z","iopub.status.idle":"2023-04-22T16:57:42.663531Z","shell.execute_reply.started":"2023-04-22T16:57:42.655866Z","shell.execute_reply":"2023-04-22T16:57:42.661905Z"},"trusted":true},"execution_count":null,"outputs":[]}]}