{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Isolated Sign Language Recognition : dive in \n\nThis notebook is the continuation of my previous notebook [here](https://www.kaggle.com/code/dorianmb/isolated-sign-language-recognition-quick-start?scriptVersionId=121909882)\n\n","metadata":{}},{"cell_type":"markdown","source":"# Credits\n\nNotebook that help me : \n- [NGHI HUYNH's notebook](https://www.kaggle.com/code/nghihuynh/gislr-eda-feature-processing)\n- [ROBERT HATCH' notbook](https://www.kaggle.com/code/roberthatch/gislr-lb-0-63-on-the-shoulders/notebook#PREPROCESSING)\n- [JVTHUNDER's notebook](https://www.kaggle.com/code/jvthunder/lstm-baseline-for-starters-sign-language)\n","metadata":{}},{"cell_type":"markdown","source":"# As newbie on kaggle, I am open to any suggestion for improvement.","metadata":{}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom sklearn.model_selection import train_test_split\n\nimport os\nimport random\nimport json","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-31T12:07:58.334697Z","iopub.execute_input":"2023-03-31T12:07:58.334997Z","iopub.status.idle":"2023-03-31T12:08:13.666549Z","shell.execute_reply.started":"2023-03-31T12:07:58.334968Z","shell.execute_reply":"2023-03-31T12:08:13.665414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install nb_black for autoformatting\n!pip install nb_black --quiet\n%load_ext lab_black","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:13.668934Z","iopub.execute_input":"2023-03-31T12:08:13.669728Z","iopub.status.idle":"2023-03-31T12:08:27.991719Z","shell.execute_reply.started":"2023-03-31T12:08:13.669677Z","shell.execute_reply":"2023-03-31T12:08:27.990632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:27.993082Z","iopub.execute_input":"2023-03-31T12:08:27.993492Z","iopub.status.idle":"2023-03-31T12:08:28.002921Z","shell.execute_reply.started":"2023-03-31T12:08:27.993443Z","shell.execute_reply":"2023-03-31T12:08:28.001904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set-up & Reproducible","metadata":{}},{"cell_type":"code","source":"# Constante\nSEED = 42\nROWS_PER_FRAME = 543\ndata_dir = \"/kaggle/input/asl-signs\"\nlandmark_fimes_dir = \"/kaggle/input/asl-signs/train_landmark_files\"","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.006362Z","iopub.execute_input":"2023-03-31T12:08:28.007205Z","iopub.status.idle":"2023-03-31T12:08:28.015108Z","shell.execute_reply.started":"2023-03-31T12:08:28.007086Z","shell.execute_reply":"2023-03-31T12:08:28.013988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_it_all(seed=SEED):\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nseed_it_all()  # Reproducible","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.016789Z","iopub.execute_input":"2023-03-31T12:08:28.017325Z","iopub.status.idle":"2023-03-31T12:08:28.028680Z","shell.execute_reply.started":"2023-03-31T12:08:28.017286Z","shell.execute_reply":"2023-03-31T12:08:28.027607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    data_columns = [\"x\", \"y\", \"z\"]\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.030963Z","iopub.execute_input":"2023-03-31T12:08:28.031874Z","iopub.status.idle":"2023-03-31T12:08:28.041930Z","shell.execute_reply.started":"2023-03-31T12:08:28.031821Z","shell.execute_reply":"2023-03-31T12:08:28.041054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_json(path):\n    with open(path, \"r\") as file:\n        json_data = json.load(file)\n    return json_data","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.043701Z","iopub.execute_input":"2023-03-31T12:08:28.044171Z","iopub.status.idle":"2023-03-31T12:08:28.054638Z","shell.execute_reply.started":"2023-03-31T12:08:28.044130Z","shell.execute_reply":"2023-03-31T12:08:28.053533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"path_train_df = pd.read_csv(data_dir + \"/train.csv\")\npath_train_df[\"path\"] = data_dir + \"/\" + path_train_df[\"path\"]\ndisplay(path_train_df.head(2)), len(path_train_df)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.056040Z","iopub.execute_input":"2023-03-31T12:08:28.056793Z","iopub.status.idle":"2023-03-31T12:08:28.322949Z","shell.execute_reply.started":"2023-03-31T12:08:28.056753Z","shell.execute_reply":"2023-03-31T12:08:28.321777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s2p_map = read_json(os.path.join(data_dir, \"sign_to_prediction_index_map.json\"))\np2s_map = {v: k for k, v in s2p_map.items()}\n\nencoder = lambda x: s2p_map.get(x)\ndecoder = lambda x: p2s_map.get(x)\n\npath_train_df[\"label\"] = path_train_df[\"sign\"].map(encoder)\nprint(f\"shape = {path_train_df.shape}\")\n\npath_train_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.324661Z","iopub.execute_input":"2023-03-31T12:08:28.325131Z","iopub.status.idle":"2023-03-31T12:08:28.390472Z","shell.execute_reply.started":"2023-03-31T12:08:28.325094Z","shell.execute_reply":"2023-03-31T12:08:28.389557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_parquet(path_train_df.path[0])\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.394939Z","iopub.execute_input":"2023-03-31T12:08:28.395217Z","iopub.status.idle":"2023-03-31T12:08:28.559144Z","shell.execute_reply.started":"2023-03-31T12:08:28.395184Z","shell.execute_reply":"2023-03-31T12:08:28.558063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**need to select usefull Landmark index**","metadata":{}},{"cell_type":"markdown","source":"# Quick EDA and Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## Distribution of number of frame per sequence over the dataset","metadata":{}},{"cell_type":"code","source":"# let's check the distribution of n_frame over the dataset\ndistribution_lenght = int(len(path_train_df) / 100)\nframes = np.zeros(distribution_lenght)\nfor index, row in tqdm(path_train_df.iterrows(), total=distribution_lenght):\n    if index > distribution_lenght - 1:\n        break\n    x = load_relevant_data_subset(row.path)\n    frames[index] = x.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:28.560824Z","iopub.execute_input":"2023-03-31T12:08:28.561494Z","iopub.status.idle":"2023-03-31T12:08:48.754366Z","shell.execute_reply.started":"2023-03-31T12:08:28.561455Z","shell.execute_reply":"2023-03-31T12:08:48.753430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"minimum frames in sequence : \", frames.min())\nprint(\"maximum frames in sequence : \", frames.max())\nprint(\"mean of frames in the first 944 sequences\", frames.mean())\nprint(\"median of frames in the first 944 sequences\", np.median(frames))","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:48.758767Z","iopub.execute_input":"2023-03-31T12:08:48.759932Z","iopub.status.idle":"2023-03-31T12:08:48.774382Z","shell.execute_reply.started":"2023-03-31T12:08:48.759872Z","shell.execute_reply":"2023-03-31T12:08:48.770913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nhist = plt.hist(frames, bins=100)\n\nmedian = np.median(frames)\nplt.plot(\n    [median, median],\n    [0, hist[0].max()],\n    \"--\",\n    c=\"red\",\n    linewidth=2,\n    label=f\"Median={median:.0f} frames\",\n)\n\nplt.title(\"Distribution of number of frame per sequence over the dataset\")\nplt.xlabel(\"n-frames in one sequence\")\nplt.ylabel(\"Number of sequence with n-frames\")\n\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:48.776287Z","iopub.execute_input":"2023-03-31T12:08:48.776991Z","iopub.status.idle":"2023-03-31T12:08:49.430470Z","shell.execute_reply.started":"2023-03-31T12:08:48.776949Z","shell.execute_reply":"2023-03-31T12:08:49.429469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int(np.percentile(frames, 25))","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:49.435274Z","iopub.execute_input":"2023-03-31T12:08:49.438046Z","iopub.status.idle":"2023-03-31T12:08:49.452008Z","shell.execute_reply.started":"2023-03-31T12:08:49.438002Z","shell.execute_reply":"2023-03-31T12:08:49.451114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Herlper function","metadata":{}},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n    x_zero_insted_of_nan = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    x_zero_for_nan_else_one = tf.where(\n        tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)\n    )\n    x_out = tf.reduce_sum(x_zero_insted_of_nan, axis=axis) / tf.reduce_sum(\n        x_zero_for_nan_else_one, axis=axis\n    )\n    return tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:49.456465Z","iopub.execute_input":"2023-03-31T12:08:49.458732Z","iopub.status.idle":"2023-03-31T12:08:49.477251Z","shell.execute_reply.started":"2023-03-31T12:08:49.458693Z","shell.execute_reply":"2023-03-31T12:08:49.475942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"DROP_Z = False\nprint(\"Drop Z in data column :\", DROP_Z, end=\"\\n\\n\")\n\n# Drop most of the face landmarks to reduce the dimensionality\nLANDMARK = [0, 9, 11, 13, 14, 17, 117, 118, 119, 199, 346, 347, 348] + list(\n    range(468, 543)\n)\n\nLENGHT_LANDMARK = len(LANDMARK)\nN_DATA = len([\"x\", \"y\"]) if DROP_Z else len([\"x\", \"y\", \"z\"])\nFIXED_FRAME = int(np.median(frames))\nSHAPE = [FIXED_FRAME, LENGHT_LANDMARK, N_DATA]\n\nprint(\"Fixed Frame (shape[0]) =\", FIXED_FRAME, end=\"\\n\\n\")\nprint(\"Shape =\", SHAPE, end=\"\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:49.481616Z","iopub.execute_input":"2023-03-31T12:08:49.484367Z","iopub.status.idle":"2023-03-31T12:08:49.506879Z","shell.execute_reply.started":"2023-03-31T12:08:49.484328Z","shell.execute_reply":"2023-03-31T12:08:49.505913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Generator","metadata":{}},{"cell_type":"code","source":"class FeatureGen(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n\n    def call(self, x):\n        if x.shape[0] is None:  # for inference model\n            n_frames = FIXED_FRAME\n        else:\n            n_frames = x.shape[0]\n\n        # Drop \"z\" column\n        if DROP_Z:\n            x = x[:, :, 0:2]\n\n        # NaN values become 0\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n\n        # Landmarks reduction\n        # Select only the usefull landmark\n        x = tf.gather(\n            x,\n            indices=LANDMARK,\n            axis=1,\n        )\n\n        if FIXED_FRAME > n_frames:\n            outputs = tf.image.resize(x, size=[SHAPE[0], SHAPE[1]], method=\"bilinear\")\n        else:\n            outputs = tf.image.resize(x, size=[SHAPE[0], SHAPE[1]], method=\"nearest\")\n\n        return outputs\n\n\nfeature_converter = FeatureGen()","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:49.511516Z","iopub.execute_input":"2023-03-31T12:08:49.513928Z","iopub.status.idle":"2023-03-31T12:08:49.547746Z","shell.execute_reply.started":"2023-03-31T12:08:49.513876Z","shell.execute_reply":"2023-03-31T12:08:49.547025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some test\nsample = load_relevant_data_subset(path_train_df.path[1])\nprepocesse_sample = feature_converter(sample)\nprepocesse_sample.shape, sample.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:49.548886Z","iopub.execute_input":"2023-03-31T12:08:49.549209Z","iopub.status.idle":"2023-03-31T12:08:54.666717Z","shell.execute_reply.started":"2023-03-31T12:08:49.549174Z","shell.execute_reply":"2023-03-31T12:08:54.665551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making data","metadata":{}},{"cell_type":"code","source":"TOTAL_DATA_LENGHT = len(path_train_df)\nDATA_LENGHT_EXPERIMENT = int(len(path_train_df) / 10)\nprint(\"Lenght of data for modeling :\", DATA_LENGHT_EXPERIMENT)\nprint(f\"Percentage of total data {DATA_LENGHT_EXPERIMENT/TOTAL_DATA_LENGHT*100:.1f}%\")","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:54.668710Z","iopub.execute_input":"2023-03-31T12:08:54.669129Z","iopub.status.idle":"2023-03-31T12:08:54.678219Z","shell.execute_reply.started":"2023-03-31T12:08:54.669090Z","shell.execute_reply":"2023-03-31T12:08:54.676915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_row(row):\n    x = load_relevant_data_subset(row.path)\n    x = feature_converter(x)\n    return x, row.label\n\n\ndef convert_and_save_data(data_lenght=DATA_LENGHT_EXPERIMENT):\n    np_features = np.zeros([data_lenght] + SHAPE)\n    np_labels = np.zeros(data_lenght)\n\n    print(f\"Total data to processe : {data_lenght}\")\n    print(f\"Percentage of total data {data_lenght/TOTAL_DATA_LENGHT*100:.2f}%\")\n    for index, row in tqdm(path_train_df.iterrows(), total=data_lenght):\n        if index > data_lenght - 1:\n            break\n\n        if index % (DATA_LENGHT_EXPERIMENT // 10) == 0:\n            print(f\"Data processed {index/data_lenght*100:.1f}%\")\n\n        data = load_relevant_data_subset(row.path)\n        feature, label = convert_row(row)\n        np_features[index] = feature\n        np_labels[index] = label\n\n    np.save(\"features.npy\", np_features)\n    np.save(\"labels.npy\", np_labels)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:54.680009Z","iopub.execute_input":"2023-03-31T12:08:54.680660Z","iopub.status.idle":"2023-03-31T12:08:54.700113Z","shell.execute_reply.started":"2023-03-31T12:08:54.680621Z","shell.execute_reply":"2023-03-31T12:08:54.699018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    features = np.load(\"/kaggle/working/features.npy\")\n    labels = np.load(\"/kaggle/working/labels.npy\")\n    print(\"Data Load successfully\")\nexcept:\n    print(\"Loading DATA has fail... \\nCreating DataSet\")\n    convert_and_save_data(DATA_LENGHT_EXPERIMENT)\n\n    features = np.load(\"features.npy\")\n    labels = np.load(\"labels.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:08:54.703509Z","iopub.execute_input":"2023-03-31T12:08:54.704396Z","iopub.status.idle":"2023-03-31T12:13:22.475789Z","shell.execute_reply.started":"2023-03-31T12:08:54.704360Z","shell.execute_reply":"2023-03-31T12:13:22.474686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(\n    features, labels, test_size=0.2, random_state=SEED\n)\n\ndel (\n    features,\n    labels,\n)  # delete, usefull with full data otherwise it fail (memorry issues)\n\nbuffer_size = int(DATA_LENGHT_EXPERIMENT / 10)\n\ntrain_data = tf.data.Dataset.from_tensor_slices((X_train, y_train))\ntrain_data = train_data.shuffle(buffer_size).batch(128).prefetch(tf.data.AUTOTUNE)\n\nval_data = tf.data.Dataset.from_tensor_slices((X_val, y_val))\nval_data = val_data.batch(128).prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:13:22.477282Z","iopub.execute_input":"2023-03-31T12:13:22.478054Z","iopub.status.idle":"2023-03-31T12:13:23.535295Z","shell.execute_reply.started":"2023-03-31T12:13:22.478012Z","shell.execute_reply":"2023-03-31T12:13:23.534018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Quick test -> after training\nquick_test_idx = np.random.randint(0, len(y_val), size=(10,))  # See after training\nquick_test_X = np.take(X_val, quick_test_idx, axis=0)\nquick_test_y = np.take(y_val, quick_test_idx, axis=0)\ndel X_train, X_val, y_train, y_val","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:13:23.536785Z","iopub.execute_input":"2023-03-31T12:13:23.537198Z","iopub.status.idle":"2023-03-31T12:13:23.550486Z","shell.execute_reply.started":"2023-03-31T12:13:23.537155Z","shell.execute_reply":"2023-03-31T12:13:23.549306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"def dense_block(units, drop):\n    fc = layers.Dense(units)\n    norm = layers.LayerNormalization()\n    act = layers.Activation(\"relu\")\n    dropout = layers.Dropout(drop)\n    return lambda x: dropout(act(norm(fc(x))))\n\n\ndef classifier(lstm_units, n_labels, drop):\n    lstm = layers.ConvLSTM1D(\n        filters=lstm_units, kernel_size=1\n    )  # RNN capable of learning long-term dependencies.\n    dropout = layers.Dropout(drop)\n    flat = layers.Flatten()\n    dense = layers.Dense(n_labels)\n    outputs = layers.Activation(\"softmax\", dtype=\"float32\", name=\"predictions\")\n    return lambda x: outputs(dense(flat(dropout(lstm(x)))))\n\n\ndef get_model(\n    encoder_units=[128, 64],\n    drop=0.5,\n    lstm_units=250,\n    n_labels=250,\n    shape=SHAPE,\n    learning_rate=0.001,\n):\n    inputs = layers.Input(shape=shape)\n    x = inputs\n\n    for units in encoder_units:\n        x = dense_block(units, drop)(x)\n\n    outputs = classifier(lstm_units, n_labels, drop)(x)\n\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n\n    model.compile(\n        loss=\"sparse_categorical_crossentropy\",\n        optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n        metrics=[\"accuracy\"],\n    )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:13:23.552030Z","iopub.execute_input":"2023-03-31T12:13:23.552950Z","iopub.status.idle":"2023-03-31T12:13:23.575503Z","shell.execute_reply.started":"2023-03-31T12:13:23.552905Z","shell.execute_reply":"2023-03-31T12:13:23.574423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_callbacks():\n    return [\n        tf.keras.callbacks.EarlyStopping(\n            monitor=\"val_accuracy\", patience=10, restore_best_weights=True\n        ),\n        tf.keras.callbacks.ReduceLROnPlateau(\n            monitor=\"val_accuracy\", factor=0.5, patience=3\n        ),\n        tf.keras.callbacks.ModelCheckpoint(\n            \"./ASL_model\",\n            save_best_only=True,\n            restore_best_weights=True,\n            monitor=\"val_accuracy\",\n            mode=\"max\",\n            verbose=False,\n        ),\n    ]\n\n\ncb_list = get_callbacks()\n\nmodel = get_model()\nmodel.summary()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-03-31T12:13:23.578534Z","iopub.execute_input":"2023-03-31T12:13:23.579485Z","iopub.status.idle":"2023-03-31T12:13:24.090571Z","shell.execute_reply.started":"2023-03-31T12:13:23.579449Z","shell.execute_reply":"2023-03-31T12:13:24.089768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\n\nplot_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:13:24.091775Z","iopub.execute_input":"2023-03-31T12:13:24.092194Z","iopub.status.idle":"2023-03-31T12:13:24.503230Z","shell.execute_reply.started":"2023-03-31T12:13:24.092149Z","shell.execute_reply":"2023-03-31T12:13:24.501958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Guessing: \", 1 / 250)\nprint(\"Data per Class:\", DATA_LENGHT_EXPERIMENT / 250)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:13:24.504946Z","iopub.execute_input":"2023-03-31T12:13:24.505293Z","iopub.status.idle":"2023-03-31T12:13:24.513018Z","shell.execute_reply.started":"2023-03-31T12:13:24.505261Z","shell.execute_reply":"2023-03-31T12:13:24.511561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit","metadata":{"execution":{"iopub.status.busy":"2023-03-25T11:00:48.764672Z","iopub.execute_input":"2023-03-25T11:00:48.765295Z","iopub.status.idle":"2023-03-25T11:00:48.798007Z","shell.execute_reply.started":"2023-03-25T11:00:48.765217Z","shell.execute_reply":"2023-03-25T11:00:48.796143Z"}}},{"cell_type":"code","source":"%%time\nhistory = model.fit(\n    train_data,\n    validation_data=val_data,\n    epochs=30,\n    callbacks=cb_list\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:13:24.520957Z","iopub.execute_input":"2023-03-31T12:13:24.522042Z","iopub.status.idle":"2023-03-31T12:29:51.262853Z","shell.execute_reply.started":"2023-03-31T12:13:24.522000Z","shell.execute_reply":"2023-03-31T12:29:51.261713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model(\"./ASL_model\")\nscore = model.evaluate(val_data)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:29:51.264631Z","iopub.execute_input":"2023-03-31T12:29:51.265266Z","iopub.status.idle":"2023-03-31T12:29:59.430446Z","shell.execute_reply.started":"2023-03-31T12:29:51.265190Z","shell.execute_reply":"2023-03-31T12:29:59.429345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(quick_test_X, verbose=False).argmax(axis=1)\n\nfor true_label_id, pred_label_id in zip(quick_test_y, predictions):\n    true_label = decoder(true_label_id)\n    pred_label = decoder(pred_label_id)\n    result = True if pred_label == true_label else False\n    print(\n        f\"Prediction on test label : {pred_label.upper():<10} => True label {true_label.upper():<10} => {int(result)}\"\n    )","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:29:59.432780Z","iopub.execute_input":"2023-03-31T12:29:59.433656Z","iopub.status.idle":"2023-03-31T12:30:00.166885Z","shell.execute_reply.started":"2023-03-31T12:29:59.433606Z","shell.execute_reply":"2023-03-31T12:30:00.165476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test on 10% data\n\n`kernel_size=1`, 192 landmark, 13 frames:\n- 2dense block => 24s 402ms/step - loss: 0.1679 - accuracy: 0.9951 - val_loss: 6.1105 - val_accuracy: 0.1402 - lr: 3.1250e-05\n- 2dense block without Z =>23s 388ms/step - loss: 0.1138 - accuracy: 0.9978 - val_loss: 5.6463 - val_accuracy: 0.1810 - lr: 6.2500e-05\n- 2dense block kernel_size=4 => 85s 1s/step - loss: 5.5149 - accuracy: 0.0060 - val_loss: 5.5236 - val_accuracy: 0.0032 - lr: 1.2500e-04 - best accuracy: 0.0069\n- 2dense block only \"bilinear\" => 27s 457ms/step - loss: 0.0523 - accuracy: 0.9996 - val_loss: 5.9966 - val_accuracy: 0.1852 - lr: 6.2500e-05\n\n---\n\n- 3dense block only \"bilinear\" => 29s 477ms/step - loss: 0.0864 - accuracy: 0.9959 - val_loss: 6.2813 - val_accuracy: 0.1884 - lr: 3.1250e-05 - best accuracy: 0.1926\n- 3dense block only \"nearest\" => 28s 471ms/step - loss: 0.6903 - accuracy: 0.8809 - val_loss: 5.1938 - val_accuracy: 0.1825 - lr: 3.1250e-05 - best accuracy: 0.1857\n- 3dense block only \"bilinear\" => 25s 421ms/step - loss: 0.5771 - accuracy: 0.9135 - val_loss: 5.7056 - val_accuracy: 0.1608 - lr: 3.1250e-05 - best accuracy: 0.1608\n- 3dense block => 26s 440ms/step - loss: 0.0279 - accuracy: 0.9997 - val_loss: 6.7141 - val_accuracy: 0.1899 - lr: 1.5625e-05 - best accuracy: 0.1952\n- 3dense block + normalisation => 27s 441ms/step - loss: 0.0011 - accuracy: 1.0000 - val_loss: 11.0349 - val_accuracy: 0.0587 - lr: 1.2500e-04 - best accuracy: 0.0614\n\n---\n\n88landmark, `kernel_size=1`, `encoder_units=[512, 256, 128]`\n- 3dense block 45frames => 57s 953ms/step - loss: 0.1020 - accuracy: 0.9794 - val_loss: 6.4184 - val_accuracy: 0.1868 - lr: 6.2500e-05 - best accuracy: 0.1884\n- 3dense block 23frames => 33s 544ms/step - loss: 0.1231 - accuracy: 0.9790 - val_loss: 5.8464 - val_accuracy: 0.1989 - lr: 1.5625e-05 - best accuracy: 0.2042\n- 2dense block 23frames `[512, 256]` => 33s 544ms/step - loss: 0.0770 - accuracy: 0.9874 - val_loss: 6.0699 - val_accuracy: 0.1974 - lr: 3.1250e-05 - best accuracy: 0.2021\n- 2dense block 23frames `[256, 128]` => 24s 407ms/step - loss: 0.0993 - accuracy: 0.9807 - val_loss: 6.0558 - val_accuracy: 0.2238 - lr: 3.1250e-05 - best accuracy: 0.2286","metadata":{}},{"cell_type":"markdown","source":"# Visualize History","metadata":{}},{"cell_type":"code","source":"def plot_history(history, zoom=0):\n    df = pd.DataFrame(history.history)\n    n = len(df.columns)\n\n    row = n // 2\n    col = n // 2 + n % 2\n\n    plt.figure(figsize=(5 * (col + 1) + zoom, 5 * row + zoom))\n    for i, column in enumerate(df.columns):\n        plt.subplot(row, col + 1, i + 1)\n        plt.plot(df[f\"{column}\"], label=f\"{column}\")\n        plt.legend()\n        plt.xlabel(\"epochs\")\n        plt.ylabel(f\"{column}\")\n        plt.tight_layout(pad=2)  # padding\n\n    plt.subplot(row, col + 1, n + 1)\n    for column in df.columns:\n        plt.plot(df[f\"{column}\"], label=f\"{column}\")\n        plt.legend()\n    plt.xlabel(\"epochs\")","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:30:00.168534Z","iopub.execute_input":"2023-03-31T12:30:00.168920Z","iopub.status.idle":"2023-03-31T12:30:00.188031Z","shell.execute_reply.started":"2023-03-31T12:30:00.168882Z","shell.execute_reply":"2023-03-31T12:30:00.186852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:30:00.189760Z","iopub.execute_input":"2023-03-31T12:30:00.190555Z","iopub.status.idle":"2023-03-31T12:30:01.439477Z","shell.execute_reply.started":"2023-03-31T12:30:00.190497Z","shell.execute_reply":"2023-03-31T12:30:01.438364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Model","metadata":{}},{"cell_type":"code","source":"def get_inference_model(model):\n    inputs = tf.keras.Input(shape=(543, 3), name=\"inputs\")\n    x = feature_converter(inputs)\n    x = tf.expand_dims(x, axis=0)\n    x = model(x)\n    output = tf.keras.layers.Activation(activation=\"linear\", name=\"outputs\")(x)\n    inference_model = tf.keras.Model(inputs=inputs, outputs=output)\n    inference_model.compile(\n        loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[\"accuracy\"]\n    )\n    return inference_model","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:30:01.440688Z","iopub.execute_input":"2023-03-31T12:30:01.442133Z","iopub.status.idle":"2023-03-31T12:30:01.457134Z","shell.execute_reply.started":"2023-03-31T12:30:01.442090Z","shell.execute_reply":"2023-03-31T12:30:01.455933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_model = get_inference_model(model)\ninference_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:30:01.458795Z","iopub.execute_input":"2023-03-31T12:30:01.459458Z","iopub.status.idle":"2023-03-31T12:30:01.989398Z","shell.execute_reply.started":"2023-03-31T12:30:01.459404Z","shell.execute_reply":"2023-03-31T12:30:01.988521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"demo_output = inference_model(load_relevant_data_subset(path_train_df.path[0]))\ndecoder(int(demo_output.numpy().argmax(axis=1)))","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:30:01.990771Z","iopub.execute_input":"2023-03-31T12:30:01.991127Z","iopub.status.idle":"2023-03-31T12:30:02.283889Z","shell.execute_reply.started":"2023-03-31T12:30:01.991088Z","shell.execute_reply":"2023-03-31T12:30:02.282780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Submission Fail**\n\n**TODO : find a way to submite**","metadata":{}},{"cell_type":"code","source":"converter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\n\"\"\"\nconverter.target_spec.supported_ops = [\n    tf.lite.OpsSet.TFLITE_BUILTINS,\n    tf.lite.OpsSet.SELECT_TF_OPS,\n]\nconverter._experimental_lower_tensor_list_ops = False\n\"\"\"\ntflite_model = converter.convert()\n\nmodel_path = \"model.tflite\"\n# Save the model.\nwith open(model_path, \"wb\") as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:30:02.285434Z","iopub.execute_input":"2023-03-31T12:30:02.285843Z","iopub.status.idle":"2023-03-31T12:30:11.784710Z","shell.execute_reply.started":"2023-03-31T12:30:02.285800Z","shell.execute_reply":"2023-03-31T12:30:11.783607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $model_path","metadata":{"execution":{"iopub.status.busy":"2023-03-31T12:30:11.786084Z","iopub.execute_input":"2023-03-31T12:30:11.786433Z","iopub.status.idle":"2023-03-31T12:30:14.262945Z","shell.execute_reply.started":"2023-03-31T12:30:11.786395Z","shell.execute_reply":"2023-03-31T12:30:14.261646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}