{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"},{"sourceId":10230497,"sourceType":"datasetVersion","datasetId":6325435},{"sourceId":10230837,"sourceType":"datasetVersion","datasetId":6325625},{"sourceId":10233219,"sourceType":"datasetVersion","datasetId":6327365}],"dockerImageVersionId":30407,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom tqdm import tqdm\n\nfrom joblib import Parallel, delayed\nimport multiprocessing as mp\nfrom multiprocessing import cpu_count\nfrom sklearn.model_selection import StratifiedGroupKFold, KFold\n\nimport tensorflow as tf\ncpu_count()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-18T18:21:32.669074Z","iopub.execute_input":"2024-12-18T18:21:32.669704Z","iopub.status.idle":"2024-12-18T18:21:46.772083Z","shell.execute_reply.started":"2024-12-18T18:21:32.669662Z","shell.execute_reply":"2024-12-18T18:21:46.770762Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/merged-islr-data/asl-signs/merged_train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:21:46.774081Z","iopub.execute_input":"2024-12-18T18:21:46.775390Z","iopub.status.idle":"2024-12-18T18:21:47.089463Z","shell.execute_reply.started":"2024-12-18T18:21:46.775342Z","shell.execute_reply":"2024-12-18T18:21:47.087910Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T18:21:51.526978Z","iopub.execute_input":"2024-12-18T18:21:51.527408Z","iopub.status.idle":"2024-12-18T18:21:51.575574Z","shell.execute_reply.started":"2024-12-18T18:21:51.527370Z","shell.execute_reply":"2024-12-18T18:21:51.574378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_parquet('/kaggle/input/merged-islr-data/asl-signs/train_landmark_files/0/00853.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:01.066116Z","iopub.execute_input":"2024-12-18T18:22:01.066531Z","iopub.status.idle":"2024-12-18T18:22:01.336513Z","shell.execute_reply.started":"2024-12-18T18:22:01.066483Z","shell.execute_reply":"2024-12-18T18:22:01.335139Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_parquet('/kaggle/input/merged-islr-data/asl-signs/train_landmark_files/0/00853.parquet')[:543].type.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:18.601946Z","iopub.execute_input":"2024-12-18T18:22:18.602370Z","iopub.status.idle":"2024-12-18T18:22:18.623587Z","shell.execute_reply.started":"2024-12-18T18:22:18.602333Z","shell.execute_reply":"2024-12-18T18:22:18.621836Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_parquet('/kaggle/input/merged-islr-data/asl-signs/train_landmark_files/0/00853.parquet')[543:543*2].type.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:22.268120Z","iopub.execute_input":"2024-12-18T18:22:22.268528Z","iopub.status.idle":"2024-12-18T18:22:22.290501Z","shell.execute_reply.started":"2024-12-18T18:22:22.268490Z","shell.execute_reply":"2024-12-18T18:22:22.289250Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:24.648029Z","iopub.execute_input":"2024-12-18T18:22:24.648878Z","iopub.status.idle":"2024-12-18T18:22:24.655075Z","shell.execute_reply.started":"2024-12-18T18:22:24.648837Z","shell.execute_reply":"2024-12-18T18:22:24.653820Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\nwith open('/kaggle/input/merged-islr-data/asl-signs/merged_sign_to_prediction_index_map.json') as json_file:\n    LABEL_DICT = json.load(json_file)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:25.011231Z","iopub.execute_input":"2024-12-18T18:22:25.011979Z","iopub.status.idle":"2024-12-18T18:22:25.022693Z","shell.execute_reply.started":"2024-12-18T18:22:25.011937Z","shell.execute_reply":"2024-12-18T18:22:25.021112Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_row(row):\n    coordinates = load_relevant_data_subset(f'/kaggle/input/merged-islr-data/asl-signs/{row.path}')\n    coordinates_encoded = coordinates.tobytes()\n    participant_id = int(row.participant_id)\n    sequence_id = int(row.sequence_id)\n    sign = int(LABEL_DICT[row.sign])\n    record_bytes = tf.train.Example(features=tf.train.Features(feature={\n                'coordinates': tf.train.Feature(bytes_list=tf.train.BytesList(value=[coordinates_encoded])),\n                'participant_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[participant_id])),\n                'sequence_id':tf.train.Feature(int64_list=tf.train.Int64List(value=[sequence_id])),\n                'sign':tf.train.Feature(int64_list=tf.train.Int64List(value=[sign])),\n                })).SerializeToString()\n    return record_bytes\n\ndef process_chunk(chunk, tfrecord_name):\n    options = tf.io.TFRecordOptions(compression_type='GZIP', compression_level=9)\n    with tf.io.TFRecordWriter(tfrecord_name, options=options) as file_writer:\n        for i, row in tqdm(chunk.iterrows()):\n            record_bytes = encode_row(row)\n            file_writer.write(record_bytes)\n            del record_bytes\n        file_writer.close()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:29.282638Z","iopub.execute_input":"2024-12-18T18:22:29.283035Z","iopub.status.idle":"2024-12-18T18:22:29.293983Z","shell.execute_reply.started":"2024-12-18T18:22:29.283002Z","shell.execute_reply":"2024-12-18T18:22:29.292633Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"row = train_df.iloc[0]\ncoordinates = load_relevant_data_subset(f'/kaggle/input/merged-islr-data/asl-signs/{row.path}')\ncoordinates_encoded = coordinates.tobytes()\nparticipant_id = int(row.participant_id)\nsequence_id = int(row.sequence_id)\nsign = int(LABEL_DICT[row.sign])","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:31.827413Z","iopub.execute_input":"2024-12-18T18:22:31.828210Z","iopub.status.idle":"2024-12-18T18:22:31.856045Z","shell.execute_reply.started":"2024-12-18T18:22:31.828163Z","shell.execute_reply":"2024-12-18T18:22:31.854633Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"\nrecord_bytes = tf.train.Example(features=tf.train.Features(feature={\n            'coordinates': tf.train.Feature(bytes_list=tf.train.BytesList(value=[coordinates_encoded])),\n            'participant_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[participant_id])),\n            'sequence_id':tf.train.Feature(int64_list=tf.train.Int64List(value=[sequence_id])),\n            'sign':tf.train.Feature(int64_list=tf.train.Int64List(value=[sign])),\n            }))","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:34.181730Z","iopub.execute_input":"2024-12-18T18:22:34.182156Z","iopub.status.idle":"2024-12-18T18:22:34.190034Z","shell.execute_reply.started":"2024-12-18T18:22:34.182117Z","shell.execute_reply":"2024-12-18T18:22:34.188664Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"N_FILES = len(train_df)\nCHUNK_SIZE = 512\nN_PART = 1\nFOLD = 4\npart = 0\n\nclass CFG:\n    seed = 42\n    n_splits = 4","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:34.437052Z","iopub.execute_input":"2024-12-18T18:22:34.438006Z","iopub.status.idle":"2024-12-18T18:22:34.443818Z","shell.execute_reply.started":"2024-12-18T18:22:34.437955Z","shell.execute_reply":"2024-12-18T18:22:34.442344Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_folds = train_df.copy()\ntrain_folds['fold']=-1\n\nnum_bins = 5\n\n# train_folds = train_folds.sample(frac=1, random_state=CFG.seed).reset_index(drop=True)\n# gkfold = StratifiedGroupKFold(n_splits=CFG.n_splits, shuffle=True, random_state=CFG.seed) \n# print(f'{CFG.n_splits}fold training', len(train_folds), 'samples')\n# for fold_idx, (train_idx, valid_idx) in enumerate(gkfold.split(train_folds, y=train_folds['sign'].values, groups=train_folds.participant_id)):\n#     train_folds.loc[valid_idx,'fold'] = fold_idx\n#     print(f'fold{fold_idx}:', 'train', len(train_idx), 'valid', len(valid_idx))\nkfold = KFold(n_splits=CFG.n_splits, shuffle=True, random_state=CFG.seed) \nprint(f'{CFG.n_splits}fold training', len(train_folds), 'samples')\nfor fold_idx, (train_idx, valid_idx) in enumerate(kfold.split(train_folds)):\n    train_folds.loc[valid_idx,'fold'] = fold_idx\n    print(f'fold{fold_idx}:', 'train', len(train_idx), 'valid', len(valid_idx))\n    \nassert not (train_folds['fold']==-1).sum()\nassert len(np.unique(train_folds['fold']))==CFG.n_splits\ntrain_folds.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:35.980242Z","iopub.execute_input":"2024-12-18T18:22:35.980644Z","iopub.status.idle":"2024-12-18T18:22:36.030087Z","shell.execute_reply.started":"2024-12-18T18:22:35.980609Z","shell.execute_reply":"2024-12-18T18:22:36.028845Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nimport json\nimport os\n\nDATASET_NAME = f'ISLR-{CFG.n_splits}fold-randsplit'\n### Create Kaggle Dataset if not exists \n# DATASET_NAME='nearestls'\n\n!rm -rf /tmp/{DATASET_NAME}\n\nos.makedirs(f'/tmp/{DATASET_NAME}', exist_ok=True)\n\nwith open('/kaggle/input/my-own-kaggle-key/kaggle.json') as f:\n    kaggle_creds = json.load(f)\n    \nos.environ['KAGGLE_USERNAME'] = kaggle_creds['username']\nos.environ['KAGGLE_KEY'] = kaggle_creds['key']\n\n!kaggle datasets init -p /tmp/{DATASET_NAME}\n\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json') as f:\n    dataset_meta = json.load(f)\n\ndataset_meta['id'] = f'carolineyyy/{DATASET_NAME}'\ndataset_meta['title'] = DATASET_NAME\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json', \"w\") as outfile:\n    json.dump(dataset_meta, outfile)\nprint(dataset_meta)\n\n!cp /tmp/{DATASET_NAME}/dataset-metadata.json /tmp/{DATASET_NAME}/meta.json\n!ls /tmp/{DATASET_NAME}\n\n!kaggle datasets create -p /tmp/{DATASET_NAME} --public","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:22:50.653487Z","iopub.execute_input":"2024-12-18T18:22:50.654391Z","iopub.status.idle":"2024-12-18T18:22:57.869577Z","shell.execute_reply.started":"2024-12-18T18:22:50.654340Z","shell.execute_reply":"2024-12-18T18:22:57.867879Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Put every image in a seperate TFRecord file\n# Make Pairs of Views as input to the model\n\n\ndef split_dataframe(df, chunk_size = 10000): \n    chunks = list()\n    num_chunks = len(df) // chunk_size + 1\n    for i in range(num_chunks):\n        chunks.append(df[i*chunk_size:(i+1)*chunk_size])\n    return chunks\n\nfor fold in range(CFG.n_splits):#[FOLD]:#range(CFG.n_splits):\n    rows = train_folds[train_folds['fold']==fold]\n    chunks = split_dataframe(rows, CHUNK_SIZE)\n    part_size = len(chunks)//N_PART\n    last = (part+1)*part_size if part != N_PART - 1 else len(chunks)+1\n    chunks = chunks[part*part_size:last]\n    \n    N = [len(x) for x in chunks]\n    _ = Parallel(n_jobs=cpu_count())(\n        delayed(process_chunk)(x, f'/tmp/{DATASET_NAME}/fold{fold}-{i}-{n}.tfrecords')\n        for i,(x,n) in enumerate(zip(chunks,N))\n    )","metadata":{"execution":{"iopub.status.busy":"2024-12-18T18:23:08.835258Z","iopub.execute_input":"2024-12-18T18:23:08.836434Z","iopub.status.idle":"2024-12-18T18:44:55.543820Z","shell.execute_reply.started":"2024-12-18T18:23:08.836380Z","shell.execute_reply":"2024-12-18T18:44:55.541588Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import datetime\nversion_name = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\nprint(version_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T18:44:55.547228Z","iopub.execute_input":"2024-12-18T18:44:55.547696Z","iopub.status.idle":"2024-12-18T18:44:55.555359Z","shell.execute_reply.started":"2024-12-18T18:44:55.547645Z","shell.execute_reply":"2024-12-18T18:44:55.554068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!kaggle datasets version -m {version_name} -p /tmp/{DATASET_NAME} -r zip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T18:44:55.557236Z","iopub.execute_input":"2024-12-18T18:44:55.557746Z","iopub.status.idle":"2024-12-18T18:59:40.016022Z","shell.execute_reply.started":"2024-12-18T18:44:55.557693Z","shell.execute_reply":"2024-12-18T18:59:40.012915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n# print(KaggleDatasets().get_gcs_path('islr-5fold'))\n# print(KaggleDatasets().get_gcs_path('islr-5fold-randsplit'))\n# print(KaggleDatasets().get_gcs_path('islr-5fold-fold0'))\n# print(KaggleDatasets().get_gcs_path('islr-5fold-fold1'))\n# print(KaggleDatasets().get_gcs_path('islr-5fold-fold2'))\n# print(KaggleDatasets().get_gcs_path('islr-5fold-fold3'))\n# print(KaggleDatasets().get_gcs_path('islr-5fold-fold4'))\n# print(KaggleDatasets().get_gcs_path('ISLR-4fold-randsplit'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T18:59:40.022701Z","iopub.execute_input":"2024-12-18T18:59:40.023377Z","iopub.status.idle":"2024-12-18T18:59:40.053611Z","shell.execute_reply.started":"2024-12-18T18:59:40.023324Z","shell.execute_reply":"2024-12-18T18:59:40.052055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}