{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow_addons as tfa\nfrom helper_functions import *\nfrom preprocessing_functions import *\nfrom utility_functions import *\nfrom tensorflow.keras import layers, models\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\n# Define the data directories\ndata_dir = '/kaggle/input/g2net-gravitational-wave-detection'\n\n# related to the train data\ntrain_dir = os.path.join(data_dir, 'train')\ntrain_labels_path = os.path.join(data_dir, 'training_labels.csv')\n\n# related to the test data\ntest_dir = os.path.join(data_dir, 'test')\n\n# working dir\n\nworking_dir = os.path.join('/kaggle/working/')\ntmp_dir = os.path.join('/kaggle/tmp/')\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output,\n# when you create a version using \"Save & Run All\"\n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"4ae2507b-bec0-4482-924e-6b96d1bfb837","_cell_guid":"749bf022-34a5-4107-8088-5e373adc3f8e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:27.182801Z","iopub.execute_input":"2021-09-29T07:13:27.183157Z","iopub.status.idle":"2021-09-29T07:13:27.190207Z","shell.execute_reply.started":"2021-09-29T07:13:27.183122Z","shell.execute_reply":"2021-09-29T07:13:27.189385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################################################\n# USEFUL VARIABLES RELATED TO THE TRAIN DATA\n###########################################################################################\n\n# get the paths of train directories\ntrain_files_path = list()\nfor dir_path, dirs, dir_files in os.walk(train_dir):\n    train_files_path.extend([os.path.join(dir_path, file) for file in dir_files])\n\n# Analyse the train labels.csv\ntrain_labels = pd.read_csv(train_labels_path, index_col='id')\n\ntrain_labels_df = train_labels\n\n# generate map of ids as keys and paths as values for each target\n\ntrain_target_0 = train_labels_df[train_labels_df['target'] == 0].index.to_list()\ntrain_target_1 = train_labels_df[train_labels_df['target'] == 1].index.to_list()\n\ntrain_target_0_dict = {item: '' for item in train_target_0}\ntrain_target_1_dict = {item: '' for item in train_target_1}\n\n# iterate over the directories and assign the paths to each ids for each label 1 and 0\nfor file_path in train_files_path:\n    f_id = os.path.basename(file_path)[:-4]\n    if f_id in train_target_0_dict:\n        train_target_0_dict[f_id] = file_path\n    if f_id in train_target_1_dict:\n        train_target_1_dict[f_id] = file_path\n\ntrain_target_0_paths = train_target_0_dict.values()\ntrain_target_1_paths = train_target_1_dict.values()\ntotal_train_files = len(train_files_path)\n\ntrain_ids = list(train_target_0_dict.keys())\ntrain_ids.extend(list(train_target_1_dict.keys()))\nprint(len(train_ids))\nprint(len(train_files_path))\n\n\n###########################################################################################\n# USEFUL VARIABLES RELATED TO THE TEST DATA\n###########################################################################################\n\ntest_files_paths = list()\nfor dir_path, dirs, f_names in os.walk(test_dir):\n    test_files_paths.extend([os.path.join(dir_path, f_name) for f_name in f_names])\n\ntotal_test_files = len(test_files_paths)","metadata":{"_uuid":"097af657-6d23-4f3a-9af6-a2cb5825382e","_cell_guid":"db189024-2509-4e4b-92af-a761e40163f1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:28.009062Z","iopub.execute_input":"2021-09-29T07:13:28.009393Z","iopub.status.idle":"2021-09-29T07:13:37.649242Z","shell.execute_reply.started":"2021-09-29T07:13:28.009361Z","shell.execute_reply":"2021-09-29T07:13:37.648382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EDA**","metadata":{"_uuid":"3d6e37d9-b116-4191-bcb1-409f70660133","_cell_guid":"9a653e96-6975-4b8f-9af3-52f9e187d9c1","trusted":true}},{"cell_type":"code","source":"###########################################################################################\n# EXPLORATORY DATA ANALYSIS\n###########################################################################################\n\n# Analyse the file counts in sub directories of train dir and test dir\n# Total files in all the train sub directories\nprint('Total files in all the train sub directories: {}'.format(total_train_files))\n\n# Total files test sub directories\nprint('Total files in all the test sub directories: {}'.format(total_test_files))\n\n# check if the total files in all the sub directories is equal to the train lables\nprint(\"Are Length of train labels and train files equal: {}\".format(total_train_files == train_labels.shape[0]))\n\n# check if all the ids present in the train directories are in the train_labels_df\ndir_ids = [os.path.basename(file_path)[:-4] for file_path in train_files_path]\ntrain_ids = train_labels_df.index.to_list()\nprint('Are ids matching in train set and labels set: {}'.format(set(dir_ids) == set(train_ids)))\ntrain_ids_set = set(dir_ids)\ntrain_label_ids_set = set(train_ids)\nprint(len(train_ids_set))\nprint(len(train_label_ids_set))\nprint('Num of Matching ids: {}'.format(len(train_ids_set.intersection(train_label_ids_set))))\nprint('Num of ids present in train dirs but not in labels: {}'.format(len(train_ids_set - train_label_ids_set)))\nprint('Num of ids present in train labels but not in train dirs: {}'.format(len(train_label_ids_set - train_ids_set)))\n\n#######################################################################################################################","metadata":{"_uuid":"3bab5486-03af-4e5c-a5e1-e96ef7032709","_cell_guid":"62dd931c-b323-4d98-9c1b-bdd79278bc1f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:37.650721Z","iopub.execute_input":"2021-09-29T07:13:37.651066Z","iopub.status.idle":"2021-09-29T07:13:38.688298Z","shell.execute_reply.started":"2021-09-29T07:13:37.651031Z","shell.execute_reply":"2021-09-29T07:13:38.687282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#############################################################################################\n# check for data imbalance. In other words check for the total number of 1's and 0's\n#############################################################################################\n# Analyse the total number of the 1's and 0's in the train labels\nprint(type(train_labels_df.target))\nlabels_cnt_0 = train_labels_df[train_labels_df.target == 0].count()\nlabels_cnt_1 = train_labels_df[train_labels_df.target == 1].count()\nlabels_cnt_nan = train_labels_df[train_labels_df.target == np.nan].count()\nprint(\"Total number of 0's in the train labels: {}\".format(labels_cnt_0))\nprint(\"Total number of 1's in the train labels: {}\".format(labels_cnt_1))\nprint(\"Total number of NaN's in the train labels: {}\".format(labels_cnt_nan))\n\nprint(\"Proportion of 1's in train labels: {}\".format(round((labels_cnt_1 * 100 / (labels_cnt_1 + labels_cnt_0)), 2)))\nprint('Are label count matching total row count: {}'\n      .format((labels_cnt_0 + labels_cnt_1 + labels_cnt_nan) == train_labels_df.shape[0]))\n\n\n##############################################################################################","metadata":{"_uuid":"2075cc3d-a50a-4a2d-9ef5-4d68845e9460","_cell_guid":"44cf632b-1b53-4f77-a576-33a6fbf24e01","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:38.690264Z","iopub.execute_input":"2021-09-29T07:13:38.690614Z","iopub.status.idle":"2021-09-29T07:13:38.736924Z","shell.execute_reply.started":"2021-09-29T07:13:38.690575Z","shell.execute_reply":"2021-09-29T07:13:38.735958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remarks**  \n\nIn the training set, total number of 0's and 1's are matching. There is **NO Class Imbalance**\nLabels 0's and 1's are almost equally distributed ~ 50%","metadata":{"_uuid":"4c2e196c-749f-46fe-9d89-bd9c5aa142c5","_cell_guid":"918b2773-660f-418d-88e0-a5fa74b4e3cd","trusted":true}},{"cell_type":"code","source":"sample_1 = np.load(train_files_path[0])\nsample_2 = np.load(train_files_path[1])\nsample_3 = np.load(train_files_path[2])\nsample_4 = np.load(train_files_path[3])\nbatch = list()\nbatch.append(sample_1)\nbatch.append(sample_2)\nbatch.append(sample_3)\nbatch.append(sample_4)\n\nsample_batch = np.array(batch)\nprint(sample_batch.shape)\n\ndef this_cross_correlation(ts1,ts2, backwards=False, adjusted=False, is_batch=False):\n    \"\"\"\n    This function is based on statsmodel package which computes only \n    forward or positive side of cross correlation. The below function \n    is made to compute both sides of lags. \n    \"\"\"\n    import statsmodels.api as sm\n    def _cross_correlation(ts1,ts2, backwards=backwards, adjusted=adjusted):\n        if not backwards:\n            return sm.tsa.stattools.ccf(ts1, ts2, adjusted=adjusted)\n\n        backwards = sm.tsa.stattools.ccf(ts1[::-1], ts2[::-1], adjusted=adjusted)[::-1]\n        forwards = sm.tsa.stattools.ccf(ts1, ts2, adjusted=adjusted)\n        ccf = np.r_[backwards[:-1], forwards]\n        return ccf\n\n    if not is_batch:\n        return _cross_correlation(ts_1, ts_2)\n\n    if not isinstance(ts1, np.ndarray) or not isinstance(ts2, np.ndarray):\n        raise TypeError(\"args: 'ts1'/'ts2' has to be of type 'np.ndarray'\")\n\n    if ts1.shape != ts2.shape:\n        raise AssertionError(\"'ts1.shape: {} not matched with ts2.shape: {}'.format(ts1.shape, ts2.shape)\")\n\n    data_to_return = [_cross_correlation(ts_1, ts_2) for ts_1, ts_2 in zip(ts1, ts2)]\n    return np.array(data_to_return)\n\n\ndef execute_pipeline(data, is_batch=True):\n    if not isinstance(data, np.ndarray) and not tf.is_tensor(data):\n        raise TypeError(\"'arg: data' should be of type 'np.ndarray' or 'Tensor'\")\n        \n    std_ts = standardize_time_series(data)\n    lstm = convert_2d_to_3d_lstm(std_ts)\n    return convert_to_tensor(lstm)\n\n# hello = execute_pipeline(sample_batch)\n\nstd_ts = standardize_time_series(sample_batch, is_batch=True)\nc12 = this_cross_correlation(std_ts[:,0,:], std_ts[:,1,:], is_batch=True, backwards=True)\nc13 = this_cross_correlation(std_ts[:,0,:], std_ts[:,2,:], is_batch=True, backwards=True)\nc23 = this_cross_correlation(std_ts[:,1,:], std_ts[:,2,:], is_batch=True, backwards=True)\nprint(c12[:5])\nprint(c13[:5])\nprint(c23[:5])\n\ncc = np.concatenate([c12, c13, c23], axis=1)\nnum_batches, data_len = cc.shape\ncc = cc.reshape((num_batches,3,-1))\ncc = numpy_cross_correlation(cc.tolist(), is_batch=True)\nprint(cc)","metadata":{"_uuid":"4c78d8a9-7c05-4d4d-9c1c-3e44b09aa509","_cell_guid":"f9d05558-9de9-403b-890c-d65fd5185c5c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:38.738375Z","iopub.execute_input":"2021-09-29T07:13:38.738919Z","iopub.status.idle":"2021-09-29T07:13:38.97087Z","shell.execute_reply.started":"2021-09-29T07:13:38.738879Z","shell.execute_reply":"2021-09-29T07:13:38.969995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"_uuid":"c94c8a09-3eb0-4222-901b-c0fb583d03e6","_cell_guid":"98cccb5f-7be9-4c8f-8568-8f851f1ae131","trusted":true}},{"cell_type":"markdown","source":"**Find the cross correlation between the signals from the 3 signals for one sample**","metadata":{"_uuid":"ad2ba0b4-77f4-4978-b410-6639c7271394","_cell_guid":"02a8e817-73f8-4ca3-9f2a-aee60ffbcce0","trusted":true}},{"cell_type":"code","source":"##############################################################################################\n# Helper functions\n##############################################################################################\ndef flow_alternate_binary_proportions(label_proportions, label_files, batch_size, shuffle=True, repeat=True):\n    if len(label_proportions) != 2:\n        raise AssertionError(\"Expected num of label proportions:2, found: {}\".format(len(label_proportions)))\n\n    if len(label_files) != 2:\n        raise AssertionError(\"Expected num of label files:2, found: {}\".format(len(label_files)))\n\n    def _swap_labels(lbl_props, lbl_files):\n        label_x, label_y = lbl_props.keys()\n        lbl_prop_x, lbl_prop_y = lbl_props.values()\n        lbl_props[label_x] = lbl_prop_y\n        lbl_props[label_y] = lbl_prop_x\n\n        lbl_files_x, lbl_files_y = lbl_files.values()\n        lbl_files[label_x] = lbl_files_y\n        lbl_files[label_y] = lbl_files_x\n        return lbl_props, lbl_files\n\n    total_files = 0\n    num_batches = total_files // batch_size\n    batch_count = 0\n    data_processed = 0\n    np_array = list()\n    while batch_count < num_batches:\n        label_proportions, label_files = _swap_labels(label_proportions, label_files)\n        batch_files, labels = batch_producer(label_proportions, label_files, batch_size, shuffle)\n        for file in batch_files:\n            img = np.load(file)\n            np_array.append(img)\n            data_processed += 1\n            if data_processed % batch_size == 0:\n                batch_count += 1\n                img_batch = np.array(np_array)\n                preprocessed_img_batch = img_batch\n                yield tf.expand_dims(preprocessed_img_batch, -1), np.array(labels)\n                np_array = list()\n\n            if batch_count == num_batches and repeat:\n                batch_count = 0\n\n\ndef flow_data_proportions(label_proportions, label_files, batch_size, shuffle=True, repeat=True):\n    total_files = 0\n    for label, files in label_files.items():\n        total_files += len(files)\n\n    num_batches = total_files // batch_size\n    batch_count = 0\n    data_processed = 0\n    np_array = list()\n    while batch_count < num_batches:\n        batch_files, labels = batch_producer(label_proportions, label_files, batch_size, shuffle)\n        for file in batch_files:\n            img = np.load(file)\n            np_array.append(img)\n            data_processed += 1\n            if data_processed % batch_size == 0:\n                batch_count += 1\n                img_batch = np.array(np_array)\n                preprocessed_img_batch = img_batch\n                yield tf.expand_dims(preprocessed_img_batch, -1), np.array(labels)\n                np_array = list()\n\n            if batch_count == num_batches and repeat:\n                batch_count = 0\n\n\ndef flow_test_data(test_files):\n    if not isinstance(test_files, list):\n        raise TypeError(\"arg: 'test_files' is not of type 'list'\")\n    for img_file in test_files:\n        img = np.load(img_file)\n        preprocessed_img = img\n        exp_dims_img = tf.expand_dims(preprocessed_img, axis=-1)\n        exp_dims_img = tf.expand_dims(exp_dims_img, axis=0)\n        yield exp_dims_img\n\n\n#################################################################################################","metadata":{"_uuid":"ebeeb5fd-afed-43b6-9cbb-35e1338e530d","_cell_guid":"39066570-9841-474c-9a7a-55aa34989202","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:38.972312Z","iopub.execute_input":"2021-09-29T07:13:38.972756Z","iopub.status.idle":"2021-09-29T07:13:38.988547Z","shell.execute_reply.started":"2021-09-29T07:13:38.972711Z","shell.execute_reply":"2021-09-29T07:13:38.987324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##############################################################################################\n# Generate auxiliary data\n##############################################################################################\ndef generate_correlation_auxiliary_data(csv_file_name, files_paths, batch_size, columns=None):\n    if columns is None:\n        columns = ['corr12', 'corr13', 'corr23']\n\n    if len(columns) != 3:\n        raise AssertionError(\"Expected columns size: 3, found:{}\".format(len(columns)))\n\n    corr_cols = columns\n\n    if batch_size > 10000:\n        raise AssertionError(\"Batch size cannot be greater than 10000\")\n\n    aux_batch_size = batch_size\n\n    if len(files_paths) <= 0:\n        raise AssertionError(\"Files Paths must be greater than 0\")\n\n    aux_data_path = os.path.join(working_dir, csv_file_name + '.csv')\n    \n    if os.path.exists(aux_data_path):\n        os.remove(aux_data_path)\n    \n    if not os.path.exists(aux_data_path):\n        header_df = pd.DataFrame(columns=corr_cols)\n        header_df.to_csv(aux_data_path, mode='w')\n \n    expected_batches = len(files_paths)//aux_batch_size\n    \n    if len(files_paths) % aux_batch_size > 0:\n        expected_batches += 1\n\n    for files_batch in tqdm(generate_files_batch(files=files_paths, batch_size=aux_batch_size, drop_remaining=False), total=expected_batches):\n        data = list()\n        for file in files_batch:\n            datum = np.load(file)\n            data.append(datum)\n        \n        \n        std_ts = standardize_time_series(np.array(data), is_batch=True)\n        c12 = this_cross_correlation(std_ts[:,0,:], std_ts[:,1,:], is_batch=True, backwards=True)\n        c13 = this_cross_correlation(std_ts[:,0,:], std_ts[:,2,:], is_batch=True, backwards=True)\n        c23 = this_cross_correlation(std_ts[:,1,:], std_ts[:,2,:], is_batch=True, backwards=True)\n\n        cc = np.concatenate([c12, c13, c23], axis=1)\n        num_batches, data_len = cc.shape\n        batch_corr_matrix = cc.reshape((num_batches,3,-1))\n        batch_corr_matrix = numpy_cross_correlation(batch_corr_matrix.tolist(), is_batch=True)\n        batch_file_ids = [os.path.basename(f_path[:-4]) for f_path in files_batch]\n        batch_corr_df = pd.DataFrame(data=batch_corr_matrix, columns=corr_cols, index=batch_file_ids)\n        if os.path.exists(aux_data_path):\n            batch_corr_df.to_csv(aux_data_path, header=False, mode='a')\n\n    return pd.read_csv(aux_data_path)\n\n\nprint(\"generating train auxiliary data\")\ntrain_aux_df = generate_correlation_auxiliary_data('train_auxiliary_data', train_files_path, batch_size=100)\nprint(\"generating test auxiliary data\")\ntest_aux_df = generate_correlation_auxiliary_data('test_auxiliary_data', test_files_paths, batch_size=100)\n\n##############################################################################################","metadata":{"_uuid":"4d9de1d1-3b75-47ac-8964-5e4d2ff157a4","_cell_guid":"2f67c96c-8c39-4d17-b401-9f450f4dd2cf","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T08:27:57.077601Z","iopub.execute_input":"2021-09-29T08:27:57.077994Z","iopub.status.idle":"2021-09-29T08:27:57.220297Z","shell.execute_reply.started":"2021-09-29T08:27:57.077913Z","shell.execute_reply":"2021-09-29T08:27:57.219175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from zipfile import ZipFile\n\nfile1 = os.path.join(working_dir, 'train_auxiliary_data.csv')\nfile2 = os.path.join(working_dir, 'test_auxiliary_data.csv')\n\nzip_file = os.path.join(working_dir, 'auxiliary_data.zip')\n # writing files to a zipfile\nwith ZipFile(zip_file,'w') as zip:\n    # writing each file one by one\n    zip.write(file1)\n    zip.write(file2)\n\nprint('All files zipped successfully!')     ","metadata":{"execution":{"iopub.status.busy":"2021-09-29T07:13:23.848671Z","iopub.status.idle":"2021-09-29T07:13:23.849404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#################################################################################################\n# Model\n#################################################################################################\n\nmodel = models.Sequential()\nmodel.add(tf.keras.Input(shape=(None, 3)))\nmodel.add(layers.Dense(16, activation='relu', kernel_constraint=tf.keras.constraints.MaxNorm(3)))\nmodel.add(layers.Dropout(0.4))\nmodel.add(layers.Dense(3, activation='relu', kernel_constraint=tf.keras.constraints.MaxNorm(3)))\nmodel.add(layers.Dropout(0.4))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n\noptimizer = tf.keras.optimizers.Adam(lr=3e-4)\nmodel.compile(optimizer=optimizer,\n              loss=tf.keras.losses.BinaryCrossentropy(),\n              metrics=[tf.keras.metrics.BinaryAccuracy(), tf.keras.metrics.AUC()])\nmodel.summary()\n###################################################################################################","metadata":{"_uuid":"794af3aa-3777-4e7c-95bb-b3871df874aa","_cell_guid":"7c9d3876-39d3-4a30-9357-6b64f18ac6ed","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:23.850512Z","iopub.status.idle":"2021-09-29T07:13:23.85572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n####################################################################################################\n# Generate Data\ntrain_label_proportions = {0: 0.5, 1: 0.5}\ntrain_label_files = {0: train_target_0_paths, 1: train_target_1_paths}\n\n\ntrain_1, train_cv_1 = cross_validation_split(files=list(train_label_files[1]), proportions=(0.8, 0.2))\ntrain_0, train_cv_0 = cross_validation_split(files=list(train_label_files[0]), proportions=(0.8, 0.2))\n\n# sample test data from the train data for model evaluation\n# generate random number between 0 to 100 for test label 1\n\ntest_proportion_0 = round(random.random(), 2)\ntest_proportion_1 = 1 - test_proportion_0\ntest_label_proportions = {0: test_proportion_0, 1: test_proportion_1}\n\ntest_x_0 = random_sample(train_label_files[0], percent=test_proportion_0 * 100)\ntest_x_1 = random_sample(train_label_files[1], percent=test_proportion_1 * 100)\ntest_label_files = {0: test_x_0, 1: test_x_1}\n\n####################################################################################################","metadata":{"_uuid":"8e80781e-c823-44b8-8099-3d1431418308","_cell_guid":"58341a33-bf3b-452d-9db9-19ecaca1413a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:23.857435Z","iopub.status.idle":"2021-09-29T07:13:23.858485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################################\n# model training\n####################################################################################################\nnum_epochs = 1\nbatch_size = 64\n\n# call backs\nes_cb = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3, restore_best_weights=True)\ntqdm_callback = tfa.callbacks.TQDMProgressBar()\n\n# Create a callback that saves the model's weights every 2 epochs\ncheck_point_path = os.path.join(working_dir, \"cp-{epoch:04d}.ckpt\")\ncp_callback = tf.keras.callbacks.ModelCheckpoint(\n    filepath=check_point_path,\n    monitor='loss',\n    verbose=1,\n    save_weights_only=True,\n    save_freq='epoch')\n\nmy_callbacks = [es_cb, cp_callback, tqdm_callback]\n\nmodel.save_weights(check_point_path.format(epoch=0))\n\ntrain_history = model.fit(\n    flow_data_proportions(train_label_proportions, label_files={0: train_0, 1: train_1}, batch_size=batch_size,\n                          shuffle=True),\n    validation_data=flow_data_proportions(train_label_proportions, label_files={0: train_cv_0, 1: train_cv_1},\n                                          batch_size=batch_size, shuffle=True),\n\n    validation_steps=(len(train_cv_0) + len(train_cv_1)) // batch_size,\n    epochs=num_epochs,\n    steps_per_epoch=(len(train_0) + len(train_1)) // batch_size,\n    verbose=2,\n    callbacks=my_callbacks)\n\nmodel.save(os.path.join(working_dir, 'model_1_v1.h5'))","metadata":{"_uuid":"cc32c0d9-9b90-4e97-a2f1-dfa61e36ae99","_cell_guid":"e47c6b07-973e-4d25-870c-eb804bcc74be","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:23.860049Z","iopub.status.idle":"2021-09-29T07:13:23.867164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################\n# plot train charts\nx_positions = pd.DataFrame(data=np.arange(num_epochs), columns=['epochs'])\nhistory_df = pd.DataFrame(data=train_history.history)\nhistory_df = history_df.join(x_positions)\nsns.lineplot(x=\"epochs\", y='value', hue='variable', data=pd.melt(history_df, [\"epochs\"]))\n#########################################################################################","metadata":{"_uuid":"70f8d547-5142-49ed-b641-29c2be8084be","_cell_guid":"5a13313b-9556-469a-95af-b22c38f8d533","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:23.86821Z","iopub.status.idle":"2021-09-29T07:13:23.868989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################\n# Evaluate  the model on the test data\nmodel.load_weights(os.path.join(working_dir, 'model_1_v1.h5'))\n\nresults = model.evaluate(\n    flow_data_proportions(test_label_proportions, test_label_files, batch_size=batch_size,\n                          shuffle=True, repeat=False),\n)\nprint(\"test loss :{}, test acc:{}\".format(results[0], results[1]))\n#########################################################################################","metadata":{"_uuid":"2f394368-75f5-434e-848d-482a591d9f4a","_cell_guid":"9449bd0a-7002-4700-a8d3-7bc82101bf31","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:23.870057Z","iopub.status.idle":"2021-09-29T07:13:23.870971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################\n# predictions\nmodel.load_weights(os.path.join(working_dir, 'model_1_v1.h5'))\npredictions = model.predict(flow_test_data(test_files=test_files_paths))\npredictions = remove_dimension(array=predictions)\ntest_file_ids = [os.path.basename(file)[:-4] for file in test_files_paths]\n##########################################################################################","metadata":{"_uuid":"4d4cb368-7499-44a1-864c-30c1da00738f","_cell_guid":"6eb3e69f-1337-400b-99fb-54c6048e5cde","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:23.872299Z","iopub.status.idle":"2021-09-29T07:13:23.873176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################################################\n# submission section\nsubmission_df = pd.DataFrame({'id': test_file_ids, 'target': np.array(predictions)})\nsubmission_df.to_csv('submission.csv', index=False)\n###########################################################################################","metadata":{"_uuid":"8b9f6535-31f4-495d-bfce-8ff696349e1e","_cell_guid":"c4c7207d-47dc-4bd1-9710-f430fe81db22","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-29T07:13:23.875113Z","iopub.status.idle":"2021-09-29T07:13:23.875846Z"},"trusted":true},"execution_count":null,"outputs":[]}]}