{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9818336,"sourceType":"datasetVersion","datasetId":5994608}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":11045.499535,"end_time":"2024-10-29T18:50:44.133491","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-29T15:46:38.633956","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"13fb9aae-bcf1-4370-81b3-6a520cae6cb9","cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\nfrom sklearn.metrics import cohen_kappa_score\nfrom scipy import stats\n\nfrom lightgbm import log_evaluation, early_stopping\nimport lightgbm as lgb\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":5.220239,"end_time":"2024-10-29T15:46:48.065265","exception":false,"start_time":"2024-10-29T15:46:42.845026","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"cf6e5d4e-3ee2-4772-b6a2-e37217ce8ec9","cell_type":"markdown","source":"### Parameters","metadata":{"papermill":{"duration":0.019118,"end_time":"2024-10-29T15:46:48.104542","exception":false,"start_time":"2024-10-29T15:46:48.085424","status":"completed"},"tags":[]}},{"id":"2fa29719-a5f1-4bf8-ab1b-b7a9c499c611","cell_type":"code","source":"\ngap_threshold = 5 # When looking at time series data, any gap larger than this parameter is considered a break in the sequence\nsequence_length = int(np.ceil ((60/gap_threshold) * 60*24)) # max possible number of reads in one day for the specific gap_threshold\n\n\n\n#### Time Series parameters ############\n\n#ts_aggregations = ['std', 'mean', 'min', 'max', 'skew'] # what aggregations we want to do on the parquet time series data\nts_aggregations = ['mean'] # ''range', 'std', 'mean', 'min', 'max', 'skew']\npivot_features = [\"week_hour\"] #[ \"3h\", \"weekday\"] # how do we want to pivot the time series data\n\nts_aggregations_diff = [ 'std'] #'std', 'mean'] #, 'min', 'max', 'skew','range',]\npivot_features_diff = [\"week_hour\"]\n\nfilter_wear_flag_on = False\n\n                      \n\ntime_series_aggregation_on = True\nmax_sequence_breaks = 1000000000\nts_params =  {'colsample_bytree': 0.9601929352092674, 'learning_rate': 0.2347885187747232, 'max_depth': 11.757995766256757, 'reg_alpha': 1.7907064273510211, 'reg_lambda': 1.1998209578340595}\nts_params =  {'colsample_bytree': 0.01, 'learning_rate': 0.3, 'max_depth': 11.313660926264719, 'reg_alpha': 0.46678721120685024, 'reg_lambda': 0.306473629655206}\n\nts_params =  {'colsample_bytree': 0.6053939548634096, 'learning_rate': 0.059451995869314544, 'max_depth': 5.715567522838075, 'reg_alpha': 1.898282219134133, 'reg_lambda': 1.9316077458183731}\n\n#skew\nts_params =  {'colsample_bytree': 0.01, 'learning_rate': 0.01, 'max_depth': 12.879810403438045, 'reg_alpha': 0.01, 'reg_lambda': 2.0}\n\n#mean+std\nts_params =  {'colsample_bytree': 0.13540154453539682, 'learning_rate': 0.1523011064987601, 'max_depth': 9.862303059496922, 'reg_alpha': 1.4151776808991248, 'reg_lambda': 1.0308510302118108}\n\nts_pbounds ={ 'learning_rate':        (0.01, 0.3),\n            'max_depth':            (5, 16) ,\n           # 'num_leaves':           (6,25),\n            'colsample_bytree':     (0.01, 0.99),\n            'reg_alpha':     (0.01, 2),\n            'reg_lambda':     (0.01, 2)\n            }\n\n\ntime_series_diff_aggregation_on = True\nmax_sequence_breaks_diff = 1000000000\n#ts_params_diff =  {'colsample_bytree': 0.7470338431149504, 'learning_rate': 0.10625217855619423, 'max_depth': 8.72568252920024, 'reg_alpha': 0.3159450522178731, 'reg_lambda': 0.12064449083256262}\n#ts_params_diff =   {'colsample_bytree': 0.13388348095481123, 'learning_rate': 0.04613209037998326, 'max_depth': 11.426714521456175, 'reg_alpha': 0.5846646649161501, 'reg_lambda': 0.6682133410804093}\n\nts_params_diff =   {'colsample_bytree': 0.5670036758072614, 'learning_rate': 0.15994593335479826, 'max_depth': 10.686235274715159, 'reg_alpha': 0.8874276627799633, 'reg_lambda': 0.9607550687386944}\n\n#std\nts_params_diff =  {'colsample_bytree': 0.3397164927400806, 'learning_rate': 0.06957188299302025, 'max_depth': 13.35387092281375, 'reg_alpha': 0.0971940790593206, 'reg_lambda': 0.8783974774996233}\n\n#skew\nts_params_diff =   {'colsample_bytree': 0.4554183749179259, 'learning_rate': 0.2916617519325446, 'max_depth': 6.299859413190621, 'reg_alpha': 0.13048542017544976, 'reg_lambda': 0.2230622093441913}\n\n\n\n#mean+std\nts_params_diff =   {'colsample_bytree': 0.9134367503226545, 'learning_rate': 0.03566282559505665, 'max_depth': 7.155811486610597, 'reg_alpha': 0.05432274313232731, 'reg_lambda': 0.32882372414799904}\nts_diff_pbounds = { 'learning_rate':        (0.01, 0.3),\n            'max_depth':            (5, 16) ,\n           # 'num_leaves':           (6,25),\n            'colsample_bytree':     (0.01, 0.99),\n            'reg_alpha':     (0.01, 0.99),\n            'reg_lambda':     (0.01, .99)\n            }\n\nbayesian_optimization_ts = True\n\n\n\n#### Feature Engineering parameters ############\n\nautoencoder_on =  False\nimputation_on =  False\npca_components = 50\n\nauto_encoder_epochs = 1000\nauto_encoder_dimensions = 50\n\n\n#### Bayesian parameters ############\n\nbayesian_init_rounds =  15\nbayesian_opt_rounds = 25\n\n#### LGBM parameters ############\n\nstepwise_features = -1 # if >0, we perform stepwise feature selection\n\nbayesian_optimization = True\n\n#lgbm_bayesian_params= {'colsample_bytree': 0.6823794822105649, 'learning_rate': 0.2301614731732811, 'max_depth': 10.801625849761404, 'reg_alpha': 1.6874131077909975, 'reg_lambda': 0.5307186645018772}\nlgbm_bayesian_params= {'colsample_bytree': 0.5392080638622293, 'learning_rate': 0.22346026672581432, 'max_depth': 6.986179522202823, 'reg_alpha': 2.296985388146088, 'reg_lambda': 0.9183713307836241}\nlgbm_bayesian_params={'colsample_bytree': 0.5375004629293539, 'learning_rate': 0.12203738871670142, 'max_depth': 5.0, 'reg_alpha': 2.6771459038523795, 'reg_lambda': 2.0405495674993808}\n\n\n#skew new LGBM bayesian params: \nlgbm_bayesian_params={'colsample_bytree': 0.5649831878655511, 'learning_rate': 0.15351602445837612, 'max_depth': 10.684214684783765, 'reg_alpha': 2.6808851558840576, 'reg_lambda': 2.904608567941819}\n\n#mean and std\nlgbm_bayesian_params={'colsample_bytree': 0.9093996009218469, 'learning_rate': 0.11632189631258541, 'max_depth': 12.279550746843778, 'reg_alpha': 0.8658183467578231, 'reg_lambda': 0.4743461498510985}\nlgbm_pbounds = { 'learning_rate':        (0.001, 0.3),\n            'max_depth':            (5, 16) ,\n           # 'num_leaves':           (6,25),\n            'colsample_bytree':     (0.01, 0.99),\n            'reg_alpha':     (0.01,3),\n            'reg_lambda':     (0.01,3)\n            }\n\n\n\n\n#### XGBoost parameters ############\nxgboost_on = False\nbayesian_optimization_xgb =  False\n                        \n#XGB_Params = {'colsample_bytree': 0.9697442313295259, 'learning_rate': 0.01, 'max_depth': 12.350315861619743, 'reg_alpha': 9.343517370820674, 'reg_lambda': 5.195984284719909}\n#XGB_Params = {'colsample_bytree': 0.7992962842349717, 'learning_rate': 0.07244114289373242, 'max_depth': 12.473050505626817, 'reg_alpha': 9.605956633784812, 'reg_lambda': 9.2658047672509}\nXGB_Params = {'colsample_bytree': 0.9203192422243004, 'learning_rate': 0.06858941956522761, 'max_depth': 12.064661182375076, 'reg_alpha': 9.642449802316628, 'reg_lambda': 9.376323929371848}\n\n# std\nXGB_Params = {'colsample_bytree': 0.6053939548634096,\n 'learning_rate': 0.017135359956360424,\n 'max_depth': 5.975773894779193,\n 'reg_alpha': 9.4893665171608,\n 'reg_lambda': 9.656664010414849}\n\n\n# skew\nXGB_Params = {'colsample_bytree': 0.9601929352092674,\n 'learning_rate': 0.07753576905377535,\n 'max_depth': 19.09248412346284,\n 'reg_alpha': 8.949325230772212,\n 'reg_lambda': 5.98302078832274}\nxgb_pbounds = { 'learning_rate':   (0.0001, .1),\n            'max_depth':            (5, 20) ,\n            #'num_leaves':           (6,25),\n            'colsample_bytree':     (0.01, 0.99),\n            'reg_alpha':     (0.01, 10),\n            'reg_lambda':     (0.01, 10)\n            }         \n               \n\n    \n#### Random Forest parameters ############\n\n\nrandomforest_on = True\nbayesian_optimization_rf =  True\n\n#skew\nrf_params = {'max_depth': 18.734914690766647,\n 'max_samples': 0.7815358749813437,\n 'min_samples_leaf': 0.01,\n 'min_samples_split': 0.01,\n 'n_estimators': 29.2777253927227}\n\n\n#mean and std\n#skew\nrf_params = {'max_depth': 20.0,\n 'max_samples': 0.6984854531443835,\n 'min_samples_leaf': 0.01,\n 'min_samples_split': 0.01,\n 'n_estimators': 31.07664004216061}\n\nrf_pbounds = {        \n        \"max_depth\":  (10, 20),\n        \"n_estimators\": (20, 100) ,\n        \"max_samples\": (0.01, 1),\n        \"min_samples_split\": (0.01, 0.99),\n        \"min_samples_leaf\": (0.01, .99)\n        #\"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 2, 600),\n    }      \n    \ncustom_objective_on = True\ncustom_metric =  True\n\nthreshold_optimization_on = True","metadata":{"papermill":{"duration":0.040654,"end_time":"2024-10-29T15:46:48.164493","exception":false,"start_time":"2024-10-29T15:46:48.123839","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"08929a6c-0f75-4416-9812-7eef70bb6f7f","cell_type":"markdown","source":"# Load training and test data","metadata":{"papermill":{"duration":0.018817,"end_time":"2024-10-29T15:46:48.244684","exception":false,"start_time":"2024-10-29T15:46:48.225867","status":"completed"},"tags":[]}},{"id":"c0205826-4ac6-423a-aee8-7bea826182b8","cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\n#df_train = df_train[~pd.isnull(df_train.sii)]\ndf_test = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\ndf_train = df_train[ [c for c in df_test.columns] + [\"sii\"]]\n\ncategorical_features = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n\n\nfor f in categorical_features: \n    df_train[f] = df_train[f].astype('category')\n    df_test[f] = df_test[f].astype('category')  \n\n    \nprint(df_train.shape, df_test.shape)","metadata":{"papermill":{"duration":0.162534,"end_time":"2024-10-29T15:46:48.427863","exception":false,"start_time":"2024-10-29T15:46:48.265329","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"e73171e7-5507-48b1-90aa-d1a8c0ba4be5","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"7c464d4e-674e-4b10-a02e-44017f45c233","cell_type":"markdown","source":"### Time series ( parquet ) files","metadata":{"papermill":{"duration":0.018905,"end_time":"2024-10-29T15:46:48.466029","exception":false,"start_time":"2024-10-29T15:46:48.447124","status":"completed"},"tags":[]}},{"id":"0aa95d06-4e7c-4b2b-89f2-e00dfd3403de","cell_type":"code","source":"# We'll gather basic metadata about the parquet files. In particular we want to understand if there are breaks in the sequence\n# so that we can decide whether the file is good enough for our analysis\n#\n# For descriptive data analysis ( mean, std, min, max) we can work with all files\n# For sequence analysis ( through CNN, LSTM ) we want to work with continous sequences only\n\n\ndef analyze_ts_files(directory_path, gap_threshold= 5.0):\n    \n    folder_names = [f for f in os.listdir(directory_path) if os.path.isdir(os.path.join(directory_path, f))]\n\n    dfs = []\n    row_count = 0\n    i = 0\n    df_results = pd.DataFrame()\n    for folder_name in folder_names:\n        folder_path = os.path.join(directory_path, folder_name)\n        parquet_files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.parquet')]\n        df_folder = pd.DataFrame()\n\n        df_folder = pd.DataFrame({\"folder\": [folder_name[3:]] * len(parquet_files), \"files\" : parquet_files})\n        df_temp = pd.read_parquet(parquet_files[0])\n        df_temp[\"time_of_day\"] = df_temp.time_of_day/1000000000\n        df_temp[\"time_of_day\"] = df_temp[\"time_of_day\"].astype(np.int32)\n        df_temp[\"time_of_day_diff\"] = df_temp[\"time_of_day\"].diff()\n\n        sequence_breaks = df_temp[df_temp.time_of_day_diff > gap_threshold].shape[0]\n\n        min_break = df_temp[(~pd.isnull(df_temp.time_of_day_diff)) & (df_temp.time_of_day_diff > gap_threshold)].time_of_day_diff.min()\n        max_break = df_temp[(~pd.isnull(df_temp.time_of_day_diff)) & (df_temp.time_of_day_diff > gap_threshold)].time_of_day_diff.max()\n        d= {\"folder_name\": folder_name, \"file\": parquet_files[0], \"sequence_breaks\": sequence_breaks, \"record_count\" : df_temp.shape[0],  \"min_break\": min_break, \"max_break\": max_break, \"threshold\" : gap_threshold}\n        df_results = pd.concat([df_results, pd.DataFrame(d, index=[0])], ignore_index=True)\n\n  \n    return df_results","metadata":{"papermill":{"duration":0.039103,"end_time":"2024-10-29T15:46:48.524235","exception":false,"start_time":"2024-10-29T15:46:48.485132","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"29dc618f-4077-47c7-b28f-ce966e54f554","cell_type":"code","source":"file_path = f\"/kaggle/input/dataset-piu/piu/timeseries_{gap_threshold}.csv\" \nif os.path.exists(file_path):\n    ts_df = pd.read_csv(file_path)\nelse:\n    directory_path = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/'\n    ts_df = analyze_ts_files(directory_path, gap_threshold)\n    ts_df.to_csv(file_path, index = None)\n\n\nts_df[\"id\"] = ts_df[\"folder_name\"].str[3:]\nts_df = ts_df.merge(df_train[[\"id\", \"sii\"]], on=\"id\", how=\"inner\")\nts_df[\"sii\"] = ts_df[\"sii\"].astype(np.int8)","metadata":{"papermill":{"duration":79.519972,"end_time":"2024-10-29T15:48:08.064038","exception":false,"start_time":"2024-10-29T15:46:48.544066","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b16b64d9-40de-4d58-954a-bf85255b7809","cell_type":"markdown","source":"# Time series feature extraction","metadata":{"papermill":{"duration":0.019466,"end_time":"2024-10-29T15:48:08.102879","exception":false,"start_time":"2024-10-29T15:48:08.083413","status":"completed"},"tags":[]}},{"id":"1eeec770-af59-4800-b390-ae652ee62bff","cell_type":"code","source":"\n# Aggregate the metrics in the parquet files \nfrom scipy import stats\nfrom scipy import stats\n\ndef timeseries_aggregation(df, aggregation, aggregation_columns):\n\n    df_ts_agg = pd.DataFrame()\n    for index, row in df.iterrows():             \n\n        file_path = row[\"file\"]\n        rowid  = row[\"id\"]\n        df = pd.read_parquet(file_path)\n        df[ \"non-wear_flag\"] = df[\"non-wear_flag\"].astype(np.int8)\n\n            \n        df[\"time_of_day\"] = df.time_of_day / 1000000000\n        df[\"time_of_day\"] = df[\"time_of_day\"].astype(np.int32)\n        df[\"id\"] =  row[\"id\"]\n        df['magnitude'] = np.sqrt(df['X']**2 + df['Y']**2 + df['Z']**2)\n        #remove the first day if not complete\n        if df.iloc[0].time_of_day > 10:\n            first_day = df[\"relative_date_PCIAT\"].iloc[0]\n            df = df[df[\"relative_date_PCIAT\"] != first_day]\n\n        if (df.shape[0] == 0): continue\n\n        #remove the last day if not complete\n        if df.iloc[-1].time_of_day < 60 * 60*24 - 10:\n            last_day = df[\"relative_date_PCIAT\"].iloc[-1]\n            df = df[df[\"relative_date_PCIAT\"] != last_day]\n\n\n        df['hour_of_day'] = (df['time_of_day'] // 3600).astype(int)\n        df['10min'] = (df['time_of_day'] // 600).astype(int)\n        df['3h'] = (df['time_of_day'] // 14400).astype(int)\n        df['week_3h'] = df['weekday'].astype(str) + \"_\" + df['3h'].astype(str)\n        df[\"week_hour\"] = df['weekday'].astype(str) + \"_\" + df['hour_of_day'].astype(str)\n\n      \n        cols = ['X', 'Y', 'Z', 'enmo', 'anglez', 'light',  'battery_voltage']\n        if filter_wear_flag_on :\n            df = df[df[\"non-wear_flag\"] == 0 ]\n        else:\n            cols.append('non-wear_flag')\n\n        if (df.shape[0] == 0): continue\n        if aggregation == 'kurt':\n            df_agg_by_date = df[cols + ['id', 'relative_date_PCIAT'] + aggregation_columns\n           ].groupby(aggregation_columns + ['id', 'relative_date_PCIAT']).agg(stats.kurtosis).reset_index()\n        \n        elif aggregation == 'range':            \n            df_agg_by_date = df[cols + ['id', 'relative_date_PCIAT'] + aggregation_columns\n           ].groupby(aggregation_columns + ['id', 'relative_date_PCIAT']).agg(['min', 'max']).reset_index()\n            df_agg_by_date.columns = list(map('_'.join, df_agg_by_date.columns.values))\n            #display(df_agg_by_date) \n            \n            for c in cols:\n                df_agg_by_date[f\"{c}\"] =  df_agg_by_date[f\"{c}_max\"] -  df_agg_by_date[f\"{c}_min\"]\n            df_agg_by_date.rename( columns = {\"id_\": 'id', 'relative_date_PCIAT_': 'relative_date_PCIAT'}, inplace = True)\n            for c in aggregation_columns:\n                df_agg_by_date.rename( columns = { f\"{c}_\" : c }, inplace = True)\n            \n            df_agg_by_date = df_agg_by_date[cols + ['id', 'relative_date_PCIAT'] + aggregation_columns]\n            \n        else:    \n            df_agg_by_date = df[cols + ['id', 'relative_date_PCIAT'] + aggregation_columns].groupby(aggregation_columns + ['id', 'relative_date_PCIAT']).agg(aggregation).reset_index()\n\n            \n        df_agg = df_agg_by_date[cols + ['id'] + aggregation_columns].groupby(aggregation_columns + ['id']).agg('mean').reset_index()\n    \n        # We are going to calculate the stats by the hour ( for example) and then average them accross days\n        df_agg['activity_range']: hourly_activity.max() - hourly_activity.min()\n        df_ts_agg = pd.concat([df_ts_agg, df_agg], ignore_index = True)\n\n    return df_ts_agg\n\n\ndef timeseries_aggregation_diff(df, aggregation, aggregation_columns, full_days_only = False):\n\n    df_ts_agg = pd.DataFrame()\n    for index, row in df.iterrows():             \n\n        file_path = row[\"file\"]\n        rowid  = row[\"id\"]\n        df = pd.read_parquet(file_path)\n        df[ \"non-wear_flag\"] = df[\"non-wear_flag\"].astype(np.int8)\n        \n        df[\"time_of_day\"] = df.time_of_day / 1000000000\n        df[\"time_of_day\"] = df[\"time_of_day\"].astype(np.int32)\n\n        df[\"id\"] =  row[\"id\"]\n\n        #remove the first day if not complete\n        if full_days_only:\n            if df.iloc[0].time_of_day > 10:\n                first_day = df[\"relative_date_PCIAT\"].iloc[0]\n                df = df[df[\"relative_date_PCIAT\"] != first_day]\n\n            if (df.shape[0] == 0): continue        \n            #remove the last day if not complete\n\n            if df.iloc[-1].time_of_day < 60 * 60*24 - 10:\n                last_day = df[\"relative_date_PCIAT\"].iloc[-1]\n                df = df[df[\"relative_date_PCIAT\"] != last_day]\n                \n            \n        df[\"time_of_day_diff\"] = df[\"time_of_day\"].diff()    \n        cols = ['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage']\n        if filter_wear_flag_on :\n            df = df[df[\"non-wear_flag\"] == 0 ]\n        else:\n            cols.append('non-wear_flag')\n        \n        #if (df.shape[0] == 0): continue\n        diff_cols = []\n        for c in cols:\n            df[f'diff_{c}'] = df[c].diff()\n            df.loc[df[\"time_of_day_diff\"] > 5, f'diff_{c}'] = 0\n            diff_cols.append(f'diff_{c}')\n            \n        df[\"distance\"] = df['diff_X'] * df['diff_X']  + df['diff_Y'] *  df['diff_Y']  + df['diff_Z'] * df['diff_Z'] \n        diff_cols.append(\"distance\")\n\n        if (df.shape[0] == 0): continue\n\n        df['hour_of_day'] = (df['time_of_day'] // 3600).astype(int)\n        df['10min'] = (df['time_of_day'] // 600).astype(int)\n        df[\"week_hour\"] = df['weekday'].astype(str) + \"_\" + df['hour_of_day'].astype(str)\n        df['3h'] = (df['time_of_day'] // 14400).astype(int)\n        df['week_3h'] =  df['weekday'].astype(str) + \"_\" + df['3h'].astype(str)\n\n        if aggregation == 'kurt':\n            df_agg_by_date = df[diff_cols + ['id', 'relative_date_PCIAT'] + aggregation_columns].groupby(aggregation_columns + ['id', 'relative_date_PCIAT']).agg(stats.kurtosis).reset_index()\n\n        elif aggregation == 'range':            \n            df_agg_by_date = df[diff_cols + ['id', 'relative_date_PCIAT'] + aggregation_columns\n           ].groupby(aggregation_columns + ['id', 'relative_date_PCIAT']).agg(['min', 'max']).reset_index()\n            df_agg_by_date.columns = list(map('_'.join, df_agg_by_date.columns.values))\n            #display(df_agg_by_date) \n            \n            for c in diff_cols:\n                df_agg_by_date[f\"{c}\"] =  df_agg_by_date[f\"{c}_max\"] -  df_agg_by_date[f\"{c}_min\"]\n            df_agg_by_date.rename( columns = {\"id_\": 'id', 'relative_date_PCIAT_': 'relative_date_PCIAT'}, inplace = True)\n            for c in aggregation_columns:\n                df_agg_by_date.rename( columns = { f\"{c}_\" : c }, inplace = True)\n            \n            df_agg_by_date = df_agg_by_date[diff_cols + ['id', 'relative_date_PCIAT'] + aggregation_columns]\n\n            #display(df_agg_by_date)\n        else: \n            df_agg_by_date = df[diff_cols + ['id', 'relative_date_PCIAT'] + aggregation_columns].groupby(aggregation_columns + ['id', 'relative_date_PCIAT']).agg(aggregation).reset_index()\n        \n        \n        # We are going to calculate the stats by the hour ( for example) and then average them accross days\n        df_agg = df_agg_by_date[diff_cols + ['id'] + aggregation_columns].groupby(aggregation_columns + ['id']).agg('mean').reset_index()\n\n        df_ts_agg = pd.concat([df_ts_agg, df_agg], ignore_index = True)\n\n    return df_ts_agg\n","metadata":{"papermill":{"duration":0.060786,"end_time":"2024-10-29T15:48:08.183807","exception":false,"start_time":"2024-10-29T15:48:08.123021","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"fa131362-0ac5-49bf-acfd-11a1a09299b4","cell_type":"code","source":"# Pivoting the values so that we have for each id, aggregations for hour 1,2,3,  or weekdays,or weekday-hour \ndef ts_pivot_by_feature(df_ts_agg, pivot_feature, cols):\n    #cols = ['X', 'Y', 'Z', 'enmo', 'anglez']\n    #cols = ['diff_X', 'diff_Y', 'diff_Z', 'diff_enmo', 'diff_anglez']\n    df_features = df_ts_agg[[\"id\"]].drop_duplicates()\n\n    for col in cols:\n        #print( \"pivoting column \", col)\n        df_temp = df_ts_agg[['id', pivot_feature, col]].pivot(index='id', columns=pivot_feature, values=col).reset_index()\n        df_temp.columns = [ 'id'] + [ str(col) + '_' + str(c) for c in df_temp.columns if c != 'id']\n        df_features = df_features.merge(df_temp, on = \"id\", how = \"left\")\n    \n    return df_features    ","metadata":{"papermill":{"duration":0.032125,"end_time":"2024-10-29T15:48:08.235051","exception":false,"start_time":"2024-10-29T15:48:08.202926","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"13fc2bd2-e1ff-456e-97d1-afb6e48aba97","cell_type":"code","source":"      \n\ndef ts_aggregation_features(ts_df, ts_aggregations, pivot_features, timeseries_aggregation_function, file_prefix, pivot_cols):\n    #caching the preprocessed data in the working directory to avoid recalculations\n    df_ts_features = ts_df[[\"id\"]].copy()\n    features = [] # We need to do this for the cached features\n    for ts_aggregation in ts_aggregations:\n        for pivot_feature in pivot_features:\n            #file_path = f\"/kaggle/input/dataset-piu/piu/{file_prefix}_{pivot_feature}_{ts_aggregation}_{gap_threshold}_{ts_df.shape[0]}_2levels.csv\" \n            file_path = f\"/kaggle/input/dataset-piu/piu/{file_prefix}_{pivot_feature}_{ts_aggregation}_{gap_threshold}_{ts_df.shape[0]}_addtional_2levels.csv\" \n\n            print(file_path)\n            if os.path.exists(file_path):\n                df_temp = pd.read_csv(file_path)\n                df_temp.columns = [ 'id'] + [ pivot_feature + '_' + ts_aggregation + '_' + c for c in df_temp.columns if c != 'id']\n                df_ts_features = df_ts_features.merge(df_temp, on = \"id\", how = \"left\")\n            else:\n                print( ts_aggregation, pivot_feature)\n                df_ts_agg =  timeseries_aggregation_function(ts_df, ts_aggregation, [pivot_feature])    \n\n\n                df_temp = ts_pivot_by_feature(df_ts_agg, pivot_feature, pivot_cols)\n                df_temp.to_csv(file_path, index = None)\n                df_temp.columns = [ 'id'] + [ pivot_feature + '_' + ts_aggregation + '_' + c for c in df_temp.columns if c != 'id']\n                df_ts_features = df_ts_features.merge(df_temp, on = \"id\", how = \"left\")\n    print(df_ts_features.shape)\n    return df_ts_features\n\n\n\n    \n            \n","metadata":{"papermill":{"duration":0.036933,"end_time":"2024-10-29T15:48:08.291178","exception":false,"start_time":"2024-10-29T15:48:08.254245","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"598c3d87-6772-48d7-902f-6a9fc4a66ac2","cell_type":"code","source":"#To start off, we'll focus only on those time series that have no breaks\n\n\n#ts_df = ts_df[ts_df.sequence_breaks == 0]\n#ts_df.head()","metadata":{"papermill":{"duration":0.02907,"end_time":"2024-10-29T15:48:08.340111","exception":false,"start_time":"2024-10-29T15:48:08.311041","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"1779043c-7a7a-4a2f-9e96-655e057a7ac6","cell_type":"code","source":"\n#This is the evaluation metric used by LGBM, see parameters\ndef quadratic_weighted_kappa(y_true, y_pred):\n    #y_true = y_true\n    y_pred = y_pred.clip(y_min, y_max).round()\n    qwk = cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n    return 'QWK', qwk, True\n\ny_min = int(df_train.sii.min())\ny_max = int(df_train.sii.max())\n\na = 2.998\nb = 1.092\n\n#still looking into this\ndef quadratic_weighted_kappa_obj(y_true, y_pred):\n    y_true1 = (y_true + a).clip(y_min, y_max).round()\n    y_pred1 = (y_pred + a).clip(y_min, y_max).round()\n    #qwk = cohen_kappa_score(y_true.clip(0, 4).round(), y_pred, weights=\"quadratic\")\n    qwk = cohen_kappa_score(y_true1, y_pred1, weights=\"quadratic\")\n    return 'QWK', qwk, True\n\n\n\ndef qwk_obj(y_true, y_pred):\n    labels = y_true + a\n    preds = y_pred + a\n    preds = preds.clip(1, 6)\n    f = 1/2*np.sum((preds-labels)**2)\n    g = 1/2*np.sum((preds-a)**2+b)\n    df = preds - labels\n    dg = preds - a\n    grad = (df/g - f*dg/g**2)*len(labels)\n    hess = np.ones(len(labels))\n    return grad, hess\n\n","metadata":{"papermill":{"duration":0.034895,"end_time":"2024-10-29T15:48:08.394110","exception":false,"start_time":"2024-10-29T15:48:08.359215","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"56828ffa-059f-4d3a-8ec7-b0edc5c637a6","cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import KFold\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import make_scorer\nfrom sklearn.metrics import cohen_kappa_score\n\ndef quadratic_kappa(y_true, y_pred):\n    \n    y_pred1 = y_pred.clip(y_min, y_max).round()\n    return cohen_kappa_score(y_true, y_pred1, weights='quadratic')\n","metadata":{"papermill":{"duration":0.038344,"end_time":"2024-10-29T15:48:08.491085","exception":false,"start_time":"2024-10-29T15:48:08.452741","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d3792b63-84ec-42d8-9b66-9ff46cd98318","cell_type":"code","source":"def train_timeseries_model(df_train, df_ts_features, params, prediction_field):\n   \n\n    df_train_ts = df_train.merge(df_ts_features, on  = \"id\", how = \"inner\")\n    features = [ c for c in df_ts_features.columns if c not in [ \"id\"]]\n    X = df_train_ts[features]\n    y = df_train_ts[\"sii\"].astype(int)\n    print(X.shape, y.shape)\n    \n    callbacks = [log_evaluation(period=100), early_stopping(stopping_rounds=75,first_metric_only=True)]\n    ts_training_predictions = np.zeros((df_train_ts.shape[0],))\n    i=1\n    ts_models = []\n    sum_kappa = 0\n    for train_index, test_index in skf.split(X, y):   \n        print('fold',i)\n        X_train_fold, X_test_fold = X.iloc[train_index], X.iloc[test_index]   \n        y_train_fold, y_test_fold = y.iloc[train_index], y.iloc[test_index]  \n        model = lgb.LGBMRegressor(\n                objective = qwk_obj if custom_objective_on else None ,\n                metrics = 'None',\n                learning_rate = params[\"learning_rate\"],\n                max_depth = round(params[\"max_depth\"]),\n                num_leaves = 2^(round(params[\"max_depth\"] ) - 1),\n                colsample_bytree=params[\"colsample_bytree\"],\n                reg_alpha = params[\"reg_alpha\"],\n                reg_lambda = params[\"reg_lambda\"],\n                n_estimators=n_estimators,\n                random_state=42,\n                extra_trees=True,\n                class_weight='balanced',\n                verbosity = - 1)\n\n        model.fit(X_train_fold,y_train_fold, \n                                  eval_names=['train', 'valid'],\n                                  eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n                                  eval_metric= quadratic_weighted_kappa if custom_metric else None,\n                                  callbacks=callbacks)\n        ts_models.append(model)\n        predictions_fold = model.predict(X_test_fold)\n        ts_training_predictions [test_index] = predictions_fold\n        f1_fold = f1_score(y_test_fold, predictions_fold.clip(y_min, y_max).round(), average='weighted')\n\n        kappa_fold = cohen_kappa_score(y_test_fold, predictions_fold.clip(y_min, y_max).round(), weights='quadratic')\n        sum_kappa += kappa_fold\n        print( f1_fold, kappa_fold)\n        i+= 1\n        \n    print(\"mean kappa\" , sum_kappa/n_splits)\n    df_train_ts[prediction_field] = ts_training_predictions\n    return df_train_ts, ts_models\n","metadata":{"papermill":{"duration":0.040564,"end_time":"2024-10-29T15:48:08.550816","exception":false,"start_time":"2024-10-29T15:48:08.510252","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0f025a79-52cf-490d-93ae-98b69ea0c74f","cell_type":"code","source":"def display_progress(event, instance):\n    iter = len(instance.res) - 1\n    print('Iteration: {} - Kappa: {} - {}'.format(iter, instance.res[iter].get('target'), instance.res[iter].get('params')))","metadata":{"papermill":{"duration":0.029573,"end_time":"2024-10-29T15:48:08.659505","exception":false,"start_time":"2024-10-29T15:48:08.629932","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"593aec3d-c79f-4420-b5b0-d53329ec08e9","cell_type":"code","source":"from bayes_opt import BayesianOptimization\nfrom bayes_opt.logger import JSONLogger\nfrom bayes_opt.event import Events\n\ndef bayesian_parameter_optimization( init_rounds, opt_rounds, parameter_bounds, evaluator):\n\n\n    optimizer = BayesianOptimization(f = evaluator, \n                                    pbounds = parameter_bounds,\n                                    random_state = 42,\n                                    verbose = 2)\n\n    logger = JSONLogger(path = f'bayesian_parameter_output.json')\n    optimizer.subscribe(\"optimization:step\", logger)\n\n    optimizer.subscribe(\"optimization:step\", \" \", display_progress)\n\n    optimizer.maximize(init_points = init_rounds, n_iter = opt_rounds)\n\n    return optimizer.max","metadata":{"papermill":{"duration":0.071909,"end_time":"2024-10-29T15:48:08.750628","exception":false,"start_time":"2024-10-29T15:48:08.678719","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"be0eb3ff-3075-42df-85de-706a50b88abc","cell_type":"code","source":"def lgbm_cv_evaluator(learning_rate, max_depth, colsample_bytree,reg_alpha, reg_lambda):\n\n    callbacks = [log_evaluation(period=100), early_stopping(stopping_rounds=75,first_metric_only=True)]\n\n    i = 0\n   # f1_scores = []\n    kappa_scores = []\n    for train_index, test_index in skf.split(X, y):\n\n        print('fold',i+1, X.shape, y.shape, type(y))\n        X_train_fold, X_test_fold = X.iloc[train_index], X.iloc[test_index]\n        \n        y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y[test_index]\n\n        model = lgb.LGBMRegressor(\n                    objective = qwk_obj if custom_objective_on else None,\n                    metrics = 'None',\n                    learning_rate = learning_rate,\n                    max_depth = round(max_depth),\n                    num_leaves = 2^ (round(max_depth) - 1),\n                    colsample_bytree=colsample_bytree,\n                    reg_alpha =reg_alpha,\n                    reg_lambda = reg_lambda,\n                    n_estimators=n_estimators,\n                    random_state=42,\n                    extra_trees=True,\n                    class_weight='balanced',\n                    verbosity = - 1,\n                    verbose_eval=False)\n\n        predictor = model.fit(X_train_fold,\n                                y_train_fold,\n                                eval_names=['train', 'valid'],\n                                eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n                                eval_metric= quadratic_weighted_kappa if custom_metric else None,\n                                callbacks=callbacks\n\n                                 )\n\n        predictions_fold = predictor.predict(X_test_fold)\n        predictions_fold = predictions_fold\n        predictions_fold = predictions_fold.clip(y_min, y_max).round()\n\n\n        kappa_fold = quadratic_kappa(y_test_fold_int, predictions_fold)\n\n        kappa_scores.append(kappa_fold)\n\n        print(f'Cohen kappa score across fold: {kappa_fold}')\n        i+=1\n\n    mean_kappa_score = np.mean(kappa_scores)\n\n\n    print(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')\n    return mean_kappa_score","metadata":{"papermill":{"duration":0.039544,"end_time":"2024-10-29T15:48:08.610879","exception":false,"start_time":"2024-10-29T15:48:08.571335","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"6d50dc5c-b4ac-41e9-9c23-299dc3849b2a","cell_type":"code","source":"if time_series_diff_aggregation_on:\n       \n    n_splits = 5\n    n_estimators = 500\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    \n    pivot_columns = ['diff_X', 'diff_Y', 'diff_Z', 'diff_enmo', 'diff_anglez', 'diff_light','distance', 'diff_battery_voltage']\n    if not filter_wear_flag_on :\n        pivot_columns.append('diff_non-wear_flag')\n\n    ts_df_diff = ts_df[ts_df.sequence_breaks <= max_sequence_breaks_diff]\n    df_ts_diff_features =  ts_aggregation_features(ts_df_diff, ts_aggregations_diff, pivot_features_diff, timeseries_aggregation_diff, \"ts_diff_features_agg\", pivot_columns)\n    print(df_ts_diff_features.shape)\n    features = ['id']\n    for pivot in pivot_columns:\n        \n        features.extend([c for c in df_ts_diff_features.columns if pivot in c])\n\n    df_ts_diff_features = df_ts_diff_features[features]\n    lgbm_params = ts_params_diff\n    if bayesian_optimization_ts:\n        df_train_ts = df_train.merge(df_ts_diff_features, on  = \"id\", how = \"inner\")\n        features = [ c for c in df_ts_diff_features.columns if c not in [ \"id\"]]\n        X = df_train_ts[features]\n        y = df_train_ts[\"sii\"].astype(int)\n        print(X.shape, y.shape)\n        max_params = bayesian_parameter_optimization( init_rounds = bayesian_init_rounds, opt_rounds = bayesian_opt_rounds, parameter_bounds = ts_diff_pbounds, evaluator = lgbm_cv_evaluator)\n        lgbm_params = max_params['params']\n        print(\"new LGBM bayesian params:\", lgbm_params)    \n    \n    df_train_diff_ts, ts_diff_models = train_timeseries_model(df_train, df_ts_diff_features, lgbm_params, \"ts_predictions_diff\")\n    df_train = df_train.merge(df_train_diff_ts[[\"ts_predictions_diff\", \"id\"]], on  = \"id\", how = \"left\")\n    ","metadata":{"papermill":{"duration":8636.370687,"end_time":"2024-10-29T18:12:05.141642","exception":false,"start_time":"2024-10-29T15:48:08.770955","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"604371e5-afbf-49d2-a0e0-895c5c73a745","cell_type":"code","source":"if time_series_aggregation_on:\n    n_splits = 5\n    n_estimators = 500\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    pivot_columns = ['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage']\n    if not filter_wear_flag_on :\n        pivot_columns.append('non-wear_flag')\n    ts_df = ts_df[ts_df.sequence_breaks <= max_sequence_breaks]\n    #ts_df_diff = ts_df.copy()\n    df_ts_features =  ts_aggregation_features(ts_df, ts_aggregations, pivot_features, timeseries_aggregation, \"ts_features_agg\", pivot_columns)\n    print(df_ts_features.shape)\n    features = ['id']\n    for pivot in pivot_columns:\n        \n        features.extend([c for c in df_ts_features.columns if pivot in c])\n\n    df_ts_features = df_ts_features[features]\n    #print(df_ts_diff_features.shape, len(features))\n    #params = {'learning_rate': 0.01, 'max_depth': 30, 'n_estimators': 225, 'num_leaves': 75}\n    df_train_ts = df_train.merge(df_ts_features, on  = \"id\", how = \"inner\")\n    features = [ c for c in df_ts_features.columns if c not in [ \"id\"]]\n    X = df_train_ts[features]\n    y = df_train_ts[\"sii\"].astype(int)\n    print(X.shape, y.shape)\n    \n    lgbm_params = ts_params\n    if bayesian_optimization_ts:\n        \n        max_params = bayesian_parameter_optimization( init_rounds = bayesian_init_rounds, opt_rounds = bayesian_opt_rounds, parameter_bounds = ts_pbounds, evaluator = lgbm_cv_evaluator)\n        lgbm_params = max_params['params']\n        print(\"new LGBM bayesian params:\", lgbm_params)    \n    df_train_ts, ts_models = train_timeseries_model(df_train, df_ts_features, lgbm_params, \"ts_predictions\")\n    \n\n    df_train = df_train.merge(df_train_ts[[\"ts_predictions\", \"id\"]], on  = \"id\", how = \"left\")\n \n","metadata":{"papermill":{"duration":2000.600842,"end_time":"2024-10-29T18:45:26.541958","exception":false,"start_time":"2024-10-29T18:12:05.941116","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4907a1cb-ba40-4989-9284-85806fccccde","cell_type":"code","source":"from sklearn.decomposition import PCA\nif pca_components > 0:\n    #n_components = 20\n    df_ts_all_features  = df_ts_diff_features.merge(df_ts_features, on = \"id\")\n    pca_numeric_cols = df_ts_all_features.select_dtypes(include=['float64', 'int64']).columns\n    print(df_ts_all_features.shape, len(pca_numeric_cols))\n\n    #display(df_ts_diff_features.loc[:, df_ts_diff_features.isnull().any()])\n\n    pca_cols = [ c for c in pca_numeric_cols if c not in [ 'id', 'sii']]\n    pca_X = df_ts_all_features[pca_cols].copy()\n    pca_X.fillna(0, inplace =  True)\n    pca = PCA(n_components=pca_components)\n    #pca.fit(pca_X)\n    reduced_X = pca.fit_transform(pca_X)\n    #PCA(n_components=2)\n\n    import seaborn as sns\n    pca_df  = pd.DataFrame(reduced_X)\n    pca_columns = [ f\"pca{i}\" for i in range(pca_components)] \n    pca_df.columns =  pca_columns\n    pca_df[\"sii\"] = df_train_diff_ts[\"sii\"]\n    pca_df[\"id\"] = df_train_diff_ts[\"id\"]\n\n    #pca.components_.shape, X.shape, newdf.shape\n\n    sns.scatterplot(pca_df, x = \"pca0\", y = \"pca1\", hue = \"sii\")\n    #newdf.head()\n\n    df_train = df_train.merge(pca_df[ pca_columns + [\"id\"]] , on  = \"id\", how = \"left\")\n    \n    \n    \n  \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"8c492c30-4b15-433d-af34-4aeac1f8eac1","cell_type":"code","source":"def tabular_feature_engineering(df):\n    #season_cols = [col for col in df.columns if 'Season' in col]\n    #df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df","metadata":{"papermill":{"duration":0.207517,"end_time":"2024-10-29T18:45:26.941346","exception":false,"start_time":"2024-10-29T18:45:26.733829","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d9d8d9a7-66e1-4a36-ab87-f683421ec03c","cell_type":"code","source":"import torch.nn as nn\nimport torch.optim as optim\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nimport torch\n\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    #\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    autoencoder = autoencoder.to(device)\n    data_tensor = data_tensor.to(device)\n    #\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            batch.to(device)\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 100 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).cpu().numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"76c8024b-4a32-47ba-aad7-bf1f1a96031c","cell_type":"code","source":"if autoencoder_on:\n    #train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n    #test_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n    #train_ts_encoded = perform_autoencoder(train_ts.drop('id', axis=1), encoding_dim=60, epochs=100, batch_size=32)\n    #test_ts_encoded = perform_autoencoder(test_ts.drop('id', axis=1), encoding_dim=60, epochs=100, batch_size=32)\n   \n    #autoenc_numeric_cols = df_ts_all_features.select_dtypes(include=['float64', 'int64']).columns\n\n    df_ts_all_features  = df_ts_diff_features.merge(df_ts_features, on = \"id\")\n    df_ts_all_features  = df_ts_diff_features #.merge(df_ts_features, on = \"id\")\n    cols = [c for c in df_ts_all_features.columns if c not in [\"id\"]]\n    #cols = [ c for c in numeric_cols if c not in [ 'id', 'sii']]\n    unencoded_df = df_ts_all_features[cols].copy()\n    print(\"Auto-encoding \", unencoded_df.shape)\n    unencoded_df.fillna(0, inplace =  True)\n    \n    train_ts_encoded = perform_autoencoder(unencoded_df, encoding_dim=auto_encoder_dimensions, epochs=auto_encoder_epochs, batch_size=16)\n    #test_ts_encoded = perform_autoencoder(df_ts_diff_features, encoding_dim=60, epochs=100, batch_size=32)\n    train_ts_encoded[\"id\"] = df_ts_all_features[\"id\"]\n    \n    time_series_cols = train_ts_encoded.columns.tolist()\n    #train_ts_encoded[\"id\"]=df_train_ts[\"id\"]\n    #test_ts_encoded['id']=test_ts[\"id\"]\n\n\n    df_train = df_train.merge(train_ts_encoded, on  = \"id\", how = \"left\")\n    print(df_train.shape)","metadata":{"papermill":{"duration":0.198062,"end_time":"2024-10-29T18:45:28.939457","exception":false,"start_time":"2024-10-29T18:45:28.741395","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"563253a7-3450-4e8d-8e60-454a570e1322","cell_type":"code","source":"df_train = df_train[~pd.isnull(df_train.sii)]\nprint(df_train.shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"6d74aaf6-fffd-4536-aa9c-fc47084be958","cell_type":"code","source":"df_train = tabular_feature_engineering(df_train)\nprint(df_train.shape)","metadata":{"papermill":{"duration":0.208828,"end_time":"2024-10-29T18:45:28.555108","exception":false,"start_time":"2024-10-29T18:45:28.346280","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4986c112-11ea-4bc0-b6e1-05128a22aeab","cell_type":"code","source":"features = [ c for c in df_train.columns if c not in [ \"id\", \"sii\"]]\nX = df_train[features]\ny = df_train[\"sii\"].astype(int).to_numpy()\nprint(X.shape, y.shape)\n\n\nnumeric_X = pd.get_dummies(X, columns = categorical_features)\nprint(numeric_X.shape)","metadata":{"papermill":{"duration":0.202996,"end_time":"2024-10-29T18:45:29.331915","exception":false,"start_time":"2024-10-29T18:45:29.128919","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"04f2dc58-78e6-4bfb-a4c5-bf52f2571b35","cell_type":"markdown","source":"# LGBM","metadata":{"papermill":{"duration":0.194135,"end_time":"2024-10-29T18:45:29.719000","exception":false,"start_time":"2024-10-29T18:45:29.524865","status":"completed"},"tags":[]}},{"id":"8b102498-be35-45b7-ad1c-e419d39769b3","cell_type":"code","source":"params = lgbm_bayesian_params\nif bayesian_optimization:\n    n_splits = 5\n    n_estimators = 500\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    max_params = bayesian_parameter_optimization( init_rounds = bayesian_init_rounds, opt_rounds = bayesian_opt_rounds, parameter_bounds = lgbm_pbounds, evaluator = lgbm_cv_evaluator)\n    params = max_params['params']\n    print(\"new LGBM bayesian params:\", params)","metadata":{"papermill":{"duration":266.791773,"end_time":"2024-10-29T18:49:56.699223","exception":false,"start_time":"2024-10-29T18:45:29.907450","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"5d4e5985-3e8d-487b-ab84-3ef3d6d89954","cell_type":"code","source":"# We want to store the predictions for each fold both real and discrete for ensemble or threshold purposes\n\ntraining_predictions = np.zeros((df_train.shape[0],))\ntraining_clipped_predictions = np.zeros((df_train.shape[0],))\ntraining_predictions.shape","metadata":{"papermill":{"duration":0.308296,"end_time":"2024-10-29T18:49:57.300555","exception":false,"start_time":"2024-10-29T18:49:56.992259","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"e3f2a8a5-e316-486f-84f4-b4eac50cb6f0","cell_type":"code","source":"if stepwise_features > 0:\n    n_splits = 5\n    n_estimators = 500\n\n    #stratified K fold based on classes\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n    f1_scores = []\n    kappa_scores = []\n    models = []\n    predictions = []\n\n\n    features = [ c for c in df_train.columns if c not in [ \"id\", \"sii\"]]\n\n\n    #y = df_train[\"sii\"].astype(int)\n    print(X.shape, y.shape)\n\n\n    callbacks = [log_evaluation(period=25), early_stopping(stopping_rounds=75,first_metric_only=True)]\n\n    y_true =  y\n    selected_features = features\n\n    mean_scores = []\n    feature_selection = []\n\n    while len(selected_features)> stepwise_features:\n        feature_importance_df = None\n        X = df_train[selected_features]\n        i=1\n        for train_index, test_index in skf.split(X, y):\n\n            print('fold',i)\n\n            X_train_fold, X_test_fold = X.iloc[train_index], X.iloc[test_index]   \n\n            y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y_true[test_index]\n\n            model = lgb.LGBMRegressor( \n                        objective = qwk_obj if custom_objective_on else None,\n                        metrics = 'None',\n                        learning_rate = params[\"learning_rate\"],\n                        max_depth = round(params[\"max_depth\"]),\n                        num_leaves = 2 ^(round(params[\"max_depth\"]) - 1),\n                        colsample_bytree=params[\"colsample_bytree\"],\n                        reg_alpha = params[\"reg_alpha\"],\n                        reg_lambda = params[\"reg_lambda\"],\n                        n_estimators=n_estimators,\n                        random_state=42,\n                        #extra_trees=True,\n                        #class_weight='balanced',\n                        verbosity = - 1,\n                        verbose_eval=False)\n\n\n            predictor = model.fit(X_train_fold,\n                                          y_train_fold,\n                                          eval_names=['train', 'valid'],\n                                          eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n                                          eval_metric= quadratic_weighted_kappa if custom_metric else None,\n                                          callbacks=callbacks,)\n\n            predictions_fold = predictor.predict(X_test_fold)\n            predictions_fold = predictions_fold.clip(y_min, y_max).round()\n\n            kappa_fold = cohen_kappa_score(y_test_fold_int, predictions_fold, weights='quadratic')\n            kappa_scores.append(kappa_fold)\n\n            print(f'Cohen kappa score across fold: {kappa_fold}')\n\n            fold_feature_imp = pd.DataFrame({'Feature':selected_features, f'Value_{i}':predictor.feature_importances_}) \n            feature_importance_df = fold_feature_imp if feature_importance_df is None else feature_importance_df.merge(fold_feature_imp, on = ['Feature'] )      \n            value_cols = [ c for c in feature_importance_df.columns if c.startswith('Value_')]\n            feature_importance_df[\"Importance\"] = np.mean(feature_importance_df[value_cols], axis = 1)\n            feature_importance_df = feature_importance_df.sort_values(by = [\"Importance\"], ascending = False)\n            i+=1\n\n\n        mean_kappa_score = np.mean(kappa_scores)\n\n        print(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')\n\n        mean_scores.append(mean_kappa_score)\n        feature_selection.append(selected_features)\n        selected_features = [f for f in selected_features if f != feature_importance_df.Feature.iloc[-1]]\n        print(\"dropping feature :\" , feature_importance_df.Feature.iloc[-1], \"selected features \", len(selected_features) )\n\n    sns.lineplot (mean_scores)\n    max_index = 5 + np.argmax(mean_scores[5:])\n    print(\"max_index\", max_index, \"score\", mean_scores [  max_index])\n    features = feature_selection[max_index]\n    X = df_train[features]\n    print(features)","metadata":{"papermill":{"duration":0.321392,"end_time":"2024-10-29T18:49:57.909780","exception":false,"start_time":"2024-10-29T18:49:57.588388","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d9e7d062-48b1-4687-b176-bd4a5eda9052","cell_type":"code","source":"","metadata":{"papermill":{"duration":0.290559,"end_time":"2024-10-29T18:49:59.071233","exception":false,"start_time":"2024-10-29T18:49:58.780674","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b0f6948c-f8cc-4542-b2ae-bef707cb0340","cell_type":"markdown","source":"### LightGBM regression","metadata":{"papermill":{"duration":0.290347,"end_time":"2024-10-29T18:49:59.713096","exception":false,"start_time":"2024-10-29T18:49:59.422749","status":"completed"},"tags":[]}},{"id":"ecce2ca0-0e21-4fba-91ec-1bfd37b8a2f5","cell_type":"code","source":"n_splits = 5\nn_estimators = 500\n\n#stratified K fold based on classes\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\nf1_scores = []\nkappa_scores = []\nmodels = []\npredictions = []\nfeature_importance_df = None\n\n# Early stopping call back to avoid overfitting\ncallbacks = [log_evaluation(period=25), early_stopping(stopping_rounds=75,first_metric_only=True)]\n\ny_true =  y\n\ni=1\nfor train_index, test_index in skf.split(X, y):\n   \n    print('fold',i)\n\n    X_train_fold, X_test_fold = X.iloc[train_index], X.iloc[test_index]   \n   \n    y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y_true[test_index]\n  \n    model = lgb.LGBMRegressor( \n                objective = qwk_obj if custom_objective_on else None,\n                metrics = 'None',\n                learning_rate = params[\"learning_rate\"],\n                max_depth = round(params[\"max_depth\"]),\n                num_leaves = 2 ^(round(params[\"max_depth\"]) - 1),\n                colsample_bytree=params[\"colsample_bytree\"],\n                reg_alpha = params[\"reg_alpha\"],\n                reg_lambda = params[\"reg_lambda\"],\n                n_estimators=n_estimators,\n                random_state=42,\n                #extra_trees=True,\n                #class_weight='balanced',\n                verbosity = - 1)\n    \n\n    \n    predictor = model.fit(X_train_fold,\n                                  y_train_fold,\n                                  eval_names=['train', 'valid'],\n                                  eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n                                  eval_metric= quadratic_weighted_kappa if custom_metric else None,\n                                  callbacks=callbacks,)\n    \n    #saving all models for inference\n    models.append(predictor)\n    predictions_fold = predictor.predict(X_test_fold)\n    #predictions_fold = predictions_fold + a\n    training_predictions [test_index]=predictions_fold\n    predictions_fold = predictions_fold.clip(y_min, y_max).round()\n    predictions.append(predictions_fold)\n    training_clipped_predictions [test_index]=predictions_fold\n    \n    f1_fold = f1_score(y_test_fold_int, predictions_fold, average='weighted')\n    f1_scores.append(f1_fold)\n        \n    kappa_fold = cohen_kappa_score(y_test_fold_int, predictions_fold, weights='quadratic')\n    kappa_scores.append(kappa_fold)\n    \n    cm = confusion_matrix(y_test_fold_int, predictions_fold, labels=[x for x in range(0,4)])\n\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n                                  display_labels=[x for x in range(0,4)])\n    disp.plot()\n    plt.show()\n    print(f'F1 score across fold: {f1_fold}')\n    print(f'Cohen kappa score across fold: {kappa_fold}')\n    \n    # Capturing the importance of each feature through the folds    \n    fold_feature_imp = pd.DataFrame({'Feature':features, f'Value_{i}':predictor.feature_importances_}) \n    feature_importance_df = fold_feature_imp if feature_importance_df is None else feature_importance_df.merge(fold_feature_imp, on = ['Feature'] )      \n    value_cols = [ c for c in feature_importance_df.columns if c.startswith('Value_')]\n    feature_importance_df[\"Importance\"] = np.mean(feature_importance_df[value_cols], axis = 1)\n    feature_importance_df = feature_importance_df.sort_values(by = [\"Importance\"], ascending = False)\n    feature_importance_df = feature_importance_df[[\"Feature\", \"Importance\"] + [ c for c in feature_importance_df.columns if c.startswith('Value_')]]\n    display(feature_importance_df.head(10))       \n    \n    i+=1\n\nmean_f1_score = np.mean(f1_scores)\nmean_kappa_score = np.mean(kappa_scores)\n\nprint(f'Mean F1 score across {n_splits} folds: {mean_f1_score}')\nprint(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')","metadata":{"papermill":{"duration":10.474742,"end_time":"2024-10-29T18:50:10.487595","exception":false,"start_time":"2024-10-29T18:50:00.012853","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"601c366e-0031-4831-8085-46722fe41268","cell_type":"code","source":"feature_importance_df.to_csv(\"/kaggle/working/feature_importance.csv\")    \nplt.figure(figsize=(20,10))\n#sns.set(font_scale = 1)\nsns.barplot(x=\"Importance\", y=\"Feature\", data= feature_importance_df[0:50])\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":1.358323,"end_time":"2024-10-29T18:50:12.149357","exception":false,"start_time":"2024-10-29T18:50:10.791034","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"287d61f3-0c1e-4195-badf-fa3a69a73037","cell_type":"code","source":"cm = confusion_matrix(y , training_clipped_predictions, labels=[x for x in range(0,4)])\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=[x for x in range(0,4)])\ndisp.plot()","metadata":{"papermill":{"duration":0.631832,"end_time":"2024-10-29T18:50:13.085676","exception":false,"start_time":"2024-10-29T18:50:12.453844","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b23f1889-5763-430f-adf4-db803c90f122","cell_type":"code","source":"","metadata":{"papermill":{"duration":0.309176,"end_time":"2024-10-29T18:50:13.697754","exception":false,"start_time":"2024-10-29T18:50:13.388578","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4074bd3f-0617-45bd-aa83-b9d73ac9aa1f","cell_type":"code","source":"sns.histplot(training_predictions)\nsns.histplot(y)","metadata":{"papermill":{"duration":0.818757,"end_time":"2024-10-29T18:50:14.882449","exception":false,"start_time":"2024-10-29T18:50:14.063692","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"9ede430f-4bf3-4aeb-9e5a-8cfdb76cef29","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"c55d3625-48a6-4789-9554-f4bd5a806c07","cell_type":"markdown","source":"# XGBoost","metadata":{}},{"id":"3bbe329d-1dc6-4936-a00e-cf5fa5231d52","cell_type":"code","source":"import xgboost\nfrom xgboost import XGBRegressor\ndef xgb_cv_evaluator(learning_rate, max_depth, colsample_bytree,reg_alpha, reg_lambda):\n\n    i = 0\n    f1_scores = []\n    kappa_scores = []\n    for train_index, test_index in skf.split(numeric_X, y):\n\n        print('fold',i+1, X.shape, y.shape)\n        X_train_fold, X_test_fold = numeric_X.iloc[train_index], numeric_X.iloc[test_index]\n        \n        y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y[test_index]\n        xgb_callbacks = [\n            xgboost.callback.EvaluationMonitor(period=100),\n            #xgboost.callback.EarlyStopping(50, metric_name=\"QWK\", maximize=True, save_best=True)\n        ]\n        xgb_model = XGBRegressor(\n                    #objective = qwk_obj if custom_objective_on else 'regression',\n                    #metrics = 'None',\n                    learning_rate = learning_rate,\n                    max_depth = round(max_depth),\n                    #num_leaves = 2 ^( round(max_depth) - 1 ),\n                    colsample_bytree=colsample_bytree,\n                    reg_alpha =reg_alpha,\n                    reg_lambda = reg_lambda,\n                    n_estimators=n_estimators,\n                    random_state=42,\n                    enable_categorical = True,\n                    tree_method = \"hist\", \n                    device = \"cuda\",\n                    callbacks=xgb_callbacks,\n                    #eval_metric=quadratic_weighted_kappa_xgb,\n                    #verbose_eval=False\n                    early_stopping_rounds = 50\n                    \n\n        )\n\n\n        predictor = xgb_model.fit(X_train_fold,\n                                      y_train_fold,\n                                      eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n                                      #eval_set= (X_test_fold, y_test_fold),\n                                      verbose = 0,\n                                      #eval_metric=quadratic_weighted_kappa_xgb\n                                 )\n        predictions_fold = predictor.predict(X_test_fold)\n        predictions_fold = predictions_fold\n        predictions_fold = predictions_fold.clip(y_min, y_max).round()\n        #f1_fold = f1_score(y_test_fold_int, predictions_fold, average='weighted')\n        f1_scores.append(f1_fold)\n\n        kappa_fold = quadratic_kappa(y_test_fold_int, predictions_fold)\n        kappa_scores.append(kappa_fold)\n        #print(f' fold {i+1} = F1 score across fold: {f1_fold}', f'Cohen kappa score across fold: {kappa_fold}')\n        print( f'Cohen kappa score across fold: {kappa_fold}')\n        i+=1\n\n    #mean_f1_score = np.mean(f1_scores)\n    mean_kappa_score = np.mean(kappa_scores)\n\n\n    #print(f'Mean F1 score across {n_splits} folds: {mean_f1_score}')\n    print(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')\n    return mean_kappa_score","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"50abd17c-76dd-4cd1-9310-2ca055f51a22","cell_type":"code","source":"          \nif bayesian_optimization_xgb:\n    n_splits = 5\n    n_estimators = 1000\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    max_params = bayesian_parameter_optimization( init_rounds = bayesian_init_rounds, opt_rounds = bayesian_opt_rounds, parameter_bounds = xgb_pbounds, evaluator =xgb_cv_evaluator)\n    XGB_Params = max_params['params']\n    print(\"new XBG bayesian params:\", XGB_Params)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"1b09de3d-429c-4f3e-aeec-d23c3829bb8f","cell_type":"code","source":"XGB_Params","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"d55bc531-1812-4b26-a084-6fe249b6352b","cell_type":"code","source":"\n\nxgb_training_predictions = np.zeros((df_train.shape[0],))\nxgb_training_clipped_predictions = np.zeros((df_train.shape[0],))\n\n#xgboost_on = True\nif xgboost_on:\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    #f1_scores = []\n    kappa_scores = []\n    xgb_models = []\n    xgb_predictions = []\n    xgb_feature_importance_df = None\n    # Early stopping call back to avoid overfitting\n    y_true =  y\n\n    i=1\n    for train_index, test_index in skf.split(numeric_X, y):\n\n        print('fold',i)\n\n        X_train_fold, X_test_fold = numeric_X.iloc[train_index], numeric_X.iloc[test_index]   \n\n        y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y_true[test_index]\n        xgb_callbacks = [\n            xgboost.callback.EvaluationMonitor(period=100),\n            #xgboost.callback.EarlyStopping(75, metric_name=\"QWK\", maximize=True, save_best=True)\n        ]\n        xgb_model = XGBRegressor(\n                                learning_rate = XGB_Params[\"learning_rate\"],\n                                max_depth = round(XGB_Params[\"max_depth\"]),\n                                colsample_bytree = XGB_Params[\"colsample_bytree\"],\n                                reg_alpha = XGB_Params[\"reg_alpha\"],\n                                reg_lambda = XGB_Params[\"reg_lambda\"],\n                                enable_categorical = True, \n                                n_estimators=n_estimators,\n                                 #device = 'cpu',\n                                 #objective = qwk_obj,\n                                #metrics = 'None',\n                                tree_method = \"hist\", \n                                device = \"cuda\",\n                                callbacks=xgb_callbacks,\n                                #eval_metric=quadratic_weighted_kappa_xgb,\n                                #verbose_eval=False\n                                early_stopping_rounds = 50\n                                )\n        \n\n        predictor = xgb_model.fit(X_train_fold,\n                                      y_train_fold,\n                                      eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n                                      #eval_metric=quadratic_weighted_kappa_xgb,\n                                      #callbacks=xgb_callbacks,\n                                      verbose = 0                              \n                                 )\n\n        #saving all models for inference\n        xgb_models.append(predictor)\n        predictions_fold = predictor.predict(X_test_fold)\n        #predictions_fold = predictions_fold + a\n        xgb_training_predictions [test_index]=predictions_fold\n        predictions_fold = predictions_fold.clip(y_min, y_max).round()\n        xgb_predictions.append(predictions_fold)\n        xgb_training_clipped_predictions [test_index]=predictions_fold\n\n        f1_fold = f1_score(y_test_fold_int, predictions_fold, average='weighted')\n        f1_scores.append(f1_fold)\n\n        kappa_fold = cohen_kappa_score(y_test_fold_int, predictions_fold, weights='quadratic')\n        kappa_scores.append(kappa_fold)\n\n        cm = confusion_matrix(y_test_fold_int, predictions_fold, labels=[x for x in range(0,4)])\n\n        disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n                                      display_labels=[x for x in range(0,4)])\n        disp.plot()\n        plt.show()\n        print(f'F1 score across fold: {f1_fold}')\n        print(f'Cohen kappa score across fold: {kappa_fold}')\n\n        # Capturing the importance of each feature through the folds    \n        fold_feature_imp = pd.DataFrame({'Feature':numeric_X.columns, f'Value_{i}':predictor.feature_importances_}) \n        xgb_feature_importance_df = fold_feature_imp if xgb_feature_importance_df is None else xgb_feature_importance_df.merge(fold_feature_imp, on = ['Feature'] )      \n        value_cols = [ c for c in xgb_feature_importance_df.columns if c.startswith('Value_')]\n        xgb_feature_importance_df[\"Importance\"] = np.mean(xgb_feature_importance_df[value_cols], axis = 1)\n        xgb_feature_importance_df = xgb_feature_importance_df.sort_values(by = [\"Importance\"], ascending = False)\n        xgb_feature_importance_df = xgb_feature_importance_df[[\"Feature\", \"Importance\"] + [ c for c in xgb_feature_importance_df.columns if c.startswith('Value_')]]\n        display(xgb_feature_importance_df.head(10))       \n\n        i+=1\n\n    mean_f1_score = np.mean(f1_scores)\n    mean_kappa_score = np.mean(kappa_scores)\n\n    print(f'Mean F1 score across {n_splits} folds: {mean_f1_score}')\n    print(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"3f6dc7d6-d2ae-4670-af56-53290b543be1","cell_type":"code","source":"if xgboost_on:\n    xgb_feature_importance_df.to_csv(\"/kaggle/working/xgb_feature_importance.csv\")    \n    plt.figure(figsize=(20,10))\n    #sns.set(font_scale = 1)\n    sns.barplot(x=\"Importance\", y=\"Feature\", data= xgb_feature_importance_df[0:50])\n    plt.title('XGB Features (avg over folds)')\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"43f72f97-7717-427b-81bf-874245849461","cell_type":"code","source":"if xgboost_on:\n    cm = confusion_matrix(y , xgb_training_clipped_predictions, labels=[x for x in range(0,4)])\n\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n                                  display_labels=[x for x in range(0,4)])\n    disp.plot()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"595bfd54-4dce-44a4-a18d-91d77e1ba2d4","cell_type":"markdown","source":"# Random Forest Regressor","metadata":{}},{"id":"15f27d07-b56e-4f10-a965-dfd2affda0a8","cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\n\n\ndef rf_cv_evaluator( max_depth, n_estimators, max_samples ,min_samples_split, min_samples_leaf):\n\n    i = 0\n    kappa_scores = []\n    for train_index, test_index in skf.split(numeric_X, y):\n\n        #print('fold',i+1, X.shape, y.shape)\n        X_train_fold, X_test_fold = numeric_X.iloc[train_index], numeric_X.iloc[test_index]\n        \n        y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y[test_index]\n        model =  RandomForestRegressor(\n                    max_depth = round(max_depth),\n                    max_samples =max_samples ,\n                    min_samples_split =min_samples_split,\n                    min_samples_leaf = min_samples_leaf,\n                    n_estimators=round(n_estimators),\n                    random_state=42,\n        )\n\n\n        predictor = model.fit(X_train_fold,\n                                      y_train_fold,\n                                 )\n        predictions_fold = predictor.predict(X_test_fold)\n        #print( predictions_fold[:20])\n\n        predictions_fold = predictions_fold.clip(y_min, y_max).round()\n        #print(np.unique(predictions_fold), predictions_fold[:20])\n        #print(y_test_fold_int[:20])\n\n        kappa_fold = quadratic_kappa(y_test_fold_int, predictions_fold)\n        kappa_scores.append(kappa_fold)\n        #print( f'Cohen kappa score across fold: {kappa_fold}')\n        i+=1\n\n    mean_kappa_score = np.mean(kappa_scores)\n\n\n    print(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')\n    return mean_kappa_score","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"d12fd0ed-e996-4b93-8422-e52711c04340","cell_type":"code","source":" \n\nnumeric_X.fillna(0, inplace = True)\nif bayesian_optimization_rf:\n    n_splits = 5\n    n_estimators = 1000\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    #max_params = bayesian_parameter_optimization( init_rounds = bayesian_init_rounds, opt_rounds = bayesian_opt_rounds, parameter_bounds = rf_pbounds, evaluator =rf_cv_evaluator)\n    max_params = bayesian_parameter_optimization( init_rounds = 20, opt_rounds = 80, parameter_bounds = rf_pbounds, evaluator =rf_cv_evaluator)\n\n    rf_params = max_params['params']\n    print(\"new RF bayesian params:\", rf_params)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"5fbeaed9-2969-4f08-af2f-15339bdc048e","cell_type":"code","source":"rf_params","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"45aff062-bb9c-4c08-a3d0-ad09b085f7ee","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"d29eba4a-87c1-4e31-aa2f-7f4ea550c3aa","cell_type":"code","source":"rf_training_predictions = np.zeros((df_train.shape[0],))\nrf_training_clipped_predictions = np.zeros((df_train.shape[0],))\n\nif randomforest_on:\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    kappa_scores = []\n    rf_models = []\n    rf_predictions = []\n    rf_feature_importance_df = None\n    y_true =  y\n\n    i=1\n    for train_index, test_index in skf.split(X, y):\n\n        print('fold',i)\n\n        X_train_fold, X_test_fold = numeric_X.iloc[train_index], numeric_X.iloc[test_index]   \n\n        y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y_true[test_index]\n\n        rf_model =  RandomForestRegressor(\n                    max_depth = round(rf_params[\"max_depth\"]),\n                    max_samples =rf_params[\"max_samples\"] ,\n                    min_samples_split =rf_params[\"min_samples_split\"],\n                    min_samples_leaf = rf_params[\"min_samples_leaf\"],\n                    n_estimators=round(rf_params[\"n_estimators\"]),\n                    random_state=42,\n        )\n\n\n        predictor = rf_model.fit(X_train_fold,\n                                      y_train_fold,\n                                 )\n\n\n        rf_models.append(predictor)\n        predictions_fold = predictor.predict(X_test_fold)\n        rf_training_predictions [test_index]=predictions_fold\n        predictions_fold = predictions_fold.clip(y_min, y_max).round()\n        rf_predictions.append(predictions_fold)\n        rf_training_clipped_predictions [test_index]=predictions_fold\n\n        kappa_fold = cohen_kappa_score(y_test_fold_int, predictions_fold, weights='quadratic')\n        kappa_scores.append(kappa_fold)\n\n        cm = confusion_matrix(y_test_fold_int, predictions_fold, labels=[x for x in range(0,4)])\n\n        disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n                                      display_labels=[x for x in range(0,4)])\n        disp.plot()\n        plt.show()\n        print(f'Cohen kappa score across fold: {kappa_fold}')\n\n        fold_feature_imp = pd.DataFrame({'Feature':numeric_X.columns, f'Value_{i}':predictor.feature_importances_}) \n        rf_feature_importance_df = fold_feature_imp if rf_feature_importance_df is None else rf_feature_importance_df.merge(fold_feature_imp, on = ['Feature'] )      \n        value_cols = [ c for c in rf_feature_importance_df.columns if c.startswith('Value_')]\n        rf_feature_importance_df[\"Importance\"] = np.mean(rf_feature_importance_df[value_cols], axis = 1)\n        rf_feature_importance_df = rf_feature_importance_df.sort_values(by = [\"Importance\"], ascending = False)\n        rf_feature_importance_df = rf_feature_importance_df[[\"Feature\", \"Importance\"] + [ c for c in rf_feature_importance_df.columns if c.startswith('Value_')]]\n        display(rf_feature_importance_df.head(10))       \n\n        i+=1\n\n    mean_kappa_score = np.mean(kappa_scores)\n\n    print(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"916012f3-43f4-430f-9f42-99da968ec953","cell_type":"code","source":"if randomforest_on:\n    rf_feature_importance_df.to_csv(\"/kaggle/working/rf_feature_importance.csv\")    \n    plt.figure(figsize=(20,10))\n    #sns.set(font_scale = 1)\n    sns.barplot(x=\"Importance\", y=\"Feature\", data= rf_feature_importance_df[0:50])\n    plt.title('XGB Features (avg over folds)')\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"0c02ea56-8c65-44f0-9898-52ac67296d1d","cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nmodel_count = 1\ndf_predictions = df_train[[\"sii\"]].copy()\ndf_predictions[\"lgbm\"]= training_predictions\n\nif xgboost_on:\n        df_predictions[\"xgb\"]= xgb_training_predictions\n\n\nif randomforest_on:\n        df_predictions[\"rf\"]= rf_training_predictions\n        \n\nens_training_predictions =  df_predictions.drop(\"sii\", axis=1).mean(axis = 1)\n\nens_clipped_predictions =  ens_training_predictions.clip(y_min, y_max).round()\nkappa_score = cohen_kappa_score(y, ens_clipped_predictions, weights='quadratic')\n\nprint(\"Ensembloe kappa:\", kappa_score)\n\ncm = confusion_matrix(y , ens_clipped_predictions, labels=[x for x in range(0,4)])\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm,\n                              display_labels=[x for x in range(0,4)])\ndisp.plot()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"df59e2ff-1e85-43d5-865c-753415c24dcd","cell_type":"markdown","source":"## Threshold optimization","metadata":{}},{"id":"3945616f-7cf6-4e8f-8e1c-cd4a3924d94d","cell_type":"code","source":"# Copied from https://www.kaggle.com/code/ichigoe/lb0-494-with-tabnet\nfrom scipy.optimize import minimize\n\ndef evaluate_predictions(thresholds, y_true, y_pred):\n    rounded_p = threshold_Rounder(y_pred, thresholds)\n    return -cohen_kappa_score(y_true, rounded_p, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ncm = confusion_matrix(y , training_clipped_predictions, labels=[x for x in range(0,4)])\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm,\n                              display_labels=[x for x in range(0,4)])\ndisp.plot()\n\n\nKappaOPtimizer = minimize(evaluate_predictions,\n                          x0=[0.5, 1.5, 2.5], args=(y, ens_training_predictions),  \n                          method='Nelder-Mead')\n\n\nassert KappaOPtimizer.success, \"Optimization did not converge.\"\n\nens_training_predictions_tuned = threshold_Rounder(ens_training_predictions, KappaOPtimizer.x)\nkappa_score = cohen_kappa_score(y, ens_training_predictions_tuned, weights='quadratic') #quadratic_weighted_kappa()\nprint(kappa_score)\n\ncm = confusion_matrix(y , ens_training_predictions_tuned, labels=[x for x in range(0,4)])\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm,\n                              display_labels=[x for x in range(0,4)])\n\nprint(KappaOPtimizer.x)\ndisp.plot()","metadata":{"papermill":{"duration":0.316071,"end_time":"2024-10-29T18:50:16.143628","exception":false,"start_time":"2024-10-29T18:50:15.827557","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"8b6576cb-5eff-4fb8-8062-210e21d0fcc7","cell_type":"markdown","source":"# Inference","metadata":{"papermill":{"duration":0.306444,"end_time":"2024-10-29T18:50:16.758034","exception":false,"start_time":"2024-10-29T18:50:16.451590","status":"completed"},"tags":[]}},{"id":"048d7c12-efa4-4086-b3d9-fc634df50e05","cell_type":"markdown","source":"### Feature extraction on test dataset","metadata":{"papermill":{"duration":0.313575,"end_time":"2024-10-29T18:50:17.378344","exception":false,"start_time":"2024-10-29T18:50:17.064769","status":"completed"},"tags":[]}},{"id":"be15e882-a246-42bd-9a3f-23b254d4cfc2","cell_type":"code","source":"# We are basically doing performing the same steps for the test dataset that we did for the training set\n\ntest_directory_path = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/'\ntest_ts_df = analyze_ts_files(test_directory_path, gap_threshold)\ntest_ts_df[\"id\"] = test_ts_df[\"folder_name\"].str[3:]\n\n\n","metadata":{"papermill":{"duration":0.495243,"end_time":"2024-10-29T18:50:18.241881","exception":false,"start_time":"2024-10-29T18:50:17.746638","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"54d8f949-fcfa-4fbe-baa6-018f062956f3","cell_type":"code","source":"if time_series_diff_aggregation_on:\n    pivot_columns = ['diff_X', 'diff_Y', 'diff_Z', 'diff_enmo', 'diff_anglez', 'diff_light','distance', 'diff_battery_voltage']\n    if not filter_wear_flag_on :\n        pivot_columns.append('diff_non-wear_flag')\n    #pivot_columns = [ 'diff_enmo']\n    test_ts_diff_df = test_ts_df[test_ts_df.sequence_breaks <= max_sequence_breaks_diff]\n    df_test_ts_diff_features = test_ts_diff_df[[\"id\"]].copy()\n    for ts_aggregation in ts_aggregations_diff:\n        for pivot_feature in pivot_features_diff:\n                df_ts_agg =  timeseries_aggregation_diff(test_ts_diff_df, ts_aggregation,[pivot_feature])    \n                df_temp = ts_pivot_by_feature(df_ts_agg, pivot_feature, pivot_columns)\n                df_temp.columns = [ 'id'] + [ pivot_feature + '_' + ts_aggregation + '_' + c for c in df_temp.columns if c != 'id']\n                df_test_ts_diff_features = df_test_ts_diff_features.merge(df_temp, on = \"id\", how = \"left\")\n\n    ts_all_predictions = []\n    for model in ts_diff_models:\n        fold_prediction = model.predict(df_test_ts_diff_features[ [c for c in df_test_ts_diff_features.columns if c not in ['id']] ])\n        ts_all_predictions.append(fold_prediction)           \n\n\n    #df_test = df_test.merge(df_test_ts_diff_features, on  = \"id\", how = \"left\")\n    test_ts_diff_df[\"ts_predictions_diff\"] = np.mean(ts_all_predictions, axis=0) \n\n    df_test = df_test.merge(test_ts_diff_df[[\"id\",\"ts_predictions_diff\"]], on  = \"id\", how = \"left\")","metadata":{"papermill":{"duration":13.464484,"end_time":"2024-10-29T18:50:32.011955","exception":false,"start_time":"2024-10-29T18:50:18.547471","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"844f688c-303b-4118-83c6-e89110c73b07","cell_type":"code","source":"if time_series_aggregation_on:\n    pivot_columns = ['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage']\n    if not filter_wear_flag_on :\n        pivot_columns.append('non-wear_flag')\n    test_ts_df = test_ts_df[test_ts_df.sequence_breaks <= max_sequence_breaks]\n    df_test_ts_features = test_ts_df[[\"id\"]].copy()\n    for ts_aggregation in ts_aggregations:\n        for pivot_feature in pivot_features:\n                df_ts_agg =  timeseries_aggregation(test_ts_df, ts_aggregation,[pivot_feature])    \n                df_temp = ts_pivot_by_feature(df_ts_agg, pivot_feature, pivot_columns)\n                df_temp.columns = [ 'id'] + [ pivot_feature + '_' + ts_aggregation + '_' + c for c in df_temp.columns if c != 'id']\n                df_test_ts_features = df_test_ts_features.merge(df_temp, on = \"id\", how = \"left\")\n\n    ts_all_predictions = []\n    for model in ts_models:\n        fold_prediction = model.predict(df_test_ts_features[ [c for c in df_test_ts_features.columns if c not in ['id']] ])\n        ts_all_predictions.append(fold_prediction)           \n\n\n    #df_test = df_test.merge(df_test_ts_features, on  = \"id\", how = \"left\")\n    test_ts_df[\"ts_predictions\"] = np.mean(ts_all_predictions, axis=0) \n\n    df_test = df_test.merge(test_ts_df[[\"id\",\"ts_predictions\"]], on  = \"id\", how = \"left\")","metadata":{"papermill":{"duration":3.288475,"end_time":"2024-10-29T18:50:35.600954","exception":false,"start_time":"2024-10-29T18:50:32.312479","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"a4c8e2c6-cee1-4cbc-944a-b5cac35ef949","cell_type":"markdown","source":"## ","metadata":{"papermill":{"duration":0.301128,"end_time":"2024-10-29T18:50:36.202232","exception":false,"start_time":"2024-10-29T18:50:35.901104","status":"completed"},"tags":[]}},{"id":"c134cac4-70f6-423e-8a8b-a495eabc9848","cell_type":"code","source":"df_test = tabular_feature_engineering(df_test)\nprint(df_test.shape)\nif autoencoder_on:\n    \n    #df_ts_test_all_features  = df_test_ts_diff_features.merge(df_test_ts_features, on = \"id\")\n    df_ts_test_all_features  = df_test_ts_diff_features #.merge(df_test_ts_features, on = \"id\")\n    cols = [c for c in df_ts_all_features.columns if c not in [\"id\"]]\n    #cols = [ c for c in numeric_cols if c not in [ 'id', 'sii']]\n    test_unencoded_df = df_ts_test_all_features[cols].copy()\n    print(\"Auto-encoding \", unencoded_df.shape[0])\n    test_unencoded_df.fillna(0, inplace =  True)\n    \n    test_ts_encoded = perform_autoencoder(test_unencoded_df, encoding_dim=auto_encoder_dimensions, epochs=auto_encoder_epochs, batch_size=64)\n    \n    \n    time_series_cols = train_ts_encoded.columns.tolist()\n    test_ts_encoded[\"id\"]=df_ts_test_all_features[\"id\"]\n    #test_ts_encoded['id']=test_ts[\"id\"]\n\n\n    df_train = df_train.merge(train_ts_encoded, on  = \"id\", how = \"left\")    \n\n    df_test = df_test.merge(test_ts_encoded, on  = \"id\", how = \"left\")","metadata":{"papermill":{"duration":0.317071,"end_time":"2024-10-29T18:50:36.820371","exception":false,"start_time":"2024-10-29T18:50:36.503300","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"c9886935-b850-45a2-ba79-f1df6674fac5","cell_type":"code","source":"print(df_test.shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"0ab7a0d0-930b-4291-a73d-fcbac863385b","cell_type":"code","source":"if pca_components > 0:\n    df_test_ts_all_features  = df_test_ts_diff_features.merge(df_test_ts_features, on = \"id\")\n    #numeric_cols = df_test_ts_all_features.select_dtypes(include=['float64', 'int64']).columns\n    print(df_test_ts_all_features.shape, len(pca_numeric_cols))\n    #display(df_ts_diff_features.loc[:, df_ts_diff_features.isnull().any()])\n    \n\n    pca_cols = [ c for c in pca_numeric_cols if c not in [ 'id', 'sii']]\n    pca_test_X = df_test_ts_all_features[pca_cols].copy()\n    pca_test_X.fillna(0, inplace =  True)\n    reduced_test_X  = pca.transform(pca_test_X)\n    #print(test_pca_components.shape)\n\n\n    test_pca_df  = pd.DataFrame(reduced_test_X)\n    test_pca_df.columns =  pca_columns\n    test_pca_df[\"id\"] = test_ts_df[\"id\"]\n\n    df_test = df_test.merge(test_pca_df[ pca_columns + [\"id\"]] , on  = \"id\", how = \"left\")\n    \n    \n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"534f5cbb-53fa-4cbe-9841-56639cf61dcf","cell_type":"markdown","source":"### Prediction","metadata":{"papermill":{"duration":0.297908,"end_time":"2024-10-29T18:50:37.481095","exception":false,"start_time":"2024-10-29T18:50:37.183187","status":"completed"},"tags":[]}},{"id":"e64105f2-6278-4c75-95c0-3effee488379","cell_type":"code","source":"all_predictions = []\n#features = [ c for c in df_train.columns if c not in [ \"id\", \"sii\"]]\ntest_numeric_X = pd.get_dummies(df_test[features], columns = categorical_features)\nfor model in models:\n    fold_prediction = model.predict(df_test[features])\n    all_predictions.append(fold_prediction)\nar = np.mean(all_predictions, axis=0)\nprint(df_test.shape,ar.shape)","metadata":{"papermill":{"duration":0.379295,"end_time":"2024-10-29T18:50:38.179813","exception":false,"start_time":"2024-10-29T18:50:37.800518","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"86570543-2b53-4e07-91dd-2b7575edfe87","cell_type":"code","source":"test_numeric_X","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"990c8d80-245c-47a6-94f3-edf4123405e9","cell_type":"code","source":"all_predictions = []\n#features = [ c for c in df_train.columns if c not in [ \"id\", \"sii\"]]\n\ntest_numeric_X = pd.get_dummies(df_test, columns = categorical_features)\ntest_numeric_X.fillna(0, inplace = True)\nfor c in numeric_X.columns:\n    if c not in test_numeric_X:\n        test_numeric_X[c] = 0\n        \ntest_numeric_X = test_numeric_X[numeric_X.columns]\nprint(test_numeric_X.shape, numeric_X.shape)\n\n#print ( [ c for c in test_numeric_X.colums if \"BIA-Season_Spring])\nfor model in models:\n    fold_prediction = model.predict(df_test[features])\n    all_predictions.append(fold_prediction)\ndf_test[\"lgbm_predictions\"] = np.mean(all_predictions, axis=0)\nprediction_cols = [\"lgbm_predictions\"]\n\n\nif xgboost_on:\n    all_predictions = []\n    for model in xgb_models:\n        fold_prediction = model.predict(test_numeric_X)\n        all_predictions.append(fold_prediction)\n    df_test[\"xgb_predictions\"]= np.mean(all_predictions, axis=0)\n    prediction_cols.append(\"xgb_predictions\")\n\n\nif randomforest_on:\n    all_predictions = []\n    for model in rf_models:\n        fold_prediction = model.predict(test_numeric_X)\n        all_predictions.append(fold_prediction)\n    df_test[\"rf_predictions\"]= np.mean(all_predictions, axis=0)\n    prediction_cols.append(\"rf_predictions\")\n    \n\nens_test_predictions =  df_test[prediction_cols].mean(axis = 1)    \nprint(ens_test_predictions.shape)\n    \n\nif threshold_optimization_on:\n    test_predictions = threshold_Rounder( ens_test_predictions, KappaOPtimizer.x)\nelse:\n    test_predictions = ens_test_predictions\n\nprint(ens_test_predictions[:20])\nclipped_predictions = test_predictions.clip(y_min, y_max).round().astype(np.int32)\nprint(clipped_predictions[:20])","metadata":{"papermill":{"duration":0.395402,"end_time":"2024-10-29T18:50:38.885183","exception":false,"start_time":"2024-10-29T18:50:38.489781","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"e9a04856-475f-44e4-9cdc-4376da842c1e","cell_type":"code","source":"#test_numeric_X [\"BIA-Season_Spring\"]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"b7ab3dd3-fd90-4fd1-8d0d-303b3a9e07cc","cell_type":"markdown","source":"### Submission","metadata":{"papermill":{"duration":0.300135,"end_time":"2024-10-29T18:50:39.491795","exception":false,"start_time":"2024-10-29T18:50:39.191660","status":"completed"},"tags":[]}},{"id":"3a87bd40-f893-430a-8879-c2af89abc08f","cell_type":"code","source":"df_submission = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv\")\ndf_submission['sii'] = clipped_predictions\ndf_submission['sii'] = df_submission['sii'].astype(int)\ndf_submission.to_csv(\"submission.csv\",index=None)\ndisplay(df_submission.head())","metadata":{"papermill":{"duration":0.330875,"end_time":"2024-10-29T18:50:40.123909","exception":false,"start_time":"2024-10-29T18:50:39.793034","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"f3943858-95a1-41f1-ba1c-afd29b1171b5","cell_type":"code","source":"sns.boxplot (x=y_true , y =training_predictions)","metadata":{"papermill":{"duration":0.594409,"end_time":"2024-10-29T18:50:41.075131","exception":false,"start_time":"2024-10-29T18:50:40.480722","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"f1b8da55-a002-4b1b-b664-54bad3114ae3","cell_type":"markdown","source":"Ucomment and run this line if you are changing any of the cached features.","metadata":{"papermill":{"duration":0.298187,"end_time":"2024-10-29T18:50:41.672501","exception":false,"start_time":"2024-10-29T18:50:41.374314","status":"completed"},"tags":[]}},{"id":"3648d1a1-a48f-4840-bb97-85ba7f1b9a10","cell_type":"code","source":"#!rm /kaggle/working/ts_diff_features*\n#!rm /kaggle/working/*","metadata":{"papermill":{"duration":0.315381,"end_time":"2024-10-29T18:50:42.288330","exception":false,"start_time":"2024-10-29T18:50:41.972949","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"1f84f389-d346-4400-8568-b994c4849810","cell_type":"code","source":"#df_ts_features.X1_12.describe()\n    \n#sns.scatterplot(x= \"enmo_18\", y = \"sii\", data = df_train)","metadata":{"papermill":{"duration":0.311937,"end_time":"2024-10-29T18:50:42.902530","exception":false,"start_time":"2024-10-29T18:50:42.590593","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}