{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":6497264,"sourceType":"datasetVersion","datasetId":3755317},{"sourceId":7992464,"sourceType":"datasetVersion","datasetId":4705028}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install \"/kaggle/input/lightgbm410/lightgbm-4.1.0-py3-none-manylinux_2_28_x86_64.whl\"","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:28:18.060675Z","iopub.execute_input":"2024-04-14T21:28:18.061069Z","iopub.status.idle":"2024-04-14T21:28:54.947404Z","shell.execute_reply.started":"2024-04-14T21:28:18.061037Z","shell.execute_reply":"2024-04-14T21:28:54.945907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport multiprocessing\nimport pickle\nimport warnings\nimport time\nimport tqdm\n\nimport plotly.express as px\nimport polars as pl\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier\nfrom sklearn.metrics import roc_auc_score\n\nwarnings.simplefilter(\"ignore\")\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\npd.set_option('display.float_format', lambda x: '%.5f' % x)\nplt.rcParams['figure.figsize']=[14,6]\npd.set_option('display.max_colwidth', 200)\ngc.enable()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:28:54.950386Z","iopub.execute_input":"2024-04-14T21:28:54.950753Z","iopub.status.idle":"2024-04-14T21:29:00.072003Z","shell.execute_reply.started":"2024-04-14T21:28:54.950720Z","shell.execute_reply":"2024-04-14T21:29:00.070848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = '/kaggle/input/home-credit-credit-risk-model-stability'\non = \"test\" #\"test\" #\"train\"","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.073665Z","iopub.execute_input":"2024-04-14T21:29:00.074483Z","iopub.status.idle":"2024-04-14T21:29:00.080266Z","shell.execute_reply.started":"2024-04-14T21:29:00.074441Z","shell.execute_reply":"2024-04-14T21:29:00.078873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/base-model-v1/base_model_v1.pkl', 'rb') as f:\n    rf_cv = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.083105Z","iopub.execute_input":"2024-04-14T21:29:00.083444Z","iopub.status.idle":"2024-04-14T21:29:00.189579Z","shell.execute_reply.started":"2024-04-14T21:29:00.083417Z","shell.execute_reply":"2024-04-14T21:29:00.188177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_columns = ['avgdpdtolclosure24_3658938P', 'firstdatedue_489D',\n       'numrejects9m_859L', 'dateofbirth_337D', 'birthdate_574D',\n       'lastrejectreason_759M', 'lastrejectdate_50D', 'pmtnum_254L',\n       'maxdpdlast12m_727P', 'numinstpaidearly3d_3546850L',\n       'numinstlsallpaid_934L', 'mobilephncnt_593L', 'monthsannuity_845L',\n       'maxdpdinstldate_3546855D', 'days180_256L', 'lastdelinqdate_224D',\n       'pmtaverage_3A', 'numinstlswithoutdpd_562L',\n       'datelastunpaid_3546854D', 'maxdbddpdtollast12m_3658940P',\n       'pctinstlsallpaidlate1d_3546856L', 'datelastinstal40dpd_247D',\n       'numinstlswithdpd10_728L', 'days90_310L', 'lastcancelreason_561M',\n       'maxdpdlast24m_143P', 'assignmentdate_238D', 'lastapprdate_640D',\n       'days120_123L', 'lastst_736L', 'price_1097A',\n       'lastapplicationdate_877D', 'responsedate_1012D',\n       'lastactivateddate_801D', 'numinsttopaygr_769L',\n       'firstclxcampaign_1125D', 'maxdpdlast3m_392P', 'education_1103M',\n       'cntpmts24_3658933L', 'numinstlallpaidearly3d_817L',\n       'maxdpdtolerance_577P', 'maxdpdlast6m_474P', 'riskassesment_302T',\n       'maxdpdlast9m_1059P', 'requesttype_4525192L', 'homephncnt_628L',\n       'pmtscount_423L', 'validfrom_1069D', 'days30_165L',\n       'inittransactioncode_186L', 'numinstunpaidmax_3546851L',\n       'isbidproduct_1095L', 'maxdebt4_972A', 'disbursedcredamount_1113A',\n       'avgdbddpdlast3m_4187120P', 'avgdbddpdlast24m_3658932P',\n       'maxdbddpdlast1m_3658939P', 'education_1138M',\n       'numincomingpmts_3546848L', 'pctinstlsallpaidlate6d_3546844L',\n       'pmtssum_45A', 'lastrejectreasonclient_4145040M',\n       'previouscontdistrict_112M', 'responsedate_4527233D',\n       'riskassesment_940T', 'amtinstpaidbefduel24m_4187115A', 'eir_270L',\n       'pctinstlsallpaidearl3d_427L', 'totalsettled_863A',\n       'disbursementtype_67L', 'currdebt_22A',\n       'maxdbddpdtollast6m_4187119P', 'maxdpdtolerance_374P',\n       'totaldebt_9A', 'numinstunpaidmaxest_4493212L',\n       'pctinstlsallpaidlat10d_839L', 'dtlastpmtallstes_4499206D',\n       'datefirstoffer_1144D', 'daysoverduetolerancedd_3976961L',\n       'avgdbdtollast24m_4525197P', 'days360_512L', 'WEEK_NUM',\n       'lastapprcommoditycat_1041M', 'avgmaxdpdlast9m_3716943P',\n       'applicationscnt_867L', 'pctinstlsallpaidlate4d_3546849L',\n       'numinstpaidearly_338L', 'interestrate_311L',\n       'maxdpdfrom6mto36m_3546853P', 'numinstmatpaidtearly2d_4499204L',\n       'annuity_780A', 'pmtnum_8L', 'familystate_726L',\n       'dateofbirth_342D', 'district_544M', 'lastrejectcommoditycat_161M',\n       'maritalst_385M', 'firstnonzeroinstldate_307D',\n       'creationdate_885D', 'num_group1', 'mainoccupationinc_437A',\n       'lastrejectcredamount_222A',\"case_id\"]","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.191813Z","iopub.execute_input":"2024-04-14T21:29:00.192499Z","iopub.status.idle":"2024-04-14T21:29:00.205735Z","shell.execute_reply.started":"2024-04-14T21:29:00.192461Z","shell.execute_reply":"2024-04-14T21:29:00.204498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_case = pd.read_parquet(f'{ROOT}/parquet_files/{on}/{on}_base.parquet')\nif on ==\"train\":\n    train_case = train_case.drop(columns=[\"date_decision\",\"MONTH\",\"target\"])\nelse:\n    train_case = train_case.drop(columns=[\"date_decision\",\"MONTH\"])\ntrain_case.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.207251Z","iopub.execute_input":"2024-04-14T21:29:00.207622Z","iopub.status.idle":"2024-04-14T21:29:00.315594Z","shell.execute_reply.started":"2024-04-14T21:29:00.207589Z","shell.execute_reply":"2024-04-14T21:29:00.314234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0_0 = pd.read_parquet(f'{ROOT}/parquet_files/{on}/{on}_static_0_0.parquet')\ntrain_static_0_0 = train_static_0_0[train_static_0_0.columns.intersection(filtered_columns)]\nprint(\"train_static_0_0\", train_static_0_0.shape)\ntrain_static_0_1 = pd.read_parquet(f'{ROOT}/parquet_files/{on}/{on}_static_0_1.parquet')\ntrain_static_0_1 = train_static_0_1[train_static_0_1.columns.intersection(filtered_columns)]\nprint(\"train_static_0_1\", train_static_0_1.shape)\ntrain_static = pd.concat([train_static_0_0,train_static_0_1])\ndel train_static_0_0\ndel train_static_0_1\ngc.collect()\nprint(\"train_static\", train_static.shape)\ntrain_static.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.317425Z","iopub.execute_input":"2024-04-14T21:29:00.317900Z","iopub.status.idle":"2024-04-14T21:29:00.561124Z","shell.execute_reply.started":"2024-04-14T21:29:00.317856Z","shell.execute_reply":"2024-04-14T21:29:00.559865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.562561Z","iopub.execute_input":"2024-04-14T21:29:00.562931Z","iopub.status.idle":"2024-04-14T21:29:00.681921Z","shell.execute_reply.started":"2024-04-14T21:29:00.562894Z","shell.execute_reply":"2024-04-14T21:29:00.680650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_cb_0 = pd.read_parquet(f'{ROOT}/parquet_files/{on}/{on}_static_cb_0.parquet')\ntrain_static_cb_0 = train_static_cb_0[train_static_cb_0.columns.intersection(filtered_columns)]\nprint(\"train_static_cb_0\", train_static_cb_0.shape)\ntrain_static = pd.merge(train_static, train_static_cb_0, left_on=\"case_id\", right_on=\"case_id\", how=\"left\")\ndel train_static_cb_0\ngc.collect()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.683477Z","iopub.execute_input":"2024-04-14T21:29:00.683837Z","iopub.status.idle":"2024-04-14T21:29:00.845197Z","shell.execute_reply.started":"2024-04-14T21:29:00.683796Z","shell.execute_reply":"2024-04-14T21:29:00.844289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_applprev_1_0 = pd.read_parquet(f'{ROOT}/parquet_files/{on}/{on}_applprev_1_0.parquet')\ntrain_applprev_1_0 = train_applprev_1_0[train_applprev_1_0.columns.intersection(filtered_columns)]\nprint(\"train_applprev_1_0\", train_applprev_1_0.shape)\ntrain_static = pd.merge(train_static, train_applprev_1_0, left_on=\"case_id\", right_on=\"case_id\", how=\"left\")\ndel train_applprev_1_0\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.849278Z","iopub.execute_input":"2024-04-14T21:29:00.849876Z","iopub.status.idle":"2024-04-14T21:29:00.981672Z","shell.execute_reply.started":"2024-04-14T21:29:00.849836Z","shell.execute_reply":"2024-04-14T21:29:00.980845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_case = pd.merge(train_case, train_static, left_on=\"case_id\", right_on=\"case_id\", how=\"left\")\ndel train_static\ngc.collect()\nprint(\"train_case\", train_case.shape)\ntrain_case.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:00.982892Z","iopub.execute_input":"2024-04-14T21:29:00.983389Z","iopub.status.idle":"2024-04-14T21:29:01.159040Z","shell.execute_reply.started":"2024-04-14T21:29:00.983361Z","shell.execute_reply":"2024-04-14T21:29:01.158229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fields_with_bad_dtypes = [\n    \"bankacctype_710L\", \"cardtype_51L\", \"credtype_322L\", \"datefirstoffer_1144D\", \n    \"datelastinstal40dpd_247D\", \"datelastunpaid_3546854D\", \"disbursementtype_67L\", \n    \"dtlastpmtallstes_4499206D\", \"equalitydataagreement_891L\", \"equalityempfrom_62L\", \n    \"firstclxcampaign_1125D\", \"firstdatedue_489D\", \"inittransactioncode_186L\", \n    \"isbidproductrequest_292L\", \"isdebitcard_729L\", \"lastactivateddate_801D\", \n    \"lastapplicationdate_877D\", \"lastapprcommoditycat_1041M\", \"lastapprcommoditytypec_5251766M\", \n    \"lastapprdate_640D\", \"lastcancelreason_561M\", \"lastdelinqdate_224D\", \n    \"lastrejectcommoditycat_161M\", \"lastrejectcommodtypec_5251769M\", \"lastrejectdate_50D\", \n    \"lastrejectreason_759M\", \"lastrejectreasonclient_4145040M\", \"lastrepayingdate_696D\", \n    \"lastst_736L\", \"maxdpdinstldate_3546855D\", \"opencred_647L\", \"paytype1st_925L\", \n    \"paytype_783L\", \"payvacationpostpone_4187118D\", \"previouscontdistrict_112M\", \n    \"twobodfilling_608L\", \"typesuite_864L\", \"validfrom_1069D\",\n    \"assignmentdate_238D\", \"assignmentdate_4527235D\", \"assignmentdate_4955616D\", \n    \"birthdate_574D\", \"dateofbirth_337D\", \"dateofbirth_342D\", \n    \"description_5085714M\", \"education_1103M\", \"education_88M\", \n    \"maritalst_385M\", \"maritalst_893M\", \"requesttype_4525192L\", \n    \"responsedate_1012D\", \"responsedate_4527233D\", \"responsedate_4917613D\", \n    \"riskassesment_302T\", \"approvaldate_319D\", \"cancelreason_3545846M\", \n    \"creationdate_885D\", \"credacc_status_367L\", \"credtype_587L\", \n    \"dateactivated_425D\", \"district_544M\", \"dtlastpmt_581D\", \n    \"dtlastpmtallstes_3545839D\", \"education_1138M\", \"employedfrom_700D\", \n    \"familystate_726L\", \"firstnonzeroinstldate_307D\", \"inittransactioncode_279L\", \n    \"isbidproduct_390L\", \"isdebitcard_527L\", \"postype_4733339M\", \n    \"profession_152M\", \"rejectreason_755M\", \"rejectreasonclient_4145042M\", \n    \"status_219L\"\n]\n\n# Convert fields to categorical data type\ncat_col = list(set(filtered_columns).intersection(fields_with_bad_dtypes))\n# cat_col\ntrain_case[cat_col] = train_case[cat_col].astype('category')\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.160414Z","iopub.execute_input":"2024-04-14T21:29:01.160953Z","iopub.status.idle":"2024-04-14T21:29:01.195030Z","shell.execute_reply.started":"2024-04-14T21:29:01.160922Z","shell.execute_reply":"2024-04-14T21:29:01.194114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_case[\"case_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.196434Z","iopub.execute_input":"2024-04-14T21:29:01.196988Z","iopub.status.idle":"2024-04-14T21:29:01.205178Z","shell.execute_reply.started":"2024-04-14T21:29:01.196956Z","shell.execute_reply":"2024-04-14T21:29:01.203807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# temp_df = train_case[[\"case_id\"]]\n# train_case.set_index(\"case_id\",inplace=True)\n# temp_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.206937Z","iopub.execute_input":"2024-04-14T21:29:01.207311Z","iopub.status.idle":"2024-04-14T21:29:01.218631Z","shell.execute_reply.started":"2024-04-14T21:29:01.207282Z","shell.execute_reply":"2024-04-14T21:29:01.217224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_predictions_custom(model, X, threshold=0.5):\n#     y_prob = model.predict_proba(X)\n#     y_pred = np.where(y_prob[:, 1] > threshold, 1, 0)\n#     y_prob = y_prob[:, 1]\n#     return y_pred, y_prob","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.220284Z","iopub.execute_input":"2024-04-14T21:29:01.220638Z","iopub.status.idle":"2024-04-14T21:29:01.231057Z","shell.execute_reply.started":"2024-04-14T21:29:01.220610Z","shell.execute_reply":"2024-04-14T21:29:01.229845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_predictions_custom1(model, X, threshold=0.5):\n#     y_prob = model.predict_proba(X)[:, 1]\n#     return y_prob[:, 1]","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.232653Z","iopub.execute_input":"2024-04-14T21:29:01.233019Z","iopub.status.idle":"2024-04-14T21:29:01.247765Z","shell.execute_reply.started":"2024-04-14T21:29:01.232991Z","shell.execute_reply":"2024-04-14T21:29:01.246500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_case","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.249461Z","iopub.execute_input":"2024-04-14T21:29:01.249835Z","iopub.status.idle":"2024-04-14T21:29:01.260793Z","shell.execute_reply.started":"2024-04-14T21:29:01.249792Z","shell.execute_reply":"2024-04-14T21:29:01.259359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# batch_size = 10000  # You can adjust this based on your computational resources\n\n# # Initialize an empty array to store predictions\n# all_predictions = []\n\n# # Divide data into batches and make predictions\n# for i in tqdm.tqdm(range(0, len(train_case), batch_size)):\n# #     print(\"processing batch:\",i)\n#     batch_data = train_case[i:i+batch_size]\n#     temp_df = batch_data[[\"case_id\"]]\n#     try:\n#     #     display(temp_df.head())\n#         batch_data.set_index(\"case_id\",inplace=True)\n#         y_train_pred1, y_train_proba1 = get_predictions_custom(rf_cv,batch_data)\n#         temp_df[\"score\"] = y_train_proba1\n#     except Exception as e:\n#         temp_df[\"score\"] = 0.5\n# #     display(temp_df.head())\n#     all_predictions.append(temp_df)\n#     del batch_data\n#     del temp_df\n#     del y_train_pred1\n#     del y_train_proba1\n#     gc.collect()\n\n# # Concatenate predictions from all batches\n# merged_predictions = pd.concat(all_predictions)\n# merged_predictions.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.262603Z","iopub.execute_input":"2024-04-14T21:29:01.263292Z","iopub.status.idle":"2024-04-14T21:29:01.274106Z","shell.execute_reply.started":"2024-04-14T21:29:01.263259Z","shell.execute_reply":"2024-04-14T21:29:01.272751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission =  train_case[[\"case_id\"]]\n# train_case.set_index(\"case_id\",inplace=True)\n# # Drop duplicate rows\n# # submission = submission.drop_duplicates()\n# #try mean of score\n\n# submission[\"score\"] = np.random.rand(len(submission))\n# submission = pd.DataFrame(submission.groupby(\"case_id\")[\"score\"].mean())\n# submission.reset_index(inplace=True) \n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.275765Z","iopub.execute_input":"2024-04-14T21:29:01.276197Z","iopub.status.idle":"2024-04-14T21:29:01.289912Z","shell.execute_reply.started":"2024-04-14T21:29:01.276164Z","shell.execute_reply":"2024-04-14T21:29:01.288526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# temp = train_case.groupby(\"case_id\").max()\n# temp.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.291728Z","iopub.execute_input":"2024-04-14T21:29:01.292151Z","iopub.status.idle":"2024-04-14T21:29:01.303015Z","shell.execute_reply.started":"2024-04-14T21:29:01.292114Z","shell.execute_reply":"2024-04-14T21:29:01.301987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission =  train_case[[\"case_id\"]]\n# # train_case.set_index(\"case_id\",inplace=True)\n# # Drop duplicate rows\n# # submission = submission.drop_duplicates()\n# #try mean of score\n# try:\n#     submission[\"score\"] = model.predict_proba(train_case)[:, 1]\n# except Exception as e:\n#     submission[\"score\"] = np.random.rand(len(submission))\n# submission = pd.DataFrame(submission.groupby(\"case_id\")[\"score\"].mean())\n# submission.reset_index(inplace=True) \n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.304262Z","iopub.execute_input":"2024-04-14T21:29:01.304578Z","iopub.status.idle":"2024-04-14T21:29:01.318042Z","shell.execute_reply.started":"2024-04-14T21:29:01.304552Z","shell.execute_reply":"2024-04-14T21:29:01.316836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 500\nall_predictions = []\n\n# Divide data into batches and make predictions\nfor i in tqdm.tqdm(range(0, len(train_case), batch_size)):\n    batch_data = train_case[i:i+batch_size]\n    temp_df = batch_data[[\"case_id\"]]\n    try:\n        batch_data.set_index(\"case_id\",inplace=True)\n        temp_df[\"score\"] = rf_cv.predict_proba(batch_data)[:, 1]\n    except Exception as e:\n        temp_df[\"score\"] = np.random.rand(len(submission))\n    all_predictions.append(temp_df)\n    del batch_data\n    del temp_df\n    gc.collect()\n\n# Concatenate predictions from all batches\nmerged_predictions = pd.concat(all_predictions)\nmerged_predictions.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.322097Z","iopub.execute_input":"2024-04-14T21:29:01.322478Z","iopub.status.idle":"2024-04-14T21:29:01.534174Z","shell.execute_reply.started":"2024-04-14T21:29:01.322445Z","shell.execute_reply":"2024-04-14T21:29:01.532998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del all_predictions\n# del train_case\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.535680Z","iopub.execute_input":"2024-04-14T21:29:01.536042Z","iopub.status.idle":"2024-04-14T21:29:01.541288Z","shell.execute_reply.started":"2024-04-14T21:29:01.536013Z","shell.execute_reply":"2024-04-14T21:29:01.540143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merged_predictions.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.542750Z","iopub.execute_input":"2024-04-14T21:29:01.543192Z","iopub.status.idle":"2024-04-14T21:29:01.554676Z","shell.execute_reply.started":"2024-04-14T21:29:01.543154Z","shell.execute_reply":"2024-04-14T21:29:01.553494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.read_csv(ROOT+\"/sample_submission.csv\")\n# submission = submission.set_index(\"case_id\")  # Set case_id as the index","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.556273Z","iopub.execute_input":"2024-04-14T21:29:01.556723Z","iopub.status.idle":"2024-04-14T21:29:01.569805Z","shell.execute_reply.started":"2024-04-14T21:29:01.556681Z","shell.execute_reply":"2024-04-14T21:29:01.568629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merged_predictions.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.571016Z","iopub.execute_input":"2024-04-14T21:29:01.571477Z","iopub.status.idle":"2024-04-14T21:29:01.580340Z","shell.execute_reply.started":"2024-04-14T21:29:01.571438Z","shell.execute_reply":"2024-04-14T21:29:01.579359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(merged_predictions.groupby(\"case_id\")[\"score\"].mean())#.values\ndel merged_predictions\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.581446Z","iopub.execute_input":"2024-04-14T21:29:01.581910Z","iopub.status.idle":"2024-04-14T21:29:01.707444Z","shell.execute_reply.started":"2024-04-14T21:29:01.581853Z","shell.execute_reply":"2024-04-14T21:29:01.706284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.714203Z","iopub.execute_input":"2024-04-14T21:29:01.714570Z","iopub.status.idle":"2024-04-14T21:29:01.719785Z","shell.execute_reply.started":"2024-04-14T21:29:01.714541Z","shell.execute_reply":"2024-04-14T21:29:01.718565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.721305Z","iopub.execute_input":"2024-04-14T21:29:01.721755Z","iopub.status.idle":"2024-04-14T21:29:01.732589Z","shell.execute_reply.started":"2024-04-14T21:29:01.721715Z","shell.execute_reply":"2024-04-14T21:29:01.731389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.734031Z","iopub.execute_input":"2024-04-14T21:29:01.734476Z","iopub.status.idle":"2024-04-14T21:29:01.745266Z","shell.execute_reply.started":"2024-04-14T21:29:01.734436Z","shell.execute_reply":"2024-04-14T21:29:01.744286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission[\"score\"] = np.clip(np.nan_to_num(submission[\"score\"],nan=0.5),0,1)\n# submission.reset_index(inplace=True) \n# # submission.fillna(0.5,inplace=True)\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.746591Z","iopub.execute_input":"2024-04-14T21:29:01.747132Z","iopub.status.idle":"2024-04-14T21:29:01.758206Z","shell.execute_reply.started":"2024-04-14T21:29:01.747090Z","shell.execute_reply":"2024-04-14T21:29:01.756846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# plt.hist(submission[\"score\"], bins=30, color='skyblue', edgecolor='black')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.760127Z","iopub.execute_input":"2024-04-14T21:29:01.760559Z","iopub.status.idle":"2024-04-14T21:29:01.769659Z","shell.execute_reply.started":"2024-04-14T21:29:01.760517Z","shell.execute_reply":"2024-04-14T21:29:01.768530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.set_index(\"case_id\",inplace=True)\nsubmission.to_csv(\"submission.csv\",index=None)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.771126Z","iopub.execute_input":"2024-04-14T21:29:01.771565Z","iopub.status.idle":"2024-04-14T21:29:01.785352Z","shell.execute_reply.started":"2024-04-14T21:29:01.771526Z","shell.execute_reply":"2024-04-14T21:29:01.784089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T21:29:01.789665Z","iopub.execute_input":"2024-04-14T21:29:01.790045Z","iopub.status.idle":"2024-04-14T21:29:01.804146Z","shell.execute_reply.started":"2024-04-14T21:29:01.790015Z","shell.execute_reply":"2024-04-14T21:29:01.802858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}