{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nfrom xgboost import XGBClassifier\n\nfrom sklearn.metrics import silhouette_score, silhouette_samples\nfrom sklearn.cluster import KMeans, MiniBatchKMeans\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-12T10:05:26.106514Z","iopub.execute_input":"2024-03-12T10:05:26.106992Z","iopub.status.idle":"2024-03-12T10:05:26.113337Z","shell.execute_reply.started":"2024-03-12T10:05:26.106962Z","shell.execute_reply":"2024-03-12T10:05:26.112001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # Manual catching dtypes:\n    float_vals=[]\n    float_vals_static_L=[\n    'clientscnt_136L',\n    'numinstlswithdpd5_4187116L',\n    'numinstmatpaidtearly2d_4499204L',\n    'numinstpaid_4499208L',\n    'numinstpaidearly3dest_4493216L',\n    'numinstpaidearly5dest_4493211L',\n    'numinstpaidearly5dobd_4499205L',\n    'numinstpaidearlyest_4493214L',\n    'numinstpaidlastcontr_4325080L',\n    'numinstregularpaidest_4493210L',\n    'numinsttopaygrest_4493213L',\n    'numinstunpaidmaxest_4493212L',\n    ]\n    float_vals.extend(float_vals_static_L)\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        if col in float_vals:\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n            \n    return df\ndef match_datasets_dtypes(df_train, df_test):\n    dtypes_train = df_train.to_pandas().dtypes\n    df_test=df_test.to_pandas().astype(dtypes_train)\n    return df_train, pl.DataFrame._from_pandas(df_test)\n    \ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df\n\n#Convert dates on bool instances, and take end title letter as F\ndef dates_to_tf(df: pl.DataFrame, columns) -> pl.DataFrame:\n    for column in columns:\n        df = df.with_columns((pl.col(column).is_not_null()).alias(f'{column[:-1]}F'))\n    return df\n\ndef sum_columns(df, column_names, sum_column_name):\n    # Seleccionar las columnas especificadas y sumarlas\n    total_sum = df.select([pl.col(col) for col in column_names]).fold(lambda accum, x: accum + x)\n    \n    # Agregar la columna de suma total al DataFrame\n    df = df.with_columns(total_sum.alias(sum_column_name))\n    \n    return df\n\ndef fill_na(df, columns,strategy=None, naval=0):\n  \n    if strategy=='median':\n        df_medians = df[columns].mean()\n        for column in columns:\n            df= df.with_columns(df[column].fill_null(df_medians[0,column]))\n    else:\n        for column in columns:\n            df = df.with_columns(df[column].fill_null(naval))\n     \n    return df\n\ndef kmean_caract_groups(df,columns,clusters,make_scaler=False,find_groups=True):\n    subset = df[columns]\n    if make_scaler:\n        subset = (subset - subset.mean()) / subset.std()\n    \n    kmean = MiniBatchKMeans(n_clusters=clusters,n_init=3)\n    print(f'Fit para k={clusters}')\n    groups = kmean.fit_predict(subset)\n    \n    if find_groups:\n        silscore = silhouette_score(subset,kmean.labels_)\n        silcoeff = silhouette_samples(subset, groups)\n        \n        print(f'Terminado para k={clusters}')\n        return clusters, kmean.inertia_, silscore, silcoeff, groups\n    else:\n        return groups","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:05:27.074942Z","iopub.execute_input":"2024-03-12T10:05:27.076148Z","iopub.status.idle":"2024-03-12T10:05:27.095575Z","shell.execute_reply.started":"2024-03-12T10:05:27.076087Z","shell.execute_reply":"2024-03-12T10:05:27.094277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_features(base_data, features):\n    for feature in features:\n        base_data=base_data.join(feature, how='left', on='case_id')\n    \n    return base_data","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:05:27.992215Z","iopub.execute_input":"2024-03-12T10:05:27.992637Z","iopub.status.idle":"2024-03-12T10:05:27.998239Z","shell.execute_reply.started":"2024-03-12T10:05:27.992605Z","shell.execute_reply":"2024-03-12T10:05:27.997015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Subsets pipelines\n## level 0 pipelines","metadata":{}},{"cell_type":"code","source":"def static_cb_pipe(df):\n    columns_sum=['firstquarter_103L','secondquarter_766L','thirdquarter_1082L','fourthquarter_440L' ]\n    fill0_columns = [\n    'contractssum_5085716L',\n    'days120_123L',\n    'days180_256L',\n    'days30_165L',\n    'days360_512L',\n    'days90_310L',\n    'numberofqueries_373L',\n    'pmtscount_423L',\n    'total_resultsL'\n     ]\n    \n    df=sum_columns(df,columns_sum,'total_resultsL')\n    df = fill_na(df,fill0_columns,0)\n    df = fill_na(df,['requesttype_4525192L'],'OTHER')\n    \n    \n    for col in fill0_columns:\n        df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    \n    total_columns= fill0_columns\n    total_columns.append('requesttype_4525192L')\n    return df, total_columns\n    \n\ndef static_pipe(df):\n    drop_columns=[\n        'bankacctype_710L',\n        'cardtype_51L',\n        'clientscnt_136L',\n        'equalitydataagreement_891L',\n        'equalityempfrom_62L',\n        'isbidproductrequest_292L',\n        'isdebitcard_729L',\n        'lastdependentsnum_448L',\n        'mastercontrelectronic_519L',\n        'mastercontrexist_109L',\n        'interestrategrace_34L',\n        'avgdbddpdlast3m_4187120P',\n        'avgdbdtollast24m_4525197P',\n        'avgdpdtolclosure24_3658938P',\n        'avgmaxdpdlast9m_3716943P',\n        'maxdpdinstlnum_3546846P',\n        'mindbdtollast24m_4525191P',\n        'avglnamtstart24m_4525187A',\n        'avgoutstandbalancel6m_4187114A',\n        'avgpmtlast12m_4525200A',\n        'inittransactionamount_650A',\n        'lastotherinc_902A',\n        'lastotherlnsexpense_631A',\n        'lastrejectcredamount_222A',\n        'maxannuity_4075009A',\n        'maxlnamtstart6m_4525199A',\n        'maxoutstandbalancel12m_4187113A',\n        'maxpmtlast3m_4525190A',\n        'sumoutstandtotalest_4493215A',\n        'totinstallast1m_4525188A',\n        'opencred_647L',\n        ]\n    fill0_columns=[\n        'amtinstpaidbefduel24m_4187115A',\n        'annuity_780A',\n        'annuitynextmonth_57A',\n        'avginstallast24m_3658937A',\n        'credamount_770A',\n        'currdebt_22A',\n        'currdebtcredtyperange_828A',\n        'disbursedcredamount_1113A',\n        'downpmt_116A',     \n    ]\n    \n    fill_median=['lastapprcredamount_781A',\n        'maininc_215A',\n        'maxannuity_159A',\n        'maxdebt4_972A',\n        'maxinstallast24m_3658928A',\n        'price_1097A',\n        'sumoutstandtotal_3546847A',\n        'totaldebt_9A',\n        'totalsettled_863A']\n    #fill_false = ['']\n\n    df=df.drop(drop_columns)\n    \n    num_cols=[]\n    L_cols=[]\n    for col in df.columns:\n        dtype = df[col].dtype\n        #df.to_pandas()[col].dtype()\n        if col[-1] == 'L' or col[-1] == 'P':\n            L_cols.append(col)\n            if str(dtype) == 'Float64':\n                #print (col)\n                num_cols.append(col)\n    for col in num_cols:\n        df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    df=fill_na(df,num_cols,0)\n    df=fill_na(df,fill0_columns,0)\n    df=fill_na(df,fill_median,strategy='median')\n    #df = fill_na(df,fill_false,naval=False)\n    L_cols.extend(fill0_columns)\n    L_cols.extend(fill_median)\n    return df, L_cols","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:05:41.270156Z","iopub.execute_input":"2024-03-12T11:05:41.270661Z","iopub.status.idle":"2024-03-12T11:05:41.287745Z","shell.execute_reply.started":"2024-03-12T11:05:41.270628Z","shell.execute_reply":"2024-03-12T11:05:41.286463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# level 1 pipelines","metadata":{}},{"cell_type":"code","source":"def person_1_pipe(df,df_base):\n    feats_df = []\n    person_columns = []\n    #Feature 1 year of birth\n    person_1_feats_1 = df.select([\"case_id\", \"num_group1\", \"birth_259D\"]).filter(\n    pl.col(\"num_group1\") == 0\n    ).drop(\"num_group1\").rename({\"birth_259D\": \"applicant_birthdate\"})\n    person_1_feats_1 = person_1_feats_1.join(df_base.select('case_id','date_decision'), how='left', on='case_id')\n    person_1_feats_1=person_1_feats_1.with_columns((pl.col('date_decision').cast(pl.Date).dt.year()-pl.col('applicant_birthdate').dt.year()).alias('years')).drop('applicant_birthdate','date_decision')\n    \n    feats_df.append(person_1_feats_1)\n    person_columns.append('years')\n    \n    #Feature 2 contaddr (location carr)\n    person_1_feats_2 = df.select([\"case_id\", \"num_group1\", \"contaddr_district_15M\",'contaddr_matchlist_1032L','contaddr_zipcode_807M']).filter(\n    pl.col(\"num_group1\") == 0\n    ).drop(\"num_group1\")\n    \n    feats_df.append(person_1_feats_2)\n    #person_columns.extend([\"contaddr_district_15M\",'contaddr_matchlist_1032L','contaddr_zipcode_807M'])\n    \n    #Feature 3 education\n    person_1_feats_3 = df.select([\"case_id\", \"num_group1\",'education_927M']).filter(\n    pl.col(\"num_group1\") == 0\n    ).drop(\"num_group1\")\n    \n    feats_df.append(person_1_feats_3)\n    person_columns.append('education_927M')\n    \n    #Feature 4 language\n    person_1_feats_4 = df.select([\"case_id\", \"num_group1\",'language1_981M']).filter(\n    pl.col(\"num_group1\") == 0\n    ).drop(\"num_group1\")\n    \n    feats_df.append(person_1_feats_4)\n    person_columns.append('language1_981M')\n    \n    #Feature 5 incomes and occupation\n    person_1_feats_5=df.select([\"case_id\", \"num_group1\",'incometype_1044T','mainoccupationinc_384A']).filter(\n    pl.col(\"num_group1\") == 0\n    ).drop(\"num_group1\")\n    \n    feats_df.append(person_1_feats_5)\n    person_columns.extend(['incometype_1044T','mainoccupationinc_384A'])\n    \n    #Feature 6 groups of workers\n    #NO APORTA MEJORA AL MODELO\n    #person_1_feats_6= person_1_feats_5.to_dummies(columns=[\"incometype_1044T\"])\n    #cols = person_1_feats_6.columns[1:]\n    #result=kmean_caract_groups(person_1_feats_6,cols,50,find_groups=False)\n    \n    #person_1_feats_6=person_1_feats_6.to_pandas()\n    #person_1_feats_6['groups_workers']=result.astype(str)\n    \n    #person_1_feats_6=pl.from_pandas(person_1_feats_6).select([\"case_id\",'groups_workers'])\n    \n    #feats_df.append(person_1_feats_6)\n    #person_columns.append('groups_workers')\n    \n    #Feature 7 and 8 type person feature evaluation\n    person_1_feats_7= df.select(['case_id','num_group1','personindex_1023L','persontype_792L']).filter(\n    pl.col('personindex_1023L').is_not_null()).group_by('case_id').agg(\n    pl.col('personindex_1023L').count().alias('personindex_1023L_count'),\n    pl.col('persontype_792L').sum().alias('persontype_792L_sum')\n    )\n    \n    feats_df.append(person_1_feats_7)\n    person_columns.extend(['personindex_1023L_count','persontype_792L_sum'])\n    \n    person_1_feats_8= df.select(['case_id','num_group1','persontype_1072L']).group_by('case_id').agg(\n     pl.col('persontype_1072L').sum().alias('persontype_1072L_sum')\n     )\n    \n    feats_df.append(person_1_feats_8)\n    person_columns.append('persontype_1072L_sum')\n    \n    #Feature 9 sex, familystate and guarantyflag\n    person_1_feats_9 = df.select([\"case_id\", \"num_group1\", 'familystate_447L','sex_738L','safeguarantyflag_411L']).filter(\n    pl.col(\"num_group1\") == 0\n    ).drop(\"num_group1\")\n    person_1_feats_9=fill_na(person_1_feats_9,['safeguarantyflag_411L'],naval=False)\n    \n    feats_df.append(person_1_feats_9)\n    person_columns.extend(['familystate_447L','sex_738L'])\n    \n    #Feature 10 relationships\n    person_1_feats_10= df.select(['case_id','num_group1','relationshiptoclient_642T'])\n    \n    relation_types={\n    None : 0,\n    'OTHER': 1,\n    'NEIGHBOR' : 2,\n    'COLLEAGUE' : 3,\n    'FRIEND' : 4,\n    'OTHER_RELATIVE' : 5,\n    'GRAND_PARENT' : 6,\n    'SIBLING' : 7,\n    'PARENT' : 8,\n    'CHILD' : 9,\n    'SPOUSE' : 10\n     }\n    \n    person_1_feats_10 = person_1_feats_10.to_pandas()\n    person_1_feats_10['relationshiptoclient_642T_num'] = person_1_feats_10['relationshiptoclient_642T'].map(relation_types)\n    \n    \n    person_1_feats_10=pl.from_pandas(person_1_feats_10)\n    aux_count = person_1_feats_10\n    \n    person_1_feats_10 = person_1_feats_10.group_by('case_id').agg(\n    pl.col('relationshiptoclient_642T_num').max().alias('relationshiptoclient_642T_nearest'),\n    pl.col('relationshiptoclient_642T_num').sum().alias('relationshiptoclient_642T_sum'),\n    )\n    aux_count = aux_count.filter(pl.col('relationshiptoclient_642T_num')>1).group_by('case_id').agg(\n    pl.col('relationshiptoclient_642T_num').count().alias('relationshiptoclient_642T_number'),\n    )\n    person_1_feats_10 = person_1_feats_10.join(aux_count,how='left', on= 'case_id')\n    \n    person_1_feats_10=fill_na(person_1_feats_10,['relationshiptoclient_642T_number'])\n    \n    feats_df.append(person_1_feats_10)\n    #person_columns.extend(['relationshiptoclient_642T_nearest','relationshiptoclient_642T_sum','relationshiptoclient_642T_number'])\n    \n    return feats_df, person_columns","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:05:45.926730Z","iopub.execute_input":"2024-03-12T11:05:45.927375Z","iopub.status.idle":"2024-03-12T11:05:45.949057Z","shell.execute_reply.started":"2024-03-12T11:05:45.927335Z","shell.execute_reply":"2024-03-12T11:05:45.947692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DF charge\n## level 0","metadata":{}},{"cell_type":"code","source":"train_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\")\ntrain_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:05:49.388498Z","iopub.execute_input":"2024-03-12T11:05:49.388944Z","iopub.status.idle":"2024-03-12T11:06:00.953240Z","shell.execute_reply.started":"2024-03-12T11:05:49.388894Z","shell.execute_reply":"2024-03-12T11:06:00.951988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_csv(dataPath + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_static_cb = pl.read_csv(dataPath + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:06:00.955245Z","iopub.execute_input":"2024-03-12T11:06:00.955713Z","iopub.status.idle":"2024-03-12T11:06:01.004849Z","shell.execute_reply.started":"2024-03-12T11:06:00.955672Z","shell.execute_reply":"2024-03-12T11:06:01.003617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# level 1","metadata":{}},{"cell_type":"code","source":"train_person_1 = pl.read_csv(dataPath + 'csv_files/train/train_person_1.csv').pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:06:40.201553Z","iopub.execute_input":"2024-03-12T11:06:40.202012Z","iopub.status.idle":"2024-03-12T11:06:42.905448Z","shell.execute_reply.started":"2024-03-12T11:06:40.201979Z","shell.execute_reply":"2024-03-12T11:06:42.903998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1 = pl.read_csv(dataPath + 'csv_files/test/test_person_1.csv').pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:06:42.907746Z","iopub.execute_input":"2024-03-12T11:06:42.908292Z","iopub.status.idle":"2024-03-12T11:06:42.918701Z","shell.execute_reply.started":"2024-03-12T11:06:42.908246Z","shell.execute_reply":"2024-03-12T11:06:42.917102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Matching dtypes after charge","metadata":{}},{"cell_type":"code","source":"train_static,test_static=match_datasets_dtypes(train_static,test_static)\ntrain_static_cb, test_static_cb= match_datasets_dtypes(train_static_cb, test_static_cb)\n\n\ntrain_person_1, test_person_1 = match_datasets_dtypes(train_person_1, test_person_1)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:06:43.186842Z","iopub.execute_input":"2024-03-12T11:06:43.187269Z","iopub.status.idle":"2024-03-12T11:06:48.447017Z","shell.execute_reply.started":"2024-03-12T11:06:43.187238Z","shell.execute_reply":"2024-03-12T11:06:48.445667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data transformation\n## level 0","metadata":{}},{"cell_type":"code","source":"conv_columns= [\n    'datefirstoffer_1144D',\n    'datelastinstal40dpd_247D',\n    'datelastunpaid_3546854D',\n    'lastactivateddate_801D',\n    'lastapprdate_640D',\n    'lastdelinqdate_224D',\n    'lastrejectdate_50D',\n    'payvacationpostpone_4187118D',\n    \n]\n#Transformation train static\ntrain_static= dates_to_tf(train_static,conv_columns)\ntest_static = dates_to_tf(test_static,conv_columns)\n\ntrain_static, columns_st = static_pipe(train_static)\ntest_static, columns_st = static_pipe(test_static)\n\n#transformation train static cb\ntrain_static_cb, columns_cb = static_cb_pipe(train_static_cb)\ntest_static_cb, columns_cb = static_cb_pipe(test_static_cb)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:06:48.448842Z","iopub.execute_input":"2024-03-12T11:06:48.449205Z","iopub.status.idle":"2024-03-12T11:06:50.475616Z","shell.execute_reply.started":"2024-03-12T11:06:48.449176Z","shell.execute_reply":"2024-03-12T11:06:50.474328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## level 1","metadata":{}},{"cell_type":"code","source":"train_person_1_feats, person_1_columns = person_1_pipe(train_person_1,train_basetable)\ntest_person_1_feats, _ = person_1_pipe(test_person_1,test_basetable)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:07:32.704968Z","iopub.execute_input":"2024-03-12T11:07:32.705402Z","iopub.status.idle":"2024-03-12T11:07:35.130285Z","shell.execute_reply.started":"2024-03-12T11:07:32.705366Z","shell.execute_reply":"2024-03-12T11:07:35.129013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_static_cols = []\nfor col in train_static.columns:\n    if col[-1] in (\"M\",'F'):\n        selected_static_cols.append(col)\n\nselected_static_cols.extend(columns_st)\n\nprint(selected_static_cols)\nprint('-'*50)\n\nselected_static_cb_cols = []\nfor col in train_static_cb.columns:\n    if col[-1] in (\"M\"):\n        selected_static_cb_cols.append(col)\n\nselected_static_cb_cols.extend(columns_cb)\nprint(selected_static_cb_cols)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:07:37.888699Z","iopub.execute_input":"2024-03-12T11:07:37.889257Z","iopub.status.idle":"2024-03-12T11:07:37.906518Z","shell.execute_reply.started":"2024-03-12T11:07:37.889213Z","shell.execute_reply":"2024-03-12T11:07:37.905180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Join dataframes\n# level 0","metadata":{}},{"cell_type":"code","source":"data = train_basetable.join(\n    train_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:10:54.757897Z","iopub.execute_input":"2024-03-12T11:10:54.758407Z","iopub.status.idle":"2024-03-12T11:10:56.289932Z","shell.execute_reply.started":"2024-03-12T11:10:54.758373Z","shell.execute_reply":"2024-03-12T11:10:56.288687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = test_basetable.join(\n    test_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:10:56.292032Z","iopub.execute_input":"2024-03-12T11:10:56.292397Z","iopub.status.idle":"2024-03-12T11:10:56.307080Z","shell.execute_reply.started":"2024-03-12T11:10:56.292368Z","shell.execute_reply":"2024-03-12T11:10:56.305620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Level 1","metadata":{}},{"cell_type":"code","source":"data = join_features(data,train_person_1_feats)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:10:56.308781Z","iopub.execute_input":"2024-03-12T11:10:56.309141Z","iopub.status.idle":"2024-03-12T11:10:58.669250Z","shell.execute_reply.started":"2024-03-12T11:10:56.309114Z","shell.execute_reply":"2024-03-12T11:10:58.667991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = join_features(data_submission,test_person_1_feats)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:10:58.671151Z","iopub.execute_input":"2024-03-12T11:10:58.671505Z","iopub.status.idle":"2024-03-12T11:10:58.679754Z","shell.execute_reply.started":"2024-03-12T11:10:58.671477Z","shell.execute_reply":"2024-03-12T11:10:58.678390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids = data[\"case_id\"].unique().shuffle(seed=1)\ncase_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.6, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\ncols_pred = []\n# Add features to use\ncols_pred.extend(selected_static_cols)\ncols_pred.extend(selected_static_cb_cols)\n#level 1 features\ncols_pred.extend(person_1_columns)\n\n\nprint(cols_pred)\n\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\nX_train = convert_strings(X_train)\nX_valid = convert_strings(X_valid)\nX_test = convert_strings(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T11:10:58.959127Z","iopub.execute_input":"2024-03-12T11:10:58.959534Z","iopub.status.idle":"2024-03-12T11:11:12.743083Z","shell.execute_reply.started":"2024-03-12T11:10:58.959503Z","shell.execute_reply":"2024-03-12T11:11:12.741558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {X_train.shape}\")\nprint(f\"Valid: {X_valid.shape}\")\nprint(f\"Test: {X_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:06:19.201557Z","iopub.execute_input":"2024-03-12T10:06:19.202768Z","iopub.status.idle":"2024-03-12T10:06:19.209519Z","shell.execute_reply.started":"2024-03-12T10:06:19.202722Z","shell.execute_reply":"2024-03-12T10:06:19.208018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_train = lgb.Dataset(X_train, label=y_train)\nlgb_valid = lgb.Dataset(X_valid, label=y_valid, reference=lgb_train)\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 3,\n    \"num_leaves\": 31,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.9,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"n_estimators\": 5000,\n    \"verbose\": -1,\n}\n\ngbm = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=lgb_valid,\n    callbacks=[lgb.log_evaluation(50), lgb.early_stopping(30)]\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:17:17.727472Z","iopub.execute_input":"2024-03-12T10:17:17.727882Z","iopub.status.idle":"2024-03-12T10:23:06.605087Z","shell.execute_reply.started":"2024-03-12T10:17:17.727854Z","shell.execute_reply":"2024-03-12T10:23:06.604034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for base, X in [(base_train, X_train), (base_valid, X_valid), (base_test, X_test)]:\n    y_pred = gbm.predict(X, num_iteration=gbm.best_iteration)\n    base[\"score\"] = y_pred\n\nprint(f'The AUC score on the train set is: {roc_auc_score(base_train[\"target\"], base_train[\"score\"])}') \nprint(f'The AUC score on the valid set is: {roc_auc_score(base_valid[\"target\"], base_valid[\"score\"])}') \nprint(f'The AUC score on the test set is: {roc_auc_score(base_test[\"target\"], base_test[\"score\"])}')  ","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:24:34.995456Z","iopub.execute_input":"2024-03-12T10:24:34.996365Z","iopub.status.idle":"2024-03-12T10:25:33.790420Z","shell.execute_reply.started":"2024-03-12T10:24:34.996325Z","shell.execute_reply":"2024-03-12T10:25:33.789501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\nstability_score_test = gini_stability(base_test)\n\nprint(f'The stability score on the train set is: {stability_score_train}') \nprint(f'The stability score on the valid set is: {stability_score_valid}') \nprint(f'The stability score on the test set is: {stability_score_test}')  ","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:25:33.791963Z","iopub.execute_input":"2024-03-12T10:25:33.792848Z","iopub.status.idle":"2024-03-12T10:25:34.824386Z","shell.execute_reply.started":"2024-03-12T10:25:33.792813Z","shell.execute_reply":"2024-03-12T10:25:34.823272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ny_submission_pred = gbm.predict(X_submission, num_iteration=gbm.best_iteration)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:25:34.825632Z","iopub.execute_input":"2024-03-12T10:25:34.826043Z","iopub.status.idle":"2024-03-12T10:25:34.930521Z","shell.execute_reply.started":"2024-03-12T10:25:34.826015Z","shell.execute_reply":"2024-03-12T10:25:34.929388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def view_cat_features_errors(view=False):\n    if view:\n        for col in X_train.select_dtypes(include='category').columns:\n            for feature in X_submission[col].value_counts().index:\n                if feature not in X_train[col].value_counts().index:\n                    print(f'{feature}---{col}')\n\nview_cat_features_errors(view=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:15:30.442289Z","iopub.execute_input":"2024-03-12T10:15:30.442661Z","iopub.status.idle":"2024-03-12T10:15:30.450071Z","shell.execute_reply.started":"2024-03-12T10:15:30.442634Z","shell.execute_reply":"2024-03-12T10:15:30.447967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:16:36.782472Z","iopub.execute_input":"2024-03-12T10:16:36.783782Z","iopub.status.idle":"2024-03-12T10:16:36.796509Z","shell.execute_reply.started":"2024-03-12T10:16:36.783726Z","shell.execute_reply":"2024-03-12T10:16:36.795526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}