{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Adding Count occurrence feature to the list","metadata":{}},{"cell_type":"markdown","source":"** As we are going to squeeze the data to the most recent or average,\nI think we can add this count feature, Lets see does adding the number of occurances of a customer really matters?**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np \nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:01:04.935733Z","iopub.execute_input":"2022-07-28T07:01:04.936146Z","iopub.status.idle":"2022-07-28T07:01:05.560205Z","shell.execute_reply.started":"2022-07-28T07:01:04.936064Z","shell.execute_reply":"2022-07-28T07:01:05.559279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\nlabels=pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\n%time","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:01:05.562212Z","iopub.execute_input":"2022-07-28T07:01:05.562474Z","iopub.status.idle":"2022-07-28T07:01:21.366847Z","shell.execute_reply.started":"2022-07-28T07:01:05.562448Z","shell.execute_reply":"2022-07-28T07:01:21.365743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"Count_occ\"]=df_train.groupby(\"customer_ID\")[\"customer_ID\"].transform(\"count\")\ndf_train = df_train.merge(labels, left_on=\"customer_ID\", right_on='customer_ID') ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:01:21.368115Z","iopub.execute_input":"2022-07-28T07:01:21.368761Z","iopub.status.idle":"2022-07-28T07:03:25.461073Z","shell.execute_reply.started":"2022-07-28T07:01:21.368734Z","shell.execute_reply":"2022-07-28T07:03:25.459414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(df_train[\"Count_occ\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:03:25.464384Z","iopub.execute_input":"2022-07-28T07:03:25.464738Z","iopub.status.idle":"2022-07-28T07:03:26.486915Z","shell.execute_reply.started":"2022-07-28T07:03:25.464706Z","shell.execute_reply":"2022-07-28T07:03:26.485585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Grouping Data by customer_ID and Taking the Most Recent Records","metadata":{}},{"cell_type":"code","source":"df_train = df_train.groupby([\"customer_ID\"]).tail(1).set_index('customer_ID')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:03:26.488422Z","iopub.execute_input":"2022-07-28T07:03:26.488792Z","iopub.status.idle":"2022-07-28T07:03:29.296767Z","shell.execute_reply.started":"2022-07-28T07:03:26.488763Z","shell.execute_reply":"2022-07-28T07:03:29.295391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets check the pearson correlation bw target and Count_occ column\nprint(df_train[\"target\"].corr(df_train[\"Count_occ\"]))","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:03:29.298419Z","iopub.execute_input":"2022-07-28T07:03:29.298785Z","iopub.status.idle":"2022-07-28T07:03:29.313018Z","shell.execute_reply.started":"2022-07-28T07:03:29.298761Z","shell.execute_reply":"2022-07-28T07:03:29.312086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets check how target is distributed among the Count_occ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T05:35:25.078121Z","iopub.execute_input":"2022-07-28T05:35:25.078519Z","iopub.status.idle":"2022-07-28T05:35:25.085980Z","shell.execute_reply.started":"2022-07-28T05:35:25.078488Z","shell.execute_reply":"2022-07-28T05:35:25.084455Z"}}},{"cell_type":"code","source":"d_new=df_train\nd_new[\"isDefault\"]=d_new[\"target\"]>0  #Boolean feature","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:03:29.319949Z","iopub.execute_input":"2022-07-28T07:03:29.320318Z","iopub.status.idle":"2022-07-28T07:03:29.332565Z","shell.execute_reply.started":"2022-07-28T07:03:29.320285Z","shell.execute_reply":"2022-07-28T07:03:29.331594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(d_new[\"isDefault\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:03:29.334839Z","iopub.execute_input":"2022-07-28T07:03:29.335217Z","iopub.status.idle":"2022-07-28T07:03:29.648608Z","shell.execute_reply.started":"2022-07-28T07:03:29.335185Z","shell.execute_reply":"2022-07-28T07:03:29.647625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ng=sns.histplot(data=d_new,x=\"Count_occ\",hue=\"isDefault\",multiple=\"dodge\",kde=True,shrink=0.7)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:06:58.350946Z","iopub.execute_input":"2022-07-28T07:06:58.351338Z","iopub.status.idle":"2022-07-28T07:07:02.909888Z","shell.execute_reply.started":"2022-07-28T07:06:58.351309Z","shell.execute_reply":"2022-07-28T07:07:02.908770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g=sns.histplot(data=d_new,x=\"Count_occ\",hue=\"isDefault\",multiple=\"dodge\",stat=\"probability\",discrete=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:12:12.870381Z","iopub.execute_input":"2022-07-28T07:12:12.870701Z","iopub.status.idle":"2022-07-28T07:12:13.465033Z","shell.execute_reply.started":"2022-07-28T07:12:12.870677Z","shell.execute_reply":"2022-07-28T07:12:13.463771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analysis down the line","metadata":{}},{"cell_type":"markdown","source":"As it can be seen as their is no major difference in customer occurences till 12 occurances.\nBut if a customer had appeared 13 times so their is a higher probabilty (~0.66) of customer not going default.\nThis also makes sense right?\nIf a customer regular for a cycle, it can be considered more loyal toward the payments.\n","metadata":{}},{"cell_type":"markdown","source":"*Does using this feature will add some changes to the score line?*\n*  Leave the discussion for experts..**","metadata":{"execution":{"iopub.status.busy":"2022-07-28T07:18:54.044994Z","iopub.execute_input":"2022-07-28T07:18:54.045354Z","iopub.status.idle":"2022-07-28T07:18:54.051949Z","shell.execute_reply.started":"2022-07-28T07:18:54.045330Z","shell.execute_reply":"2022-07-28T07:18:54.050391Z"}}}]}