{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib\nimport seaborn as sns\nimport plotly.express as px\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T03:58:02.374230Z","iopub.execute_input":"2022-07-15T03:58:02.374945Z","iopub.status.idle":"2022-07-15T03:58:05.050162Z","shell.execute_reply.started":"2022-07-15T03:58:02.374839Z","shell.execute_reply":"2022-07-15T03:58:05.048743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/home-credit-default-risk/application_train.csv\")\ntest = pd.read_csv(\"../input/home-credit-default-risk/application_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:05.052687Z","iopub.execute_input":"2022-07-15T03:58:05.053104Z","iopub.status.idle":"2022-07-15T03:58:12.717616Z","shell.execute_reply.started":"2022-07-15T03:58:05.053070Z","shell.execute_reply":"2022-07-15T03:58:12.716399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Exploratory Data Analysis**","metadata":{}},{"cell_type":"markdown","source":"**First 5 rows**","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:12.719653Z","iopub.execute_input":"2022-07-15T03:58:12.720109Z","iopub.status.idle":"2022-07-15T03:58:12.765351Z","shell.execute_reply.started":"2022-07-15T03:58:12.720065Z","shell.execute_reply":"2022-07-15T03:58:12.763913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Shape of the data**","metadata":{}},{"cell_type":"code","source":"print(\"The application_train.csv has {} entires.\".format(train.shape))\nprint(\"The application_test.csv has {} entires.\".format(test.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:12.768384Z","iopub.execute_input":"2022-07-15T03:58:12.768742Z","iopub.status.idle":"2022-07-15T03:58:12.776992Z","shell.execute_reply.started":"2022-07-15T03:58:12.768712Z","shell.execute_reply":"2022-07-15T03:58:12.775446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Available columns and total number of columns**","metadata":{}},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:12.778937Z","iopub.execute_input":"2022-07-15T03:58:12.779264Z","iopub.status.idle":"2022-07-15T03:58:12.790135Z","shell.execute_reply.started":"2022-07-15T03:58:12.779234Z","shell.execute_reply":"2022-07-15T03:58:12.788831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Checking the datatypes**","metadata":{}},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:12.792149Z","iopub.execute_input":"2022-07-15T03:58:12.792811Z","iopub.status.idle":"2022-07-15T03:58:12.805588Z","shell.execute_reply.started":"2022-07-15T03:58:12.792774Z","shell.execute_reply":"2022-07-15T03:58:12.804487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.select_dtypes(include=['object']).columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:12.807576Z","iopub.execute_input":"2022-07-15T03:58:12.808007Z","iopub.status.idle":"2022-07-15T03:58:12.871633Z","shell.execute_reply.started":"2022-07-15T03:58:12.807960Z","shell.execute_reply":"2022-07-15T03:58:12.870736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check out the stats**","metadata":{}},{"cell_type":"code","source":"train.describe(include=\"all\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:12.872655Z","iopub.execute_input":"2022-07-15T03:58:12.872976Z","iopub.status.idle":"2022-07-15T03:58:15.941052Z","shell.execute_reply.started":"2022-07-15T03:58:12.872946Z","shell.execute_reply":"2022-07-15T03:58:15.939900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Overview of the data**","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:15.942591Z","iopub.execute_input":"2022-07-15T03:58:15.942928Z","iopub.status.idle":"2022-07-15T03:58:15.962398Z","shell.execute_reply.started":"2022-07-15T03:58:15.942898Z","shell.execute_reply":"2022-07-15T03:58:15.961364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Who is the highest borrower? Male or Female?**","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10,7))\nsns.countplot(x='CODE_GENDER',data=train)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:15.966290Z","iopub.execute_input":"2022-07-15T03:58:15.966903Z","iopub.status.idle":"2022-07-15T03:58:16.566959Z","shell.execute_reply.started":"2022-07-15T03:58:15.966858Z","shell.execute_reply":"2022-07-15T03:58:16.565546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Females are the highest borrowers with counts:\\n{}\".format(train.CODE_GENDER.value_counts()))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:16.568848Z","iopub.execute_input":"2022-07-15T03:58:16.569243Z","iopub.status.idle":"2022-07-15T03:58:16.614173Z","shell.execute_reply.started":"2022-07-15T03:58:16.569210Z","shell.execute_reply":"2022-07-15T03:58:16.612504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **How is the distribution of target labels? - Did most people return on time ?**\n\n0: Loan was repaid       1: Loan was not repaid ","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10,7))\nsns.countplot(x ='TARGET',data=train, hue='TARGET',palette=\"Set1\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:16.616723Z","iopub.execute_input":"2022-07-15T03:58:16.617232Z","iopub.status.idle":"2022-07-15T03:58:16.851187Z","shell.execute_reply.started":"2022-07-15T03:58:16.617184Z","shell.execute_reply":"2022-07-15T03:58:16.850086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['TARGET'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:16.852551Z","iopub.execute_input":"2022-07-15T03:58:16.853167Z","iopub.status.idle":"2022-07-15T03:58:16.863025Z","shell.execute_reply.started":"2022-07-15T03:58:16.853135Z","shell.execute_reply":"2022-07-15T03:58:16.862021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Based on the description, most people returned the money. Very clearly the target label is imbalanced.","metadata":{}},{"cell_type":"markdown","source":"### **Who are the major borrowers? - What are their occupations?**","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,7))\nsns.countplot(x='OCCUPATION_TYPE',data=train)\nplt.xlabel(\"Occupation Type\")\nplt.xticks(rotation=70)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:16.864737Z","iopub.execute_input":"2022-07-15T03:58:16.865936Z","iopub.status.idle":"2022-07-15T03:58:17.507215Z","shell.execute_reply.started":"2022-07-15T03:58:16.865896Z","shell.execute_reply":"2022-07-15T03:58:17.506295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the clients are laborers and the least of the clients are IT Staff.","metadata":{}},{"cell_type":"markdown","source":"\n### **How economically stable are clients? Who are the most and least stable?**","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,7))\nsns.barplot(x='OCCUPATION_TYPE',y='AMT_INCOME_TOTAL',data=train)\nplt.xticks(rotation=70)\nplt.xlabel(\"Occupation Type\")\nplt.ylabel(\"Average Annual family income\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:17.508717Z","iopub.execute_input":"2022-07-15T03:58:17.509005Z","iopub.status.idle":"2022-07-15T03:58:20.778699Z","shell.execute_reply.started":"2022-07-15T03:58:17.508978Z","shell.execute_reply":"2022-07-15T03:58:20.777173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Managers are the most earning borrowers while cleaning staff are the least earning borrowers - Based on the annual family income.","metadata":{}},{"cell_type":"markdown","source":"### **Which category of occupants repay on time and are better clients for company to lend money?**","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,7))\nsns.countplot(x='OCCUPATION_TYPE',hue='TARGET',data=train,palette=\"Set2\")\nplt.xticks(rotation=70)\nplt.xlabel(\"Occupation Type\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:20.780210Z","iopub.execute_input":"2022-07-15T03:58:20.781259Z","iopub.status.idle":"2022-07-15T03:58:21.721190Z","shell.execute_reply.started":"2022-07-15T03:58:20.781186Z","shell.execute_reply":"2022-07-15T03:58:21.719883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nRight off the bat, it seems as if the labourers have the highest difficulty in repaying. Also it seems lending to Reality agents, IT staff, HR staff is the safest.\n\n**This is not a better way to conclude, because this contains baised number of applicants.**\n\n**A better way is to find a metric that incorporates relative relationship between applicants count and repayers count.**\n\n\n### Let us look at the number of repayer's to number of applicants ratio in every occupation category.","metadata":{}},{"cell_type":"code","source":"# get the number of people having occupation type and target grouped.\nOccupation_df = pd.DataFrame(data=train.groupby(['OCCUPATION_TYPE','TARGET']).count()['SK_ID_CURR'])\nOccupation_df","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:21.723014Z","iopub.execute_input":"2022-07-15T03:58:21.723417Z","iopub.status.idle":"2022-07-15T03:58:22.625062Z","shell.execute_reply.started":"2022-07-15T03:58:21.723352Z","shell.execute_reply":"2022-07-15T03:58:22.623680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reset the multiindex organization of dataframe.\nOccupation_df = Occupation_df.reset_index() \nOccupation_df","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:22.626677Z","iopub.execute_input":"2022-07-15T03:58:22.627017Z","iopub.status.idle":"2022-07-15T03:58:22.646295Z","shell.execute_reply.started":"2022-07-15T03:58:22.626987Z","shell.execute_reply":"2022-07-15T03:58:22.645443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the number of people grouped on type of occupation and target in an array form.\nvalue_counts = Occupation_df['SK_ID_CURR'].values\nvalue_counts","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:22.647704Z","iopub.execute_input":"2022-07-15T03:58:22.648040Z","iopub.status.idle":"2022-07-15T03:58:22.657169Z","shell.execute_reply.started":"2022-07-15T03:58:22.648008Z","shell.execute_reply":"2022-07-15T03:58:22.655871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def repayers_to_applicants_ratio(values):\n    \"\"\"\n    Finds the ratio of Repayers to Applicants. This kind of is a \n    measure for safety. Larger the value better the applicant - More \n    safe for the company to lend loan to this category of workers.\n    \n    values: array of entires whose counts are given\n    returns the repayers to applicants ratio. \n    \n    precondition: The counts are such that the targets alligned are\n    in order 0 and 1\n    \"\"\"\n    flag = 1\n    ratios = []\n    for count in range(len(values)):\n        if flag == 1:\n            current_number = values[count]\n            next_number = values[count+1]\n            ratios.append(current_number/(current_number+next_number))\n            ratios.append(current_number/(current_number+next_number))\n        flag=flag*-1\n    return ratios       ","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:22.659234Z","iopub.execute_input":"2022-07-15T03:58:22.659738Z","iopub.status.idle":"2022-07-15T03:58:22.669986Z","shell.execute_reply.started":"2022-07-15T03:58:22.659691Z","shell.execute_reply":"2022-07-15T03:58:22.668876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find the ratios from the array values\nOccupation_df['Ratio R/A'] = repayers_to_applicants_ratio(value_counts)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:22.671085Z","iopub.execute_input":"2022-07-15T03:58:22.671473Z","iopub.status.idle":"2022-07-15T03:58:22.689363Z","shell.execute_reply.started":"2022-07-15T03:58:22.671440Z","shell.execute_reply":"2022-07-15T03:58:22.688381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Repayment ratio based on Occupation Type.**","metadata":{}},{"cell_type":"code","source":"# get the ratio and values based on the order of saftety.\n\nOccupation_ratio_df = Occupation_df.groupby(['OCCUPATION_TYPE','Ratio R/A']).count().drop(['TARGET', 'SK_ID_CURR'],axis=1)\nOccupation_ratio_df = Occupation_ratio_df.reset_index() \nOccupation_ratio_df = Occupation_ratio_df.sort_values(['Ratio R/A'],ascending=False)\nOccupation_ratio_df","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:22.691265Z","iopub.execute_input":"2022-07-15T03:58:22.691693Z","iopub.status.idle":"2022-07-15T03:58:22.717508Z","shell.execute_reply.started":"2022-07-15T03:58:22.691650Z","shell.execute_reply":"2022-07-15T03:58:22.716562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Occupation type and occupation based repayment to applicants ratio.\nfig,ax = plt.subplots(figsize = (15,7))\nsns.barplot(x='OCCUPATION_TYPE',y='Ratio R/A',data=Occupation_ratio_df,palette=sns.color_palette(\"GnBu_d\"))\nplt.xticks(rotation=70)\nplt.xlabel(\"Occupation Type\")\nplt.ylabel(\"Mean R/A Ratio\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:22.718886Z","iopub.execute_input":"2022-07-15T03:58:22.719691Z","iopub.status.idle":"2022-07-15T03:58:23.013197Z","shell.execute_reply.started":"2022-07-15T03:58:22.719654Z","shell.execute_reply":"2022-07-15T03:58:23.011472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"According to the ratio of Number of repayers to Number of applicants in every occupation type, we see that it is most safe to lend money to Accountants with an R/A ratio of 0.9516 and it is least safe to lend money to low skilled labourers with an R/A ratio of 0.8284","metadata":{}},{"cell_type":"markdown","source":"### **How is the distribution of males and females in terms of loan safety given that they belong to a specific occupation?**\n**find the probabilities of repaying given a specific gender and a specific occupation type.**","metadata":{}},{"cell_type":"code","source":"# merge the new column 'Ratio R/A' to the train dataframe.\ntrain = pd.merge(left=train,right=Occupation_ratio_df,on='OCCUPATION_TYPE')\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:23.015005Z","iopub.execute_input":"2022-07-15T03:58:23.015358Z","iopub.status.idle":"2022-07-15T03:58:24.970808Z","shell.execute_reply.started":"2022-07-15T03:58:23.015327Z","shell.execute_reply":"2022-07-15T03:58:24.969478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax = plt.subplots(figsize = (15,7))\nsns.countplot(x='CODE_GENDER',data=train,hue='TARGET',palette=sns.color_palette(\"GnBu_d\"))\nplt.xticks(rotation=70)\nplt.xlabel(\"Gender\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:24.972818Z","iopub.execute_input":"2022-07-15T03:58:24.973994Z","iopub.status.idle":"2022-07-15T03:58:25.441968Z","shell.execute_reply.started":"2022-07-15T03:58:24.973943Z","shell.execute_reply":"2022-07-15T03:58:25.440806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find out what is the probability that an applicant will return given that he/she is a male/Female respectively.\npd.DataFrame(train.groupby(['CODE_GENDER','TARGET']).count()['SK_ID_CURR']).reset_index() ","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:25.443774Z","iopub.execute_input":"2022-07-15T03:58:25.444415Z","iopub.status.idle":"2022-07-15T03:58:26.127786Z","shell.execute_reply.started":"2022-07-15T03:58:25.444348Z","shell.execute_reply":"2022-07-15T03:58:26.126279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### To find out the probability here's what we have to do:\nprint(\"probability that an applicant will repay the given that he is a male P(R|M): 73260/(73260+8576) = 0.8952\") \nprint(\"probability that an applicant will repay the given that she is a female P(R|F): 119311/(119311+9971) = 0.9228\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:26.129556Z","iopub.execute_input":"2022-07-15T03:58:26.130698Z","iopub.status.idle":"2022-07-15T03:58:26.137971Z","shell.execute_reply.started":"2022-07-15T03:58:26.130658Z","shell.execute_reply":"2022-07-15T03:58:26.136473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let us create a new dataframe where the probabilites of repaying based on gender is included. GR/A stands\n# for Gender based repayment ratio.\ngender_repay_ratio = pd.DataFrame({\"CODE_GENDER\":['M','F'],\"GR/A\":[0.8952,0.9228]})\ngender_repay_ratio ","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:26.146023Z","iopub.execute_input":"2022-07-15T03:58:26.146470Z","iopub.status.idle":"2022-07-15T03:58:26.161297Z","shell.execute_reply.started":"2022-07-15T03:58:26.146430Z","shell.execute_reply":"2022-07-15T03:58:26.160399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge this dataframe with the old train dataframe\ntrain = pd.merge(left=train,right=gender_repay_ratio,on='CODE_GENDER')\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:26.163247Z","iopub.execute_input":"2022-07-15T03:58:26.164033Z","iopub.status.idle":"2022-07-15T03:58:27.834735Z","shell.execute_reply.started":"2022-07-15T03:58:26.163972Z","shell.execute_reply":"2022-07-15T03:58:27.833257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets create a new column that's indicative of repayment with gender and occupation type which is just the product of Ratio R/A with G R/A.\n# EGR/A stands for employment gender repayment ratio.\ntrain['EGR/A'] = train['Ratio R/A']*train['GR/A']","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:27.836783Z","iopub.execute_input":"2022-07-15T03:58:27.837143Z","iopub.status.idle":"2022-07-15T03:58:27.845322Z","shell.execute_reply.started":"2022-07-15T03:58:27.837111Z","shell.execute_reply":"2022-07-15T03:58:27.844476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax = plt.subplots(figsize = (19,10))\nplt.xticks(rotation=70)\nsns.barplot(x='OCCUPATION_TYPE',y='EGR/A',hue='CODE_GENDER',data=train)\nplt.legend(loc=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:27.846778Z","iopub.execute_input":"2022-07-15T03:58:27.847645Z","iopub.status.idle":"2022-07-15T03:58:31.818432Z","shell.execute_reply.started":"2022-07-15T03:58:27.847611Z","shell.execute_reply":"2022-07-15T03:58:31.816480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nSo, in every occupation type, females are more likely to repay the loan on time.","metadata":{}},{"cell_type":"markdown","source":"### **Which occupation category are the highest loan recipients?**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,10))\nsns.boxplot(x='OCCUPATION_TYPE',y='AMT_CREDIT',data=train,hue='CODE_GENDER')\nplt.xticks(rotation=70)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:58:31.820668Z","iopub.execute_input":"2022-07-15T03:58:31.821221Z","iopub.status.idle":"2022-07-15T03:58:33.093963Z","shell.execute_reply.started":"2022-07-15T03:58:31.821172Z","shell.execute_reply":"2022-07-15T03:58:33.092469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Accountants and Managers are the highest amount recipents, while low skilled laborers are the least recipents (let me make it clear- labourers are highest volume based applicants, but not large recipents ). \n- It makes sense because accountants are more likely to get a large credit approved as opposed to low skilled laborers - which was kinda explained through Ratio R/A.","metadata":{}}]}