{"cells":[{"metadata":{"trusted":true,"_uuid":"ca638966a69188432919ac1587276d90116b3bf5"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c8bb377ee2e681ba8d7017598e65ca60df7084a"},"cell_type":"code","source":"import scipy.stats as stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebedfbdba98ab085bade96d2985e4f682212ba2a"},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d8a55b7255981cc20c76adf4a781b63e6087002"},"cell_type":"code","source":"df_test = pd.read_csv('../input/test.csv')\ndf_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06a1fc3722e0ed26a5f20f666f959f35952bbe60"},"cell_type":"code","source":"df = pd.concat((df_train, df_test), ignore_index= True, sort = False)\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc6366efd0a43a5d692fe5beed0c54632a8072d8"},"cell_type":"markdown","source":"Check the number of household heads exist in the train, test datasets."},{"metadata":{"trusted":true,"_uuid":"ccbbfdf4afc5c141249971fd56f7e9d2512ce6df"},"cell_type":"code","source":"unique_hh_heads = (df_train['parentesco1'] == 1).sum()\nunique_hh = len(df_train['idhogar'].unique())\n\nprint ('There are {} unique households and the dataset contain {} records of household heads'.format(unique_hh, unique_hh_heads))\nprint ('There is {} households without household head'.format(unique_hh - unique_hh_heads))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a471385f3d5e4f1b2540d99d6bbcc2dea789649"},"cell_type":"code","source":"unique_hh_heads = (df_test['parentesco1'] == 1).sum()\nunique_hh = len(df_test['idhogar'].unique())\n\nprint ('There are {} unique households and the dataset contain {} records of household heads'.format(unique_hh, unique_hh_heads))\nprint ('There is {} households without household head'.format(unique_hh - unique_hh_heads))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d85bb59aab009ea462386f37ad911f52987f0ad5"},"cell_type":"code","source":"df.info(verbose=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb5efa5ffa02afeda0b20ead3208d2b4c59665b4"},"cell_type":"markdown","source":"We can see that there is nine columns with dtype float, five with dtype object and 129 with dtype int."},{"metadata":{"trusted":true,"_uuid":"775581784cb5e781f6f76235d4e13b124fbf38fb"},"cell_type":"code","source":"df.columns[df.dtypes == object]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73a69eb470315fe07724486e4c7f9c462af5101a"},"cell_type":"markdown","source":"`dependency`, `edjefe` and `edjefa` should be `numeric`, rather than `object`"},{"metadata":{"trusted":true,"_uuid":"43cbb86e6d497e5b38744a2c45078f45352c4ba7"},"cell_type":"code","source":"df.columns[df.dtypes == float]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"813777dce9ad012edfc1a1a1ebb3493b2c498d17"},"cell_type":"code","source":"# check columns for null values\ndf.isnull().sum()[df.isnull().sum() > 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9190469d7ad124e844d92f4abd249cdb720ab8e"},"cell_type":"code","source":"df['Target'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70f970a5109ccd1a612aa60ef1d90340f7b71715"},"cell_type":"code","source":"sns.countplot(x = 'Target', data = df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93be04bc348da7593943233383c559331a782fcf"},"cell_type":"code","source":"hh_heads = set(df['idhogar'][df['parentesco1'] == 1])\nhouseholds = set(df['idhogar'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3648e138f4bed696039dcc54217dfea3df67b67b"},"cell_type":"code","source":"'''\nmissing_hh =  households.difference(hh_heads)\nrows_to_delete = df[df['idhogar'].isin(missing_hh)].index\ndf.drop(index= rows_to_delete, inplace = True) '''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4769980bdb4bb38cdc2eccabaad5b3941994a807"},"cell_type":"markdown","source":"### EDA"},{"metadata":{"_uuid":"435374b7cc7bd4f6ad706360b0aa495056ea67e3"},"cell_type":"markdown","source":"#### Monthly Rent"},{"metadata":{"trusted":true,"_uuid":"1832dc84b2d7b1168c1b1732f68e0fa09d147e09"},"cell_type":"code","source":"# Number of records with no rent amount\ndf['v2a1'].isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e927708d2f51432ef24ff180759500595c332ba"},"cell_type":"markdown","source":"Whether or not rent is applicable for an household depends on the ownership type of the house. Let's obtain how the household ownership is distributed in the combined dataset"},{"metadata":{"trusted":true,"_uuid":"0e638a7e56fac039e4d95549b61e217f8669c932"},"cell_type":"code","source":"# \ncol = [i for i in df.columns if i.startswith('tipovivi')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0d1c26bb21e8534965b5b24cc81b849f63b4aae"},"cell_type":"code","source":"# Create temporary column to identify the home ownership type.\ndf['temp_tipovivi'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b88cb8f5b8010306aea91054972d7df5e3fe1c3c"},"cell_type":"code","source":"## Identify the home ownership status of the hh with zero rent\ndf['temp_tipovivi'][df['v2a1'].isnull()].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd54523c9190f843ac49bb79cfcb60717823b77d"},"cell_type":"markdown","source":"The NA's only occure when the house ownership is own house('tipovivi1'), xxxxx ('tipovivi5') or xxx ('tipovivi4'). We can guess that the NA's are due to no rent being applicable for such households. So we can fill out the NA values with 0"},{"metadata":{"trusted":true,"_uuid":"00744be001f6e404cffa537df01989a3b83790a2"},"cell_type":"code","source":"df['v2a1'].fillna(value = 0, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72a317dd660a5294c9d37d3fbb287a0c9163d7ae"},"cell_type":"markdown","source":"We should also verify that the zero rent only occure in households when the ownership type is in 'tipovivi1', 'tipovivi5', 'tipovivi4'."},{"metadata":{"trusted":true,"_uuid":"7a1923d37625e7fcbc41dc6d1f066ae0bae4cccf"},"cell_type":"code","source":"df['temp_tipovivi'][df['v2a1'] == 0].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d35319b73b114aaa3880e441d5259680564b1045"},"cell_type":"markdown","source":"As the above output shows there are 97 households that have zero rent, but the house ownership is recorded as 'tipovivi2' and 'tipovivi3'. However, zero rent implies that the the household does not pay rent so we should change the the home ownership type to be consistant."},{"metadata":{"trusted":true,"_uuid":"16f4640d78012338577f99923cb174948e964a0b"},"cell_type":"code","source":"# Change the homeownership type to  be consistant with the rent amount.\ntipovivi2 = (df['v2a1'] == 0)&(df['tipovivi2'] == 1)\ntipovivi3 = (df['v2a1'] == 0)&(df['tipovivi3'] == 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86758186c824d4085ff597bcf2e80df0e0d5d3c6"},"cell_type":"code","source":"df.loc[tipovivi2,'tipovivi1'] = 1\ndf.loc[tipovivi3,'tipovivi1'] = 1\n\ndf.loc[tipovivi2,'tipovivi2'] = 0\ndf.loc[tipovivi3,'tipovivi3'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fef8bec69b3bc1bdadfa9ed98f6a7a5f38b0a09"},"cell_type":"code","source":"## Update temp_tipovivi to reflect change\ndf['temp_tipovivi'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9118e7b298cfd7f444f4237d1aa5113de55a2b3"},"cell_type":"code","source":"df[col][df['v2a1'] == 0].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8504d6972a0b1d91b03fdfd8ad9059b01ec1532"},"cell_type":"code","source":"sns.distplot(df['v2a1'],fit = stats.norm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e38a8638141567ff1327041094dc29bcebf4df19"},"cell_type":"code","source":"## Seperate out the records where the households does not pay rent\ndf['RentPaying'] = (df['v2a1'] > 0)*1\n## Log transfrom to make distribution normal\ndf['v2a1'] = np.log1p(df['v2a1'])\nsns.distplot(df['v2a1'][df['RentPaying'] == 1],fit = stats.norm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48472406cb0b65e1952695d013374e0d66b66009"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02908347b8fb91599adb13d15c0e5d82d724b960"},"cell_type":"code","source":"df.pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_tipovivi', aggfunc= 'count', margins= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"053896374955c98dc79f28de93a651c293b0cb26"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_tipovivi', aggfunc= 'count')\ncat = df['temp_tipovivi'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0c8192692a24017326edf4b89e920d3a07b9b6c"},"cell_type":"markdown","source":"This suggest that where the household owns and fully paid house, they are more likely (86%) to be in 'non-vulnerable household' and only 2% of families having 'own and fully paid' houses tend to live in extreme poverty."},{"metadata":{"_uuid":"16f2b28006dcc9ac12ea437c3ac9674b099c17b3"},"cell_type":"markdown","source":"#### Tablet Ownership"},{"metadata":{"trusted":true,"_uuid":"2c7e7460b10821582e48adc7f34daec65dad1823"},"cell_type":"code","source":"df['v18q'][df['v18q1'].isnull()].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6af20da3b476341ffc627947b4ce54f3af1f20f2"},"cell_type":"markdown","source":"We can see that `NA` occure in the `v18q1` only when the household does not own any tablets. We can easily fill out these `NA`s with zeros."},{"metadata":{"trusted":true,"_uuid":"136743b4b7a1268529a6303f6b46915925a4bfab"},"cell_type":"code","source":"df['v18q1'].fillna(value = 0, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed88a36121ebf775b7834d7fd46aa6af16d1fa22"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['v18q'].mean()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"404732d8e8d8bd3881f8bb4b00b2e3c2129eb418"},"cell_type":"markdown","source":"We can see that the likelihood of a household owning a tablet increases as their income level increases."},{"metadata":{"trusted":true,"_uuid":"aacec8b0e974843976aecb8de7926f9b8185341e"},"cell_type":"code","source":"temp = df[(df['parentesco1'] == 1)&(df['Target'].notnull())].pivot_table(index = 'Target', columns = 'v18q1', values = 'idhogar', aggfunc='count')\ncat = df['v18q1'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"33dc7e8c20d8151073287f2b550ce96c8b61ebe6"},"cell_type":"markdown","source":"#### Schooling "},{"metadata":{"trusted":true,"_uuid":"5d83fa9575aaf714e93be42609f58ae46dc3e88d"},"cell_type":"code","source":"#### Years behind in school\ndf['rez_esc'].value_counts(dropna = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"405c1964c0bb3af4ac9d2aa934c82d9b86f3540f"},"cell_type":"markdown","source":"For a vast majority of instances the 'rez_esc' is null. We can guess that the years behind in school is related to the age of the individual, so let's check when this value is applicable."},{"metadata":{"trusted":true,"_uuid":"ce916ea586560c9ace7e39440a0919c05af91622"},"cell_type":"code","source":"df['age'][df['rez_esc'].notna()].value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec9aa3b6e5e64d0ad6c06fbcb230f24d40505eb6"},"cell_type":"markdown","source":"So the years behind in school variable is only applicable for children whose age is between 7 and 18 (< 18). i.e the typical school going age range. We can simply fill in the NAs with 0."},{"metadata":{"trusted":true,"_uuid":"e5b1e5b2ba76fcb785baeb36f9846ff31c9f7c2e"},"cell_type":"code","source":"df['rez_esc'].fillna(value = 0, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a11f993d0e678270a4284181d9f6d43151de8064"},"cell_type":"code","source":"## Fixing the large age behind in school value\ndf.loc[df['rez_esc'] > 50, 'rez_esc'] = 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"261d006f3fc4d1f0ed05dc424fc1dbca2592b845"},"cell_type":"markdown","source":"#### Mean Education"},{"metadata":{"trusted":true,"_uuid":"1b376a12b4d2c6904563e2a07a264a02424e4138"},"cell_type":"code","source":"## Obtain a list of households where the average years schooled is NA\nna_mean_households = df['idhogar'][df['meaneduc'].isna()].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d65b2063ab88b4051f630beabab0ef21fdb93a4"},"cell_type":"code","source":"## Checking if there are 18+ persons in households \ndf[df['meaneduc'].isna()].groupby('idhogar')['age'].max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dea90563904914357c694749e81bbb6eb0490ae5"},"cell_type":"markdown","source":"The `meaneduc` is calculated by taking the mean of individuals aged 18 or above. Out of the households that have missing `meaneduc` values all except two households have individuals aged 18 or above. For the households c31f9f3a0 and c49af2e64 we do not have any individuals over the age of 18. So we'll set the `meaneduc` value to zero for these households. For the rest we can recompute the applicable value."},{"metadata":{"trusted":true,"_uuid":"b73f36864e72b891962e6a72b5870715edf680d5"},"cell_type":"code","source":"## recompute meaneduc for households.\nmapper = df[df['age'] >= 18].groupby('idhogar')['escolari'].mean().to_dict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1fb38598a1c1a6fa574408589d15ec94f1facfd"},"cell_type":"code","source":"df['meaneduc'] = df.apply(lambda x: mapper.get('idhogar', 0) if np.isnan(x['meaneduc']) else x['meaneduc'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50bf2fe4f9c8554e7cafa099d513d9b70b7416b1"},"cell_type":"code","source":"df['meaneduc'].isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ea9f5cccb38cd5488562ecacd31648ee1f60676"},"cell_type":"code","source":"df['SQBmeaned'] = df['meaneduc']**2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29ac9896c16f2b8590071d9184d16257ecfe79b6"},"cell_type":"markdown","source":"#### Overcrowding"},{"metadata":{"trusted":true,"_uuid":"47dfd2f63255f542e26573594baf39fcc59908d3"},"cell_type":"code","source":"sns.countplot(x = 'Target', data = df[df['hacdor'] == 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f740d7106434806d0c16dcd66e7f6aef64762f7e"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['hacdor'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72ad76662b5f4055bea2c9098d345ba8242626bd"},"cell_type":"code","source":"## Since overcrowing occures only ~3% of the time this is a possible candidates for deletion\n## df.drop(column = ['hacdor', 'hacapo'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31714df1a2088908d0ea9ace1637ecc1a7d8dde0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"564c7f02333d4270662d34e794848fff0f040e16"},"cell_type":"markdown","source":"#### Rooms"},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"ad975b96f9d50a0c2e27cdfdb9e27285573a4ff3"},"cell_type":"code","source":"### IGNORE !!!\n### For the time being we'll calculate the likelihood based on the household head only.\ntemp = df[df['Target'].notnull()].pivot_table(index = 'Target', columns = 'v18q1', values = 'idhogar', aggfunc='count')\ncat = df['v18q1'][df['Target'].notnull()].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05b87964c3ff0e2e7350c9115bfe38efd96c4728"},"cell_type":"markdown","source":"#### Household Members"},{"metadata":{"_uuid":"dc9b76ed1a9c5da40614b4a757a4fab5d6051570"},"cell_type":"markdown","source":"The data set several features to give the number of individuals in the household:\n* tamhog (size of the household)\n* tamviv (number of persons living in the household)\n* r4t3 (Total persons in the household)\n* hhsize (household size)\n* hogar_total (# of total individuals in the household)"},{"metadata":{"trusted":true,"_uuid":"e08bf04158dffb7306fa61a87c3de77992f9b371"},"cell_type":"code","source":"df[['tamhog', 'tamviv', 'r4t3', 'hhsize', 'hogar_total']][df['r4t3'] != df['hhsize']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a43cd49a35bdbf24f16aa5cb0b8fdd974facec2f"},"cell_type":"code","source":"df.drop(columns= ['tamhog', 'hogar_total', 'r4t3'], inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7a1ef8ed0f26065ece0ac55a38222f20e73343b"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"337c6909ade9a1f9a98a175608099f34e1323207"},"cell_type":"markdown","source":"#### Bathroom"},{"metadata":{"trusted":true,"_uuid":"62e3f5b9f5fba34fe2a876fd00f04bd407fbaf8b"},"cell_type":"code","source":"(df['v14a'][df['parentesco1']== 1]).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc9bbeb1584e70108fac0a783be211f5fd072260"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['v14a'].mean()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"623bfb0cdf4c9c30e1b00ab066f958a4d6b4b67a"},"cell_type":"markdown","source":"If we look at each income level the likelyhood of the household having an bathroom changes only slighly between the income levels. "},{"metadata":{"_uuid":"cf077cd19a80f685002dcbcfa471d04149c3f58d"},"cell_type":"markdown","source":"#### Refrigerator "},{"metadata":{"trusted":true,"_uuid":"9730fb00ad97037518e4b2bfa5fb40ec0cbe89a2"},"cell_type":"code","source":"(df['refrig'][df['parentesco1']== 1]).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd0ca89d50d4d62ddc4484463be552c214acc066"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['refrig'].mean()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5739552c1a82782b9f94e996713ffe3df14b111"},"cell_type":"markdown","source":"#### Tablet Ownership"},{"metadata":{"_uuid":"c2df31d7b7273c06c53fc010db4e8c8bf10afcfd"},"cell_type":"markdown","source":"#### outside wall"},{"metadata":{"trusted":true,"_uuid":"a926712d9d8a109145ca3ad4db089e81ad254d34"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('pared')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2189eef337774d168ddb8029bfb12da79b4d36fb"},"cell_type":"code","source":"df['temp_pared'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e0b8a8490213f4a371b9d1095008646af71cdd3"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_pared', aggfunc= 'count')\ncat = df['temp_pared'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20316dac9f883a9a69db8c8b9b6578bbcad8ef97"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"257d453cdf72701281ec9b0a1dc604653a3e872c"},"cell_type":"markdown","source":"#### Floor"},{"metadata":{"trusted":true,"_uuid":"e31ada06485ab7ae237b377a49d84d33f9fac73c"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('piso')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6db3b60a0de3372812863ceabae270aa45bef47"},"cell_type":"code","source":"df['temp_piso'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a67c7d96a867caa9742f34c07b59bea7cef2961"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_piso', aggfunc= 'count')\ncat = df['temp_piso'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c580cf3d742cd0056dbc26f0865dc44545e8fe63"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50ceaf5cd73c20ac7f2ad38c0845d65e9ac8fc16"},"cell_type":"markdown","source":"#### Roof"},{"metadata":{"trusted":true,"_uuid":"9d05c2f54dcd6c80f53d69d3b44f10557195f4e9"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('techo')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd12c2beeacd081dbcff6d19936b715032a33ca8"},"cell_type":"code","source":"df['temp_techo'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5681ae4ad56423ac59907640d52d8fc9d7dad42e"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_techo', aggfunc= 'count')\ncat = df['temp_techo'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec71c64636b7d053fd254fb32818745846bea761"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c752a8d5f026a8795628d77d685fb22031e70fea"},"cell_type":"markdown","source":"#### ceiling"},{"metadata":{"trusted":true,"_uuid":"f0c1deadf10a6000d6dbb32272f9b7c1d210905f"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['cielorazo'].mean()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66993f70eda1f2764279089b092d6d21d3f808b5"},"cell_type":"markdown","source":"#### Plumbing"},{"metadata":{"trusted":true,"_uuid":"2ad0b2d16e3dc0b5bcb4629855c6da74e9885804"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('abastagua')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fad1914c5db4248f45dd1d4b3228645a45b9e1df"},"cell_type":"code","source":"df['temp_abastagua'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f00ac36730eaceea6acac604b3d402d14f55d5f9"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_abastagua', aggfunc= 'count')\ncat = df['temp_abastagua'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"909976fc0260d1deb119a1c034565837bcffa422"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abc40d5a6486df1d8f89ee7756561faff0683671"},"cell_type":"markdown","source":"#### Electricity"},{"metadata":{"trusted":true,"_uuid":"7f9fabe4455a095dc10a0f77135b1ef2fc470920"},"cell_type":"code","source":"df['temp_electricity'] = df[['public', 'planpri', 'noelec', 'coopele']].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30798e8dc1f54265acd2eec8e3457eedf98c4d82"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_electricity', aggfunc= 'count')\ncat = df['temp_electricity'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66c2c9cd2c89e59c8b21655c3ff3ab0a1e1204ee"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08b776596631cd98e4e1c54d48b61f9e3b6a6386"},"cell_type":"markdown","source":"#### Toilet"},{"metadata":{"trusted":true,"_uuid":"bb20150a344ae0de8065d0b610588c2709e7f3e6"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('sanit')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9df80fffec8e4ae0181a6599103de8da4e426bf0"},"cell_type":"code","source":"df['temp_sanitario'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d85cc22e21afc88b0b71844f04f8da4b7fa6ad4"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_sanitario', aggfunc= 'count')\ncat = df['temp_sanitario'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcfe3840c914b3939f16c9877eb775710d90c9cc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0940b8c1869cfefea8840c4b6328e91c39de5184"},"cell_type":"markdown","source":"#### Cooking"},{"metadata":{"trusted":true,"_uuid":"cb01d3a9e61c1f0025209feb17cbac4d794d891a"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('energcocinar')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6042d9e29021ac675665b5e407b98c8eeac6e6f0"},"cell_type":"code","source":"df['temp_energcocinar'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5bffccfed198947b7224fedeb954ecdc6801fb2"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_energcocinar', aggfunc= 'count')\ncat = df['temp_energcocinar'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5df5f0a8aa9f4db8addc2aa7752451f0e353bcf5"},"cell_type":"code","source":"df['cooking_lowEng'] = ((df['energcocinar1'] == 1)|(df['energcocinar4'] == 1))*1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"42f45ed1ba1d0166cb7a25a42c9b2be42ef9816a"},"cell_type":"markdown","source":"#### Garbage Disposal"},{"metadata":{"trusted":true,"_uuid":"f9d20c6d75c376dfec05e283c922e82f4407ae3b"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('elimbasu')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fa150087088c0171a2a7996b57befd3bbaf252d"},"cell_type":"code","source":"df['temp_elimbasu'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"092c209c0393f1b08376ecbd4032a8b379469232"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_elimbasu', aggfunc= 'count')\ncat = df['temp_elimbasu'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5654b0d59786b0de6a44e4fa4365f421eeeb2b5b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3549d9d41e1663b3460a6bce34deff78a7daa25b"},"cell_type":"markdown","source":"#### Wall Condition"},{"metadata":{"trusted":true,"_uuid":"dfde1eaec8c9bb161c445a1f69803187e0035b90"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('epared')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06af3b44c127fa0f374e13e0775bdece20d7a9bd"},"cell_type":"code","source":"df['temp_epared'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec667bd1dc809212651ded0d6fbd2422c9b905ba"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_epared', aggfunc= 'count')\ncat = df['temp_epared'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"260a5b8051110bc64f62cef321ea24ea1b9d53af"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae13ef50e280bf63a778e23d5d962a0b4290b43a"},"cell_type":"markdown","source":"#### Roof Quality"},{"metadata":{"trusted":true,"_uuid":"8ddcc5d9e7a8768236ec89520bbbdf3c13e71b7a"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('etecho')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"409d552dad97771eb2db5bb84b8106e6ffef9b63"},"cell_type":"code","source":"df['temp_etecho'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb0a06ae2f4e512b559e00b8b35158f548715bd2"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_etecho', aggfunc= 'count')\ncat = df['temp_etecho'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a525ca171085c58a6c3b3de5cbf092fe91e7f70"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad3dd6d428144050a2434024d776309a18ded130"},"cell_type":"markdown","source":"#### Floor Quality"},{"metadata":{"trusted":true,"_uuid":"1941e232a2c3491a1575eb652acf64765e8fc028"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('eviv')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1156672e90086825d373463d704a0fb1589925e4"},"cell_type":"code","source":"df['temp_eviv'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ab88c05137241b9f0daef8c230112f90e6ac468"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_eviv', aggfunc= 'count')\ncat = df['temp_eviv'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94bf6ec236c0d00aa9754f37458c2dc9c593cf34"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a742fe6d8990b101dfbf4139ba12a2028f2e3e6"},"cell_type":"markdown","source":"#### "},{"metadata":{"trusted":true,"_uuid":"6179513f92b5d9d91b3185cf8bf74765a2caaacc"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['dis'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b396c9d46e11c6c87976b19be3f1541b4babe0ce"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['male'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f783f50abf376bb9570fbd44978983e587e93be"},"cell_type":"code","source":"df.drop(columns= 'female', inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c19e979f9873482ef92541b6f7d2bc97750fa82f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"093806056af04f1b6d0f681a89643757e2ff2974"},"cell_type":"markdown","source":"#### Civil Status"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"5f607f369b38b6e1ffd9fceddf4f92797b5e7953"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('estadocivil')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3e75bb2775ed3903b3f9407f9588379356ec3c3"},"cell_type":"code","source":"df['temp_estadocivil'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d86d55750bbe98ca17385efa6711acc3e8a97966"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_estadocivil', aggfunc= 'count')\ncat = df['temp_estadocivil'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5859c6a3070a649832487387ff6996cfb8daa089"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b19446969576b8cbfdd310e087074ed06d519902"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2797328769e5a18786284627644a560c4322f8ad"},"cell_type":"markdown","source":"#### Level of Education"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"7c33f24eef50012fac1acbcb041e4cbae6e5abee"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('instlevel')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f81278e3e95aec2e3ff8913ac75090ce4df8661"},"cell_type":"code","source":"df['temp_instlevel'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1054ad8ff7f6a67f66ba07fbbdacc92457a588c2"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_instlevel', aggfunc= 'count')\ncat = df['temp_instlevel'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/(cat.T), vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"048efc3baa5d5f2b3fc85f25fb8540a3e1eba213"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"df0d01873c781541a0e2e1059e6e2b4dbc4d6bf7"},"cell_type":"markdown","source":"#### Region"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"c7566377722cdf33667146a2a02d8455648d92c3"},"cell_type":"code","source":"col = [i for i in df.columns if i.startswith('lugar')]\ndf.loc[:, col].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09bb1b58a4bf0e5acc578b2c196cf9808af233cb"},"cell_type":"code","source":"df['temp_lugar'] = df[col].idxmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8855c9c8c4d82a679ef08719f474a3f1ff195443"},"cell_type":"code","source":"temp = df[df['parentesco1'] == 1 ].pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_lugar', aggfunc= 'count')\ncat = df['temp_lugar'][(df['Target'].notnull())&(df['parentesco1'] == 1)].value_counts()\n\n##np.divide(temp, cat.values)\nsns.heatmap(temp/cat.T, vmin= 0, vmax= 1, cmap = 'viridis', annot= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9284bfd96144dc42a7672c0724e93c84c18f5d40"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04ae8e13258b3147bc68b535bc1218d0ea7be1a0"},"cell_type":"code","source":"df[df['parentesco1'] == 1].groupby('Target')['area1'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0d2a097c29a51db649daed5f25bd2576f454ce9"},"cell_type":"code","source":"df.drop(columns= 'area2', inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ecd0ca00c58a3ee1e1f4bc5f0a3c7382bc368f6c"},"cell_type":"markdown","source":"#### Dependency"},{"metadata":{"trusted":true,"_uuid":"05b6c04e33963b4a8a513ef033f697370a20555c"},"cell_type":"code","source":"df['hogar_workingAge'] = df['hogar_adul'] - df['hogar_mayor']\ndf['hogar_dependent'] = df['hogar_nin'] + df['hogar_mayor']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2e11db8c5680f1088899c19f633f56bb183eff3"},"cell_type":"markdown","source":"dependency feature \nyes = 1, i.e hogar_workingAge == hogar_dependent\nno  = 0, hogar_dependent = 0\n8 = inf, hogar_workingAge = 0"},{"metadata":{"trusted":true,"_uuid":"4c61e86004eb2e4964db8da2d9265af3505a11bc"},"cell_type":"code","source":"## df[['hogar_nin', 'hogar_adul','hogar_mayor', 'hogar_workingAge', 'hogar_dependent','dependency']][df['dependency'] == 'no']\n\ndf[['hogar_nin', 'hogar_adul','hogar_mayor', 'hogar_workingAge', 'hogar_dependent','dependency']][df['dependency'] == '8']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31a5c63d9f987349bddb9b68c2c63e77acd6cea3"},"cell_type":"code","source":"df['dependency'] = df['dependency'].replace({'yes': 1, 'no': 0}).astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"809d43d5f62914a17438a8504613fa88f19ec5ce"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4ad8819dead0958e1fbce5a448696b8692d8c2a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d46a53510bf4f5779d44da028eaa4a83d9216f7"},"cell_type":"markdown","source":"#### Years of Education"},{"metadata":{"trusted":true,"_uuid":"30f89e704dfffbed92ff65cd0d6d57493ff8c19b"},"cell_type":"code","source":"df['edjefe'] = df['edjefe'].replace({'no': 0, 'yes': 1})\ndf['edjefa'] = df['edjefa'].replace({'no': 0, 'yes': 1})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60e26654edbc6e48d01177fe942926c37050ebee"},"cell_type":"code","source":"df['edjefe'] = df['edjefe'].astype(int)\ndf['edjefa'] = df['edjefa'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a98cfe7f3f96a7ee99b8f57d932f42268586d22"},"cell_type":"code","source":"df['median_schooling'] = df['escolari'].groupby(df['idhogar']).transform('median')\ndf['max_schooling'] = df['escolari'].groupby(df['idhogar']).transform('max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16964931874c651b8b07892a7a29d76f62815c7e"},"cell_type":"code","source":"df['eduForHeadofHH'] = 0\ndf.loc[(df['parentesco1']== 1), 'eduForHeadofHH'] = df['escolari']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb04f045ad3377cb9fa843c2828192707183db50"},"cell_type":"code","source":"df['eduForHeadofHH'] = df['eduForHeadofHH'].groupby(df['idhogar']).transform('max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d13552fe8761d3c0ca37f5d8424e5d9624075fb1"},"cell_type":"code","source":"df['SecondaryEduLess'] = ((df[['instlevel1','instlevel2', 'instlevel3', 'instlevel4']] == 1).any(axis = 1)&(df['age'] > 19))*1\ndf['SecondaryEduMore'] = ((df[['instlevel5','instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']] == 1).any(axis = 1)&(df['age'] > 19))*1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7451cd35fe14f78674de9f68a83e9f7edd04cab"},"cell_type":"code","source":"df['MembersWithSecEdu']  = df['SecondaryEduMore'].groupby(df['idhogar']).transform('sum')\ndf['MembersWithPrimEdu']  = df['SecondaryEduLess'].groupby(df['idhogar']).transform('sum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2aab54e99e5b19fb58231b6afed0d87b2bb3cfa3"},"cell_type":"code","source":"df['Educated_Gap'] = (df['MembersWithSecEdu'] - df['MembersWithPrimEdu'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f173587e428df8f7b2ae5c78a0c14ea5755816e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3088cde0710ec746ee142f75b2895bf6160b356"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5e22b1cc18f81272249ea941e6b980b4a7d23e5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1d0b3857eaef1bc57d56ea346236f08d5d7d471"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc74447d063e74bd0d068401f7ae2ff32fcb7483"},"cell_type":"code","source":"df['marital_status'] = (((df['estadocivil3'] ==1)|(df['estadocivil4'] == 1))&(df['parentesco1'] == 1))*1\n\ndf['marital_status'] = df['marital_status'].groupby(df['idhogar']).transform('max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2a8e4bc3d6ebb318d1f80999c2220d37d90f995"},"cell_type":"code","source":"df['FemaleHousehold'] = ((df['male'] == 0)&(df['parentesco1'] == 1))*1\ndf['FemaleHousehold'] = df['FemaleHousehold'].groupby(df['idhogar']).transform('max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"975b71f966c64cd932da724e4fa3dfb126508c01"},"cell_type":"code","source":"df['phones_percap'] = df['qmobilephone'] / df['tamviv']\ndf['tablets_percap'] = df['v18q1'] / df['tamviv']\ndf['rooms_percap'] = df['rooms'] / df['tamviv']\ndf['rent_percap'] = df['v2a1'] / df['tamviv']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08b82770338a67b9b3e46747d9d1bd914f47a4ff"},"cell_type":"code","source":"df['minors_ratio'] = df['hogar_nin']/df['tamviv']\ndf['elder_ratio'] = df['hogar_mayor']/df['tamviv']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f14583cdd81ff415475075f9da13da1c8e5c1c09"},"cell_type":"code","source":"df['child_ratio'] = df['r4t1']/ df['tamviv']\ndf['malefemale_ratio'] = df['r4h3'] -  df['r4m3'] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"767f14f6688392e6395009a90919d5f2a3e2b42d"},"cell_type":"code","source":"df['ismale_only'] =  (df['r4m3'] == 0)*1\ndf['isfemale_only'] = (df['r4h3'] == 0)*1\ndf['no_adultmale'] = (df['r4h2'] == 0)*1\ndf['no_adultfemale'] = (df['r4m2'] == 0)*1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b85b5168bafd87051aaf91efa08db3ccfe9d2ea"},"cell_type":"code","source":"df['rent_per_room'] = df['v2a1']/df['rooms']\ndf['bedroom_per_room'] = df['bedrooms']/df['rooms']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a298d9631c722da4a59a2670fc61c673ab8cb06"},"cell_type":"code","source":"df['rent_per_room'] = df['v2a1'] / df['rooms']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd05e039363c1e987cfe01dd28a401e23e9c22de"},"cell_type":"code","source":"df['total_disabled'] = df.groupby('idhogar')['dis'].transform(lambda x: x.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"154a709a97fdac213f51d8cb3ffb92328d267b26"},"cell_type":"code","source":"df['average_age'] = df.groupby('idhogar')['age'].transform(lambda x: x.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb9cd90b74c15a9ea35a0720a582814050c260af"},"cell_type":"code","source":"df['disable_ratio'] = df['total_disabled']/df['tamviv']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fe5e9a9aef563d184ad991f61fb055af2f19e04"},"cell_type":"code","source":"df['info_accessibility'] = df[['mobilephone', 'television', 'computer', 'v18q']].any(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e47e4df4ebe17cffc473dd335922a003dd919cf"},"cell_type":"code","source":"df.select_dtypes(include = 'number').columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12c031b3c2fd12cea77b5f7d8ca0db5a45525371"},"cell_type":"code","source":"df.to_csv('./processed.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c886c513a551b94a8372baf6697b83b03ea7353"},"cell_type":"markdown","source":"### Model Creation"},{"metadata":{"trusted":true,"_uuid":"ca09d02bd34b21a1367f57293036975acf9436e2"},"cell_type":"code","source":"df.drop(columns=['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"855ae635e796c60b5ce8658504607b7ee16b8f69"},"cell_type":"code","source":"training_df = df.select_dtypes(include = 'number')[df['Target'].notnull()]\ntest_df = df.select_dtypes(include = 'number')[df['Target'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4b0a53fa9a2ca6df834eb794919777274a520f6"},"cell_type":"code","source":"training_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49a9a7e7c9c64949f96dc91f736464e3dbe41cea"},"cell_type":"code","source":"features = [col for col in training_df.columns if col != 'Target']\nX, y = training_df[features], training_df['Target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2389d74cb2df3c1c96cf6367ce661840c9fb544"},"cell_type":"code","source":"test_df.drop(columns = 'Target', inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c229255e752c623c428794aa28d295a61250e85"},"cell_type":"code","source":"from sklearn.metrics import f1_score, make_scorer, confusion_matrix\nfrom sklearn.model_selection import cross_val_score, cross_val_predict\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2df3e73bac7905a9c58a0f7d24afbd9828097039"},"cell_type":"code","source":"tree = DecisionTreeClassifier(max_features= 75, class_weight='balanced')\ntree.fit(X,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"15a09424d2320fa976387716017224680555bcf5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"449afb287cdffd2e14f541ced9e54d6118e2ada9"},"cell_type":"code","source":"model = RandomForestClassifier(n_estimators=100, random_state=10, max_features= 75, n_jobs = -1 ,class_weight= 'balanced')\ncv_score = cross_val_score(model, X, y, cv = 10, scoring = 'f1_macro')\ncv_score.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddc9ebca674e7aac55ca2b86792b82eb94e26e5e"},"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingClassifier, VotingClassifier\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54ac421967be1c547a8a489ad5c78d3bbebcf008"},"cell_type":"code","source":"ets = []\nfor i in range(10):\n    rf = RandomForestClassifier(random_state=217+i, n_jobs=4, n_estimators=700, min_impurity_decrease=1e-3, min_samples_leaf=2, verbose=0, class_weight= 'balanced')\n    ets.append(('rf{}'.format(i), rf)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"544390f45edc064ba7622eef9819d2eb68594ac2"},"cell_type":"code","source":"vclf = VotingClassifier(ets, voting= 'soft')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86d1bb23fd408b4d74a903875bc97292f787b0bb"},"cell_type":"code","source":"### Score CV results\ncv_score = cross_val_score(vclf, X, y, cv= 5, scoring = 'f1_macro')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cd325afd71be983b47f79380b72d56fb4979383"},"cell_type":"code","source":"cv_score.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1ee14f620c261b495c29b9737b2cee0ebb1bb21"},"cell_type":"code","source":"cv_predict = cross_val_predict(vclf, X, y, cv = 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ee05e60e0df12168062119a2ac04afee2c49c3e"},"cell_type":"code","source":"confusion_matrix(y, cv_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"75945704beee04b3be4f9cc7030fb092256e9630"},"cell_type":"code","source":"f1_score()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"035125dde926f71b93a82e6d8562266a889c7d61"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"210ba026d55bce5711fbe25118bed76ec487fbc2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1be88fc27d835c37df27daf237a4795ef87dfa3"},"cell_type":"code","source":"vclf = VotingClassifier(ets, voting= 'hard')\ncv_score = cross_val_score(vclf, X, y, cv = 5, scoring = 'f1_macro')\ncv_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cabc92ef7835e2e1c0b55cf91a49d766b5f5f58a"},"cell_type":"code","source":"cv_score.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e053d199fe3fb1b4d4845241fe3a56eaeb3294de"},"cell_type":"code","source":"vclf.fit(X,y)\nvclf_hardvoting = vclf.predict(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81881e5b4fc0721c84ac10ff0136556e267502e1"},"cell_type":"code","source":"len(vclf_hardvoting)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e91aa303c975c6ae89f300c99c2b863accf598b"},"cell_type":"markdown","source":"### LGB "},{"metadata":{"trusted":true,"_uuid":"152aea89531c4451b6d0342d496bbab90f6a1226"},"cell_type":"code","source":"import lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b57046e9e21694db315ecd6bf99240783601c99"},"cell_type":"code","source":"##clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',\n                                 random_state=None, silent=True, metric='None', \n                                 n_jobs=4, n_estimators=500, class_weight='balanced',\n                                 colsample_bytree =  0.89, min_child_samples = 90, num_leaves = 56, subsample = 0.96)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35e1869a93a42f8d2e92a31042da0914755ddb75"},"cell_type":"code","source":"## cv_score = cross_val_score(clf, X, y, cv = 3, scoring = 'f1_macro')\n## cv_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bb6f5a9531d1c6713abafde8f456a552586103a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"451aab002b30450908e7ff5f002634f02967da4e"},"cell_type":"code","source":"### prediction = [model].predict(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"753b26ec9efb09fbf294190f977e9e1173fe23e0"},"cell_type":"code","source":"submit=pd.DataFrame({'Id': df['Id'][df['Target'].isna()] , 'Target': vclf_hardvoting.astype(int)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6968bc71155a607f110acb453f57f9925d3b7cd6"},"cell_type":"code","source":"submit['Target'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29cc20928fb0fd92f8a309854ab884498de33ee0"},"cell_type":"code","source":"submit.to_csv('./submission.csv', index= False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"552f6d381c9c8a8752cff9d238c257f56afbd89d"},"cell_type":"code","source":"##training_df_hhO = df.select_dtypes(include = 'number')[(df['Target'].notnull())&(df['parentesco1'] == 1)]\n##test_df_hhO = df.select_dtypes(include = 'number')[(df['Target'].isnull())&(df['parentesco1'] == 1)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"39e4def5f64162699b90714db6a4a1541b82e185"},"cell_type":"code","source":"##features = [col for col in training_df.columns if col != 'Target']\n##X_hhO, y_hhO = training_df_hhO[features], training_df_hhO['Target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4de4efbb1a19ad012f2211280d0e1c843ecd78a4"},"cell_type":"code","source":"##cv_score = cross_val_score(vclf, X_hhO, y_hhO, cv = 5, scoring = 'f1_macro')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"07fcfc29a24666dcdf483d0f4db61aa805cad93b"},"cell_type":"code","source":"##cv_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"96777c09d575796cc5c45d1d65cdb1ecb0f3666f"},"cell_type":"raw","source":"array([0.4596173 , 0.42617731, 0.38660971, 0.35653499, 0.37714824])  \narray([0.46729109, 0.42841538, 0.38395209, 0.35942352, 0.37480922])"},{"metadata":{"trusted":false,"_uuid":"af158c74f00fdf389312cce90b0684d9aded2204"},"cell_type":"code","source":"## np.array([0.4596173 , 0.42617731, 0.38660971, 0.35653499, 0.37714824]).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"55d78e9252001341c558b52f4bae326958e1e188"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"}},"nbformat":4,"nbformat_minor":1}