{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Quick overview\n\nIam using the parquet data of Sanskar","metadata":{}},{"cell_type":"code","source":"%%time\ntrain = pd.read_parquet('../input/amex-parquet/train_data.parquet')\ntrain_labels=pd.read_csv('../input/amex-default-prediction/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:30:55.750729Z","iopub.execute_input":"2022-05-26T00:30:55.751199Z","iopub.status.idle":"2022-05-26T00:31:04.438159Z","shell.execute_reply.started":"2022-05-26T00:30:55.751162Z","shell.execute_reply":"2022-05-26T00:31:04.437442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# missing table function\ndef missing_table(df):\n    missing_df=pd.DataFrame({'Share in %':round((df.isnull().sum()/len(df))*100,1).values},index=[list(df.columns)])\n    missing_df=missing_df.sort_values(by='Share in %',ascending=False)\n    return missing_df\n\n# show top 5 missing variables\nmissing_table(train).head(5)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:00:29.476787Z","iopub.execute_input":"2022-05-26T00:00:29.478527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check types of columns\ntrain.dtypes.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look up mean values for targets in train_labels\nprint(f'{round((train_labels.target.mean())*100,1)} % of customers default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge train with targets\ntemp=pd.merge(train[['customer_ID']],train_labels,on='customer_ID',how='left')\ndel train","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:32:30.468060Z","iopub.execute_input":"2022-05-26T00:32:30.468574Z","iopub.status.idle":"2022-05-26T00:32:31.969609Z","shell.execute_reply.started":"2022-05-26T00:32:30.468532Z","shell.execute_reply":"2022-05-26T00:32:31.968519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# frequencys and target distribution\ntemp1=pd.DataFrame(temp.customer_ID.value_counts()).reset_index().rename({'index':'customer_ID','customer_ID':'cust_bins'},axis=1)\ntemp2=pd.merge(temp1,train_labels,on='customer_ID',how='right')\ntemp2.groupby('cust_bins').target.agg(customer_amount_in_bins='count',target_share_in_bins='mean')","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:32:48.857189Z","iopub.execute_input":"2022-05-26T00:32:48.857655Z","iopub.status.idle":"2022-05-26T00:32:50.260505Z","shell.execute_reply.started":"2022-05-26T00:32:48.857616Z","shell.execute_reply":"2022-05-26T00:32:50.259519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"to be continued","metadata":{}}]}