{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Reading full data 15gb Train data without any transformation within 16gb kernel\n# would advise to transform train and test further to do EDA.\n\nimport time\nimport pandas as pd\nimport plotly.express as px\n\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-15T11:37:44.401685Z","iopub.execute_input":"2022-06-15T11:37:44.402118Z","iopub.status.idle":"2022-06-15T11:37:45.917876Z","shell.execute_reply.started":"2022-06-15T11:37:44.402074Z","shell.execute_reply":"2022-06-15T11:37:45.916744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  defining data types\ndtypes = {'customer_ID':'O',\n'S_2':'O',\n'P_2':'float32',\n'D_39':'float32',\n'B_1':'float32',\n'B_2':'float32',\n'R_1':'float32',\n'S_3':'float32',\n'D_41':'float32',\n'B_3':'float32',\n'D_42':'float32',\n'D_43':'float32',\n'D_44':'float32',\n'B_4':'float32',\n'D_45':'float32',\n'B_5':'float32',\n'R_2':'float32',\n'D_46':'float32',\n'D_47':'float32',\n'D_48':'float32',\n'D_49':'float32',\n'B_6':'float32',\n'B_7':'float32',\n'B_8':'float32',\n'D_50':'float32',\n'D_51':'float32',\n'B_9':'float32',\n'R_3':'float32',\n'D_52':'float32',\n'P_3':'float32',\n'B_10':'float32',\n'D_53':'float32',\n'S_5':'float32',\n'B_11':'float32',\n'S_6':'float32',\n'D_54':'float32',\n'R_4':'float32',\n'S_7':'float32',\n'B_12':'float32',\n'S_8':'float32',\n'D_55':'float32',\n'D_56':'float32',\n'B_13':'float32',\n'R_5':'float32',\n'D_58':'float32',\n'S_9':'float32',\n'B_14':'float32',\n'D_59':'float32',\n'D_60':'float32',\n'D_61':'float32',\n'B_15':'float32',\n'S_11':'float32',\n'D_62':'float32',\n'D_63':'category',\n'D_64':'category',\n'D_65':'float32',\n'B_16':'float32',\n'B_17':'float32',\n'B_18':'float32',\n'B_19':'float32',\n'D_66':'category',\n'B_20':'float32',\n'D_68':'category',\n'S_12':'float32',\n'R_6':'float32',\n'S_13':'float32',\n'B_21':'float32',\n'D_69':'float32',\n'B_22':'float32',\n'D_70':'float32',\n'D_71':'float32',\n'D_72':'float32',\n'S_15':'float32',\n'B_23':'float32',\n'D_73':'float32',\n'P_4':'float32',\n'D_74':'float32',\n'D_75':'float32',\n'D_76':'float32',\n'B_24':'float32',\n'R_7':'float32',\n'D_77':'float32',\n'B_25':'float32',\n'B_26':'float32',\n'D_78':'float32',\n'D_79':'float32',\n'R_8':'float32',\n'R_9':'float32',\n'S_16':'float32',\n'D_80':'float32',\n'R_10':'float32',\n'R_11':'float32',\n'B_27':'float32',\n'D_81':'float32',\n'D_82':'float32',\n'S_17':'float32',\n'R_12':'float32',\n'B_28':'float32',\n'R_13':'float32',\n'D_83':'float32',\n'R_14':'float32',\n'R_15':'float32',\n'D_84':'float32',\n'R_16':'float32',\n'B_29':'float32',\n'B_30':'category',\n'S_18':'float32',\n'D_86':'float32',\n'D_87':'float32',\n'R_17':'float32',\n'R_18':'float32',\n'D_88':'float32',\n'B_31':'int32',\n'S_19':'float32',\n'R_19':'float32',\n'B_32':'float32',\n'S_20':'float32',\n'R_20':'float32',\n'R_21':'float32',\n'B_33':'float32',\n'D_89':'float32',\n'R_22':'float32',\n'R_23':'float32',\n'D_91':'float32',\n'D_92':'float32',\n'D_93':'float32',\n'D_94':'float32',\n'R_24':'float32',\n'R_25':'float32',\n'D_96':'float32',\n'S_22':'float32',\n'S_23':'float32',\n'S_24':'float32',\n'S_25':'float32',\n'S_26':'float32',\n'D_102':'float32',\n'D_103':'float32',\n'D_104':'float32',\n'D_105':'float32',\n'D_106':'float32',\n'D_107':'float32',\n'B_36':'float32',\n'B_37':'float32',\n'R_26':'float32',\n'R_27':'float32',\n'B_38':'category',\n'D_108':'float32',\n'D_109':'float32',\n'D_110':'float32',\n'D_111':'float32',\n'B_39':'float32',\n'D_112':'float32',\n'B_40':'float32',\n'S_27':'float32',\n'D_113':'float32',\n'D_114':'category',\n'D_115':'float32',\n'D_116':'category',\n'D_117':'category',\n'D_118':'float32',\n'D_119':'float32',\n'D_120':'category',\n'D_121':'float32',\n'D_122':'float32',\n'D_123':'float32',\n'D_124':'float32',\n'D_125':'float32',\n'D_126':'category',\n'D_127':'float32',\n'D_128':'float32',\n'D_129':'float32',\n'B_41':'float32',\n'B_42':'float32',\n'D_130':'float32',\n'D_131':'float32',\n'D_132':'float32',\n'D_133':'float32',\n'R_28':'float32',\n'D_134':'float32',\n'D_135':'float32',\n'D_136':'float32',\n'D_137':'float32',\n'D_138':'float32',\n'D_139':'float32',\n'D_140':'float32',\n'D_141':'float32',\n'D_142':'float32',\n'D_143':'float32',\n'D_144':'float32',\n'D_145':'float32'}","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:37:45.920316Z","iopub.execute_input":"2022-06-15T11:37:45.920665Z","iopub.status.idle":"2022-06-15T11:37:45.946111Z","shell.execute_reply.started":"2022-06-15T11:37:45.920619Z","shell.execute_reply":"2022-06-15T11:37:45.945152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\ntp = pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', iterator=True, chunksize=1000000,dtype = dtypes)  # gives TextFileReader\ndf = pd.concat(tp, ignore_index=True)\nprint('time taken to read',time.time()-start)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:37:45.94728Z","iopub.execute_input":"2022-06-15T11:37:45.947575Z","iopub.status.idle":"2022-06-15T11:44:13.908298Z","shell.execute_reply.started":"2022-06-15T11:37:45.947548Z","shell.execute_reply":"2022-06-15T11:44:13.904538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info(verbose=False, memory_usage=\"deep\") # to check the memory","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:44:13.915594Z","iopub.execute_input":"2022-06-15T11:44:13.916495Z","iopub.status.idle":"2022-06-15T11:44:16.118953Z","shell.execute_reply.started":"2022-06-15T11:44:13.916425Z","shell.execute_reply":"2022-06-15T11:44:16.118066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe().apply(lambda s: s.apply('{0:.5f}'.format))","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:48:12.367678Z","iopub.execute_input":"2022-06-15T11:48:12.367993Z","iopub.status.idle":"2022-06-15T11:48:55.095825Z","shell.execute_reply.started":"2022-06-15T11:48:12.367966Z","shell.execute_reply":"2022-06-15T11:48:55.094764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['customer_ID'].nunique()\n# total customers 458913","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets read train labels\n# as the file size is not so huge thus no need to use data types or chunksize\nstart = time.time()\ntrain_labels_df = pd.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')  # gives TextFileReader\nprint('time taken to read',time.time()-start)\ntrain_labels_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge the train data with train labels\nstart = time.time()\ntrain_df = pd.merge(df,train_labels_df,left_on='customer_ID',right_on='customer_ID',how='left')\nprint('time taken to read',time.time()-start)\ntrain_df.shape","metadata":{"execution":{"iopub.status.idle":"2022-06-15T11:45:29.821924Z","shell.execute_reply.started":"2022-06-15T11:45:03.521752Z","shell.execute_reply":"2022-06-15T11:45:29.820868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check distribution\ntrain_df['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:45:29.825Z","iopub.execute_input":"2022-06-15T11:45:29.825343Z","iopub.status.idle":"2022-06-15T11:45:29.864698Z","shell.execute_reply.started":"2022-06-15T11:45:29.825314Z","shell.execute_reply":"2022-06-15T11:45:29.863709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train_df['target'], x=\"target\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:45:29.866808Z","iopub.execute_input":"2022-06-15T11:45:29.867266Z","iopub.status.idle":"2022-06-15T11:45:56.955497Z","shell.execute_reply.started":"2022-06-15T11:45:29.867213Z","shell.execute_reply":"2022-06-15T11:45:56.954401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:45:56.956814Z","iopub.execute_input":"2022-06-15T11:45:56.957165Z","iopub.status.idle":"2022-06-15T11:45:59.88402Z","shell.execute_reply.started":"2022-06-15T11:45:56.957133Z","shell.execute_reply":"2022-06-15T11:45:59.883046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum() / len(df)","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:46:43.996598Z","iopub.execute_input":"2022-06-15T11:46:43.997421Z","iopub.status.idle":"2022-06-15T11:46:46.923592Z","shell.execute_reply.started":"2022-06-15T11:46:43.997374Z","shell.execute_reply":"2022-06-15T11:46:46.922663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T11:46:46.924972Z","iopub.execute_input":"2022-06-15T11:46:46.925404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}