{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip install pyspark","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-16T11:37:08.736167Z","iopub.execute_input":"2022-07-16T11:37:08.736645Z","iopub.status.idle":"2022-07-16T11:37:55.239598Z","shell.execute_reply.started":"2022-07-16T11:37:08.736609Z","shell.execute_reply":"2022-07-16T11:37:55.238154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom pyspark.sql import SparkSession\nfrom pyspark.sql.functions import col,isnan, when, count\nfrom pyspark.ml  import Pipeline   \nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:37:55.241988Z","iopub.execute_input":"2022-07-16T11:37:55.242348Z","iopub.status.idle":"2022-07-16T11:37:58.359630Z","shell.execute_reply.started":"2022-07-16T11:37:55.242298Z","shell.execute_reply":"2022-07-16T11:37:58.358686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark = SparkSession.builder.appName(\"prediction\").getOrCreate()\nspark","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:37:58.360946Z","iopub.execute_input":"2022-07-16T11:37:58.361310Z","iopub.status.idle":"2022-07-16T11:38:05.336315Z","shell.execute_reply.started":"2022-07-16T11:37:58.361275Z","shell.execute_reply":"2022-07-16T11:38:05.335273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf=spark.read.csv('../input/amex-default-prediction/train_data.csv',\n                  inferSchema=True,header=True)\nspark.catalog.clearCache()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:38:05.342059Z","iopub.execute_input":"2022-07-16T11:38:05.344373Z","iopub.status.idle":"2022-07-16T11:43:35.179945Z","shell.execute_reply.started":"2022-07-16T11:38:05.344333Z","shell.execute_reply":"2022-07-16T11:43:35.178922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time \n# dataframe = pd.read_csv(\"../input/amex-default-prediction/train_data.csv\")\n# ddataframe.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T06:31:24.071531Z","iopub.execute_input":"2022-07-15T06:31:24.072212Z","iopub.status.idle":"2022-07-15T06:31:24.076906Z","shell.execute_reply.started":"2022-07-15T06:31:24.072174Z","shell.execute_reply":"2022-07-15T06:31:24.075828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n- Your notebook tried to allocate more memory than is available. It has restarted.\n","metadata":{}},{"cell_type":"code","source":"print((df.count(), len(df.columns)))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T06:50:34.848251Z","iopub.execute_input":"2022-07-16T06:50:34.848592Z","iopub.status.idle":"2022-07-16T06:52:20.521992Z","shell.execute_reply.started":"2022-07-16T06:50:34.848559Z","shell.execute_reply":"2022-07-16T06:52:20.520975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.limit(5).toPandas()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T06:52:20.523559Z","iopub.execute_input":"2022-07-16T06:52:20.524475Z","iopub.status.idle":"2022-07-16T06:52:21.456002Z","shell.execute_reply.started":"2022-07-16T06:52:20.524434Z","shell.execute_reply":"2022-07-16T06:52:21.455036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_list = df.columns\nSpending_col = [i for i in col_list if \"S\" in i]\nprint(\"Spending Columns\",len(Spending_col))\n\nDelinquency_col = [i for i in col_list if \"D\" in i]\nprint(\"Delinquency Columns\",len(Delinquency_col))\n\nPayment_col = [i for i in col_list if \"P\" in i]\nprint(\"Payment Columns\",len(Payment_col))\n\nBalance_col = [i for i in col_list if \"B\" in i]\nprint(\"Balance Columns\",len(Balance_col))\n\nRisk_col = [i for i in col_list if \"R\" in i]\nprint(\"Risk Columns\",len(Risk_col))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:51:53.234034Z","iopub.execute_input":"2022-07-16T11:51:53.234634Z","iopub.status.idle":"2022-07-16T11:51:53.290903Z","shell.execute_reply.started":"2022-07-16T11:51:53.234598Z","shell.execute_reply":"2022-07-16T11:51:53.289917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select(Payment_col).show(10)\nspark.catalog.clearCache()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:51:56.741764Z","iopub.execute_input":"2022-07-16T11:51:56.742133Z","iopub.status.idle":"2022-07-16T11:51:57.421407Z","shell.execute_reply.started":"2022-07-16T11:51:56.742102Z","shell.execute_reply":"2022-07-16T11:51:57.418955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"balance_null_cols = df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in Balance_col]\n   ).toPandas()\nbalance_null_cols.head().T","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:41:00.090769Z","iopub.execute_input":"2022-07-16T07:41:00.091112Z","iopub.status.idle":"2022-07-16T07:43:47.795118Z","shell.execute_reply.started":"2022-07-16T07:41:00.091082Z","shell.execute_reply":"2022-07-16T07:43:47.794199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"risk_null_cols = df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in Risk_col]\n   ).toPandas()\nrisk_null_cols.head().T","metadata":{"execution":{"iopub.status.busy":"2022-07-16T08:06:20.367239Z","iopub.execute_input":"2022-07-16T08:06:20.367616Z","iopub.status.idle":"2022-07-16T08:08:56.160833Z","shell.execute_reply.started":"2022-07-16T08:06:20.367578Z","shell.execute_reply":"2022-07-16T08:08:56.159850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npayment_null_cols = df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in Payment_col]\n   ).toPandas()\npayment_null_cols.head().T","metadata":{"execution":{"iopub.status.busy":"2022-07-16T08:08:56.162273Z","iopub.execute_input":"2022-07-16T08:08:56.163250Z","iopub.status.idle":"2022-07-16T08:11:13.635180Z","shell.execute_reply.started":"2022-07-16T08:08:56.163213Z","shell.execute_reply":"2022-07-16T08:11:13.634213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delinquency_null_cols = df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in Delinquency_col]\n   ).toPandas()\ndelinquency_null_cols.head().T","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:53:20.295110Z","iopub.execute_input":"2022-07-16T11:53:20.295491Z","iopub.status.idle":"2022-07-16T11:57:52.158489Z","shell.execute_reply.started":"2022-07-16T11:53:20.295460Z","shell.execute_reply":"2022-07-16T11:57:52.157419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_col_list = [\"B_42\",\"B_39\",\"B_29\",\"R_9\",\"R_26\",\"D_42\",\"D_49\",\n                        \"D_56\",\"D_142\",\"D_136\",\"D_137\",\"D_138\",\"D_76\",\"D_73\",\n                        \"D_132\",\"D_134\",\"D_135\",\"D_66\",\"D_77\",\"D_82\",\"D_87\",\"D_88\",\n                        \"D_106\",\"D_108\",\"D_110\",\"D_111\",\"D_105\",\"S_9\"]\ndf1 = df.drop(*null_col_list)\nprint((df1.count(), len(df1.columns)))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T13:20:34.817617Z","iopub.execute_input":"2022-07-16T13:20:34.818010Z","iopub.status.idle":"2022-07-16T13:22:27.445808Z","shell.execute_reply.started":"2022-07-16T13:20:34.817954Z","shell.execute_reply":"2022-07-16T13:22:27.444812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = df1.na.drop()\nprint((df2.count(), len(df2.columns)))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T13:24:04.385864Z","iopub.execute_input":"2022-07-16T13:24:04.386518Z","iopub.status.idle":"2022-07-16T13:29:04.289123Z","shell.execute_reply.started":"2022-07-16T13:24:04.386484Z","shell.execute_reply":"2022-07-16T13:29:04.288108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3 = df.drop(\"S_2\")\ncol_list1 = df3.columns\nSpending_col1 = [i for i in col_list1 if \"S\" in i]\nspending_null_cols1 = df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in Spending_col1]\n   ).toPandas()\nspending_null_cols1.head().T","metadata":{"execution":{"iopub.status.busy":"2022-07-16T13:14:11.483210Z","iopub.execute_input":"2022-07-16T13:14:11.484205Z","iopub.status.idle":"2022-07-16T13:16:49.152860Z","shell.execute_reply.started":"2022-07-16T13:14:11.484160Z","shell.execute_reply":"2022-07-16T13:16:49.151834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in df[Spending_col\n]]).toPandas()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:10:28.023087Z","iopub.execute_input":"2022-07-15T18:10:28.023609Z","iopub.status.idle":"2022-07-15T18:10:28.190879Z","shell.execute_reply.started":"2022-07-15T18:10:28.023570Z","shell.execute_reply":"2022-07-15T18:10:28.188017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in df.columns]\n   ).show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df = df.join(df1, df.customer_ID == df.customer_ID, \n               \"outer\").show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:56:46.911122Z","iopub.execute_input":"2022-07-15T07:56:46.911565Z","iopub.status.idle":"2022-07-15T17:53:02.996322Z","shell.execute_reply.started":"2022-07-15T07:56:46.911527Z","shell.execute_reply":"2022-07-15T17:53:02.992641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark.catalog.clearCache()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:53:03.089178Z","iopub.status.idle":"2022-07-15T17:53:03.089628Z","shell.execute_reply.started":"2022-07-15T17:53:03.089390Z","shell.execute_reply":"2022-07-15T17:53:03.089413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:53:03.450689Z","iopub.execute_input":"2022-07-15T17:53:03.451318Z","iopub.status.idle":"2022-07-15T17:53:03.459388Z","shell.execute_reply.started":"2022-07-15T17:53:03.451274Z","shell.execute_reply":"2022-07-15T17:53:03.458536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"heell\"","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:53:03.469887Z","iopub.execute_input":"2022-07-15T17:53:03.470227Z","iopub.status.idle":"2022-07-15T17:53:03.476161Z","shell.execute_reply.started":"2022-07-15T17:53:03.470194Z","shell.execute_reply":"2022-07-15T17:53:03.475233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"None","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:53:03.491195Z","iopub.execute_input":"2022-07-15T17:53:03.491540Z","iopub.status.idle":"2022-07-15T17:53:03.495457Z","shell.execute_reply.started":"2022-07-15T17:53:03.491513Z","shell.execute_reply":"2022-07-15T17:53:03.494665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}