{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Install Pyspark","metadata":{}},{"cell_type":"code","source":"!pip install pyspark","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:30:52.312826Z","iopub.execute_input":"2022-05-26T02:30:52.313216Z","iopub.status.idle":"2022-05-26T02:31:37.003106Z","shell.execute_reply.started":"2022-05-26T02:30:52.313168Z","shell.execute_reply":"2022-05-26T02:31:37.001488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create a Spark Session","metadata":{}},{"cell_type":"code","source":"from pyspark.sql import SparkSession\n \nspark = SparkSession.builder.appName(\n    'AmexChallenger').getOrCreate()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:46:22.823661Z","iopub.execute_input":"2022-05-26T02:46:22.823997Z","iopub.status.idle":"2022-05-26T02:46:22.833270Z","shell.execute_reply.started":"2022-05-26T02:46:22.823973Z","shell.execute_reply":"2022-05-26T02:46:22.832262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load csv to pyspark dataframe and define schema","metadata":{}},{"cell_type":"code","source":"from pyspark.sql.types import FloatType, IntegerType, StringType, StructType, StructField\n\n # define the schema, corresponding to a line in the csv data file.\nschema = StructType([\nStructField(\"customer_ID\", StringType(), nullable = True),\nStructField(\"S_2\", StringType(), nullable = True),\nStructField(\"P_2\", FloatType(), nullable = True),\nStructField(\"D_39\", FloatType(), nullable = True),\nStructField(\"B_1\", FloatType(), nullable = True),\nStructField(\"B_2\", FloatType(), nullable = True),\nStructField(\"R_1\", FloatType(), nullable = True),\nStructField(\"S_3\", FloatType(), nullable = True),\nStructField(\"D_41\", FloatType(), nullable = True),\nStructField(\"B_3\", FloatType(), nullable = True),\nStructField(\"D_42\", FloatType(), nullable = True),\nStructField(\"D_43\", FloatType(), nullable = True),\nStructField(\"D_44\", FloatType(), nullable = True),\nStructField(\"B_4\", FloatType(), nullable = True),\nStructField(\"D_45\", FloatType(), nullable = True),\nStructField(\"B_5\", FloatType(), nullable = True),\nStructField(\"R_2\", FloatType(), nullable = True),\nStructField(\"D_46\", FloatType(), nullable = True),\nStructField(\"D_47\", FloatType(), nullable = True),\nStructField(\"D_48\", FloatType(), nullable = True),\nStructField(\"D_49\", FloatType(), nullable = True),\nStructField(\"B_6\", FloatType(), nullable = True),\nStructField(\"B_7\", FloatType(), nullable = True),\nStructField(\"B_8\", FloatType(), nullable = True),\nStructField(\"D_50\", FloatType(), nullable = True),\nStructField(\"D_51\", FloatType(), nullable = True),\nStructField(\"B_9\", FloatType(), nullable = True),\nStructField(\"R_3\", FloatType(), nullable = True),\nStructField(\"D_52\", FloatType(), nullable = True),\nStructField(\"P_3\", FloatType(), nullable = True),\nStructField(\"B_10\", FloatType(), nullable = True),\nStructField(\"D_53\", FloatType(), nullable = True),\nStructField(\"S_5\", FloatType(), nullable = True),\nStructField(\"B_11\", FloatType(), nullable = True),\nStructField(\"S_6\", FloatType(), nullable = True),\nStructField(\"D_54\", FloatType(), nullable = True),\nStructField(\"R_4\", FloatType(), nullable = True),\nStructField(\"S_7\", FloatType(), nullable = True),\nStructField(\"B_12\", FloatType(), nullable = True),\nStructField(\"S_8\", FloatType(), nullable = True),\nStructField(\"D_55\", FloatType(), nullable = True),\nStructField(\"D_56\", FloatType(), nullable = True),\nStructField(\"B_13\", FloatType(), nullable = True),\nStructField(\"R_5\", FloatType(), nullable = True),\nStructField(\"D_58\", FloatType(), nullable = True),\nStructField(\"S_9\", FloatType(), nullable = True),\nStructField(\"B_14\", FloatType(), nullable = True),\nStructField(\"D_59\", FloatType(), nullable = True),\nStructField(\"D_60\", FloatType(), nullable = True),\nStructField(\"D_61\", FloatType(), nullable = True),\nStructField(\"B_15\", FloatType(), nullable = True),\nStructField(\"S_11\", FloatType(), nullable = True),\nStructField(\"D_62\", FloatType(), nullable = True),\nStructField(\"D_63\", StringType(), nullable = True),\nStructField(\"D_64\", StringType(), nullable = True),\nStructField(\"D_65\", FloatType(), nullable = True),\nStructField(\"B_16\", FloatType(), nullable = True),\nStructField(\"B_17\", FloatType(), nullable = True),\nStructField(\"B_18\", FloatType(), nullable = True),\nStructField(\"B_19\", FloatType(), nullable = True),\nStructField(\"D_66\", FloatType(), nullable = True),\nStructField(\"B_20\", FloatType(), nullable = True),\nStructField(\"D_68\", FloatType(), nullable = True),\nStructField(\"S_12\", FloatType(), nullable = True),\nStructField(\"R_6\", FloatType(), nullable = True),\nStructField(\"S_13\", FloatType(), nullable = True),\nStructField(\"B_21\", FloatType(), nullable = True),\nStructField(\"D_69\", FloatType(), nullable = True),\nStructField(\"B_22\", FloatType(), nullable = True),\nStructField(\"D_70\", FloatType(), nullable = True),\nStructField(\"D_71\", FloatType(), nullable = True),\nStructField(\"D_72\", FloatType(), nullable = True),\nStructField(\"S_15\", FloatType(), nullable = True),\nStructField(\"B_23\", FloatType(), nullable = True),\nStructField(\"D_73\", FloatType(), nullable = True),\nStructField(\"P_4\", FloatType(), nullable = True),\nStructField(\"D_74\", FloatType(), nullable = True),\nStructField(\"D_75\", FloatType(), nullable = True),\nStructField(\"D_76\", FloatType(), nullable = True),\nStructField(\"B_24\", FloatType(), nullable = True),\nStructField(\"R_7\", FloatType(), nullable = True),\nStructField(\"D_77\", FloatType(), nullable = True),\nStructField(\"B_25\", FloatType(), nullable = True),\nStructField(\"B_26\", FloatType(), nullable = True),\nStructField(\"D_78\", FloatType(), nullable = True),\nStructField(\"D_79\", FloatType(), nullable = True),\nStructField(\"R_8\", FloatType(), nullable = True),\nStructField(\"R_9\", FloatType(), nullable = True),\nStructField(\"S_16\", FloatType(), nullable = True),\nStructField(\"D_80\", FloatType(), nullable = True),\nStructField(\"R_10\", FloatType(), nullable = True),\nStructField(\"R_11\", FloatType(), nullable = True),\nStructField(\"B_27\", FloatType(), nullable = True),\nStructField(\"D_81\", FloatType(), nullable = True),\nStructField(\"D_82\", FloatType(), nullable = True),\nStructField(\"S_17\", FloatType(), nullable = True),\nStructField(\"R_12\", FloatType(), nullable = True),\nStructField(\"B_28\", FloatType(), nullable = True),\nStructField(\"R_13\", FloatType(), nullable = True),\nStructField(\"D_83\", FloatType(), nullable = True),\nStructField(\"R_14\", FloatType(), nullable = True),\nStructField(\"R_15\", FloatType(), nullable = True),\nStructField(\"D_84\", FloatType(), nullable = True),\nStructField(\"R_16\", FloatType(), nullable = True),\nStructField(\"B_29\", FloatType(), nullable = True),\nStructField(\"B_30\", FloatType(), nullable = True),\nStructField(\"S_18\", FloatType(), nullable = True),\nStructField(\"D_86\", FloatType(), nullable = True),\nStructField(\"D_87\", FloatType(), nullable = True),\nStructField(\"R_17\", FloatType(), nullable = True),\nStructField(\"R_18\", FloatType(), nullable = True),\nStructField(\"D_88\", FloatType(), nullable = True),\nStructField(\"B_31\", IntegerType(), nullable = True),\nStructField(\"S_19\", FloatType(), nullable = True),\nStructField(\"R_19\", FloatType(), nullable = True),\nStructField(\"B_32\", FloatType(), nullable = True),\nStructField(\"S_20\", FloatType(), nullable = True),\nStructField(\"R_20\", FloatType(), nullable = True),\nStructField(\"R_21\", FloatType(), nullable = True),\nStructField(\"B_33\", FloatType(), nullable = True),\nStructField(\"D_89\", FloatType(), nullable = True),\nStructField(\"R_22\", FloatType(), nullable = True),\nStructField(\"R_23\", FloatType(), nullable = True),\nStructField(\"D_91\", FloatType(), nullable = True),\nStructField(\"D_92\", FloatType(), nullable = True),\nStructField(\"D_93\", FloatType(), nullable = True),\nStructField(\"D_94\", FloatType(), nullable = True),\nStructField(\"R_24\", FloatType(), nullable = True),\nStructField(\"R_25\", FloatType(), nullable = True),\nStructField(\"D_96\", FloatType(), nullable = True),\nStructField(\"S_22\", FloatType(), nullable = True),\nStructField(\"S_23\", FloatType(), nullable = True),\nStructField(\"S_24\", FloatType(), nullable = True),\nStructField(\"S_25\", FloatType(), nullable = True),\nStructField(\"S_26\", FloatType(), nullable = True),\nStructField(\"D_102\", FloatType(), nullable = True),\nStructField(\"D_103\", FloatType(), nullable = True),\nStructField(\"D_104\", FloatType(), nullable = True),\nStructField(\"D_105\", FloatType(), nullable = True),\nStructField(\"D_106\", FloatType(), nullable = True),\nStructField(\"D_107\", FloatType(), nullable = True),\nStructField(\"B_36\", FloatType(), nullable = True),\nStructField(\"B_37\", FloatType(), nullable = True),\nStructField(\"R_26\", FloatType(), nullable = True),\nStructField(\"R_27\", FloatType(), nullable = True),\nStructField(\"B_38\", FloatType(), nullable = True),\nStructField(\"D_108\", FloatType(), nullable = True),\nStructField(\"D_109\", FloatType(), nullable = True),\nStructField(\"D_110\", FloatType(), nullable = True),\nStructField(\"D_111\", FloatType(), nullable = True),\nStructField(\"B_39\", FloatType(), nullable = True),\nStructField(\"D_112\", FloatType(), nullable = True),\nStructField(\"B_40\", FloatType(), nullable = True),\nStructField(\"S_27\", FloatType(), nullable = True),\nStructField(\"D_113\", FloatType(), nullable = True),\nStructField(\"D_114\", FloatType(), nullable = True),\nStructField(\"D_115\", FloatType(), nullable = True),\nStructField(\"D_116\", FloatType(), nullable = True),\nStructField(\"D_117\", FloatType(), nullable = True),\nStructField(\"D_118\", FloatType(), nullable = True),\nStructField(\"D_119\", FloatType(), nullable = True),\nStructField(\"D_120\", FloatType(), nullable = True),\nStructField(\"D_121\", FloatType(), nullable = True),\nStructField(\"D_122\", FloatType(), nullable = True),\nStructField(\"D_123\", FloatType(), nullable = True),\nStructField(\"D_124\", FloatType(), nullable = True),\nStructField(\"D_125\", FloatType(), nullable = True),\nStructField(\"D_126\", FloatType(), nullable = True),\nStructField(\"D_127\", FloatType(), nullable = True),\nStructField(\"D_128\", FloatType(), nullable = True),\nStructField(\"D_129\", FloatType(), nullable = True),\nStructField(\"B_41\", FloatType(), nullable = True),\nStructField(\"B_42\", FloatType(), nullable = True),\nStructField(\"D_130\", FloatType(), nullable = True),\nStructField(\"D_131\", FloatType(), nullable = True),\nStructField(\"D_132\", FloatType(), nullable = True),\nStructField(\"D_133\", FloatType(), nullable = True),\nStructField(\"R_28\", FloatType(), nullable = True),\nStructField(\"D_134\", FloatType(), nullable = True),\nStructField(\"D_135\", FloatType(), nullable = True),\nStructField(\"D_136\", FloatType(), nullable = True),\nStructField(\"D_137\", FloatType(), nullable = True),\nStructField(\"D_138\", FloatType(), nullable = True),\nStructField(\"D_139\", FloatType(), nullable = True),\nStructField(\"D_140\", FloatType(), nullable = True),\nStructField(\"D_141\", FloatType(), nullable = True),\nStructField(\"D_142\", FloatType(), nullable = True),\nStructField(\"D_143\", FloatType(), nullable = True),\nStructField(\"D_144\", FloatType(), nullable = True),\nStructField(\"D_145\", FloatType(), nullable = True)]\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T03:20:49.182695Z","iopub.execute_input":"2022-05-26T03:20:49.183066Z","iopub.status.idle":"2022-05-26T03:20:49.253103Z","shell.execute_reply.started":"2022-05-26T03:20:49.183038Z","shell.execute_reply":"2022-05-26T03:20:49.252241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = spark.read.csv('../input/amex-default-prediction/train_data.csv', sep=',',\n                         header=True, schema=schema).cache()\n \ntrain_df.printSchema()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T03:21:59.127102Z","iopub.execute_input":"2022-05-26T03:21:59.132175Z","iopub.status.idle":"2022-05-26T03:22:00.112965Z","shell.execute_reply.started":"2022-05-26T03:21:59.132012Z","shell.execute_reply":"2022-05-26T03:22:00.110679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train shape: \", train_df.count(), len(train_df.columns))","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:39:26.032786Z","iopub.execute_input":"2022-05-26T02:39:26.033175Z","iopub.status.idle":"2022-05-26T02:40:23.024962Z","shell.execute_reply.started":"2022-05-26T02:39:26.033142Z","shell.execute_reply":"2022-05-26T02:40:23.024051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inspect first five rows\ntrain_df.take(5)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T03:23:51.431869Z","iopub.execute_input":"2022-05-26T03:23:51.432401Z","iopub.status.idle":"2022-05-26T03:23:56.589496Z","shell.execute_reply.started":"2022-05-26T03:23:51.432362Z","shell.execute_reply":"2022-05-26T03:23:56.587313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark.stop()","metadata":{},"execution_count":null,"outputs":[]}]}