{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","metadata":{},"source":"# Data Science Portfolio Project: DISTRIBUTED_COMPUTING\n\nThis notebook demonstrates applied machine learning and data science techniques. The code below spans data processing, modeling, and evaluation."},{"cell_type":"markdown","metadata":{},"source":"## Distributed Big Data Processing with PySpark\n**Portfolio Concept:** Standard Pandas DataFrames are restricted to single-node memory. Here, PySpark is utilized to process data in a distributed, resilient manner across the JVM. Operations are lazily evaluated, allowing the Spark optimizer (Catalyst) to generate highly efficient query execution plans."},{"cell_type":"code","source":"!pip install pyspark","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-16T01:37:59.451422Z","iopub.execute_input":"2022-07-16T01:37:59.452103Z","iopub.status.idle":"2022-07-16T01:38:50.975273Z","shell.execute_reply.started":"2022-07-16T01:37:59.451987Z","shell.execute_reply":"2022-07-16T01:38:50.974081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport itertools\nimport numpy as np \nimport pandas as pd\n\npd.set_option('display.max_rows',None)\npd.set_option('display.max_columns',None)\npd.set_option('display.width',500)\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom tqdm.auto import tqdm\n\nfrom pyspark import SparkContext\nfrom pyspark.sql import SparkSession\nfrom pyspark.sql.functions import *","metadata":{"execution":{"iopub.status.busy":"2022-07-16T01:38:51.187836Z","iopub.execute_input":"2022-07-16T01:38:51.189076Z","iopub.status.idle":"2022-07-16T01:38:51.19822Z","shell.execute_reply.started":"2022-07-16T01:38:51.188996Z","shell.execute_reply":"2022-07-16T01:38:51.197067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc = SparkContext(appName = \"AMEX_default_pred\")\nspark = SparkSession.Builder().getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T01:38:51.200328Z","iopub.execute_input":"2022-07-16T01:38:51.201062Z","iopub.status.idle":"2022-07-16T01:38:57.148796Z","shell.execute_reply.started":"2022-07-16T01:38:51.200994Z","shell.execute_reply":"2022-07-16T01:38:57.147434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = spark.read.csv('../input/amex-default-prediction/train_data.csv',header=True,dateFormat='yyyy-MM-dd',inferSchema=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T01:39:25.791975Z","iopub.execute_input":"2022-07-16T01:39:25.792481Z","iopub.status.idle":"2022-07-16T01:42:49.292706Z","shell.execute_reply.started":"2022-07-16T01:39:25.792439Z","shell.execute_reply":"2022-07-16T01:42:49.290931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = spark.read.csv('../input/amex-default-prediction/test_data.csv',header=True,dateFormat='yyyy-MM-dd',inferSchema=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T01:42:49.296903Z","iopub.execute_input":"2022-07-16T01:42:49.297453Z","iopub.status.idle":"2022-07-16T01:49:18.402855Z","shell.execute_reply.started":"2022-07-16T01:42:49.297408Z","shell.execute_reply":"2022-07-16T01:49:18.401427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data.show(1,vertical=True)\n#train_data.printSchema()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:27:33.201153Z","iopub.execute_input":"2022-07-14T23:27:33.201642Z","iopub.status.idle":"2022-07-14T23:27:34.407433Z","shell.execute_reply.started":"2022-07-14T23:27:33.201603Z","shell.execute_reply":"2022-07-14T23:27:34.406564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data.sort('S_2')\ntrain_data = train_data.repartition(50, \"S_2\")\n\ntrain_data.write.parquet('./output')","metadata":{"execution":{"iopub.status.busy":"2022-07-16T01:49:18.404639Z","iopub.execute_input":"2022-07-16T01:49:18.405125Z","iopub.status.idle":"2022-07-16T01:56:19.372404Z","shell.execute_reply.started":"2022-07-16T01:49:18.405074Z","shell.execute_reply":"2022-07-16T01:56:19.370116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_list = [] \n\nfor x in os.listdir(\"./output\"):\n    if x.startswith('part'):\n        parquet_list.append(x)\n        \n#print(parquet_list)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T01:56:19.37616Z","iopub.execute_input":"2022-07-16T01:56:19.37781Z","iopub.status.idle":"2022-07-16T01:56:19.387322Z","shell.execute_reply.started":"2022-07-16T01:56:19.377749Z","shell.execute_reply":"2022-07-16T01:56:19.386383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_list = sorted(parquet_list)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T01:56:19.413561Z","iopub.execute_input":"2022-07-16T01:56:19.414176Z","iopub.status.idle":"2022-07-16T01:56:19.425003Z","shell.execute_reply.started":"2022-07-16T01:56:19.414141Z","shell.execute_reply":"2022-07-16T01:56:19.423936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_list","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:37:40.410039Z","iopub.execute_input":"2022-07-15T00:37:40.410487Z","iopub.status.idle":"2022-07-15T00:37:40.418914Z","shell.execute_reply.started":"2022-07-15T00:37:40.410456Z","shell.execute_reply":"2022-07-15T00:37:40.418083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = spark.read.parquet('./output/'+parquet_list[1])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T17:55:40.273932Z","iopub.execute_input":"2022-07-15T17:55:40.274411Z","iopub.status.idle":"2022-07-15T17:55:40.375724Z","shell.execute_reply.started":"2022-07-15T17:55:40.274316Z","shell.execute_reply":"2022-07-15T17:55:40.374231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdf = df.toPandas()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:46:27.280921Z","iopub.execute_input":"2022-07-15T00:46:27.281314Z","iopub.status.idle":"2022-07-15T00:46:31.018655Z","shell.execute_reply.started":"2022-07-15T00:46:27.281283Z","shell.execute_reply":"2022-07-15T00:46:31.017649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:46:40.908687Z","iopub.execute_input":"2022-07-15T00:46:40.909227Z","iopub.status.idle":"2022-07-15T00:46:41.082669Z","shell.execute_reply.started":"2022-07-15T00:46:40.909165Z","shell.execute_reply":"2022-07-15T00:46:41.081515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdf.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:47:08.010592Z","iopub.execute_input":"2022-07-15T00:47:08.011022Z","iopub.status.idle":"2022-07-15T00:47:08.019465Z","shell.execute_reply.started":"2022-07-15T00:47:08.010988Z","shell.execute_reply":"2022-07-15T00:47:08.018087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in zip(range(len(parquet_list)),parquet_list):\n    df+str(x[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:42:43.150357Z","iopub.execute_input":"2022-07-15T00:42:43.150871Z","iopub.status.idle":"2022-07-15T00:42:43.160585Z","shell.execute_reply.started":"2022-07-15T00:42:43.150832Z","shell.execute_reply":"2022-07-15T00:42:43.158823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_df = spark.read.parquet('./output/part-00093-d08fa89d-b158-4ee7-84fb-ff753e9fdbce-c000.snappy.parquet')\nx_df.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:57:30.890318Z","iopub.execute_input":"2022-07-14T23:57:30.890806Z","iopub.status.idle":"2022-07-14T23:57:32.282583Z","shell.execute_reply.started":"2022-07-14T23:57:30.890766Z","shell.execute_reply":"2022-07-14T23:57:32.281407Z"},"trusted":true},"execution_count":null,"outputs":[]}]}