{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-27T13:41:07.327206Z","iopub.execute_input":"2022-05-27T13:41:07.327903Z","iopub.status.idle":"2022-05-27T13:41:07.337747Z","shell.execute_reply.started":"2022-05-27T13:41:07.327867Z","shell.execute_reply":"2022-05-27T13:41:07.33655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Please upvote if you find this JNB useful !!!","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:07.354386Z","iopub.execute_input":"2022-05-27T13:41:07.354817Z","iopub.status.idle":"2022-05-27T13:41:07.359361Z","shell.execute_reply.started":"2022-05-27T13:41:07.354783Z","shell.execute_reply":"2022-05-27T13:41:07.358336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyspark\n","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:07.380263Z","iopub.execute_input":"2022-05-27T13:41:07.381222Z","iopub.status.idle":"2022-05-27T13:41:50.436706Z","shell.execute_reply.started":"2022-05-27T13:41:07.381183Z","shell.execute_reply":"2022-05-27T13:41:50.435761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nfrom pyspark import SparkConf, SparkContext\nfrom pyspark.sql import SparkSession, SQLContext\n\nfrom pyspark.sql.types import *\nimport pyspark.sql.functions as F\nfrom pyspark.sql.functions import udf, col\n\nfrom pyspark.ml.regression import LinearRegression\nfrom pyspark.mllib.evaluation import RegressionMetrics\n\nfrom pyspark.ml.tuning import ParamGridBuilder, CrossValidator, CrossValidatorModel\nfrom pyspark.ml.feature import VectorAssembler, StandardScaler\nfrom pyspark.ml.evaluation import RegressionEvaluator","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:50.438928Z","iopub.execute_input":"2022-05-27T13:41:50.439306Z","iopub.status.idle":"2022-05-27T13:41:50.626632Z","shell.execute_reply.started":"2022-05-27T13:41:50.439271Z","shell.execute_reply":"2022-05-27T13:41:50.625759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:50.627603Z","iopub.execute_input":"2022-05-27T13:41:50.628517Z","iopub.status.idle":"2022-05-27T13:41:51.695196Z","shell.execute_reply.started":"2022-05-27T13:41:50.628481Z","shell.execute_reply":"2022-05-27T13:41:51.69419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualization\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\n\npd.set_option('display.max_columns', 200)\npd.set_option('display.max_colwidth', 400)\n\nfrom matplotlib import rcParams\nsns.set(context='notebook', style='whitegrid', rc={'figure.figsize': (18,4)})\nrcParams['figure.figsize'] = 18,4\n\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:51.697012Z","iopub.execute_input":"2022-05-27T13:41:51.697371Z","iopub.status.idle":"2022-05-27T13:41:51.71867Z","shell.execute_reply.started":"2022-05-27T13:41:51.69734Z","shell.execute_reply":"2022-05-27T13:41:51.717934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.sql import SparkSession\nspark = SparkSession.builder.appName('classification').getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:51.720167Z","iopub.execute_input":"2022-05-27T13:41:51.720904Z","iopub.status.idle":"2022-05-27T13:41:57.658955Z","shell.execute_reply.started":"2022-05-27T13:41:51.720842Z","shell.execute_reply":"2022-05-27T13:41:57.658029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import chain\nfrom pyspark.sql.functions import count, mean, when, lit, create_map, regexp_extract","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:57.660143Z","iopub.execute_input":"2022-05-27T13:41:57.660487Z","iopub.status.idle":"2022-05-27T13:41:57.665768Z","shell.execute_reply.started":"2022-05-27T13:41:57.660455Z","shell.execute_reply":"2022-05-27T13:41:57.664602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = spark.read.csv('../input/amex-default-prediction/train_data.csv',\\\n                     header=True, inferSchema=True)\ndf2 = spark.read.csv('../input/amex-default-prediction/test_data.csv', \\\n                     header=True, inferSchema=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T13:41:57.666697Z","iopub.execute_input":"2022-05-27T13:41:57.667469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.show(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.printSchema()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\npd.DataFrame(df1.take(5), columns = df1.columns).transpose()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.printSchema()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"str_features = [t[0] for t in df1.dtypes if t[1] == 'string']\nstr_features\ndf1.select(str_features).show(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.groupBy('D_63').count().show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssdf = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssdf.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssdf.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssdf['prediction'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainlabels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ntrainlabels.head(5)                        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainlabels.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.count(), len(df1.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ps = spark.read.csv('../input/amex-default-prediction/train_labels.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf = df1.crossJoin(ps)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.show(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.printSchema()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.ml.feature import StringIndexer, VectorAssembler\nfrom pyspark.ml.classification import LogisticRegression,\\\n                    RandomForestClassifier, GBTClassifier\nfrom pyspark.ml.evaluation import MulticlassClassificationEvaluator\nfrom pyspark.ml import Pipeline\nfrom pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Please upvote if you find this JNB useful !!!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}