{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!pip install pyspark","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:53.861084Z","iopub.execute_input":"2022-08-12T01:35:53.861542Z","iopub.status.idle":"2022-08-12T01:35:53.867612Z","shell.execute_reply.started":"2022-08-12T01:35:53.861509Z","shell.execute_reply":"2022-08-12T01:35:53.866262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd\nfrom pyspark.sql import SparkSession #Import our spark session\nfrom pyspark.sql.types import IntegerType\n\nspark = SparkSession \\\n    .builder \\\n    .appName(\"Getting Started with Titanic\") \\\n    .getOrCreate()\nspark.sparkContext.setLogLevel('WARN')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-12T01:35:53.877104Z","iopub.execute_input":"2022-08-12T01:35:53.878261Z","iopub.status.idle":"2022-08-12T01:35:53.894540Z","shell.execute_reply.started":"2022-08-12T01:35:53.878182Z","shell.execute_reply":"2022-08-12T01:35:53.892995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import train and test data","metadata":{}},{"cell_type":"code","source":"train_data = spark \\\n        .read.option(\"header\", True) \\\n        .csv(\"/kaggle/input/titanic/train.csv\",inferSchema = True)\ntrain_data = train_data.na.fill(value=-10000)\ntrain_data.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:53.951138Z","iopub.execute_input":"2022-08-12T01:35:53.952096Z","iopub.status.idle":"2022-08-12T01:35:54.349376Z","shell.execute_reply.started":"2022-08-12T01:35:53.952051Z","shell.execute_reply":"2022-08-12T01:35:54.347978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = spark \\\n            .read \\\n            .option(\"header\", True) \\\n            .csv(\"/kaggle/input/titanic/test.csv\",inferSchema = True)\ntest_data = test_data.na.fill(value=-10000)\ntest_data.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:54.352235Z","iopub.execute_input":"2022-08-12T01:35:54.352725Z","iopub.status.idle":"2022-08-12T01:35:54.662574Z","shell.execute_reply.started":"2022-08-12T01:35:54.352679Z","shell.execute_reply":"2022-08-12T01:35:54.661333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"women = train_data \\\n        .select(train_data.Survived) \\\n        .where(train_data.Sex == 'female')\n\nrate_women = women \\\n            .groupby() \\\n            .sum().collect()[0][0] / women.count()\n\nprint(\"% of women who survived: \", rate_women)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:54.663882Z","iopub.execute_input":"2022-08-12T01:35:54.664340Z","iopub.status.idle":"2022-08-12T01:35:55.001222Z","shell.execute_reply.started":"2022-08-12T01:35:54.664297Z","shell.execute_reply":"2022-08-12T01:35:54.999904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"men = train_data \\\n    .select(train_data.Survived) \\\n    .where(train_data.Sex == 'male')\nrate_men = men.groupby().sum().collect()[0][0] / men.count()\n\nprint(\"% of men who survived: \", rate_men)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:55.004229Z","iopub.execute_input":"2022-08-12T01:35:55.004702Z","iopub.status.idle":"2022-08-12T01:35:55.382620Z","shell.execute_reply.started":"2022-08-12T01:35:55.004660Z","shell.execute_reply":"2022-08-12T01:35:55.381301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-processing feature\n\nTransform columns to vector","metadata":{}},{"cell_type":"code","source":"from pyspark.ml.feature import VectorAssembler\n\nfeature_cols = [c for c in train_data.columns if c not in {'PassengerId', 'Name', 'Ticket', 'Cabin', 'Embarked', 'Survived', 'Sex'}]\n\nassembler = VectorAssembler(inputCols = feature_cols, outputCol = \"features\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:55.384099Z","iopub.execute_input":"2022-08-12T01:35:55.385591Z","iopub.status.idle":"2022-08-12T01:35:55.399054Z","shell.execute_reply.started":"2022-08-12T01:35:55.385534Z","shell.execute_reply":"2022-08-12T01:35:55.397677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_merged = assembler.transform(train_data)\ntest_data_merged = assembler.transform(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:55.401317Z","iopub.execute_input":"2022-08-12T01:35:55.402109Z","iopub.status.idle":"2022-08-12T01:35:55.498846Z","shell.execute_reply.started":"2022-08-12T01:35:55.402071Z","shell.execute_reply":"2022-08-12T01:35:55.497410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(train_data_merged.take(3), columns = train_data_merged.columns).transpose()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:55.500317Z","iopub.execute_input":"2022-08-12T01:35:55.500779Z","iopub.status.idle":"2022-08-12T01:35:55.666866Z","shell.execute_reply.started":"2022-08-12T01:35:55.500736Z","shell.execute_reply":"2022-08-12T01:35:55.665658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"rf = RandomForestClassifier(featuresCol = 'features', labelCol = 'Survived')\nrf_model = rf.fit(train_data_merged)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:55.674470Z","iopub.execute_input":"2022-08-12T01:35:55.680294Z","iopub.status.idle":"2022-08-12T01:35:57.451322Z","shell.execute_reply.started":"2022-08-12T01:35:55.680226Z","shell.execute_reply":"2022-08-12T01:35:57.449871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = rf_model.transform(test_data_merged)\noutput_pred = predictions.select('PassengerId','prediction')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:57.460943Z","iopub.execute_input":"2022-08-12T01:35:57.467031Z","iopub.status.idle":"2022-08-12T01:35:57.600677Z","shell.execute_reply.started":"2022-08-12T01:35:57.466961Z","shell.execute_reply":"2022-08-12T01:35:57.599242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_pred.printSchema()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:57.606371Z","iopub.execute_input":"2022-08-12T01:35:57.607161Z","iopub.status.idle":"2022-08-12T01:35:57.614861Z","shell.execute_reply.started":"2022-08-12T01:35:57.607114Z","shell.execute_reply":"2022-08-12T01:35:57.613586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_pred = output_pred.select('PassengerId', col(\"prediction\").cast('int').alias(\"Survived\"))\npassenger_id = output_pred.select('PassengerId') \\\n                .rdd \\\n                .flatMap(lambda x: x) \\\n                .collect()\n\npredictions = output_pred.select('Survived') \\\n                .rdd \\\n                .flatMap(lambda x: x) \\\n                .collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:57.616681Z","iopub.execute_input":"2022-08-12T01:35:57.617476Z","iopub.status.idle":"2022-08-12T01:35:58.005413Z","shell.execute_reply.started":"2022-08-12T01:35:57.617428Z","shell.execute_reply":"2022-08-12T01:35:58.004208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save submission","metadata":{}},{"cell_type":"code","source":"output_data = {'PassengerId': passenger_id, 'Survived': predictions}\noutput_tag = pd.DataFrame(data = output_data)\noutput_tag.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T01:35:58.006738Z","iopub.execute_input":"2022-08-12T01:35:58.009635Z","iopub.status.idle":"2022-08-12T01:35:58.019605Z","shell.execute_reply.started":"2022-08-12T01:35:58.009564Z","shell.execute_reply":"2022-08-12T01:35:58.018418Z"},"trusted":true},"execution_count":null,"outputs":[]}]}