{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-09T07:14:32.671020Z","iopub.execute_input":"2022-10-09T07:14:32.671494Z","iopub.status.idle":"2022-10-09T07:14:32.688690Z","shell.execute_reply.started":"2022-10-09T07:14:32.671466Z","shell.execute_reply":"2022-10-09T07:14:32.687243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install pyspark --target=/kaggle/working/mysitepackages","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:14:44.999178Z","iopub.execute_input":"2022-10-09T07:14:44.999514Z","iopub.status.idle":"2022-10-09T07:14:45.004471Z","shell.execute_reply.started":"2022-10-09T07:14:44.999487Z","shell.execute_reply":"2022-10-09T07:14:45.003341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/working/mysitepackages')","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:14:46.429226Z","iopub.execute_input":"2022-10-09T07:14:46.429574Z","iopub.status.idle":"2022-10-09T07:14:46.435892Z","shell.execute_reply.started":"2022-10-09T07:14:46.429533Z","shell.execute_reply":"2022-10-09T07:14:46.434131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.sql import SparkSession\nspark = SparkSession.builder.appName('test2').getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:14:47.817204Z","iopub.execute_input":"2022-10-09T07:14:47.817532Z","iopub.status.idle":"2022-10-09T07:14:47.826302Z","shell.execute_reply.started":"2022-10-09T07:14:47.817508Z","shell.execute_reply":"2022-10-09T07:14:47.824631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:01.145791Z","iopub.execute_input":"2022-10-09T06:48:01.146148Z","iopub.status.idle":"2022-10-09T06:48:01.159795Z","shell.execute_reply.started":"2022-10-09T06:48:01.146125Z","shell.execute_reply":"2022-10-09T06:48:01.158530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = spark.read.csv('../input/tabular-playground-series-oct-2022/train_0.csv',inferSchema=True,header=True,nullValue='NA')","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:08.633667Z","iopub.execute_input":"2022-10-09T06:48:08.634016Z","iopub.status.idle":"2022-10-09T06:48:26.445422Z","shell.execute_reply.started":"2022-10-09T06:48:08.633991Z","shell.execute_reply":"2022-10-09T06:48:26.444332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:26.447687Z","iopub.execute_input":"2022-10-09T06:48:26.447958Z","iopub.status.idle":"2022-10-09T06:48:26.491504Z","shell.execute_reply.started":"2022-10-09T06:48:26.447933Z","shell.execute_reply":"2022-10-09T06:48:26.490591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.count()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:26.494492Z","iopub.execute_input":"2022-10-09T06:48:26.494941Z","iopub.status.idle":"2022-10-09T06:48:28.276990Z","shell.execute_reply.started":"2022-10-09T06:48:26.494907Z","shell.execute_reply":"2022-10-09T06:48:28.276135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:28.279429Z","iopub.execute_input":"2022-10-09T06:48:28.280542Z","iopub.status.idle":"2022-10-09T06:48:28.948717Z","shell.execute_reply.started":"2022-10-09T06:48:28.280508Z","shell.execute_reply":"2022-10-09T06:48:28.947783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select('ball_pos_x','ball_pos_y','ball_pos_z').show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:28.949672Z","iopub.execute_input":"2022-10-09T06:48:28.949981Z","iopub.status.idle":"2022-10-09T06:48:29.145700Z","shell.execute_reply.started":"2022-10-09T06:48:28.949952Z","shell.execute_reply":"2022-10-09T06:48:29.144767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.cache()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:29.146671Z","iopub.execute_input":"2022-10-09T06:48:29.146978Z","iopub.status.idle":"2022-10-09T06:48:29.196656Z","shell.execute_reply.started":"2022-10-09T06:48:29.146947Z","shell.execute_reply":"2022-10-09T06:48:29.195773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:41:43.922308Z","iopub.execute_input":"2022-10-09T06:41:43.922762Z","iopub.status.idle":"2022-10-09T06:41:43.927748Z","shell.execute_reply.started":"2022-10-09T06:41:43.922692Z","shell.execute_reply":"2022-10-09T06:41:43.926431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.printSchema()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:48:32.446072Z","iopub.execute_input":"2022-10-09T06:48:32.446410Z","iopub.status.idle":"2022-10-09T06:48:32.454125Z","shell.execute_reply.started":"2022-10-09T06:48:32.446385Z","shell.execute_reply":"2022-10-09T06:48:32.453079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:49:01.456518Z","iopub.execute_input":"2022-10-09T06:49:01.456872Z","iopub.status.idle":"2022-10-09T06:49:01.467071Z","shell.execute_reply.started":"2022-10-09T06:49:01.456847Z","shell.execute_reply":"2022-10-09T06:49:01.465721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe().show()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:15:03.190382Z","iopub.execute_input":"2022-10-09T07:15:03.191790Z","iopub.status.idle":"2022-10-09T07:16:06.492988Z","shell.execute_reply.started":"2022-10-09T07:15:03.191738Z","shell.execute_reply":"2022-10-09T07:16:06.492227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.na.drop(how = 'any')","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:54:08.869811Z","iopub.execute_input":"2022-10-09T06:54:08.870148Z","iopub.status.idle":"2022-10-09T06:54:08.909641Z","shell.execute_reply.started":"2022-10-09T06:54:08.870123Z","shell.execute_reply":"2022-10-09T06:54:08.908488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.count()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:54:10.920089Z","iopub.execute_input":"2022-10-09T06:54:10.920443Z","iopub.status.idle":"2022-10-09T06:54:15.568609Z","shell.execute_reply.started":"2022-10-09T06:54:10.920418Z","shell.execute_reply":"2022-10-09T06:54:15.567641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df = df.na.fill(value=,subset=)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:55:01.920221Z","iopub.execute_input":"2022-10-09T06:55:01.920571Z","iopub.status.idle":"2022-10-09T06:55:01.925438Z","shell.execute_reply.started":"2022-10-09T06:55:01.920529Z","shell.execute_reply":"2022-10-09T06:55:01.924267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.ml.feature import Imputer","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:55:47.800826Z","iopub.execute_input":"2022-10-09T06:55:47.801201Z","iopub.status.idle":"2022-10-09T06:55:47.806946Z","shell.execute_reply.started":"2022-10-09T06:55:47.801166Z","shell.execute_reply":"2022-10-09T06:55:47.805663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:56:29.299490Z","iopub.execute_input":"2022-10-09T06:56:29.299893Z","iopub.status.idle":"2022-10-09T06:56:29.311102Z","shell.execute_reply.started":"2022-10-09T06:56:29.299865Z","shell.execute_reply":"2022-10-09T06:56:29.309415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.filter('team_A_scoring_within_10sec=1').select('team_A_scoring_within_10sec','team_B_scoring_within_10sec').show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:00:10.380173Z","iopub.execute_input":"2022-10-09T07:00:10.380604Z","iopub.status.idle":"2022-10-09T07:00:10.707789Z","shell.execute_reply.started":"2022-10-09T07:00:10.380551Z","shell.execute_reply":"2022-10-09T07:00:10.706882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select('team_A_scoring_within_10sec').show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T06:58:12.810620Z","iopub.execute_input":"2022-10-09T06:58:12.810981Z","iopub.status.idle":"2022-10-09T06:58:13.192368Z","shell.execute_reply.started":"2022-10-09T06:58:12.810955Z","shell.execute_reply":"2022-10-09T06:58:13.191720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.where('team_B_scoring_within_10sec=1').select('team_A_scoring_within_10sec','team_B_scoring_within_10sec').show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:01:37.399799Z","iopub.execute_input":"2022-10-09T07:01:37.400145Z","iopub.status.idle":"2022-10-09T07:01:37.766082Z","shell.execute_reply.started":"2022-10-09T07:01:37.400120Z","shell.execute_reply":"2022-10-09T07:01:37.765089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby('team_A_scoring_within_10sec').mean().select('team_A_scoring_within_10sec','avg(ball_pos_x)', 'avg(ball_pos_y)', 'avg(ball_pos_z)').show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:09:25.367158Z","iopub.execute_input":"2022-10-09T07:09:25.367499Z","iopub.status.idle":"2022-10-09T07:09:29.939102Z","shell.execute_reply.started":"2022-10-09T07:09:25.367474Z","shell.execute_reply":"2022-10-09T07:09:29.937963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby('team_B_scoring_within_10sec').mean().select('team_B_scoring_within_10sec','avg(ball_pos_x)', 'avg(ball_pos_y)', 'avg(ball_pos_z)').show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T07:10:01.247095Z","iopub.execute_input":"2022-10-09T07:10:01.249079Z","iopub.status.idle":"2022-10-09T07:10:08.153817Z","shell.execute_reply.started":"2022-10-09T07:10:01.249022Z","shell.execute_reply":"2022-10-09T07:10:08.152797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}