{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-28T12:12:35.472743Z","iopub.execute_input":"2022-07-28T12:12:35.473313Z","iopub.status.idle":"2022-07-28T12:12:35.510439Z","shell.execute_reply.started":"2022-07-28T12:12:35.473204Z","shell.execute_reply":"2022-07-28T12:12:35.508656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyspark\n\n!pip install git+https://github.com/dllllb/pytorch-lifestream.git@main","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:13:25.265394Z","iopub.execute_input":"2022-07-28T12:13:25.266145Z","iopub.status.idle":"2022-07-28T12:14:18.656911Z","shell.execute_reply.started":"2022-07-28T12:13:25.266087Z","shell.execute_reply":"2022-07-28T12:14:18.655012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.sql import SparkSession\nfrom pyspark import SparkConf\nimport pyspark.sql.types as T\nimport pyspark.sql.functions as F","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:18.659365Z","iopub.execute_input":"2022-07-28T12:14:18.659959Z","iopub.status.idle":"2022-07-28T12:14:18.748403Z","shell.execute_reply.started":"2022-07-28T12:14:18.659903Z","shell.execute_reply":"2022-07-28T12:14:18.746769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark_conf = SparkConf()\nspark_conf.setMaster(\"local[*]\")\nspark_conf.set(\"spark.driver.memory\", \"12g\")\nspark_conf.set(\"spark.driver.memoryOverhead\", \"2g\")\nspark_conf.set(\"spark.local.dir\", \"/kaggle/temp/\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:18.751452Z","iopub.execute_input":"2022-07-28T12:14:18.752054Z","iopub.status.idle":"2022-07-28T12:14:18.762252Z","shell.execute_reply.started":"2022-07-28T12:14:18.752010Z","shell.execute_reply":"2022-07-28T12:14:18.761288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark = SparkSession.builder.config(conf=spark_conf).getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:18.763652Z","iopub.execute_input":"2022-07-28T12:14:18.763971Z","iopub.status.idle":"2022-07-28T12:14:24.577593Z","shell.execute_reply.started":"2022-07-28T12:14:18.763944Z","shell.execute_reply":"2022-07-28T12:14:24.576459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_data = spark.read.csv('../input/amex-default-prediction/train_data.csv', header=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:24.578951Z","iopub.execute_input":"2022-07-28T12:14:24.579333Z","iopub.status.idle":"2022-07-28T12:14:31.249515Z","shell.execute_reply.started":"2022-07-28T12:14:24.579297Z","shell.execute_reply":"2022-07-28T12:14:31.248278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLS_CATEGORY = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:31.250911Z","iopub.execute_input":"2022-07-28T12:14:31.251348Z","iopub.status.idle":"2022-07-28T12:14:31.258136Z","shell.execute_reply.started":"2022-07-28T12:14:31.251306Z","shell.execute_reply":"2022-07-28T12:14:31.256524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```\nStructField('customer_ID', StringType(), True)\nStructField('S_2', TimestampType(), True)\nStructField('D_63', StringType(), True)\nStructField('D_64', StringType(), True)\nStructField('B_31', IntegerType(), True)\n```","metadata":{}},{"cell_type":"code","source":"def col_type(col):\n    if col in ('customer_ID', 'D_63', 'D_64'):\n        return T.StringType()\n    if col == 'S_2':\n        return T.TimestampType()\n    return T.FloatType()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:31.259843Z","iopub.execute_input":"2022-07-28T12:14:31.261467Z","iopub.status.idle":"2022-07-28T12:14:31.271584Z","shell.execute_reply.started":"2022-07-28T12:14:31.261418Z","shell.execute_reply":"2022-07-28T12:14:31.270651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"schema = T.StructType([T.StructField(col, col_type(col)) for col in df_train_data.columns])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:31.275042Z","iopub.execute_input":"2022-07-28T12:14:31.275921Z","iopub.status.idle":"2022-07-28T12:14:31.371789Z","shell.execute_reply.started":"2022-07-28T12:14:31.275874Z","shell.execute_reply":"2022-07-28T12:14:31.370476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_data = spark.read.csv('../input/amex-default-prediction/train_data.csv', header=True, schema=schema)\ndf_test_data = spark.read.csv('../input/amex-default-prediction/test_data.csv', header=True, schema=schema)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:31.376013Z","iopub.execute_input":"2022-07-28T12:14:31.376413Z","iopub.status.idle":"2022-07-28T12:14:31.665331Z","shell.execute_reply.started":"2022-07-28T12:14:31.376372Z","shell.execute_reply":"2022-07-28T12:14:31.663868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = [c for c in df_train_data.columns if c not in COLS_CATEGORY + ['customer_ID', 'S_2']]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_target = spark.read.csv('../input/amex-default-prediction/train_labels.csv', header=True, inferSchema=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:31.666854Z","iopub.execute_input":"2022-07-28T12:14:31.667352Z","iopub.status.idle":"2022-07-28T12:14:33.666426Z","shell.execute_reply.started":"2022-07-28T12:14:31.667292Z","shell.execute_reply":"2022-07-28T12:14:33.665070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_target","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:33.667965Z","iopub.execute_input":"2022-07-28T12:14:33.668456Z","iopub.status.idle":"2022-07-28T12:14:33.689047Z","shell.execute_reply.started":"2022-07-28T12:14:33.668410Z","shell.execute_reply":"2022-07-28T12:14:33.687283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ptls.preprocessing import PysparkDataPreprocessor","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:33.692617Z","iopub.execute_input":"2022-07-28T12:14:33.693580Z","iopub.status.idle":"2022-07-28T12:14:35.959105Z","shell.execute_reply.started":"2022-07-28T12:14:33.693530Z","shell.execute_reply":"2022-07-28T12:14:35.957935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor = PysparkDataPreprocessor(\n    col_id='customer_ID',\n    col_event_time='S_2',\n    cols_category=COLS_CATEGORY,\n    cols_identity=num_cols,\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:35.960483Z","iopub.execute_input":"2022-07-28T12:14:35.961052Z","iopub.status.idle":"2022-07-28T12:14:35.972913Z","shell.execute_reply.started":"2022-07-28T12:14:35.961020Z","shell.execute_reply":"2022-07-28T12:14:35.972044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_train = preprocessor.fit_transform(df_train_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:14:35.974117Z","iopub.execute_input":"2022-07-28T12:14:35.974676Z","iopub.status.idle":"2022-07-28T12:29:04.908195Z","shell.execute_reply.started":"2022-07-28T12:14:35.974645Z","shell.execute_reply":"2022-07-28T12:29:04.905220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = data_train.withColumn('num_cols', F.array(*num_cols)).drop(*num_cols)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = data_train.join(df_target, on='customer_ID', how='inner')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:29:04.914639Z","iopub.execute_input":"2022-07-28T12:29:04.917367Z","iopub.status.idle":"2022-07-28T12:29:05.521853Z","shell.execute_reply.started":"2022-07-28T12:29:04.917288Z","shell.execute_reply":"2022-07-28T12:29:05.521005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_train.write.parquet('/kaggle/working/data_train.parquet', mode='overwrite')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:29:05.522948Z","iopub.execute_input":"2022-07-28T12:29:05.523299Z","iopub.status.idle":"2022-07-28T12:44:48.936599Z","shell.execute_reply.started":"2022-07-28T12:29:05.523270Z","shell.execute_reply":"2022-07-28T12:44:48.931830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_test = preprocessor.transform(df_test_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:44:48.945469Z","iopub.execute_input":"2022-07-28T12:44:48.949246Z","iopub.status.idle":"2022-07-28T12:45:34.895376Z","shell.execute_reply.started":"2022-07-28T12:44:48.949181Z","shell.execute_reply":"2022-07-28T12:45:34.893971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test = data_test.withColumn('num_cols', F.array(*num_cols)).drop(*num_cols)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_test.write.parquet('/kaggle/working/data_test.parquet', mode='overwrite')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T12:45:34.896742Z","iopub.execute_input":"2022-07-28T12:45:34.897128Z","iopub.status.idle":"2022-07-28T13:16:52.522501Z","shell.execute_reply.started":"2022-07-28T12:45:34.897089Z","shell.execute_reply":"2022-07-28T13:16:52.520577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh /kaggle/working/*.parquet","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:16:52.527211Z","iopub.execute_input":"2022-07-28T13:16:52.527955Z","iopub.status.idle":"2022-07-28T13:16:53.354779Z","shell.execute_reply.started":"2022-07-28T13:16:52.527895Z","shell.execute_reply":"2022-07-28T13:16:53.353110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor.get_category_dictionary_sizes()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:16:55.237091Z","iopub.execute_input":"2022-07-28T13:16:55.237617Z","iopub.status.idle":"2022-07-28T13:16:55.248327Z","shell.execute_reply.started":"2022-07-28T13:16:55.237568Z","shell.execute_reply":"2022-07-28T13:16:55.247047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\nwith open('/kaggle/working/preprocessor.pickle', 'wb') as f:\n    pickle.dump(preprocessor, f)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:16:55.676916Z","iopub.execute_input":"2022-07-28T13:16:55.677447Z","iopub.status.idle":"2022-07-28T13:16:55.686433Z","shell.execute_reply.started":"2022-07-28T13:16:55.677400Z","shell.execute_reply":"2022-07-28T13:16:55.684775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark.stop()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:59:27.246978Z","iopub.execute_input":"2022-07-28T13:59:27.247494Z","iopub.status.idle":"2022-07-28T13:59:27.767686Z","shell.execute_reply.started":"2022-07-28T13:59:27.247457Z","shell.execute_reply":"2022-07-28T13:59:27.766706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}