{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Thank you for visit\n\n- We have a loooot of data in this competition.\n- However, we have limited RAM.\n- We can save memory with PySpark in EDA.\n- So, I'll show some simple usage with PySpark.\n\n### If you like this notebook, upvote please 😉","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/ppsheth91/introductory-tutorial-1-pyspark-for-beginners","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyspark > /dev/null","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyspark\nfrom pyspark.sql import SparkSession\n\n\nspark = SparkSession.builder.appName('h-and-m-personalized-fashion-recommendations').getOrCreate()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    TRANSACTION_PATH = '../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv'\n    ARTICLE_PATH = '../input/h-and-m-personalized-fashion-recommendations/articles.csv'\n    CUSTOMER_PATH = '../input/h-and-m-personalized-fashion-recommendations/customers.csv'\n    SAMPLE_PATH = '../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv'\n    IMAGE_PATH = '../input/h-and-m-personalized-fashion-recommendations/images'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Data","metadata":{}},{"cell_type":"code","source":"train = spark.read.option('header','true').csv(CFG.TRANSACTION_PATH)\n\nprint(train.count())\ntrain.show(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article = spark.read.option('header','true').csv(CFG.ARTICLE_PATH)\n\nprint(article.count())\narticle.show(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer = spark.read.option('header','true').csv(CFG.CUSTOMER_PATH)\n\nprint(customer.count())\ncustomer.show(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Register Tables","metadata":{}},{"cell_type":"code","source":"train.createOrReplaceTempView('transactions_train')\narticle.createOrReplaceTempView('articles')\ncustomer.createOrReplaceTempView('customers')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA with Spark SQL","metadata":{}},{"cell_type":"code","source":"# simple read data\n\nsample = spark.sql('''\nselect\n  * \nfrom transactions_train \nlimit 100\n''')\n\nsample.show(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group and get stats\n\ngrouped = spark.sql('''\nselect\n  customer_id\n  , collect_list(article_id) as article_id_list\n  , avg(price) as avg_price\n  , max(price) as max_price\n  , min(price) as min_price\nfrom transactions_train \ngroup by customer_id\nlimit 100\n''')\n\n# to pandas dataframe\ndf_grouped = grouped.toPandas()\n\nprint(df_grouped.shape)\ndf_grouped.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# how to join\n\nmerged = spark.sql('''\nselect\n  * \nfrom transactions_train as tt\nleft join customers as cs on cs.customer_id = tt.customer_id\nleft join articles as ar on ar.article_id = tt.article_id\nlimit 100\n''')\n\nprint(merged.count())\nprint(merged.columns)\n\n\ndf_merged = merged.toPandas()\n\nprint(df_merged.shape)\ndf_merged.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}