{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<center> <h1 style=\"background-color:seagreen; color:white\" >Load Data</h1> ","metadata":{}},{"cell_type":"code","source":"# Install PySpark\n!pip install pyspark > /dev/null","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-14T17:11:15.812659Z","iopub.execute_input":"2022-02-14T17:11:15.812980Z","iopub.status.idle":"2022-02-14T17:11:26.383932Z","shell.execute_reply.started":"2022-02-14T17:11:15.812948Z","shell.execute_reply":"2022-02-14T17:11:26.382642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Req libraries\nimport os\nimport numpy as np\nimport pandas as pd\n\nfrom pyspark.sql.types import *\nfrom pyspark.sql.functions import *\nfrom pyspark.sql.window import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-14T17:11:26.386172Z","iopub.execute_input":"2022-02-14T17:11:26.386457Z","iopub.status.idle":"2022-02-14T17:11:26.394076Z","shell.execute_reply.started":"2022-02-14T17:11:26.386418Z","shell.execute_reply":"2022-02-14T17:11:26.392988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Spark Session\nimport pyspark\nfrom pyspark.sql import SparkSession\n\nspark = SparkSession.builder.appName('fashion-recommendations').getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T17:11:26.395223Z","iopub.execute_input":"2022-02-14T17:11:26.395518Z","iopub.status.idle":"2022-02-14T17:11:26.409850Z","shell.execute_reply.started":"2022-02-14T17:11:26.395488Z","shell.execute_reply":"2022-02-14T17:11:26.408632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Data\ntransaction = spark.read.option('header','true').csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-14T17:11:26.412909Z","iopub.execute_input":"2022-02-14T17:11:26.413302Z","iopub.status.idle":"2022-02-14T17:11:26.674086Z","shell.execute_reply.started":"2022-02-14T17:11:26.413255Z","shell.execute_reply":"2022-02-14T17:11:26.673120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center> <h1 style=\"background-color:seagreen; color:white\" >Feature Engineering</h1> ","metadata":{}},{"cell_type":"code","source":"# filter only 2020\nsales =  transaction.withColumn('t_dat', transaction['t_dat'].cast('string'))\nsales = sales.withColumn('date', from_unixtime(unix_timestamp('t_dat', 'yyyy-MM-dd')))\nsales = sales.withColumn('year', year(col('date')))\nsales = sales.withColumn('month', month(col('date')))\n\n# Let's filter the data to start with\nsales = sales[sales['year'] == 2020]\nsales = sales[sales['month'] == 1]\n\ntransaction.unpersist()\n\n# Prepare the dataset\nsales = sales.groupby('customer_id', 'article_id').count()\nsales.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T17:11:48.443043Z","iopub.execute_input":"2022-02-14T17:11:48.443360Z","iopub.status.idle":"2022-02-14T17:11:48.515788Z","shell.execute_reply.started":"2022-02-14T17:11:48.443311Z","shell.execute_reply":"2022-02-14T17:11:48.515113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center> <h1 style=\"background-color:seagreen; color:white\" >Alternative Least Sqaure</h1>","metadata":{}},{"cell_type":"code","source":"from pyspark.ml.evaluation import RegressionEvaluator\nfrom pyspark.ml.recommendation import ALS","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:50:18.578852Z","iopub.execute_input":"2022-02-14T16:50:18.57913Z","iopub.status.idle":"2022-02-14T16:50:18.944831Z","shell.execute_reply.started":"2022-02-14T16:50:18.579093Z","shell.execute_reply":"2022-02-14T16:50:18.944123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center> <h1 style=\"background-color:DarkKhaki; color:white\" >Converting String to Index</h1>","metadata":{}},{"cell_type":"code","source":"from pyspark.ml.feature import StringIndexer\nfrom pyspark.ml import Pipeline\nfrom pyspark.sql.functions import col\n\nindexer = [StringIndexer(inputCol=column, outputCol=column+\"_index\") for column in list(set(sales.columns)-set(['count'])) ]\npipeline = Pipeline(stages=indexer)\ntransformed = pipeline.fit(sales).transform(sales)\ntransformed.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:50:18.946637Z","iopub.execute_input":"2022-02-14T16:50:18.946962Z","iopub.status.idle":"2022-02-14T16:54:47.4971Z","shell.execute_reply.started":"2022-02-14T16:50:18.946919Z","shell.execute_reply":"2022-02-14T16:54:47.496216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center> <h1 style=\"background-color:DarkKhaki; color:white\" >Creating ALS model and fitting data</h1>","metadata":{}},{"cell_type":"code","source":"(training,test)=transformed.randomSplit([0.8, 0.2])\n\nals=ALS(maxIter=5,regParam=0.09,rank=25,userCol=\"customer_id_index\",itemCol=\"article_id_index\",ratingCol=\"count\",coldStartStrategy=\"drop\",nonnegative=True)\nmodel=als.fit(training)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:59:30.967869Z","iopub.execute_input":"2022-02-14T16:59:30.968197Z","iopub.status.idle":"2022-02-14T17:03:48.992375Z","shell.execute_reply.started":"2022-02-14T16:59:30.968151Z","shell.execute_reply":"2022-02-14T17:03:48.991414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluator=RegressionEvaluator(metricName=\"rmse\",labelCol=\"count\",predictionCol=\"prediction\")\npredictions=model.transform(test)\nrmse=evaluator.evaluate(predictions)\nprint(\"RMSE=\"+str(rmse))\npredictions.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T17:03:48.997714Z","iopub.execute_input":"2022-02-14T17:03:48.998314Z","iopub.status.idle":"2022-02-14T17:06:36.878465Z","shell.execute_reply.started":"2022-02-14T17:03:48.998238Z","shell.execute_reply":"2022-02-14T17:06:36.877167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}