{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pyspark","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:51:38.157200Z","iopub.execute_input":"2022-03-30T01:51:38.157458Z","iopub.status.idle":"2022-03-30T01:52:19.213358Z","shell.execute_reply.started":"2022-03-30T01:51:38.157431Z","shell.execute_reply":"2022-03-30T01:52:19.212559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\narticles = pd.read_csv ('../input/h-and-m-personalized-fashion-recommendations/articles.csv')\ntransactions = pd.read_csv ('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\ncustomers = pd.read_csv ('../input/h-and-m-personalized-fashion-recommendations/customers.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:52:19.223878Z","iopub.execute_input":"2022-03-30T01:52:19.224655Z","iopub.status.idle":"2022-03-30T01:53:25.897625Z","shell.execute_reply.started":"2022-03-30T01:52:19.224615Z","shell.execute_reply":"2022-03-30T01:53:25.896841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')\nsample","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:25.899635Z","iopub.execute_input":"2022-03-30T01:53:25.899894Z","iopub.status.idle":"2022-03-30T01:53:30.456917Z","shell.execute_reply.started":"2022-03-30T01:53:25.899860Z","shell.execute_reply":"2022-03-30T01:53:30.456240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:30.460795Z","iopub.execute_input":"2022-03-30T01:53:30.462664Z","iopub.status.idle":"2022-03-30T01:53:30.491217Z","shell.execute_reply.started":"2022-03-30T01:53:30.462625Z","shell.execute_reply":"2022-03-30T01:53:30.490615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('min date: {}, max date: {}'.format(transactions.t_dat.min(), transactions.t_dat.max()))","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:30.494826Z","iopub.execute_input":"2022-03-30T01:53:30.496718Z","iopub.status.idle":"2022-03-30T01:53:38.417858Z","shell.execute_reply.started":"2022-03-30T01:53:30.496680Z","shell.execute_reply":"2022-03-30T01:53:38.416877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#select one day and only online channel\nts_1d = transactions.loc[((transactions['t_dat']=='2020-09-21') & (transactions['sales_channel_id']==2))]\nts_1d","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:38.418982Z","iopub.execute_input":"2022-03-30T01:53:38.419401Z","iopub.status.idle":"2022-03-30T01:53:42.836475Z","shell.execute_reply.started":"2022-03-30T01:53:38.419364Z","shell.execute_reply":"2022-03-30T01:53:42.835821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:42.837572Z","iopub.execute_input":"2022-03-30T01:53:42.838745Z","iopub.status.idle":"2022-03-30T01:53:43.373906Z","shell.execute_reply.started":"2022-03-30T01:53:42.838703Z","shell.execute_reply":"2022-03-30T01:53:43.372444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation","metadata":{}},{"cell_type":"markdown","source":"### create dataframe from Spark","metadata":{}},{"cell_type":"code","source":"from pyspark.sql import SparkSession\n#Create PySpark SparkSession\nspark = SparkSession.builder \\\n    .master(\"local[1]\") \\\n    .appName(\"Recommendation\") \\\n    .getOrCreate()\n#Create PySpark DataFrame from Pandas\ndf=spark.createDataFrame(ts_1d) \ndf.printSchema()\ndf.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:43.375353Z","iopub.execute_input":"2022-03-30T01:53:43.375626Z","iopub.status.idle":"2022-03-30T01:53:56.397212Z","shell.execute_reply.started":"2022-03-30T01:53:43.375576Z","shell.execute_reply":"2022-03-30T01:53:56.394707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df.groupBy('customer_id').count()\ndf1.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:56.401263Z","iopub.execute_input":"2022-03-30T01:53:56.402048Z","iopub.status.idle":"2022-03-30T01:53:58.104455Z","shell.execute_reply.started":"2022-03-30T01:53:56.402007Z","shell.execute_reply":"2022-03-30T01:53:58.103712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = df.groupby('customer_id','article_id').count()\ndf2.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:58.105429Z","iopub.execute_input":"2022-03-30T01:53:58.106967Z","iopub.status.idle":"2022-03-30T01:53:59.008881Z","shell.execute_reply.started":"2022-03-30T01:53:58.106928Z","shell.execute_reply":"2022-03-30T01:53:59.008101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### convert string id to interger starting from number 0","metadata":{}},{"cell_type":"code","source":"from pyspark.ml.feature import StringIndexer\n\ndef str2int(df,col_name):\n    for col in col_name:\n        indexer = StringIndexer(inputCol=col, outputCol=col+\"_index\")\n        model = indexer.fit(df)\n        df = model.transform(df)\n    return df\n\nstr2int_col = ['customer_id','article_id']\ndf_idx = str2int(df2,str2int_col)\ndf_idx.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:53:59.010263Z","iopub.execute_input":"2022-03-30T01:53:59.011562Z","iopub.status.idle":"2022-03-30T01:54:03.749227Z","shell.execute_reply.started":"2022-03-30T01:53:59.011525Z","shell.execute_reply":"2022-03-30T01:54:03.748526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### split the modelling dataset into training and testing sets ","metadata":{}},{"cell_type":"code","source":"(training,test)=df_idx.randomSplit([0.8, 0.2])","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:54:03.750218Z","iopub.execute_input":"2022-03-30T01:54:03.750452Z","iopub.status.idle":"2022-03-30T01:54:03.787259Z","shell.execute_reply.started":"2022-03-30T01:54:03.750417Z","shell.execute_reply":"2022-03-30T01:54:03.786503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Modelling using ALS","metadata":{}},{"cell_type":"code","source":"from pyspark.ml.evaluation import RegressionEvaluator\nfrom pyspark.ml.recommendation import ALS\nfrom pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n\n\n#create ALS model\nals=ALS(userCol=\"customer_id_index\",itemCol=\"article_id_index\",ratingCol=\"count\",coldStartStrategy=\"drop\",nonnegative=True)\n\n#tune model using ParamGridBuilder\nparam_grid = ParamGridBuilder()\\\n            .addGrid(als.rank, [15,20,25])\\\n            .addGrid(als.maxIter,[5,10,15])\\\n            .addGrid(als.regParam,[0.09,0.14,0.19])\\\n            .build()\n\n#define evaluator as RMSE\nevaluator = RegressionEvaluator(metricName = \"rmse\",labelCol = 'count', predictionCol = 'prediction')\n\n#Build cross validation using CrossValidator\ncv = CrossValidator(estimator=als,estimatorParamMaps=param_grid, evaluator=evaluator,numFolds=3)\n\n\n#Fit ALS model to training data\nmodel = cv.fit(training)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-30T01:54:27.670085Z","iopub.execute_input":"2022-03-30T01:54:27.670398Z","iopub.status.idle":"2022-03-30T02:34:28.694728Z","shell.execute_reply.started":"2022-03-30T01:54:27.670359Z","shell.execute_reply":"2022-03-30T02:34:28.693926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Extract best model from the tuning exercise using ParamGridBuilder\nbest_model = model.bestModel\n\n#Generate predictions and evaluate using RMSE\npredictions = best_model.transform(test)\nrmse = evaluator.evaluate(predictions)\n\n#print evaluation metrics and model parameters\nprint(\"RMSE =\" + str(rmse))\nprint(\"**Best Model**\")\nprint(\"Rank : {}\".format(best_model.rank))\nprint(\"MaxIter: {}\".format(best_model._java_obj.parent().getMaxIter()))\nprint(\"RegParam: {}\".format(best_model._java_obj.parent().getRegParam()))","metadata":{"execution":{"iopub.status.busy":"2022-03-30T03:08:44.528992Z","iopub.execute_input":"2022-03-30T03:08:44.529254Z","iopub.status.idle":"2022-03-30T03:08:46.400129Z","shell.execute_reply.started":"2022-03-30T03:08:44.529226Z","shell.execute_reply":"2022-03-30T03:08:46.399323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T02:37:01.747260Z","iopub.execute_input":"2022-03-30T02:37:01.747653Z","iopub.status.idle":"2022-03-30T02:37:03.845356Z","shell.execute_reply.started":"2022-03-30T02:37:01.747572Z","shell.execute_reply":"2022-03-30T02:37:03.844666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_recom = best_model.recommendForAllUsers(10)\ndf_recom.show(10)","metadata":{"execution":{"iopub.status.busy":"2022-03-30T03:47:18.107102Z","iopub.execute_input":"2022-03-30T03:47:18.107402Z","iopub.status.idle":"2022-03-30T03:47:30.594673Z","shell.execute_reply.started":"2022-03-30T03:47:18.107368Z","shell.execute_reply":"2022-03-30T03:47:30.587635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_recom = df_recom.select(\"customer_id_index\",\"recommendations.article_id_index\")\ndf_recom.show(10)\ndf_recom = df_recom.toPandas()","metadata":{"execution":{"iopub.status.busy":"2022-03-30T03:47:30.595982Z","iopub.execute_input":"2022-03-30T03:47:30.596206Z","iopub.status.idle":"2022-03-30T03:47:52.116962Z","shell.execute_reply.started":"2022-03-30T03:47:30.596175Z","shell.execute_reply":"2022-03-30T03:47:52.116170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_recom.sort_values('customer_id_index')","metadata":{"execution":{"iopub.status.busy":"2022-03-30T03:48:59.930003Z","iopub.execute_input":"2022-03-30T03:48:59.930548Z","iopub.status.idle":"2022-03-30T03:48:59.947198Z","shell.execute_reply.started":"2022-03-30T03:48:59.930510Z","shell.execute_reply":"2022-03-30T03:48:59.946521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"md=df_idx.select(df_idx['article_id'],df_idx['article_id_index'],df_idx['customer_id'],df_idx['customer_id_index'])\nmd=md.toPandas()\nmd","metadata":{"execution":{"iopub.status.busy":"2022-03-30T03:14:10.049333Z","iopub.execute_input":"2022-03-30T03:14:10.049720Z","iopub.status.idle":"2022-03-30T03:14:11.395130Z","shell.execute_reply.started":"2022-03-30T03:14:10.049682Z","shell.execute_reply":"2022-03-30T03:14:11.394428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict1 =dict(zip(md['article_id_index'],md['article_id']))\ndict2=dict(zip(md['customer_id_index'],md['customer_id']))\ndf_recom['article_id'] = df_recom['article_id_index'].map(lambda x: [dict1[y] for y in x if y in dict1])\ndf_recom['customer_id']=df_recom['customer_id_index'].map(dict2)\ndf_recom","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:00:03.604741Z","iopub.execute_input":"2022-03-30T04:00:03.605451Z","iopub.status.idle":"2022-03-30T04:00:03.666906Z","shell.execute_reply.started":"2022-03-30T04:00:03.605395Z","shell.execute_reply":"2022-03-30T04:00:03.666212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recom_final = df_recom.drop(['customer_id_index','article_id_index'], axis = 1)\nrecom_final","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:13:09.514853Z","iopub.execute_input":"2022-03-30T04:13:09.515130Z","iopub.status.idle":"2022-03-30T04:13:09.533409Z","shell.execute_reply.started":"2022-03-30T04:13:09.515095Z","shell.execute_reply":"2022-03-30T04:13:09.532546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\ndef plot_img(prev_items):\n    fig = plt.figure(figsize=(20, 10))\n    for item, i in zip(prev_items, range(1, len(prev_items)+1)):\n        item = '0' + str(item)\n        sub = item[:3]\n        image = path + \"/\"+ sub + \"/\"+ item +\".jpg\"\n        image = plt.imread(image)\n        fig.add_subplot(2, 5, i)\n        plt.imshow(image)\n        if i == 10:\n            break","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:39:13.975722Z","iopub.execute_input":"2022-03-30T04:39:13.976357Z","iopub.status.idle":"2022-03-30T04:39:13.984120Z","shell.execute_reply.started":"2022-03-30T04:39:13.976318Z","shell.execute_reply":"2022-03-30T04:39:13.983434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/h-and-m-personalized-fashion-recommendations/images\"\ncustomer1 = recom_final.loc[10,:]\ncustomer_id = customer1[0]\npredict_articles = customer1[1]\nactual_articles = ts_1d.loc[ts_1d['customer_id']==customer_id].article_id.tolist()\n\nprint('customer id: {}'.format(customer1[0]))\nprint('actual articles')\nplot_img(actual_articles)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:39:40.415330Z","iopub.execute_input":"2022-03-30T04:39:40.415614Z","iopub.status.idle":"2022-03-30T04:39:44.829050Z","shell.execute_reply.started":"2022-03-30T04:39:40.415568Z","shell.execute_reply":"2022-03-30T04:39:44.825873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('predicted article')\nplot_img(predict_articles)","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:39:44.830627Z","iopub.execute_input":"2022-03-30T04:39:44.831106Z","iopub.status.idle":"2022-03-30T04:39:48.645879Z","shell.execute_reply.started":"2022-03-30T04:39:44.831066Z","shell.execute_reply":"2022-03-30T04:39:48.645237Z"},"trusted":true},"execution_count":null,"outputs":[]}]}