{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install pyspark","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-10-15T11:36:32.949483Z","iopub.execute_input":"2022-10-15T11:36:32.950063Z","iopub.status.idle":"2022-10-15T11:37:35.591119Z","shell.execute_reply.started":"2022-10-15T11:36:32.949922Z","shell.execute_reply":"2022-10-15T11:37:35.589949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install comet_ml -qqq","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:37:35.593680Z","iopub.execute_input":"2022-10-15T11:37:35.594106Z","iopub.status.idle":"2022-10-15T11:37:53.945259Z","shell.execute_reply.started":"2022-10-15T11:37:35.594051Z","shell.execute_reply":"2022-10-15T11:37:53.943960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import comet_ml\n\nexperiment = comet_ml.Experiment(\n    api_key=\"YOUR_API_KEY\",\n     project_name=\"spark\"\n \n)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:22.715071Z","iopub.execute_input":"2022-10-15T12:14:22.715436Z","iopub.status.idle":"2022-10-15T12:14:27.560674Z","shell.execute_reply.started":"2022-10-15T12:14:22.715401Z","shell.execute_reply":"2022-10-15T12:14:27.559053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pyspark.ml.evaluation import RegressionEvaluator\nfrom pyspark.ml.recommendation import ALS\nfrom pyspark.sql import SparkSession\nfrom sklearn import preprocessing\n\n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-15T11:37:57.578579Z","iopub.execute_input":"2022-10-15T11:37:57.579008Z","iopub.status.idle":"2022-10-15T11:37:58.980346Z","shell.execute_reply.started":"2022-10-15T11:37:57.578966Z","shell.execute_reply":"2022-10-15T11:37:58.978998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:37:58.982194Z","iopub.execute_input":"2022-10-15T11:37:58.982510Z","iopub.status.idle":"2022-10-15T11:38:00.276378Z","shell.execute_reply.started":"2022-10-15T11:37:58.982474Z","shell.execute_reply":"2022-10-15T11:38:00.275272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:38:00.277893Z","iopub.execute_input":"2022-10-15T11:38:00.278200Z","iopub.status.idle":"2022-10-15T11:38:00.326316Z","shell.execute_reply.started":"2022-10-15T11:38:00.278166Z","shell.execute_reply":"2022-10-15T11:38:00.325522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntransactions_train = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:38:00.327625Z","iopub.execute_input":"2022-10-15T11:38:00.328583Z","iopub.status.idle":"2022-10-15T11:39:17.088243Z","shell.execute_reply.started":"2022-10-15T11:38:00.328536Z","shell.execute_reply":"2022-10-15T11:39:17.087001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:39:17.089866Z","iopub.execute_input":"2022-10-15T11:39:17.090177Z","iopub.status.idle":"2022-10-15T11:39:17.102487Z","shell.execute_reply.started":"2022-10-15T11:39:17.090142Z","shell.execute_reply":"2022-10-15T11:39:17.101255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dff = articles.merge(transactions_train,on=\"article_id\")","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:39:17.104050Z","iopub.execute_input":"2022-10-15T11:39:17.104344Z","iopub.status.idle":"2022-10-15T11:41:38.386453Z","shell.execute_reply.started":"2022-10-15T11:39:17.104313Z","shell.execute_reply":"2022-10-15T11:41:38.384475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dff.tail(1)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:38.392778Z","iopub.execute_input":"2022-10-15T11:41:38.393204Z","iopub.status.idle":"2022-10-15T11:41:38.426951Z","shell.execute_reply.started":"2022-10-15T11:41:38.393160Z","shell.execute_reply":"2022-10-15T11:41:38.425965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# these will all get logged\nparams = {\n    \"appname\": \"fashion\",\n    \"master\": \"local[1]\",\n    \"sample_size\": 100,\n    \"training_size\": 0.7,\n    \"test_size\":0.3,\n    \"random_state\": 0,\n    \"maxIter\": 5,\n    \"regParam\": 0.01,\n    \"userCol\":\"customer_id\",\n    \"itemCol\": \"article_id\",\n    \"ratingCol\":\"price\",\n    \"coldStartStrategy\": \"drop\",\n    \"metricName\":\"rmse\",\n    \"predictionCol\": \"prediction\"\n}\n\nexperiment.log_parameters(params)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:48.545231Z","iopub.execute_input":"2022-10-15T12:14:48.545686Z","iopub.status.idle":"2022-10-15T12:14:48.557697Z","shell.execute_reply.started":"2022-10-15T12:14:48.545639Z","shell.execute_reply":"2022-10-15T12:14:48.556415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create PySpark SparkSession\nspark = SparkSession.builder \\\n    .master(params['master']) \\\n    .appName(params['appname']) \\\n    .getOrCreate()\n","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:38.439068Z","iopub.execute_input":"2022-10-15T11:41:38.439344Z","iopub.status.idle":"2022-10-15T11:41:46.416483Z","shell.execute_reply.started":"2022-10-15T11:41:38.439289Z","shell.execute_reply":"2022-10-15T11:41:46.415668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taking a few records due to Kaggle memory contraints \ndata = dff.head(params['sample_size'])[['customer_id','article_id','price']]","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:46.417932Z","iopub.execute_input":"2022-10-15T11:41:46.418182Z","iopub.status.idle":"2022-10-15T11:41:46.430590Z","shell.execute_reply.started":"2022-10-15T11:41:46.418152Z","shell.execute_reply":"2022-10-15T11:41:46.429944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = preprocessing.LabelEncoder()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:46.432053Z","iopub.execute_input":"2022-10-15T11:41:46.432510Z","iopub.status.idle":"2022-10-15T11:41:46.441675Z","shell.execute_reply.started":"2022-10-15T11:41:46.432477Z","shell.execute_reply":"2022-10-15T11:41:46.440865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['customer_id']= le.fit_transform(data['customer_id'])","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:46.443764Z","iopub.execute_input":"2022-10-15T11:41:46.444210Z","iopub.status.idle":"2022-10-15T11:41:46.461273Z","shell.execute_reply.started":"2022-10-15T11:41:46.444174Z","shell.execute_reply":"2022-10-15T11:41:46.460323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.sample()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:46.462986Z","iopub.execute_input":"2022-10-15T11:41:46.463416Z","iopub.status.idle":"2022-10-15T11:41:46.482000Z","shell.execute_reply.started":"2022-10-15T11:41:46.463371Z","shell.execute_reply":"2022-10-15T11:41:46.480968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create PySpark DataFrame from Pandas\nsparkDF=spark.createDataFrame(data) ","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:46.484107Z","iopub.execute_input":"2022-10-15T11:41:46.484865Z","iopub.status.idle":"2022-10-15T11:41:50.573589Z","shell.execute_reply.started":"2022-10-15T11:41:46.484819Z","shell.execute_reply":"2022-10-15T11:41:50.572581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sparkDF","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:50.574923Z","iopub.execute_input":"2022-10-15T11:41:50.575543Z","iopub.status.idle":"2022-10-15T11:41:50.587644Z","shell.execute_reply.started":"2022-10-15T11:41:50.575495Z","shell.execute_reply":"2022-10-15T11:41:50.586504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sparkDF.printSchema()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:50.589223Z","iopub.execute_input":"2022-10-15T11:41:50.589972Z","iopub.status.idle":"2022-10-15T11:41:50.638929Z","shell.execute_reply.started":"2022-10-15T11:41:50.589916Z","shell.execute_reply":"2022-10-15T11:41:50.638003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training, testing = sparkDF.randomSplit([params['training_size'], params['test_size']], seed=params['random_state'])","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:50.640250Z","iopub.execute_input":"2022-10-15T11:41:50.640554Z","iopub.status.idle":"2022-10-15T11:41:50.703213Z","shell.execute_reply.started":"2022-10-15T11:41:50.640511Z","shell.execute_reply":"2022-10-15T11:41:50.702489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T11:41:50.704170Z","iopub.execute_input":"2022-10-15T11:41:50.704406Z","iopub.status.idle":"2022-10-15T11:41:53.849503Z","shell.execute_reply.started":"2022-10-15T11:41:50.704376Z","shell.execute_reply":"2022-10-15T11:41:53.848690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://spark.apache.org/docs/3.0.0/ml-collaborative-filtering.html\n# According to https://spark.apache.org/docs/2.2.0/api/python/pyspark.ml.html#pyspark.ml.recommendation.ALS\n# ALS attempts to estimate the ratings matrix R as the product of two lower-rank matrices, X and Y, i.e. X * Yt = R.\n# Typically these approximations are called ‘factor’ matrices. The general approach is iterative. During each iteration,\n# one of the factor matrices is held constant, while the other is solved for using least squares. The newly-solved factor\n# matrix is then held constant while solving for the other factor matrix.\nals = ALS(maxIter=params['maxIter'], regParam=params['regParam'], userCol=params['userCol'], itemCol=params['itemCol'], ratingCol=params['ratingCol'],\n          coldStartStrategy=params['coldStartStrategy']  )\nmodel = als.fit(training)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:55.172848Z","iopub.execute_input":"2022-10-15T12:14:55.173194Z","iopub.status.idle":"2022-10-15T12:14:57.661528Z","shell.execute_reply.started":"2022-10-15T12:14:55.173162Z","shell.execute_reply":"2022-10-15T12:14:57.660405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model by computing the RMSE on the test data\npredictions = model.transform(testing)\nevaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=params['ratingCol'], predictionCol=params['predictionCol'])\nrmse = evaluator.evaluate(predictions)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:57.665140Z","iopub.execute_input":"2022-10-15T12:14:57.665935Z","iopub.status.idle":"2022-10-15T12:14:58.361208Z","shell.execute_reply.started":"2022-10-15T12:14:57.665859Z","shell.execute_reply":"2022-10-15T12:14:58.360191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment.log_metric(\"rmse\",rmse)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:58.362592Z","iopub.execute_input":"2022-10-15T12:14:58.362933Z","iopub.status.idle":"2022-10-15T12:14:58.369668Z","shell.execute_reply.started":"2022-10-15T12:14:58.362870Z","shell.execute_reply":"2022-10-15T12:14:58.368614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.show(3)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:58.373801Z","iopub.execute_input":"2022-10-15T12:14:58.374496Z","iopub.status.idle":"2022-10-15T12:14:58.998347Z","shell.execute_reply.started":"2022-10-15T12:14:58.374444Z","shell.execute_reply":"2022-10-15T12:14:58.997285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmse","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:58.999639Z","iopub.execute_input":"2022-10-15T12:14:58.999971Z","iopub.status.idle":"2022-10-15T12:14:59.018279Z","shell.execute_reply.started":"2022-10-15T12:14:58.999929Z","shell.execute_reply":"2022-10-15T12:14:59.016952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate top 10 article recommendations for each user\nuserRecs = model.recommendForAllUsers(10)\n# Generate top 10 user recommendations for each article\narticleRecs = model.recommendForAllItems(10)\nuserRecs.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:14:59.021211Z","iopub.execute_input":"2022-10-15T12:14:59.029057Z","iopub.status.idle":"2022-10-15T12:15:01.136991Z","shell.execute_reply.started":"2022-10-15T12:14:59.028830Z","shell.execute_reply":"2022-10-15T12:15:01.135966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment.log_dataframe_profile(userRecs.toPandas())\nexperiment.log_dataframe_profile(articleRecs.toPandas())","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:15:01.138348Z","iopub.execute_input":"2022-10-15T12:15:01.139233Z","iopub.status.idle":"2022-10-15T12:15:11.291732Z","shell.execute_reply.started":"2022-10-15T12:15:01.139177Z","shell.execute_reply":"2022-10-15T12:15:11.290700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articleRecs.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:15:11.297995Z","iopub.execute_input":"2022-10-15T12:15:11.298414Z","iopub.status.idle":"2022-10-15T12:15:13.181935Z","shell.execute_reply.started":"2022-10-15T12:15:11.298361Z","shell.execute_reply":"2022-10-15T12:15:13.181121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate top 5 article recommendations for a specified set of users\nusers = sparkDF.select(als.getUserCol()).distinct().limit(3)\nuserSubsetRecs = model.recommendForUserSubset(users, 5)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:15:13.182996Z","iopub.execute_input":"2022-10-15T12:15:13.183722Z","iopub.status.idle":"2022-10-15T12:15:13.345042Z","shell.execute_reply.started":"2022-10-15T12:15:13.183646Z","shell.execute_reply":"2022-10-15T12:15:13.344240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"userSubsetRecs.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:15:13.347093Z","iopub.execute_input":"2022-10-15T12:15:13.348684Z","iopub.status.idle":"2022-10-15T12:15:15.398149Z","shell.execute_reply.started":"2022-10-15T12:15:13.348638Z","shell.execute_reply":"2022-10-15T12:15:15.397256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate top 10 user recommendations for a specified set of articles\narticles = sparkDF.select(als.getItemCol()).distinct().limit(3)\narticleSubSetRecs = model.recommendForItemSubset(articles, 10)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:15:15.399559Z","iopub.execute_input":"2022-10-15T12:15:15.399936Z","iopub.status.idle":"2022-10-15T12:15:15.551284Z","shell.execute_reply.started":"2022-10-15T12:15:15.399894Z","shell.execute_reply":"2022-10-15T12:15:15.550333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articleSubSetRecs.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:15:15.552723Z","iopub.execute_input":"2022-10-15T12:15:15.553475Z","iopub.status.idle":"2022-10-15T12:15:17.610221Z","shell.execute_reply.started":"2022-10-15T12:15:15.553411Z","shell.execute_reply":"2022-10-15T12:15:17.609458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment.end()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T12:15:17.611718Z","iopub.execute_input":"2022-10-15T12:15:17.612993Z","iopub.status.idle":"2022-10-15T12:15:19.808898Z","shell.execute_reply.started":"2022-10-15T12:15:17.612941Z","shell.execute_reply":"2022-10-15T12:15:19.808090Z"},"trusted":true},"execution_count":null,"outputs":[]}]}