{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-22T07:41:23.561556Z","iopub.execute_input":"2022-04-22T07:41:23.561926Z","iopub.status.idle":"2022-04-22T07:41:23.567908Z","shell.execute_reply.started":"2022-04-22T07:41:23.561892Z","shell.execute_reply":"2022-04-22T07:41:23.567001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# About this pipeline\nThe idea is simply to collect the most sold items in the last two weeks of observation, grouped by age range of customers.\n\nIn order to do this I will create a \"age_range\" feature on customers table. Then I will use the transactions table, limited to the two last weeks, to rank the most sold items (grouped by age range).\n\nLastly the recommendations are collected in a list for each age_range and joined to the application table.\n\nFor a more complex solution, please feel free to check my [LightGBM.Ranker model proposal](https://www.kaggle.com/code/lorenzopagliaro01/h-m-ranker-pyspark-lgbmranker).","metadata":{}},{"cell_type":"code","source":"!pip install pyspark -q\nimport pyspark\nfrom pyspark.sql import SparkSession\nfrom pyspark.sql import SQLContext\nfrom pyspark.sql import functions as F\nfrom pyspark.sql import Window\nfrom pyspark.sql.types import StructType,StructField, StringType, IntegerType, ArrayType, DoubleType, BooleanType\n\nsc = SparkSession.builder.appName(\"Recommendations\").config(\"spark.sql.files.maxPartitionBytes\", 5000000).getOrCreate()\nspark = SparkSession(sc)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:41:23.652969Z","iopub.execute_input":"2022-04-22T07:41:23.653296Z","iopub.status.idle":"2022-04-22T07:41:36.615491Z","shell.execute_reply.started":"2022-04-22T07:41:23.653259Z","shell.execute_reply":"2022-04-22T07:41:36.614078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = spark.read.option(\"header\",True) \\\n                .csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomers = spark.read.option(\"header\",True) \\\n                .csv(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")\ntransactions = spark.read.option(\"header\",True) \\\n                .csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:41:36.618105Z","iopub.execute_input":"2022-04-22T07:41:36.618491Z","iopub.status.idle":"2022-04-22T07:41:37.076178Z","shell.execute_reply.started":"2022-04-22T07:41:36.618441Z","shell.execute_reply":"2022-04-22T07:41:37.075226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add age_range feature to customers","metadata":{}},{"cell_type":"code","source":"customers = customers\\\n    .fillna({'age': '27'})\\\n    .withColumn('age_range', \n                 F.when(F.col('age') < 20, 'under_20')\\\n                  .when((F.col('age') >= 20) & (F.col('age') <= 25), '20_25')\\\n                  .when((F.col('age') >= 26) & (F.col('age') <= 30), '26_30')\\\n                  .when((F.col('age') >= 31) & (F.col('age') <= 35), '31_35')\\\n                  .when((F.col('age') >= 36) & (F.col('age') <= 40), '36_40')\\\n                  .when((F.col('age') >= 41) & (F.col('age') <= 45), '41_45')\\\n                  .when((F.col('age') >= 46) & (F.col('age') <= 50), '46_50')\\\n                  .when((F.col('age') >= 51) & (F.col('age') <= 55), '51_55')\\\n                  .when((F.col('age') >= 56) & (F.col('age') <= 60), '56_60')\\\n                  .when((F.col('age') >= 61) & (F.col('age') <= 65), '61_65')\\\n                  .when((F.col('age') >= 66) & (F.col('age') <= 70), '66_70')\\\n                  .otherwise('over_70'))\\\n    .drop('age','FN', 'Active', 'club_member_status', 'fashion_news_frequency', 'postal_code')\n\ncustomers.show(5)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:41:37.077316Z","iopub.execute_input":"2022-04-22T07:41:37.077546Z","iopub.status.idle":"2022-04-22T07:41:37.294503Z","shell.execute_reply.started":"2022-04-22T07:41:37.077517Z","shell.execute_reply":"2022-04-22T07:41:37.293673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Limit data to only last 2 weeks","metadata":{}},{"cell_type":"code","source":"transactions = transactions\\\n    .withColumn('week1', F.date_trunc('week', transactions.t_dat))\\\n    .withColumn('week', F.to_date('week1', 'yyyy-MM-dd'))\\\n    .drop('week1')\\\n    .filter(F.col('week').isin(['2020-09-21', '2020-09-14']))\\\n    .withColumn('article_id_int', transactions['article_id'].cast(IntegerType()))\\\n    .drop('price', 'sales_channel_id', 'week', 't_dat')\\\n    .join(customers, 'customer_id', 'left')\n\ntransactions.show(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:41:37.296415Z","iopub.execute_input":"2022-04-22T07:41:37.296737Z","iopub.status.idle":"2022-04-22T07:42:30.11742Z","shell.execute_reply.started":"2022-04-22T07:41:37.296688Z","shell.execute_reply":"2022-04-22T07:42:30.11597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Articles rank section - top items 12 for last 2 weeks, for each age range","metadata":{}},{"cell_type":"code","source":"articles_orders = transactions\\\n    .groupBy('article_id', 'age_range').count().orderBy('count', ascending=False)\\\n    .withColumnRenamed('count', 'articles_order_count')\n\narticles_orders.show(50)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:42:30.118791Z","iopub.execute_input":"2022-04-22T07:42:30.119099Z","iopub.status.idle":"2022-04-22T07:43:18.569557Z","shell.execute_reply.started":"2022-04-22T07:42:30.119053Z","shell.execute_reply":"2022-04-22T07:43:18.568662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Keep only top 12 sold items in last 2 weeks","metadata":{}},{"cell_type":"code","source":"w_articles = Window.partitionBy(articles_orders.age_range).orderBy(articles_orders.articles_order_count.desc())\n\narticles_orders = articles_orders\\\n    .withColumn('rn', F.row_number().over(w_articles))\\\n    .filter(F.col('rn') <= 12)\n\narticles_orders.show(24)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:43:18.570918Z","iopub.execute_input":"2022-04-22T07:43:18.571242Z","iopub.status.idle":"2022-04-22T07:44:20.007039Z","shell.execute_reply.started":"2022-04-22T07:43:18.571202Z","shell.execute_reply":"2022-04-22T07:44:20.005692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a list column, for each age range, that contains the top 12 sold items for each age_range","metadata":{}},{"cell_type":"code","source":"listina = articles_orders\\\n    .groupBy('age_range')\\\n    .agg(F.collect_list('article_id').alias('sorted_list'))\\\n    .withColumn('prediction', F.concat_ws(' ', 'sorted_list'))\\\n    .drop('sorted_list')\n    \nlistina.show(20)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:44:20.008486Z","iopub.execute_input":"2022-04-22T07:44:20.008901Z","iopub.status.idle":"2022-04-22T07:45:09.677227Z","shell.execute_reply.started":"2022-04-22T07:44:20.008854Z","shell.execute_reply":"2022-04-22T07:45:09.676464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load application file","metadata":{}},{"cell_type":"code","source":"application = spark.read.option(\"header\",True) \\\n                .csv(\"../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:45:09.67839Z","iopub.execute_input":"2022-04-22T07:45:09.678715Z","iopub.status.idle":"2022-04-22T07:45:09.868479Z","shell.execute_reply.started":"2022-04-22T07:45:09.678673Z","shell.execute_reply":"2022-04-22T07:45:09.867674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"join customers to add age_range column to application, then join prediction","metadata":{}},{"cell_type":"code","source":"application = application\\\n    .drop('prediction')\\\n    .join(customers, 'customer_id', 'left')\\\n    .join(listina, 'age_range', 'left')\\\n    .drop('age_range')\\\n\napplication.show(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:45:09.869402Z","iopub.execute_input":"2022-04-22T07:45:09.869636Z","iopub.status.idle":"2022-04-22T07:45:09.965016Z","shell.execute_reply.started":"2022-04-22T07:45:09.8696Z","shell.execute_reply":"2022-04-22T07:45:09.964061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Export the prediction","metadata":{}},{"cell_type":"code","source":"my_pred = application.toPandas()\nmy_pred.to_csv('my_pred.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-22T07:45:09.967475Z","iopub.execute_input":"2022-04-22T07:45:09.967823Z","iopub.status.idle":"2022-04-22T07:46:48.147593Z","shell.execute_reply.started":"2022-04-22T07:45:09.96778Z","shell.execute_reply":"2022-04-22T07:46:48.146681Z"},"trusted":true},"execution_count":null,"outputs":[]}]}