{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"}],"dockerImageVersionId":30157,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"---\n* In this notebook I demonstrate the following topics:\n\n    <a id=\"toc\"></a>\n    - [1. Parquet: How to reduce load time](#1)\n    - [2. EDA: Mini-insights](#2)\n    - [3. Cold Start: User cold-start problem](#3)\n---","metadata":{}},{"cell_type":"code","source":"# import packages\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib \nimport matplotlib.pyplot as plt\nfrom matplotlib import gridspec\nimport matplotlib.lines as lines\n\nimport seaborn as sns\n\nfrom scipy.signal import periodogram\nfrom statsmodels.graphics.tsaplots import plot_pacf\nimport random\n\nfrom pathlib import Path\n\nimport time\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\npd.set_option('display.max_colwidth', False)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-05-10T04:58:02.199031Z","iopub.execute_input":"2025-05-10T04:58:02.199306Z","iopub.status.idle":"2025-05-10T04:58:03.330445Z","shell.execute_reply.started":"2025-05-10T04:58:02.199231Z","shell.execute_reply":"2025-05-10T04:58:03.329647Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# runtime configuration of matplotlib\nplt.style.use(\"Solarize_Light2\")\nplt.rc(\"figure\", \n    autolayout=True, \n    figsize=(20, 10)\n)\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=20,\n    titlepad=10,\n)\n\n# periodogram\ndef plot_periodogram(ts, detrend='linear', ax=None, title=\"Periodogram\"):\n    from scipy.signal import periodogram\n    fs = pd.Timedelta(\"1Y\") / pd.Timedelta(\"1D\")\n    freqencies, spectrum = periodogram(\n        ts,\n        fs=fs,\n        detrend=detrend,\n        window=\"boxcar\",\n        scaling='spectrum',\n    )\n    if ax is None:\n        _, ax = plt.subplots()\n    ax.step(freqencies, spectrum, color=\"purple\")\n    ax.set_xscale(\"log\")\n    ax.set_xticks([1, 2, 4, 6, 12, 26, 52, 104])\n    ax.set_xticklabels(\n        [\n            \"Annual (1)\",\n            \"Semiannual (2)\",\n            \"Quarterly (4)\",\n            \"Bimonthly (6)\",\n            \"Monthly (12)\",\n            \"Biweekly (26)\",\n            \"Weekly (52)\",\n            \"Semiweekly (104)\",\n        ],\n        rotation=30,\n    )\n    ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n    ax.set_ylabel(\"Variance\")\n    ax.set_title(title)\n    return ax","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-05-10T04:58:51.210271Z","iopub.execute_input":"2025-05-10T04:58:51.210562Z","iopub.status.idle":"2025-05-10T04:58:51.220100Z","shell.execute_reply.started":"2025-05-10T04:58:51.210529Z","shell.execute_reply":"2025-05-10T04:58:51.219012Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n<a id=\"1\"></a>\n### Parquet\n\n* *Step 1*: Load the data `articles`, `customers` and `transactions_train` in `csv` format. This you have to do only once.\n* *Step 2*: Save the dataframes in `parquet` format using `.to_parquet()` method of pandas. This you have to do only once.\n* *Step 3*: Load the `parquet` data files. ","metadata":{}},{"cell_type":"code","source":"data_path = Path('/kaggle/input/h-and-m-personalized-fashion-recommendations/')\n\n# This you have to do just once.\nstart = time.time()\n\ntransactions = pd.read_csv(data_path/'transactions_train.csv', \n                           dtype={'article_id': str} ,low_memory=True)\n\narticles = pd.read_csv(data_path / 'articles.csv', dtype={'article_id': str},low_memory=True)\n\nend = time.time()\n\nprint(f\"Loading time for csv format: {np.round(end - start)}\")","metadata":{"execution":{"iopub.status.busy":"2025-05-10T04:58:59.087222Z","iopub.execute_input":"2025-05-10T04:58:59.087508Z","iopub.status.idle":"2025-05-10T05:00:10.641784Z","shell.execute_reply.started":"2025-05-10T04:58:59.087476Z","shell.execute_reply":"2025-05-10T05:00:10.640954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save the csv file to parquet. This you have to do just once.\ntransactions.to_parquet('transactions_parquet.parquet')\narticles.to_parquet('articles_parquet.parquet')\n","metadata":{"execution":{"iopub.status.busy":"2025-05-10T05:04:50.972119Z","iopub.execute_input":"2025-05-10T05:04:50.974010Z","iopub.status.idle":"2025-05-10T05:05:12.932422Z","shell.execute_reply.started":"2025-05-10T05:04:50.973938Z","shell.execute_reply":"2025-05-10T05:05:12.931474Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer = pd.read_csv(data_path / 'customers.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T05:05:56.662607Z","iopub.execute_input":"2025-05-10T05:05:56.662939Z","iopub.status.idle":"2025-05-10T05:06:02.407283Z","shell.execute_reply.started":"2025-05-10T05:05:56.662909Z","shell.execute_reply":"2025-05-10T05:06:02.406491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv(data_path / 'sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T05:06:45.546090Z","iopub.execute_input":"2025-05-10T05:06:45.546387Z","iopub.status.idle":"2025-05-10T05:06:51.580779Z","shell.execute_reply.started":"2025-05-10T05:06:45.546355Z","shell.execute_reply":"2025-05-10T05:06:51.579888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Read the parquet data.\nstart = time.time()\n\ntransactions_parquet = pd.read_parquet('transactions_parquet.parquet')\narticles_parquet = pd.read_parquet('articles_parquet.parquet')\n\n\nend = time.time()\n\nprint(f\"Loading time for parquet format: {np.round((end - start),2)}\")","metadata":{"execution":{"iopub.status.busy":"2025-05-10T05:09:34.707087Z","iopub.execute_input":"2025-05-10T05:09:34.707757Z","iopub.status.idle":"2025-05-10T05:09:48.661295Z","shell.execute_reply.started":"2025-05-10T05:09:34.707714Z","shell.execute_reply":"2025-05-10T05:09:48.660445Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transactions = transactions_parquet.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T05:10:03.481776Z","iopub.execute_input":"2025-05-10T05:10:03.482196Z","iopub.status.idle":"2025-05-10T05:10:06.122931Z","shell.execute_reply.started":"2025-05-10T05:10:03.482159Z","shell.execute_reply":"2025-05-10T05:10:06.122116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transactions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T05:10:11.989985Z","iopub.execute_input":"2025-05-10T05:10:11.990829Z","iopub.status.idle":"2025-05-10T05:10:12.005208Z","shell.execute_reply.started":"2025-05-10T05:10:11.990786Z","shell.execute_reply":"2025-05-10T05:10:12.004142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"articles = articles_parquet.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T05:10:44.463232Z","iopub.execute_input":"2025-05-10T05:10:44.463518Z","iopub.status.idle":"2025-05-10T05:10:44.554743Z","shell.execute_reply.started":"2025-05-10T05:10:44.463491Z","shell.execute_reply":"2025-05-10T05:10:44.553989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"articles","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-10T05:10:52.730432Z","iopub.execute_input":"2025-05-10T05:10:52.730719Z","iopub.status.idle":"2025-05-10T05:10:52.869314Z","shell.execute_reply.started":"2025-05-10T05:10:52.730687Z","shell.execute_reply":"2025-05-10T05:10:52.868442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n<a id=\"2\"></a>\n### EDA - Mini Insights","metadata":{}},{"cell_type":"code","source":"# Convert date to date object\ntransactions = transactions_parquet.copy()\ntransactions[\"date\"] = pd.to_datetime(transactions[\"t_dat\"]).dt.date\ntransactions.drop(columns=[\"t_dat\"], inplace=True)\n\n# Extract product name\ndf_plot = transactions.merge(articles[[\"article_id\", \"prod_name\"]],\\\n     how='left', on=None, left_on='article_id', right_on='article_id', suffixes=('_x', '_y'))\n\n# Calculate the order of the transcation per customer\norder_number = df_plot[[\"date\", \"customer_id\"]].groupby([\"date\", \"customer_id\"]).count()\norder_number.reset_index([\"date\", \"customer_id\"], inplace=True)\n\norder_number['nth_order'] = order_number.sort_values([\"customer_id\",'date'], ascending=True)\\\n             .groupby(['customer_id'])\\\n             .cumcount() + 1\norder_number.loc[order_number[\"customer_id\"]==\"000058a12d5b43e67d225668fa1f8d618c13dc232df0cad8ffe7ad4a1091e318\",:]\n\n# Join transaction with order of transaction from previous step\ndf_plot = df_plot.merge(order_number[[\"date\", \"customer_id\", \"nth_order\"]],\\\n     how='left', on=None, left_on=[\"date\", \"customer_id\"], right_on=[\"date\", \"customer_id\"], suffixes=('_x', '_y'))\n\n# prepare data for eda\ndf_plot[\"flag\"] = 1\n\ny_channel= df_plot[[\"date\",\"sales_channel_id\", \"customer_id\",\"flag\"]]\\\n    .groupby([\"date\",\"sales_channel_id\", \"customer_id\"]).max(\"flag\")\ny_channel.reset_index(level=\"sales_channel_id\", inplace=True)\n\ny_channel_2= df_plot.groupby([\"date\",\"sales_channel_id\"]).agg({\"customer_id\": lambda num: num.nunique()}) #total price per customer\ny_channel_2.columns = ['nb_visitors']\ny_channel_2.reset_index(level=[\"date\",\"sales_channel_id\"], inplace=True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-28T10:22:55.561531Z","iopub.execute_input":"2022-02-28T10:22:55.561781Z","iopub.status.idle":"2022-02-28T10:26:23.284836Z","shell.execute_reply.started":"2022-02-28T10:22:55.561751Z","shell.execute_reply":"2022-02-28T10:26:23.2839Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# placeholders for min and max of the axis\nxmin = y_channel_2[\"date\"].min()\nxmax = y_channel_2[\"date\"].max()\n\nymin = y_channel_2[\"nb_visitors\"].min() - 1000\nymax = y_channel_2[\"nb_visitors\"].max() + 1000","metadata":{"execution":{"iopub.status.busy":"2022-02-28T10:26:23.286134Z","iopub.execute_input":"2022-02-28T10:26:23.286394Z","iopub.status.idle":"2022-02-28T10:26:23.295109Z","shell.execute_reply.started":"2022-02-28T10:26:23.286365Z","shell.execute_reply":"2022-02-28T10:26:23.294104Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top12= pd.value_counts(df_plot[\"prod_name\"]).iloc[:12]\n\n# plot\nfig = plt.figure(constrained_layout=False)\nspec = gridspec.GridSpec(ncols=2, nrows=3, figure=fig)\n\nax1 = fig.add_subplot(spec[:2, 0])\nsns.countplot(y=\"prod_name\", ax=ax1, data=df_plot, order=top12.index);\nax1.set(xlabel=\"Quantity Sold\", ylabel = \"\")\nplt.setp(ax1.get_yticklabels(), rotation=45);\nax1.set_title('Top 12 popular articles')\n\nax3 = fig.add_subplot(spec[:2, 1]);\nsns.countplot(x=\"sales_channel_id\", ax=ax3, data=y_channel, palette=[\"orange\", \"green\"]);\nax3.set(xlabel=\"Channel\", ylabel = \"Visitors\");\nax3.ticklabel_format(style='plain', useOffset=False, axis='y');\nax3.set_xticklabels([\"offline\", \"online\"])\nax3.set_title('Visitors per channel');\n\nax5 = fig.add_subplot(spec[2, :]);\nax5.set(xlabel=\"Date\", ylabel = \"Visitors\");\nsns.scatterplot(data=y_channel_2.loc[y_channel_2[\"sales_channel_id\"]==1,:],  x=\"date\", y=\"nb_visitors\", color=['orange'], label=\"offline\", ax=ax5)\nsns.scatterplot(data=y_channel_2.loc[y_channel_2[\"sales_channel_id\"]==2,:],  x=\"date\", y=\"nb_visitors\", color=['green'], label=\"online\", ax=ax5)\nax5.ticklabel_format(style='plain', useOffset=False, axis='y');\nax5.legend(title=\"Channel\")\nax5.set_title('Daily Visitors');\nax5.set_ylim(ymin, ymax)\nax5.set_xlim(xmin, xmax)\n\nax5.fill_betweenx([ymin,ymax],18343, 18384, color=\"gray\", alpha=0.3)\n\nprops = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\nax5.annotate(\"Missing offline \\ntransaction period\", (18342, 15000), (18270, 27000), \\\n    arrowprops={\"arrowstyle\": \"->\", \"color\":\"C1\"},\n    bbox=props,\n    fontproperties='italic'\n    );","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-28T10:26:23.296588Z","iopub.execute_input":"2022-02-28T10:26:23.296919Z","iopub.status.idle":"2022-02-28T10:26:56.423662Z","shell.execute_reply.started":"2022-02-28T10:26:23.296876Z","shell.execute_reply":"2022-02-28T10:26:56.422654Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Takeaways\n\n    - From the second and third plot we can conclude that there are more Online visitors than offline.\n    - From the third plot it can be observed that for `April-2020` daily transactions are missing for the offline channel.\n\n---\n\n<a id=\"3\"></a>\n### Cold Start: User cold-start problem\n\nThe recommender systems face a problem in recommending items to users in case there is very little data available related to the user or item. This is called the cold-start problem. There different methods to mitigate this problem. Because in the case of H&M we have absolutely no information wrt the new users except that they are online users this leaves us with one possible solution that is the past behaviour of current online user base.  \n\nSo, what I will do next is visualize the top 12 products for new online visitors(that is the first purchase of online visitors) and compare that to the top 12 popular products for returning customers and see if there is a difference.","metadata":{}},{"cell_type":"code","source":"# \npalette_dict = [\"#003f5c\", \"#2f4b7c\", \"#665191\", \"#a05195\", \"#d45087\", \"#f95d6a\", \"#ff7c43\", \"#ffa600\"]\n\nonline_transactions = df_plot.loc[df_plot[\"sales_channel_id\"]==2,:]\n\nfirst_order = pd.value_counts(online_transactions.loc[online_transactions[\"nth_order\"]==1,\"prod_name\"]).iloc[:12].index\n\npalette_1 = [\n'lightblue'for d in first_order if d in ('Luna skinny RW', 'Jade HW Skinny Denim TRS', 'Timeless Midrise Brief','Tilly (1)') \n] + random.sample(palette_dict, 8)\n\nnot_first_order = pd.value_counts(df_plot.loc[df_plot[\"nth_order\"]!=1,\"prod_name\"]).iloc[:12].index\n\npalette_2 = [\n'lightblue'for d in not_first_order if d in ('Luna skinny RW', 'Jade HW Skinny Denim TRS', 'Timeless Midrise Brief','Tilly (1)') \n] + random.sample(palette_dict, 8)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-28T10:26:56.425301Z","iopub.execute_input":"2022-02-28T10:26:56.425809Z","iopub.status.idle":"2022-02-28T10:27:04.397244Z","shell.execute_reply.started":"2022-02-28T10:26:56.42577Z","shell.execute_reply":"2022-02-28T10:27:04.396253Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = plt.figure(constrained_layout=False)\nspec = gridspec.GridSpec(ncols=2, nrows=1, figure=fig)\n\nax1 = fig.add_subplot(spec[0])\nsns.countplot(y=\"prod_name\", ax=ax1, \\\n     data=online_transactions.loc[online_transactions[\"nth_order\"]==1,:],\\\n     order=first_order,\n     palette=palette_1\n     )\nax1.set(xlabel=\"Quantity Sold\", ylabel = \"\")\nplt.setp(ax1.get_xticklabels(), rotation=90)\nax1.set_title('Popular Products - New visitors Online')\n\nax2 = fig.add_subplot(spec[1])\nsns.countplot(y=\"prod_name\", ax=ax2, \\\n     data=df_plot.loc[df_plot[\"nth_order\"]!=1,:],\\\n     order=not_first_order,\n     palette=palette_2)\nax2.set(xlabel=\"Quantity Sold\", ylabel = \"\")\nplt.setp(ax2.get_xticklabels(), rotation=90)\nax2.set_title('Popular Products - Returning visitors')\n\nfig.add_artist(lines.Line2D([0, 1], [0.667, 0.667], color=\"black\", linestyle=\"-.\"));","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-28T10:27:04.398797Z","iopub.execute_input":"2022-02-28T10:27:04.399077Z","iopub.status.idle":"2022-02-28T10:27:34.777828Z","shell.execute_reply.started":"2022-02-28T10:27:04.399045Z","shell.execute_reply":"2022-02-28T10:27:34.776875Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* Takeaway:\n\n    From the two plots one can conclude that there are two 'type' of products:\n\n    - *Type1*: The first four popular products are the same for both new and returning custmers. So these are products that are always popular no matter if you are new or returning customer. \n        \n    - *Type2*: For the new customers 5 of the last 8 popular products are different from the returning customers. So the second group consists of products that is dependent on the type of customer namely new or returning.\n\nSo for the rest of analysis I will only consider the preferences of the first purchase of online customers to predict recommendation for new online visiting customers. \n\n---\n\n### Time Series analysis\n\nSo, now that we know on which popular products to focus the last thing that we have to do is to check for seasonality and serial correlation for these products. But what is seasonality and serial correlation?\n\n\n*Seasonality*: A time series exhibits seasonality whenever there is a regular, periodic change in the mean of the series. Seasonal changes generally follow the clock and calendar -- repetitions over a day, a week, or a year are common. Seasonality is often driven by the cycles of the natural world over days and years or by conventions of social behavior surrounding dates and times. With the help of a `periodogram` one can calculate the significance of different frequencies in time-series data to identify any intrinsic periodic signals.\n\n*Serial correlation*: are patterns of growth and decay in a time series associated with how the value in a series at one time depends on values at previous times, but not necessarily on the time step itself. This cyclic or serial correlation behavior is characteristic of systems that can affect themselves or whose reactions persist over time. Economies, epidemics, animal populations, volcano eruptions, and similar natural phenomena often display cyclic behavior. With the help of a `Partial Auto Correlation Function (PACF)` one can calculate the lag dependencies.","metadata":{}},{"cell_type":"code","source":"new_customers_online = df_plot.loc[(df_plot[\"nth_order\"]==1)&(df_plot[\"sales_channel_id\"]==2),:]\nnew_y_online= new_customers_online.groupby(\"date\").agg({\"article_id\": lambda num: num.nunique()})\nnew_y_online.columns = ['products_sold']\nnew_y_online.index = pd.to_datetime(new_y_online.index)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-28T10:27:34.77935Z","iopub.execute_input":"2022-02-28T10:27:34.779631Z","iopub.status.idle":"2022-02-28T10:27:37.093456Z","shell.execute_reply.started":"2022-02-28T10:27:34.779598Z","shell.execute_reply":"2022-02-28T10:27:37.092511Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = plt.figure(constrained_layout=False, tight_layout=True)\nspec = gridspec.GridSpec(nrows=2, ncols=2, figure=fig)\n\nax1 = fig.add_subplot(spec[0,:])\nsns.scatterplot(data=new_y_online,  x=\"date\", y=\"products_sold\", color=['green'], ax=ax1);\nax1.set_title('Daily Product Sales');\nax1.set(xlabel=\"Date\", ylabel = \"Number Sold\");\n\nax5 = fig.add_axes([0.77, 0.75, 0.2, 0.2]);\nax5.hist(new_y_online[\"products_sold\"]);\n\n# place a text box in upper left in axes coords\nprops = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\nax5.text(6000, 250, \"Histogram daily product sales\",fontsize=8, verticalalignment='top', bbox=props)\n\nax2 = fig.add_subplot(spec[1,0])\nplot_periodogram(new_y_online[\"products_sold\"], 'linear', ax2, title=\"Periodogram\");\n\nax3 = fig.add_subplot(spec[1,1])\nplot_pacf(new_y_online[\"products_sold\"], ax3, lags=10, title=\"PACF\");\nax3.set_xlabel(\"Lags\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-28T10:27:49.057279Z","iopub.execute_input":"2022-02-28T10:27:49.057955Z","iopub.status.idle":"2022-02-28T10:27:50.446076Z","shell.execute_reply.started":"2022-02-28T10:27:49.05791Z","shell.execute_reply":"2022-02-28T10:27:50.445217Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* Takeaway\n    - The first plot is very interesting. What it is saying is that the number of new online purchasers significntly decreased in the last two months of `2018`. So H&M is attracting lesser new customer through the online channel.\n    -  The periodogram plot suggests that ther is `annual` and `monthy` seasonality.\n    - `PACF` suggests serial correlation of upto 8 lags, although a weak one. But the first three lags are strong.\n\n---\n\n*Next steps*: When recommending products to new customes how do one factor in seasonality? This is something that I would want to work on as next steps. *To be continued...*","metadata":{}}]}