{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"# Data processing \nimport numpy as np\nimport pandas as pd\n\n# Data visualisation\nimport seaborn as sns\n\n%matplotlib inline\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nmpl.rc('axes',  labelsize=14)\nmpl.rc('xtick', labelsize=12)\nmpl.rc('ytick', labelsize=12)\n\n# Commons\nimport os\nimport math\nimport random\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-29T07:23:27.621068Z","iopub.execute_input":"2022-03-29T07:23:27.621416Z","iopub.status.idle":"2022-03-29T07:23:27.633881Z","shell.execute_reply.started":"2022-03-29T07:23:27.621379Z","shell.execute_reply":"2022-03-29T07:23:27.632542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constants","metadata":{}},{"cell_type":"code","source":"ARTICLES_CSV_PATH     = '../input/h-and-m-personalized-fashion-recommendations/articles.csv'\nCUSTOMERS_CSV_PATH    = '../input/h-and-m-personalized-fashion-recommendations/customers.csv'\nTRANSACTIONS_CSV_PATH = '../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv'\nARTICLES_IMAGES_PATH  = '../input/h-and-m-personalized-fashion-recommendations/images'\n\nCOLORS = ['grey', 'lightcoral', 'tomato', 'coral','chocolate', 'tan',\n          'lawngreen', 'aquamarine', 'darkcyan', 'deepskyblue', 'navy',\n          'crimson', 'orchid', 'mediumorchid', 'orange', 'slateblue',\n          'cadetblue', 'mediumspringgreen', 'lightseagreen']","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:23:27.636188Z","iopub.execute_input":"2022-03-29T07:23:27.636472Z","iopub.status.idle":"2022-03-29T07:23:27.649108Z","shell.execute_reply.started":"2022-03-29T07:23:27.636438Z","shell.execute_reply":"2022-03-29T07:23:27.647852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils functions","metadata":{}},{"cell_type":"code","source":"def stop_execution():\n    \"\"\"\n    Stop the execution of the program\n    \"\"\"\n    raise SystemExit()\n    \n\ndef csv_to_pd(csv_path):\n    \"\"\" \n    Convert a `csv` file to `pandas Dataframe`\n    If the given file path is not correct, then `None` is returned\n    \n    @param `csv_path`: Path to the csv file\n    @return: Parsed csv file or `None`\n    @rtype: pandas.Dataframe    \n    \"\"\"\n    \n    if not os.path.isfile(csv_path):\n        print(f\"The file '{csv_path}' doesn't exist!\")\n        return None\n    return pd.read_csv(csv_path)\n\n\ndef plot_missing_values(data):\n    \"\"\"\n    Plot the NaNs percentages of each column of the pd df\n    \n    @param `data`: Pandas Dataframe to be analysed\n    \"\"\"\n    \n    if data.isna().sum().sum() == 0:\n        print('Zero missing values')\n        return\n    \n    # Compute the df with the NAs and drop the non-missing values\n    na_df = (data.isna().sum() / len(data)) * 100      \n    na_df = na_df                       \\\n        .drop(na_df[na_df == 0].index)  \\\n        .sort_values(ascending=False)\n    \n    # Plot the NAs df\n    missing_data = pd.DataFrame({'Missing Ratio %': na_df})\n    missing_data.plot(kind='barh')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:23:27.651140Z","iopub.execute_input":"2022-03-29T07:23:27.651441Z","iopub.status.idle":"2022-03-29T07:23:27.668778Z","shell.execute_reply.started":"2022-03-29T07:23:27.651406Z","shell.execute_reply":"2022-03-29T07:23:27.667654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_outliers(data, threshold=0.95):\n    \"\"\"\n    Filter the data. Remove the least (1-threshold) semnificative instances\n    \n    @param `data`: Input table (pd df)\n    @param `threshold`: The strength of the filtering (1-threshold) \n    \"\"\"\n    \n    min_count = data.loc[data['count'].idxmin()][1]\n    max_count = data.loc[data['count'].idxmax()][1]\n    computed_threshold = (max_count - min_count) * (1 - threshold)\n    return data[data['count'] > computed_threshold]\n\n    \ndef group_data_by(data, groupby, countby):\n    \"\"\"\n    Group data based on the column `groupby`\n    and count values by after the `countby` param\n    \n    @param `data`: Pandas Dataframe to be grouped\n    @param `groupby`: Key used to group the data\n    @param `countby`: Count the instances based on this key\n    @return: pd df with 2 columns: `groupby` and `count`\n    @rtype: Pandas Dataframe\n    \"\"\"\n    \n    # Check if the param `groupby` is a nested list \n    if not any(isinstance(x, list) for x in groupby):\n        group_by = groupby\n    else:\n        group_by = [groupby]\n    \n    grouped_data = data                  \\\n        .groupby(group_by)               \\\n        .count()[countby]                \\\n        .sort_values(ascending=False)    \\\n        .reset_index()\n    return grouped_data.rename(columns={countby: 'count'})","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:23:27.670348Z","iopub.execute_input":"2022-03-29T07:23:27.670630Z","iopub.status.idle":"2022-03-29T07:23:27.683356Z","shell.execute_reply.started":"2022-03-29T07:23:27.670590Z","shell.execute_reply":"2022-03-29T07:23:27.682637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_sns_lineplot(x, y, x_label='', y_label='', draw_pareto_line=False):\n    \"\"\"\n    Plot a function given the x and y values\n    \n    @param `x`: X-axis discrete points\n    @param `y`: Y-axis discrete values (e.g: f(x))\n    @param `x_label`: Label of the X-axis plot\n    @param `y_label`: Label of the Y-axis plot\n    @param `draw_pareto_line`: Mark the 20-80 lines on the plot\n    \"\"\"\n    \n    fig, ax = plt.subplots(figsize=(8, 8))\n    ax = sns.lineplot(x=x, y=y)\n    \n    if draw_pareto_line:\n        ax = sns.lineplot(x=[0.2, 0.2], y=[min(y), 0.8], linewidth=3, color='r', estimator=None)\n        ax = sns.lineplot(x=[min(x), 0.2], y=[0.8, 0.8], linewidth=3, color='r', estimator=None)\n    \n    ax.set_xlabel(x_label)\n    ax.set_ylabel(y_label)\n    \n    plt.tight_layout()\n    plt.show()\n\n\ndef plot_sns_hist(data, x, x_label='', y_label='', vertical=False, **kwargs):\n    \"\"\" \n    Plot a histogram to show distributions of the dataset\n    \n    @param `data`: Input data structure (e.g: pd df)\n    @param `x`: Variable that specify position on the y axes (e.g: pd df column name)\n    @param `x_label`: Label of the X-axis histogram plot\n    @param `y_label`: Label of the Y-axis histogram plot\n    @param `**kwargs`: Other parameters for `histplot` function\n    \"\"\"\n        \n    if vertical:\n        kwargs['x'] = x\n    else:\n        kwargs['y'] = x\n        x_label, y_label = y_label, x_label\n    \n    fig, ax = plt.subplots(figsize=(20, 8))\n    ax = sns.histplot(data=data, **kwargs)\n    \n    ax.set_xlabel(x_label)\n    ax.set_ylabel(y_label)\n    \n    plt.tight_layout()\n    plt.show()\n    \n\ndef plot_sns_bar(x, y, x_label='', y_label='', vertical=False, **kwargs):\n    \"\"\" \n    Graph a bar plot\n    \n    @param `x`: X-axis values\n    @param `y`: Y-axis values\n    @param `x_label`: Label of the X-axis bar plot\n    @param `y_label`: Label of the Y-axis bar plot\n    @param `**kwargs`: Other parameters for `barplot` function\n    \"\"\"\n    \n    # Choose a random color if not given\n    if 'color' not in kwargs:\n        kwargs['color'] = random.choice(COLORS)\n    \n    # Compute the figure size\n    x_sz = 10\n    y_sz = math.floor(0.25 * len(y)) if len(y) > 10 else 10\n    if vertical:\n        x, y = y, x\n        x_sz, y_sz = y_sz, x_sz\n        kwargs['orient'] = 'v'\n    \n    fig, ax = plt.subplots(figsize=(x_sz, y_sz))\n    ax = sns.barplot(x=x, y=y, **kwargs)\n    ax.set_xlabel(x_label)\n    ax.set_ylabel(y_label)\n    \n    if not x_label:\n        ax.set(xticklabels=[])\n        \n    if not y_label:\n        ax.set(yticklabels=[])\n    \n    plt.tight_layout()\n    plt.show()\n    \n    \ndef plot_sns_boxplot(data, x='', y=''):\n    \"\"\" \n    Graph a box plot\n    \n    @param `data`: Input table (pd df)\n    @param `x`: X-axis values (column's name)\n    @param `y`: Y-axis values (for multiple boxplots)\n    \"\"\"\n        \n    fig, ax = plt.subplots(figsize=(15, 6))\n    if not y:\n        ax = sns.boxplot(data=data, x=x, color='xkcd:cerulean')\n    else:\n        sns.set_style('darkgrid')\n        ax = sns.boxplot(data=data, x=x, y=y)\n    ax.set_xlabel(x)\n    plt.show()\n    \n\ndef plot_sns_color_pallete(data, labels=None):\n    \"\"\" \n    Graph a pie chart\n    \n    @param `data`: Values of labels\n    @param `labels`: Labels to plot on pie\n    \"\"\"\n    \n    fig, ax = plt.subplots(figsize=(20, 8))\n    colors = sns.color_palette('pastel')\n    \n    ax.pie(data, labels=labels, colors=colors, autopct='%1.1f%%')\n    ax.set_facecolor('lightgrey')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:23:27.685834Z","iopub.execute_input":"2022-03-29T07:23:27.686606Z","iopub.status.idle":"2022-03-29T07:23:27.711645Z","shell.execute_reply.started":"2022-03-29T07:23:27.686535Z","shell.execute_reply":"2022-03-29T07:23:27.710584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_articles_images(data):\n    \"\"\"\n    Plot images of certain articles\n    \n    @param `data`: Input Pandas Dataframe for plotting the images \n    \"\"\"\n    \n    fig, ax = plt.subplots(1, data.shape[0], figsize=(20, 10))\n    i = 0\n\n    # For each article\n    for _, data in data.iterrows():\n        # Compute description with a new line after 5 words\n        description = articles[articles['article_id'] == data['article_id']]['detail_desc'].iloc[0]\n        description_list = description.split(' ')\n        for j, elem in enumerate(description_list):\n            if j > 0 and j % 5 == 0:\n                description_list[j] = description_list[j] + '\\n'\n        description = ' '.join(description_list)\n\n        # Plot the image\n        img = mpimg.imread(f\"{ARTICLES_IMAGES_PATH}/0{str(data['article_id'])[:2]}/0{int(data['article_id'])}.jpg\")\n        ax[i].imshow(img)\n\n        # Add title and description\n        ax[i].set_title(f'price: {data.price:.4f}')\n        ax[i].set_xlabel(description, fontsize=10)\n\n        # Disable axes and grid\n        ax[i].set_xticks([], [])\n        ax[i].set_yticks([], [])\n        ax[i].grid(False)\n\n        # Move to the next article\n        i += 1\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:23:27.756628Z","iopub.execute_input":"2022-03-29T07:23:27.756922Z","iopub.status.idle":"2022-03-29T07:23:27.769434Z","shell.execute_reply.started":"2022-03-29T07:23:27.756892Z","shell.execute_reply":"2022-03-29T07:23:27.768093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the data","metadata":{}},{"cell_type":"code","source":"articles     = csv_to_pd(ARTICLES_CSV_PATH)\ncustomers    = csv_to_pd(CUSTOMERS_CSV_PATH)\ntransactions = csv_to_pd(TRANSACTIONS_CSV_PATH)\n\n# Sanity check\nif any(pd_df is None for pd_df in [articles,\n                                   customers,\n                                   transactions]):\n    stop_execution()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:23:27.771556Z","iopub.execute_input":"2022-03-29T07:23:27.772634Z","iopub.status.idle":"2022-03-29T07:24:20.467817Z","shell.execute_reply.started":"2022-03-29T07:23:27.772580Z","shell.execute_reply":"2022-03-29T07:24:20.466782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's tackle each dataframe","metadata":{}},{"cell_type":"code","source":"# Show all columns in the dataframes\npd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:20.469248Z","iopub.execute_input":"2022-03-29T07:24:20.469496Z","iopub.status.idle":"2022-03-29T07:24:20.474144Z","shell.execute_reply.started":"2022-03-29T07:24:20.469466Z","shell.execute_reply":"2022-03-29T07:24:20.472817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Articles","metadata":{}},{"cell_type":"code","source":"# ~100k articles and 25 attributes\narticles.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:20.480769Z","iopub.execute_input":"2022-03-29T07:24:20.482005Z","iopub.status.idle":"2022-03-29T07:24:20.496942Z","shell.execute_reply.started":"2022-03-29T07:24:20.481931Z","shell.execute_reply":"2022-03-29T07:24:20.496272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can group the attributes as follows:\n\n# (product_code, product_name)\n# (product_type_no, product_type_name)\n# product_group_name\n# (graphical_appearance_no, graphical_appearance_name)\n# (colour_group_code, colour_group_name)\n# (perceived_colour_value_id, perceived_colour_value_name)\n# (perceived_colour_master_id, perceived_colour_master_name)\n# (department_no, department_name)\n# (index_code, index_name)\n# (index_group_no, index_group_name)\n# (section_no, section_name)\n# (garment_group_no, garment_group_name)\n# detail_desc\n\n# You can see a lot of pairs of type (id, name).\n# That means we can only use the `numerical attributes` or the `categorical attributes` when we will train the models.\n# The `name` fields are very important for human readability when investigating the data. ","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:20.498209Z","iopub.execute_input":"2022-03-29T07:24:20.499239Z","iopub.status.idle":"2022-03-29T07:24:20.510124Z","shell.execute_reply.started":"2022-03-29T07:24:20.499182Z","shell.execute_reply":"2022-03-29T07:24:20.509308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:20.511683Z","iopub.execute_input":"2022-03-29T07:24:20.512632Z","iopub.status.idle":"2022-03-29T07:24:20.557949Z","shell.execute_reply.started":"2022-03-29T07:24:20.512574Z","shell.execute_reply":"2022-03-29T07:24:20.556990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:20.559331Z","iopub.execute_input":"2022-03-29T07:24:20.559926Z","iopub.status.idle":"2022-03-29T07:24:20.756164Z","shell.execute_reply.started":"2022-03-29T07:24:20.559891Z","shell.execute_reply":"2022-03-29T07:24:20.755335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only the `detail_desc` has missing values (0.4%), which is good\nplot_missing_values(articles)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:20.757444Z","iopub.execute_input":"2022-03-29T07:24:20.757863Z","iopub.status.idle":"2022-03-29T07:24:21.359282Z","shell.execute_reply.started":"2022-03-29T07:24:20.757815Z","shell.execute_reply":"2022-03-29T07:24:21.358231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### index_name","metadata":{}},{"cell_type":"code","source":"# From the plot below, we can see that H&M has a lot of `ladieswear` and very little `sportwear`\nplot_sns_hist(data=articles,\n              x='index_name',\n              x_label='index name',\n              y_label='count',\n              color='navy')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:21.360900Z","iopub.execute_input":"2022-03-29T07:24:21.361766Z","iopub.status.idle":"2022-03-29T07:24:21.886441Z","shell.execute_reply.started":"2022-03-29T07:24:21.361711Z","shell.execute_reply":"2022-03-29T07:24:21.885167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# `Jersey Fancy` is the most frequent garment, especially for children and women\n# The `Sport` group is very sparse\nplot_sns_hist(data=articles,\n              x='garment_group_name',\n              hue='index_group_name',\n              multiple='stack')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:21.891068Z","iopub.execute_input":"2022-03-29T07:24:21.891879Z","iopub.status.idle":"2022-03-29T07:24:23.121833Z","shell.execute_reply.started":"2022-03-29T07:24:21.891830Z","shell.execute_reply":"2022-03-29T07:24:23.120666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### product_group","metadata":{}},{"cell_type":"code","source":"# Let's group the articles by `product_group_name` and count how many articles we have in each group.\nproduct_group_name_sorted = group_data_by(data=articles,\n                                          groupby='product_group_name',\n                                          countby='article_id')\n\n# Then, graph a bar plot\nplot_sns_bar(x=product_group_name_sorted['count'],\n             y=product_group_name_sorted['product_group_name'],\n             x_label='count',\n             y_label='product group name')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:23.123622Z","iopub.execute_input":"2022-03-29T07:24:23.124527Z","iopub.status.idle":"2022-03-29T07:24:23.825689Z","shell.execute_reply.started":"2022-03-29T07:24:23.124469Z","shell.execute_reply":"2022-03-29T07:24:23.824692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's plot a pie chart with the most semnificative `product_groups`\npie_data = filter_outliers(data=product_group_name_sorted, threshold=0.95)\npie_data","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:23.827465Z","iopub.execute_input":"2022-03-29T07:24:23.827758Z","iopub.status.idle":"2022-03-29T07:24:23.841701Z","shell.execute_reply.started":"2022-03-29T07:24:23.827726Z","shell.execute_reply":"2022-03-29T07:24:23.840611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_sns_color_pallete(data=pie_data['count'],\n                       labels=pie_data['product_group_name'])","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:23.843600Z","iopub.execute_input":"2022-03-29T07:24:23.843856Z","iopub.status.idle":"2022-03-29T07:24:24.183144Z","shell.execute_reply.started":"2022-03-29T07:24:23.843826Z","shell.execute_reply":"2022-03-29T07:24:24.182351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### product_type","metadata":{}},{"cell_type":"code","source":"# Let's group the articles by `product_type_name` and count how many articles we have of each type.\nproduct_type_name_sorted = group_data_by(data=articles,\n                                         groupby='product_type_name',\n                                         countby='article_id')\n\n# Filter the outliers (look only at the most relevant `product_types`)\nproduct_type_name_filtered = filter_outliers(data=product_type_name_sorted, threshold=0.97)\n\n# Then, graph a bar plot\nplot_sns_bar(x=product_type_name_filtered['count'],\n             y=product_type_name_filtered['product_type_name'],\n             x_label='count',\n             y_label='product type name',\n             color='navy')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:24.184962Z","iopub.execute_input":"2022-03-29T07:24:24.185430Z","iopub.status.idle":"2022-03-29T07:24:25.293536Z","shell.execute_reply.started":"2022-03-29T07:24:24.185396Z","shell.execute_reply":"2022-03-29T07:24:25.292527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### graphical_appearance","metadata":{}},{"cell_type":"code","source":"# Let's group the articles by `graphical_apperance_name` and count how many articles we have in each group.\ngraphical_appearance_sorted = group_data_by(data=articles,\n                                            groupby='graphical_appearance_name',\n                                            countby='article_id')\n\n# Then, graph a bar plot\nplot_sns_bar(x=graphical_appearance_sorted['count'],\n             y=graphical_appearance_sorted['graphical_appearance_name'],\n             x_label='count',\n             y_label='graphical appearance name',\n             color='darkcyan')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:25.295182Z","iopub.execute_input":"2022-03-29T07:24:25.295430Z","iopub.status.idle":"2022-03-29T07:24:25.970570Z","shell.execute_reply.started":"2022-03-29T07:24:25.295401Z","shell.execute_reply":"2022-03-29T07:24:25.969173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can also plot a pie chart with the most semnificative `graphical_appearances`\ngraphical_appearance_filtered = filter_outliers(data=graphical_appearance_sorted)\nplot_sns_color_pallete(data=graphical_appearance_filtered['count'],\n                       labels=graphical_appearance_filtered['graphical_appearance_name'])","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:25.972315Z","iopub.execute_input":"2022-03-29T07:24:25.972599Z","iopub.status.idle":"2022-03-29T07:24:26.149441Z","shell.execute_reply.started":"2022-03-29T07:24:25.972538Z","shell.execute_reply":"2022-03-29T07:24:26.148413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### color_group","metadata":{}},{"cell_type":"code","source":"# Let's group the articles by `color_group_name` and count how many articles we have in each group.\ncolor_group_sorted = group_data_by(data=articles,\n                                   groupby='colour_group_name',\n                                   countby='article_id')\n\n# Then, graph a bar plot\nplot_sns_bar(x=color_group_sorted['count'],\n             y=color_group_sorted['colour_group_name'],\n             x_label='count',\n             y_label='colour group name')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:26.151579Z","iopub.execute_input":"2022-03-29T07:24:26.152381Z","iopub.status.idle":"2022-03-29T07:24:27.763043Z","shell.execute_reply.started":"2022-03-29T07:24:26.152326Z","shell.execute_reply":"2022-03-29T07:24:27.760751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Customers","metadata":{}},{"cell_type":"code","source":"# ~1.3mil customers and 7 attributes\ncustomers.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:27.771859Z","iopub.execute_input":"2022-03-29T07:24:27.773950Z","iopub.status.idle":"2022-03-29T07:24:27.785446Z","shell.execute_reply.started":"2022-03-29T07:24:27.773879Z","shell.execute_reply":"2022-03-29T07:24:27.783946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:27.787973Z","iopub.execute_input":"2022-03-29T07:24:27.789217Z","iopub.status.idle":"2022-03-29T07:24:27.825551Z","shell.execute_reply.started":"2022-03-29T07:24:27.788856Z","shell.execute_reply":"2022-03-29T07:24:27.824800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The `customers` df has a lot of missing values on the `FN` and `Active` columns\nplot_missing_values(customers)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:27.827018Z","iopub.execute_input":"2022-03-29T07:24:27.827405Z","iopub.status.idle":"2022-03-29T07:24:29.817052Z","shell.execute_reply.started":"2022-03-29T07:24:27.827373Z","shell.execute_reply":"2022-03-29T07:24:29.815759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.describe()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:29.819155Z","iopub.execute_input":"2022-03-29T07:24:29.819614Z","iopub.status.idle":"2022-03-29T07:24:30.056643Z","shell.execute_reply.started":"2022-03-29T07:24:29.819544Z","shell.execute_reply":"2022-03-29T07:24:30.055641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### age","metadata":{}},{"cell_type":"code","source":"# The distribution is not a Gaussian one, but we can look at it within 2 ranges: [10, 40] and [40, 80]\n# If we take a look at those 2 distributions separately, we can compare them with a Normal Distribution\n# The common ages are 20-25 in the first frame and 50-55 in the second one\nplot_sns_hist(data=customers,\n              x='age',\n              x_label='age',\n              y_label='count',\n              vertical=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:30.058255Z","iopub.execute_input":"2022-03-29T07:24:30.058637Z","iopub.status.idle":"2022-03-29T07:24:31.074626Z","shell.execute_reply.started":"2022-03-29T07:24:30.058589Z","shell.execute_reply":"2022-03-29T07:24:31.073680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### age_outliers","metadata":{}},{"cell_type":"code","source":"# We can see that the data is `right skewed`. It's also obvious from the above distribution.\n# It means that the data constitute higher frequency of low valued scores.\nplot_sns_boxplot(data=customers, x='age')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:31.076277Z","iopub.execute_input":"2022-03-29T07:24:31.076637Z","iopub.status.idle":"2022-03-29T07:24:31.326486Z","shell.execute_reply.started":"2022-03-29T07:24:31.076590Z","shell.execute_reply":"2022-03-29T07:24:31.325321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### club_member_status","metadata":{}},{"cell_type":"code","source":"# Let's group the customers by `club_member_status` and count how many customers we have in each group.\nclub_member_status_sorted = group_data_by(data=customers,\n                                          groupby='club_member_status',\n                                          countby='customer_id')\n\n# Then, graph a bar plot\nplot_sns_bar(x=club_member_status_sorted['count'],\n             y=club_member_status_sorted['club_member_status'],\n             x_label='count',\n             y_label='club member status',\n             vertical=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:31.328249Z","iopub.execute_input":"2022-03-29T07:24:31.328540Z","iopub.status.idle":"2022-03-29T07:24:32.394479Z","shell.execute_reply.started":"2022-03-29T07:24:31.328504Z","shell.execute_reply":"2022-03-29T07:24:32.393366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### fashion_news_frequency","metadata":{}},{"cell_type":"code","source":"# Let's group the customers by `fashion_news_frequency` and count how many customers we have in each group.\nfashion_news_frequency_sorted = group_data_by(data=customers,\n                                              groupby='fashion_news_frequency',\n                                              countby='customer_id')\nfashion_news_frequency_sorted.drop([3], inplace=True)\n\n# Then, graph a bar plot\nplot_sns_bar(x=fashion_news_frequency_sorted['count'],\n             y=fashion_news_frequency_sorted['fashion_news_frequency'],\n             x_label='count',\n             y_label='fashion news frequency',\n             vertical=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:32.396263Z","iopub.execute_input":"2022-03-29T07:24:32.396743Z","iopub.status.idle":"2022-03-29T07:24:33.496889Z","shell.execute_reply.started":"2022-03-29T07:24:32.396692Z","shell.execute_reply":"2022-03-29T07:24:33.495963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As we can see, customers prefer not to get any messages about the news\nplot_sns_color_pallete(data=fashion_news_frequency_sorted['count'],\n                       labels=fashion_news_frequency_sorted['fashion_news_frequency'])","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:33.498870Z","iopub.execute_input":"2022-03-29T07:24:33.499469Z","iopub.status.idle":"2022-03-29T07:24:33.698026Z","shell.execute_reply.started":"2022-03-29T07:24:33.499421Z","shell.execute_reply":"2022-03-29T07:24:33.696991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Transactions","metadata":{}},{"cell_type":"code","source":"# ~31mil transactions and 5 attributes\ntransactions.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:33.699833Z","iopub.execute_input":"2022-03-29T07:24:33.700176Z","iopub.status.idle":"2022-03-29T07:24:33.709359Z","shell.execute_reply.started":"2022-03-29T07:24:33.700129Z","shell.execute_reply":"2022-03-29T07:24:33.708357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# `customer_id` is also present in the `customers` df and `article_id` in the `articles` df\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:33.718673Z","iopub.execute_input":"2022-03-29T07:24:33.719275Z","iopub.status.idle":"2022-03-29T07:24:33.737631Z","shell.execute_reply.started":"2022-03-29T07:24:33.719229Z","shell.execute_reply":"2022-03-29T07:24:33.736432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_missing_values(transactions)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:33.739924Z","iopub.execute_input":"2022-03-29T07:24:33.741805Z","iopub.status.idle":"2022-03-29T07:24:40.922909Z","shell.execute_reply.started":"2022-03-29T07:24:33.741741Z","shell.execute_reply":"2022-03-29T07:24:40.921773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.describe()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:40.924958Z","iopub.execute_input":"2022-03-29T07:24:40.925723Z","iopub.status.idle":"2022-03-29T07:24:45.063989Z","shell.execute_reply.started":"2022-03-29T07:24:40.925661Z","shell.execute_reply":"2022-03-29T07:24:45.062917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:45.065802Z","iopub.execute_input":"2022-03-29T07:24:45.066155Z","iopub.status.idle":"2022-03-29T07:24:59.783938Z","shell.execute_reply.started":"2022-03-29T07:24:45.066107Z","shell.execute_reply":"2022-03-29T07:24:59.782974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:59.785454Z","iopub.execute_input":"2022-03-29T07:24:59.785771Z","iopub.status.idle":"2022-03-29T07:24:59.801538Z","shell.execute_reply.started":"2022-03-29T07:24:59.785734Z","shell.execute_reply":"2022-03-29T07:24:59.800814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's convert the `t_dat` attribute into pd df datetime\ntransactions['t_dat'] = pd.to_datetime(transactions['t_dat'])","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:24:59.802959Z","iopub.execute_input":"2022-03-29T07:24:59.803693Z","iopub.status.idle":"2022-03-29T07:25:06.103106Z","shell.execute_reply.started":"2022-03-29T07:24:59.803652Z","shell.execute_reply":"2022-03-29T07:25:06.102052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:06.104912Z","iopub.execute_input":"2022-03-29T07:25:06.105196Z","iopub.status.idle":"2022-03-29T07:25:06.116872Z","shell.execute_reply.started":"2022-03-29T07:25:06.105165Z","shell.execute_reply":"2022-03-29T07:25:06.115730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:06.118834Z","iopub.execute_input":"2022-03-29T07:25:06.119234Z","iopub.status.idle":"2022-03-29T07:25:06.141104Z","shell.execute_reply.started":"2022-03-29T07:25:06.119178Z","shell.execute_reply":"2022-03-29T07:25:06.140109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.tail(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:06.143414Z","iopub.execute_input":"2022-03-29T07:25:06.144228Z","iopub.status.idle":"2022-03-29T07:25:06.162381Z","shell.execute_reply.started":"2022-03-29T07:25:06.144170Z","shell.execute_reply":"2022-03-29T07:25:06.161194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_t_dat = transactions['t_dat'].min()\nmax_t_dat = transactions['t_dat'].max()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:06.163926Z","iopub.execute_input":"2022-03-29T07:25:06.164228Z","iopub.status.idle":"2022-03-29T07:25:06.434137Z","shell.execute_reply.started":"2022-03-29T07:25:06.164190Z","shell.execute_reply":"2022-03-29T07:25:06.433095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The dataset contains transactions carried out over a period of 2 years\nprint('Start: ', min_t_dat)\nprint('Stop:  ', max_t_dat)\nprint('Total: ', max_t_dat - min_t_dat)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:06.435507Z","iopub.execute_input":"2022-03-29T07:25:06.435765Z","iopub.status.idle":"2022-03-29T07:25:06.446769Z","shell.execute_reply.started":"2022-03-29T07:25:06.435736Z","shell.execute_reply":"2022-03-29T07:25:06.445518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:06.448364Z","iopub.execute_input":"2022-03-29T07:25:06.448716Z","iopub.status.idle":"2022-03-29T07:25:06.469488Z","shell.execute_reply.started":"2022-03-29T07:25:06.448672Z","shell.execute_reply":"2022-03-29T07:25:06.468513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### sales_channel_id","metadata":{}},{"cell_type":"code","source":"# Let's group the transactions by `sales_cahnnel_id` and count how many transactions we have in each group.\ntransactions_channel_sorted = group_data_by(data=transactions.sample(frac=0.1),\n                                            groupby='sales_channel_id',\n                                            countby='article_id')\n\n# We can see that almost twice as many sales were done through the second channel\nplot_sns_bar(x=transactions_channel_sorted['count'],\n             y=transactions_channel_sorted['sales_channel_id'],\n             x_label='sales channel id',\n             y_label='count',\n             vertical=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:06.471289Z","iopub.execute_input":"2022-03-29T07:25:06.471549Z","iopub.status.idle":"2022-03-29T07:25:13.849502Z","shell.execute_reply.started":"2022-03-29T07:25:06.471519Z","shell.execute_reply":"2022-03-29T07:25:13.848289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### price","metadata":{}},{"cell_type":"code","source":"price_filtered = transactions.loc[transactions['price'] < 0.1]\nplot_sns_hist(data=price_filtered,\n              x='price',\n              x_label='price',\n              y_label='count',\n              vertical=True,\n              bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:13.851601Z","iopub.execute_input":"2022-03-29T07:25:13.852208Z","iopub.status.idle":"2022-03-29T07:25:30.969685Z","shell.execute_reply.started":"2022-03-29T07:25:13.852155Z","shell.execute_reply":"2022-03-29T07:25:30.968667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### price_outliers","metadata":{}},{"cell_type":"code","source":"# From this plotbox we can see that the prices are fairly centered, but only in the interval [0, 0.06]\n# The median (Q2) is placed in the center of the interval [Q1, Q3], which is good\n# In this interval, the distribution is almost Normal\n\n# But, we can also see that the dataset has a lot of outliers, especially high prices\n# That's the reason I've filtered the prices so much in the previous histogram\n\nplot_sns_boxplot(data=transactions, x='price')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:30.971578Z","iopub.execute_input":"2022-03-29T07:25:30.972103Z","iopub.status.idle":"2022-03-29T07:25:36.128122Z","shell.execute_reply.started":"2022-03-29T07:25:30.972054Z","shell.execute_reply":"2022-03-29T07:25:36.127345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:36.129411Z","iopub.execute_input":"2022-03-29T07:25:36.129801Z","iopub.status.idle":"2022-03-29T07:25:36.152666Z","shell.execute_reply.started":"2022-03-29T07:25:36.129763Z","shell.execute_reply":"2022-03-29T07:25:36.151932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's select the most relevant attributes from the `articles` df\narticles_red = articles[['article_id', 'prod_name', 'product_type_name',\n                         'product_group_name', 'index_name', 'garment_group_name']]\narticles_red.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:36.154178Z","iopub.execute_input":"2022-03-29T07:25:36.154460Z","iopub.status.idle":"2022-03-29T07:25:36.189530Z","shell.execute_reply.started":"2022-03-29T07:25:36.154420Z","shell.execute_reply":"2022-03-29T07:25:36.188456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now, merge the `articles_red` df with `transactions` df\ntransactions_merged = transactions.merge(articles_red, on='article_id')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:25:36.191127Z","iopub.execute_input":"2022-03-29T07:25:36.191497Z","iopub.status.idle":"2022-03-29T07:26:05.496001Z","shell.execute_reply.started":"2022-03-29T07:25:36.191447Z","shell.execute_reply":"2022-03-29T07:26:05.495083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_merged.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:26:05.497615Z","iopub.execute_input":"2022-03-29T07:26:05.497913Z","iopub.status.idle":"2022-03-29T07:26:05.515451Z","shell.execute_reply.started":"2022-03-29T07:26:05.497880Z","shell.execute_reply":"2022-03-29T07:26:05.514485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In the following boxplot, we can see outliers for `group_name` prices\n# Lower/Upper/Full body have a lot of outliers (maybe because we can think of some unique collections vs. casual garment)\n# Accessories have also some high price variance\n\nplot_sns_boxplot(data=transactions_merged.sample(frac=0.1),\n                 x='price',\n                 y='product_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:26:05.516857Z","iopub.execute_input":"2022-03-29T07:26:05.517134Z","iopub.status.idle":"2022-03-29T07:26:28.515497Z","shell.execute_reply.started":"2022-03-29T07:26:05.517103Z","shell.execute_reply":"2022-03-29T07:26:28.514490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's look inside the `Garment Upper Body` category\nupper_body = transactions_merged[transactions_merged['product_group_name'].str.contains('Upper')]\nupper_body.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:26:28.517275Z","iopub.execute_input":"2022-03-29T07:26:28.517600Z","iopub.status.idle":"2022-03-29T07:26:50.622573Z","shell.execute_reply.started":"2022-03-29T07:26:28.517547Z","shell.execute_reply":"2022-03-29T07:26:50.621401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Jackets, Coats, Shirts or Blouses have a high price variance\n# That's ok, some special collections are much more expensive than regular clothes\n# Looking at the Bodysuits, we can see very little variance\n\nplot_sns_boxplot(data=upper_body.sample(frac=0.25),\n                 x='price',\n                 y='product_type_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:26:50.624294Z","iopub.execute_input":"2022-03-29T07:26:50.624685Z","iopub.status.idle":"2022-03-29T07:27:00.820793Z","shell.execute_reply.started":"2022-03-29T07:26:50.624637Z","shell.execute_reply":"2022-03-29T07:27:00.819954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### total_daily_sales","metadata":{}},{"cell_type":"code","source":"transactions_merged.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:00.822286Z","iopub.execute_input":"2022-03-29T07:27:00.822990Z","iopub.status.idle":"2022-03-29T07:27:00.841366Z","shell.execute_reply.started":"2022-03-29T07:27:00.822945Z","shell.execute_reply":"2022-03-29T07:27:00.839936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_t_dat_sorted = transactions     \\\n    .groupby('t_dat')['price']               \\\n    .agg(['sum', 'mean'])                    \\\n    .sort_values(by='t_dat', ascending=True) \\\n    .reset_index()\n    \ntransactions_t_dat_sorted.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:00.843231Z","iopub.execute_input":"2022-03-29T07:27:00.843514Z","iopub.status.idle":"2022-03-29T07:27:01.883920Z","shell.execute_reply.started":"2022-03-29T07:27:00.843482Z","shell.execute_reply":"2022-03-29T07:27:01.882747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_t_dates  = transactions_t_dat_sorted.shape[0]\nstep_t_dates = 10\nidx_list     = [idx for idx in range(1, len_t_dates, step_t_dates)]\nprint('Sampled days: ', idx_list)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:01.886503Z","iopub.execute_input":"2022-03-29T07:27:01.886934Z","iopub.status.idle":"2022-03-29T07:27:01.894456Z","shell.execute_reply.started":"2022-03-29T07:27:01.886883Z","shell.execute_reply":"2022-03-29T07:27:01.893693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first_day = transactions_t_dat_sorted['t_dat'][idx_list[0]]\nlast_day  = transactions_t_dat_sorted['t_dat'][idx_list[len(idx_list)-1]]\nprint(f'Graph of sales starting from date {first_day}, until {last_day} with step of size {step_t_dates} days')\n\nplot_sns_bar(x=transactions_t_dat_sorted['sum'][idx_list],\n             y=transactions_t_dat_sorted['t_dat'][idx_list],\n             x_label='',\n             y_label='sum of sales',\n             vertical=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:01.896015Z","iopub.execute_input":"2022-03-29T07:27:01.896598Z","iopub.status.idle":"2022-03-29T07:27:02.752755Z","shell.execute_reply.started":"2022-03-29T07:27:01.896514Z","shell.execute_reply":"2022-03-29T07:27:02.751660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### total_monthly_sales","metadata":{}},{"cell_type":"code","source":"transactions_merged.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:02.754452Z","iopub.execute_input":"2022-03-29T07:27:02.755362Z","iopub.status.idle":"2022-03-29T07:27:02.775541Z","shell.execute_reply.started":"2022-03-29T07:27:02.755301Z","shell.execute_reply":"2022-03-29T07:27:02.774362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's create a new pd df for dividing the sales in months\nmonths = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',\n          'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']\nmonths_dict = dict(zip(range(len(months)), months))\n\nmonths_sales = pd.DataFrame()\nmonths_sales['month'] = transactions_merged['t_dat'].dt.month.map(months_dict)\nmonths_sales['price'] = transactions_merged['price']\n\nmonths_sales_sorted = months_sales          \\\n    .groupby('month')['price']              \\\n    .agg(['sum'])                           \\\n    .sort_values(by='sum', ascending=False) \\\n    .reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:02.777446Z","iopub.execute_input":"2022-03-29T07:27:02.778147Z","iopub.status.idle":"2022-03-29T07:27:15.813038Z","shell.execute_reply.started":"2022-03-29T07:27:02.778102Z","shell.execute_reply":"2022-03-29T07:27:15.811835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_sns_bar(x=months_sales_sorted['month'],\n             y=months_sales_sorted['sum'],\n             x_label='month of the year',\n             y_label='sum of sales per month')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:15.814614Z","iopub.execute_input":"2022-03-29T07:27:15.814912Z","iopub.status.idle":"2022-03-29T07:27:16.137646Z","shell.execute_reply.started":"2022-03-29T07:27:15.814880Z","shell.execute_reply":"2022-03-29T07:27:16.136652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### total_weekly_sales","metadata":{}},{"cell_type":"code","source":"transactions_merged.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:16.139239Z","iopub.execute_input":"2022-03-29T07:27:16.140176Z","iopub.status.idle":"2022-03-29T07:27:16.156884Z","shell.execute_reply.started":"2022-03-29T07:27:16.140121Z","shell.execute_reply":"2022-03-29T07:27:16.155953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's create a new pd df for dividing the sales in days\ndays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']\ndays_dict = dict(zip(range(len(days)), days))\n\ndays_sales = pd.DataFrame()\ndays_sales['day']   = transactions_merged['t_dat'].dt.weekday.map(days_dict)\ndays_sales['price'] = transactions_merged['price']\n\ndays_sales_sorted = days_sales              \\\n    .groupby('day')['price']                \\\n    .agg(['sum'])                           \\\n    .sort_values(by='sum', ascending=False) \\\n    .reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:16.158396Z","iopub.execute_input":"2022-03-29T07:27:16.159008Z","iopub.status.idle":"2022-03-29T07:27:29.146974Z","shell.execute_reply.started":"2022-03-29T07:27:16.158954Z","shell.execute_reply":"2022-03-29T07:27:29.145882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_sns_bar(x=days_sales_sorted['day'],\n             y=days_sales_sorted['sum'],\n             x_label='day of the week',\n             y_label='sum of sales per day')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:29.148378Z","iopub.execute_input":"2022-03-29T07:27:29.148649Z","iopub.status.idle":"2022-03-29T07:27:29.481390Z","shell.execute_reply.started":"2022-03-29T07:27:29.148618Z","shell.execute_reply":"2022-03-29T07:27:29.480115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Change of price in time for some articles","metadata":{}},{"cell_type":"code","source":"transactions_merged.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:29.483425Z","iopub.execute_input":"2022-03-29T07:27:29.483884Z","iopub.status.idle":"2022-03-29T07:27:29.504013Z","shell.execute_reply.started":"2022-03-29T07:27:29.483829Z","shell.execute_reply":"2022-03-29T07:27:29.503173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:29.505876Z","iopub.execute_input":"2022-03-29T07:27:29.506244Z","iopub.status.idle":"2022-03-29T07:27:29.538673Z","shell.execute_reply.started":"2022-03-29T07:27:29.506201Z","shell.execute_reply":"2022-03-29T07:27:29.537513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['product_group_name'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:29.540340Z","iopub.execute_input":"2022-03-29T07:27:29.540682Z","iopub.status.idle":"2022-03-29T07:27:29.559589Z","shell.execute_reply.started":"2022-03-29T07:27:29.540646Z","shell.execute_reply":"2022-03-29T07:27:29.558533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_list = ['Garment Upper body', 'Garment Lower body',\n                'Garment Full body', 'Accessories', 'Underwear', 'Shoes']\n\nfig_cols = 2\nfig_rows = math.ceil(len(product_list) / fig_cols)\nfig, ax  = plt.subplots(fig_rows, fig_cols, figsize=(20, 15))\n\nk = 0\nfor i in range(fig_rows):\n    for j in range(fig_cols):\n        # Sanity check\n        if k >= len(product_list):\n            ax[i, j].set_visible(False)\n            continue\n        \n        # Current product in the list\n        product = product_list[k]\n        articles_curr_prod = transactions_merged[transactions_merged['product_group_name'] == product]\n        \n        # Compute the mean price over time\n        series_mean = articles_curr_prod[['t_dat', 'price']]         \\\n            .groupby(pd.Grouper(key='t_dat', freq='M'))              \\\n            .mean()                                                  \\\n            .fillna(0)                                               \\\n\n        # Compute the std deviation from the mean\n        series_std  = articles_curr_prod[['t_dat', 'price']]         \\\n            .groupby(pd.Grouper(key='t_dat', freq='M'))              \\\n            .std()                                                   \\\n            .fillna(0)                                               \\\n\n        # Plot the main line\n        rnd_color = random.choice(COLORS)\n        ax[i, j].plot(series_mean, linewidth=5, color=rnd_color)\n\n        # Plot the 2 standard deviations\n        ax[i, j].fill_between(series_mean.index,                                      \\\n                              (series_mean.values - 2 * series_std.values).flatten(), \\\n                              (series_mean.values + 2 * series_std.values).flatten(), \\\n                              color=rnd_color,                                        \\\n                              alpha=0.1)                                              \\\n\n        # Set title and labels\n        ax[i, j].set_title(f'`{product}` price in time')\n\n        # Move to the next product\n        k += 1\n            \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:27:29.561290Z","iopub.execute_input":"2022-03-29T07:27:29.561737Z","iopub.status.idle":"2022-03-29T07:28:25.215069Z","shell.execute_reply.started":"2022-03-29T07:27:29.561600Z","shell.execute_reply":"2022-03-29T07:28:25.213718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pareto principle","metadata":{}},{"cell_type":"code","source":"pareto_df = transactions_merged[['article_id', 'price']]\npareto_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:25.217297Z","iopub.execute_input":"2022-03-29T07:28:25.218404Z","iopub.status.idle":"2022-03-29T07:28:25.534955Z","shell.execute_reply.started":"2022-03-29T07:28:25.218353Z","shell.execute_reply":"2022-03-29T07:28:25.533692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sales = transactions_merged['price'].sum()\ntotal_sales","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:25.537759Z","iopub.execute_input":"2022-03-29T07:28:25.538224Z","iopub.status.idle":"2022-03-29T07:28:25.630669Z","shell.execute_reply.started":"2022-03-29T07:28:25.538168Z","shell.execute_reply":"2022-03-29T07:28:25.629461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sort the `pareto_df` in descending order by `price`\npareto_sorted = pareto_df           \\\n    .groupby('article_id')['price'] \\\n    .sum()                          \\\n    .sort_values(ascending=False)   \\","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:25.632444Z","iopub.execute_input":"2022-03-29T07:28:25.632827Z","iopub.status.idle":"2022-03-29T07:28:26.468282Z","shell.execute_reply.started":"2022-03-29T07:28:25.632783Z","shell.execute_reply":"2022-03-29T07:28:26.467275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pareto_sorted.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:26.470141Z","iopub.execute_input":"2022-03-29T07:28:26.470745Z","iopub.status.idle":"2022-03-29T07:28:26.479075Z","shell.execute_reply.started":"2022-03-29T07:28:26.470704Z","shell.execute_reply":"2022-03-29T07:28:26.478027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pareto_sorted.tail()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:26.480361Z","iopub.execute_input":"2022-03-29T07:28:26.480626Z","iopub.status.idle":"2022-03-29T07:28:26.495859Z","shell.execute_reply.started":"2022-03-29T07:28:26.480594Z","shell.execute_reply":"2022-03-29T07:28:26.494961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's see how much top 1% products contribute to the overall sales\ntop_1p_articles = pareto_sorted.head(int(pareto_sorted.shape[0] * 0.01))\nprint(f'~{round(top_1p_articles.sum() / total_sales * 100, 2)}% of the sales belongs from the top 1% products.')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:26.497407Z","iopub.execute_input":"2022-03-29T07:28:26.498125Z","iopub.status.idle":"2022-03-29T07:28:26.509591Z","shell.execute_reply.started":"2022-03-29T07:28:26.498082Z","shell.execute_reply":"2022-03-29T07:28:26.508477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's generalize: top i% articles contribute to the overall sales \ntop_np_articles = []\nfor i in range(1, 101):\n    top_articles = pareto_sorted.head(int(pareto_sorted.shape[0] * 0.01 * i))\n    top_np_articles.append(top_articles.sum() / total_sales)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:26.511161Z","iopub.execute_input":"2022-03-29T07:28:26.512163Z","iopub.status.idle":"2022-03-29T07:28:26.543146Z","shell.execute_reply.started":"2022-03-29T07:28:26.512122Z","shell.execute_reply":"2022-03-29T07:28:26.542061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can now plot the graph describing the event: `top n% articles contribute to the overall sales with  x%`\nplot_sns_lineplot(x=[i/100 for i in range(1, 101)],\n                  y=top_np_articles,\n                  x_label='top n% articles (normalized)',\n                  y_label='sales (normalized)',\n                  draw_pareto_line=True)\n\n# As we can see, the top 20% products contribute to the overall sales with 80%\n# \"The Pareto principle states that for many outcomes, roughly 80% of consequences come from 20% of causes\"","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:26.544730Z","iopub.execute_input":"2022-03-29T07:28:26.545014Z","iopub.status.idle":"2022-03-29T07:28:26.951250Z","shell.execute_reply.started":"2022-03-29T07:28:26.544981Z","shell.execute_reply":"2022-03-29T07:28:26.950146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### articles_with_images","metadata":{}},{"cell_type":"code","source":"# Select the most 5 expensive articles in the last 3 days\nmax_price_articles = transactions[transactions.t_dat >= transactions.t_dat.max() - pd.Timedelta(days=3)].sort_values('price', ascending=False).iloc[:5][['article_id', 'price']]\nplot_articles_images(max_price_articles)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:26.953091Z","iopub.execute_input":"2022-03-29T07:28:26.953406Z","iopub.status.idle":"2022-03-29T07:28:29.080177Z","shell.execute_reply.started":"2022-03-29T07:28:26.953369Z","shell.execute_reply":"2022-03-29T07:28:29.078798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select the cheapest 5 articles in the last 3 days\nmin_price_articles = transactions[transactions.t_dat >= transactions.t_dat.max() - pd.Timedelta(days=3)].sort_values('price', ascending=True).iloc[:5][['article_id', 'price']]\nplot_articles_images(min_price_articles)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T07:28:29.081974Z","iopub.execute_input":"2022-03-29T07:28:29.083006Z","iopub.status.idle":"2022-03-29T07:28:31.008535Z","shell.execute_reply.started":"2022-03-29T07:28:29.082936Z","shell.execute_reply":"2022-03-29T07:28:31.007440Z"},"trusted":true},"execution_count":null,"outputs":[]}]}