{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom glob import glob\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nimport random\nfrom PIL import Image\nimport math\nimport itertools\nfrom plotly.graph_objects import treemap\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.io as pio\nimport plotly.subplots as sp\nimport gc\nfrom collections import Counter\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.io as pio\nimport plotly.subplots as sp\n\ndef count_items(series):\n    return len(series[~series.isna()])\n\ndef show_clear_plt():\n    plt.tight_layout()\n    plt.show()\n    plt.clf()\n\n\nsns.set(font_scale=1.5)\nsns.set_style(style='darkgrid')\nplt.rcParams['figure.figsize'] = (10, 6)\nplt.rcParams['legend.facecolor'] = 'white'\n\ndef reduce_memory_usage(df, columns, verbose=True):\n    numerics = [\"int8\", \"int16\", \"int32\", \"int64\", \"float16\", \"float32\", \"float64\"]\n    start_mem = df.memory_usage().sum() / 1024 ** 2\n    for col in columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if (\n                        c_min > np.finfo(np.float16).min\n                        and c_max < np.finfo(np.float16).max\n                ):\n                    df[col] = df[col].astype(np.float16)\n                elif (\n                        c_min > np.finfo(np.float32).min\n                        and c_max < np.finfo(np.float32).max\n                ):\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024 ** 2\n    if verbose:\n        print(\n            \"Mem. usage decreased to {:.2f} Mb ({:.1f}% reduction)\".format(\n                end_mem, 100 * (start_mem - end_mem) / start_mem\n            )\n        )\n    return df","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-19T03:25:37.778464Z","iopub.execute_input":"2022-02-19T03:25:37.778726Z","iopub.status.idle":"2022-02-19T03:25:37.797311Z","shell.execute_reply.started":"2022-02-19T03:25:37.778699Z","shell.execute_reply":"2022-02-19T03:25:37.796345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some plots of which types of items / colours are popular depending on the time of year.\n\nNot sure how useful this is just found it interesting to explore briefly. We can see how some types of clothing or some colours are much more popular at certain times of year.\n\nI haven't read all the EDA notebooks for this comp so apologies to anyone in advance who has already done this in a public notebook...","metadata":{}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    KAGGLE = os.path.exists('../input/h-and-m-personalized-fashion-recommendations/')\n\n    if KAGGLE:\n        PATH = '../input/h-and-m-personalized-fashion-recommendations/'\n        print('running on Kaggle')\n    else:\n        PATH = 'NA'\n        print('not running on Kaggle')\n\n    DEBUG = False\n    DEBUG_PC = 0.1\n\n    print(f'debugging / reduce data rows is {DEBUG}')\n\n    EXAMPLE_LIMIT = 10","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-19T03:24:57.509523Z","iopub.execute_input":"2022-02-19T03:24:57.509725Z","iopub.status.idle":"2022-02-19T03:24:57.51632Z","shell.execute_reply.started":"2022-02-19T03:24:57.509702Z","shell.execute_reply":"2022-02-19T03:24:57.515464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = pd.read_parquet('../input/h-m-save-to-parquet/transactions_train.parquet',\n                     )\ncustomers = pd.read_parquet('../input/h-m-save-to-parquet/customers.parquet')\narticles = pd.read_parquet('../input/h-m-save-to-parquet/articles.parquet',\n                      )\n\nprint(transactions.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:24:57.517448Z","iopub.execute_input":"2022-02-19T03:24:57.517709Z","iopub.status.idle":"2022-02-19T03:25:05.481497Z","shell.execute_reply.started":"2022-02-19T03:24:57.51768Z","shell.execute_reply":"2022-02-19T03:25:05.48067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data subsampling","metadata":{}},{"cell_type":"markdown","source":"(this reduces data size if DEBUG selected in CONFIG)","metadata":{}},{"cell_type":"code","source":"if CONFIG.DEBUG:\n    random.seed(42)\n\n    # subsample the articles\n    unique_articles = articles['article_id'].unique().tolist()\n    sample_articles = random.sample(unique_articles, int(CONFIG.DEBUG_PC * len(unique_articles)))\n    print(f'number of sample articles {len(sample_articles)}')\n\n    print(articles.shape)\n    articles = articles[articles['article_id'].isin(sample_articles)].reset_index(drop=True)\n    print(articles.shape)\n\n    # subsample the customers\n    unique_customers = customers['customer_id'].unique().tolist()\n    sample_customers = random.sample(unique_customers, int(CONFIG.DEBUG_PC * len(unique_customers)))\n    print(f'number of sample customers {len(sample_customers)}')\n\n    print(customers.shape)\n    customers = customers[customers['customer_id'].isin(sample_customers)].reset_index(drop=True)\n    print(customers.shape)\n\n    print(f'original train transactions shape {transactions.shape}')\n    transactions = transactions[(transactions['customer_id'].isin(sample_customers)) &\n                                (transactions['article_id'].isin(sample_articles))].reset_index(drop=True)\n\n    print(f'reduced train transactions shape {transactions.shape}')\n    \nelse:\n    print('running with all train data')","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-19T03:25:05.483037Z","iopub.execute_input":"2022-02-19T03:25:05.483368Z","iopub.status.idle":"2022-02-19T03:25:05.493574Z","shell.execute_reply.started":"2022-02-19T03:25:05.48332Z","shell.execute_reply":"2022-02-19T03:25:05.492761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check seasonality of product groups, colours","metadata":{}},{"cell_type":"markdown","source":"Step 1 - use cyclic encoding on dates","metadata":{}},{"cell_type":"code","source":"transactions['day_of_year'] = transactions['t_dat'].dt.dayofyear\ntransactions['week'] = transactions['t_dat'].dt.isocalendar().week\ntransactions['quarter'] = transactions['t_dat'].dt.quarter\ntransactions['month'] = transactions['t_dat'].dt.month\n\ndef cyclic_encode(df, column):\n    df[f'{column}_sin'] = np.sin(2 * np.pi * df[column] / df[column].max())\n    df[f'{column}_cos'] = np.cos(2 * np.pi * df[column] / df[column].max())\n    return df\n\n\nencode_cols = [\n    'day_of_year',\n]\n\nfor ec in encode_cols:\n    transactions[ec] = transactions[ec] - transactions[ec].min()\n    transactions = cyclic_encode(transactions, ec)\n\n# display example\n\ncyclic_outputs = ['day_of_year_sin', 'day_of_year_cos']\n\ndaily_transactions = transactions.groupby(['t_dat'])[cyclic_outputs].mean()\n\nfor c in daily_transactions.columns:\n    sns.lineplot(x=daily_transactions[c].resample('w').mean().index,\n                 y=daily_transactions[c].resample('w').mean().values,\n                 linewidth=4)\nplt.title('Time of year cyclic encoding')\nplt.legend(daily_transactions.columns.tolist())\nplt.ylabel('Cyclic encoding')\nshow_clear_plt()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:33:53.925211Z","iopub.execute_input":"2022-02-19T03:33:53.925543Z","iopub.status.idle":"2022-02-19T03:34:14.798145Z","shell.execute_reply.started":"2022-02-19T03:33:53.92551Z","shell.execute_reply":"2022-02-19T03:34:14.797198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below demonstrates the position of months 1-12 in the cycle\n\nWinter (for northern hemisphere - which looks to be the location of this data) is in the upper part of the outputs.","metadata":{}},{"cell_type":"code","source":"qs = transactions.groupby(['month'])[cyclic_outputs].median()\nfig, axes = plt.subplots(figsize=(8, 8))\nplt.scatter(x=qs[cyclic_outputs[0]],\n            y=qs[cyclic_outputs[1]])\n\nfor i in qs.index:\n    plt.annotate(f'Month {i}', (qs.loc[i, cyclic_outputs[0]],\n                     qs.loc[i, cyclic_outputs[1]]),\n                 ha='center',\n                 va='center',\n                 fontsize=20)\nplt.xlabel(cyclic_outputs[0])\nplt.ylabel(cyclic_outputs[1])\nplt.xlim(-1.2,1.2)\nplt.ylim(-1.2,1.2)\nplt.title('Months as cycle',\n          fontsize=24)\nshow_clear_plt()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:34:14.799443Z","iopub.execute_input":"2022-02-19T03:34:14.799797Z","iopub.status.idle":"2022-02-19T03:34:16.766147Z","shell.execute_reply.started":"2022-02-19T03:34:14.799764Z","shell.execute_reply":"2022-02-19T03:34:16.765606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_category_seasonality(column, n_examples=5):\n\n    transactions[column] = transactions['article_id'].map(dict(zip(articles['article_id'],\n                                                               articles[column])))\n\n    feature_dict = {cyclic_outputs[0]: [np.mean],\n                    cyclic_outputs[1]: [np.mean],\n                    'article_id': [count_items],}\n\n    top_examples = transactions[column].value_counts().sort_values(ascending=False).index[:n_examples].tolist()\n\n    cat_seasonality = transactions[transactions[column].isin(top_examples)].groupby([column]).agg(feature_dict)\n    cat_seasonality.columns = [f'{a}_{b}' for a,b in cat_seasonality.columns]\n\n    fig = px.scatter(cat_seasonality, x=cat_seasonality[f'{cyclic_outputs[0]}_mean'],\n                     y=cat_seasonality[f'{cyclic_outputs[1]}_mean'],\n                     size=cat_seasonality['article_id_count_items'],\n                      color=cat_seasonality[f'{cyclic_outputs[1]}_mean'],\n                     text=cat_seasonality.index,\n                     log_x=False,\n                     size_max=150,\n                     color_continuous_scale='solar_r',\n                     )\n\n    fig.update_layout(\n        title=dict(text=f'<b>Examples for category {column} weighted by time of year (as cycle)<b>',\n                   font=dict(\n                       family=\"Arial\",\n                       size=20,\n                       color='#000000'\n                   )),\n\n        xaxis_title=\"Autumn (left) to Spring (right)\",\n        yaxis_title=\"Summer (lower) to Winter (upper)\",\n        height=600,\n        width=900,\n        font=dict(\n            family=\"Arial\",\n            size=18,\n            color='#000000'\n        )\n    )\n\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:34:19.165986Z","iopub.execute_input":"2022-02-19T03:34:19.166273Z","iopub.status.idle":"2022-02-19T03:34:19.175746Z","shell.execute_reply.started":"2022-02-19T03:34:19.166227Z","shell.execute_reply":"2022-02-19T03:34:19.174818Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_category_seasonality('product_type_name', n_examples=CONFIG.EXAMPLE_LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:34:19.530838Z","iopub.execute_input":"2022-02-19T03:34:19.53143Z","iopub.status.idle":"2022-02-19T03:34:34.69271Z","shell.execute_reply.started":"2022-02-19T03:34:19.531396Z","shell.execute_reply":"2022-02-19T03:34:34.691777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_category_seasonality('product_group_name', n_examples=CONFIG.EXAMPLE_LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:30:09.940289Z","iopub.execute_input":"2022-02-19T03:30:09.940516Z","iopub.status.idle":"2022-02-19T03:30:32.359537Z","shell.execute_reply.started":"2022-02-19T03:30:09.94049Z","shell.execute_reply":"2022-02-19T03:30:32.358586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_category_seasonality('index_group_name', n_examples=CONFIG.EXAMPLE_LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:30:32.361645Z","iopub.execute_input":"2022-02-19T03:30:32.362032Z","iopub.status.idle":"2022-02-19T03:30:54.946297Z","shell.execute_reply.started":"2022-02-19T03:30:32.361991Z","shell.execute_reply":"2022-02-19T03:30:54.945489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_category_seasonality('garment_group_name', n_examples=CONFIG.EXAMPLE_LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:30:54.947244Z","iopub.execute_input":"2022-02-19T03:30:54.94748Z","iopub.status.idle":"2022-02-19T03:31:16.91052Z","shell.execute_reply.started":"2022-02-19T03:30:54.947453Z","shell.execute_reply":"2022-02-19T03:31:16.909503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_category_seasonality('index_name', n_examples=CONFIG.EXAMPLE_LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:35:27.12796Z","iopub.execute_input":"2022-02-19T03:35:27.128326Z","iopub.status.idle":"2022-02-19T03:35:53.972352Z","shell.execute_reply.started":"2022-02-19T03:35:27.128276Z","shell.execute_reply":"2022-02-19T03:35:53.971479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_category_seasonality('colour_group_name', n_examples=CONFIG.EXAMPLE_LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:31:48.237384Z","iopub.execute_input":"2022-02-19T03:31:48.237651Z","iopub.status.idle":"2022-02-19T03:32:11.105835Z","shell.execute_reply.started":"2022-02-19T03:31:48.237623Z","shell.execute_reply":"2022-02-19T03:32:11.105097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_category_seasonality('perceived_colour_value_name', n_examples=CONFIG.EXAMPLE_LIMIT)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T03:32:11.107507Z","iopub.execute_input":"2022-02-19T03:32:11.1133Z","iopub.status.idle":"2022-02-19T03:32:30.218833Z","shell.execute_reply.started":"2022-02-19T03:32:11.11322Z","shell.execute_reply":"2022-02-19T03:32:30.218022Z"},"trusted":true},"execution_count":null,"outputs":[]}]}