{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-25T16:22:26.572098Z","iopub.execute_input":"2022-10-25T16:22:26.572967Z","iopub.status.idle":"2022-10-25T16:24:03.102643Z","shell.execute_reply.started":"2022-10-25T16:22:26.572863Z","shell.execute_reply":"2022-10-25T16:24:03.101587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:24:03.104523Z","iopub.execute_input":"2022-10-25T16:24:03.105242Z","iopub.status.idle":"2022-10-25T16:24:03.817883Z","shell.execute_reply.started":"2022-10-25T16:24:03.105202Z","shell.execute_reply":"2022-10-25T16:24:03.816794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv')\ncustomers = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/customers.csv')\ntransactions = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:24:03.819269Z","iopub.execute_input":"2022-10-25T16:24:03.819803Z","iopub.status.idle":"2022-10-25T16:25:31.661867Z","shell.execute_reply.started":"2022-10-25T16:24:03.819760Z","shell.execute_reply":"2022-10-25T16:25:31.660656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:31.665073Z","iopub.execute_input":"2022-10-25T16:25:31.666364Z","iopub.status.idle":"2022-10-25T16:25:31.729231Z","shell.execute_reply.started":"2022-10-25T16:25:31.666269Z","shell.execute_reply":"2022-10-25T16:25:31.728249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pecentage of missing values\n(articles.isnull().sum()/articles.shape[0])*100","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:31.730504Z","iopub.execute_input":"2022-10-25T16:25:31.731043Z","iopub.status.idle":"2022-10-25T16:25:31.811418Z","shell.execute_reply.started":"2022-10-25T16:25:31.731009Z","shell.execute_reply":"2022-10-25T16:25:31.810105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:31.812817Z","iopub.execute_input":"2022-10-25T16:25:31.813596Z","iopub.status.idle":"2022-10-25T16:25:31.821509Z","shell.execute_reply.started":"2022-10-25T16:25:31.813557Z","shell.execute_reply":"2022-10-25T16:25:31.820102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_desc = articles[articles['detail_desc'].isna()]","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:31.823210Z","iopub.execute_input":"2022-10-25T16:25:31.823708Z","iopub.status.idle":"2022-10-25T16:25:31.846893Z","shell.execute_reply.started":"2022-10-25T16:25:31.823649Z","shell.execute_reply":"2022-10-25T16:25:31.845634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_desc.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:31.848155Z","iopub.execute_input":"2022-10-25T16:25:31.849227Z","iopub.status.idle":"2022-10-25T16:25:31.881719Z","shell.execute_reply.started":"2022-10-25T16:25:31.849178Z","shell.execute_reply":"2022-10-25T16:25:31.879980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_value_counts_plot(df, column):\n    count_df = df[column].value_counts(sort=True).reset_index().rename(columns = {'index':column, column:'count'})\n    count_df = count_df.iloc[:20,:][::-1]\n    plt.figure(figsize=(10,6))\n    plt.barh(count_df[column],count_df['count'])\n    plt.title(\"count plot of \" + column)\n    plt.xlabel('Count')\n    plt.ylabel(column)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:31.884017Z","iopub.execute_input":"2022-10-25T16:25:31.884819Z","iopub.status.idle":"2022-10-25T16:25:31.892654Z","shell.execute_reply.started":"2022-10-25T16:25:31.884777Z","shell.execute_reply":"2022-10-25T16:25:31.891388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'prod_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:31.896775Z","iopub.execute_input":"2022-10-25T16:25:31.897230Z","iopub.status.idle":"2022-10-25T16:25:32.294645Z","shell.execute_reply.started":"2022-10-25T16:25:31.897194Z","shell.execute_reply":"2022-10-25T16:25:32.293115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'product_type_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:32.296714Z","iopub.execute_input":"2022-10-25T16:25:32.297774Z","iopub.status.idle":"2022-10-25T16:25:32.606911Z","shell.execute_reply.started":"2022-10-25T16:25:32.297719Z","shell.execute_reply":"2022-10-25T16:25:32.605551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'product_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:32.608399Z","iopub.execute_input":"2022-10-25T16:25:32.608771Z","iopub.status.idle":"2022-10-25T16:25:32.939661Z","shell.execute_reply.started":"2022-10-25T16:25:32.608736Z","shell.execute_reply":"2022-10-25T16:25:32.938495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'department_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:32.941472Z","iopub.execute_input":"2022-10-25T16:25:32.942192Z","iopub.status.idle":"2022-10-25T16:25:33.250924Z","shell.execute_reply.started":"2022-10-25T16:25:32.942149Z","shell.execute_reply":"2022-10-25T16:25:33.249513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'colour_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:33.253250Z","iopub.execute_input":"2022-10-25T16:25:33.253814Z","iopub.status.idle":"2022-10-25T16:25:33.554288Z","shell.execute_reply.started":"2022-10-25T16:25:33.253778Z","shell.execute_reply":"2022-10-25T16:25:33.552928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'section_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:33.556267Z","iopub.execute_input":"2022-10-25T16:25:33.556780Z","iopub.status.idle":"2022-10-25T16:25:33.892335Z","shell.execute_reply.started":"2022-10-25T16:25:33.556726Z","shell.execute_reply":"2022-10-25T16:25:33.890895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'garment_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:33.893647Z","iopub.execute_input":"2022-10-25T16:25:33.893995Z","iopub.status.idle":"2022-10-25T16:25:34.206632Z","shell.execute_reply.started":"2022-10-25T16:25:33.893964Z","shell.execute_reply":"2022-10-25T16:25:34.205195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(articles,'index_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:34.207940Z","iopub.execute_input":"2022-10-25T16:25:34.208293Z","iopub.status.idle":"2022-10-25T16:25:34.470113Z","shell.execute_reply.started":"2022-10-25T16:25:34.208261Z","shell.execute_reply":"2022-10-25T16:25:34.468863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles.groupby([\"product_group_name\"])[\"product_type_name\"].nunique()\ntemp_df = pd.DataFrame({'Product Group': temp.index,\n                   'Product Types': temp.values\n                  })\ntemp_df = temp_df.sort_values(['Product Types'], ascending=False)\nplt.figure(figsize = (8,6))\nplt.title('Number of Product Types per each Product Group')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Product Group', y=\"Product Types\", data=temp_df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:34.471847Z","iopub.execute_input":"2022-10-25T16:25:34.472756Z","iopub.status.idle":"2022-10-25T16:25:34.853473Z","shell.execute_reply.started":"2022-10-25T16:25:34.472716Z","shell.execute_reply":"2022-10-25T16:25:34.852144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_stacked_count_bar(df, column1, column2):\n    f, ax = plt.subplots(figsize=(15, 7))\n    ax = sns.histplot(data=df, y=column1, color='orange', hue=column2, multiple=\"stack\")\n    ax.set_xlabel('count by ' + column1)\n    ax.set_ylabel(column1)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:34.854866Z","iopub.execute_input":"2022-10-25T16:25:34.855246Z","iopub.status.idle":"2022-10-25T16:25:34.862767Z","shell.execute_reply.started":"2022-10-25T16:25:34.855213Z","shell.execute_reply":"2022-10-25T16:25:34.861437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_stacked_count_bar(articles,'garment_group_name', 'index_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:34.864578Z","iopub.execute_input":"2022-10-25T16:25:34.865124Z","iopub.status.idle":"2022-10-25T16:25:36.123518Z","shell.execute_reply.started":"2022-10-25T16:25:34.865045Z","shell.execute_reply":"2022-10-25T16:25:36.122172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_stacked_count_bar(articles,'product_group_name', 'index_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:36.125331Z","iopub.execute_input":"2022-10-25T16:25:36.126241Z","iopub.status.idle":"2022-10-25T16:25:37.173548Z","shell.execute_reply.started":"2022-10-25T16:25:36.126199Z","shell.execute_reply":"2022-10-25T16:25:37.171986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_stacked_count_bar(articles,'garment_group_name', 'index_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:37.175224Z","iopub.execute_input":"2022-10-25T16:25:37.176190Z","iopub.status.idle":"2022-10-25T16:25:38.014206Z","shell.execute_reply.started":"2022-10-25T16:25:37.176147Z","shell.execute_reply":"2022-10-25T16:25:38.013349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_stacked_count_bar(articles,'perceived_colour_master_name', 'index_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:38.015405Z","iopub.execute_input":"2022-10-25T16:25:38.016014Z","iopub.status.idle":"2022-10-25T16:25:38.803347Z","shell.execute_reply.started":"2022-10-25T16:25:38.015978Z","shell.execute_reply":"2022-10-25T16:25:38.802397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:38.804978Z","iopub.execute_input":"2022-10-25T16:25:38.805569Z","iopub.status.idle":"2022-10-25T16:25:38.823591Z","shell.execute_reply.started":"2022-10-25T16:25:38.805531Z","shell.execute_reply":"2022-10-25T16:25:38.822292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(customers.isna().sum()/customers.shape[0])*100","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:38.825086Z","iopub.execute_input":"2022-10-25T16:25:38.825439Z","iopub.status.idle":"2022-10-25T16:25:39.122560Z","shell.execute_reply.started":"2022-10-25T16:25:38.825406Z","shell.execute_reply":"2022-10-25T16:25:39.121123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:39.123941Z","iopub.execute_input":"2022-10-25T16:25:39.124307Z","iopub.status.idle":"2022-10-25T16:25:39.132345Z","shell.execute_reply.started":"2022-10-25T16:25:39.124265Z","shell.execute_reply":"2022-10-25T16:25:39.131129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(customers['age'], kde=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:39.134909Z","iopub.execute_input":"2022-10-25T16:25:39.135377Z","iopub.status.idle":"2022-10-25T16:25:45.996459Z","shell.execute_reply.started":"2022-10-25T16:25:39.135338Z","shell.execute_reply":"2022-10-25T16:25:45.995139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(customers, 'club_member_status')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:46.004598Z","iopub.execute_input":"2022-10-25T16:25:46.005011Z","iopub.status.idle":"2022-10-25T16:25:46.295290Z","shell.execute_reply.started":"2022-10-25T16:25:46.004979Z","shell.execute_reply":"2022-10-25T16:25:46.294333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(customers, 'fashion_news_frequency')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:46.296766Z","iopub.execute_input":"2022-10-25T16:25:46.297658Z","iopub.status.idle":"2022-10-25T16:25:46.580854Z","shell.execute_reply.started":"2022-10-25T16:25:46.297615Z","shell.execute_reply":"2022-10-25T16:25:46.579707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers['fashion_news_frequency'].replace('None','NONE',inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:46.582727Z","iopub.execute_input":"2022-10-25T16:25:46.583316Z","iopub.status.idle":"2022-10-25T16:25:46.629077Z","shell.execute_reply.started":"2022-10-25T16:25:46.583278Z","shell.execute_reply":"2022-10-25T16:25:46.627710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_value_counts_plot(customers, 'fashion_news_frequency')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:46.630731Z","iopub.execute_input":"2022-10-25T16:25:46.632756Z","iopub.status.idle":"2022-10-25T16:25:46.914938Z","shell.execute_reply.started":"2022-10-25T16:25:46.632704Z","shell.execute_reply":"2022-10-25T16:25:46.913324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=customers, x='age', hue='club_member_status', multiple='stack', kde=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:46.916634Z","iopub.execute_input":"2022-10-25T16:25:46.917182Z","iopub.status.idle":"2022-10-25T16:25:56.176036Z","shell.execute_reply.started":"2022-10-25T16:25:46.917132Z","shell.execute_reply":"2022-10-25T16:25:56.174501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=customers, x='age', hue='fashion_news_frequency', multiple='stack', kde=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:25:56.177543Z","iopub.execute_input":"2022-10-25T16:25:56.178199Z","iopub.status.idle":"2022-10-25T16:26:05.217573Z","shell.execute_reply.started":"2022-10-25T16:25:56.178156Z","shell.execute_reply":"2022-10-25T16:26:05.216147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(customers['postal_code'].value_counts())\nprint(len(customers['postal_code'].value_counts()))","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:05.219393Z","iopub.execute_input":"2022-10-25T16:26:05.219879Z","iopub.status.idle":"2022-10-25T16:26:06.792643Z","shell.execute_reply.started":"2022-10-25T16:26:05.219824Z","shell.execute_reply":"2022-10-25T16:26:06.791158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:06.794470Z","iopub.execute_input":"2022-10-25T16:26:06.795800Z","iopub.status.idle":"2022-10-25T16:26:06.810624Z","shell.execute_reply.started":"2022-10-25T16:26:06.795741Z","shell.execute_reply":"2022-10-25T16:26:06.809072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:06.812618Z","iopub.execute_input":"2022-10-25T16:26:06.813009Z","iopub.status.idle":"2022-10-25T16:26:06.822688Z","shell.execute_reply.started":"2022-10-25T16:26:06.812973Z","shell.execute_reply":"2022-10-25T16:26:06.821253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['t_dat'] = pd.to_datetime(transactions['t_dat'])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:06.824023Z","iopub.execute_input":"2022-10-25T16:26:06.824394Z","iopub.status.idle":"2022-10-25T16:26:12.039920Z","shell.execute_reply.started":"2022-10-25T16:26:06.824361Z","shell.execute_reply":"2022-10-25T16:26:12.038538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:12.041400Z","iopub.execute_input":"2022-10-25T16:26:12.041794Z","iopub.status.idle":"2022-10-25T16:26:12.050995Z","shell.execute_reply.started":"2022-10-25T16:26:12.041750Z","shell.execute_reply":"2022-10-25T16:26:12.049584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(transactions['price'])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:12.052635Z","iopub.execute_input":"2022-10-25T16:26:12.053641Z","iopub.status.idle":"2022-10-25T16:26:12.994965Z","shell.execute_reply.started":"2022-10-25T16:26:12.053587Z","shell.execute_reply":"2022-10-25T16:26:12.993388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_transactions = articles[['article_id', 'prod_name', 'product_type_name', 'product_group_name','graphical_appearance_name','colour_group_name','perceived_colour_value_name','perceived_colour_master_name','department_name','index_group_name', 'index_name','section_name','garment_group_name']]\narticle_transactions = transactions[['customer_id', 'article_id', 'price', 't_dat']].merge(article_transactions, on='article_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:12.996451Z","iopub.execute_input":"2022-10-25T16:26:12.996809Z","iopub.status.idle":"2022-10-25T16:26:32.884569Z","shell.execute_reply.started":"2022-10-25T16:26:12.996776Z","shell.execute_reply":"2022-10-25T16:26:32.883034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:32.886106Z","iopub.execute_input":"2022-10-25T16:26:32.886462Z","iopub.status.idle":"2022-10-25T16:26:32.910342Z","shell.execute_reply.started":"2022-10-25T16:26:32.886429Z","shell.execute_reply":"2022-10-25T16:26:32.908782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_box_plot(df, x, y):\n    f, ax = plt.subplots(figsize=(25,15))\n    ax = sns.boxplot(data=df, x=x, y=y, showmeans=True)\n    ax.set_xlabel(x + ' outliers', fontsize=22)\n    ax.set_ylabel(y, fontsize=22)\n    ax.xaxis.set_tick_params(labelsize=22)\n    ax.yaxis.set_tick_params(labelsize=22)\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:32.912003Z","iopub.execute_input":"2022-10-25T16:26:32.912510Z","iopub.status.idle":"2022-10-25T16:26:32.924443Z","shell.execute_reply.started":"2022-10-25T16:26:32.912470Z","shell.execute_reply":"2022-10-25T16:26:32.922668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_box_plot(article_transactions, 'price', 'product_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:32.926601Z","iopub.execute_input":"2022-10-25T16:26:32.926986Z","iopub.status.idle":"2022-10-25T16:26:54.781474Z","shell.execute_reply.started":"2022-10-25T16:26:32.926953Z","shell.execute_reply":"2022-10-25T16:26:54.779954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_box_plot(article_transactions, 'price','index_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:26:54.783230Z","iopub.execute_input":"2022-10-25T16:26:54.783732Z","iopub.status.idle":"2022-10-25T16:27:15.451464Z","shell.execute_reply.started":"2022-10-25T16:26:54.783676Z","shell.execute_reply":"2022-10-25T16:27:15.450081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_box_plot(article_transactions, 'price','index_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:27:15.453341Z","iopub.execute_input":"2022-10-25T16:27:15.453749Z","iopub.status.idle":"2022-10-25T16:27:36.836477Z","shell.execute_reply.started":"2022-10-25T16:27:15.453714Z","shell.execute_reply":"2022-10-25T16:27:36.835161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_box_plot(article_transactions, 'price','perceived_colour_master_name')","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:27:36.838144Z","iopub.execute_input":"2022-10-25T16:27:36.839358Z","iopub.status.idle":"2022-10-25T16:27:58.520904Z","shell.execute_reply.started":"2022-10-25T16:27:36.839299Z","shell.execute_reply":"2022-10-25T16:27:58.519993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_list = ['Shoes', 'Garment Full body', 'Bags', 'Garment Lower body', 'Underwear/nightwear']\ncolors = ['cadetblue', 'orange', 'mediumspringgreen', 'tomato', 'lightseagreen']\nk = 0\nf, ax = plt.subplots(3, 2, figsize=(20, 15))\nfor i in range(3):\n    for j in range(2):\n        try:\n            product = product_list[k]\n            articles_for_merge_product = article_transactions[article_transactions.product_group_name == product_list[k]]\n            series_mean = articles_for_merge_product[['t_dat', 'price']].groupby(pd.Grouper(key=\"t_dat\", freq='M')).mean().fillna(0)\n            series_std = articles_for_merge_product[['t_dat', 'price']].groupby(pd.Grouper(key=\"t_dat\", freq='M')).std().fillna(0)\n            ax[i, j].plot(series_mean, linewidth=4, color=colors[k])\n            ax[i, j].fill_between(series_mean.index, (series_mean.values-2*series_std.values).ravel(), \n                             (series_mean.values+2*series_std.values).ravel(), color=colors[k], alpha=.1)\n            ax[i, j].set_title(f'Mean {product_list[k]} price in time')\n            ax[i, j].set_xlabel('month')\n            ax[i, j].set_xlabel(f'{product_list[k]}')\n            k += 1\n        except IndexError:\n            ax[i, j].set_visible(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:27:58.522603Z","iopub.execute_input":"2022-10-25T16:27:58.523306Z","iopub.status.idle":"2022-10-25T16:28:35.737420Z","shell.execute_reply.started":"2022-10-25T16:27:58.523263Z","shell.execute_reply":"2022-10-25T16:28:35.735921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_group_list = ['Ladieswear', 'Divided', 'Menswear', 'Baby/Children', 'Sport']\ncolors = ['cadetblue', 'orange', 'mediumspringgreen', 'tomato', 'lightseagreen']\nk = 0\nf, ax = plt.subplots(3, 2, figsize=(20, 15))\nfor i in range(3):\n    for j in range(2):\n        try:\n            product = index_group_list[k]\n            articles_for_merge_product = article_transactions[article_transactions.index_group_name == index_group_list[k]]\n            series_mean = articles_for_merge_product[['t_dat', 'price']].groupby(pd.Grouper(key=\"t_dat\", freq='M')).mean().fillna(0)\n            series_std = articles_for_merge_product[['t_dat', 'price']].groupby(pd.Grouper(key=\"t_dat\", freq='M')).std().fillna(0)\n            ax[i, j].plot(series_mean, linewidth=4, color=colors[k])\n            ax[i, j].fill_between(series_mean.index, (series_mean.values-2*series_std.values).ravel(), \n                             (series_mean.values+2*series_std.values).ravel(), color=colors[k], alpha=.1)\n            ax[i, j].set_title(f'Mean {index_group_list[k]} price in time')\n            ax[i, j].set_xlabel('month')\n            ax[i, j].set_xlabel(f'{index_group_list[k]}')\n            k += 1\n        except IndexError:\n            ax[i, j].set_visible(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:28:35.739571Z","iopub.execute_input":"2022-10-25T16:28:35.739965Z","iopub.status.idle":"2022-10-25T16:29:03.706844Z","shell.execute_reply.started":"2022-10-25T16:28:35.739927Z","shell.execute_reply":"2022-10-25T16:29:03.705633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_transactions = article_transactions.merge(customers[['customer_id','club_member_status','fashion_news_frequency','age']], on='customer_id',how='left')\narticle_transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:29:03.708617Z","iopub.execute_input":"2022-10-25T16:29:03.708988Z","iopub.status.idle":"2022-10-25T16:29:50.261936Z","shell.execute_reply.started":"2022-10-25T16:29:03.708954Z","shell.execute_reply":"2022-10-25T16:29:50.260755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"age_price = article_transactions[['age','price']]\nage_price.drop_duplicates(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:29:50.263487Z","iopub.execute_input":"2022-10-25T16:29:50.263843Z","iopub.status.idle":"2022-10-25T16:30:11.332363Z","shell.execute_reply.started":"2022-10-25T16:29:50.263810Z","shell.execute_reply":"2022-10-25T16:30:11.331128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter('age','price',data=age_price)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:11.334226Z","iopub.execute_input":"2022-10-25T16:30:11.334599Z","iopub.status.idle":"2022-10-25T16:30:12.376708Z","shell.execute_reply.started":"2022-10-25T16:30:11.334567Z","shell.execute_reply":"2022-10-25T16:30:12.375080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del age_price","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:12.378670Z","iopub.execute_input":"2022-10-25T16:30:12.379047Z","iopub.status.idle":"2022-10-25T16:30:12.383948Z","shell.execute_reply.started":"2022-10-25T16:30:12.379014Z","shell.execute_reply":"2022-10-25T16:30:12.382469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check=transactions.groupby('customer_id').count()[['article_id']].sort_values('article_id', ascending=False)\nnewcheck=check.head(10)\nnewcheck=newcheck.reset_index()[::-1]\nnewcheck","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:12.385594Z","iopub.execute_input":"2022-10-25T16:30:12.386044Z","iopub.status.idle":"2022-10-25T16:30:27.506845Z","shell.execute_reply.started":"2022-10-25T16:30:12.386007Z","shell.execute_reply":"2022-10-25T16:30:27.505443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.barh('customer_id','article_id',data=newcheck)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:27.508723Z","iopub.execute_input":"2022-10-25T16:30:27.509278Z","iopub.status.idle":"2022-10-25T16:30:27.861617Z","shell.execute_reply.started":"2022-10-25T16:30:27.509238Z","shell.execute_reply":"2022-10-25T16:30:27.860417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.image as mpimg\nmax_price_ids = transactions[transactions.t_dat==transactions.t_dat.max()].sort_values('price', ascending=False).iloc[:6][['article_id', 'price']]\nmin_price_ids = transactions[transactions.t_dat==transactions.t_dat.min()].sort_values('price', ascending=True).iloc[:6][['article_id', 'price']]","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:27.863207Z","iopub.execute_input":"2022-10-25T16:30:27.863906Z","iopub.status.idle":"2022-10-25T16:30:28.534134Z","shell.execute_reply.started":"2022-10-25T16:30:27.863832Z","shell.execute_reply":"2022-10-25T16:30:28.532299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(1, 6, figsize=(20,10))\ni = 0\nfor _, data in max_price_ids.iterrows():\n    desc = articles[articles['article_id'] == data['article_id']]['detail_desc'].iloc[0]\n    desc_list = desc.split(' ')\n    for j, elem in enumerate(desc_list):\n        if j > 0 and j % 5 == 0:\n            desc_list[j] = desc_list[j] + '\\n'\n    desc = ' '.join(desc_list)\n    img = mpimg.imread(f'../input/h-and-m-personalized-fashion-recommendations/images/0{str(data.article_id)[:2]}/0{int(data.article_id)}.jpg')\n    ax[i].imshow(img)\n    ax[i].set_title(f'price: {data.price:.2f}')\n    ax[i].set_xticks([], [])\n    ax[i].set_yticks([], [])\n    ax[i].grid(False)\n    ax[i].set_xlabel(desc, fontsize=10)\n    i += 1\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:28.536659Z","iopub.execute_input":"2022-10-25T16:30:28.537166Z","iopub.status.idle":"2022-10-25T16:30:31.152429Z","shell.execute_reply.started":"2022-10-25T16:30:28.537119Z","shell.execute_reply":"2022-10-25T16:30:31.151137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(1, 6, figsize=(20,10))\ni = 0\nfor _, data in min_price_ids.iterrows():\n    desc = articles[articles['article_id'] == data['article_id']]['detail_desc'].iloc[0]\n    desc_list = desc.split(' ')\n    for j, elem in enumerate(desc_list):\n        if j > 0 and j % 4 == 0:\n            desc_list[j] = desc_list[j] + '\\n'\n    desc = ' '.join(desc_list)\n    img = mpimg.imread(f'../input/h-and-m-personalized-fashion-recommendations/images/0{str(data.article_id)[:2]}/0{int(data.article_id)}.jpg')\n    ax[i].imshow(img)\n    ax[i].set_title(f'price: {data.price:.4f}')\n    ax[i].set_xlabel(desc, fontsize=10)\n    ax[i].set_xticks([], [])\n    ax[i].set_yticks([], [])\n    ax[i].grid(False)\n    i += 1\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:31.154114Z","iopub.execute_input":"2022-10-25T16:30:31.155289Z","iopub.status.idle":"2022-10-25T16:30:33.801547Z","shell.execute_reply.started":"2022-10-25T16:30:31.155237Z","shell.execute_reply":"2022-10-25T16:30:33.799984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nfrom sklearn.utils import shuffle\nfrom sklearn.preprocessing import LabelBinarizer\nfrom keras.applications.xception import Xception,preprocess_input\nimport tensorflow as tf\nfrom keras.preprocessing import image\nfrom keras.layers import Input\nfrom keras.backend import reshape\nfrom sklearn.neighbors import NearestNeighbors\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:33.803271Z","iopub.execute_input":"2022-10-25T16:30:33.803652Z","iopub.status.idle":"2022-10-25T16:30:41.711303Z","shell.execute_reply.started":"2022-10-25T16:30:33.803617Z","shell.execute_reply":"2022-10-25T16:30:41.709969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_dir = '../input/h-and-m-personalized-fashion-recommendations/images'","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:41.713178Z","iopub.execute_input":"2022-10-25T16:30:41.713894Z","iopub.status.idle":"2022-10-25T16:30:41.720432Z","shell.execute_reply.started":"2022-10-25T16:30:41.713856Z","shell.execute_reply":"2022-10-25T16:30:41.718490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getImagePaths(path):\n    image_names = []\n    for dirname, _, filenames in os.walk(path):\n        for filename in filenames:\n            fullpath = os.path.join(dirname, filename)\n            image_names.append(fullpath)\n    return image_names\n\ndef preprocess_img(img_path):\n    dsize = (225,225)\n    new_image=cv2.imread(img_path)\n    new_image=cv2.resize(new_image,dsize,interpolation=cv2.INTER_NEAREST)  \n    new_image=np.expand_dims(new_image,axis=0)\n    new_image=preprocess_input(new_image)\n    return new_image\n\ndef load_data():\n    output=[]\n    output=getImagePaths(images_dir)[:10000]\n    return output","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:41.722065Z","iopub.execute_input":"2022-10-25T16:30:41.722511Z","iopub.status.idle":"2022-10-25T16:30:41.734917Z","shell.execute_reply.started":"2022-10-25T16:30:41.722456Z","shell.execute_reply":"2022-10-25T16:30:41.733428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model():\n    model=Xception(weights='imagenet',include_top=False)\n    for layer in model.layers:\n        layer.trainable=False\n        #model.summary()\n    return model\n\ndef feature_extraction(image_data,model):\n    features=model.predict(image_data)\n    features=np.array(features)\n    features=features.flatten()\n    return features","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:41.736658Z","iopub.execute_input":"2022-10-25T16:30:41.737305Z","iopub.status.idle":"2022-10-25T16:30:41.748694Z","shell.execute_reply.started":"2022-10-25T16:30:41.737251Z","shell.execute_reply":"2022-10-25T16:30:41.747313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def result_vector_cosine(model,feature_vector,new_img):\n    new_feature = model.predict(new_img)\n    new_feature = np.array(new_feature)\n    new_feature = new_feature.flatten()\n    N_result = 12\n    nbrs = NearestNeighbors(n_neighbors=N_result, metric=\"cosine\").fit(feature_vector)\n    distances, indices = nbrs.kneighbors([new_feature])\n    return(indices)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:41.750342Z","iopub.execute_input":"2022-10-25T16:30:41.751920Z","iopub.status.idle":"2022-10-25T16:30:41.761314Z","shell.execute_reply.started":"2022-10-25T16:30:41.751865Z","shell.execute_reply":"2022-10-25T16:30:41.760045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def input_show(data):\n    plt.title(\"Query Image\")\n    plt.imshow(data)\n  \ndef show_result(data,result):\n    fig = plt.figure(figsize=(12,8))\n    for i in range(0,12):\n        index_result=result[0][i]\n        plt.subplot(3,4,i+1)\n        plt.imshow(cv2.imread(data[index_result]))\n    plt.show()\n\ndef main():  \n    features=[]\n    output=load_data()\n    main_model=model()\n    #Limiting the data for training\n    for i in output[:3000]:\n        new_img=preprocess_img(i)\n        features.append(feature_extraction(new_img,main_model))\n    feature_vec = np.array(features)\n    for ind in [4512, 5700, 7654, 3522]:\n        print(\"Suggestions for the product: \")\n        result=result_vector_cosine(main_model,feature_vec,preprocess_img(output[ind]))\n        input_show(cv2.imread(output[ind]))\n        show_result(output,result)\n  \n\nif __name__=='__main__':\n    main()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:30:41.763330Z","iopub.execute_input":"2022-10-25T16:30:41.763830Z","iopub.status.idle":"2022-10-25T16:43:24.458729Z","shell.execute_reply.started":"2022-10-25T16:30:41.763781Z","shell.execute_reply":"2022-10-25T16:43:24.457282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/mayukh18/reco","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:43:24.461144Z","iopub.execute_input":"2022-10-25T16:43:24.461655Z","iopub.status.idle":"2022-10-25T16:44:08.730419Z","shell.execute_reply.started":"2022-10-25T16:43:24.461614Z","shell.execute_reply":"2022-10-25T16:44:08.727916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport reco\nfrom tqdm import tqdm\nimport datetime\nfrom collections import Counter","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:44:08.733985Z","iopub.execute_input":"2022-10-25T16:44:08.734523Z","iopub.status.idle":"2022-10-25T16:44:08.752207Z","shell.execute_reply.started":"2022-10-25T16:44:08.734463Z","shell.execute_reply":"2022-10-25T16:44:08.750719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"All Transactions Date Range: {} to {}\".format(transactions['t_dat'].min(), transactions['t_dat'].max()))","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:44:08.754181Z","iopub.execute_input":"2022-10-25T16:44:08.755142Z","iopub.status.idle":"2022-10-25T16:44:09.469216Z","shell.execute_reply.started":"2022-10-25T16:44:08.755090Z","shell.execute_reply":"2022-10-25T16:44:09.467822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions[\"t_dat\"] = pd.to_datetime(transactions[\"t_dat\"])\ntrain1 = transactions.loc[(transactions[\"t_dat\"] >= datetime.datetime(2020,8,1)) & (transactions['t_dat'] < datetime.datetime(2020,8,16))]\ntrain2 = transactions.loc[(transactions[\"t_dat\"] >= datetime.datetime(2020,7,15)) & (transactions['t_dat'] < datetime.datetime(2020,8,1))]\ntrain3 = transactions.loc[(transactions[\"t_dat\"] >= datetime.datetime(2020,7,1)) & (transactions['t_dat'] < datetime.datetime(2020,7,15))]\ntrain4 = transactions.loc[(transactions[\"t_dat\"] >= datetime.datetime(2020,6,15)) & (transactions['t_dat'] < datetime.datetime(2020,7,1))]\n\nval = transactions.loc[transactions[\"t_dat\"] >= datetime.datetime(2020,8,16)]","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:44:09.471593Z","iopub.execute_input":"2022-10-25T16:44:09.472134Z","iopub.status.idle":"2022-10-25T16:44:12.127915Z","shell.execute_reply.started":"2022-10-25T16:44:09.472078Z","shell.execute_reply":"2022-10-25T16:44:12.126587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_items_per_user1 = train1.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user2 = train2.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user3 = train3.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user4 = train4.groupby(['customer_id'])['article_id'].apply(list)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:44:12.129872Z","iopub.execute_input":"2022-10-25T16:44:12.130319Z","iopub.status.idle":"2022-10-25T16:44:27.716064Z","shell.execute_reply.started":"2022-10-25T16:44:12.130245Z","shell.execute_reply":"2022-10-25T16:44:27.714330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([train1, train2, train3, train4], axis=0)\n\n#time decay popularity of each article\ntrain['pop_factor'] = train['t_dat'].apply(lambda x: 1/(datetime.datetime(2020,8,16) - x).days**2)\npopular_items_group = train.groupby(['article_id'])['pop_factor'].sum()\n\n# purchase count of each article\nitems_total_count = train.groupby(['article_id'])['article_id'].count()\n# purchase count of each user\nusers_total_count = train.groupby(['customer_id'])['customer_id'].count()\n\n\ntrain['feedback'] = 1\ntrain = train.groupby(['customer_id', 'article_id']).sum().reset_index()\ntrain['feedback'] = train.apply(lambda row: row['feedback']/popular_items_group[row['article_id']], axis=1)\n\ntrain['feedback'] = train['feedback'].apply(lambda x: 5.0 if x>5.0 else x)\ntrain.drop(['price', 'sales_channel_id'], axis=1, inplace=True)\n\n# shuffling\ntrain = train.sample(frac=1).reset_index(drop=True)\ntrain['feedback'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:44:27.717967Z","iopub.execute_input":"2022-10-25T16:44:27.718423Z","iopub.status.idle":"2022-10-25T16:46:15.314486Z","shell.execute_reply.started":"2022-10-25T16:44:27.718383Z","shell.execute_reply":"2022-10-25T16:46:15.313099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pop = transactions.loc[(transactions[\"t_dat\"] >= datetime.datetime(2020,8,1)) & (transactions['t_dat'] < datetime.datetime(2020,8,16))]\ntrain_pop['pop_factor'] = train_pop['t_dat'].apply(lambda x: 1/(datetime.datetime(2020,8,16) - x).days)\npopular_items_group = train_pop.groupby(['article_id'])['pop_factor'].sum()\n\n_, popular_items = zip(*sorted(zip(popular_items_group, popular_items_group.keys()))[::-1])\n\ntrain_pop['pop_factor'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:46:15.316146Z","iopub.execute_input":"2022-10-25T16:46:15.316518Z","iopub.status.idle":"2022-10-25T16:46:24.213986Z","shell.execute_reply.started":"2022-10-25T16:46:15.316476Z","shell.execute_reply":"2022-10-25T16:46:24.212475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_most_freq_next_item(user_group):\n    next_items = {}\n    for user in tqdm(user_group.keys()):\n        items = user_group[user]\n        for i,item in enumerate(items[:-1]):\n            if item not in next_items:\n                next_items[item] = []\n            if item != items[i+1]:\n                next_items[item].append(items[i+1])\n\n    pred_next = {}\n    for item in next_items:\n        if len(next_items[item]) >= 5:\n            most_common = Counter(next_items[item]).most_common()\n            ratio = most_common[0][1]/len(next_items[item])\n            if ratio >= 0.1:\n                pred_next[item] = most_common[0][0]\n            \n    return pred_next\n\nuser_group = train.groupby(['customer_id'])['article_id'].apply(list)\npred_next = get_most_freq_next_item(user_group)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:46:24.215784Z","iopub.execute_input":"2022-10-25T16:46:24.216732Z","iopub.status.idle":"2022-10-25T16:46:42.515191Z","shell.execute_reply.started":"2022-10-25T16:46:24.216691Z","shell.execute_reply":"2022-10-25T16:46:42.513641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from reco.recommender import FunkSVD\nfrom reco.metrics import rmse\n\n# k = number of dimensions of the latent embedding. formatizer dict takes in names of the columns\n# for user, item and values/feedback/ratings respectively.\n\nsvd = FunkSVD(k=8, learning_rate=0.008, regularizer = .01, iterations = 100, method = 'stochastic', bias=True)\nsvd.fit(X=train, formatizer={'user':'customer_id', 'item':'article_id', 'value':'feedback'},verbose=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:46:42.517143Z","iopub.execute_input":"2022-10-25T16:46:42.517514Z","iopub.status.idle":"2022-10-25T16:48:40.569392Z","shell.execute_reply.started":"2022-10-25T16:46:42.517480Z","shell.execute_reply":"2022-10-25T16:48:40.567869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apk(actual, predicted, k=12):\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12):\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:48:40.571788Z","iopub.execute_input":"2022-10-25T16:48:40.572194Z","iopub.status.idle":"2022-10-25T16:48:40.582851Z","shell.execute_reply.started":"2022-10-25T16:48:40.572157Z","shell.execute_reply":"2022-10-25T16:48:40.581299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_items_val = val.groupby(['customer_id'])['article_id'].apply(list)\nval_users = positive_items_val.keys()\nval_items = []\n\nfor i,user in tqdm(enumerate(val_users)):\n    val_items.append(positive_items_val[user])\n    \nprint(\"Total users in validation:\", len(val_users))","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:48:40.584720Z","iopub.execute_input":"2022-10-25T16:48:40.585628Z","iopub.status.idle":"2022-10-25T16:48:48.119510Z","shell.execute_reply.started":"2022-10-25T16:48:40.585577Z","shell.execute_reply":"2022-10-25T16:48:48.118207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = []\ncnt = 0\n\npopular_items = list(popular_items)\nuserindexes = {svd.users[i]:i for i in range(len(svd.users))}\n\nfor user in tqdm(val_users):\n    user_output = []\n    if user in positive_items_per_user1.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user1[user]).most_common()}\n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    if user in positive_items_per_user2.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user2[user]).most_common()}\n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    if user in positive_items_per_user3.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user3[user]).most_common()}\n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    if user in positive_items_per_user4.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user4[user]).most_common()}\n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    user_output += [pred_next[item] for item in user_output if item in pred_next and pred_next[item] not in user_output]      \n    \n    user_output += list(popular_items[:12 - len(user_output)])\n    outputs.append(user_output)\n    \nprint(\"mAP Score on Validation set:\", mapk(val_items, outputs))","metadata":{"execution":{"iopub.status.busy":"2022-10-25T16:48:48.121132Z","iopub.execute_input":"2022-10-25T16:48:48.121506Z","iopub.status.idle":"2022-10-25T17:02:05.754638Z","shell.execute_reply.started":"2022-10-25T16:48:48.121470Z","shell.execute_reply":"2022-10-25T17:02:05.753087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\", nrows=5000)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:02:05.756784Z","iopub.execute_input":"2022-10-25T17:02:05.757237Z","iopub.status.idle":"2022-10-25T17:02:05.819585Z","shell.execute_reply.started":"2022-10-25T17:02:05.757194Z","shell.execute_reply":"2022-10-25T17:02:05.818529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = []\ncnt = 0\n\npopular_items = list(popular_items)\nuserindexes = {svd.users[i]:i for i in range(len(svd.users))}\n\nfor user in tqdm(submission['customer_id']):\n    user_output = []\n    if user in positive_items_per_user1.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user1[user]).most_common()}\n        \n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    if user in positive_items_per_user2.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user2[user]).most_common()}\n        \n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    if user in positive_items_per_user3.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user3[user]).most_common()}\n        \n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    if user in positive_items_per_user4.keys():\n        most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user4[user]).most_common()}\n        \n        user_index = userindexes[user]\n        new_order = {}\n        for k in list(most_common_items_of_user.keys())[:20]:\n            try:\n                itemindex = svd.items.index(k)\n                pred_value = np.dot(svd.userfeatures[user_index], svd.itemfeatures[itemindex].T) + svd.item_bias[0, itemindex]\n            except:\n                pred_value = most_common_items_of_user[k]\n            new_order[k] = pred_value\n        user_output += [k for k, v in sorted(new_order.items(), key=lambda item: item[1])][:12]\n        \n    user_output += [pred_next[item] for item in user_output if item in pred_next and pred_next[item] not in user_output]      \n    \n    user_output += list(popular_items[:12 - len(user_output)])\n    outputs.append(user_output)\n    \nstr_outputs = []\nfor output in outputs:\n    str_outputs.append(\" \".join([str(x) for x in output]))","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:02:05.821352Z","iopub.execute_input":"2022-10-25T17:02:05.821976Z","iopub.status.idle":"2022-10-25T17:02:19.426752Z","shell.execute_reply.started":"2022-10-25T17:02:05.821938Z","shell.execute_reply":"2022-10-25T17:02:19.425477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['prediction'] = str_outputs\nsubmission.to_csv(\"submissions.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:02:19.428358Z","iopub.execute_input":"2022-10-25T17:02:19.428746Z","iopub.status.idle":"2022-10-25T17:02:20.278463Z","shell.execute_reply.started":"2022-10-25T17:02:19.428710Z","shell.execute_reply":"2022-10-25T17:02:20.276980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:02:20.280741Z","iopub.execute_input":"2022-10-25T17:02:20.281262Z","iopub.status.idle":"2022-10-25T17:02:20.294481Z","shell.execute_reply.started":"2022-10-25T17:02:20.281218Z","shell.execute_reply":"2022-10-25T17:02:20.292583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Suggested articles\nsubmission.iloc[0,1]","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:45:25.342398Z","iopub.execute_input":"2022-10-25T17:45:25.342832Z","iopub.status.idle":"2022-10-25T17:45:25.351413Z","shell.execute_reply.started":"2022-10-25T17:45:25.342798Z","shell.execute_reply":"2022-10-25T17:45:25.350185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.iloc[5,0]","metadata":{"execution":{"iopub.status.busy":"2022-10-25T18:49:26.084891Z","iopub.execute_input":"2022-10-25T18:49:26.085318Z","iopub.status.idle":"2022-10-25T18:49:26.093514Z","shell.execute_reply.started":"2022-10-25T18:49:26.085284Z","shell.execute_reply":"2022-10-25T18:49:26.092145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_recommendations = submission.iloc[0,1].split()\ncustomer_recommendations","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:45:30.957502Z","iopub.execute_input":"2022-10-25T17:45:30.958312Z","iopub.status.idle":"2022-10-25T17:45:30.967463Z","shell.execute_reply.started":"2022-10-25T17:45:30.958270Z","shell.execute_reply":"2022-10-25T17:45:30.966217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_recommendations = [str(0) + i for i in customer_recommendations]\ncustomer_recommendations","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:45:33.168709Z","iopub.execute_input":"2022-10-25T17:45:33.169161Z","iopub.status.idle":"2022-10-25T17:45:33.177898Z","shell.execute_reply.started":"2022-10-25T17:45:33.169125Z","shell.execute_reply":"2022-10-25T17:45:33.176864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_names = []\nfor dirname, _, filenames in os.walk(images_dir):\n    for filename in filenames:\n        fullpath = os.path.join(dirname, filename)\n        image_names.append(fullpath)","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:36:39.999510Z","iopub.execute_input":"2022-10-25T17:36:40.000731Z","iopub.status.idle":"2022-10-25T17:37:38.019766Z","shell.execute_reply.started":"2022-10-25T17:36:40.000678Z","shell.execute_reply":"2022-10-25T17:37:38.018452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_recommendations = [images_dir + '/' + i[:3] + '/' + i + '.jpg' for i in customer_recommendations]\ncustomer_recommendations","metadata":{"execution":{"iopub.status.busy":"2022-10-25T17:45:53.357822Z","iopub.execute_input":"2022-10-25T17:45:53.358261Z","iopub.status.idle":"2022-10-25T17:45:53.367596Z","shell.execute_reply.started":"2022-10-25T17:45:53.358225Z","shell.execute_reply":"2022-10-25T17:45:53.366190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    fig = plt.figure(figsize=(12,8))\n    for i in range(0,len(customer_recommendations)):\n        img = mpimg.imread(customer_recommendations[i])\n        plt.subplot(3,4,i+1)\n        plt.imshow(img)\n    plt.show()\nexcept OSError:\n    pass","metadata":{"execution":{"iopub.status.busy":"2022-10-25T18:09:21.654208Z","iopub.execute_input":"2022-10-25T18:09:21.654707Z","iopub.status.idle":"2022-10-25T18:09:24.081525Z","shell.execute_reply.started":"2022-10-25T18:09:21.654671Z","shell.execute_reply":"2022-10-25T18:09:24.080302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def customer_recommendations(customer_id):\n    cust_rec = submission[submission['customer_id'] == customer_id]['prediction'].iloc[0].split()\n    cust_rec = [str(0) + i for i in cust_rec]\n    cust_rec = [images_dir + '/' + i[:3] + '/' + i + '.jpg' for i in cust_rec]\n    fig = plt.figure(figsize=(12,8))\n    for i in range(0,len(cust_rec)):\n        try:\n            img = mpimg.imread(cust_rec[i])\n            plt.subplot(3,4,i+1)\n            plt.imshow(img)\n        except OSError:\n            pass\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-25T18:47:52.294900Z","iopub.execute_input":"2022-10-25T18:47:52.296246Z","iopub.status.idle":"2022-10-25T18:47:52.305970Z","shell.execute_reply.started":"2022-10-25T18:47:52.296174Z","shell.execute_reply":"2022-10-25T18:47:52.304327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_recommendations(submission.iloc[0,0])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T18:47:52.530488Z","iopub.execute_input":"2022-10-25T18:47:52.530949Z","iopub.status.idle":"2022-10-25T18:47:56.749836Z","shell.execute_reply.started":"2022-10-25T18:47:52.530910Z","shell.execute_reply":"2022-10-25T18:47:56.748444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_recommendations(submission.iloc[154,0])","metadata":{"execution":{"iopub.status.busy":"2022-10-25T18:50:37.301403Z","iopub.execute_input":"2022-10-25T18:50:37.301883Z","iopub.status.idle":"2022-10-25T18:50:42.311623Z","shell.execute_reply.started":"2022-10-25T18:50:37.301842Z","shell.execute_reply":"2022-10-25T18:50:42.310325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}