{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\nfrom datetime import datetime\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:19:38.283738Z","iopub.execute_input":"2022-05-26T02:19:38.284103Z","iopub.status.idle":"2022-05-26T02:19:39.411176Z","shell.execute_reply.started":"2022-05-26T02:19:38.283999Z","shell.execute_reply":"2022-05-26T02:19:39.410284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nprint(f\"files and folders: {os.listdir('/kaggle/input/h-and-m-personalized-fashion-recommendations/')}\")\nprint(\"Subfolders in images folder: \", len(list(os.listdir(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/images\"))))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:19:39.412966Z","iopub.execute_input":"2022-05-26T02:19:39.413330Z","iopub.status.idle":"2022-05-26T02:19:39.442176Z","shell.execute_reply.started":"2022-05-26T02:19:39.413282Z","shell.execute_reply":"2022-05-26T02:19:39.441043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_folders = total_files = 0\nfolder_info = []\nimages_names = []\nfor base, dirs, files in tqdm(os.walk('/kaggle/input/h-and-m-personalized-fashion-recommendations/')):\n    for directories in dirs:\n        folder_info.append((directories, len(os.listdir(os.path.join(base, directories)))))\n        total_folders += 1\n    for _files in files:\n        total_files += 1\n        if len(_files.split(\".jpg\"))==2:\n            images_names.append(_files.split(\".jpg\")[0])","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:19:39.443523Z","iopub.execute_input":"2022-05-26T02:19:39.443818Z","iopub.status.idle":"2022-05-26T02:22:01.089611Z","shell.execute_reply.started":"2022-05-26T02:19:39.443778Z","shell.execute_reply":"2022-05-26T02:22:01.088730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nprint(f\"Total number of folders: {total_folders}\\nTotal number of files: {total_files}\")\nfolder_info_df = pd.DataFrame(folder_info, columns=[\"folder\", \"files count\"])\nfolder_info_df.sort_values([\"files count\"], ascending=False).head()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:22:01.092380Z","iopub.execute_input":"2022-05-26T02:22:01.092911Z","iopub.status.idle":"2022-05-26T02:22:01.128660Z","shell.execute_reply.started":"2022-05-26T02:22:01.092864Z","shell.execute_reply":"2022-05-26T02:22:01.127679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nprint(\"folder names: \", list(folder_info_df.folder.unique()))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:22:01.129986Z","iopub.execute_input":"2022-05-26T02:22:01.130336Z","iopub.status.idle":"2022-05-26T02:22:01.137949Z","shell.execute_reply.started":"2022-05-26T02:22:01.130299Z","shell.execute_reply":"2022-05-26T02:22:01.137221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomers_df = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv\")\nsample_submission_df = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:22:01.138958Z","iopub.execute_input":"2022-05-26T02:22:01.139756Z","iopub.status.idle":"2022-05-26T02:22:13.445067Z","shell.execute_reply.started":"2022-05-26T02:22:01.139708Z","shell.execute_reply":"2022-05-26T02:22:13.444072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_df = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:22:13.446223Z","iopub.execute_input":"2022-05-26T02:22:13.446554Z","iopub.status.idle":"2022-05-26T02:23:17.317379Z","shell.execute_reply.started":"2022-05-26T02:22:13.446512Z","shell.execute_reply":"2022-05-26T02:23:17.316498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.318540Z","iopub.execute_input":"2022-05-26T02:23:17.318764Z","iopub.status.idle":"2022-05-26T02:23:17.343725Z","shell.execute_reply.started":"2022-05-26T02:23:17.318737Z","shell.execute_reply":"2022-05-26T02:23:17.342851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ncustomers_df.head()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.358480Z","iopub.execute_input":"2022-05-26T02:23:17.358903Z","iopub.status.idle":"2022-05-26T02:23:17.371807Z","shell.execute_reply.started":"2022-05-26T02:23:17.358870Z","shell.execute_reply":"2022-05-26T02:23:17.371236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nsample_submission_df.head()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.374641Z","iopub.execute_input":"2022-05-26T02:23:17.374865Z","iopub.status.idle":"2022-05-26T02:23:17.385961Z","shell.execute_reply.started":"2022-05-26T02:23:17.374840Z","shell.execute_reply":"2022-05-26T02:23:17.385233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.386987Z","iopub.execute_input":"2022-05-26T02:23:17.387513Z","iopub.status.idle":"2022-05-26T02:23:17.404203Z","shell.execute_reply.started":"2022-05-26T02:23:17.387476Z","shell.execute_reply":"2022-05-26T02:23:17.403399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def missing_data(data):\n    total = data.isnull().sum().sort_values(ascending = False)\n    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)\n    return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.405608Z","iopub.execute_input":"2022-05-26T02:23:17.406514Z","iopub.status.idle":"2022-05-26T02:23:17.416950Z","shell.execute_reply.started":"2022-05-26T02:23:17.406404Z","shell.execute_reply":"2022-05-26T02:23:17.416099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unique_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columns = ['Total']\n    uniques = []\n    for col in data.columns:\n        unique = data[col].nunique()\n        uniques.append(unique)\n    tt['Uniques'] = uniques\n    return tt","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.418105Z","iopub.execute_input":"2022-05-26T02:23:17.418368Z","iopub.status.idle":"2022-05-26T02:23:17.434141Z","shell.execute_reply.started":"2022-05-26T02:23:17.418305Z","shell.execute_reply":"2022-05-26T02:23:17.433527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\narticles_df.info()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.435282Z","iopub.execute_input":"2022-05-26T02:23:17.436137Z","iopub.status.idle":"2022-05-26T02:23:17.524890Z","shell.execute_reply.started":"2022-05-26T02:23:17.436092Z","shell.execute_reply":"2022-05-26T02:23:17.524259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.526211Z","iopub.execute_input":"2022-05-26T02:23:17.526703Z","iopub.status.idle":"2022-05-26T02:23:17.616529Z","shell.execute_reply.started":"2022-05-26T02:23:17.526650Z","shell.execute_reply":"2022-05-26T02:23:17.615963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmissing_data(articles_df)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.617614Z","iopub.execute_input":"2022-05-26T02:23:17.617983Z","iopub.status.idle":"2022-05-26T02:23:17.827227Z","shell.execute_reply.started":"2022-05-26T02:23:17.617946Z","shell.execute_reply":"2022-05-26T02:23:17.826426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nIn the article data, the only missing data is for the detailed description of the article (0.4% missing data).\n","metadata":{}},{"cell_type":"code","source":"customers_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:17.828458Z","iopub.execute_input":"2022-05-26T02:23:17.828750Z","iopub.status.idle":"2022-05-26T02:23:18.117311Z","shell.execute_reply.started":"2022-05-26T02:23:17.828712Z","shell.execute_reply":"2022-05-26T02:23:18.116352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_data(customers_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:18.118718Z","iopub.execute_input":"2022-05-26T02:23:18.119063Z","iopub.status.idle":"2022-05-26T02:23:18.957281Z","shell.execute_reply.started":"2022-05-26T02:23:18.119022Z","shell.execute_reply":"2022-05-26T02:23:18.956663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only customer id and postal code are completely filled. Age, fashion news frequency have arounfd 1% misssing data, FN has 65% missing and Active has 66% missing data.","metadata":{}},{"cell_type":"code","source":"customers_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:18.958206Z","iopub.execute_input":"2022-05-26T02:23:18.958573Z","iopub.status.idle":"2022-05-26T02:23:19.243969Z","shell.execute_reply.started":"2022-05-26T02:23:18.958524Z","shell.execute_reply":"2022-05-26T02:23:19.243174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmissing_data(transactions_train_df)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:19.245100Z","iopub.execute_input":"2022-05-26T02:23:19.245322Z","iopub.status.idle":"2022-05-26T02:23:27.879185Z","shell.execute_reply.started":"2022-05-26T02:23:19.245295Z","shell.execute_reply":"2022-05-26T02:23:27.878588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values(articles_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:27.880165Z","iopub.execute_input":"2022-05-26T02:23:27.880492Z","iopub.status.idle":"2022-05-26T02:23:28.088071Z","shell.execute_reply.started":"2022-05-26T02:23:27.880466Z","shell.execute_reply":"2022-05-26T02:23:28.087450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nWe observe that features for which we expect to have the same number of unique value, like:\n\n    product_type_no and product_type_name,\n    departmant_no and department_name,\n    section_no and section_name have different number of unique values, which might means that we might have categories with same name. Others, like:\n    index_code and index_name,\n    garment_group_no and garment_group_name have the same number of unique values.\n\n","metadata":{}},{"cell_type":"code","source":"unique_values(customers_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:28.089197Z","iopub.execute_input":"2022-05-26T02:23:28.089539Z","iopub.status.idle":"2022-05-26T02:23:29.687869Z","shell.execute_reply.started":"2022-05-26T02:23:28.089512Z","shell.execute_reply":"2022-05-26T02:23:29.687348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values(transactions_train_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:29.688825Z","iopub.execute_input":"2022-05-26T02:23:29.689119Z","iopub.status.idle":"2022-05-26T02:23:42.861111Z","shell.execute_reply.started":"2022-05-26T02:23:29.689094Z","shell.execute_reply":"2022-05-26T02:23:42.860353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"out of 31.7M transactions, and for 1.3M customers, buying 104K different articles. Same for the dates, there are only 734 different dates.","metadata":{}},{"cell_type":"markdown","source":"\nArticles data\n","metadata":{}},{"cell_type":"code","source":"temp = articles_df.groupby([\"product_group_name\"])[\"product_type_name\"].nunique()\ndf = pd.DataFrame({'Product Group': temp.index,\n                   'Product Types': temp.values\n                  })\ndf = df.sort_values(['Product Types'], ascending=False)\nplt.figure(figsize = (8,6))\nplt.title('Number of Product Types per each Product Group')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Product Group', y=\"Product Types\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:42.862451Z","iopub.execute_input":"2022-05-26T02:23:42.862651Z","iopub.status.idle":"2022-05-26T02:23:43.181498Z","shell.execute_reply.started":"2022-05-26T02:23:42.862627Z","shell.execute_reply":"2022-05-26T02:23:43.180510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:43.182814Z","iopub.execute_input":"2022-05-26T02:23:43.183050Z","iopub.status.idle":"2022-05-26T02:23:43.193356Z","shell.execute_reply.started":"2022-05-26T02:23:43.183009Z","shell.execute_reply":"2022-05-26T02:23:43.192284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STOPWORDS","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:43.194465Z","iopub.execute_input":"2022-05-26T02:23:43.194670Z","iopub.status.idle":"2022-05-26T02:23:43.210384Z","shell.execute_reply.started":"2022-05-26T02:23:43.194647Z","shell.execute_reply":"2022-05-26T02:23:43.209216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='white',\n        stopwords=stopwords,\n        max_words=200,\n        max_font_size=40, \n        scale=5,\n        random_state=1\n    ).generate(str(data))\n\n    fig = plt.figure(1, figsize=(10,10))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=14)\n        fig.subplots_adjust(top=2.3)\n\n    plt.imshow(wordcloud)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:43.211999Z","iopub.execute_input":"2022-05-26T02:23:43.212524Z","iopub.status.idle":"2022-05-26T02:23:43.225309Z","shell.execute_reply.started":"2022-05-26T02:23:43.212480Z","shell.execute_reply":"2022-05-26T02:23:43.224301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df[\"prod_name\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:43.229899Z","iopub.execute_input":"2022-05-26T02:23:43.230478Z","iopub.status.idle":"2022-05-26T02:23:43.242231Z","shell.execute_reply.started":"2022-05-26T02:23:43.230433Z","shell.execute_reply":"2022-05-26T02:23:43.241381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nshow_wordcloud(articles_df[\"prod_name\"], \"Wordcloud from product name\")\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:43.243938Z","iopub.execute_input":"2022-05-26T02:23:43.244432Z","iopub.status.idle":"2022-05-26T02:23:43.844029Z","shell.execute_reply.started":"2022-05-26T02:23:43.244387Z","shell.execute_reply":"2022-05-26T02:23:43.843417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"product_group_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Product Group': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (8,6))\nplt.title('Number of Articles per each Product Group')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Product Group', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:43.844922Z","iopub.execute_input":"2022-05-26T02:23:43.845266Z","iopub.status.idle":"2022-05-26T02:23:44.123070Z","shell.execute_reply.started":"2022-05-26T02:23:43.845237Z","shell.execute_reply":"2022-05-26T02:23:44.122192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"product_type_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Product Type': temp.index,\n                   'Articles': temp.values\n                  })\ntotal_types = len(df['Product Type'].unique())\ndf = df.sort_values(['Articles'], ascending=False)[0:50]\nplt.figure(figsize = (16,6))\nplt.title(f'Number of Articles per each Product Type (top 50 from total: {total_types})')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Product Type', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:44.124252Z","iopub.execute_input":"2022-05-26T02:23:44.124560Z","iopub.status.idle":"2022-05-26T02:23:44.863980Z","shell.execute_reply.started":"2022-05-26T02:23:44.124520Z","shell.execute_reply":"2022-05-26T02:23:44.862849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"department_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Department Name': temp.index,\n                   'Articles': temp.values\n                  })\ntotal_depts = len(df['Department Name'].unique())\ndf = df.sort_values(['Articles'], ascending=False).head(50)\nplt.figure(figsize = (16,6))\nplt.title(f'Number of Articles per each Department (top 50 from total: {total_depts})')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Department Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:44.865070Z","iopub.execute_input":"2022-05-26T02:23:44.865331Z","iopub.status.idle":"2022-05-26T02:23:45.817399Z","shell.execute_reply.started":"2022-05-26T02:23:44.865304Z","shell.execute_reply":"2022-05-26T02:23:45.816483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"graphical_appearance_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Graphical Appearance Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False).head(50)\nplt.figure(figsize = (16,6))\nplt.title(f'Number of Articles per each Graphical Appearance Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Graphical Appearance Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:45.818589Z","iopub.execute_input":"2022-05-26T02:23:45.818902Z","iopub.status.idle":"2022-05-26T02:23:46.165383Z","shell.execute_reply.started":"2022-05-26T02:23:45.818869Z","shell.execute_reply":"2022-05-26T02:23:46.164569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"index_group_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Index Group Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (6,6))\nplt.title(f'Number of Articles per each Index Group Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Index Group Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:46.166423Z","iopub.execute_input":"2022-05-26T02:23:46.166727Z","iopub.status.idle":"2022-05-26T02:23:46.376715Z","shell.execute_reply.started":"2022-05-26T02:23:46.166697Z","shell.execute_reply":"2022-05-26T02:23:46.375210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"colour_group_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Colour Group Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (12,6))\nplt.title(f'Number of Articles per each Colour Group Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Colour Group Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:46.377644Z","iopub.execute_input":"2022-05-26T02:23:46.378175Z","iopub.status.idle":"2022-05-26T02:23:47.111179Z","shell.execute_reply.started":"2022-05-26T02:23:46.378145Z","shell.execute_reply":"2022-05-26T02:23:47.110350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"perceived_colour_value_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Perceived Colour Group Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (6,6))\nplt.title(f'Number of Articles per each Perceived Colour Group Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Perceived Colour Group Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:47.112659Z","iopub.execute_input":"2022-05-26T02:23:47.112953Z","iopub.status.idle":"2022-05-26T02:23:47.300976Z","shell.execute_reply.started":"2022-05-26T02:23:47.112914Z","shell.execute_reply":"2022-05-26T02:23:47.300405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntemp = articles_df.groupby([\"perceived_colour_master_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Perceived Colour Master Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (12,6))\nplt.title(f'Number of Articles per each Perceived Colour Master Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Perceived Colour Master Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:47.302005Z","iopub.execute_input":"2022-05-26T02:23:47.302313Z","iopub.status.idle":"2022-05-26T02:23:47.561522Z","shell.execute_reply.started":"2022-05-26T02:23:47.302279Z","shell.execute_reply":"2022-05-26T02:23:47.560704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df[\"perceived_colour_master_name\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:47.562946Z","iopub.execute_input":"2022-05-26T02:23:47.563414Z","iopub.status.idle":"2022-05-26T02:23:47.572084Z","shell.execute_reply.started":"2022-05-26T02:23:47.563372Z","shell.execute_reply":"2022-05-26T02:23:47.571185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntemp = articles_df.groupby([\"index_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Index Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (8,6))\nplt.title(f'Number of Articles per each Index Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Index Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:47.573162Z","iopub.execute_input":"2022-05-26T02:23:47.573532Z","iopub.status.idle":"2022-05-26T02:23:47.776355Z","shell.execute_reply.started":"2022-05-26T02:23:47.573495Z","shell.execute_reply":"2022-05-26T02:23:47.775613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"garment_group_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Garment Group Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (12,6))\nplt.title(f'Number of Articles per each Garment Group Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Garment Group Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:47.777514Z","iopub.execute_input":"2022-05-26T02:23:47.777854Z","iopub.status.idle":"2022-05-26T02:23:48.060127Z","shell.execute_reply.started":"2022-05-26T02:23:47.777826Z","shell.execute_reply":"2022-05-26T02:23:48.059321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntemp = articles_df.groupby([\"section_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Section Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (16,6))\nplt.title(f'Number of Articles per each Section Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Section Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:48.061648Z","iopub.execute_input":"2022-05-26T02:23:48.062099Z","iopub.status.idle":"2022-05-26T02:23:49.165704Z","shell.execute_reply.started":"2022-05-26T02:23:48.062057Z","shell.execute_reply":"2022-05-26T02:23:49.164859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = articles_df.groupby([\"section_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Section Name': temp.index,\n                   'Articles': temp.values\n                  })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (16,6))\nplt.title(f'Number of Articles per each Section Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Section Name', y=\"Articles\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:49.167217Z","iopub.execute_input":"2022-05-26T02:23:49.167729Z","iopub.status.idle":"2022-05-26T02:23:50.110019Z","shell.execute_reply.started":"2022-05-26T02:23:49.167684Z","shell.execute_reply":"2022-05-26T02:23:50.108804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_df[\"detail_desc\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:50.111864Z","iopub.execute_input":"2022-05-26T02:23:50.112233Z","iopub.status.idle":"2022-05-26T02:23:50.120936Z","shell.execute_reply.started":"2022-05-26T02:23:50.112176Z","shell.execute_reply":"2022-05-26T02:23:50.119908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_wordcloud(articles_df[\"detail_desc\"], \"Wordcloud from detailed description of articles\")","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:50.122261Z","iopub.execute_input":"2022-05-26T02:23:50.122769Z","iopub.status.idle":"2022-05-26T02:23:50.790751Z","shell.execute_reply.started":"2022-05-26T02:23:50.122734Z","shell.execute_reply":"2022-05-26T02:23:50.789855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Customers data","metadata":{}},{"cell_type":"code","source":"temp = customers_df.groupby([\"age\"])[\"customer_id\"].count()\ndf = pd.DataFrame({'Age': temp.index,\n                   'Customers': temp.values\n                  })\ndf = df.sort_values(['Age'], ascending=False)\nplt.figure(figsize = (16,6))\nplt.title(f'Number of Customers per each Age')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Age', y=\"Customers\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:50.791848Z","iopub.execute_input":"2022-05-26T02:23:50.792067Z","iopub.status.idle":"2022-05-26T02:23:51.856299Z","shell.execute_reply.started":"2022-05-26T02:23:50.792040Z","shell.execute_reply":"2022-05-26T02:23:51.855352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = customers_df.groupby([\"fashion_news_frequency\"])[\"customer_id\"].count()\ndf = pd.DataFrame({'Fashion News Frequency': temp.index,\n                   'Customers': temp.values\n                  })\ndf = df.sort_values(['Customers'], ascending=False)\nplt.figure(figsize = (6,6))\nplt.title(f'Number of Customers per each Fashion News Frequency')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Fashion News Frequency', y=\"Customers\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:51.857476Z","iopub.execute_input":"2022-05-26T02:23:51.857712Z","iopub.status.idle":"2022-05-26T02:23:52.157655Z","shell.execute_reply.started":"2022-05-26T02:23:51.857684Z","shell.execute_reply":"2022-05-26T02:23:52.156770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntemp = customers_df.groupby([\"club_member_status\"])[\"customer_id\"].count()\ndf = pd.DataFrame({'Club Member Status': temp.index,\n                   'Customers': temp.values\n                  })\ndf = df.sort_values(['Customers'], ascending=False)\nplt.figure(figsize = (6,6))\nplt.title(f'Number of Customers per each Club Member Status')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Club Member Status', y=\"Customers\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:52.158938Z","iopub.execute_input":"2022-05-26T02:23:52.159904Z","iopub.status.idle":"2022-05-26T02:23:52.459103Z","shell.execute_reply.started":"2022-05-26T02:23:52.159857Z","shell.execute_reply":"2022-05-26T02:23:52.458438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transactions data","metadata":{}},{"cell_type":"code","source":"transactions_train_df.sample(100_000)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:52.460211Z","iopub.execute_input":"2022-05-26T02:23:52.461260Z","iopub.status.idle":"2022-05-26T02:23:54.352944Z","shell.execute_reply.started":"2022-05-26T02:23:52.461179Z","shell.execute_reply":"2022-05-26T02:23:54.352124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = transactions_train_df.sample(100_000)\nfig, ax = plt.subplots(1, 1, figsize=(14, 7))\nsns.kdeplot(np.log(df.loc[df[\"sales_channel_id\"]==1].price.value_counts()))\nsns.kdeplot(np.log(df.loc[df[\"sales_channel_id\"]==2].price.value_counts()))\nax.legend(labels=['Sales channel 1', 'Sales channel 1'])\nplt.title(\"Logaritmic distribution of price frequency in transactions, grouped per sales channel (100k sample)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:54.354323Z","iopub.execute_input":"2022-05-26T02:23:54.355027Z","iopub.status.idle":"2022-05-26T02:23:56.383280Z","shell.execute_reply.started":"2022-05-26T02:23:54.354981Z","shell.execute_reply":"2022-05-26T02:23:56.382331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = transactions_train_df.sample(100_000).groupby([\"t_dat\"])[\"article_id\"].count().reset_index()\ndf[\"t_dat\"] = df[\"t_dat\"].apply(lambda x: datetime.strptime(x, '%Y-%m-%d'))\ndf.columns = [\"Date\", \"Transactions\"]\nfig, ax = plt.subplots(1, 1, figsize=(16,6))\nplt.plot(df[\"Date\"], df[\"Transactions\"], color=\"Darkgreen\")\nplt.xlabel(\"Date\")\nplt.ylabel(\"Transactions\")\nplt.title(f\"Transactions per day (100k sample; to get the real volume, please consider that real transaction count is {round(transactions_train_df.shape[0]/10.e6,2)}M)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:56.384422Z","iopub.execute_input":"2022-05-26T02:23:56.384685Z","iopub.status.idle":"2022-05-26T02:23:58.264055Z","shell.execute_reply.started":"2022-05-26T02:23:56.384652Z","shell.execute_reply":"2022-05-26T02:23:58.263199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = transactions_train_df.sample(100_000).groupby([\"t_dat\", \"sales_channel_id\"])[\"article_id\"].count().reset_index()\ndf[\"t_dat\"] = df[\"t_dat\"].apply(lambda x: datetime.strptime(x, '%Y-%m-%d'))\ndf.columns = [\"Date\", \"Sales Channel Id\", \"Transactions\"]\nfig, ax = plt.subplots(1, 1, figsize=(16,6))\ng1 = ax.plot(df.loc[df[\"Sales Channel Id\"]==1, \"Date\"], df.loc[df[\"Sales Channel Id\"]==1, \"Transactions\"], label=\"Sales Channel 1\", color=\"Darkblue\")\ng2 = ax.plot(df.loc[df[\"Sales Channel Id\"]==2, \"Date\"], df.loc[df[\"Sales Channel Id\"]==2, \"Transactions\"], label=\"Sales Channel 2\", color=\"Magenta\")\nplt.xlabel(\"Date\")\nplt.ylabel(\"Transactions\")\nax.legend()\nplt.title(f\"Transactions per day, grouped by Sales Channel (100k sample; to get the real volume, please consider that real transaction count is {round(transactions_train_df.shape[0]/10.e6,2)}M)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:23:58.265263Z","iopub.execute_input":"2022-05-26T02:23:58.265486Z","iopub.status.idle":"2022-05-26T02:24:00.159412Z","shell.execute_reply.started":"2022-05-26T02:23:58.265460Z","shell.execute_reply":"2022-05-26T02:24:00.158368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = transactions_train_df.groupby([\"t_dat\", \"sales_channel_id\"])[\"article_id\"].nunique().reset_index()\ndf[\"t_dat\"] = df[\"t_dat\"].apply(lambda x: datetime.strptime(x, '%Y-%m-%d'))\ndf.columns = [\"Date\", \"Sales Channel Id\", \"Unique Articles\"]\nfig, ax = plt.subplots(1, 1, figsize=(16,6))\ng1 = ax.plot(df.loc[df[\"Sales Channel Id\"]==1, \"Date\"], df.loc[df[\"Sales Channel Id\"]==1, \"Unique Articles\"], label=\"Sales Channel 1\", color=\"Blue\")\ng2 = ax.plot(df.loc[df[\"Sales Channel Id\"]==2, \"Date\"], df.loc[df[\"Sales Channel Id\"]==2, \"Unique Articles\"], label=\"Sales Channel 2\", color=\"Green\")\nplt.xlabel(\"Date\")\nplt.ylabel(\"Unique Articles / Day\")\nax.legend()\nplt.title(f\"Unique articles per day, grouped by Sales Channel\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:00.161362Z","iopub.execute_input":"2022-05-26T02:24:00.162007Z","iopub.status.idle":"2022-05-26T02:24:15.145290Z","shell.execute_reply.started":"2022-05-26T02:24:00.161970Z","shell.execute_reply":"2022-05-26T02:24:15.144408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nImage data","metadata":{}},{"cell_type":"code","source":"image_name_df = pd.DataFrame(images_names, columns = [\"image_name\"])\nimage_name_df[\"article_id\"] = image_name_df[\"image_name\"].apply(lambda x: int(x[1:]))","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.146914Z","iopub.execute_input":"2022-05-26T02:24:15.147199Z","iopub.status.idle":"2022-05-26T02:24:15.218709Z","shell.execute_reply.started":"2022-05-26T02:24:15.147164Z","shell.execute_reply":"2022-05-26T02:24:15.217576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_name_df","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.220178Z","iopub.execute_input":"2022-05-26T02:24:15.220576Z","iopub.status.idle":"2022-05-26T02:24:15.231362Z","shell.execute_reply.started":"2022-05-26T02:24:15.220519Z","shell.execute_reply":"2022-05-26T02:24:15.230574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_article_df = articles_df[[\"article_id\", \"product_code\", \"product_group_name\", \"product_type_name\"]].merge(image_name_df, on=[\"article_id\"], how=\"left\")\nprint(image_article_df.shape)\nimage_article_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.232948Z","iopub.execute_input":"2022-05-26T02:24:15.233447Z","iopub.status.idle":"2022-05-26T02:24:15.296807Z","shell.execute_reply.started":"2022-05-26T02:24:15.233369Z","shell.execute_reply":"2022-05-26T02:24:15.295985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_no_image_df = image_article_df.loc[image_article_df.image_name.isna()]\nprint(article_no_image_df.shape)\narticle_no_image_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.298128Z","iopub.execute_input":"2022-05-26T02:24:15.299071Z","iopub.status.idle":"2022-05-26T02:24:15.327192Z","shell.execute_reply.started":"2022-05-26T02:24:15.299027Z","shell.execute_reply":"2022-05-26T02:24:15.326291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Product codes with some missing images: \", article_no_image_df.product_code.nunique())\nprint(\"Product groups with some missing images: \", list(article_no_image_df.product_group_name.unique()))","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.328850Z","iopub.execute_input":"2022-05-26T02:24:15.329138Z","iopub.status.idle":"2022-05-26T02:24:15.336383Z","shell.execute_reply.started":"2022-05-26T02:24:15.329099Z","shell.execute_reply":"2022-05-26T02:24:15.335274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image_samples(image_article_df, product_group_name, cols=1, rows=-1):\n    image_path = \"/kaggle/input/h-and-m-personalized-fashion-recommendations/images/\"\n    _df = image_article_df.loc[image_article_df.product_group_name==product_group_name]\n    article_ids = _df.article_id.values[0:cols*rows]\n    plt.figure(figsize=(2 + 3 * cols, 2 + 4 * rows))\n    for i in range(cols * rows):\n        article_id = (\"0\" + str(article_ids[i]))[-10:]\n        plt.subplot(rows, cols, i + 1)\n        plt.axis('off')\n        plt.title(f\"{product_group_name} {article_id[:3]}\\n{article_id}.jpg\")\n        image = Image.open(f\"{image_path}{article_id[:3]}/{article_id}.jpg\")\n        plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.337910Z","iopub.execute_input":"2022-05-26T02:24:15.338706Z","iopub.status.idle":"2022-05-26T02:24:15.346723Z","shell.execute_reply.started":"2022-05-26T02:24:15.338664Z","shell.execute_reply":"2022-05-26T02:24:15.346132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nprint(image_article_df.product_group_name.unique())\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.347826Z","iopub.execute_input":"2022-05-26T02:24:15.348271Z","iopub.status.idle":"2022-05-26T02:24:15.373673Z","shell.execute_reply.started":"2022-05-26T02:24:15.348235Z","shell.execute_reply":"2022-05-26T02:24:15.372722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nplot_image_samples(image_article_df, \"Garment Lower body\", 4, 2)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:15.374748Z","iopub.execute_input":"2022-05-26T02:24:15.375431Z","iopub.status.idle":"2022-05-26T02:24:17.535589Z","shell.execute_reply.started":"2022-05-26T02:24:15.375397Z","shell.execute_reply":"2022-05-26T02:24:17.534523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nplot_image_samples(image_article_df, \"Stationery\", 4, 1)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:17.536884Z","iopub.execute_input":"2022-05-26T02:24:17.537135Z","iopub.status.idle":"2022-05-26T02:24:18.756634Z","shell.execute_reply.started":"2022-05-26T02:24:17.537106Z","shell.execute_reply":"2022-05-26T02:24:18.755650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Initial submission","metadata":{}},{"cell_type":"code","source":"\n\ntransactions_train_df = transactions_train_df.sort_values([\"customer_id\", \"t_dat\"], ascending=False)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:18.757786Z","iopub.execute_input":"2022-05-26T02:24:18.758022Z","iopub.status.idle":"2022-05-26T02:24:42.689769Z","shell.execute_reply.started":"2022-05-26T02:24:18.757994Z","shell.execute_reply":"2022-05-26T02:24:42.688927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:42.691442Z","iopub.execute_input":"2022-05-26T02:24:42.691792Z","iopub.status.idle":"2022-05-26T02:24:42.704516Z","shell.execute_reply.started":"2022-05-26T02:24:42.691749Z","shell.execute_reply":"2022-05-26T02:24:42.703416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nlast_date = transactions_train_df.t_dat.max()\nprint(last_date)\nprint(transactions_train_df.loc[transactions_train_df.t_dat==last_date].shape)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:42.705854Z","iopub.execute_input":"2022-05-26T02:24:42.706134Z","iopub.status.idle":"2022-05-26T02:24:46.826056Z","shell.execute_reply.started":"2022-05-26T02:24:42.706095Z","shell.execute_reply":"2022-05-26T02:24:46.824929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_df.loc[transactions_train_df.t_dat==last_date]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:46.827364Z","iopub.execute_input":"2022-05-26T02:24:46.827684Z","iopub.status.idle":"2022-05-26T02:24:48.796481Z","shell.execute_reply.started":"2022-05-26T02:24:46.827642Z","shell.execute_reply":"2022-05-26T02:24:48.795659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_df.loc[transactions_train_df.t_dat==last_date].article_id.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:48.799594Z","iopub.execute_input":"2022-05-26T02:24:48.799846Z","iopub.status.idle":"2022-05-26T02:24:50.769966Z","shell.execute_reply.started":"2022-05-26T02:24:48.799817Z","shell.execute_reply":"2022-05-26T02:24:50.769399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmost_frequent_articles = list(transactions_train_df.loc[transactions_train_df.t_dat==last_date].article_id.value_counts()[0:12].index)\nart_list = []\nfor art in most_frequent_articles:\n    art = \"0\"+str(art)\n    art_list.append(art)\nart_str = \" \".join(art_list)\nprint(\"Frequent articles bought recently: \", art_str)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:50.771065Z","iopub.execute_input":"2022-05-26T02:24:50.771788Z","iopub.status.idle":"2022-05-26T02:24:52.787646Z","shell.execute_reply.started":"2022-05-26T02:24:50.771745Z","shell.execute_reply":"2022-05-26T02:24:52.786740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"art_list","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:30:59.216040Z","iopub.execute_input":"2022-05-26T02:30:59.216420Z","iopub.status.idle":"2022-05-26T02:30:59.224018Z","shell.execute_reply.started":"2022-05-26T02:30:59.216383Z","shell.execute_reply":"2022-05-26T02:30:59.223224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transactions_train_df.groupby([\"customer_id\"])[\"article_id\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:52.788898Z","iopub.execute_input":"2022-05-26T02:24:52.789142Z","iopub.status.idle":"2022-05-26T02:24:57.612915Z","shell.execute_reply.started":"2022-05-26T02:24:52.789111Z","shell.execute_reply":"2022-05-26T02:24:57.611932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_df[['customer_id','article_id']]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:26:38.316916Z","iopub.execute_input":"2022-05-26T02:26:38.318298Z","iopub.status.idle":"2022-05-26T02:26:39.556763Z","shell.execute_reply.started":"2022-05-26T02:26:38.318215Z","shell.execute_reply":"2022-05-26T02:26:39.555891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df = transactions_train_df.groupby([\"customer_id\"])[\"article_id\"].agg(lambda x: str(x.values[0:12])[1:-1]).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:24:57.614387Z","iopub.execute_input":"2022-05-26T02:24:57.615111Z","iopub.status.idle":"2022-05-26T02:26:22.204284Z","shell.execute_reply.started":"2022-05-26T02:24:57.615061Z","shell.execute_reply":"2022-05-26T02:26:22.203530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:26:49.598476Z","iopub.execute_input":"2022-05-26T02:26:49.598874Z","iopub.status.idle":"2022-05-26T02:26:49.616392Z","shell.execute_reply.started":"2022-05-26T02:26:49.598839Z","shell.execute_reply":"2022-05-26T02:26:49.615263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df['customer_id'].iloc[0]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:28:12.953245Z","iopub.execute_input":"2022-05-26T02:28:12.954409Z","iopub.status.idle":"2022-05-26T02:28:12.960751Z","shell.execute_reply.started":"2022-05-26T02:28:12.954362Z","shell.execute_reply":"2022-05-26T02:28:12.960059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_df[transactions_train_df['customer_id']=='00000dbacae5abe5e23885899a1fa44253a17956c6d1c3d25f88aa139fdfc657']","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:28:26.547978Z","iopub.execute_input":"2022-05-26T02:28:26.548606Z","iopub.status.idle":"2022-05-26T02:28:29.598073Z","shell.execute_reply.started":"2022-05-26T02:28:26.548566Z","shell.execute_reply":"2022-05-26T02:28:29.597386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### adding top most frequency read article with the old articles\ndef padding_articles(x):\n    if x:\n        xl = x.split()\n        x = []\n        for xi in xl:\n            x.append(\"0\"+xi)\n        dimm_x = len(x)\n        if dimm_x < 12:\n            x.extend(art_list[:12-dimm_x])\n        return(\" \".join(x))","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:28:36.470435Z","iopub.execute_input":"2022-05-26T02:28:36.470806Z","iopub.status.idle":"2022-05-26T02:28:36.476275Z","shell.execute_reply.started":"2022-05-26T02:28:36.470771Z","shell.execute_reply":"2022-05-26T02:28:36.475616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df[\"article_id\"] = agg_df[\"article_id\"].apply(lambda x: padding_articles(x))","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:28:43.704942Z","iopub.execute_input":"2022-05-26T02:28:43.706010Z","iopub.status.idle":"2022-05-26T02:28:46.962717Z","shell.execute_reply.started":"2022-05-26T02:28:43.705957Z","shell.execute_reply":"2022-05-26T02:28:46.961659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:28:53.656180Z","iopub.execute_input":"2022-05-26T02:28:53.656556Z","iopub.status.idle":"2022-05-26T02:28:53.672934Z","shell.execute_reply.started":"2022-05-26T02:28:53.656513Z","shell.execute_reply":"2022-05-26T02:28:53.671832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df[agg_df['customer_id']==\"00000dbacae5abe5e23885899a1fa44253a17956c6d1c3d25f88aa139fdfc657\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:34:53.923212Z","iopub.execute_input":"2022-05-26T02:34:53.924266Z","iopub.status.idle":"2022-05-26T02:34:54.522684Z","shell.execute_reply.started":"2022-05-26T02:34:53.924205Z","shell.execute_reply":"2022-05-26T02:34:54.521974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nprint(\"Aggregated transaction history: \", agg_df.customer_id.nunique())\nprint(\"Submission sample: \", sample_submission_df.customer_id.nunique())\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:32:22.596762Z","iopub.execute_input":"2022-05-26T02:32:22.597877Z","iopub.status.idle":"2022-05-26T02:32:24.529292Z","shell.execute_reply.started":"2022-05-26T02:32:22.597822Z","shell.execute_reply":"2022-05-26T02:32:24.528158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_submission_df.shape)\nsample_submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:32:37.531384Z","iopub.execute_input":"2022-05-26T02:32:37.531657Z","iopub.status.idle":"2022-05-26T02:32:37.544765Z","shell.execute_reply.started":"2022-05-26T02:32:37.531628Z","shell.execute_reply":"2022-05-26T02:32:37.543902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df['customer_id'].iloc[0]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:34:31.691155Z","iopub.execute_input":"2022-05-26T02:34:31.691611Z","iopub.status.idle":"2022-05-26T02:34:31.699234Z","shell.execute_reply.started":"2022-05-26T02:34:31.691565Z","shell.execute_reply":"2022-05-26T02:34:31.698487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"replace the values in sample submission with the existent in aggregated transactions data ","metadata":{}},{"cell_type":"code","source":"submission_df = agg_df.merge(sample_submission_df[[\"customer_id\"]], how=\"right\")\nsubmission_df.columns = [\"customer_id\", \"prediction\"]\nprint(submission_df.shape)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:35:04.679501Z","iopub.execute_input":"2022-05-26T02:35:04.680525Z","iopub.status.idle":"2022-05-26T02:35:06.748172Z","shell.execute_reply.started":"2022-05-26T02:35:04.680459Z","shell.execute_reply":"2022-05-26T02:35:06.747163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Rows with missing data in submission: \", submission_df.loc[submission_df.prediction.isna()].shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:35:30.935786Z","iopub.execute_input":"2022-05-26T02:35:30.936102Z","iopub.status.idle":"2022-05-26T02:35:31.081640Z","shell.execute_reply.started":"2022-05-26T02:35:30.936070Z","shell.execute_reply":"2022-05-26T02:35:31.080755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df[submission_df['prediction'].isnull()]","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:36:05.907923Z","iopub.execute_input":"2022-05-26T02:36:05.908751Z","iopub.status.idle":"2022-05-26T02:36:06.056453Z","shell.execute_reply.started":"2022-05-26T02:36:05.908692Z","shell.execute_reply":"2022-05-26T02:36:06.055164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.loc[submission_df.prediction.isna(), [\"prediction\"]] = art_str","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:36:15.844611Z","iopub.execute_input":"2022-05-26T02:36:15.845366Z","iopub.status.idle":"2022-05-26T02:36:15.953043Z","shell.execute_reply.started":"2022-05-26T02:36:15.845305Z","shell.execute_reply":"2022-05-26T02:36:15.952172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"replace the missing data with the most frequently bought articles, from recent days","metadata":{}},{"cell_type":"code","source":"print(\"Rows with missing data in submission: \", submission_df.loc[submission_df.prediction.isna()].shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:36:40.681774Z","iopub.execute_input":"2022-05-26T02:36:40.682460Z","iopub.status.idle":"2022-05-26T02:36:40.825673Z","shell.execute_reply.started":"2022-05-26T02:36:40.682371Z","shell.execute_reply":"2022-05-26T02:36:40.823116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T02:36:45.629955Z","iopub.execute_input":"2022-05-26T02:36:45.630671Z","iopub.status.idle":"2022-05-26T02:36:53.029145Z","shell.execute_reply.started":"2022-05-26T02:36:45.630606Z","shell.execute_reply":"2022-05-26T02:36:53.028161Z"},"trusted":true},"execution_count":null,"outputs":[]}]}