{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This NB has been heavily Inspired and referenced from the following Kernels :\n* https://www.kaggle.com/code/remekkinas/h-m-eda-first-look-into-data/notebook\nIf you have gone through the above NB then this kernel won't be much of a new to you.","metadata":{}},{"cell_type":"code","source":"from termcolor import colored\nimport pandas as pd\nfrom glob import glob\nimport os\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport cv2\nimport random","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-06T08:48:36.884000Z","iopub.execute_input":"2022-04-06T08:48:36.884287Z","iopub.status.idle":"2022-04-06T08:48:38.270951Z","shell.execute_reply.started":"2022-04-06T08:48:36.884255Z","shell.execute_reply":"2022-04-06T08:48:38.269794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('darkgrid')","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:48:38.272874Z","iopub.execute_input":"2022-04-06T08:48:38.274815Z","iopub.status.idle":"2022-04-06T08:48:38.278719Z","shell.execute_reply.started":"2022-04-06T08:48:38.274780Z","shell.execute_reply":"2022-04-06T08:48:38.277925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv',dtype={'article_id':str})\narticle = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv',dtype={'article_id':str})\ncustomer = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/customers.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:48:38.279797Z","iopub.execute_input":"2022-04-06T08:48:38.280168Z","iopub.status.idle":"2022-04-06T08:50:00.484477Z","shell.execute_reply.started":"2022-04-06T08:48:38.280131Z","shell.execute_reply":"2022-04-06T08:50:00.483229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset DESC\n> 31m Transaction,105k Unique Article & 1m Customers\n\n> Image for almost Each Article","metadata":{}},{"cell_type":"code","source":"print(f\"Number of observations in TRANSACTIONS: {colored(train.shape, 'yellow')}\")\nprint(f\"Number of observations in Articles: {colored(article.shape, 'yellow')}\")\nprint(f\"Number of observations in Customers: {colored(customer.shape, 'yellow')}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:00.487019Z","iopub.execute_input":"2022-04-06T08:50:00.487288Z","iopub.status.idle":"2022-04-06T08:50:00.494290Z","shell.execute_reply.started":"2022-04-06T08:50:00.487253Z","shell.execute_reply":"2022-04-06T08:50:00.493421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **ARTICLES**","metadata":{}},{"cell_type":"markdown","source":"# Check \n* For how many Articles do we have corresponding Images and for how many are we missing.\n* Missing Articles Images category wise distribution\n* Transactions involving Article with/without Images\n\n# Inferred\n* Majority of Product are unique will some of them are almost change but slightly differnet in design and coloring scheme.\nImages are placed in subfolders starting with the first three digits of the article_id","metadata":{}},{"cell_type":"code","source":"display(article.columns,article.nunique(),article.head(2),article.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:00.495902Z","iopub.execute_input":"2022-04-06T08:50:00.496174Z","iopub.status.idle":"2022-04-06T08:50:00.803705Z","shell.execute_reply.started":"2022-04-06T08:50:00.496138Z","shell.execute_reply":"2022-04-06T08:50:00.803114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)\npd.set_option('display.max_colwidth', 40)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:00.804769Z","iopub.execute_input":"2022-04-06T08:50:00.805572Z","iopub.status.idle":"2022-04-06T08:50:00.809296Z","shell.execute_reply.started":"2022-04-06T08:50:00.805526Z","shell.execute_reply":"2022-04-06T08:50:00.808512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in article.columns:\n    print(i,article[i].nunique(),article[i].unique()[:5])","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:00.810694Z","iopub.execute_input":"2022-04-06T08:50:00.810910Z","iopub.status.idle":"2022-04-06T08:50:01.119566Z","shell.execute_reply.started":"2022-04-06T08:50:00.810884Z","shell.execute_reply":"2022-04-06T08:50:01.118720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_csv = []\narticles_photo = []\nfor i in tqdm(glob('../input/h-and-m-personalized-fashion-recommendations/images/*')):\n    j = glob(os.path.join(i,'*'))\n    j = [x.split('/')[-1].split('.')[0] for x in j]\n    articles_photo+=j\n    images_csv.append([i.split('/')[-1],len(j)])\nimages_csv = pd.DataFrame(images_csv,columns=['article_img','num_image'])\nmissing_photo = set(article.article_id) - set(articles_photo)\narticle.loc[:,'img_present'] = article.article_id.apply(lambda x:0 if x in missing_photo else 1)\nprint('We have Images for',len(glob('../input/h-and-m-personalized-fashion-recommendations/images/*/*')),'Articles')\nprint(f\"We are missing Images for {colored(len(missing_photo),'red')} Articles\")\nprint(colored(\"Missing Article ID categorization\",\"red\"))\ndisplay(article.query('article_id in @missing_photo').groupby('product_group_name').article_id.nunique())","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:07.360906Z","iopub.execute_input":"2022-04-06T08:50:07.361181Z","iopub.status.idle":"2022-04-06T08:50:11.759957Z","shell.execute_reply.started":"2022-04-06T08:50:07.361151Z","shell.execute_reply":"2022-04-06T08:50:11.759180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train.query('article_id in @missing_photo'))*100/len(train),colored('% of Total Transactions are of Non ImageArticle','blue'))\nprint(len(train.query('article_id in @articles_photo'))*100/len(train),colored('% of Total Transactions are of ImageArticle','blue'))\ndisplay(article.groupby(['product_group_name','img_present']).article_id.nunique())","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:11.761690Z","iopub.execute_input":"2022-04-06T08:50:11.761967Z","iopub.status.idle":"2022-04-06T08:50:21.625600Z","shell.execute_reply.started":"2022-04-06T08:50:11.761937Z","shell.execute_reply":"2022-04-06T08:50:21.624756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This code was borrowed from https://www.kaggle.com/ishandutta/v7-shopee-indepth-eda-one-stop-for-all-your-needs\ndef getImagePaths(path):\n    \"\"\"\n    Function to Combine Directory Path with individual Image Paths\n    \n    parameters: path(string) - Path of directory\n    returns: image_names(string) - Full Image Path\n    \"\"\"\n    image_names = []\n    for dirname, _, filenames in os.walk(path):\n        for filename in filenames:\n            fullpath = os.path.join(dirname, filename)\n            image_names.append(fullpath)\n    return image_names\n\ndef display_multiple_img(images_paths, rows, cols):\n    \"\"\"\n    Function to Display Images from Dataset.\n    \n    parameters: images_path(string) - Paths of Images to be displayed\n                rows(int) - No. of Rows in Output\n                cols(int) - No. of Columns in Output\n    \"\"\"\n    figure, ax = plt.subplots(nrows=rows,ncols=cols,figsize=(16,8) )\n    for ind,image_path in enumerate(images_paths):\n        image=cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) \n        curr_id = image_path.split('/')[-1].split('.')[0]\n        image_title = article.query('article_id == @curr_id').product_group_name.values[0]\n        try:\n            ax.ravel()[ind].imshow(image)\n            ax.ravel()[ind].set_title(image_title)\n            ax.ravel()[ind].set_axis_off()\n        except:\n            continue;\n    plt.tight_layout()\n    plt.show()\n\ndef plot_distribution(x, data, title):\n        fig = px.histogram(\n        data, \n        x = x,\n        width = 800,\n        height = 500,\n        title = title\n        )\n\n        fig.show()\n        \ndef disply_multiple_img_ids(idx, rows, cols):\n    figure, ax = plt.subplots(nrows=rows,ncols=cols,figsize=(12,60))#,figsize=(16,8)\n    for ind, im_id in enumerate(idx):\n\n        image_path = f'{images_dir}/{str(im_id)[:3]}/{im_id}.jpg'\n        try:\n            image=cv2.imread(image_path)\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) \n            ax.ravel()[ind].imshow(image)\n            ax.ravel()[ind].set_axis_off()\n        except:\n            continue;\n    plt.tight_layout()\n    plt.show()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-04-06T08:50:21.627705Z","iopub.execute_input":"2022-04-06T08:50:21.628021Z","iopub.status.idle":"2022-04-06T08:50:21.642209Z","shell.execute_reply.started":"2022-04-06T08:50:21.627976Z","shell.execute_reply":"2022-04-06T08:50:21.641134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_dir = '../input/h-and-m-personalized-fashion-recommendations/images'\nimages_path = getImagePaths(images_dir)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:21.643580Z","iopub.execute_input":"2022-04-06T08:50:21.643955Z","iopub.status.idle":"2022-04-06T08:50:42.430663Z","shell.execute_reply.started":"2022-04-06T08:50:21.643910Z","shell.execute_reply":"2022-04-06T08:50:42.429882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_multiple_img(random.sample(images_path,50), 5, 8)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:42.432307Z","iopub.execute_input":"2022-04-06T08:50:42.432525Z","iopub.status.idle":"2022-04-06T08:50:53.858325Z","shell.execute_reply.started":"2022-04-06T08:50:42.432499Z","shell.execute_reply":"2022-04-06T08:50:53.857195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Product Code","metadata":{}},{"cell_type":"code","source":"print(colored(f'Count of Unique Product Code is {article.product_code.nunique()}','blue'))\nprint(colored(f'Total Number of Articles {len(article)}','blue'))\n# fig = plt.figure(figsize=(100,80))\nfor i in list(article.groupby('product_code').article_id.nunique().sort_values(ascending=False).index[:10]):\n    tmp = article.query(\"product_code == @i\").article_id\n    disply_multiple_img_ids(tmp, 1, min(20,len(tmp)))","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:50:53.859649Z","iopub.execute_input":"2022-04-06T08:50:53.859904Z","iopub.status.idle":"2022-04-06T08:51:49.674962Z","shell.execute_reply.started":"2022-04-06T08:50:53.859873Z","shell.execute_reply":"2022-04-06T08:51:49.674071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,15))\nsns.countplot(article.product_code.value_counts())\nplt.title('Distribution of Number of Articles falling under same product code')\nplt.xlabel('Number of Unique Articles')","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:49.676341Z","iopub.execute_input":"2022-04-06T08:51:49.676644Z","iopub.status.idle":"2022-04-06T08:51:50.358211Z","shell.execute_reply.started":"2022-04-06T08:51:49.676604Z","shell.execute_reply":"2022-04-06T08:51:50.357348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Product Group**","metadata":{}},{"cell_type":"code","source":"a = article.product_group_name.value_counts()\na_len = a / len(article.index) * 100\ndisplay(pd.concat([a,a_len],axis=1))","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:50.359241Z","iopub.execute_input":"2022-04-06T08:51:50.359452Z","iopub.status.idle":"2022-04-06T08:51:50.378554Z","shell.execute_reply.started":"2022-04-06T08:51:50.359426Z","shell.execute_reply":"2022-04-06T08:51:50.377948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Customers**","metadata":{}},{"cell_type":"code","source":"print(f\"{colored(customer.shape,'red')}\")\ndisplay(customer.columns,customer.nunique(),customer.head(2),customer.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:50.379474Z","iopub.execute_input":"2022-04-06T08:51:50.380004Z","iopub.status.idle":"2022-04-06T08:51:52.075597Z","shell.execute_reply.started":"2022-04-06T08:51:50.379970Z","shell.execute_reply":"2022-04-06T08:51:52.074914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in customer.columns:\n    print(i,customer[i].nunique(),customer[i].unique()[:5])","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:52.076791Z","iopub.execute_input":"2022-04-06T08:51:52.077029Z","iopub.status.idle":"2022-04-06T08:51:54.766363Z","shell.execute_reply.started":"2022-04-06T08:51:52.077000Z","shell.execute_reply":"2022-04-06T08:51:54.765429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer.FN.fillna(0,inplace=True)\ncustomer.Active.fillna(0,inplace=True)\ncustomer.club_member_status.fillna(\"UNK\",inplace=True)\ncustomer.fashion_news_frequency.replace('None','none',inplace=True)\ncustomer.fashion_news_frequency.fillna(\"UNK\",inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:54.768550Z","iopub.execute_input":"2022-04-06T08:51:54.768785Z","iopub.status.idle":"2022-04-06T08:51:54.951809Z","shell.execute_reply.started":"2022-04-06T08:51:54.768757Z","shell.execute_reply":"2022-04-06T08:51:54.950902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(customer.FN.value_counts()/len(customer))\ndisplay(customer.Active.value_counts()/len(customer))\ndisplay(customer.club_member_status.value_counts()/len(customer))\ndisplay(customer.fashion_news_frequency.value_counts()/len(customer))","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:54.953118Z","iopub.execute_input":"2022-04-06T08:51:54.953403Z","iopub.status.idle":"2022-04-06T08:51:55.182793Z","shell.execute_reply.started":"2022-04-06T08:51:54.953372Z","shell.execute_reply":"2022-04-06T08:51:55.181849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have 2 peaks at age 24 and 54.","metadata":{}},{"cell_type":"code","source":"display(customer.age.describe().apply(lambda x: format(x, 'f')))\nsns.histplot(customer.age)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:55.184140Z","iopub.execute_input":"2022-04-06T08:51:55.184455Z","iopub.status.idle":"2022-04-06T08:51:56.817501Z","shell.execute_reply.started":"2022-04-06T08:51:55.184410Z","shell.execute_reply":"2022-04-06T08:51:56.815699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Transactions**","metadata":{}},{"cell_type":"code","source":"display(train.info())\ndisplay(train.columns,train.nunique(),train.head(2),train.isnull().sum())\ntrain.t_dat = pd.to_datetime(train.t_dat)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:51:56.818634Z","iopub.execute_input":"2022-04-06T08:51:56.818855Z","iopub.status.idle":"2022-04-06T08:52:20.737369Z","shell.execute_reply.started":"2022-04-06T08:51:56.818828Z","shell.execute_reply":"2022-04-06T08:52:20.736306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(train.sales_channel_id)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:52:20.739174Z","iopub.execute_input":"2022-04-06T08:52:20.740085Z","iopub.status.idle":"2022-04-06T08:52:23.873570Z","shell.execute_reply.started":"2022-04-06T08:52:20.740032Z","shell.execute_reply":"2022-04-06T08:52:23.872602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(20,8))\nplt.plot(train.t_dat.value_counts().sort_index())","metadata":{"execution":{"iopub.status.busy":"2022-04-06T08:52:23.874853Z","iopub.execute_input":"2022-04-06T08:52:23.875083Z","iopub.status.idle":"2022-04-06T08:52:24.508018Z","shell.execute_reply.started":"2022-04-06T08:52:23.875054Z","shell.execute_reply":"2022-04-06T08:52:24.507076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Some days have more transactions than others.","metadata":{}},{"cell_type":"code","source":"display(train.groupby('sales_channel_id')['price'].describe())","metadata":{"execution":{"iopub.status.busy":"2022-04-06T09:00:31.671118Z","iopub.execute_input":"2022-04-06T09:00:31.671411Z","iopub.status.idle":"2022-04-06T09:00:34.263193Z","shell.execute_reply.started":"2022-04-06T09:00:31.671381Z","shell.execute_reply":"2022-04-06T09:00:34.262381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.countplot(train.price)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T21:59:53.568504Z","iopub.execute_input":"2022-04-05T21:59:53.568899Z","iopub.status.idle":"2022-04-05T21:59:53.574471Z","shell.execute_reply.started":"2022-04-05T21:59:53.568864Z","shell.execute_reply":"2022-04-05T21:59:53.573574Z"},"trusted":true},"execution_count":null,"outputs":[]}]}