{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom IPython import display","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-07T18:17:23.065424Z","iopub.execute_input":"2022-09-07T18:17:23.066239Z","iopub.status.idle":"2022-09-07T18:17:23.095764Z","shell.execute_reply.started":"2022-09-07T18:17:23.066113Z","shell.execute_reply":"2022-09-07T18:17:23.094630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip  uninstall tridentx -y \n!pip install ../input/trident/tridentx-0.7.5-py3-none-any.whl --upgrade","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:23.098284Z","iopub.execute_input":"2022-09-07T18:17:23.099053Z","iopub.status.idle":"2022-09-07T18:17:38.619488Z","shell.execute_reply.started":"2022-09-07T18:17:23.099018Z","shell.execute_reply":"2022-09-07T18:17:38.618121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport numpy as np\nimport pandas as pd\nos.environ['TRIDENT_BACKEND']='pytorch'\nimport trident as T\nfrom trident import *","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:38.621916Z","iopub.execute_input":"2022-09-07T18:17:38.622469Z","iopub.status.idle":"2022-09-07T18:17:45.217914Z","shell.execute_reply.started":"2022-09-07T18:17:38.622414Z","shell.execute_reply":"2022-09-07T18:17:45.216563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_articles=pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv')\ndf_articles.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:45.221432Z","iopub.execute_input":"2022-09-07T18:17:45.222875Z","iopub.status.idle":"2022-09-07T18:17:46.249366Z","shell.execute_reply.started":"2022-09-07T18:17:45.222812Z","shell.execute_reply":"2022-09-07T18:17:46.248163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"product_group_name=df_articles['product_group_name'].unique()\nprint(product_group_name)\nproduct_type_name=df_articles['product_type_name'].unique()\nprint(product_type_name)\nindex_group_name=df_articles['index_group_name'].unique()\nprint(index_group_name)\nsection_name=df_articles['section_name'].unique()\nprint(section_name)\n\nproduct_type_name_mapping = {id:i for i, id in enumerate(product_type_name)}\n\nindex_group_name_mapping = {id:i for i, id in enumerate(index_group_name)}","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:46.250609Z","iopub.execute_input":"2022-09-07T18:17:46.250934Z","iopub.status.idle":"2022-09-07T18:17:46.301418Z","shell.execute_reply.started":"2022-09-07T18:17:46.250905Z","shell.execute_reply":"2022-09-07T18:17:46.299932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keys=df_articles['article_id'].to_numpy()\nprint(keys[:10])\nprint(keys.dtype)\nvalues=df_articles['product_type_name'].to_numpy()\nvalues2=df_articles['index_group_name'].to_numpy()\nproduct_type_dict=OrderedDict(zip(keys,values))\nlist(product_type_dict.items())[:5]\n\nindex_group_dict=OrderedDict(zip(keys,values2))","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:46.303289Z","iopub.execute_input":"2022-09-07T18:17:46.304210Z","iopub.status.idle":"2022-09-07T18:17:46.441109Z","shell.execute_reply.started":"2022-09-07T18:17:46.304163Z","shell.execute_reply":"2022-09-07T18:17:46.439728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs=glob.glob('../input/h-and-m-personalized-fashion-recommendations/images/*/*.*g')\nimages=[]\nlabels=[]\nlabels2=[]\nfor img in imgs:\n    try:\n        folder,filename,ext=split_path(img)\n        filename=int(filename)\n        if filename in product_type_dict:\n           \n            labels.append(product_type_name_mapping[product_type_dict[filename]])\n            labels2.append(index_group_name_mapping[index_group_dict[filename]])\n            images.append(img)\n        else:\n            print('{0} not in product_type_dict'.format(filename))\n    except Exception as e:\n        \n        print(e)\n#         PrintException()\n#         break\n\nprint(len(images),len(labels)) ","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:46.443085Z","iopub.execute_input":"2022-09-07T18:17:46.443567Z","iopub.status.idle":"2022-09-07T18:17:52.596722Z","shell.execute_reply.started":"2022-09-07T18:17:46.443519Z","shell.execute_reply":"2022-09-07T18:17:52.595047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds1=ImageDataset(images,symbol='images')\nds2=ImageDataset(images,symbol='target')\nds3=LabelDataset(labels,symbol='labels')\nds4=LabelDataset(labels2,symbol='labels2')\ndata_provider=DataProvider(traindata=Iterator(data=ds1,label=ZipDataset(ds2,ds3,ds4)))\ndata_provider.paired_transform_funcs=[\n    Resize((128,128),background_color=(236,235,233)),\n    RandomTransformAffine(rotation_range=10, zoom_range=(1.1,1.4), shift_range=0.02, shear_range=0.0, random_flip=0.0,border_mode='constant',background_color=(236,235,233), interpolation=cv2.INTER_AREA,keep_prob=0.2)]\nds1.image_transform_funcs=[\n    RandomAdjustBrightness(scale=(0,0.15)),\n    RandomAdjustContrast(scale=(0.8,1.2)),\n    RandomAdjustSaturation(scale=(0.8,1.2)),\n    SaltPepperNoise(0.002),\n    Normalize(127.5,127.5)]\nds2.image_transform_funcs=[\n    Normalize(127.5,127.5)]\n\nprint(data_provider.signature)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:52.598654Z","iopub.execute_input":"2022-09-07T18:17:52.599489Z","iopub.status.idle":"2022-09-07T18:17:52.821580Z","shell.execute_reply.started":"2022-09-07T18:17:52.599443Z","shell.execute_reply":"2022-09-07T18:17:52.820243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_provider.preview_images()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:52.823207Z","iopub.execute_input":"2022-09-07T18:17:52.824743Z","iopub.status.idle":"2022-09-07T18:17:54.020527Z","shell.execute_reply.started":"2022-09-07T18:17:52.824691Z","shell.execute_reply":"2022-09-07T18:17:54.019578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_data,_target,_label,_label2=data_provider.next()\nprint(_data.shape)\nprint(_label.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:54.023874Z","iopub.execute_input":"2022-09-07T18:17:54.024910Z","iopub.status.idle":"2022-09-07T18:17:55.094815Z","shell.execute_reply.started":"2022-09-07T18:17:54.024868Z","shell.execute_reply":"2022-09-07T18:17:55.094008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder=Sequential(\n    Conv2d_Block((5,5),32,strides=1,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(32,128,128)\n    Conv2d_Block((3,3),64,strides=2,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(64,64,64)\n    Conv2d_Block((3,3),64,strides=2,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(64,32,32)\n    Conv2d_Block((3,3),96,strides=2,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False,dropout_rate=0.2),#(128,16,16)\n    Conv2d_Block((3,3),96,strides=2,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(128,8,8)\n    Conv2d_Block((3,3),128,strides=2,auto_pad=True,activation='leaky_relu',normalization=None,use_bias=False),#(128,4,4)\n    Flatten(), #(2048)\n    Dense(256,activation=None,use_bias=False),\n    L2Norm()\n)\n\n\ndecoder=Sequential(\n    Dense(128*8*8,activation=None,use_bias=False), #(2048))\n    L2Norm(),\n    Reshape((128,8,8)), #(128,8,8)\n    Conv2d_Block((3,3),128,strides=1,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(128,16,16)\n    Upsampling2d(mode='pixel_shuffle',scale_factor=2),\n    Conv2d_Block((3,3),128,strides=1,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False,dropout_rate=0.2),#(96,32,32)\n    Upsampling2d(mode='pixel_shuffle',scale_factor=2),\n    Conv2d_Block((3,3),128,strides=1,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(64,64,64)\n    Upsampling2d(mode='pixel_shuffle',scale_factor=2),\n    Conv2d_Block((3,3),128,strides=1,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(64,128,128)\n    Upsampling2d(mode='pixel_shuffle',scale_factor=2),\n    Conv2d_Block((3,3),128,strides=1,auto_pad=True,activation='leaky_relu',normalization='batch',use_bias=False),#(64,128,128)\n    Conv2d((3,3),3,strides=1,auto_pad=True,activation='tanh',use_bias=False)\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:55.096327Z","iopub.execute_input":"2022-09-07T18:17:55.096942Z","iopub.status.idle":"2022-09-07T18:17:55.154755Z","shell.execute_reply.started":"2022-09-07T18:17:55.096906Z","shell.execute_reply":"2022-09-07T18:17:55.153556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resultdict=ModuleDict(\n    {\n        'decoder':decoder,\n        'classifier':Dense(len(product_type_name), activation=SoftMax()),\n        'classifier2':Dense(len(index_group_name), activation=SoftMax()),\n    },is_multicasting=True\n)\n\n\nautoencoder=Sequential(\n    encoder,\n    resultdict\n)\nae_model=Model(input_shape=(3,128,128),output=autoencoder)\nif os.path.exists('./Models/h_and_m_articles.pth'):\n    ae_model.load_model('./Models/h_and_m_articles.pth')\nelif os.path.exists('../input/h-m-articles-autoencoder/Models/h_and_m_articles.pth.tar'):\n    ae_model.load_model('../input/h-m-articles-autoencoder/Models/h_and_m_articles.pth.tar')\n#ae_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:55.156349Z","iopub.execute_input":"2022-09-07T18:17:55.156720Z","iopub.status.idle":"2022-09-07T18:17:55.486081Z","shell.execute_reply.started":"2022-09-07T18:17:55.156673Z","shell.execute_reply":"2022-09-07T18:17:55.485015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmse_new(decoder,target):\n    return rmse(decoder,target)\ndef accuracy_new(classifier,labels):\n    return accuracy(classifier,labels)\ndef accuracy_new2(classifier2,labels2):\n    return accuracy(classifier2,labels2)\n\ndata_feed=OrderedDict()\ndata_feed['input']='images'\ndata_feed['target']='target'\ndata_feed['output']='decoder'\n\nctx=get_session()\n\nctx.l2_fn=L2Loss('mean')\ndef ae_l2_loss(decoder,target):\n    return ctx.l2_fn(decoder*10,target*10)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:55.487725Z","iopub.execute_input":"2022-09-07T18:17:55.488176Z","iopub.status.idle":"2022-09-07T18:17:55.496374Z","shell.execute_reply.started":"2022-09-07T18:17:55.488139Z","shell.execute_reply":"2022-09-07T18:17:55.495318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ae_model.with_optimizer(optimizer=DiffGrad,lr=5e-5,betas=(0.9, 0.999),gradient_centralization='all')\\\n    .with_loss(ae_l2_loss,name='ae_l2_loss')\\\n    .with_loss(L2Loss,input_names=['decoder','target'],enable_ohem=True,ohem_thresh=0.1,loss_weight=0.5,name='l2_loss_ohem')\\\n    .with_loss(CrossEntropyLoss,input_names=['classifier','labels'],loss_weight=0.5,name='crossentropy')\\\n    .with_loss(CrossEntropyLoss,input_names=['classifier2','labels2'],loss_weight=1,name='crossentropy2')\\\n    .with_metric(rmse_new,name='rmse')\\\n    .with_metric(accuracy_new,name='accuracy')\\\n    .with_metric(accuracy_new2,name='accuracy2')\\\n    .with_regularizer('l2',reg_weight=1e-5)\\\n    .with_grad_clipping(3)\\\n    .with_accumulate_grads(5)\\\n    .with_model_save_path('Models/h_and_m_articles.pth')\\\n    .with_learning_rate_scheduler(PolyLR(period=3000,unit='batch',max_lr=2e-4, min_lr=1e-5,power=2.0,cycle=True))\\\n    .with_callbacks(TileImageCallback(frequency=100,unit='batch',data_feed=data_feed,include_input=True,include_output=True,include_target=True,imshow=True))\\\n    .with_automatic_mixed_precision_training()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:55.497786Z","iopub.execute_input":"2022-09-07T18:17:55.498397Z","iopub.status.idle":"2022-09-07T18:17:55.522001Z","shell.execute_reply.started":"2022-09-07T18:17:55.498362Z","shell.execute_reply":"2022-09-07T18:17:55.521197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plan=TrainingPlan()\\\n    .add_training_item(ae_model)\\\n    .with_data_loader(data_provider)\\\n    .repeat_epochs(60)\\\n    .with_batch_size(16)\\\n    .print_progress_scheduling(5,unit='batch')\\\n    .print_gpu_utilization(200,unit='batch')\\\n    .save_model_scheduling(50,unit='batch')\\\n    .display_loss_metric_curve_scheduling(frequency=200,unit='batch',imshow=True)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:55.525396Z","iopub.execute_input":"2022-09-07T18:17:55.526248Z","iopub.status.idle":"2022-09-07T18:17:55.534594Z","shell.execute_reply.started":"2022-09-07T18:17:55.526212Z","shell.execute_reply":"2022-09-07T18:17:55.533630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plan.start_now()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T18:17:55.535840Z","iopub.execute_input":"2022-09-07T18:17:55.536799Z"},"trusted":true},"execution_count":null,"outputs":[]}]}