{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pandas as pd\nimport pandas.io.sql as psql\nimport numpy as np\nimport numpy.random as rd\nimport gc\nimport multiprocessing as mp\nimport os\nimport sys\nimport pickle\nfrom collections import defaultdict\nfrom glob import glob\nimport math\nfrom datetime import datetime as dt\nfrom pathlib import Path\nimport scipy.stats as st\nimport re\nimport shutil\nfrom tqdm import tqdm_notebook as tqdm\nimport datetime\nts_conv = np.vectorize(datetime.datetime.fromtimestamp) # 秒ut(10桁) ⇒ 日付\n\n\nimport matplotlib\nfrom matplotlib import font_manager\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nfrom matplotlib import rc\n\nfrom matplotlib import animation as ani\nfrom IPython.display import Image\n\nplt.rcParams[\"patch.force_edgecolor\"] = True\n#rc('text', usetex=True)\nfrom IPython.display import display # Allows the use of display() for DataFrames\nimport seaborn as sns\nsns.set(style=\"whitegrid\", palette=\"muted\", color_codes=True)\nsns.set_style(\"whitegrid\", {'grid.linestyle': '--'})\nred = sns.xkcd_rgb[\"light red\"]\ngreen = sns.xkcd_rgb[\"medium green\"]\nblue = sns.xkcd_rgb[\"denim blue\"]\n\n#カラム内の文字数。デフォルトは50\npd.set_option(\"display.max_colwidth\", 100)\n\n#行数\npd.set_option(\"display.max_rows\", None)\npd.set_option(\"display.max_columns\", None)\n#\npd.options.display.float_format = '{:,.5f}'.format\n\n%matplotlib inline","execution_count":1,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\", index_col=0)\ndf_labels = pd.read_csv(\"../input/labels.csv\", index_col=0)\nprint(f\"df_train.shape:{df_train.shape}, df_labels.shape:{df_labels.shape}\")","execution_count":2,"outputs":[{"output_type":"stream","text":"df_train.shape:(109237, 1), df_labels.shape:(1103, 1)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_class = df_labels.shape[0]\nprint(f\"nb_class: {nb_class}\")","execution_count":3,"outputs":[{"output_type":"stream","text":"nb_class: 1103\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":" attribute_ids\nid \n1000483014d91860 147 616 813\n1000fe2e667721fe 51 616 734 813\n1001614cb89646ee 776\n10041eb49b297c08 51 671 698 813 1092\n100501c227f8beea 13 404 492 903 1093","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
attribute_ids
id
1000483014d91860147 616 813
1000fe2e667721fe51 616 734 813
1001614cb89646ee776
10041eb49b297c0851 671 698 813 1092
100501c227f8beea13 404 492 903 1093
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_label_len = df_train.attribute_ids.str.split(\" \").apply(len)","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(25, 4))\ndf_label_len.value_counts().plot.bar()\nplt.title(f\"# of label for each instance.\")","execution_count":6,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":"Text(0.5, 1.0, '# of label for each instance.')"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAABbMAAAEPCAYAAAB4PFkLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X14nGd94PuvkzgmCqkIkWjlgpSsTH42WA4w2wNtSVnOtXSXhZzS0lJgw+uWNoWW7Z522w2nlL5sE9rSLQVCk2XLO6WnlPJSAn1hTylQ+rZDg+xi30EullPLFInEXkDFEbHOHzNSlGDHGluje2Z+3891+Yr0PDOa362vwcmtx89sWV5eRpIkSZIkSZKkXnZe7QEkSZIkSZIkSToTN7MlSZIkSZIkST3PzWxJkiRJkiRJUs9zM1uSJEmSJEmS1PPczJYkSZIkSZIk9Tw3syVJkiRJkiRJPc/NbEmSJPWFiPjmiPh4RHw5In79FOffGhH/dZ1f62MR8UNnOcdpnxsRWyLiLRFxV0T8zdl8/c3Qyfoj4uqIKN2eSZIkSTqTC2oPIEmSpMHW3tS9Fvg68PullMed5Zf6YWAB+KZSyvJGzbfBngg8BXh4KeWrtYfZCKWUTwBxrl8nIg4BP1RK+ei5fi1JkiTl5JXZkiRJ6pqI2ApMAJ8DGsCnz+HLTQCf7eGNbGjNeOhsNrIjwgtNJEmSpAfgvzBLkiSpm3bT3oCOiH/JGTazI+I7gN8ErgRuB/5jKeVTEfFW4N8DyxHxE8AzHugK34i4FHgH8Hha/877F8B1pZR/XPOwyfZV4zuBPwNeVEq5s/38JwD/DXgUMNue42NnmP0/ADcBWyPiK8Cvl1JeFREvAX4GeCjwyfYcc+3nLAM/BvxEe84rTvF1TztLRLwI+Gng4cA88CullFvWPPd7gF8A/kX7/MtKKX/UPj0REX8B7AH+EnhuKWXhFK//r4B3llIe3v78EPAG4Pm0Nu//CHhBKeVrETECvJXWFeongb8HngS8DRgH/jAi7gF+sZTyqxHxHuBq4CLgM8CPllL+vv06bwW+ClwOfBfw2faMB9vnHw28ltYPSZaA3yyl3BAR57W/Jy8BHgL8z/b3/M77r02SJEn9xSuzJUmStOEi4kURcYzWJvK3tz/+SeBXIuJYRJxq0/ahwK3A64DLaG3g3hoRl5VSXgi8C/jVUsqD13GrivOAt9DabB0H/pnWBuxazwdeDIzRugXK69pzfGt7jv9KawP6p4D3RsToA71gKeW3geuAv2zP+KqI+D+BG4FntV9nFvjd+z31GbQ23R91iu/JmWb5IvB04JuAFwG/ERGPaz/3/wDeDvxnWpu63wUcWvPln9t+zsOAC9tfe72eBfxbWpvve4AXto//JPCPwCjwzcArgOVSyvOAw8A17e/Nr7Yf/xHgke0ZPk2r8VrPprUZfykwA/xye22XAB+ltZG+HdhBa9Ma4MdpfU+f1D53F60fMkiSJKnPeWW2JEmSNlwp5S3AWyLiE7Q2F+8EPgg89gFuE/I04HOllHe0P393RLwcuIbW1b6dvP6XgPeufB4Rv0zr6uu13lFK2dc+/0rgtoh4Aa37e3+4lPLh9uP+NCL+F/DvaF1h3Il/D7y5lPLp9utcD9wVEZeXUg61H3PjA1w1/ICzlFJuXfPYP4+IP6F1pfOngf/Qfu0/bZ8/cr+v/ZZSyu3tuX4P+L86WNfr1lxd/ofAY9rHl2ht2k+UUmaATzzQFymlvHnl44j4eVrfm+FSyvH24feVUv6mff5dtH7AAa0N/C+UUlbeCPRrwF+3P74O+LGVq/DbX/dwRDyvlPL1DtYoSZKkHuNmtiRJkjZU+wrrfwC2AA8GPgZsa5++KyJ+vpTy2lM8dTutK5fXmgW+9SxmGAJ+g9bVw5e2D18SEeeXUu5pf37H/V5nKzBC62ruH4iIa9ac38o3boavx3bW3FqllPKViPgSrTUdOsUc9/eAs0TEU4FX0boty3nAELC3/bhHAB/m9L6w5uNFWq3W6/7P3d7++NeAnwf+JCIA/nsp5dWn+gIRcT6tK61/gNaV3Cfbp0aAlc3s0834CODgaWabAN4XESfXHLuH1pXi99/QlyRJUh9xM1uSJEkbqn2V8UMi4tnAk0spPxIR7wNuOsPtQeZobUSuNU7rVhKd+kkggMeXUr4QEY8B/o7WBvuKR9zvdZaABVqby+8opbzkLF73/u6zpoi4mNYtVNZuqj7QG1qedpaI2Ebr6vPnAx8opSxFxPu5d413AJPnNn5nSilfpvW9/8mI2A38fxHxt6WU/8k3rvO5wPcA/5rWxv4wrVuCbOHM7qB1C5LTnXtxKeUvOl+BJEmSepmb2ZIkSeqWBvdelfxYoHmGx38YeH1EPBf4PeCZtO4j/aGzeO1LaN0n+1j7SvFXneIx10bE22ltpP4i8PullHsi4p3A30bEv6F1X+atwBOAmfu9geR6vJvW7VJ+B9gP3AD89ZpbjJzJaWehdfXyNlpv7Pj19lXa3w3saz/3t2ldIf0hWldyjwGXlFIOdLiGdYuIpwMHaF01fZzWFdErV0j/E603olxxCXAC+BKtK8pv6OClPgT8t/abgf4WrXt+P6qU8tfAzcAvR8QLSimz7fuLf0cp5QNnvzJJkiT1At8AUpIkSd3SAD4dEZcB95RS7nqgB7fvc/10Wlf2fgn4aeDppZSFs3jt1wIX0brS+q849dXd76B1L+4vAA8CXt6e4w5aVwy/gtZG8R203kSx4393bl+J/kpaV1AfpXWl9OmuKD7V8087S/sq6JfT2vi/i9aVzh9c89y/of2mkLQ2lv+cb7zyfaM9ktam+1eAvwTeWEpZuT3LjcDPtt8A9KdovTnlLK2r1D9Lq9O6tNf+FFr3U/8C8Dngye3Tv0nr+/AnEfHl9td9/MpzI+IrEXH1Wa9QkiRJ1WxZXn6gv9UoSZIkSZIkSVJ9XpktSZIkSZIkSep5bmZLkiRJkiRJknqem9mSJEmSJEmSpJ7nZrYkSZIkSZIkqeddUHuAjdRsNrcB30brneLvqTyOJEmSJEmSJOnUzgfGgL9tNBon1vOEgdrMprWR/YnaQ0iSJEmSJEmS1uVq4JPreeCgbWYfBbjyyiu58MILa8+y6RYXFxkaGqo9hiqwfV62z8v2Odk9L9vnZfu8bJ+X7XOye16Z2999993cfvvt0N7TXY9B28y+B+DCCy9k27ZttWfZdPv27aPRaNQeQxXYPi/b52X7nOyel+3zsn1ets/L9jnZPS/bAx3cLto3gJQkSZIkSZIk9Tw3syVJkiRJkiRJPc/N7AEyNjZWewRVYvu8bJ+X7XOye162z8v2edk+L9vnZPe8bN+ZLcvLy7Vn2DDNZvNy4PO7d+9Oec9sSZIkSZIkSeoHJ06cYN++fQBXNBqNQ+t5jldmD5Dp6enaI6gS2+dl+7xsn5Pd87J9XrbPy/Z52T4nu+dl+864mT1AlpaWao+gSmyfl+3zsn1Ods/L9nnZPi/b52X7nOyel+0742a2JEmSJEmSJKnnuZm9ge5eOln19RuNRtXXr73+zIaGhmqPoEpsn5ftc7J7XrbPy/Z52T4v2+dk97xs3xnfAHKDPfX6z1R53V7wkRuvqj2CJEmSJEmSpD7gG0BKSc3OztYeQZXYPi/b52T3vGyfl+3zsn1ets/J7nnZvjNuZksDYGFhofYIqsT2edk+J7vnZfu8bJ+X7fOyfU52z8v2nXEzW5IkSZIkSZLU89zMliRJkiRJkiT1PDezpQEwNTVVewRVYvu8bJ+T3fOyfV62z8v2edk+J7vnZfvOuJktDYDFxcXaI6gS2+dl+5zsnpft87J9XrbPy/Y52T0v23fGzWxpABw8eLD2CKrE9nnZPie752X7vGyfl+3zsn1Ods/L9p1xM1uSJEmSJEmS1PMuWM+DIuL9wBXASeArwI+XUm6LiEPA19q/AH6mlPLH7ec8AbgFuAg4BFxbSvniuZyTJEmSJEmSJOW03iuzX1BKuaqU8ljgNcCb15z7/lLKY9q/VjayzwPeCbyslHIl8HHg1edyTtLpjY+P1x5Bldg+L9vnZPe8bJ+X7fOyfV62z8nuedm+M+vazC6lHF/z6TCtK7QfSAP4Winlk+3PbwaedY7nJJ3G6Oho7RFUie3zsn1Ods/L9nnZPi/b52X7nOyel+07s67bjABExP8AvhvYAvzbNafeFRFbgE8CryilHAPGgdmVB5RSFiLivIh46NmeK6Xcud5Z9+3bt/rxzp07AThw4MDqsbGxMbZv38709DRLS0sADA0NsWvXLmZnZ1lYWFh97NTUFIuLi/e5Gfv4+Dijo6M0m83VY8PDw+zYsWO9Iw6std+TkZERJiYm2L9//+o7s27dupU9e/YwNzfH0aNHVx+72Z1mZmY4fvzen9E0Gg3m5+c5fPjw6rHJyUmGhobYu3dvz6/p+PHjq/8clDUNYqdurGlpaYmxsbGBWtMgdurGmlZmGKQ1DWIn1+SaXJNrck2uyTW5JtfkmlyTa3JNp17TkSNH6NSW5eXljp4QEc8DnlNK+XcR8YhSyh0RsQ14LXBJKeXaiHgm8OJSytPWPG8ReDjw5LM5t57N7GazeTnw+d27d7Nt27aO1rVRnnr9Z6q8bi/4yI1X1R4hrWazSaPRqD2GKrB9XrbPye552T4v2+dl+7xsn5Pd88rc/sSJEysXJV/RaDQOrec5671n9qpSyjuAJ0fEZaWUO9rHTgBvBL6z/bDDwMTKcyJiBDjZ3pA+23OSJEmSJEmSpKTOuJkdEQ+OiEes+fwa4E7gaxEx3D62BXg2cFv7YU3gooh4Yvvz64D3nOM5SacxPDxcewRVYvu8bJ+T3fOyfV62z8v2edk+J7vnZfvOnPE2IxHxzcAHgIuBe2htZP8UcAx4L3B++9dngZeXUo62n/cdwC3Ag4BDwLWllH86l3Nn4m1G6vI2I5IkSZIkSZLW42xuM3LGN4BsbyQ/4TSnH/sAz/sUMLWR5ySd2szMjG9AmpTt87J9TnbPy/Z52T4v2+dl+5zsnpftO9PxPbMl9Z6170arXGyfl+1zsntets/L9nnZPi/b52T3vGzfGTezJUmSJEmSJEk9z81sSZIkSZIkSVLPO+MbQPYT3wCyLt8AUpIkSZIkSdJ6nM0bQHpltjQA5ufna4+gSmyfl+1zsntets/L9nnZPi/b52T3vGzfGTezpQFw+PDh2iOoEtvnZfuc7J6X7fOyfV62z8v2Odk9L9t3xs1sSZIkSZIkSVLPczNbkiRJkiRJktTz3MyWBsDk5GTtEVSJ7fOyfU52z8v2edk+L9vnZfuc7J6X7TvjZrY0AIaGhmqPoEpsn5ftc7J7XrbPy/Z52T4v2+dk97xs3xk3s6UBsHfv3tojqBLb52X7nOyel+3zsn1ets/L9jnZPS/bd8bNbEmSJEmSJElSz3MzW5IkSZIkSZLU89zMlgbAyMhI7RFUie3zsn1Ods/L9nnZPi/b52X7nOyel+0742a2NAAmJiZqj6BKbJ+X7XOye162z8v2edk+L9vnZPe8bN8ZN7OlAbB///7aI6gS2+dl+5zsnpft87J9XrbPy/Y52T0v23fGzWxpACwuLtYeQZXYPi/b52T3vGyfl+3zsn1ets/J7nnZvjNuZkuSJEmSJEmSet4F63lQRLwfuAI4CXwF+PFSym0RcSXwNuAy4EvA80spn2s/Z8PPSTq1rVu31h5Bldg+L9vnZPe8bJ+X7fOyfV62z8nuedm+M+u9MvsFpZSrSimPBV4DvLl9/GbgplLKlcBNwC1rntONc5JOYc+ePbVHUCW2z8v2Odk9L9vnZfu8bJ+X7XOye16278y6NrNLKcfXfDoMnIyIhwGPA97dPv5u4HERMdqNc2e3PCmHubm52iOoEtvnZfuc7J6X7fOyfV62z8v2Odk9L9t3Zt33zI6I/xERh4FfBl4APAI4Ukq5B6D9z7n28W6ck3QaR48erT2CKrF9XrbPye552T4v2+dl+7xsn5Pd87J9Z9Z1z2yAUsoPAUTE84BfA17ZraHO1b59+1Y/3rlzJwAHDhxYPTY2Nsb27duZnp5maWkJgKGhIXbt2sXs7CwLCwurj52ammJxcZGDBw+uHhsfH2d0dJRms7l6bHh4mB07dnRtTf1i7fdkZGSEiYkJ9u/fv/rOrFu3bmXPnj3Mzc3d53+sm91pZmaG48fv/QsHjUaD+fl5Dh8+vHpscnKSoaEh9u7d2/NrAgZuTYPYqRtrAgZuTYPYqRtrAjh27NhArWkQO3VjTceOHRu4NQ1ip26saW5ubuDWNIidurGmmZmZgVvTIHbqxpqmp6cHbk2D2Kkba2o2mwO3pkHstJFrgnv3VAZlTYPYqRtrgnvbD8qa1tvpyJEjdGrL8vJyx0+KiH8GLgcKcFkp5Z6IOJ/WGzY+EtgC3L6R50op82eaq9lsXg58fvfu3Wzbtq3jdW2Ep17/mSqv2ws+cuNVtUdIq9ls0mg0ao+hCmyfl+1zsntets/L9nnZPi/b52T3vDK3P3HixMpFyVc0Go1D63nOGW8zEhEPjohHrPn8GuBO4IvAbcBz2qeeA/xdKWW+lLLh59azGCmrtT/NUy62z8v2Odk9L9vnZfu8bJ+X7XOye16278x6bjNyMfCeiLgYuIfWRvY1pZTliLgOeFtE/BxwF/D8Nc/rxjlJkiRJkiRJUkJn3MwupfwT8ITTnDsAPH6zzkm9avnrd7Plggurvf7FF19c7bWh/vozO3DgQNq/jpSd7XOye162z8v2edk+L9vnZPe8bN+Zdb8BpKTT23LBhdx1w9Nqj1HNpa+4tfYIkiRJkiRJGnBnvGe2JEmSJEmSJEm1uZktSX1sbGys9giqxPY52T0v2+dl+7xsn5ftc7J7XrbvjJvZktTHtm/fXnsEVWL7nOyel+3zsn1ets/L9jnZPS/bd8bNbEnqY9PT07VHUCW2z8nuedk+L9vnZfu8bJ+T3fOyfWfczJakPra0tFR7BFVi+5zsnpft87J9XrbPy/Y52T0v23fGzWxJkiRJkiRJUs9zM1uS+tjQ0FDtEVSJ7XOye162z8v2edk+L9vnZPe8bN8ZN7MlqY/t2rWr9giqxPY52T0v2+dl+7xsn5ftc7J7XrbvjJvZktTHZmdna4+gSmyfk93zsn1ets/L9nnZPie752X7zriZLUl9bGFhofYIqsT2Odk9L9vnZfu8bJ+X7XOye16274yb2ZIkSZIkSZKknudmtiRJkiRJkiSp57mZLUl9bGpqqvYIqsT2Odk9L9vnZfu8bJ+X7XOye16274yb2ZLUxxYXF2uPoEpsn5Pd87J9XrbPy/Z52T4nu+dl+864mS1JfezgwYO1R1Alts/J7nnZPi/b52X7vGyfk93zsn1n3MyWJEmSJEmSJPU8N7MlSZIkSZIkST3PzWxJ6mPj4+O1R1Alts/J7nnZPi/b52X7vGyfk93zsn1nLjjTAyLiMuAdwCRwN/A54EdKKfMRsQzsBU62H/68Usre9vOuAX6t/RpN4EWllMVzOSdJuq/R0dHaI6gS2+dk97xsn5ft87J9XrbPye552b4z67kyexn41VJKlFKmgIPAq9ec/45SymPav1Y2sh8MvAm4ppSyA/gy8FPnck6S9I2azWbtEVSJ7XOye162z8v2edk+L9vnZPe8bN+ZM25ml1LuLKV8bM2hvwImzvC0pwL/q5TyufbnNwM/eI7nJEmSJEmSJElJnfE2I2tFxHnAjwIfXHP4YxFxAfAR4OdLKSeAcWB2zWMOA49of3y25yRJkiRJkiRJSXW0mQ28HvgK8Ib25+OllDsi4pto3Vf7lcDPbuB8Z2Xfvn2rH+/cuROAAwcOrB4bGxtj+/btTE9Ps7S0BMDQ0BC7du1idnaWhYWF1cdOTU2xuLjIwYMHV4+Nj48zOjp6n78GMDw8zI4dO7q2pn6x9nsyMjLCxMQE+/fvZ3GxddvzrVu3smfPHubm5jh69OjqYze708zMDMePH1893mg0mJ+f5/Dhw6vHJicnGRoaYu/evetaU3b90mnQfu8NDw8P3JoGsVM31nTJJZdw7NixgVrTIHbqxpqOHTs2cGsaxE7dWNPc3NzArWkQO3VjTTMzMwO3pkHs1I01TU9PD9yaBrFTN9bUbDYHbk2D2Gkj1wT37qkMypoGsVM31gT3th+UNa2305EjR+jUluXl5XU9MCJeA+yhdT/rE6c4fw3wf5dSnhwRPwC8oJTy9Pa5fwm8rZTy6LM9t54Zm83m5cDnd+/ezbZt29a1ro321Os/U+V1e8FHbryq9ghV3XXD02qPUM2lr7i19giSJEmSJEnqIydOnFi5KPmKRqNxaD3PWc8bQBIRNwAN4BkrG9kRcWlEXNT++ALg+4Hb2k/5I+DbIuKR7c+vA37vHM9Jku5nZmam9giqxPY52T0v2+dl+7xsn5ftc7J7XrbvzBk3syPi0cD1wHbgUxFxW0S8D9gJ/HVEfAaYBpZo3WaEUsqXgR8GPhQRM8Aw8JpzOSdJ+kZr/4qQcrF9TnbPy/Z52T4v2+dl+5zsnpftO3PGe2aXUv4e2HKa06e9UXAp5QPABzbynCRJkiRJkiQpp3XdZkSSJEmSJEmSpJrczJakPtZoNGqPoEpsn5Pd87J9XrbPy/Z52T4nu+dl+864mS1JfWx+fr72CKrE9jnZPS/b52X7vGyfl+1zsntetu+Mm9mS1McOHz5cewRVYvuc7J6X7fOyfV62z8v2Odk9L9t3xs1sSZIkSZIkSVLPczNbkiRJkiRJktTz3MyWpD42OTlZewRVYvuc7J6X7fOyfV62z8v2Odk9L9t3xs1sSepjQ0NDtUdQJbbPye552T4v2+dl+7xsn5Pd87J9Z9zMlqQ+tnfv3tojqBLb52T3vGyfl+3zsn1ets/J7nnZvjNuZkuSJEmSJEmSep6b2ZIkSZIkSZKknudmtiT1sZGRkdojqBLb52T3vGyfl+3zsn1ets/J7nnZvjNuZktSH5uYmKg9giqxfU52z8v2edk+L9vnZfuc7J6X7TvjZrYk9bH9+/fXHkGV2D4nu+dl+7xsn5ft87J9TnbPy/adcTNbkvrY4uJi7RFUie1zsntets/L9nnZPi/b52T3vGzfGTezJUmSJEmSJEk9z81sSepjW7durT2CKrF9TnbPy/Z52T4v2+dl+5zsnpftO+NmtiT1sT179tQeQZXYPie752X7vGyfl+3zsn1Ods/L9p1xM1uS+tjc3FztEVSJ7XOye162z8v2edk+L9vnZPe8bN+ZC870gIi4DHgHMAncDXwO+JFSynxEPAG4BbgIOARcW0r5Yvt5G35OknRfR48eZfv27bXHUAW2z8nuedk+L9vnZfu8bJ+T3fOyfWfWc2X2MvCrpZQopUwBB4FXR8R5wDuBl5VSrgQ+DrwaoBvnJEmSJEmSJEl5nXEzu5RyZynlY2sO/RUwATSAr5VSPtk+fjPwrPbH3TgnST1n6eRS1ddvNBrVXrv22iVJkiRJUi5nvM3IWu0rp38U+CAwDsyunCulLETEeRHx0G6cK6Xcud459+3bt/rxzp07AThw4MDqsbGxMbZv38709DRLS63NmKGhIXbt2sXs7CwLCwurj52ammJxcZGDBw+uHhsfH2d0dJRms7l6bHh4mB07dqx3xIG19nsyMjLCxMQE+/fvZ3FxEWi9Q+uePXuYm5vj6NGjq4/d7E4zMzMcP3589Xij0WB+fp7Dhw+vHpucnGRoaIi9e/eua03Z9Uunjf6912g0eFnzJWf5XetvNzXeBNAXnQbx997k5CTHjh0bqDUNYqdurOnYsWMDt6ZB7NSNNc3NzQ3cmgaxUzfWNDMzM3BrGsRO3VjT9PT0wK1pEDt1Y03NZnPg1jSInTZyTXDvnsqgrGkQO3VjTXBv+0FZ03o7HTlyhE5tWV5eXveDI+Im4FuB7wO+F3hxKeVpa84vAg8HnrzR59azmd1sNi8HPr979262bdu27nVtpKde/5kqr9sLPnLjVbVHqOquG5525gcNqEtfcWvtEarKvpmtOr761a9y8cUX1x5Dm8zuedk+L9vnZfu8bJ+T3fPK3P7EiRMrFyVf0Wg0Dq3nOeu5ZzYAEfEa4JHAD5ZSTgKHad1uZOX8CHCyvencjXOSJKlt7U+6lYfd87J9XrbPy/Z52T4nu+dl+86sazM7Im6gdT/rZ5RSTrQPN4GLIuKJ7c+vA97TxXOSJEmSJEmSpKTOeM/siHg0cD1wO/CpiAD4fCnleyPiecAtEfEg4BBwLUAp5eRGn5MkSZIkSZIk5XXGzexSyt8DW05z7lPA1GadkyRJLWNjY7VHUAV2z8v2edk+L9vnZfuc7J6X7Tuz7ntmS5Kk3rF9+/baI6gCu+dl+7xsn5ft87J9TnbPy/adcTNbkqQ+ND09XXsEVWD3vGyfl+3zsn1ets/J7nnZvjNuZkuS1IeWlpZqj6AK7J6X7fOyfV62z8v2Odk9L9t3xs1sSZIkSZIkSVLPczNbkqQ+NDQ0VHsEVWD3vGyfl+3zsn1ets/J7nnZvjNuZkuS1Id27dpVewRVYPe8bJ+X7fOyfV62z8nuedm+M25mS5LUh2ZnZ2uPoArsnpft87J9XrbPy/Y52T0v23fGzWxJkvrQwsJC7RFUgd3zsn1ets/L9nnZPie752X7zriZLUmSJEmSJEnqeW5mS5IkSZIkSZJ6npvZkiT1oampqdojqAK752X7vGyfl+3zsn1Ods/L9p1xM1uSpD60uLhYewRVYPe8bJ+X7fOyfV62z8nuedm+M25mS5LUhw4ePFh7BFVg97xsn5ft87J9XrbPye552b4zbmZLkiRJkiRJknqem9mSJEmSJEmSpJ7nZrYkSX1ofHy89giqwO552T4v2+dl+7xsn5Pd87J9Z9zMliSpD42OjtYeQRXYPS/b52X7vGyfl+1zsntetu+Mm9mSJPWhZrNZewRVYPe8bJ+X7fOyfV62z8nuedm+Mxes50ER8RrgmcDlwFQpZV/7+CHga+1fAD9TSvnj9rknALcAFwGHgGtLKV88l3OSJEmSJEmSpJzWe2X2+4HvAmZPce77SymPaf9a2cg+D3gn8LJSypXAx4FEkuDhAAAVvElEQVRXn8s5SZIkSZIkSVJe69rMLqV8spRyRwdftwF8rZTyyfbnNwPPOsdzkiSpbXh4uPYIqsDuedk+L9vnZfu8bJ+T3fOyfWc24p7Z74qI6Yh4Y0Q8pH1snDVXcZdSFoDzIuKh53BOkiS17dixo/YIqsDuedk+L9vnZfu8bJ+T3fOyfWfWdc/sB3B1KeWOiNgGvBZ4A3DtuY91bvbt27f68c6dOwE4cODA6rGxsTG2b9/O9PQ0S0tLAAwNDbFr1y5mZ2dZWFhYfezU1BSLi4scPHhw9dj4+Dijo6P3uUH78PCwv/m4703rR0ZGmJiYYP/+/SwuLgKwdetW9uzZw9zcHEePHl197GZ3mpmZ4fjx46vHG40G8/PzHD58ePXY5OQkQ0ND7N27d11ryq5fOm30771Go3GW37HB0Q+dBvH33iWXXMLDHvawgVrTIHZyTa7JNbkm1+SaXJNrck2uyTW5Jtd06jUdOXKETm1ZXl5e94Pbb/j49JU3gLzfuSngg6WUKyLi24C3lFJ2t8+NAIdKKQ8+23Prma/ZbF4OfH737t1s27Zt3evaSE+9/jNVXrcXfOTGq2qPUNVdNzyt9gjVXPqKW2uPUNXLmi+pPUIVNzXeVHuE1JrNpj9MScjuedk+L9vnZfu8bJ+T3fPK3P7EiRMrFyVf0Wg0Dq3nOWd9m5GIuDgihtsfbwGeDdzWPt0ELoqIJ7Y/vw54zzmekyRJkiRJkiQlta7bjETE64DvA74F+GhEfAm4BnhvRJwPnA98FngpQCnlZEQ8D7glIh4EHKJ9+5GzPSdJkiRJkiRJyquj24z0Om8zUpe3GfE2I1l5mxFJkiRJkqTObOptRiRJUj3z8/O1R1AFds/L9nnZPi/b52X7nOyel+0742a2JEl9aO07TisPu+dl+7xsn5ft87J9TnbPy/adcTNbkiRJkiRJktTz3MyWJEmSJEmSJPU8N7MlSepDk5OTtUdQBXbPy/Z52T4v2+dl+5zsnpftO+NmtiRJfWhoaKj2CKrA7nnZPi/b52X7vGyfk93zsn1n3MyWJKkP7d27t/YIqsDuedk+L9vnZfu8bJ+T3fOyfWfczJYkSZIkSZIk9Tw3syVJkiRJkiRJPc/NbEmS+tDIyEjtEVSB3fOyfV62z8v2edk+J7vnZfvOuJktSVIfmpiYqD2CKrB7XrbPy/Z52T4v2+dk97xs3xk3syVJ6kP79++vPYIqsHtets/L9nnZPi/b52T3vGzfGTezJUnqQ4uLi7VHUAV2z8v2edk+L9vnZfuc7J6X7TvjZrYkSZIkSZIkqee5mS1JUh/aunVr7RFUgd3zsn1ets/L9nnZPie752X7zriZLUlSH9qzZ0/tEVSB3fOyfV62z8v2edk+J7vnZfvOuJktSVIfmpubqz2CKrB7XrbPy/Z52T4v2+dk97xs3xk3syVJ6kNHjx6tPYIqsHtets/L9nnZPi/b52T3vGzfGTezJUk6Cyfvvrvq6zcajWqvXXvtkiRJkqScLjjTAyLiNcAzgcuBqVLKvvbxK4G3AZcBXwKeX0r5XLfOSZLUS8678EL2v/CFtceoYtdb31p7BEmSJElSQuu5Mvv9wHcBs/c7fjNwUynlSuAm4JYun5MkSUpt586dtUdQJbbPy/Z52T4v2+dk97xs35kzXpldSvkkQESsHouIhwGPA57SPvRu4A0RMQps2ehzpZT5s12gJEmSJEmSJKn/ne09sx8BHCml3APQ/udc+3g3zkmSJKV34MCB2iOoEtvnZfu8bJ+X7XOye16278wZr8zuR/v27Vv9eOVS/bW/McbGxti+fTvT09MsLS0BMDQ0xK5du5idnWVhYWH1sVNTUywuLnLw4MHVY+Pj44yOjtJsNlePDQ8Ps2PHjq6tqV+s/Z6MjIwwMTHB/v37WVxcBGDr1q3s2bOHubm5+7xb62Z3mpmZ4fjx46vHG40G8/PzHD58ePXY5OQkQ0ND7N27d11ryq5fOm30772ab8LXK/qhUzd+79mevug0iL/3jh07NnBrGsRO3VjT3NzcwK1pEDt1Y00zMzMDt6ZB7NSNNU1PTw/cmgaxUzfW1Gw2B25Ng9hpI9cE9+6pDMqaBrFTN9YE97YflDWtt9ORI0fo1Jbl5eV1PTAiDgFPL6Xsa99m5HbgslLKPRFxPq03bHwkrduFbOi59d5mpNlsXg58fvfu3Wzbtm1d69poT73+M1Vetxd85Marao9Q1V03PK32CNVc+opba49Q1cuaL6k9QhU3Nd5Ue4TqfANIbbZms+kPUpKyfV62z8v2edk+J7vnlbn9iRMnVi5KvqLRaBxaz3PO6jYjpZQvArcBz2kfeg7wd6WU+W6cO5sZJUmSBs3Y2FjtEVSJ7fOyfV62z8v2Odk9L9t35oy3GYmI1wHfB3wL8NGI+FIp5dHAdcDbIuLngLuA5695WjfOSZIkpbZ9+/baI6gS2+dl+7xsn5ftc7J7XrbvzBk3s0spLwdeforjB4DHn+Y5G35OkiQpu5V7pyof2+dl+7xsn5ftc7J7XrbvzFndZkSSJEmbb+UNU5SP7fOyfV62z8v2Odk9L9t3xs1sSZIkSZIkSVLPczNbkiSpTwwNDdUeQZXYPi/b52X7vGyfk93zsn1n3MyWJEnqE7t27ao9giqxfV62z8v2edk+J7vnZfvOuJktSZLUJ2ZnZ2uPoEpsn5ft87J9XrbPye552b4zbmZLkiT1iYWFhdojqBLb52X7vGyfl+1zsntetu+Mm9mSJEmSJEmSpJ7nZrYkSZIkSZIkqee5mS1JktQnpqamao+gSmyfl+3zsn1ets/J7nnZvjNuZkuSJPWJxcXF2iOoEtvnZfu8bJ+X7XOye16274yb2ZIkSX3i4MGDtUdQJbbPy/Z52T4v2+dk97xs3xk3syVJkiRJkiRJPc/NbEmSJEmSJElSz3MzW5IkqU+Mj4/XHkGV2D4v2+dl+7xsn5Pd87J9Z9zMliRJ6hOjo6O1R1Alts/L9nnZPi/b52T3vGzfGTezJUmS+kSz2aw9giqxfV62z8v2edk+J7vnZfvOuJktSZLUga8v3VPttRuNRrXXhrprlyRJkqQLag8gSZLUTy7Yej5v/PFba49RxUtf/7TaI0iSJElKzCuzJUmSpB43PDxcewRVYvu8bJ+X7XOye16278w5X5kdEYeAr7V/AfxMKeWPI+IJwC3ARcAh4NpSyhfbzzmrc5IkSVJGO3bsqD2CKrF9XrbPy/Y52T0v23dmo67M/v5SymPav/44Is4D3gm8rJRyJfBx4NUAZ3tOkiRJympmZqb2CKrE9nnZPi/b52T3vGzfmW7dZqQBfK2U8sn25zcDzzrHc5IkSVJKx48frz2CKrF9XrbPy/Y52T0v23dmozaz3xUR0xHxxoh4CDAOzK6cLKUsAOdFxEPP4ZwkSZIkSZIkKalzvmc2cHUp5Y6I2Aa8FngD8L4N+Lpnbd++fasf79y5E4ADBw6sHhsbG2P79u1MT0+ztLQEwNDQELt27WJ2dpaFhYXVx05NTbG4uMjBgwdXj42PjzM6Okqz2Vw9Njw87D1u4D7fk5GRESYmJti/fz+Li4sAbN26lT179jA3N8fRo0dXH7vZnWZmZu7zk69Go8H8/DyHDx9ePTY5OcnQ0BB79+5d15qy65dOG/17r9FonOV3bHD0Q6du/N6zPX3RqRu/97K3v/vuu/ui0yD+3pubmxu4NQ1ip26saWZmZuDWNIidurGm6enpgVvTIHbqxpqazebArWkQO23kmuDePZVBWdMgdurGmuDe9oOypvV2OnLkCJ3asry83PGTTicipoAP0ro1yFtKKbvbx0eAQ6WUB0fEt53NufW8frPZvBz4/O7du9m2bduGrasTT73+M1Vetxd85Marao9Q1V03PK32CNVc+opba49Q1cuaL6k9QhU3Nd5Ue4Tq9r/whbVHqGLXW99ae4Tq3vjjOf9/76Wvz/tnnSRJkqSNdeLEiZWLkq9oNBqH1vOcc7rNSERcHBHD7Y+3AM8GbgOawEUR8cT2Q68D3tP++GzPSZIkSSmtXLGlfGyfl+3zsn1Ods/L9p0513tmfzPwsYiYBvYBVwIvLaWcBJ4H/FZEfA54EvBfAM72nCRJklTL15dOVH390dHRqq9fe/2Zrf2rv8rF9nnZPie752X7zpzTPbNLKf8APPY05z4FTG3kOUmSJKmGC7Zu49dffHXtMar5yTd/ovYIkiRJ0jlfmS1JkiRJkiRJUte5mS1JkiRJPWpycrL2CKrE9nnZPie752X7zriZLUmSJEk9amhoqPYIqsT2edk+J7vnZfvOuJktSZIkST1q7969tUdQJbbPy/Y52T0v23fGzWxJkiRJkiRJUs9zM1uSJEmSJEmS1PPczJYkSZKkHjUyMlJ7BFVi+7xsn5Pd87J9Z9zMliRJkqQeNTExUXsEVWL7vGyfk93zsn1n3MyWJEmSpB61f//+2iOoEtvnZfuc7J6X7TvjZrYkSZIk9ajFxcXaI6gS2+dl+5zsnpftO+NmtiRJkiRJkiSp57mZLUmSJEk9auvWrbVHUCW2z8v2Odk9L9t3xs1sSZIkSepRe/bsqT2CKrF9XrbPye552b4zbmZLkiRJUo+am5urPYIqsX1ets/J7nnZvjNuZkuSJElSjzp69GjtEVSJ7fOyfU52z8v2nXEzW5IkSZIkSZLU89zMliRJkiRJkiT1PDezJUmSJOk0Ti6drPr6jUaj6uvXXn9mO3furD2CKrF9TnbPy/aduaD2AJIkSZLUq87beh4HX/XntceoZvIXnlR7BEmSpFU9eWV2RFwZEX8ZEbe3//nI2jNJkiRJkrRZDhw4UHsEVWL7nOyel+0705Ob2cDNwE2llCuBm4BbKs8jSZIkSZIkSaqo524zEhEPAx4HPKV96N3AGyJitJQyf4annw9w9913d3HCB/aQoeVqr13biRMnao9Q1dKDHlJ7hGqyt38wl9QeoYrs3QFOXmL7rC588Pm1R6gie/sHXfLQ2iNUk739PRdtqT1CNdnb1+b3Py/b52T3vLK2X7OHu+7/wNqyvNxbm68R0QDeXkp59JpjnwWuLaV8+oGe22w2nwh8ossjSpIkSZIkSZI2xtWNRuOT63lgz12ZfY7+FrgaOArcU3kWSZIkSZIkSdKpnQ+M0drTXZdevDL7YcDtwGWllHsi4nzgS8Aj13GbEUmSJEmSJEnSAOq5N4AspXwRuA14TvvQc4C/cyNbkiRJkiRJkvLquSuzASJiJ/A24FLgLuD5pZRSdypJkiRJkiRJUi09uZktSZIkSZIkSdJaPXebEUmSJEmSJEmS7s/NbEmSJEmSJElSz3MzW5IkSZIkSZLU89zMliRJkiRJkiT1PDezJUmSJEmSJEk974LaA0g6dxFxaSnlrtpzaHNFxL8upXy09hySNkdEPBi4Epgppfzv2vNIkronIoaAXcDBUsqx2vOouyLiocA48HVazf+58kiS1LO2LC8v155BZyEiLgN+hdYfeB8opdy05tx7SynPrDacuioirgLeDNwDvAB4DfBk4EvANaWU2yqOpy6JiEed4vAfA98NbCmlfHaTR9ImiYinlFL+tP3xMPAG4DuA24CXllL+qeZ86p6IuBl4ZSllPiK+E/gDYAEYBa4tpfxJ1QElbbiIWAB+B3iz/06XS0R8L/A2YA54PvAe4KvAw4AXlVL+sOJ46pKImABuBv4NsAwcAy4Cfgu4vpRyd8XxJKkneZuR/nULcCetP/ieERF/EBErV9r/i3pjaRO8DvgFWhtafwT8TillCHgprY1tDaZ9wIeAW9f8+hbgw+3jGly/subjXwa+DHwPcIDW/x9ocH17KWW+/fEv0fqB5aOBJwI31BtLNUXE3tozqKu+TOuChT+JiE9HxI9FxKW1h9Km+DngO4EfpvXvd88ppTwKuBr4xZqDqaveCrwTuAz4CVr/jXc5MAz8RrWptKki4rKIeEz712W155F6nbcZ6V+PLKV8P0BEvI/WH3ofiohn1B1Lm+CSUsoHASLil0op7wIopfxhRPgvuoPrF4DHA9eVUg4DRMTnSylX1B1Lm2DLmo+fCHxbKWUJ+H/c1Bp4F635+JJSyt8AlFJuj4gLK82kTXCav42zwv/IHWx3lVL+U0T8NK0fXL4IuDEibgV+e+Vv6mgwlVL2AkTEV0opn2of2x8RdQdTNz105b/ngNdHxN+UUl4VET8MlJqDqfsiYhL478DjaP2tDIDtEfFpWv/d97lqw6mKiNhbSpmqPUevczO7f63+R2wpZRl4WUT8Gq2rNR9UbSpthrUbW/f/K+b+bYsBVUr5hYh4LPC7EfH2UsrNtP4qogbftojYRet/+8vtjewV91SaSZvjoxHx68ArgT+LiB8spfy/EfEUWreW0uDaBxzivn/mrxjZ3FFUQ/v/638f+P2I2A68EHg9sLPmXOqq5faf9w8BLo6IJ5RS/ioirgTOrzybuufrETFZSjkYEQ3gBEAp5WRELJ3huep/bwfeCDyllHISICLOA57bPvftFWdTl3jRwrlzM7t//UNEfFcp5eMrB0op/zkibgB+puJc6r5DEXFJKeXLpZSXrByMiIcDixXnUpeVUv4uIv4V8IsR8VHW/FBLA22I1g8qtwBExLeWUo5ExDcBJ6tOpm77T8CvAUdobV7/VES8Hfgz4MU1B1PXHQKuLqUcuf+JiLhj88fRJvqGH2CUUuZo3VrI2wsNtp8D/oLWD6p/EPiliBgDHg78aM3B1FU/B/xVRHyB1i0EfxAgIr6Z1u8HDbbL1lyZD7R+kAG8MyJ+ttJM6j4vWjhHbmb3r+dxiqsySymviIh3VphHm6SU8r2nOXUXrb+OqgHWfhOY/xIRTwCeVHsedV8p5fLTnPo64Jv9DrBSygng5RFxPTBJ68q8w6UUr8oefO8FJmj9IOP+/mCTZ9Hm8paBSZVSPgQ8dOXziPhz4DHAP/pmz4OrlHJrRDwS2AHcXkr53+3j/wS85AGfrEFwZ0Q8B/jd9t+4JyK20Loy+1jVydRNh/CihXOyZXnZv6UuSZIkSZIkbZb2DzJuBh7LvT/A/lbgNuBHSyneN30AtW8R/L6V90a437nfLKX8xwpj9RU3syVJkiRJkqQKImIUeET70ztKKfM155F6nZvZkiRJkiRJUo+IiL2llKnac2hz2X19vGe2JEmSJEmStIki4lGnObUFuGwzZ9HmeYDuYPd1cTNbkiRJkiRJ2lz7aL0Z4JZTnBvZ3FG0iex+jtzMliRJkiRJkjbXIeDqUsqR+5+IiDs2fxxtkkPY/ZycV3sASZIkSZIkKZn3AhOnOfcHmzmINpXdz5FvAClJkiRJkiRJ6nlemS1JkiRJkiRJ6nluZkuSJEmSJEmSep6b2ZIkSZIkSZKknudmtiRJkiRJkiSp57mZLUmSJEmSJEnqef8/xKIFiKBbPRoAAAAASUVORK5CYII=\n"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"# Target"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_all = df_train.attribute_ids.values.flatten()\ndf_cnt_label_set = pd.Series(labels_all).value_counts()\ndf_cnt_label_set.shape","execution_count":7,"outputs":[{"output_type":"execute_result","execution_count":7,"data":{"text/plain":"(50238,)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# There are frequent combinations.\ndf_cnt_label_set.head(30)","execution_count":8,"outputs":[{"output_type":"execute_result","execution_count":8,"data":{"text/plain":"13 405 896 1092 1158\n813 896 586\n194 1034 489\n13 552 482\n121 1059 465\n121 433 425\n13 626 365\n79 1059 352\n13 813 896 339\n121 1039 332\n189 1034 329\n896 1092 328\n121 962 290\n79 1062 261\n147 671 780 1034 245\n1034 369 241\n1059 234\n188 1034 230\n684 221\n194 1059 221\n121 724 955 219\n79 487 218\n147 813 896 210\n304 487 201\n813 201\n1092 200\n147 813 195\n121 432 724 191\n13 519 188\n615 813 896 187\ndtype: int64"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnt_each_label = np.hstack(df_train.attribute_ids.str.split(\" \").values)","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# top 30 frequent labels\ntarget_each_cnt = pd.Series(cnt_each_label).value_counts()\ntarget_each_cnt.head(30)","execution_count":11,"outputs":[{"output_type":"execute_result","execution_count":11,"data":{"text/plain":"813 19970\n1092 14281\n147 13522\n189 10375\n13 9151\n671 8419\n51 7615\n194 7394\n1059 6564\n121 6542\n896 5955\n1046 5591\n79 5382\n780 5259\n156 5163\n369 4416\n744 3890\n477 3692\n738 3665\n1034 3570\n188 3500\n835 3005\n903 2552\n420 2548\n1099 2327\n552 2180\n485 2097\n776 2075\n161 2050\n489 2045\ndtype: int64"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_each_cnt.tail()","execution_count":14,"outputs":[{"output_type":"execute_result","execution_count":14,"data":{"text/plain":"199 1\n281 1\n396 1\n366 1\n11 1\ndtype: int64"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_each_cnt.shape","execution_count":12,"outputs":[{"output_type":"execute_result","execution_count":12,"data":{"text/plain":"(1103,)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnt_each_label.astype(int).min(), cnt_each_label.astype(int).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Co-occurence of labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"def func(row):\n base = np.zeros(nb_class)\n for r in row:\n base[int(r)] = 1\n return base.astype(int)\nattr_oo = df_train.attribute_ids.str.split(\" \").apply(func)\nattr_oo = np.vstack(np.array(attr_oo))\nattr_oo.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy.sparse as sp\nattr_oo_sp = sp.coo_matrix(attr_oo)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"co_occur = attr_oo_sp.T.dot(attr_oo_sp)\nco_occur = co_occur.toarray()\nco_occur.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"co_occur","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,20))\nsns.heatmap(co_occur, vmin=0, vmax=100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"culture = df_labels.attribute_name[df_labels.attribute_name.str.contains(\"culture::\")]\ntag = df_labels.attribute_name[df_labels.attribute_name.str.contains(\"tag::\")]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* attribute_id 0 - 397 -> culture\n* attribute_id 398 - 1102 -> tag"},{"metadata":{"trusted":true},"cell_type":"code","source":"culture","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels_split = df_labels.attribute_name.str.split(\"::\", expand=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels_split.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# There are 2 types of category\ndf_labels_split[0].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# all names are unique\ndf_labels_split[1].value_counts().sort_values(ascending=False).iloc[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_labels_split[1][:30]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}