{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### 사용한 kaggle notebook\n- 'GLR 2021 🗽🌁⛺🌋 EDA + EfficientNetB0 inference'","metadata":{}},{"cell_type":"code","source":"import os\n\n\nimport random\nimport seaborn as sns\nimport cv2\nimport copy\n\n# General packages\nimport pandas as pd\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport PIL\nimport IPython.display as ipd\nimport glob\nimport h5py\nimport plotly.graph_objs as go\nimport plotly.express as px\nfrom PIL import Image\nfrom tempfile import mktemp\n\n\nfrom bokeh.plotting import figure, output_notebook, show\nfrom math import pi\n\noutput_notebook()\n\nfrom IPython.display import Image, display\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:22.964367Z","iopub.execute_input":"2021-09-29T08:54:22.965265Z","iopub.status.idle":"2021-09-29T08:54:26.158441Z","shell.execute_reply.started":"2021-09-29T08:54:22.965198Z","shell.execute_reply":"2021-09-29T08:54:26.157542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to display two tables side by side\nfrom IPython.display import display_html\ndef display_side_by_side(*args):\n    html_str=''\n    for df in args:\n        html_str+=df.to_html()\n    display_html(html_str.replace('table','table style=\"display:inline\"'),raw=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:26.159841Z","iopub.execute_input":"2021-09-29T08:54:26.16012Z","iopub.status.idle":"2021-09-29T08:54:26.16524Z","shell.execute_reply.started":"2021-09-29T08:54:26.160092Z","shell.execute_reply":"2021-09-29T08:54:26.164042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input/landmark-recognition-2021/')","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:26.166989Z","iopub.execute_input":"2021-09-29T08:54:26.167356Z","iopub.status.idle":"2021-09-29T08:54:26.184053Z","shell.execute_reply.started":"2021-09-29T08:54:26.167326Z","shell.execute_reply":"2021-09-29T08:54:26.182924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATASET_DIR = '../input/landmark-recognition-2021'\n\nTRAIN_IMAGE_DIR = f'{DATASET_DIR}/train'\nTEST_IMAGE_DIR = f'{DATASET_DIR}/test'\ntrain = pd.read_csv(f'{DATASET_DIR}/train.csv')\nSUB = pd.read_csv(f'{DATASET_DIR}/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:26.186429Z","iopub.execute_input":"2021-09-29T08:54:26.186913Z","iopub.status.idle":"2021-09-29T08:54:28.062035Z","shell.execute_reply.started":"2021-09-29T08:54:26.186862Z","shell.execute_reply":"2021-09-29T08:54:28.060968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Outlier detection\n## 1. 이미지 갯수별 상위 1~100위 Landmark_id 확인","metadata":{"execution":{"iopub.status.busy":"2021-09-16T07:19:24.816526Z","iopub.execute_input":"2021-09-16T07:19:24.816938Z","iopub.status.idle":"2021-09-16T07:19:24.822158Z","shell.execute_reply.started":"2021-09-16T07:19:24.816903Z","shell.execute_reply":"2021-09-16T07:19:24.820754Z"}}},{"cell_type":"code","source":"#landmark 전체 이미지 갯수, unique id 갯수 (1,250,470개, 81,313개)\ndisplay(train.head(3))\nnum_train = train.shape[0]\nnum_ids = train.landmark_id.nunique()\nprint(\"Shape of train_data :\", train.shape)\nprint('The number of whole train images :', format(num_train, \",\"))\nprint('The number of unique ids :', format(num_ids, \",\"))","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:28.064457Z","iopub.execute_input":"2021-09-29T08:54:28.064928Z","iopub.status.idle":"2021-09-29T08:54:28.134724Z","shell.execute_reply.started":"2021-09-29T08:54:28.064881Z","shell.execute_reply":"2021-09-29T08:54:28.133743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#unique id가 총 81,313개\nlandmark = train.landmark_id.value_counts()\n\n#unique id마다 할당된 사진들의 갯수를 정리하여 가장 image갯수가 가장 많은 상위 100개만 dataframe으로 정리\nlandmark_df = pd.DataFrame({'landmark_id':landmark.index, 'frequency':landmark.values}).head(100)\n\n#['landmark_id']의 elements를 'landmark_id_***' 모양으로 변경. (생략시 y축의 id명을 카테고리가 아닌 숫자값으로 인식해버림.)\nlandmark_df['landmark_id'] =   landmark_df.landmark_id.apply(lambda x: f'id_{x}')\n\n#각 landmark ID의 image 갯수 비교 bar 그래프 (top1~top100)\nfig = px.bar(landmark_df, x=\"frequency\", y=\"landmark_id\",color='landmark_id', #import plotly.express as px의 px.bar 이용\n             hover_data=[\"landmark_id\", \"frequency\"],\n             height=1000,\n             title='Number of images per landmark_id (Top 100 landmark_ids)',\n             color_discrete_sequence=px.colors.sequential.RdBu)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:28.13635Z","iopub.execute_input":"2021-09-29T08:54:28.136778Z","iopub.status.idle":"2021-09-29T08:54:29.940248Z","shell.execute_reply.started":"2021-09-29T08:54:28.136733Z","shell.execute_reply":"2021-09-29T08:54:29.939203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#각 id별 비중, 1~100위가 전체에서 차지하는 비중 확인\nlandmark_100 = copy.copy(landmark_df)\nlandmark_100['ratio'] = landmark_100['frequency']/num_train #전체 이미지에서 해당 id가 차지하는 비중\nlandmark_100['accu_ratio'] = landmark_100['ratio'].cumsum() #누적 비율(accumulation ratio)\ndisplay_side_by_side(landmark_100.head(), landmark_100.tail())","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:29.941609Z","iopub.execute_input":"2021-09-29T08:54:29.941934Z","iopub.status.idle":"2021-09-29T08:54:29.972546Z","shell.execute_reply.started":"2021-09-29T08:54:29.941902Z","shell.execute_reply":"2021-09-29T08:54:29.971586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train set 특징\n\n|설명|값|비중|\n|------|---:|:---:|\n|전체 이미지|1,580,470개|-|\n|landmark_id 고유값|81,313개|-|\n|1위(138982)|6,272개|0.39%|\n|100위(144036)|364개|0.02%|\n|이미지 1,000개 이하로 떨어지는 지점|8위(971개)||\n|1-5위가 전체에서 차지하는 비중||0.83%|\n|<b>1-100위가 전체에서 차지하는 비중</b>||<b>4.18%</b>|","metadata":{}},{"cell_type":"markdown","source":"## 2. 전체 데이터 범위에서 통계적 이상치 확인\n### 2.1. Landmark_ids 순위$^*$에 따른 이미지량의 변화 ($^*$images를 많이 보유한 순)","metadata":{"execution":{"iopub.status.busy":"2021-09-21T12:40:33.176211Z","iopub.execute_input":"2021-09-21T12:40:33.176618Z","iopub.status.idle":"2021-09-21T12:40:33.179964Z","shell.execute_reply.started":"2021-09-21T12:40:33.176585Z","shell.execute_reply":"2021-09-21T12:40:33.179256Z"}}},{"cell_type":"code","source":"landmarks_fold = pd.DataFrame(train['landmark_id'].value_counts()) #image많은 순으로 내림차순 정렬\nlandmarks_fold.reset_index(inplace=True)\nlandmarks_fold.columns = ['landmark_id','count']\n\nlandmarks_fold['ratio'] = landmarks_fold['count']/num_train #전체 이미지에서 해당 id가 차지하는 비중\nlandmarks_fold['accu_ratio'] = landmarks_fold['ratio'].cumsum()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:29.974992Z","iopub.execute_input":"2021-09-29T08:54:29.975846Z","iopub.status.idle":"2021-09-29T08:54:30.031885Z","shell.execute_reply.started":"2021-09-29T08:54:29.975795Z","shell.execute_reply":"2021-09-29T08:54:30.030922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_side_by_side(landmarks_fold.head(10),landmarks_fold.tail(10))\nlandmarks_fold['count'].describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:30.033224Z","iopub.execute_input":"2021-09-29T08:54:30.033554Z","iopub.status.idle":"2021-09-29T08:54:30.058893Z","shell.execute_reply.started":"2021-09-29T08:54:30.033521Z","shell.execute_reply":"2021-09-29T08:54:30.057705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#line plot\nsns.set()\nplt.title('Training set: number of images per class(line plot)') \nsns.set_color_codes(\"pastel\")\n# 참고: counts 값이 큰 것 부터 내림차순으로 정리되어있음. ex index:0,id:138982,count:6272\nlandmarks_fold = pd.DataFrame(train['landmark_id'].value_counts()) \nlandmarks_fold.reset_index(inplace=True)\nlandmarks_fold.columns = ['landmark_id','count']\nax = landmarks_fold['count'].plot(logy=True, grid=True) #logy = T: y축(counts 값)은 log 스케일로 조정 \nlocs, labels = plt.xticks()\nplt.setp(labels, rotation=30)\nax.set(xlabel=\"Landmark rankings (the 1st to the last)\", ylabel=\"Number of images\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:30.060188Z","iopub.execute_input":"2021-09-29T08:54:30.06053Z","iopub.status.idle":"2021-09-29T08:54:30.815938Z","shell.execute_reply.started":"2021-09-29T08:54:30.060494Z","shell.execute_reply":"2021-09-29T08:54:30.814613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Boxplot을 통한 통계적 outlier 탐색\n- 통계적 판단의 이상치: 상하위 0.3%이거나 <u>상자그림의 ``1.5*IQR``을 벗어난 경우</u> <br>\n- 박스플롯 해석을 위해 도움되는 사이트\n    * [boxplot 해설](https://boxnwhis.kr/2019/02/19/boxplot.html)\n    * [outlier detector 코드](https://hong-yp-ml-records.tistory.com/15)\n    * [사분위 범위 설명 및 수식](https://truman.tistory.com/105)","metadata":{}},{"cell_type":"code","source":"Image('../input/picture1/boxplot1.JPG')","metadata":{"execution":{"iopub.status.busy":"2021-09-23T09:34:46.952472Z","iopub.execute_input":"2021-09-23T09:34:46.952759Z","iopub.status.idle":"2021-09-23T09:34:46.966828Z","shell.execute_reply.started":"2021-09-23T09:34:46.95273Z","shell.execute_reply":"2021-09-23T09:34:46.965811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 박스플롯을 통한 outlier 확인\n# landmark images들의 최소한계치, 최대한계치, 25%(1사분위), 75%(3사분위)\nsns.set()\nax = landmarks_fold.boxplot(column='count')\nax.set_yscale('log')","metadata":{"execution":{"iopub.status.busy":"2021-09-23T09:34:46.967956Z","iopub.execute_input":"2021-09-23T09:34:46.968376Z","iopub.status.idle":"2021-09-23T09:34:47.363812Z","shell.execute_reply.started":"2021-09-23T09:34:46.968341Z","shell.execute_reply":"2021-09-23T09:34:47.362798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# box플롯의 최대한계치&최소한계치 벗어난 outlier 탐색\n\nfrom collections import Counter \n\ndef detect_outliers(df, features):\n    outlier_indices = []\n    for col in features:\n        Q1 = np.percentile(df[col], 25) # 1사분위수\n        Q3 = np.percentile(df[col], 75) # 3사분위수\n        IQR = Q3 - Q1 # inter-quatile range = 3사분위수 - 1사분위수 \n        \n        outlier_step = 1.5 * IQR # 최저한계치와 최고한계치 구하기 위한 step\n        \n        outlier_list_col = df[(df[col] < Q1 - outlier_step) | (df[col] > Q3 + outlier_step)].index\n        outlier_indices.extend(outlier_list_col)\n#    outlier_indices = Counter(outlier_indices)\n#    multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)\n    \n    return outlier_indices","metadata":{"execution":{"iopub.status.busy":"2021-09-23T09:34:47.364792Z","iopub.execute_input":"2021-09-23T09:34:47.365067Z","iopub.status.idle":"2021-09-23T09:34:47.371632Z","shell.execute_reply.started":"2021-09-23T09:34:47.365039Z","shell.execute_reply":"2021-09-23T09:34:47.370466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#최고한계치(upper fence) 밖에 있는 outlier(이상치)들\nOutliers_to_drop = detect_outliers(landmarks_fold, [\"count\"])\nlandmarks_fold.loc[Outliers_to_drop]\n\n#'boxplot outlier'를 제거한 df\n#landmarks_fold_bxout = landmarks_fold.drop(Outliers_to_drop, axis = 0).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T09:34:47.372661Z","iopub.execute_input":"2021-09-23T09:34:47.372908Z","iopub.status.idle":"2021-09-23T09:34:47.402708Z","shell.execute_reply.started":"2021-09-23T09:34:47.372885Z","shell.execute_reply":"2021-09-23T09:34:47.401715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- id마다 이미지 갯수가 2 ~ 6272로 넓게 분포하고, 중앙값 9, 평균 19로 landmark_id 간 데이터양이 극심하게 차이가 남.\n- 특히 데이터가 50장 이하인 id가 전체 id중 절반을 차지함.\n- 사진 갯수로만 outlier를 제거하면 8053개의 id(사진 양으로는 train set의 50%)를 지우게 됨.\n- 기존에 존재하는 랜드마크들(81,313가지) 중에서 약 10%에 해당하는 카테고리가 날아감 --> 손실이 커 보임 \n- 단순 통계적 극단치를 버리는 것은 옳은 방법이 아니어보임.\n<br>\n<br>\n- **최종 대안: 기존에 고려한 top100중에서 잘못된 사진이 많이 섞인 몇 가지 카테고리를 버리거나, 많은 사진을 가진 카테고리(랜드마크)마다 각각 outlier(잘못된 사진)를 줄이는 게 적합한 방향일 것으로 생각됨.**\n- **정리: 카테고리의 크기는 최대한 유지/hold 하고 학습할 데이터를 줄이자??**","metadata":{}},{"cell_type":"markdown","source":"## 3. Top1~top100 이미지 Visualizing (랜덤하게 100개씩)","metadata":{}},{"cell_type":"code","source":"import PIL\nfrom PIL import Image, ImageDraw\n\n\ndef display_images(images, title=None): \n    \"\"\"\n    func for display images \n    Thank you @rohitsingh9990 for this fucntion\n    \"\"\"\n    f, ax = plt.subplots(10,10, figsize=(18,22))\n    if title:\n        f.suptitle(title, fontsize = 30)\n\n    for i, image_id in enumerate(images):\n        image_path = os.path.join(TRAIN_IMAGE_DIR, f'{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg')\n        image = Image.open(image_path)\n        \n        ax[i//10, i%10].imshow(image) \n        image.close()       \n        ax[i//10, i%10].axis('off')\n\n        landmark_id = train[train.id==image_id.split('.')[0]].landmark_id.values[0]\n        ax[i//10, i%10].set_title(f\"ID: {image_id.split('.')[0]}\\nLandmark_id: {landmark_id}\", fontsize=\"12\")\n\n    plt.show() ","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-09-29T08:54:32.624208Z","iopub.execute_input":"2021-09-29T08:54:32.624596Z","iopub.status.idle":"2021-09-29T08:54:32.636247Z","shell.execute_reply.started":"2021-09-29T08:54:32.624561Z","shell.execute_reply":"2021-09-29T08:54:32.635397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#samples = train.sample(100).id.values #train set에서 100개의 id를 랜덤추출하여 id값을 저장\n#display_images(samples, 'Random') #samples에 들어간 id값에 해당하는 이미지를 TRAIN_IMAGE_DIR에서 가져와서 보여줌","metadata":{"execution":{"iopub.status.busy":"2021-09-23T09:34:47.414911Z","iopub.execute_input":"2021-09-23T09:34:47.415326Z","iopub.status.idle":"2021-09-23T09:34:47.426632Z","shell.execute_reply.started":"2021-09-23T09:34:47.415294Z","shell.execute_reply":"2021-09-23T09:34:47.425781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# top m에서 top n까지 각각 랜덤으로 100개 이미지 보기\nfor i in range(6):\n    samples = train[train.landmark_id == landmarks_fold.iloc[i].landmark_id].sample(100).id.values\n    display_images(samples, f'Top {i+1}')","metadata":{"execution":{"iopub.status.busy":"2021-09-29T08:54:49.38893Z","iopub.execute_input":"2021-09-29T08:54:49.389804Z","iopub.status.idle":"2021-09-29T08:58:45.370753Z","shell.execute_reply.started":"2021-09-29T08:54:49.389741Z","shell.execute_reply":"2021-09-29T08:58:45.369647Z"},"trusted":true},"execution_count":null,"outputs":[]}]}