{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 9.4 향후 판매량 예측 경진대회 모델 성능 개선","metadata":{"papermill":{"duration":0.057648,"end_time":"2021-09-11T07:56:43.901354","exception":false,"start_time":"2021-09-11T07:56:43.843706","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\n\nwarnings.filterwarnings(action='ignore') # 경고 메시지 생략\n\n# 데이터 경로\ndata_path = '/kaggle/input/competitive-data-science-predict-future-sales/'\n\nsales_train = pd.read_csv(data_path + 'sales_train.csv')\nshops = pd.read_csv(data_path + 'shops.csv')\nitems = pd.read_csv(data_path + 'items.csv')\nitem_categories = pd.read_csv(data_path + 'item_categories.csv')\ntest = pd.read_csv(data_path + 'test.csv')\nsubmission = pd.read_csv(data_path + 'sample_submission.csv')","metadata":{"papermill":{"duration":2.978519,"end_time":"2021-09-11T07:56:47.058138","exception":false,"start_time":"2021-09-11T07:56:44.079619","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.1 피처 엔지니어링 I : 피처명 한글화와 데이터 다운캐스팅","metadata":{"papermill":{"duration":0.055122,"end_time":"2021-09-11T07:56:47.169837","exception":false,"start_time":"2021-09-11T07:56:47.114715","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sales_train = sales_train.rename(columns={'date': '날짜', \n                                          'date_block_num': '월ID',\n                                          'shop_id': '상점ID',\n                                          'item_id': '상품ID',\n                                          'item_price': '판매가',\n                                          'item_cnt_day': '판매량'})\n\nshops = shops.rename(columns={'shop_name': '상점명',\n                              'shop_id': '상점ID'})\n\nitems = items.rename(columns={'item_name': '상품명',\n                              'item_id': '상품ID',\n                              'item_category_id': '상품분류ID'})\n\nitem_categories = item_categories.rename(columns=\n                                         {'item_category_name': '상품분류명',\n                                          'item_category_id': '상품분류ID'})\n\ntest = test.rename(columns={'shop_id': '상점ID',\n                            'item_id': '상품ID'})","metadata":{"papermill":{"duration":0.146746,"end_time":"2021-09-11T07:56:47.372356","exception":false,"start_time":"2021-09-11T07:56:47.22561","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def downcast(df, verbose=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        dtype_name = df[col].dtype.name\n        if dtype_name == 'object':\n            pass\n        elif dtype_name == 'bool':\n            df[col] = df[col].astype('int8')\n        elif dtype_name.startswith('int') or (df[col].round() == df[col]).all():\n            df[col] = pd.to_numeric(df[col], downcast='integer')\n        else:\n            df[col] = pd.to_numeric(df[col], downcast='float')\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        print('{:.1f}% 압축됨'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\nall_df = [sales_train, shops, items, item_categories, test]\nfor df in all_df:\n    df = downcast(df)","metadata":{"papermill":{"duration":0.441517,"end_time":"2021-09-11T07:56:47.869725","exception":false,"start_time":"2021-09-11T07:56:47.428208","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.2 피처 엔지니어링 II : 개별 데이터 피처 엔지니어링","metadata":{"papermill":{"duration":0.055465,"end_time":"2021-09-11T07:56:47.981771","exception":false,"start_time":"2021-09-11T07:56:47.926306","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### sales_train 이상치 제거 및 전처리","metadata":{"papermill":{"duration":0.056092,"end_time":"2021-09-11T07:56:48.09438","exception":false,"start_time":"2021-09-11T07:56:48.038288","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 판매가가 0보다 큰 데이터 추출\nsales_train = sales_train[sales_train['판매가'] > 0]\n# 판매가가 50,000보다 작은 데이터 추출\nsales_train = sales_train[sales_train['판매가'] < 50000]\n\n# 판매량이 0보다 큰 데이터 추출\nsales_train = sales_train[sales_train['판매량'] > 0]\n# 판매량이 1,000보다 작은 데이터 추출\nsales_train = sales_train[sales_train['판매량'] < 1000]","metadata":{"papermill":{"duration":0.450085,"end_time":"2021-09-11T07:56:48.601325","exception":false,"start_time":"2021-09-11T07:56:48.15124","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(shops['상점명'][0], '||', shops['상점명'][57])\nprint(shops['상점명'][1], '||', shops['상점명'][58])\nprint(shops['상점명'][10], '||', shops['상점명'][11])\nprint(shops['상점명'][39], '||', shops['상점명'][40])","metadata":{"papermill":{"duration":0.06892,"end_time":"2021-09-11T07:56:48.726727","exception":false,"start_time":"2021-09-11T07:56:48.657807","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sales_train 데이터에서 상점ID 수정\nsales_train.loc[sales_train['상점ID'] == 0, '상점ID'] = 57\nsales_train.loc[sales_train['상점ID'] == 1, '상점ID'] = 58\nsales_train.loc[sales_train['상점ID'] == 10, '상점ID'] = 11\nsales_train.loc[sales_train['상점ID'] == 39, '상점ID'] = 40\n\n# test 데이터에서 상점ID 수정\ntest.loc[test['상점ID'] == 0, '상점ID'] = 57\ntest.loc[test['상점ID'] == 1, '상점ID'] = 58\ntest.loc[test['상점ID'] == 10, '상점ID'] = 11\ntest.loc[test['상점ID'] == 39, '상점ID'] = 40","metadata":{"papermill":{"duration":0.085782,"end_time":"2021-09-11T07:56:48.869244","exception":false,"start_time":"2021-09-11T07:56:48.783462","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### shops 파생 피처 생성 및 인코딩","metadata":{"papermill":{"duration":0.058082,"end_time":"2021-09-11T07:56:48.985548","exception":false,"start_time":"2021-09-11T07:56:48.927466","status":"completed"},"tags":[]}},{"cell_type":"code","source":"shops['도시'] = shops['상점명'].apply(lambda x: x.split()[0])","metadata":{"papermill":{"duration":0.066925,"end_time":"2021-09-11T07:56:49.109263","exception":false,"start_time":"2021-09-11T07:56:49.042338","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops['도시'].unique()","metadata":{"papermill":{"duration":0.068266,"end_time":"2021-09-11T07:56:49.234286","exception":false,"start_time":"2021-09-11T07:56:49.16602","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops.loc[shops['도시'] =='!Якутск', '도시'] = 'Якутск'","metadata":{"papermill":{"duration":0.065275,"end_time":"2021-09-11T07:56:49.357208","exception":false,"start_time":"2021-09-11T07:56:49.291933","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# 레이블 인코더 생성\nlabel_encoder = LabelEncoder()\n# 도시 피처 레이블 인코딩\nshops['도시'] = label_encoder.fit_transform(shops['도시'])","metadata":{"papermill":{"duration":0.957376,"end_time":"2021-09-11T07:56:50.371508","exception":false,"start_time":"2021-09-11T07:56:49.414132","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상점명 피처 제거\nshops = shops.drop('상점명', axis=1)\n\nshops.head()","metadata":{"papermill":{"duration":0.074567,"end_time":"2021-09-11T07:56:50.502728","exception":false,"start_time":"2021-09-11T07:56:50.428161","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### itmes 파생 피처 생성","metadata":{"papermill":{"duration":0.056839,"end_time":"2021-09-11T07:56:50.618204","exception":false,"start_time":"2021-09-11T07:56:50.561365","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 상품명 피처 제거\nitems = items.drop(['상품명'], axis=1)","metadata":{"papermill":{"duration":0.065571,"end_time":"2021-09-11T07:56:50.74091","exception":false,"start_time":"2021-09-11T07:56:50.675339","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품이 맨 처음 팔린 날을 피처로 추가\nitems['첫 판매월'] = sales_train.groupby('상품ID').agg({'월ID': 'min'})['월ID']\n\nitems.head()","metadata":{"papermill":{"duration":0.143514,"end_time":"2021-09-11T07:56:50.941579","exception":false,"start_time":"2021-09-11T07:56:50.798065","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items[items['첫 판매월'].isna()]","metadata":{"papermill":{"duration":0.074982,"end_time":"2021-09-11T07:56:51.075149","exception":false,"start_time":"2021-09-11T07:56:51.000167","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 첫 판매월 피처의 결측값을 34로 대체\nitems['첫 판매월'] = items['첫 판매월'].fillna(34)","metadata":{"papermill":{"duration":0.066123,"end_time":"2021-09-11T07:56:51.200603","exception":false,"start_time":"2021-09-11T07:56:51.13448","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### item_categories 파생 피처 생성 및 인코딩","metadata":{"papermill":{"duration":0.057838,"end_time":"2021-09-11T07:56:51.31738","exception":false,"start_time":"2021-09-11T07:56:51.259542","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 상품분류명의 첫 단어를 대분류로 추출\nitem_categories['대분류'] = item_categories['상품분류명'].apply(lambda x: x.split()[0])  ","metadata":{"papermill":{"duration":0.067002,"end_time":"2021-09-11T07:56:51.443562","exception":false,"start_time":"2021-09-11T07:56:51.37656","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_categories['대분류'].value_counts()","metadata":{"papermill":{"duration":0.069546,"end_time":"2021-09-11T07:56:51.570898","exception":false,"start_time":"2021-09-11T07:56:51.501352","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_etc(x):\n    if len(item_categories[item_categories['대분류']==x]) >= 5:\n        return x\n    else:\n        return 'etc'\n\n# 대분류의 고윳값 개수가 5개 미만이면 'etc'로 바꾸기\nitem_categories['대분류'] = item_categories['대분류'].apply(make_etc)","metadata":{"papermill":{"duration":0.098477,"end_time":"2021-09-11T07:56:51.728407","exception":false,"start_time":"2021-09-11T07:56:51.62993","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_categories.head()","metadata":{"papermill":{"duration":0.07161,"end_time":"2021-09-11T07:56:51.858657","exception":false,"start_time":"2021-09-11T07:56:51.787047","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 레이블 인코더 생성\nlabel_encoder = LabelEncoder()\n\n# 대분류 피처 레이블 인코딩\nitem_categories['대분류'] = label_encoder.fit_transform(item_categories['대분류'])\n\n# 상품분류명 피처 제거\nitem_categories = item_categories.drop('상품분류명', axis=1)","metadata":{"papermill":{"duration":0.070027,"end_time":"2021-09-11T07:56:51.987464","exception":false,"start_time":"2021-09-11T07:56:51.917437","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.3 피처 엔지니어링 III : 데이터 조합 및 파생 피처 생성","metadata":{"papermill":{"duration":0.060883,"end_time":"2021-09-11T07:56:52.107747","exception":false,"start_time":"2021-09-11T07:56:52.046864","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 데이터 조합","metadata":{"papermill":{"duration":0.059127,"end_time":"2021-09-11T07:56:52.226748","exception":false,"start_time":"2021-09-11T07:56:52.167621","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from itertools import product\n\ntrain = []\n# 월ID, 상점ID, 상품ID 조합 생성\nfor i in sales_train['월ID'].unique():\n    all_shop = sales_train.loc[sales_train['월ID']==i, '상점ID'].unique()\n    all_item = sales_train.loc[sales_train['월ID']==i, '상품ID'].unique()\n    train.append(np.array(list(product([i], all_shop, all_item))))\n\nidx_features = ['월ID', '상점ID', '상품ID'] # 기준 피처\ntrain = pd.DataFrame(np.vstack(train), columns=idx_features)","metadata":{"papermill":{"duration":21.945919,"end_time":"2021-09-11T07:57:14.233056","exception":false,"start_time":"2021-09-11T07:56:52.287137","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 파생 피처 생성","metadata":{"papermill":{"duration":0.060105,"end_time":"2021-09-11T07:57:14.354016","exception":false,"start_time":"2021-09-11T07:57:14.293911","status":"completed"},"tags":[]}},{"cell_type":"code","source":"group = sales_train.groupby(idx_features).agg({'판매량': 'sum',\n                                               '판매가': 'mean'})\ngroup = group.reset_index()\ngroup = group.rename(columns={'판매량': '월간 판매량', '판매가': '평균 판매가'})\n\ntrain = train.merge(group, on=idx_features, how='left')\n\ntrain.head()","metadata":{"papermill":{"duration":5.181073,"end_time":"2021-09-11T07:57:19.594651","exception":false,"start_time":"2021-09-11T07:57:14.413578","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\n# group 변수 가비지 컬렉션\ndel group\ngc.collect();","metadata":{"papermill":{"duration":0.168339,"end_time":"2021-09-11T07:57:19.822312","exception":false,"start_time":"2021-09-11T07:57:19.653973","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품 판매건수 피처 추가\ngroup = sales_train.groupby(idx_features).agg({'판매량': 'count'})\ngroup = group.reset_index()\ngroup = group.rename(columns={'판매량': '판매건수'})\n\ntrain = train.merge(group, on=idx_features, how='left')\n\n# 가비지 컬렉션\ndel group, sales_train\ngc.collect()\n\ntrain.head()","metadata":{"papermill":{"duration":5.102554,"end_time":"2021-09-11T07:57:24.9844","exception":false,"start_time":"2021-09-11T07:57:19.881846","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.4 피처 엔지니어링 IV : 데이터 합치기","metadata":{}},{"cell_type":"markdown","source":"### 테스트 데이터 이어붙이기","metadata":{"papermill":{"duration":0.060197,"end_time":"2021-09-11T07:57:25.104765","exception":false,"start_time":"2021-09-11T07:57:25.044568","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 테스트 데이터 월ID를 34로 설정\ntest['월ID'] = 34\n\n# train과 test 이어붙이기\nall_data = pd.concat([train, test.drop('ID', axis=1)],\n                     ignore_index=True,\n                     keys=idx_features)\n# 결측값을 0으로 대체\nall_data = all_data.fillna(0)\n\nall_data.head()","metadata":{"papermill":{"duration":0.606687,"end_time":"2021-09-11T07:57:25.771611","exception":false,"start_time":"2021-09-11T07:57:25.164924","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 모든 데이터 병합","metadata":{"papermill":{"duration":0.060196,"end_time":"2021-09-11T07:57:25.892499","exception":false,"start_time":"2021-09-11T07:57:25.832303","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 나머지 데이터 병합\nall_data = all_data.merge(shops, on='상점ID', how='left')\nall_data = all_data.merge(items, on='상품ID', how='left')\nall_data = all_data.merge(item_categories, on='상품분류ID', how='left')\n\n# 데이터 다운캐스팅\nall_data = downcast(all_data)","metadata":{"papermill":{"duration":6.676043,"end_time":"2021-09-11T07:57:32.629511","exception":false,"start_time":"2021-09-11T07:57:25.953468","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 가비지 컬렉션\ndel shops, items, item_categories\ngc.collect();","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.5 피처 엔지니어링 V : 시차 피처 생성","metadata":{"papermill":{"duration":0.060531,"end_time":"2021-09-11T07:57:32.758651","exception":false,"start_time":"2021-09-11T07:57:32.69812","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 기준 피처별 월간 평균 판매량 파생 피처 생성","metadata":{"papermill":{"duration":0.060497,"end_time":"2021-09-11T07:57:32.879801","exception":false,"start_time":"2021-09-11T07:57:32.819304","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def add_mean_features(df, mean_features, idx_features):\n    # 기준 피처 확인 \n    assert (idx_features[0] == '월ID') and \\\n           len(idx_features) in [2, 3]\n    \n    # 파생 피처명 설정 \n    if len(idx_features) == 2:\n        feature_name = idx_features[1] + '별 평균 판매량'\n    else:\n        feature_name = idx_features[1] + ' ' + idx_features[2] + '별 평균 판매량'\n    \n    # 기준 피처를 토대로 그룹화해 월간 평균 판매량 구하기 \n    group = df.groupby(idx_features).agg({'월간 판매량': 'mean'})\n    group = group.reset_index()\n    group = group.rename(columns={'월간 판매량': feature_name})\n    \n    # df와 group 병합 \n    df = df.merge(group, on=idx_features, how='left')\n    # 데이터 다운캐스팅 \n    df = downcast(df, verbose=False)\n    # 새로 만든 feature_name 피처명을 mean_features 리스트에 추가 \n    mean_features.append(feature_name)\n    \n    # 가비지 컬렉션\n    del group\n    gc.collect()\n    \n    return df, mean_features","metadata":{"papermill":{"duration":0.07212,"end_time":"2021-09-11T07:57:33.012509","exception":false,"start_time":"2021-09-11T07:57:32.940389","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 그룹화 기준 피처 중 '상품ID'가 포함된 파생 피처명을 담을 리스트\nitem_mean_features = []\n\n# ['월ID', '상품ID']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, item_mean_features = add_mean_features(df=all_data,\n                                                 mean_features=item_mean_features,\n                                                 idx_features=['월ID', '상품ID'])\n\n# ['월ID', '상품ID', '도시']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, item_mean_features = add_mean_features(df=all_data,\n                                                 mean_features=item_mean_features,\n                                                 idx_features=['월ID', '상품ID', '도시'])","metadata":{"papermill":{"duration":20.809118,"end_time":"2021-09-11T07:57:53.883065","exception":false,"start_time":"2021-09-11T07:57:33.073947","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_mean_features","metadata":{"papermill":{"duration":0.070964,"end_time":"2021-09-11T07:57:54.018665","exception":false,"start_time":"2021-09-11T07:57:53.947701","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 그룹화 기준 피처 중 '상점ID'가 포함된 파생 피처명을 담을 리스트\nshop_mean_features = []\n\n# ['월ID', '상점ID', '상품분류ID']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, shop_mean_features = add_mean_features(df=all_data, \n                                                 mean_features=shop_mean_features,\n                                                 idx_features=['월ID', '상점ID', '상품분류ID'])","metadata":{"papermill":{"duration":3.67184,"end_time":"2021-09-11T07:57:57.752094","exception":false,"start_time":"2021-09-11T07:57:54.080254","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shop_mean_features","metadata":{"papermill":{"duration":0.071104,"end_time":"2021-09-11T07:57:57.884062","exception":false,"start_time":"2021-09-11T07:57:57.812958","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 시차 피처 생성 원리와 구현","metadata":{"papermill":{"duration":0.060624,"end_time":"2021-09-11T07:57:58.006218","exception":false,"start_time":"2021-09-11T07:57:57.945594","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def add_lag_features(df, lag_features_to_clip, idx_features, \n                     lag_feature, nlags=3, clip=False):\n    # 시차 피처 생성에 필요한 DataFrame 부분만 복사 \n    df_temp = df[idx_features + [lag_feature]].copy() \n\n    # 시차 피처 생성 \n    for i in range(1, nlags+1):\n        # 시차 피처명 \n        lag_feature_name = lag_feature +'_시차' + str(i)\n        # df_temp 열 이름 설정 \n        df_temp.columns = idx_features + [lag_feature_name]\n        # df_temp의 date_block_num 피처에 i 더하기 \n        df_temp['월ID'] += i\n        # idx_feature를 기준으로 df와 df_temp 병합하기 \n        df = df.merge(df_temp.drop_duplicates(), \n                      on=idx_features, \n                      how='left')\n        # 결측값 0으로 대체 \n        df[lag_feature_name] = df[lag_feature_name].fillna(0)\n        # 0 ~ 20 사이로 제한할 시차 피처명을 lag_features_to_clip에 추가 \n        if clip: \n            lag_features_to_clip.append(lag_feature_name)\n    \n    # 데이터 다운캐스팅\n    df = downcast(df, False)\n    # 가비지 컬렉션\n    del df_temp\n    gc.collect()\n    \n    return df, lag_features_to_clip","metadata":{"papermill":{"duration":0.073715,"end_time":"2021-09-11T07:57:58.140953","exception":false,"start_time":"2021-09-11T07:57:58.067238","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 시차 피처 생성 I : 월간 판매량","metadata":{"papermill":{"duration":0.060783,"end_time":"2021-09-11T07:57:58.263326","exception":false,"start_time":"2021-09-11T07:57:58.202543","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lag_features_to_clip = [] # 0 ~ 20 사이로 제한할 시차 피처명을 담을 리스트\nidx_features = ['월ID', '상점ID', '상품ID'] # 기준 피처\n\n# idx_features를 기준으로 월간 판매량의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='월간 판매량', \n                                                  nlags=3,\n                                                  clip=True) # 값을 0 ~ 20 사이로 제한","metadata":{"papermill":{"duration":37.909878,"end_time":"2021-09-11T07:58:36.235916","exception":false,"start_time":"2021-09-11T07:57:58.326038","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head().T","metadata":{"papermill":{"duration":0.082799,"end_time":"2021-09-11T07:58:36.380301","exception":false,"start_time":"2021-09-11T07:58:36.297502","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lag_features_to_clip","metadata":{"papermill":{"duration":0.072365,"end_time":"2021-09-11T07:58:36.515673","exception":false,"start_time":"2021-09-11T07:58:36.443308","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 시차 피처 생성 II : 판매건수, 평균 판매가","metadata":{"papermill":{"duration":0.062285,"end_time":"2021-09-11T07:58:36.640942","exception":false,"start_time":"2021-09-11T07:58:36.578657","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# idx_features를 기준으로 판매건수 피처의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='판매건수', \n                                                  nlags=3)\n\n# idx_features를 기준으로 평균 판매가 피처의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='평균 판매가', \n                                                  nlags=3)","metadata":{"papermill":{"duration":73.828586,"end_time":"2021-09-11T07:59:50.532069","exception":false,"start_time":"2021-09-11T07:58:36.703483","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 시차 피처 생성 III : 평균 판매량","metadata":{"papermill":{"duration":0.063176,"end_time":"2021-09-11T07:59:50.657913","exception":false,"start_time":"2021-09-11T07:59:50.594737","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# idx_features를 기준으로 item_mean_features 요소별 시차 피처 생성\nfor item_mean_feature in item_mean_features:\n    all_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                      lag_features_to_clip=lag_features_to_clip, \n                                                      idx_features=idx_features, \n                                                      lag_feature=item_mean_feature, \n                                                      nlags=3,\n                                                      clip=True)\n# item_mean_features 피처 제거\nall_data = all_data.drop(item_mean_features, axis=1)","metadata":{"papermill":{"duration":76.147571,"end_time":"2021-09-11T08:01:06.867256","exception":false,"start_time":"2021-09-11T07:59:50.719685","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ['월ID', '상점ID', '상품분류ID']를 기준으로 shop_mean_features 요소별 시차 피처 생성\nfor shop_mean_feature in shop_mean_features:\n    all_data, lag_features_to_clip = add_lag_features(df=all_data,\n                                                      lag_features_to_clip=lag_features_to_clip, \n                                                      idx_features=['월ID', '상점ID', '상품분류ID'], \n                                                      lag_feature=shop_mean_feature, \n                                                      nlags=3,\n                                                      clip=True)\n# shop_mean_features 피처 제거\nall_data = all_data.drop(shop_mean_features, axis=1)","metadata":{"papermill":{"duration":13.966763,"end_time":"2021-09-11T08:01:20.897471","exception":false,"start_time":"2021-09-11T08:01:06.930708","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 시차 피처 생성 마무리 : 결측값 처리","metadata":{"papermill":{"duration":0.063434,"end_time":"2021-09-11T08:01:21.025137","exception":false,"start_time":"2021-09-11T08:01:20.961703","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 월ID 3미만인 데이터 제거\nall_data = all_data.drop(all_data[all_data['월ID'] < 3].index)","metadata":{"papermill":{"duration":2.75113,"end_time":"2021-09-11T08:01:23.838924","exception":false,"start_time":"2021-09-11T08:01:21.087794","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.6 피처 엔지니어링 VI : 기타 피처 엔지니어링","metadata":{"papermill":{"duration":0.062687,"end_time":"2021-09-11T08:01:24.051697","exception":false,"start_time":"2021-09-11T08:01:23.98901","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 기타 피처 추가","metadata":{"papermill":{"duration":0.062908,"end_time":"2021-09-11T08:01:24.178846","exception":false,"start_time":"2021-09-11T08:01:24.115938","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"#### 월간 판매량 시차 피처들의 평균","metadata":{}},{"cell_type":"code","source":"all_data['월간 판매량 시차평균'] = all_data[['월간 판매량_시차1',\n                                          '월간 판매량_시차2', \n                                          '월간 판매량_시차3']].mean(axis=1)","metadata":{"papermill":{"duration":0.22849,"end_time":"2021-09-11T08:01:24.47048","exception":false,"start_time":"2021-09-11T08:01:24.24199","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0 ~ 20 사이로 값 제한\nall_data[lag_features_to_clip + ['월간 판매량', '월간 판매량 시차평균']] = all_data[lag_features_to_clip + ['월간 판매량', '월간 판매량 시차평균']].clip(0, 20)","metadata":{"papermill":{"duration":3.113688,"end_time":"2021-09-11T08:01:27.646811","exception":false,"start_time":"2021-09-11T08:01:24.533123","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 시차 변화량","metadata":{"papermill":{"duration":0.063065,"end_time":"2021-09-11T08:01:27.773421","exception":false,"start_time":"2021-09-11T08:01:27.710356","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_data['시차변화량1'] = all_data['월간 판매량_시차1']/all_data['월간 판매량_시차2']\nall_data['시차변화량1'] = all_data['시차변화량1'].replace([np.inf, -np.inf], \n                                                        np.nan).fillna(0)\n\nall_data['시차변화량2'] = all_data['월간 판매량_시차2']/all_data['월간 판매량_시차3']\nall_data['시차변화량2'] = all_data['시차변화량2'].replace([np.inf, -np.inf], \n                                                        np.nan).fillna(0)","metadata":{"papermill":{"duration":0.5739,"end_time":"2021-09-11T08:01:28.410926","exception":false,"start_time":"2021-09-11T08:01:27.837026","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 신상 여부","metadata":{"papermill":{"duration":0.06189,"end_time":"2021-09-11T08:01:28.535914","exception":false,"start_time":"2021-09-11T08:01:28.474024","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_data['신상여부'] = all_data['첫 판매월'] == all_data['월ID']","metadata":{"papermill":{"duration":0.076882,"end_time":"2021-09-11T08:01:28.675245","exception":false,"start_time":"2021-09-11T08:01:28.598363","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 첫 판매 후 경과 기간","metadata":{"papermill":{"duration":0.062165,"end_time":"2021-09-11T08:01:28.799679","exception":false,"start_time":"2021-09-11T08:01:28.737514","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_data['첫 판매 후 기간'] = all_data['월ID'] - all_data['첫 판매월']","metadata":{"papermill":{"duration":0.077744,"end_time":"2021-09-11T08:01:28.940538","exception":false,"start_time":"2021-09-11T08:01:28.862794","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 월(month)","metadata":{"papermill":{"duration":0.062473,"end_time":"2021-09-11T08:01:29.066322","exception":false,"start_time":"2021-09-11T08:01:29.003849","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_data['월'] = all_data['월ID'] % 12","metadata":{"papermill":{"duration":0.112824,"end_time":"2021-09-11T08:01:29.242176","exception":false,"start_time":"2021-09-11T08:01:29.129352","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 필요 없는 피처 제거","metadata":{"papermill":{"duration":0.06234,"end_time":"2021-09-11T08:01:29.367872","exception":false,"start_time":"2021-09-11T08:01:29.305532","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 첫 판매월, 평균 판매가, 판매건수 피처 제거\nall_data = all_data.drop(['첫 판매월', '평균 판매가', '판매건수'], axis=1)","metadata":{"papermill":{"duration":0.831694,"end_time":"2021-09-11T08:01:30.263547","exception":false,"start_time":"2021-09-11T08:01:29.431853","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = downcast(all_data, False) # 데이터 다운캐스팅","metadata":{"papermill":{"duration":1.813467,"end_time":"2021-09-11T08:01:32.265633","exception":false,"start_time":"2021-09-11T08:01:30.452166","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.7 피처 엔지니어링 VII : 마무리","metadata":{"papermill":{"duration":0.062394,"end_time":"2021-09-11T08:01:32.390893","exception":false,"start_time":"2021-09-11T08:01:32.328499","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_data.info()","metadata":{"papermill":{"duration":0.075994,"end_time":"2021-09-11T08:01:32.531333","exception":false,"start_time":"2021-09-11T08:01:32.455339","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 훈련 데이터 (피처)\nX_train = all_data[all_data['월ID'] < 33]\nX_train = X_train.drop(['월간 판매량'], axis=1)\n# 검증 데이터 (피처)\nX_valid = all_data[all_data['월ID'] == 33]\nX_valid = X_valid.drop(['월간 판매량'], axis=1)\n# 테스트 데이터 (피처)\nX_test = all_data[all_data['월ID'] == 34]\nX_test = X_test.drop(['월간 판매량'], axis=1)\n\n# 훈련 데이터 (타깃값)\ny_train = all_data[all_data['월ID'] < 33]['월간 판매량']\n# 검증 데이터 (타깃값)\ny_valid = all_data[all_data['월ID'] == 33]['월간 판매량']\n\n# 가비지 컬렉션\ndel all_data\ngc.collect();","metadata":{"papermill":{"duration":2.84926,"end_time":"2021-09-11T08:01:35.443256","exception":false,"start_time":"2021-09-11T08:01:32.593996","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.8 모델 훈련 및 성능 검증","metadata":{"papermill":{"duration":0.062794,"end_time":"2021-09-11T08:01:35.569011","exception":false,"start_time":"2021-09-11T08:01:35.506217","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgb\n\n# LightGBM 하이퍼파라미터\nparams = {'metric': 'rmse', \n          'num_leaves': 255,\n          'learning_rate': 0.005,\n          'feature_fraction': 0.75,\n          'bagging_fraction': 0.75,\n          'bagging_freq': 5,\n          'force_col_wise': True,\n          'random_state': 10}\n\ncat_features = ['상점ID', '도시', '상품분류ID', '대분류', '월']\n\n# LightGBM 훈련 및 검증 데이터셋\ndtrain = lgb.Dataset(X_train, y_train)\ndvalid = lgb.Dataset(X_valid, y_valid)\n \n# LightGBM 모델 훈련\nlgb_model = lgb.train(params=params,\n                      train_set=dtrain,\n                      num_boost_round=1500,\n                      valid_sets=(dtrain, dvalid),\n                      early_stopping_rounds=150,\n                      categorical_feature=cat_features,\n                      verbose_eval=100)      ","metadata":{"papermill":{"duration":938.063212,"end_time":"2021-09-11T08:17:13.694683","exception":false,"start_time":"2021-09-11T08:01:35.631471","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.4.9 예측 및 결과 제출","metadata":{"papermill":{"duration":0.071373,"end_time":"2021-09-11T08:17:13.838493","exception":false,"start_time":"2021-09-11T08:17:13.76712","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 예측\npreds = lgb_model.predict(X_test).clip(0, 20)\n\n# 제출 파일 생성\nsubmission['item_cnt_month'] = preds\nsubmission.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":12.206423,"end_time":"2021-09-11T08:17:26.112209","exception":false,"start_time":"2021-09-11T08:17:13.905786","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train, y_train, X_valid, y_valid, X_test, lgb_model, dtrain, dvalid\ngc.collect();","metadata":{"papermill":{"duration":0.212648,"end_time":"2021-09-11T08:17:26.392011","exception":false,"start_time":"2021-09-11T08:17:26.179363","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}