{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 향후 판매량 예측 경진대회 실전 문제\n(훈련 데이터에서 테스트 데이터에 있는 상점ID만 추출)","metadata":{"papermill":{"duration":0.045071,"end_time":"2021-09-11T07:53:59.320006","exception":false,"start_time":"2021-09-11T07:53:59.274935","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\n\nwarnings.filterwarnings(action='ignore') # 경고 메시지 생략\n\n# 데이터 경로\ndata_path = '/kaggle/input/competitive-data-science-predict-future-sales/'\n\nsales_train = pd.read_csv(data_path + 'sales_train.csv')\nshops = pd.read_csv(data_path + 'shops.csv')\nitems = pd.read_csv(data_path + 'items.csv')\nitem_categories = pd.read_csv(data_path + 'item_categories.csv')\ntest = pd.read_csv(data_path + 'test.csv')\nsubmission = pd.read_csv(data_path + 'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales_train = sales_train.rename(columns={'date': '날짜', \n                                          'date_block_num': '월ID',\n                                          'shop_id': '상점ID',\n                                          'item_id': '상품ID',\n                                          'item_price': '판매가',\n                                          'item_cnt_day': '판매량'})\n\nshops = shops.rename(columns={'shop_name': '상점명',\n                              'shop_id': '상점ID'})\n\nitems = items.rename(columns={'item_name': '상품명',\n                              'item_id': '상품ID',\n                              'item_category_id': '상품분류ID'})\n\nitem_categories = item_categories.rename(columns=\n                                         {'item_category_name': '상품분류명',\n                                          'item_category_id': '상품분류ID'})\n\ntest = test.rename(columns={'shop_id': '상점ID',\n                            'item_id': '상품ID'})","metadata":{"papermill":{"duration":0.171343,"end_time":"2021-09-11T07:54:02.82152","exception":false,"start_time":"2021-09-11T07:54:02.650177","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def downcast(df, verbose=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        dtype_name = df[col].dtype.name\n        if dtype_name == 'object':\n            pass\n        elif dtype_name == 'bool':\n            df[col] = df[col].astype('int8')\n        elif dtype_name.startswith('int') or (df[col].round() == df[col]).all():\n            df[col] = pd.to_numeric(df[col], downcast='integer')\n        else:\n            df[col] = pd.to_numeric(df[col], downcast='float')\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        print('{:.1f}% 압축됨'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\nall_df = [sales_train, shops, items, item_categories, test]\nfor df in all_df:\n    df = downcast(df)","metadata":{"papermill":{"duration":0.422472,"end_time":"2021-09-11T07:54:03.287596","exception":false,"start_time":"2021-09-11T07:54:02.865124","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 판매가가 0보다 큰 데이터 추출\nsales_train = sales_train[sales_train['판매가'] > 0]\n# 판매가가 50,000보다 작은 데이터 추출\nsales_train = sales_train[sales_train['판매가'] < 50000]\n# 판매량이 0보다 큰 데이터 추출\nsales_train = sales_train[sales_train['판매량'] > 0]\n# 판매량이 1,000보다 작은 데이터 추출\nsales_train = sales_train[sales_train['판매량'] < 1000]","metadata":{"papermill":{"duration":0.525625,"end_time":"2021-09-11T07:54:03.858663","exception":false,"start_time":"2021-09-11T07:54:03.333038","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(shops['상점명'][0], '||', shops['상점명'][57])\nprint(shops['상점명'][1], '||', shops['상점명'][58])\nprint(shops['상점명'][10], '||', shops['상점명'][11])\nprint(shops['상점명'][39], '||', shops['상점명'][40])","metadata":{"papermill":{"duration":0.058305,"end_time":"2021-09-11T07:54:03.962839","exception":false,"start_time":"2021-09-11T07:54:03.904534","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sales_train 데이터에서 상점ID 수정\nsales_train.loc[sales_train['상점ID'] == 0, '상점ID'] = 57\nsales_train.loc[sales_train['상점ID'] == 1, '상점ID'] = 58\nsales_train.loc[sales_train['상점ID'] == 10, '상점ID'] = 11\nsales_train.loc[sales_train['상점ID'] == 39, '상점ID'] = 40\n\n# test 데이터에서 상점ID 수정\ntest.loc[test['상점ID'] == 0, '상점ID'] = 57\ntest.loc[test['상점ID'] == 1, '상점ID'] = 58\ntest.loc[test['상점ID'] == 10, '상점ID'] = 11\ntest.loc[test['상점ID'] == 39, '상점ID'] = 40","metadata":{"papermill":{"duration":0.071697,"end_time":"2021-09-11T07:54:04.080763","exception":false,"start_time":"2021-09-11T07:54:04.009066","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 테스트 데이터에 있는 상점ID만 추출","metadata":{"papermill":{"duration":0.045234,"end_time":"2021-09-11T07:54:04.171455","exception":false,"start_time":"2021-09-11T07:54:04.126221","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 테스트 데이터에 있는 상점ID만 추출\nunique_test_shop_id = test['상점ID'].unique()\nsales_train = sales_train[sales_train['상점ID'].isin(unique_test_shop_id)]","metadata":{"papermill":{"duration":0.157325,"end_time":"2021-09-11T07:54:04.375122","exception":false,"start_time":"2021-09-11T07:54:04.217797","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops['도시'] = shops['상점명'].apply(lambda x: x.split()[0])","metadata":{"papermill":{"duration":0.056743,"end_time":"2021-09-11T07:54:04.477338","exception":false,"start_time":"2021-09-11T07:54:04.420595","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops['도시'].unique()","metadata":{"papermill":{"duration":0.08467,"end_time":"2021-09-11T07:54:04.60678","exception":false,"start_time":"2021-09-11T07:54:04.52211","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shops.loc[shops['도시'] =='!Якутск', '도시'] = 'Якутск'","metadata":{"papermill":{"duration":0.062415,"end_time":"2021-09-11T07:54:04.725994","exception":false,"start_time":"2021-09-11T07:54:04.663579","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# 레이블 인코더 생성\nlabel_encoder = LabelEncoder()\n# 도시 피처 레이블 인코딩\nshops['도시'] = label_encoder.fit_transform(shops['도시'])","metadata":{"papermill":{"duration":0.929735,"end_time":"2021-09-11T07:54:05.703749","exception":false,"start_time":"2021-09-11T07:54:04.774014","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상점명 피처 제거\nshops = shops.drop('상점명', axis=1)\n\nshops.head()","metadata":{"papermill":{"duration":0.063621,"end_time":"2021-09-11T07:54:05.813109","exception":false,"start_time":"2021-09-11T07:54:05.749488","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품명 피처 제거\nitems = items.drop(['상품명'], axis=1)","metadata":{"papermill":{"duration":0.053707,"end_time":"2021-09-11T07:54:05.911886","exception":false,"start_time":"2021-09-11T07:54:05.858179","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품이 맨 처음 팔린 날을 피처로 추가\nitems['첫 판매월'] = sales_train.groupby('상품ID').agg({'월ID': 'min'})['월ID']\n\nitems.head()","metadata":{"papermill":{"duration":0.124783,"end_time":"2021-09-11T07:54:06.081871","exception":false,"start_time":"2021-09-11T07:54:05.957088","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items[items['첫 판매월'].isna()]","metadata":{"papermill":{"duration":0.064072,"end_time":"2021-09-11T07:54:06.19249","exception":false,"start_time":"2021-09-11T07:54:06.128418","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 첫 판매월 피처의 결측값을 34로 대체\nitems['첫 판매월'] = items['첫 판매월'].fillna(34)","metadata":{"papermill":{"duration":0.055152,"end_time":"2021-09-11T07:54:06.294514","exception":false,"start_time":"2021-09-11T07:54:06.239362","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품분류명의 첫 단어를 대분류로 추출\nitem_categories['대분류'] = item_categories['상품분류명'].apply(lambda x: x.split()[0])  ","metadata":{"papermill":{"duration":0.058487,"end_time":"2021-09-11T07:54:06.399546","exception":false,"start_time":"2021-09-11T07:54:06.341059","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_categories['대분류'].value_counts()","metadata":{"papermill":{"duration":0.059877,"end_time":"2021-09-11T07:54:06.506786","exception":false,"start_time":"2021-09-11T07:54:06.446909","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_etc(x):\n    if len(item_categories[item_categories['대분류']==x]) >= 5:\n        return x\n    else:\n        return 'etc'\n\n# 대분류의 고윳 개수가 5개 미만이면 'etc'로 바꾸기\nitem_categories['대분류'] = item_categories['대분류'].apply(make_etc)","metadata":{"papermill":{"duration":0.087865,"end_time":"2021-09-11T07:54:06.641637","exception":false,"start_time":"2021-09-11T07:54:06.553772","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_categories.head()","metadata":{"papermill":{"duration":0.06168,"end_time":"2021-09-11T07:54:06.750493","exception":false,"start_time":"2021-09-11T07:54:06.688813","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 레이블 인코더 생성\nlabel_encoder = LabelEncoder()\n# 대분류 피처 레이블 인코딩\nitem_categories['대분류'] = label_encoder.fit_transform(item_categories['대분류'])\n\n# 상품분류명 피처 제거\nitem_categories = item_categories.drop('상품분류명', axis=1)","metadata":{"papermill":{"duration":0.05765,"end_time":"2021-09-11T07:54:06.856297","exception":false,"start_time":"2021-09-11T07:54:06.798647","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import product\n\ntrain = []\n# 월ID, 상점ID, 상품ID 조합 생성\nfor i in sales_train['월ID'].unique():\n    all_shop = sales_train.loc[sales_train['월ID']==i, '상점ID'].unique()\n    all_item = sales_train.loc[sales_train['월ID']==i, '상품ID'].unique()\n    train.append(np.array(list(product([i], all_shop, all_item))))\n\nidx_features = ['월ID', '상점ID', '상품ID'] # 기준 피처\ntrain = pd.DataFrame(np.vstack(train), columns=idx_features)","metadata":{"papermill":{"duration":17.704679,"end_time":"2021-09-11T07:54:24.610052","exception":false,"start_time":"2021-09-11T07:54:06.905373","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group = sales_train.groupby(idx_features).agg({'판매량': 'sum',\n                                               '판매가': 'mean'})\ngroup = group.reset_index()\ngroup = group.rename(columns={'판매량': '월간 판매량', '판매가': '평균 판매가'})\n\ntrain = train.merge(group, on=idx_features, how='left')\n\ntrain.head()","metadata":{"papermill":{"duration":5.084102,"end_time":"2021-09-11T07:54:29.741605","exception":false,"start_time":"2021-09-11T07:54:24.657503","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\n# group 변수 가비지 컬렉션\ndel group\ngc.collect();","metadata":{"papermill":{"duration":0.206373,"end_time":"2021-09-11T07:54:29.996939","exception":false,"start_time":"2021-09-11T07:54:29.790566","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상품 판매건수 피처 추가\ngroup = sales_train.groupby(idx_features).agg({'판매량': 'count'})\ngroup = group.reset_index()\ngroup = group.rename(columns={'판매량': '판매건수'})\n\ntrain = train.merge(group, on=idx_features, how='left')\n\n# 가비지 컬렉션\ndel group, sales_train\ngc.collect()\n\ntrain.head()","metadata":{"papermill":{"duration":4.809101,"end_time":"2021-09-11T07:54:34.854265","exception":false,"start_time":"2021-09-11T07:54:30.045164","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 테스트 데이터 월ID를 34로 설정\ntest['월ID'] = 34\n\n# train과 test 이어붙이기\nall_data = pd.concat([train, test.drop('ID', axis=1)],\n                     ignore_index=True,\n                     keys=idx_features)\n# 결측값을 0으로 대체\nall_data = all_data.fillna(0)\n\nall_data.head()","metadata":{"papermill":{"duration":0.489766,"end_time":"2021-09-11T07:54:35.39284","exception":false,"start_time":"2021-09-11T07:54:34.903074","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 나머지 데이터 병합\nall_data = all_data.merge(shops, on='상점ID', how='left')\nall_data = all_data.merge(items, on='상품ID', how='left')\nall_data = all_data.merge(item_categories, on='상품분류ID', how='left')\n\n# 데이터 다운캐스팅\nall_data = downcast(all_data)\n\n# 가비지 컬렉션\ndel shops, items, item_categories\ngc.collect();","metadata":{"papermill":{"duration":6.839293,"end_time":"2021-09-11T07:54:42.280558","exception":false,"start_time":"2021-09-11T07:54:35.441265","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_mean_features(df, mean_features, idx_features):\n    # 기준 피처 확인\n    assert (idx_features[0] == '월ID') and \\\n           len(idx_features) in [2, 3]\n    \n    # 파생 피처명 설정 \n    if len(idx_features) == 2:\n        feature_name = idx_features[1] + '별 평균 판매량'\n    else:\n        feature_name = idx_features[1] + ' ' + idx_features[2] + '별 평균 판매량'\n    \n    # 기준 피처를 토대로 그룹화해 월간 평균 판매량 구하기\n    group = df.groupby(idx_features).agg({'월간 판매량': 'mean'})\n    group = group.reset_index()\n    group = group.rename(columns={'월간 판매량': feature_name})\n    \n    # df와 group 병합 \n    df = df.merge(group, on=idx_features, how='left')\n    # 데이터 다운캐스팅 \n    df = downcast(df, verbose=False)\n    # 새로 만든 feature_name 피처명을 mean_features 리스트에 추가 \n    mean_features.append(feature_name)\n    \n    # 가비지 컬렉션\n    del group\n    gc.collect()\n    \n    return df, mean_features","metadata":{"papermill":{"duration":0.061429,"end_time":"2021-09-11T07:54:42.391138","exception":false,"start_time":"2021-09-11T07:54:42.329709","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 그룹화 기준 피처 중 '상품ID'가 포함된 파생 피처명을 담을 리스트\nitem_mean_features = []\n\n# ['월ID', '상품ID']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, item_mean_features = add_mean_features(df=all_data,\n                                                 mean_features=item_mean_features,\n                                                 idx_features=['월ID', '상품ID'])\n\n# ['월ID', '상품ID', '도시']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, item_mean_features = add_mean_features(df=all_data,\n                                                 mean_features=item_mean_features,\n                                                 idx_features=['월ID', '상품ID', '도시'])","metadata":{"papermill":{"duration":14.281785,"end_time":"2021-09-11T07:54:56.721833","exception":false,"start_time":"2021-09-11T07:54:42.440048","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 그룹화 기준 피처 중 '상점ID'가 포함된 파생 피처명을 담을 리스트\nshop_mean_features = []\n\n# ['월ID', '상점ID', '상품분류ID']로 그룹화한 월간 평균 판매량 파생 피처 생성\nall_data, shop_mean_features = add_mean_features(df=all_data, \n                                                 mean_features=shop_mean_features,\n                                                 idx_features=['월ID', '상점ID', '상품분류ID'])","metadata":{"papermill":{"duration":3.694415,"end_time":"2021-09-11T07:55:00.465365","exception":false,"start_time":"2021-09-11T07:54:56.77095","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_lag_features(df, lag_features_to_clip, idx_features, \n                     lag_feature, nlags=3, clip=False):\n    # 시차 피처 생성에 필요한 DataFrame 부분만 복사 \n    df_temp = df[idx_features + [lag_feature]].copy() \n\n    # 시차 피처 생성 \n    for i in range(1, nlags+1):\n        # 시차 피처명 \n        lag_feature_name = lag_feature +'_시차' + str(i)\n        # df_temp 열 이름 설정 \n        df_temp.columns = idx_features + [lag_feature_name]\n        # df_temp의 date_block_num 피처에 i 더하기 \n        df_temp['월ID'] += i\n        # idx_feature를 기준으로 df와 df_temp 병합하기\n        df = df.merge(df_temp.drop_duplicates(), \n                      on=idx_features, \n                      how='left')\n        # 결측값 0으로 대체 \n        df[lag_feature_name] = df[lag_feature_name].fillna(0)\n        # 0 ~ 20 사이로 제한할 시차 피처명을 lag_features_to_clip에 추가\n        if clip: \n            lag_features_to_clip.append(lag_feature_name)\n    \n    # 데이터 다운캐스팅\n    df = downcast(df, False)\n    # 가비지 컬렉션\n    del df_temp\n    gc.collect()\n    \n    return df, lag_features_to_clip","metadata":{"papermill":{"duration":0.06144,"end_time":"2021-09-11T07:55:00.575477","exception":false,"start_time":"2021-09-11T07:55:00.514037","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lag_features_to_clip = [] # 0 ~ 20 사이로 제한할 시차 피처명을 담을 리스트\nidx_features = ['월ID', '상점ID', '상품ID'] # 기준 피처\n\n# idx_features를 기준으로 월간 판매량의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='월간 판매량', \n                                                  nlags=3,\n                                                  clip=True)","metadata":{"papermill":{"duration":29.072942,"end_time":"2021-09-11T07:55:29.696778","exception":false,"start_time":"2021-09-11T07:55:00.623836","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head().T","metadata":{"papermill":{"duration":0.068651,"end_time":"2021-09-11T07:55:29.814402","exception":false,"start_time":"2021-09-11T07:55:29.745751","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lag_features_to_clip","metadata":{"papermill":{"duration":0.060314,"end_time":"2021-09-11T07:55:29.923872","exception":false,"start_time":"2021-09-11T07:55:29.863558","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idx_features를 기준으로 판매건수 피처의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='판매건수', \n                                                  nlags=3)\n\n# idx_features를 기준으로 평균 판매가 피처의 세 달치 시차 피처 생성\nall_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                  lag_features_to_clip=lag_features_to_clip,\n                                                  idx_features=idx_features,\n                                                  lag_feature='평균 판매가', \n                                                  nlags=3)","metadata":{"papermill":{"duration":70.188378,"end_time":"2021-09-11T07:56:40.162392","exception":false,"start_time":"2021-09-11T07:55:29.974014","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_temp = all_data[all_data['월ID'] == 34]\nX_test_temp[item_mean_features].sum()","metadata":{"papermill":{"duration":0.381471,"end_time":"2021-09-11T07:56:40.59783","exception":false,"start_time":"2021-09-11T07:56:40.216359","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idx_features를 기준으로 item_mean_features 요소별 시차 피처 생성\nfor item_mean_feature in item_mean_features:\n    all_data, lag_features_to_clip = add_lag_features(df=all_data, \n                                                      lag_features_to_clip=lag_features_to_clip, \n                                                      idx_features=idx_features, \n                                                      lag_feature=item_mean_feature, \n                                                      nlags=3,\n                                                      clip=True)\n# item_mean_features 피처 제거\nall_data = all_data.drop(item_mean_features, axis=1)","metadata":{"papermill":{"duration":67.950839,"end_time":"2021-09-11T07:57:48.601381","exception":false,"start_time":"2021-09-11T07:56:40.650542","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shop_mean_features","metadata":{"papermill":{"duration":0.061003,"end_time":"2021-09-11T07:57:48.71523","exception":false,"start_time":"2021-09-11T07:57:48.654227","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ['날월ID짜ID', '상점ID', '상품분류ID']를 기준으로 shop_mean_features 요소별 시차 피처 생성\nfor shop_mean_feature in shop_mean_features:\n    all_data, lag_features_to_clip = add_lag_features(df=all_data,\n                                                      lag_features_to_clip=lag_features_to_clip, \n                                                      idx_features=['월ID', '상점ID', '상품분류ID'], \n                                                      lag_feature=shop_mean_feature, \n                                                      nlags=3,\n                                                      clip=True)\n# shop_mean_features 피처 제거\nall_data = all_data.drop(shop_mean_features, axis=1)","metadata":{"papermill":{"duration":12.127324,"end_time":"2021-09-11T07:58:00.893865","exception":false,"start_time":"2021-09-11T07:57:48.766541","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 월ID 3미만인 데이터 제거\nall_data = all_data.drop(all_data[all_data['월ID'] < 3].index)","metadata":{"papermill":{"duration":2.561672,"end_time":"2021-09-11T07:58:03.508126","exception":false,"start_time":"2021-09-11T07:58:00.946454","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['월간 판매량 시차평균'] = all_data[['월간 판매량_시차1',\n                                          '월간 판매량_시차2', \n                                          '월간 판매량_시차3']].mean(axis=1)","metadata":{"papermill":{"duration":0.230902,"end_time":"2021-09-11T07:58:03.789932","exception":false,"start_time":"2021-09-11T07:58:03.55903","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0 ~ 20 사이로 값 제한\nall_data[lag_features_to_clip + ['월간 판매량', '월간 판매량 시차평균']] = all_data[lag_features_to_clip +['월간 판매량', '월간 판매량 시차평균']].clip(0, 20)","metadata":{"papermill":{"duration":3.005294,"end_time":"2021-09-11T07:58:06.846142","exception":false,"start_time":"2021-09-11T07:58:03.840848","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['시차변화량1'] = all_data['월간 판매량_시차1']/all_data['월간 판매량_시차2']\nall_data['시차변화량1'] = all_data['시차변화량1'].replace([np.inf, -np.inf], \n                                                        np.nan).fillna(0)\n\nall_data['시차변화량2'] = all_data['월간 판매량_시차2']/all_data['월간 판매량_시차3']\nall_data['시차변화량2'] = all_data['시차변화량2'].replace([np.inf, -np.inf], \n                                                        np.nan).fillna(0)","metadata":{"papermill":{"duration":0.613439,"end_time":"2021-09-11T07:58:07.510465","exception":false,"start_time":"2021-09-11T07:58:06.897026","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['신상여부'] = all_data['첫 판매월'] == all_data['월ID']","metadata":{"papermill":{"duration":0.06782,"end_time":"2021-09-11T07:58:07.628495","exception":false,"start_time":"2021-09-11T07:58:07.560675","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['첫 판매 후 기간'] = all_data['월ID'] - all_data['첫 판매월']","metadata":{"papermill":{"duration":0.068014,"end_time":"2021-09-11T07:58:07.748119","exception":false,"start_time":"2021-09-11T07:58:07.680105","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['월'] = all_data['월ID'] % 12","metadata":{"papermill":{"duration":0.094063,"end_time":"2021-09-11T07:58:07.892834","exception":false,"start_time":"2021-09-11T07:58:07.798771","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 첫 판매월, 평균 판매가, 판매건수 피처 제거\nall_data = all_data.drop(['첫 판매월', '평균 판매가', '판매건수'], axis=1)","metadata":{"papermill":{"duration":0.950936,"end_time":"2021-09-11T07:58:08.893811","exception":false,"start_time":"2021-09-11T07:58:07.942875","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = downcast(all_data, False) # 데이터 다운캐스팅","metadata":{"papermill":{"duration":1.812834,"end_time":"2021-09-11T07:58:10.759095","exception":false,"start_time":"2021-09-11T07:58:08.946261","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.info()","metadata":{"papermill":{"duration":0.070224,"end_time":"2021-09-11T07:58:10.87972","exception":false,"start_time":"2021-09-11T07:58:10.809496","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 훈련 데이터 (피처)\nX_train = all_data[all_data['월ID'] < 33]\nX_train = X_train.drop(['월간 판매량'], axis=1)\n# 검증 데이터 (피처)\nX_valid = all_data[all_data['월ID'] == 33]\nX_valid = X_valid.drop(['월간 판매량'], axis=1)\n# 테스트 데이터 (피처)\nX_test = all_data[all_data['월ID'] == 34]\nX_test = X_test.drop(['월간 판매량'], axis=1)\n\n# 훈련 데이터 (타깃값)\ny_train = all_data[all_data['월ID'] < 33]['월간 판매량']\n# 검증 데이터 (타깃값)\ny_valid = all_data[all_data['월ID'] == 33]['월간 판매량']\n\n# 가비지 컬렉션\ndel all_data\ngc.collect();","metadata":{"papermill":{"duration":2.713052,"end_time":"2021-09-11T07:58:13.643914","exception":false,"start_time":"2021-09-11T07:58:10.930862","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\n\n# LightGBM 하이퍼파라미터\nparams = {'metric': 'rmse',\n          'num_leaves': 255,\n          'learning_rate': 0.005,\n          'feature_fraction': 0.75,\n          'bagging_fraction': 0.75,\n          'bagging_freq': 5,\n          'force_col_wise': True,\n          'random_state': 10}\n\ncat_features = ['상점ID', '도시', '상품분류ID', '대분류', '월']\n\n# LightGBM 훈련 및 검증 데이터셋\ndtrain = lgb.Dataset(X_train, y_train)\ndvalid = lgb.Dataset(X_valid, y_valid)\n \n# LightGBM 모델 훈련\nlgb_model = lgb.train(params=params,\n                      train_set=dtrain,\n                      num_boost_round=1500,\n                      valid_sets=(dtrain, dvalid),\n                      early_stopping_rounds=150,\n                      categorical_feature=cat_features,\n                      verbose_eval=100)      ","metadata":{"papermill":{"duration":809.626892,"end_time":"2021-09-11T08:11:43.322732","exception":false,"start_time":"2021-09-11T07:58:13.69584","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 예측\npreds = lgb_model.predict(X_test).clip(0, 20)\n\n# 제출 파일 생성\nsubmission['item_cnt_month'] = preds\nsubmission.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":12.512026,"end_time":"2021-09-11T08:11:55.890917","exception":false,"start_time":"2021-09-11T08:11:43.378891","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train, y_train, X_valid, y_valid, X_test, lgb_model, dtrain, dvalid\ngc.collect();","metadata":{"papermill":{"duration":0.2264,"end_time":"2021-09-11T08:11:56.175099","exception":false,"start_time":"2021-09-11T08:11:55.948699","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}