{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 参考","metadata":{"id":"82GWIx3pEGoY"}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/lucasmorin/amex-lgbm-features-eng/notebook","metadata":{}},{"cell_type":"code","source":"# https://zenn.dev/mst8823/articles/da505dcf45474f","metadata":{"id":"s7FuteO5shzg"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pipeline","metadata":{"id":"XAed8tBOEzyO"}},{"cell_type":"markdown","source":"## Library","metadata":{"id":"iDON_httEjM0"}},{"cell_type":"code","source":"!pip install --upgrade lightgbm","metadata":{"id":"7KelnyuQt5kZ","outputId":"c79103c0-1664-400b-8198-c7e6c71dd6c9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport warnings\nimport shutil\nimport logging\nimport joblib\nimport random\nimport datetime\nimport pytz\nimport sys\nimport re\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tqdm.notebook import tqdm\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import accuracy_score, mean_squared_error\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom tensorflow.keras import backend as K\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom tensorflow.keras.utils import plot_model\n\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n\nimport pickle\nimport glob\n\n# import shap\nimport xgboost\nfrom scipy.stats import spearmanr\nfrom sklearn.ensemble import (\n    ExtraTreesRegressor,\n    GradientBoostingRegressor,\n    RandomForestRegressor,\n)\n\nimport lightgbm as lgb\nfrom lightgbm import log_evaluation\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"id":"WA-bmarGgH2b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLAB = \"google.colab\" in sys.modules","metadata":{"id":"_Ivy662-g91t"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{"id":"ySSZB4IzELyJ"}},{"cell_type":"code","source":"class Config:\n    # notebookのタイトル取得\n    if COLAB:\n        from requests import get\n        name = get('http://172.28.0.2:9000/api/sessions').json()[0]['name'].split('.')[0]  \n    else:\n        name = \"baseline_lgbm001\"  # kaggle環境ならば自分で記入  \n\n    # 予測のみ/学習+予測の選択\n    only_inference = False\n    if only_inference:\n        task = 'infer'\n    else:\n        task = 'train'\n\n    # クロスバリデーション設定\n    n_fold = 5\n    trn_fold = list(range(n_fold))\n\n    seed = 42\n    \n    target_col = \"target\"  # ⚠️コンペごとに更新\n    categ_feats = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n    # debug = False\n\n\n    # lgb params\n    def get_lgb_params() -> dict:\n        lgb_params = {\n            'objective': 'binary',\n            'metric': \"binary_logloss\",\n            'boosting': 'goss',  # 'goss', 'gbdt', 'dart'\n            'seed': Config.seed,\n            'num_leaves': 100,\n            'learning_rate': 0.01,\n            'feature_fraction': 0.20,\n            'bagging_freq': 10,\n            'bagging_fraction': 0.50,\n            'n_jobs': -1,\n            'lambda_l2': 2,\n            'min_data_in_leaf': 40,\n            }\n        return lgb_params\n    \"\"\"\n    # DNN params\n    model_name = \"roberta-base\"\n    learning_rate = 1e-5\n    max_length = 256\n    epochs = 8\n    batch_size = 16\n    \"\"\"\n\n    # 解凍ファイル\n    # zip_file = 'foursquare-location-matching.zip'  # ⚠️コンペごとに更新\n\n    # Colab Env\n    upload_from_colab = True\n    api_path = \"/content/drive/MyDrive/kaggle/kaggle.json\"\n    drive_path = \"/content/drive/MyDrive/kaggle/AmericanExpress\"  # ⚠️コンペごとに更新\n    \n    # Kaggle Env\n    kaggle_input_path = \"../input/expression-chinchilla\"  # ⚠️コンペごとに更新\n    kaggle_dataset_path = None","metadata":{"id":"jB5W0vW9fsLz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{"id":"rTotr8nyEYld"}},{"cell_type":"code","source":"# log を txtファイルに出力させるためのクラス\nclass Logger:\n    # 参考) https://github.com/ghmagazine/kagglebook/blob/master/ch04-model-interface/code/util.py\n    def __init__(self, path, log_title='Experiment'):\n        self.general_logger = logging.getLogger(path)\n        stream_handler = logging.StreamHandler()\n        file_general_handler = logging.FileHandler(os.path.join(path, f'{log_title}.log'))\n        if len(self.general_logger.handlers) == 0:\n            self.general_logger.addHandler(stream_handler)\n            self.general_logger.addHandler(file_general_handler)\n            self.general_logger.setLevel(logging.INFO)\n\n    def info(self, message):\n        # display time\n        self.general_logger.info(f'[{self.now_string()}] - {message}')\n\n    @staticmethod\n    def now_string():\n        return str(datetime.datetime.now(pytz.timezone('Asia/Tokyo')).strftime('%Y-%m-%d %H:%M:%S'))","metadata":{"id":"t4FkO6Cpg5zP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# シード固定用関数\ndef seed_everything(seed=42):\n#  参考) https://qiita.com/kaggle_grandmaster-arai-san/items/d59b2fb7142ec7e270a5\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)","metadata":{"id":"2fRR9p6A2LQD"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SetUp","metadata":{"id":"gUeiMunXEv8i"}},{"cell_type":"markdown","source":"### 環境ごとのセットアップ","metadata":{"id":"hY7xPwMVhJce"}},{"cell_type":"code","source":"# -------------------------------colab 環境の場合-------------------------------\nif COLAB:\n    print(\"-------------------------------This environment is Google Colab-------------------------------\")\n    \n    # mount\n    from google.colab import drive\n    if not os.path.isdir(\"/content/drive\"):\n        drive.mount('/content/drive') \n\n    # my-modules のPath設定\n    import sys\n    sys.path.append('/content/drive/MyDrive/Colab Notebooks/my-modules')\n\n    # use kaggle api (need kaggle token)\n    f = open(Config.api_path, 'r')\n    json_data = json.load(f) \n    os.environ[\"KAGGLE_USERNAME\"] = json_data[\"username\"]\n    os.environ[\"KAGGLE_KEY\"] = json_data[\"key\"]\n    \n    # set dirs\n    DRIVE = Config.drive_path\n    EXP = (Config.name if Config.name is not None \n           else get(\"http://172.28.0.2:9000/api/sessions\").json()[0][\"name\"][:-6])\n    INPUT = os.path.join(DRIVE, \"INPUT\")\n    FEATURES = os.path.join(INPUT, \"FEATURES\")\n    OUTPUT = os.path.join(DRIVE, \"OUTPUT\")\n    SUBMISSION = os.path.join(DRIVE, \"SUBMISSION\")\n    OUTPUT_EXP = os.path.join(OUTPUT, EXP) \n    EXP_MODEL = os.path.join(OUTPUT_EXP, \"MODEL\")\n    EXP_FIG = os.path.join(OUTPUT_EXP, \"FIG\")\n    EXP_PREDS = os.path.join(OUTPUT_EXP, \"PREDS\")\n\n    # make dirs\n    for d in [INPUT, FEATURES, SUBMISSION, EXP_MODEL, EXP_FIG, EXP_PREDS]:\n        os.makedirs(d, exist_ok=True)\n\n    # if not os.path.isfile(os.path.join(INPUT, Config.zip_file)):\n    #     # download dataset\n    #     # kaggle をインストール\n    #     # アクセスパーミッションのため、以下を打ち込みます。\n    #     ! chmod 600 /root/.kaggle/kaggle.json\n    #     ! pip install kaggle\n    #     ! kaggle competitions download -c foursquare-location-matching -p $INPUT  # ⚠️コンペごとに更新\n    #     # 上記でdownloadしてきたZIPファイルを解凍\n    #     ! apt-get install p7zip-full -y\n    #     ! 7za x os.path.join(INPUT, Config.zip_file)\n    # else:\n    #     print('DS for competition has been already installed.') \n    \n    # utils\n    logger = Logger(OUTPUT_EXP)\n    \n    sys.path.append('/content/drive/MyDrive/Colab Notebooks/my-modules')\n\n\n# -------------------------------kaggle 環境の場合-------------------------------\nelse:\n    print(\"-------------------------------This environment is Kaggle Kernel-------------------------------\")\n    \n    # set dirs\n    INPUT = Config.kaggle_input_path  # ⚠️コンペごとに更新\n    EXP, OUTPUT, SUBMISSION = \"./\", \"./\", \"./\"\n    EXP_MODEL = os.path.join(EXP, \"model\")\n    EXP_FIG = os.path.join(EXP, \"fig\")\n    EXP_PREDS = os.path.join(EXP, \"preds\")\n    \n    # copy dirs\n    if Config.kaggle_dataset_path is not None:\n        KD_MODEL = os.path.join(Config.kaggle_dataset_path, \"model\")\n        KD_EXP_PREDS = os.path.join(Config.kaggle_dataset_path, \"preds\")\n        shutil.copytree(KD_MODEL, EXP_MODEL)\n        shutil.copytree(KD_EXP_PREDS, EXP_PREDS)\n\n    # make dirs\n    for d in [EXP_MODEL, EXP_FIG, EXP_PREDS]:\n        os.makedirs(d, exist_ok=True)\n        \n    # utils\n    logger = Logger(EXP)\n\n# utils\nwarnings.filterwarnings(\"ignore\")\nsns.set(style='whitegrid')\nseed_everything(seed=Config.seed)","metadata":{"id":"ay6hwrE3hTFM","outputId":"f4cb3006-d4e0-4566-9a71-711656bdaf97"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## コンペ説明","metadata":{"id":"CVGI0JLYZrBz"}},{"cell_type":"markdown","source":"レストランでの食事やコンサートのチケット購入など、現代の生活では日々の買い物にクレジットカードの利便性が欠かせません。\nクレジットカードがあれば、多額の現金を持ち歩く必要がなく、また、買い物の全額を前払いして、長期にわたって支払うことができます。\nしかし、カード発行会社は、私たちが請求した金額をきちんと返済してくれることをどうやって確認するのでしょうか？\nこの問題は複雑で、多くの解決策がありますが、このコンペティションでは、さらに多くの改善策が検討されています。\n\n貸し倒れ予測は、消費者金融ビジネスのリスク管理の中心的存在です。\n貸し倒れを予測することで、貸し出しの決定を最適化し、より良い顧客体験と健全なビジネス経済を実現することができます。\n現在のモデルは、リスク管理を支援するために存在しています。\nしかし、現在使用されているモデルを凌駕する、より優れたモデルを作成することは可能です。\n\nアメリカン・エキスプレスは、世界的に統合された決済企業です。\n世界最大の決済カード発行会社である同社は、生活を豊かにし、ビジネスの成功をもたらす商品、洞察、体験へのアクセスを顧客に提供しています。\n\nこのコンペティションでは、機械学習のスキルを応用して、クレジット・デフォルトを予測します。\n具体的には、産業界規模のデータセットを活用し、現在の生産モデルに挑戦する機械学習モデルを構築していただきます。\nトレーニング、検証、テストの各データセットには、時系列行動データおよび匿名化された顧客プロファイル情報が含まれます。\n特徴量の作成から、モデル内でのデータの有機的な利用まで、最も強力なモデルを作るためのあらゆる手法を自由に探求することができます。\n\n成功すれば、クレジットカードの審査が通りやすくなり、カード会員にとってより良い顧客体験の創造に貢献できます。\n優れたソリューションは、世界最大のクレジットカード発行会社が使用しているクレジットデフォルト予測モデルに挑戦し、\n賞金やアメリカン・エキスプレスとの面接の機会、そしてやりがいのある新しいキャリアを獲得する可能性があります。","metadata":{"id":"zfG4HQptZj_A"}},{"cell_type":"markdown","source":"## Load Data","metadata":{"id":"-2RFUO1Qk0Wr"}},{"cell_type":"code","source":"train = pd.read_pickle(os.path.join(INPUT, 'train_data_agg.pkl'))\ntrain_labels = pd.read_csv(os.path.join(INPUT, 'train_labels.csv')).set_index('customer_ID').loc[train.index]\n\ntest = pd.read_pickle(os.path.join(INPUT, 'test_data_agg.pkl'))\nsample_submission = pd.read_csv(os.path.join(INPUT, 'sample_submission.csv'))  # parquetにすべき？\n","metadata":{"id":"OvOoQCrji4uk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.merge(train, train_labels, left_index=True, right_index=True)\ntrain.reset_index()","metadata":{"id":"e9QVCkvQLh38","outputId":"0e72a8c8-1c46-4c37-d756-b0b13d130961"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習・予測に使う特徴量のみ抽出する関数\ndef get_feat_cols(train: pd.DataFrame, *drop_cols) -> list:\n    return list(train.drop(list(drop_cols), axis=1).columns)","metadata":{"id":"vqdbJjSLzqsq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['target']\nfeat_cols = get_feat_cols(train, *drop_cols)\nfeat_cols","metadata":{"id":"0GGiYFgb6kGb","outputId":"90012f5f-8f46-450a-8111-b307d886598c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[[0, 1]]","metadata":{"id":"a1r6XuvbTo2n","outputId":"d79ba64b-32d9-449d-ccef-1b6b7a0f5f45"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Models","metadata":{"id":"zSZdiLGpWBxB"}},{"cell_type":"markdown","source":"### CV split","metadata":{"id":"phEpMOEodj6e"}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\ntrain['fold'] = -1\n\nskf = StratifiedKFold(n_splits=Config.n_fold,\n                      shuffle=True,\n                      random_state=42\n                      )\nskf_split = list(skf.split(X=train,\n                        y=train[Config.target_col]))\n\nfor i_fold, (trn_, val_) in enumerate(skf_split):\n    if i_fold in Config.trn_fold:\n        train.loc[val_, 'fold'] = i_fold","metadata":{"id":"mbfY_ipOgHME","outputId":"0fc888fa-8c2d-49b7-cc89-122dce639ec5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"id":"qXMUMG-DWAuX","outputId":"44623bfc-c88e-4b38-c3f6-b51ac3cfec49"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Calculate functions","metadata":{"id":"N4N2xY28iO6T"}},{"cell_type":"code","source":"def compute_recall_at4(y_true: np.array, y_pred: np.array) -> float:\n    \n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n    \n    # desc sorting by prediction values\n    indices = np.argsort(y_pred)[::-1]\n    target = y_true[indices]\n    \n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_mask = cum_norm_weight <= 0.04\n    \n    # default rate captured at 4%\n    d = target[four_pct_mask].sum() / n_pos\n    \n    return d\n\ndef compute_normalized_gini(y_true: np.array, y_pred: np.array) -> float:\n    \n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting desc by prediction values\n    indices = np.argsort(y_pred)[::-1]\n    target = y_true[indices]\n\n    # weighted gini coefficient\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n    \n    return g\n    \ndef compute_amex_metric(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting desc by prediction values\n    indices = np.argsort(y_pred)[::-1]\n    target = y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_filter = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = target[four_pct_filter].sum() / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)","metadata":{"id":"VWSKaUyAiMZO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# metrics in lgbm format\n\ndef metric_recall_at4(y_pred: np.ndarray, data: lgb.Dataset):\n    y_true = data.get_label()\n    # name, result, is_higher_better\n    return 'recall_at4', compute_recall_at4(y_true, y_pred), True\n\ndef metric_normalized_gini(y_pred: np.ndarray, data: lgb.Dataset):\n    y_true = data.get_label()\n    # name, result, is_higher_better\n    return 'norm_gini', compute_normalized_gini(y_true, y_pred), True\n\ndef metric_amex(y_pred: np.ndarray, data: lgb.Dataset):\n    y_true = data.get_label()\n    # name, result, is_higher_better\n    return 'amex_metric', compute_amex_metric(y_true, y_pred), True","metadata":{"id":"S9SBHD-55Hmk"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LGBM","metadata":{"id":"MqNcM7A4WBxB"}},{"cell_type":"code","source":"def make_lgb_ds(X, y):\n    return lgb.Dataset(data=X, \n                       label=y, \n                       feature_name='auto',  # 列名を自動で認識\n                    #    categorical_feature=Config.categ_feats,\n                       free_raw_data=False)","metadata":{"id":"EWdTznZPtciw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_lgbm(df: pd.DataFrame, folds: int, params: dict) -> list:\n    models = []\n    \n    for fold in tqdm(range(folds)):\n        model_path = os.path.join(EXP_MODEL, f\"{Config.name}-seed{Config.seed}-fold{fold}\")\n        # modelが保存されていない場合はtrainning\n        if not os.path.isfile(model_path):\n            # train, valid毎の入出力を用意\n            X_train = df[df.fold != fold][feat_cols]\n            y_train = df[df.fold != fold][Config.target_col]\n            X_valid = df[df.fold == fold][feat_cols]\n            y_valid = df[df.fold == fold][Config.target_col]\n            # train, valid毎にdsへ格納\n            train_ds = make_lgb_ds(X_train, y_train)\n            valid_ds = make_lgb_ds(X_valid, y_valid)\n            # modelの用意\n            model = lgb.train(params=params,\n                              train_set=train_ds,\n                              valid_sets=[train_ds, valid_ds],\n                              feval=[metric_amex, \n                                     metric_recall_at4, \n                                     metric_normalized_gini],\n                              early_stopping_rounds=100,\n                              num_boost_round=10500,  # 最大の分岐回数\n                              verbose_eval=500,\n                            #   callbacks=[lgb.log_evaluation(period=50), lgb.early_stopping(50)]\n                              )\n            # validモードの予測とスコアを計算\n            oof_preds = model.predict(X_valid)\n            oof_score = compute_amex_metric(y_valid.values, oof_preds)\n            # modelsへmodelを追加\n            models.append(model)\n            # fold毎のmodelをpklファイルとして保存\n            pickle.dump(model, open(model_path, 'wb'))\n            print(f\"{Config.name}-seed{Config.seed}-fold{fold} has been saved.\")\n            # fold毎にモデル名とスコアを表示\n            logger.info(f\"model_name:{Config.name}-seed:{Config.seed}-fold:{fold}\\\n                        \\n-X_cols:{X_train.columns.values} >>>>> Score={oof_score}\")\n            print(f'fold_{fold} has finished.')\n            print('-----------------------------')\n            if fold == (Config.n_fold - 1):\n                logger.info(f\"Training finished.\")\n        # 既に保存済みの場合は保存しない\n        else:\n            print(f'fold_{fold}: No model trained.')\n            print(f'{model_path} -> loaded.')\n            model = pickle.load(open(model_path, 'rb'))\n            models.append(model)\n        \n    return models","metadata":{"execution":{"iopub.status.busy":"2022-04-09T02:16:45.55676Z","iopub.execute_input":"2022-04-09T02:16:45.557577Z","iopub.status.idle":"2022-04-09T02:16:45.565817Z","shell.execute_reply.started":"2022-04-09T02:16:45.557518Z","shell.execute_reply":"2022-04-09T02:16:45.564655Z"},"id":"XP-mMZvMtciw","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlgbm_models = train_lgbm(train, folds=Config.n_fold, params=Config.get_lgb_params())","metadata":{"execution":{"iopub.status.busy":"2022-04-09T02:16:57.334568Z","iopub.execute_input":"2022-04-09T02:16:57.335341Z","iopub.status.idle":"2022-04-09T02:16:59.214364Z","shell.execute_reply.started":"2022-04-09T02:16:57.3353Z","shell.execute_reply":"2022-04-09T02:16:59.213631Z"},"id":"Blu7GZyvtnxi","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Predictions & Submit","metadata":{"id":"tBWWrbijWBxC"}},{"cell_type":"code","source":"def inference_lgbm(models: list, feat_df: pd.DataFrame) -> np.array:\n    pred = np.array([model.predict(feat_df) for model in models])\n    pred = np.mean(pred, axis=0)\n    return pred","metadata":{"id":"zeO_4PyosnXO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"id":"nYbrL7bIwJ0O"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = lgbm_models[0]\npred = model.predict(test[feat_cols])","metadata":{"id":"fDiObudkw4mG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = inference_lgbm(models=lgbm_models, feat_df=test[feat_cols])","metadata":{"id":"2VFt3z8atPeM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_pred = pd.DataFrame(pred)\n# df_pred.to_csv(os.path.join(EXP_PREDS, 'pred.csv'))","metadata":{"id":"5efQe31e-m2I"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pred_to_submission(pred, sample_submission):\n    sample_submission['prediction'] = pred\n    return submission","metadata":{"id":"lOGVX1z0nTn-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(pred)","metadata":{"id":"T1wdcfbsO_SY","outputId":"1c551474-7f67-4f61-d867-18913b0481b8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.describe()","metadata":{"id":"8a91LxTP1MuZ","outputId":"1a31bc8d-fe2e-4a0a-90d3-4d789c087cf2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['prediction'] = pred\nsample_submission","metadata":{"id":"b0mn5eWetQAE","outputId":"c837d8ed-ce48-4469-91bf-fca09de7eac1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(os.path.join(EXP_PREDS, 'submission.csv'), index=False)","metadata":{"id":"eEu3kMvwuAw-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"tCYm9ovRvrB9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# kaggle.jsonの設定\n# kaggle.jsonをGoogle Driveの任意なフォルダに保存しておけば、Kaggle API with Colabで紹介された方法でColabへコピーできます。\n\nfrom googleapiclient.discovery import build\nimport io, os\nfrom googleapiclient.http import MediaIoBaseDownload\nfrom google.colab import auth\n\nauth.authenticate_user()\n\ndrive_service = build('drive', 'v3')\nresults = drive_service.files().list(\n        q=\"name = 'kaggle.json'\", fields=\"files(id)\").execute()\nkaggle_api_key = results.get('files', [])\n\nfilename = \"/root/.kaggle/kaggle.json\"\nos.makedirs(os.path.dirname(filename), exist_ok=True)\n\nrequest = drive_service.files().get_media(fileId=kaggle_api_key[0]['id'])\nfh = io.FileIO(filename, 'wb')\ndownloader = MediaIoBaseDownload(fh, request)\ndone = False\nwhile done is False:\n    status, done = downloader.next_chunk()\n    print(\"Download %d%%.\" % int(status.progress() * 100))\nos.chmod(filename, 600)","metadata":{"id":"2xS67zFSfSre"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!kaggle competitions submit -c \"amex-default-prediction\" -f '/content/drive/MyDrive/kaggle/AmericanExpress/OUTPUT/baseline-lgbm001/PREDS/submission.csv' -m \"Message\"","metadata":{"id":"2Xio-XyaGPUR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Others","metadata":{"id":"pIxLBypaED0h"}},{"cell_type":"code","source":"# folderのディレクトリ構造可視化ツール\nimport pathlib\nimport glob\nimport os\n\ndef tree(path, layer=0, is_last=False, indent_current='　'):\n    if not pathlib.Path(path).is_absolute():\n        path = str(pathlib.Path(path).resolve())\n\n    # カレントディレクトリの表示\n    current = path.split('/')[::-1][0]\n    if layer == 0:\n        print('<'+current+'>')\n    else:\n        branch = '└' if is_last else '├'\n        print('{indent}{branch}<{dirname}>'.format(indent=indent_current, branch=branch, dirname=current))\n\n    # 下の階層のパスを取得\n    paths = [p for p in glob.glob(path+'/*') if os.path.isdir(p) or os.path.isfile(p)]\n    def is_last_path(i):\n        return i == len(paths)-1\n\n    # 再帰的に表示\n    for i, p in enumerate(paths):\n\n        indent_lower = indent_current\n        if layer != 0:\n            indent_lower += '　　' if is_last else '│　'\n\n        if os.path.isfile(p):\n            branch = '└' if is_last_path(i) else '├'\n            print('{indent}{branch}{filename}'.format(indent=indent_lower, branch=branch, filename=p.split('/')[::-1][0]))\n        if os.path.isdir(p):\n            tree(p, layer=layer+1, is_last=is_last_path(i), indent_current=indent_lower)","metadata":{"id":"H2PA3JjCx4nw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree('/content/drive/MyDrive/kaggle/JPXTokyoStock')","metadata":{"id":"pvjGkkw77uLp","outputId":"1f085457-7ab3-43e5-f538-6f535afa20a9"},"execution_count":null,"outputs":[]}]}