# This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) # Input data files are available in the read-only "../input/" directory # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory import os for dirname, _, filenames in os.walk('/kaggle/input'): for filename in filenames: print(os.path.join(dirname, filename)) # You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session ### import os from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter from functools import partial import numpy as np import pandas as pd import random import math from tqdm.notebook import tqdm import matplotlib.pyplot as plt import seaborn as sns from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold from sklearn.metrics import mean_squared_error import category_encoders as ce from PIL import Image import cv2 import pydicom import torch import lightgbm as lgb from sklearn.linear_model import Ridge ################## import warnings warnings.filterwarnings("ignore") # from pandas.core.nanops import nanmean as pd_nanmean #from statsmodels.tsa.api import Holt pd.set_option('display.max_columns', 90) pd.set_option('display.max_rows', 290) #pd.set_option('display.precision', 3) # точность pd.set_option('display.width', 240) ################################# def get_logger(filename='log'): logger = getLogger(__name__) logger.setLevel(INFO) handler1 = StreamHandler() handler1.setFormatter(Formatter("%(message)s")) handler2 = FileHandler(filename=f"{filename}.log") handler2.setFormatter(Formatter("%(message)s")) logger.addHandler(handler1) logger.addHandler(handler2) return logger logger = get_logger() def seed_everything(seed=777): random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True #################################################### OUTPUT_DICT = './' ROOT = "../input/osic-pulmonary-fibrosis-progression" ID = 'Patient_Week' TARGET = 'FVC' SEED = 222 seed_everything(seed=SEED) N_FOLD = 4 ###################### train = pd.read_csv(f"{ROOT}/train.csv") train[ID] = train['Patient'].astype(str) + '_' + train['Weeks'].astype(str) print(train.shape) train.head() ##################################### output = pd.DataFrame() gb = train.groupby('Patient') tk0 = tqdm(gb, total=len(gb)) for _, usr_df in tk0: usr_output = pd.DataFrame() for week, tmp in usr_df.groupby('Weeks'): rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'} tmp = tmp.drop(columns='Patient_Week').rename(columns=rename_cols) drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent'] _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient') _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week'] usr_output = pd.concat([usr_output, _usr_output]) output = pd.concat([output, usr_output]) train = output[output['Week_passed']!=0].reset_index(drop=True) print(train.shape) train.head() ###################################### test = pd.read_csv(f"{ROOT}/test.csv")\ .rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'}) submission = pd.read_csv(f"{ROOT}/sample_submission.csv") submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0]) submission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int) test = submission.drop(columns=['FVC', 'Confidence']).merge(test, on='Patient') test['Week_passed'] = test['predict_Week'] - test['base_Week'] print(test.shape) test.head() ################################### submission = pd.read_csv(f"{ROOT}/sample_submission.csv") print(submission.shape) submission.head() folds = train[[ID, 'Patient', TARGET]].copy() #Fold = KFold(n_splits=N_FOLD, shuffle=True, random_state=SEED) Fold = GroupKFold(n_splits=N_FOLD) groups = folds['Patient'].values for n, (train_index, val_index) in enumerate(Fold.split(folds, folds[TARGET], groups)): folds.loc[val_index, 'fold'] = int(n) folds['fold'] = folds['fold'].astype(int) folds.head() ############################################ #=========================================================== # model #=========================================================== def run_single_ridge(param, train_df, test_df, folds, features, target, fold_num=0): trn_idx = folds[folds.fold!=fold_num].index val_idx = folds[folds.fold==fold_num].index y_tr = target.iloc[trn_idx].values X_tr = train_df.iloc[trn_idx][features].values y_val = target.iloc[val_idx].values X_val = train_df.iloc[val_idx][features].values oof = np.zeros(len(train_df)) predictions = np.zeros(len(test_df)) clf = Ridge(**param) clf.fit(X_tr, y_tr) oof[val_idx] = clf.predict(X_val) predictions += clf.predict(test_df[features]) logger.info("fold{} score: {:<8.5f}" .format(fold_num, np.sqrt(mean_squared_error(target[val_idx], oof[val_idx])))) return oof, predictions def run_kfold_ridge(param, train, test, folds, features, target, n_fold=5): oof = np.zeros(len(train)) predictions = np.zeros(len(test)) feature_importance_df = pd.DataFrame() for fold_ in range(n_fold): logger.info("Fold {}".format(fold_)) _oof, _predictions = run_single_ridge(param, train, test, folds, features, target, fold_num=fold_) oof += _oof predictions += _predictions/n_fold logger.info("CV score: {:<8.5f}" .format(np.sqrt(mean_squared_error(target, oof)))) return oof, predictions ####################################### target = train[TARGET] test[TARGET] = np.nan # features cat_features = ['Sex', 'SmokingStatus'] num_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)] features = num_features + cat_features drop_features = [ID, TARGET, 'predict_Week', 'base_Week'] features = [c for c in features if c not in drop_features] if cat_features: ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute') ce_oe.fit(train) train = ce_oe.transform(train) test = ce_oe.transform(test) ridge_param = { 'alpha': 1, 'random_state': SEED, } oof, predictions = run_kfold_ridge(ridge_param, train, test, folds, features, target, n_fold=N_FOLD) ####################################### train['FVC_pred'] = oof test['FVC_pred'] = predictions # baseline score train['Confidence'] = 100 train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70)) train['diff'] = abs(train['FVC'] - train['FVC_pred']) train['delta'] = train['diff'].apply(lambda x: min(x, 1000)) train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped']) score = train['score'].mean() print(score) import scipy as sp def loss_func(weight, row): confidence = weight sigma_clipped = max(confidence, 70) diff = abs(row['FVC'] - row['FVC_pred']) delta = min(diff, 1000) score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped) return -score results = [] tk0 = tqdm(train.iterrows(), total=len(train)) for _, row in tk0: loss_partial = partial(loss_func, row=row) weight = [100] #bounds = [(70, 100)] #result = sp.optimize.minimize(loss_partial, weight, method='SLSQP', bounds=bounds) result = sp.optimize.minimize(loss_partial, weight, method='SLSQP') x = result['x'] results.append(x[0]) # optimized score train['Confidence'] = results train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70)) train['diff'] = abs(train['FVC'] - train['FVC_pred']) train['delta'] = train['diff'].apply(lambda x: min(x, 1000)) train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped']) score = train['score'].mean() print(score) TARGET = 'Confidence' target = train[TARGET] test[TARGET] = np.nan # features cat_features = ['Sex', 'SmokingStatus'] num_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)] features = num_features + cat_features drop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred'] features = [c for c in features if c not in drop_features] ridge_param = { 'alpha': 1, 'random_state': SEED, } oof, predictions = run_kfold_ridge(ridge_param, train, test, folds, features, target, n_fold=N_FOLD) train['Confidence'] = oof train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70)) train['diff'] = abs(train['FVC'] - train['FVC_pred']) train['delta'] = train['diff'].apply(lambda x: min(x, 1000)) train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped']) score = train['score'].mean() print(score) def lb_metric(train): train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70)) train['diff'] = abs(train['FVC'] - train['FVC_pred']) train['delta'] = train['diff'].apply(lambda x: min(x, 1000)) train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped']) score = train['score'].mean() return score score = lb_metric(train) logger.info(f'Local Score: {score}') test['Confidence'] = predictions sub = submission.drop(columns=['FVC', 'Confidence']).merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week') sub.columns = submission.columns submission.to_csv('submission.csv', index=False) #sub.head()