{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":10380805,"sourceType":"datasetVersion","datasetId":6409825}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"* Here's Part 1 before the Meme Stock Craze\n  \nhttps://www.kaggle.com/code/jazivxt/the-market-is-reactive\n\n* Here's Part 2 the Speculation Pump on Efficiency and Crypto\n\n![DOGE](https://upload.wikimedia.org/wikipedia/commons/f/fb/Musk_DOGE_logo.jpg)","metadata":{}},{"cell_type":"code","source":"#Trained on TPU VM v3-8 Accelerator, Internet On Required Temporarily\n#!pip3 install polars\n#!pip3 install xgboost","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn import *\nimport polars as pl\nimport gc; gc.enable()\nimport kaggle_evaluation.jane_street_inference_server\n\np = '/kaggle/input/jane-street-real-time-market-data-forecasting/'\nfeatures = pd.read_csv(p+'features.csv')\n#responders = pd.read_csv(p+'responders.csv')\n#sub = pd.read_csv(p+'sample_submission.csv')\n#lags = pd.read_parquet(p+'lags.parquet/date_id=0/part-0.parquet')\n#test = pd.read_parquet(p+'test.parquet/date_id=0/part-0.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:48:43.487827Z","iopub.execute_input":"2025-01-05T21:48:43.488250Z","iopub.status.idle":"2025-01-05T21:48:45.856552Z","shell.execute_reply.started":"2025-01-05T21:48:43.488212Z","shell.execute_reply":"2025-01-05T21:48:45.855397Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Features","metadata":{}},{"cell_type":"code","source":"df = features[['tag_'+str(i) for i in range(17)]]\ndf = pd.get_dummies(df).idxmax(1)\nfeatures['tags'] = df.values\ndgroups = {}\nfor f, t in features[['feature','tags']].values:\n    if t in dgroups:\n        dgroups[t].append(f)\n    else:\n        dgroups[t] = [f]\n#print(dgroups)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:48:50.498967Z","iopub.execute_input":"2025-01-05T21:48:50.499385Z","iopub.status.idle":"2025-01-05T21:48:50.528626Z","shell.execute_reply.started":"2025-01-05T21:48:50.499351Z","shell.execute_reply":"2025-01-05T21:48:50.527413Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"#train = pd.read_parquet(p+'train.parquet/partition_id=0/part-0.parquet')\n#print(0, train.shape)\n#for i in range(1,10):\n#    train = pd.concat([train, pd.read_parquet(p+'train.parquet/partition_id=' + str(i) + '/part-0.parquet')], axis=0)\n#    print(i, train.shape)\n# 9 (47_127_338, 92)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Lags","metadata":{}},{"cell_type":"code","source":"#lags = train[['date_id','time_id','symbol_id']+['responder_' + str(i) for i in range(9)]]\n#lags['date_id'] = lags['date_id'] + 1\n#lags.columns = ['date_id','time_id','symbol_id']+['responder_' + str(i) + '_lag_1' for i in range(9)]\n#train = pd.merge(train, lags, how='left', on=['date_id','time_id','symbol_id'])\n#train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Avoid Overfits","metadata":{}},{"cell_type":"code","source":"#for i in range(9):\n#    train = train[train['responder_' + str(i) + '_lag_1'] != train['responder_6']]\n#    print(i, train.shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/code/yunsuxiaozi/js-ridge-baseline?scriptVersionId=202739388\ndef custom_metric(y_true,y_pred,weight):\n    weighted_r2=1-(np.sum(weight*(y_true-y_pred)**2)/np.sum(weight*y_true**2))\n    return weighted_r2\n\n#XGBoost\ndef xwr(preds, dtrain):\n    y = dtrain.get_label()\n    w = dtrain.get_weight()\n    r = custom_metric(y,preds,w)\n    return 'WR2', r","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:48:56.054865Z","iopub.execute_input":"2025-01-05T21:48:56.055252Z","iopub.status.idle":"2025-01-05T21:48:56.061560Z","shell.execute_reply.started":"2025-01-05T21:48:56.055221Z","shell.execute_reply":"2025-01-05T21:48:56.060357Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Validation Sets","metadata":{}},{"cell_type":"code","source":"#dftrain = []\n#dfval = []\n#for s in train.symbol_id.unique():\n#    df = train[train['symbol_id']==s].reset_index(drop=True)\n#    df = df.tail(1_000_000)\n#    split = int(len(df) * 0.3) #Reversed the Split so we keep latest for training\n#    dftrain.append(df[split:])\n#    dfval.append(df[:split])\n\n#dftrain = pd.concat(dftrain, axis=0).reset_index(drop=True)\n#dfval = pd.concat(dfval, axis=0).reset_index(drop=True) #Randomize after\n#dftrain.shape, dfval.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#del train\n#gc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessed Means","metadata":{}},{"cell_type":"code","source":"#means = train.mean().to_dict()\nmeans = {'feature_00': 0.5738325119018555, 'feature_01': 0.010191998444497585,\n 'feature_02': 0.5731021165847778, 'feature_03': 0.5727452635765076,\n 'feature_04': -0.00085223134374246, 'feature_05': -0.03837984800338745,\n 'feature_06': -0.005057959817349911, 'feature_07': -0.01383929792791605,\n 'feature_08': 0.08246718347072601, 'feature_09': 32.76299157826398,\n 'feature_10': 4.875799350262474, 'feature_11': 170.6931200527388,\n 'feature_12': -0.038973260670900345, 'feature_13': -0.026642438024282455,\n 'feature_14': -0.04121825471520424, 'feature_15': -0.2512187361717224,\n 'feature_16': -0.20489005744457245, 'feature_17': -0.2428077757358551,\n 'feature_18': -0.008926504291594028, 'feature_19': -0.0318937748670578,\n 'feature_20': -0.2097044736146927, 'feature_21': 0.037686582654714584,\n 'feature_22': 0.06790614873170853, 'feature_23': 0.07220280170440674,\n 'feature_24': 0.57228684425354, 'feature_25': 0.0925765112042427,\n 'feature_26': -0.04924396425485611, 'feature_27': -0.23443453013896942,\n 'feature_28': -0.1555429995059967, 'feature_29': -0.2153135985136032,\n 'feature_30': -0.25759994983673096, 'feature_31': 0.036165494471788406,\n 'feature_32': 0.42054930329322815, 'feature_33': -0.015208646655082703,\n 'feature_34': 0.4469282031059265, 'feature_35': 0.44508907198905945,\n 'feature_36': 0.0018062674207612872, 'feature_37': 0.037253618240356445,\n 'feature_38': 0.04561939463019371, 'feature_39': 0.03008369728922844,\n 'feature_40': 0.015741169452667236, 'feature_41': 0.04192369431257248,\n 'feature_42': -0.06303924322128296, 'feature_43': -0.05390552431344986,\n 'feature_44': -0.08035079389810562, 'feature_45': 0.02337714470922947,\n 'feature_46': -0.04295474663376808, 'feature_47': -0.01057121530175209,\n 'feature_48': -0.0017195119289681315, 'feature_49': -0.00401264289394021,\n 'feature_50': -0.07570816576480865, 'feature_51': -0.0975748673081398,\n 'feature_52': -0.10389170795679092, 'feature_53': 0.07496077567338943,\n 'feature_54': 0.048526789993047714, 'feature_55': 0.08681382983922958,\n 'feature_56': -0.028327863663434982, 'feature_57': 0.027874287217855453,\n 'feature_58': -0.015197671949863434, 'feature_59': -0.0025453190319240093,\n 'feature_60': -0.0066503798589110374, 'feature_61': 0.01238778792321682,\n 'feature_62': -0.17957818508148193, 'feature_63': -0.14525409042835236,\n 'feature_64': -0.166556254029274, 'feature_65': -0.0431523360311985,\n 'feature_66': -0.05011758580803871, 'feature_67': -0.04260352998971939,\n 'feature_68': -0.023995323106646538, 'feature_69': -0.04050201177597046,\n 'feature_70': -0.032991524785757065, 'feature_71': -0.02115715853869915,\n 'feature_72': -0.03668196499347687, 'feature_73': -0.01655886322259903,\n 'feature_74': -0.023102542385458946, 'feature_75': -0.0022374617401510477,\n 'feature_76': -0.00802241824567318, 'feature_77': -0.006589713506400585,\n 'feature_78': -0.012558902613818645, 'responder_0': -0.0015451457584276795,\n 'responder_1': -0.0008655996643938124, 'responder_2': -0.00014196030679158866,\n 'responder_3': -0.01638009212911129, 'responder_4': -0.01244197878986597,\n 'responder_5': -0.016624215990304947, 'responder_6': -0.0021406463347375393,\n 'responder_7': 0.001476110192015767, 'responder_8': -0.0011136182583868504,}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:49:01.771726Z","iopub.execute_input":"2025-01-05T21:49:01.772144Z","iopub.status.idle":"2025-01-05T21:49:01.785984Z","shell.execute_reply.started":"2025-01-05T21:49:01.772109Z","shell.execute_reply":"2025-01-05T21:49:01.784268Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Feature Sets","metadata":{}},{"cell_type":"code","source":"def getFeatures(df):\n    print(df.shape)\n    df['null_count'] = df.isnull().sum(axis=1)\n    for g in dgroups:\n        df[g + '_nulls'] = df[dgroups[g]].isnull().sum(axis=1)\n        df[g + '_sums'] = df[dgroups[g]].sum(axis=1)\n    df.fillna(means, inplace=True) #Means Only\n    df.fillna(0, inplace=True) #All Others\n    print(df.shape)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:49:06.269762Z","iopub.execute_input":"2025-01-05T21:49:06.270191Z","iopub.status.idle":"2025-01-05T21:49:06.277011Z","shell.execute_reply.started":"2025-01-05T21:49:06.270156Z","shell.execute_reply":"2025-01-05T21:49:06.275866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#dftrain = getFeatures(dftrain)\n#dfval = getFeatures(dfval)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#REMOVED: 'date_id', 'time_id', 'weight', 'responder_0', 'responder_1', 'responder_2', 'responder_3', 'responder_4', 'responder_5', 'responder_6', 'responder_7', 'responder_8',\ncols = ['symbol_id', 'null_count', 'tag_2_nulls', 'tag_2_sums', 'tag_3_nulls', 'tag_3_sums', 'tag_1_nulls', 'tag_1_sums', 'tag_9_nulls', 'tag_9_sums', 'tag_6_nulls', 'tag_6_sums', 'tag_4_nulls', 'tag_4_sums', 'tag_5_nulls', 'tag_5_sums', 'tag_0_nulls', 'tag_0_sums', 'tag_15_nulls', 'tag_15_sums', 'tag_8_nulls', 'tag_8_sums']\ncols += ['feature_' + str(i).zfill(2) for i in range(79)]\ncols += ['responder_' + str(i) + '_lag_1' for i in range(9)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:49:09.630161Z","iopub.execute_input":"2025-01-05T21:49:09.631097Z","iopub.status.idle":"2025-01-05T21:49:09.636735Z","shell.execute_reply.started":"2025-01-05T21:49:09.631057Z","shell.execute_reply":"2025-01-05T21:49:09.635713Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Models","metadata":{}},{"cell_type":"code","source":"#import pickle\n\n#ardr = linear_model.ARDRegression()\n#elcv = linear_model.ElasticNetCV(random_state=99, n_jobs=-1)\n#lscv = linear_model.LassoCV(random_state=99, n_jobs=-1)\n#ridg = linear_model.Ridge(max_iter=5000, tol=0.0001, random_state=99)\n#models = [ardr, elcv, lscv, ridg]\n#names = ['ardr', 'elcv', 'lscv', 'ridg']\n\n#for i in range(len(models)):\n#    models[i] = models[i].fit(dftrain[cols], dftrain['responder_6'])\n#    preds = models[i].predict(dfval[cols]).clip(-5.0, 5.0)\n#    m = custom_metric(dfval['responder_6'], preds, dfval['weight'])\n#    print(names[i], m)\n#    #Save Model\n#    pickle.dump(models[i], open(names[i] + '.pkl', 'wb'))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle\n\nardr = pickle.load(open('/kaggle/input/jane-street-rtmdf-models/ardr.pkl', 'rb'))\nelcv = pickle.load(open('/kaggle/input/jane-street-rtmdf-models/elcv.pkl', 'rb'))\nlscv = pickle.load(open('/kaggle/input/jane-street-rtmdf-models/lscv.pkl', 'rb'))\nridg = pickle.load(open('/kaggle/input/jane-street-rtmdf-models/ridg.pkl', 'rb'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:49:17.169113Z","iopub.execute_input":"2025-01-05T21:49:17.170017Z","iopub.status.idle":"2025-01-05T21:49:17.192488Z","shell.execute_reply.started":"2025-01-05T21:49:17.169979Z","shell.execute_reply":"2025-01-05T21:49:17.191424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for m in [ardr, elcv, lscv, ridg]:\n    co = m.coef_\n    fe = pd.DataFrame({'Feature': cols, 'Importance': np.abs(co)})\n    fe = fe.sort_values('Importance', ascending=False)\n    fe.plot(x='Feature', y='Importance', kind='barh', figsize=(10, 6))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Make Predictions","metadata":{}},{"cell_type":"code","source":"def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    global models\n    global lags_\n    \n    test = test.to_pandas()\n    \n    if lags is not None:\n        lags_ = lags.to_pandas()\n        test = pd.merge(test, lags_, how='left', on=['date_id','time_id','symbol_id'])\n    else:\n        for c in ['responder_' + str(i) + '_lag_1' for i in range(9)]:\n            test[c] = 0.0\n\n    test = getFeatures(test)\n    test['responder_6'] = 0.0\n    test['responder_6'] = (ardr.predict(test[cols]) * 0.15) + (elcv.predict(test[cols]) * 0.30) + (lscv.predict(test[cols]) * 0.30) + (ridg.predict(test[cols]) * 0.25)\n    test['responder_6'] = test['responder_6'].clip(-5.0, 5.0) \n    predictions =  test[['row_id', 'responder_6']]\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:49:22.507553Z","iopub.execute_input":"2025-01-05T21:49:22.507971Z","iopub.status.idle":"2025-01-05T21:49:22.515726Z","shell.execute_reply.started":"2025-01-05T21:49:22.507937Z","shell.execute_reply":"2025-01-05T21:49:22.514436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_evaluation import jane_street_inference_server\nimport os\n\ninference_server = jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T21:49:25.987507Z","iopub.execute_input":"2025-01-05T21:49:25.988321Z","iopub.status.idle":"2025-01-05T21:49:26.608299Z","shell.execute_reply.started":"2025-01-05T21:49:25.988273Z","shell.execute_reply":"2025-01-05T21:49:26.607242Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ｈ𝐀𝑷𝑷𝓎 🇰𝗮𝘨𝘨🇱𝖎Ｎɢ 💯","metadata":{}}]}