{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81000,"databundleVersionId":8812083,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # plotting\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-20T15:38:20.300214Z","iopub.execute_input":"2024-06-20T15:38:20.300720Z","iopub.status.idle":"2024-06-20T15:38:21.725719Z","shell.execute_reply.started":"2024-06-20T15:38:20.300677Z","shell.execute_reply":"2024-06-20T15:38:21.724323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Calculating basic features","metadata":{}},{"cell_type":"code","source":"def load_data(crop: str, mode: str=\"train\"):\n    # note that years represent an offset from model spinup;\n    # soil co2 dataset has real year;\n    # 0-30 are days before sowing, 31-238 are days after sowing\n    #tasmax = pd.read_parquet(f\"/kaggle/input/the-future-crop-challenge/tasmax_{crop}_{mode}.parquet\")\n    #tasmin = pd.read_parquet(f\"/kaggle/input/the-future-crop-challenge/tasmin_{crop}_{mode}.parquet\")\n    tas = pd.read_parquet(f\"/kaggle/input/the-future-crop-challenge/tas_{crop}_{mode}.parquet\")\n    pr = pd.read_parquet(f\"/kaggle/input/the-future-crop-challenge/pr_{crop}_{mode}.parquet\")\n    rsds = pd.read_parquet(f\"/kaggle/input/the-future-crop-challenge/rsds_{crop}_{mode}.parquet\")\n    soil_co2 = pd.read_parquet(f\"/kaggle/input/the-future-crop-challenge/soil_co2_{crop}_{mode}.parquet\")\n    target = pd.read_parquet(f\"/kaggle/input/the-future-crop-challenge/{mode}_solutions_{crop}.parquet\") if mode == \"train\" else None\n    return {\n        #'tasmax': tasmax,\n        #'tasmin': tasmin,\n        'tas': tas,\n        'pr': pr,\n        'rsds': rsds,\n        'soil_co2': soil_co2,\n        'target': target,\n    }","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:38:24.327523Z","iopub.execute_input":"2024-06-20T15:38:24.328149Z","iopub.status.idle":"2024-06-20T15:38:24.336778Z","shell.execute_reply.started":"2024-06-20T15:38:24.328113Z","shell.execute_reply":"2024-06-20T15:38:24.335534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_simple_features(data_dict):\n    for variable in ['tas','pr','rsds']:\n        data_dict[variable][f'mean_{variable}'] = data_dict[variable].iloc[:,35:215].mean(axis=1)\n        data_dict[variable][f'min_{variable}'] = data_dict[variable].iloc[:,35:215].min(axis=1)\n        data_dict[variable][f'max_{variable}'] = data_dict[variable].iloc[:,35:215].max(axis=1)\n\n    data_dict['features'] = data_dict['soil_co2'][['year','crop','texture_class', 'co2', 'nitrogen']]\n\n    for variable in ['tas','pr','rsds']:\n        data_dict['features'] = data_dict['features'].join(data_dict[variable][[f'mean_{variable}', f'min_{variable}', f'max_{variable}']])\n\n    data_dict['features']['pr_sum'] = data_dict['pr'].iloc[:,35:215].sum(axis=1)\n    return data_dict['features']","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:38:24.520834Z","iopub.execute_input":"2024-06-20T15:38:24.521278Z","iopub.status.idle":"2024-06-20T15:38:24.531870Z","shell.execute_reply.started":"2024-06-20T15:38:24.521246Z","shell.execute_reply":"2024-06-20T15:38:24.530541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maize_train = load_data(\"maize\", \"train\")\nmaize_train.keys()","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:38:24.762390Z","iopub.execute_input":"2024-06-20T15:38:24.762828Z","iopub.status.idle":"2024-06-20T15:38:42.891539Z","shell.execute_reply.started":"2024-06-20T15:38:24.762791Z","shell.execute_reply":"2024-06-20T15:38:42.890260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maize_train['features'] = get_simple_features(maize_train)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:38:42.894328Z","iopub.execute_input":"2024-06-20T15:38:42.894743Z","iopub.status.idle":"2024-06-20T15:38:47.569626Z","shell.execute_reply.started":"2024-06-20T15:38:42.894710Z","shell.execute_reply":"2024-06-20T15:38:47.568241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maize_train['features']","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:38:47.571499Z","iopub.execute_input":"2024-06-20T15:38:47.571889Z","iopub.status.idle":"2024-06-20T15:38:47.615538Z","shell.execute_reply.started":"2024-06-20T15:38:47.571857Z","shell.execute_reply":"2024-06-20T15:38:47.614268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wheat_train = load_data(\"wheat\", \"train\")\nwheat_train.keys()","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:38:47.619156Z","iopub.execute_input":"2024-06-20T15:38:47.619528Z","iopub.status.idle":"2024-06-20T15:39:02.677648Z","shell.execute_reply.started":"2024-06-20T15:38:47.619498Z","shell.execute_reply":"2024-06-20T15:39:02.676457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wheat_train['features'] = get_simple_features(wheat_train)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:39:02.678904Z","iopub.execute_input":"2024-06-20T15:39:02.679275Z","iopub.status.idle":"2024-06-20T15:39:06.322592Z","shell.execute_reply.started":"2024-06-20T15:39:02.679243Z","shell.execute_reply":"2024-06-20T15:39:06.321290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wheat_train['features']","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:39:06.323961Z","iopub.execute_input":"2024-06-20T15:39:06.324331Z","iopub.status.idle":"2024-06-20T15:39:06.357054Z","shell.execute_reply.started":"2024-06-20T15:39:06.324302Z","shell.execute_reply":"2024-06-20T15:39:06.355757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Split data","metadata":{}},{"cell_type":"code","source":"x_train = pd.concat([maize_train['features'][maize_train['features']['year']<410].drop(columns=['year']),\n                     wheat_train['features'][wheat_train['features']['year']<410].drop(columns=['year'])])\nx_test = pd.concat([maize_train['features'][maize_train['features']['year']>=410].drop(columns=['year']),\n                    wheat_train['features'][wheat_train['features']['year']>=410].drop(columns=['year'])])","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:39:06.359068Z","iopub.execute_input":"2024-06-20T15:39:06.359546Z","iopub.status.idle":"2024-06-20T15:39:06.468100Z","shell.execute_reply.started":"2024-06-20T15:39:06.359504Z","shell.execute_reply":"2024-06-20T15:39:06.466806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Map crop column to a binary:","metadata":{}},{"cell_type":"code","source":"x_train['crop'] = x_train['crop'].map({'maize' : 0, 'wheat' : 1})\nx_test['crop'] = x_test['crop'].map({'maize' : 0, 'wheat' : 1})","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:39:06.469498Z","iopub.execute_input":"2024-06-20T15:39:06.470015Z","iopub.status.idle":"2024-06-20T15:39:06.563989Z","shell.execute_reply.started":"2024-06-20T15:39:06.469978Z","shell.execute_reply":"2024-06-20T15:39:06.562245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = x_train.join(pd.concat([maize_train['target'], wheat_train['target']]))['yield']\ny_test = x_test.join(pd.concat([maize_train['target'], wheat_train['target']]))['yield']","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:39:06.565382Z","iopub.execute_input":"2024-06-20T15:39:06.565741Z","iopub.status.idle":"2024-06-20T15:39:06.661500Z","shell.execute_reply.started":"2024-06-20T15:39:06.565709Z","shell.execute_reply":"2024-06-20T15:39:06.660164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train a model","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:42:51.049575Z","iopub.execute_input":"2024-06-20T15:42:51.050132Z","iopub.status.idle":"2024-06-20T15:42:51.056110Z","shell.execute_reply.started":"2024-06-20T15:42:51.050087Z","shell.execute_reply":"2024-06-20T15:42:51.054957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ExtraTreesRegressor(random_state=0, n_jobs=-1, max_depth=30, n_estimators=30)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:42:57.579057Z","iopub.execute_input":"2024-06-20T15:42:57.579538Z","iopub.status.idle":"2024-06-20T15:42:57.585163Z","shell.execute_reply.started":"2024-06-20T15:42:57.579500Z","shell.execute_reply":"2024-06-20T15:42:57.583871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:42:58.966858Z","iopub.execute_input":"2024-06-20T15:42:58.967432Z","iopub.status.idle":"2024-06-20T15:43:29.818036Z","shell.execute_reply.started":"2024-06-20T15:42:58.967391Z","shell.execute_reply":"2024-06-20T15:43:29.816801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:43:29.820308Z","iopub.execute_input":"2024-06-20T15:43:29.820690Z","iopub.status.idle":"2024-06-20T15:43:30.457515Z","shell.execute_reply.started":"2024-06-20T15:43:29.820658Z","shell.execute_reply":"2024-06-20T15:43:30.456083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_squared_error(y_test, y_pred, squared=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:43:30.459394Z","iopub.execute_input":"2024-06-20T15:43:30.459914Z","iopub.status.idle":"2024-06-20T15:43:30.472378Z","shell.execute_reply.started":"2024-06-20T15:43:30.459865Z","shell.execute_reply":"2024-06-20T15:43:30.471125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(pd.concat([x_train, x_test]), pd.concat([y_train, y_test]))","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:43:30.475204Z","iopub.execute_input":"2024-06-20T15:43:30.475685Z","iopub.status.idle":"2024-06-20T15:44:06.351503Z","shell.execute_reply.started":"2024-06-20T15:43:30.475638Z","shell.execute_reply":"2024-06-20T15:44:06.349690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Generate predictions for the test set","metadata":{}},{"cell_type":"code","source":"maize_test = load_data(\"maize\", \"test\")\nwheat_test = load_data(\"wheat\", \"test\")","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:44:12.336181Z","iopub.execute_input":"2024-06-20T15:44:12.336662Z","iopub.status.idle":"2024-06-20T15:45:33.990030Z","shell.execute_reply.started":"2024-06-20T15:44:12.336622Z","shell.execute_reply":"2024-06-20T15:45:33.988566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maize_test['features'] = get_simple_features(maize_test)\nwheat_test['features'] = get_simple_features(wheat_test)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:45:33.992410Z","iopub.execute_input":"2024-06-20T15:45:33.992826Z","iopub.status.idle":"2024-06-20T15:45:50.530184Z","shell.execute_reply.started":"2024-06-20T15:45:33.992793Z","shell.execute_reply":"2024-06-20T15:45:50.528291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_final = pd.concat([maize_test['features'].drop(columns=['year']),\n                     wheat_test['features'].drop(columns=['year'])])","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:45:50.531812Z","iopub.execute_input":"2024-06-20T15:45:50.532328Z","iopub.status.idle":"2024-06-20T15:45:50.673467Z","shell.execute_reply.started":"2024-06-20T15:45:50.532284Z","shell.execute_reply":"2024-06-20T15:45:50.672269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_final['crop'] = x_final['crop'].map({'maize' : 0, 'wheat' : 1})","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:45:50.676363Z","iopub.execute_input":"2024-06-20T15:45:50.677703Z","iopub.status.idle":"2024-06-20T15:45:50.852514Z","shell.execute_reply.started":"2024-06-20T15:45:50.677650Z","shell.execute_reply":"2024-06-20T15:45:50.850426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_pred = model.predict(x_final)","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:45:50.854471Z","iopub.execute_input":"2024-06-20T15:45:50.855308Z","iopub.status.idle":"2024-06-20T15:45:55.984380Z","shell.execute_reply.started":"2024-06-20T15:45:50.855258Z","shell.execute_reply":"2024-06-20T15:45:55.982804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_final['yield'] = final_pred","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:46:22.655222Z","iopub.execute_input":"2024-06-20T15:46:22.656404Z","iopub.status.idle":"2024-06-20T15:46:22.664898Z","shell.execute_reply.started":"2024-06-20T15:46:22.656354Z","shell.execute_reply":"2024-06-20T15:46:22.663359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_final['yield']","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:46:29.684191Z","iopub.execute_input":"2024-06-20T15:46:29.684655Z","iopub.status.idle":"2024-06-20T15:46:29.698376Z","shell.execute_reply.started":"2024-06-20T15:46:29.684619Z","shell.execute_reply":"2024-06-20T15:46:29.696694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_final[['yield']].to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-06-20T15:46:36.667899Z","iopub.execute_input":"2024-06-20T15:46:36.668465Z","iopub.status.idle":"2024-06-20T15:46:40.566186Z","shell.execute_reply.started":"2024-06-20T15:46:36.668423Z","shell.execute_reply":"2024-06-20T15:46:40.564846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}