{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Open Problems in Single Cell Perturbations - Kaggle Competition","metadata":{"_uuid":"57dad6d7-2015-4197-9798-276efcb7f704","_cell_guid":"601b07cf-bc1b-43de-95d6-5de7ff691625","trusted":true}},{"cell_type":"markdown","source":"## Import Necessary Libraries","metadata":{"_uuid":"f76a0f68-0e75-4228-86fa-6401391e8d65","_cell_guid":"050c6f5c-9bfa-4841-955e-ef55a53d75d9","trusted":true}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom lightgbm import LGBMRegressor\nfrom sklearn.preprocessing import LabelEncoder\nimport gc","metadata":{"_uuid":"f6ad3336-feba-4263-b4d2-9e2945d8996a","_cell_guid":"c259d2f1-53ca-44c0-8375-3db2e2bc6b08","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-16T03:29:30.986733Z","iopub.execute_input":"2023-09-16T03:29:30.987888Z","iopub.status.idle":"2023-09-16T03:29:30.993409Z","shell.execute_reply.started":"2023-09-16T03:29:30.987838Z","shell.execute_reply":"2023-09-16T03:29:30.992517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load and Preprocess Data","metadata":{"_uuid":"a3d9f6fc-9566-49fe-b594-ec33fa2df03f","_cell_guid":"dfd5590f-74c1-4f30-8a80-e68f31463c64","trusted":true}},{"cell_type":"code","source":"# Load the columns ['obs_id', 'gene', 'normalized_count']\ngene_expression_data = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/adata_train.parquet', columns=['obs_id', 'gene', 'normalized_count'])\n\n# Pivot the data to get a wide format\npivot_data = gene_expression_data.pivot(index='obs_id', columns='gene', values='normalized_count').reset_index()\n\n# Load metadata\nmetadata = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/adata_train.parquet', columns=['obs_id', 'cell_type', 'sm_name', 'SMILES', 'sm_lincs_id', 'control'])\n\n# Merge with metadata\ntrain_df = metadata.merge(pivot_data, on='obs_id')\n\n# Convert categorical columns to string type and encode\ncategorical_cols = ['cell_type', 'sm_name', 'SMILES', 'sm_lincs_id']\nfor col in categorical_cols:\n    train_df[col] = train_df[col].astype(str)\n\nlabel_encoders = {}\nfor col in categorical_cols:\n    le = LabelEncoder()\n    train_df[col] = le.fit_transform(train_df[col])\n    label_encoders[col] = le\n\n# Cleanup\ndel gene_expression_data, pivot_data, metadata\ngc.collect()\n\n# Concatenate processed chunks\ninitial_df = pd.concat(list_of_dataframes, axis=0)\n\n# Convert categorical columns to string type\ncategorical_cols = ['cell_type', 'sm_name', 'SMILES', 'sm_lincs_id']\nfor col in categorical_cols:\n    initial_df[col] = initial_df[col].astype(str)\n\n# Encoding categorical columns\nlabel_encoders = {}\nfor col in categorical_cols:\n    le = LabelEncoder()\n    initial_df[col] = le.fit_transform(initial_df[col])\n    label_encoders[col] = le\n\n# Now, load gene expression data in chunks and merge with the initial_df\ngenes = []\nfor chunk in pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/adata_train.parquet', chunksize=chunksize):\n    genes.extend([col for col in chunk.columns if col not in cols_to_load])\n    initial_df = pd.concat([initial_df, chunk[genes].astype('float32')], axis=1)\n    del chunk\n    gc.collect()\n\ntrain_df = initial_df\ndel initial_df\ngc.collect()","metadata":{"_uuid":"dce83459-d317-48bc-93f6-0993d19bb58f","_cell_guid":"dfe4002a-8fa7-4873-8d73-fbdb9f80d7a8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-16T03:29:31.000037Z","iopub.execute_input":"2023-09-16T03:29:31.000415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyarrow.parquet as pq\n\n# Check the columns in the parquet file\nfile_metadata = pq.read_metadata('/kaggle/input/open-problems-single-cell-perturbations/adata_train.parquet')\nall_columns = file_metadata.schema.names\n\nprint(all_columns)","metadata":{"_uuid":"e74a5528-edbc-4ab7-b30d-82e0514f65e1","_cell_guid":"edc816f9-c97b-4222-b90d-e9cb87253c1e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Training","metadata":{"_uuid":"c4e6ee7b-f3cc-4cbf-ba58-8960b1bb9e86","_cell_guid":"383f6280-a62c-4815-89ec-70c3b8308b28","trusted":true}},{"cell_type":"code","source":"# Splitting data for training individual models for each gene\nX_train, X_val, y_train, y_val = train_test_split(train_df.drop(columns=genes), train_df[genes], test_size=0.2, random_state=42)\n\n# Train individual models for each gene\nmodels = {}\nfor gene in genes:\n    model = LGBMRegressor()\n    model.fit(X_train, y_train[gene])\n    models[gene] = model\n    # Free up memory after each model training\n    gc.collect()\n\n# Validate models\nval_preds = {}\nfor gene, model in models.items():\n    val_preds[gene] = model.predict(X_val)\n\n# Calculate overall validation score\noverall_val_score = mean_squared_error(y_val, pd.DataFrame(val_preds))","metadata":{"_uuid":"01948534-6115-4be3-a7a4-72c2612382b9","_cell_guid":"a2a41f87-8c4a-4377-aeda-39130fc27171","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare the Test Data","metadata":{"_uuid":"f163f1aa-6775-4275-bb46-17fa1e95e123","_cell_guid":"612976cc-e9d5-43fc-8c95-f9f0e20b9a81","trusted":true}},{"cell_type":"code","source":"# Load the id_map data\nid_map = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/id_map.csv')\n\n# Ensure consistent data types before merging\nfor col in categorical_cols:\n    id_map[col] = id_map[col].astype(str)\n    train_df[col] = train_df[col].astype(str)\n\n# Merge the data\ntest_df = train_df.merge(id_map, on=categorical_cols, how='right')\n\n# Encoding the categorical columns using the label encoders from training data\nfor col, le in label_encoders.items():\n    test_df[col] = le.transform(test_df[col])","metadata":{"_uuid":"30357364-802f-4609-bcae-de3c1dce9f8b","_cell_guid":"5ec3f290-f0cf-4598-a9fa-c377b5c4ef86","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate Predictions for Submission","metadata":{"_uuid":"1ea620f2-9ead-41aa-9a62-333a08325755","_cell_guid":"71a73fe4-ab2a-4ae0-b31f-2eb73b6b3681","trusted":true}},{"cell_type":"code","source":"# Making predictions\npredictions = pd.DataFrame(index=test_df.index, columns=genes)\nfor gene, model in models.items():\n    predictions[gene] = model.predict(test_df)\n\n# Creating the submission dataframe\nsubmission = predictions.melt(id_vars=['id'], value_vars=genes, \n                              var_name='gene', value_name='predicted_expression')\nsubmission = submission.sort_values(by=['id', 'gene']).reset_index(drop=True)\n\n# Save the submission file\nsubmission.to_csv('submission.csv', index=False)","metadata":{"_uuid":"a44d9928-b371-46e3-82a2-a376ad531454","_cell_guid":"d7518f8f-1820-46cf-8a75-3bd79d29e716","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## End of Notebook","metadata":{"_uuid":"2b469e42-c85d-47c3-84e1-cfe800695fda","_cell_guid":"c604c2b1-1662-47da-bd4d-8e1286da95be","trusted":true}}]}