{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hi everyone,\n\nHere, we propose a little improvement of the kaggle script for Giba : https://www.kaggle.com/titericz/0-525-tabular-xgboost-gpu-fft-gpu-cuml-fast about the Kaggle competition : \"Rainforest Connection Species Audio Detection\"\n\nThe code is nearly the same as Giba but we decid to focus the Fourrier transformed only when specy songs are mentioned, and we remove too low frequency and too high frequency for the classification. \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport gc\nimport time\nimport xgboost as xgb\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom scipy.stats import rankdata\nimport soundfile as sf\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score, label_ranking_average_precision_score\nimport cuml as cm\nimport cupy as cp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Loading files"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainfiles = glob.glob( '../input/rfcx-species-audio-detection/train/*.flac' )\ntestfiles = glob.glob( '../input/rfcx-species-audio-detection/test/*.flac' )\nprint('Size of training files :', len(trainfiles))\nprint('Size of training files :', len(testfiles))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. Separation in training set : True Positive (TP) and False Positive (FP)"},{"metadata":{"trusted":true},"cell_type":"code","source":"Table_train_tp = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\nTable_train_fp = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\nprint('Number of True positive songs in training Data:',Table_train_tp.shape)\nprint('Number of False positive songs in training Data:',Table_train_fp.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(Table_train_tp.head())\ndisplay(Table_train_fp.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Table_train_tp.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Table_train_fp.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We observe that the lowest frequency of specy songs is located at ~90 Hz and the highest at ~13700 Hz. "},{"metadata":{},"cell_type":"markdown","source":"3. Example of processing on 1 file : "},{"metadata":{"trusted":true},"cell_type":"code","source":"# File Example : \nfile_test='/kaggle/input/rfcx-species-audio-detection/train/c12e0a62b.flac'\n#file_test='/kaggle/input/rfcx-species-audio-detection/train/03b96f209.flac'\n#file_test='/kaggle/input/rfcx-species-audio-detection/train/00204008d.flac'\n#file_test='/kaggle/input/rfcx-species-audio-detection/train/003bec244.flac'\n#file_test='/kaggle/input/rfcx-species-audio-detection/train/006ab765f.flac'\n#file_test='/kaggle/input/rfcx-species-audio-detection/train/003b04435.flac'\n\nx , sr = librosa.load(file_test)\n\n\n# Plot file \nplt.figure(figsize=(14, 5))\nlibrosa.display.waveplot(x, sr=sr)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Audio representation of file :"},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(file_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data, samplerate = sf.read(file_test)\n\nprint('The samplerate is ',samplerate,' samples per second (=',samplerate,'Hz)'  )\nprint('Data is composed by ',samplerate*60,' samples (the file audio lasts 1 min)')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We're looking for the detected bird periods in file "},{"metadata":{"trusted":true},"cell_type":"code","source":"Name_file=file_test.split('/')[-1:][0].split('.')[0]\n\n# We look at the species observed in the file test.\nTable_file_species=Table_train_tp[Table_train_tp['recording_id']==Name_file].reset_index(drop=True)\n\ndisplay(Table_file_species)\n\n#We create the x-axis for the data \nN = data.shape[0] # number of signal points\nT = 1.0/samplerate #(samplerate = sample spacing)\nx = np.linspace(0.0, N*T, N)\n\n#We create the y-axis for the data \ny = data #Data\n\n\n#We search in data when species appeared\nindex_birds=[]\nfor n in range(0,Table_file_species.shape[0]):\n    index_bird= np.where((x > Table_file_species['t_min'][n]) & (x<Table_file_species['t_max'][n]))[0]\n    index_birds=np.concatenate([index_birds, index_bird])\n    \nindex_birds=index_birds.astype(int)\n\ny_birds= y.copy()*0\ny_birds[index_birds]=y[index_birds]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We represent the signal and the specy song periods."},{"metadata":{"trusted":true},"cell_type":"code","source":"# We represent the original data (in blue) and locations of specy songs (in red)\nplt.figure(figsize=(20, 5))\nplt.plot(x, y,label=\"Audio retranscription\")\nplt.plot(x, y_birds,'r',label=\"Period of bird songs\")\nplt.xlabel('Time (in s)')\nplt.ylabel('Amplitude')\nplt.legend(loc='upper right')\nplt.title('Audio file ')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then, we apply the Fourrier transform to transform signal in frequency domain and also, only during birds periods."},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom scipy.fft import fft, fftfreq, fftshift, ifft, rfft, irfft\n\n#--------------- TF and example  -----------#\n# Here, we transform data in frequency domain \n\n# we create the x_axis of FFT. \nxf = fftfreq(N, T)\nxf_final=xf[:(len(xf)//2)]\n\n# we tansform the original signal with FFT\nyf = fft(y)\nyf_final=np.abs(yf[:(len(xf)//2)])\n\n# we tansform the signal with only the specy songs with FFT\ny_birds = cp.array(y_birds)\nyf_birds = cp.abs(cp.fft.fft(y_birds))\nyf_birds_final=cp.asnumpy(yf_birds[:(len(xf)//2)])\n\n\n\n#--------------- Plot TF  -----------#\nplt.figure(figsize=(20, 5))\nplt.plot(xf_final,yf_final,label=\"for all points\")\nplt.plot(xf_final.reshape( (1000,1440) ).mean(axis=1),yf_final.reshape( (1000,1440) ).mean(axis=1),'k--', label=\"averaged points\")\nplt.xlabel('Freq (Hz)')\nplt.ylabel('Amplitude')\nplt.title('FFT of original data')\nplt.legend(loc='upper right')\nplt.grid()\nplt.show()\n\nplt.figure(figsize=(20, 5))\nplt.plot(xf_final,yf_birds_final,'r',label=\"for all points\")\nplt.plot(xf_final.reshape( (1000,1440) ).mean(axis=1),yf_birds_final.reshape( (1000,1440) ).mean(axis=1),'k--', label=\"averaged points\")\nplt.xlabel('Freq (Hz)')\nplt.ylabel('Amplitude')\nplt.title('FFT with only songs detected')\nplt.legend(loc='upper right')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"On the figure, we can observe : \n- the second representation (in red) have the quite same shape than the original data but with lower amplitude. It is due to the fact that we take only few parts of the original data. \n- a large pic in x=0 represent the mean of original signal. \n- the FFT is represented only for frequency in 0 to Fe/2 (Fe = 48000) because the FFT have the same shape (but with mirroiring effect for Frequency between Fe/2 to Fe)"},{"metadata":{},"cell_type":"markdown","source":"Function creation to extract TFF from file but only when birds are detected\n\nIn this function, we decide to remove too high frequencies (i.e. higher than 14000 Hz) and too low frequencies (i.e. lower than 90 Hz) "},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_fft_for_birds(fn,Table_train):\n    \n    data, samplerate = sf.read(fn)\n    data = cp.array(data)\n    \n\n    Name_file=fn.split('/')[-1:][0].split('.')[0]\n\n    Table_file_species=Table_train[Table_train['recording_id']==Name_file].reset_index(drop=True)\n\n    # number of signal points\n    N = data.shape[0]\n    \n    # sample spacing\n    T = 1.0/samplerate\n    x = np.linspace(0.0, N*T, N)\n\n    index_birds=[]\n    for n in range(0,Table_file_species.shape[0]):\n        index_bird= np.where((x > Table_file_species['t_min'][n]) & (x<Table_file_species['t_max'][n]))[0]\n        index_birds=np.concatenate([index_birds, index_bird])\n\n    index_birds=index_birds.astype(int)\n\n    y_birds= data.copy()*0\n    y_birds[index_birds]=y[index_birds]\n    \n    \n    #--------------- TF and example  -----------#\n    xf_final = fftfreq(N, T)[:(len(xf)//2)]\n    \n    tff_data_final= cp.abs(cp.fft.fft(data)[:(len(xf)//2)])\n\n    tff_y_birds_final = cp.abs(cp.fft.fft(y_birds)[:(len(xf)//2)])\n\n    ## We remove too low frequencies (90 Hz) and too high frequencies(14000 Hz) :\n    xf_too_low= np.where(xf_final < 90)[0]\n    xf_too_high= np.where(xf_final > 14000)[0]\n\n    tff_y_birds_final[xf_too_low]=0\n    tff_y_birds_final[xf_too_high]=0\n    \n    \n    return (cp.asnumpy(xf_final.reshape( (1000,1440) ).mean(axis=1)), cp.asnumpy(tff_y_birds_final.reshape( (1000,1440) ).mean(axis=1)), cp.asnumpy(tff_data_final.reshape( (1000,1440) ).mean(axis=1))  ) #We take the average values every 1000 features.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test function \"extract_fft_for_birds\""},{"metadata":{"trusted":true},"cell_type":"code","source":"#We verify our fonction on file_test :\nfn=file_test\n\nfrequency_test,data_test,data_test_origin=extract_fft_for_birds(fn,Table_train_tp)\n\nplt.figure(figsize=(20, 5))\nplt.plot(frequency_test,data_test_origin)\nplt.plot(frequency_test,data_test,'r')\nplt.title('FFT with only specy songs detected in file : '+ fn)\nplt.xlabel('Freq (Hz)')\nplt.ylabel('Amplitude')\nplt.legend(loc='upper right')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We run our function on Training Data : \n- first on True positive songs\n- secondy on False positive songs "},{"metadata":{"trusted":true},"cell_type":"code","source":"# This loop runs in 3min using cupy(GPU)\nFT = []\nfor fn in tqdm(Table_train_tp.recording_id.values):\n    FT.append( extract_fft_for_birds( '../input/rfcx-species-audio-detection/train/'+fn+'.flac',Table_train_tp ) )\nFT = np.stack(FT)\ngc.collect()\n\nFT.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This loop runs in 17min using cupy(GPU)\n\nFF = []\nfor fn in tqdm(Table_train_fp.recording_id.values):\n    FF.append( extract_fft_for_birds( '../input/rfcx-species-audio-detection/train/'+fn+'.flac',Table_train_fp ) )\nFF = np.stack(FF)\ngc.collect()\n\nFF.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Combine True Positives and False Positives\n\nTRAIN_ALL = np.vstack( (FT, FF) )\n\ndel FT, FF\ngc.collect()\nTRAIN_ALL.shape\n\nTRAIN= TRAIN_ALL[:,1,:]\ndisplay(TRAIN.shape)\n\nTRAIN_Original= TRAIN_ALL[:,2,:]\ndisplay(TRAIN_Original.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = Table_train_tp[['recording_id','species_id']].copy()\ntf = Table_train_fp[['recording_id','species_id']].copy()\ntf['species_id'] = -1\n\nTRAIN_TAB = pd.concat( (tt, tf) )\n\nfor i in range(24):\n    TRAIN_TAB['s'+str(i)] = 0\n    TRAIN_TAB.loc[TRAIN_TAB.species_id==i,'s'+str(i)] = 1\n\nTRAIN_TAB=TRAIN_TAB.reset_index(drop=True)\nTRAIN_TAB.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_fft_test(fn):\n    data, samplerate = sf.read(fn)\n\n    N = data.shape[0] # number of signal points\n    T = 1.0/samplerate # sample spacing\n    \n    data = cp.array(data)\n    \n    #--------------- TF and example  -----------#\n    xf_final = fftfreq(N, T)[:(len(xf)//2)]\n\n    tff_y_final = cp.abs(cp.fft.fft(data)[:(len(xf)//2)])\n    \n    xf_too_low= np.where(xf_final < 90)[0]\n    xf_too_high= np.where(xf_final > 14000)[0]\n\n    tff_y_final[xf_too_low]=0\n    tff_y_final[xf_too_high]=0\n    \n\n    return (cp.asnumpy(xf_final.reshape( (1000,1440) ).mean(axis=1)), cp.asnumpy(tff_y_final.reshape( (1000,1440) ).mean(axis=1)) ) #We take the average values every 1000 features.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This loop runs in 3min using cupy(GPU)\n\nTEST_ALL = []\nfor fn in tqdm(testfiles):\n    TEST_ALL.append( extract_fft_test(fn) )\nTEST_ALL = np.stack(TEST_ALL)\ngc.collect()\n\nTEST_ALL.shape\nTEST= TEST_ALL[:,1,:]\ndisplay(TEST.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nstd = StandardScaler()\nstd.fit( np.vstack((TRAIN,TEST)) )\n\nTRAIN = std.transform(TRAIN)\nTEST  = std.transform(TEST)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Here we test 4 differents models XGBclassifier, Logistic Regression, SVC and KNN.\nfrom cuml.linear_model import MBSGDClassifier\n\nsub = pd.DataFrame({'recording_id': [f.split('/')[-1].split('.')[0] for f in testfiles] })\nsub1=sub.copy()\nsub2=sub.copy()\nsub3=sub.copy()\nsub4=sub.copy()\n\ngkf = GroupKFold(5)\n\nSCORE = []\nSCORE1=[]\nSCORE2=[]\nSCORE3=[]\nSCORE4=[]\n\nTRAIN_TAB1=TRAIN_TAB.copy()\nTRAIN_TAB2=TRAIN_TAB.copy()\nTRAIN_TAB3=TRAIN_TAB.copy()\nTRAIN_TAB4=TRAIN_TAB.copy()\n\ngroups = TRAIN_TAB['recording_id'].values\nfor tgt in range(0,24):\n    starttime = time.time()\n    target = TRAIN_TAB['s'+str(tgt)].values\n\n    ytrain = np.zeros(TRAIN.shape[0])\n    ytest = np.zeros(TEST.shape[0])\n    \n    ytrain1=ytrain.copy()\n    ytrain2=ytrain.copy()\n    ytrain3=ytrain.copy()\n    ytrain4=ytrain.copy()\n    \n    ytest1=ytest.copy()\n    ytest2=ytest.copy()\n    ytest3=ytest.copy()\n    ytest4=ytest.copy()\n    for ind_train, ind_valid in gkf.split( TRAIN, target, groups ):\n        \n        # Define 4 models\n        model1 = xgb.XGBClassifier(n_estimators=1000,\n                                   max_depth=4,\n                                   learning_rate=0.05,\n                                   verbosity=0,\n                                   objective='binary:logistic',\n                                   subsample=0.95,\n                                   colsample_bytree=0.95,\n                                   random_state=2021,\n                                   tree_method='gpu_hist',\n                                   predictor='gpu_predictor',\n                                   n_jobs=2,\n                                   scale_pos_weight = np.sum(target==0) / np.sum(target==1),\n                                  )\n        model2 = cm.LogisticRegression( C=1, max_iter=5000 )\n        #model3 = cm.svm.SVC(C=1.0, class_weight='balanced', probability=True, kernel='rbf', gamma='scale')\n        model3 = cm.ensemble.RandomForestClassifier(n_estimators=1000)\n        model4 = cm.neighbors.KNeighborsClassifier(n_neighbors=100)\n\n        # Train using GPUs\n        model1.fit( X=TRAIN[ind_train], y=target[ind_train], eval_set=[(TRAIN[ind_valid], target[ind_valid])], eval_metric='auc', early_stopping_rounds=30, verbose=False )\n        model2.fit( TRAIN[ind_train], target[ind_train] )\n        model3.fit( TRAIN[ind_train].astype(np.float32), target[ind_train] )\n        model4.fit( TRAIN[ind_train], target[ind_train] )\n        \n        # Predict valid and test sets\n        yvalid1 = model1.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid2 = model2.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid3 = model3.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid4 = model4.predict_proba(TRAIN[ind_valid])[:,1]\n        ytest1 = model1.predict_proba(TEST)[:,1]\n        ytest2 = model2.predict_proba(TEST)[:,1]\n        ytest3 = model3.predict_proba(TEST)[:,1]\n        ytest4 = model4.predict_proba(TEST)[:,1]\n        \n        #Rank predictions\n        SZ = len(ind_valid) + len(ytest1)\n        yvalid1 = rankdata( np.concatenate((yvalid1,ytest1)) )[:len(ind_valid)] / SZ\n        yvalid2 = rankdata( np.concatenate((yvalid2,ytest2)) )[:len(ind_valid)] / SZ\n        yvalid3 = rankdata( np.concatenate((yvalid3,ytest3)) )[:len(ind_valid)] / SZ\n        yvalid4 = rankdata( np.concatenate((yvalid4,ytest4)) )[:len(ind_valid)] / SZ\n        ytest1 = rankdata( np.concatenate((yvalid1,ytest1)) )[len(ind_valid):] / SZ\n        ytest2 = rankdata( np.concatenate((yvalid2,ytest2)) )[len(ind_valid):] / SZ\n        ytest3 = rankdata( np.concatenate((yvalid3,ytest3)) )[len(ind_valid):] / SZ\n        ytest4 = rankdata( np.concatenate((yvalid4,ytest4)) )[len(ind_valid):] / SZ\n        \n        #Weighted average models\n        \n        ytrain1[ind_valid] =  yvalid1\n        ytest1            +=  ytest1 / (1.*5)\n        \n        ytrain2[ind_valid] =  yvalid2\n        ytest2            +=  ytest2 / (1.*5)\n        \n        ytrain3[ind_valid] = yvalid3\n        ytest3            += ytest3 / (1.*5)\n        \n        ytrain4[ind_valid] = yvalid4\n        ytest4            += ytest4 / (1.*5)\n        \n        ytrain[ind_valid] = (0.6*yvalid1 +0.2*yvalid2+ 0.1*yvalid3 +0.1*yvalid4)/4\n        ytest += (0.6*ytest1 +0.2*ytest2+ 0.1*ytest3 +0.1*ytest4) / (4.*5)\n\n    score1 = roc_auc_score(target, ytrain1)\n    print( 'Target AUC M1', tgt, score1, time.time()-starttime )\n    SCORE1.append(score1)\n    \n    score2 = roc_auc_score(target, ytrain2)\n    print( 'Target AUC M2', tgt, score2, time.time()-starttime )\n    SCORE2.append(score2)\n    \n    score3 = roc_auc_score(target, ytrain3)\n    print( 'Target AUC M3', tgt, score3, time.time()-starttime )\n    SCORE3.append(score3)\n    \n    score4 = roc_auc_score(target, ytrain4)\n    print( 'Target AUC M4', tgt, score4, time.time()-starttime )\n    SCORE4.append(score4)\n    print( '--------------------------------------')\n    \n    TRAIN_TAB['y'+str(tgt)] = ytrain\n    sub['s'+str(tgt)] = ytest\n    \n    TRAIN_TAB1['y'+str(tgt)] = ytrain1\n    sub1['s'+str(tgt)] = ytest1\n    \n    TRAIN_TAB2['y'+str(tgt)] = ytrain2\n    sub2['s'+str(tgt)] = ytest2\n    \n    TRAIN_TAB3['y'+str(tgt)] = ytrain3\n    sub3['s'+str(tgt)] = ytest3\n\n    TRAIN_TAB4['y'+str(tgt)] = ytrain4\n    sub4['s'+str(tgt)] = ytest4\n    \nprint('Overall Score M1:', np.mean(SCORE1) )\nprint('Overall Score M2:', np.mean(SCORE2) )\nprint('Overall Score M3:', np.mean(SCORE3) )\nprint('Overall Score M4:', np.mean(SCORE4) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\npd.set_option(\"display.max_rows\", 50,'display.max_columns', 50)\ndisplay(TRAIN_TAB[0:50])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Comparison between Observed and probabilities on Training Data example.\n\nimport random\nTRAIN_TAB1_true_species=TRAIN_TAB1[TRAIN_TAB1['species_id']>-1]\nTRAIN_TAB2_true_species=TRAIN_TAB2[TRAIN_TAB2['species_id']>-1]\nTRAIN_TAB3_true_species=TRAIN_TAB3[TRAIN_TAB3['species_id']>-1]\nTRAIN_TAB4_true_species=TRAIN_TAB4[TRAIN_TAB4['species_id']>-1]\n\nrd_index=random.randint(0, len(TRAIN_TAB1_true_species))\n\nTRAIN_TAB1_test=TRAIN_TAB1_true_species.iloc[rd_index,:]\nTRAIN_TAB2_test=TRAIN_TAB2_true_species.iloc[rd_index,:]\nTRAIN_TAB3_test=TRAIN_TAB3_true_species.iloc[rd_index,:]\nTRAIN_TAB4_test=TRAIN_TAB4_true_species.iloc[rd_index,:]\n\n#print(TRAIN_TAB1_test)\n\nplt.figure(figsize=(20, 5))\nplt.plot(range(0,24),TRAIN_TAB1_test[2:26],'ro')\nplt.plot(range(0,24),TRAIN_TAB1_test[26:51],'b--')\nplt.plot(range(0,24),TRAIN_TAB2_test[26:51],'g--')\nplt.plot(range(0,24),TRAIN_TAB3_test[26:51],'k--')\nplt.plot(range(0,24),TRAIN_TAB4_test[26:51],'y--')\nplt.xlabel('Species')\nplt.ylabel('Proba')\nplt.legend(loc='upper right')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"XGBClassifier is the best model. We keep it for the submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_final=sub1.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Final Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_final.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}