{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Hello again, Kagglers! Hope you will like my work","metadata":{}},{"cell_type":"markdown","source":"## [Dataset](https://www.kaggle.com/datasets/ilyaryabov/fasttext-model-for-google-ai4code) with a pretrained models\n## [Notebook](https://www.kaggle.com/ilyaryabov/fasttext-public-model-teaching) that describes how to create a fasttext model","metadata":{}},{"cell_type":"markdown","source":"# The following notebook describes an algorithm for sorting cells using cosine distance in vector space","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nimport fasttext\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.stem import WordNetLemmatizer\nfrom pathlib import Path\n\npd.options.display.max_rows = 100","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:04:39.228347Z","iopub.execute_input":"2022-05-23T19:04:39.228984Z","iopub.status.idle":"2022-05-23T19:04:41.121206Z","shell.execute_reply.started":"2022-05-23T19:04:39.228845Z","shell.execute_reply":"2022-05-23T19:04:41.12048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"src = '/kaggle/input/AI4Code/'\ndata_dir = Path('../input/AI4Code')\nfasttext_model = '/kaggle/input/fasttext-model-for-google-ai4code/model140000.bin'","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:04:48.146731Z","iopub.execute_input":"2022-05-23T19:04:48.147022Z","iopub.status.idle":"2022-05-23T19:04:48.151629Z","shell.execute_reply.started":"2022-05-23T19:04:48.146988Z","shell.execute_reply":"2022-05-23T19:04:48.150884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"src = '../input/AI4Code/'\ntrain_orders_df = pd.read_csv(src + 'train_orders.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:04:51.6062Z","iopub.execute_input":"2022-05-23T19:04:51.606718Z","iopub.status.idle":"2022-05-23T19:04:53.165363Z","shell.execute_reply.started":"2022-05-23T19:04:51.606683Z","shell.execute_reply":"2022-05-23T19:04:53.164415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer = WordNetLemmatizer()\n\ndef preprocess_text(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        #return document\n\n        # Lemmatization\n        tokens = document.split()\n        tokens = [stemmer.lemmatize(word) for word in tokens]\n        tokens = [word for word in tokens if len(word) > 3]\n\n        preprocessed_text = ' '.join(tokens)\n        return preprocessed_text\n\n    \ndef preprocess_df(df):\n    \"\"\"\n    This function is for processing sorce of notebook\n    returns preprocessed dataframe\n    \"\"\"\n    return [preprocess_text(message) for message in df.source]\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:04:53.167239Z","iopub.execute_input":"2022-05-23T19:04:53.167651Z","iopub.status.idle":"2022-05-23T19:04:53.178131Z","shell.execute_reply.started":"2022-05-23T19:04:53.167602Z","shell.execute_reply":"2022-05-23T19:04:53.177106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = fasttext.load_model(fasttext_model)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:04:53.449432Z","iopub.execute_input":"2022-05-23T19:04:53.449746Z","iopub.status.idle":"2022-05-23T19:05:08.406139Z","shell.execute_reply.started":"2022-05-23T19:04:53.44971Z","shell.execute_reply":"2022-05-23T19:05:08.404636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = paths_test = list((data_dir / 'test').glob('*.json'))\n#dfs = [ (pd.read_json(file)) for file in test_files]","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:05:08.40818Z","iopub.execute_input":"2022-05-23T19:05:08.408496Z","iopub.status.idle":"2022-05-23T19:05:08.425518Z","shell.execute_reply.started":"2022-05-23T19:05:08.408461Z","shell.execute_reply":"2022-05-23T19:05:08.424257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example of applying fasttext model:","metadata":{}},{"cell_type":"code","source":"df = pd.read_json(test_files[0])\ndf.source = df.source.apply(preprocess_text)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:05:08.426923Z","iopub.execute_input":"2022-05-23T19:05:08.427934Z","iopub.status.idle":"2022-05-23T19:05:10.62923Z","shell.execute_reply.started":"2022-05-23T19:05:08.427893Z","shell.execute_reply":"2022-05-23T19:05:10.62817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some useful functions:","metadata":{}},{"cell_type":"code","source":"def check(result, file):\n    \"\"\"\n    This function shows how notebook looks with predicted cell order\n    returns nothing\n    \"\"\"\n    notebook_df = pd.read_json(file,\n                                dtype={'cell_type': 'category', 'source': 'str'}\n                                ).rename_axis('cell_id')\n    cells = result\n    df = notebook_df.loc[cells]\n    display(df)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:05:10.630929Z","iopub.execute_input":"2022-05-23T19:05:10.631162Z","iopub.status.idle":"2022-05-23T19:05:10.637427Z","shell.execute_reply.started":"2022-05-23T19:05:10.631134Z","shell.execute_reply":"2022-05-23T19:05:10.636547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_notebook_from_train_orders(file):\n    \"\"\"\n    This function reads a notebook from i-th line of train_orders.csv with a correct cell order\n    \"\"\"\n    id_ = file.split('/')[-1][:-5]\n    _, cell_order = train_orders_df[train_orders_df.id == id_].values[0] #train_orders_df.iloc[i]\n    path = src + 'train/' + id_ +'.json'\n    cell_order = cell_order.split( )\n    #print(cell_order)\n    notebook_df = pd.read_json(\n                            path,\n                            dtype={'cell_type': 'category', 'source': 'str'}\n                            ).rename_axis('cell_id')\n    return notebook_df.loc[cell_order], cell_order # put cells in a correct cell order\n\n\n\ndef visualize_corr_matrix(true_table, axs, k):\n    true_table.source = true_table.source.apply(preprocess_text)\n    true_vectors = []\n    for i in range(len(true_table)):\n        sentence = true_table.source[i]\n        sentence = preprocess_text(sentence)\n        vector = model.get_sentence_vector(sentence)\n        true_vectors.append(vector)\n    matrix2 = cosine_similarity(true_vectors, true_vectors)\n    axs[k].imshow(matrix2)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:05:10.638596Z","iopub.execute_input":"2022-05-23T19:05:10.638823Z","iopub.status.idle":"2022-05-23T19:05:10.658235Z","shell.execute_reply.started":"2022-05-23T19:05:10.638795Z","shell.execute_reply":"2022-05-23T19:05:10.657025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Functions to avaluate the result: ###\n\ndef count_inversions_slowly(ranks):\n    inversions = 0\n    size = len(ranks)\n    for i in range(size):\n        for j in range(i+1, size):\n            if ranks[i] > ranks[j]:\n                total += 1\n    return total\n\nfrom bisect import bisect\n\n# Actually O(N^2), but fast in practice for our data\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):  # O(N)\n        j = bisect(sorted_so_far, u)  # O(log N)\n        inversions += i - j\n        sorted_so_far.insert(j, u)  # O(N)\n    return inversions\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0  # total inversions in predicted ranks across all instances\n    total_2max = 0  # maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:05:10.659734Z","iopub.execute_input":"2022-05-23T19:05:10.660135Z","iopub.status.idle":"2022-05-23T19:05:10.673253Z","shell.execute_reply.started":"2022-05-23T19:05:10.660095Z","shell.execute_reply":"2022-05-23T19:05:10.672221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The first mardown in the very first cell is often some greetings, desriptions of the following notebok or forks to another notebooks. It is not related to the following code cell and it lies far from it in a vector space, thus the model will not place it correctly.\n## Let's hace a look at such markdowns:","metadata":{}},{"cell_type":"code","source":"train_orders = pd.read_csv(src + 'train_orders.csv')\nfirst_markdowns = []\n\nfor i in tqdm(range(10000)):\n    id_, cell_order = train_orders.iloc[i]\n    cell = (cell_order.split(' ')[0])\n    first_cell = (pd.read_json(src + 'train/' + id_ + '.json').loc[cell])\n    if first_cell.cell_type == 'markdown':\n        first_markdowns.append(first_cell.source)\n        \nprint(len(first_markdowns))\nfirst_markdowns[:5]","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:06:25.89324Z","iopub.execute_input":"2022-05-23T19:06:25.893589Z","iopub.status.idle":"2022-05-23T19:08:33.265369Z","shell.execute_reply.started":"2022-05-23T19:06:25.893553Z","shell.execute_reply":"2022-05-23T19:08:33.264378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\nstopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='white',\n        stopwords=stopwords,\n        max_words=200,\n        max_font_size=40, \n        scale=2,\n        random_state=1).generate(str(data))\n    fig = plt.figure(1, figsize=(18, 12))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=20)\n        fig.subplots_adjust(top=2.3)\n    plt.imshow(wordcloud)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:18:22.317452Z","iopub.execute_input":"2022-05-23T19:18:22.318347Z","iopub.status.idle":"2022-05-23T19:18:22.325583Z","shell.execute_reply.started":"2022-05-23T19:18:22.31824Z","shell.execute_reply":"2022-05-23T19:18:22.324858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Word cloud for markdowns in the very beginning of notebooks:\")\nshow_wordcloud(first_markdowns)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:18:23.25813Z","iopub.execute_input":"2022-05-23T19:18:23.258829Z","iopub.status.idle":"2022-05-23T19:18:27.371895Z","shell.execute_reply.started":"2022-05-23T19:18:23.258784Z","shell.execute_reply":"2022-05-23T19:18:27.370945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"serch_list = ['hello', 'kagglers', 'this notebook', 'kaggle', 'welcome', \n              'competition', 'kernel', 'introduction', 'data analysis', \n              'https', 'data science', 'nbsp'] # These words almost sure stands in the markdowns at the very beginning of a notebook","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:10:46.951005Z","iopub.execute_input":"2022-05-23T19:10:46.95135Z","iopub.status.idle":"2022-05-23T19:10:46.957066Z","shell.execute_reply.started":"2022-05-23T19:10:46.951288Z","shell.execute_reply":"2022-05-23T19:10:46.956335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_first_markdown(df):\n    \"\"\"\n    This function finds a cell that should stand in the very beginning of notebooks\n    retunrs cell index and a key word\n    \"\"\"\n    table = df[df.cell_type == 'markdown']\n    for word in serch_list:\n        for i, line in enumerate(table.source):\n            if re.search(word, line.lower()):\n                return table.index[i], word\n    return None, None","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:15:44.658294Z","iopub.execute_input":"2022-05-23T19:15:44.658674Z","iopub.status.idle":"2022-05-23T19:15:44.666953Z","shell.execute_reply.started":"2022-05-23T19:15:44.658638Z","shell.execute_reply":"2022-05-23T19:15:44.665259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example below shows how we can find such markdown from dataframe\n## Markdown \"This notebook illustrate how to speedup...\" should definitely stay in the beginning","metadata":{}},{"cell_type":"code","source":"df = pd.read_json(test_files[2])\ncell, word = find_first_markdown(df)\nprint(cell, word)\ndf.tail(1)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:16:19.022364Z","iopub.execute_input":"2022-05-23T19:16:19.022835Z","iopub.status.idle":"2022-05-23T19:16:19.057053Z","shell.execute_reply.started":"2022-05-23T19:16:19.022802Z","shell.execute_reply":"2022-05-23T19:16:19.055968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main algorithm","metadata":{}},{"cell_type":"code","source":"def overall_algo(file, to_check=False, to_plot = False, train = False):\n    \"\"\"\n    This is the main sorting algorithm\n    It computes matrix of cosine distance between cells in a vector space, \n    and than it sorts them by finding the perfect matches between markdown and code cells\n    returns the cell order\n    \"\"\"\n    df = (pd.read_json(file))\n    df.source = df.source.apply(preprocess_text)\n    vectors = []\n\n    for i in range(len(df)):\n        sentence = df.source[i]\n        sentence = preprocess_text(sentence)\n        vector = model.get_sentence_vector(sentence)\n        vectors.append(vector)\n\n    matrix = cosine_similarity(vectors, vectors)\n    if to_plot:\n        f, axs = plt.subplots(1,3,figsize=(21,7)) # if you try the algo on a train dataset with true cell order\n        #f, axs = plt.subplots(1,2,figsize=(20,10)) # if you try the algo on a test dataset you have just two graphs\n        #axs[0].figure(figsize=(8,8))\n        axs[0].imshow(matrix)\n        axs[0].title.set_text('Initial Table')\n\n    \n    ### True notebook ###\n    #id_ = file.split('/')[-1][:-5]\n    if train:\n        true_table, cell_order = read_notebook_from_train_orders(str(file))\n\n    if to_plot:\n        visualize_corr_matrix(true_table, axs, 2)\n        axs[2].title.set_text('True Table')\n     \n    \n    indexes_code = list(df[df.cell_type == 'code'].index)\n    indexes_markdown = list(df[df.cell_type == 'markdown'].index)\n\n    result = indexes_code.copy()\n\n    #indexes_code , indexes_markdown\n    N = len(indexes_code)\n    K = len(indexes_markdown)\n    #print(N, K)\n\n    order = dict.fromkeys(np.arange(K))\n\n\n\n    #### ALGORITHM WORKABLE #########\n\n    markdowns_submatrix = matrix[N:,:-K].copy()\n    order = dict.fromkeys(np.arange(K))\n    indexes_to_order = list(np.arange(K))\n\n    # find the markdown in the very beginning of the notebook and place it there\n    cell, word = find_first_markdown(df)\n    if cell != None:\n        idx = indexes_markdown.index(cell)\n        order[idx] = 0\n        markdowns_submatrix[idx] = np.zeros(N)\n        indexes_to_order.remove(idx)\n        \n        \n    initial_markdowns_submatrix = markdowns_submatrix.copy()\n\n    for i in range(K):\n        for i in range(K):\n            if i in indexes_to_order:\n                most_similar_code = np.argmax(markdowns_submatrix[i])\n                most_similar_markdown = np.argmax(markdowns_submatrix.T[most_similar_code])\n                if most_similar_markdown == i:\n                    #print(i, ' congrats')\n                    order[i] = most_similar_code\n                    indexes_to_order.remove(i)\n                    markdowns_submatrix[most_similar_markdown] = np.zeros(N)\n                    markdowns_submatrix[:, most_similar_code] = np.zeros(K)\n        if (np.max(markdowns_submatrix) == 0): # stop creteria - all cells are sorted\n            break\n        markdowns_submatrix_old = markdowns_submatrix.copy()\n\n\n    # if some mardowns left (number of mardown cells is greater than codes)\n    for key, value in zip(indexes_to_order, np.argmax(initial_markdowns_submatrix[indexes_to_order], axis=1)):\n        order[key] = value \n\n    indexes_code = list(df[df.cell_type == 'code'].index)\n    indexes_markdown = list(df[df.cell_type == 'markdown'].index)\n\n    result = indexes_code.copy()\n\n    for i in range(K-1, -1, -1):\n        markdown_cell = indexes_markdown[i]\n        corresponding_code_cell = indexes_code[order[i]]\n        new_index = result.index(corresponding_code_cell)\n        result.insert(new_index, markdown_cell)\n\n    if to_check:\n        print(\"Algorithm: \")\n        check(result, file)\n        print(\"True notebook: \")\n        #table, gt = read_notebook_from_train_orders(id_)\n        display(true_table)\n        \n    if to_plot:\n        visualize_corr_matrix(df.loc[result], axs, 1)\n        axs[1].title.set_text('My Result')\n    return result","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:23:04.875572Z","iopub.execute_input":"2022-05-23T19:23:04.876002Z","iopub.status.idle":"2022-05-23T19:23:04.89862Z","shell.execute_reply.started":"2022-05-23T19:23:04.875965Z","shell.execute_reply":"2022-05-23T19:23:04.897645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# An example of algorithm with graph:","metadata":{}},{"cell_type":"code","source":"file =  src + 'train/0001daf4c2c76d.json'\nfile =  src + 'train/000bbb79a2fe3c.json' # not bad\nfile =  src + 'train/000c0a9b2fef4d.json' # far not bad\n\n\npredictions = overall_algo(file, to_check = True, to_plot = True, train = True)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:23:06.897262Z","iopub.execute_input":"2022-05-23T19:23:06.897585Z","iopub.status.idle":"2022-05-23T19:23:07.598994Z","shell.execute_reply.started":"2022-05-23T19:23:06.897553Z","shell.execute_reply":"2022-05-23T19:23:07.598072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Accuracy = 87% for table of 70 rows with 21 markdowns","metadata":{}},{"cell_type":"code","source":"true_table, ground_truth = read_notebook_from_train_orders(str(file))\nscore = kendall_tau([ground_truth], [predictions])\nprint(f\"score = {score}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:23:22.911494Z","iopub.execute_input":"2022-05-23T19:23:22.911856Z","iopub.status.idle":"2022-05-23T19:23:22.955719Z","shell.execute_reply.started":"2022-05-23T19:23:22.911809Z","shell.execute_reply":"2022-05-23T19:23:22.954578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = (pd.read_json(file))\nlen(df), len(df[df.cell_type == 'markdown'])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-23T19:23:23.994267Z","iopub.execute_input":"2022-05-23T19:23:23.995151Z","iopub.status.idle":"2022-05-23T19:23:24.01052Z","shell.execute_reply.started":"2022-05-23T19:23:23.995109Z","shell.execute_reply":"2022-05-23T19:23:24.009134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Accuracy evaluation","metadata":{}},{"cell_type":"code","source":"def compare_algo_with_gt(file, to_check=True):\n    id_ = file.split('/')[-1][:-5]\n    if to_check:\n        print(i, file, id_)\n        print(\"Algorithm: \")\n    algo_result = overall_algo(file, to_check=to_check)\n    if to_check:\n        print(\"True notebook: \")\n    table, gt = read_notebook_from_train_orders(file)\n    if to_check:\n        display(table)\n    return algo_result, gt","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:23:51.971945Z","iopub.execute_input":"2022-05-23T19:23:51.972766Z","iopub.status.idle":"2022-05-23T19:23:51.979224Z","shell.execute_reply.started":"2022-05-23T19:23:51.972718Z","shell.execute_reply":"2022-05-23T19:23:51.978208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '../input/AI4Code/train/'\ntrain_files = os.listdir(train_dir)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:23:53.291918Z","iopub.execute_input":"2022-05-23T19:23:53.292201Z","iopub.status.idle":"2022-05-23T19:23:55.784112Z","shell.execute_reply.started":"2022-05-23T19:23:53.292172Z","shell.execute_reply":"2022-05-23T19:23:55.783373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\nground_truth = []\n\nN = 140\n\nfor i in tqdm(range(N)):\n    file = train_dir + train_files[i]\n    algo_result, cell_order = compare_algo_with_gt(file, to_check=False)\n    ground_truth.append(cell_order)\n    predictions.append(algo_result)\n\nfinal_score = kendall_tau(ground_truth, predictions)\nprint(f\"the accuracy on {N/1400}% of the data is {round(final_score, 3)}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:31:56.282553Z","iopub.execute_input":"2022-05-23T19:31:56.282953Z","iopub.status.idle":"2022-05-23T19:32:07.459048Z","shell.execute_reply.started":"2022-05-23T19:31:56.282917Z","shell.execute_reply":"2022-05-23T19:32:07.455796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 71.8% for now","metadata":{}},{"cell_type":"code","source":"df_result = pd.DataFrame(columns = ['id', 'cell_order'])\n#df_example = pd.read_csv(src + 'sample_submission.csv')\n\nfor i, file in enumerate(test_files):\n    print(i, file)\n    id_ = str(test_files[i]).split('/')[-1][:-5]\n    df = pd.read_json(file)\n    example = df.index.tolist()\n    result = overall_algo(file)\n    try:\n        result = overall_algo(file)\n        if len(result) != len(df):\n            result = ' '.join(example) #df_example.cell_order[i] # if result is weird (can't imagine how it's possible)\n        elif (len(set(result) - set(example)) != 0) or (len( set(example) - set(result) ) != 0 ):\n            result = ' '.join(example) #df_example.cell_order[i]\n        else:\n            result = ' '.join(result)\n    except:\n        result = ' '.join(example) # if algo failed put from example # never happened but just in case to avoid submission error \n      \n    to_add = result\n    df_result = df_result.append({'id':id_, 'cell_order':to_add}, ignore_index=True)\n\ndf_result\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:32:28.584827Z","iopub.execute_input":"2022-05-23T19:32:28.58515Z","iopub.status.idle":"2022-05-23T19:32:28.987261Z","shell.execute_reply.started":"2022-05-23T19:32:28.585115Z","shell.execute_reply":"2022-05-23T19:32:28.986192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T19:32:33.736328Z","iopub.execute_input":"2022-05-23T19:32:33.736904Z","iopub.status.idle":"2022-05-23T19:32:33.744685Z","shell.execute_reply.started":"2022-05-23T19:32:33.736856Z","shell.execute_reply":"2022-05-23T19:32:33.743796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cat submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-05-17T20:26:39.74725Z","iopub.execute_input":"2022-05-17T20:26:39.747511Z","iopub.status.idle":"2022-05-17T20:26:39.750849Z","shell.execute_reply.started":"2022-05-17T20:26:39.747481Z","shell.execute_reply":"2022-05-17T20:26:39.750146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}