{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":129276,"databundleVersionId":15506988,"sourceType":"competition"},{"sourceId":14832902,"sourceType":"datasetVersion","datasetId":9486587}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Cell 1 — Install packages","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip -q install transformers accelerate torchaudio soundfile librosa jiwer\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T13:30:23.611645Z","iopub.execute_input":"2026-02-11T13:30:23.612726Z","iopub.status.idle":"2026-02-11T13:30:27.136879Z","shell.execute_reply.started":"2026-02-11T13:30:23.612693Z","shell.execute_reply":"2026-02-11T13:30:27.13602Z"}},"outputs":[],"execution_count":4},{"cell_type":"markdown","source":"# Cell 2 — Imports + paths (your dataset)","metadata":{}},{"cell_type":"code","source":"import os, re, gc, time\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nimport soundfile as sf\nimport librosa\nimport torch\nimport torchaudio\n\nfrom transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline\n\nDATA_ROOT = Path(\"/kaggle/input/dl-sprint-4-0-bengali-long-form-speech-recognition/transcription/transcription\")\nTRAIN_AUDIO_DIR = DATA_ROOT / \"train\" / \"audio\"\nTRAIN_TEXT_DIR = DATA_ROOT / \"train\" / \"annotation\"\nTEST_AUDIO_DIR = DATA_ROOT / \"test\" / \"audio\"\n\nSAMPLE_SUB_PATH = Path(\"/kaggle/input/dl-sprint-4-0-bengali-long-form-speech-recognition/sample_submission .csv\")\n\nprint(\"Train audio dir:\", TRAIN_AUDIO_DIR.exists(), TRAIN_AUDIO_DIR)\nprint(\"Train text dir :\", TRAIN_TEXT_DIR.exists(), TRAIN_TEXT_DIR)\nprint(\"Test audio dir :\", TEST_AUDIO_DIR.exists(), TEST_AUDIO_DIR)\nprint(\"Sample sub :\", SAMPLE_SUB_PATH.exists(), SAMPLE_SUB_PATH)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T13:30:33.425406Z","iopub.execute_input":"2026-02-11T13:30:33.426112Z","iopub.status.idle":"2026-02-11T13:30:33.432663Z","shell.execute_reply.started":"2026-02-11T13:30:33.426081Z","shell.execute_reply":"2026-02-11T13:30:33.431799Z"}},"outputs":[{"name":"stdout","text":"Train audio dir: True /kaggle/input/dl-sprint-4-0-bengali-long-form-speech-recognition/transcription/transcription/train/audio\nTrain text dir : True /kaggle/input/dl-sprint-4-0-bengali-long-form-speech-recognition/transcription/transcription/train/annotation\nTest audio dir : True /kaggle/input/dl-sprint-4-0-bengali-long-form-speech-recognition/transcription/transcription/test/audio\nSample sub : True /kaggle/input/dl-sprint-4-0-bengali-long-form-speech-recognition/sample_submission .csv\n","output_type":"stream"}],"execution_count":6},{"cell_type":"markdown","source":"# Cell 3 — List test files (multi-format) + confirm submission columns","metadata":{}},{"cell_type":"code","source":"AUDIO_EXTS = {\".wav\", \".mp3\", \".flac\", \".m4a\", \".ogg\", \".aac\", \".wma\", \".opus\"}\n\ntest_paths = sorted([p for p in TEST_AUDIO_DIR.iterdir() if p.suffix.lower() in AUDIO_EXTS])\nprint(\"Found test files:\", len(test_paths))\nprint(\"First 10:\", [p.name for p in test_paths[:10]])\n\nsample = pd.read_csv(SAMPLE_SUB_PATH)\nprint(\"Sample columns:\", sample.columns.tolist())\ndisplay(sample.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T13:30:39.89071Z","iopub.execute_input":"2026-02-11T13:30:39.891583Z","iopub.status.idle":"2026-02-11T13:30:39.948675Z","shell.execute_reply.started":"2026-02-11T13:30:39.891551Z","shell.execute_reply":"2026-02-11T13:30:39.947992Z"}},"outputs":[{"name":"stdout","text":"Found test files: 24\nFirst 10: ['test_001.wav', 'test_002.wav', 'test_003.wav', 'test_004.wav', 'test_005.wav', 'test_006.wav', 'test_008.wav', 'test_009.wav', 'test_010.wav', 'test_011.wav']\nSample columns: ['filename', 'transcript']\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":" filename transcript\n0 test_001 এটি একটি নমুনা প্রতিলিপি\n1 test_002 এটি একটি নমুনা প্রতিলিপি\n2 test_003 এটি একটি নমুনা প্রতিলিপি\n3 test_004 এটি একটি নমুনা প্রতিলিপি\n4 test_005 এটি একটি নমুনা প্রতিলিপি","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
filenametranscript
0test_001এটি একটি নমুনা প্রতিলিপি
1test_002এটি একটি নমুনা প্রতিলিপি
2test_003এটি একটি নমুনা প্রতিলিপি
3test_004এটি একটি নমুনা প্রতিলিপি
4test_005এটি একটি নমুনা প্রতিলিপি
\n
"},"metadata":{}}],"execution_count":7},{"cell_type":"markdown","source":"# Cell 4 — Safe Bangla normalization (do NOT over-clean)","metadata":{}},{"cell_type":"code","source":"_PUNC = r\"\"\"[!\"#$%&'()*+,\\-./:;<=>?@\\[\\]^_`{|}~।…“”‘’—–]\"\"\"\n_ZW = r\"[\\u200b\\u200c\\u200d\\uFEFF]\"\n\ndef normalize_bn(text: str) -> str:\n if text is None:\n return \"\"\n text = re.sub(_ZW, \"\", text)\n text = re.sub(_PUNC, \" \", text)\n text = re.sub(r\"\\s+\", \" \", text).strip()\n return text\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T13:30:45.548869Z","iopub.execute_input":"2026-02-11T13:30:45.549515Z","iopub.status.idle":"2026-02-11T13:30:45.554297Z","shell.execute_reply.started":"2026-02-11T13:30:45.549486Z","shell.execute_reply":"2026-02-11T13:30:45.553352Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"from collections import Counter\n\ndef hallucination_signature(words, window=90):\n \"\"\"Return (triggered, dbg) for intro hallucination detection.\"\"\"\n if len(words) < window + 10:\n return False, {\"reason\": \"short\"}\n\n w0 = words[:window]\n uniq0 = len(set(w0)) / len(w0)\n top1_frac = Counter(w0).most_common(1)[0][1] / len(w0)\n bigrams = list(zip(w0, w0[1:]))\n bigram_rep = sum(v-1 for v in Counter(bigrams).values() if v > 1)\n\n triggered = (uniq0 <= 0.55) or (top1_frac >= 0.10) or (bigram_rep >= 10)\n dbg = {\"uniq0\": round(uniq0,3), \"top1_frac\": round(top1_frac,3), \"bigram_rep\": int(bigram_rep), \"triggered\": triggered}\n return triggered, dbg\n\n# Strict decoding ONLY for the intro (reduces music hallucinations)\nINTRO_SECONDS = 90 # 60–120 works; 90 is a good default\n\nINTRO_GEN_KWARGS = {\n \"no_repeat_ngram_size\": 4,\n \"repetition_penalty\": 1.20,\n \"num_beams\": 2, # keep low to avoid “confident hallucination”\n \"max_new_tokens\": 128,\n}\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Cell 5 — Robust audio loader (supports wav + non-wav)","metadata":{}},{"cell_type":"code","source":"TARGET_SR = 16000\n\ndef load_audio_any(path: str):\n ext = Path(path).suffix.lower()\n\n # Try soundfile first (fast for wav/flac)\n if ext in {\".wav\", \".flac\"}:\n wav, sr = sf.read(path)\n if wav.ndim > 1:\n wav = wav.mean(axis=1)\n wav = wav.astype(np.float32)\n if sr != TARGET_SR:\n wav = librosa.resample(wav, orig_sr=sr, target_sr=TARGET_SR).astype(np.float32)\n return wav, TARGET_SR\n\n # Fallback: librosa handles many formats (mp3/m4a/ogg/etc)\n wav, sr = librosa.load(path, sr=TARGET_SR, mono=True)\n return wav.astype(np.float32), sr\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T15:24:09.230867Z","iopub.execute_input":"2026-02-11T15:24:09.231412Z","iopub.status.idle":"2026-02-11T15:24:09.236814Z","shell.execute_reply.started":"2026-02-11T15:24:09.231384Z","shell.execute_reply":"2026-02-11T15:24:09.235934Z"}},"outputs":[],"execution_count":29},{"cell_type":"markdown","source":"# Cell 5.5 (right after Cell 5 audio loader)","metadata":{}},{"cell_type":"code","source":"# Cell 5.5 — Dynamic intro trimming (music-safe) using Silero VAD\n\nimport torch\n\ntorch.set_num_threads(1)\n\nvad_model, vad_utils = torch.hub.load(\n repo_or_dir=\"snakers4/silero-vad\",\n model=\"silero_vad\",\n trust_repo=True\n)\n(get_speech_timestamps, _, _, _, _) = vad_utils\n\ndef trim_music_intro(\n wav: np.ndarray,\n sr: int,\n max_check_s: float = 90.0,\n vad_threshold: float = 0.70,\n min_speech_ms: int = 900,\n min_silence_ms: int = 300,\n pad_back_s: float = 0.4,\n trim_if_start_after_s: float = 8.0,\n stability_window_s: float = 12.0,\n stability_min_segments: int = 3,\n # NEW knobs:\n gap_for_new_start_s: float = 0.6, # require a gap before we accept \"real speech start\"\n ignore_initial_long_seg_s: float = 20.0 # if first segment starts ~0 and lasts long, treat as music\n):\n \"\"\"\n Returns (wav_trimmed, trim_start_seconds).\n Trims only when it finds a likely \"real speech start\" after music intro.\n \"\"\"\n if sr != 16000:\n wav = librosa.resample(wav, orig_sr=sr, target_sr=16000).astype(np.float32)\n sr = 16000\n\n n_check = min(len(wav), int(max_check_s * sr))\n head = wav[:n_check]\n\n head_t = torch.from_numpy(head)\n ts = get_speech_timestamps(\n head_t,\n vad_model,\n sampling_rate=sr,\n threshold=vad_threshold,\n min_speech_duration_ms=min_speech_ms,\n min_silence_duration_ms=min_silence_ms\n )\n\n if not ts:\n return wav, 0.0\n\n # Convert to seconds\n segs = [(t[\"start\"]/sr, t[\"end\"]/sr) for t in ts]\n\n # --- NEW: detect \"music-like\" initial long segment starting near 0 ---\n # If the first segment starts very early and is very long, it’s often music mis-tagged as speech.\n if segs[0][0] <= 1.0 and (segs[0][1] - segs[0][0]) >= ignore_initial_long_seg_s:\n # We will NOT accept this as the start; look for the next \"restart\" after a real gap.\n pass\n\n # Build candidate starts:\n # A candidate start is a segment that begins after a gap >= gap_for_new_start_s\n candidates = []\n for i, (s, e) in enumerate(segs):\n if i == 0:\n # allow segment 0 only if it doesn't look like long music\n if not (segs[0][0] <= 1.0 and (segs[0][1] - segs[0][0]) >= ignore_initial_long_seg_s):\n candidates.append(s)\n else:\n prev_end = segs[i-1][1]\n gap = s - prev_end\n if gap >= gap_for_new_start_s:\n candidates.append(s)\n\n if not candidates:\n return wav, 0.0\n\n # pick the earliest candidate that is \"late enough\" to justify trimming\n chosen = None\n for start_s in candidates:\n if start_s < trim_if_start_after_s:\n continue\n\n # Stability check: enough segments start within the next stability_window_s\n win_end = start_s + stability_window_s\n count = sum(1 for (s, e) in segs if start_s <= s <= win_end)\n if count >= stability_min_segments:\n chosen = start_s\n break\n\n if chosen is None:\n return wav, 0.0\n\n trim_start = max(0.0, chosen - pad_back_s)\n trim_idx = int(trim_start * sr)\n return wav[trim_idx:], trim_start\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T15:44:11.525242Z","iopub.execute_input":"2026-02-11T15:44:11.526092Z","iopub.status.idle":"2026-02-11T15:44:12.112097Z","shell.execute_reply.started":"2026-02-11T15:44:11.526063Z","shell.execute_reply":"2026-02-11T15:44:12.111397Z"}},"outputs":[{"name":"stderr","text":"Using cache found in /root/.cache/torch/hub/snakers4_silero-vad_master\n","output_type":"stream"}],"execution_count":37},{"cell_type":"markdown","source":"# Cell 6 — tugstugi whisper-medium)","metadata":{}},{"cell_type":"code","source":"MODEL_ID = \"bengaliAI/tugstugi_bengaliai-asr_whisper-medium\"\n\ndevice = 0 if torch.cuda.is_available() else -1\ndtype = torch.float16 if torch.cuda.is_available() else torch.float32\n\nprocessor = AutoProcessor.from_pretrained(MODEL_ID)\nmodel = AutoModelForSpeechSeq2Seq.from_pretrained(MODEL_ID, torch_dtype=dtype).to(\"cuda\" if device == 0 else \"cpu\")\n# Ensure generation config is loaded (some fine-tunes ship outdated generation_config)\nmodel.generation_config = model.generation_config.from_pretrained(MODEL_ID)\n\n\nasr = pipeline(\n task=\"automatic-speech-recognition\",\n model=model,\n tokenizer=processor.tokenizer,\n feature_extractor=processor.feature_extractor,\n device=device,\n)\nprint(\"Loaded model:\", MODEL_ID, \"| device:\", \"cuda\" if device == 0 else \"cpu\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T15:24:36.218901Z","iopub.execute_input":"2026-02-11T15:24:36.219225Z","iopub.status.idle":"2026-02-11T15:24:42.221428Z","shell.execute_reply.started":"2026-02-11T15:24:36.219198Z","shell.execute_reply":"2026-02-11T15:24:42.22068Z"}},"outputs":[{"name":"stderr","text":"Device set to use cuda:0\n","output_type":"stream"},{"name":"stdout","text":"Loaded model: bengaliAI/tugstugi_bengaliai-asr_whisper-medium | device: cuda\n","output_type":"stream"}],"execution_count":32},{"cell_type":"markdown","source":"# Cell 7 — Baseline long-form settings","metadata":{}},{"cell_type":"code","source":"CHUNK_LENGTH_S = 30.0\nSTRIDE_LENGTH_S = 5.0 # keep baseline first; tune later\n\n# Anti-repetition controls (WER-focused):\n# - no_repeat_ngram_size prevents repeating phrases\n# - repetition_penalty discourages loops\n# - length_penalty discourages overly long rambles\n# - num_beams improves accuracy (slower but lower WER)\nGEN_KWARGS = {\n \"no_repeat_ngram_size\": 4,\n \"repetition_penalty\": 1.15,\n \"length_penalty\": 1.0,\n \"num_beams\": 4,\n}\n\nprint(\"chunk_length_s:\", CHUNK_LENGTH_S, \"| stride_length_s:\", STRIDE_LENGTH_S)\nprint(\"GEN_KWARGS:\", GEN_KWARGS)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T15:24:46.487562Z","iopub.execute_input":"2026-02-11T15:24:46.488356Z","iopub.status.idle":"2026-02-11T15:24:46.493102Z","shell.execute_reply.started":"2026-02-11T15:24:46.488315Z","shell.execute_reply":"2026-02-11T15:24:46.492435Z"}},"outputs":[{"name":"stdout","text":"chunk_length_s: 30.0 | stride_length_s: 5.0\nGEN_KWARGS: {'no_repeat_ngram_size': 4, 'repetition_penalty': 1.15, 'length_penalty': 1.0, 'num_beams': 4}\n","output_type":"stream"}],"execution_count":33},{"cell_type":"markdown","source":"# Cell 8 — Sanity test on 1 file (fast check)","metadata":{}},{"cell_type":"code","source":"# Cell 8 — Sanity test (UPDATED)\n\np = test_paths[0]\nwav, sr = load_audio_any(str(p))\n\n# NEW: trim intro if needed\nwav2, trim_s = trim_music_intro(wav, sr)\n\nt0 = time.time()\nout = asr(\n {\"array\": wav2, \"sampling_rate\": sr},\n chunk_length_s=CHUNK_LENGTH_S,\n stride_length_s=STRIDE_LENGTH_S,\n generate_kwargs=GEN_KWARGS,\n ignore_warning=True,\n)\npred = normalize_bn(out[\"text\"])\n\nprint(\"File:\", p.name)\nprint(\"Trimmed seconds:\", round(trim_s, 2))\nprint(\"Seconds:\", round(time.time() - t0, 2))\nprint(\"Chars:\", len(pred))\nprint(pred[:400])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T15:44:17.663615Z","iopub.execute_input":"2026-02-11T15:44:17.664377Z","iopub.status.idle":"2026-02-11T15:56:30.038355Z","shell.execute_reply.started":"2026-02-11T15:44:17.664346Z","shell.execute_reply":"2026-02-11T15:56:30.037626Z"}},"outputs":[{"name":"stdout","text":"File: test_001.wav\nTrimmed seconds: 0.0\nSeconds: 730.41\nChars: 19966\nডাকাতির নির্বাচনের জন্য তিনি ছিলেন নিম্ন প্রতিষ্ঠানের প্রধানমন্ত্রী শ্রোতা বিশ্টারিং প্ল্যান্টিং প্রোগ্রেস ট্রেনের সাথে স্কুলের প্রশিক্ এটা নিতে পারে এটা আপনাকে বেশ ভালো মানাবে এমনিতে গিয়ে ভীষণ সুন্দর মানুষ আপনি যেটা পড়বেন সেটাই ভুল লাগবে তাই হুম থ্যাংক ইউ বার্কাম ইয়ে আপনি জানতেন না যে আপনি এত বেশি সুন্দর দেখছি আমি ড্রেস টুস করছিলাম আপনি হেল্প করলেন তাই থ্যাংকে দিয়েছি আর আমি আয়না দেখি সো আমি জানি \n","output_type":"stream"}],"execution_count":38},{"cell_type":"markdown","source":"# Cell 9 — Full inference + resume-safe saving ","metadata":{}},{"cell_type":"code","source":"\nOUT_PATH = Path(\"/kaggle/working/submission4.csv\")\n\n# Build DF from actual test files\nsub = pd.DataFrame({\n \"filename\": [p.stem for p in test_paths],\n \"transcript\": [\"\"] * len(test_paths)\n})\n\n# Resume if exists\nif OUT_PATH.exists():\n prev = pd.read_csv(OUT_PATH)\n if set(prev.columns) == {\"filename\", \"transcript\"}:\n sub = sub.merge(prev, on=\"filename\", how=\"left\", suffixes=(\"\", \"_prev\"))\n sub[\"transcript\"] = sub[\"transcript_prev\"].fillna(sub[\"transcript\"])\n sub = sub[[\"filename\", \"transcript\"]]\n print(\"Resuming from existing submission.csv\")\n\ndone = set(sub.loc[sub[\"transcript\"].fillna(\"\").str.len() > 0, \"filename\"].tolist())\nprint(\"Already done:\", len(done), \"of\", len(sub))\n\nfor i, p in enumerate(test_paths, 1):\n fid = p.stem\n if fid in done:\n print(f\"[skip {i}/{len(test_paths)}] {p.name}\")\n continue\n\n print(f\"[{i}/{len(test_paths)}] START {p.name}\")\n\n try:\n # 1) Load audio\n wav, sr = load_audio_any(str(p))\n\n # 2) Dynamic intro trim (music-safe)\n wav2, trim_s = trim_music_intro(wav, sr)\n\n # 3) ASR (chunk+stride + anti-repetition)\n out = asr(\n {\"array\": wav2, \"sampling_rate\": sr},\n chunk_length_s=CHUNK_LENGTH_S,\n stride_length_s=STRIDE_LENGTH_S,\n generate_kwargs=GEN_KWARGS,\n ignore_warning=True,\n )\n\n pred = normalize_bn(out[\"text\"])\n\n # Store + save\n sub.loc[sub[\"filename\"] == fid, \"transcript\"] = pred\n sub.to_csv(OUT_PATH, index=False)\n\n print(f\"[{i}/{len(test_paths)}] DONE {p.name} trim={trim_s:.1f}s chars={len(pred)} (saved)\")\n\n except Exception as e:\n # Don't crash the whole run\n print(f\"[{i}/{len(test_paths)}] ERROR {p.name}: {repr(e)}\")\n\n sub.loc[sub[\"filename\"] == fid, \"transcript\"] = \"\"\n sub.to_csv(OUT_PATH, index=False)\n print(f\"[{i}/{len(test_paths)}] SAVED empty transcript for {p.name}\")\n\n # periodic cleanup\n if i % 2 == 0:\n gc.collect()\n if torch.cuda.is_available():\n torch.cuda.empty_cache()\n\nprint(\"Saved final:\", OUT_PATH)\ndisplay(sub.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T15:59:55.130532Z","iopub.execute_input":"2026-02-11T15:59:55.130987Z","iopub.status.idle":"2026-02-11T21:58:23.10521Z","shell.execute_reply.started":"2026-02-11T15:59:55.130956Z","shell.execute_reply":"2026-02-11T21:58:23.104567Z"}},"outputs":[{"name":"stdout","text":"Already done: 0 of 24\n[1/24] START test_001.wav\n[1/24] DONE test_001.wav trim=0.0s chars=19966 (saved)\n[2/24] START test_002.wav\n[2/24] DONE test_002.wav trim=45.1s chars=21847 (saved)\n[3/24] START test_003.wav\n[3/24] DONE test_003.wav trim=20.1s chars=31618 (saved)\n[4/24] START test_004.wav\n[4/24] DONE test_004.wav trim=48.7s chars=34927 (saved)\n[5/24] START test_005.wav\n[5/24] DONE test_005.wav trim=15.6s chars=24781 (saved)\n[6/24] START test_006.wav\n[6/24] DONE test_006.wav trim=20.8s chars=23465 (saved)\n[7/24] START test_008.wav\n[7/24] DONE test_008.wav trim=8.2s chars=30730 (saved)\n[8/24] START test_009.wav\n[8/24] DONE test_009.wav trim=0.0s chars=15864 (saved)\n[9/24] START test_010.wav\n[9/24] DONE test_010.wav trim=16.0s chars=16483 (saved)\n[10/24] START test_011.wav\n[10/24] DONE test_011.wav trim=9.4s chars=31894 (saved)\n[11/24] START test_012.wav\n[11/24] DONE test_012.wav trim=11.3s chars=23646 (saved)\n[12/24] START test_013.wav\n[12/24] DONE test_013.wav trim=60.1s chars=13178 (saved)\n[13/24] START test_016.wav\n[13/24] DONE test_016.wav trim=0.0s chars=17663 (saved)\n[14/24] START test_018.wav\n[14/24] DONE test_018.wav trim=12.9s chars=31734 (saved)\n[15/24] START test_019.wav\n[15/24] DONE test_019.wav trim=27.2s chars=24694 (saved)\n[16/24] START test_020.wav\n[16/24] DONE test_020.wav trim=15.5s chars=17832 (saved)\n[17/24] START test_021.wav\n[17/24] DONE test_021.wav trim=26.3s chars=25671 (saved)\n[18/24] START test_022.wav\n[18/24] DONE test_022.wav trim=0.0s chars=33762 (saved)\n[19/24] START test_023.wav\n[19/24] DONE test_023.wav trim=16.3s chars=29433 (saved)\n[20/24] START test_024.wav\n[20/24] DONE test_024.wav trim=14.1s chars=25860 (saved)\n[21/24] START test_027.wav\n[21/24] DONE test_027.wav trim=9.8s chars=33433 (saved)\n[22/24] START test_029.wav\n[22/24] DONE test_029.wav trim=32.1s chars=19429 (saved)\n[23/24] START test_030.wav\n[23/24] DONE test_030.wav trim=0.0s chars=19774 (saved)\n[24/24] START test_032.wav\n[24/24] DONE test_032.wav trim=38.0s chars=18998 (saved)\nSaved final: /kaggle/working/submission4.csv\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":" filename transcript\n0 test_001 ডাকাতির নির্বাচনের জন্য তিনি ছিলেন নিম্ন প্রতি...\n1 test_002 ঠিক আছে তোমরা ওঠো ওঠো দাঁড়াও দাঁড়োও দূরে যাও ত...\n2 test_003 তোর নাম কানাই তো তুমি কে কোন জাদুকর নাকি আজই শ...\n3 test_004 কিছু সময়েও একদিন যাবত পাওনা ছুটি নিয়ে বসে রয়েছ...\n4 test_005 কী মেঘ ডাকছে রে বাবা এই দেখছি ঝনঝন করে নামল কো...","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
filenametranscript
0test_001ডাকাতির নির্বাচনের জন্য তিনি ছিলেন নিম্ন প্রতি...
1test_002ঠিক আছে তোমরা ওঠো ওঠো দাঁড়াও দাঁড়োও দূরে যাও ত...
2test_003তোর নাম কানাই তো তুমি কে কোন জাদুকর নাকি আজই শ...
3test_004কিছু সময়েও একদিন যাবত পাওনা ছুটি নিয়ে বসে রয়েছ...
4test_005কী মেঘ ডাকছে রে বাবা এই দেখছি ঝনঝন করে নামল কো...
\n
"},"metadata":{}}],"execution_count":39},{"cell_type":"markdown","source":"# Cell A — Install KenLM tools + python bindings + rapidfuzz","metadata":{}},{"cell_type":"code","source":"!apt-get -qq update\n!apt-get -qq install -y build-essential cmake git zlib1g-dev libboost-all-dev\n\n!pip -q install -U --no-cache-dir rapidfuzz\n!pip -q install -U --no-cache-dir https://github.com/kpu/kenlm/archive/master.zip\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:54:40.047369Z","iopub.execute_input":"2026-02-13T21:54:40.048103Z","iopub.status.idle":"2026-02-13T21:55:54.454931Z","shell.execute_reply.started":"2026-02-13T21:54:40.048073Z","shell.execute_reply":"2026-02-13T21:55:54.454255Z"}},"outputs":[{"name":"stdout","text":"W: Skipping acquire of configured file 'main/source/Sources' as repository 'https://r2u.stat.illinois.edu/ubuntu jammy InRelease' does not seem to provide it (sources.list entry misspelt?)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.2/3.2 MB\u001b[0m \u001b[31m41.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[2K \u001b[32m\\\u001b[0m \u001b[32m553.6 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m \u001b[33m0:00:00\u001b[0m\n\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n Building wheel for kenlm (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","output_type":"stream"}],"execution_count":5},{"cell_type":"markdown","source":"# Cell B — Build KenLM binaries (lmplz / build_binary)","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n!rm -rf kenlm\n!git clone --depth 1 https://github.com/kpu/kenlm.git\n%cd /kaggle/working/kenlm\n!mkdir -p build\n%cd /kaggle/working/kenlm/build\n!cmake ..\n!make -j 4\n\n!ls -la /kaggle/working/kenlm/build/bin\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:56:54.646864Z","iopub.execute_input":"2026-02-13T21:56:54.647683Z","iopub.status.idle":"2026-02-13T21:57:45.806397Z","shell.execute_reply.started":"2026-02-13T21:56:54.647636Z","shell.execute_reply":"2026-02-13T21:57:45.8057Z"}},"outputs":[{"name":"stdout","text":"/kaggle/working\nCloning into 'kenlm'...\nremote: Enumerating objects: 319, done.\u001b[K\nremote: Counting objects: 100% (319/319), done.\u001b[K\nremote: Compressing objects: 100% (310/310), done.\u001b[K\nremote: Total 319 (delta 15), reused 227 (delta 5), pack-reused 0 (from 0)\u001b[K\nReceiving objects: 100% (319/319), 493.07 KiB | 5.94 MiB/s, done.\nResolving deltas: 100% (15/15), done.\n/kaggle/working/kenlm\n/kaggle/working/kenlm/build\n-- The C compiler identification is GNU 11.4.0\n-- The CXX compiler identification is GNU 11.4.0\n-- Detecting C compiler ABI info\n-- Detecting C compiler ABI info - done\n-- Check for working C compiler: /usr/bin/cc - skipped\n-- Detecting C compile features\n-- Detecting C compile features - done\n-- Detecting CXX compiler ABI info\n-- Detecting CXX compiler ABI info - done\n-- Check for working CXX compiler: /usr/bin/c++ - skipped\n-- Detecting CXX compile features\n-- Detecting CXX compile features - done\n-- Could NOT find Eigen3 (missing: Eigen3_DIR)\n\u001b[33mCMake Warning (dev) at CMakeLists.txt:101 (find_package):\n Policy CMP0167 is not set: The FindBoost module is removed. Run \"cmake\n --help-policy CMP0167\" for policy details. Use the cmake_policy command to\n set the policy and suppress this warning.\n\nThis warning is for project developers. Use -Wno-dev to suppress it.\n\u001b[0m\n-- Found Boost: /usr/lib/x86_64-linux-gnu/cmake/Boost-1.74.0/BoostConfig.cmake (found suitable version \"1.74.0\", minimum required is \"1.41.0\") found components: program_options system thread unit_test_framework\n-- Found Threads: TRUE\n-- Found ZLIB: /usr/lib/x86_64-linux-gnu/libz.so (found version \"1.2.11\")\n-- Found BZip2: /usr/lib/x86_64-linux-gnu/libbz2.so (found version \"1.0.8\")\n-- Looking for BZ2_bzCompressInit\n-- Looking for BZ2_bzCompressInit - found\n-- Looking for lzma_auto_decoder in /usr/lib/x86_64-linux-gnu/liblzma.so\n-- Looking for lzma_auto_decoder in /usr/lib/x86_64-linux-gnu/liblzma.so - found\n-- Looking for lzma_easy_encoder in /usr/lib/x86_64-linux-gnu/liblzma.so\n-- Looking for lzma_easy_encoder in /usr/lib/x86_64-linux-gnu/liblzma.so - found\n-- Looking for lzma_lzma_preset in /usr/lib/x86_64-linux-gnu/liblzma.so\n-- Looking for lzma_lzma_preset in /usr/lib/x86_64-linux-gnu/liblzma.so - found\n-- Found LibLZMA: /usr/lib/x86_64-linux-gnu/liblzma.so (found version \"5.2.5\")\n-- Looking for clock_gettime in rt\n-- Looking for clock_gettime in rt - found\n-- Configuring done (1.2s)\n-- Generating done (0.1s)\n-- Build files have been written to: /kaggle/working/kenlm/build\n[ 1%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/bignum-dtoa.cc.o\u001b[0m\n[ 2%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/bignum.cc.o\u001b[0m\n[ 3%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/cached-powers.cc.o\u001b[0m\n[ 5%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/fast-dtoa.cc.o\u001b[0m\n[ 6%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/fixed-dtoa.cc.o\u001b[0m\n[ 7%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/strtod.cc.o\u001b[0m\n[ 8%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/double-to-string.cc.o\u001b[0m\n[ 10%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/string-to-double.cc.o\u001b[0m\n[ 11%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/chain.cc.o\u001b[0m\n[ 12%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/count_records.cc.o\u001b[0m\n[ 13%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/io.cc.o\u001b[0m\n[ 15%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/line_input.cc.o\u001b[0m\n[ 16%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/multi_progress.cc.o\u001b[0m\n[ 17%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/rewindable_stream.cc.o\u001b[0m\n[ 18%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/bit_packing.cc.o\u001b[0m\n[ 20%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/ersatz_progress.cc.o\u001b[0m\n[ 21%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/exception.cc.o\u001b[0m\n[ 22%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/file.cc.o\u001b[0m\n[ 23%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/file_piece.cc.o\u001b[0m\n[ 25%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/float_to_string.cc.o\u001b[0m\n[ 26%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/integer_to_string.cc.o\u001b[0m\n[ 27%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/mmap.cc.o\u001b[0m\n[ 28%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/murmur_hash.cc.o\u001b[0m\n[ 30%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/parallel_read.cc.o\u001b[0m\n[ 31%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/pool.cc.o\u001b[0m\n[ 32%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/read_compressed.cc.o\u001b[0m\n[ 33%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/scoped.cc.o\u001b[0m\n[ 35%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/spaces.cc.o\u001b[0m\n[ 36%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/string_piece.cc.o\u001b[0m\n[ 37%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/usage.cc.o\u001b[0m\n[ 38%] \u001b[32m\u001b[1mLinking CXX static library ../lib/libkenlm_util.a\u001b[0m\n[ 38%] Built target kenlm_util\n[ 40%] \u001b[32mBuilding CXX object util/CMakeFiles/probing_hash_table_benchmark.dir/probing_hash_table_benchmark_main.cc.o\u001b[0m\n[ 41%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/bhiksha.cc.o\u001b[0m\n[ 42%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/kenlm_filter.dir/arpa_io.cc.o\u001b[0m\n[ 43%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/kenlm_filter.dir/phrase.cc.o\u001b[0m\n[ 45%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/binary_format.cc.o\u001b[0m\n[ 46%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/config.cc.o\u001b[0m\n[ 47%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/lm_exception.cc.o\u001b[0m\n[ 48%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/model.cc.o\u001b[0m\n[ 50%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/kenlm_filter.dir/vocab.cc.o\u001b[0m\n[ 51%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/quantize.cc.o\u001b[0m\n[ 52%] \u001b[32m\u001b[1mLinking CXX static library ../../lib/libkenlm_filter.a\u001b[0m\n[ 52%] Built target kenlm_filter\n[ 53%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/read_arpa.cc.o\u001b[0m\n[ 55%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/search_hashed.cc.o\u001b[0m\n[ 56%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/search_trie.cc.o\u001b[0m\n[ 57%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/sizes.cc.o\u001b[0m\n[ 58%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/trie.cc.o\u001b[0m\n[ 60%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/trie_sort.cc.o\u001b[0m\n[ 61%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/value_build.cc.o\u001b[0m\n[ 62%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/virtual_interface.cc.o\u001b[0m\n[ 63%] \u001b[32m\u001b[1mLinking CXX executable ../bin/probing_hash_table_benchmark\u001b[0m\n[ 63%] Built target probing_hash_table_benchmark\n[ 65%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/vocab.cc.o\u001b[0m\n[ 66%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/model_buffer.cc.o\u001b[0m\n[ 67%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/print.cc.o\u001b[0m\n[ 68%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/renumber.cc.o\u001b[0m\n[ 70%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/size_option.cc.o\u001b[0m\n[ 71%] \u001b[32m\u001b[1mLinking CXX static library ../lib/libkenlm.a\u001b[0m\n[ 71%] Built target kenlm\n[ 73%] \u001b[32mBuilding CXX object lm/CMakeFiles/fragment.dir/fragment_main.cc.o\u001b[0m\n[ 73%] \u001b[32mBuilding CXX object lm/CMakeFiles/query.dir/query_main.cc.o\u001b[0m\n[ 75%] \u001b[32mBuilding CXX object lm/CMakeFiles/build_binary.dir/build_binary_main.cc.o\u001b[0m\n[ 76%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm_benchmark.dir/kenlm_benchmark_main.cc.o\u001b[0m\n[ 77%] \u001b[32m\u001b[1mLinking CXX executable ../bin/fragment\u001b[0m\n[ 77%] Built target fragment\n[ 78%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/adjust_counts.cc.o\u001b[0m\n[ 80%] \u001b[32m\u001b[1mLinking CXX executable ../bin/build_binary\u001b[0m\n[ 80%] Built target build_binary\n[ 81%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/corpus_count.cc.o\u001b[0m\n[ 82%] \u001b[32m\u001b[1mLinking CXX executable ../bin/query\u001b[0m\n[ 82%] Built target query\n[ 83%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/initial_probabilities.cc.o\u001b[0m\n[ 85%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/filter.dir/filter_main.cc.o\u001b[0m\n[ 86%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/phrase_table_vocab.dir/phrase_table_vocab_main.cc.o\u001b[0m\n[ 87%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/interpolate.cc.o\u001b[0m\n[ 88%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/phrase_table_vocab\u001b[0m\n[ 88%] Built target phrase_table_vocab\n[ 90%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/output.cc.o\u001b[0m\n[ 91%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/pipeline.cc.o\u001b[0m\n[ 92%] \u001b[32m\u001b[1mLinking CXX executable ../bin/kenlm_benchmark\u001b[0m\n[ 92%] Built target kenlm_benchmark\n[ 93%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/filter\u001b[0m\n[ 93%] Built target filter\n[ 95%] \u001b[32m\u001b[1mLinking CXX static library ../../lib/libkenlm_builder.a\u001b[0m\n[ 95%] Built target kenlm_builder\n[ 96%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/lmplz.dir/lmplz_main.cc.o\u001b[0m\n[ 97%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/count_ngrams.dir/count_ngrams_main.cc.o\u001b[0m\n[ 98%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/lmplz\u001b[0m\n[ 98%] Built target lmplz\n[100%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/count_ngrams\u001b[0m\n[100%] Built target count_ngrams\ntotal 5672\ndrwxr-xr-x 2 root root 4096 Feb 13 21:57 .\ndrwxr-xr-x 7 root root 4096 Feb 13 21:56 ..\n-rwxr-xr-x 1 root root 638248 Feb 13 21:57 build_binary\n-rwxr-xr-x 1 root root 524288 Feb 13 21:57 count_ngrams\n-rwxr-xr-x 1 root root 615616 Feb 13 21:57 filter\n-rwxr-xr-x 1 root root 609312 Feb 13 21:57 fragment\n-rwxr-xr-x 1 root root 1019472 Feb 13 21:57 kenlm_benchmark\n-rwxr-xr-x 1 root root 1255328 Feb 13 21:57 lmplz\n-rwxr-xr-x 1 root root 183600 Feb 13 21:57 phrase_table_vocab\n-rwxr-xr-x 1 root root 284568 Feb 13 21:57 probing_hash_table_benchmark\n-rwxr-xr-x 1 root root 658536 Feb 13 21:57 query\n","output_type":"stream"}],"execution_count":6},{"cell_type":"markdown","source":"# Cell C — Create LM training text from your train annotations","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport re\n\nTRAIN_TEXT_DIR = Path(\"/kaggle/input/dl-sprint-4-0-bengali-long-form-speech-recognition/transcription/transcription/train/annotation\")\nlm_text_path = Path(\"/kaggle/working/train_text.txt\")\n\n_PUNC = r\"\"\"[!\"#$%&'()*+,\\-./:;<=>?@\\[\\]^_`{|}~।…“”‘’—–]\"\"\"\n_ZW = r\"[\\u200b\\u200c\\u200d\\uFEFF]\"\n\ndef normalize_bn(text: str) -> str:\n text = re.sub(_ZW, \"\", text)\n text = re.sub(_PUNC, \" \", text)\n text = re.sub(r\"\\s+\", \" \", text).strip()\n return text\n\nlines = []\nfor p in sorted(TRAIN_TEXT_DIR.glob(\"*.txt\")):\n t = normalize_bn(p.read_text(encoding=\"utf-8\", errors=\"ignore\"))\n if t:\n lines.append(t)\n\nwith open(lm_text_path, \"w\", encoding=\"utf-8\") as f:\n for t in lines:\n f.write(t + \"\\n\")\n\nprint(\"LM lines:\", len(lines))\nprint(\"Saved:\", lm_text_path)\nprint(\"Sample:\", lines[0][:200])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:57:56.666326Z","iopub.execute_input":"2026-02-13T21:57:56.667061Z","iopub.status.idle":"2026-02-13T21:57:57.899391Z","shell.execute_reply.started":"2026-02-13T21:57:56.667025Z","shell.execute_reply":"2026-02-13T21:57:57.898588Z"}},"outputs":[{"name":"stdout","text":"LM lines: 113\nSaved: /kaggle/working/train_text.txt\nSample: গল্পতর চ্যানেলে আপনাদের সাথে আছি আমি রাজ পড়ছিলাম হুমায়ুন আহমেদের উপন্যাস মধ্যান্য পড়ছি এর চতুর্থ পর্ব রঙিলা নটিবাড়ি সোহাগঞ্জ বাজারের শেষ মাথায় মাছের আড়ত পার হয়েও আট দশ মিনিট হাঁটতে হয় রাস্তার \n","output_type":"stream"}],"execution_count":7},{"cell_type":"markdown","source":"# Cell D — Train LM (ARPA + binary)","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\nlm_dir = Path(\"/kaggle/working/lm\")\nlm_dir.mkdir(exist_ok=True)\n\narpa_path = lm_dir / \"bn_5gram.arpa\"\nbin_path = lm_dir / \"bn_5gram.binary\"\n\nlmplz = \"/kaggle/working/kenlm/build/bin/lmplz\"\nbuild_binary = \"/kaggle/working/kenlm/build/bin/build_binary\"\n\n!{lmplz} -o 5 --discount_fallback < {lm_text_path} > {arpa_path}\n!{build_binary} {arpa_path} {bin_path}\n\nprint(\"LM binary:\", bin_path)\nprint(\"Size MB:\", bin_path.stat().st_size/1e6)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:58:28.083197Z","iopub.execute_input":"2026-02-13T21:58:28.083807Z","iopub.status.idle":"2026-02-13T21:58:34.291812Z","shell.execute_reply.started":"2026-02-13T21:58:28.08378Z","shell.execute_reply":"2026-02-13T21:58:34.291074Z"}},"outputs":[{"name":"stdout","text":"=== 1/5 Counting and sorting n-grams ===\nReading /kaggle/working/train_text.txt\n----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n****************************************************************************************************\nUnigram tokens 647387 types 43412\n=== 2/5 Calculating and sorting adjusted counts ===\nChain sizes: 1:520944 2:2627264000 3:4926120448 4:7881792000 5:11494281216\nStatistics:\n1 43412 D1=0.650104 D2=1.03725 D3+=1.39558\n2 378014 D1=0.819134 D2=1.14798 D3+=1.45022\n3 594942 D1=0.94345 D2=1.41244 D3+=1.5835\n4 634331 D1=0.987112 D2=1.6483 D3+=1.84297\n5 640539 D1=0.988505 D2=1.60064 D3+=1.92019\nMemory estimate for binary LM:\ntype MB\nprobing 48 assuming -p 1.5\nprobing 58 assuming -r models -p 1.5\ntrie 23 without quantization\ntrie 12 assuming -q 8 -b 8 quantization \ntrie 21 assuming -a 22 array pointer compression\ntrie 10 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n=== 3/5 Calculating and sorting initial probabilities ===\nChain sizes: 1:520944 2:6048224 3:11898840 4:15223944 5:17935092\n----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n####################################################################################################\n=== 4/5 Calculating and writing order-interpolated probabilities ===\nChain sizes: 1:520944 2:6048224 3:11898840 4:15223944 5:17935092\n----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n####################################################################################################\n=== 5/5 Writing ARPA model ===\n----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n****************************************************************************************************\nName:lmplz\tVmPeak:26470460 kB\tVmRSS:7536 kB\tRSSMax:4684292 kB\tuser:2.57545\tsys:2.7724\tCPU:5.34791\treal:4.58405\nReading /kaggle/working/lm/bn_5gram.arpa\n----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n****************************************************************************************************\nSUCCESS\nLM binary: /kaggle/working/lm/bn_5gram.binary\nSize MB: 52.112797\n","output_type":"stream"}],"execution_count":8},{"cell_type":"markdown","source":"# Cell E — Load LM + build vocab + define correct_text_lm","metadata":{}},{"cell_type":"code","source":"import kenlm\nfrom rapidfuzz import process, fuzz\n\nlm = kenlm.Model(str(bin_path))\n\n# vocab from train text\nvocab = set()\nfor t in lines:\n for w in t.split():\n if 2 <= len(w) <= 20:\n vocab.add(w)\nvocab = sorted(vocab)\n\ndef lm_score(sentence: str) -> float:\n return lm.score(sentence, bos=True, eos=True)\n\ndef correct_text_lm(text: str, max_changes=60, sim_thresh=85, gain_thresh=1.0):\n \"\"\"\n Conservative word-level correction using KenLM scoring.\n - sim_thresh: minimum fuzzy similarity to consider replacement\n - gain_thresh: minimum LM score improvement to accept replacement\n \"\"\"\n text = normalize_bn(text)\n words = text.split()\n if len(words) < 5:\n return text\n\n base_words = words\n base_sent = \" \".join(base_words)\n base_score = lm_score(base_sent)\n\n changes = 0\n for i in range(len(base_words)):\n w = base_words[i]\n if len(w) < 2:\n continue\n\n cands = process.extract(w, vocab, scorer=fuzz.ratio, limit=6)\n best_word = None\n best_score = base_score\n\n for cand, sim, _ in cands:\n if sim < sim_thresh or cand == w:\n continue\n tmp = base_words.copy()\n tmp[i] = cand\n sent = \" \".join(tmp)\n sc = lm_score(sent)\n if sc > best_score + gain_thresh:\n best_score = sc\n best_word = cand\n\n if best_word is not None:\n base_words[i] = best_word\n base_sent = \" \".join(base_words)\n base_score = best_score\n changes += 1\n if changes >= max_changes:\n break\n\n return base_sent\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T22:00:49.611502Z","iopub.execute_input":"2026-02-13T22:00:49.612165Z","iopub.status.idle":"2026-02-13T22:00:49.839381Z","shell.execute_reply.started":"2026-02-13T22:00:49.612131Z","shell.execute_reply":"2026-02-13T22:00:49.838834Z"}},"outputs":[],"execution_count":9},{"cell_type":"markdown","source":"# ✅ Cell F — Apply to your existing submission.csv (no GPU)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom tqdm import tqdm\n\nsub = pd.read_csv(\"/kaggle/input/datasets/tanmoymridha/submissionfile/submission4.csv\")\nprint(\"Loaded:\", sub.shape)\ndisplay(sub.head())\n\nsub[\"transcript_lm\"] = [\n correct_text_lm(t) for t in tqdm(sub[\"transcript\"].fillna(\"\").tolist())\n]\n\n# Save corrected submission in required format (overwrite transcript)\nout = sub[[\"filename\", \"transcript_lm\"]].rename(columns={\"transcript_lm\":\"transcript\"})\nout.to_csv(\"/kaggle/working/submission_lm.csv\", index=False)\n\nprint(\"Saved: /kaggle/working/submission_lm.csv\")\ndisplay(out.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T22:00:56.816122Z","iopub.execute_input":"2026-02-13T22:00:56.816767Z","iopub.status.idle":"2026-02-13T22:02:53.166094Z","shell.execute_reply.started":"2026-02-13T22:00:56.816741Z","shell.execute_reply":"2026-02-13T22:02:53.165471Z"}},"outputs":[{"name":"stdout","text":"Loaded: (24, 2)\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":" filename transcript\n0 test_001 ডাকাতির নির্বাচনের জন্য তিনি ছিলেন নিম্ন প্রতি...\n1 test_002 ঠিক আছে তোমরা ওঠো ওঠো দাঁড়াও দাঁড়োও দূরে যাও ত...\n2 test_003 তোর নাম কানাই তো তুমি কে কোন জাদুকর নাকি আজই শ...\n3 test_004 কিছু সময়েও একদিন যাবত পাওনা ছুটি নিয়ে বসে রয়েছ...\n4 test_005 কী মেঘ ডাকছে রে বাবা এই দেখছি ঝনঝন করে নামল কো...","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
filenametranscript
0test_001ডাকাতির নির্বাচনের জন্য তিনি ছিলেন নিম্ন প্রতি...
1test_002ঠিক আছে তোমরা ওঠো ওঠো দাঁড়াও দাঁড়োও দূরে যাও ত...
2test_003তোর নাম কানাই তো তুমি কে কোন জাদুকর নাকি আজই শ...
3test_004কিছু সময়েও একদিন যাবত পাওনা ছুটি নিয়ে বসে রয়েছ...
4test_005কী মেঘ ডাকছে রে বাবা এই দেখছি ঝনঝন করে নামল কো...
\n
"},"metadata":{}},{"name":"stderr","text":"100%|██████████| 24/24 [01:56<00:00, 4.84s/it]","output_type":"stream"},{"name":"stdout","text":"Saved: /kaggle/working/submission_lm.csv\n","output_type":"stream"},{"name":"stderr","text":"\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":" filename transcript\n0 test_001 ডাকাতি নির্বাচনের জন্য তিনি ছিলেন নিম্ন প্রতিষ...\n1 test_002 ঠিক আছে তোমরা ওঠো ওঠো দাঁড়াও দাঁড়োও দূরে যাও ত...\n2 test_003 তোর নাম কানা তো তুমি কে কোন জাদুকরের নাকি আজই ...\n3 test_004 কিছু সময়েও একদিন যাব পাওনা ছুটি নিয়ে বসে রয়েছে...\n4 test_005 কী মেঘ ডাকছে রে বাবা এই দেখি ঝনঝন করে নাম কোথা...","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
filenametranscript
0test_001ডাকাতি নির্বাচনের জন্য তিনি ছিলেন নিম্ন প্রতিষ...
1test_002ঠিক আছে তোমরা ওঠো ওঠো দাঁড়াও দাঁড়োও দূরে যাও ত...
2test_003তোর নাম কানা তো তুমি কে কোন জাদুকরের নাকি আজই ...
3test_004কিছু সময়েও একদিন যাব পাওনা ছুটি নিয়ে বসে রয়েছে...
4test_005কী মেঘ ডাকছে রে বাবা এই দেখি ঝনঝন করে নাম কোথা...
\n
"},"metadata":{}}],"execution_count":10}]}