{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"},{"sourceId":5158870,"sourceType":"datasetVersion","datasetId":2997548},{"sourceId":5194802,"sourceType":"datasetVersion","datasetId":3020507},{"sourceId":5594542,"sourceType":"datasetVersion","datasetId":3218684},{"sourceId":11377281,"sourceType":"datasetVersion","datasetId":7123132}],"dockerImageVersionId":30475,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**It takes around 6~7 hours to train a model with single fold.**\n\nSo, you should run this notebook four time (for each seed=42,43,44,45) to get all 4 seed weights of the model.\n\nOr you can just press 'Save and Run All' to get the result of the fold 0 model.\n\nThere are two issues with this notebook:\n\n- Training time issue: The weird thing is that it takes around 3 hours in Colab TPU (v2-8), which is supposed to be slower than Kaggle's TPU (v3-8). If you know a lot about tf+TPU frameworks, please let me know how to debug this issue.\n\n- Training unstability: I Changed some minor configurations from the final solution. I changed epoch 400 -> 300, clipvalue=1. -> None, label_smoothing=0 -> label_smoothing=0.1 for more stable reproducibility(with slightly lower accuracy). unstability mainly caused by high lambda value of the AWP(0.2) - it will cause nan loss somtimes(around once out of five times). you can switch it to 0.1 or lower the learning rate and still can get fairly high accuracy. Also, if you have any idea related to this issue, please let me know.","metadata":{}},{"cell_type":"code","source":"!pip install -q /kaggle/input/tensorflow-2120/tensorflow-2.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n#!pip install -q tensorflow==2.12.0\n!pip install -q tensorflow-addons==0.20.0\n!pip install -q git+https://github.com/hoyso48/tf-utils@main","metadata":{"id":"IpEQKDrDqAFP","outputId":"2affcd98-7204-4d46-c030-ce8daefe9ad4","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport json\nimport os\nfrom multiprocessing import cpu_count\n\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport tensorflow.keras.mixed_precision as mixed_precision\n\nfrom tqdm.autonotebook import tqdm\nimport sklearn\n\nfrom tf_utils.schedules import OneCycleLR, ListedLR\nfrom tf_utils.callbacks import SWA\nfrom tf_utils.learners import FGM, AWP\n\nimport os\nimport time\nimport pickle\nimport math\nimport random\nimport sys\nimport cv2\nimport gc\nimport h5py\nimport glob\nimport datetime\n\ndef read_json_file(file_path):\n    \"\"\"Read a JSON file and parse it into a Python object.\n\n    Args:\n        file_path (str): The path to the JSON file to read.\n\n    Returns:\n        dict: A dictionary object representing the JSON data.\n        \n    Raises:\n        FileNotFoundError: If the specified file path does not exist.\n        ValueError: If the specified file path does not contain valid JSON data.\n    \"\"\"\n    try:\n        # Open the file and load the JSON data into a Python object\n        with open(file_path, 'r') as file:\n            json_data = json.load(file)\n        return json_data\n    except FileNotFoundError:\n        # Raise an error if the file path does not exist\n        raise FileNotFoundError(f\"File not found: {file_path}\")\n    except ValueError:\n        # Raise an error if the file does not contain valid JSON data\n        raise ValueError(f\"Invalid JSON data in file: {file_path}\")\n\ncpu_count()","metadata":{"id":"at4Io1dNtSLh","trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:09:56.494199Z","iopub.execute_input":"2025-04-09T15:09:56.494548Z","iopub.status.idle":"2025-04-09T15:10:01.858388Z","shell.execute_reply.started":"2025-04-09T15:09:56.494509Z","shell.execute_reply":"2025-04-09T15:10:01.857525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Snapshot(tf.keras.callbacks.Callback):\n    def __init__(self, save_prefix, save_freq=5, keep_max=3):\n        super().__init__()\n        self.save_prefix = save_prefix\n        self.save_freq = save_freq\n        self.keep_max = keep_max\n        self.saved_files = []\n\n    def on_epoch_end(self, epoch, logs=None):\n        if (epoch + 1) % self.save_freq == 0:\n            # TPU-safe文件操作\n            filename = f\"{self.save_prefix}_epoch{epoch+1}.weights.h5\"\n            \n            # 删除已存在的同名文件\n            if tf.io.gfile.exists(filename):\n                tf.io.gfile.remove(filename)\n                \n            # 异步保存权重\n            self.model.save_weights(filename)\n            self.saved_files.append(filename)\n            \n            # 清理旧文件\n            while len(self.saved_files) > self.keep_max:\n                old_file = self.saved_files.pop(0)\n                if tf.io.gfile.exists(old_file):\n                    tf.io.gfile.remove(old_file)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/asl-signs/train.csv')\nprint(\"\\n\\n... LOAD SIGN TO PREDICTION INDEX MAP FROM JSON FILE ...\\n\")\ns2p_map = {k.lower():v for k,v in read_json_file(os.path.join(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\")).items()}\np2s_map = {v:k for k,v in read_json_file(os.path.join(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\")).items()}\nencoder = lambda x: s2p_map.get(x.lower())\ndecoder = lambda x: p2s_map.get(x)\n# print(s2p_map)\ntrain_df['label'] = train_df.sign.map(encoder)","metadata":{"id":"pRnjnF6Cj9nY","trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:04.352723Z","iopub.execute_input":"2025-04-09T15:10:04.353318Z","iopub.status.idle":"2025-04-09T15:10:04.58736Z","shell.execute_reply.started":"2025-04-09T15:10:04.353293Z","shell.execute_reply":"2025-04-09T15:10:04.586533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\nMAX_LEN = 384\nCROP_LEN = MAX_LEN\nNUM_CLASSES  = 250\nPAD = -100.\nNOSE=[\n    1,2,98,327\n]\nLNOSE = [98]\nRNOSE = [327]\nLIP = [ 0, \n    61, 185, 40, 39, 37, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nLLIP = [84,181,91,146,61,185,40,39,37,87,178,88,95,78,191,80,81,82]\nRLIP = [314,405,321,375,291,409,270,269,267,317,402,318,324,308,415,310,311,312]\n\nPOSE = [500, 502, 504, 501, 503, 505, 512, 513]\nLPOSE = [513,505,503,501]\nRPOSE = [512,504,502,500]\n\nREYE = [\n    33, 7, 163, 144, 145, 153, 154, 155, 133,\n    246, 161, 160, 159, 158, 157, 173,\n]\nLEYE = [\n    263, 249, 390, 373, 374, 380, 381, 382, 362,\n    466, 388, 387, 386, 385, 384, 398,\n]\n\nLHAND = np.arange(468, 489).tolist()\nRHAND = np.arange(522, 543).tolist()\n\nPOINT_LANDMARKS = LIP + LHAND + RHAND + NOSE + REYE + LEYE #+POSE\n\nNUM_NODES = len(POINT_LANDMARKS)\nCHANNELS = 6*NUM_NODES\n\nprint(NUM_NODES)\nprint(CHANNELS)\n\ndef tf_nan_mean(x, axis=0, keepdims=False):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis, keepdims=keepdims) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis, keepdims=keepdims)\n\ndef tf_nan_std(x, center=None, axis=0, keepdims=False):\n    if center is None:\n        center = tf_nan_mean(x, axis=axis,  keepdims=True)\n    d = x - center\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis, keepdims=keepdims))\n\nclass Preprocess(tf.keras.layers.Layer):\n    def __init__(self, max_len=MAX_LEN, point_landmarks=POINT_LANDMARKS, **kwargs):\n        super().__init__(**kwargs)\n        self.max_len = max_len\n        self.point_landmarks = point_landmarks\n\n    def call(self, inputs):\n        if tf.rank(inputs) == 3:\n            x = inputs[None,...]\n        else:\n            x = inputs\n        \n        mean = tf_nan_mean(tf.gather(x, [17], axis=2), axis=[1,2], keepdims=True)\n        mean = tf.where(tf.math.is_nan(mean), tf.constant(0.5,x.dtype), mean)\n        x = tf.gather(x, self.point_landmarks, axis=2) #N,T,P,C\n        std = tf_nan_std(x, center=mean, axis=[1,2], keepdims=True)\n        std = tf.where(std == 0, tf.constant(1e-6, dtype=std.dtype), std)  # 避免除以零\n        x = (x - mean)/std\n\n        if self.max_len is not None:\n            x = x[:,:self.max_len]\n        length = tf.shape(x)[1]\n        x = x[...,:2]\n\n        dx = tf.cond(tf.shape(x)[1]>1,lambda:tf.pad(x[:,1:] - x[:,:-1], [[0,0],[0,1],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n\n        dx2 = tf.cond(tf.shape(x)[1]>2,lambda:tf.pad(x[:,2:] - x[:,:-2], [[0,0],[0,2],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n\n        x = tf.concat([\n            tf.reshape(x, (-1,length,2*len(self.point_landmarks))),\n            tf.reshape(dx, (-1,length,2*len(self.point_landmarks))),\n            tf.reshape(dx2, (-1,length,2*len(self.point_landmarks))),\n        ], axis = -1)\n        \n        x = tf.where(tf.math.is_nan(x),tf.constant(0.,x.dtype),x)\n        \n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:06.600045Z","iopub.execute_input":"2025-04-09T15:10:06.600708Z","iopub.status.idle":"2025-04-09T15:10:06.617264Z","shell.execute_reply.started":"2025-04-09T15:10:06.600677Z","shell.execute_reply":"2025-04-09T15:10:06.616439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ECA(tf.keras.layers.Layer):\n    def __init__(self, kernel_size=5, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n    def call(self, inputs, mask=None):\n        nn = tf.keras.layers.GlobalAveragePooling1D()(inputs, mask=mask)\n        nn = tf.expand_dims(nn, -1)\n        nn = self.conv(nn)\n        nn = tf.squeeze(nn, -1)\n        nn = tf.nn.sigmoid(nn)\n        nn = nn[:,None,:]\n        return inputs * nn\n\nclass LateDropout(tf.keras.layers.Layer):\n    def __init__(self, rate, noise_shape=None, start_step=0, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.rate = rate\n        self.start_step = start_step\n        self.dropout = tf.keras.layers.Dropout(rate, noise_shape=noise_shape)\n      \n    def build(self, input_shape):\n        super().build(input_shape)\n        agg = tf.VariableAggregation.ONLY_FIRST_REPLICA\n        self._train_counter = tf.Variable(0, dtype=\"int64\", aggregation=agg, trainable=False)\n\n    def call(self, inputs, training=False):\n        x = tf.cond(self._train_counter < self.start_step, lambda:inputs, lambda:self.dropout(inputs, training=training))\n        if training:\n            self._train_counter.assign_add(1)\n        return x\n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n    def __init__(self, \n        kernel_size=17,\n        dilation_rate=1,\n        use_bias=False,\n        depthwise_initializer='glorot_uniform',\n        name='', **kwargs):\n        super().__init__(name=name,**kwargs)\n        self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n        self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n                            kernel_size,\n                            strides=1,\n                            dilation_rate=dilation_rate,\n                            padding='valid',\n                            use_bias=use_bias,\n                            depthwise_initializer=depthwise_initializer,\n                            name=name + '_dwconv')\n        self.supports_masking = True\n        \n    def call(self, inputs):\n        x = self.causal_pad(inputs)\n        x = self.dw_conv(x)\n        return x\n\ndef Conv1DBlock(channel_size,\n          kernel_size,\n          dilation_rate=1,\n          drop_rate=0.0,\n          expand_ratio=2,\n          se_ratio=0.25,\n          activation='swish',\n          name=None):\n    '''\n    efficient conv1d block, @hoyso48\n    '''\n    if name is None:\n        name = str(tf.keras.backend.get_uid(\"mbblock\"))\n    # Expansion phase\n    def apply(inputs):\n        channels_in = tf.keras.backend.int_shape(inputs)[-1]\n        channels_expand = channels_in * expand_ratio\n\n        skip = inputs\n\n        x = tf.keras.layers.Dense(\n            channels_expand,\n            use_bias=True,\n            activation=activation,\n            name=name + '_expand_conv')(inputs)\n\n        # Depthwise Convolution\n        x = CausalDWConv1D(kernel_size,\n            dilation_rate=dilation_rate,\n            use_bias=False,\n            name=name + '_dwconv')(x)\n\n        x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_bn')(x)\n\n        x  = ECA()(x)\n\n        x = tf.keras.layers.Dense(\n            channel_size,\n            use_bias=True,\n            name=name + '_project_conv')(x)\n\n        if drop_rate > 0:\n            x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1), name=name + '_drop')(x)\n\n        if (channels_in == channel_size):\n            x = tf.keras.layers.add([x, skip], name=name + '_add')\n        return x\n\n    return apply","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:09.239461Z","iopub.execute_input":"2025-04-09T15:10:09.240267Z","iopub.status.idle":"2025-04-09T15:10:09.254316Z","shell.execute_reply.started":"2025-04-09T15:10:09.240235Z","shell.execute_reply":"2025-04-09T15:10:09.253303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MultiHeadSelfAttention(tf.keras.layers.Layer):\n    def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n        super().__init__(**kwargs)\n        self.dim = dim\n        self.scale = self.dim ** -0.5\n        self.num_heads = num_heads\n        self.qkv = tf.keras.layers.Dense(3 * dim, use_bias=False)\n        self.drop1 = tf.keras.layers.Dropout(dropout)\n        self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n        self.supports_masking = True\n\n    def call(self, inputs, mask=None):\n        qkv = self.qkv(inputs)\n        qkv = tf.keras.layers.Permute((2, 1, 3))(tf.keras.layers.Reshape((-1, self.num_heads, self.dim * 3 // self.num_heads))(qkv))\n        q, k, v = tf.split(qkv, [self.dim // self.num_heads] * 3, axis=-1)\n\n        attn = tf.matmul(q, k, transpose_b=True) * self.scale\n\n        if mask is not None:\n            mask = mask[:, None, None, :]\n\n        attn = tf.keras.layers.Softmax(axis=-1)(attn, mask=mask)\n        attn = self.drop1(attn)\n\n        x = attn @ v\n        x = tf.keras.layers.Reshape((-1, self.dim))(tf.keras.layers.Permute((2, 1, 3))(x))\n        x = self.proj(x)\n        return x\n\n\ndef TransformerBlock(dim=256, num_heads=4, expand=4, attn_dropout=0.2, drop_rate=0.2, activation='swish'):\n    def apply(inputs):\n        x = inputs\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = MultiHeadSelfAttention(dim=dim,num_heads=num_heads,dropout=attn_dropout)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        attn_out = x\n\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([attn_out, x])\n        return x\n    return apply","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:12.179223Z","iopub.execute_input":"2025-04-09T15:10:12.18002Z","iopub.status.idle":"2025-04-09T15:10:12.190224Z","shell.execute_reply.started":"2025-04-09T15:10:12.17999Z","shell.execute_reply":"2025-04-09T15:10:12.189428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(max_len=MAX_LEN, dropout_step=0, dim=192):\n    inp = tf.keras.Input((max_len,CHANNELS))\n    #x = tf.keras.layers.Masking(mask_value=PAD,input_shape=(max_len,CHANNELS))(inp) #we don't need masking layer with inference\n    x = inp\n    ksize = 17\n    x = tf.keras.layers.Dense(dim, use_bias=False,name='stem_conv')(x)\n    x = tf.keras.layers.BatchNormalization(momentum=0.95,name='stem_bn')(x)\n\n    x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n    x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n    x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n    x = TransformerBlock(dim,expand=2)(x)\n\n    x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n    x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n    x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n    x = TransformerBlock(dim,expand=2)(x)\n\n    if dim == 384: #for the 4x sized model\n        x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n        x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n        x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n        x = TransformerBlock(dim,expand=2)(x)\n\n        x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n        x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n        x = Conv1DBlock(dim,ksize,drop_rate=0.2)(x)\n        x = TransformerBlock(dim,expand=2)(x)\n\n    x = tf.keras.layers.Dense(dim*2,activation=None,name='top_conv')(x)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    x = LateDropout(0.8, start_step=dropout_step)(x)\n    x = tf.keras.layers.Dense(NUM_CLASSES,name='classifier')(x)\n    return tf.keras.Model(inp, x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:38.720109Z","iopub.execute_input":"2025-04-09T15:10:38.72073Z","iopub.status.idle":"2025-04-09T15:10:38.728674Z","shell.execute_reply.started":"2025-04-09T15:10:38.720701Z","shell.execute_reply":"2025-04-09T15:10:38.727718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Seed all random number generators\ndef seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\ndef get_strategy(device='TPU-VM'):\n    IS_TPU=False\n    if \"TPU\" in device:\n        tpu = 'local' if device=='TPU-VM' else None\n        print(\"connecting to TPU...\")\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu)\n        strategy = tf.distribute.TPUStrategy(tpu)\n        IS_TPU = True\n\n    if device == \"GPU\"  or device==\"CPU\":\n        ngpu = len(tf.config.experimental.list_physical_devices('GPU'))\n        if ngpu>1:\n            print(\"Using multi GPU\")\n            strategy = tf.distribute.MirroredStrategy()\n        elif ngpu==1:\n            print(\"Using single GPU\")\n            strategy = tf.distribute.get_strategy()\n        else:\n            print(\"Using CPU\")\n            strategy = tf.distribute.get_strategy()\n            CFG.device = \"CPU\"\n\n    if device == \"GPU\":\n        print(\"Num GPUs Available: \", ngpu)\n\n    AUTO     = tf.data.experimental.AUTOTUNE\n    REPLICAS = strategy.num_replicas_in_sync\n    print(f'REPLICAS: {REPLICAS}')\n\n    return strategy, REPLICAS, IS_TPU\n\nSTRATEGY, N_REPLICAS, IS_TPU = get_strategy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:19.833651Z","iopub.execute_input":"2025-04-09T15:10:19.834307Z","iopub.status.idle":"2025-04-09T15:10:22.96423Z","shell.execute_reply.started":"2025-04-09T15:10:19.834273Z","shell.execute_reply":"2025-04-09T15:10:22.963349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class LoRADense(tf.keras.layers.Layer):\n    def __init__(self, original_layer, rank=8, alpha_initial=1.0, name=None, **kwargs):\n        \"\"\"\n        LoRADense layer for efficient low-rank adaptation of attention weights.\n        \n        Args:\n            original_layer: The original Dense layer to be adapted.\n            rank: The rank for low-rank approximation.\n            alpha_initial: Initial scaling factor for LoRA adjustments.\n            name: Name of the layer (used to ensure unique weight names).\n        \"\"\"\n        super().__init__(name=name, **kwargs)  # 确保层名由外部指定\n        self.original_layer = original_layer\n        self.rank = rank\n        self.alpha_initial = alpha_initial\n\n        # Get the original weight matrix\n        original_weights = original_layer.get_weights()\n        if not original_weights:\n            raise ValueError(\"Original layer weights are required for LoRA initialization.\")\n        \n        W_orig = original_weights[0]  # Shape: [in_dim, out_dim]\n        in_dim, out_dim = W_orig.shape\n\n        # Initialize A with small random values and B with zeros\n        A_init = np.random.normal(0, 0.02, size=(in_dim, rank))  # Small random initialization\n        B_init = np.zeros((rank, out_dim))  # Zero initialization\n        \n        # Use the layer's name as a prefix to ensure unique weight names\n        self.A = self.add_weight(\n            shape=(in_dim, rank),\n            initializer=tf.constant_initializer(A_init),\n            trainable=True,\n            name=f'{self.name}_lora_A'  # 动态生成唯一名称\n        )\n        self.B = self.add_weight(\n            shape=(rank, out_dim),\n            initializer=tf.constant_initializer(B_init),\n            trainable=True,\n            name=f'{self.name}_lora_B'  # 动态生成唯一名称\n        )\n        self.alpha = self.add_weight(\n            shape=(1,),\n            initializer=tf.constant_initializer(alpha_initial),\n            trainable=True,\n            name=f'{self.name}_lora_alpha'  # 动态生成唯一名称\n        )\n\n        # Freeze the original layer weights\n        self.original_layer.trainable = False\n\n    def call(self, inputs):\n        # Original kernel (frozen)\n        W_orig = self.original_layer.kernel\n        if W_orig is None:\n            raise ValueError(\"Original layer kernel is None. Ensure the original layer is properly initialized.\")\n        \n        # Compute LoRA adjustment\n        delta_w = self.alpha * tf.matmul(self.A, self.B)  # Shape: [in_dim, out_dim]\n        updated_weights = W_orig + delta_w\n        \n        # Apply the updated weights to inputs\n        bias = self.original_layer.bias or 0.0\n        return tf.matmul(inputs, updated_weights) + bias","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:42.832683Z","iopub.execute_input":"2025-04-09T15:10:42.832999Z","iopub.status.idle":"2025-04-09T15:10:42.842154Z","shell.execute_reply.started":"2025-04-09T15:10:42.832975Z","shell.execute_reply":"2025-04-09T15:10:42.841224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_lora_to_model(model, rank=8, alpha_initial=1.0):\n    for layer in model.layers:\n        # Check if the layer is MultiHeadSelfAttention\n        if isinstance(layer, MultiHeadSelfAttention):\n            if isinstance(layer.qkv, tf.keras.layers.Dense):  # Replace qkv with LoRADense\n                print(f'Replacing {layer.name}.qkv with LoRADense')\n                layer.qkv = LoRADense(\n                    layer.qkv,\n                    rank=rank,\n                    alpha_initial=alpha_initial,\n                    name=f\"{layer.name}_qkv_lora\"  # 确保唯一名称\n                )\n            if isinstance(layer.proj, tf.keras.layers.Dense):  # Replace proj with LoRADense\n                print(f'Replacing {layer.name}.proj with LoRADense')\n                layer.proj = LoRADense(\n                    layer.proj,\n                    rank=rank,\n                    alpha_initial=alpha_initial,\n                    name=f\"{layer.name}_proj_lora\"  # 确保唯一名称\n                )\n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_pretrained_weights(model, weight_path=None):\n    \"\"\"Load pretrained weights into a model.\"\"\"\n    layer_weights = {}\n    if weight_path is None:\n        weight_path = '/kaggle/input/iris-models/averaged_weights.h5'\n\n    # Step 1: Collect all weights into a dictionary\n    with h5py.File(weight_path, 'r') as f:\n        def _collect(name, obj):\n            if isinstance(obj, h5py.Dataset):\n                parts = name.replace(':0', '').split('/')\n                param_type = parts[-1]  # kernel, depthwise_kernel, etc.\n                # Handle nested structures\n                if param_type in ['kernel', 'depthwise_kernel', 'bias', 'beta', 'gamma', 'moving_mean', 'moving_variance']:\n                    # For dwconv: 1_dwconv/1_dwconv/1_dwconv_dwconv/depthwise_kernel → 1_dwconv\n                    # For eca: eca/eca/conv1d/kernel → eca\n                    # For mhsa: multi_head_self_attention/.../dense/kernel → multi_head_self_attention\n                    layer_name = parts[0]  # Take top-level name\n                    # Special case for nested dense layers in attention\n                    if 'multi_head_self_attention' in name:\n                        if 'dense/kernel' in name:\n                            param_type = 'qkv.kernel'\n                        elif 'dense_1/kernel' in name:\n                            param_type = 'proj.kernel'\n                        elif 'dense_4/kernel' in name:\n                            param_type = 'qkv.kernel'\n                        elif 'dense_5/kernel' in name:\n                            param_type = 'proj.kernel'\n                    key = f\"{layer_name}.{param_type}\"\n                    layer_weights[key] = obj[()]\n        f.visititems(_collect)\n\n    # Step 2: Assign weights to model layers\n    for layer in model.layers:\n        if isinstance(layer, tf.keras.layers.BatchNormalization):\n            params = ['gamma', 'beta', 'moving_mean', 'moving_variance']\n            weights = [layer_weights.get(f\"{layer.name}.{p}\") for p in params]\n            weights = [w for w in weights if w is not None]\n            if weights:\n                layer.set_weights(weights)\n                print(f\"Loaded weights for {layer.name}: {params[:len(weights)]}\")\n            else:\n                print(f\"No weights found for BN layer {layer.name}\")\n\n        elif isinstance(layer, (tf.keras.layers.Conv1D, tf.keras.layers.Dense)):\n            kernel = layer_weights.get(f\"{layer.name}.kernel\")\n            bias = layer_weights.get(f\"{layer.name}.bias\")\n            weights = [k for k in [kernel, bias] if k is not None]\n            if weights:\n                try:\n                    layer.set_weights(weights)\n                    print(f\"Loaded weights for {layer.name}: {['kernel', 'bias'][:len(weights)]}\")\n                except ValueError as e:\n                    print(f\"Error setting weights for {layer.name}: {e}\")\n            else:\n                print(f\"No weights found for Conv1D/Dense layer {layer.name}\")\n\n        elif isinstance(layer, CausalDWConv1D):\n            depthwise_kernel = layer_weights.get(f\"{layer.name}.depthwise_kernel\")\n            weights = [depthwise_kernel] if depthwise_kernel is not None else []\n            if weights:\n                try:\n                    layer.dw_conv.set_weights(weights)\n                    print(f\"Loaded weights for {layer.name}.dw_conv: ['depthwise_kernel']\")\n                except ValueError as e:\n                    print(f\"Error setting weights for {layer.name}.dw_conv: {e}\")\n            else:\n                print(f\"No weights found for CausalDWConv1D layer {layer.name}\")\n\n        elif isinstance(layer, ECA):\n            kernel = layer_weights.get(f\"{layer.name}.kernel\")\n            weights = [kernel] if kernel is not None else []\n            if weights:\n                try:\n                    layer.conv.set_weights(weights)\n                    print(f\"Loaded weights for {layer.name}.conv: ['kernel']\")\n                except ValueError as e:\n                    print(f\"Error setting weights for {layer.name}.conv: {e}\")\n            else:\n                print(f\"No weights found for ECA layer {layer.name}\")\n\n        elif isinstance(layer, MultiHeadSelfAttention):\n            qkv_kernel = layer_weights.get(f\"{layer.name}.qkv.kernel\")\n            proj_kernel = layer_weights.get(f\"{layer.name}.proj.kernel\")\n            weights = [k for k in [qkv_kernel, proj_kernel] if k is not None]\n            if weights:\n                try:\n                    # Assign weights in order: qkv, proj\n                    if qkv_kernel is not None:\n                        layer.qkv.set_weights([qkv_kernel])\n                    if proj_kernel is not None:\n                        layer.proj.set_weights([proj_kernel])\n                    print(f\"Loaded weights for {layer.name}: {['qkv.kernel', 'proj.kernel'][:len(weights)]}\")\n                except ValueError as e:\n                    print(f\"Error setting weights for {layer.name}: {e}\")\n            else:\n                print(f\"No weights found for MultiHeadSelfAttention layer {layer.name}\")\n\n        elif isinstance(layer, LateDropout):\n            # No trainable weights to load\n            print(f\"Skipping layer {layer.name}: No trainable weights (LateDropout)\")\n\n        else:\n            print(f\"Skipping layer {layer.name}: No kernel, BN, or custom trainable parameters\")\n\n    print(\"✅ Weight loading completed\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def initialize_lora_model(base_model, weight_paths=None, rank=8, alpha_initial=1.0):\n    \"\"\"\n    Initialize a model with LoRA structure and load pretrained weights.\n\n    Args:\n        base_model: The base model to modify.\n        weight_paths: List of paths to the pre-trained weights files. If None, default paths will be used.\n        rank: Rank for LoRADense.\n        alpha_initial: Initial scaling factor for LoRADense layers.\n\n    Returns:\n        The base model with LoRA structure applied and weights loaded.\n    \"\"\"\n    # Step 1: Load pretrained weights into the base model\n    load_pretrained_weights(base_model, weight_paths)\n\n    # Step 2: Apply LoRA structure\n    apply_lora_to_model(base_model, rank=rank, alpha_initial=alpha_initial)\n\n    # Step 3: Freeze non-LoRA parameters\n    for layer in base_model.layers:\n        if isinstance(layer, (MultiHeadSelfAttention, LoRADense)):\n            # Ensure LoRA layers are trainable\n            if hasattr(layer, 'qkv') and isinstance(layer.qkv, LoRADense):\n                layer.qkv.trainable = True\n                print(f'Set {layer.name}.qkv trainable')\n            if hasattr(layer, 'proj') and isinstance(layer.proj, LoRADense):\n                layer.proj.trainable = True\n                print(f'Set {layer.name}.proj trainable')\n        else:\n            # Freeze non-LoRA layers\n            layer.trainable = False\n\n    return base_model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_lora_fold(CFG, fold, train_files, valid_files=None, strategy=STRATEGY, summary=True):\n    \"\"\"LoRA fine-tuning function\"\"\"\n\n    # Initialize environment\n    seed_everything(CFG.seed)\n    tf.keras.backend.clear_session()\n    gc.collect()\n    tf.config.optimizer.set_jit(True)\n\n    # Mixed precision setup\n    if CFG.fp16:\n        try:\n            policy = mixed_precision.Policy('mixed_bfloat16')\n            mixed_precision.set_global_policy(policy)\n        except:\n            policy = mixed_precision.Policy('mixed_float16')\n            mixed_precision.set_global_policy(policy)\n    else:\n        policy = mixed_precision.Policy('float32')\n        mixed_precision.set_global_policy(policy)\n\n    # Dataset setup\n    if fold != 'all':\n        train_ds = get_tfrec_dataset(train_files, batch_size=CFG.batch_size, max_len=CFG.max_len, drop_remainder=True, augment=True, repeat=True, shuffle=32768)\n        valid_ds = get_tfrec_dataset(valid_files, batch_size=CFG.batch_size, max_len=CFG.max_len, drop_remainder=False, repeat=False, shuffle=False)\n    else:\n        train_ds = get_tfrec_dataset(train_files, batch_size=CFG.batch_size, max_len=CFG.max_len, drop_remainder=False, augment=True, repeat=True, shuffle=32768)\n        valid_ds = None\n        valid_files = []\n\n    num_train = count_data_items(train_files)\n    num_valid = count_data_items(valid_files) if valid_files else 0\n    steps_per_epoch = num_train // CFG.batch_size\n\n    with strategy.scope():\n        dropout_step = CFG.dropout_start_epoch * steps_per_epoch\n        base_model = get_model(max_len=CFG.max_len, dropout_step=dropout_step, dim=CFG.dim)\n\n        # Initialize model with LoRA structure and pretrained weights\n        weight_path = f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-last.h5' if CFG.resume else None\n        print(\"Applying LoRA structure...\")\n        lora_model = initialize_lora_model(base_model, weight_path, rank=CFG.lora_rank)\n        for i, w in enumerate(lora_model.weights):\n            w._handle_name = w.name + \"_\" + str(i)\n\n        # Learning rate schedule\n        schedule = OneCycleLR(CFG.lora_lr, CFG.lora_epoch, warmup_epochs=CFG.lora_epoch * CFG.warmup, steps_per_epoch=steps_per_epoch, resume_epoch=CFG.resume,\n                              decay_epochs=CFG.lora_epoch, lr_min=CFG.lora_lr_min, decay_type=CFG.lora_decay_type, warmup_type='linear')\n        weight_decay_schedule = OneCycleLR(CFG.lora_lr * CFG.lora_weight_decay, CFG.lora_epoch, warmup_epochs=CFG.lora_epoch * CFG.warmup,\n                                           steps_per_epoch=steps_per_epoch, resume_epoch=CFG.resume, decay_epochs=CFG.lora_epoch,\n                                           lr_min=CFG.lora_lr_min * CFG.lora_weight_decay, decay_type=CFG.lora_decay_type, warmup_type='linear')\n\n        # Adversarial training components: FGM and AWP\n        awp_step = CFG.awp_start_epoch * steps_per_epoch\n        if CFG.fgm:\n            lora_model = FGM(lora_model.input, lora_model.output, delta=CFG.awp_lambda, eps=0., start_step=awp_step)\n        elif CFG.awp:\n            lora_model = AWP(lora_model.input, lora_model.output, delta=CFG.awp_lambda, eps=0., start_step=awp_step)\n        \n        # Optimizer setup\n        opt = tfa.optimizers.RectifiedAdam(learning_rate=schedule, weight_decay=weight_decay_schedule, sma_threshold=4, clipvalue=CFG.clipvalue)\n        opt = tfa.optimizers.Lookahead(opt, sync_period=5)\n\n        # Compile LoRA model\n        lora_model.compile(\n            optimizer=opt,\n            loss=[tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=CFG.label_smoothing)],\n            metrics=[[tf.keras.metrics.CategoricalAccuracy()]],\n            steps_per_execution=steps_per_epoch\n        )\n\n    # Model summary\n    if summary:\n        print()\n        lora_model.summary()\n        print()\n        print(train_ds, valid_ds)\n        print()\n        schedule.plot()\n        print()\n    print(f'---------fold{fold}---------')\n    print(f'train:{num_train} valid:{num_valid}')\n    print()\n\n    # Resume from checkpoint\n    resume_path = f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-last.h5'\n    if CFG.resume and os.path.exists(resume_path):\n        print(f'Resuming from epoch {CFG.resume} using {resume_path}')\n        lora_model.load_weights(resume_path)\n        if train_ds is not None:\n            lora_model.evaluate(train_ds.take(steps_per_epoch))\n        if valid_ds is not None:\n            lora_model.evaluate(valid_ds)\n    elif CFG.resume:\n        print(f'Resume path {resume_path} does not exist, starting from scratch')\n\n    # Clear existing weight files before training to avoid conflicts\n    best_path = f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-best.h5'\n    if os.path.exists(best_path):\n        os.remove(best_path)\n        print(f\"Removed existing file: {best_path}\")\n    last_path = f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-last.h5'\n    if os.path.exists(last_path):\n        os.remove(last_path)\n        print(f\"Removed existing file: {last_path}\")\n\n    # Callbacks\n    logger = tf.keras.callbacks.CSVLogger(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-logs.csv')\n    sv_loss = tf.keras.callbacks.ModelCheckpoint(\n        best_path,\n        monitor='val_loss',\n        verbose=1,  # Enable logging to track saves\n        save_best_only=True,\n        save_weights_only=True,\n        mode='min',\n        save_freq='epoch'\n    )\n    snap = Snapshot(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}')\n    swa = SWA(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}', CFG.swa_epochs, strategy=strategy, train_ds=train_ds, valid_ds=valid_ds,\n              valid_steps=-(num_valid // -CFG.batch_size))\n    callbacks = []\n    if CFG.save_output:\n        callbacks.append(logger)\n        callbacks.append(snap)\n        callbacks.append(swa)\n        if fold != 'all':\n            callbacks.append(sv_loss)\n\n    # Training loop\n    history = lora_model.fit(\n        train_ds,\n        epochs=CFG.lora_epoch - CFG.resume,\n        steps_per_epoch=steps_per_epoch,\n        callbacks=callbacks,\n        validation_data=valid_ds,\n        verbose=CFG.verbose,\n        validation_steps=-(num_valid // -CFG.batch_size)\n    )\n\n    # Save final weights\n    if CFG.save_output:\n        try:\n            lora_model.save_weights(last_path)\n            print(f\"Saved final weights to {last_path}\")\n            if os.path.exists(best_path):\n                lora_model.load_weights(best_path)\n                print(f\"Loaded best weights from {best_path} for evaluation\")\n        except Exception as e:\n            print(f\"Error saving or loading final weights: {e}\")\n\n    if fold != 'all':\n        cv = lora_model.evaluate(valid_ds, verbose=CFG.verbose, steps=-(num_valid // -CFG.batch_size))\n    else:\n        cv = None\n\n    return lora_model, cv, history\n\ndef train_lora_folds(CFG, folds, strategy=STRATEGY, summary=True):\n    \"\"\"LoRA multi-fold training\"\"\"\n    for fold in folds:\n        if fold != 'all':\n            all_files = TRAIN_FILENAMES\n            train_files = [x for x in all_files if f'fold{fold}' not in x]\n            valid_files = [x for x in all_files if f'fold{fold}' in x]\n        else:\n            train_files = TRAIN_FILENAMES\n            valid_files = None\n\n        train_lora_fold(CFG, fold, train_files, valid_files, strategy=strategy, summary=summary)\n    return","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_lora_fold(CFG, fold, train_files, valid_files=None, strategy=STRATEGY, summary=True):\n    \"\"\"LoRA fine-tuning function\"\"\"\n\n    # Initialize environment\n    seed_everything(CFG.seed)\n    tf.keras.backend.clear_session()\n    gc.collect()\n    tf.config.optimizer.set_jit(True)\n\n    # Mixed precision setup\n    if CFG.fp16:\n        try:\n            policy = mixed_precision.Policy('mixed_bfloat16')\n            mixed_precision.set_global_policy(policy)\n        except:\n            policy = mixed_precision.Policy('mixed_float16')\n            mixed_precision.set_global_policy(policy)\n    else:\n        policy = mixed_precision.Policy('float32')\n        mixed_precision.set_global_policy(policy)\n\n    # Dataset setup\n    if fold != 'all':\n        train_ds = get_tfrec_dataset(train_files, batch_size=CFG.batch_size, max_len=CFG.max_len, drop_remainder=True, augment=True, repeat=True, shuffle=32768)\n        valid_ds = get_tfrec_dataset(valid_files, batch_size=CFG.batch_size, max_len=CFG.max_len, drop_remainder=False, repeat=False, shuffle=False)\n    else:\n        train_ds = get_tfrec_dataset(train_files, batch_size=CFG.batch_size, max_len=CFG.max_len, drop_remainder=False, augment=True, repeat=True, shuffle=32768)\n        valid_ds = None\n        valid_files = []\n\n    num_train = count_data_items(train_files)\n    num_valid = count_data_items(valid_files) if valid_files else 0\n    steps_per_epoch = num_train // CFG.batch_size\n\n    with strategy.scope():\n        dropout_step = CFG.dropout_start_epoch * steps_per_epoch\n        base_model = get_model(max_len=CFG.max_len, dropout_step=dropout_step, dim=CFG.dim)\n\n        # Initialize model with LoRA structure and pretrained weights\n        weight_path = f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-last.h5' if CFG.resume else None\n        print(\"Applying LoRA structure...\")\n        lora_model = initialize_lora_model(base_model, weight_path, rank=CFG.lora_rank)\n        for i, w in enumerate(lora_model.weights):\n            w._handle_name = w.name + \"_\" + str(i)\n\n        # Learning rate schedule\n        schedule = OneCycleLR(CFG.lora_lr, CFG.lora_epoch, warmup_epochs=CFG.lora_epoch * CFG.warmup, steps_per_epoch=steps_per_epoch, resume_epoch=CFG.resume,\n                              decay_epochs=CFG.lora_epoch, lr_min=CFG.lora_lr_min, decay_type=CFG.lora_decay_type, warmup_type='linear')\n        weight_decay_schedule = OneCycleLR(CFG.lora_lr * CFG.lora_weight_decay, CFG.lora_epoch, warmup_epochs=CFG.lora_epoch * CFG.warmup,\n                                           steps_per_epoch=steps_per_epoch, resume_epoch=CFG.resume, decay_epochs=CFG.lora_epoch,\n                                           lr_min=CFG.lora_lr_min * CFG.lora_weight_decay, decay_type=CFG.lora_decay_type, warmup_type='linear')\n\n        # Adversarial training components: FGM and AWP\n        awp_step = CFG.awp_start_epoch * steps_per_epoch\n        if CFG.fgm:\n            pissa_model = FGM(lora_model.input, lora_model.output, delta=CFG.awp_lambda, eps=0., start_step=awp_step)\n        elif CFG.awp:\n            pissa_model = AWP(lora_model.input, lora_model.output, delta=CFG.awp_lambda, eps=0., start_step=awp_step)\n        \n        # Optimizer setup\n        opt = tfa.optimizers.RectifiedAdam(learning_rate=schedule, weight_decay=weight_decay_schedule, sma_threshold=4, clipvalue=CFG.clipvalue)\n        opt = tfa.optimizers.Lookahead(opt, sync_period=5)\n\n        # Compile LoRA model\n        lora_model.compile(\n            optimizer=opt,\n            loss=[tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=CFG.label_smoothing)],\n            metrics=[[tf.keras.metrics.CategoricalAccuracy()]],\n            steps_per_execution=steps_per_epoch\n        )\n\n    # Model summary\n    if summary:\n        print()\n        lora_model.summary()\n        print()\n        print(train_ds, valid_ds)\n        print()\n        schedule.plot()\n        print()\n    print(f'---------fold{fold}---------')\n    print(f'train:{num_train} valid:{num_valid}')\n    print()\n\n    # Resume from checkpoint\n    if CFG.resume:\n        print(f'resume from epoch{CFG.resume}')\n        if os.path.exists(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-last.h5'):\n            os.remove(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-last.h5')  # 删除旧的权重文件\n        lora_model.load_weights(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-last.h5')\n        if train_ds is not None:\n            lora_model.evaluate(train_ds.take(steps_per_epoch))\n        if valid_ds is not None:\n            lora_model.evaluate(valid_ds)\n\n    # Callbacks\n    logger = tf.keras.callbacks.CSVLogger(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-logs.csv')\n    sv_loss = tf.keras.callbacks.ModelCheckpoint(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-best.h5', monitor='val_loss', verbose=0, save_best_only=True,\n                                                 save_weights_only=True, mode='min')\n    snap = Snapshot(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}')\n    swa = SWA(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}', CFG.swa_epochs, strategy=strategy, train_ds=train_ds, valid_ds=valid_ds,\n              valid_steps=-(num_valid // -CFG.batch_size))\n    callbacks = []\n    if CFG.save_output:\n        callbacks.append(logger)\n        callbacks.append(snap)\n        callbacks.append(swa)\n        if fold != 'all':\n            callbacks.append(sv_loss)\n\n    # Training loop\n    history = lora_model.fit(\n        train_ds,\n        epochs=CFG.lora_epoch - CFG.resume,\n        steps_per_epoch=steps_per_epoch,\n        callbacks=callbacks,\n        validation_data=valid_ds,\n        verbose=CFG.verbose,\n        validation_steps=-(num_valid // -CFG.batch_size)\n    )\n\n    # 绘制损失曲线\n    plt.plot(history.history['loss'], label='Training Loss')\n    if 'val_loss' in history.history:\n        plt.plot(history.history['val_loss'], label='Validation Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.title(f'Loss Curve - Fold {fold} - Seed {CFG.seed}')\n    plt.grid(True)\n    plt.legend()\n    plt.show()\n\n    # Save final weights\n    if CFG.save_output:\n        try:\n            if os.path.exists(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-best.h5'):\n                os.remove(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-best.h5')  # 删除旧的权重文件\n            lora_model.load_weights(f'{CFG.output_dir}/lora_{CFG.comment}-fold{fold}-best.h5')\n        except:\n            pass\n    if fold != 'all':\n        cv = lora_model.evaluate(valid_ds, verbose=CFG.verbose, steps=-(num_valid // -CFG.batch_size))\n    else:\n        cv = None\n\n    return lora_model, cv, history\n\ndef train_lora_folds(CFG, folds, strategy=STRATEGY, summary=True):\n    \"\"\"LoRA multi-fold training\"\"\"\n    for fold in folds:\n        if fold != 'all':\n            all_files = TRAIN_FILENAMES\n            train_files = [x for x in all_files if f'fold{fold}' not in x]\n            valid_files = [x for x in all_files if f'fold{fold}' in x]\n        else:\n            train_files = TRAIN_FILENAMES\n            valid_files = None\n\n        train_lora_fold(CFG, fold, train_files, valid_files, strategy=strategy, summary=summary)\n    return","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:10:56.28371Z","iopub.execute_input":"2025-04-09T15:10:56.283997Z","iopub.status.idle":"2025-04-09T15:10:56.302287Z","shell.execute_reply.started":"2025-04-09T15:10:56.283977Z","shell.execute_reply":"2025-04-09T15:10:56.301313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    n_splits = 5\n    save_output = True\n    output_dir = '/kaggle/working'\n\n    seed = 42\n    verbose = 2 #0) silent 1) progress bar 2) one line per epoch\n\n    max_len = 384\n    replicas = 8\n    lr = 5e-4 * replicas\n    weight_decay = 0.1\n    lr_min = 1e-6\n    epoch = 300 #400\n    warmup = 0\n    batch_size = 64 * replicas\n    snapshot_epochs = 5\n    swa_epochs = [] #list(range(epoch//2,epoch+1))\n\n    fp16 = True\n    fgm = False\n    awp = False\n    awp_lambda = 0.05\n    awp_start_epoch = 15\n    dropout_start_epoch = 15\n    resume = 0\n    dim = 192\n    lora_epoch = 30\n    lora_decay_type = 'cosine'\n    lora_weight_decay = 0.1\n    lora_lr_min = 1e-6\n    lora_lr = 5e-4\n    label_smoothing = 0.05\n    lora_rank = 8\n    clipvalue = 1.0\n    niter = 5\n    comment = f'islr-fp16-192-8-lora-r{lora_rank}-seed{seed}'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:11:17.361649Z","iopub.execute_input":"2025-04-09T15:11:17.362263Z","iopub.status.idle":"2025-04-09T15:11:17.368177Z","shell.execute_reply.started":"2025-04-09T15:11:17.362232Z","shell.execute_reply":"2025-04-09T15:11:17.367363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_FILENAMES = glob.glob('/kaggle/input/islr-5fold/*.tfrecords')\nprint(len(TRAIN_FILENAMES))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:11:22.083934Z","iopub.execute_input":"2025-04-09T15:11:22.08487Z","iopub.status.idle":"2025-04-09T15:11:22.111694Z","shell.execute_reply.started":"2025-04-09T15:11:22.084838Z","shell.execute_reply":"2025-04-09T15:11:22.110975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def interp1d_(x, target_len, method='random'):\n    length = tf.shape(x)[1]\n    target_len = tf.maximum(1,target_len)\n    if method == 'random':\n        if tf.random.uniform(()) < 0.33:\n            x = tf.image.resize(x, (target_len,tf.shape(x)[1]),'bilinear')\n        else:\n            if tf.random.uniform(()) < 0.5:\n                x = tf.image.resize(x, (target_len,tf.shape(x)[1]),'bicubic')\n            else:\n                x = tf.image.resize(x, (target_len,tf.shape(x)[1]),'nearest')\n    else:\n        x = tf.image.resize(x, (target_len,tf.shape(x)[1]),method)\n    return x\n\ndef tf_nan_mean(x, axis=0, keepdims=False):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis, keepdims=keepdims) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis, keepdims=keepdims)\n\ndef tf_nan_std(x, center=None, axis=0, keepdims=False):\n    if center is None:\n        center = tf_nan_mean(x, axis=axis,  keepdims=True)\n    d = x - center\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis, keepdims=keepdims))\n\nclass Preprocess(tf.keras.layers.Layer):\n    def __init__(self, max_len=MAX_LEN, point_landmarks=POINT_LANDMARKS, **kwargs):\n        super().__init__(**kwargs)\n        self.max_len = max_len\n        self.point_landmarks = point_landmarks\n\n    def call(self, inputs):\n        if tf.rank(inputs) == 3:\n            x = inputs[None,...]\n        else:\n            x = inputs\n\n        mean = tf_nan_mean(tf.gather(x, [17], axis=2), axis=[1,2], keepdims=True)\n        mean = tf.where(tf.math.is_nan(mean), tf.constant(0.5,x.dtype), mean)\n        x = tf.gather(x, self.point_landmarks, axis=2) #N,T,P,C\n        std = tf_nan_std(x, center=mean, axis=[1,2], keepdims=True)\n\n        x = (x - mean)/std\n\n        if self.max_len is not None:\n            x = x[:,:self.max_len]\n        length = tf.shape(x)[1]\n        x = x[...,:2]\n\n        dx = tf.cond(tf.shape(x)[1]>1,lambda:tf.pad(x[:,1:] - x[:,:-1], [[0,0],[0,1],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n\n        dx2 = tf.cond(tf.shape(x)[1]>2,lambda:tf.pad(x[:,2:] - x[:,:-2], [[0,0],[0,2],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n\n        x = tf.concat([\n            tf.reshape(x, (-1,length,2*len(self.point_landmarks))),\n            tf.reshape(dx, (-1,length,2*len(self.point_landmarks))),\n            tf.reshape(dx2, (-1,length,2*len(self.point_landmarks))),\n        ], axis = -1)\n\n        x = tf.where(tf.math.is_nan(x),tf.constant(0.,x.dtype),x)\n\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:11:26.204446Z","iopub.execute_input":"2025-04-09T15:11:26.20511Z","iopub.status.idle":"2025-04-09T15:11:26.218447Z","shell.execute_reply.started":"2025-04-09T15:11:26.205075Z","shell.execute_reply":"2025-04-09T15:11:26.217359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def decode_tfrec(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'coordinates': tf.io.FixedLenFeature([], tf.string),\n        'sign': tf.io.FixedLenFeature([], tf.int64),\n    })\n    out = {}\n    out['coordinates']  = tf.reshape(tf.io.decode_raw(features['coordinates'], tf.float32), (-1,ROWS_PER_FRAME,3))\n    out['sign'] = features['sign']\n    return out\n\ndef filter_nans_tf(x, ref_point=POINT_LANDMARKS):\n    mask = tf.math.logical_not(tf.reduce_all(tf.math.is_nan(tf.gather(x,ref_point,axis=1)), axis=[-2,-1]))\n    x = tf.boolean_mask(x, mask, axis=0)\n    return x\n\ndef preprocess(x, augment=False, max_len=MAX_LEN):\n    coord = x['coordinates']\n    coord = filter_nans_tf(coord)\n    if augment:\n        coord = augment_fn(coord, max_len=max_len)\n    coord = tf.ensure_shape(coord, (None,ROWS_PER_FRAME,3))\n\n    return tf.cast(Preprocess(max_len=max_len)(coord)[0],tf.float32), tf.one_hot(x['sign'], NUM_CLASSES)\n\ndef flip_lr(x):\n    x,y,z = tf.unstack(x, axis=-1)\n    x = 1-x\n    new_x = tf.stack([x,y,z], -1)\n    new_x = tf.transpose(new_x, [1,0,2])\n    lhand = tf.gather(new_x, LHAND, axis=0)\n    rhand = tf.gather(new_x, RHAND, axis=0)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(LHAND)[...,None], rhand)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(RHAND)[...,None], lhand)\n    llip = tf.gather(new_x, LLIP, axis=0)\n    rlip = tf.gather(new_x, RLIP, axis=0)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(LLIP)[...,None], rlip)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(RLIP)[...,None], llip)\n    lpose = tf.gather(new_x, LPOSE, axis=0)\n    rpose = tf.gather(new_x, RPOSE, axis=0)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(LPOSE)[...,None], rpose)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(RPOSE)[...,None], lpose)\n    leye = tf.gather(new_x, LEYE, axis=0)\n    reye = tf.gather(new_x, REYE, axis=0)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(LEYE)[...,None], reye)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(REYE)[...,None], leye)\n    lnose = tf.gather(new_x, LNOSE, axis=0)\n    rnose = tf.gather(new_x, RNOSE, axis=0)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(LNOSE)[...,None], rnose)\n    new_x = tf.tensor_scatter_nd_update(new_x, tf.constant(RNOSE)[...,None], lnose)\n    new_x = tf.transpose(new_x, [1,0,2])\n    return new_x\n\ndef resample(x, rate=(0.8,1.2)):\n    rate = tf.random.uniform((), rate[0], rate[1])\n    length = tf.shape(x)[0]\n    new_size = tf.cast(rate*tf.cast(length,tf.float32), tf.int32)\n    new_x = interp1d_(x, new_size)\n    return new_x\n\ndef spatial_random_affine(xyz,\n    scale  = (0.8,1.2),\n    shear = (-0.15,0.15),\n    shift  = (-0.1,0.1),\n    degree = (-30,30),\n):\n    center = tf.constant([0.5,0.5])\n    if scale is not None:\n        scale = tf.random.uniform((),*scale)\n        xyz = scale*xyz\n\n    if shear is not None:\n        xy = xyz[...,:2]\n        z = xyz[...,2:]\n        shear_x = shear_y = tf.random.uniform((),*shear)\n        if tf.random.uniform(()) < 0.5:\n            shear_x = 0.\n        else:\n            shear_y = 0.\n        shear_mat = tf.identity([\n            [1.,shear_x],\n            [shear_y,1.]\n        ])\n        xy = xy @ shear_mat\n        center = center + [shear_y, shear_x]\n        xyz = tf.concat([xy,z], axis=-1)\n\n    if degree is not None:\n        xy = xyz[...,:2]\n        z = xyz[...,2:]\n        xy -= center\n        degree = tf.random.uniform((),*degree)\n        radian = degree/180*np.pi\n        c = tf.math.cos(radian)\n        s = tf.math.sin(radian)\n        rotate_mat = tf.identity([\n            [c,s],\n            [-s, c],\n        ])\n        xy = xy @ rotate_mat\n        xy = xy + center\n        xyz = tf.concat([xy,z], axis=-1)\n\n    if shift is not None:\n        shift = tf.random.uniform((),*shift)\n        xyz = xyz + shift\n\n    return xyz\n\ndef temporal_crop(x, length=MAX_LEN):\n    l = tf.shape(x)[0]\n    offset = tf.random.uniform((), 0, tf.clip_by_value(l-length,1,length), dtype=tf.int32)\n    x = x[offset:offset+length]\n    return x\n\ndef temporal_mask(x, size=(0.2,0.4), mask_value=float('nan')):\n    l = tf.shape(x)[0]\n    mask_size = tf.random.uniform((), *size)\n    mask_size = tf.cast(tf.cast(l, tf.float32) * mask_size, tf.int32)\n    mask_offset = tf.random.uniform((), 0, tf.clip_by_value(l-mask_size,1,l), dtype=tf.int32)\n    x = tf.tensor_scatter_nd_update(x,tf.range(mask_offset, mask_offset+mask_size)[...,None],tf.fill([mask_size,543,3],mask_value))\n    return x\n\ndef spatial_mask(x, size=(0.2,0.4), mask_value=float('nan')):\n    mask_offset_y = tf.random.uniform(())\n    mask_offset_x = tf.random.uniform(())\n    mask_size = tf.random.uniform((), *size)\n    mask_x = (mask_offset_x<x[...,0]) & (x[...,0] < mask_offset_x + mask_size)\n    mask_y = (mask_offset_y<x[...,1]) & (x[...,1] < mask_offset_y + mask_size)\n    mask = mask_x & mask_y\n    x = tf.where(mask[...,None], mask_value, x)\n    return x\n\ndef augment_fn(x, always=False, max_len=None):\n    if tf.random.uniform(())<0.8 or always:\n        x = resample(x, (0.5,1.5))\n    if tf.random.uniform(())<0.5 or always:\n        x = flip_lr(x)\n    if max_len is not None:\n        x = temporal_crop(x, max_len)\n    if tf.random.uniform(())<0.75 or always:\n        x = spatial_random_affine(x)\n    if tf.random.uniform(())<0.5 or always:\n        x = temporal_mask(x)\n    if tf.random.uniform(())<0.5 or always:\n        x = spatial_mask(x)\n    return x\n\ndef get_tfrec_dataset(tfrecords, batch_size=64, max_len=64, drop_remainder=False, augment=False, shuffle=False, repeat=False):\n    # Initialize dataset with TFRecords\n    ds = tf.data.TFRecordDataset(tfrecords, num_parallel_reads=tf.data.AUTOTUNE, compression_type='GZIP')\n    ds = ds.map(decode_tfrec, tf.data.AUTOTUNE)\n    ds = ds.map(lambda x: preprocess(x, augment=augment, max_len=max_len), tf.data.AUTOTUNE)\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(shuffle)\n        options = tf.data.Options()\n        options.experimental_deterministic = (False)\n        ds = ds.with_options(options)\n\n    if batch_size:\n        ds = ds.padded_batch(batch_size, padding_values=PAD, padded_shapes=([max_len,CHANNELS],[NUM_CLASSES]), drop_remainder=drop_remainder)\n\n    ds = ds.prefetch(tf.data.AUTOTUNE)\n\n    return ds\n\nds = get_tfrec_dataset(TRAIN_FILENAMES, augment=True, batch_size=1024)\nfor x in ds:\n    temp_train = x\n    break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:11:28.273405Z","iopub.execute_input":"2025-04-09T15:11:28.273747Z","iopub.status.idle":"2025-04-09T15:11:33.938076Z","shell.execute_reply.started":"2025-04-09T15:11:28.273722Z","shell.execute_reply":"2025-04-09T15:11:33.937357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename.split('/')[-1]).group(1)) for filename in filenames]\n    return np.sum(n)\nprint(count_data_items(TRAIN_FILENAMES), len(train_df))\nassert count_data_items(TRAIN_FILENAMES) == len(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T14:46:41.173871Z","iopub.execute_input":"2025-04-09T14:46:41.174575Z","iopub.status.idle":"2025-04-09T14:46:41.181205Z","shell.execute_reply.started":"2025-04-09T14:46:41.174541Z","shell.execute_reply":"2025-04-09T14:46:41.180079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 创建测试模型\ntest_model = get_model(max_len=CFG.max_len)  # 加载基础模型\ntest_model = apply_lora_to_model(test_model, rank=8)  # 注入LoRA结构\n\n# 创建虚拟输入样本\ndummy_input = tf.random.normal((1, CFG.max_len, CHANNELS))  # 调整维度与实际情况匹配\n\n# 定义转换验证函数\ndef validate_tflite_conversion(model, dummy_input):\n    try:\n        # 运行一次推理以构建模型\n        _ = model.predict(dummy_input)\n        \n        keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(model)\n        keras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\n        keras_model_converter.target_spec.supported_types = [tf.float16]\n        tflite_model = keras_model_converter.convert()\n        \n        # 验证模型可加载性\n        interpreter = tf.lite.Interpreter(model_content=tflite_model)\n        interpreter.allocate_tensors()\n        \n        # 输入/输出维度验证\n        input_details = interpreter.get_input_details()\n        output_details = interpreter.get_output_details()\n        print(f\"✅ 转换成功 | 输入维度: {input_details[0]['shape']} | 输出维度: {output_details[0]['shape']}\")\n        \n        return True\n    except Exception as e:\n        print(f\"❌ 转换失败: {str(e)}\")\n        return False\n\n# 执行验证\nvalidate_tflite_conversion(test_model, dummy_input)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T15:12:10.128774Z","iopub.execute_input":"2025-04-09T15:12:10.129715Z","iopub.status.idle":"2025-04-09T15:12:16.184206Z","shell.execute_reply.started":"2025-04-09T15:12:10.129683Z","shell.execute_reply":"2025-04-09T15:12:16.183271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CFG.seed = 42\nCFG.fold = 0\nCFG.comment = f'islr-fp16-192-8-lora-fold{CFG.fold}-seed{CFG.seed}' \ntrain_lora_folds(CFG, [0], summary=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-09T14:39:55.802108Z","iopub.execute_input":"2025-04-09T14:39:55.802899Z","execution_failed":"2025-04-09T14:41:26.677Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CFG.seed = 43 \nCFG.fold = 1\nCFG.comment = f'islr-fp16-192-8-lora-fold{CFG.fold}-seed{CFG.seed}' \ntrain_lora_folds(CFG, [1], summary=True)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CFG.seed = 44\nCFG.fold = 2\nCFG.comment = f'islr-fp16-192-8-lora-fold{CFG.fold}-seed{CFG.seed}' \ntrain_lora_folds(CFG, [2], summary=True)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CFG.seed = 45 \nCFG.fold = 3\nCFG.comment = f'islr-fp16-192-8-lora-fold{CFG.fold}-seed{CFG.seed}'  \ntrain_lora_folds(CFG, [3], summary=True)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n    \"\"\"\n    TensorFlow Lite model that takes input tensors and applies:\n        – a preprocessing model\n        – the ISLR model \n    \"\"\"\n\n    def __init__(self, islr_models):\n        \"\"\"\n        Initializes the TFLiteModel with the specified preprocessing model and ISLR model.\n        \"\"\"\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.prep_inputs = Preprocess()\n        self.islr_models   = islr_models\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs):\n        \"\"\"\n        Applies the feature generation model and main model to the input tensors.\n\n        Args:\n            inputs: Input tensor with shape [batch_size, 543, 3].\n\n        Returns:\n            A dictionary with a single key 'outputs' and corresponding output tensor.\n        \"\"\"\n        x = self.prep_inputs(tf.cast(inputs, dtype=tf.float32))\n        outputs = [model(x) for model in self.islr_models]\n        outputs = tf.keras.layers.Average()(outputs)[0]\n        return {'outputs': outputs}","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet('/kaggle/input/asl-signs/' + pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tflite_keras_model = TFLiteModel(islr_models=lora_models)\ndemo_output = tflite_keras_model(load_relevant_data_subset(train_df.path[0]))[\"outputs\"]\ndecoder(np.argmax(demo_output.numpy(), axis=-1))","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\nkeras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\nkeras_model_converter.target_spec.supported_types = [tf.float16]\ntflite_model = keras_model_converter.convert()\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n!zip submission.zip /kaggle/working/model.tflite","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#check inference time\n#code from @hengck23\nmode = 's' #'d'ebug #'s'ubmit\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport shutil\nfrom datetime import datetime\nfrom timeit import default_timer as timer\n\n\nif mode in ['d']:  \n    try:\n        import tflite_runtime\n    except:\n        !pip install tflite-runtime\n\n    import tflite_runtime.interpreter as tflite   \n    import tflite_runtime\n    print(tflite_runtime.__version__)\n\nprint('import ok')\n'''\nYour model must also require less than 40 MB in memory and \nperform inference with less than 100 milliseconds of latency per video. \nExpect to see approximately 40,000 videos in the test set. \nWe allow an additional 10 minute buffer for loading the data and miscellaneous overhead.\n\n'''\ndef time_to_str(t, mode='min'):\n    if mode=='min':\n        t  = int(t)/60\n        hr = t//60\n        min = t%60\n        return '%2d hr %02d min'%(hr,min)\n\n    elif mode=='sec':\n        t   = int(t)\n        min = t//60\n        sec = t%60\n        return '%2d min %02d sec'%(min,sec)\n\n    else:\n        raise NotImplementedError\n\n        \nROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\nif mode in ['d']: \n \n    interpreter = tflite.Interpreter('/kaggle/working/model.tflite')\n    prediction_fn = interpreter.get_signature_runner('serving_default')\n#     valid_df = pd.read_csv('/kaggle/input/asl-demo/train_prepared.csv') \n#     valid_df = train_df[train_df.fold==0].reset_index(drop=True)\n#     valid_df = valid_df[:1000]\n    valid_df = train_df[:1000]\n    valid_num = len(valid_df)\n    valid = {\n        'sign':[],\n    }\n\n    start_timer = timer()\n    for t, d in valid_df.iterrows():\n\n        pq_file = f'/kaggle/input/asl-signs/{d.path}'\n        #print(pq_file)\n        xyz = load_relevant_data_subset(pq_file)\n\n        output = prediction_fn(inputs=xyz)\n        p = output['outputs'].reshape(-1)\n\n        valid['sign'].append(p)\n\n        #---\n        if t%100==0:\n            time_taken = timer() - start_timer\n            print('\\r %8d / %d  %s'%(t,valid_num,time_to_str(time_taken,'sec')),end='',flush=True)\n\n    print('\\n')\n\n\n    truth = valid_df.label.values\n    sign  = np.stack(valid['sign'])\n    predict = np.argsort(-sign, -1)\n    correct = predict==truth.reshape(valid_num,1)\n    topk = correct.cumsum(-1).mean(0)[:5]\n\n\n    print(f'time_taken = {time_to_str(time_taken,\"sec\")}')\n    print(f'time_taken for LB = {time_taken*1000/valid_num:05f} msec\\n')\n    for i in range(5):\n        print(f'topk[{i}] = {topk[i]}')  \n    print('----- end -----\\n')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-09T13:03:34.284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}