Upload 4 files

Meso_4.ipynb

@@ -0,0 +1,525 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "h3_RFpXiPQR6"
+      },
+      "outputs": [],
+      "source": [
+        "from tensorflow.keras import Model\n",
+        "from tensorflow.keras import Input\n",
+        "from tensorflow.keras.layers import Conv2D, ReLU, ELU, LeakyReLU, Dropout, Dense, MaxPooling2D, Flatten, BatchNormalization\n",
+        "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
+        "from tensorflow.keras.optimizers import Adam\n",
+        "from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping\n",
+        "from tensorflow.keras.utils import plot_model\n",
+        "from tensorflow.keras.models import load_model\n",
+        "from sklearn.metrics import classification_report\n",
+        "\n",
+        "import numpy as np\n",
+        "import matplotlib.pyplot as plt\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "ADU3Hu_TAFvG"
+      },
+      "outputs": [],
+      "source": [
+        "IMG_WIDTH = 256"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "id": "NyTzubIUjeCR"
+      },
+      "outputs": [],
+      "source": [
+        "def get_datagen(use_default_augmentation=True, **kwargs):\n",
+        "    kwargs.update({'rescale': 1./255})\n",
+        "    if use_default_augmentation:\n",
+        "        kwargs.update({\n",
+        "            'rotation_range': 15,\n",
+        "            'zoom_range': 0.2,\n",
+        "            'brightness_range': (0.8, 1.2),\n",
+        "            'channel_shift_range': 30,\n",
+        "            'horizontal_flip': True,\n",
+        "        })\n",
+        "    return ImageDataGenerator(**kwargs)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {
+        "id": "nrH7Fz6EM4mk"
+      },
+      "outputs": [],
+      "source": [
+        "def get_train_data_generator(\n",
+        "    train_data_dir, \n",
+        "    batch_size, \n",
+        "    validation_split=None, \n",
+        "    use_default_augmentation=True,\n",
+        "    augmentations=None\n",
+        "):\n",
+        "    if not augmentations:\n",
+        "        augmentations = {}\n",
+        "\n",
+        "    train_datagen = get_datagen(\n",
+        "        use_default_augmentation=use_default_augmentation,\n",
+        "        validation_split=validation_split if validation_split else 0.0,\n",
+        "        **augmentations\n",
+        "    )\n",
+        "   \n",
+        "    train_generator = train_datagen.flow_from_directory(\n",
+        "        directory=train_data_dir,\n",
+        "        target_size=(IMG_WIDTH, IMG_WIDTH),\n",
+        "        batch_size=batch_size,\n",
+        "        class_mode='binary',\n",
+        "        subset='training',\n",
+        "    )\n",
+        "\n",
+        "    validation_generator = None\n",
+        "\n",
+        "    if validation_split:\n",
+        "        validation_generator = train_datagen.flow_from_directory(\n",
+        "            directory=train_data_dir,\n",
+        "            target_size=(IMG_WIDTH, IMG_WIDTH),\n",
+        "            batch_size=batch_size,\n",
+        "            class_mode='binary',\n",
+        "            subset='validation'\n",
+        "        )\n",
+        "\n",
+        "    return train_generator, validation_generator"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {
+        "id": "6G7tVf0wNHvd"
+      },
+      "outputs": [],
+      "source": [
+        "def get_test_data_generator(test_data_dir, batch_size, shuffle=False):\n",
+        "    test_datagen = get_datagen(use_default_augmentation=False)\n",
+        "    return test_datagen.flow_from_directory(\n",
+        "        directory=test_data_dir,\n",
+        "        target_size=(IMG_WIDTH, IMG_WIDTH),\n",
+        "        batch_size=batch_size,\n",
+        "        class_mode='binary',\n",
+        "        shuffle=shuffle\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "metadata": {
+        "id": "kSWlbqMv-TK4"
+      },
+      "outputs": [],
+      "source": [
+        "def activation_layer(ip, activation, *args):\n",
+        "    return {'relu': ReLU(*args)(ip),\n",
+        "            'elu': ELU(*args)(ip),\n",
+        "            'lrelu': LeakyReLU(*args)(ip)}[activation]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "metadata": {
+        "id": "CvF1f4Y28oPM"
+      },
+      "outputs": [],
+      "source": [
+        "def conv2D(ip,\n",
+        "           filters,\n",
+        "           kernel_size,\n",
+        "           activation,\n",
+        "           padding='same',\n",
+        "           pool_size=(2, 2)):\n",
+        "    layer = Conv2D(filters,\n",
+        "                   kernel_size=kernel_size,\n",
+        "                   padding=padding)(ip)\n",
+        "\n",
+        "    layer = activation_layer(layer, activation=activation)\n",
+        "\n",
+        "    layer = BatchNormalization()(layer)\n",
+        "\n",
+        "    return MaxPooling2D(pool_size=pool_size, padding=padding)(layer)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {
+        "id": "d-4--jRd-bz1"
+      },
+      "outputs": [],
+      "source": [
+        "def fully_connected_layer(ip,\n",
+        "                          hidden_activation,\n",
+        "                          dropout):\n",
+        "    layer = Dense(16)(ip)\n",
+        "    layer = activation_layer(layer, hidden_activation, *[0.1,])\n",
+        "    return Dropout(rate=dropout)(layer)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "id": "1Cp48aFy_k4G"
+      },
+      "outputs": [],
+      "source": [
+        "def build_model(ip=Input(shape=(IMG_WIDTH, IMG_WIDTH, 3)),\n",
+        "                activation='relu',\n",
+        "                dropout=0.5,\n",
+        "                hidden_activation='lrelu'):\n",
+        "    \n",
+        "    layer = conv2D(ip, filters=8, kernel_size=(3, 3), activation=activation)\n",
+        "\n",
+        "    layer = conv2D(layer, filters=8, kernel_size=(5, 5), activation=activation)\n",
+        "\n",
+        "    layer = conv2D(layer, filters=16, kernel_size=(5, 5), activation=activation)\n",
+        "\n",
+        "    layer = conv2D(layer, filters=16, kernel_size=(5, 5), activation=activation, pool_size=(4, 4))\n",
+        "\n",
+        "    layer = Flatten()(layer)\n",
+        "    layer = Dropout(rate=dropout)(layer)\n",
+        "\n",
+        "    layer = fully_connected_layer(layer, hidden_activation=hidden_activation, dropout=dropout)\n",
+        "\n",
+        "    op_layer = Dense(1, activation='sigmoid')(layer)\n",
+        "\n",
+        "    model = Model(ip, op_layer)\n",
+        "\n",
+        "    return model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {
+        "id": "ZdoMu0LbDGMC"
+      },
+      "outputs": [],
+      "source": [
+        "def evaluate_model(model, test_data_dir, batch_size):\n",
+        "    data = get_test_data_generator(test_data_dir, batch_size)\n",
+        "    return model.evaluate(data)\n",
+        "\n",
+        "\n",
+        "def predict(model, data, steps=None, threshold=0.5):\n",
+        "    predictions = model.predict(data, steps=steps, verbose=1)\n",
+        "    return predictions, np.where(predictions >= threshold, 1, 0)\n",
+        "\n",
+        "\n",
+        "def save_model_history(history, filename):\n",
+        "    with open(filename, 'wb') as f:\n",
+        "        pickle.dump(history.history, f)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "id": "fuXsZWxke_ic"
+      },
+      "outputs": [],
+      "source": [
+        "def get_activation_model(model, conv_idx):\n",
+        "    conv_layers = [layer for layer in model.layers if 'conv' in layer.name]\n",
+        "    selected_layers = [layer for index, layer in enumerate(conv_layers) if index in conv_idx]\n",
+        "    activation_model = Model(\n",
+        "        inputs=model.inputs,\n",
+        "        outputs=[layer.output for layer in selected_layers]\n",
+        "    )\n",
+        "    return activation_model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 15,
+      "metadata": {
+        "id": "W0Sda_34HCzQ"
+      },
+      "outputs": [],
+      "source": [
+        "def train_model(model,\n",
+        "                train_data_dir,\n",
+        "                validation_split=None,\n",
+        "                batch_size=32,\n",
+        "                use_default_augmentation=True,\n",
+        "                augmentations=None,\n",
+        "                epochs=30,\n",
+        "                lr=1e-3,\n",
+        "                loss='binary_crossentropy',\n",
+        "                compile=True,\n",
+        "                lr_decay=True,\n",
+        "                decay_rate=0.10,\n",
+        "                decay_limit=1e-6,\n",
+        "                checkpoint=True,\n",
+        "                stop_early=True,\n",
+        "                monitor='val_accuracy',\n",
+        "                mode='max',\n",
+        "                patience=20,\n",
+        "                tensorboard=True,\n",
+        "                loss_curve=True):\n",
+        "    \n",
+        "\n",
+        "    train_generator, validation_generator = get_train_data_generator(\n",
+        "        train_data_dir=train_data_dir,\n",
+        "        batch_size=batch_size,\n",
+        "        validation_split=validation_split,\n",
+        "        use_default_augmentation=use_default_augmentation,\n",
+        "        augmentations=augmentations\n",
+        "    )\n",
+        "\n",
+        "    callbacks = []\n",
+        "    if checkpoint:\n",
+        "        filepath = f'mesonet_trained.hdf5'\n",
+        "        model_checkpoint = ModelCheckpoint(\n",
+        "            filepath, monitor='val_accuracy', verbose=1,\n",
+        "            save_best_only=True\n",
+        "        )\n",
+        "        callbacks.append(model_checkpoint)\n",
+        "\n",
+        "    if stop_early:\n",
+        "        callbacks.append(\n",
+        "            EarlyStopping(\n",
+        "                monitor=monitor,\n",
+        "                mode=mode,\n",
+        "                patience=patience,\n",
+        "                verbose=1\n",
+        "            )\n",
+        "        )\n",
+        "\n",
+        "\n",
+        "    history = model.fit(\n",
+        "        train_generator,\n",
+        "        epochs=epochs,\n",
+        "        verbose=1,\n",
+        "        callbacks=callbacks,\n",
+        "        validation_data=validation_generator,\n",
+        "        steps_per_epoch=train_generator.samples // batch_size,\n",
+        "        validation_steps=validation_generator.samples // batch_size if validation_generator else None,\n",
+        "    )\n",
+        "\n",
+        "    return history"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 14,
+      "metadata": {
+        "id": "aZtXFYPDoLZp"
+      },
+      "outputs": [
+        {
+          "ename": "OSError",
+          "evalue": "Unable to open file (file signature not found)",
+          "output_type": "error",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
+            "Input \u001b[0;32mIn [14]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m model_exp \u001b[38;5;241m=\u001b[39m \u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m/Users/jarvis/pymycod/Deepfakes/Meso_4.ipynb\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
+            "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/keras/utils/traceback_utils.py:67\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:  \u001b[38;5;66;03m# pylint: disable=broad-except\u001b[39;00m\n\u001b[1;32m     66\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m---> 67\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     68\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     69\u001b[0m   \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
+            "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/h5py/_hl/files.py:507\u001b[0m, in \u001b[0;36mFile.__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, **kwds)\u001b[0m\n\u001b[1;32m    502\u001b[0m     fapl \u001b[38;5;241m=\u001b[39m make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0,\n\u001b[1;32m    503\u001b[0m                      locking, page_buf_size, min_meta_keep, min_raw_keep, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds)\n\u001b[1;32m    504\u001b[0m     fcpl \u001b[38;5;241m=\u001b[39m make_fcpl(track_order\u001b[38;5;241m=\u001b[39mtrack_order, fs_strategy\u001b[38;5;241m=\u001b[39mfs_strategy,\n\u001b[1;32m    505\u001b[0m                      fs_persist\u001b[38;5;241m=\u001b[39mfs_persist, fs_threshold\u001b[38;5;241m=\u001b[39mfs_threshold,\n\u001b[1;32m    506\u001b[0m                      fs_page_size\u001b[38;5;241m=\u001b[39mfs_page_size)\n\u001b[0;32m--> 507\u001b[0m     fid \u001b[38;5;241m=\u001b[39m make_fid(name, mode, userblock_size, fapl, fcpl, swmr\u001b[38;5;241m=\u001b[39mswmr)\n\u001b[1;32m    509\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(libver, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m    510\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_libver \u001b[38;5;241m=\u001b[39m libver\n",
+            "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/h5py/_hl/files.py:220\u001b[0m, in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m    218\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m swmr \u001b[38;5;129;01mand\u001b[39;00m swmr_support:\n\u001b[1;32m    219\u001b[0m         flags \u001b[38;5;241m|\u001b[39m\u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mACC_SWMR_READ\n\u001b[0;32m--> 220\u001b[0m     fid \u001b[38;5;241m=\u001b[39m \u001b[43mh5f\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    221\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr+\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m    222\u001b[0m     fid \u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mopen(name, h5f\u001b[38;5;241m.\u001b[39mACC_RDWR, fapl\u001b[38;5;241m=\u001b[39mfapl)\n",
+            "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
+            "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
+            "File \u001b[0;32mh5py/h5f.pyx:106\u001b[0m, in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n",
+            "\u001b[0;31mOSError\u001b[0m: Unable to open file (file signature not found)"
+          ]
+        }
+      ],
+      "source": [
+        "model_exp = load_model('/Users/jarvis/pymycod/Deepfakes/Meso_4.ipynb')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "IC4V7HflFZC2",
+        "outputId": "c86ee986-baeb-449c-b024-d6ad7ecdaf2a"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Found 1945 images belonging to 2 classes.\n",
+            "31/31 [==============================] - 7s 235ms/step - loss: 0.0998 - accuracy: 0.9625\n"
+          ]
+        },
+        {
+          "data": {
+            "text/plain": [
+              "[0.09982584416866302, 0.9624678492546082]"
+            ]
+          },
+          "execution_count": 124,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "evaluate_model(model_exp, 'data/test', 64)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "fG6V0lOzFeg2",
+        "outputId": "db1fec59-cd05-440f-9ae6-acaea21db0ea"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Found 1945 images belonging to 2 classes.\n",
+            "31/31 [==============================] - 7s 235ms/step\n",
+            "              precision    recall  f1-score   support\n",
+            "\n",
+            "           0       0.96      0.94      0.95       773\n",
+            "           1       0.96      0.97      0.97      1172\n",
+            "\n",
+            "    accuracy                           0.96      1945\n",
+            "   macro avg       0.96      0.96      0.96      1945\n",
+            "weighted avg       0.96      0.96      0.96      1945\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(get_classification_report(model_exp, 'data/test'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cjHCL4fiF29I"
+      },
+      "outputs": [],
+      "source": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "_z00HkaPG76d",
+        "outputId": "f3d07ca5-29dd-4191-ef04-e6ef04bf592a"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Found 1945 images belonging to 2 classes.\n",
+            "31/31 [==============================] - 8s 241ms/step - loss: 0.2321 - accuracy: 0.9080\n"
+          ]
+        },
+        {
+          "data": {
+            "text/plain": [
+              "[0.23209184408187866, 0.9079691767692566]"
+            ]
+          },
+          "execution_count": 129,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "evaluate_model(model_exp, 'data/test', 64)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "v3ZQN53bHBEf",
+        "outputId": "14bd2bd7-0c83-4e52-ee12-054a4e52ca9f"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Found 1945 images belonging to 2 classes.\n",
+            "31/31 [==============================] - 7s 234ms/step\n",
+            "              precision    recall  f1-score   support\n",
+            "\n",
+            "           0       0.90      0.87      0.88       773\n",
+            "           1       0.91      0.93      0.92      1172\n",
+            "\n",
+            "    accuracy                           0.91      1945\n",
+            "   macro avg       0.91      0.90      0.90      1945\n",
+            "weighted avg       0.91      0.91      0.91      1945\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(get_classification_report(model_exp, 'data/test'))"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "Meso-4.ipynb",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.13"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

first.ipynb

@@ -0,0 +1,265 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import cv2\n",
+    "import torch\n",
+    "from onnx2pytorch import ConvertModel\n",
+    "from keras.models import load_model\n",
+    "import onnx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Metal device set to: Apple M1\n",
+      "\n",
+      "systemMemory: 8.00 GB\n",
+      "maxCacheSize: 2.67 GB\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-02-01 19:38:26.414359: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
+      "2024-02-01 19:38:26.414541: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "model1 = load_model('/Users/jarvis/pymycod/Deepfakes/DeepDetect/mesonet_trained.hdf5')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1/1 [==============================] - 0s 198ms/step\n",
+      "the image is realllll boii\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-02-01 14:03:47.045296: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
+      "2024-02-01 14:03:47.118034: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import keras.utils as image\n",
+    "\n",
+    "img_width, img_height = 256,256\n",
+    "img = image.load_img(f'/Users/jarvis/Downloads/im6.jpeg', target_size = (img_width, img_height))\n",
+    "img = image.img_to_array(img)\n",
+    "img = np.expand_dims(img, axis = 0)\n",
+    "ans = model1.predict(img)\n",
+    "if ans[0] ==0:\n",
+    "    print(\"the image is fake afff\")\n",
+    "else:\n",
+    "    print(\"the image is realllll boii\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "DecodeError",
+     "evalue": "Wrong wire type in tag.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mDecodeError\u001b[0m                               Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [11]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m onnx_model \u001b[38;5;241m=\u001b[39m \u001b[43monnx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m/Users/jarvis/pymycod/Deepfakes/multimodal_deepfake_detection/checkpoints/efficientnet.onnx\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/onnx/__init__.py:208\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(f, format, load_external_data)\u001b[0m\n\u001b[1;32m    187\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_model\u001b[39m(\n\u001b[1;32m    188\u001b[0m     f: IO[\u001b[38;5;28mbytes\u001b[39m] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m|\u001b[39m os\u001b[38;5;241m.\u001b[39mPathLike,\n\u001b[1;32m    189\u001b[0m     \u001b[38;5;28mformat\u001b[39m: _SupportedFormat \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    190\u001b[0m     load_external_data: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    191\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ModelProto:\n\u001b[1;32m    192\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Loads a serialized ModelProto into memory.\u001b[39;00m\n\u001b[1;32m    193\u001b[0m \n\u001b[1;32m    194\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[38;5;124;03m        Loaded in-memory ModelProto.\u001b[39;00m\n\u001b[1;32m    207\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 208\u001b[0m     model \u001b[38;5;241m=\u001b[39m \u001b[43m_get_serializer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdeserialize_proto\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_load_bytes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mModelProto\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    210\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m load_external_data:\n\u001b[1;32m    211\u001b[0m         model_filepath \u001b[38;5;241m=\u001b[39m _get_file_path(f)\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/onnx/serialization.py:118\u001b[0m, in \u001b[0;36m_ProtobufSerializer.deserialize_proto\u001b[0;34m(self, serialized, proto)\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(serialized, \u001b[38;5;28mbytes\u001b[39m):\n\u001b[1;32m    115\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m    116\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mParameter \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mserialized\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m must be bytes, but got type: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(serialized)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    117\u001b[0m     )\n\u001b[0;32m--> 118\u001b[0m decoded \u001b[38;5;241m=\u001b[39m typing\u001b[38;5;241m.\u001b[39mcast(Optional[\u001b[38;5;28mint\u001b[39m], \u001b[43mproto\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mParseFromString\u001b[49m\u001b[43m(\u001b[49m\u001b[43mserialized\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    119\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decoded \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m decoded \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(serialized):\n\u001b[1;32m    120\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m google\u001b[38;5;241m.\u001b[39mprotobuf\u001b[38;5;241m.\u001b[39mmessage\u001b[38;5;241m.\u001b[39mDecodeError(\n\u001b[1;32m    121\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mProtobuf decoding consumed too few bytes: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdecoded\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m out of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(serialized)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    122\u001b[0m     )\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/google/protobuf/message.py:202\u001b[0m, in \u001b[0;36mMessage.ParseFromString\u001b[0;34m(self, serialized)\u001b[0m\n\u001b[1;32m    194\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Parse serialized protocol buffer data into this message.\u001b[39;00m\n\u001b[1;32m    195\u001b[0m \n\u001b[1;32m    196\u001b[0m \u001b[38;5;124;03mLike :func:`MergeFromString()`, except we clear the object first.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;124;03m  message.DecodeError if the input cannot be parsed.\u001b[39;00m\n\u001b[1;32m    200\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    201\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mClear()\n\u001b[0;32m--> 202\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mMergeFromString\u001b[49m\u001b[43m(\u001b[49m\u001b[43mserialized\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/google/protobuf/internal/python_message.py:1128\u001b[0m, in \u001b[0;36m_AddMergeFromStringMethod.<locals>.MergeFromString\u001b[0;34m(self, serialized)\u001b[0m\n\u001b[1;32m   1126\u001b[0m length \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(serialized)\n\u001b[1;32m   1127\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1128\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_InternalParse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mserialized\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlength\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m!=\u001b[39m length:\n\u001b[1;32m   1129\u001b[0m     \u001b[38;5;66;03m# The only reason _InternalParse would return early is if it\u001b[39;00m\n\u001b[1;32m   1130\u001b[0m     \u001b[38;5;66;03m# encountered an end-group tag.\u001b[39;00m\n\u001b[1;32m   1131\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m message_mod\u001b[38;5;241m.\u001b[39mDecodeError(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUnexpected end-group tag.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m   1132\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mIndexError\u001b[39;00m, \u001b[38;5;167;01mTypeError\u001b[39;00m):\n\u001b[1;32m   1133\u001b[0m   \u001b[38;5;66;03m# Now ord(buf[p:p+1]) == ord('') gets TypeError.\u001b[39;00m\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/google/protobuf/internal/python_message.py:1181\u001b[0m, in \u001b[0;36m_AddMergeFromStringMethod.<locals>.InternalParse\u001b[0;34m(self, buffer, pos, end)\u001b[0m\n\u001b[1;32m   1179\u001b[0m \u001b[38;5;66;03m# TODO(jieluo): remove old_pos.\u001b[39;00m\n\u001b[1;32m   1180\u001b[0m old_pos \u001b[38;5;241m=\u001b[39m new_pos\n\u001b[0;32m-> 1181\u001b[0m (data, new_pos) \u001b[38;5;241m=\u001b[39m \u001b[43mdecoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_DecodeUnknownField\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1182\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_pos\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwire_type\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[1;32m   1183\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m new_pos \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m   1184\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m pos\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/google/protobuf/internal/decoder.py:965\u001b[0m, in \u001b[0;36m_DecodeUnknownField\u001b[0;34m(buffer, pos, wire_type)\u001b[0m\n\u001b[1;32m    963\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m (\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m    964\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 965\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m _DecodeError(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mWrong wire type in tag.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m    967\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m (data, pos)\n",
+      "\u001b[0;31mDecodeError\u001b[0m: Wrong wire type in tag."
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "onnx_model = onnx.load('/Users/jarvis/pymycod/Deepfakes/multimodal_deepfake_detection/checkpoints/efficientnet.onnx')\n",
+    "# pytorch_model = ConvertModel(onnx_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "def load_img_modality_model(args):\n",
+    "    '''Loads image modality model.'''\n",
+    "    rgb_encoder = pytorch_model\n",
+    "\n",
+    "    ckpt = torch.load('checkpoints/model.pth', map_location = torch.device('cpu'))\n",
+    "    rgb_encoder.load_state_dict(ckpt['rgb_encoder'], strict = True)\n",
+    "    rgb_encoder.eval()\n",
+    "    return rgb_encoder\n",
+    "img_model = load_img_modality_model(args)\n",
+    "\n",
+    "def preprocess_img(face):\n",
+    "    face = face / 255\n",
+    "    face = cv2.resize(face, (256, 256))\n",
+    "    # face = face.transpose(2, 0, 1) #(W, H, C) -> (C, W, H)\n",
+    "    face_pt = torch.unsqueeze(torch.Tensor(face), dim = 0) \n",
+    "    return face_pt\n",
+    "def preprocess_video(input_video, n_frames = 3):\n",
+    "    v_cap = cv2.VideoCapture(input_video)\n",
+    "    v_len = int(v_cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
+    "\n",
+    "    # Pick 'n_frames' evenly spaced frames to sample\n",
+    "    if n_frames is None:\n",
+    "        sample = np.arange(0, v_len)\n",
+    "    else:\n",
+    "        sample = np.linspace(0, v_len - 1, n_frames).astype(int)\n",
+    "\n",
+    "    #Loop through frames.\n",
+    "    frames = []\n",
+    "    for j in range(v_len):\n",
+    "        success = v_cap.grab()\n",
+    "        if j in sample:\n",
+    "            # Load frame\n",
+    "            success, frame = v_cap.retrieve()\n",
+    "            if not success:\n",
+    "                continue\n",
+    "            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
+    "            frame = preprocess_img(frame)\n",
+    "            frames.append(frame)\n",
+    "    v_cap.release()\n",
+    "    return frames\n",
+    "\n",
+    "\n",
+    "def deepfakes_video_predict(input_video):\n",
+    "    '''Perform inference on a video.'''\n",
+    "    video_frames = preprocess_video(input_video)\n",
+    "    real_faces_list = []\n",
+    "    fake_faces_list = []\n",
+    "\n",
+    "    for face in video_frames:\n",
+    "        # face = preprocess_img(face)\n",
+    "\n",
+    "        img_grads = img_model.forward(face)\n",
+    "        img_grads = img_grads.cpu().detach().numpy()\n",
+    "        img_grads_np = np.squeeze(img_grads)\n",
+    "        real_faces_list.append(img_grads_np[0])\n",
+    "        fake_faces_list.append(img_grads_np[1])\n",
+    "\n",
+    "    real_faces_mean = np.mean(real_faces_list)\n",
+    "    fake_faces_mean = np.mean(fake_faces_list)\n",
+    "\n",
+    "    if real_faces_mean > 0.5:\n",
+    "        preds = round(real_faces_mean * 100, 3)\n",
+    "        text2 = f\"The video is REAL. \\nConfidence score is: {preds}%\"\n",
+    "\n",
+    "    else:\n",
+    "        preds = round(fake_faces_mean * 100, 3)\n",
+    "        text2 = f\"The video is FAKE. \\nConfidence score is: {preds}%\"\n",
+    "\n",
+    "    return text2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "'Functional' object has no attribute 'forward'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [25]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdeepfakes_video_predict\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/Users/jarvis/Documents/Ss/ras_df.mov\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "Input \u001b[0;32mIn [24]\u001b[0m, in \u001b[0;36mdeepfakes_video_predict\u001b[0;34m(input_video)\u001b[0m\n\u001b[1;32m     37\u001b[0m fake_faces_list \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     39\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m face \u001b[38;5;129;01min\u001b[39;00m video_frames:\n\u001b[1;32m     40\u001b[0m     \u001b[38;5;66;03m# face = preprocess_img(face)\u001b[39;00m\n\u001b[0;32m---> 42\u001b[0m     img_grads \u001b[38;5;241m=\u001b[39m \u001b[43mmodel1\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m(face)\n\u001b[1;32m     43\u001b[0m     img_grads \u001b[38;5;241m=\u001b[39m img_grads\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[1;32m     44\u001b[0m     img_grads_np \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39msqueeze(img_grads)\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'Functional' object has no attribute 'forward'"
+     ]
+    }
+   ],
+   "source": [
+    "deepfakes_video_predict(\"/Users/jarvis/Documents/Ss/ras_df.mov\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ValidationError",
+     "evalue": "Unable to parse proto from file: /Users/jarvis/pymycod/Deepfakes/multimodal_deepfake_detection/checkpoints/efficientnet.onnx. Please check if it is a valid protobuf file of proto. ",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mValidationError\u001b[0m                           Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [10]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01monnx\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m checker\n\u001b[0;32m----> 3\u001b[0m \u001b[43mchecker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/Users/jarvis/pymycod/Deepfakes/multimodal_deepfake_detection/checkpoints/efficientnet.onnx\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/tensor/lib/python3.8/site-packages/onnx/checker.py:137\u001b[0m, in \u001b[0;36mcheck_model\u001b[0;34m(model, full_check, skip_opset_compatibility_check)\u001b[0m\n\u001b[1;32m    135\u001b[0m \u001b[38;5;66;03m# If model is a path instead of ModelProto\u001b[39;00m\n\u001b[1;32m    136\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(model, (\u001b[38;5;28mstr\u001b[39m, os\u001b[38;5;241m.\u001b[39mPathLike)):\n\u001b[0;32m--> 137\u001b[0m     \u001b[43mC\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_model_path\u001b[49m\u001b[43m(\u001b[49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfspath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfull_check\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mskip_opset_compatibility_check\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    138\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    139\u001b[0m     protobuf_string \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    140\u001b[0m         model \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(model, \u001b[38;5;28mbytes\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m model\u001b[38;5;241m.\u001b[39mSerializeToString()\n\u001b[1;32m    141\u001b[0m     )\n",
+      "\u001b[0;31mValidationError\u001b[0m: Unable to parse proto from file: /Users/jarvis/pymycod/Deepfakes/multimodal_deepfake_detection/checkpoints/efficientnet.onnx. Please check if it is a valid protobuf file of proto. "
+     ]
+    }
+   ],
+   "source": [
+    "from onnx import checker\n",
+    "\n",
+    "checker.check_model(\"/Users/jarvis/pymycod/Deepfakes/multimodal_deepfake_detection/checkpoints/efficientnet.onnx\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "tensor",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

gradio_api.py

@@ -0,0 +1,10 @@
+from gradio_client import Client
+
+client = Client("http://127.0.0.1:7860/")
+# result = client.predict(
+# 		"C:\\Users\\hp\\Downloads\\im1.jpeg",api_name="/predict"	
+# )
+
+result = client.predict({"video":"C:\\Users\\hp\\Downloads\\ras_df.mp4",
+                        "subtitles":None},	api_name="/predict_1")
+print(result)

inference_2.py

@@ -0,0 +1,153 @@
+import cv2
+import onnx
+import torch
+import argparse
+import numpy as np
+from models import image
+
+import warnings
+from onnx2pytorch import ConvertModel
+
+warnings.filterwarnings("ignore", message="The given NumPy array is not writable")
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", message="The given NumPy array is not writable")
+    onnx_model = onnx.load('models/efficientnet.onnx')
+    pytorch_model = ConvertModel(onnx_model)
+torch.manual_seed(42)
+
+
+audio_args = { 'nb_samp': 64600, 'first_conv': 1024, 'in_channels': 1, 'filts': [20, [20, 20], [20, 128], [128, 128]], 'blocks': [2, 4],'nb_fc_node': 1024,'gru_node': 1024, 'nb_gru_layer': 3, 'nb_classes': 2}
+
+
+def get_args(parser):
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--data_dir", type=str, default="datasets/train/fakeavceleb*")
+    parser.add_argument("--LOAD_SIZE", type=int, default=256)
+    parser.add_argument("--FINE_SIZE", type=int, default=224)
+    parser.add_argument("--dropout", type=float, default=0.2)
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=1)
+    parser.add_argument("--hidden", nargs="*", type=int, default=[])
+    parser.add_argument("--hidden_sz", type=int, default=768)
+    parser.add_argument("--img_embed_pool_type", type=str, default="avg", choices=["max", "avg"])
+    parser.add_argument("--img_hidden_sz", type=int, default=1024)
+    parser.add_argument("--include_bn", type=int, default=True)
+    parser.add_argument("--lr", type=float, default=1e-4)
+    parser.add_argument("--lr_factor", type=float, default=0.3)
+    parser.add_argument("--lr_patience", type=int, default=10)
+    parser.add_argument("--max_epochs", type=int, default=500)
+    parser.add_argument("--n_workers", type=int, default=12)
+    parser.add_argument("--name", type=str, default="MMDF")
+    parser.add_argument("--num_image_embeds", type=int, default=1)
+    parser.add_argument("--patience", type=int, default=20)
+    parser.add_argument("--savedir", type=str, default="./savepath/")
+    parser.add_argument("--seed", type=int, default=1)
+    parser.add_argument("--n_classes", type=int, default=2)
+    parser.add_argument("--annealing_epoch", type=int, default=10)
+    parser.add_argument("--device", type=str, default='cpu')
+    parser.add_argument("--pretrained_image_encoder", type=bool, default = False)
+    parser.add_argument("--freeze_image_encoder", type=bool, default = False)
+    parser.add_argument("--pretrained_audio_encoder", type = bool, default=False)
+    parser.add_argument("--freeze_audio_encoder", type = bool, default = False)
+    parser.add_argument("--augment_dataset", type = bool, default = True)
+
+    for key, value in audio_args.items():
+        parser.add_argument(f"--{key}", type=type(value), default=value)
+
+def load_img_modality_model(args):
+    rgb_encoder = pytorch_model
+    ckpt = torch.load('models/model.pth', map_location = torch.device('cpu'))
+    rgb_encoder.load_state_dict(ckpt['rgb_encoder'], strict = True)
+    rgb_encoder.eval()
+    return rgb_encoder
+
+def load_spec_modality_model(args):
+    spec_encoder = image.RawNet(args)
+    ckpt = torch.load('models/model.pth', map_location = torch.device('cpu'))
+    spec_encoder.load_state_dict(ckpt['spec_encoder'], strict = True)
+    spec_encoder.eval()
+    return spec_encoder
+
+parser = argparse.ArgumentParser(description="Inference models")
+get_args(parser)
+args, remaining_args = parser.parse_known_args()
+assert remaining_args == [], remaining_args
+spec_model = load_spec_modality_model(args)
+img_model = load_img_modality_model(args)
+
+def preprocess_img(face):
+    face = face / 255
+    face = cv2.resize(face, (256, 256))
+    face_pt = torch.unsqueeze(torch.Tensor(face), dim = 0) 
+    return face_pt
+
+def preprocess_audio(audio_file):
+    audio_pt = torch.unsqueeze(torch.Tensor(audio_file), dim = 0)
+    return audio_pt
+
+def df_spec_pred(input_audio):
+    x, _ = input_audio
+    audio = preprocess_audio(x)
+    spec_grads = spec_model.forward(audio)
+    spec_grads_inv = np.exp(spec_grads.cpu().detach().numpy().squeeze())
+    max_value = np.argmax(spec_grads_inv)
+    if max_value > 0.5:
+        preds = round(100 - (max_value*100), 3)
+        text2 = f"The audio is REAL."
+    else:
+        preds = round(max_value*100, 3)
+        text2 = f"The audio is FAKE."
+    return text2
+
+def df_img_pred(input_image):
+    face = preprocess_img(input_image)
+    print(f"Face shape is: {face.shape}")
+    img_grads = img_model.forward(face)
+    img_grads = img_grads.cpu().detach().numpy()
+    img_grads_np = np.squeeze(img_grads)
+    if img_grads_np[0] > 0.5:
+        preds = round(img_grads_np[0] * 100, 3)
+        text2 = f"The image is REAL. \nConfidence score is: {preds}"
+    else:
+        preds = round(img_grads_np[1] * 100, 3)
+        text2 = f"The image is FAKE. \nConfidence score is: {preds}"
+    return text2
+
+def preprocess_video(input_video, n_frames = 3):
+    v_cap = cv2.VideoCapture(input_video)
+    v_len = int(v_cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    if n_frames is None:
+        sample = np.arange(0, v_len)
+    else:
+        sample = np.linspace(0, v_len - 1, n_frames).astype(int)
+    frames = []
+    for j in range(v_len):
+        success = v_cap.grab()
+        if j in sample:
+            success, frame = v_cap.retrieve()
+            if not success:
+                continue
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            frame = preprocess_img(frame)
+            frames.append(frame)
+    v_cap.release()
+    return frames
+
+def df_video_pred(input_video):
+    video_frames = preprocess_video(input_video)
+    real_faces_list = []
+    fake_faces_list = []
+    for face in video_frames:
+        img_grads = img_model.forward(face)
+        img_grads = img_grads.cpu().detach().numpy()
+        img_grads_np = np.squeeze(img_grads)
+        real_faces_list.append(img_grads_np[0])
+        fake_faces_list.append(img_grads_np[1])
+    real_faces_mean = np.mean(real_faces_list)
+    fake_faces_mean = np.mean(fake_faces_list)
+    if real_faces_mean > 0.5:
+        preds = round(real_faces_mean * 100, 3)
+        text2 = f"The video is REAL. \nConfidence score is: {preds}%"
+    else:
+        preds = round(fake_faces_mean * 100, 3)
+        text2 = f"The video is FAKE. \nConfidence score is: {preds}%"
+    return text2