Well_known_algorithm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Well_known_algorithm.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMV5+TTnNV7vvqi4ey6NgEl",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShakutaiGit/Final_Project_Applied_Data_science/blob/main/Well_known_algorithm.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "40yCnCufvuBl"
      },
      "source": [
        "relevant imports "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "luTxD34ave3s"
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_datasets as tfds\n",
        "from tensorflow.keras.layers.experimental import preprocessing\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.utils import plot_model \n",
        "from tensorflow.python.keras.utils import np_utils\n",
        "from tensorflow.python.keras.models import Sequential, Model\n",
        "from tensorflow.python.keras.layers import Dense, Dropout, Flatten, Activation, BatchNormalization\n",
        "from tensorflow.python.keras.layers.convolutional import Conv2D, MaxPooling2D \n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "!pip install optuna\n",
        "import optuna\n",
        "from sklearn.model_selection import KFold\n",
        "from skimage.transform import resize\n",
        "from skimage.transform import rescale\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.models import Sequential\n",
        "import cv2\n",
        "from statistics import mean\n",
        "from google.colab import files\n",
        "import time \n",
        "import csv\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8OCUqyJev-QR"
      },
      "source": [
        "global variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J0rBEpzfwA4x"
      },
      "source": [
        "optimizer = None\n",
        "size = 128 \n",
        "batch_size = 128 # like the original paper \n",
        "classes = 0\n",
        "AUTOTUNE = tf.data.AUTOTUNE\n",
        "x_train = []\n",
        "x_test = []\n",
        "y_train = []\n",
        "y_test = []\n",
        "data_shape = None \n",
        "algorithem_name = 'Well_known'\n",
        "database_name = None"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xdFZNHN6v1D-"
      },
      "source": [
        "loading the dataset and pre processing "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tLsH9p4Kv47W"
      },
      "source": [
        "train_ds ,train_info= tfds.load('oxford_flowers102', split='train[:80%]', with_info=True)\n",
        "test_ds , test_info = tfds.load('oxford_flowers102', split='train[80%:]', with_info=True)\n",
        "## pre processing \n",
        "database_name = train_info.name\n",
        "resize_and_rescale = tf.keras.Sequential([\n",
        "layers.experimental.preprocessing.Resizing(size, size),\n",
        "layers.experimental.preprocessing.Rescaling(1./255)\n",
        "])\n",
        "\n",
        "def pre_process(data):\n",
        "  pictures=[]\n",
        "  labels = []\n",
        "  for i in data:\n",
        "    temp = resize_and_rescale(i['image'])\n",
        "    temp = np.asarray(temp)\n",
        "    pictures.append(temp)\n",
        "    labels.append(int(i['label']))\n",
        "  return np.asarray(pictures),np.asarray(labels)\n",
        "\n",
        "x_train,y_train =pre_process(train_ds)\n",
        "data_shape= x_train[0].shape\n",
        "x_test,y_test =pre_process(test_ds)\n",
        "classes = train_info.features['label'].num_classes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zCrTW96UZRhV"
      },
      "source": [
        "creating the model as described in the report"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dpE2w6Liv8tt"
      },
      "source": [
        "def creating_the_model():\n",
        "  # creating input layer\n",
        "  inputs = layers.Input(shape=(size, size, 3))\n",
        "  # loading the pretrained model \n",
        "  pre_trained_model = keras.applications.ResNet50V2(include_top=False, weights='imagenet',input_tensor=inputs,pooling='avg')\n",
        "  pre_trained_model.trainable= False\n",
        "  #normlizing the out put of the model \n",
        "  x = layers.BatchNormalization()(pre_trained_model.output)\n",
        "  top_dropout_rate = 0.5\n",
        "  # adding dropout layer\n",
        "  x = layers.Dropout(top_dropout_rate, name=\"top_dropout\")(x)\n",
        "  outputs = layers.Dense(classes, activation=\"softmax\", name=\"pred\")(x)\n",
        "  model = tf.keras.Model(inputs, outputs, name=\"ResNet\")\n",
        "  #unfreeze some layers of the pretrained model\n",
        "  for layer in model.layers[-10:]:\n",
        "    if not isinstance(layer, layers.BatchNormalization):\n",
        "        layer.trainable = True\n",
        "  return model\n",
        "\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xTZZEyzTax2T"
      },
      "source": [
        "the obejctive function that describe in the report"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8FiaPOxPdUIE"
      },
      "source": [
        "def objective(trial,outer_x_train,outer_y_train):\n",
        "\n",
        "# inner cross validation loop \n",
        "  internal_kfold = KFold(n_splits=3, shuffle=True)\n",
        "  fold_acc = []  \n",
        "  for inner_train_index,inner_test_index in internal_kfold.split(outer_x_train,outer_y_train):\n",
        "    inner_x_train, inner_x_test = outer_x_train[inner_train_index], outer_x_train[inner_test_index]\n",
        "    inner_y_train, inner_y_test = outer_y_train[inner_train_index], outer_y_train[inner_test_index]\n",
        "    model = creating_the_model()\n",
        "    model.compile(\n",
        "        # hyper paramters \n",
        "    loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(\n",
        "  learning_rate=trial.suggest_float(name=\"lr\", low = 0.00001,high= 0.5, log=True),\n",
        "  beta_1=trial.suggest_float(name=\"beta_1\", low = 0,high=1), \n",
        "  beta_2=trial.suggest_float(name=\"beta_2\", low = 0,high=1)\n",
        "      ), metrics=[\"accuracy\"]\n",
        "  )\n",
        "    # training the model \n",
        "    model.fit(\n",
        "        x=inner_x_train,\n",
        "        y=to_categorical(inner_y_train,num_classes=classes),\n",
        "        # validation_data= (inner_x_test,to_categorical(,inner_y_test)),\n",
        "        # shuffle=True,\n",
        "        validation_split=0.1,\n",
        "        batch_size=batch_size,\n",
        "        epochs=5\n",
        "    )\n",
        "    score = model.evaluate(inner_x_test,to_categorical(inner_y_test, num_classes=classes) , verbose=0)\n",
        "    fold_acc.append(score[1] * 100)\n",
        "  return mean(fold_acc)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0q4hZH1l3SOL"
      },
      "source": [
        "def fold_dict_creator(Hyper_Parameters_Values,score,training_time,inference_time,fold):\n",
        "  fold_dict = {}\n",
        "  fold_dict['Dataset Name'] = database_name\n",
        "  fold_dict['Algorithm Name'] = algorithem_name\n",
        "  fold_dict['Cross Validation [1-10]'] = fold\n",
        "  fold_dict['Hyper-Parameters Values'] = Hyper_Parameters_Values\n",
        "  fold_dict['Accuracy'] = score[1]\n",
        "  fold_dict['TPR'] = score[2]\n",
        "  fold_dict['FPR'] = score[3]\n",
        "  fold_dict['Precision'] = score[4]\n",
        "  fold_dict['AUC'] = score[5]\n",
        "  fold_dict['PR-Curve'] = score[6]\n",
        "  fold_dict['Training Time'] = training_time\n",
        "  fold_dict['Inference Time'] = inference_time\n",
        "  return fold_dict\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UhxOlNiUc4Ki"
      },
      "source": [
        "outer_kfold = KFold(n_splits=10, shuffle=True)\n",
        "fold_num = 0\n",
        "dict_data= []\n",
        "# the outer cross validation loop \n",
        "for outer_train_index,outer_test_index in outer_kfold.split(x_train, y_train):\n",
        "  fold_num += 1\n",
        "  outer_x_train, outer_x_test = x_train[outer_train_index], x_train[outer_test_index]\n",
        "  outer_y_train, outer_y_test = y_train[outer_train_index], y_train[outer_test_index]\n",
        "  study = optuna.create_study(direction=\"maximize\")\n",
        "  # creating the study as described in the report \n",
        "  study.optimize(lambda trail:objective(trail,outer_x_train,outer_y_train), n_trials=50)\n",
        "  model = creating_the_model()\n",
        "  model.compile(\n",
        "      # compiling the model with the parameters that produce the best results \n",
        "    loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(\n",
        "  learning_rate=study.best_params[\"lr\"],\n",
        "  beta_1=study.best_params[\"beta_1\"], \n",
        " beta_2=study.best_params[\"beta_2\"])\n",
        "    # describing which metrics we gunna use \n",
        "      , metrics=[\"accuracy\", tf.keras.metrics.SensitivityAtSpecificity(0.5),\n",
        "      tf.keras.metrics.SpecificityAtSensitivity(0.5), \n",
        "      tf.keras.metrics.Precision(),\n",
        "      tf.keras.metrics.AUC(),\n",
        "      tf.keras.metrics.AUC(curve = 'pr')]\n",
        "  )\n",
        "  start_time = time.time()\n",
        "  model_info = model.fit(\n",
        "    outer_x_train,\n",
        "    to_categorical(outer_y_train,num_classes=classes),\n",
        "    validation_split= 0.15,\n",
        "    shuffle=True,\n",
        "    batch_size=batch_size,\n",
        "    epochs=5,\n",
        "  )\n",
        "  training_time = time.time() - start_time\n",
        "  start_time = time.time()\n",
        "  score = model.evaluate(outer_x_test,to_categorical(outer_y_test,num_classes=classes))\n",
        "  print(model.metrics_names)\n",
        "  inference_time = time.time() - start_time\n",
        "  dict_data.append(fold_dict_creator(str(study.best_params), score, training_time , inference_time, fold_num))\n",
        "csv_file = \"results_algo_{}_data_set{}.csv\".format(algorithem_name,database_name)\n",
        " # writing to csv \n",
        "with open(csv_file, 'w') as csvfile:\n",
        "    writer = csv.DictWriter(csvfile, fieldnames=dict_data[0].keys())\n",
        "    writer.writeheader()\n",
        "    for data in dict_data:\n",
        "        writer.writerow(data)\n",
        "files.download(csv_file)  "
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}