{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pASl9O_Gb4x1"
   },
   "source": [
    "# Auto-encoder with Neural Networks\n",
    "\n",
    "Titouan Vayer, titouan.vayer@inria.fr \n",
    "Aurélien Garivier, aurelien.garivier@ens-lyon.fr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2WZvU5Ocb4x5"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kS_YZhkUb4yT"
   },
   "source": [
    "This hands-on session deals with the construction of an auto-encoder for a dataset of digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SjPoRCFKb4yU"
   },
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7kfz-If-b4yU"
   },
   "outputs": [],
   "source": [
    "digits = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2tAsdFUXb4yV",
    "outputId": "5deb1cdf-05f6-4250-be7e-cdfe362ff6f9"
   },
   "outputs": [],
   "source": [
    "_, axes = plt.subplots(nrows=1, ncols=6, figsize=(10, 3))\n",
    "for ax, image, label in zip(axes, digits.images, digits.target):\n",
    "    ax.set_axis_off()\n",
    "    ax.imshow(image, cmap=plt.cm.gray_r, interpolation=\"nearest\")\n",
    "    ax.set_title(\"Training: %i\" % label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Io7dStGrb4yV"
   },
   "outputs": [],
   "source": [
    "n_samples = len(digits.images)\n",
    "data = digits.images.reshape((n_samples, -1))\n",
    "X, y = data, digits.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KFfaDkb2b4yV",
    "outputId": "efa6f33e-cb17-47e2-c697-cd0bd886f250"
   },
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PdWUXzERb4yV"
   },
   "source": [
    "Let us define our auto-encoder as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WQ-E_wV4b4yW"
   },
   "outputs": [],
   "source": [
    "# 64 ==> 2 ==> 64\n",
    "class AE(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.encoder = torch.nn.Sequential(\n",
    "            torch.nn.Linear(64, 32),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(32, 16),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(16, 2)\n",
    "        )\n",
    "         \n",
    "        self.decoder = torch.nn.Sequential(\n",
    "            torch.nn.Linear(2, 16),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(16, 32),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(32, 64)\n",
    "        )\n",
    " \n",
    "    def forward(self, x):\n",
    "        encoded = self.encoder(x)\n",
    "        decoded = self.decoder(encoded)\n",
    "        return decoded"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4LnUegg2b4yW"
   },
   "source": [
    "#### Q: Comment on this neural network. What is different from the previous nets we have used?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y5wLcSHab4yX"
   },
   "source": [
    "We now train the network by batch gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xZ7_0q8eb4yX"
   },
   "outputs": [],
   "source": [
    "# Initialization\n",
    "model = AE()\n",
    " \n",
    "loss_function = torch.nn.MSELoss()\n",
    " \n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = 5e-3)\n",
    "\n",
    "ds = torch.from_numpy(X).type(torch.float)\n",
    "\n",
    "loader = torch.utils.data.DataLoader(dataset = ds, batch_size = 32, shuffle = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dqQTtZlZb4yX"
   },
   "source": [
    "#### Q: Can you recall the principles of batch gradient descent?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-Ooi-TN6b4yX"
   },
   "outputs": [],
   "source": [
    "epochs = 300 #ca risque de chauffer un peu\n",
    "outputs = []\n",
    "losses = []\n",
    "for epoch in range(epochs):\n",
    "    for img in loader:\n",
    "        # Output of Autoencoder\n",
    "        reconstructed = model(img)\n",
    "\n",
    "        # Calculating the loss function\n",
    "        loss = loss_function(reconstructed, img)\n",
    "\n",
    "        # The gradients are set to zero,\n",
    "        # the gradient is computed and stored.\n",
    "        # .step() performs parameter update\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # Storing the losses in a list for plotting\n",
    "        losses.append(loss.item())\n",
    "    outputs.append((epochs, image, reconstructed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "p5HZQEAXb4yX",
    "outputId": "2e38e42a-e5ae-41ff-83a4-6769d67943e9"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize = (10, 5))\n",
    "plt.xlabel('Iterations', fontsize = 15)\n",
    "plt.ylabel('Loss', fontsize = 15)\n",
    "plt.plot(losses, lw = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8wSDhm8bb4yY",
    "outputId": "75e785aa-fb9e-46f7-c416-1827c2849b91"
   },
   "outputs": [],
   "source": [
    "losses[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C1mXtW5zb4yY"
   },
   "source": [
    "We obtain the low-dimension representation of the data as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DOjmptQEb4yY"
   },
   "outputs": [],
   "source": [
    "XAE = model.encoder(ds).detach().numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ksG_owTKb4yY"
   },
   "source": [
    "Let us compute the average reconstruction error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dBTBhogVb4yY"
   },
   "outputs": [],
   "source": [
    "error_ae = np.sum((X - model.forward(ds).detach().numpy()) ** 2, axis=1).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YUkX6oFhb4yZ",
    "outputId": "a0c3eba9-ce71-4511-c44a-9940634a3991"
   },
   "outputs": [],
   "source": [
    "error_ae"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iVrcKqbTb4yZ"
   },
   "source": [
    "#### We can compare this dimensionality reduction method with the well-known PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fvh8_ILSb4yZ"
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components = 2)\n",
    "Xpca = pca.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZQQatSnKb4yZ"
   },
   "outputs": [],
   "source": [
    "error_pca = np.sum((X - pca.inverse_transform(Xpca)) ** 2, axis=1).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "H4-UXlu1b4ya",
    "outputId": "8c11ff97-2f3c-43b8-d29d-d0a5c9d0dc8e"
   },
   "outputs": [],
   "source": [
    "error_pca"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WzN8nn7Nb4ya"
   },
   "source": [
    "Let us now display the projected data in the two-dimensional embedding space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LXnC0Pd_b4ya"
   },
   "outputs": [],
   "source": [
    "scatter_kwargs_digits = {'s': 5, 'alpha': 0.9, 'c': y, 'cmap': plt.get_cmap('tab10')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "01zZN-uLb4ya",
    "outputId": "c7982718-f00e-47cc-b468-8c3c32c31161"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize = (15, 5))\n",
    "plt.subplot(1,2,1)\n",
    "plt.scatter(Xpca[:,0], Xpca[:,1], **scatter_kwargs_digits)\n",
    "plt.title('PCA embedding', fontsize = 15)\n",
    "plt.subplot(1,2,2)\n",
    "plt.scatter(XAE[:,0], XAE[:,1], **scatter_kwargs_digits)\n",
    "plt.title('AE embedding', fontsize = 15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "drMIKlB9b4yf"
   },
   "source": [
    "#### Q: What can you say about this model?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bu-O0aaLb4yf"
   },
   "source": [
    "#### Q: Choose some digits, and print their reconstruction "
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
