{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Guessing missing values with an auto-encoder\n",
    "\n",
    "Titouan Vayer, titouan.vayer@inria.fr\n",
    "Aurélien Garivier, aurelien.garivier@ens-lyon.fr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "digits = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "digits.images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, axes = plt.subplots(nrows=1, ncols=6, figsize=(10, 3))\n",
    "for ax, image, label in zip(axes, digits.images, digits.target):\n",
    "    ax.set_axis_off()\n",
    "    ax.imshow(image, cmap=plt.cm.gray_r, interpolation=\"nearest\")\n",
    "    ax.set_title(\"Training: %i\" % label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples = len(digits.images)\n",
    "data = digits.images.reshape((n_samples, -1))\n",
    "X, y = data, digits.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.1, shuffle=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 64 ==> 2 ==> 64\n",
    "class AE(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.encoder = torch.nn.Sequential(\n",
    "            torch.nn.Linear(64, 32),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(32, 16),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(16, 8)\n",
    "        )\n",
    "         \n",
    "        self.decoder = torch.nn.Sequential(\n",
    "            torch.nn.Linear(8, 16),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(16, 32),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(32, 64)\n",
    "        )\n",
    " \n",
    "    def forward(self, x):\n",
    "        encoded = self.encoder(x)\n",
    "        decoded = self.decoder(encoded)\n",
    "        return decoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Init du modèle\n",
    "model = AE()\n",
    " \n",
    "loss_function = torch.nn.MSELoss()\n",
    " \n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = 1e-2, weight_decay= 1e-4)\n",
    "\n",
    "ds = torch.from_numpy(X_train).type(torch.float)\n",
    "\n",
    "loader = torch.utils.data.DataLoader(dataset = ds, batch_size = 128, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 2200 #ca risque de chauffer un peu\n",
    "outputs = []\n",
    "losses = []\n",
    "for epoch in range(epochs):\n",
    "    for img in loader:\n",
    "        # Output of Autoencoder\n",
    "        reconstructed = model(img)\n",
    "\n",
    "        # Calculating the loss function\n",
    "        loss = loss_function(reconstructed, img)\n",
    "\n",
    "        # The gradients are set to zero,\n",
    "        # the gradient is computed and stored.\n",
    "        # .step() performs parameter update\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # Storing the losses in a list for plotting\n",
    "        losses.append(loss.item())\n",
    "    outputs.append((epochs, image, reconstructed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (10, 5))\n",
    "plt.xlabel('Iterations', fontsize = 15)\n",
    "plt.ylabel('Loss', fontsize = 15)\n",
    "plt.semilogy(losses, lw = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_reconstructed = model.forward(ds).detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_ae = np.sum((X_train - X_train_reconstructed) ** 2, axis=1).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_ae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_reconstructed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, axes = plt.subplots(nrows=1, ncols=9, figsize=(10, 3))\n",
    "for ax, image, label in zip(axes, X_train_reconstructed.reshape(y_train.shape[0], 8, 8), y_train):\n",
    "    ax.set_axis_off()\n",
    "    ax.imshow(image, cmap='viridis', interpolation=\"nearest\")\n",
    "    ax.set_title(\"Training: %i\" % label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, axes = plt.subplots(nrows=1, ncols=9, figsize=(10, 3))\n",
    "for ax, image, label in zip(axes, X_train.reshape(y_train.shape[0], 8, 8), y_train):\n",
    "    ax.set_axis_off()\n",
    "    ax.imshow(image, cmap='viridis', interpolation=\"nearest\")\n",
    "    ax.set_title(\"Training: %i\" % label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning imputed variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_without = np.random.randint(0, 63, size = 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test2 = X_test.copy()\n",
    "X_test2[:,idx_without] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "k = 5\n",
    "current_cmap = matplotlib.cm.get_cmap()\n",
    "current_cmap.set_bad(color='white')\n",
    "plt.subplot(1,2,1)\n",
    "plt.imshow(X_test[k].reshape(8,8))\n",
    "plt.subplot(1,2,2)\n",
    "plt.imshow(X_test2[k].reshape(8,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = np.zeros(X_test2.shape)\n",
    "X_completed = X_test2.copy()\n",
    "for j in idx_without:\n",
    "    M[:,j] = np.ones(X_test2.shape[0])\n",
    "    X_completed[:,j] = np.random.randn(X_test2.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Mt = torch.Tensor(M)\n",
    "X_completed_t = torch.Tensor(X_completed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_to_impute = torch.rand(X_test2.shape)*Mt #we will only impute the part where there is missing value\n",
    "X_to_impute.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam([X_to_impute], lr = 1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_function = torch.nn.MSELoss()\n",
    "epochs = 1000\n",
    "losses = []\n",
    "err = []\n",
    "for i in range(epochs):\n",
    "    optimizer.zero_grad()\n",
    "    X = (1-Mt)*X_completed_t+Mt*X_to_impute\n",
    "    reconstructed = model(X)\n",
    "    loss = loss_function(reconstructed, X)\n",
    "    losses.append(loss.item())\n",
    "    X_imputed = (1-Mt)*X_completed_t+Mt*X_to_impute.detach().clone()\n",
    "    X_imputed_np = X_imputed.numpy()\n",
    "    err.append(np.linalg.norm(X_imputed_np-X_test))\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (10,5))\n",
    "plt.subplot(1,2,1)\n",
    "plt.semilogy(losses, label = 'Loss')\n",
    "plt.legend()\n",
    "plt.subplot(1,2,2)\n",
    "plt.semilogy(err, 'r', label = 'Reconstruction Error')\n",
    "plt.legend()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "err[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_imputed = (1-Mt)*X_completed_t+Mt*X_to_impute.detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_imputed_np = X_imputed.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "k = 2\n",
    "current_cmap = matplotlib.cm.get_cmap()\n",
    "current_cmap.set_bad(color='white')\n",
    "plt.figure(figsize = (10,5))\n",
    "plt.subplot(1,3,1)\n",
    "plt.imshow(X_test[k].reshape(8,8))\n",
    "plt.title('Original image')\n",
    "plt.subplot(1,3,2)\n",
    "plt.imshow(X_test2[k].reshape(8,8))\n",
    "plt.title('Corrupted image')\n",
    "plt.subplot(1,3,3)\n",
    "plt.imshow(X_imputed_np[k].reshape(8,8))\n",
    "plt.title('Imputed image')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
