{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Session 4: Model selection \n",
    "\n",
    "Titouan Vayer, titouan.vayer@inria.fr\n",
    "\n",
    "Material taken from the Euroscipy 2019 scikit-learn tutorial by Loic Esteve and Olivier Grisel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inline plots\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the adult census dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use data from the \"Current Population adult_census\" from 1994 that we\n",
    "downloaded from [OpenML](http://openml.org/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult_census = pd.read_csv(\"https://www.openml.org/data/get_csv/1595261/adult-census.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>Private</td>\n",
       "      <td>226802</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>89814</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Farming-fishing</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>336951</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44</td>\n",
       "      <td>Private</td>\n",
       "      <td>160323</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>7688</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>?</td>\n",
       "      <td>103497</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>?</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age   workclass  fnlwgt      education  education-num       marital-status  \\\n",
       "0   25     Private  226802           11th              7        Never-married   \n",
       "1   38     Private   89814        HS-grad              9   Married-civ-spouse   \n",
       "2   28   Local-gov  336951     Assoc-acdm             12   Married-civ-spouse   \n",
       "3   44     Private  160323   Some-college             10   Married-civ-spouse   \n",
       "4   18           ?  103497   Some-college             10        Never-married   \n",
       "\n",
       "           occupation relationship    race      sex  capital-gain  \\\n",
       "0   Machine-op-inspct    Own-child   Black     Male             0   \n",
       "1     Farming-fishing      Husband   White     Male             0   \n",
       "2     Protective-serv      Husband   White     Male             0   \n",
       "3   Machine-op-inspct      Husband   Black     Male          7688   \n",
       "4                   ?    Own-child   White   Female             0   \n",
       "\n",
       "   capital-loss  hours-per-week  native-country   class  \n",
       "0             0              40   United-States   <=50K  \n",
       "1             0              50   United-States   <=50K  \n",
       "2             0              40   United-States    >50K  \n",
       "3             0              40   United-States    >50K  \n",
       "4             0              30   United-States   <=50K  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_census.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The column named **class** is our target variable (i.e., the variable which\n",
    "we want to predict). The two possible classes are `<= 50K` (low-revenue) and\n",
    "`> 50K` (high-revenue)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " <=50K    37155\n",
       " >50K     11687\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_column = 'class'\n",
    "adult_census[target_column].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: classes are slighly imbalanced. Class imbalance happens often in\n",
    "practice and may need special techniques for machine learning. For example in\n",
    "a medical setting, if we are trying to predict whether patients will develop\n",
    "a rare disease, there will be a lot more sane patients than ill patients in\n",
    "the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_columns = ['age', 'education-num', 'capital-gain', 'capital-loss',\n",
    "                     'hours-per-week']\n",
    "categorical_columns = ['workclass', 'education', 'marital-status', 'occupation',\n",
    "                       'relationship', 'race', 'sex', 'native-country']\n",
    "all_columns = numerical_columns + categorical_columns + [target_column]\n",
    "\n",
    "adult_census = adult_census[all_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_name = \"class\"\n",
    "target = adult_census[target_name]\n",
    "data = adult_census.drop(columns=[target_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset contains 48842 samples and 13 features\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    f\"The dataset contains {data.shape[0]} samples and {data.shape[1]} \"\n",
    "    \"features\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "age                int64\n",
       "education-num      int64\n",
       "capital-gain       int64\n",
       "capital-loss       int64\n",
       "hours-per-week     int64\n",
       "workclass         object\n",
       "education         object\n",
       "marital-status    object\n",
       "occupation        object\n",
       "relationship      object\n",
       "race              object\n",
       "sex               object\n",
       "native-country    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "data_numeric = data.select_dtypes(include=np.number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>education-num</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44</td>\n",
       "      <td>10</td>\n",
       "      <td>7688</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48837</th>\n",
       "      <td>27</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48838</th>\n",
       "      <td>40</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48839</th>\n",
       "      <td>58</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48840</th>\n",
       "      <td>22</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48841</th>\n",
       "      <td>52</td>\n",
       "      <td>9</td>\n",
       "      <td>15024</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>48842 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       age  education-num  capital-gain  capital-loss  hours-per-week\n",
       "0       25              7             0             0              40\n",
       "1       38              9             0             0              50\n",
       "2       28             12             0             0              40\n",
       "3       44             10          7688             0              40\n",
       "4       18             10             0             0              30\n",
       "...    ...            ...           ...           ...             ...\n",
       "48837   27             12             0             0              38\n",
       "48838   40              9             0             0              40\n",
       "48839   58              9             0             0              40\n",
       "48840   22              9             0             0              20\n",
       "48841   52              9         15024             0              40\n",
       "\n",
       "[48842 rows x 5 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_numeric"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're going to train various models. How can we choose the best? \n",
    "- choice of metric (here OK: classification)\n",
    "- applied to what?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The training dataset contains 36631 samples and 5 features\n",
      "The testing dataset contains 12211 samples and 5 features\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "data_train, data_test, target_train, target_test = train_test_split(\n",
    "    data_numeric, target, random_state=2408\n",
    ")\n",
    "\n",
    "print(\n",
    "    f\"The training dataset contains {data_train.shape[0]} samples and \"\n",
    "    f\"{data_train.shape[1]} features\"\n",
    ")\n",
    "print(\n",
    "    f\"The testing dataset contains {data_test.shape[0]} samples and \"\n",
    "    f\"{data_test.shape[1]} features\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will build a linear classification model called \"Logistic Regression\". The\n",
    "`fit` method is called to train the model from the input and target data. Only\n",
    "the training data should be given for this purpose.\n",
    "\n",
    "In addition, when checking the time required to train the model and internally\n",
    "check the number of iterations done by the solver to find a solution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model LogisticRegression was trained in 0.170 seconds for [84] iterations\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "import time\n",
    "\n",
    "model = LogisticRegression(solver='lbfgs', max_iter=1000)\n",
    "start = time.time()\n",
    "model.fit(data_train, target_train)\n",
    "elapsed_time = time.time() - start\n",
    "\n",
    "print(\n",
    "    f\"The model {model.__class__.__name__} was trained in \"\n",
    "    f\"{elapsed_time:.3f} seconds for {model.n_iter_} iterations\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>education-num</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>predicted-class</th>\n",
       "      <th>expected-class</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14897</th>\n",
       "      <td>49</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11053</th>\n",
       "      <td>53</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2967</th>\n",
       "      <td>59</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&gt;50K</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11113</th>\n",
       "      <td>25</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9832</th>\n",
       "      <td>30</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29094</th>\n",
       "      <td>19</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4090</th>\n",
       "      <td>32</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>&gt;50K</td>\n",
       "      <td>&gt;50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31408</th>\n",
       "      <td>51</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&gt;50K</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26508</th>\n",
       "      <td>34</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23800</th>\n",
       "      <td>22</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>&lt;=50K</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       age  education-num  capital-gain  capital-loss  hours-per-week  \\\n",
       "14897   49             13             0             0              40   \n",
       "11053   53              9             0             0              60   \n",
       "2967    59              9             0             0              40   \n",
       "11113   25             13             0             0              20   \n",
       "9832    30             13             0             0              50   \n",
       "29094   19              7             0             0              40   \n",
       "4090    32             13             0             0              70   \n",
       "31408   51             10             0             0              40   \n",
       "26508   34             13             0             0              20   \n",
       "23800   22              9             0             0              40   \n",
       "\n",
       "      predicted-class expected-class  correct  \n",
       "14897           <=50K          <=50K     True  \n",
       "11053           <=50K          <=50K     True  \n",
       "2967            <=50K           >50K    False  \n",
       "11113           <=50K          <=50K     True  \n",
       "9832            <=50K          <=50K     True  \n",
       "29094           <=50K          <=50K     True  \n",
       "4090             >50K           >50K     True  \n",
       "31408           <=50K           >50K    False  \n",
       "26508           <=50K          <=50K     True  \n",
       "23800           <=50K          <=50K     True  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_predicted = model.predict(data_test)\n",
    "predictions = data_test.copy()\n",
    "predictions['predicted-class'] = target_predicted\n",
    "predictions['expected-class'] = target_test\n",
    "predictions['correct'] = target_predicted == target_test\n",
    "predictions.head(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The test accuracy using a LogisticRegression is 0.81156\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    f\"The test accuracy using a {model.__class__.__name__} is \"\n",
    "    f\"{model.score(data_test, target_test):.5f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 1\n",
    "\n",
    "- What would be the score of a model that always predicts `' >50K'`?\n",
    "- What would be the score of a model that always predicts `' <= 50K'`?\n",
    "- Is 81% or 82% accuracy a good score for this problem?\n",
    "\n",
    "Hint: You can compute the cross-validated of a [DummyClassifier](https://scikit-learn.org/stable/modules/model_evaluation.html#dummy-estimators) the performance of such baselines."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.dummy import DummyClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " <=50K    27895\n",
       " >50K      8736\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_train.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DummyClassifier(random_state=0, strategy='most_frequent')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_model = DummyClassifier(strategy='most_frequent', random_state=0)\n",
    "dummy_model.fit(data_train, target_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The test accuracy using a DummyClassifier is 0.75833\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    f\"The test accuracy using a {dummy_model.__class__.__name__} is \"\n",
    "    f\"{dummy_model.score(data_test, target_test):.5f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also look at the cross-validation score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The different scores obtained are: \n",
      "[0.81216092 0.8096018  0.81337019 0.81337019 0.82217445]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "scores = cross_val_score(model, data_numeric, target, cv=5)\n",
    "print(f\"The different scores obtained are: \\n{scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean cross-validation accuracy is: 0.814 +/- 0.004\n"
     ]
    }
   ],
   "source": [
    "print(f\"The mean cross-validation accuracy is: \"\n",
    "      f\"{scores.mean():.3f} +/- {scores.std():.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And for the dummy model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The different scores obtained are: \n",
      "[0.76067151 0.76067151 0.76074939 0.76074939 0.76074939]\n"
     ]
    }
   ],
   "source": [
    "scores_dummy = cross_val_score(dummy_model, data_numeric, target, cv=5)\n",
    "print(f\"The different scores obtained are: \\n{scores_dummy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean cross-validation accuracy is: 0.761 +/- 0.000\n"
     ]
    }
   ],
   "source": [
    "print(f\"The mean cross-validation accuracy is: \"\n",
    "      f\"{scores_dummy.mean():.3f} +/- {scores_dummy.std():.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with categorical variables\n",
    "\n",
    "As we have seen in the previous section, a numerical variable is a continuous\n",
    "quantity represented by a real or integer number. Those variables can be\n",
    "naturally handled by machine learning algorithms that typically composed of\n",
    "a sequence of arithmetic instructions such as additions and multiplications.\n",
    "\n",
    "By opposition, categorical variables have discrete values typically represented\n",
    "by string labels taken in a finite list of possible choices. For instance, the\n",
    "variable `native-country` in our dataset is a categorical variable because it\n",
    "encodes the data using a finite list of possible countries (along with the `?`\n",
    "marker when this information is missing):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " United-States                 43832\n",
       " Mexico                          951\n",
       " ?                               857\n",
       " Philippines                     295\n",
       " Germany                         206\n",
       " Puerto-Rico                     184\n",
       " Canada                          182\n",
       " El-Salvador                     155\n",
       " India                           151\n",
       " Cuba                            138\n",
       " England                         127\n",
       " China                           122\n",
       " South                           115\n",
       " Jamaica                         106\n",
       " Italy                           105\n",
       " Dominican-Republic              103\n",
       " Japan                            92\n",
       " Guatemala                        88\n",
       " Poland                           87\n",
       " Vietnam                          86\n",
       " Columbia                         85\n",
       " Haiti                            75\n",
       " Portugal                         67\n",
       " Taiwan                           65\n",
       " Iran                             59\n",
       " Greece                           49\n",
       " Nicaragua                        49\n",
       " Peru                             46\n",
       " Ecuador                          45\n",
       " France                           38\n",
       " Ireland                          37\n",
       " Hong                             30\n",
       " Thailand                         30\n",
       " Cambodia                         28\n",
       " Trinadad&Tobago                  27\n",
       " Laos                             23\n",
       " Yugoslavia                       23\n",
       " Outlying-US(Guam-USVI-etc)       23\n",
       " Scotland                         21\n",
       " Honduras                         20\n",
       " Hungary                          19\n",
       " Holand-Netherlands                1\n",
       "Name: native-country, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"native-country\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_categorical = adult_census.select_dtypes(include=[object]).drop(columns=[\"class\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>workclass</th>\n",
       "      <th>education</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>native-country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Private</td>\n",
       "      <td>11th</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Private</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Farming-fishing</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Local-gov</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Private</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>?</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>?</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    workclass      education       marital-status          occupation  \\\n",
       "0     Private           11th        Never-married   Machine-op-inspct   \n",
       "1     Private        HS-grad   Married-civ-spouse     Farming-fishing   \n",
       "2   Local-gov     Assoc-acdm   Married-civ-spouse     Protective-serv   \n",
       "3     Private   Some-college   Married-civ-spouse   Machine-op-inspct   \n",
       "4           ?   Some-college        Never-married                   ?   \n",
       "\n",
       "  relationship    race      sex  native-country  \n",
       "0    Own-child   Black     Male   United-States  \n",
       "1      Husband   White     Male   United-States  \n",
       "2      Husband   White     Male   United-States  \n",
       "3      Husband   Black     Male   United-States  \n",
       "4    Own-child   White   Female   United-States  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_categorical.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The datasets is composed of 8 features\n"
     ]
    }
   ],
   "source": [
    "print(f\"The datasets is composed of {data_categorical.shape[1]} features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding ordinal categories\n",
    "\n",
    "The most intuitive strategy is to encode each category with a number.\n",
    "The `OrdinalEncoder` will transform the data in such manner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset encoded contains 8 features\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 4.,  1.,  4.,  7.,  3.,  2.,  1., 39.],\n",
       "       [ 4., 11.,  2.,  5.,  0.,  4.,  1., 39.],\n",
       "       [ 2.,  7.,  2., 11.,  0.,  4.,  1., 39.],\n",
       "       [ 4., 15.,  2.,  7.,  0.,  2.,  1., 39.],\n",
       "       [ 0., 15.,  4.,  0.,  3.,  4.,  0., 39.]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "\n",
    "encoder = OrdinalEncoder()\n",
    "data_encoded = encoder.fit_transform(data_categorical)\n",
    "\n",
    "print(f\"The dataset encoded contains {data_encoded.shape[1]} features\")\n",
    "data_encoded[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that all categories have been encoded for each feature\n",
    "independently. We can also notice that the number of features before and after\n",
    "the encoding is the same.\n",
    "\n",
    "However, one has to be careful when using this encoding strategy. Using this\n",
    "integer representation can lead the downstream models to make the assumption\n",
    "that the categories are ordered: 0 is smaller than 1 which is smaller than 2,\n",
    "etc.\n",
    "\n",
    "By default, `OrdinalEncoder` uses a lexicographical strategy to map string\n",
    "category labels to integers. This strategy is completely arbitrary and often be\n",
    "meaningless. For instance suppose the dataset has a categorical variable named\n",
    "\"size\" with categories such as \"S\", \"M\", \"L\", \"XL\". We would like the integer\n",
    "representation to respect the meaning of the sizes by mapping them to increasing\n",
    "integers such as 0, 1, 2, 3. However lexicographical strategy used by default\n",
    "would map the labels \"S\", \"M\", \"L\", \"XL\" to 2, 1, 0, 3.\n",
    "\n",
    "The `OrdinalEncoder` class accepts a \"categories\" constructor argument to pass\n",
    "an the correct ordering explicitly.\n",
    "\n",
    "If a categorical variable does not carry any meaningful order information then\n",
    "this encoding might be misleading to downstream statistical models and you might\n",
    "consider using one-hot encoding instead (see below).\n",
    "\n",
    "Note however that the impact a violation of this ordering assumption is really\n",
    "dependent on the downstream models (for instance linear models are much more\n",
    "sensitive than models built from a ensemble of decision trees).\n",
    "\n",
    "### Encoding nominal categories (without assuming any order)\n",
    "\n",
    "`OneHotEncoder` is an alternative encoder that can prevent the dowstream\n",
    "models to make a false assumption about the ordering of categories. For a\n",
    "given feature, it will create as many new columns as there are possible\n",
    "categories. For a given sample, the value of the column corresponding to the\n",
    "category will be set to `1` while all the columns of the other categories will\n",
    "be set to `0`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset encoded contains 102 features\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 1., ..., 1., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 0., ..., 1., 0., 0.]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "encoder = OneHotEncoder(sparse=False)\n",
    "data_encoded = encoder.fit_transform(data_categorical)\n",
    "print(f\"The dataset encoded contains {data_encoded.shape[1]} features\")\n",
    "data_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>workclass_ ?</th>\n",
       "      <th>workclass_ Federal-gov</th>\n",
       "      <th>workclass_ Local-gov</th>\n",
       "      <th>workclass_ Never-worked</th>\n",
       "      <th>workclass_ Private</th>\n",
       "      <th>workclass_ Self-emp-inc</th>\n",
       "      <th>workclass_ Self-emp-not-inc</th>\n",
       "      <th>workclass_ State-gov</th>\n",
       "      <th>workclass_ Without-pay</th>\n",
       "      <th>education_ 10th</th>\n",
       "      <th>...</th>\n",
       "      <th>native-country_ Portugal</th>\n",
       "      <th>native-country_ Puerto-Rico</th>\n",
       "      <th>native-country_ Scotland</th>\n",
       "      <th>native-country_ South</th>\n",
       "      <th>native-country_ Taiwan</th>\n",
       "      <th>native-country_ Thailand</th>\n",
       "      <th>native-country_ Trinadad&amp;Tobago</th>\n",
       "      <th>native-country_ United-States</th>\n",
       "      <th>native-country_ Vietnam</th>\n",
       "      <th>native-country_ Yugoslavia</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   workclass_ ?  workclass_ Federal-gov  workclass_ Local-gov  \\\n",
       "0           0.0                     0.0                   0.0   \n",
       "1           0.0                     0.0                   0.0   \n",
       "2           0.0                     0.0                   1.0   \n",
       "3           0.0                     0.0                   0.0   \n",
       "4           1.0                     0.0                   0.0   \n",
       "\n",
       "   workclass_ Never-worked  workclass_ Private  workclass_ Self-emp-inc  \\\n",
       "0                      0.0                 1.0                      0.0   \n",
       "1                      0.0                 1.0                      0.0   \n",
       "2                      0.0                 0.0                      0.0   \n",
       "3                      0.0                 1.0                      0.0   \n",
       "4                      0.0                 0.0                      0.0   \n",
       "\n",
       "   workclass_ Self-emp-not-inc  workclass_ State-gov  workclass_ Without-pay  \\\n",
       "0                          0.0                   0.0                     0.0   \n",
       "1                          0.0                   0.0                     0.0   \n",
       "2                          0.0                   0.0                     0.0   \n",
       "3                          0.0                   0.0                     0.0   \n",
       "4                          0.0                   0.0                     0.0   \n",
       "\n",
       "   education_ 10th  ...  native-country_ Portugal  \\\n",
       "0              0.0  ...                       0.0   \n",
       "1              0.0  ...                       0.0   \n",
       "2              0.0  ...                       0.0   \n",
       "3              0.0  ...                       0.0   \n",
       "4              0.0  ...                       0.0   \n",
       "\n",
       "   native-country_ Puerto-Rico  native-country_ Scotland  \\\n",
       "0                          0.0                       0.0   \n",
       "1                          0.0                       0.0   \n",
       "2                          0.0                       0.0   \n",
       "3                          0.0                       0.0   \n",
       "4                          0.0                       0.0   \n",
       "\n",
       "   native-country_ South  native-country_ Taiwan  native-country_ Thailand  \\\n",
       "0                    0.0                     0.0                       0.0   \n",
       "1                    0.0                     0.0                       0.0   \n",
       "2                    0.0                     0.0                       0.0   \n",
       "3                    0.0                     0.0                       0.0   \n",
       "4                    0.0                     0.0                       0.0   \n",
       "\n",
       "   native-country_ Trinadad&Tobago  native-country_ United-States  \\\n",
       "0                              0.0                            1.0   \n",
       "1                              0.0                            1.0   \n",
       "2                              0.0                            1.0   \n",
       "3                              0.0                            1.0   \n",
       "4                              0.0                            1.0   \n",
       "\n",
       "   native-country_ Vietnam  native-country_ Yugoslavia  \n",
       "0                      0.0                         0.0  \n",
       "1                      0.0                         0.0  \n",
       "2                      0.0                         0.0  \n",
       "3                      0.0                         0.0  \n",
       "4                      0.0                         0.0  \n",
       "\n",
       "[5 rows x 102 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns_encoded = encoder.get_feature_names_out(data_categorical.columns)\n",
    "pd.DataFrame(data_encoded, columns=columns_encoded).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The different scores obtained are: \n",
      "[0.83275333 0.83172979 0.83824734 0.83271908 0.82841933 0.83046683\n",
      " 0.83476658 0.83271908 0.83804259 0.83271908]\n"
     ]
    }
   ],
   "source": [
    "model = make_pipeline(\n",
    "    OneHotEncoder(handle_unknown='ignore'),\n",
    "    LogisticRegression(solver='lbfgs', max_iter=1000)\n",
    ")\n",
    "scores = cross_val_score(model, data_categorical, target, cv=10)\n",
    "print(f\"The different scores obtained are: \\n{scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy is: 0.833 +/- 0.00291\n"
     ]
    }
   ],
   "source": [
    "print(f\"The accuracy is: {scores.mean():.3f} +/- {scores.std():.5f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using all the data at hand and specifying which columns to transform how"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "binary_encoding_columns = ['sex']\n",
    "one_hot_encoding_columns = ['workclass', 'education', 'marital-status',\n",
    "                            'occupation', 'relationship',\n",
    "                            'race', 'native-country']\n",
    "scaling_columns = ['age', 'education-num', 'hours-per-week',\n",
    "                   'capital-gain', 'capital-loss']\n",
    "\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "preprocessor = ColumnTransformer([\n",
    "    ('binary-encoder', OrdinalEncoder(), binary_encoding_columns),\n",
    "    ('one-hot-encoder', OneHotEncoder(handle_unknown='ignore'),\n",
    "     one_hot_encoding_columns),\n",
    "    ('standard-scaler', StandardScaler(), scaling_columns)\n",
    "])\n",
    "model = make_pipeline(\n",
    "    preprocessor,\n",
    "    LogisticRegression(solver='lbfgs', max_iter=1000)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pipelines, as all sklearn models, can be displayed!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('columntransformer',\n",
       "                 ColumnTransformer(transformers=[('binary-encoder',\n",
       "                                                  OrdinalEncoder(), ['sex']),\n",
       "                                                 ('one-hot-encoder',\n",
       "                                                  OneHotEncoder(handle_unknown='ignore'),\n",
       "                                                  ['workclass', 'education',\n",
       "                                                   'marital-status',\n",
       "                                                   'occupation', 'relationship',\n",
       "                                                   'race', 'native-country']),\n",
       "                                                 ('standard-scaler',\n",
       "                                                  StandardScaler(),\n",
       "                                                  ['age', 'education-num',\n",
       "                                                   'hours-per-week',\n",
       "                                                   'capital-gain',\n",
       "                                                   'capital-loss'])])),\n",
       "                ('logisticregression', LogisticRegression(max_iter=1000))])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The different scores obtained are: \n",
      "[0.85105947 0.8498311  0.84756347 0.85268223 0.85513923]\n"
     ]
    }
   ],
   "source": [
    "scores = cross_val_score(model, data, target, cv=5)\n",
    "print(f\"The different scores obtained are: \\n{scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy is: 0.851 +- 0.003\n"
     ]
    }
   ],
   "source": [
    "print(f\"The accuracy is: {scores.mean():.3f} +- {scores.std():.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The compound model has a higher predictive accuracy than the\n",
    "two models that used numerical and categorical variables in\n",
    "isolation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The issue of finding the best model parameters\n",
    "\n",
    "In the previous example, we created a `LogisticRegression` classifier using\n",
    "the default parameters by omitting setting explicitly these parameters.\n",
    "\n",
    "For this classifier, the parameter `C` governes the penalty; in other\n",
    "words, how much our model should \"trust\" (or fit) the training data.\n",
    "\n",
    "Therefore, the default value of `C` is never certified to give the best\n",
    "performing model.\n",
    "\n",
    "We can make a quick experiment by changing the value of `C` and see the\n",
    "impact of this parameter on the model performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train, data_test, target_train, target_test = train_test_split(\n",
    "    data, target, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy score using a Pipeline is 0.85 with C=1\n",
      "The accuracy score using a Pipeline is 0.76 with C=1e-05\n"
     ]
    }
   ],
   "source": [
    "C = 1\n",
    "model = make_pipeline(\n",
    "    preprocessor, LogisticRegression(C=C, max_iter=1000, solver='lbfgs')\n",
    ")\n",
    "model.fit(data_train, target_train)\n",
    "print(\n",
    "    f\"The accuracy score using a {model.__class__.__name__} is \"\n",
    "    f\"{model.score(data_test, target_test):.2f} with C={C}\"\n",
    ")\n",
    "\n",
    "C = 1e-5\n",
    "model = make_pipeline(\n",
    "    preprocessor, LogisticRegression(C=C, max_iter=1000, solver='lbfgs')\n",
    ")\n",
    "model.fit(data_train, target_train)\n",
    "print(\n",
    "    f\"The accuracy score using a {model.__class__.__name__} is \"\n",
    "    f\"{model.score(data_test, target_test):.2f} with C={C}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Finding the best model hyper-parameters via exhaustive parameters search\n",
    "\n",
    "We see that the parameter `C` as a significative impact on the model\n",
    "performance. This parameter should be tuned to get the best cross-validation\n",
    "score, so as to avoid over-fitting problems.\n",
    "\n",
    "In short, we will set the parameter, train our model on some data, and\n",
    "evaluate the model performance on some left out data. Ideally, we will select\n",
    "the parameter leading to the optimal performance on the testing set.\n",
    "Scikit-learn provides a `GridSearchCV` estimator which will handle the\n",
    "cross-validation and hyper-parameter search for us."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "model = make_pipeline(\n",
    "    preprocessor, LogisticRegression(max_iter=1000, solver='lbfgs')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will see that we need to provide the name of the parameter to be set.\n",
    "Thus, we can use the method `get_params()` to have the list of the parameters\n",
    "of the model which can set during the grid-search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The hyper-parameters are for a logistic regression model are:\n",
      "C\n",
      "class_weight\n",
      "dual\n",
      "fit_intercept\n",
      "intercept_scaling\n",
      "l1_ratio\n",
      "max_iter\n",
      "multi_class\n",
      "n_jobs\n",
      "penalty\n",
      "random_state\n",
      "solver\n",
      "tol\n",
      "verbose\n",
      "warm_start\n",
      "The hyper-parameters are for the full-pipeline are:\n",
      "memory\n",
      "steps\n",
      "verbose\n",
      "columntransformer\n",
      "logisticregression\n",
      "columntransformer__n_jobs\n",
      "columntransformer__remainder\n",
      "columntransformer__sparse_threshold\n",
      "columntransformer__transformer_weights\n",
      "columntransformer__transformers\n",
      "columntransformer__verbose\n",
      "columntransformer__verbose_feature_names_out\n",
      "columntransformer__binary-encoder\n",
      "columntransformer__one-hot-encoder\n",
      "columntransformer__standard-scaler\n",
      "columntransformer__binary-encoder__categories\n",
      "columntransformer__binary-encoder__dtype\n",
      "columntransformer__binary-encoder__handle_unknown\n",
      "columntransformer__binary-encoder__unknown_value\n",
      "columntransformer__one-hot-encoder__categories\n",
      "columntransformer__one-hot-encoder__drop\n",
      "columntransformer__one-hot-encoder__dtype\n",
      "columntransformer__one-hot-encoder__handle_unknown\n",
      "columntransformer__one-hot-encoder__sparse\n",
      "columntransformer__standard-scaler__copy\n",
      "columntransformer__standard-scaler__with_mean\n",
      "columntransformer__standard-scaler__with_std\n",
      "logisticregression__C\n",
      "logisticregression__class_weight\n",
      "logisticregression__dual\n",
      "logisticregression__fit_intercept\n",
      "logisticregression__intercept_scaling\n",
      "logisticregression__l1_ratio\n",
      "logisticregression__max_iter\n",
      "logisticregression__multi_class\n",
      "logisticregression__n_jobs\n",
      "logisticregression__penalty\n",
      "logisticregression__random_state\n",
      "logisticregression__solver\n",
      "logisticregression__tol\n",
      "logisticregression__verbose\n",
      "logisticregression__warm_start\n"
     ]
    }
   ],
   "source": [
    "print(\"The hyper-parameters are for a logistic regression model are:\")\n",
    "for param_name in LogisticRegression().get_params().keys():\n",
    "    print(param_name)\n",
    "\n",
    "print(\"The hyper-parameters are for the full-pipeline are:\")\n",
    "for param_name in model.get_params().keys():\n",
    "    print(param_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The parameter `'logisticregression__C'` is the parameter for which we would\n",
    "like different values. Let see how to use the `GridSearchCV` estimator for\n",
    "doing such search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy score using a GridSearchCV is 0.85 in 6.819 seconds\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "\n",
    "param_grid = {'logisticregression__C': (0.1, 1.0, 10.0)}\n",
    "model_grid_search = GridSearchCV(model, param_grid=param_grid, n_jobs=4, cv=5)\n",
    "start = time.time()\n",
    "model_grid_search.fit(data_train, target_train)\n",
    "elapsed_time = time.time() - start\n",
    "print(\n",
    "    f\"The accuracy score using a {model_grid_search.__class__.__name__} is \"\n",
    "    f\"{model_grid_search.score(data_test, target_test):.2f} in \"\n",
    "    f\"{elapsed_time:.3f} seconds\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `GridSearchCV` estimator takes a `param_grid` parameter which defines\n",
    "all possible parameters combination. Once the grid-search fitted, it can be\n",
    "used as any other predictor by calling `predict` and `predict_proba`.\n",
    "Internally, it will use the model with the best parameters found during\n",
    "`fit`. You can know about these parameters by looking at the `best_params_`\n",
    "attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best set of parameters is: {'logisticregression__C': 1.0}\n"
     ]
    }
   ],
   "source": [
    "print(f\"The best set of parameters is: {model_grid_search.best_params_}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the `GridSearchCV` estimator, the parameters need to be specified\n",
    "explicitely. Instead, one could randomly generate (following a specific\n",
    "distribution) the parameter candidates. The `RandomSearchCV` allows for such\n",
    "stochastic search. It is used similarly to the `GridSearchCV` but the\n",
    "sampling distributions need to be specified instead of the parameter values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy score using a RandomizedSearchCV is 0.85\n",
      "The best set of parameters is: {'logisticregression__C': 126.08022261126433}\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import uniform\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "param_distributions = {'logisticregression__C': uniform(loc=50, scale=100)}\n",
    "model_grid_search = RandomizedSearchCV(\n",
    "    model, param_distributions=param_distributions, n_iter=3, n_jobs=4, cv=5\n",
    ")\n",
    "model_grid_search.fit(data_train, target_train)\n",
    "print(\n",
    "    f\"The accuracy score using a {model_grid_search.__class__.__name__} is \"\n",
    "    f\"{model_grid_search.score(data_test, target_test):.2f}\"\n",
    ")\n",
    "print(f\"The best set of parameters is: {model_grid_search.best_params_}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "EnvPose3D",
   "language": "python",
   "name": "envpose3d"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
