{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pylab import *\n",
    "from sklearn.datasets import fetch_20newsgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ntrain =  2243\n"
     ]
    }
   ],
   "source": [
    "categories = ['rec.autos', 'comp.graphics', 'misc.forsale', 'alt.atheism'] #['talk.politics.guns', 'rec.autos', 'comp.graphics', 'sci.space']\n",
    "newsgroups_train = fetch_20newsgroups(subset='train', categories=categories)\n",
    "category_numbers = [newsgroups_train.target_names.index(c) for c in categories]\n",
    "\n",
    "ntrain = shape(newsgroups_train.data)[0]\n",
    "print(\"ntrain = \", ntrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From: ddeciacco@cix.compulink.co.uk (David Deciacco)\n",
      "Subject: Re: Another CVIEW question (wa\n",
      "Reply-To: ddeciacco@cix.compulink.co.uk\n",
      "Lines: 5\n",
      "\n",
      "\n",
      "In-Reply-To: <20APR199312262902@rigel.tamu.edu> lmp8913@rigel.tamu.edu (PRESTON, LISA M)\n",
      "\n",
      "I have a trident card and fullview works real gif jpg try it#\n",
      "dave\n",
      "\n",
      "1 comp.graphics\n"
     ]
    }
   ],
   "source": [
    "k = randint(ntrain)\n",
    "print(newsgroups_train.data[k])\n",
    "print(newsgroups_train.target[k], newsgroups_train.target_names[newsgroups_train.target[k]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2243, 32848)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer()\n",
    "vectors = vectorizer.fit_transform(newsgroups_train.data)\n",
    "vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_clusters = len(categories)\n",
    "kmeans = KMeans(n_clusters, random_state=0).fit(vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = []\n",
    "for j in range(n_clusters):\n",
    "    cat.append([i for i in range(ntrain) if kmeans.labels_[i] == j] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0.,   7., 446.,   8.],\n",
       "       [108.,   0.,   0.,   0.],\n",
       "       [273.,  66.,  32., 255.],\n",
       "       [ 99., 511., 107., 331.]])"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusionMatrix = zeros((n_clusters, len(categories)))\n",
    "for j in range(n_clusters):\n",
    "    confusionMatrix[j,:] = histogram([newsgroups_train.target[i] for i in cat[j]], bins = range(1+len(categories)))[0]\n",
    "confusionMatrix"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
