Project using Feature Selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "cols = ['age','workclass','fnlwgt','education','education-num','marital-status','occupation','relationship'\n",
    "        ,'race','sex','capital-gain','capital-loss','hours-per-week','native-country','Salary']\n",
    "adult_data = pd.read_csv('https://raw.githubusercontent.com/zekelabs/data-science-complete-tutorial/master/Data/adult.data.txt', names=cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "      <th>Salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age          workclass  fnlwgt   education  education-num  \\\n",
       "0   39          State-gov   77516   Bachelors             13   \n",
       "1   50   Self-emp-not-inc   83311   Bachelors             13   \n",
       "2   38            Private  215646     HS-grad              9   \n",
       "3   53            Private  234721        11th              7   \n",
       "4   28            Private  338409   Bachelors             13   \n",
       "\n",
       "        marital-status          occupation    relationship    race      sex  \\\n",
       "0        Never-married        Adm-clerical   Not-in-family   White     Male   \n",
       "1   Married-civ-spouse     Exec-managerial         Husband   White     Male   \n",
       "2             Divorced   Handlers-cleaners   Not-in-family   White     Male   \n",
       "3   Married-civ-spouse   Handlers-cleaners         Husband   Black     Male   \n",
       "4   Married-civ-spouse      Prof-specialty            Wife   Black   Female   \n",
       "\n",
       "   capital-gain  capital-loss  hours-per-week  native-country  Salary  \n",
       "0          2174             0              40   United-States   <=50K  \n",
       "1             0             0              13   United-States   <=50K  \n",
       "2             0             0              40   United-States   <=50K  \n",
       "3             0             0              40   United-States   <=50K  \n",
       "4             0             0              40            Cuba   <=50K  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline, Pipeline\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler, OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.feature_selection import SelectKBest, chi2, f_classif, mutual_info_classif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_cols = adult_data.select_dtypes(include=['object']).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_cols = list(cat_cols)\n",
    "cat_cols.remove('Salary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols = adult_data.select_dtypes(exclude=['object']).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_feature_selector = SelectKBest(k=5, score_func=mutual_info_classif)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "\n",
    "for col in cat_cols:\n",
    "    le = LabelEncoder()\n",
    "    df[col]  = le.fit_transform(adult_data[col])\n",
    "    \n",
    "X1 = cat_feature_selector.fit_transform(df, adult_data.Salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols = num_cols.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_feature_selector = SelectKBest(k=3, score_func=f_classif)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "X2 = num_feature_selector.fit_transform(df,adult_data.Salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.hstack([X1,X2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt = DecisionTreeClassifier()\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/awantik/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.fit(X, adult_data.Salary)\n",
    "lr.fit(X, adult_data.Salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8356930069715304"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.score(X, adult_data.Salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7547986855440558"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.score(X, adult_data.Salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_num = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='median')),\n",
    "    ('num_feature_selector',num_feature_selector),\n",
    "    ('scaling',MinMaxScaler())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_cat = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),\n",
    "    #('cat_feature_selector', cat_feature_selector),\n",
    "    ('encoding', OneHotEncoder(handle_unknown='ignore')),\n",
    "    ('cat_feature_selector', cat_feature_selector)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('num', pipeline_num, num_cols),\n",
    "        ('cat', pipeline_cat, cat_cols)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline(steps=[('preprocessor',preprocessor),\n",
    "                ('classifier', MultinomialNB())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['age', 'workclass', 'fnlwgt', 'education', 'education-num',\n",
       "       'marital-status', 'occupation', 'relationship', 'race', 'sex',\n",
       "       'capital-gain', 'capital-loss', 'hours-per-week', 'native-country',\n",
       "       'Salary'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('preprocessor', ColumnTransformer(n_jobs=None, remainder='drop', sparse_threshold=0.3,\n",
       "         transformer_weights=None,\n",
       "         transformers=[('num', Pipeline(memory=None,\n",
       "     steps=[('imputer', SimpleImputer(copy=True, fill_value=None, missing_values=nan,\n",
       "       strategy='median', verbo..., 'native-country'])])), ('classifier', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(adult_data, adult_data.Salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 32561 entries, 0 to 32560\n",
      "Data columns (total 15 columns):\n",
      "age               32561 non-null int64\n",
      "workclass         32561 non-null object\n",
      "fnlwgt            32561 non-null int64\n",
      "education         32561 non-null object\n",
      "education-num     32561 non-null int64\n",
      "marital-status    32561 non-null object\n",
      "occupation        32561 non-null object\n",
      "relationship      32561 non-null object\n",
      "race              32561 non-null object\n",
      "sex               32561 non-null object\n",
      "capital-gain      32561 non-null int64\n",
      "capital-loss      32561 non-null int64\n",
      "hours-per-week    32561 non-null int64\n",
      "native-country    32561 non-null object\n",
      "Salary            32561 non-null object\n",
      "dtypes: int64(6), object(9)\n",
      "memory usage: 3.7+ MB\n"
     ]
    }
   ],
   "source": [
    "adult_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8745"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_data[adult_data['native-country'].isnull()].size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "for col in cat_cols:\n",
    "    adult_data[col] = adult_data[col].str.strip().replace('?',np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 32561 entries, 0 to 32560\n",
      "Data columns (total 15 columns):\n",
      "age               32561 non-null int64\n",
      "workclass         30725 non-null object\n",
      "fnlwgt            32561 non-null int64\n",
      "education         32561 non-null object\n",
      "education-num     32561 non-null int64\n",
      "marital-status    32561 non-null object\n",
      "occupation        30718 non-null object\n",
      "relationship      32561 non-null object\n",
      "race              32561 non-null object\n",
      "sex               32561 non-null object\n",
      "capital-gain      32561 non-null int64\n",
      "capital-loss      32561 non-null int64\n",
      "hours-per-week    32561 non-null int64\n",
      "native-country    31978 non-null object\n",
      "Salary            32561 non-null object\n",
      "dtypes: int64(6), object(9)\n",
      "memory usage: 3.7+ MB\n"
     ]
    }
   ],
   "source": [
    "adult_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('num', Pipeline(memory=None,\n",
       "       steps=[('imputer', SimpleImputer(copy=True, fill_value=None, missing_values=nan,\n",
       "         strategy='median', verbose=0)), ('num_feature_selector', SelectKBest(k=3, score_func=<function f_classif at 0x7f070efa2ae8>)), ('scaling', MinMaxScaler(copy=True, feature_range=(0, 1)))]), ['age',\n",
       "   'fnlwgt',\n",
       "   'education-num',\n",
       "   'capital-gain',\n",
       "   'capital-loss',\n",
       "   'hours-per-week']),\n",
       " ('cat', Pipeline(memory=None,\n",
       "       steps=[('imputer', SimpleImputer(copy=True, fill_value='missing', missing_values=nan,\n",
       "         strategy='constant', verbose=0)), ('encoding', OneHotEncoder(categorical_features=None, categories=None,\n",
       "         dtype=<class 'numpy.float64'>, handle_unknown='ignore',\n",
       "         n_values=None, sparse=True)), ('cat_feature_selector', SelectKBest(k=5, score_func=<function mutual_info_classif at 0x7f070f071ea0>))]), ['workclass',\n",
       "   'education',\n",
       "   'marital-status',\n",
       "   'occupation',\n",
       "   'relationship',\n",
       "   'race',\n",
       "   'sex',\n",
       "   'native-country'])]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.steps[0][1].transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_params = {'preprocessor__num__num_feature_selector__k':[5,4,6],\n",
    "               'preprocessor__num__num_feature_selector__score_func':[chi2,mutual_info_classif]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs = GridSearchCV(pipeline, grid_params,cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('preprocessor', ColumnTransformer(n_jobs=None, remainder='drop', sparse_threshold=0.3,\n",
       "         transformer_weights=None,\n",
       "         transformers=[('num', Pipeline(memory=None,\n",
       "     steps=[('imputer', SimpleImputer(copy=True, fill_value=None, missing_values=nan,\n",
       "       strategy='median', verbo..., 'native-country'])])), ('classifier', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'preprocessor__num__num_feature_selector__k': [5, 4, 6], 'preprocessor__num__num_feature_selector__score_func': [<function chi2 at 0x7f070efa2bf8>, <function mutual_info_classif at 0x7f070f071ea0>]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.fit(adult_data, adult_data.Salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'preprocessor__num__num_feature_selector__k': 4,\n",
       " 'preprocessor__num__num_feature_selector__score_func': <function sklearn.feature_selection.univariate_selection.chi2(X, y)>}"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7197260526396609"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}