Machine Learning with Python - Day 6¶
Cognizant, Bangalore
March, 2015
Jigsaw Academy
Instructor: Anand Chitipothu
This live notes are avaialble online at http://bit.ly/cognizant-py.
Classification¶
We are going to look at various classification algorithms today.
Datasets¶
Wine Quality - Download
Wine recognition data from UCI machine learning datasets.
Utility Functions¶
We are going to use the following function for drawing the clasification boundaries to get a visual sense of how the classfication algorithm is working on the given data.
In [115]:
import os
from IPython.display import Image
from sklearn import tree
# https://gist.github.com/anandology/772d44d291a9daa198d4
def plot_decision_boundaries(X, y, model_class, **model_params):
"""Function to plot the decision boundaries of a classification model.
This uses just the first two columns of the data for fitting
the model as we need to find the predicted value for every point in
scatter plot.
One possible improvement could be to use all columns fot fitting
and using the first 2 columns and median of all other columns
for predicting.
Adopted from:
http://scikit-learn.org/stable/auto_examples/ensemble/plot_voting_decision_regions.html
http://scikit-learn.org/stable/auto_examples/cluster/plot_kmeans_digits.html
"""
reduced_data = X[:, :2]
model = model_class(**model_params)
model.fit(reduced_data, y)
# Step size of the mesh. Decrease to increase the quality of the VQ.
h = .02 # point in the mesh [x_min, m_max]x[y_min, y_max].
# Plot the decision boundary. For that, we will assign a color to each
x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
# Obtain labels for each point in mesh using the model.
Z = model.predict(np.c_[xx.ravel(), yy.ravel()])
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
np.arange(y_min, y_max, 0.1))
Z = model.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
plt.contourf(xx, yy, Z, alpha=0.4)
plt.scatter(X[:, 0], X[:, 1], c=y, alpha=0.8)
return plt
def export_tree(model, filename="tree.dot", simple=False):
"""Exports the decision tree as a graphviz file.
"""
params = {}
if not simple:
params = dict(filled=True, rounded=True, special_characters=True)
with open(filename, "w") as f:
tree.export_graphviz(model, out_file=f,
class_names=iris.target_names,
**params)
def show_tree(model, filename="tree.dot", simple=False, text=False):
"""Displays the tree as image.
This requires the graphviz package to be installed on
the computer. If it is not installed it displays the
graphviz file as text.
"""
dot_exists = os.system("dot -V") == 0
if not dot_exists or text:
simple = True
export_tree(model, filename, simple=simple)
pngfile = filename.replace(".dot", ".png")
if dot_exists and not text:
os.system("dot -Tpng {} -o {}".format(filename, pngfile))
return Image(pngfile)
else:
txt = open(filename).read().replace("\\n", "\n ").replace(";", ";\n")
print(txt)
Logistic Regression¶
$y = b + b1 * X$
$logit(p) = b + b1*X$
where:
$logit(p) = log(\frac {p}{1-p})$
If you work hard, you can figure out that:
$p = \frac{1}{1 + e^{\beta_0 + \beta_1X}}$
In [13]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
Lets take the iris data.
In [11]:
from sklearn.datasets import load_iris
iris = load_iris()
# convert the data into two classes
X = iris.data
y = (iris.target > 0).astype(np.int8)
In [17]:
plt.scatter(X[:, 0], X[:, 1], c=y, cmap="summer")
Out[17]:
In [18]:
pd.Series(y).value_counts()
Out[18]:
In [19]:
from sklearn.linear_model import LogisticRegression
model = LogisticRegression()
In [20]:
model.fit(X, y)
Out[20]:
In [21]:
model.predict(X[:5])
Out[21]:
In [22]:
y[:5]
Out[22]:
In [23]:
model.predict_proba(X[:5])
Out[23]:
Problem: Find the class for sample [5, 4, 3, 1]. What is the probablity for it to belong to class 0?
In [24]:
X[:5]
Out[24]:
In [25]:
X[:1]
Out[25]:
Lets try to plot the probability of being in class 0 for every point.
In [26]:
p0 = model.predict_proba(X)[:, 0]
In [27]:
plt.plot(sorted(p0))
Out[27]:
In [30]:
iris.target_names
Out[30]:
In [32]:
plot_decision_boundaries(X, y, LogisticRegression)
Out[32]:
Q: Can we try with three classes?
In [38]:
plot_decision_boundaries(X, iris.target, LogisticRegression)
Out[38]:
Problem: Take the wine data and try to fit a logistic regression model.
The first column is the class label.
In [49]:
wine = pd.read_csv("datasets/wine.data", header=None)
In [50]:
wine.shape
Out[50]:
In [51]:
wine.columns
Out[51]:
In [52]:
wine.head()
Out[52]:
In [53]:
wine.columns = ["C"+str(n) for n in wine.columns]
In [54]:
wine.head()
Out[54]:
In [55]:
X = wine.drop("C0", axis=1)
y = wine.C0
In [56]:
X.shape
Out[56]:
In [57]:
y.shape
Out[57]:
In [59]:
plot_decision_boundaries(X.values, y.values, LogisticRegression)
Out[59]:
In [60]:
wine.C0.value_counts()
Out[60]:
Metrics¶
In [61]:
model = LogisticRegression()
model.fit(X,y)
Out[61]:
In [62]:
yp = model.predict(X)
In [63]:
from sklearn.metrics import accuracy_score
In [64]:
accuracy_score(y, yp)
Out[64]:
In [65]:
y.shape
Out[65]:
In [66]:
(y==yp).sum()
Out[66]:
In [67]:
173/178.0
Out[67]:
Split the data into train and test.
In [68]:
from sklearn.cross_validation import train_test_split
In [69]:
Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.3)
In [70]:
Xtrain.shape
Out[70]:
In [71]:
Xtest.shape
Out[71]:
In [74]:
pd.Series(ytrain).value_counts()
Out[74]:
In [75]:
pd.Series(ytest).value_counts()
Out[75]:
In [76]:
model = LogisticRegression()
model.fit(Xtrain, ytrain)
Out[76]:
In [78]:
ytrainp = model.predict(Xtrain)
print("accurary on training data is ", accuracy_score(ytrain, ytrainp))
In [79]:
ytestp = model.predict(Xtest)
print("accurary on training data is ", accuracy_score(ytest, ytestp))
In [80]:
from sklearn.metrics import confusion_matrix
In [81]:
confusion_matrix(ytest, ytestp)
Out[81]:
In [82]:
pd.Series(ytest).value_counts()
Out[82]:
In [86]:
pd.Series(ytestp).value_counts()
Out[86]:
Decision Trees¶
In [87]:
from sklearn.tree import DecisionTreeClassifier
In [88]:
X = iris.data[:, [2, 3]] # petal-length and petal-width
y = iris.target
In [95]:
plt.scatter(X[:, 0], X[:, 1], c=y, cmap="hot")
Out[95]:
In [96]:
plot_decision_boundaries(X, y, DecisionTreeClassifier, max_depth=1)
Out[96]:
In [107]:
plot_decision_boundaries(X, y, DecisionTreeClassifier, max_depth=2, random_state=0)
Out[107]:
In [98]:
plot_decision_boundaries(X, y, DecisionTreeClassifier, max_depth=3)
Out[98]:
In [108]:
model = DecisionTreeClassifier(max_depth=2, random_state=0)
model.fit(X, y)
Out[108]:
In [109]:
show_tree(model)
Out[109]:
In [111]:
yp = model.predict(X)
In [112]:
accuracy_score(y, yp)
Out[112]:
In [113]:
confusion_matrix(y, yp)
Out[113]:
In [116]:
show_tree(model, text=True)
Lets see what happens if we allow the decision tree to go very deep.
In [117]:
plot_decision_boundaries(X, y, DecisionTreeClassifier, max_depth=5)
Out[117]:
Problem: Build a decision tree model using all the features of iris. split the data into training and test and see what is the model accuracy on training and test datasets.
Find out at what max_depth, the model gives best accuracy.
In [167]:
X = iris.data
y = iris.target
Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.3, stratify=y)
def try_model(max_depth):
model = DecisionTreeClassifier(max_depth=max_depth)
model.fit(Xtrain, ytrain)
ytrainp = model.predict(Xtrain)
ytestp = model.predict(Xtest)
return [max_depth,
accuracy_score(ytrain, ytrainp),
accuracy_score(ytest, ytestp)]
data = pd.DataFrame([try_model(i) for i in range(1, 10)])
In [168]:
data.set_index(0, inplace=True)
In [169]:
data.plot()
Out[169]:
In [170]:
model = DecisionTreeClassifier(max_depth=2)
model.fit(Xtrain, ytrain)
Out[170]:
In [171]:
show_tree(model)
Out[171]:
DecisionTree Regression¶
In [172]:
from sklearn.tree import DecisionTreeRegressor
In [173]:
d = pd.read_csv("datasets/chirps.tsv", delimiter="\t")
d.head()
Out[173]:
In [174]:
model = DecisionTreeRegressor()
model.fit(d[["temp"]], d["chirps"])
Out[174]:
In [175]:
show_tree(model)
Out[175]:
In [176]:
yp = model.predict(d[["temp"]])
Wine Dataset¶
In [178]:
!cat datasets/wine.names
Preprocessing¶
In [179]:
from sklearn.preprocessing import StandardScaler
In [180]:
scaler = StandardScaler()
In [181]:
scaler.fit(iris.data)
Out[181]:
In [182]:
data2 = scaler.transform(iris.data)
In [183]:
d = pd.DataFrame(data2)
In [184]:
d.describe()
Out[184]:
In [185]:
x1 = [3, 4, 5, 2]
In [186]:
scaler.transform([x1])
Out[186]:
In [187]:
scaler.mean_
Out[187]:
In [188]:
scaler.std_
Out[188]:
In [189]:
from sklearn.pipeline import Pipeline
In [190]:
pipe = Pipeline([('scale', StandardScaler()),
('dt', DecisionTreeClassifier())])
In [192]:
pipe.fit(X, y)
Out[192]:
In [193]:
yp = pipe.predict(X)
In [194]:
accuracy_score(y, yp)
Out[194]:
Support Vector Machines¶
In [195]:
from sklearn.datasets import make_circles
In [196]:
X, y = make_circles(n_samples=1000, factor=0.2, noise=0.1)
In [197]:
plt.scatter(X[:, 0], X[:, 1], c=y)
Out[197]:
In [198]:
d = pd.DataFrame(X, columns=["x1", "x2"])
In [199]:
d["y"] = y
In [201]:
d.head()
Out[201]:
In [205]:
d.plot(kind="scatter", x="x1", y="x2", c="y", cmap="summer")
Out[205]:
In [206]:
plt.scatter(d.x1, d.x1*d.x1+d.x2*d.x2, c=d.y)
Out[206]:
In [207]:
plt.scatter(d.x1, (d.x1*d.x1+d.x2*d.x2)**0.5, c=d.y)
Out[207]:
In [208]:
d["x3"] = (d.x1*d.x1+d.x2*d.x2)**0.5
In [216]:
from sklearn.linear_model import LogisticRegression
In [217]:
model = LogisticRegression()
model.fit(d[["x1", "x2", "x3"]], d.y)
Out[217]:
In [218]:
yp = model.predict(d[["x1", "x2", "x3"]])
In [219]:
accuracy_score(y, yp)
Out[219]:
In [221]:
plot_decision_boundaries(d[["x1", "x3"]].values, d.y.values, LogisticRegression)
Out[221]:
In [222]:
X[:5]
Out[222]:
In [223]:
plot_decision_boundaries(X, y, LogisticRegression)
Out[223]:
In [224]:
from sklearn.svm import SVC
In [225]:
plot_decision_boundaries(X, y, SVC, kernel="linear")
Out[225]:
In [226]:
plot_decision_boundaries(X, y, SVC, kernel="rbf")
Out[226]:
Problem: There is another interesting generated dataset called moons. Try it with all the classification algorithms and see how it works.
In [231]:
from sklearn.datasets import make_moons
In [228]:
X, y = make_moons(1000)
In [230]:
plt.scatter(X[:, 0], X[:, 1], c = y)
Out[230]:
In [237]:
X, y = make_circles(n_samples=1000, factor=0.2, noise=0.1)
d = pd.DataFrame(X, columns=["x1", "x2"])
d['y'] = y
d['x3'] = (d.x1*d.x1+d.x2*d.x2)**0.5
model = SVC(kernel="linear")
model.fit(d[['x1', 'x3']], y)
Out[237]:
In [239]:
yp = model.predict(d[['x1', 'x3']])
d['yp'] = yp
In [243]:
plt.scatter(d.x1, d.x3, c=d.yp)
plt.scatter(model.support_vectors_[:, 0], model.support_vectors_[:, 1], c='y')
Out[243]:
In [241]:
model.support_vectors_
Out[241]:
Q Can we get three concentric rings?
In [258]:
def scatter_circles(n, factor, noise=0.05):
X, y = make_circles(n, factor=factor, noise=noise)
plt.scatter(X[:, 0], X[:, 1], c=y)
return X, y
In [259]:
X1, y1 = scatter_circles(1000, 0.1)
X2, y2 = scatter_circles(1000, 0.6)
In [260]:
X1[:5], y1[:5]
Out[260]:
In [261]:
X2[:5], y2[:5]
Out[261]:
In [262]:
plt.scatter(X1[:, 0], X1[:, 1], c=y1)
Out[262]:
In [264]:
plt.scatter(X2[:, 0], X2[:, 1], c=y2)
Out[264]:
In [266]:
d = pd.DataFrame(X1, columns=["x1", "x2"])
In [268]:
d["y"] = y1
In [270]:
d[y1==0].plot(x="x1", y="x2", kind="scatter")
Out[270]:
In [272]:
d[y1==1].plot(x="x1", y="x2", kind="scatter")
Out[272]:
In [273]:
d3 = d[y1==1]
In [281]:
d3.y.head()
Out[281]:
In [282]:
d2 = pd.DataFrame(X2, columns=["x1", "x2"])
In [283]:
d2["y"] = y2
In [284]:
d2.head()
Out[284]:
In [285]:
d3.head()
Out[285]:
In [288]:
d4 = np.hstack?
In [292]:
d4 = d2.append(d3)
In [293]:
d4.plot(kind="scatter", x="x1", y="x2", c="y")
Out[293]:
In [297]:
def circles(n, factor):
X, y = make_circles(n, factor=factor, noise=0.05)
d = pd.DataFrame(X, columns=["x1", "x2"])
d["y"] = y
return d
In [308]:
d1 = circles(1000, 0.2)
d2 = circles(1000, 0.6)
d3 = d2[d2.y==1]
d3.y = 2
d4 = d1.append(d3)
In [312]:
d4.plot(kind="scatter", x="x1", y="x2", c="y", cmap="cool")
Out[312]:
In [313]:
plot_decision_boundaries(d4[["x1", "x2"]].values, d4.y.values, SVC)
Out[313]:
In [319]:
d4["x12"] = (d4.x1*d4.x1+d4.x2*d4.x2)
In [320]:
plot_decision_boundaries(d4[["x1", "x12"]].values, d4.y.values, LogisticRegression)
Out[320]:
In [321]:
model = SVC()
model.fit(d4[["x1", "x2"]], d4.y)
plt.scatter(d4.x1, d4.x2, c=d4.y)
plt.scatter(model.support_vectors_[:, 0], model.support_vectors_[:, 1], c='y')
Out[321]:
In [327]:
plot_decision_boundaries(d4[["x1", "x2"]].values, d4.y.values,
DecisionTreeClassifier, max_depth=8)
Out[327]:
In [331]:
xx, yy = np.meshgrid(np.arange(5), np.arange(10, 15))
In [333]:
xx
Out[333]:
In [334]:
xx.ravel()
Out[334]:
In [335]:
xx.reshape(-1)
Out[335]:
In [342]:
np.array([[x, y] for x in np.arange(5) for y in np.arange(10, 15)])
Out[342]:
In [341]:
np.c_[xx.reshape(-1), yy.reshape(-1)]
Out[341]:
In [ ]: