Logistic Regression¶
Faisal Qureshi
faisal.qureshi@ontariotechu.ca
http://vclab.science.ontariotechu.ca
In [1]:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
%matplotlib inline
Generate some data¶
In [3]:
np.random.seed(0)
from sklearn import datasets
n_samples = 120
x, y = datasets.make_blobs(n_samples=n_samples, n_features=2, centers=[(0,5),(3,0)], random_state=0)
In [4]:
plt.figure(figsize=(7,7))
plt.scatter(x[:,0], x[:,1], c=y, cmap=cm.viridis)
plt.xlabel('$x_1$')
plt.ylabel('$x_2$')
Out[4]:
Text(0, 0.5, '$x_2$')
Some useful functions¶
In [4]:
def sigm(x, th):
"""
Sigmoid function
Parameters:
x -- N times M+1 design matrix. N=1 is also valid.
th -- theta M+1 vector of paratmeters.
Returns:
N-dimensional vector -- contains reponse of the sigmoid function specified by parameters th
at locations specified by x
"""
return 1. / (1. + np.exp( - np.dot(x, th) ))
def prediction(y):
"""
Predictor for logistic regression.
Arguments:
y -- a vector containing values between 0 and 1 (responses of a sigmoid function)
Returns:
A vector containg 1 or 0, depending upon whether or not a
particular response is greater or equal to .5 or not
"""
z = np.zeros(y.shape)
z[np.where(y >= .5)] = 1
return z
def error(x, y, th):
"""
Error computation for logistic regression
"""
return np.sum(abs(y - prediction(sigm(x, th)))) / 120.
def make_design_matrix(x, n_samples):
"""
Constructs a design matrix from data points. Assumes that x is N-times-M.
Here N refers to the number of data points and M is the length (number of dimensions)
for each data point.
Paramters:
x - N-times-M matrix containing N M-dimensional data points
n_sample - number of samples (this is equal to N)
Returns:
a N-times-(M+1) matrix, whose first column is bias (equal to 1)
"""
return np.append(np.ones([n_samples, 1]), x, 1)
Gradient descent¶
In [6]:
n_itr = 1000 # Number of iterations, try 10000
learning_rate = 0.01 # Learning rate, try between 0.0001 to 0.1
th = np.random.uniform(0,1,[3]) # Initial guess for the model parameters
errs = np.empty([n_itr])
# Stochastic gradient descent
X = make_design_matrix(x, n_samples)
for i in range(n_itr):
for j in range(n_samples):
# derivative of sigmoid w.r.t. the parameters
# Check slides to see how it is derived
th = th + learning_rate * X[j,:] * (y[j] - sigm(X[j,:], th) )
errs[i] = error(X, y, th)
print ('theta', th)
theta [ 1.47282216 3.66615419 -3.61893537]
Plotting the decision boundary¶
$\theta_0 + \theta_1 x_1 + \theta_2 x_2 = 0$
So we can plot this line by constructing $y = mx + c$ form as follows
$x_2 = - \frac{\theta_0}{\theta_2} - \frac{\theta_1}{\theta_2} x_1$
In [7]:
x1 = np.linspace(-2,5,100)
y1 = - th[0]/th[2] - th[1]*x1/th[2]
plt.figure(figsize=(7,7))
plt.plot(x1,y1,'-')
plt.scatter(x[:,0], x[:,1], c=y, cmap=cm.viridis)
plt.xlabel('$x_1$')
plt.ylabel('$x_2$')
Out[7]:
Text(0, 0.5, '$x_2$')
Visualizing the sigmoid function¶
In [8]:
x1 = np.linspace(-3,6,100)
x2 = np.linspace(-3,8,100)
xv1, xv2 = np.meshgrid(x1,x2)
xx = np.vstack((xv1.ravel(), xv2.ravel()))
zz = sigm(np.vstack( (np.ones([10000]), xx )).T, th)
z_at_zero = np.zeros([n_samples]) # this to also plot the data points
In [9]:
from mpl_toolkits.mplot3d import axes3d
from matplotlib import cm
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
ax.view_init(30, 45)
ax.scatter(x[:,0], x[:,1], z_at_zero, c=y, cmap=cm.viridis)
ax.plot_surface(xv1, xv2, zz.reshape(100,100)-0.5, rstride=1, cstride=1, cmap=cm.coolwarm, linewidth=0, antialiased=False, alpha=.3)
ax.plot_wireframe(xv1, xv2, np.zeros([100,100]), rstride=10, cstride=10, linewidth=1)
ax.contour(xv1, xv2, zz.reshape(100,100)-0.5, [0], c='red')
ax.set_xlabel('$x_2$')
ax.set_ylabel('$x_1$')
ax.set_zlabel('sigm(.)')
/var/folders/kz/70dhvx_s3rbgqq52pg06cvr80000gn/T/ipykernel_72451/917121760.py:9: UserWarning: The following kwargs were not used by contour: 'c' ax.contour(xv1, xv2, zz.reshape(100,100)-0.5, [0], c='red')
Out[9]:
Text(0.5, 0, 'sigm(.)')
In [10]:
plt.figure(figsize=(7,7))
plt.plot(errs)
plt.xlabel('iterations')
plt.ylabel('error')
Out[10]:
Text(0, 0.5, 'error')
In [11]:
for i in range(n_samples):
print (sigm(X[i,:], th), y[i])
Input In [11] print sigm(X[i,:], th), y[i] ^ SyntaxError: invalid syntax
In [12]:
from mpl_toolkits.mplot3d import axes3d
from matplotlib import cm
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
ax.view_init(40, 35)
ax.scatter(x[:,0], x[:,1], z_at_zero, c=y, cmap=cm.viridis)
ax.plot_surface(xv1, xv2, zz.reshape(100,100)-.5, rstride=1, cstride=1, cmap=cm.coolwarm, linewidth=0, antialiased=False, alpha=.3)
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')
ax.set_zlabel('sigm($x . \theta$)')
Out[12]:
Text(0.5, 0, 'sigm($x . \theta$)')
In [13]:
from mpl_toolkits.mplot3d import axes3d
from matplotlib import cm
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
ax.view_init(40, 15)
ax.scatter(x[:,0], x[:,1], z_at_zero, c=y, cmap=cm.viridis)
ax.plot_surface(xv1, xv2, zz.reshape(100,100)-.5, rstride=1, cstride=1, cmap=cm.coolwarm, linewidth=0, antialiased=False, alpha=.3)
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')
ax.set_zlabel('sigm($x . \theta$)')
Out[13]:
Text(0.5, 0, 'sigm($x . \theta$)')
In [14]:
from mpl_toolkits.mplot3d import axes3d
from matplotlib import cm
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
ax.view_init(0, 45)
ax.scatter(x[:,0], x[:,1], z_at_zero, c=y, cmap=cm.viridis)
ax.plot_surface(xv1, xv2, zz.reshape(100,100)-0.5, rstride=1, cstride=1, cmap=cm.coolwarm, linewidth=0, antialiased=False, alpha=.3)
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')
ax.set_zlabel('sigm($x . \theta$)')
Out[14]:
Text(0.5, 0, 'sigm($x . \theta$)')
In [ ]: