part 2

Author: matthieurouif

assignment1.pdf

@@ -18,14 +18,23 @@ def gradcheck_naive(f, x):
     it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
     while not it.finished:
         ix = it.multi_index
-
+        
         ### try modifying x[ix] with h defined above to compute numerical gradients
         ### make sure you call random.setstate(rndstate) before calling f(x) each time, this will make it 
         ### possible to test cost functions with built in randomness later
         ### YOUR CODE HERE:
-        raise NotImplementedError
-        ### END YOUR CODE
+        epsilon = np.zeros(x.shape)
+        epsilon[ix] = h
+
+        random.setstate(rndstate)
+        f_m, grad_m = f(x - epsilon)
+
+        random.setstate(rndstate)
+        f_p, grad_p = f(x + epsilon)
 
+        numgrad = (f_p - f_m) / (2 * h)
+
+        ### END YOUR CODE
         # Compare gradients
         reldiff = abs(numgrad - grad[ix]) / max(1, abs(numgrad), abs(grad[ix]))
         if reldiff > 1e-5:
@@ -35,7 +44,6 @@ def gradcheck_naive(f, x):
             return
 
         it.iternext() # Step to next dimension
-
     print "Gradient check passed!"
 
 def sanity_check():

q1_softmax.pyc

@@ -16,6 +16,7 @@ def forward_backward_prop(data, labels, params, dimensions):
     ### Unpack network parameters (do not modify)
     ofs = 0
     Dx, H, Dy = (dimensions[0], dimensions[1], dimensions[2])
+    N = labels.shape[0]
 
     W1 = np.reshape(params[ofs:ofs+ Dx * H], (Dx, H))
     ofs += Dx * H
@@ -26,17 +27,35 @@ def forward_backward_prop(data, labels, params, dimensions):
     b2 = np.reshape(params[ofs:ofs + Dy], (1, Dy))
 
     ### YOUR CODE HERE: forward propagation
-    raise NotImplementedError
+    x = data
+    a1 = x
+    z1 = x
+    z2 = np.dot(a1,W1) + b1
+    a2 = sigmoid(z2)
+    z3 = np.dot(a2, W2) + b2
+
+    a3 = softmax(z3) #normalized for probabilty
+    #loss function would be here ? https://en.wikipedia.org/wiki/Cross_entropy or lecture 6
+    cost = - np.sum(np.log(np.sum(labels * a3, axis=1))) / N #this labels * a3 multiplication only keeps the non 0 vector, we use the softmax function here
+
     ### END YOUR CODE
 
     ### YOUR CODE HERE: backward propagation
-    raise NotImplementedError
+    error = (a3 - labels) / N
+    
+    gradW2 = np.dot(np.transpose(a2), error)
+    gradb2 = np.sum(error, axis=0)
+
+    delta2 = sigmoid_grad(a2) * np.dot(error, np.transpose(W2))
+    gradW1 = np.dot( np.transpose(x), delta2)
+    gradb1 = np.sum(delta2, axis=0)
+
     ### END YOUR CODE
 
     ### Stack gradients (do not modify)
     grad = np.concatenate((gradW1.flatten(), gradb1.flatten(), 
         gradW2.flatten(), gradb2.flatten()))
-    
+        
     return cost, grad
 
 def sanity_check():
@@ -46,12 +65,12 @@ def sanity_check():
     """
     print "Running sanity check..."
 
-    N = 20
+    N = 20 #number of window to classify
     dimensions = [10, 5, 10]
     data = np.random.randn(N, dimensions[0])   # each row will be a datum
     labels = np.zeros((N, dimensions[2]))
     for i in xrange(N):
-        labels[i,random.randint(0,dimensions[2]-1)] = 1
+        labels[i,random.randint(0,dimensions[2]-1)] = 1 # give probability 1 to each row, give a 100% label to each row
 
     params = np.random.randn((dimensions[0] + 1) * dimensions[1] + (
         dimensions[1] + 1) * dimensions[2], )
@@ -73,4 +92,4 @@ def your_sanity_checks():
 
 if __name__ == "__main__":
     sanity_check()
-    your_sanity_checks()
+    your_sanity_checks()

q2_gradcheck.py

@@ -4,11 +4,7 @@ def sigmoid(x):
     """
     Compute the sigmoid function for the input here.
     """
-    
-    ### YOUR CODE HERE
-    raise NotImplementedError
-    ### END YOUR CODE
-    
+    x = 1 / (1 + np.exp(-x))
     return x
 
 def sigmoid_grad(f):
@@ -17,10 +13,7 @@ def sigmoid_grad(f):
     for this implementation, the input f should be the sigmoid
     function value of your original input x. 
     """
-    
-    ### YOUR CODE HERE
-    raise NotImplementedError
-    ### END YOUR CODE
+    f = (1-f)*(f)
 
     return f
 

q2_gradcheck.pyc

@@ -10,7 +10,8 @@ def normalizeRows(x):
     # Implement a function that normalizes each row of a matrix to have unit length
 
     ### YOUR CODE HERE
-    raise NotImplementedError
+    norm = np.sqrt(np.sum(x * x,axis = 1, keepdims=True))
+    x = x/norm
     ### END YOUR CODE
 
     return x
@@ -195,4 +196,4 @@ def getRandomContext(C):
 
 if __name__ == "__main__":
     test_normalize_rows()
-    test_word2vec()
+    test_word2vec()