Change the way how polynomials and sinusoids are generated.

Author: trekhleb

homemade/linear_regression/linear_regression.py

@@ -6,30 +6,34 @@
 
 
 class LinearRegression:
+    # pylint: disable=too-many-instance-attributes
     """Linear Regression Class"""
 
-    def __init__(self, data, labels, polynomial_degree=0, sinusoid_degree=0):
+    def __init__(self, data, labels, polynomial_degree=0, sinusoid_degree=0, normalize_data=True):
+        # pylint: disable=too-many-arguments
         """Linear regression constructor.
 
         :param data: training set.
         :param labels: training set outputs (correct values).
         :param polynomial_degree: degree of additional polynomial features.
         :param sinusoid_degree: multipliers for sinusoidal features.
+        :param normalize_data: flag that indicates that features should be normalized.
         """
 
         # Normalize features and add ones column.
         (
             data_processed,
             features_mean,
             features_deviation
-        ) = prepare_for_training(data, polynomial_degree, sinusoid_degree)
+        ) = prepare_for_training(data, polynomial_degree, sinusoid_degree, normalize_data)
 
         self.data = data_processed
         self.labels = labels
         self.features_mean = features_mean
         self.features_deviation = features_deviation
         self.polynomial_degree = polynomial_degree
         self.sinusoid_degree = sinusoid_degree
+        self.normalize_data = normalize_data
 
         # Initialize model parameters.
         num_features = self.data.shape[1]
@@ -113,7 +117,8 @@ def get_cost(self, data, labels, lambda_param):
         data_processed = prepare_for_training(
             data,
             self.polynomial_degree,
-            self.sinusoid_degree
+            self.sinusoid_degree,
+            self.normalize_data,
         )[0]
 
         return self.cost_function(data_processed, labels, lambda_param)
@@ -155,7 +160,8 @@ def predict(self, data):
         data_processed = prepare_for_training(
             data,
             self.polynomial_degree,
-            self.sinusoid_degree
+            self.sinusoid_degree,
+            self.normalize_data,
         )[0]
 
         # Do predictions using model hypothesis.

homemade/logistic_regression/logistic_regression.py

@@ -7,11 +7,11 @@
 
 
 class LogisticRegression:
-    # pylint: disable=R0902
+    # pylint: disable=too-many-instance-attributes
     """Logistic Regression Class"""
 
     def __init__(self, data, labels, polynomial_degree=0, sinusoid_degree=0, normalize_data=False):
-        # pylint: disable=R0913
+        # pylint: disable=too-many-arguments
         """Logistic regression constructor.
 
         :param data: training set.

homemade/neural_network/multilayer_perceptron.py

@@ -8,7 +8,7 @@
 class MultilayerPerceptron:
     """Multilayer Perceptron Class"""
 
-    # pylint: disable=R0913
+    # pylint: disable=too-many-arguments
     def __init__(self, data, labels, layers, epsilon, normalize_data=False):
         """Multilayer perceptron constructor.
 
@@ -72,7 +72,7 @@ def predict(self, data):
     def gradient_descent(
             data, labels, unrolled_theta, layers, regularization_param, max_iteration, alpha
     ):
-        # pylint: disable=R0913
+        # pylint: disable=too-many-arguments
         """Gradient descent function.
 
         Iteratively optimizes theta model parameters.

homemade/utils/features/__init__.py

@@ -1,6 +1,6 @@
 """Dataset Features Related Utils"""
 
 from .normalize import normalize
-from .add_polynomials import add_polynomials
-from .add_sinusoids import add_sinusoids
+from .generate_polynomials import generate_polynomials
+from .generate_sinusoids import generate_sinusoids
 from .prepare_for_training import prepare_for_training

homemade/utils/features/add_polynomials.py

@@ -0,0 +1,60 @@
+"""Add polynomial features to the features set"""
+
+import numpy as np
+from .normalize import normalize
+
+
+def generate_polynomials(dataset, polynomial_degree, normalize_data=False):
+    """Extends data set with polynomial features of certain degree.
+
+    Returns a new feature array with more features, comprising of
+    x1, x2, x1^2, x2^2, x1*x2, x1*x2^2, etc.
+
+    :param dataset: dataset that we want to generate polynomials for.
+    :param polynomial_degree: the max power of new features.
+    :param normalize_data: flag that indicates whether polynomials need to normalized or not.
+    """
+
+    # Split features on two halves.
+    features_split = np.array_split(dataset, 2, axis=1)
+    dataset_1 = features_split[0]
+    dataset_2 = features_split[1]
+
+    # Extract sets parameters.
+    (num_examples_1, num_features_1) = dataset_1.shape
+    (num_examples_2, num_features_2) = dataset_2.shape
+
+    # Check if two sets have equal amount of rows.
+    if num_examples_1 != num_examples_2:
+        raise ValueError('Can not generate polynomials for two sets with different number of rows')
+
+    # Check if at list one set has features.
+    if num_features_1 == 0 and num_features_2 == 0:
+        raise ValueError('Can not generate polynomials for two sets with no columns')
+
+    # Replace empty set with non-empty one.
+    if num_features_1 == 0:
+        dataset_1 = dataset_2
+    elif num_features_2 == 0:
+        dataset_2 = dataset_1
+
+    # Make sure that sets have the same number of features in order to be able to multiply them.
+    num_features = num_features_1 if num_features_1 < num_examples_2 else num_features_2
+    dataset_1 = dataset_1[:, :num_features]
+    dataset_2 = dataset_2[:, :num_features]
+
+    # Create polynomials matrix.
+    polynomials = np.empty((num_examples_1, 0))
+
+    # Generate polynomial features of specified degree.
+    for i in range(1, polynomial_degree + 1):
+        for j in range(i + 1):
+            polynomial_feature = (dataset_1 ** (i - j)) * (dataset_2 ** j)
+            polynomials = np.concatenate((polynomials, polynomial_feature), axis=1)
+
+    # Normalize polynomials if needed.
+    if normalize_data:
+        polynomials = normalize(polynomials)[0]
+
+    # Return generated polynomial features.
+    return polynomials

homemade/utils/features/generate_polynomials.py

@@ -3,7 +3,7 @@
 import numpy as np
 
 
-def add_sinusoids(dataset, sinusoid_degree):
+def generate_sinusoids(dataset, sinusoid_degree):
     """Extends data set with sinusoid features.
 
     Returns a new feature array with more features, comprising of
@@ -13,10 +13,14 @@ def add_sinusoids(dataset, sinusoid_degree):
     :param sinusoid_degree: multiplier for sinusoid parameter multiplications
     """
 
-    sinusoids = np.empty((dataset.shape[0], 0))
+    # Create sinusoids matrix.
+    num_examples = dataset.shape[0]
+    sinusoids = np.empty((num_examples, 0))
 
-    for degree in range(1, sinusoid_degree):
+    # Generate sinusoid features of specified degree.
+    for degree in range(1, sinusoid_degree + 1):
         sinusoid_features = np.sin(degree * dataset)
         sinusoids = np.concatenate((sinusoids, sinusoid_features), axis=1)
 
+    # Return generated sinusoidal features.
     return sinusoids

homemade/utils/features/add_sinusoids.py renamed to homemade/utils/features/generate_sinusoids.py

@@ -1,10 +1,9 @@
 """Prepares the dataset for training"""
 
-import math
 import numpy as np
 from .normalize import normalize
-from .add_sinusoids import add_sinusoids
-from .add_polynomials import add_polynomials
+from .generate_sinusoids import generate_sinusoids
+from .generate_polynomials import generate_polynomials
 
 
 def prepare_for_training(data, polynomial_degree=0, sinusoid_degree=0, normalize_data=True):
@@ -19,29 +18,27 @@ def prepare_for_training(data, polynomial_degree=0, sinusoid_degree=0, normalize
     # Normalize data set.
     features_mean = 0
     features_deviation = 0
+    data_normalized = data_processed
     if normalize_data:
         (
-            data_processed,
+            data_normalized,
             features_mean,
             features_deviation
         ) = normalize(data_processed)
 
+        # Replace processed data with normalized processed data.
+        # We need to have normalized data below while we will adding polynomials and sinusoids.
+        data_processed = data_normalized
+
     # Add sinusoidal features to the dataset.
-    if sinusoid_degree:
-        data_processed = add_sinusoids(data_processed, sinusoid_degree)
+    if sinusoid_degree > 0:
+        sinusoids = generate_sinusoids(data_normalized, sinusoid_degree)
+        data_processed = np.concatenate((data_processed, sinusoids), axis=1)
 
     # Add polynomial features to data set.
-    if polynomial_degree >= 2:
-        current_features_num = data_processed.shape[1]
-        middle_feature_index = math.floor(current_features_num / 2)
-
-        # Split features on halves.
-        features_split = np.split(data_processed, [middle_feature_index], axis=1)
-        first_half = features_split[0]
-        second_half = features_split[1]
-
-        # Generate polynomials.
-        data_processed = add_polynomials(first_half, second_half, polynomial_degree)
+    if polynomial_degree > 0:
+        polynomials = generate_polynomials(data_normalized, polynomial_degree, normalize_data)
+        data_processed = np.concatenate((data_processed, polynomials), axis=1)
 
     # Add a column of ones to X.
     data_processed = np.hstack((np.ones((num_examples, 1)), data_processed))