fix: align Lion learning rate with spec

Author: ooples

src/Optimizers/LionOptimizer.cs

@@ -213,7 +213,7 @@ protected override IFullModel<T, TInput, TOutput> UpdateSolution(IFullModel<T, T
     {
         var parameters = currentSolution.GetParameters();
         var weightDecay = NumOps.FromDouble(_options.WeightDecay);
-        var effectiveLearningRate = GetEffectiveLearningRate();
+        var effectiveLearningRate = _currentLearningRate;
 
         for (int i = 0; i < gradient.Length; i++)
         {
@@ -273,7 +273,7 @@ public override Vector<T> UpdateParameters(Vector<T> parameters, Vector<T> gradi
         _t++;
 
         var weightDecay = NumOps.FromDouble(_options.WeightDecay);
-        var effectiveLearningRate = GetEffectiveLearningRate();
+        var effectiveLearningRate = _currentLearningRate;
         var updatedParams = new Vector<T>(parameters.Length);
 
         for (int i = 0; i < parameters.Length; i++)
@@ -332,7 +332,7 @@ public override Matrix<T> UpdateParameters(Matrix<T> parameters, Matrix<T> gradi
         _t++;
 
         var weightDecay = NumOps.FromDouble(_options.WeightDecay);
-        var effectiveLearningRate = GetEffectiveLearningRate();
+        var effectiveLearningRate = _currentLearningRate;
         var updatedMatrix = new Matrix<T>(parameters.Rows, parameters.Columns);
         int index = 0;
 
@@ -377,12 +377,6 @@ public override Matrix<T> UpdateParameters(Matrix<T> parameters, Matrix<T> gradi
         return updatedMatrix;
     }
 
-    private T GetEffectiveLearningRate()
-    {
-        var beta1Factor = NumOps.Subtract(NumOps.One, _currentBeta1);
-        var beta2Factor = NumOps.Subtract(NumOps.One, _currentBeta2);
-        return NumOps.Multiply(_currentLearningRate, NumOps.Multiply(beta1Factor, beta2Factor));
-    }
 
     /// <summary>
     /// Resets the optimizer's internal state.

tests/AiDotNet.Tests/UnitTests/Optimizers/LionOptimizerTests.cs

@@ -414,7 +414,7 @@ public void UpdateParameters_Vector_DifferentBeta1Values_ProducesDifferentResult
             var options1 = new LionOptimizerOptions<double, Vector<double>, Vector<double>>
             {
                 LearningRate = 0.1,
-                Beta1 = 0.5,
+                Beta1 = 0.2,
                 Beta2 = 0.99
             };
             var options2 = new LionOptimizerOptions<double, Vector<double>, Vector<double>>
@@ -428,15 +428,12 @@ public void UpdateParameters_Vector_DifferentBeta1Values_ProducesDifferentResult
             var optimizer2 = new LionOptimizer<double, Vector<double>, Vector<double>>(null, options2);
 
             var parameters = new Vector<double>(new double[] { 1.0, 2.0, 3.0 });
-            var gradient1 = new Vector<double>(new double[] { 0.5, 0.5, 0.5 });
-            var gradient2 = new Vector<double>(new double[] { 1.0, 1.0, 1.0 });
+            var gradient1 = new Vector<double>(new double[] { 100.0, 100.0, 100.0 });
+            var gradient2 = new Vector<double>(new double[] { -1.0, -1.0, -1.0 });
 
-            // Act - Build momentum then update with different gradient
-            optimizer1.UpdateParameters(parameters, gradient1);
-            var updated1 = optimizer1.UpdateParameters(parameters, gradient2);
-
-            optimizer2.UpdateParameters(parameters, gradient1);
-            var updated2 = optimizer2.UpdateParameters(parameters, gradient2);
+            // Act - Build momentum then flip gradient sign; beta1 affects the sign of the interpolated update.
+            var updated1 = optimizer1.UpdateParameters(optimizer1.UpdateParameters(parameters, gradient1), gradient2);
+            var updated2 = optimizer2.UpdateParameters(optimizer2.UpdateParameters(parameters, gradient1), gradient2);
 
             // Assert - Different beta1 values should produce different interpolations
             // and thus different results
@@ -477,21 +474,16 @@ public void UpdateParameters_Vector_DifferentBeta2Values_ProducesDifferentMoment
             var optimizer2 = new LionOptimizer<double, Vector<double>, Vector<double>>(null, options2);
 
             var parameters = new Vector<double>(new double[] { 1.0, 2.0, 3.0 });
-            var gradient = new Vector<double>(new double[] { 1.0, 1.0, 1.0 });
-
-            // Act - Multiple updates to see momentum effect
-            var params1 = new Vector<double>(parameters);
-            var params2 = new Vector<double>(parameters);
+            var gradientPos = new Vector<double>(new double[] { 1.0, 1.0, 1.0 });
+            var gradientNeg = new Vector<double>(new double[] { -1.0, -1.0, -1.0 });
 
-            for (int i = 0; i < 3; i++)
-            {
-                params1 = optimizer1.UpdateParameters(params1, gradient);
-                params2 = optimizer2.UpdateParameters(params2, gradient);
-            }
+            // Act - Build different momentum states with the same positive gradient, then flip gradient sign.
+            // Beta2 affects m_{t-1}, which in turn can change the sign of the interpolated update.
+            var params1 = optimizer1.UpdateParameters(optimizer1.UpdateParameters(parameters, gradientPos), gradientNeg);
+            var params2 = optimizer2.UpdateParameters(optimizer2.UpdateParameters(parameters, gradientPos), gradientNeg);
 
-            // Assert - Both should update, but momentum behavior differs
-            Assert.NotEqual(parameters, params1);
-            Assert.NotEqual(parameters, params2);
+            // Assert - Momentum behavior differs between the two beta2 values
+            Assert.NotEqual(params1, params2);
 
             // Verify that different Beta2 values produce different momentum behavior
             bool anyDifferent = false;