fix: Prevent extra backpropagation

Author: robertleeplummerjr

src/layer/add.ts

@@ -46,7 +46,6 @@ export class Add extends Operator {
       this.inputLayer1.weights,
       this.inputLayer2.weights
     ) as Texture;
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/base-layer.ts

@@ -57,6 +57,7 @@ export interface ILayerSettings {
   initPraxis?:
     | ((layerTemplate: ILayer, settings?: IPraxisSettings) => IPraxis)
     | null;
+  cleanupDeltas?: boolean;
 }
 
 export const baseLayerDefaultSettings: ILayerSettings = {
@@ -67,6 +68,7 @@ export const baseLayerDefaultSettings: ILayerSettings = {
   deltas: null,
   praxis: null,
   praxisOpts: null,
+  cleanupDeltas: true,
 };
 
 export type BaseLayerType = new (settings?: Partial<ILayerSettings>) => ILayer;
@@ -95,6 +97,9 @@ export class BaseLayer implements ILayer {
 
   set weights(weights: KernelOutput | Input) {
     this.settings.weights = weights as KernelOutput;
+    if (this.settings.cleanupDeltas && this.deltas) {
+      clear(this.deltas);
+    }
   }
 
   get deltas(): KernelOutput {
@@ -245,7 +250,6 @@ export class BaseLayer implements ILayer {
     if (!this.praxis) throw new Error('this.praxis not defined');
     this.weights = this.praxis.run(this, learningRate as number);
     release(oldWeights);
-    clear(this.deltas);
   }
 
   toArray(): TextureArrayOutput {

src/layer/convolution.ts

@@ -464,7 +464,6 @@ export class Convolution extends Filter {
     const { weights: oldWeights } = this;
     this.weights = (this.praxis as IPraxis).run(this, learningRate);
     release(oldWeights);
-    clear(this.deltas);
   }
 }
 

src/layer/input.ts

@@ -61,7 +61,6 @@ export class Input extends EntryPoint {
     } else {
       throw new Error('Inputs are not of sized correctly');
     }
-    clear(this.deltas);
   }
 
   predict1D(inputs: KernelOutput): void {
@@ -71,7 +70,6 @@ export class Input extends EntryPoint {
     } else {
       this.weights = inputs;
     }
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/leaky-relu.ts

@@ -75,7 +75,6 @@ export class LeakyRelu extends Activation {
     this.weights = (this.predictKernel as IKernelRunShortcut)(
       this.inputLayer.weights
     );
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/multiply-element.ts

@@ -61,7 +61,6 @@ export class MultiplyElement extends Operator {
       this.inputLayer1.weights,
       this.inputLayer2.weights
     );
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/multiply.ts

@@ -124,7 +124,6 @@ export class Multiply extends Operator {
       this.inputLayer1.weights,
       this.inputLayer2.weights
     ) as Texture;
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/recurrent-zeros.ts

@@ -54,7 +54,6 @@ export class RecurrentZeros extends Internal implements IRecurrentInput {
     this.weights = (this.praxis as IPraxis).run(this, learningRate);
     // this.deltas = deltas;
     release(oldWeights);
-    clear(this.deltas);
   }
 
   // validate(): void {

src/layer/relu.ts

@@ -76,7 +76,6 @@ export class Relu extends Activation {
     this.weights = (this.predictKernel as IKernelRunShortcut)(
       this.inputLayer.weights
     );
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/sigmoid.ts

@@ -75,7 +75,6 @@ export class Sigmoid extends Activation {
     this.weights = (this.predictKernel as IKernelRunShortcut)(
       this.inputLayer.weights
     );
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/tanh.ts

@@ -75,7 +75,6 @@ export class Tanh extends Activation {
     this.weights = (this.predictKernel as IKernelRunShortcut)(
       this.inputLayer.weights
     );
-    clear(this.deltas);
   }
 
   compare(): void {

src/layer/target.ts

@@ -68,7 +68,6 @@ export class Target extends BaseLayer {
     // NOTE: this looks like it shouldn't be, but the weights are immutable, and this is where they are reused.
     release(this.weights);
     this.weights = clone(this.inputLayer.weights as KernelOutput);
-    clear(this.deltas);
   }
 
   compare(targetValues: KernelOutput): void {

src/layer/transpose.ts

@@ -36,7 +36,6 @@ export class Transpose extends Modifier {
     this.weights = (this.predictKernel as IKernelRunShortcut)(
       this.inputLayer.weights
     );
-    clear(this.deltas);
   }
 
   compare(): void {

src/praxis/momentum-root-mean-squared-propagation.ts

@@ -90,7 +90,7 @@ export interface IMomentumRootMeanSquaredPropagationSettings
 
 export const defaults: IMomentumRootMeanSquaredPropagationSettings = {
   decayRate: 0.999,
-  regularizationStrength: 0.0001,
+  regularizationStrength: 0.000001,
   learningRate: 0.01,
   smoothEps: 1e-8,
   clipValue: 5,

src/recurrent.end-to-end.test.ts

@@ -86,7 +86,7 @@ describe('Recurrent Class: End to End', () => {
       recurrentNet: Recurrent<number[]>;
     } {
       const timeStep: RNNTimeStep = new RNNTimeStep({
-        regc: 0.001,
+        regc: 0.000001,
         inputSize: 1,
         hiddenLayers: [3],
         outputSize: 1,
@@ -693,11 +693,13 @@ describe('Recurrent Class: End to End', () => {
           expect(asArrayOfArrayOfNumber(model[2].weights)[0][0]).toBe(
             timeStep.model.allMatrices[2].weights[0]
           );
-          expect(asArrayOfArrayOfNumber(model[2].weights)[1][0]).toBe(
-            timeStep.model.allMatrices[2].weights[1]
+          expect(asArrayOfArrayOfNumber(model[2].weights)[1][0]).toBeCloseTo(
+            timeStep.model.allMatrices[2].weights[1],
+            0.00000000009
           );
-          expect(asArrayOfArrayOfNumber(model[2].weights)[2][0]).toBe(
-            timeStep.model.allMatrices[2].weights[2]
+          expect(asArrayOfArrayOfNumber(model[2].weights)[2][0]).toBeCloseTo(
+            timeStep.model.allMatrices[2].weights[2],
+            0.00000000009
           );
           expect(asArrayOfArrayOfNumber(model[3].weights)[0][0]).toBe(
             timeStep.model.allMatrices[3].weights[0]
@@ -1313,7 +1315,7 @@ describe('Recurrent Class: End to End', () => {
       inputLayer: () => input({ height: 1 }),
       hiddenLayers: [
         (inputLayer: ILayer, recurrentInput: IRecurrentInput) =>
-          lstmCell({ height: 3 }, inputLayer, recurrentInput),
+          lstmCell({ height: 10 }, inputLayer, recurrentInput),
       ],
       outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer),
     });
@@ -1325,7 +1327,7 @@ describe('Recurrent Class: End to End', () => {
     ];
     const errorThresh = 0.03;
     const iterations = 5000;
-    const status = net.train(xorNetValues);
+    const status = net.train(xorNetValues, { errorThresh, iterations });
     // expect(
     //   status.error <= errorThresh || status.iterations <= iterations
     // ).toBeTruthy();
@@ -1335,8 +1337,8 @@ describe('Recurrent Class: End to End', () => {
     console.log(net.run([[1], [0.001]]));
     console.log(net.run([[1], [1]]));
     expect(net.run([[0.001], [0.001]])[0][0]).toBeLessThan(0.1);
-    expect(net.run([[0.001], [1]])[0][0]).toBeGreaterThan(9);
-    expect(net.run([[1], [0.001]])[0][0]).toBeGreaterThan(9);
+    expect(net.run([[0.001], [1]])[0][0]).toBeGreaterThan(0.9);
+    expect(net.run([[1], [0.001]])[0][0]).toBeGreaterThan(0.9);
     expect(net.run([[1], [1]])[0][0]).toBeLessThan(0.1);
   });
   test('can learn 1,2,3', () => {

src/recurrent.ts

@@ -412,7 +412,7 @@ export class Recurrent<
         throw new Error('this.meanSquaredError not setup');
       }
       let error: KernelOutput = new Float32Array(1);
-      for (let i = 0, max = inputs.length - 1; i <= max; i++) {
+      for (let i = 0, max = inputs.length - 2; i <= max; i++) {
         const layerSet = this._layerSets[i];
         const lastLayer = layerSet[layerSet.length - 1];
         const prevError: KernelOutput = error;