Skip to content

Commit 891c20a

Browse files
JulioJerezJulioJerez
committed
Update ndHandWrittenDigits.cpp
clean up mnist test script.
1 parent 9c327e0 commit 891c20a

File tree

1 file changed

+127
-74
lines changed

1 file changed

+127
-74
lines changed

‎newton-4.00/applications/ndSandbox/toolbox/ndHandWrittenDigits.cpp

Lines changed: 127 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,40 @@
1212
#include "ndSandboxStdafx.h"
1313
#include "ndTestDeepBrain.h"
1414

15+
//#define MNIST_USE_MINIST_CONVOLUTIONAL_LAYERS
16+
17+
//#define MINIST_MINIBATCH_BUFFER_SIZE 64
18+
#define MINIST_MINIBATCH_BUFFER_SIZE 256
19+
//#define MINIST_MINIBATCH_BUFFER_SIZE 2
20+
21+
#define MNIST_CONVOLUTIONAL_FEATURE_MAPS 32
22+
//#define MIN_TRAIN_SCORE 0.9999f
23+
24+
#define MINIST_NUMBER_OF_EPOCKS 70
25+
26+
#ifdef MNIST_USE_MINIST_CONVOLUTIONAL_LAYERS
27+
#if 1
28+
#define MINIST_CONVOLUTIONAL_LAYER ndBrainLayerConvolutional_2d
29+
#else
30+
#define MINIST_CONVOLUTIONAL_LAYER ndBrainLayerConvolutionalWithDropOut_2d
31+
#endif
32+
#endif
33+
34+
#define MINIST_LINEAR_LAYERS_NEURONS 256
35+
#define MINIST_LINEAR_DROPOUT_RATE ndFloat32 (0.05f)
36+
37+
#define MINIST_ACTIVATION_TYPE ndBrainLayerActivationRelu
38+
//#define MINIST_ACTIVATION_TYPE ndBrainLayerActivationElu
39+
//#define MINIST_ACTIVATION_TYPE ndBrainLayerActivationSigmoidLinear
40+
41+
//#if 1
42+
// //#define MINIST_DIGIT_FILTER_LAYER_TYPE ndBrainLayerConvolutional_2d
43+
// #define MINIST_DIGIT_FILTER_LAYER_TYPE ndBrainLayerConvolutionalWithDropOut_2d
44+
//#else
45+
// #define MINIST_DIGIT_FILTER_LAYER_TYPE ndBrainLayerCrossCorrelation_2d
46+
//#endif
47+
48+
1549
static ndBrainMatrix* LoadMnistLabelData(const char* const filename)
1650
{
1751
ndBrainMatrix* labelData = nullptr;
@@ -107,43 +141,6 @@ static ndBrainMatrix* LoadMnistSampleData(const char* const filename)
107141

108142
static void MnistTrainingSet()
109143
{
110-
//#define USE_CONVOLUTIONAL_LAYERS
111-
112-
//#define MINIBATCH_BUFFER_SIZE 64
113-
#define MINIBATCH_BUFFER_SIZE 256
114-
//#define MINIBATCH_BUFFER_SIZE 2
115-
116-
#define CONVOLUTIONAL_FEATURE_MAPS 32
117-
#define MIN_TRAIN_SCORE 0.9999f
118-
119-
#define NUMBER_OF_EPOCKS 150
120-
121-
#ifdef USE_CONVOLUTIONAL_LAYERS
122-
#if 1
123-
#define CONVOLUTIONAL_LAYER ndBrainLayerConvolutional_2d
124-
#else
125-
#define CONVOLUTIONAL_LAYER ndBrainLayerConvolutionalWithDropOut_2d
126-
#endif
127-
#endif
128-
129-
#define LINEAR_LAYERS_NEURONS 256
130-
#define LINEAR_DROPOUT_RATE ndFloat32 (0.05f)
131-
132-
#if 0
133-
#define ACTIVATION_TYPE ndBrainLayerActivationTanh
134-
#else
135-
#define ACTIVATION_TYPE ndBrainLayerActivationRelu
136-
//#define ACTIVATION_TYPE ndBrainLayerActivationElu
137-
//#define ACTIVATION_TYPE ndBrainLayerActivationSigmoidLinear
138-
#endif
139-
140-
//#if 1
141-
// //#define DIGIT_FILTER_LAYER_TYPE ndBrainLayerConvolutional_2d
142-
// #define DIGIT_FILTER_LAYER_TYPE ndBrainLayerConvolutionalWithDropOut_2d
143-
//#else
144-
// #define DIGIT_FILTER_LAYER_TYPE ndBrainLayerCrossCorrelation_2d
145-
//#endif
146-
147144
ndSharedPtr<ndBrainMatrix> trainingLabels (LoadMnistLabelData("mnistDatabase/train-labels.idx1-ubyte"));
148145
ndSharedPtr<ndBrainMatrix> trainingDigits (LoadMnistSampleData("mnistDatabase/train-images.idx3-ubyte"));
149146

@@ -158,7 +155,7 @@ static void MnistTrainingSet()
158155
,m_context()
159156
,m_prioritySamples()
160157
,m_learnRate(ndReal(5.0e-4f))
161-
,m_miniBatchSize(MINIBATCH_BUFFER_SIZE)
158+
,m_miniBatchSize(MINIST_MINIBATCH_BUFFER_SIZE)
162159
,m_minCombinedScore(ndInt64(1000000) * ndInt64(1000000))
163160
,m_minValidationFail(ndInt64(1000000)* ndInt64(1000000))
164161
,m_hasGpuSupport(m_brain->IsGpuReady())
@@ -272,7 +269,7 @@ static void MnistTrainingSet()
272269
m_bestBrain = ndSharedPtr<ndBrain>(new ndBrain (**trainer->GetBrain()));
273270

274271
ndBrainLossCategoricalCrossEntropy loss(outputSize);
275-
for (ndInt32 epoch = 0; epoch < NUMBER_OF_EPOCKS; ++epoch)
272+
for (ndInt32 epoch = 0; epoch < MINIST_NUMBER_OF_EPOCKS; ++epoch)
276273
{
277274
shuffleBuffer.RandomShuffle(shuffleBuffer.GetCount());
278275
for (ndInt32 batchStart = 0; batchStart < batchesSize; batchStart += m_miniBatchSize)
@@ -366,7 +363,7 @@ static void MnistTrainingSet()
366363
ndSharedPtr<ndBrain> m_bestBrain;
367364
ndSharedPtr<ndBrainTrainer> m_trainer;
368365
ndSharedPtr<ndBrainContext> m_context;
369-
ndFixSizeArray<ndFixSizeArray<ndUnsigned32, 16>, MINIBATCH_BUFFER_SIZE> m_prioritySamples;
366+
ndFixSizeArray<ndFixSizeArray<ndUnsigned32, 16>, MINIST_MINIBATCH_BUFFER_SIZE> m_prioritySamples;
370367
ndReal m_learnRate;
371368
ndInt32 m_miniBatchSize;
372369
ndInt64 m_minCombinedScore;
@@ -379,44 +376,44 @@ static void MnistTrainingSet()
379376
ndSharedPtr<ndBrain> brain(new ndBrain);
380377
ndFixSizeArray<ndBrainLayer*, 32> layers;
381378

382-
#ifdef USE_CONVOLUTIONAL_LAYERS
379+
#ifdef MNIST_USE_MINIST_CONVOLUTIONAL_LAYERS
383380
ndInt32 height = 28;
384381
ndInt32 width = trainingDigits->GetColumns() / height;
385382
ndAssert((height * width) == trainingDigits->GetColumns());
386383

387-
const CONVOLUTIONAL_LAYER* conv;
384+
const MINIST_CONVOLUTIONAL_LAYER* conv;
388385
const ndBrainLayerImagePolling_2x2* pooling;
389386

390-
layers.PushBack(new CONVOLUTIONAL_LAYER(width, height, 1, 3, CONVOLUTIONAL_FEATURE_MAPS));
391-
conv = (CONVOLUTIONAL_LAYER*)(layers[layers.GetCount() - 1]);
392-
layers.PushBack(new ACTIVATION_TYPE(conv->GetOutputSize()));
387+
layers.PushBack(new MINIST_CONVOLUTIONAL_LAYER(width, height, 1, 3, MNIST_CONVOLUTIONAL_FEATURE_MAPS));
388+
conv = (MINIST_CONVOLUTIONAL_LAYER*)(layers[layers.GetCount() - 1]);
389+
layers.PushBack(new MINIST_ACTIVATION_TYPE(conv->GetOutputSize()));
393390
layers.PushBack(new ndBrainLayerImagePolling_2x2(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
394391
pooling = (ndBrainLayerImagePolling_2x2*)(layers[layers.GetCount() - 1]);
395392

396-
layers.PushBack(new CONVOLUTIONAL_LAYER(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, CONVOLUTIONAL_FEATURE_MAPS));
397-
conv = (CONVOLUTIONAL_LAYER*)(layers[layers.GetCount() - 1]);
398-
layers.PushBack(new ACTIVATION_TYPE(conv->GetOutputSize()));
393+
layers.PushBack(new MINIST_CONVOLUTIONAL_LAYER(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, MNIST_CONVOLUTIONAL_FEATURE_MAPS));
394+
conv = (MINIST_CONVOLUTIONAL_LAYER*)(layers[layers.GetCount() - 1]);
395+
layers.PushBack(new MINIST_ACTIVATION_TYPE(conv->GetOutputSize()));
399396
layers.PushBack(new ndBrainLayerImagePolling_2x2(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
400397
pooling = (ndBrainLayerImagePolling_2x2*)(layers[layers.GetCount() - 1]);
401398

402-
layers.PushBack(new CONVOLUTIONAL_LAYER(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, CONVOLUTIONAL_FEATURE_MAPS));
403-
conv = (CONVOLUTIONAL_LAYER*)(layers[layers.GetCount() - 1]);
404-
layers.PushBack(new ACTIVATION_TYPE(conv->GetOutputSize()));
399+
layers.PushBack(new MINIST_CONVOLUTIONAL_LAYER(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, MNIST_CONVOLUTIONAL_FEATURE_MAPS));
400+
conv = (MINIST_CONVOLUTIONAL_LAYER*)(layers[layers.GetCount() - 1]);
401+
layers.PushBack(new MINIST_ACTIVATION_TYPE(conv->GetOutputSize()));
405402
layers.PushBack(new ndBrainLayerImagePolling_2x2(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels()));
406403
pooling = (ndBrainLayerImagePolling_2x2*)(layers[layers.GetCount() - 1]);
407404

408405
#else
409-
layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), LINEAR_LAYERS_NEURONS));
406+
layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), MINIST_LINEAR_LAYERS_NEURONS));
410407
layers.PushBack(new ndBrainLayerLinearWithDropOut(layers[layers.GetCount() - 1]->GetOutputSize()));
411-
layers.PushBack(new ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize()));
408+
layers.PushBack(new MINIST_ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize()));
412409

413-
layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), LINEAR_LAYERS_NEURONS));
410+
layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), MINIST_LINEAR_LAYERS_NEURONS));
414411
layers.PushBack(new ndBrainLayerLinearWithDropOut(layers[layers.GetCount() - 1]->GetOutputSize()));
415-
layers.PushBack(new ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize()));
412+
layers.PushBack(new MINIST_ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize()));
416413

417-
layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), LINEAR_LAYERS_NEURONS));
414+
layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), MINIST_LINEAR_LAYERS_NEURONS));
418415
layers.PushBack(new ndBrainLayerLinearWithDropOut(layers[layers.GetCount() - 1]->GetOutputSize()));
419-
layers.PushBack(new ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize()));
416+
layers.PushBack(new MINIST_ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize()));
420417
#endif
421418

422419
layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), trainingLabels->GetColumns()));
@@ -437,7 +434,7 @@ static void MnistTrainingSet()
437434
time = ndGetTimeInMicroseconds() - time;
438435

439436
char path[256];
440-
#ifdef USE_CONVOLUTIONAL_LAYERS
437+
#ifdef MNIST_USE_MINIST_CONVOLUTIONAL_LAYERS
441438
ndGetWorkingFileName("mnistDatabase/mnist-cnn.dnn", path);
442439
#else
443440
ndGetWorkingFileName("mnistDatabase/mnist.dnn", path);
@@ -473,32 +470,88 @@ static void MnistTestSet()
473470
if (testLabels && testDigits)
474471
{
475472
char path[256];
476-
#ifdef USE_CONVOLUTIONAL_LAYERS
473+
#ifdef MNIST_USE_MINIST_CONVOLUTIONAL_LAYERS
477474
ndGetWorkingFileName("mnistDatabase/mnist-cnn.dnn", path);
478475
#else
479476
ndGetWorkingFileName("mnistDatabase/mnist.dnn", path);
480477
#endif
481478

482-
//ndSharedPtr<ndBrain> brain (ndBrainLoad::Load(path));
483-
//ndInt32 numbeOfParam = brain->GetNumberOfParameters();
484-
//ndUnsigned64 time = ndGetTimeInMicroseconds();
485-
//ndExpandTraceMessage("mnist database, number of Parameters %d\n", numbeOfParam);
486-
//if (ndBrainGpuContext::HasGpuSupport())
487-
//{
488-
// ValidateDataGpu("test data", *(*brain), *testLabels, *testDigits);
489-
//}
490-
//else
491-
//{
492-
// ValidateData("test data", *(*brain), *testLabels, *testDigits);
493-
//}
494-
//time = ndGetTimeInMicroseconds() - time;
495-
//ndExpandTraceMessage("time %f (sec)\n\n", ndFloat64(time) / 1000000.0f);
479+
ndSharedPtr<ndBrain> brain (ndBrainLoad::Load(path));
480+
ndInt32 numbeOfParam = brain->GetNumberOfParameters();
481+
482+
ndInt32 threadCount = ndMin(ndBrainThreadPool::GetMaxThreads(), 8);
483+
ndSharedPtr<ndBrainContext> context(new ndBrainCpuContext);
484+
context->GetAsCpuContext()->SetThreadCount(threadCount);
485+
486+
ndInt32 minibatchSize = 256;
487+
ndSharedPtr<ndBrainTrainer> inference(ndSharedPtr<ndBrainTrainer>(new ndBrainTrainerCpuInference(brain, context, minibatchSize)));
488+
489+
ndBrainVector groundTruth;
490+
ndBrainVector miniBatchInput;
491+
ndBrainVector miniBatchOutput;
492+
493+
ndInt32 inputSize = testDigits->GetColumns();
494+
ndInt32 outputSize = testLabels->GetColumns();
495+
496+
miniBatchInput.SetCount(inputSize * minibatchSize);
497+
groundTruth.SetCount(outputSize * minibatchSize);
498+
499+
ndUnsigned64 time = ndGetTimeInMicroseconds();
500+
501+
ndInt32 failCount = 0;
502+
ndInt32 batchesCount = testDigits->GetRows() / minibatchSize;
503+
ndInt32 batchesSize = batchesCount * minibatchSize;
504+
for (ndInt32 batchStart = 0; batchStart < batchesSize; batchStart += minibatchSize)
505+
{
506+
for (ndInt32 i = 0; i < minibatchSize; ++i)
507+
{
508+
ndBrainMemVector input(&miniBatchInput[i * inputSize], inputSize);
509+
input.SetCount(inputSize);
510+
input.Set((**testDigits)[batchStart + i]);
511+
}
512+
inference->MakePrediction(miniBatchInput);
513+
inference->GetOutput(miniBatchOutput);
514+
515+
for (ndInt32 i = 0; i < minibatchSize; ++i)
516+
{
517+
ndBrainMemVector truth(&groundTruth[i * outputSize], outputSize);
518+
const ndBrainMemVector output(&miniBatchOutput[i * outputSize], outputSize);
519+
520+
truth.SetCount(outputSize);
521+
truth.Set((**testLabels)[batchStart + i]);
522+
523+
ndInt32 maxProbIndex = -1;
524+
ndBrainFloat maxProbability = ndBrainFloat(-1.0f);
525+
for (ndInt32 j = 0; j < output.GetCount(); j++)
526+
{
527+
if (output[j] > maxProbability)
528+
{
529+
maxProbIndex = j;
530+
maxProbability = output[j];
531+
}
532+
}
533+
534+
ndAssert(maxProbIndex >= 0);
535+
if (truth[maxProbIndex] == ndReal(0.0f))
536+
{
537+
failCount++;
538+
}
539+
}
540+
}
541+
542+
time = ndGetTimeInMicroseconds() - time;
543+
544+
ndFloat32 score = (ndFloat32)(batchesSize - failCount) / (ndFloat32)batchesSize;
545+
ndExpandTraceMessage("mnist database, number of Parameters %d\n", numbeOfParam);
546+
ndExpandTraceMessage(" success rate:%f%%", score * 100.0f);
547+
ndExpandTraceMessage(" test fail count:%d\n", failCount);
548+
ndExpandTraceMessage("time %f (sec)\n\n", ndFloat64(time) / 1000000.0f);
496549
}
497550
}
498551

499552
void ndHandWrittenDigits()
500553
{
501554
ndSetRandSeed(53);
502555
MnistTrainingSet();
503-
//MnistTestSet();
556+
MnistTestSet();
504557
}

0 commit comments

Comments
 (0)