[speech-commands] Changes to API and model location (#100)

* [speech-commands] Changes to API and model location

- startStreaming() --> listen()
- stopStreaming() --> stopListening()
- isStreaming() --> isListening()
- Update model location

Author: caisq

speech-commands/README.md

@@ -39,7 +39,7 @@ A speech command recognizer can be used in two ways:
 ### Online streaming recognition
 
 To use the speech-command recognizer, first create a recognizer instance,
-then start the streaming recognition by calling its `startStreaming()` method.
+then start the streaming recognition by calling its `listen()` method.
 
 ```js
 import * as tf from '@tensorflow/tfjs';
@@ -59,12 +59,13 @@ await recognizer.ensureModelLoaded();
 // See the array of words that the recognizer is trained to recognize.
 console.log(recognizer.wordLabels());
 
-// `startStreaming()` takes two arguments:
+// `listen()` takes two arguments:
 // 1. A callback function that is invoked anytime a word is recognized.
 // 2. A configuration object with adjustable fields such a
 //    - includeSpectrogram
 //    - probabilityThreshold
-recognizer.startStreaming(result => {
+//    - includeEmbedding
+recognizer.listen(result => {
   // - result.scores contains the probability scores that correspond to
   //   recognizer.wordLabels().
   // - result.spectrogram contains the spectrogram of the recognized word.
@@ -100,7 +101,7 @@ Currently, the supported vocabularies are:
 #### Parameters for online streaming recognition
 
 As the example above shows, you can specify optional parameters when calling
-`startStreaming()`. The supported parameters are:
+`listen()`. The supported parameters are:
 
 * `overlapFactor`: Controls how often the recognizer performs prediction on
   spectrograms. Must be >=0 and <1 (default: 0.5). For example,
@@ -114,6 +115,11 @@ As the example above shows, you can specify optional parameters when calling
 * `invokeCallbackOnNoiseAndUnknown`: Whether the callback function will be
   invoked if the "word" with the maximum probability score is the "unknown"
   or "background noise" token. Default: `false`.
+* `includeEmbedding`: Whether an internal activation from the underlying model
+  will be included in the callback argument, in addition to the probability
+  scores. Note: if this field is set as `true`, the value of
+  `invokeCallbackOnNoiseAndUnknown` will be overridden to `true` and the
+  value of `probabilityThreshold` will be overridden to `0`.
 
 ### Offline recognition
 
@@ -154,11 +160,16 @@ tf.tidy(() => {
 });
 ```
 
+Note that you must provide a spectrogram value to the `recognize()` call
+in order to perform the offline recognition. If `recognzie()` as called
+without a first argument, it will perform one-shot online recognition
+by collecting a frame of audio via WebAudio.
+
 ### Preloading model
 
 By default, a recognizer object will load the underlying
 tf.Model via HTTP requests to a centralized location, when its
-`startStreaming()` or `recognize()` method is called the first time.
+`listen()` or `recognize()` method is called the first time.
 You can pre-load the model to reduce the latency of the first calls
 to these methods. To do that, use the `ensureModelLoaded()` method of the
 recognizer object. The `ensureModelLoaded()` method also "warms up" model after
@@ -236,7 +247,7 @@ await transferRecognizer.train({
 
 // After the transfer learning completes, you can start online streaming
 // recognition using the new model.
-await transferRecognizer.startStreaming(result => {
+await transferRecognizer.listen(result => {
   // - result.scores contains the scores for the new vocabulary, which
   //   can be checked with:
   const words = transferRecognizer.wordLabels();

speech-commands/demo/index.js

@@ -49,7 +49,7 @@ let transferRecognizer;
 
   // Make sure the tf.Model is loaded through HTTP. If this is not
   // called here, the tf.Model will be loaded the first time
-  // `startStreaming()` is called.
+  // `listen()` is called.
   recognizer.ensureModelLoaded()
       .then(() => {
         startButton.disabled = false;
@@ -79,7 +79,7 @@ startButton.addEventListener('click', () => {
   populateCandidateWords(activeRecognizer.wordLabels());
 
   activeRecognizer
-      .startStreaming(
+      .listen(
           result => {
             plotPredictions(
                 predictionCanvas, activeRecognizer.wordLabels(), result.scores,
@@ -104,7 +104,7 @@ startButton.addEventListener('click', () => {
 stopButton.addEventListener('click', () => {
   const activeRecognizer =
       transferRecognizer == null ? recognizer : transferRecognizer;
-  activeRecognizer.stopStreaming()
+  activeRecognizer.stopListening()
       .then(() => {
         startButton.disabled = false;
         stopButton.disabled = true;

speech-commands/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tensorflow-models/speech-commands",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "description": "Speech-command recognizer in TensorFlow.js",
   "main": "dist/index.js",
   "unpkg": "dist/speech-commands.min.js",

speech-commands/src/browser_fft_recognizer.ts

@@ -26,6 +26,11 @@ export const UNKNOWN_TAG = '_unknown_';
 
 let streaming = false;
 
+export function getMajorAndMinorVersion(version: string) {
+  const versionItems = version.split('.');
+  return versionItems.slice(0, 2).join('.');
+}
+
 /**
  * Speech-Command Recognizer using browser-native (WebAudio) spectral featutres.
  */
@@ -35,8 +40,8 @@ export class BrowserFftSpeechCommandRecognizer implements
   static readonly DEFAULT_VOCABULARY_NAME = '18w';
 
   readonly MODEL_URL_PREFIX =
-      `https://storage.googleapis.com/tfjs-speech-commands-models/v${
-          version}/browser_fft`;
+      `https://storage.googleapis.com/tfjs-models/tfjs/speech-commands/v${
+         getMajorAndMinorVersion(version)}/browser_fft`;
 
   private readonly SAMPLE_RATE_HZ = 44100;
   private readonly FFT_SIZE = 1024;
@@ -107,7 +112,7 @@ export class BrowserFftSpeechCommandRecognizer implements
   /**
    * Start streaming recognition.
    *
-   * To stop the recognition, use `stopStreaming()`.
+   * To stop the recognition, use `stopListening()`.
    *
    * Example: TODO(cais): Add exapmle code snippet.
    *
@@ -132,9 +137,8 @@ export class BrowserFftSpeechCommandRecognizer implements
    * @throws Error, if streaming recognition is already started or
    *   if `config` contains invalid values.
    */
-  async startStreaming(
-      callback: RecognizerCallback,
-      config?: StreamingRecognitionConfig): Promise<void> {
+  async listen(callback: RecognizerCallback,
+               config?: StreamingRecognitionConfig): Promise<void> {
     if (streaming) {
       throw new Error(
           'Cannot start streaming again when streaming is ongoing.');
@@ -355,7 +359,7 @@ export class BrowserFftSpeechCommandRecognizer implements
    *
    * @throws Error if there is not ongoing streaming recognition.
    */
-  async stopStreaming(): Promise<void> {
+  async stopListening(): Promise<void> {
     if (!streaming) {
       throw new Error('Cannot stop streaming when streaming is not ongoing.');
     }
@@ -366,7 +370,7 @@ export class BrowserFftSpeechCommandRecognizer implements
   /**
    * Check if streaming recognition is ongoing.
    */
-  isStreaming(): boolean {
+  isListening(): boolean {
     return streaming;
   }
 
@@ -397,7 +401,7 @@ export class BrowserFftSpeechCommandRecognizer implements
     if (this.model == null) {
       throw new Error(
           'Model has not been loaded yet. Load model by calling ' +
-          'ensureModelLoaded(), recognizer(), or startStreaming().');
+          'ensureModelLoaded(), recognize(), or listen().');
     }
     return this.model.inputs[0].shape;
   }
@@ -479,6 +483,15 @@ export class BrowserFftSpeechCommandRecognizer implements
       output.scores = await Promise.all(scorePromises) as Float32Array[];
       tf.dispose(unstacked);
     }
+
+    if (config.includeSpectrogram) {
+      output.spectrogram = {
+        data: (input instanceof tf.Tensor ?
+            await input.data() : input) as Float32Array,
+        frameSize: this.nonBatchInputShape[1],
+      };
+    }
+
     return output;
   }
 
@@ -507,7 +520,7 @@ export class BrowserFftSpeechCommandRecognizer implements
     if (this.model == null) {
       throw new Error(
           'Model has not been loaded yet. Load model by calling ' +
-          'ensureModelLoaded(), recognizer(), or startStreaming().');
+          'ensureModelLoaded(), recognizer(), or listen().');
     }
     tf.util.assert(
         name != null && typeof name === 'string' && name.length > 1,