Merge pull request #174 from MycroftAI/revert-173-revert-172-feature/issues-128

Revert "Revert "Listener improvements (Fixes #128)""

Author: Ryan Sipes

mycroft/client/enclosure/enclosure.py

@@ -171,16 +171,17 @@ def __init_serial(self):
 
     def __register_events(self):
         self.client.on('mycroft.paired', self.__update_events)
-        self.client.on('recognizer_loop:wakeword', self.eyes.blink)
         self.__register_mouth_events()
 
     def __register_mouth_events(self):
-        self.client.on('recognizer_loop:listening', self.mouth.listen)
+        self.client.on('recognizer_loop:record_begin', self.mouth.listen)
+        self.client.on('recognizer_loop:record_end', self.mouth.reset)
         self.client.on('recognizer_loop:audio_output_start', self.mouth.talk)
         self.client.on('recognizer_loop:audio_output_end', self.mouth.reset)
 
     def __remove_mouth_events(self):
-        self.client.remove('recognizer_loop:listening', self.mouth.listen)
+        self.client.remove('recognizer_loop:record_begin', self.mouth.listen)
+        self.client.remove('recognizer_loop:record_end', self.mouth.reset)
         self.client.remove('recognizer_loop:audio_output_start',
                            self.mouth.talk)
         self.client.remove('recognizer_loop:audio_output_end',

mycroft/client/speech/listener.py

@@ -24,13 +24,12 @@
 import speech_recognition as sr
 
 from mycroft.client.speech.local_recognizer import LocalRecognizer
-from mycroft.client.speech.mic import MutableMicrophone, Recognizer
+from mycroft.client.speech.mic import MutableMicrophone, ResponsiveRecognizer
 from mycroft.client.speech.recognizer_wrapper import \
     RemoteRecognizerWrapperFactory
-from mycroft.client.speech.word_extractor import WordExtractor
 from mycroft.configuration import ConfigurationManager
 from mycroft.messagebus.message import Message
-from mycroft.metrics import MetricsAggregator, Stopwatch
+from mycroft.metrics import MetricsAggregator
 from mycroft.session import SessionManager
 from mycroft.util import CerberusAccessDenied
 from mycroft.util.log import getLogger
@@ -62,8 +61,7 @@ def run(self):
             self.recognizer.adjust_for_ambient_noise(source)
             while self.state.running:
                 try:
-                    self.emitter.emit("recognizer_loop:listening")
-                    audio = self.recognizer.listen(source)
+                    audio = self.recognizer.listen(source, self.emitter)
                     self.queue.put(audio)
                 except IOError, ex:
                     # NOTE: Audio stack on raspi is slightly different, throws
@@ -105,73 +103,27 @@ def _audio_length(audio):
             audio.sample_rate * audio.sample_width)
 
     def read_audio(self):
-        timer = Stopwatch()
-        audio = self.queue.get()
-        self.metrics.timer("mycroft.recognizer.audio.length_s",
-                           self._audio_length(audio))
-        self.queue.task_done()
-        timer.start()
+        audio_data = self.queue.get()
 
         if self.state.sleeping:
-            self.process_wake_up(audio)
-        elif self.state.skip_wakeword:
-            self.process_skip_wake_word(audio)
+            self.try_wake_up(audio_data)
         else:
-            self.process_wake_word(audio, timer)
+            self.process_audio(audio_data)
 
-        self.metrics.flush()
-
-    def process_wake_up(self, audio):
+    def try_wake_up(self, audio):
         if self.wakeup_recognizer.is_recognized(audio.frame_data,
                                                 self.metrics):
             SessionManager.touch()
             self.state.sleeping = False
             self.__speak("I'm awake.")  # TODO: Localization
             self.metrics.increment("mycroft.wakeup")
 
-    def process_wake_word(self, audio, timer):
-        hyp = self.mycroft_recognizer.transcribe(audio.frame_data,
-                                                 self.metrics)
-
-        if self.mycroft_recognizer.contains(hyp):
-            extractor = WordExtractor(audio, self.mycroft_recognizer,
-                                      self.metrics)
-            timer.lap()
-            extractor.calculate_range()
-            self.metrics.timer("mycroft.recognizer.extractor.time_s",
-                               timer.lap())
-            audio_before = extractor.get_audio_data_before()
-            self.metrics.timer("mycroft.recognizer.audio_extracted.length_s",
-                               self._audio_length(audio_before))
-            audio_after = extractor.get_audio_data_after()
-            self.metrics.timer("mycroft.recognizer.audio_extracted.length_s",
-                               self._audio_length(audio_after))
-
-            SessionManager.touch()
-            payload = {
-                'utterance': hyp.hypstr,
-                'session': SessionManager.get().session_id,
-                'pos_begin': extractor.begin,
-                'pos_end': extractor.end
-            }
-            self.emitter.emit("recognizer_loop:wakeword", payload)
-
-            try:
-                self.transcribe([audio_before, audio_after])
-            except sr.UnknownValueError:
-                self.__speak("Go ahead")
-                self.state.skip_wakeword = True
-                self.metrics.increment("mycroft.wakeword")
-
-    def process_skip_wake_word(self, audio):
-        SessionManager.touch()
+    def process_audio(self, audio):
         try:
             self.transcribe([audio])
-        except sr.UnknownValueError:
+        except sr.UnknownValueError:  # TODO: Localization
             logger.warn("Speech Recognition could not understand audio")
             self.__speak("Sorry, I didn't catch that.")
-            self.metrics.increment("mycroft.recognizer.error")
-        self.state.skip_wakeword = False
 
     def __speak(self, utterance):
         payload = {
@@ -184,18 +136,18 @@ def _create_remote_stt_runnable(self, audio, utterances):
         def runnable():
             try:
                 text = self.remote_recognizer.transcribe(
-                        audio, metrics=self.metrics).lower()
+                    audio, metrics=self.metrics).lower()
             except sr.UnknownValueError:
                 pass
             except sr.RequestError as e:
                 logger.error(
-                        "Could not request results from Speech Recognition "
-                        "service; {0}".format(e))
+                    "Could not request results from Speech Recognition "
+                    "service; {0}".format(e))
             except CerberusAccessDenied as e:
                 logger.error("AccessDenied from Cerberus proxy.")
                 self.__speak(
-                        "Your device is not registered yet. To start pairing, "
-                        "login at cerberus dot mycroft dot A.I")
+                    "Your device is not registered yet. To start pairing, "
+                    "login at cerberus dot mycroft dot A.I")
                 utterances.append("pair my device")
             except Exception as e:
                 logger.error("Unexpected exception: {0}".format(e))
@@ -253,7 +205,7 @@ def __init__(self, channels=int(speech_config.get('channels')),
         self.mycroft_recognizer = LocalRecognizer(sample_rate, lang)
         # TODO - localization
         self.wakeup_recognizer = LocalRecognizer(sample_rate, lang, "wake up")
-        self.remote_recognizer = Recognizer()
+        self.remote_recognizer = ResponsiveRecognizer(self.mycroft_recognizer)
         self.state = RecognizerLoopState()
 
     def start_async(self):
@@ -270,7 +222,7 @@ def start_async(self):
                       self.wakeup_recognizer,
                       self.mycroft_recognizer,
                       RemoteRecognizerWrapperFactory.wrap_recognizer(
-                              self.remote_recognizer)).start()
+                          self.remote_recognizer)).start()
 
     def stop(self):
         self.state.running = False

mycroft/client/speech/local_recognizer.py

@@ -59,5 +59,5 @@ def is_recognized(self, byte_data, metrics):
         hyp = self.transcribe(byte_data, metrics)
         return hyp and self.key_phrase in hyp.hypstr.lower()
 
-    def contains(self, hypothesis):
+    def found_wake_word(self, hypothesis):
         return hypothesis and self.key_phrase in hypothesis.hypstr.lower()

mycroft/client/speech/main.py

@@ -35,14 +35,14 @@
 config = ConfigurationManager.get()
 
 
-def handle_listening():
-    logger.info("Listening...")
-    client.emit(Message('recognizer_loop:listening'))
+def handle_record_begin():
+    logger.info("Begin Recording...")
+    client.emit(Message('recognizer_loop:record_begin'))
 
 
-def handle_wakeword(event):
-    logger.info("Wakeword Detected: " + event['utterance'])
-    client.emit(Message('recognizer_loop:wakeword', event))
+def handle_record_end():
+    logger.info("End Recording...")
+    client.emit(Message('recognizer_loop:record_end'))
 
 
 def handle_utterance(event):
@@ -93,9 +93,9 @@ def main():
     if device_index:
         device_index = int(device_index)
     loop = RecognizerLoop(device_index=device_index)
-    loop.on('recognizer_loop:listening', handle_listening)
-    loop.on('recognizer_loop:wakeword', handle_wakeword)
     loop.on('recognizer_loop:utterance', handle_utterance)
+    loop.on('recognizer_loop:record_begin', handle_record_begin)
+    loop.on('recognizer_loop:record_end', handle_record_end)
     loop.on('speak', handle_speak)
     client.on('speak', handle_speak)
     client.on(