- use native speech detector

bad_network_start.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-# Oneliner to find script's directory. Please note - last path component should NOT be symlink.
-SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
-
-/usr/bin/python3 ${SCRIPT_DIR}/src/utils_network_impairment.py --start
-

bad_network_stop.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-# Oneliner to find script's directory. Please note - last path component should NOT be symlink.
-SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
-
-/usr/bin/python3 ${SCRIPT_DIR}/src/utils_network_impairment.py --stop

config/agent.in.yaml

@@ -25,8 +25,8 @@ rabbitmq:
 cache_dir:              cache
 audio:
     # Silence prefix & suffix lengths (in seconds)
-    silence_prefix:     30
+    silence_prefix:     10
-    silence_suffix:     30
+    silence_suffix:     10
     
     bluetooth_mac:      "MAC_ADDRESS"
 

run_agent.sh

@@ -3,6 +3,12 @@
 # Oneliner to find script's directory. Please note - last path component should NOT be symlink.
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
 
+for pid in $(pidof -x run_agent.sh); do
+    if [ $pid != $$ ]; then
+        echo "[$(date)] : run_agent.sh : Process is already running with PID $pid. Exiting."
+        exit 1
+    fi
+done
 
 while :
 do

src/agent_config.py

@@ -36,8 +36,8 @@ class AgentConfig:
     # Should the first task run immediately ?
     ForceRun = False
 
-    # Use silence eraser or not (speech detector is used in this case)
+    # Use external speech detector if needed
-    UseSilenceEraser = True
+    UseSpeechDetector = False
 
     # Path to log file
     LogPath : Path = None
@@ -90,7 +90,7 @@ class AgentConfig:
 
             if 'speech_detector' in config:
                 if config['speech_detector']:
-                    self.UseSilenceEraser = False
+                    self.UseSpeechDetector = True
 
             if 'audio' in config:
                 audio = config['audio']

src/agent_gsm.py

@@ -68,7 +68,10 @@ def detect_degraded_signal(file_test: Path, file_reference: Path) -> SignalBound
         # Seems some problem with recording, return zero boundaries
         return SignalBoundaries()
     
-    r = bt_signal.find_reference_signal(file_test)
+    if CONFIG.UseSpeechDetector:
+        r = bt_signal.find_reference_signal_via_speechdetector(file_test)
+    else:
+        r = bt_signal.find_reference_signal(file_test)
     
     if r.offset_start == 0.0 and is_caller:
         r.offset_start = 5.0 # Skip ringing tones
@@ -78,7 +81,10 @@ def detect_degraded_signal(file_test: Path, file_reference: Path) -> SignalBound
 
 def detect_reference_signal(file_reference: Path) -> SignalBoundaries:
     # Run silence eraser on reference file as well
-    result = bt_signal.find_reference_signal(file_reference)
+    if CONFIG.UseSpeechDetector:
+        result = bt_signal.find_reference_signal_via_speechdetector(file_reference)
+    else:
+        result = bt_signal.find_reference_signal(file_reference)
     return result
 
 
@@ -92,38 +98,29 @@ def upload_results():
         # Path to audio
         path_audio = t[1]
 
-        utils.log(f'Found {path_report.name} and {path_audio.name} files.')
+        utils.log(f'Found {t} report pair.')
-        try:
+        if path_report is not None and path_report.exists():
-            with open(path_report, 'rt') as f:
+            try:
-                report = json.loads(f.read())
+                with open(path_report, 'rt') as f:
-        except:
+                    report = json.loads(f.read())
-            utils.log_error(f'Error when processing {path_report.name}')
+            except:
-            continue
+                utils.log_error(f'Error when processing {path_report.name}')
+                continue
 
-        upload_id, success = BACKEND.upload_report(report, cache=None) 
+            upload_id, success = BACKEND.upload_report(report, cache=None) 
-        if success:
+            if success:
-            utils.log(f'Report {upload_id} is uploaded ok.')
+                utils.log(f'Report {upload_id} is uploaded ok.')
             
-            # Rename files to make sync audio filename with reported ones
-            # path_report_fixed = CACHE.dir / f'{upload_id}.json'
-            # path_report = path_report.rename(path_report_fixed)
-
-            # path_audio_fixed = CACHE.dir / f'{upload_id}.wav'
-            # path_audio = path_audio.rename(path_audio_fixed)
-            if path_audio.exists():
-                utils.log(f'Uploading {path_audio.name} file...')
-                # Upload recorded audio
-                upload_result = BACKEND.upload_audio(upload_id, path_audio)
-                if upload_result:
-                    utils.log(f'  Recorded audio {upload_id}.wav is uploaded ok.')
-                os.remove(path_audio)
-            else:
-                utils.log(f'No recorded audio file found, skipping audio upload.')
             os.remove(path_report)
+
+        if path_audio is not None and path_audio.exists():
+            utils.log(f'Uploading {path_audio.name} file...')
+            # Upload recorded audio
+            upload_result = BACKEND.upload_audio(path_audio.stem, path_audio)
+            if upload_result:
+                utils.log(f'  Recorded audio {path_audio.stem}.wav is uploaded ok.')
+            os.remove(path_audio)
             
-        else:
-            utils.log(f'Failed to upload report {path_report.name}')
-            break
         
 def run_analyze(file_test: str, file_reference: str, number: str) -> bool:
     global CALL_COUNTER
@@ -147,8 +144,8 @@ def run_analyze(file_test: str, file_reference: str, number: str) -> bool:
         utils.log(f'Recorded audio call duration: {test_audio_length}s, reference audio length: {ref_audio_length}s')
         
         # Check if audio length is strange - skip such calls. Usually this is missed call.
-        is_caller_audio_big = is_caller and test_audio_length > ref_audio_length * 1.5
+        is_caller_audio_big = is_caller and test_audio_length > ref_audio_length * 3
-        is_answerer_audio_big = is_answerer and test_audio_length > ref_audio_length * 1.5
+        is_answerer_audio_big = is_answerer and test_audio_length > ref_audio_length * 3
 
         if is_caller_audio_big or is_answerer_audio_big:
             utils.log_error(f'Recorded call is too big - looks like mobile operator prompt, skipping analysis')

src/bt_signal.py

@@ -3,33 +3,22 @@
 import sys
 import os
 import pathlib
+from utils_types import SignalBoundaries
+from utils_sevana import speech_detector
 
 from pydub import silence, AudioSegment
 
-class SignalBoundaries:
+SILENCE_DELTA = 16
-    # Offset from start (in seconds)
-    offset_start: float
-
-    # Offset from finish (in seconds)
-    offset_finish: float
-
-    def __init__(self, offset_start = 0.0, offset_finish = 0.0) -> None:
-        self.offset_start = offset_start
-        self.offset_finish = offset_finish
-
-    def __repr__(self) -> str:
-        return f'[offset_start: {round(self.offset_start, 3)}, offset_finish : {round(self.offset_finish, 3)}]'
-
 
 def find_reference_signal(input_file: pathlib.Path, output_file: pathlib.Path = None, use_end_offset: bool = True) -> SignalBoundaries:
     myaudio = AudioSegment.from_wav(str(input_file))
     dBFS = myaudio.dBFS
     
     # Find silence intervals
-    intervals = silence.detect_nonsilent(myaudio, min_silence_len=1000, silence_thresh=dBFS-17, seek_step=50)
+    intervals = silence.detect_nonsilent(myaudio, min_silence_len=1000, silence_thresh=dBFS-SILENCE_DELTA, seek_step=50)
     
     # Translate to seconds
-    intervals = [((start/1000),(stop/1000)) for start,stop in intervals] #in sec
+    intervals = [((start/1000),(stop/1000)) for start,stop in intervals] # in sec
 
     # print(intervals)
 
@@ -48,6 +37,12 @@ def find_reference_signal(input_file: pathlib.Path, output_file: pathlib.Path =
 
     return SignalBoundaries()
 
+
+def find_reference_signal_via_speechdetector(input_file: pathlib.Path) -> SignalBoundaries:
+    bounds = speech_detector(str(input_file))
+    r = SignalBoundaries(bounds[0], bounds[1])
+    return bounds
+
 if __name__ == '__main__':
     if len(sys.argv) < 2:
         print(f'Please specify input filename.')

src/utils_cache.py

@@ -80,11 +80,16 @@ class InfoCache:
         lst = os.listdir(self.dir)
         for n in lst:
             p = self.dir / n
-            if self.is_valid_uuid(p.stem) and n.endswith('.json'):
+            if self.is_valid_uuid(p.stem) and (n.endswith('.json') or n.endswith(".wav")):
                 # Probe found
+                p_json = p.with_suffix('.json')
                 p_audio = p.with_suffix('.wav')
-                if p_audio.exists():
+                if p_json.exists() and p_audio.exists():
-                    r.append((p, p.with_suffix('.wav')))
+                    r.append((p_json, p_audio))
+                elif p_json.exists():
+                    r.append((p_json, None))
+                elif p_audio.exists():
+                    r.append((None, p_audio))
         
         return r
 

src/utils_qualtest.py

@@ -58,6 +58,7 @@ TRACE_TOTAL_TIMEOUT = 30
 # a webpage is mostly I/O bound, it's not going to be significant.
 
 def trace_function(frame, event, arg):
+    global TRACE_START_TIME
     if time.time() - TRACE_START_TIME > TRACE_TOTAL_TIMEOUT:
         raise Exception('Timed out!') # Use whatever exception you consider appropriate.
 
@@ -114,6 +115,7 @@ class QualtestBackend:
         
 
     def upload_audio(self, probe_id, path_recorded: Path):
+        global TRACE_START_TIME
         result = False
 
         # Log about upload attempt

src/utils_types.py

@@ -6,12 +6,25 @@ import utils
 import json
 from crontab import CronTab
 
-
-
 # Exit codes
 EXIT_OK = 0
 EXIT_ERROR = 1
 
+class SignalBoundaries:
+    # Offset from start (in seconds)
+    offset_start: float
+
+    # Offset from finish (in seconds)
+    offset_finish: float
+
+    def __init__(self, offset_start = 0.0, offset_finish = 0.0) -> None:
+        self.offset_start = offset_start
+        self.offset_finish = offset_finish
+
+    def __repr__(self) -> str:
+        return f'[offset_start: {round(self.offset_start, 3)}, offset_finish : {round(self.offset_finish, 3)}]'
+
+
 class Phone:
     identifier: int = 0
     name: str = ""