- fix the problem with the

config/agent.in.yaml

@@ -48,6 +48,10 @@ audio:
     bluetooth:          yes
     bluetooth_mac:      "MAC_ADDRESS"
    
+    # Relative pathes starts at agent_gsm root
+    backup_dir:         backup
+    temp_dir:           /dev/shm
+
 log:
     # Log file path (otherwise log will be sent to syslog)
     path:               
@@ -57,9 +61,3 @@ log:
     
     # Log ADB output
     adb:                yes
-    
-    # Upload full audio recordings
-    audio:              yes
-    
-    # Where to keep audio
-    audio_dir:          /dev/shm

config/systemd/agent_gsm.in.service

@@ -19,7 +19,7 @@ KillSignal=SIGQUIT
 
 # make sure log directory exists and owned by syslog
 PermissionsStartOnly=true
-ExecStartPre=/usr/bin/rm -f ABSOLUTE_INSTALL_DIR/qualtest.pid
+ExecStartPre=/usr/bin/rm -f /dev/shm/qualtest.pid
 
 #ExecStartPre=/bin/chown syslog:adm /var/log/sleepservice
 #ExecStartPre=/bin/chmod 755 /var/log/sleepservice

src/agent_gsm.py

@@ -15,6 +15,8 @@ import utils_sevana
 import utils_mcon
 import utils_logcat
 import utils
+import utils_cache
+
 from bt_controller import Bluetoothctl
 import bt_call_controller
 import bt_signal
@@ -57,15 +59,14 @@ CALL_LIMIT = 0
 
 # Find script's directory
 DIR_THIS = Path(__file__).resolve().parent
+DIR_PROJECT = DIR_THIS.parent
+
+# Backup directory (to run without internet)
+DIR_CACHE = None
+CACHE = utils_cache.InfoCache(None)
 
 # PID file name
-QUALTEST_PID = DIR_THIS / "qualtest.pid"
+QUALTEST_PID = "/dev/shm/qualtest.pid"
-
-# Keep the recorded audio in the directory 
-LOG_AUDIO = False
-
-# Recorded audio directory
-LOG_AUDIO_DIR = DIR_THIS.parent / 'log_audio'
 
 # Should the first task run immediately ?
 FORCE_RUN = False
@@ -74,6 +75,7 @@ FORCE_RUN = False
 EXIT_OK = 0
 EXIT_ERROR = 1
 
+
 # Use silence eraser or not (speech detector is used in this case)
 USE_SILENCE_ERASER = True
 
@@ -87,7 +89,7 @@ def remove_oldest_log_audio():
 
 
 def detect_degraded_signal(file_test: Path, file_reference: Path) -> SignalBoundaries:
-    global USE_SILENCE_ERASER, LOG_AUDIO, LOG_AUDIO_DIR, BackendServer
+    global USE_SILENCE_ERASER, LOG_AUDIO_DIR, BackendServer
 
     is_caller : bool = 'caller' in BackendServer.phone.role
     is_answerer : bool = 'answer' in BackendServer.phone.role
@@ -105,9 +107,7 @@ def detect_degraded_signal(file_test: Path, file_reference: Path) -> SignalBound
 
 
 def detect_reference_signal(file_reference: Path) -> SignalBoundaries:
-    global USE_SILENCE_ERASER, LOG_AUDIO, LOG_AUDIO_DIR
     # Run silence eraser on reference file as well
-
     result = bt_signal.find_reference_signal(file_reference)
 
     return result
@@ -138,12 +138,7 @@ def run_analyze(file_test: str, file_reference: str, number: str) -> bool:
             bounds_signal : SignalBoundaries = detect_degraded_signal(Path(file_test), Path(file_reference))
             # bounds_signal.offset_start = 0
             # bounds_signal.offset_finish = 0
-
             print(f'Found signal bounds: {bounds_signal}')
-            # Check if there is a time to remove oldest files
-            if LOG_AUDIO:
-                remove_oldest_log_audio()
-                remove_oldest_log_audio()
 
             # PVQA report
             pvqa_mos, pvqa_report, pvqa_rfactor = utils_sevana.find_pvqa_mos(file_test, bounds_signal.offset_start, bounds_signal.offset_finish)
@@ -188,8 +183,8 @@ def run_analyze(file_test: str, file_reference: str, number: str) -> bool:
             r['task_name'] = CURRENT_TASK
             
             # Upload report
-            upload_id = BackendServer.upload_report(r, []) 
+            upload_id, success = BackendServer.upload_report(r) 
-            if upload_id != None:
+            if upload_id != None and success:
                 utils.log('Report is uploaded ok.')
 
                 # Upload recorded audio
@@ -316,10 +311,15 @@ def run_probe():
     while True:
         # Get task list update
         tasks = BackendServer.load_tasks()
+        if tasks is None:
+            # Check in cache
+            tasks = CACHE.get_tasks(BackendServer.phone.name)
+
         # Did we fetch anything ?
         if tasks:
             # Merge with existing ones. Some tasks can be removed, some can be add.
             changed = TASK_LIST.merge_with(tasks)
+            CACHE.put_tasks(changed)
         else:
             utils.log_verbose(f"No task list assigned, exiting.")
             sys.exit(EXIT_ERROR)
@@ -467,24 +467,16 @@ if config['log']['adb']:
     utils.log('Enabled adb logcat output')
 
 # Audio directories
-if 'audio_dir' in config['log']:
+if 'cache_dir' in config:
-    if config['log']['audio_dir']:
+    DIR_CACHE = Path(config['cache_dir'])
-        LOG_AUDIO_DIR = config['log']['audio_dir']
+    if not DIR_CACHE.is_absolute():
-
+        DIR_CACHE = DIR_CACHE / config['cache_dir']
-        # Ensure subdirectory log_audio exists
-        if not os.path.exists(LOG_AUDIO_DIR):
-            utils.log(f'Creating {LOG_AUDIO_DIR}')
-            os.mkdir(LOG_AUDIO_DIR)
-
-if 'audio' in config['log']:
-    if config['log']['audio']:
-        LOG_AUDIO = True
 
+    CACHE = utils_cache.InfoCache(dir=DIR_CACHE)
 
 
 # Update path to pvqa/aqua-wb
-dir_script = os.path.dirname(os.path.realpath(__file__))
+utils_sevana.find_binaries(DIR_PROJECT / 'bin')
-utils_sevana.find_binaries(os.path.join(dir_script, "../bin"))
 utils.log('Analyzer binaries are found')
 
 # Load latest licenses & configs - this requires utils_sevana.find_binaries() to be called before
@@ -518,7 +510,11 @@ with open(QUALTEST_PID, "w") as f:
 try:
     # Load information about phone
     utils.log(f'Loading information about the node {BackendServer.instance} from {BackendServer.address}')
-    BackendServer.preload()
+    BackendServer.preload(CACHE.dir)
+    if BackendServer.phone is None:
+        utils.log_error(f'Failed to obtain information about {BackendServer.instance}. Exiting.')
+        exit(EXIT_ERROR)
+    
 
     if 'answerer' in BackendServer.phone.role:
         # Check if task name is specified
@@ -533,8 +529,12 @@ try:
 
         # Load reference audio
         utils.log('Loading reference audio...')
-        if not BackendServer.load_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO):
+        if BackendServer.load_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO):
-            utils.log_error('Audio is not available, exiting.')
+            CACHE.add_reference_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO)
+        else:
+            utils.log_error('Audio is not available online.')
+            if not CACHE.get_reference_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO):
+                utils.log_error('Reference audio is not cached, sorry. Exiting.')
                 sys.exit(EXIT_ERROR)
 
         # Preparing reference audio

src/utils.py

@@ -26,7 +26,7 @@ verbose_logging: bool = False
 the_log = None
 
 # 1 minute network timeout
-NETWORK_TIMEOUT = 60
+NETWORK_TIMEOUT = 15
 
 
 def open_log_file(path: str, mode: str):

src/utils_qualtest.py

@@ -17,6 +17,7 @@ import requests
 from socket import timeout
 from crontab import CronTab
 from pathlib import Path
+from utils_cache import InfoCache
 
 start_system_time = time.time()
 start_monotonic_time = time.monotonic()
@@ -149,6 +150,32 @@ class Phone:
         self.attributes = dict()
         self.audio_id = 0
 
+    def to_dict(self) -> dict:
+        return {
+            'id': self.identifier,
+            'name': self.name,
+            'role': self.role,
+            'attr': self.attributes,
+            'audio_id': self.audio_id
+        }
+
+    def make(d: dict):
+        r = Phone()
+
+        r.identifier = d['id']
+        r.name = d['name']
+        r.role = d['role']
+        if 'attr' in d:
+            r.attr = d['attr']
+        else:
+            r.attr = None
+
+        if 'audio_id' in d:
+            r.audio_id = d['audio_id']
+        else:
+            r.audio_id = None
+
+        return r
     
 class QualtestBackend:
     address: str
@@ -165,34 +192,31 @@ class QualtestBackend:
         return self.__phone
 
 
-    def preload(self):
+    def preload(self, cache_dir: Path):
-        self.__phone = self.load_phone()
+        self.__phone = self.load_phone(cache_dir)
 
 
-    def upload_report(self, report, files) -> str:
+    def upload_report(self, report, cache: InfoCache) -> (str, bool):
         # UUID string as result
-        result = None
+        result = (None, False)
         
         # Log about upload attempt
-        utils.log_verbose(f"Uploading to {self.address} files {files} and report: {json.dumps(report, indent=4)}")
+        utils.log_verbose(f"Uploading to {self.address} report: {json.dumps(report, indent=4)}")
-
-        # POST will be sent to args.qualtest_server with args.qualtest_instance ID
-        json_content = json.dumps(report, indent=4).encode('utf8')
-        
-        # Find URL for uploading
         url = utils.join_host_and_path(self.address, "/probes/")
-        try:
-            # Step 1 - upload result record
-            req = urllib.request.Request(url,
-                                        data=json_content,
-                                        headers={'content-type': 'application/json'})
-            response = urllib.request.urlopen(req, timeout=utils.NETWORK_TIMEOUT)
-            result = response.read().decode('utf8')
-            utils.log_verbose(f"Response (probe ID): {result}")
-            utils.log_verbose(f"Upload to {self.address} finished.")
         
+        try:
+            r = requests.post(url=url, json=report, timeout=utils.NETWORK_TIMEOUT)
+            utils.log_verbose(f"Upload report finished. Response (probe ID): {r.content}")
+            if r.status_code != 200:
+                raise RuntimeError(f'Server returned code {r.status_code} and content {r.content}')
+            
+            result = (r.content.decode().strip(), True)
         except Exception as e:
-            utils.log_error(f"Upload to {self.address} finished with error.", err=e)
+            utils.log_error(f"Upload report to {self.address} finished with error.", err=e)
+            
+            # Backup probe result
+            probe_id = cache.add_report(report)
+            result = (probe_id, False)
         
         return result
         
@@ -216,8 +240,7 @@ class QualtestBackend:
             if response.status_code != 200:
                 utils.log_error(f"Upload audio to {self.address} finished with error {response.status_code}", None)
             else:
-                utils.log_verbose(f"Response (audio ID): {response.text}")
+                utils.log_verbose(f"Upload audio finished. Response (audio ID): {response.text}")
-                utils.log_verbose(f"Upload audio to {self.address} finished.")
                 result = True
         except Exception as e:
             utils.log_error(f"Upload audio to {self.address} finished with error.", err=e)
@@ -248,7 +271,8 @@ class QualtestBackend:
             return None
 
     
-    def load_phone(self) -> dict:
+    def load_phone(self, cache: InfoCache) -> dict:
+        result = None
         try:
             # Build query for both V1 & V2 API
             instance = urllib.parse.urlencode({"phone_id": self.instance, "phone_name": self.instance})
@@ -257,16 +281,21 @@ class QualtestBackend:
             url = utils.join_host_and_path(self.address, "/phones/?") + instance
 
             # Get response from server
+            try:
                 response = urllib.request.urlopen(url, timeout=utils.NETWORK_TIMEOUT)
                 if response.getcode() != 200:
                     utils.log_error("Failed to get task list. Error code: %s" % response.getcode())
                     return None
+            except Exception as e:
+                utils.log_error(f'Problem when loading the phone definition.')
+                return cache.get_phone(self.instance)
             
-            result: Phone = Phone()
+            # Get possible list of phones
             phones = json.loads(response.read().decode())
             if len(phones) == 0:
-                return result
+                return None
 
+            # But use first one
             phone = phones[0]
 
             attr_dict = dict()
@@ -290,6 +319,7 @@ class QualtestBackend:
             if 'sip_useproxy' not in attr_dict:
                 attr_dict['sip_useproxy'] = True
 
+            result = Phone()
             result.attributes = attr_dict
             result.identifier = phone['id']
             result.name = phone['instance']

src/utils_sevana.py

@@ -13,6 +13,7 @@ import urllib
 
 from pathlib import Path
 from colorama import Fore, Style
+from utils_cache import InfoCache
 
 PVQA_CMD = "{pvqa} --license {pvqa_lic} --config {pvqa_cfg} --mode analysis --channel 0 " \
            "--report {output} --input {input} --cut-begin {cut_begin} --cut-end {cut_end}"
@@ -77,7 +78,7 @@ def load_config_and_licenses(server: str):
     load_file(utils.join_host_and_path(server, '/deploy/aqua-wb.lic'), AQUA_LIC_PATH)
     
 
-def find_binaries(directory: str, license_server: str = None):
+def find_binaries(bin_directory: Path, license_server: str = None):
     # Update path to pvqa/aqua-wb
     global PVQA_CFG_PATH, PVQA_LIC_PATH, AQUA_LIC_PATH, PVQA_PATH, AQUA_PATH, PVQA_CMD, AQUA_CMD, SILER_PATH, SPEECH_DETECTOR_PATH
 
@@ -86,8 +87,6 @@ def find_binaries(directory: str, license_server: str = None):
     if utils.is_raspberrypi():
         platform_prefix = 'rpi'
         
-    bin_directory = Path(directory)
-
     PVQA_PATH =             bin_directory / platform_prefix / PVQA_PATH
     PVQA_LIC_PATH =         bin_directory / PVQA_LIC_PATH
     PVQA_CFG_PATH =         bin_directory / PVQA_CFG_PATH