From 0c5652f98a14987f8ac3f2d971b85169c89564fd Mon Sep 17 00:00:00 2001 From: Dmytro Bogovych Date: Sat, 19 Aug 2023 14:10:57 +0300 Subject: [PATCH] - fix the problem with the --- config/agent.in.yaml | 10 ++-- config/systemd/agent_gsm.in.service | 2 +- src/agent_gsm.py | 70 +++++++++++------------ src/utils.py | 2 +- src/utils_qualtest.py | 88 +++++++++++++++++++---------- src/utils_sevana.py | 5 +- 6 files changed, 102 insertions(+), 75 deletions(-) diff --git a/config/agent.in.yaml b/config/agent.in.yaml index ecc450f..d026de8 100644 --- a/config/agent.in.yaml +++ b/config/agent.in.yaml @@ -47,6 +47,10 @@ audio: bluetooth: yes bluetooth_mac: "MAC_ADDRESS" + + # Relative pathes starts at agent_gsm root + backup_dir: backup + temp_dir: /dev/shm log: # Log file path (otherwise log will be sent to syslog) @@ -57,9 +61,3 @@ log: # Log ADB output adb: yes - - # Upload full audio recordings - audio: yes - - # Where to keep audio - audio_dir: /dev/shm diff --git a/config/systemd/agent_gsm.in.service b/config/systemd/agent_gsm.in.service index cd465b3..db12d9c 100644 --- a/config/systemd/agent_gsm.in.service +++ b/config/systemd/agent_gsm.in.service @@ -19,7 +19,7 @@ KillSignal=SIGQUIT # make sure log directory exists and owned by syslog PermissionsStartOnly=true -ExecStartPre=/usr/bin/rm -f ABSOLUTE_INSTALL_DIR/qualtest.pid +ExecStartPre=/usr/bin/rm -f /dev/shm/qualtest.pid #ExecStartPre=/bin/chown syslog:adm /var/log/sleepservice #ExecStartPre=/bin/chmod 755 /var/log/sleepservice diff --git a/src/agent_gsm.py b/src/agent_gsm.py index af3c079..c6fa62b 100644 --- a/src/agent_gsm.py +++ b/src/agent_gsm.py @@ -15,6 +15,8 @@ import utils_sevana import utils_mcon import utils_logcat import utils +import utils_cache + from bt_controller import Bluetoothctl import bt_call_controller import bt_signal @@ -57,15 +59,14 @@ CALL_LIMIT = 0 # Find script's directory DIR_THIS = Path(__file__).resolve().parent +DIR_PROJECT = DIR_THIS.parent + +# Backup directory (to run without internet) +DIR_CACHE = None +CACHE = utils_cache.InfoCache(None) # PID file name -QUALTEST_PID = DIR_THIS / "qualtest.pid" - -# Keep the recorded audio in the directory -LOG_AUDIO = False - -# Recorded audio directory -LOG_AUDIO_DIR = DIR_THIS.parent / 'log_audio' +QUALTEST_PID = "/dev/shm/qualtest.pid" # Should the first task run immediately ? FORCE_RUN = False @@ -74,6 +75,7 @@ FORCE_RUN = False EXIT_OK = 0 EXIT_ERROR = 1 + # Use silence eraser or not (speech detector is used in this case) USE_SILENCE_ERASER = True @@ -87,7 +89,7 @@ def remove_oldest_log_audio(): def detect_degraded_signal(file_test: Path, file_reference: Path) -> SignalBoundaries: - global USE_SILENCE_ERASER, LOG_AUDIO, LOG_AUDIO_DIR, BackendServer + global USE_SILENCE_ERASER, LOG_AUDIO_DIR, BackendServer is_caller : bool = 'caller' in BackendServer.phone.role is_answerer : bool = 'answer' in BackendServer.phone.role @@ -105,9 +107,7 @@ def detect_degraded_signal(file_test: Path, file_reference: Path) -> SignalBound def detect_reference_signal(file_reference: Path) -> SignalBoundaries: - global USE_SILENCE_ERASER, LOG_AUDIO, LOG_AUDIO_DIR # Run silence eraser on reference file as well - result = bt_signal.find_reference_signal(file_reference) return result @@ -138,12 +138,7 @@ def run_analyze(file_test: str, file_reference: str, number: str) -> bool: bounds_signal : SignalBoundaries = detect_degraded_signal(Path(file_test), Path(file_reference)) # bounds_signal.offset_start = 0 # bounds_signal.offset_finish = 0 - print(f'Found signal bounds: {bounds_signal}') - # Check if there is a time to remove oldest files - if LOG_AUDIO: - remove_oldest_log_audio() - remove_oldest_log_audio() # PVQA report pvqa_mos, pvqa_report, pvqa_rfactor = utils_sevana.find_pvqa_mos(file_test, bounds_signal.offset_start, bounds_signal.offset_finish) @@ -188,8 +183,8 @@ def run_analyze(file_test: str, file_reference: str, number: str) -> bool: r['task_name'] = CURRENT_TASK # Upload report - upload_id = BackendServer.upload_report(r, []) - if upload_id != None: + upload_id, success = BackendServer.upload_report(r) + if upload_id != None and success: utils.log('Report is uploaded ok.') # Upload recorded audio @@ -316,10 +311,15 @@ def run_probe(): while True: # Get task list update tasks = BackendServer.load_tasks() + if tasks is None: + # Check in cache + tasks = CACHE.get_tasks(BackendServer.phone.name) + # Did we fetch anything ? if tasks: # Merge with existing ones. Some tasks can be removed, some can be add. changed = TASK_LIST.merge_with(tasks) + CACHE.put_tasks(changed) else: utils.log_verbose(f"No task list assigned, exiting.") sys.exit(EXIT_ERROR) @@ -467,24 +467,16 @@ if config['log']['adb']: utils.log('Enabled adb logcat output') # Audio directories -if 'audio_dir' in config['log']: - if config['log']['audio_dir']: - LOG_AUDIO_DIR = config['log']['audio_dir'] - - # Ensure subdirectory log_audio exists - if not os.path.exists(LOG_AUDIO_DIR): - utils.log(f'Creating {LOG_AUDIO_DIR}') - os.mkdir(LOG_AUDIO_DIR) - -if 'audio' in config['log']: - if config['log']['audio']: - LOG_AUDIO = True +if 'cache_dir' in config: + DIR_CACHE = Path(config['cache_dir']) + if not DIR_CACHE.is_absolute(): + DIR_CACHE = DIR_CACHE / config['cache_dir'] + CACHE = utils_cache.InfoCache(dir=DIR_CACHE) # Update path to pvqa/aqua-wb -dir_script = os.path.dirname(os.path.realpath(__file__)) -utils_sevana.find_binaries(os.path.join(dir_script, "../bin")) +utils_sevana.find_binaries(DIR_PROJECT / 'bin') utils.log('Analyzer binaries are found') # Load latest licenses & configs - this requires utils_sevana.find_binaries() to be called before @@ -518,7 +510,11 @@ with open(QUALTEST_PID, "w") as f: try: # Load information about phone utils.log(f'Loading information about the node {BackendServer.instance} from {BackendServer.address}') - BackendServer.preload() + BackendServer.preload(CACHE.dir) + if BackendServer.phone is None: + utils.log_error(f'Failed to obtain information about {BackendServer.instance}. Exiting.') + exit(EXIT_ERROR) + if 'answerer' in BackendServer.phone.role: # Check if task name is specified @@ -533,9 +529,13 @@ try: # Load reference audio utils.log('Loading reference audio...') - if not BackendServer.load_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO): - utils.log_error('Audio is not available, exiting.') - sys.exit(EXIT_ERROR) + if BackendServer.load_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO): + CACHE.add_reference_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO) + else: + utils.log_error('Audio is not available online.') + if not CACHE.get_reference_audio(BackendServer.phone.audio_id, REFERENCE_AUDIO): + utils.log_error('Reference audio is not cached, sorry. Exiting.') + sys.exit(EXIT_ERROR) # Preparing reference audio utils.log('Running answering loop...') diff --git a/src/utils.py b/src/utils.py index af19d76..e793a73 100644 --- a/src/utils.py +++ b/src/utils.py @@ -26,7 +26,7 @@ verbose_logging: bool = False the_log = None # 1 minute network timeout -NETWORK_TIMEOUT = 60 +NETWORK_TIMEOUT = 15 def open_log_file(path: str, mode: str): diff --git a/src/utils_qualtest.py b/src/utils_qualtest.py index e7dfcbf..334a541 100644 --- a/src/utils_qualtest.py +++ b/src/utils_qualtest.py @@ -17,6 +17,7 @@ import requests from socket import timeout from crontab import CronTab from pathlib import Path +from utils_cache import InfoCache start_system_time = time.time() start_monotonic_time = time.monotonic() @@ -149,6 +150,32 @@ class Phone: self.attributes = dict() self.audio_id = 0 + def to_dict(self) -> dict: + return { + 'id': self.identifier, + 'name': self.name, + 'role': self.role, + 'attr': self.attributes, + 'audio_id': self.audio_id + } + + def make(d: dict): + r = Phone() + + r.identifier = d['id'] + r.name = d['name'] + r.role = d['role'] + if 'attr' in d: + r.attr = d['attr'] + else: + r.attr = None + + if 'audio_id' in d: + r.audio_id = d['audio_id'] + else: + r.audio_id = None + + return r class QualtestBackend: address: str @@ -165,34 +192,31 @@ class QualtestBackend: return self.__phone - def preload(self): - self.__phone = self.load_phone() + def preload(self, cache_dir: Path): + self.__phone = self.load_phone(cache_dir) - def upload_report(self, report, files) -> str: + def upload_report(self, report, cache: InfoCache) -> (str, bool): # UUID string as result - result = None + result = (None, False) # Log about upload attempt - utils.log_verbose(f"Uploading to {self.address} files {files} and report: {json.dumps(report, indent=4)}") - - # POST will be sent to args.qualtest_server with args.qualtest_instance ID - json_content = json.dumps(report, indent=4).encode('utf8') - - # Find URL for uploading + utils.log_verbose(f"Uploading to {self.address} report: {json.dumps(report, indent=4)}") url = utils.join_host_and_path(self.address, "/probes/") + try: - # Step 1 - upload result record - req = urllib.request.Request(url, - data=json_content, - headers={'content-type': 'application/json'}) - response = urllib.request.urlopen(req, timeout=utils.NETWORK_TIMEOUT) - result = response.read().decode('utf8') - utils.log_verbose(f"Response (probe ID): {result}") - utils.log_verbose(f"Upload to {self.address} finished.") + r = requests.post(url=url, json=report, timeout=utils.NETWORK_TIMEOUT) + utils.log_verbose(f"Upload report finished. Response (probe ID): {r.content}") + if r.status_code != 200: + raise RuntimeError(f'Server returned code {r.status_code} and content {r.content}') + result = (r.content.decode().strip(), True) except Exception as e: - utils.log_error(f"Upload to {self.address} finished with error.", err=e) + utils.log_error(f"Upload report to {self.address} finished with error.", err=e) + + # Backup probe result + probe_id = cache.add_report(report) + result = (probe_id, False) return result @@ -216,8 +240,7 @@ class QualtestBackend: if response.status_code != 200: utils.log_error(f"Upload audio to {self.address} finished with error {response.status_code}", None) else: - utils.log_verbose(f"Response (audio ID): {response.text}") - utils.log_verbose(f"Upload audio to {self.address} finished.") + utils.log_verbose(f"Upload audio finished. Response (audio ID): {response.text}") result = True except Exception as e: utils.log_error(f"Upload audio to {self.address} finished with error.", err=e) @@ -248,7 +271,8 @@ class QualtestBackend: return None - def load_phone(self) -> dict: + def load_phone(self, cache: InfoCache) -> dict: + result = None try: # Build query for both V1 & V2 API instance = urllib.parse.urlencode({"phone_id": self.instance, "phone_name": self.instance}) @@ -257,16 +281,21 @@ class QualtestBackend: url = utils.join_host_and_path(self.address, "/phones/?") + instance # Get response from server - response = urllib.request.urlopen(url, timeout=utils.NETWORK_TIMEOUT) - if response.getcode() != 200: - utils.log_error("Failed to get task list. Error code: %s" % response.getcode()) - return None - - result: Phone = Phone() + try: + response = urllib.request.urlopen(url, timeout=utils.NETWORK_TIMEOUT) + if response.getcode() != 200: + utils.log_error("Failed to get task list. Error code: %s" % response.getcode()) + return None + except Exception as e: + utils.log_error(f'Problem when loading the phone definition.') + return cache.get_phone(self.instance) + + # Get possible list of phones phones = json.loads(response.read().decode()) if len(phones) == 0: - return result + return None + # But use first one phone = phones[0] attr_dict = dict() @@ -290,6 +319,7 @@ class QualtestBackend: if 'sip_useproxy' not in attr_dict: attr_dict['sip_useproxy'] = True + result = Phone() result.attributes = attr_dict result.identifier = phone['id'] result.name = phone['instance'] diff --git a/src/utils_sevana.py b/src/utils_sevana.py index ff4c823..9af0635 100644 --- a/src/utils_sevana.py +++ b/src/utils_sevana.py @@ -13,6 +13,7 @@ import urllib from pathlib import Path from colorama import Fore, Style +from utils_cache import InfoCache PVQA_CMD = "{pvqa} --license {pvqa_lic} --config {pvqa_cfg} --mode analysis --channel 0 " \ "--report {output} --input {input} --cut-begin {cut_begin} --cut-end {cut_end}" @@ -77,7 +78,7 @@ def load_config_and_licenses(server: str): load_file(utils.join_host_and_path(server, '/deploy/aqua-wb.lic'), AQUA_LIC_PATH) -def find_binaries(directory: str, license_server: str = None): +def find_binaries(bin_directory: Path, license_server: str = None): # Update path to pvqa/aqua-wb global PVQA_CFG_PATH, PVQA_LIC_PATH, AQUA_LIC_PATH, PVQA_PATH, AQUA_PATH, PVQA_CMD, AQUA_CMD, SILER_PATH, SPEECH_DETECTOR_PATH @@ -86,8 +87,6 @@ def find_binaries(directory: str, license_server: str = None): if utils.is_raspberrypi(): platform_prefix = 'rpi' - bin_directory = Path(directory) - PVQA_PATH = bin_directory / platform_prefix / PVQA_PATH PVQA_LIC_PATH = bin_directory / PVQA_LIC_PATH PVQA_CFG_PATH = bin_directory / PVQA_CFG_PATH