- improved setup script

config/ap/etc/default/hostapd

@@ -0,0 +1,23 @@
+# Defaults for hostapd initscript
+#
+# WARNING: The DAEMON_CONF setting has been deprecated and will be removed
+#          in future package releases.
+#
+# See /usr/share/doc/hostapd/README.Debian for information about alternative
+# methods of managing hostapd.
+#
+# Uncomment and set DAEMON_CONF to the absolute path of a hostapd configuration
+# file and hostapd will be started during system boot. An example configuration
+# file can be found at /usr/share/doc/hostapd/examples/hostapd.conf.gz
+#
+DAEMON_CONF="/etc/hostapd/hostapd.conf"
+
+# Additional daemon options to be appended to hostapd command:-
+# 	-d   show more debug messages (-dd for even more)
+# 	-K   include key data in debug messages
+# 	-t   include timestamps in some debug messages
+#
+# Note that -B (daemon mode) and -P (pidfile) options are automatically
+# configured by the init.d script and must not be added to DAEMON_OPTS.
+#
+#DAEMON_OPTS=""

config/ap/etc/dhcpcd.conf

@@ -0,0 +1,64 @@
+# A sample configuration for dhcpcd.
+# See dhcpcd.conf(5) for details.
+
+# Allow users of this group to interact with dhcpcd via the control socket.
+#controlgroup wheel
+
+# Inform the DHCP server of our hostname for DDNS.
+hostname
+
+# Use the hardware address of the interface for the Client ID.
+clientid
+# or
+# Use the same DUID + IAID as set in DHCPv6 for DHCPv4 ClientID as per RFC4361.
+# Some non-RFC compliant DHCP servers do not reply with this set.
+# In this case, comment out duid and enable clientid above.
+#duid
+
+# Persist interface configuration when dhcpcd exits.
+persistent
+
+# Rapid commit support.
+# Safe to enable by default because it requires the equivalent option set
+# on the server to actually work.
+option rapid_commit
+
+# A list of options to request from the DHCP server.
+option domain_name_servers, domain_name, domain_search, host_name
+option classless_static_routes
+# Respect the network MTU. This is applied to DHCP routes.
+option interface_mtu
+
+# Most distributions have NTP support.
+#option ntp_servers
+
+# A ServerID is required by RFC2131.
+require dhcp_server_identifier
+
+# Generate SLAAC address using the Hardware Address of the interface
+#slaac hwaddr
+# OR generate Stable Private IPv6 Addresses based from the DUID
+slaac private
+
+# Example static IP configuration:
+#interface eth0
+#static ip_address=192.168.0.10/24
+#static ip6_address=fd51:42f8:caae:d92e::ff/64
+#static routers=192.168.0.1
+#static domain_name_servers=192.168.0.1 8.8.8.8 fd51:42f8:caae:d92e::1
+
+# It is possible to fall back to a static IP if DHCP fails:
+# define static profile
+#profile static_eth0
+#static ip_address=192.168.1.23/24
+#static routers=192.168.1.1
+#static domain_name_servers=192.168.1.1
+
+# fallback to static profile on eth0
+#interface eth0
+#fallback static_eth0
+#
+#
+interface wlan0
+static ip_address=192.168.45.1/24
+static routers=192.168.45.1

config/ap/etc/dnsmasq.conf

@@ -0,0 +1,5 @@
+interface=wlan0				# Listening interface
+dhcp-range=192.168.45.10,192.168.45.20,255.255.255.0,24h	# Pool of IP addresses for wireless clients
+domain=wlan				# Domain
+address=/gw.wlan/192.168.45.1		# Alias for router
+

config/ap/etc/hostapd.conf

@@ -0,0 +1,8 @@
+country_code=GB
+interface=wlan0
+ssid=AGENT_GSM
+hw_mode=g
+channel=7
+macaddr_acl=0
+auth_algs=1
+ignore_broadcast_ssid=0

config/ap/lib/systemd/system/dnsmasq.service

@@ -0,0 +1,31 @@
+[Unit]
+Description=dnsmasq - A lightweight DHCP and caching DNS server
+Requires=network.target
+Wants=network-online.target
+Before=nss-lookup.target
+After=network-online.target
+
+[Service]
+Type=forking
+PIDFile=/run/dnsmasq/dnsmasq.pid
+
+# Test the config file and refuse starting if it is not valid.
+ExecStartPre=/usr/sbin/dnsmasq --test
+
+# We run dnsmasq via the /etc/init.d/dnsmasq script which acts as a
+# wrapper picking up extra configuration files and then execs dnsmasq
+# itself, when called with the "systemd-exec" function.
+ExecStart=/etc/init.d/dnsmasq systemd-exec
+
+# The systemd-*-resolvconf functions configure (and deconfigure)
+# resolvconf to work with the dnsmasq DNS server. They're called like
+# this to get correct error handling (ie don't start-resolvconf if the 
+# dnsmasq daemon fails to start.
+ExecStartPost=/etc/init.d/dnsmasq systemd-start-resolvconf
+ExecStop=/etc/init.d/dnsmasq systemd-stop-resolvconf
+
+
+ExecReload=/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target

setup/setup_agent_gsm.sh

@@ -10,8 +10,8 @@ INSTALL_DIR=agent_gsm
 GIT_SOURCE=https://git.sevana.biz/public/agent_gsm
 
 # Install prerequisites
-sudo apt install --assume-yes git mc python3 sox vim libffi-dev screen python3-pip python3-numpy
+sudo apt install --assume-yes git mc python3 sox vim libffi-dev screen python3-pip python3-numpy dnsmasq hostapd screen
-sudo pip3 install pyyaml sox pyrabbit soundfile dbus_python pexpect pydub requests rabbitpy pydub
+sudo pip3 install pyyaml sox pyrabbit soundfile dbus_python pexpect requests rabbitpy bottle
 
 if [ -f "$INSTALL_DIR" ]; then
     rm -rf "$INSTALL_DIR"
@@ -45,7 +45,7 @@ cp config/agent.in.yaml config/agent.yaml
 mkdir -p ~/.config/mc
 cp config/mc/ini ~/.config/mc
 
-# Replace the values
+# Replace the values - finish preparing the agent configuration file
 if [[ $BACKEND_URL != "" ]]; then
   sed -i "s|BACKEND|$BACKEND|" config/agent.yaml
 fi
@@ -56,11 +56,45 @@ fi
 
 sed -i "s|TASK_NAME|$TASK_NAME|" config/agent.yaml
 
-# Update systemD unit file
-cp config/systemd/agent_gsm.in.service config/systemd/agent_gsm.service
 ABSOLUTE_INSTALL_DIR=`realpath .`
 
-sed -i "s|ABSOLUTE_INSTALL_DIR|$ABSOLUTE_INSTALL_DIR|" config/systemd/agent_gsm.service
+# Update systemD unit file
+# cp config/systemd/agent_gsm.in.service config/systemd/agent_gsm.service
+# sed -i "s|ABSOLUTE_INSTALL_DIR|$ABSOLUTE_INSTALL_DIR|" config/systemd/agent_gsm.service
+
+install_ap() {
+  # $1 is AP name
+  sudo cp $ABSOLUTE_INSTALL_DIR/config/ap/etc/dhcpcd.conf /etc
+  sudo cp $ABSOLUTE_INSTALL_DIR/config/ap/etc/dnsmasq.conf /etc
+  sudo mkdir -p /etc/hostapd
+  sudo cp $ABSOLUTE_INSTALL_DIR/config/ap/etc/hostapd.conf /etc/hostapd
+  sudo sed -i "s|AGENT_GSM|$1|" /etc/hostapt/hostapd.conf
+  sudo cp $ABSOLUTE_INSTALL_DIR/config/ap/etc/default/hostapt /etd/default
+  sudo systemctl enable dnsmasq
+  sudo systemctl enable hostapd
+  sudo systemctl start dnsmasq
+  sudo systemctl start hostapd
+}
+
+function enable_autologin() {
+  sudo systemctl --quiet set-default multi-user.target
+  sudo cat > /etc/systemd/system/getty@tty1.service.d/autologin.conf << EOF
+[Service]
+ExecStart=
+ExecStart=-/sbin/agetty --autologin $USER --noclear %I \$TERM
+EOF
+
+}
+
+# ToDo:
+# - allow autologin in console mode for 'pi' user
+enable_autologin
+
+# - add $ABSOLUTE_INSTALL_DIR/run_agent_screen.sh to ~/.bashrc
+echo "$ABSOLUTE_INSTALL_DIR/run_agent_screen.sh" >> ~/.bashrc
+
+# - install wifi AP with name $PHONE_NAME
+install_ap $PHONE_NAME
 
 echo "Now the remaining prerequisites will be installed and system will reboot."
 echo "You can connect the phone via Bluetooth after the reboot." 

src/agent_gsm.py

@@ -24,6 +24,7 @@ from bt_signal import SignalBoundaries
 from bt_call_controller import INTERRUPT_SIGNAL
 import bt_call_controller
 
+import agent_point
 
 CONFIG = AgentConfig()
 
@@ -68,23 +69,23 @@ def detect_degraded_signal(file_test: Path, file_reference: Path) -> SignalBound
         # Seems some problem with recording, return zero boundaries
         return SignalBoundaries()
     
+    r = SignalBoundaries()
     if CONFIG.UseSpeechDetector:
         r = bt_signal.find_reference_signal_via_speechdetector(file_test)
-    else:
-        r = bt_signal.find_reference_signal(file_test)
     
     if r.offset_start == 0.0 and is_caller:
         r.offset_start = 5.0 # Skip ringing tones
+
     return r
 
 
 
 def detect_reference_signal(file_reference: Path) -> SignalBoundaries:
     # Run silence eraser on reference file as well
+    result = SignalBoundaries()
     if CONFIG.UseSpeechDetector:
         result = bt_signal.find_reference_signal_via_speechdetector(file_reference)
-    else:
+    
-        result = bt_signal.find_reference_signal(file_reference)
     return result
 
 
@@ -354,10 +355,10 @@ def run_caller_task(t):
     make_call(target_addr)
             
 
-
 # Runs caller probe - load task list and perform calls
 def run_probe():
     global TASK_LIST, CURRENT_TASK
+    offline_mode : bool = False
 
     while True:
         # Get task list update
@@ -366,6 +367,7 @@ def run_probe():
             # Check in cache
             utils.log('Checking for task list in cache...')
             new_tasks = CACHE.get_tasks(BACKEND.phone.name)
+            offline_mode = True
 
         # Did we fetch anything ?
         if new_tasks:
@@ -386,7 +388,7 @@ def run_probe():
         if TASK_LIST.tasks is not None:
             utils.log_verbose(f"Resulting task list: {TASK_LIST.tasks}")
 
-
+        # Run test immediately if specified
         if CONFIG.ForceRun and len(TASK_LIST.tasks) > 0:
             run_caller_task(TASK_LIST.tasks[0])
             break
@@ -401,8 +403,10 @@ def run_probe():
                         # Remove sheduled time
                         del t['scheduled_time']
                         
-                        # Run task
+                        # Run task if we are online
-                        run_caller_task(t)
+                        # Otherwise tasks run from the API point - via helper .apk
+                        if not offline_mode:
+                            run_caller_task(t)
                         
                         utils.log_verbose(f'Call #{CALL_COUNTER.value} finished')
                         if CALL_COUNTER.value >= CONFIG.TaskLimit and CONFIG.TaskLimit > 0:
@@ -413,15 +417,35 @@ def run_probe():
                     except Exception as err:
                         utils.log_error(message="Unexpected error.", err=err)
 
-            spent_time = utils.get_monotonic_time() - start_time
+        # Sleep for 
+        spent_time = utils.get_monotonic_time() - start_time
+
+        # Wait 1 minute
+        if spent_time < 60:
+            timeout_time = 60 - spent_time
+        else:
+            timeout_time = 0
+
+        # Try to get next task
+        try:
+            if agent_point.WEB_QUEUE is None:
+                utils.log('Web task queue is None')
+
+            task = agent_point.WEB_QUEUE.get(block=True, timeout=timeout_time)
+            
+            if task is not None:
+                run_caller_task(task)
+        
+        except multiprocessing.Queue.empty:
+            # Ignore this exception, this is normal
+            pass
+        except Exception as err:
+            utils.log_error(message='Error when running t')
             
-            # Wait 1 minute
-            if spent_time < 60:
-                time.sleep(60 - spent_time)
 
         # In case of empty task list wait 1 minute before refresh
-        if len(TASK_LIST.tasks) == 0:
+        # if len(TASK_LIST.tasks) == 0:
-            time.sleep(60)
+        #    time.sleep(60)
 
 
 def remove_pid_on_exit():
@@ -436,6 +460,9 @@ def receive_signal(signal_number, frame):
     # Delete PID file
     remove_pid_on_exit()
     
+    # Stop optional access point
+    agent_point.stop()
+
     # Debugging info
     print(f'Got signal {signal_number} from {frame}')
     
@@ -447,7 +474,6 @@ def receive_signal(signal_number, frame):
     return
 
 
-
 # Check if Python version is ok
 assert sys.version_info >= (3, 6)
 
@@ -471,6 +497,14 @@ if __name__ == '__main__':
     signal.signal(signal.SIGINT, receive_signal)
     signal.signal(signal.SIGQUIT, receive_signal)
 
+    if CONFIG.CacheDir:
+        CACHE = utils_cache.InfoCache(dir=CONFIG.CacheDir)
+
+    # Start own hotspot and API server
+    agent_point.CONFIG = CONFIG
+    agent_point.CACHE = CACHE
+    agent_point.start()
+
     # Preconnect the phone
     if CONFIG.BT_MAC:
         # Connect to phone before
@@ -482,7 +516,7 @@ if __name__ == '__main__':
         utils.log_error(f'No BT MAC specified, cannot connect. Exiting.')
         raise SystemExit(EXIT_ERROR)
     
-    # Init BT modem
+    # Init BT modem - here we wait for it
     bt_call_controller.init()
 
     # Logging settings
@@ -491,18 +525,12 @@ if __name__ == '__main__':
     if CONFIG.LogPath:
         utils.open_log_file(CONFIG.LogPath, 'at')
 
-
-    if CONFIG.CacheDir:
-        CACHE = utils_cache.InfoCache(dir=CONFIG.CacheDir)
-
-
     # Update path to pvqa/aqua-wb
     VOICE_QUALITY_AVAILABLE = utils_sevana.find_binaries(DIR_PROJECT / 'bin')
 
     # Load latest licenses & configs - this requires utils_sevana.find_binaries() to be called before
     # utils_sevana.load_config_and_licenses(config['backend'])
 
-
     # Limit number of calls
     if CONFIG.TaskLimit:
         utils.log(f'Limiting number of calls to {CONFIG.TaskLimit}')
@@ -570,4 +598,7 @@ if __name__ == '__main__':
     # Close log file
     utils.close_log_file()
 
+    # Stop optional access point
+    agent_point.stop()
+
     sys.exit(EXIT_OK)

src/agent_point.py

@@ -0,0 +1,133 @@
+#!/usr/bin/python3
+import bottle
+import multiprocessing
+import time
+import os
+import json
+import utils_cache
+from agent_config import AgentConfig
+
+class AccessPoint:
+    active: bool = False
+
+    def __init__(self) -> None:
+        pass
+
+
+    def start(self):
+        pass
+        
+    def stop(self):
+        pass
+
+# Just a stub for now
+ACCESS_POINT = AccessPoint()
+
+# Web server process
+SERVER_PROCESS = None
+
+# Good status response
+RESPONSE_OK = {'status': 'ok'}
+
+# Available information in cache
+CACHE : utils_cache.InfoCache = None
+CONFIG: AgentConfig = None
+
+# Web queue
+WEB_QUEUE = multiprocessing.Manager().Queue()
+
+@bottle.route('/status')
+def web_status():
+    print(f'Serving /status request...')
+    
+    r = RESPONSE_OK
+    if CONFIG is not None:
+        r['name'] = CONFIG.Name
+        r['backend'] = CONFIG.Backend
+        r['bt_mac'] = CONFIG.BT_MAC
+        
+        if CACHE is not None:
+            print('Cache is found...')
+            # Phone information
+            phone = CACHE.get_phone(CONFIG.Name)
+            if phone is not None:
+                print('Phone information is found...')
+                r['phone'] = phone.to_dict()
+
+            # Task list information
+            task_list = CACHE.get_tasks(CONFIG.Name)
+            if task_list is not None and task_list.tasks is not None:
+                r['task_list'] = task_list.tasks
+        else:
+            print('Cache not found.')
+    return r
+
+@bottle.route('/reboot')
+def web_reboot():
+    os.system('sudo reboot')
+    return RESPONSE_OK
+
+@bottle.route('/halt')
+def web_reboot():
+    os.system('sudo halt')
+    return RESPONSE_OK
+
+@bottle.route('/cache')
+def web_list_cache():
+    result = []
+    if CACHE is None:
+        return result
+    
+    # Iterate cache and return available files list
+    for f in os.listdir(CACHE.dir):
+        result.append(f)
+
+    return result
+
+@bottle.route('/call', method=['POST'])
+def web_call():
+    global WEB_QUEUE
+    try:
+        data = bottle.request.json
+
+        # Send task definition
+        print('Sending data to ougoing queue...')
+        WEB_QUEUE.put_nowait(data)
+
+        print('Returning OK response.')
+        return RESPONSE_OK
+    except Exception as e:
+        print(f'{str(e)}')
+        return RESPONSE_OK
+
+
+def web_process(mp_queue: multiprocessing.Queue):
+    #global WEB_QUEUE
+    #WEB_QUEUE = mp_queue 
+
+    print(f'Run web process...')
+    bottle.run(host='0.0.0.0', port=8080)
+
+def start():
+    global ACCESS_POINT, SERVER_PROCESS
+
+    ACCESS_POINT.start()
+    SERVER_PROCESS = multiprocessing.Process(target=web_process, args=(None,), name='agent_gsm_web')
+    SERVER_PROCESS.start()
+
+def stop():
+    global ACCESS_POINT, SERVER_PROCESS
+
+    ACCESS_POINT.stop()
+    SERVER_PROCESS.kill()
+
+
+if __name__ == '__main__':
+    # Start test stuff
+    start()
+
+    # Wait 120 seconds for tests
+    time.sleep(120.0)
+
+    # Stop test
+    stop()

src/bt_signal.py

@@ -6,36 +6,36 @@ import pathlib
 from utils_types import SignalBoundaries
 from utils_sevana import speech_detector
 
-from pydub import silence, AudioSegment
+# from pydub import silence, AudioSegment
 
 SILENCE_DELTA = 16
 
-def find_reference_signal(input_file: pathlib.Path, output_file: pathlib.Path = None, use_end_offset: bool = True) -> SignalBoundaries:
+# def find_reference_signal(input_file: pathlib.Path, output_file: pathlib.Path = None, use_end_offset: bool = True) -> SignalBoundaries:
-    myaudio = AudioSegment.from_wav(str(input_file))
+#     myaudio = AudioSegment.from_wav(str(input_file))
-    dBFS = myaudio.dBFS
+#     dBFS = myaudio.dBFS
     
-    # Find silence intervals
+#     # Find silence intervals
-    intervals = silence.detect_nonsilent(myaudio, min_silence_len=1000, silence_thresh=dBFS-SILENCE_DELTA, seek_step=50)
+#     intervals = silence.detect_nonsilent(myaudio, min_silence_len=1000, silence_thresh=dBFS-SILENCE_DELTA, seek_step=50)
     
-    # Translate to seconds
+#     # Translate to seconds
-    intervals = [((start/1000),(stop/1000)) for start,stop in intervals] # in sec
+#     intervals = [((start/1000),(stop/1000)) for start,stop in intervals] # in sec
 
-    # print(intervals)
+#     # print(intervals)
 
-    # Example of intervals: [(5.4, 6.4), (18.7, 37.05)]
+#     # Example of intervals: [(5.4, 6.4), (18.7, 37.05)]
-    for p in intervals:
+#     for p in intervals:
-        if p[1] - p[0] > 17:
+#         if p[1] - p[0] > 17:
-            bounds = SignalBoundaries(offset_start=p[0], offset_finish=p[1])
+#             bounds = SignalBoundaries(offset_start=p[0], offset_finish=p[1])
-            if output_file is not None:
+#             if output_file is not None:
-                signal = myaudio[bounds.offset_start * 1000 : bounds.offset_finish * 1000]
+#                 signal = myaudio[bounds.offset_start * 1000 : bounds.offset_finish * 1000]
-                signal.export(str(output_file), format='wav', parameters=['-ar', '44100', '-sample_fmt', 's16'])
+#                 signal.export(str(output_file), format='wav', parameters=['-ar', '44100', '-sample_fmt', 's16'])
 
-            if use_end_offset:
+#             if use_end_offset:
-                bounds.offset_finish = myaudio.duration_seconds - bounds.offset_finish
+#                 bounds.offset_finish = myaudio.duration_seconds - bounds.offset_finish
 
-            return bounds
+#             return bounds
 
-    return SignalBoundaries()
+#     return SignalBoundaries()
 
 
 def find_reference_signal_via_speechdetector(input_file: pathlib.Path) -> SignalBoundaries:

src/utils_cache.py

@@ -19,8 +19,6 @@ class InfoCache:
                 utils.log_error(str(e))
                 self.dir = None                
 
-
-
     def is_active(self) -> bool:
         return self.dir is not None
 

src/utils_qualtest.py

@@ -66,12 +66,13 @@ def trace_function(frame, event, arg):
 class QualtestBackend:
     address: str
     instance: str
+    online: bool
 
     def __init__(self):
         self.address = ""
         self.instance = ""
         self.__phone = None
-
+        self.online = False
 
     @property
     def phone(self) -> Phone:
@@ -190,8 +191,17 @@ class QualtestBackend:
                 response = urllib.request.urlopen(url, timeout=utils.NETWORK_TIMEOUT)
                 if response.getcode() != 200:
                     raise RuntimeError(f'Failed to load phone definition from server. Error code: {response.getcode()}')
+                
+                # Consider backend as working one
+                self.online = True
+            
             except Exception as e:
                 utils.log_error(f'Problem when loading the phone definition from backend. Error: {str(e)}')
+                
+                # Consider backend as non-working
+                self.online = False
+
+                # Try to get data from the cache
                 r = cache.get_phone(self.instance)
                 if r is None:
                     raise RuntimeError(f'No cached phone definition.')