Add url resolving for ML backends using HOSTNAME passed from /setup.

Better support for NeMo ASR.

Author: makseq

label_studio/examples/audio_transcribe/config.xml

@@ -2,10 +2,9 @@
 <View>
   <Header value="Listen to the audio and write the transcription" />
   <AudioPlus name="audio" value="$audio" />
-  <TextArea name="transcription" toName="audio"
+  <TextArea name="transcription" toName="audio" editable="true"
             rows="4" transcription="true" maxSubmissions="1" />
 
-
   <Style>
   [dataneedsupdate]>div:first-child{flex-grow:1;order:2}
   [dataneedsupdate]>div:last-child{margin-top:0 !important;margin-right:1em}

label_studio/ml/api.py

@@ -43,7 +43,8 @@ def _setup():
     project = data.get('project')
     schema = data.get('schema')
     force_reload = data.get('force_reload', False)
-    model = _manager.fetch(project, schema, force_reload)
+    hostname = data.get('hostname', '')  # host name for uploaded files and building urls
+    model = _manager.fetch(project, schema, force_reload, hostname=hostname)
     logger.debug('Fetch model version: {}'.format(model.model_version))
     return jsonify({'model_version': model.model_version})
 

label_studio/ml/examples/nemo/README.md

@@ -20,3 +20,21 @@ With ASR models, you can do audio pre-annotations drawn within a text area, aka
 After this app starts on the default 9090 port, configure the template for ASR:
 1. In Label Studio, open the project settings page.
 2. From the templates list, select `Speech Transcription`. You can also create your own with `<TextArea>` and `<Audio>` tags. 
+
+Or copy this labeling config into LS: 
+```
+<View>
+  <Header value="Listen to the audio and write the transcription" />
+  <AudioPlus name="audio" value="$audio" />
+  <TextArea name="transcription" toName="audio" editable="true"
+            rows="4" transcription="true" maxSubmissions="1" />
+
+
+  <Style>
+  [dataneedsupdate]>div:first-child{flex-grow:1;order:2}
+  [dataneedsupdate]>div:last-child{margin-top:0 !important;margin-right:1em}
+  </Style>
+</View>
+```
+
+> Note: The NeMo engine downloads models automatically. This can take some time and could cause Label Studio UI to hang on the Model page while the models download.  

label_studio/ml/model.py

@@ -16,7 +16,6 @@
 
 from label_studio.utils.misc import parse_config
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -34,6 +33,7 @@ def __init__(self, label_config=None, train_output=None, **kwargs):
         self.label_config = label_config
         self.parsed_label_config = parse_config(self.label_config)
         self.train_output = train_output or {}
+        self.hostname = kwargs.get('hostname', '')
 
     @abstractmethod
     def predict(self, tasks, **kwargs):
@@ -42,6 +42,10 @@ def predict(self, tasks, **kwargs):
     def fit(self, completions, workdir=None, **kwargs):
         return {}
 
+    def get_local_path(self, url, project_dir=None):
+        from label_studio.ml.utils import get_local_path
+        return get_local_path(url, project_dir=project_dir, hostname=self.hostname)
+
 
 class LabelStudioMLManager(object):
 

label_studio/ml/server.py

@@ -120,3 +120,7 @@ def main():
         create_dir(args)
     elif args.command == 'start':
         start_server(args, subargs)
+
+
+if __name__ == '__main__':
+    main()

label_studio/ml/utils.py

@@ -76,6 +76,10 @@ def get_choice(completion):
 
 
 def get_image_local_path(url, image_cache_dir=None, project_dir=None):
+    return get_local_path(url, image_cache_dir, project_dir)
+
+
+def get_local_path(url, cache_dir=None, project_dir=None, hostname=None):
     is_local_file = url.startswith('/data/') and '?d=' in url
     is_uploaded_file = url.startswith('/data/upload')
 
@@ -86,27 +90,32 @@ def get_image_local_path(url, image_cache_dir=None, project_dir=None):
         filepath = os.path.join(dir_path, filename)
         if not os.path.exists(filepath):
             raise FileNotFoundError(filepath)
+        return filepath
 
     # File uploaded via import UI
-    elif is_uploaded_file:
-        if not project_dir or not os.path.exists(project_dir):
+    elif is_uploaded_file and project_dir is not None:
+        if not os.path.exists(project_dir):
             raise FileNotFoundError(
                 "Can't find uploaded file by URL {url}: you need to pass a valid project_dir".format(url=url))
         filepath = os.path.join(project_dir, 'upload', os.path.basename(url))
+        return filepath
+
+    elif is_uploaded_file and hostname:
+        url = hostname + url
+        logger.info('Resolving url using hostname [' + hostname + '] from LSB: ' + url)
 
     # File specified by remote URL - download and cache it
-    else:
-        image_cache_dir = image_cache_dir or get_cache_dir()
-        parsed_url = urlparse(url)
-        url_filename = os.path.basename(parsed_url.path)
-        url_hash = hashlib.md5(url.encode()).hexdigest()[:6]
-        filepath = os.path.join(image_cache_dir, url_hash + '__' + url_filename)
-        if not os.path.exists(filepath):
-            logger.info('Download {url} to {filepath}'.format(url=url, filepath=filepath))
-            r = requests.get(url, stream=True)
-            r.raise_for_status()
-            with io.open(filepath, mode='wb') as fout:
-                fout.write(r.content)
+    cache_dir = cache_dir or get_cache_dir()
+    parsed_url = urlparse(url)
+    url_filename = os.path.basename(parsed_url.path)
+    url_hash = hashlib.md5(url.encode()).hexdigest()[:6]
+    filepath = os.path.join(cache_dir, url_hash + '__' + url_filename)
+    if not os.path.exists(filepath):
+        logger.info('Download {url} to {filepath}'.format(url=url, filepath=filepath))
+        r = requests.get(url, stream=True)
+        r.raise_for_status()
+        with io.open(filepath, mode='wb') as fout:
+            fout.write(r.content)
     return filepath
 
 

label_studio/utils/models.py

@@ -17,7 +17,7 @@
 from requests.adapters import HTTPAdapter
 from .io import get_data_dir
 from .exceptions import ValidationError
-from .functions import _LABEL_CONFIG_SCHEMA_DATA
+from .functions import _LABEL_CONFIG_SCHEMA_DATA, get_external_hostname
 
 DEFAULT_PROJECT_ID = 1
 logger = logging.getLogger(__name__)
@@ -376,7 +376,8 @@ def setup(self, project):
         """
         return self._post('setup', request={
             'project': self._create_project_uid(project),
-            'schema': project.label_config_line
+            'schema': project.label_config_line,
+            'hostname': get_external_hostname()
         })
 
     def delete(self, project):