From ee382f2e8f718f38fbe3c8eba68a26bb94f572b0 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Tue, 14 Jul 2020 19:42:41 +0530 Subject: [PATCH 01/42] modular test cases cut2 --- tests/performance/README.md | 145 +++++++++---- tests/performance/agents/config.ini | 2 +- tests/performance/agents/metrics/__init__.py | 25 ++- tests/performance/agents/utils/process.py | 5 +- tests/performance/requirements.txt | 3 +- tests/performance/run_performance_suite.py | 12 +- tests/performance/runs/compare.py | 49 +++-- tests/performance/runs/context.py | 23 +- tests/performance/runs/storage.py | 12 +- tests/performance/runs/taurus/__init__.py | 22 +- tests/performance/runs/taurus/reader.py | 37 +++- tests/performance/runs/taurus/x2junit.py | 170 +++++++++++++-- .../tests/api_description/api_description.jmx | 2 +- .../api_description/api_description.yaml | 75 ++----- .../api_description/environments/xlarge.yaml | 54 ++++- .../batch_and_single_inference.jmx | 6 +- .../batch_and_single_inference.yaml | 92 +------- .../environments/xlarge.yaml | 56 ++++- .../tests/batch_inference/batch_inference.jmx | 4 +- .../batch_inference/batch_inference.yaml | 78 +------ .../batch_inference/environments/xlarge.yaml | 51 ++++- .../environments/xlarge.yaml | 8 +- .../examples_local_criteria.jmx | 6 +- .../examples_local_criteria.yaml | 51 ++++- .../examples_local_monitoring.jmx | 6 +- .../examples_local_monitoring.yaml | 44 +++- .../environments/xlarge.yaml | 3 + .../examples_remote_criteria.jmx | 6 +- .../examples_remote_criteria.yaml | 55 +++-- .../examples_remote_monitoring.jmx | 6 +- .../examples_remote_monitoring.yaml | 44 +++- .../examples_starter/examples_starter.jmx | 6 +- .../examples_starter/examples_starter.yaml | 49 +++-- tests/performance/tests/global_config.yaml | 196 +++++++++++++++++- .../health_check/environments/xlarge.yaml | 49 ++++- .../tests/health_check/health_check.jmx | 2 +- .../tests/health_check/health_check.yaml | 62 +----- .../environments/xlarge.yaml | 51 ++++- .../inference_multiple_models.jmx | 6 +- .../inference_multiple_models.yaml | 63 +----- .../environments/xlarge.yaml | 51 ++++- .../inference_multiple_worker.jmx | 4 +- .../inference_multiple_worker.yaml | 57 +---- .../environments/xlarge.yaml | 51 ++++- .../inference_single_worker.jmx | 4 +- .../inference_single_worker.yaml | 56 +---- .../list_models/environments/xlarge.yaml | 48 ++++- .../tests/list_models/list_models.jmx | 4 +- .../tests/list_models/list_models.yaml | 57 +---- .../environments/xlarge.yaml | 51 ++++- .../model_description/model_description.jmx | 4 +- .../model_description/model_description.yaml | 55 +---- .../environments/xlarge.yaml | 56 ++++- .../multiple_inference_and_scaling.jmx | 70 ++++++- .../multiple_inference_and_scaling.yaml | 92 ++------ .../environments/xlarge.yaml | 54 ++++- .../register_unregister.jmx | 6 +- .../register_unregister.yaml | 67 +----- .../environments/xlarge.yaml | 57 ++++- .../register_unregister_multiple.jmx | 6 +- .../register_unregister_multiple.yaml | 72 ++----- .../environments/xlarge.yaml | 24 ++- .../scale_down_workers/scale_down_workers.jmx | 4 +- .../scale_down_workers.yaml | 116 +++++------ .../scale_up_workers/environments/xlarge.yaml | 24 ++- .../scale_up_workers/scale_up_workers.jmx | 4 +- .../scale_up_workers/scale_up_workers.yaml | 110 +++++----- tests/performance/utils/fs.py | 16 +- tests/performance/utils/pyshell.py | 13 +- tests/performance/utils/timer.py | 4 + 70 files changed, 1723 insertions(+), 1150 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index 32f3c59f9..88fd6e0bc 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -9,7 +9,7 @@ The salient features of the performance regression suite are * Non-intrusive - Does not need any code-changes or instrumentation on the server being monitored. * It can be used to monitor a wide variety of server metrics - memory, cpu, io - in addition to traditional API level metrics such as latency, throughput etc. -* It is easy to add custom metrics. For example, in MMS server, `the number of workers spawned` would be an interesting +* It is easy to add custom metrics. For example, in Model server, `the number of workers spawned` would be an interesting metric to track. The platform allows for easy addition of these metrics. * Test cases are specified in human readable yaml files. Every test case has a pass or fail status. This is determined by evaluating expressions specified in the test case. Every expression checks metrics against threshold values. For @@ -19,7 +19,7 @@ possible to specify multiple compute environments against which the test cases w environment, will have its own threshold values. * This suite leverages the open source [Taurus framework](https://gettaurus.org/). * This suite extends the Taurus framework in the following ways - * Adds resource monitoring service. This allows MMS specific metrics to be added. + * Adds resource monitoring service. This allows Model Server specific metrics to be added. * Environments as described earlier. * Specification of pass/fail criterion between two commits. For example, memory consumed by workers should not increase by more than 10% between two commits for the given test case. @@ -38,21 +38,21 @@ The building blocks of the performance regression suite and flow is captured in ``` 2. Install performance regression suite dependencies. ```bash - export MMS_HOME= - pip install -r $MMS_HOME/tests/performance/requirements.txt + export MODEL_SERVER_HOME= + pip install -r $MODEL_SERVER_HOME/tests/performance/requirements.txt ``` -3. Make sure that `git` is installed and the test suites are run from the MMS working directory. +3. Make sure that `git` is installed and the test suites are run from the Model Server working directory. ### B. Running the test suite -1. Make sure parameters set in [tests/common/global_config.yaml](tests/performance/tests/global_config.yaml) are correct. +1. Make sure parameters set in [tests/global_config.yaml](tests/performance/tests/global_config.yaml) are correct. 2. To run the test suite execute [run_performance_suite.py](run_performance_suite.py) with the following parameters * `--artifacts-dir` or `-a` is a directory where the test case results will be stored. The default value is -`$MMS_HOME/tests/performance/run_artifacts`. +`$MODEL_SERVER_HOME/tests/performance/run_artifacts`. * `--test-dir` or `-t` is a directory containing the test cases. The default value is -`$MMS_HOME/tests/performance/tests`. +`$MODEL_SERVER_HOME/tests/performance/tests`. * `--pattern` or `-p` glob pattern picks up certain test cases for execution within the `test-dir`. The default value picks up all test cases. @@ -64,23 +64,34 @@ The default value excludes nothing. the file (minus the extension) found inside the environments folder in each test case. They encapsulate parameter values which are specific to the execution environment. This is a mandatory parameter. + * `--compare-local` or `--no-compare-local` specifies whether to do comparison with run artifacts data available on local machine + or the data available on S3 bucket. + + * `--compare-with` or `-c` specifies the commit id compare against. The default value is 'HEAD~1'. The branch name, tag, + can also be specified. The comparison happens if the run artifacts folder for the commit_id and env is available. + + + + The script does the following: 1. Starts the metrics monitoring server. - 2. Collects all the tests from test-dir satisfying the pattern - 3. Executes the tests + 2. Collects all the tests from test-dir satisfying the pattern, excluding exclude pattern and test starting with 'skip' + 3. Executes the collected tests 4. Generates artifacts in the artifacts-dir against each test case. + 5. Generate Pass Fail report for test cases + 6. Generate comparison report for specified commit id -3. Check the console logs, $artifacts-dir$//performance_results.html report, comparison.csv, comparison.html +3. Check the console logs, $artifacts-dir$//performance_results.html report, comparison_result.csv, comparison_result.html and other artifacts. **Steps are provided below** ```bash -export MMS_HOME= -cd $MMS_HOME/tests/performance +export MODEL_SERVER_HOME= +cd $MODEL_SERVER_HOME/tests/performance -# Note that MMS server started and stopped by the individual test suite. -# check variables such as MMS server PORT etc +# Note that Model server started and stopped by the individual test suite. +# check variables such as Model server PORT etc # vi tests/common/global_config.yaml #all tests @@ -92,17 +103,55 @@ python -m run_performance_suite -e xlarge -p inference_single_worker ``` ### C. Understanding the test suite artifacts and reports -1. The $artifacts-dir$//performance_results.html is a summary report of the test run. +1. The $artifacts-dir//performance_results.html is a summary report of the test run. 2. Each test yaml is treated as a test suite. Each criteria in the test suite is treated as a test case. If the test suite does not specify any criteria, then the test suite is reported as skipped with 0 test cases. 3. For each test suite, a sub-directory is created containing relevant run artifacts. Important files in this directory are * metrics.csv -- contains the values of the various system-monitored metrics over time + * metrics_agg.csv -- contains percentile values for columns in metrics.csv * finals_stats.csv -- contains the values of the various api metrics over time -4. The $artifacts-dir$//comparison_results.html is a summary report which shows performance difference between +4. The $artifacts-dir//comparison_results.html is a summary report which shows performance difference between the last two commits. 5. The run completes with a console summary of the performance and comparision suites which have failed ![](assets/console.png) +### D. Understanding the test case components +A Test Case consists of the test.yaml, test.jmx, environments/*.yaml files and a global_config.yaml. +Below is the sample folder structure for 'api_description' test case: +```bash +tests + -- api_description + --- environments + ---- xlarge.yaml + ---- mac_xlarge.yaml + --- api_description.jmx + --- api_description.yaml + -- global_config.yaml +``` + +1. global_config.yaml + - It is a master template for all_comm the test cases and is shared across all the tests. + - It contains all the common yaml sections, criteria, monitoring metrics etc. + - It also contain variables in the format ${variable} for metric thresholds and other test specific attributes. + +2. environments/*.yaml + - A test case can have multiple environment files. If you have a environment dependent metrics you can create an environment + yaml file. For ex. macos_xlarge, ubuntu_xlarge etc. + - The environment file contains values for all the variables mentioned in global_config.yaml and test.yaml. + +3. test.yaml + - The test.yaml is main yaml for a test case. Note the name of the yaml should be same as the test folder. + - It inherits the master template global_config.yaml. + And it usually contains the scenario, specific pre-processing commands (if any), and special criteria (if any) applicable for that test case only. + - If you want a behavior other than defined in the master template, It is possible to override sections of global_config.yaml in the individual test case. + The global_config.yaml's top-level sections can be overridden, merged, or appended based on below rules: + 1. By default the dictionaries get merged. + 2. If the dictionary key is prepended with '~' it will get overridden. + 3. The list gets appended. +4. test.jmx + - The JMeter test scenario file. The test.yaml runs the scenarion mentioned in the .jmx file. + + ## Add a new test Follow these three steps to add a new test case to the test suite. @@ -110,6 +159,7 @@ Follow these three steps to add a new test case to the test suite. 1. Add scenario (a.k.a test suite) 2. Add metrics to monitor 3. Add pass/fail criteria (a.k.a test case) +4. Add compare criteria (a.k.a compare test cases) #### 1. Add scenario (a.k.a test suite) @@ -139,8 +189,8 @@ Please note that various global configuration settings used by examples_starter. To execute this test suite, run the following command ```bash - export MMS_HOME= - cd $MMS_HOME/tests/performance + export MODEL_SERVER_HOME= + cd $MODEL_SERVER_HOME/tests/performance python -m run_performance_suite -p examples_starter -e xlarge ``` @@ -154,15 +204,15 @@ Specify the metrics of interest in the services/monitoring section of the yaml. 1. Standalone monitoring server - Use this technique if MMS and the tests execute on different machines. Before running the test cases, + Use this technique if Model Server and the tests execute on different machines. Before running the test cases, please start the [metrics_monitoring_server.py](metrics_monitoring_server.py) script. It will communicate server metric data with the test client over sockets. The monitoring server runs on port 9009 by default. - To start the monitoring server, run the following commands on the MMS host: + To start the monitoring server, run the following commands on the Model Server host: ```bash - export MMS_HOME= - pip install -r $MMS_HOME/tests/performance/requirements.txt - python $MMS_HOME/tests/performance/metrics_monitoring_server.py --start + export MODEL_SERVER_HOME= + pip install -r $MODEL_SERVER_HOME/tests/performance/requirements.txt + python $MODEL_SERVER_HOME/tests/performance/metrics_monitoring_server.py --start ``` The monitoring section configuration is shown below. @@ -171,29 +221,29 @@ Specify the metrics of interest in the services/monitoring section of the yaml. services: - module: monitoring server-agent: - - address: :9009 # metric monitoring service address - label: mms-inference-server # Specified label will be used in reports instead of ip:port + - address: :9009 # metric monitoring service address + label: Model-Server-inference-server # Specified label will be used in reports instead of ip:port interval: 1s # polling interval logging: True # those logs will be saved to "SAlogs_192.168.0.1_9009.csv" in the artifacts dir metrics: # metrics should be supported by monitoring service - - sum_cpu_percent # cpu percent used by all the mms server processes and workers + - sum_cpu_percent # cpu percent used by all the Model server processes and workers - sum_memory_percent - sum_num_handles - - server_workers # no of mms workers + - server_workers # no of Model Server workers ``` The complete yaml can be found [here](tests/examples_remote_monitoring/examples_remote_monitoring.yaml) Use the command below to run the test suite. ```bash - export MMS_HOME= - cd $MMS_HOME/tests/performance + export MODEL_SERVER_HOME= + cd $MODEL_SERVER_HOME/tests/performance python -m run_performance_suite -p examples_remote_monitoring -e xlarge ``` 2. Local monitoring plugin - Use this technique if both MMS and the tests run on the same host. + Use this technique if both Model Server and the tests run on the same host. The monitoring section configuration is shown below. ```yaml @@ -218,8 +268,8 @@ Specify the metrics of interest in the services/monitoring section of the yaml. Use the command below to run the test suite. ```bash - export MMS_HOME= - cd $MMS_HOME/tests/performance + export MODEL_SERVER_HOME= + cd $MODEL_SERVER_HOME/tests/performance python -m run_performance_suite -p examples_local_monitoring -e xlarge ``` @@ -235,7 +285,7 @@ pass-fail module from Taurus to achieve this functionality. More details can be - module: passfail criteria: - class: bzt.modules.monitoring.MonitoringCriteria - subject: mms-inference-server/sum_num_handles + subject: model-server/sum_num_handles condition: '>' threshold: 180 timeframe: 1s @@ -255,19 +305,18 @@ specified in the pass/fail criterion are used for comparison with the previous r - module: passfail criteria: - class: bzt.modules.monitoring.MonitoringCriteria - subject: mms-inference-server/sum_num_handles + subject: model-server/sum_num_handles condition: '>' threshold: 180 timeframe: 1s fail: true stop: true - diff_percent : 30 ``` Note that 1. At least one test suite run on the same environment should have happened in order to do the comparison. 2. The $artifacts-dir$//comparison_results.html is a summary report which shows performance difference - between the last two commits. + between the current run and user specified compare_with commit_id run. 3. The test case fails if the diff_percent is greater than the specified value across runs. 3. Metrics available for pass-fail criteria @@ -307,18 +356,34 @@ specified in the pass/fail criterion are used for comparison with the previous r * total_workers - Total number of workers spawned * orphans - Total number of orphan processes +4. Add compare criteria: +There are two types of compare criteria you can add for metrics: + 1. diff_percent_run + This criteria is used to check the percent difference between first and last value of the metric for a run. + In other words it is used to verify if metrics values are same before and after the scenario run. + 2. diff_percent_previous + Compare the metric aggregate values with previous run. Here we take aggregate min, max and avg of metric values for current run + and previous run and check if percentage difference is not greater than diff_percent_previous. + +Note formula for percentage difference is abs(value1 - value2)/((value1 + value2)/2) * 100 + +## Guidelines for writing good test cases: +1. The 'timeframe' duration to check values for threshold criteria should be sufficiently large at least 5 sec. +2. The duration specified using 'hold-for' property should also be sufficiently large at least 5 min. +3. When you use diff_percent_run, make sure that scenario (JMX script) results in deterministic state across different runs. + ## Test Strategy & Cases More details about our testing strategy and test cases can be found [here](TESTS.md) ## FAQ -Q1. Is it possible to use the performance regression framework to test MMS on Python2.7? +Q1. Is it possible to use the performance regression framework to test Model Server on Python2.7? Yes. Even though, the performance regression framework needs Python 3.7+ (as Taurus requires Python 3.7+), there are two possible ways to achieve this -* Please create a Python 2.7 virtual env which runs MMS and a Python 3.7 virtual env which runs +* Please create a Python 2.7 virtual env which runs Model Server and a Python 3.7 virtual env which runs the test framework and test cases. -* Alternatively, deploy the standalone monitoring agent on the MMS instance and run the test cases against the remote +* Alternatively, deploy the standalone monitoring agent on the Model Server instance and run the test cases against the remote server. Note that the standalone monitoring agent works on both Python 2/3. diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index aacbe97e1..f2caaebcd 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -6,4 +6,4 @@ HOST = PORT = 9009 [suite] -s3_bucket = mms-performance-regression-reports \ No newline at end of file +s3_bucket = torchserve-performance-regression-reports \ No newline at end of file diff --git a/tests/performance/agents/metrics/__init__.py b/tests/performance/agents/metrics/__init__.py index 9d7bf7eb2..642976be5 100644 --- a/tests/performance/agents/metrics/__init__.py +++ b/tests/performance/agents/metrics/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -""" Customised system and mms process metrics for monitoring and pass-fail criteria in taurus""" +""" Customised system and Model Server process metrics for monitoring and pass-fail criteria in taurus""" # Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. # Licensed under the Apache License, Version 2.0 (the "License"). @@ -19,7 +19,7 @@ class ProcessType(Enum): - """ Type of MMS processes to compute metrics on """ + """ Type of Server processes to compute metrics on """ FRONTEND = 1 WORKER = 2 ALL = 3 @@ -64,7 +64,8 @@ class ProcessType(Enum): misc_metrics = { 'total_processes': None, 'total_workers': None, - 'orphans': None + 'orphans': None, + 'zombies': None } AVAILABLE_METRICS = list(system_metrics) + list(misc_metrics) @@ -85,6 +86,7 @@ class ProcessType(Enum): AVAILABLE_METRICS.append('{}_{}_{}'.format(op, PNAME, metric)) children = set() +zombie_children = set() def get_metrics(server_process, child_processes, logger): @@ -118,7 +120,7 @@ def update_metric(metric_name, proc_type, stats): processes_stats.append({'type': ProcessType.FRONTEND, 'stats': server_process.as_dict()}) except: pass - for child in children: + for child in children | zombie_children: try: child_cmdline = child.cmdline() if psutil.pid_exists(child.pid) and len(child_cmdline) >= 2 and WORKER_NAME in child_cmdline[1]: @@ -126,12 +128,18 @@ def update_metric(metric_name, proc_type, stats): else: reclaimed_pids.append(child) logger.debug('child {0} no longer available'.format(child.pid)) - except (NoSuchProcess, ZombieProcess): + except ZombieProcess: + zombie_children.add(child) + except NoSuchProcess: reclaimed_pids.append(child) logger.debug('child {0} no longer available'.format(child.pid)) for p in reclaimed_pids: - children.remove(p) + if p in children: + children.remove(p) + if p in zombie_children: + zombie_children.remove(p) + ### PROCESS METRICS ### worker_stats = list(map(lambda x: x['stats'], \ @@ -147,10 +155,11 @@ def update_metric(metric_name, proc_type, stats): # Total processes result['total_processes'] = len(worker_stats) + 1 - result['total_workers'] = max(len(worker_stats) - 1, 0) + result['total_workers'] = max(len(worker_stats), 0) result['orphans'] = len(list(filter(lambda p: p['ppid'] == 1, worker_stats))) + result['zombies'] = len(zombie_children) - ### SYSTEM METRICS ### + # ###SYSTEM METRICS ### result['system_disk_used'] = psutil.disk_usage('/').used result['system_memory_percent'] = psutil.virtual_memory().percent system_disk_io_counters = psutil.disk_io_counters() diff --git a/tests/performance/agents/utils/process.py b/tests/performance/agents/utils/process.py index 8bdfb5078..c9a8e98de 100644 --- a/tests/performance/agents/utils/process.py +++ b/tests/performance/agents/utils/process.py @@ -56,9 +56,7 @@ def get_child_processes(process): def get_server_processes(server_process_pid): - """ It caches the main server and child processes at module level. - Ensure that you call this process so that MMS process - """ + """get psutil Process object from process id """ try: server_process = psutil.Process(server_process_pid) except Exception as e: @@ -68,4 +66,5 @@ def get_server_processes(server_process_pid): def get_server_pidfile(file): + """get temp server pid file""" return os.path.join(tempfile.gettempdir(), ".{}".format(file)) diff --git a/tests/performance/requirements.txt b/tests/performance/requirements.txt index 2fa19c26a..55a4486fe 100644 --- a/tests/performance/requirements.txt +++ b/tests/performance/requirements.txt @@ -8,4 +8,5 @@ awscli==1.18.80 click==7.1.2 tabulate==0.8.7 pandas==1.0.3 -termcolor==1.1.0 \ No newline at end of file +termcolor==1.1.0 +bzt== 1.14.2 \ No newline at end of file diff --git a/tests/performance/run_performance_suite.py b/tests/performance/run_performance_suite.py index 45948dff4..66bc41713 100755 --- a/tests/performance/run_performance_suite.py +++ b/tests/performance/run_performance_suite.py @@ -71,8 +71,9 @@ def validate_env(ctx, param, value): @click.option('--monit/--no-monit', help='Start Monitoring server', default=True) @click.option('--compare-local/--no-compare-local', help='Compare with previous run with files stored' ' in artifacts directory', default=True) +@click.option('-c', '--compare-with', help='Compare with commit id, branch, tag, HEAD~N.', default="HEAD~1") def run_test_suite(artifacts_dir, test_dir, pattern, exclude_pattern, - jmeter_path, env_name, monit, compare_local): + jmeter_path, env_name, monit, compare_local, compare_with): """Collect test suites, run them and generate reports""" logger.info("Artifacts will be stored in directory %s", artifacts_dir) @@ -84,7 +85,7 @@ def run_test_suite(artifacts_dir, test_dir, pattern, exclude_pattern, else: logger.info("Collected tests %s", test_dirs) - with ExecutionEnv(MONITORING_AGENT, artifacts_dir, env_name, compare_local, monit) as prt: + with ExecutionEnv(MONITORING_AGENT, artifacts_dir, env_name, compare_local, compare_with, monit) as prt: pre_command = 'export PYTHONPATH={}:$PYTHONPATH;'.format(os.path.join(str(ROOT_PATH), "agents")) for suite_name in tqdm(test_dirs, desc="Test Suites"): with Timer("Test suite {} execution time".format(suite_name)) as t: @@ -95,10 +96,13 @@ def run_test_suite(artifacts_dir, test_dir, pattern, exclude_pattern, test_file = os.path.join(test_dir, suite_name, "{}.yaml".format(suite_name)) with x2junit.X2Junit(suite_name, suite_artifacts_dir, prt.reporter, t, env_name) as s: s.code, s.err = run_process("{} bzt {} {} {} {}".format(pre_command, options_str, - test_file, env_yaml_path, - GLOBAL_CONFIG_PATH)) + GLOBAL_CONFIG_PATH, test_file, + env_yaml_path)) update_taurus_metric_files(suite_artifacts_dir, test_file) + sys.exit(prt.exit_code) + + if __name__ == "__main__": run_test_suite() diff --git a/tests/performance/runs/compare.py b/tests/performance/runs/compare.py index 8ebbb4b67..ea5d4bf6a 100644 --- a/tests/performance/runs/compare.py +++ b/tests/performance/runs/compare.py @@ -35,12 +35,13 @@ class CompareReportGenerator(): - def __init__(self, path, env_name, local_run): + def __init__(self, path, env_name, local_run, compare_with): self.artifacts_dir = path self.current_run_name = os.path.basename(path) self.env_name = env_name + self.comare_with = compare_with storage_class = LocalStorage if local_run else S3Storage - self.storage = storage_class(self.artifacts_dir, self.env_name) + self.storage = storage_class(self.artifacts_dir, self.env_name, compare_with) self.junit_reporter = None self.pandas_result = None self.pass_fail = True @@ -87,7 +88,7 @@ def add_test_case(self, name, msg, type): def get_log_file(dir, sub_dir): """Get metric monitoring log files""" - metrics_file = os.path.join(dir, sub_dir, "metrics.csv") + metrics_file = os.path.join(dir, sub_dir, "metrics_agg.csv") return metrics_file if os.path.exists(metrics_file) else None @@ -102,11 +103,21 @@ def get_aggregate_val(df, agg_func, col): return val +def get_centile_val(df, agg_func, col): + """Get aggregate values of a pandas dataframe coulmn for given aggregate function""" + + val = None + if "metric_name" in df and agg_func in df: + val = df[df["metric_name"] == col][agg_func] + val = val[0] if len(val) else None + return val + + def compare_values(val1, val2, diff_percent, run_name1, run_name2): """ Compare percentage diff values of val1 and val2 """ if pd.isna(val1) or pd.isna(val2): - msg = "Either of the value can not be determined. The run1 value is '{}' and " \ - "run2 value is {}.".format(val1, val2) + msg = "Either of the value can not be determined. run1_value='{}' and " \ + "run2_value='{}'.".format(val1, val2) pass_fail, diff, msg = "error", "NA", msg else: try: @@ -116,15 +127,15 @@ def compare_values(val1, val2, diff_percent, run_name1, run_name2): if diff < float(diff_percent): pass_fail, diff, msg = "pass", diff, "passed" else: - msg = "The diff_percent criteria has failed. The expected diff_percent is '{}' and actual " \ - "diff percent is '{}' and the '{}' run value is '{}' and '{}' run value is '{}'. ". \ + msg = "The diff_percent criteria has failed. Expected='{}', actual='{}' " \ + "run1='{}', run1_value='{}', run2='{}', run2_value='{}' ". \ format(diff_percent, diff, run_name1, val1, run_name2, val2) pass_fail, diff, msg = "fail", diff, msg else: # special case of 0 pass_fail, diff, msg = "pass", 0, "" except Exception as e: - msg = "error while calculating the diff for val1={} and val2={}." \ + msg = "error while calculating the diff for val1='{}' and val2='{}'." \ "Error is: {}".format(val1, val2, str(e)) logger.info(msg) pass_fail, diff, msg = "pass", "NA", msg @@ -139,7 +150,7 @@ def compare_artifacts(dir1, dir2, run_name1, run_name2): sub_dirs_1 = get_sub_dirs(dir1) over_all_pass = True - aggregates = ["mean", "max", "min"] + aggregates = ["first_value", "last_value"] header = ["run_name1", "run_name2", "test_suite", "metric", "run1", "run2", "percentage_diff", "expected_diff", "result", "message"] rows = [header] @@ -161,15 +172,18 @@ def compare_artifacts(dir1, dir2, run_name1, run_name2): metrics_from_file1 = pd.read_csv(metrics_file1) metrics_from_file2 = pd.read_csv(metrics_file2) - metrics, diff_percents = taurus_reader.get_compare_metric_list(dir1, sub_dir1) + metrics = taurus_reader.get_compare_metric_list(dir1, sub_dir1) - for col, diff_percent in zip(metrics, diff_percents): + for metric_values in metrics: + col = metric_values[0] + diff_percent = metric_values[1] + if diff_percent is None: + continue for agg_func in aggregates: name = "{}_{}".format(agg_func, str(col)) - val1 = get_aggregate_val(metrics_from_file1, agg_func, col) - val2 = get_aggregate_val(metrics_from_file2, agg_func, col) - + val2 = get_centile_val(metrics_from_file2, agg_func, col) + val1 = get_centile_val(metrics_from_file2, agg_func, col) diff, pass_fail, msg = compare_values(val1, val2, diff_percent, run_name1, run_name2) if over_all_pass: @@ -188,3 +202,10 @@ def compare_artifacts(dir1, dir2, run_name1, run_name2): dataframe = pd.DataFrame(rows[1:], columns=rows[0]) return reporter, dataframe + +if __name__ == "__main__": + compare_artifacts( + "/Users/demo/git/serve/test/performance/run_artifacts/xlarge__45b6399__1594725947", + "/Users/demo/git/serve/test/performance/run_artifacts/xlarge__45b6399__1594725717", + "xlarge__45b6399__1594725947", "xlarge__45b6399__1594725717" + ) \ No newline at end of file diff --git a/tests/performance/runs/context.py b/tests/performance/runs/context.py index a204c67bc..860b35454 100644 --- a/tests/performance/runs/context.py +++ b/tests/performance/runs/context.py @@ -19,6 +19,7 @@ import os import sys import time +import subprocess import webbrowser from termcolor import colored @@ -32,20 +33,27 @@ logging.basicConfig(stream=sys.stdout, format="%(message)s", level=logging.INFO) +def get_git_commit_id(compare_with): + return subprocess.check_output('git rev-parse --short {}'.format(compare_with).split()).decode( + "utf-8")[:-1] + + class ExecutionEnv(object): """ Context Manager class to run the performance regression suites """ - def __init__(self, agent, artifacts_dir, env, local_run, use=True, check_mms_server_status=False): + def __init__(self, agent, artifacts_dir, env, local_run, compare_with, use=True, check_model_server_status=False): self.monitoring_agent = agent self.artifacts_dir = artifacts_dir self.use = use self.env = env self.local_run = local_run - self.check_mms_server_status = check_mms_server_status + self.compare_with = get_git_commit_id(compare_with) + self.check_model_server_status = check_model_server_status self.reporter = JUnitXml() - self.compare_reporter_generator = CompareReportGenerator(self.artifacts_dir, self.env, self.local_run) + self.compare_reporter_generator = CompareReportGenerator(self.artifacts_dir, self.env, self.local_run, compare_with) + self.exit_code = 1 def __enter__(self): if self.use: @@ -63,7 +71,7 @@ def open_report(file_path): @staticmethod def report_summary(reporter, suite_name): if reporter and os.path.exists(reporter.junit_html_path): - status = reporter.junit_xml.errors or reporter.junit_xml.failures or reporter.junit_xml.skipped + status = reporter.junit_xml.errors or reporter.junit_xml.failures status, code, color = ("failed", 3, "red") if status else ("passed", 0, "green") msg = "{} run has {}.".format(suite_name, status) @@ -95,4 +103,9 @@ def __exit__(self, type, value, traceback): compare_exit_code = ExecutionEnv.report_summary(junit_compare_reporter, "Comparison Test suite") exit_code = ExecutionEnv.report_summary(junit_reporter, "Performance Regression Test suite") - sys.exit(0 if 0 == exit_code == compare_exit_code else 3) + self.exit_code = 0 if 0 == exit_code == compare_exit_code else 3 + + # Return True needed so that __exit__ method do no ignore the exception + # otherwise exception are not reported + return False + diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 6db69716c..1f7e0c421 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -35,10 +35,11 @@ class Storage(): """Class to store and retrieve artifacts""" - def __init__(self, path, env_name): + def __init__(self, path, env_name, compare_with): self.artifacts_dir = path self.current_run_name = os.path.basename(path) self.env_name = env_name + self.compare_with = compare_with def get_dir_to_compare(self): """get the artifacts dir to compare to""" @@ -47,7 +48,7 @@ def store_results(self): """Store the results""" @staticmethod - def get_latest(names, env_name, exclude_name): + def get_latest(names, env_name, exclude_name, compare_with): """ Get latest directory for same env_name name given a list of them. :param names: list of folder names in the format env_name___commitid__timestamp @@ -59,7 +60,8 @@ def get_latest(names, env_name, exclude_name): latest_run = '' for run_name in names: run_name_list = run_name.split('__') - if env_name == run_name_list[0] and run_name != exclude_name: + if env_name == run_name_list[0] and compare_with == run_name_list[1]\ + and run_name != exclude_name: if int(run_name_list[2]) > max_ts: max_ts = int(run_name_list[2]) latest_run = run_name @@ -76,7 +78,7 @@ def get_dir_to_compare(self): """Get latest run directory name to be compared with""" parent_dir = pathlib.Path(self.artifacts_dir).parent names = [di for di in os.listdir(parent_dir) if os.path.isdir(os.path.join(parent_dir, di))] - latest_run = self.get_latest(names, self.env_name, self.current_run_name) + latest_run = self.get_latest(names, self.env_name, self.current_run_name, self.compare_with) return os.path.join(parent_dir, latest_run), latest_run @@ -96,7 +98,7 @@ def get_dir_to_compare(self): for o in result.get('CommonPrefixes'): run_names.append(o.get('Prefix')[:-1]) - latest_run = self.get_latest(run_names, self.env_name, self.current_run_name) + latest_run = self.get_latest(run_names, self.env_name, self.current_run_name, self.compare_with) if not latest_run: logger.info("No run found for env_id %s", self.env_name) return '', '' diff --git a/tests/performance/runs/taurus/__init__.py b/tests/performance/runs/taurus/__init__.py index 4a07717ec..973acf64d 100644 --- a/tests/performance/runs/taurus/__init__.py +++ b/tests/performance/runs/taurus/__init__.py @@ -18,8 +18,14 @@ import glob import shutil import os +import sys +import logging from .reader import get_mon_metrics_list +from utils.pyshell import run_process + +logger = logging.getLogger(__name__) +logging.basicConfig(stream=sys.stdout, format="%(message)s", level=logging.INFO) def get_taurus_options(artifacts_dir, jmeter_path=None): @@ -29,7 +35,7 @@ def get_taurus_options(artifacts_dir, jmeter_path=None): options.append('-o modules.jmeter.path={}'.format(jmeter_path)) options.append('-o settings.artifacts-dir={}'.format(artifacts_dir)) options.append('-o modules.console.disable=true') - options.append('-o settings.env.BASEDIR={}'.format(artifacts_dir)) + options.append('-o settings.env.ARTIFACTS_DIR={}'.format(artifacts_dir)) options_str = ' '.join(options) return options_str @@ -61,3 +67,17 @@ def update_taurus_metric_files(suite_artifacts_dir, test_file): metrics_log_file = os.path.join(suite_artifacts_dir, "local_monitoring_logs.csv") if os.path.exists(metrics_log_file): os.rename(metrics_log_file, metrics_new_file) + + KEEP_LINES = 10000 + + def handle_big_files(name): + report_file = os.path.join(suite_artifacts_dir, name) + report_tmp_file = os.path.join(suite_artifacts_dir, "{}_tmp".format(name)) + if os.path.exists(report_file) and os.stat(report_file).st_size > 1e+7: #10MB + logger.info("Keeping first {} records from file {} as it is >10MB".format(KEEP_LINES, report_file)) + run_process("head -{0} {1} > {2}; mv {2} {1};".format(KEEP_LINES, report_file, report_tmp_file)) + + handle_big_files("error.jtl") + handle_big_files("kpi.jtl") + + diff --git a/tests/performance/runs/taurus/reader.py b/tests/performance/runs/taurus/reader.py index 2222afdbe..7abfdf7cc 100644 --- a/tests/performance/runs/taurus/reader.py +++ b/tests/performance/runs/taurus/reader.py @@ -34,9 +34,18 @@ def get_mon_metrics_list(test_yaml_path): return metrics -def get_compare_metric_list(dir, sub_dir): +def parse_criterion_sec(criterion): + subject = criterion["subject"] + metric = subject.rsplit('/', 1) + metric = metric[1] if len(metric) == 2 else metric[0] + diff_percent_prev = criterion.get("diff_percent_previous", None) + diff_percent_run = criterion.get("diff_percent_run", None) + + return [metric, diff_percent_prev, diff_percent_run] + + +def get_compare_metric_list_taurus(dir, sub_dir): """Utility method to get list of compare monitoring metrics identified by diff_percent property""" - diff_percents = [] metrics = [] test_yaml = os.path.join(dir, sub_dir, "effective.yml") with open(test_yaml) as test_yaml: @@ -45,13 +54,21 @@ def get_compare_metric_list(dir, sub_dir): if rep_section.get('module', None) == 'passfail': for criterion in rep_section.get('criteria', []): if isinstance(criterion, dict) and 'monitoring' in criterion.get('class', ''): - subject = criterion["subject"] - metric = subject.rsplit('/', 1) - metric = metric[1] if len(metric) == 2 else metric[0] - diff_percent = criterion.get("diff_percent", None) + metrics.append(parse_criterion_sec(criterion)) + + return metrics + - if diff_percent: - metrics.append(metric) - diff_percents.append(diff_percent) +def get_compare_metric_list(dir, sub_dir): + """Utility method to get list of compare monitoring metrics identified by diff_percent property""" + metrics = [] + test_yaml = os.path.join(dir, sub_dir, "effective.yml") + with open(test_yaml) as test_yaml: + test_yaml = yaml.safe_load(test_yaml) + sec = test_yaml.get('compare_criteria', []) + if sec: + for criterion in sec: + if criterion: + metrics.append(parse_criterion_sec(criterion)) - return metrics, diff_percents + return metrics diff --git a/tests/performance/runs/taurus/x2junit.py b/tests/performance/runs/taurus/x2junit.py index 0219ef76d..a209a7a5e 100644 --- a/tests/performance/runs/taurus/x2junit.py +++ b/tests/performance/runs/taurus/x2junit.py @@ -17,7 +17,12 @@ import os +import pandas as pd +from runs.taurus.reader import get_compare_metric_list +import html +import tabulate +from bzt.modules.passfail import DataCriterion from junitparser import TestCase, TestSuite, JUnitXml, Skipped, Error, Failure @@ -33,44 +38,183 @@ def __init__(self, name, artifacts_dir, junit_xml, timer, env_name): self.timer = timer self.artifacts_dir = artifacts_dir self.env_name = env_name + self.metrics = None + self.metrics_agg_dict = {} + + self.code = 0 + self.err = "" + + self.ts.tests, self.ts.failures, self.ts.skipped, self.ts.errors = 0, 0, 0, 0 def __enter__(self): return self + def add_compare_tests(self): + compare_list = get_compare_metric_list(self.artifacts_dir, "") + for metric_values in compare_list: + col = metric_values[0] + diff_percent = metric_values[2] + tc = TestCase("{}_diff_run > {}".format(col, diff_percent)) + if diff_percent is None: + tc.result = Skipped("diff_percent_run value is not mentioned") + self.ts.skipped += 1 + elif self.metrics is None: + tc.result = Skipped("Metrics are not captured") + self.ts.skipped += 1 + else: + col_metric_values = getattr(self.metrics, col, None) + if col_metric_values is None: + tc.result = Error("Metric {} is not captured".format(col)) + self.ts.errors += 1 + elif len(col_metric_values) < 2: + tc.result = Skipped("Enough values are not captured") + self.ts.errors += 1 + else: + first_value = col_metric_values.iloc[0] + last_value = col_metric_values.iloc[-1] + + try: + if last_value == first_value == 0: + diff_actual = 0 + else: + diff_actual = (abs(last_value - first_value) / ((last_value + first_value) / 2)) * 100 + + if float(diff_actual) <= float(diff_percent): + self.ts.tests += 1 + else: + tc.result = Failure("The first value and last value of run are {}, {} " + "with percent diff {}".format(first_value, last_value, diff_actual)) + + except Exception as e: + tc.result = Error("Error while comparing values {}".format(str(e))) + self.ts.errors += 1 + + self.ts.add_testcase(tc) + + @staticmethod + def casename_to_criteria(test_name): + metric = None + if ' of ' not in test_name: + test_name = "label of {}".format(test_name) + try: + test_name = html.unescape(html.unescape(test_name)) + criteria = DataCriterion.string_to_config(test_name) + except Exception as e: + return None + + label = criteria["label"].split('/') + if len(label) == 2: + metric = label[1] + return metric + + def percentile_values(self, metric_name): + values = {} + if self.metrics is not None and metric_name is not None: + metric_vals = getattr(self.metrics, metric_name, None) + if metric_vals is not None: + centile_values = [0, 0.5, 0.9, 0.95, 0.99, 0.999, 1] + for centile in centile_values: + val = getattr(metric_vals, 'quantile')(centile) + values.update({str(centile * 100)+"%": val}) + + return values + + def update_metrics(self): + metrics_file = os.path.join(self.artifacts_dir, "metrics.csv") + rows = [] + agg_dict = {} + if os.path.exists(metrics_file): + self.metrics = pd.read_csv(metrics_file) + centile_values = [0, 0.5, 0.9, 0.95, 0.99, 0.999, 1] + header_names = ['test_name', 'metric_name'] + header_names.extend([str(colname * 100) + "%" for colname in centile_values]) + header_names.extend(['first_value', 'last_value']) + if self.metrics.size: + for col in self.metrics.columns: + row = [self.name, str(col)] + metric_vals = getattr(self.metrics, str(col), None) + for centile in centile_values: + row.append(getattr(metric_vals, 'quantile')(centile)) + row.extend([metric_vals.iloc[0], metric_vals.iloc[-1]]) + agg_dict.update({row[0]: dict(zip(header_names, row[1:]))}) + rows.append(row) + + dataframe = pd.DataFrame(rows, columns=header_names) + print("Metric percentile values:\n") + print(tabulate.tabulate(rows, headers=header_names, tablefmt="grid")) + dataframe.to_csv(os.path.join(self.artifacts_dir, "metrics_agg.csv"), index=False) + + self.metrics_agg_dict = agg_dict + def __exit__(self, type, value, traceback): + print("error code is "+str(self.code)) + + self.update_metrics() xunit_file = os.path.join(self.artifacts_dir, "xunit.xml") - tests, failures, skipped, errors = 0, 0, 0, 0 - if os.path.exists(xunit_file): + if self.code == 1: + tc = TestCase(self.name) + tc.result = Error(self.err) + self.ts.add_testcase(tc) + elif os.path.exists(xunit_file): xml = JUnitXml.fromfile(xunit_file) for i, suite in enumerate(xml): for case in suite: name = "scenario_{}: {}".format(i, case.name) result = case.result + + metric_name = X2Junit.casename_to_criteria(case.name) + values = self.metrics_agg_dict.get(metric_name, None) + msg = result.message if result else "" + if values: + val_msg = "Actual percentile values are {}".format(values) + msg = "{}. {}".format(msg, val_msg) + if isinstance(result, Error): - failures += 1 - result = Failure(result.message, result.type) + self.ts.failures += 1 + result = Failure(msg, result.type) elif isinstance(result, Failure): - errors += 1 - result = Error(result.message, result.type) + self.ts.errors += 1 + result = Error(msg, result.type) elif isinstance(result, Skipped): - skipped += 1 + self.ts.skipped += 1 + result = Skipped(msg, result.type) else: - tests += 1 + self.ts.tests += 1 tc = TestCase(name) tc.result = result self.ts.add_testcase(tc) else: tc = TestCase(self.name) - tc.result = Skipped() + tc.result = Skipped("Skipped criteria test cases as Taurus XUnit file is not generated.") self.ts.add_testcase(tc) + self.add_compare_tests() + self.ts.hostname = self.env_name self.ts.timestamp = self.timer.start self.ts.time = self.timer.diff() - self.ts.tests = tests - self.ts.failures = failures - self.ts.skipped = skipped - self.ts.errors = errors self.ts.update_statistics() self.junit_xml.add_testsuite(self.ts) + + # Return False needed so that __exit__ method do no ignore the exception + # otherwise exception are not reported + return False + +if __name__ == "__main__": + from utils.timer import Timer + with Timer("ads") as t: + test_folder = '/Users/demo/git/serve/test/performance/'\ + 'run_artifacts/xlarge__2dc700f__1594662587/scale_down_workers' + x = X2Junit("test", test_folder, JUnitXml(), t, "xlarge") + + # x.update_metrics() + # + # x.add_compare_tests() + + x.__exit__(None, None, None) + x.ts + + print("a") + + diff --git a/tests/performance/tests/api_description/api_description.jmx b/tests/performance/tests/api_description/api_description.jmx index a026cb312..69a31b5cf 100644 --- a/tests/performance/tests/api_description/api_description.jmx +++ b/tests/performance/tests/api_description/api_description.jmx @@ -1,7 +1,7 @@ - + false true diff --git a/tests/performance/tests/api_description/api_description.yaml b/tests/performance/tests/api_description/api_description.yaml index da3316228..bb2ec1e88 100644 --- a/tests/performance/tests/api_description/api_description.yaml +++ b/tests/performance/tests/api_description/api_description.yaml @@ -1,68 +1,19 @@ ---- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 30s - scenario: api_description - -scenarios: - api_description: - script: api_description.jmx - -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - -services: - - module: shellexec - prepare: - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss reporting: - module: passfail criteria: # Inbuilt Criteria - - success of ManagementAPIDescription<${MGMT_DESC_SUCC}, stop as failed - - success of InferenceAPIDescription<${INFR_DESC_SUCC}, stop as failed - - avg-rt of ManagementAPIDescription>${MGMT_DESC_RT}, stop as failed - - avg-rt of InferenceAPIDescription>${INFR_DESC_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '<' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 5s - stop : true - fail : true + - success of ManagementAPIDescription<${MGMT_DESC_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of ManagementAPIDescription>${MGMT_DESC_AVG_RT}, ${STOP_ALIAS} as failed +# # Custom Criteria # - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/sum_all_memory_rss -# condition: '>' -# threshold: ${TOTAL_MEM} -# timeframe: 5s -# stop : true -# fail : true \ No newline at end of file +# subject: ServerLocalClient/total_processes +# condition: '<' +# threshold: ${TOTAL_PROCS} +# timeframe: 1s +# stop : ${STOP} +# fail : true + +scenarios: + ~scenario_0: + script: api_description.jmx \ No newline at end of file diff --git a/tests/performance/tests/api_description/environments/xlarge.yaml b/tests/performance/tests/api_description/environments/xlarge.yaml index f7bc5561d..be4e38930 100644 --- a/tests/performance/tests/api_description/environments/xlarge.yaml +++ b/tests/performance/tests/api_description/environments/xlarge.yaml @@ -1,10 +1,52 @@ --- settings: env: - MGMT_DESC_SUCC: 100% - INFR_DESC_SUCC: 100% - MGMT_DESC_RT : 10ms - INFR_DESC_RT : 10ms + MGMT_DESC_SUCC: 80% + MGMT_DESC_AVG_RT: 30ms + + API_LABEL : ManagementAPIDescription + API_SUCCESS : 80% + API_AVG_RT : 30ms + + TOTAL_WORKERS: 0 + TOTAL_WORKERS_MEM: 0 + TOTAL_WORKERS_FDS: 0 + + TOTAL_MEM : 1500098304 TOTAL_PROCS : 1 - TOTAL_FDS : 73 - TOTAL_MEM: 100000000 #100MB + TOTAL_FDS : 10 + + FRNTEND_MEM: 1500098304 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 0 + TOTAL_WORKERS_FDS_RUN_DIFF: 0 + TOTAL_MEM_RUN_DIFF: 185 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 185 + FRNTEND_MEM_RUN_DIFF: 30 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 30s + SCRIPT : api_description.jmx + + STOP : '' + STOP_ALIAS: continue \ No newline at end of file diff --git a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.jmx b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.jmx index d119e9b7f..de4e2a00d 100644 --- a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.jmx +++ b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,12 +16,12 @@ model1 - ${__P(model_name1,resnet-152)} + ${__P(RESNET_152_BATCH_NAME,resnet-152-batch)} = model2 - ${__P(model_name2,squeezenet_v1.1)} + ${__P(SQZNET_NAME,squeezenet1_1)} = diff --git a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml index 73e9ab957..54d6b8475 100644 --- a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml +++ b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml @@ -1,96 +1,22 @@ --- -execution: -- concurrency: 10 - ramp-up: 5s - hold-for: 20s - scenario: Inference - scenarios: - Inference: + scenario_0: script: batch_and_single_inference.jmx -modules: - server_local_monitoring: - # metrics_monitoring_inproc and dependencies should be in python path - class : metrics_monitoring_inproc.Monitor # monitoring class. services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/examples/resnet-152-batching/resnet-152.mar&batch_size=8&max_batch_delay=50" + - "curl -s -X POST http://localhost:8081/models?url=${RESNET_152_BATCH_URL}&batch_size=8&max_batch_delay=50" # uncomment below and comment prev and use downloaded model with model-store - #- curl -s -X POST "http://localhost:8081/models?url=resnet-152.mar&batch_size=8&max_batch_delay=60&initial_workers=1" - - "curl -s -X PUT http://localhost:8081/models/resnet-152?min_worker=2&synchronous=true" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=2&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring # should be added in modules section - ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor - - interval: 1s - logging : True - metrics: - - sum_workers_memory_rss - - sum_workers_file_descriptors - - total_workers - - orphans + #- curl -s -X POST "http://localhost:8081/models?url=${RESNET_152_BATCH_NAME}.mar&batch_size=8&max_batch_delay=60&initial_workers=1" + - "curl -s -X PUT http://localhost:8081/models/${RESNET_152_BATCH_NAME}?min_worker=2&synchronous=true" + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=2&synchronous=true" reporting: - module: passfail criteria: - - subject: avg-rt # required - label: 'Inference1' # optional, default is '' - condition: '>' # required - threshold: ${INFR1_RT} # required - logic: for # optional, logic to aggregate values within timeframe. - # Default 'for' means take latest, - # 'within' and 'over' means take sum/avg of all values within interval - timeframe: 1s # optional, default is none - stop: true # optional, default is true. false for nonstop testing until the end - fail: true # optional, default is true - - subject: avg-rt # required - label: 'Inference2' # optional, default is '' - condition: '>' # required - threshold: ${INFR2_RT} # required - logic: for # optional, logic to aggregate values within timeframe. - # Default 'for' means take latest, - # 'within' and 'over' means take sum/avg of all values within interval - timeframe: 1s # optional, default is none - stop: true # optional, default is true. false for nonstop testing until the end - fail: true # optional, default is true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_memory_rss - condition: '>' - threshold: ${TOTAL_WORKERS_MEM} - timeframe: 1s - stop : true - fail : true - diff_percent : 30 - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/orphans - condition: '>' - threshold: ${TOTAL_ORPHANS} - timeframe: 1s - stop : true - fail : true - diff_percent : 0 - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_workers - condition: '>' - threshold: ${TOTAL_WORKERS} - timeframe: 1s - stop: true - fail: true - diff_percent: 0 - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_file_descriptors - condition: '>' - threshold: ${TOTAL_WORKERS_FDS} - timeframe: 1s - stop: true - fail: true - diff_percent: 30 + # Inbuilt Criteria + - success of ManagementAPIDescription<${INF2_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of ManagementAPIDescription>${INF2_AVG_RT}, ${STOP_ALIAS} as failed \ No newline at end of file diff --git a/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml b/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml index 97307b690..cd22dcdb3 100644 --- a/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml +++ b/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml @@ -1,9 +1,55 @@ --- settings: env: - INFR1_RT : 6s - INFR2_RT : 0.08s - TOTAL_WORKERS_MEM : 4000000000 #4GB - TOTAL_WORKERS : 9 + API_LABEL : Inference1 + API_SUCCESS : 80% + API_AVG_RT : 30ms + + INF2_SUCC: 80% + INF2_AVG_RT: 30ms + + TOTAL_WORKERS: 5 + TOTAL_WORKERS_MEM: 999686400 + TOTAL_WORKERS_FDS: 60 + + TOTAL_MEM : 1292481024 + TOTAL_PROCS : 7 + TOTAL_FDS : 230 + + FRNTEND_MEM: 435241216 + TOTAL_ORPHANS : 0 - TOTAL_WORKERS_FDS : 78 + TOTAL_ZOMBIES : 0 + + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 45 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 45 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 45 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : batch_and_single_inference.jmx + + STOP : '' + STOP_ALIAS: continue + + diff --git a/tests/performance/tests/batch_inference/batch_inference.jmx b/tests/performance/tests/batch_inference/batch_inference.jmx index 885fac295..111799a57 100644 --- a/tests/performance/tests/batch_inference/batch_inference.jmx +++ b/tests/performance/tests/batch_inference/batch_inference.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ model - ${__P(model_name,resnet-152)} + ${__P(RESNET_152_BATCH_NAME,resnet-152-batch)} = diff --git a/tests/performance/tests/batch_inference/batch_inference.yaml b/tests/performance/tests/batch_inference/batch_inference.yaml index 7c4485c06..ef8c4c700 100644 --- a/tests/performance/tests/batch_inference/batch_inference.yaml +++ b/tests/performance/tests/batch_inference/batch_inference.yaml @@ -1,84 +1,14 @@ --- -execution: -- concurrency: 10 - ramp-up: 5s - hold-for: 20s - scenario: Inference - scenarios: - Inference: + scenario_0: script: batch_inference.jmx -modules: - server_local_monitoring: - # metrics_monitoring_inproc and dependencies should be in python path - class : metrics_monitoring_inproc.Monitor # monitoring class. services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/examples/resnet-152-batching/resnet-152.mar&batch_size=8&max_batch_delay=50" + - "curl -s -X POST http://localhost:8081/models?url=${RESNET_152_BATCH_URL}&batch_size=8&max_batch_delay=50" # uncomment below and comment prev and use downloaded model with model-store - #- "curl -s -X POST http://localhost:8081/models?url=resnet-152.mar&batch_size=8&max_batch_delay=60&initial_workers=1" - - "curl -s -X PUT http://localhost:8081/models/resnet-152?min_worker=2&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring # should be added in modules section - ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor - - interval: 1s - logging : True - metrics: - - sum_workers_memory_rss - - sum_workers_file_descriptors - - total_workers - - orphans + #- "curl -s -X POST http://localhost:8081/models?url=${RESNET_152_BATCH_NAME}.mar&batch_size=8&max_batch_delay=60&initial_workers=1" + - "curl -s -X PUT http://localhost:8081/models/${RESNET_152_BATCH_NAME}?min_worker=2&synchronous=true" -reporting: -- module: passfail - criteria: - - subject: avg-rt # required - label: 'Inference' # optional, default is '' - condition: '>' # required - threshold: ${INFR_RT} # required - logic: for # optional, logic to aggregate values within timeframe. - # Default 'for' means take latest, - # 'within' and 'over' means take sum/avg of all values within interval - timeframe: 1s # optional, default is none - stop: true # optional, default is true. false for nonstop testing until the end - fail: true # optional, default is true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_memory_rss - condition: '>' - threshold: ${TOTAL_WORKERS_MEM} - timeframe: 1s - stop : true - fail : true - diff_percent : 30 - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/orphans - condition: '>' - threshold: ${TOTAL_ORPHANS} - timeframe: 1s - stop : true - fail : true - diff_percent : 0 - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_workers - condition: '>' - threshold: ${TOTAL_WORKERS} - timeframe: 1s - stop: true - fail: true - diff_percent: 0 - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_file_descriptors - condition: '>' - threshold: ${TOTAL_WORKERS_FDS} - timeframe: 1s - stop: true - fail: true - diff_percent: 30 diff --git a/tests/performance/tests/batch_inference/environments/xlarge.yaml b/tests/performance/tests/batch_inference/environments/xlarge.yaml index 23a443aaf..5ee95f963 100644 --- a/tests/performance/tests/batch_inference/environments/xlarge.yaml +++ b/tests/performance/tests/batch_inference/environments/xlarge.yaml @@ -1,8 +1,51 @@ --- settings: env: - INFR_RT : 1.5s - TOTAL_WORKERS_MEM : 3000000000 #3GB - TOTAL_WORKERS : 4 + API_LABEL : Inference + API_SUCCESS : 80% + API_AVG_RT : 30ms + + TOTAL_WORKERS: 4 + TOTAL_WORKERS_MEM: 3000000000 + TOTAL_WORKERS_FDS: 400 + + TOTAL_MEM : 4000000000 + TOTAL_PROCS : 7 + TOTAL_FDS : 200 + + FRNTEND_MEM: 1000000000 + TOTAL_ORPHANS : 0 - TOTAL_WORKERS_FDS : 38 \ No newline at end of file + TOTAL_ZOMBIES : 0 + + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 50 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 45 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 80 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : batch_inference.jmx + + STOP : '' + STOP_ALIAS: continue + + diff --git a/tests/performance/tests/examples_local_criteria/environments/xlarge.yaml b/tests/performance/tests/examples_local_criteria/environments/xlarge.yaml index 6c3835292..6cbbedc7d 100644 --- a/tests/performance/tests/examples_local_criteria/environments/xlarge.yaml +++ b/tests/performance/tests/examples_local_criteria/environments/xlarge.yaml @@ -4,5 +4,9 @@ settings: FAIL : 100% P90 : 290ms AVG_RT : 1s - TOTAL_WORKERS_MEM : 132000000 - PERCENT_DIFF_TOTAL_WORKERS_MEM : 5 + TOTAL_WORKERS_MEM : 135000000 + TOTAL_WORKERS_MEM_DIFF : 5 + + STOP : false + +~compare_criteria: diff --git a/tests/performance/tests/examples_local_criteria/examples_local_criteria.jmx b/tests/performance/tests/examples_local_criteria/examples_local_criteria.jmx index 0d60d304c..618b88095 100644 --- a/tests/performance/tests/examples_local_criteria/examples_local_criteria.jmx +++ b/tests/performance/tests/examples_local_criteria/examples_local_criteria.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ cnn_url - https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar + ${__P(SQZNET_URL,https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar)} = The url from where to fetch noop model from @@ -34,7 +34,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = diff --git a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml index 9d8b87907..3adbf0a48 100644 --- a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml +++ b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml @@ -1,11 +1,11 @@ --- -execution: +~execution: - concurrency: 1 ramp-up: 5s hold-for: 20s scenario: Inference -scenarios: +~scenarios: Inference: script: examples_local_criteria.jmx @@ -14,15 +14,34 @@ modules: # metrics_monitoring_inproc and dependencies should be in python path class : metrics_monitoring_inproc.Monitor # monitoring class. -services: +~services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" + - "curl -s -O $INPUT_IMG_URL" + - "mkdir /tmp/ts_model_store" + - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "sleep 20s" post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" + - "$SERVER_CMD --stop > /dev/null 2>&1" + - "rm $INPUT_IMG_PATH" + - "rm -r /tmp/ts_model_store" + - "mv logs ${ARTIFACTS_DIR}/model_server_logs" + + env: + SERVER_CMD : ${SERVER_CMD} + ARTIFACTS_DIR : ${ARTIFACTS_DIR} + SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} + INPUT_IMG_URL : ${INPUT_IMG_URL} + INPUT_IMG_PATH : ${INPUT_IMG_PATH} + + RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} + RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} + SQZNET_URL: ${SQZNET_URL} + SQZNET_NAME: ${SQZNET_NAME} + RESNET_URL: ${RESNET_URL} + RESNET_NAME: ${RESNET_NAME} + - module: server_local_monitoring # should be added in modules section ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor - interval: 1s @@ -33,7 +52,14 @@ services: - mem - sum_workers_memory_rss -reporting: +~reporting: +- module: passfail # this is to enable passfail module +- module: junit-xml + data-source: pass-fail +- module: junit-xml + data-source: sample-labels +- module: final-stats + dump-csv : ${ARTIFACTS_DIR}/final_stats.csv - module: passfail criteria: - fail >${FAIL}, stop as failed @@ -44,6 +70,9 @@ reporting: condition: '>' threshold: ${TOTAL_WORKERS_MEM} timeframe: 1s - stop : true + stop : ${STOP} fail : true - diff_percent : ${PERCENT_DIFF_TOTAL_WORKERS_MEM} + diff_percent_previous : ${TOTAL_WORKERS_MEM_DIFF} + +~compare_criteria: + - \ No newline at end of file diff --git a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.jmx b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.jmx index 0d60d304c..618b88095 100644 --- a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.jmx +++ b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ cnn_url - https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar + ${__P(SQZNET_URL,https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar)} = The url from where to fetch noop model from @@ -34,7 +34,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = diff --git a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml index d00226470..ee8fdac7c 100644 --- a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml +++ b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml @@ -1,10 +1,11 @@ --- -execution: +~execution: - concurrency: 1 ramp-up: 5s hold-for: 20s scenario: Inference -scenarios: + +~scenarios: Inference: script: examples_local_monitoring.jmx @@ -13,15 +14,34 @@ modules: # metrics_monitoring_inproc and dependencies should be in python path class : metrics_monitoring_inproc.Monitor # monitoring class. -services: +~services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" + - "curl -s -O $INPUT_IMG_URL" + - "mkdir /tmp/ts_model_store" + - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "sleep 20s" post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" + - "$SERVER_CMD --stop > /dev/null 2>&1" + - "rm $INPUT_IMG_PATH" + - "rm -r /tmp/ts_model_store" + - "mv logs ${ARTIFACTS_DIR}/model_server_logs" + + env: + SERVER_CMD : ${SERVER_CMD} + ARTIFACTS_DIR : ${ARTIFACTS_DIR} + SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} + INPUT_IMG_URL : ${INPUT_IMG_URL} + INPUT_IMG_PATH : ${INPUT_IMG_PATH} + + RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} + RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} + SQZNET_URL: ${SQZNET_URL} + SQZNET_NAME: ${SQZNET_NAME} + RESNET_URL: ${RESNET_URL} + RESNET_NAME: ${RESNET_NAME} + - module: server_local_monitoring # should be added in modules section ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor - interval: 1s @@ -29,4 +49,10 @@ services: - cpu - disk-space - mem - - sum_workers_memory_percent \ No newline at end of file + - sum_workers_memory_percent + +~reporting: + - module: passfail + +~compare_criteria: + - diff --git a/tests/performance/tests/examples_remote_criteria/environments/xlarge.yaml b/tests/performance/tests/examples_remote_criteria/environments/xlarge.yaml index 674a6c1ff..6ad41860d 100644 --- a/tests/performance/tests/examples_remote_criteria/environments/xlarge.yaml +++ b/tests/performance/tests/examples_remote_criteria/environments/xlarge.yaml @@ -5,3 +5,6 @@ settings: P90 : 250ms AVG_RT : 1s TOTAL_WORKERS_FDS : 80 + TOTAL_WORKERS_FDS_DIFF : 35 + +~compare_criteria: \ No newline at end of file diff --git a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.jmx b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.jmx index 0d60d304c..618b88095 100644 --- a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.jmx +++ b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ cnn_url - https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar + ${__P(SQZNET_URL,https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar)} = The url from where to fetch noop model from @@ -34,7 +34,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = diff --git a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml index 0c3c206d1..10b028895 100644 --- a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml +++ b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml @@ -1,47 +1,74 @@ -execution: +~execution: - concurrency: 4 ramp-up: 1s hold-for: 20s scenario: Inference -scenarios: +~scenarios: Inference: script: examples_remote_criteria.jmx -services: +~services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" + - "curl -s -O $INPUT_IMG_URL" + - "mkdir /tmp/ts_model_store" + - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "sleep 20s" post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" + - "$SERVER_CMD --stop > /dev/null 2>&1" + - "rm $INPUT_IMG_PATH" + - "rm -r /tmp/ts_model_store" + - "mv logs ${ARTIFACTS_DIR}/model_server_logs" + + env: + SERVER_CMD : ${SERVER_CMD} + ARTIFACTS_DIR : ${ARTIFACTS_DIR} + SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} + INPUT_IMG_URL : ${INPUT_IMG_URL} + INPUT_IMG_PATH : ${INPUT_IMG_PATH} + + RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} + RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} + SQZNET_URL: ${SQZNET_URL} + SQZNET_NAME: ${SQZNET_NAME} + RESNET_URL: ${RESNET_URL} + RESNET_NAME: ${RESNET_NAME} + - module: monitoring server-agent: - address: localhost:9009 # metric monitoring service address - label: mms-inference-server # if you specify label, it will be used in reports instead of ip:port + label: model-server # if you specify label, it will be used in reports instead of ip:port interval: 1s # polling interval logging: True # those logs will be saved to "SAlogs_192.168.0.1_9009.csv" in the artifacts dir metrics: # metrics should be supported by monitoring service - - sum_workers_cpu_percent # cpu percent used by all the mms server processes and workers + - sum_workers_cpu_percent # cpu percent used by all the Model Server server processes and workers - sum_workers_memory_percent - sum_workers_file_descriptors - - total_workers # no of mms workers + - total_workers # no of Model Server workers -reporting: +~reporting: +- module: passfail # this is to enable passfail module +- module: junit-xml + data-source: pass-fail +- module: junit-xml + data-source: sample-labels +- module: final-stats + dump-csv : ${ARTIFACTS_DIR}/final_stats.csv - module: passfail criteria: - fail >${FAIL}, stop as failed - p90 >${P90} , stop as failed - avg-rt >${AVG_RT} , stop as failed - class: bzt.modules.monitoring.MonitoringCriteria - subject: mms-inference-server/sum_workers_file_descriptors + subject: model-server/sum_workers_file_descriptors condition: '>' threshold: ${TOTAL_WORKERS_FDS} timeframe: 1s fail: true stop: true - diff_percent : 35 \ No newline at end of file + diff_percent_previous : ${TOTAL_WORKERS_FDS_DIFF} + diff --git a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.jmx b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.jmx index 0d60d304c..618b88095 100644 --- a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.jmx +++ b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ cnn_url - https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar + ${__P(SQZNET_URL,https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar)} = The url from where to fetch noop model from @@ -34,7 +34,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = diff --git a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml index 235c3b803..e3b71cf4e 100644 --- a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml +++ b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml @@ -1,34 +1,56 @@ -execution: +~execution: - concurrency: 4 ramp-up: 1s hold-for: 20s scenario: Inference -scenarios: +~scenarios: Inference: script: examples_remote_monitoring.jmx -services: +~services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" + - "curl -s -O $INPUT_IMG_URL" + - "mkdir /tmp/ts_model_store" + - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "sleep 20s" post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" + - "$SERVER_CMD --stop > /dev/null 2>&1" + - "rm $INPUT_IMG_PATH" + - "rm -r /tmp/ts_model_store" + - "mv logs ${ARTIFACTS_DIR}/model_server_logs" + + env: + SERVER_CMD : ${SERVER_CMD} + ARTIFACTS_DIR : ${ARTIFACTS_DIR} + SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} + INPUT_IMG_URL : ${INPUT_IMG_URL} + INPUT_IMG_PATH : ${INPUT_IMG_PATH} + + RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} + RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} + SQZNET_URL: ${SQZNET_URL} + SQZNET_NAME: ${SQZNET_NAME} + RESNET_URL: ${RESNET_URL} + RESNET_NAME: ${RESNET_NAME} - module: monitoring server-agent: - address: localhost:9009 # metric monitoring service address - label: mms-inference-server # if you specify label, it will be used in reports instead of ip:port + label: model-server # if you specify label, it will be used in reports instead of ip:port interval: 1s # polling interval logging: True # those logs will be saved to "SAlogs_192.168.0.1_9009.csv" in the artifacts dir metrics: # metrics should be supported by monitoring service - - sum_all_cpu_percent # cpu percent used by all the mms server processes and workers + - sum_all_cpu_percent # cpu percent used by all the Model server processes and workers - sum_workers_memory_percent - frontend_file_descriptors - - total_workers # no of mms workers + - total_workers # no of Model Server workers + +~reporting: + - module: passfail +~compare_criteria: diff --git a/tests/performance/tests/examples_starter/examples_starter.jmx b/tests/performance/tests/examples_starter/examples_starter.jmx index 0d60d304c..618b88095 100644 --- a/tests/performance/tests/examples_starter/examples_starter.jmx +++ b/tests/performance/tests/examples_starter/examples_starter.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ cnn_url - https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar + ${__P(SQZNET_URL,https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar)} = The url from where to fetch noop model from @@ -34,7 +34,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = diff --git a/tests/performance/tests/examples_starter/examples_starter.yaml b/tests/performance/tests/examples_starter/examples_starter.yaml index ac6aaa50b..ee191afaf 100644 --- a/tests/performance/tests/examples_starter/examples_starter.yaml +++ b/tests/performance/tests/examples_starter/examples_starter.yaml @@ -1,20 +1,45 @@ --- -execution: -- concurrency: 1 - ramp-up: 1s - hold-for: 40s - scenario: Inference -scenarios: +~execution: + - concurrency: 1 + ramp-up: 1s + hold-for: 40s + scenario: Inference + +~scenarios: Inference: script: examples_starter.jmx -services: +~services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" + - "curl -s -O $INPUT_IMG_URL" + - "mkdir /tmp/ts_model_store" + - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "sleep 20s" post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" + - "$SERVER_CMD --stop > /dev/null 2>&1" + - "rm $INPUT_IMG_PATH" + - "rm -r /tmp/ts_model_store" + - "mv logs ${ARTIFACTS_DIR}/model_server_logs" + + env: + SERVER_CMD : ${SERVER_CMD} + ARTIFACTS_DIR : ${ARTIFACTS_DIR} + SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} + INPUT_IMG_URL : ${INPUT_IMG_URL} + INPUT_IMG_PATH : ${INPUT_IMG_PATH} + + RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} + RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} + SQZNET_URL: ${SQZNET_URL} + SQZNET_NAME: ${SQZNET_NAME} + RESNET_URL: ${RESNET_URL} + RESNET_NAME: ${RESNET_NAME} + + +~reporting: + - module: passfail +~compare_criteria: + - diff --git a/tests/performance/tests/global_config.yaml b/tests/performance/tests/global_config.yaml index 94731f4a3..7fd5acfc7 100644 --- a/tests/performance/tests/global_config.yaml +++ b/tests/performance/tests/global_config.yaml @@ -1,17 +1,74 @@ +--- +execution: +- concurrency: ${CONCURRENCY} + ramp-up: ${RAMP-UP} + hold-for: ${HOLD-FOR} + scenario: scenario_0 + +scenarios: + scenario_0: + script: ${SCRIPT} + modules: jmeter: # These are JMeter test case properties. These variables are used in jmx files. # Change the vaues as per your setup properties: - hostname : 127.0.0.1 # MMS properties - port : 8080 - management_port : 8081 - protocol : http - input_filepath : kitten.jpg # make sure jpg is available at this path + hostname: 127.0.0.1 # Model Server properties + port: 8080 + management_port: 8081 + protocol: http + input_filepath: kitten.jpg # make sure jpg is available at this path # if relative path is provided this will be relative to current working directory -# DO-NOT change properties below unless you know what you are doing. -# They are needed for performance test suite runner script. + server_local_monitoring: + # metrics_monitoring_inproc and dependencies should be in python path + class : metrics_monitoring_inproc.Monitor # monitoring class. + +services: + - module: shellexec + prepare: + - "curl -s -O ${INPUT_IMG_URL}" + - "mkdir /tmp/ts_model_store" + - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "${SERVER_CMD} --start --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "sleep 20s" + post-process: + - "${SERVER_CMD} --stop > /dev/null 2>&1" + - "rm ${INPUT_IMG_PATH}" + - "rm -r /tmp/ts_model_store" + - "mv logs ${ARTIFACTS_DIR}/model_server_logs" + + env: + SERVER_CMD : ${SERVER_CMD} + ARTIFACTS_DIR : ${ARTIFACTS_DIR} + SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} + INPUT_IMG_URL : ${INPUT_IMG_URL} + INPUT_IMG_PATH : ${INPUT_IMG_PATH} + + RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} + RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} + SQZNET_URL: ${SQZNET_URL} + SQZNET_NAME: ${SQZNET_NAME} + RESNET_URL: ${RESNET_URL} + RESNET_NAME: ${RESNET_NAME} + + + - module: server_local_monitoring # should be added in modules section + ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor + - interval: 1s + logging : True + metrics: + - sum_workers_memory_rss + - sum_workers_file_descriptors + - total_workers + - orphans + - zombies + - frontend_memory_rss + - sum_all_memory_rss + - total_processes + - sum_all_file_descriptors + reporting: - module: passfail # this is to enable passfail module - module: junit-xml @@ -19,8 +76,129 @@ reporting: - module: junit-xml data-source: sample-labels - module: final-stats - dump-csv : ${BASEDIR}/final_stats.csv + dump-csv : ${ARTIFACTS_DIR}/final_stats.csv + +- module: passfail + criteria: + # API requests KPI crieteria + - success of ${API_LABEL}<${API_SUCCESS} for 10s, stop as failed + - avg-rt of ${API_LABEL}>${API_AVG_RT}, ${STOP_ALIAS} as failed +# +# # Monitoring metrics criteria +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/total_workers +# condition: '>' +# threshold: ${TOTAL_WORKERS} +# timeframe: 5s +# stop: ${STOP} +# fail: true +# diff_percent_previous: ${TOTAL_WORKERS_DIFF} +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/sum_workers_memory_rss +# condition: '>' +# threshold: ${TOTAL_WORKERS_MEM} +# timeframe: 5s +# stop : ${STOP} +# fail : true +# diff_percent_previous : ${TOTAL_WORKERS_MEM_DIFF} +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/sum_workers_file_descriptors +# condition: '>' +# threshold: ${TOTAL_WORKERS_FDS} +# timeframe: 5s +# stop: ${STOP} +# fail: true +# diff_percent_previous: ${TOTAL_WORKERS_FDS_DIFF} +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/sum_all_memory_rss +# condition: '>' +# threshold: ${TOTAL_MEM} +# timeframe: 5s +# stop : ${STOP} +# fail : true +# diff_percent_previous: ${TOTAL_MEM_DIFF} +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/total_processes +# condition: '>' +# threshold: ${TOTAL_PROCS} +# timeframe: 5s +# stop: ${STOP} +# fail: true +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/sum_all_file_descriptors +# condition: '>' +# threshold: ${TOTAL_FDS} +# timeframe: 1s +# stop: ${STOP} +# fail: true +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/frontend_memory_rss +# condition: '>' +# threshold: ${FRNTEND_MEM} +# timeframe: 5s +# stop: ${STOP} +# fail: true +# diff_percent_previous: ${FRNTEND_MEM_DIFF} +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/orphans +# condition: '>' +# threshold: ${TOTAL_ORPHANS} +# timeframe: 5s +# stop: ${STOP} +# fail: true +# diff_percent_previous: ${TOTAL_ORPHANS_DIFF} +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/zombies +# condition: '>' +# threshold: ${TOTAL_ZOMBIES} +# timeframe: 5s +# stop: ${STOP} +# fail: true +# diff_percent_previous: ${TOTAL_ZOMBIES_DIFF} + +compare_criteria: + # Monitoring metrics criteria + - subject: ServerLocalClient/total_workers + diff_percent_previous: ${TOTAL_WORKERS_PREV_DIFF} + diff_percent_run: ${TOTAL_WORKERS_RUN_DIFF} + - subject: ServerLocalClient/sum_workers_memory_rss + diff_percent_previous: ${TOTAL_WORKERS_MEM_PREV_DIFF} + diff_percent_run : ${TOTAL_WORKERS_MEM_RUN_DIFF} + - subject: ServerLocalClient/sum_workers_file_descriptors + diff_percent_previous: ${TOTAL_WORKERS_FDS_PREV_DIFF} + diff_percent_run: ${TOTAL_WORKERS_FDS_RUN_DIFF} +# - subject: ServerLocalClient/sum_all_memory_rss +# diff_percent_previous: ${TOTAL_MEM_PREV_DIFF} +# diff_percent_run: ${TOTAL_MEM_RUN_DIFF} + - subject: ServerLocalClient/total_processes + diff_percent_previous: ${TOTAL_PROCS_PREV_DIFF} + diff_percent_run: ${TOTAL_PROCS_RUN_DIFF} + - subject: ServerLocalClient/sum_all_file_descriptors + diff_percent_previous : ${TOTAL_FDS_PREV_DIFF} + diff_percent_run : ${TOTAL_FDS_RUN_DIFF} +# - subject: ServerLocalClient/frontend_memory_rss +# diff_percent_previous: ${FRNTEND_MEM_PREV_DIFF} +# diff_percent_run: ${FRNTEND_MEM_RUN_DIFF} + - subject: ServerLocalClient/orphans + diff_percent_previous: ${TOTAL_ORPHANS_PREV_DIFF} + diff_percent_run: ${TOTAL_ORPHANS_RUN_DIFF} + - subject: ServerLocalClient/zombies + diff_percent_previous: ${TOTAL_ZOMBIES_PREV_DIFF} + diff_percent_run: ${TOTAL_ZOMBIES_RUN_DIFF} + settings: env: - BASEDIR : '.' + ARTIFACTS_DIR : '.' + SERVER_CMD : "multi-model-server" + SERVER_PROCESS_NAME : "[c]om.amazonaws.ml.mms.ModelServer" + INPUT_IMG_URL: "https://s3.amazonaws.com/model-server/inputs/kitten.jpg" + INPUT_IMG_PATH: "kitten.jpg" + + RESNET_152_BATCH_URL : "https://s3.amazonaws.com/model-server/model_archive_1.0/examples/resnet-152-batching/resnet-152.mar" + RESNET_152_BATCH_NAME : "resnet-152-batch" + SQZNET_URL : "https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" + SQZNET_NAME : "squeezenet_v1.1" + RESNET_URL : "https://s3.amazonaws.com/model-server/model_archive_1.0/resnet-18.mar" + RESNET_NAME : "resnet-18" + diff --git a/tests/performance/tests/health_check/environments/xlarge.yaml b/tests/performance/tests/health_check/environments/xlarge.yaml index 689a5d66b..2a8e2332d 100644 --- a/tests/performance/tests/health_check/environments/xlarge.yaml +++ b/tests/performance/tests/health_check/environments/xlarge.yaml @@ -1,8 +1,49 @@ + --- settings: env: - HLTH_CHK_SUCC : 100% - HLTH_CHK_RT : 14ms + API_LABEL : HealthCheck + API_SUCCESS : 80% + API_AVG_RT : 30ms + + TOTAL_WORKERS: 0 + TOTAL_WORKERS_MEM: 0 + TOTAL_WORKERS_FDS: 0 + + TOTAL_MEM : 1500098304 TOTAL_PROCS : 1 - TOTAL_FDS : 67 - TOTAL_MEM : 750000000 #750MB \ No newline at end of file + TOTAL_FDS : 73 + + FRNTEND_MEM: 1500098304 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 0 + TOTAL_WORKERS_FDS_RUN_DIFF: 0 + TOTAL_MEM_RUN_DIFF: 200 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 200 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : health_check.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/health_check/health_check.jmx b/tests/performance/tests/health_check/health_check.jmx index 422c45cf9..dc699a6be 100644 --- a/tests/performance/tests/health_check/health_check.jmx +++ b/tests/performance/tests/health_check/health_check.jmx @@ -1,7 +1,7 @@ - + false true diff --git a/tests/performance/tests/health_check/health_check.yaml b/tests/performance/tests/health_check/health_check.yaml index 2c8785e3c..3f2d38636 100644 --- a/tests/performance/tests/health_check/health_check.yaml +++ b/tests/performance/tests/health_check/health_check.yaml @@ -1,66 +1,18 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 30s - scenario: health_check - -scenarios: - health_check: - script: health_check.jmx - -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - -services: - - module: shellexec - prepare: - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss - reporting: - module: passfail criteria: - # Inbuilt Criteria - - success of HealthCheck<${HLTH_CHK_SUCC}, stop as failed - - avg-rt of HealthCheck>${HLTH_CHK_RT}, stop as failed # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} timeframe: 5s - stop : true + stop : ${STOP} fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/sum_all_memory_rss -# condition: '>' -# threshold: ${TOTAL_MEM} -# timeframe: 5s -# stop : true -# fail : true \ No newline at end of file + +scenarios: + ~scenario_0: + script: health_check.jmx + + diff --git a/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml b/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml index 36b7dc0ad..c854de1cd 100644 --- a/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml +++ b/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml @@ -1,10 +1,51 @@ + --- settings: env: - INFR1_SUCC : 100% + API_LABEL : Inference1 + API_SUCCESS : 80% + API_AVG_RT : 30ms + INFR2_SUCC: 100% - INFR1_RT : 290ms INFR2_RT: 450ms - TOTAL_PROCS : 5 - TOTAL_FDS : 107 - TOTAL_MEM : 600000000 #600MB \ No newline at end of file + + TOTAL_WORKERS: 2 + TOTAL_WORKERS_MEM: 600000000 + TOTAL_WORKERS_FDS: 150 + + TOTAL_MEM : 1400000000 + TOTAL_PROCS : 3 + TOTAL_FDS : 150 + + FRNTEND_MEM: 800000000 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + TOTAL_WORKERS_PREV_DIFF: 30 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 30 + TOTAL_WORKERS_MEM_RUN_DIFF: 30 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 30 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 40 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : inference_multiple_models.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/inference_multiple_models/inference_multiple_models.jmx b/tests/performance/tests/inference_multiple_models/inference_multiple_models.jmx index 1ceeaf2c2..67a5d689a 100644 --- a/tests/performance/tests/inference_multiple_models/inference_multiple_models.jmx +++ b/tests/performance/tests/inference_multiple_models/inference_multiple_models.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,13 +16,13 @@ model1 - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model1 Name model2 - resnet-18 + ${__P(RESNET_NAME,resnet-18)} Model2 Name = diff --git a/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml b/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml index 3244c4d8f..047d4d168 100644 --- a/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml +++ b/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml @@ -1,74 +1,27 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 30s - scenario: inference_multiple_models - scenarios: - inference_multiple_models: + scenario_0: script: inference_multiple_models.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/resnet-18.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true" - - "curl -s -X PUT http://localhost:8081/models/resnet-18?min_worker=1&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X POST http://localhost:8081/models?url=${RESNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=1&synchronous=true" + - "curl -s -X PUT http://localhost:8081/models/${RESNET_NAME}?min_worker=1&synchronous=true" reporting: - module: passfail criteria: # Inbuilt Criteria - - success of Inference1<${INFR1_SUCC}, stop as failed - - success of Inference2<${INFR2_SUCC}, stop as failed - - avg-rt of Inference1>${INFR1_RT}, stop as failed - - avg-rt of Inference2>${INFR2_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true + - success of Inference2<${INFR2_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of Inference2>${INFR2_RT}, ${STOP_ALIAS} as failed - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} timeframe: 5s - stop : true + stop : ${STOP} fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/sum_all_memory_rss -# condition: '>' -# threshold: ${TOTAL_MEM} -# timeframe: 5s -# stop : true -# fail : true diff --git a/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml b/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml index 5b9fd6c0a..7f7887242 100644 --- a/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml +++ b/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml @@ -1,8 +1,49 @@ + --- settings: env: - INFR_SUCC : 100% - INFR_RT : 140ms - TOTAL_PROCS : 6 - TOTAL_FDS : 126 - TOTAL_MEM : 750000000 #750MB \ No newline at end of file + API_LABEL : Inference + API_SUCCESS : 80% + API_AVG_RT : 140ms + + TOTAL_WORKERS: 4 + TOTAL_WORKERS_MEM: 600000000 + TOTAL_WORKERS_FDS: 40 + + TOTAL_MEM : 1400000000 + TOTAL_PROCS : 5 + TOTAL_FDS : 150 + + FRNTEND_MEM: 800000000 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 30 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 35 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 60 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : inference_multiple_worker.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.jmx b/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.jmx index 1251a56b8..5d0816ae2 100644 --- a/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.jmx +++ b/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model Name diff --git a/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.yaml b/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.yaml index 5d73624a6..5f98bbd2c 100644 --- a/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.yaml +++ b/tests/performance/tests/inference_multiple_worker/inference_multiple_worker.yaml @@ -1,71 +1,22 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 1m - iterations: 100 - scenario: inference_multiple_worker - scenarios: inference_multiple_worker: script: inference_multiple_worker.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=4&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=4&synchronous=true" reporting: - module: passfail criteria: - # Inbuilt Criteria - - success of Inference<${INFR_SUCC}, stop as failed - - avg-rt of Inference>${INFR_RT}, stop as failed # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 1s - stop : true + timeframe: 5s + stop : ${STOP} fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/sum_all_memory_rss -# condition: '>' -# threshold: ${TOTAL_MEM} -# timeframe: 5s -# stop : true -# fail : true \ No newline at end of file diff --git a/tests/performance/tests/inference_single_worker/environments/xlarge.yaml b/tests/performance/tests/inference_single_worker/environments/xlarge.yaml index c945e1f91..f160a1bcf 100644 --- a/tests/performance/tests/inference_single_worker/environments/xlarge.yaml +++ b/tests/performance/tests/inference_single_worker/environments/xlarge.yaml @@ -1,8 +1,49 @@ + --- settings: env: - INFR_SUCC : 100% - INFR_RT : 290ms - TOTAL_PROCS : 3 - TOTAL_FDS : 90 - TOTAL_MEM : 290000000 #290MB \ No newline at end of file + API_LABEL : Inference + API_SUCCESS : 80% + API_AVG_RT : 140ms + + TOTAL_WORKERS: 1 + TOTAL_WORKERS_MEM: 300000000 + TOTAL_WORKERS_FDS: 150 + + TOTAL_MEM : 1000000000 + TOTAL_PROCS : 2 + TOTAL_FDS : 150 + + FRNTEND_MEM: 600000000 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 30 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 60 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 90 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : inference_single_worker.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/inference_single_worker/inference_single_worker.jmx b/tests/performance/tests/inference_single_worker/inference_single_worker.jmx index ea05cc1ef..5124dbc6e 100644 --- a/tests/performance/tests/inference_single_worker/inference_single_worker.jmx +++ b/tests/performance/tests/inference_single_worker/inference_single_worker.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model Name diff --git a/tests/performance/tests/inference_single_worker/inference_single_worker.yaml b/tests/performance/tests/inference_single_worker/inference_single_worker.yaml index ece9e5b74..1bf6cce30 100644 --- a/tests/performance/tests/inference_single_worker/inference_single_worker.yaml +++ b/tests/performance/tests/inference_single_worker/inference_single_worker.yaml @@ -1,71 +1,21 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 1m - iterations: 100 - scenario: inference_single_worker - scenarios: inference_single_worker: script: inference_single_worker.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=1&synchronous=true" reporting: - module: passfail criteria: - # Inbuilt Criteria - - success of Inference<${INFR_SUCC}, stop as failed - - avg-rt of Inference>${INFR_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS} timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 1s - stop : true + stop : ${STOP} fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/sum_all_memory_rss -# condition: '>' -# threshold: ${TOTAL_MEM} -# timeframe: 5s -# stop : true -# fail : true diff --git a/tests/performance/tests/list_models/environments/xlarge.yaml b/tests/performance/tests/list_models/environments/xlarge.yaml index 611624824..bed934bf9 100644 --- a/tests/performance/tests/list_models/environments/xlarge.yaml +++ b/tests/performance/tests/list_models/environments/xlarge.yaml @@ -1,8 +1,48 @@ --- settings: env: - LST_MODLS_SUCC : 100% - LST_MODLS_RT : 14ms + API_LABEL : ListModels + API_SUCCESS : 80% + API_AVG_RT : 14ms + + TOTAL_WORKERS: 2 + TOTAL_WORKERS_MEM: 600000000 + TOTAL_WORKERS_FDS: 40 + + TOTAL_MEM : 1400000000 TOTAL_PROCS : 3 - TOTAL_FDS : 86 - TOTAL_MEM : 185000000 #185MB \ No newline at end of file + TOTAL_FDS : 150 + + FRNTEND_MEM: 800000000 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 30 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 30 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 30 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : list_models.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/list_models/list_models.jmx b/tests/performance/tests/list_models/list_models.jmx index cd5490dc4..0323fcee8 100644 --- a/tests/performance/tests/list_models/list_models.jmx +++ b/tests/performance/tests/list_models/list_models.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model name diff --git a/tests/performance/tests/list_models/list_models.yaml b/tests/performance/tests/list_models/list_models.yaml index 81dd8ada7..60d29551c 100644 --- a/tests/performance/tests/list_models/list_models.yaml +++ b/tests/performance/tests/list_models/list_models.yaml @@ -1,68 +1,21 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 30s - scenario: list_models - scenarios: - list_models: + scenario_0: script: list_models.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - services: - module: shellexec prepare: - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/shufflenet.mar" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X POST http://localhost:8081/models?url=${RESNET_URL}" reporting: - module: passfail criteria: - # Inbuilt Criteria - - success of ListModels<${LST_MODLS_SUCC}, stop as failed - - avg-rt of ListModels>${LST_MODLS_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 1s - stop : true + timeframe: 5s + stop : {STOP} fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/sum_all_memory_rss -# condition: '>' -# threshold: ${TOTAL_MEM} -# timeframe: 5s -# stop : true -# fail : true \ No newline at end of file diff --git a/tests/performance/tests/model_description/environments/xlarge.yaml b/tests/performance/tests/model_description/environments/xlarge.yaml index 00e0aac87..f62c5e282 100644 --- a/tests/performance/tests/model_description/environments/xlarge.yaml +++ b/tests/performance/tests/model_description/environments/xlarge.yaml @@ -1,8 +1,49 @@ + --- settings: env: - MODL_DESC_SUCC : 100% - MODL_DESC_RT : 14ms - TOTAL_PROCS : 3 - TOTAL_FDS : 90 - TOTAL_MEM : 300000000 #300MB \ No newline at end of file + API_LABEL : ModelDescription + API_SUCCESS : 80% + API_AVG_RT : 14ms + + TOTAL_WORKERS: 1 + TOTAL_WORKERS_MEM: 150205952 + TOTAL_WORKERS_FDS: 40 + + TOTAL_MEM : 1400000000 + TOTAL_PROCS : 2 + TOTAL_FDS : 150 + + FRNTEND_MEM: 800000000 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 50 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 30 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 30 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 30 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : model_description.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/model_description/model_description.jmx b/tests/performance/tests/model_description/model_description.jmx index 4d8898adb..5cb68f9ac 100644 --- a/tests/performance/tests/model_description/model_description.jmx +++ b/tests/performance/tests/model_description/model_description.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,7 +16,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model name diff --git a/tests/performance/tests/model_description/model_description.yaml b/tests/performance/tests/model_description/model_description.yaml index 05358a53c..ef429dd5e 100644 --- a/tests/performance/tests/model_description/model_description.yaml +++ b/tests/performance/tests/model_description/model_description.yaml @@ -1,68 +1,21 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 30s - scenario: model_description - scenarios: - model_description: + scenario_0: script: model_description.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - services: - module: shellexec prepare: - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=1&synchronous=true" reporting: - module: passfail criteria: - # Inbuilt Criteria - - success of ModelDescription<${MODL_DESC_SUCC}, stop as failed - - avg-rt of ModelDescription>${MODL_DESC_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 1s - stop : true - fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS} timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 1s - stop : true + stop : ${STOP} fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/sum_all_memory_rss -# condition: '>' -# threshold: ${TOTAL_MEM} -# timeframe: 5s -# stop : true -# fail : true \ No newline at end of file diff --git a/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml b/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml index 1671213d1..704576848 100644 --- a/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml +++ b/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml @@ -1,12 +1,56 @@ + --- settings: env: - INFR1_SUCC : 100% - INFR2_SUCC: 100% - INFR1_RT : 290ms + API_LABEL : Inference1 + API_SUCCESS : 80% + API_AVG_RT : 290ms + + INFR2_SUCC: 80% INFR2_RT: 450ms - TOTAL_PROCS : 14 + SCALEUP1_RT : 500ms + SCALEUP2_RT : 500ms + SCALEDOWN1_RT : 100ms + SCALEDOWN2_RT : 100ms + + TOTAL_WORKERS: 9 + TOTAL_WORKERS_MEM: 2668554752 + TOTAL_WORKERS_FDS: 100 + + TOTAL_MEM : 2000000000 + TOTAL_PROCS : 11 TOTAL_FDS : 300 - TOTAL_MEM : 2000000000 #~2GB + + FRNTEND_MEM: 1000000000 + TOTAL_ORPHANS : 0 - FRNTEND_MEM : 1000000000 #~1GB \ No newline at end of file + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 120 + TOTAL_WORKERS_MEM_RUN_DIFF: 135 + TOTAL_WORKERS_FDS_RUN_DIFF: 130 + TOTAL_MEM_RUN_DIFF: 130 + TOTAL_PROCS_RUN_DIFF: 100 + TOTAL_FDS_RUN_DIFF: 40 + FRNTEND_MEM_RUN_DIFF: 130 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : multiple_inference_and_scaling.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.jmx b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.jmx index cbff8debc..cbff660c4 100644 --- a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.jmx +++ b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,13 +16,13 @@ model1 - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model1 Name model2 - resnet-18 + ${__P(RESNET_NAME,resnet-18)} Model2 Name = @@ -117,7 +117,7 @@ - + @@ -148,7 +148,35 @@ - + + + + + ${__P(input_filepath)} + data + image/jpeg + + + + + + + + + + + /predictions/${model1} + POST + true + false + true + true + + + + + + @@ -234,7 +262,7 @@ - + @@ -265,7 +293,35 @@ - + + + + + ${__P(input_filepath)} + data + image/jpeg + + + + + + + + + + + /predictions/${model2} + POST + true + false + true + true + + + + + + diff --git a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml index 8f3324f2a..ce5c9725b 100644 --- a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml +++ b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml @@ -1,83 +1,33 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 300s - scenario: inference_multiple_models - scenarios: - inference_multiple_models: + scenario_0: script: multiple_inference_and_scaling.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/resnet-18.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true" - - "curl -s -X PUT http://localhost:8081/models/resnet-18?min_worker=1&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - sum_all_memory_rss - - frontend_memory_rss - - orphans + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X POST http://localhost:8081/models?url=${RESNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=1&synchronous=true" + - "curl -s -X PUT http://localhost:8081/models/${RESNET_NAME}?min_worker=1&synchronous=true" + reporting: - module: passfail criteria: # Inbuilt Criteria - - success of Inference1<${INFR1_SUCC}, stop as failed - - success of Inference2<${INFR2_SUCC}, stop as failed - - avg-rt of Inference1>${INFR1_RT}, stop as failed - - avg-rt of Inference2>${INFR2_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 10s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 5s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_memory_rss - condition: '>' - threshold: ${TOTAL_MEM} - timeframe: 5s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/orphans - condition: '>' - threshold: ${TOTAL_ORPHANS} - timeframe: 1s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/frontend_memory_rss - condition: '>' - threshold: ${FRNTEND_MEM} - timeframe: 5s - stop : true - fail : true + - success of Inference2<${INFR2_SUCC} for 10s, stop as failed + - avg-rt of Inference2>${INFR2_RT}, ${STOP_ALIAS} as failed + - success of Inference11<${INFR1_SUCC} for 10s, stop as failed + - success of Inference21<${INFR2_SUCC} for 10s, stop as failed + - avg-rt of Inference11>${INFR1_RT}, ${STOP_ALIAS} as failed + - avg-rt of Inference21>${INFR2_RT}, ${STOP_ALIAS} as failed + - success of ScaleUp1<${INFR2_SUCC} for 10s, stop as failed + - avg-rt of ScaleUp1>${SCALEUP1_RT}, ${STOP_ALIAS} as failed + - success of ScaleUp2<${INFR2_SUCC} for 10s, stop as failed + - avg-rt of ScaleUp2>${SCALEUP2_RT}, ${STOP_ALIAS} as failed + - success of ScaleDown1<${INFR2_SUCC} for 10s, stop as failed + - avg-rt of ScaleDown1>${SCALEDOWN1_RT}, ${STOP_ALIAS} as failed + - success of ScaleDown2<${INFR2_SUCC} for 10s, stop as failed + - avg-rt of ScaleDown2>${SCALEDOWN2_RT}, ${STOP_ALIAS} as failed + diff --git a/tests/performance/tests/register_unregister/environments/xlarge.yaml b/tests/performance/tests/register_unregister/environments/xlarge.yaml index 0e099afed..1407c6336 100644 --- a/tests/performance/tests/register_unregister/environments/xlarge.yaml +++ b/tests/performance/tests/register_unregister/environments/xlarge.yaml @@ -1,11 +1,51 @@ --- settings: env: - REG_SUCC : 100% - UNREG_SUCC: 100% - REG_RT : 15s - UNREG_RT: 10ms - TOTAL_PROCS : 1 - TOTAL_FDS : 66 + API_LABEL : RegisterModel + API_SUCCESS : 80% + API_AVG_RT : 290ms + + UNREG_SUCC: 80% + UNREG_RT: 290ms + + TOTAL_WORKERS: 1 + TOTAL_WORKERS_MEM: 14054528 + TOTAL_WORKERS_FDS: 50 + + TOTAL_MEM : 1400000000 + TOTAL_PROCS : 2 + TOTAL_FDS : 100 + + FRNTEND_MEM: 1200000000 + TOTAL_ORPHANS : 0 - FRNTEND_MEM : 75000000 #75MB \ No newline at end of file + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 220 + TOTAL_WORKERS_FDS_RUN_DIFF: 220 + TOTAL_MEM_RUN_DIFF: 150 + TOTAL_PROCS_RUN_DIFF: 70 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 140 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 1 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : register_unregister.jmx + + STOP : false #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/register_unregister/register_unregister.jmx b/tests/performance/tests/register_unregister/register_unregister.jmx index 504c3ae8c..aa420f88e 100644 --- a/tests/performance/tests/register_unregister/register_unregister.jmx +++ b/tests/performance/tests/register_unregister/register_unregister.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,13 +16,13 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model name model_url - https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar + ${__P(SQZNET_URL,https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar)} URL to model store on s3 = diff --git a/tests/performance/tests/register_unregister/register_unregister.yaml b/tests/performance/tests/register_unregister/register_unregister.yaml index 1feb22487..6892e3531 100644 --- a/tests/performance/tests/register_unregister/register_unregister.yaml +++ b/tests/performance/tests/register_unregister/register_unregister.yaml @@ -1,72 +1,11 @@ --- -execution: -- concurrency: 1 - ramp-up: 0s -# hold-for: 5h - iterations: 5 - scenario: register_unregister - scenarios: - register_unregister: + scenario_0: script: register_unregister.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - -services: - - module: shellexec - prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 10s" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - frontend_memory_rss - - orphans - reporting: - module: passfail criteria: # Inbuilt Criteria - - success of RegisterModel<${REG_SUCC}, stop as failed - - success of UnregisterModel<${UNREG_SUCC}, stop as failed - - avg-rt of RegisterModel>${REG_RT}, stop as failed - - avg-rt of UnregisterModel>${UNREG_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 5s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 5s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/orphans - condition: '>' - threshold: ${TOTAL_ORPHANS} - timeframe: 1s - stop : true - fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/frontend_memory_rss -# condition: '>' -# threshold: ${FRNTEND_MEM} -# timeframe: 5s -# stop : true -# fail : true \ No newline at end of file + - success of UnregisterModel<${UNREG_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of UnregisterModel>${UNREG_RT}, ${STOP_ALIAS} as failed diff --git a/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml b/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml index 24c07f5cf..4affbc8ef 100644 --- a/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml +++ b/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml @@ -1,12 +1,55 @@ --- settings: env: - REG_SUCC : 100% - SCL_UP_SUCC: 100% - UNREG_SUCC: 100% - REG_RT : 15s + API_LABEL : RegisterModel + API_SUCCESS : 80% + API_AVG_RT : 15s + + SCL_UP_SUCC: 80% + UNREG_SUCC: 80% SCL_UP_RT: 1.5s UNREG_RT: 18ms - TOTAL_PROCS : 2 - TOTAL_FDS : 73 - FRNTEND_MEM : 120000000 #120MB \ No newline at end of file + + TOTAL_WORKERS: 4 + TOTAL_WORKERS_MEM: 100000000 + TOTAL_WORKERS_FDS: 200 + + + TOTAL_MEM : 2000000000 + TOTAL_PROCS : 5 + TOTAL_FDS : 200 + + FRNTEND_MEM: 100000000 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 200 + TOTAL_WORKERS_MEM_RUN_DIFF: 200 + TOTAL_WORKERS_FDS_RUN_DIFF: 200 + TOTAL_MEM_RUN_DIFF: 200 + TOTAL_PROCS_RUN_DIFF: 150 + TOTAL_FDS_RUN_DIFF: 200 + FRNTEND_MEM_RUN_DIFF: 200 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + + CONCURRENCY : 1 + RAMP-UP : 1s + HOLD-FOR : 300s + SCRIPT : register_unregister_multiple.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.jmx b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.jmx index 1dac0d5fe..876e51513 100644 --- a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.jmx +++ b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.jmx @@ -1,7 +1,7 @@ - + false true @@ -16,13 +16,13 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model name model_url - https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar + ${__P(SQZNET_URL,https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar)} URL to model store on s3 = diff --git a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml index 5c8fcb85e..def6b57e3 100644 --- a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml +++ b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml @@ -1,66 +1,28 @@ --- -execution: -- concurrency: 1 - ramp-up: 0s - iterations: 5 - scenario: register_unregister_multiple - scenarios: - register_unregister_multiple: + scenario_0: script: register_unregister_multiple.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor - services: - module: shellexec prepare: - - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg" - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/resnet-18.mar" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - "rm kitten.jpg" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - sum_all_file_descriptors - - frontend_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${RESNET_URL}" + -reporting: +~reporting: +- module: passfail # this is to enable passfail module +- module: junit-xml + data-source: pass-fail +- module: junit-xml + data-source: sample-labels +- module: final-stats + dump-csv : ${ARTIFACTS_DIR}/final_stats.csv - module: passfail criteria: # Inbuilt Criteria - - success of RegisterModel<${REG_SUCC}, stop as failed - - success of ScaleUp<${SCL_UP_SUCC}, stop as failed - - success of UnregisterModel<${UNREG_SUCC}, stop as failed - - avg-rt of RegisterModel>${REG_RT}, stop as failed - - avg-rt of ScaleUp>${SCL_UP_RT}, stop as failed - - avg-rt of UnregisterModel>${UNREG_RT}, stop as failed - # Custom Criteria - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/total_processes - condition: '>' - threshold: ${TOTAL_PROCS} - timeframe: 5s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_all_file_descriptors - condition: '>' - threshold: ${TOTAL_FDS} - timeframe: 5s - stop : true - fail : true -# - class: bzt.modules.monitoring.MonitoringCriteria -# subject: ServerLocalClient/frontend_memory_rss -# condition: '>' -# threshold: ${FRNTEND_MEM} -# timeframe: 5s -# stop : true -# fail : true \ No newline at end of file + - success of ${API_LABEL}<${API_SUCCESS} for 10s, ${STOP_ALIAS} as failed + - avg-rt of ${API_LABEL}>${API_AVG_RT}, ${STOP_ALIAS} as failed + - success of ScaleUp<${SCL_UP_SUCC} for 10s, ${STOP_ALIAS} as failed + - success of UnregisterModel<${UNREG_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of ScaleUp>${SCL_UP_RT}, ${STOP_ALIAS} as failed + - avg-rt of UnregisterModel>${UNREG_RT}, ${STOP_ALIAS} as failed diff --git a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml index 6d43899b3..e02d16bc0 100644 --- a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml @@ -1,15 +1,25 @@ --- settings: env: - SCL_DWN_SUCC : 100% + SCL_DWN_SUCC : 80% SCL_DWN_RT : 10ms - TOTAL_PROCS_B4_SCL_DWN : 6 - TOTAL_PROCS_AFTR_SCL_DWN : 4 + TOTAL_PROCS_B4_SCL_DWN : 5 + TOTAL_PROCS_AFTR_SCL_DWN : 3 TOTAL_WRKRS_B4_SCL_DWN : 4 TOTAL_WRKRS_AFTR_SCL_DWN : 2 FRNTEND_FDS : 78 TOTAL_WRKRS_FDS_B4_SCL_DWN: 38 - TOTAL_WRKRS_FDS_AFTR_SCL_DWN: 23 - FRNTEND_MEM : 290000000 #290MB - TOTAL_WRKRS_MEM_B4_SCL_DWN : 450000000 #450MB - TOTAL_WRKRS_MEM_AFTR_SCL_DWN : 210000000 #210MB \ No newline at end of file + FRNTEND_MEM : 1000000000 + TOTAL_WRKRS_MEM_B4_SCL_DWN : 650000000 + TOTAL_WRKRS_MEM_AFTR_SCL_DWN : 200000000 + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + + CONCURRENCY: 10 + RAMP-UP: 1s + HOLD-FOR: 300s + SCRIPT: scale_down_workers.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/scale_down_workers/scale_down_workers.jmx b/tests/performance/tests/scale_down_workers/scale_down_workers.jmx index 512444b07..c995a47fd 100644 --- a/tests/performance/tests/scale_down_workers/scale_down_workers.jmx +++ b/tests/performance/tests/scale_down_workers/scale_down_workers.jmx @@ -1,7 +1,7 @@ - + false true @@ -22,7 +22,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model name diff --git a/tests/performance/tests/scale_down_workers/scale_down_workers.yaml b/tests/performance/tests/scale_down_workers/scale_down_workers.yaml index dc8cc1382..7e99ba2d6 100644 --- a/tests/performance/tests/scale_down_workers/scale_down_workers.yaml +++ b/tests/performance/tests/scale_down_workers/scale_down_workers.yaml @@ -1,113 +1,109 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 30s - scenario: scaledown scenarios: - scaledown: + scenario_0: script: scale_down_workers.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor services: - module: shellexec prepare: - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=4&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - total_workers - - frontend_file_descriptors - - sum_workers_file_descriptors - - frontend_memory_rss - - sum_workers_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=4&synchronous=true" + - "sleep 10s" -reporting: +~reporting: +- module: passfail # this is to enable passfail module +- module: junit-xml + data-source: pass-fail +- module: junit-xml + data-source: sample-labels +- module: final-stats + dump-csv : ${ARTIFACTS_DIR}/final_stats.csv - module: passfail criteria: # Inbuilt Criteria - - success of ScaleDown<${SCL_DWN_SUCC}, stop as failed - - avg-rt of ScaleDown>${SCL_DWN_RT}, stop as failed + - success of ScaleDown<${SCL_DWN_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of ScaleDown>${SCL_DWN_RT}, ${STOP_ALIAS} as failed # Custom Criteria - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '>' threshold: ${TOTAL_PROCS_B4_SCL_DWN} timeframe: 1s - stop : true + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS_AFTR_SCL_DWN} - timeframe: 1s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_workers condition: '>' threshold: ${TOTAL_WRKRS_B4_SCL_DWN} - timeframe: 1s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_workers condition: '<' threshold: ${TOTAL_WRKRS_AFTR_SCL_DWN} - timeframe: 1s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/frontend_file_descriptors condition: '>' threshold: ${FRNTEND_FDS} - timeframe: 5s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/sum_workers_file_descriptors condition: '>' threshold: ${TOTAL_WRKRS_FDS_B4_SCL_DWN} - timeframe: 5s - stop : true - fail : true - - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_file_descriptors - condition: '<' - threshold: ${TOTAL_WRKRS_FDS_AFTR_SCL_DWN} - timeframe: 5s - stop : true + timeframe: 10s + stop : ${STOP} fail : true # - class: bzt.modules.monitoring.MonitoringCriteria # subject: ServerLocalClient/frontend_memory_rss # condition: '>' # threshold: ${FRNTEND_MEM} -# timeframe: 5s -# stop : true +# timeframe: 10s +# stop : ${STOP} +# fail : true +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/sum_workers_memory_rss +# condition: '>' +# threshold: ${TOTAL_WRKRS_MEM_B4_SCL_DWN} +# timeframe: 10s +# stop : ${STOP} +# fail : true +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/sum_workers_memory_rss +# condition: '<' +# threshold: ${TOTAL_WRKRS_MEM_AFTR_SCL_DWN} +# timeframe: 10s +# stop : ${STOP} # fail : true - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_memory_rss + subject: ServerLocalClient/orphans condition: '>' - threshold: ${TOTAL_WRKRS_MEM_B4_SCL_DWN} - timeframe: 5s - stop : true - fail : true + threshold: ${TOTAL_ORPHANS} + timeframe: 10s + stop: ${STOP} + fail: true + diff_percent_previous: ${TOTAL_ORPHANS_DIFF} - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_memory_rss - condition: '<' - threshold: ${TOTAL_WRKRS_MEM_AFTR_SCL_DWN} - timeframe: 5s - stop : true - fail : true \ No newline at end of file + subject: ServerLocalClient/zombies + condition: '>' + threshold: ${TOTAL_ZOMBIES} + timeframe: 10s + stop: ${STOP} + fail: true + +~compare_criteria: + - \ No newline at end of file diff --git a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml index 9e3182c3e..83ef339b6 100644 --- a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml @@ -1,15 +1,25 @@ --- settings: env: - SCL_UP_SUCC : 100% + SCL_UP_SUCC : 80% SCL_UP_RT : 10ms - TOTAL_PROCS_AFTR_SCL_UP : 6 - TOTAL_PROCS_B4_SCL_UP : 3 + TOTAL_PROCS_AFTR_SCL_UP : 5 + TOTAL_PROCS_B4_SCL_UP : 2 TOTAL_WRKRS_AFTR_SCL_UP : 4 TOTAL_WRKRS_B4_SCL_UP : 1 FRNTEND_FDS : 88 TOTAL_WRKRS_FDS_AFTR_SCL_UP : 38 - TOTAL_WRKRS_FDS_B4_SCL_UP : 11 - FRNTEND_MEM : 290000000 #290MB - TOTAL_WRKRS_MEM_AFTR_SCL_UP : 450000000 #450MB - TOTAL_WRKRS_MEM_B4_SCL_UP : 115000000 #115MB \ No newline at end of file + FRNTEND_MEM : 1000000000 + TOTAL_WRKRS_MEM_AFTR_SCL_UP : 796492032 + TOTAL_WRKRS_MEM_B4_SCL_UP : 115000000 #115MB + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + + CONCURRENCY: 10 + RAMP-UP: 1s + HOLD-FOR: 300s + SCRIPT: scale_up_workers.jmx + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/scale_up_workers/scale_up_workers.jmx b/tests/performance/tests/scale_up_workers/scale_up_workers.jmx index 997547d66..d875872e8 100644 --- a/tests/performance/tests/scale_up_workers/scale_up_workers.jmx +++ b/tests/performance/tests/scale_up_workers/scale_up_workers.jmx @@ -1,7 +1,7 @@ - + false true @@ -22,7 +22,7 @@ model - squeezenet_v1.1 + ${__P(SQZNET_NAME,squeezenet1_1)} = Model name diff --git a/tests/performance/tests/scale_up_workers/scale_up_workers.yaml b/tests/performance/tests/scale_up_workers/scale_up_workers.yaml index 125aff830..051122d92 100644 --- a/tests/performance/tests/scale_up_workers/scale_up_workers.yaml +++ b/tests/performance/tests/scale_up_workers/scale_up_workers.yaml @@ -1,113 +1,109 @@ --- -execution: -- concurrency: 10 - ramp-up: 1s - hold-for: 30s - scenario: scaleup - scenarios: - scaleup: + scenario_0: script: scale_up_workers.jmx -modules: - server_local_monitoring: - class : metrics_monitoring_inproc.Monitor services: - module: shellexec prepare: - - "multi-model-server --start > /dev/null 2>&1" - - "sleep 20s" - - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" - - "curl -s -X PUT http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true" - post-process: - - "multi-model-server --stop > /dev/null 2>&1" - - module: server_local_monitoring - ServerLocalClient: - - interval: 1s - logging : True - metrics: - - total_processes - - total_workers - - frontend_file_descriptors - - sum_workers_file_descriptors - - frontend_memory_rss - - sum_workers_memory_rss + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=1&synchronous=true" + - "sleep 10s" + -reporting: +~reporting: +- module: passfail # this is to enable passfail module +- module: junit-xml + data-source: pass-fail +- module: junit-xml + data-source: sample-labels +- module: final-stats + dump-csv : ${ARTIFACTS_DIR}/final_stats.csv - module: passfail criteria: # Inbuilt Criteria - - success of ScaleUp<${SCL_UP_SUCC}, stop as failed - - avg-rt of ScaleUp>${SCL_UP_RT}, stop as failed + - success of ScaleUp<${SCL_UP_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of ScaleUp>${SCL_UP_RT}, ${STOP_ALIAS} as failed # Custom Criteria - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '>' threshold: ${TOTAL_PROCS_AFTR_SCL_UP} - timeframe: 1s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' threshold: ${TOTAL_PROCS_B4_SCL_UP} - timeframe: 1s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_workers condition: '>' threshold: ${TOTAL_WRKRS_AFTR_SCL_UP} - timeframe: 1s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_workers condition: '<' threshold: ${TOTAL_WRKRS_B4_SCL_UP} - timeframe: 1s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/frontend_file_descriptors condition: '>' threshold: ${FRNTEND_FDS} - timeframe: 5s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/sum_workers_file_descriptors condition: '>' threshold: ${TOTAL_WRKRS_FDS_AFTR_SCL_UP} - timeframe: 5s - stop : true + timeframe: 10s + stop : ${STOP} fail : true - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/sum_workers_file_descriptors condition: '<' threshold: ${TOTAL_WRKRS_FDS_B4_SCL_UP} - timeframe: 5s - stop : true + timeframe: 10s + stop : ${STOP} fail : true # - class: bzt.modules.monitoring.MonitoringCriteria # subject: ServerLocalClient/frontend_memory_rss # condition: '>' # threshold: ${FRNTEND_MEM} -# timeframe: 5s -# stop : true +# timeframe: 10s +# stop : ${STOP} +# fail : true +# - class: bzt.modules.monitoring.MonitoringCriteria +# subject: ServerLocalClient/sum_workers_memory_rss +# condition: '>' +# threshold: ${TOTAL_WRKRS_MEM_AFTR_SCL_UP} +# timeframe: 10s +# stop : ${STOP} # fail : true - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_memory_rss + subject: ServerLocalClient/orphans condition: '>' - threshold: ${TOTAL_WRKRS_MEM_AFTR_SCL_UP} - timeframe: 5s - stop : true - fail : true + threshold: ${TOTAL_ORPHANS} + timeframe: 10s + stop: ${STOP} + fail: true + diff_percent_previous: ${TOTAL_ORPHANS_DIFF} - class: bzt.modules.monitoring.MonitoringCriteria - subject: ServerLocalClient/sum_workers_memory_rss - condition: '<' - threshold: ${TOTAL_WRKRS_MEM_B4_SCL_UP} - timeframe: 5s - stop : true - fail : true \ No newline at end of file + subject: ServerLocalClient/zombiesx + condition: '>' + threshold: ${TOTAL_ZOMBIES} + timeframe: 10s + stop: ${STOP} + fail: true + +~compare_criteria: + - \ No newline at end of file diff --git a/tests/performance/utils/fs.py b/tests/performance/utils/fs.py index 2a2a73178..8ade8d5bb 100644 --- a/tests/performance/utils/fs.py +++ b/tests/performance/utils/fs.py @@ -23,7 +23,7 @@ logging.basicConfig(stream=sys.stdout, format="%(message)s", level=logging.INFO) -def get_sub_dirs(dir, exclude_list=['comp_data'], include_pattern='*', exclude_pattern=None): +def get_sub_dirs(dir, exclude_list=[], include_pattern='*', exclude_pattern=None): """Utility method to get list of folders in a directory""" dir = dir.strip() if not os.path.exists(dir): @@ -32,8 +32,16 @@ def get_sub_dirs(dir, exclude_list=['comp_data'], include_pattern='*', exclude_p raise Exception(msg) pattern_list = glob.glob(dir + "/" + include_pattern) - exclude_pattern_list = glob.glob(dir + "/" + exclude_pattern) if exclude_pattern is not None else [] - return list([x for x in os.listdir(dir) if os.path.isdir(dir + "/" + x) + exclude_pattern_list, exclude_pattern = (glob.glob(dir + "/" + exclude_pattern), exclude_pattern)\ + if exclude_pattern is not None else ([], '') + skip_pattern = "/skip*" + skip_list = glob.glob(dir + skip_pattern) + + exclude_patterns = exclude_list + exclude_patterns.extend([skip_pattern, exclude_pattern]) + logger.info("Excluding the tests with name patterns '{}'.".format("','".join(exclude_patterns))) + return sorted(list([x for x in os.listdir(dir) if os.path.isdir(dir + "/" + x) and x not in exclude_list and dir + "/" + x in pattern_list - and dir + "/" + x not in exclude_pattern_list]) + and dir + "/" + x not in exclude_pattern_list + and dir + "/" + x not in skip_list])) diff --git a/tests/performance/utils/pyshell.py b/tests/performance/utils/pyshell.py index 108cb9253..6178fb2a9 100644 --- a/tests/performance/utils/pyshell.py +++ b/tests/performance/utils/pyshell.py @@ -38,9 +38,20 @@ def run_process(cmd, wait=True): if not line: break lines.append(line) + if len(lines) > 20: + lines = lines[1:] logger.info(line) - return process.returncode, '\n'.join(lines) + process.communicate() + code = process.returncode + error_msg = "" + if code: + error_msg = "Error (error_code={}) while executing command : {}. ".format(code, cmd) + logger.info(error_msg) + error_msg += "\n\n$$$$Here are the last 20 lines of the logs." \ + " For more details refer log file.$$$$\n\n" + error_msg += '\n'.join(lines) + return code, error_msg else: process = subprocess.Popen(cmd, shell=True) return process.returncode, '' diff --git a/tests/performance/utils/timer.py b/tests/performance/utils/timer.py index 1fa47d086..8beb03c29 100644 --- a/tests/performance/utils/timer.py +++ b/tests/performance/utils/timer.py @@ -37,5 +37,9 @@ def __enter__(self): def __exit__(self, type, value, traceback): logger.info("%s: %ss", self.description, self.diff()) + # Return False needed so that __exit__ method do no ignore the exception + # otherwise exception are not reported + return False + def diff(self): return int(time.time()) - self.start From 3e638cdd0ef03c2355237ccaa9c198b4875c3ea2 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Tue, 14 Jul 2020 19:46:28 +0530 Subject: [PATCH 02/42] change S3 bucket name --- tests/performance/agents/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index f2caaebcd..aacbe97e1 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -6,4 +6,4 @@ HOST = PORT = 9009 [suite] -s3_bucket = torchserve-performance-regression-reports \ No newline at end of file +s3_bucket = mms-performance-regression-reports \ No newline at end of file From 525afacbf2ad6e1634791b552cc38a35ab15ef12 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 09:04:54 +0530 Subject: [PATCH 03/42] fix for worker processes --- tests/performance/agents/metrics/__init__.py | 22 +++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/performance/agents/metrics/__init__.py b/tests/performance/agents/metrics/__init__.py index 642976be5..b814e8c2d 100644 --- a/tests/performance/agents/metrics/__init__.py +++ b/tests/performance/agents/metrics/__init__.py @@ -155,7 +155,7 @@ def update_metric(metric_name, proc_type, stats): # Total processes result['total_processes'] = len(worker_stats) + 1 - result['total_workers'] = max(len(worker_stats), 0) + result['total_workers'] = max(len(worker_stats) -1 , 0) result['orphans'] = len(list(filter(lambda p: p['ppid'] == 1, worker_stats))) result['zombies'] = len(zombie_children) @@ -169,3 +169,23 @@ def update_metric(metric_name, proc_type, stats): result['system_write_bytes'] = system_disk_io_counters.write_bytes return result + + +if __name__ == "__main__": + import logging + import sys + from agents.utils.process import * + from agents import configuration + + logger = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout, format="%(message)s", level=logging.INFO) + + PID_FILE = configuration.get('server', 'pid_file', 'model_server.pid') + server_pid = get_process_pid_from_file(get_server_pidfile(PID_FILE)) + server_process = get_server_processes(server_pid) + children = get_child_processes(server_process) + + metrics = get_metrics(server_process, children, logger) + + + print(metrics) \ No newline at end of file From e4e976b0fdd558e7ad1bc46db44c604ab82f473b Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 09:17:54 +0530 Subject: [PATCH 04/42] jmeter property --- tests/performance/tests/global_config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/performance/tests/global_config.yaml b/tests/performance/tests/global_config.yaml index 7fd5acfc7..4f7a68bbc 100644 --- a/tests/performance/tests/global_config.yaml +++ b/tests/performance/tests/global_config.yaml @@ -21,6 +21,13 @@ modules: input_filepath: kitten.jpg # make sure jpg is available at this path # if relative path is provided this will be relative to current working directory + RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} + RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} + SQZNET_URL: ${SQZNET_URL} + SQZNET_NAME: ${SQZNET_NAME} + RESNET_URL: ${RESNET_URL} + RESNET_NAME: ${RESNET_NAME} + server_local_monitoring: # metrics_monitoring_inproc and dependencies should be in python path class : metrics_monitoring_inproc.Monitor # monitoring class. From 7bc198257d9b820fc8743a180090fa9866cdd05c Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 09:34:34 +0530 Subject: [PATCH 05/42] modular --- .../tests/scale_down_workers/environments/xlarge.yaml | 4 ++-- .../tests/scale_up_workers/environments/xlarge.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml index e02d16bc0..e30b12c7b 100644 --- a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml @@ -3,8 +3,8 @@ settings: env: SCL_DWN_SUCC : 80% SCL_DWN_RT : 10ms - TOTAL_PROCS_B4_SCL_DWN : 5 - TOTAL_PROCS_AFTR_SCL_DWN : 3 + TOTAL_PROCS_B4_SCL_DWN : 6 + TOTAL_PROCS_AFTR_SCL_DWN : 4 TOTAL_WRKRS_B4_SCL_DWN : 4 TOTAL_WRKRS_AFTR_SCL_DWN : 2 FRNTEND_FDS : 78 diff --git a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml index 83ef339b6..8ed1da50e 100644 --- a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml @@ -3,7 +3,7 @@ settings: env: SCL_UP_SUCC : 80% SCL_UP_RT : 10ms - TOTAL_PROCS_AFTR_SCL_UP : 5 + TOTAL_PROCS_AFTR_SCL_UP : 6 TOTAL_PROCS_B4_SCL_UP : 2 TOTAL_WRKRS_AFTR_SCL_UP : 4 TOTAL_WRKRS_B4_SCL_UP : 1 From 0a904f913444556df56abac0cedde13de93c5884 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 12:54:29 +0530 Subject: [PATCH 06/42] fixes --- tests/performance/runs/compare.py | 16 ++++++------ tests/performance/runs/taurus/x2junit.py | 7 ++++-- .../examples_local_criteria.yaml | 24 ++++-------------- .../examples_local_monitoring.yaml | 24 ++++-------------- .../examples_remote_criteria.yaml | 24 ++++-------------- .../examples_remote_monitoring.yaml | 23 ++++------------- .../examples_starter/examples_starter.yaml | 25 ++++--------------- tests/performance/tests/global_config.yaml | 24 ++++-------------- .../scale_up_workers/environments/xlarge.yaml | 1 + 9 files changed, 44 insertions(+), 124 deletions(-) diff --git a/tests/performance/runs/compare.py b/tests/performance/runs/compare.py index ea5d4bf6a..982ea63c4 100644 --- a/tests/performance/runs/compare.py +++ b/tests/performance/runs/compare.py @@ -44,7 +44,7 @@ def __init__(self, path, env_name, local_run, compare_with): self.storage = storage_class(self.artifacts_dir, self.env_name, compare_with) self.junit_reporter = None self.pandas_result = None - self.pass_fail = True + self.pass_fail = True def gen(self): """Driver method to get comparison directory, do the comparison of it with current run directory @@ -53,7 +53,7 @@ def gen(self): compare_dir, compare_run_name = self.storage.get_dir_to_compare() if compare_run_name: self.junit_reporter, self.pandas_result = compare_artifacts(self.storage.artifacts_dir, compare_dir, - self.storage.current_run_name, compare_run_name) + self.storage.current_run_name, compare_run_name) self.pandas_result.to_csv(os.path.join(self.artifacts_dir, "comparison_result.csv")) else: logger.warning("The latest run not found for env.") @@ -108,8 +108,8 @@ def get_centile_val(df, agg_func, col): val = None if "metric_name" in df and agg_func in df: - val = df[df["metric_name"] == col][agg_func] - val = val[0] if len(val) else None + val = df[df["metric_name"] == col][agg_func] + val = val[0] if len(val) else None return val @@ -205,7 +205,7 @@ def compare_artifacts(dir1, dir2, run_name1, run_name2): if __name__ == "__main__": compare_artifacts( - "/Users/demo/git/serve/test/performance/run_artifacts/xlarge__45b6399__1594725947", - "/Users/demo/git/serve/test/performance/run_artifacts/xlarge__45b6399__1594725717", - "xlarge__45b6399__1594725947", "xlarge__45b6399__1594725717" - ) \ No newline at end of file + "./run_artifacts/xlarge__45b6399__1594725947", + "./run_artifacts/xlarge__45b6399__1594725717", + "xlarge__45b6399__1594725947", "xlarge__45b6399__1594725717" + ) diff --git a/tests/performance/runs/taurus/x2junit.py b/tests/performance/runs/taurus/x2junit.py index a209a7a5e..7d1185e0d 100644 --- a/tests/performance/runs/taurus/x2junit.py +++ b/tests/performance/runs/taurus/x2junit.py @@ -54,6 +54,10 @@ def add_compare_tests(self): for metric_values in compare_list: col = metric_values[0] diff_percent = metric_values[2] + try: + diff_percent = float(diff_percent) + except Exception as e: + diff_percent = None tc = TestCase("{}_diff_run > {}".format(col, diff_percent)) if diff_percent is None: tc.result = Skipped("diff_percent_run value is not mentioned") @@ -204,8 +208,7 @@ def __exit__(self, type, value, traceback): if __name__ == "__main__": from utils.timer import Timer with Timer("ads") as t: - test_folder = '/Users/demo/git/serve/test/performance/'\ - 'run_artifacts/xlarge__2dc700f__1594662587/scale_down_workers' + test_folder = './run_artifacts/xlarge__7bc1982__1594795786/scale_up_workers' x = X2Junit("test", test_folder, JUnitXml(), t, "xlarge") # x.update_metrics() diff --git a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml index 3adbf0a48..dd9864ac1 100644 --- a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml +++ b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml @@ -17,31 +17,17 @@ modules: ~services: - module: shellexec prepare: - - "curl -s -O $INPUT_IMG_URL" + - "curl -s -O ${INPUT_IMG_URL}" - "mkdir /tmp/ts_model_store" - - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" post-process: - - "$SERVER_CMD --stop > /dev/null 2>&1" - - "rm $INPUT_IMG_PATH" + - "${SERVER_STOP_CMD} > /dev/null 2>&1" + - "rm ${INPUT_IMG_PATH}" - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - env: - SERVER_CMD : ${SERVER_CMD} - ARTIFACTS_DIR : ${ARTIFACTS_DIR} - SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} - INPUT_IMG_URL : ${INPUT_IMG_URL} - INPUT_IMG_PATH : ${INPUT_IMG_PATH} - - RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} - RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} - SQZNET_URL: ${SQZNET_URL} - SQZNET_NAME: ${SQZNET_NAME} - RESNET_URL: ${RESNET_URL} - RESNET_NAME: ${RESNET_NAME} - - module: server_local_monitoring # should be added in modules section ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor - interval: 1s diff --git a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml index ee8fdac7c..da615c04d 100644 --- a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml +++ b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml @@ -17,31 +17,17 @@ modules: ~services: - module: shellexec prepare: - - "curl -s -O $INPUT_IMG_URL" + - "curl -s -O ${INPUT_IMG_URL}" - "mkdir /tmp/ts_model_store" - - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" post-process: - - "$SERVER_CMD --stop > /dev/null 2>&1" - - "rm $INPUT_IMG_PATH" + - "${SERVER_STOP_CMD} > /dev/null 2>&1" + - "rm ${INPUT_IMG_PATH}" - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - env: - SERVER_CMD : ${SERVER_CMD} - ARTIFACTS_DIR : ${ARTIFACTS_DIR} - SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} - INPUT_IMG_URL : ${INPUT_IMG_URL} - INPUT_IMG_PATH : ${INPUT_IMG_PATH} - - RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} - RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} - SQZNET_URL: ${SQZNET_URL} - SQZNET_NAME: ${SQZNET_NAME} - RESNET_URL: ${RESNET_URL} - RESNET_NAME: ${RESNET_NAME} - - module: server_local_monitoring # should be added in modules section ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor - interval: 1s diff --git a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml index 10b028895..4fbbe31d0 100644 --- a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml +++ b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml @@ -12,31 +12,17 @@ ~services: - module: shellexec prepare: - - "curl -s -O $INPUT_IMG_URL" + - "curl -s -O ${INPUT_IMG_URL}" - "mkdir /tmp/ts_model_store" - - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" post-process: - - "$SERVER_CMD --stop > /dev/null 2>&1" - - "rm $INPUT_IMG_PATH" + - "${SERVER_STOP_CMD} > /dev/null 2>&1" + - "rm ${INPUT_IMG_PATH}" - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - env: - SERVER_CMD : ${SERVER_CMD} - ARTIFACTS_DIR : ${ARTIFACTS_DIR} - SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} - INPUT_IMG_URL : ${INPUT_IMG_URL} - INPUT_IMG_PATH : ${INPUT_IMG_PATH} - - RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} - RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} - SQZNET_URL: ${SQZNET_URL} - SQZNET_NAME: ${SQZNET_NAME} - RESNET_URL: ${RESNET_URL} - RESNET_NAME: ${RESNET_NAME} - - module: monitoring server-agent: - address: localhost:9009 # metric monitoring service address diff --git a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml index e3b71cf4e..407a1e6ac 100644 --- a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml +++ b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml @@ -14,30 +14,17 @@ ~services: - module: shellexec prepare: - - "curl -s -O $INPUT_IMG_URL" + - "curl -s -O ${INPUT_IMG_URL}" - "mkdir /tmp/ts_model_store" - - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" post-process: - - "$SERVER_CMD --stop > /dev/null 2>&1" - - "rm $INPUT_IMG_PATH" + - "${SERVER_STOP_CMD} > /dev/null 2>&1" + - "rm ${INPUT_IMG_PATH}" - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - env: - SERVER_CMD : ${SERVER_CMD} - ARTIFACTS_DIR : ${ARTIFACTS_DIR} - SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} - INPUT_IMG_URL : ${INPUT_IMG_URL} - INPUT_IMG_PATH : ${INPUT_IMG_PATH} - - RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} - RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} - SQZNET_URL: ${SQZNET_URL} - SQZNET_NAME: ${SQZNET_NAME} - RESNET_URL: ${RESNET_URL} - RESNET_NAME: ${RESNET_NAME} - module: monitoring server-agent: - address: localhost:9009 # metric monitoring service address diff --git a/tests/performance/tests/examples_starter/examples_starter.yaml b/tests/performance/tests/examples_starter/examples_starter.yaml index ee191afaf..5a8ebe8db 100644 --- a/tests/performance/tests/examples_starter/examples_starter.yaml +++ b/tests/performance/tests/examples_starter/examples_starter.yaml @@ -12,32 +12,17 @@ ~services: - module: shellexec prepare: - - "curl -s -O $INPUT_IMG_URL" + - "curl -s -O ${INPUT_IMG_URL}" - "mkdir /tmp/ts_model_store" - - "ps aux | grep '$SERVER_PROCESS_NAME' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - - "$SERVER_CMD --start --ncs --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" + - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" post-process: - - "$SERVER_CMD --stop > /dev/null 2>&1" - - "rm $INPUT_IMG_PATH" + - "${SERVER_STOP_CMD} > /dev/null 2>&1" + - "rm ${INPUT_IMG_PATH}" - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - env: - SERVER_CMD : ${SERVER_CMD} - ARTIFACTS_DIR : ${ARTIFACTS_DIR} - SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} - INPUT_IMG_URL : ${INPUT_IMG_URL} - INPUT_IMG_PATH : ${INPUT_IMG_PATH} - - RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} - RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} - SQZNET_URL: ${SQZNET_URL} - SQZNET_NAME: ${SQZNET_NAME} - RESNET_URL: ${RESNET_URL} - RESNET_NAME: ${RESNET_NAME} - - ~reporting: - module: passfail diff --git a/tests/performance/tests/global_config.yaml b/tests/performance/tests/global_config.yaml index 4f7a68bbc..471a16a04 100644 --- a/tests/performance/tests/global_config.yaml +++ b/tests/performance/tests/global_config.yaml @@ -38,29 +38,14 @@ services: - "curl -s -O ${INPUT_IMG_URL}" - "mkdir /tmp/ts_model_store" - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - - "${SERVER_CMD} --start --model-store /tmp/ts_model_store > /dev/null 2>&1" + - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" post-process: - - "${SERVER_CMD} --stop > /dev/null 2>&1" + - "${SERVER_STOP_CMD} > /dev/null 2>&1" - "rm ${INPUT_IMG_PATH}" - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - env: - SERVER_CMD : ${SERVER_CMD} - ARTIFACTS_DIR : ${ARTIFACTS_DIR} - SERVER_PROCESS_NAME : ${SERVER_PROCESS_NAME} - INPUT_IMG_URL : ${INPUT_IMG_URL} - INPUT_IMG_PATH : ${INPUT_IMG_PATH} - - RESNET_152_BATCH_URL: ${RESNET_152_BATCH_URL} - RESNET_152_BATCH_NAME: ${RESNET_152_BATCH_NAME} - SQZNET_URL: ${SQZNET_URL} - SQZNET_NAME: ${SQZNET_NAME} - RESNET_URL: ${RESNET_URL} - RESNET_NAME: ${RESNET_NAME} - - - module: server_local_monitoring # should be added in modules section ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor - interval: 1s @@ -197,13 +182,14 @@ compare_criteria: settings: env: ARTIFACTS_DIR : '.' - SERVER_CMD : "multi-model-server" + SERVER_START_CMD : "multi-model-server --start " + SERVER_STOP_CMD : "multi-model-server --stop " SERVER_PROCESS_NAME : "[c]om.amazonaws.ml.mms.ModelServer" INPUT_IMG_URL: "https://s3.amazonaws.com/model-server/inputs/kitten.jpg" INPUT_IMG_PATH: "kitten.jpg" RESNET_152_BATCH_URL : "https://s3.amazonaws.com/model-server/model_archive_1.0/examples/resnet-152-batching/resnet-152.mar" - RESNET_152_BATCH_NAME : "resnet-152-batch" + RESNET_152_BATCH_NAME : "resnet-152" SQZNET_URL : "https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar" SQZNET_NAME : "squeezenet_v1.1" RESNET_URL : "https://s3.amazonaws.com/model-server/model_archive_1.0/resnet-18.mar" diff --git a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml index 8ed1da50e..32004800e 100644 --- a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml @@ -14,6 +14,7 @@ settings: TOTAL_WRKRS_MEM_B4_SCL_UP : 115000000 #115MB TOTAL_ORPHANS : 0 TOTAL_ZOMBIES : 0 + TOTAL_WRKRS_FDS_B4_SCL_UP : 0 CONCURRENCY: 10 From 75a06fe08b80c346be5c365581dfdb68dd3765e6 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 14:11:40 +0530 Subject: [PATCH 07/42] pylint fixes --- tests/performance/README.md | 76 +++++++++++++++----- tests/performance/agents/metrics/__init__.py | 10 +-- tests/performance/agents/utils/process.py | 1 - tests/performance/run_performance_suite.py | 10 ++- tests/performance/runs/compare.py | 7 +- tests/performance/runs/context.py | 8 ++- tests/performance/runs/junit.py | 7 +- tests/performance/runs/storage.py | 4 +- tests/performance/runs/taurus/__init__.py | 15 ++-- tests/performance/runs/taurus/x2junit.py | 54 +++++++------- tests/performance/utils/fs.py | 10 +-- 11 files changed, 123 insertions(+), 79 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index 88fd6e0bc..8f6258682 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -78,8 +78,8 @@ values which are specific to the execution environment. This is a mandatory para 2. Collects all the tests from test-dir satisfying the pattern, excluding exclude pattern and test starting with 'skip' 3. Executes the collected tests 4. Generates artifacts in the artifacts-dir against each test case. - 5. Generate Pass Fail report for test cases - 6. Generate comparison report for specified commit id + 5. Generates Pass/Fail report for test cases + 6. Generates comparison report for specified commit id 3. Check the console logs, $artifacts-dir$//performance_results.html report, comparison_result.csv, comparison_result.html and other artifacts. @@ -130,25 +130,63 @@ tests ``` 1. global_config.yaml - - It is a master template for all_comm the test cases and is shared across all the tests. - - It contains all the common yaml sections, criteria, monitoring metrics etc. - - It also contain variables in the format ${variable} for metric thresholds and other test specific attributes. + - It is a master store for common items across all the tests. + - It contains the common sections, criteria, monitoring metrics etc. + - It also contain variables in the format ${variable} for metric thresholds and other test specific attributes. 2. environments/*.yaml - - A test case can have multiple environment files. If you have a environment dependent metrics you can create an environment - yaml file. For ex. macos_xlarge, ubuntu_xlarge etc. - - The environment file contains values for all the variables mentioned in global_config.yaml and test.yaml. - -3. test.yaml - - The test.yaml is main yaml for a test case. Note the name of the yaml should be same as the test folder. - - It inherits the master template global_config.yaml. - And it usually contains the scenario, specific pre-processing commands (if any), and special criteria (if any) applicable for that test case only. - - If you want a behavior other than defined in the master template, It is possible to override sections of global_config.yaml in the individual test case. - The global_config.yaml's top-level sections can be overridden, merged, or appended based on below rules: - 1. By default the dictionaries get merged. - 2. If the dictionary key is prepended with '~' it will get overridden. - 3. The list gets appended. + - It stores values specific to an environment. An environment reflects the underlying compute characteristics. For e.g. macos_xlarge, ubuntu_xlarge etc. + - A test case can have multiple environments. + - The environment file can override variable values defined in global_config.yaml and test.yaml. + +3. test_name.yaml + - The central file for a test case. Note the name of the yaml should be same as the test folder. + - It contains the scenario, specific pre-processing commands (optional) and special criteria (optional) relevant for the test case. + - It inherits the settings defined global_config.yaml. global_config.yaml's top-level sections can be overridden, merged, or appended based on following rules + 1. By default the test cases configurations get merged with the global configuration. + 2. If the dictionary key is pre-pended with '~', it will get overridden. + 3. The lists in yaml section gets appended. + - Below are the sample yamls to demonstrate the merging of global_config and test_name yamls. The list in "services" section in global_config will + get appended by list in 'services' section of test yaml. The 'reporting' section will get replaced by '~reporting' section from test yaml. + Refer test case for more details [tests/scale_down_workers/scale_down_workers.yaml](tests/scale_down_workers/scale_down_workers.yaml) and [global_config.yaml](tests/global_config.yaml) + + ```yaml + #global_config.yaml + + services: + - module: shellexec + prepare: + - "curl -s -O ${INPUT_IMG_URL}" + - "mkdir /tmp/ts_model_store" + + reporting: + - module: passfail + criteria: + # API requests KPI crieteria + - success of ${API_LABEL}<${API_SUCCESS} for 10s, stop as failed + - avg-rt of ${API_LABEL}>${API_AVG_RT}, ${STOP_ALIAS} as failed + + ``` + + ```yaml + #test.yaml + + services: + - module: shellexec + prepare: + - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" + + + ~reporting: + - module: passfail + criteria: + # Inbuilt Criteria + - success of ScaleDown<${SCL_DWN_SUCC} for 10s, ${STOP_ALIAS} as failed + - avg-rt of ScaleDown>${SCL_DWN_RT}, ${STOP_ALIAS} as failed + ``` + 4. test.jmx + - The JMeter test scenario file. The test.yaml runs the scenario mentioned in the .jmx file.4. test.jmx - The JMeter test scenario file. The test.yaml runs the scenarion mentioned in the .jmx file. @@ -362,7 +400,7 @@ There are two types of compare criteria you can add for metrics: This criteria is used to check the percent difference between first and last value of the metric for a run. In other words it is used to verify if metrics values are same before and after the scenario run. 2. diff_percent_previous - Compare the metric aggregate values with previous run. Here we take aggregate min, max and avg of metric values for current run + Compare first and last values of previous run of compare_with commit_id. Here we compare first and last value of metrics for current run and previous run and check if percentage difference is not greater than diff_percent_previous. Note formula for percentage difference is abs(value1 - value2)/((value1 + value2)/2) * 100 diff --git a/tests/performance/agents/metrics/__init__.py b/tests/performance/agents/metrics/__init__.py index b814e8c2d..8ecd97ab0 100644 --- a/tests/performance/agents/metrics/__init__.py +++ b/tests/performance/agents/metrics/__init__.py @@ -10,6 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. +# pylint: disable=redefined-builtin, redefined-outer-name, broad-except, unused-variable from enum import Enum from statistics import mean @@ -118,8 +119,9 @@ def update_metric(metric_name, proc_type, stats): try: # as_dict() gets all stats in one shot processes_stats.append({'type': ProcessType.FRONTEND, 'stats': server_process.as_dict()}) - except: + except Exception as e: pass + for child in children | zombie_children: try: child_cmdline = child.cmdline() @@ -140,7 +142,6 @@ def update_metric(metric_name, proc_type, stats): if p in zombie_children: zombie_children.remove(p) - ### PROCESS METRICS ### worker_stats = list(map(lambda x: x['stats'], \ filter(lambda x: x['type'] == ProcessType.WORKER, processes_stats))) @@ -155,7 +156,7 @@ def update_metric(metric_name, proc_type, stats): # Total processes result['total_processes'] = len(worker_stats) + 1 - result['total_workers'] = max(len(worker_stats) -1 , 0) + result['total_workers'] = max(len(worker_stats) - 1, 0) result['orphans'] = len(list(filter(lambda p: p['ppid'] == 1, worker_stats))) result['zombies'] = len(zombie_children) @@ -187,5 +188,4 @@ def update_metric(metric_name, proc_type, stats): metrics = get_metrics(server_process, children, logger) - - print(metrics) \ No newline at end of file + print(metrics) diff --git a/tests/performance/agents/utils/process.py b/tests/performance/agents/utils/process.py index c9a8e98de..02cf5d528 100644 --- a/tests/performance/agents/utils/process.py +++ b/tests/performance/agents/utils/process.py @@ -17,7 +17,6 @@ import os import tempfile - import psutil diff --git a/tests/performance/run_performance_suite.py b/tests/performance/run_performance_suite.py index 66bc41713..7f4b89af6 100755 --- a/tests/performance/run_performance_suite.py +++ b/tests/performance/run_performance_suite.py @@ -1,5 +1,3 @@ - - # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. # Licensed under the Apache License, Version 2.0 (the "License"). # You may not use this file except in compliance with the License. @@ -13,20 +11,20 @@ """ Run Performance Regression Test Cases and Generate Reports """ -# pylint: disable=redefined-builtin, no-value-for-parameter +# pylint: disable=redefined-builtin, no-value-for-parameter, unused-argument import logging import os import subprocess import sys import time +import pathlib import click -import pathlib -from runs.context import ExecutionEnv -from runs.taurus import get_taurus_options, x2junit, update_taurus_metric_files from tqdm import tqdm +from runs.context import ExecutionEnv +from runs.taurus import get_taurus_options, x2junit, update_taurus_metric_files from utils import run_process, Timer, get_sub_dirs logger = logging.getLogger(__name__) diff --git a/tests/performance/runs/compare.py b/tests/performance/runs/compare.py index 982ea63c4..a9ae49b0e 100644 --- a/tests/performance/runs/compare.py +++ b/tests/performance/runs/compare.py @@ -15,9 +15,6 @@ """ # pylint: disable=redefined-builtin, self-assigning-variable, broad-except - -import csv -import glob import logging import sys import os @@ -34,6 +31,7 @@ class CompareReportGenerator(): + """Wrapper class to generate the compare report""" def __init__(self, path, env_name, local_run, compare_with): self.artifacts_dir = path @@ -182,8 +180,9 @@ def compare_artifacts(dir1, dir2, run_name1, run_name2): for agg_func in aggregates: name = "{}_{}".format(agg_func, str(col)) + val1 = get_centile_val(metrics_from_file1, agg_func, col) val2 = get_centile_val(metrics_from_file2, agg_func, col) - val1 = get_centile_val(metrics_from_file2, agg_func, col) + diff, pass_fail, msg = compare_values(val1, val2, diff_percent, run_name1, run_name2) if over_all_pass: diff --git a/tests/performance/runs/context.py b/tests/performance/runs/context.py index 860b35454..c90d83dc6 100644 --- a/tests/performance/runs/context.py +++ b/tests/performance/runs/context.py @@ -35,7 +35,7 @@ def get_git_commit_id(compare_with): return subprocess.check_output('git rev-parse --short {}'.format(compare_with).split()).decode( - "utf-8")[:-1] + "utf-8")[:-1] class ExecutionEnv(object): @@ -52,7 +52,8 @@ def __init__(self, agent, artifacts_dir, env, local_run, compare_with, use=True, self.compare_with = get_git_commit_id(compare_with) self.check_model_server_status = check_model_server_status self.reporter = JUnitXml() - self.compare_reporter_generator = CompareReportGenerator(self.artifacts_dir, self.env, self.local_run, compare_with) + self.compare_reporter_generator = CompareReportGenerator(self.artifacts_dir, self.env, self.local_run, + compare_with) self.exit_code = 1 def __enter__(self): @@ -64,12 +65,14 @@ def __enter__(self): @staticmethod def open_report(file_path): + """Open html report in browser """ if os.path.exists(file_path): return webbrowser.open_new_tab('file://' + os.path.realpath(file_path)) return False @staticmethod def report_summary(reporter, suite_name): + """Create a report summary """ if reporter and os.path.exists(reporter.junit_html_path): status = reporter.junit_xml.errors or reporter.junit_xml.failures status, code, color = ("failed", 3, "red") if status else ("passed", 0, "green") @@ -108,4 +111,3 @@ def __exit__(self, type, value, traceback): # Return True needed so that __exit__ method do no ignore the exception # otherwise exception are not reported return False - diff --git a/tests/performance/runs/junit.py b/tests/performance/runs/junit.py index 8ff8f1951..fb3c41c41 100644 --- a/tests/performance/runs/junit.py +++ b/tests/performance/runs/junit.py @@ -19,13 +19,16 @@ import html import textwrap import tabulate -from utils import run_process from junitparser import JUnitXml +from utils import run_process + + header = ["suite_name", "test_case", "result", "message"] class JunitConverter(): + """Convert JUnit XML object to XML and HTML report""" def __init__(self, junit_xml, out_dir, report_name): self.junit_xml = junit_xml @@ -50,7 +53,7 @@ def pretty_text(data): def junit2array(junit_xml): """convert junit xml junitparser.JUnitXml object to 2d array """ rows = [header] - for i, suite in enumerate(junit_xml): + for _, suite in enumerate(junit_xml): if len(suite) == 0: rows.append([suite.name, "", "skipped", "No criteria specified or there is an error."]) diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 1f7e0c421..648687b9c 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -20,11 +20,11 @@ import os import sys import shutil +import pathlib import boto3 -import pathlib -from agents import configuration +from agents import configuration from utils import run_process logger = logging.getLogger(__name__) diff --git a/tests/performance/runs/taurus/__init__.py b/tests/performance/runs/taurus/__init__.py index 973acf64d..d2dfd5012 100644 --- a/tests/performance/runs/taurus/__init__.py +++ b/tests/performance/runs/taurus/__init__.py @@ -21,8 +21,9 @@ import sys import logging -from .reader import get_mon_metrics_list from utils.pyshell import run_process +from .reader import get_mon_metrics_list + logger = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout, format="%(message)s", level=logging.INFO) @@ -64,20 +65,18 @@ def update_taurus_metric_files(suite_artifacts_dir, test_file): os.rename(metrics_log_file[0], metrics_new_file) else: - metrics_log_file = os.path.join(suite_artifacts_dir, "local_monitoring_logs.csv") - if os.path.exists(metrics_log_file): - os.rename(metrics_log_file, metrics_new_file) + metrics_log_file = os.path.join(suite_artifacts_dir, "local_monitoring_logs.csv") + if os.path.exists(metrics_log_file): + os.rename(metrics_log_file, metrics_new_file) KEEP_LINES = 10000 def handle_big_files(name): report_file = os.path.join(suite_artifacts_dir, name) report_tmp_file = os.path.join(suite_artifacts_dir, "{}_tmp".format(name)) - if os.path.exists(report_file) and os.stat(report_file).st_size > 1e+7: #10MB - logger.info("Keeping first {} records from file {} as it is >10MB".format(KEEP_LINES, report_file)) + if os.path.exists(report_file) and os.stat(report_file).st_size > 1e+7: # 10MB + logger.info("Keeping first %s records from file %s as it is >10MB", KEEP_LINES, report_file) run_process("head -{0} {1} > {2}; mv {2} {1};".format(KEEP_LINES, report_file, report_tmp_file)) handle_big_files("error.jtl") handle_big_files("kpi.jtl") - - diff --git a/tests/performance/runs/taurus/x2junit.py b/tests/performance/runs/taurus/x2junit.py index 7d1185e0d..37227870b 100644 --- a/tests/performance/runs/taurus/x2junit.py +++ b/tests/performance/runs/taurus/x2junit.py @@ -13,24 +13,26 @@ """ Convert the Taurus Test suite XML to Junit XML """ -# pylint: disable=redefined-builtin +# pylint: disable=redefined-builtin, unused-variable, broad-except import os -import pandas as pd -from runs.taurus.reader import get_compare_metric_list - import html + +import pandas as pd import tabulate from bzt.modules.passfail import DataCriterion from junitparser import TestCase, TestSuite, JUnitXml, Skipped, Error, Failure +from runs.taurus.reader import get_compare_metric_list + class X2Junit(object): """ Context Manager class to do convert Taurus Test suite XML report which is in Xunit specifications to JUnit XML report. """ + def __init__(self, name, artifacts_dir, junit_xml, timer, env_name): self.ts = TestSuite(name) self.name = name @@ -50,6 +52,9 @@ def __enter__(self): return self def add_compare_tests(self): + """Add compare test for a run. + Compare actual percentage difference between fist value and last value against provided difference.""" + compare_list = get_compare_metric_list(self.artifacts_dir, "") for metric_values in compare_list: col = metric_values[0] @@ -97,6 +102,7 @@ def add_compare_tests(self): @staticmethod def casename_to_criteria(test_name): + """Extract metric from Taurus pass/fail criteria string""" metric = None if ' of ' not in test_name: test_name = "label of {}".format(test_name) @@ -112,6 +118,7 @@ def casename_to_criteria(test_name): return metric def percentile_values(self, metric_name): + """Calculate percentile values for metric_name column in self.metrics pandas df""" values = {} if self.metrics is not None and metric_name is not None: metric_vals = getattr(self.metrics, metric_name, None) @@ -119,11 +126,12 @@ def percentile_values(self, metric_name): centile_values = [0, 0.5, 0.9, 0.95, 0.99, 0.999, 1] for centile in centile_values: val = getattr(metric_vals, 'quantile')(centile) - values.update({str(centile * 100)+"%": val}) + values.update({str(centile * 100) + "%": val}) return values def update_metrics(self): + """ Update self.mertics and self.metrics_agg_dict""" metrics_file = os.path.join(self.artifacts_dir, "metrics.csv") rows = [] agg_dict = {} @@ -134,24 +142,24 @@ def update_metrics(self): header_names.extend([str(colname * 100) + "%" for colname in centile_values]) header_names.extend(['first_value', 'last_value']) if self.metrics.size: - for col in self.metrics.columns: - row = [self.name, str(col)] - metric_vals = getattr(self.metrics, str(col), None) - for centile in centile_values: - row.append(getattr(metric_vals, 'quantile')(centile)) - row.extend([metric_vals.iloc[0], metric_vals.iloc[-1]]) - agg_dict.update({row[0]: dict(zip(header_names, row[1:]))}) - rows.append(row) - - dataframe = pd.DataFrame(rows, columns=header_names) - print("Metric percentile values:\n") - print(tabulate.tabulate(rows, headers=header_names, tablefmt="grid")) - dataframe.to_csv(os.path.join(self.artifacts_dir, "metrics_agg.csv"), index=False) + for col in self.metrics.columns: + row = [self.name, str(col)] + metric_vals = getattr(self.metrics, str(col), None) + for centile in centile_values: + row.append(getattr(metric_vals, 'quantile')(centile)) + row.extend([metric_vals.iloc[0], metric_vals.iloc[-1]]) + agg_dict.update({row[0]: dict(zip(header_names, row[1:]))}) + rows.append(row) + + dataframe = pd.DataFrame(rows, columns=header_names) + print("Metric percentile values:\n") + print(tabulate.tabulate(rows, headers=header_names, tablefmt="grid")) + dataframe.to_csv(os.path.join(self.artifacts_dir, "metrics_agg.csv"), index=False) self.metrics_agg_dict = agg_dict def __exit__(self, type, value, traceback): - print("error code is "+str(self.code)) + print("error code is " + str(self.code)) self.update_metrics() xunit_file = os.path.join(self.artifacts_dir, "xunit.xml") @@ -205,19 +213,17 @@ def __exit__(self, type, value, traceback): # otherwise exception are not reported return False + if __name__ == "__main__": from utils.timer import Timer + with Timer("ads") as t: test_folder = './run_artifacts/xlarge__7bc1982__1594795786/scale_up_workers' x = X2Junit("test", test_folder, JUnitXml(), t, "xlarge") # x.update_metrics() - # # x.add_compare_tests() x.__exit__(None, None, None) - x.ts - + print(x.ts) print("a") - - diff --git a/tests/performance/utils/fs.py b/tests/performance/utils/fs.py index 8ade8d5bb..eb0cce149 100644 --- a/tests/performance/utils/fs.py +++ b/tests/performance/utils/fs.py @@ -32,7 +32,7 @@ def get_sub_dirs(dir, exclude_list=[], include_pattern='*', exclude_pattern=None raise Exception(msg) pattern_list = glob.glob(dir + "/" + include_pattern) - exclude_pattern_list, exclude_pattern = (glob.glob(dir + "/" + exclude_pattern), exclude_pattern)\ + exclude_pattern_list, exclude_pattern = (glob.glob(dir + "/" + exclude_pattern), exclude_pattern) \ if exclude_pattern is not None else ([], '') skip_pattern = "/skip*" skip_list = glob.glob(dir + skip_pattern) @@ -41,7 +41,7 @@ def get_sub_dirs(dir, exclude_list=[], include_pattern='*', exclude_pattern=None exclude_patterns.extend([skip_pattern, exclude_pattern]) logger.info("Excluding the tests with name patterns '{}'.".format("','".join(exclude_patterns))) return sorted(list([x for x in os.listdir(dir) if os.path.isdir(dir + "/" + x) - and x not in exclude_list - and dir + "/" + x in pattern_list - and dir + "/" + x not in exclude_pattern_list - and dir + "/" + x not in skip_list])) + and x not in exclude_list + and dir + "/" + x in pattern_list + and dir + "/" + x not in exclude_pattern_list + and dir + "/" + x not in skip_list])) From fd211555593b47e5483f03bbea13c7b31541cfce Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 14:31:28 +0530 Subject: [PATCH 08/42] modular --- tests/performance/runs/compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/performance/runs/compare.py b/tests/performance/runs/compare.py index a9ae49b0e..01885fd8c 100644 --- a/tests/performance/runs/compare.py +++ b/tests/performance/runs/compare.py @@ -54,7 +54,8 @@ def gen(self): self.storage.current_run_name, compare_run_name) self.pandas_result.to_csv(os.path.join(self.artifacts_dir, "comparison_result.csv")) else: - logger.warning("The latest run not found for env.") + logger.info("The latest run for comparison was not found for env='%s' and commit_id='%s'.", + self.env_name, self.comare_with) self.storage.store_results() return self.junit_reporter From 84299b58c3c6369b0198f8a7b1b463d2a185f6ee Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 14:32:01 +0530 Subject: [PATCH 09/42] compare_with logging --- tests/performance/runs/context.py | 8 ++++++-- tests/performance/runs/storage.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/performance/runs/context.py b/tests/performance/runs/context.py index c90d83dc6..ef0051061 100644 --- a/tests/performance/runs/context.py +++ b/tests/performance/runs/context.py @@ -34,8 +34,12 @@ def get_git_commit_id(compare_with): - return subprocess.check_output('git rev-parse --short {}'.format(compare_with).split()).decode( - "utf-8")[:-1] + """Get short commit id for compare_with commit, branch, tag""" + cmd = 'git rev-parse --short {}'.format(compare_with) + logger.info("Running command: %s", cmd) + commit_id = subprocess.check_output(cmd.split()).decode("utf-8")[:-1] + logger.info("Commit id for compare_with='%s' is '%s'", compare_with, commit_id) + return commit_id class ExecutionEnv(object): diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 648687b9c..6098cb4b6 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -100,7 +100,7 @@ def get_dir_to_compare(self): latest_run = self.get_latest(run_names, self.env_name, self.current_run_name, self.compare_with) if not latest_run: - logger.info("No run found for env_id %s", self.env_name) + logger.info("No run artifacts folder found for env_id %s", self.env_name) return '', '' if not os.path.exists(comp_data_path): From 32e994989f6d1cdbc4aef315c0504dddcb9bf7d7 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 17:04:52 +0530 Subject: [PATCH 10/42] Fix on fail actual values --- tests/performance/runs/taurus/x2junit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/runs/taurus/x2junit.py b/tests/performance/runs/taurus/x2junit.py index 37227870b..d8e5c864b 100644 --- a/tests/performance/runs/taurus/x2junit.py +++ b/tests/performance/runs/taurus/x2junit.py @@ -148,7 +148,7 @@ def update_metrics(self): for centile in centile_values: row.append(getattr(metric_vals, 'quantile')(centile)) row.extend([metric_vals.iloc[0], metric_vals.iloc[-1]]) - agg_dict.update({row[0]: dict(zip(header_names, row[1:]))}) + agg_dict.update({row[1]: dict(zip(header_names[2:], row[2:]))}) rows.append(row) dataframe = pd.DataFrame(rows, columns=header_names) From f3860381f723eba06a43b8ba565258c4a99dbc90 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 18:52:48 +0530 Subject: [PATCH 11/42] compare_with fix --- tests/performance/runs/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/runs/context.py b/tests/performance/runs/context.py index ef0051061..d1d98383f 100644 --- a/tests/performance/runs/context.py +++ b/tests/performance/runs/context.py @@ -57,7 +57,7 @@ def __init__(self, agent, artifacts_dir, env, local_run, compare_with, use=True, self.check_model_server_status = check_model_server_status self.reporter = JUnitXml() self.compare_reporter_generator = CompareReportGenerator(self.artifacts_dir, self.env, self.local_run, - compare_with) + self.compare_with) self.exit_code = 1 def __enter__(self): From 5c35d9808518e8a5e1d22ce7c2fb2b59e1df0217 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 19:00:53 +0530 Subject: [PATCH 12/42] hold-for api description --- .../performance/tests/api_description/environments/xlarge.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/tests/api_description/environments/xlarge.yaml b/tests/performance/tests/api_description/environments/xlarge.yaml index be4e38930..24e9b4dc4 100644 --- a/tests/performance/tests/api_description/environments/xlarge.yaml +++ b/tests/performance/tests/api_description/environments/xlarge.yaml @@ -45,7 +45,7 @@ settings: CONCURRENCY : 10 RAMP-UP : 1s - HOLD-FOR : 30s + HOLD-FOR : 300s SCRIPT : api_description.jmx STOP : '' From e8b58ce5238b21f3b325a0aeef2bf724210baeaa Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 19:32:01 +0530 Subject: [PATCH 13/42] fix for compare --- tests/performance/runs/compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/performance/runs/compare.py b/tests/performance/runs/compare.py index 01885fd8c..28f9068c6 100644 --- a/tests/performance/runs/compare.py +++ b/tests/performance/runs/compare.py @@ -108,7 +108,7 @@ def get_centile_val(df, agg_func, col): val = None if "metric_name" in df and agg_func in df: val = df[df["metric_name"] == col][agg_func] - val = val[0] if len(val) else None + val = val.iloc[0] if len(val) >= 1 else None return val @@ -205,7 +205,7 @@ def compare_artifacts(dir1, dir2, run_name1, run_name2): if __name__ == "__main__": compare_artifacts( - "./run_artifacts/xlarge__45b6399__1594725947", - "./run_artifacts/xlarge__45b6399__1594725717", - "xlarge__45b6399__1594725947", "xlarge__45b6399__1594725717" + "./run_artifacts/xlarge__5c35d98__1594819866", + "./run_artifacts/xlarge__f386038__1594819700", + "xlarge__5c35d98__1594819866", "xlarge__f386038__1594819700" ) From 7428eac398939c8dbf398202ee5e81a5022ee6ce Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 20:50:06 +0530 Subject: [PATCH 14/42] handle error in compare report --- tests/performance/runs/context.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/performance/runs/context.py b/tests/performance/runs/context.py index d1d98383f..edd8d1a1b 100644 --- a/tests/performance/runs/context.py +++ b/tests/performance/runs/context.py @@ -101,11 +101,14 @@ def __exit__(self, type, value, traceback): junit_reporter = JunitConverter(self.reporter, self.artifacts_dir, 'performance_results') junit_reporter.generate_junit_report() - junit_compare = self.compare_reporter_generator.gen() junit_compare_reporter = None - if junit_compare: - junit_compare_reporter = JunitConverter(junit_compare, self.artifacts_dir, 'comparison_results') - junit_compare_reporter.generate_junit_report() + try: + junit_compare = self.compare_reporter_generator.gen() + if junit_compare: + junit_compare_reporter = JunitConverter(junit_compare, self.artifacts_dir, 'comparison_results') + junit_compare_reporter.generate_junit_report() + except Exception as e: + logger.info("Exception has occured while comparing results", exc_info=1) compare_exit_code = ExecutionEnv.report_summary(junit_compare_reporter, "Comparison Test suite") exit_code = ExecutionEnv.report_summary(junit_reporter, "Performance Regression Test suite") From 0b133f2375ff4745136e6d319b0c7bfc6535f7c8 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 20:52:16 +0530 Subject: [PATCH 15/42] pylint --- tests/performance/runs/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/runs/context.py b/tests/performance/runs/context.py index edd8d1a1b..35905ba9b 100644 --- a/tests/performance/runs/context.py +++ b/tests/performance/runs/context.py @@ -13,7 +13,7 @@ """ Start and stop monitoring server """ -# pylint: disable=redefined-builtin +# pylint: disable=redefined-builtin, broad-except import logging import os From a4fbeb09edcb846c89fb3eb197909389b0e383ad Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Wed, 15 Jul 2020 22:18:17 +0530 Subject: [PATCH 16/42] modular --- tests/performance/runs/taurus/x2junit.py | 2 +- .../tests/api_description/api_description.yaml | 2 +- .../tests/api_description/environments/xlarge.yaml | 2 +- .../batch_and_single_inference.yaml | 2 +- .../examples_local_criteria.yaml | 2 +- .../examples_local_monitoring.yaml | 2 +- .../examples_remote_criteria.yaml | 2 +- .../examples_remote_monitoring.yaml | 2 +- .../tests/examples_starter/examples_starter.yaml | 2 +- tests/performance/tests/global_config.yaml | 4 ++-- .../inference_multiple_models.yaml | 2 +- .../multiple_inference_and_scaling.yaml | 14 +++++++------- .../register_unregister/register_unregister.yaml | 2 +- .../register_unregister_multiple.yaml | 6 +++--- .../scale_down_workers/environments/xlarge.yaml | 2 +- .../scale_down_workers/scale_down_workers.yaml | 2 +- .../scale_up_workers/environments/xlarge.yaml | 2 +- .../tests/scale_up_workers/scale_up_workers.yaml | 2 +- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/performance/runs/taurus/x2junit.py b/tests/performance/runs/taurus/x2junit.py index d8e5c864b..71320a46d 100644 --- a/tests/performance/runs/taurus/x2junit.py +++ b/tests/performance/runs/taurus/x2junit.py @@ -163,7 +163,7 @@ def __exit__(self, type, value, traceback): self.update_metrics() xunit_file = os.path.join(self.artifacts_dir, "xunit.xml") - if self.code == 1: + if self.code not in [0, 3]: # 0-no error, 3-pass/fail tc = TestCase(self.name) tc.result = Error(self.err) self.ts.add_testcase(tc) diff --git a/tests/performance/tests/api_description/api_description.yaml b/tests/performance/tests/api_description/api_description.yaml index bb2ec1e88..873104ba6 100644 --- a/tests/performance/tests/api_description/api_description.yaml +++ b/tests/performance/tests/api_description/api_description.yaml @@ -4,7 +4,7 @@ reporting: criteria: # Inbuilt Criteria - success of ManagementAPIDescription<${MGMT_DESC_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ManagementAPIDescription>${MGMT_DESC_AVG_RT}, ${STOP_ALIAS} as failed + - avg-rt of ManagementAPIDescription>${MGMT_DESC_AVG_RT} for 10s, ${STOP_ALIAS} as failed # # Custom Criteria # - class: bzt.modules.monitoring.MonitoringCriteria # subject: ServerLocalClient/total_processes diff --git a/tests/performance/tests/api_description/environments/xlarge.yaml b/tests/performance/tests/api_description/environments/xlarge.yaml index 24e9b4dc4..be4e38930 100644 --- a/tests/performance/tests/api_description/environments/xlarge.yaml +++ b/tests/performance/tests/api_description/environments/xlarge.yaml @@ -45,7 +45,7 @@ settings: CONCURRENCY : 10 RAMP-UP : 1s - HOLD-FOR : 300s + HOLD-FOR : 30s SCRIPT : api_description.jmx STOP : '' diff --git a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml index 54d6b8475..60373dcde 100644 --- a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml +++ b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml @@ -19,4 +19,4 @@ reporting: criteria: # Inbuilt Criteria - success of ManagementAPIDescription<${INF2_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ManagementAPIDescription>${INF2_AVG_RT}, ${STOP_ALIAS} as failed \ No newline at end of file + - avg-rt of ManagementAPIDescription>${INF2_AVG_RT} for 10s, ${STOP_ALIAS} as failed \ No newline at end of file diff --git a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml index dd9864ac1..8234a8dd7 100644 --- a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml +++ b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml @@ -18,7 +18,7 @@ modules: - module: shellexec prepare: - "curl -s -O ${INPUT_IMG_URL}" - - "mkdir /tmp/ts_model_store" + - "mkdir -p /tmp/ts_model_store" - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" diff --git a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml index da615c04d..86ac99ac9 100644 --- a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml +++ b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml @@ -18,7 +18,7 @@ modules: - module: shellexec prepare: - "curl -s -O ${INPUT_IMG_URL}" - - "mkdir /tmp/ts_model_store" + - "mkdir -p /tmp/ts_model_store" - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" diff --git a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml index 4fbbe31d0..384f6bf2b 100644 --- a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml +++ b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml @@ -13,7 +13,7 @@ - module: shellexec prepare: - "curl -s -O ${INPUT_IMG_URL}" - - "mkdir /tmp/ts_model_store" + - "mkdir -p /tmp/ts_model_store" - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" diff --git a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml index 407a1e6ac..f899ec98f 100644 --- a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml +++ b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml @@ -15,7 +15,7 @@ - module: shellexec prepare: - "curl -s -O ${INPUT_IMG_URL}" - - "mkdir /tmp/ts_model_store" + - "mkdir -p /tmp/ts_model_store" - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" diff --git a/tests/performance/tests/examples_starter/examples_starter.yaml b/tests/performance/tests/examples_starter/examples_starter.yaml index 5a8ebe8db..a94ccc5c8 100644 --- a/tests/performance/tests/examples_starter/examples_starter.yaml +++ b/tests/performance/tests/examples_starter/examples_starter.yaml @@ -13,7 +13,7 @@ - module: shellexec prepare: - "curl -s -O ${INPUT_IMG_URL}" - - "mkdir /tmp/ts_model_store" + - "mkdir -p /tmp/ts_model_store" - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" diff --git a/tests/performance/tests/global_config.yaml b/tests/performance/tests/global_config.yaml index 471a16a04..c88e1a2e4 100644 --- a/tests/performance/tests/global_config.yaml +++ b/tests/performance/tests/global_config.yaml @@ -36,7 +36,7 @@ services: - module: shellexec prepare: - "curl -s -O ${INPUT_IMG_URL}" - - "mkdir /tmp/ts_model_store" + - "mkdir -p /tmp/ts_model_store" - "ps aux | grep '${SERVER_PROCESS_NAME}' | awk '{print $2}' | xargs kill -9 2> /dev/null || true" - "${SERVER_START_CMD} --model-store /tmp/ts_model_store > /dev/null 2>&1" - "sleep 20s" @@ -74,7 +74,7 @@ reporting: criteria: # API requests KPI crieteria - success of ${API_LABEL}<${API_SUCCESS} for 10s, stop as failed - - avg-rt of ${API_LABEL}>${API_AVG_RT}, ${STOP_ALIAS} as failed + - avg-rt of ${API_LABEL}>${API_AVG_RT} for 10s, ${STOP_ALIAS} as failed # # # Monitoring metrics criteria # - class: bzt.modules.monitoring.MonitoringCriteria diff --git a/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml b/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml index 047d4d168..92a8d155c 100644 --- a/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml +++ b/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml @@ -17,7 +17,7 @@ reporting: criteria: # Inbuilt Criteria - success of Inference2<${INFR2_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of Inference2>${INFR2_RT}, ${STOP_ALIAS} as failed + - avg-rt of Inference2>${INFR2_RT} for 10s, ${STOP_ALIAS} as failed - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' diff --git a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml index ce5c9725b..d5a38242b 100644 --- a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml +++ b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml @@ -17,17 +17,17 @@ reporting: criteria: # Inbuilt Criteria - success of Inference2<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of Inference2>${INFR2_RT}, ${STOP_ALIAS} as failed + - avg-rt of Inference2>${INFR2_RT} for 10s, ${STOP_ALIAS} as failed - success of Inference11<${INFR1_SUCC} for 10s, stop as failed - success of Inference21<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of Inference11>${INFR1_RT}, ${STOP_ALIAS} as failed - - avg-rt of Inference21>${INFR2_RT}, ${STOP_ALIAS} as failed + - avg-rt of Inference11>${INFR1_RT} for 10s, ${STOP_ALIAS} as failed + - avg-rt of Inference21>${INFR2_RT} for 10s, ${STOP_ALIAS} as failed - success of ScaleUp1<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleUp1>${SCALEUP1_RT}, ${STOP_ALIAS} as failed + - avg-rt of ScaleUp1>${SCALEUP1_RT} for 10s, ${STOP_ALIAS} as failed - success of ScaleUp2<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleUp2>${SCALEUP2_RT}, ${STOP_ALIAS} as failed + - avg-rt of ScaleUp2>${SCALEUP2_RT} for 10s, ${STOP_ALIAS} as failed - success of ScaleDown1<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleDown1>${SCALEDOWN1_RT}, ${STOP_ALIAS} as failed + - avg-rt of ScaleDown1>${SCALEDOWN1_RT} for 10s, ${STOP_ALIAS} as failed - success of ScaleDown2<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleDown2>${SCALEDOWN2_RT}, ${STOP_ALIAS} as failed + - avg-rt of ScaleDown2>${SCALEDOWN2_RT} for 10s, ${STOP_ALIAS} as failed diff --git a/tests/performance/tests/register_unregister/register_unregister.yaml b/tests/performance/tests/register_unregister/register_unregister.yaml index 6892e3531..35778217d 100644 --- a/tests/performance/tests/register_unregister/register_unregister.yaml +++ b/tests/performance/tests/register_unregister/register_unregister.yaml @@ -8,4 +8,4 @@ reporting: criteria: # Inbuilt Criteria - success of UnregisterModel<${UNREG_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of UnregisterModel>${UNREG_RT}, ${STOP_ALIAS} as failed + - avg-rt of UnregisterModel>${UNREG_RT} for 10s, ${STOP_ALIAS} as failed diff --git a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml index def6b57e3..b87aa88a9 100644 --- a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml +++ b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml @@ -21,8 +21,8 @@ services: criteria: # Inbuilt Criteria - success of ${API_LABEL}<${API_SUCCESS} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ${API_LABEL}>${API_AVG_RT}, ${STOP_ALIAS} as failed + - avg-rt of ${API_LABEL}>${API_AVG_RT} for 10s, ${STOP_ALIAS} as failed - success of ScaleUp<${SCL_UP_SUCC} for 10s, ${STOP_ALIAS} as failed - success of UnregisterModel<${UNREG_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ScaleUp>${SCL_UP_RT}, ${STOP_ALIAS} as failed - - avg-rt of UnregisterModel>${UNREG_RT}, ${STOP_ALIAS} as failed + - avg-rt of ScaleUp>${SCL_UP_RT} for 10s, ${STOP_ALIAS} as failed + - avg-rt of UnregisterModel>${UNREG_RT} for 10s, ${STOP_ALIAS} as failed diff --git a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml index e30b12c7b..2a0c9101a 100644 --- a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml @@ -18,7 +18,7 @@ settings: CONCURRENCY: 10 RAMP-UP: 1s - HOLD-FOR: 300s + HOLD-FOR: 30s SCRIPT: scale_down_workers.jmx STOP : '' #possible values true, false. Bug in bzt so for false use '' diff --git a/tests/performance/tests/scale_down_workers/scale_down_workers.yaml b/tests/performance/tests/scale_down_workers/scale_down_workers.yaml index 7e99ba2d6..13924e484 100644 --- a/tests/performance/tests/scale_down_workers/scale_down_workers.yaml +++ b/tests/performance/tests/scale_down_workers/scale_down_workers.yaml @@ -24,7 +24,7 @@ services: criteria: # Inbuilt Criteria - success of ScaleDown<${SCL_DWN_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ScaleDown>${SCL_DWN_RT}, ${STOP_ALIAS} as failed + - avg-rt of ScaleDown>${SCL_DWN_RT} for 10s, ${STOP_ALIAS} as failed # Custom Criteria - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes diff --git a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml index 32004800e..bb08a703b 100644 --- a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml @@ -19,7 +19,7 @@ settings: CONCURRENCY: 10 RAMP-UP: 1s - HOLD-FOR: 300s + HOLD-FOR: 30s SCRIPT: scale_up_workers.jmx STOP : '' #possible values true, false. Bug in bzt so for false use '' diff --git a/tests/performance/tests/scale_up_workers/scale_up_workers.yaml b/tests/performance/tests/scale_up_workers/scale_up_workers.yaml index 051122d92..da139df6d 100644 --- a/tests/performance/tests/scale_up_workers/scale_up_workers.yaml +++ b/tests/performance/tests/scale_up_workers/scale_up_workers.yaml @@ -24,7 +24,7 @@ services: criteria: # Inbuilt Criteria - success of ScaleUp<${SCL_UP_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ScaleUp>${SCL_UP_RT}, ${STOP_ALIAS} as failed + - avg-rt of ScaleUp>${SCL_UP_RT} for 10s, ${STOP_ALIAS} as failed # Custom Criteria - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes From 0adf6ac6fbe10cf579d2613100b585e845582f79 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Thu, 16 Jul 2020 14:39:05 +0530 Subject: [PATCH 17/42] fix links --- tests/performance/README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index 8f6258682..5fbcfdb86 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -44,7 +44,7 @@ The building blocks of the performance regression suite and flow is captured in 3. Make sure that `git` is installed and the test suites are run from the Model Server working directory. ### B. Running the test suite -1. Make sure parameters set in [tests/global_config.yaml](tests/performance/tests/global_config.yaml) are correct. +1. Make sure parameters set in [tests/global_config.yaml](tests/global_config.yaml) are correct. 2. To run the test suite execute [run_performance_suite.py](run_performance_suite.py) with the following parameters @@ -92,7 +92,7 @@ cd $MODEL_SERVER_HOME/tests/performance # Note that Model server started and stopped by the individual test suite. # check variables such as Model server PORT etc -# vi tests/common/global_config.yaml +# vi tests/global_config.yaml #all tests python -m run_performance_suite -e xlarge @@ -205,11 +205,10 @@ Create a folder for the test under `test_dir` location. A test generally compris load scenario and a yaml file which contains test scenarios specifying the conditions for failure or success. The file-names should be identical to the folder name with their respective extensions. -An example [jmeter script](tests/examples_starter/examples_starter.jmx) -and a [scenario](tests/examples_starter/examples_starter.yaml) is provided as a template to get started. +An example [jmeter script](tests/examples_starter/examples_starter.jmx) and [scenario](tests/examples_starter/examples_starter.yaml) is provided as a template to get started. Please note that various global configuration settings used by examples_starter.jmx script are specified in -[tests/global_config.yaml](tests/performance/tests/global_config.yaml) file. +[tests/global_config.yaml](tests/global_config.yaml) file. ```tests/examples_starter/examples_starter.yaml execution: @@ -243,7 +242,7 @@ Specify the metrics of interest in the services/monitoring section of the yaml. 1. Standalone monitoring server Use this technique if Model Server and the tests execute on different machines. Before running the test cases, - please start the [metrics_monitoring_server.py](metrics_monitoring_server.py) script. It will communicate server + please start the [metrics_monitoring_server.py](agents/metrics_monitoring_server.py) script. It will communicate server metric data with the test client over sockets. The monitoring server runs on port 9009 by default. To start the monitoring server, run the following commands on the Model Server host: @@ -424,5 +423,3 @@ possible ways to achieve this * Alternatively, deploy the standalone monitoring agent on the Model Server instance and run the test cases against the remote server. Note that the standalone monitoring agent works on both Python 2/3. - - From 6cacaf43a268ba0ec2ee65a7c2d55a1b6aff5738 Mon Sep 17 00:00:00 2001 From: Mahesh Ambule Date: Thu, 16 Jul 2020 19:55:31 +0530 Subject: [PATCH 18/42] modular --- .../tests/inference_single_worker/inference_single_worker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/tests/inference_single_worker/inference_single_worker.yaml b/tests/performance/tests/inference_single_worker/inference_single_worker.yaml index 1bf6cce30..fdba969e0 100644 --- a/tests/performance/tests/inference_single_worker/inference_single_worker.yaml +++ b/tests/performance/tests/inference_single_worker/inference_single_worker.yaml @@ -1,6 +1,6 @@ --- scenarios: - inference_single_worker: + scenario_0: script: inference_single_worker.jmx services: From 7866a95696dfeada79f1005469f747d8ac21aed7 Mon Sep 17 00:00:00 2001 From: Prashant Sail Date: Fri, 17 Jul 2020 17:46:11 +0530 Subject: [PATCH 19/42] AB support in PRT (#28) --- tests/performance/README.md | 31 ++++- tests/performance/run_performance_suite.py | 2 +- .../runs/taurus/override/__init__.py | 0 .../runs/taurus/override/apache_bench.py | 129 ++++++++++++++++++ .../override}/metrics_monitoring_inproc.py | 6 +- .../environments/xlarge.yaml | 48 +++++++ .../examples_apache_bench.yaml | 43 ++++++ tests/performance/tests/global_config.yaml | 3 + 8 files changed, 254 insertions(+), 8 deletions(-) create mode 100644 tests/performance/runs/taurus/override/__init__.py create mode 100644 tests/performance/runs/taurus/override/apache_bench.py rename tests/performance/{agents => runs/taurus/override}/metrics_monitoring_inproc.py (94%) create mode 100644 tests/performance/tests/examples_apache_bench/environments/xlarge.yaml create mode 100644 tests/performance/tests/examples_apache_bench/examples_apache_bench.yaml diff --git a/tests/performance/README.md b/tests/performance/README.md index 5fbcfdb86..89025dab8 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -24,6 +24,7 @@ environment, will have its own threshold values. * Specification of pass/fail criterion between two commits. For example, memory consumed by workers should not increase by more than 10% between two commits for the given test case. * Custom reporting of results. + * Apache Benchmark executor which supports GET, POST, PUT, OPTIONS, DELETE methods The building blocks of the performance regression suite and flow is captured in the following drawing @@ -201,6 +202,8 @@ Follow these three steps to add a new test case to the test suite. #### 1. Add scenario (a.k.a test suite) +> By default, all scenarios are triggered using _jmeter_ as the underlying executor. + Create a folder for the test under `test_dir` location. A test generally comprises of a jmeter file - containing the load scenario and a yaml file which contains test scenarios specifying the conditions for failure or success. The file-names should be identical to the folder name with their respective extensions. @@ -222,7 +225,6 @@ Please note that various global configuration settings used by examples_starter. script: examples_starter.jmx ``` - To execute this test suite, run the following command ```bash @@ -231,9 +233,30 @@ To execute this test suite, run the following command python -m run_performance_suite -p examples_starter -e xlarge ``` -**Note**: -Taurus provides support for different executors such as JMeter. Supported executor types can be found [here](https://gettaurus.org/docs/ExecutionSettings/). -Details about how to use an existing JMeter script are provided [here](https://gettaurus.org/docs/JMeter/). +**Using Apache Benchmark** + +To execute a scenario using _apache benchmark_ as the executor; In the yaml - +1. Override the `execution` section and explicitly specify "apache_bench" as the value of `executor` +2. Override the `scenarios` section and specify the request details under `requests` section + +``` +~execution: + - executor: apache_bench + concurrency: 10 + hold-for: 300s +~scenarios: + demo: + requests: + - url: http://127.0.0.1:8080/predictions/squeezenet1_1 + label: MyInference + method: POST + file-path: /Users/johndoe/demo/kitten.jpg +``` +Refer to [examples_apache_bench](tests/examples_apache_bench/examples_apache_bench.yaml) for the complete scenario. + +> **Note**: +> Taurus provides support for different executors such as JMeter, Apache Benchmark, etc. Supported executor types can be found [here](https://gettaurus.org/docs/ExecutionSettings/). +> Details about how to use an existing JMeter script are provided [here](https://gettaurus.org/docs/JMeter/). #### 2. Add metrics to monitor diff --git a/tests/performance/run_performance_suite.py b/tests/performance/run_performance_suite.py index 7f4b89af6..11d56d315 100755 --- a/tests/performance/run_performance_suite.py +++ b/tests/performance/run_performance_suite.py @@ -84,7 +84,7 @@ def run_test_suite(artifacts_dir, test_dir, pattern, exclude_pattern, logger.info("Collected tests %s", test_dirs) with ExecutionEnv(MONITORING_AGENT, artifacts_dir, env_name, compare_local, compare_with, monit) as prt: - pre_command = 'export PYTHONPATH={}:$PYTHONPATH;'.format(os.path.join(str(ROOT_PATH), "agents")) + pre_command = 'export PYTHONPATH={}:$PYTHONPATH;'.format(os.path.join(str(ROOT_PATH), "runs", "taurus", "override")) for suite_name in tqdm(test_dirs, desc="Test Suites"): with Timer("Test suite {} execution time".format(suite_name)) as t: suite_artifacts_dir = os.path.join(artifacts_dir, suite_name) diff --git a/tests/performance/runs/taurus/override/__init__.py b/tests/performance/runs/taurus/override/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/performance/runs/taurus/override/apache_bench.py b/tests/performance/runs/taurus/override/apache_bench.py new file mode 100644 index 000000000..026cbb119 --- /dev/null +++ b/tests/performance/runs/taurus/override/apache_bench.py @@ -0,0 +1,129 @@ +""" +Module add support for POST, PUT, OPTIONS and DELETE methods to Apache Benchmark +""" +import mimetypes +import os + +from math import ceil +from distutils.version import LooseVersion + +from bzt import TaurusConfigError +from bzt.modules.aggregator import ConsolidatingAggregator +from bzt.six import iteritems +from bzt.utils import dehumanize_time +from bzt.modules.ab import ApacheBenchmarkExecutor, TSVDataReader + + +class ApacheBenchmarkExecutor(ApacheBenchmarkExecutor): + """ + Apache Benchmark executor module + """ + + def prepare(self): + super(ApacheBenchmarkExecutor, self).prepare() + self.scenario = self.get_scenario() + self.install_required_tools() + + self._tsv_file = self.engine.create_artifact("ab", ".tsv") + + self.stdout = open(self.engine.create_artifact("ab", ".out"), 'w') + self.stderr = open(self.engine.create_artifact("ab", ".err"), 'w') + + self.reader = TSVDataReader(self._tsv_file, self.log) + if isinstance(self.engine.aggregator, ConsolidatingAggregator): + self.engine.aggregator.add_underling(self.reader) + + def startup(self): + args = [self.tool.tool_path] + load = self.get_load() + load_iterations = load.iterations or 1 + load_concurrency = load.concurrency or 1 + + if load.hold: + hold = int(ceil(dehumanize_time(load.hold))) + args += ['-t', str(hold)] + else: + args += ['-n', str(load_iterations * load_concurrency)] # ab waits for total number of iterations + + timeout = self.get_scenario().get("timeout", None) + if timeout: + args += ['-s', str(ceil(dehumanize_time(timeout)))] + + args += ['-c', str(load_concurrency)] + args += ['-d'] # do not print 'Processed *00 requests' every 100 requests or so + args += ['-r'] # do not crash on socket level errors + + if self.tool.version and LooseVersion(self.tool.version) >= LooseVersion("2.4.7"): + args += ['-l'] # accept variable-len responses + + args += ['-g', str(self._tsv_file)] # dump stats to TSV file + + # add global scenario headers + for key, val in iteritems(self.scenario.get_headers()): + args += ['-H', "%s: %s" % (key, val)] + + requests = self.scenario.get_requests() + if not requests: + raise TaurusConfigError("You must specify at least one request for ab") + if len(requests) > 1: + self.log.warning("ab doesn't support multiple requests. Only first one will be used.") + request = self.__first_http_request() + if request is None: + raise TaurusConfigError("ab supports only HTTP requests, while scenario doesn't have any") + + # add request-specific headers + for key, val in iteritems(request.headers): + args += ['-H', "%s: %s" % (key, val)] + + # if request.method != 'GET': + # raise TaurusConfigError("ab supports only GET requests, but '%s' is found" % request.method) + + if request.method == 'HEAD': + args += ['-i'] + elif request.method in ['POST', 'PUT']: + options = {'POST': '-p', 'PUT': '-u'} + file_path = request.config['file-path'] + if not file_path: + file_path = os.devnull + self.log.warning("No file path specified, dev null will be used instead") + args += [options[request.method], file_path] + content_type = request.config['content-type'] or mimetypes.guess_type(file_path)[0] + if content_type: + args += ['-T', content_type] + else: # 'GET', 'OPTIONS', 'DELETE', etc + args += ['-m', request.method] + + if request.priority_option('keepalive', default=True): + args += ['-k'] + + args += [request.url] + + self.reader.setup(load_concurrency, request.label) + + self.log.info('Executing command : ' + ' '.join(arg for arg in args)) + self.process = self._execute(args) + + +class TSVDataReader(TSVDataReader): + def _read(self, last_pass=False): + lines = self.file.get_lines(size=1024 * 1024, last_pass=last_pass) + + for line in lines: + if not self.skipped_header: + self.skipped_header = True + continue + log_vals = [val.strip() for val in line.split('\t')] + + _error = None + # _rstatus = None + _rstatus = '' #Hack to trick taurus into computing aggreated stats + + _url = self.url_label + _concur = self.concurrency + _tstamp = int(log_vals[1]) # timestamp - moment of request sending + _con_time = float(log_vals[2]) / 1000.0 # connection time + _etime = float(log_vals[4]) / 1000.0 # elapsed time + _latency = float(log_vals[5]) / 1000.0 # latency (aka waittime) + _bytes = None + + yield _tstamp, _url, _concur, _etime, _con_time, _latency, _rstatus, _error, '', _bytes \ No newline at end of file diff --git a/tests/performance/agents/metrics_monitoring_inproc.py b/tests/performance/runs/taurus/override/metrics_monitoring_inproc.py similarity index 94% rename from tests/performance/agents/metrics_monitoring_inproc.py rename to tests/performance/runs/taurus/override/metrics_monitoring_inproc.py index ed5788187..e922dfb2a 100644 --- a/tests/performance/agents/metrics_monitoring_inproc.py +++ b/tests/performance/runs/taurus/override/metrics_monitoring_inproc.py @@ -24,9 +24,9 @@ from bzt.modules import monitoring from bzt.utils import dehumanize_time -import configuration -from metrics import get_metrics, AVAILABLE_METRICS as AVAILABLE_SERVER_METRICS -from utils.process import get_process_pid_from_file, get_server_processes, \ +from agents import configuration +from agents.metrics import get_metrics, AVAILABLE_METRICS as AVAILABLE_SERVER_METRICS +from agents.utils.process import get_process_pid_from_file, get_server_processes, \ get_child_processes, get_server_pidfile diff --git a/tests/performance/tests/examples_apache_bench/environments/xlarge.yaml b/tests/performance/tests/examples_apache_bench/environments/xlarge.yaml new file mode 100644 index 000000000..f06a0540b --- /dev/null +++ b/tests/performance/tests/examples_apache_bench/environments/xlarge.yaml @@ -0,0 +1,48 @@ + +--- +settings: + env: + API_LABEL : Inference + API_SUCCESS : 80% + API_AVG_RT : 140ms + + TOTAL_WORKERS: 1 + TOTAL_WORKERS_MEM: 300000000 + TOTAL_WORKERS_FDS: 150 + + TOTAL_MEM : 1000000000 + TOTAL_PROCS : 3 + TOTAL_FDS : 150 + + FRNTEND_MEM: 600000000 + + TOTAL_ORPHANS : 0 + TOTAL_ZOMBIES : 0 + + ## Percent diff values to do a compare across runs + TOTAL_WORKERS_PREV_DIFF: 0 + TOTAL_WORKERS_MEM_PREV_DIFF: 30 + TOTAL_WORKERS_FDS_PREV_DIFF: 30 + TOTAL_MEM_PREV_DIFF: 30 + TOTAL_PROCS_PREV_DIFF: 30 + TOTAL_FDS_PREV_DIFF: 30 + FRNTEND_MEM_PREV_DIFF: 30 + TOTAL_ORPHANS_PREV_DIFF: 0 + TOTAL_ZOMBIES_PREV_DIFF: 0 + + TOTAL_WORKERS_RUN_DIFF: 0 + TOTAL_WORKERS_MEM_RUN_DIFF: 30 + TOTAL_WORKERS_FDS_RUN_DIFF: 30 + TOTAL_MEM_RUN_DIFF: 60 + TOTAL_PROCS_RUN_DIFF: 30 + TOTAL_FDS_RUN_DIFF: 30 + FRNTEND_MEM_RUN_DIFF: 90 + TOTAL_ORPHANS_RUN_DIFF: 0 + TOTAL_ZOMBIES_RUN_DIFF: 0 + + CONCURRENCY : 10 + RAMP-UP : 1s + HOLD-FOR : 300s + + STOP : '' #possible values true, false. Bug in bzt so for false use '' + STOP_ALIAS: continue #possible values continue, stop \ No newline at end of file diff --git a/tests/performance/tests/examples_apache_bench/examples_apache_bench.yaml b/tests/performance/tests/examples_apache_bench/examples_apache_bench.yaml new file mode 100644 index 000000000..87af61a32 --- /dev/null +++ b/tests/performance/tests/examples_apache_bench/examples_apache_bench.yaml @@ -0,0 +1,43 @@ +~execution: + - executor: apache_bench + concurrency: ${CONCURRENCY} + ramp-up: ${RAMP-UP} + hold-for: ${HOLD-FOR} + scenario: scenario_0 + +~scenarios: + scenario_0: + requests: + - url: http://127.0.0.1:8080/predictions/${SQZNET_NAME} + label: ${API_LABEL} + method: POST + file-path: ${INPUT_IMG_PATH} + +services: + - module: shellexec + prepare: + - "curl -s -X POST http://localhost:8081/models?url=${SQZNET_URL}" + - "curl -s -X PUT http://localhost:8081/models/${SQZNET_NAME}?min_worker=1&synchronous=true" + + +reporting: +- module: passfail + criteria: + # Inbuilt Criteria - cannot be used with Apache Benchmark + # - success of MyLabel<${INFR_SUCC}, stop as failed + # - avg-rt of MyLabel>${INFR_RT}, stop as failed + # Custom Criteria + - class: bzt.modules.monitoring.MonitoringCriteria + subject: ServerLocalClient/total_processes + condition: '>' + threshold: ${TOTAL_PROCS} + timeframe: 1s + stop : true + fail : true + - class: bzt.modules.monitoring.MonitoringCriteria + subject: ServerLocalClient/total_processes + condition: '<' + threshold: ${TOTAL_PROCS} + timeframe: 1s + stop : true + fail : true \ No newline at end of file diff --git a/tests/performance/tests/global_config.yaml b/tests/performance/tests/global_config.yaml index c88e1a2e4..1b8da6de3 100644 --- a/tests/performance/tests/global_config.yaml +++ b/tests/performance/tests/global_config.yaml @@ -32,6 +32,9 @@ modules: # metrics_monitoring_inproc and dependencies should be in python path class : metrics_monitoring_inproc.Monitor # monitoring class. + apache_bench: + class: apache_bench.ApacheBenchmarkExecutor + services: - module: shellexec prepare: From 62ba5bf3ee6c2c72272e32b084491d4c1601f0e2 Mon Sep 17 00:00:00 2001 From: Prashant Sail Date: Wed, 22 Jul 2020 11:24:46 +0530 Subject: [PATCH 20/42] support for remote exeuction and logging (#29) --- tests/performance/README.md | 8 ++++---- tests/performance/run_performance_suite.py | 2 +- tests/performance/runs/taurus/__init__.py | 3 ++- ...monitoring_inproc.py => metrics_monitoring.py} | 15 +++++++++++++++ tests/performance/runs/taurus/reader.py | 6 +++--- .../examples_local_criteria.yaml | 10 +++++----- .../examples_local_monitoring.yaml | 10 +++++----- .../examples_remote_criteria.yaml | 4 ++-- .../examples_remote_monitoring.yaml | 4 ++-- tests/performance/tests/global_config.yaml | 10 +++++----- 10 files changed, 44 insertions(+), 28 deletions(-) rename tests/performance/runs/taurus/override/{metrics_monitoring_inproc.py => metrics_monitoring.py} (87%) diff --git a/tests/performance/README.md b/tests/performance/README.md index 89025dab8..b90125099 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -279,8 +279,8 @@ Specify the metrics of interest in the services/monitoring section of the yaml. ```yaml services: - - module: monitoring - server-agent: + - module: server_monitoring + ServerRemoteClient: - address: :9009 # metric monitoring service address label: Model-Server-inference-server # Specified label will be used in reports instead of ip:port interval: 1s # polling interval @@ -308,12 +308,12 @@ Specify the metrics of interest in the services/monitoring section of the yaml. ```yaml modules: - server_local_monitoring: + server_monitoring: # metrics_monitoring_taurus and dependencies should be in python path class : metrics_monitoring_taurus.Monitor # monitoring class. services: - - module: server_local_monitoring # should be added in modules section + - module: server_monitoring # should be added in modules section ServerLocalClient: # keyword from metrics_monitoring_taurus.Monitor - interval: 1s metrics: diff --git a/tests/performance/run_performance_suite.py b/tests/performance/run_performance_suite.py index 11d56d315..4e0d05bfb 100755 --- a/tests/performance/run_performance_suite.py +++ b/tests/performance/run_performance_suite.py @@ -97,7 +97,7 @@ def run_test_suite(artifacts_dir, test_dir, pattern, exclude_pattern, GLOBAL_CONFIG_PATH, test_file, env_yaml_path)) - update_taurus_metric_files(suite_artifacts_dir, test_file) + update_taurus_metric_files(suite_artifacts_dir) sys.exit(prt.exit_code) diff --git a/tests/performance/runs/taurus/__init__.py b/tests/performance/runs/taurus/__init__.py index d2dfd5012..cadc5af82 100644 --- a/tests/performance/runs/taurus/__init__.py +++ b/tests/performance/runs/taurus/__init__.py @@ -42,13 +42,14 @@ def get_taurus_options(artifacts_dir, jmeter_path=None): return options_str -def update_taurus_metric_files(suite_artifacts_dir, test_file): +def update_taurus_metric_files(suite_artifacts_dir): """ It renames the server and local metric monitoring log files to metrics.csv. The order of the columns in header of server metric monitoring SALogs file generated by taurus is not inline with data. So as a work around this function rewrites the header based on order defined in the test yaml. """ + test_file = os.path.join(suite_artifacts_dir, "effective.yml") metrics_new_file = os.path.join(suite_artifacts_dir, "metrics.csv") server_metric_file_pattern = os.path.join(suite_artifacts_dir, "SAlogs_*") diff --git a/tests/performance/runs/taurus/override/metrics_monitoring_inproc.py b/tests/performance/runs/taurus/override/metrics_monitoring.py similarity index 87% rename from tests/performance/runs/taurus/override/metrics_monitoring_inproc.py rename to tests/performance/runs/taurus/override/metrics_monitoring.py index e922dfb2a..859b2ea3a 100644 --- a/tests/performance/runs/taurus/override/metrics_monitoring_inproc.py +++ b/tests/performance/runs/taurus/override/metrics_monitoring.py @@ -42,6 +42,7 @@ class Monitor(monitoring.Monitoring): def __init__(self): super(Monitor, self).__init__() self.client_classes.update({'ServerLocalClient': ServerLocalClient}) + self.client_classes.update({'ServerRemoteClient': ServerRemoteClient}) class ServerLocalClient(monitoring.LocalClient): @@ -86,6 +87,20 @@ def connect(self): logs_writer.writerow(metrics) +class ServerRemoteClient(monitoring.ServerAgentClient): + """Custom server remote client """ + def get_data(self): + result = super().get_data() + # Logging for custom metric values + msg = [] + for res in result: + for metric_name in self.config.get("metrics"): + metric_value = res[metric_name] + msg.append("{0} : {1}".format(metric_name, metric_value)) + self.log.info("{0}".format(" -- ".join(msg))) + return result + + class ServerLocalMonitor(monitoring.LocalMonitor): """Custom server local monitor""" diff --git a/tests/performance/runs/taurus/reader.py b/tests/performance/runs/taurus/reader.py index 7abfdf7cc..71ac05b48 100644 --- a/tests/performance/runs/taurus/reader.py +++ b/tests/performance/runs/taurus/reader.py @@ -21,13 +21,13 @@ def get_mon_metrics_list(test_yaml_path): - """Utility method to get list of server-agent metrics which are being monitored from a test yaml file""" + """Utility method to get list of ServerRemoteClient metrics which are being monitored from a test yaml file""" metrics = [] with open(test_yaml_path) as test_yaml: test_yaml = yaml.safe_load(test_yaml) for rep_section in test_yaml.get('services', []): - if rep_section.get('module', None) == 'monitoring' and "server-agent" in rep_section: - for mon_section in rep_section.get('server-agent', []): + if rep_section.get('module', None) == 'monitoring' and "ServerRemoteClient" in rep_section: + for mon_section in rep_section.get('ServerRemoteClient', []): if isinstance(mon_section, dict): metrics.extend(mon_section.get('metrics', [])) diff --git a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml index 8234a8dd7..72a539d26 100644 --- a/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml +++ b/tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml @@ -10,9 +10,9 @@ script: examples_local_criteria.jmx modules: - server_local_monitoring: - # metrics_monitoring_inproc and dependencies should be in python path - class : metrics_monitoring_inproc.Monitor # monitoring class. + server_monitoring: + # metrics_monitoring and dependencies should be in python path + class : metrics_monitoring.Monitor # monitoring class. ~services: - module: shellexec @@ -28,8 +28,8 @@ modules: - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - - module: server_local_monitoring # should be added in modules section - ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor + - module: server_monitoring # should be added in modules section + ServerLocalClient: # keyword from metrics_monitoring.Monitor - interval: 1s logging : True metrics: diff --git a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml index 86ac99ac9..b48ea0d81 100644 --- a/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml +++ b/tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml @@ -10,9 +10,9 @@ script: examples_local_monitoring.jmx modules: - server_local_monitoring: - # metrics_monitoring_inproc and dependencies should be in python path - class : metrics_monitoring_inproc.Monitor # monitoring class. + server_monitoring: + # metrics_monitoring and dependencies should be in python path + class : metrics_monitoring.Monitor # monitoring class. ~services: - module: shellexec @@ -28,8 +28,8 @@ modules: - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - - module: server_local_monitoring # should be added in modules section - ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor + - module: server_monitoring # should be added in modules section + ServerLocalClient: # keyword from metrics_monitoring.Monitor - interval: 1s metrics: - cpu diff --git a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml index 384f6bf2b..487c825ae 100644 --- a/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml +++ b/tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml @@ -23,8 +23,8 @@ - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - - module: monitoring - server-agent: + - module: server_monitoring + ServerRemoteClient: - address: localhost:9009 # metric monitoring service address label: model-server # if you specify label, it will be used in reports instead of ip:port interval: 1s # polling interval diff --git a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml index f899ec98f..bf531d571 100644 --- a/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml +++ b/tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml @@ -25,8 +25,8 @@ - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - - module: monitoring - server-agent: + - module: server_monitoring + ServerRemoteClient: - address: localhost:9009 # metric monitoring service address label: model-server # if you specify label, it will be used in reports instead of ip:port interval: 1s # polling interval diff --git a/tests/performance/tests/global_config.yaml b/tests/performance/tests/global_config.yaml index 1b8da6de3..98c343d86 100644 --- a/tests/performance/tests/global_config.yaml +++ b/tests/performance/tests/global_config.yaml @@ -28,9 +28,9 @@ modules: RESNET_URL: ${RESNET_URL} RESNET_NAME: ${RESNET_NAME} - server_local_monitoring: - # metrics_monitoring_inproc and dependencies should be in python path - class : metrics_monitoring_inproc.Monitor # monitoring class. + server_monitoring: + # metrics_monitoring and dependencies should be in python path + class : metrics_monitoring.Monitor # monitoring class. apache_bench: class: apache_bench.ApacheBenchmarkExecutor @@ -49,8 +49,8 @@ services: - "rm -r /tmp/ts_model_store" - "mv logs ${ARTIFACTS_DIR}/model_server_logs" - - module: server_local_monitoring # should be added in modules section - ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor + - module: server_monitoring # should be added in modules section + ServerLocalClient: # keyword from metrics_monitoring.Monitor - interval: 1s logging : True metrics: From 37877672b4b30cc9e60356d240af59100e3b92f6 Mon Sep 17 00:00:00 2001 From: Prashant Sail Date: Wed, 22 Jul 2020 21:11:24 +0530 Subject: [PATCH 21/42] Perf cut2 (#30) * corrected section - needed for header sequence in metrics.csv * fixed workers count --- tests/performance/agents/metrics/__init__.py | 2 +- tests/performance/runs/taurus/reader.py | 2 +- .../tests/batch_and_single_inference/environments/xlarge.yaml | 2 +- .../tests/batch_inference/environments/xlarge.yaml | 4 ++-- .../tests/inference_multiple_models/environments/xlarge.yaml | 4 ++-- .../tests/inference_multiple_worker/environments/xlarge.yaml | 4 ++-- .../tests/inference_single_worker/environments/xlarge.yaml | 4 ++-- .../tests/model_description/environments/xlarge.yaml | 4 ++-- .../multiple_inference_and_scaling/environments/xlarge.yaml | 4 ++-- .../tests/register_unregister/environments/xlarge.yaml | 4 ++-- .../register_unregister_multiple/environments/xlarge.yaml | 4 ++-- .../tests/scale_down_workers/environments/xlarge.yaml | 4 ++-- .../tests/scale_up_workers/environments/xlarge.yaml | 4 ++-- 13 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/performance/agents/metrics/__init__.py b/tests/performance/agents/metrics/__init__.py index 8ecd97ab0..21266ffd2 100644 --- a/tests/performance/agents/metrics/__init__.py +++ b/tests/performance/agents/metrics/__init__.py @@ -156,7 +156,7 @@ def update_metric(metric_name, proc_type, stats): # Total processes result['total_processes'] = len(worker_stats) + 1 - result['total_workers'] = max(len(worker_stats) - 1, 0) + result['total_workers'] = len(worker_stats) result['orphans'] = len(list(filter(lambda p: p['ppid'] == 1, worker_stats))) result['zombies'] = len(zombie_children) diff --git a/tests/performance/runs/taurus/reader.py b/tests/performance/runs/taurus/reader.py index 71ac05b48..700039f04 100644 --- a/tests/performance/runs/taurus/reader.py +++ b/tests/performance/runs/taurus/reader.py @@ -26,7 +26,7 @@ def get_mon_metrics_list(test_yaml_path): with open(test_yaml_path) as test_yaml: test_yaml = yaml.safe_load(test_yaml) for rep_section in test_yaml.get('services', []): - if rep_section.get('module', None) == 'monitoring' and "ServerRemoteClient" in rep_section: + if rep_section.get('module', None) == 'server_monitoring' and "ServerRemoteClient" in rep_section: for mon_section in rep_section.get('ServerRemoteClient', []): if isinstance(mon_section, dict): metrics.extend(mon_section.get('metrics', [])) diff --git a/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml b/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml index cd22dcdb3..d42bbd30a 100644 --- a/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml +++ b/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml @@ -8,7 +8,7 @@ settings: INF2_SUCC: 80% INF2_AVG_RT: 30ms - TOTAL_WORKERS: 5 + TOTAL_WORKERS: 6 TOTAL_WORKERS_MEM: 999686400 TOTAL_WORKERS_FDS: 60 diff --git a/tests/performance/tests/batch_inference/environments/xlarge.yaml b/tests/performance/tests/batch_inference/environments/xlarge.yaml index 5ee95f963..c1f8df5df 100644 --- a/tests/performance/tests/batch_inference/environments/xlarge.yaml +++ b/tests/performance/tests/batch_inference/environments/xlarge.yaml @@ -5,12 +5,12 @@ settings: API_SUCCESS : 80% API_AVG_RT : 30ms - TOTAL_WORKERS: 4 + TOTAL_WORKERS: 3 TOTAL_WORKERS_MEM: 3000000000 TOTAL_WORKERS_FDS: 400 TOTAL_MEM : 4000000000 - TOTAL_PROCS : 7 + TOTAL_PROCS : 4 TOTAL_FDS : 200 FRNTEND_MEM: 1000000000 diff --git a/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml b/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml index c854de1cd..b68f5c08e 100644 --- a/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml +++ b/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml @@ -9,12 +9,12 @@ settings: INFR2_SUCC: 100% INFR2_RT: 450ms - TOTAL_WORKERS: 2 + TOTAL_WORKERS: 4 TOTAL_WORKERS_MEM: 600000000 TOTAL_WORKERS_FDS: 150 TOTAL_MEM : 1400000000 - TOTAL_PROCS : 3 + TOTAL_PROCS : 5 TOTAL_FDS : 150 FRNTEND_MEM: 800000000 diff --git a/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml b/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml index 7f7887242..2fa0f5b6d 100644 --- a/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml +++ b/tests/performance/tests/inference_multiple_worker/environments/xlarge.yaml @@ -6,12 +6,12 @@ settings: API_SUCCESS : 80% API_AVG_RT : 140ms - TOTAL_WORKERS: 4 + TOTAL_WORKERS: 5 TOTAL_WORKERS_MEM: 600000000 TOTAL_WORKERS_FDS: 40 TOTAL_MEM : 1400000000 - TOTAL_PROCS : 5 + TOTAL_PROCS : 6 TOTAL_FDS : 150 FRNTEND_MEM: 800000000 diff --git a/tests/performance/tests/inference_single_worker/environments/xlarge.yaml b/tests/performance/tests/inference_single_worker/environments/xlarge.yaml index f160a1bcf..f70048f05 100644 --- a/tests/performance/tests/inference_single_worker/environments/xlarge.yaml +++ b/tests/performance/tests/inference_single_worker/environments/xlarge.yaml @@ -6,12 +6,12 @@ settings: API_SUCCESS : 80% API_AVG_RT : 140ms - TOTAL_WORKERS: 1 + TOTAL_WORKERS: 2 TOTAL_WORKERS_MEM: 300000000 TOTAL_WORKERS_FDS: 150 TOTAL_MEM : 1000000000 - TOTAL_PROCS : 2 + TOTAL_PROCS : 3 TOTAL_FDS : 150 FRNTEND_MEM: 600000000 diff --git a/tests/performance/tests/model_description/environments/xlarge.yaml b/tests/performance/tests/model_description/environments/xlarge.yaml index f62c5e282..370ba7324 100644 --- a/tests/performance/tests/model_description/environments/xlarge.yaml +++ b/tests/performance/tests/model_description/environments/xlarge.yaml @@ -6,12 +6,12 @@ settings: API_SUCCESS : 80% API_AVG_RT : 14ms - TOTAL_WORKERS: 1 + TOTAL_WORKERS: 2 TOTAL_WORKERS_MEM: 150205952 TOTAL_WORKERS_FDS: 40 TOTAL_MEM : 1400000000 - TOTAL_PROCS : 2 + TOTAL_PROCS : 3 TOTAL_FDS : 150 FRNTEND_MEM: 800000000 diff --git a/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml b/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml index 704576848..21eaaee4b 100644 --- a/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml +++ b/tests/performance/tests/multiple_inference_and_scaling/environments/xlarge.yaml @@ -13,12 +13,12 @@ settings: SCALEDOWN1_RT : 100ms SCALEDOWN2_RT : 100ms - TOTAL_WORKERS: 9 + TOTAL_WORKERS: 8 TOTAL_WORKERS_MEM: 2668554752 TOTAL_WORKERS_FDS: 100 TOTAL_MEM : 2000000000 - TOTAL_PROCS : 11 + TOTAL_PROCS : 9 TOTAL_FDS : 300 FRNTEND_MEM: 1000000000 diff --git a/tests/performance/tests/register_unregister/environments/xlarge.yaml b/tests/performance/tests/register_unregister/environments/xlarge.yaml index 1407c6336..cbe49892b 100644 --- a/tests/performance/tests/register_unregister/environments/xlarge.yaml +++ b/tests/performance/tests/register_unregister/environments/xlarge.yaml @@ -8,12 +8,12 @@ settings: UNREG_SUCC: 80% UNREG_RT: 290ms - TOTAL_WORKERS: 1 + TOTAL_WORKERS: 0 TOTAL_WORKERS_MEM: 14054528 TOTAL_WORKERS_FDS: 50 TOTAL_MEM : 1400000000 - TOTAL_PROCS : 2 + TOTAL_PROCS : 1 TOTAL_FDS : 100 FRNTEND_MEM: 1200000000 diff --git a/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml b/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml index 4affbc8ef..eb4d0e024 100644 --- a/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml +++ b/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml @@ -10,13 +10,13 @@ settings: SCL_UP_RT: 1.5s UNREG_RT: 18ms - TOTAL_WORKERS: 4 + TOTAL_WORKERS: 1 TOTAL_WORKERS_MEM: 100000000 TOTAL_WORKERS_FDS: 200 TOTAL_MEM : 2000000000 - TOTAL_PROCS : 5 + TOTAL_PROCS : 2 TOTAL_FDS : 200 FRNTEND_MEM: 100000000 diff --git a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml index 2a0c9101a..913da23ee 100644 --- a/tests/performance/tests/scale_down_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_down_workers/environments/xlarge.yaml @@ -5,8 +5,8 @@ settings: SCL_DWN_RT : 10ms TOTAL_PROCS_B4_SCL_DWN : 6 TOTAL_PROCS_AFTR_SCL_DWN : 4 - TOTAL_WRKRS_B4_SCL_DWN : 4 - TOTAL_WRKRS_AFTR_SCL_DWN : 2 + TOTAL_WRKRS_B4_SCL_DWN : 5 + TOTAL_WRKRS_AFTR_SCL_DWN : 3 FRNTEND_FDS : 78 TOTAL_WRKRS_FDS_B4_SCL_DWN: 38 FRNTEND_MEM : 1000000000 diff --git a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml index bb08a703b..5beca12dc 100644 --- a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml @@ -5,8 +5,8 @@ settings: SCL_UP_RT : 10ms TOTAL_PROCS_AFTR_SCL_UP : 6 TOTAL_PROCS_B4_SCL_UP : 2 - TOTAL_WRKRS_AFTR_SCL_UP : 4 - TOTAL_WRKRS_B4_SCL_UP : 1 + TOTAL_WRKRS_AFTR_SCL_UP : 5 + TOTAL_WRKRS_B4_SCL_UP : 3 FRNTEND_FDS : 88 TOTAL_WRKRS_FDS_AFTR_SCL_UP : 38 FRNTEND_MEM : 1000000000 From f62a7fac8c340b67c00e0b35dfa964ef67079400 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Fri, 7 Aug 2020 15:39:45 +0530 Subject: [PATCH 22/42] codebuild checkin - moved test --> tests/ - updated shell scripts with proper exit code --- .circleci/README.md | 79 -------- .circleci/config.yml | 189 ------------------ .circleci/scripts/linux_test_api.sh | 77 ------- .circleci/scripts/linux_test_benchmark.sh | 14 -- .circleci/scripts/linux_test_modelarchiver.sh | 17 -- .../scripts/linux_test_perf_regression.sh | 28 --- .circleci/scripts/linux_test_python.sh | 7 - ci/buildspec-PRT.yml | 17 ++ ci/buildspec-nightly.yml | 26 +++ ci/buildspec-smoke.yml | 20 ++ .../codebuild_images}/Dockerfile.python2.7 | 0 .../codebuild_images}/Dockerfile.python3.6 | 0 {.circleci => ci}/scripts/linux_build.sh | 0 ci/scripts/linux_test_api.sh | 112 +++++++++++ ci/scripts/linux_test_modelarchiver.sh | 26 +++ ci/scripts/linux_test_perf_regression.sh | 25 +++ ci/scripts/linux_test_python.sh | 14 ++ .../imageInputModelPlan.jmx.yaml | 51 ----- run_circleci_tests.py | 168 ---------------- test/regression_tests.sh | 100 --------- test/resources/kitten.jpg | Bin 110969 -> 0 bytes {test => tests/api}/README.md | 30 ++- {test => tests/api}/postman/environment.json | 0 .../api}/postman/https_test_collection.json | 2 +- .../inference_api_test_collection.json | 0 .../api}/postman/inference_data.json | 56 +++--- .../management_api_test_collection.json | 0 tests/api/regression_tests.sh | 49 +++++ {test => tests/api}/resources/certs.pem | 0 .../api}/resources/config.properties | 4 +- {test => tests/api}/resources/key.pem | 0 {test => tests/api}/screenshot/postman.png | Bin 32 files changed, 339 insertions(+), 772 deletions(-) delete mode 100644 .circleci/README.md delete mode 100644 .circleci/config.yml delete mode 100755 .circleci/scripts/linux_test_api.sh delete mode 100755 .circleci/scripts/linux_test_benchmark.sh delete mode 100755 .circleci/scripts/linux_test_modelarchiver.sh delete mode 100755 .circleci/scripts/linux_test_perf_regression.sh delete mode 100755 .circleci/scripts/linux_test_python.sh create mode 100644 ci/buildspec-PRT.yml create mode 100644 ci/buildspec-nightly.yml create mode 100644 ci/buildspec-smoke.yml rename {.circleci/images => ci/codebuild_images}/Dockerfile.python2.7 (100%) rename {.circleci/images => ci/codebuild_images}/Dockerfile.python3.6 (100%) rename {.circleci => ci}/scripts/linux_build.sh (100%) create mode 100755 ci/scripts/linux_test_api.sh create mode 100755 ci/scripts/linux_test_modelarchiver.sh create mode 100755 ci/scripts/linux_test_perf_regression.sh create mode 100755 ci/scripts/linux_test_python.sh delete mode 100644 performance_regression/imageInputModelPlan.jmx.yaml delete mode 100755 run_circleci_tests.py delete mode 100644 test/regression_tests.sh delete mode 100644 test/resources/kitten.jpg rename {test => tests/api}/README.md (69%) rename {test => tests/api}/postman/environment.json (100%) rename {test => tests/api}/postman/https_test_collection.json (99%) rename {test => tests/api}/postman/inference_api_test_collection.json (100%) rename {test => tests/api}/postman/inference_data.json (95%) rename {test => tests/api}/postman/management_api_test_collection.json (100%) create mode 100644 tests/api/regression_tests.sh rename {test => tests/api}/resources/certs.pem (100%) rename {test => tests/api}/resources/config.properties (50%) rename {test => tests/api}/resources/key.pem (100%) rename {test => tests/api}/screenshot/postman.png (100%) diff --git a/.circleci/README.md b/.circleci/README.md deleted file mode 100644 index d37c13a27..000000000 --- a/.circleci/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Multi Model Server CircleCI build -Model Server uses CircleCI for builds. This folder contains the config and scripts that are needed for CircleCI. - -## config.yml -_config.yml_ contains MMS build logic which will be used by CircleCI. - -## Workflows and Jobs -Currently, following _workflows_ are available - -1. smoke -2. nightly -3. weekly - -Following _jobs_ are executed under each workflow - -1. **build** : Builds _frontend/model-server.jar_ and executes tests from gradle -2. **modelarchiver** : Builds and tests modelarchiver module -3. **python-tests** : Executes pytests from _mms/tests/unit_tests/_ -4. **benchmark** : Executes latency benchmark using resnet-18 model -5. (NEW!) **api-tests** : Executes newman test suite for API testing - -Following _executors_ are available for job execution - -1. py27 -2. py36 - -> Please check the _workflows_, _jobs_ and _executors_ section in _config.yml_ for an up to date list - -## scripts -Instead of using inline commands inside _config.yml_, job steps are configured as shell scripts. -This is easier for maintenance and reduces chances of error in config.yml - -## images -MMS uses customized docker image for its CircleCI build. -To make sure MMS is compatible with both Python2 and Python3, we use two build projects. -We have published two docker images on docker hub for code build -* prashantsail/mms-build:python2.7 -* prashantsail/mms-build:python3.6 - -Following files in the _images_ folder are used to create the docker images -* Dockerfile.python2.7 - Dockerfile for prashantsail/mms-build:python2.7 -* Dockerfile.python3.6 - Dockerfile for prashantsail/mms-build:python3.6 - -## Local CircleCI cli -To make it easy for developers to debug build issues locally, MMS supports CircleCI cli for running a job in a container on your machine. - -#### Dependencies -1. CircleCI cli ([Quick Install](https://circleci.com/docs/2.0/local-cli/#quick-installation)) -2. PyYAML (pip install PyYaml) -3. docker (installed and running) - -#### Command -Developers can use the following command to build MMS locally: -**./run_circleci_tests.py -j -e ** - -- _workflow_name_ -This is a madatory parameter - -- _-j, --job job_name_ -If specified, executes only the specified job name (along with the required parents). -If not specified, all jobs in the workflow are executed sequentially. - -- _-e, --executor executor_name_ -If specified, job is executed only on the specified executor(docker image). -If not specified, job is executed on all the available executors. - -```bash -$ cd multi-model-server -$ ./run_circleci_tests.py smoke -$ ./run_circleci_tests.py smoke -j modelarchiver -$ ./run_circleci_tests.py smoke -e py36 -$ ./run_circleci_tests.py smoke -j modelarchiver -e py36 -``` - -###### Checklist -> 1. Make sure docker is running before you start local execution. -> 2. Docker containers to have **at least 4GB RAM, 2 CPU**. -> 3. If you are on a network with low bandwidth, we advise you to explicitly pull the docker images - -> docker pull prashantsail/mms-build:python2.7 -> docker pull prashantsail/mms-build:python3.6 - -`To avoid Pull Request build failures on github, developers should always make sure that their local builds pass.` diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index c8022a037..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,189 +0,0 @@ -version: 2.1 - - -executors: - py36: - docker: - - image: prashantsail/mms-build:python3.6 - environment: - _JAVA_OPTIONS: "-Xmx2048m" - - py27: - docker: - - image: prashantsail/mms-build:python2.7 - environment: - _JAVA_OPTIONS: "-Xmx2048m" - - -commands: - attach-mms-workspace: - description: "Attach the MMS directory which was saved into workspace" - steps: - - attach_workspace: - at: . - - install-mms-server: - description: "Install MMS server from a wheel" - steps: - - run: - name: Install MMS - command: pip install dist/*.whl - - exeucute-api-tests: - description: "Execute API tests from a collection" - parameters: - collection: - type: enum - enum: [management, inference, https] - default: management - steps: - - run: - name: Start MMS, Execute << parameters.collection >> API Tests, Stop MMS - command: .circleci/scripts/linux_test_api.sh << parameters.collection >> - - store_artifacts: - name: Store server logs from << parameters.collection >> API tests - path: mms_<< parameters.collection >>.log - - store_artifacts: - name: Store << parameters.collection >> API test results - path: test/<< parameters.collection >>-api-report.html - - -jobs: - build: - parameters: - executor: - type: executor - executor: << parameters.executor >> - steps: - - checkout - - run: - name: Build frontend - command: .circleci/scripts/linux_build.sh - - store_artifacts: - name: Store gradle testng results - path: frontend/server/build/reports/tests/test - - persist_to_workspace: - root: . - paths: - - . - - python-tests: - parameters: - executor: - type: executor - executor: << parameters.executor >> - steps: - - attach-mms-workspace - - run: - name: Execute python unit tests - command: .circleci/scripts/linux_test_python.sh - - store_artifacts: - name: Store python Test results - path: htmlcov - - api-tests: - parameters: - executor: - type: executor - executor: << parameters.executor >> - steps: - - attach-mms-workspace - - install-mms-server - - exeucute-api-tests: - collection: management - - exeucute-api-tests: - collection: inference - - exeucute-api-tests: - collection: https - - benchmark: - parameters: - executor: - type: executor - executor: << parameters.executor >> - steps: - - attach-mms-workspace - - install-mms-server - - run: - name: Start MMS, Execute benchmark tests, Stop MMS - command: .circleci/scripts/linux_test_benchmark.sh - - store_artifacts: - name: Store server logs from benchmark tests - path: mms.log - - store_artifacts: - name: Store Benchmark Latency resnet-18 results - path: /tmp/MMSBenchmark/out/latency/resnet-18/report/ - destination: benchmark-latency-resnet-18 - - modelarchiver: - parameters: - executor: - type: executor - executor: << parameters.executor >> - steps: - - checkout - - run: - name: Execute lint, unit and integration tests - command: .circleci/scripts/linux_test_modelarchiver.sh - - store_artifacts: - name: Store unit tests results from model archiver tests - path: model-archiver/results_units - destination: units - - -workflows: - version: 2 - - smoke: - jobs: - - &build - build: - name: build-<< matrix.executor >> - matrix: &matrix - parameters: - executor: ["py27", "py36"] - - &modelarchiver - modelarchiver: - name: modelarchiver-<< matrix.executor >> - matrix: *matrix - - &python-tests - python-tests: - name: python-tests-<< matrix.executor >> - requires: - - build-<< matrix.executor >> - matrix: *matrix - - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - *build - - *modelarchiver - - *python-tests - - &api-tests - api-tests: - name: api-tests-<< matrix.executor >> - requires: - - build-<< matrix.executor >> - matrix: *matrix - - weekly: - triggers: - - schedule: - cron: "0 0 * * 0" - filters: - branches: - only: - - master - jobs: - - *build - - benchmark: - name: benchmark-<< matrix.executor >> - requires: - - build-<< matrix.executor >> - matrix: *matrix diff --git a/.circleci/scripts/linux_test_api.sh b/.circleci/scripts/linux_test_api.sh deleted file mode 100755 index 23f908abb..000000000 --- a/.circleci/scripts/linux_test_api.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -MODEL_STORE_DIR='test/model_store' - -MMS_LOG_FILE_MANAGEMENT='mms_management.log' -MMS_LOG_FILE_INFERENCE='mms_inference.log' -MMS_LOG_FILE_HTTPS='mms_https.log' -MMS_CONFIG_FILE_HTTPS='test/resources/config.properties' - -POSTMAN_ENV_FILE='test/postman/environment.json' -POSTMAN_COLLECTION_MANAGEMENT='test/postman/management_api_test_collection.json' -POSTMAN_COLLECTION_INFERENCE='test/postman/inference_api_test_collection.json' -POSTMAN_COLLECTION_HTTPS='test/postman/https_test_collection.json' -POSTMAN_DATA_FILE_INFERENCE='test/postman/inference_data.json' - -REPORT_FILE_MANAGEMENT='test/management-api-report.html' -REPORT_FILE_INFERENCE='test/inference-api-report.html' -REPORT_FILE_HTTPS='test/https-api-report.html' - -start_mms_server() { - multi-model-server --start --model-store $1 >> $2 2>&1 - sleep 10 -} - -start_mms_secure_server() { - multi-model-server --start --mms-config $MMS_CONFIG_FILE_HTTPS --model-store $1 >> $2 2>&1 - sleep 10 -} - -stop_mms_server() { - multi-model-server --stop -} - -trigger_management_tests(){ - start_mms_server $MODEL_STORE_DIR $MMS_LOG_FILE_MANAGEMENT - newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_MANAGEMENT \ - -r cli,html --reporter-html-export $REPORT_FILE_MANAGEMENT --verbose - stop_mms_server -} - -trigger_inference_tests(){ - start_mms_server $MODEL_STORE_DIR $MMS_LOG_FILE_INFERENCE - newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_INFERENCE -d $POSTMAN_DATA_FILE_INFERENCE \ - -r cli,html --reporter-html-export $REPORT_FILE_INFERENCE --verbose - stop_mms_server -} - -trigger_https_tests(){ - start_mms_secure_server $MODEL_STORE_DIR $MMS_LOG_FILE_HTTPS - newman run --insecure -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_HTTPS \ - -r cli,html --reporter-html-export $REPORT_FILE_HTTPS --verbose - stop_mms_server -} - -mkdir -p $MODEL_STORE_DIR - -case $1 in - 'management') - trigger_management_tests - ;; - 'inference') - trigger_inference_tests - ;; - 'https') - trigger_https_tests - ;; - 'ALL') - trigger_management_tests - trigger_inference_tests - trigger_https_tests - ;; - *) - echo $1 'Invalid' - echo 'Please specify any one of - management | inference | https | ALL' - exit 1 - ;; -esac \ No newline at end of file diff --git a/.circleci/scripts/linux_test_benchmark.sh b/.circleci/scripts/linux_test_benchmark.sh deleted file mode 100755 index 73c66f58e..000000000 --- a/.circleci/scripts/linux_test_benchmark.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Hack needed to make it work with existing benchmark.py -# benchmark.py expects jmeter to be present at a very specific location -mkdir -p /home/ubuntu/.linuxbrew/Cellar/jmeter/5.3/libexec/bin/ -ln -s /opt/apache-jmeter-5.3/bin/jmeter /home/ubuntu/.linuxbrew/Cellar/jmeter/5.3/libexec/bin/jmeter - -multi-model-server --start >> mms.log 2>&1 -sleep 30 - -cd benchmarks -python benchmark.py latency - -multi-model-server --stop \ No newline at end of file diff --git a/.circleci/scripts/linux_test_modelarchiver.sh b/.circleci/scripts/linux_test_modelarchiver.sh deleted file mode 100755 index bdae61a65..000000000 --- a/.circleci/scripts/linux_test_modelarchiver.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -cd model-archiver/ - -# Lint test -pylint -rn --rcfile=./model_archiver/tests/pylintrc model_archiver/. - -# Execute python unit tests -python -m pytest --cov-report html:results_units --cov=./ model_archiver/tests/unit_tests - - -# Install model archiver module -pip install . - -# Execute integration tests -python -m pytest model_archiver/tests/integ_tests -# ToDo - Report for Integration tests ? \ No newline at end of file diff --git a/.circleci/scripts/linux_test_perf_regression.sh b/.circleci/scripts/linux_test_perf_regression.sh deleted file mode 100755 index 6d3f80764..000000000 --- a/.circleci/scripts/linux_test_perf_regression.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -multi-model-server --start \ - --models squeezenet=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar \ - >> mms.log 2>&1 -sleep 90 - -cd performance_regression - -# Only on a python 2 environment - -PY_MAJOR_VER=$(python -c 'import sys; major = sys.version_info.major; print(major);') -if [ $PY_MAJOR_VER -eq 2 ]; then - # Hack to use python 3.6.5 for bzt installation and execution - export PATH="/root/.pyenv/bin:/root/.pyenv/shims:$PATH" - pyenv local 3.6.5 -fi - -# Install dependencies -pip install bzt - -curl -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg -bzt -o modules.jmeter.path=/opt/apache-jmeter-5.3/bin/jmeter \ - -o settings.artifacts-dir=/tmp/mms-performance-regression/ \ - -o modules.console.disable=true \ - imageInputModelPlan.jmx.yaml \ - -report - -multi-model-server --stop \ No newline at end of file diff --git a/.circleci/scripts/linux_test_python.sh b/.circleci/scripts/linux_test_python.sh deleted file mode 100755 index 9af6d1e5b..000000000 --- a/.circleci/scripts/linux_test_python.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Lint Test -pylint -rn --rcfile=./mms/tests/pylintrc mms/. - -# Execute python tests -python -m pytest --cov-report html:htmlcov --cov=mms/ mms/tests/unit_tests/ \ No newline at end of file diff --git a/ci/buildspec-PRT.yml b/ci/buildspec-PRT.yml new file mode 100644 index 000000000..9fcd60892 --- /dev/null +++ b/ci/buildspec-PRT.yml @@ -0,0 +1,17 @@ +# Build Spec for AWS CodeBuild CI - Performance Regression Suite + +version: 0.2 + +phases: + build: + commands: + # create and install build + - ci/scripts/linux_build.sh + - pip install dist/* + # run PRT + - ci/scripts/linux_test_perf_regression.sh + +artifacts: + files: + - tests/performance/logs/**/* + - tests/performance/run_artifacts/**/* diff --git a/ci/buildspec-nightly.yml b/ci/buildspec-nightly.yml new file mode 100644 index 000000000..ce916c8e1 --- /dev/null +++ b/ci/buildspec-nightly.yml @@ -0,0 +1,26 @@ +# Build Spec for AWS CodeBuild CI - Nightly + +version: 0.2 + +phases: + build: + commands: + # create and install build + - ci/scripts/linux_build.sh + - pip install dist/* + # modelarchiver tests + - ci/scripts/linux_test_modelarchiver.sh + # run python tests + - ci/scripts/linux_test_python.sh + # run api tests + - ci/scripts/linux_test_api.sh management + - ci/scripts/linux_test_api.sh inference + - ci/scripts/linux_test_api.sh https + +artifacts: + files: + - frontend/server/build/reports/tests/test/**/* + - model-archiver/results_units/**/* + - results_units/**/* + - tests/api/artifacts/**/* + name: MMS-NIGHTLY-$(date +%Y-%m-%d) diff --git a/ci/buildspec-smoke.yml b/ci/buildspec-smoke.yml new file mode 100644 index 000000000..2c2f477a2 --- /dev/null +++ b/ci/buildspec-smoke.yml @@ -0,0 +1,20 @@ +# Build Spec for AWS CodeBuild CI - SMOKE + +version: 0.2 + +phases: + build: + commands: + # create and install build + - ci/scripts/linux_build.sh + - pip install dist/* + # modelarchiver tests + - ci/scripts/linux_test_modelarchiver.sh + # run python tests + - ci/scripts/linux_test_python.sh + +artifacts: + files: + - frontend/server/build/reports/tests/test/**/* + - model-archiver/results_units/**/* + - results_units/**/* diff --git a/.circleci/images/Dockerfile.python2.7 b/ci/codebuild_images/Dockerfile.python2.7 similarity index 100% rename from .circleci/images/Dockerfile.python2.7 rename to ci/codebuild_images/Dockerfile.python2.7 diff --git a/.circleci/images/Dockerfile.python3.6 b/ci/codebuild_images/Dockerfile.python3.6 similarity index 100% rename from .circleci/images/Dockerfile.python3.6 rename to ci/codebuild_images/Dockerfile.python3.6 diff --git a/.circleci/scripts/linux_build.sh b/ci/scripts/linux_build.sh similarity index 100% rename from .circleci/scripts/linux_build.sh rename to ci/scripts/linux_build.sh diff --git a/ci/scripts/linux_test_api.sh b/ci/scripts/linux_test_api.sh new file mode 100755 index 000000000..d8d128a02 --- /dev/null +++ b/ci/scripts/linux_test_api.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +BASE_DIR="tests/api" +MODEL_STORE_DIR="model_store" + +ARTIFACTS_BASE_DIR="artifacts" +ARTIFACTS_MANAGEMENT_DIR="$ARTIFACTS_BASE_DIR/management" +ARTIFACTS_INFERENCE_DIR="$ARTIFACTS_BASE_DIR/inference" +ARTIFACTS_HTTPS_DIR="$ARTIFACTS_BASE_DIR/https" + +MMS_CONSOLE_LOG_FILE="mms_console.log" +MMS_CONFIG_FILE_HTTPS="resources/config.properties" + +POSTMAN_ENV_FILE="postman/environment.json" +POSTMAN_COLLECTION_MANAGEMENT="postman/management_api_test_collection.json" +POSTMAN_COLLECTION_INFERENCE="postman/inference_api_test_collection.json" +POSTMAN_COLLECTION_HTTPS="postman/https_test_collection.json" +POSTMAN_DATA_FILE_INFERENCE="postman/inference_data.json" + +REPORT_FILE="report.html" + +start_mms_server() { + multi-model-server --start --model-store $1 >> $2 2>&1 + sleep 10 +} + +start_mms_secure_server() { + multi-model-server --start --mms-config $MMS_CONFIG_FILE_HTTPS --model-store $1 >> $2 2>&1 + sleep 10 +} + +stop_mms_server() { + multi-model-server --stop + sleep 10 +} + +cleanup_model_store(){ + rm -rf $MODEL_STORE_DIR/* +} + +move_logs(){ + mv $1 logs/ + mv logs/ $2 +} + +trigger_management_tests(){ + start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE + newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_MANAGEMENT \ + -r cli,html --reporter-html-export $ARTIFACTS_MANAGEMENT_DIR/$REPORT_FILE --verbose + local EXIT_CODE=$? + stop_mms_server + move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_MANAGEMENT_DIR + cleanup_model_store + return $EXIT_CODE +} + +trigger_inference_tests(){ + start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE + newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_INFERENCE -d $POSTMAN_DATA_FILE_INFERENCE \ + -r cli,html --reporter-html-export $ARTIFACTS_INFERENCE_DIR/$REPORT_FILE --verbose + local EXIT_CODE=$? + stop_mms_server + move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_INFERENCE_DIR + cleanup_model_store + return $EXIT_CODE +} + +trigger_https_tests(){ + start_mms_secure_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE + newman run --insecure -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_HTTPS \ + -r cli,html --reporter-html-export $ARTIFACTS_HTTPS_DIR/$REPORT_FILE --verbose + local EXIT_CODE=$? + stop_mms_server + move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_HTTPS_DIR + cleanup_model_store + return $EXIT_CODE +} + +cd $BASE_DIR +mkdir -p $MODEL_STORE_DIR $ARTIFACTS_MANAGEMENT_DIR $ARTIFACTS_INFERENCE_DIR $ARTIFACTS_HTTPS_DIR + +case $1 in + 'management') + trigger_management_tests + exit $? + ;; + 'inference') + trigger_inference_tests + exit $? + ;; + 'https') + trigger_https_tests + exit $? + ;; + 'ALL') + trigger_management_tests + MGMT_EXIT_CODE=$? + trigger_inference_tests + INFR_EXIT_CODE=$? + trigger_https_tests + HTTPS_EXIT_CODE=$? + # If any one of the tests fail, exit with error + if [ "$MGMT_EXIT_CODE" -ne 0 ] || [ "$INFR_EXIT_CODE" -ne 0 ] || [ "$HTTPS_EXIT_CODE" -ne 0 ] + then exit 1 + fi + ;; + *) + echo $1 'Invalid' + echo 'Please specify any one of - management | inference | https | ALL' + exit 1 + ;; +esac \ No newline at end of file diff --git a/ci/scripts/linux_test_modelarchiver.sh b/ci/scripts/linux_test_modelarchiver.sh new file mode 100755 index 000000000..d08ad476d --- /dev/null +++ b/ci/scripts/linux_test_modelarchiver.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +cd model-archiver/ + +# Lint test +pylint -rn --rcfile=./model_archiver/tests/pylintrc model_archiver/. +PY_LINT_EXIT_CODE=$? + +# Execute python unit tests +python -m pytest --cov-report html:result_units --cov=./ model_archiver/tests/unit_tests +PY_UNITS_EXIT_CODE=$? + + +# Install model archiver module +python setup.py bdist_wheel --universal && \ +pip install dist/*.whl +MODL_ARCHVR_EXIT_CODE=$? + +# Execute integration tests +python -m pytest model_archiver/tests/integ_tests # ToDo - Report for Integration tests ? +PY_INTEG_EXIT_CODE=$? + +# If any one of the steps fail, exit with error +if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PY_UNITS_EXIT_CODE" -ne 0 ] || [ "$MODL_ARCHVR_EXIT_CODE" -ne 0 ] || [ "$PY_INTEG_EXIT_CODE" -ne 0 ] +then exit 1 +fi \ No newline at end of file diff --git a/ci/scripts/linux_test_perf_regression.sh b/ci/scripts/linux_test_perf_regression.sh new file mode 100755 index 000000000..070b10a07 --- /dev/null +++ b/ci/scripts/linux_test_perf_regression.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +JMETER_PATH='/opt/apache-jmeter-5.3/bin/jmeter' + +cd tests/performance + +# Only on a python 2 environment - +PY_MAJOR_VER=$(python -c 'import sys; major = sys.version_info.major; print(major);') +if [ $PY_MAJOR_VER -eq 2 ]; then + # Hack to use python 3.6.5 for bzt installation and execution + # While MMS continues to use system python which is at 2.7.x + export PATH="/root/.pyenv/bin:/root/.pyenv/shims:$PATH" + pyenv local 3.6.5 system +fi + +# Install dependencies +pip install -r requirements.txt +pip install bzt + +# Execute performance test suite and store exit code +./run_performance_suite.py -j $JMETER_PATH -e ci_linux_medium -x example* --no-compare-local --no-monit +EXIT_CODE=$? + +# Exit with the same error code as that of test execution +exit $EXIT_CODE \ No newline at end of file diff --git a/ci/scripts/linux_test_python.sh b/ci/scripts/linux_test_python.sh new file mode 100755 index 000000000..5bcedce95 --- /dev/null +++ b/ci/scripts/linux_test_python.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Lint Test +pylint -rn --rcfile=./mms/tests/pylintrc mms/. +PY_LINT_EXIT_CODE=$? + +# Execute python tests +python -m pytest --cov-report html:result_units --cov=mms/ mms/tests/unit_tests/ +PYTEST_EXIT_CODE=$? + +# If any one of the tests fail, exit with error +if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PYTEST_EXIT_CODE" -ne 0 ] +then exit 1 +fi \ No newline at end of file diff --git a/performance_regression/imageInputModelPlan.jmx.yaml b/performance_regression/imageInputModelPlan.jmx.yaml deleted file mode 100644 index c662874d0..000000000 --- a/performance_regression/imageInputModelPlan.jmx.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -execution: -- concurrency: 1 - ramp-up: 5s - hold-for: 1m - scenario: Inference -scenarios: - Inference: - default-address: ${__P(protocol,https)}://${__P(hostname,127.0.0.1)}:${__P(port,8443)}/ - requests: - - follow-redirects: true - label: Inference Request - method: POST - upload-files: - - mime-type: image/jpeg - param: data - path: ${__P(input_filepath)} - url: ${__P(protocol,http)}://${__P(hostname,127.0.0.1)}:${__P(port,8080)}/predictions/${model} - store-cache: false - store-cookie: false - use-dns-cache-mgr: false - variables: - cnn_url: ${__P(url, https://s3.amazonaws.com/model-server/models/squeezenet_v1.1/squeezenet_v1.1.model)} - model: ${__P(model_name,squeezenet_v1.1)} - scale_down_workers: '0' - scale_up_workers: ${__P(min_workers,1)} -modules: - jmeter: - properties: - input_filepath : kitten.jpg - model_name : squeezenet -services: - - module: monitoring - local: - - interval: 2s - logging: True - metrics: - - cpu - - disk-space - - mem -reporting: -- module: passfail - criteria: - - class: bzt.modules.monitoring.MonitoringCriteria - subject: local/cpu - condition: '>' - threshold: 100 - timeframe: 6s -- module: junit-xml - filename: ${TAURUS_ARTIFACTS_DIR}/output/results.xml - data-source: pass-fail diff --git a/run_circleci_tests.py b/run_circleci_tests.py deleted file mode 100755 index 5001c0006..000000000 --- a/run_circleci_tests.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python -""" -- This script helps to execute circleci jobs in a container on developer's local machine. -- The script accepts workflow(mandatory), job(optional) and executor(optional) arguments. -- The script used circleci cli's process command to generate a processed yaml. -- The processed yaml, is parsed and twekaed to generate a new transformed yaml. -- The transformed yaml contains a single job, which is merged and ordered list of job steps -from the specfied and requird parent jobs. -""" - -# Make sure you have following dependencies installed on your local machine -# 1. PyYAML (pip install PyYaml) -# 2. CircleCI cli from - https://circleci.com/docs/2.0/local-cli/#installation -# 3. docker - -from collections import OrderedDict -from functools import reduce - -import subprocess -import sys -import copy -import argparse -import yaml - -parser = argparse.ArgumentParser(description='Execute circleci jobs in a container \ - on your local machine') -parser.add_argument('workflow', type=str, help='Workflow name from config.yml') -parser.add_argument('-j', '--job', type=str, help='Job name from config.yml') -parser.add_argument('-e', '--executor', type=str, help='Executor name from config.yml') -args = parser.parse_args() - -workflow = args.workflow -job = args.job -executor = args.executor - -CCI_CONFIG_FILE = '.circleci/config.yml' -PROCESSED_FILE = '.circleci/processed.yml' -XFORMED_FILE = '.circleci/xformed.yml' -CCI_CONFIG = {} -PROCESSED_CONFIG = {} -XFORMED_CONFIG = {} -XFORMED_JOB_NAME = 'mms_xformed_job' -BLACKLISTED_STEPS = ['persist_to_workspace', 'attach_workspace', 'store_artifacts'] - -# Read CircleCI's config -with open(CCI_CONFIG_FILE) as fstream: - try: - CCI_CONFIG = yaml.safe_load(fstream) - except yaml.YAMLError as err: - print(err) - -# Create processed YAML using circleci cli's 'config process' commands -PROCESS_CONFIG_CMD = 'circleci config process {} > {}'.format(CCI_CONFIG_FILE, PROCESSED_FILE) -print("Executing command : ", PROCESS_CONFIG_CMD) -subprocess.check_call(PROCESS_CONFIG_CMD, shell=True) - -# Read the processed config -with open(PROCESSED_FILE) as fstream: - try: - PROCESSED_CONFIG = yaml.safe_load(fstream) - except yaml.YAMLError as err: - print(err) - -# All executors available in the config file -available_executors = list(CCI_CONFIG['executors']) - -# All jobs available under the specified workflow -jobs_in_workflow = PROCESSED_CONFIG['workflows'][workflow]['jobs'] - - -def get_processed_job_sequence(processed_job_name): - """ Recursively iterate over jobs in the workflow to generate an ordered list of parent jobs """ - jobs_in_sequence = [] - - job_dict = next((jd for jd in jobs_in_workflow \ - if isinstance(jd, dict) and processed_job_name == list(jd)[0]), None) - if job_dict: - # Find all parent jobs, recurse to find their respective ancestors - parent_jobs = job_dict[processed_job_name].get('requires', []) - for pjob in parent_jobs: - jobs_in_sequence += get_processed_job_sequence(pjob) - - return jobs_in_sequence + [processed_job_name] - - -def get_jobs_to_exec(job_name): - """ Returns a dictionary of executors and a list of jobs to be executed in them """ - jobs_dict = {} - executors = [executor] if executor else available_executors - - for exectr_name in executors: - if job_name is None: - # List of all job names(as string) - jobs_dict[exectr_name] = map(lambda j: j if isinstance(j, str) \ - else list(j)[0], jobs_in_workflow) - # Filter processed job names as per the executor - # "job_name-executor_name" is a convention set in config.yml - # pylint: disable=cell-var-from-loop - jobs_dict[exectr_name] = filter(lambda j: exectr_name in j, jobs_dict[exectr_name]) - else: - # The list might contain duplicate parent jobs due to multiple fan-ins like config - # - Remove the duplicates - # "job_name-executor_name" is a convention set in config.yml - jobs_dict[exectr_name] = \ - OrderedDict.fromkeys(get_processed_job_sequence(job_name + '-' + exectr_name)) - jobs_dict[exectr_name] = list(jobs_dict[exectr_name]) - - return jobs_dict - - -# jobs_to_exec is a dict, with executor(s) as the key and list of jobs to be executed as its value -jobs_to_exec = get_jobs_to_exec(job) - - -def get_jobs_steps(steps, job_name): - """ Merge all the steps from list of jobs to execute """ - job_steps = PROCESSED_CONFIG['jobs'][job_name]['steps'] - filtered_job_steps = list(filter(lambda step: list(step)[0] not in BLACKLISTED_STEPS, \ - job_steps)) - return steps + filtered_job_steps - - -result_dict = {} - -for exectr, jobs in jobs_to_exec.items(): - merged_steps = reduce(get_jobs_steps, jobs, []) - - # Create a new job, using the first job as a reference - # This ensures configs like executor, environment, etc are maintained from the first job - first_job = jobs[0] - xformed_job = copy.deepcopy(PROCESSED_CONFIG['jobs'][first_job]) - - # Add the merged steps to this newly introduced job - xformed_job['steps'] = merged_steps - - # Create a duplicate config(transformed) with the newly introduced job as the only job in config - XFORMED_CONFIG = copy.deepcopy(PROCESSED_CONFIG) - XFORMED_CONFIG['jobs'] = {} - XFORMED_CONFIG['jobs'][XFORMED_JOB_NAME] = xformed_job - - # Create a transformed yaml - with open(XFORMED_FILE, 'w+') as fstream: - yaml.dump(XFORMED_CONFIG, fstream) - - try: - # Locally execute the newly created job - # This newly created job has all the steps (ordered and merged from steps in parent job(s)) - LOCAL_EXECUTE_CMD = 'circleci local execute -c {} --job {}'.format(XFORMED_FILE, \ - XFORMED_JOB_NAME) - print('Executing command : ', LOCAL_EXECUTE_CMD) - result_dict[exectr] = subprocess.check_call(LOCAL_EXECUTE_CMD, shell=True) - except subprocess.CalledProcessError as err: - result_dict[exectr] = err.returncode - -# Clean up, remove the processed and transformed yml files -CLEANUP_CMD = 'rm {} {}'.format(PROCESSED_FILE, XFORMED_FILE) -print('Executing command : ', CLEANUP_CMD) -subprocess.check_call(CLEANUP_CMD, shell=True) - -# Print job execution details -for exectr, retcode in result_dict.items(): - colorcode, status = ('\033[0;37;42m', 'successful') if retcode == 0 \ - else ('\033[0;37;41m', 'failed') - print("{} Job execution {} using {} executor \x1b[0m".format(colorcode, status, exectr)) - -# Exit as per overall status -SYS_EXIT_CODE = 0 if all(retcode == 0 for exectr, retcode in result_dict.items()) else 1 -sys.exit(SYS_EXIT_CODE) diff --git a/test/regression_tests.sh b/test/regression_tests.sh deleted file mode 100644 index 8825ee6ad..000000000 --- a/test/regression_tests.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -set -x -set -e - -MMS_REPO="https://github.com/awslabs/multi-model-server.git" -BRANCH=${1:-master} -ROOT_DIR="/workspace/" -CODEBUILD_WD=$(pwd) -MODEL_STORE=$ROOT_DIR"/model_store" -MMS_LOG_FILE="/tmp/mms.log" -TEST_EXECUTION_LOG_FILE="/tmp/test_exec.log" - -install_mms_from_source() { - echo "Cloning & Building Multi Model Server Repo from " $1 - - sudo apt-get -y install nodejs-dev node-gyp libssl1.0-dev - sudo apt-get -y install npm - sudo npm install -g n - sudo n latest - export PATH="$PATH" - sudo npm install -g newman newman-reporter-html - pip install mxnet-mkl - # Clone & Build MMS - echo "Installing MMS from source" - git clone -b $2 $1 - cd multi-model-server - pip install . - cd - - echo "MMS Succesfully installed" - -} - - -start_mms() { - - # Start MMS with Model Store - multi-model-server --start --model-store $1 &>> $2 - sleep 10 - curl http://127.0.0.1:8081/models - -} - -stop_mms_serve() { - multi-model-server --stop -} - -start_secure_mms() { - - # Start MMS with Model Store - multi-model-server --start --mms-config test/resources/config.properties --model-store $1 &>> $2 - sleep 10 - curl --insecure -X GET https://127.0.0.1:8444/models -} - - -run_postman_test() { - # Run Postman Scripts - mkdir $ROOT_DIR/report/ - cd $CODEBUILD_WD/ - set +e - # Run Management API Tests - stop_mms_serve - start_mms $MODEL_STORE $MMS_LOG_FILE - newman run -e test/postman/environment.json --bail --verbose test/postman/management_api_test_collection.json \ - -r cli,html --reporter-html-export $ROOT_DIR/report/management_report.html >>$1 2>&1 - - # Run Inference API Tests after Restart - stop_mms_serve - start_mms $MODEL_STORE $MMS_LOG_FILE - newman run -e test/postman/environment.json --bail --verbose test/postman/inference_api_test_collection.json \ - -d test/postman/inference_data.json -r cli,html --reporter-html-export $ROOT_DIR/report/inference_report.html >>$1 2>&1 - - - # Run Https test cases - stop_mms_serve - start_secure_mms $MODEL_STORE $MMS_LOG_FILE - newman run --insecure -e test/postman/environment.json --bail --verbose test/postman/https_test_collection.json \ - -r cli,html --reporter-html-export $ROOT_DIR/report/MMS_https_test_report.html >>$1 2>&1 - - stop_mms_serve - set -e - cd - -} - - -sudo rm -rf $ROOT_DIR && sudo mkdir $ROOT_DIR -sudo chown -R $USER:$USER $ROOT_DIR -cd $ROOT_DIR -mkdir $MODEL_STORE - -sudo rm -f $TEST_EXECUTION_LOG_FILE $MMS_LOG_FILE - -echo "** Execuing MMS Regression Test Suite executon for " $MMS_REPO " **" - -install_mms_from_source $MMS_REPO $BRANCH -run_postman_test $TEST_EXECUTION_LOG_FILE - -echo "** Tests Complete ** " -exit 0 diff --git a/test/resources/kitten.jpg b/test/resources/kitten.jpg deleted file mode 100644 index ffcd2be2c674fddc4886117c12f3b29e4367aba8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 110969 zcmb5VcUY4_vo{=iuY!VvW@w=(-9YFiK#%|dsftwTCG;X4M37EEg@j%Oq(}fokRnwD z450`}7YH4s34$*^=R4e-{AEaF`wpKypRmB;o+T z-))jgBNuOfZ$B4rA3mAek^qQ?o&h;I;EG6+fh1M{GA=uj~Ml`T+!8{QZ3t@7(dU3-8h z{zoFI%UEPz8L7bZSc` zmSbApr)s;zbVXyiE1kDM91J%TuHXL{3(26dP%uz`P`Gd{IlLP2%DGe6u!Y^SZJG^T zO#?M;)PjBaIdLeSw|#OI>0G(pkwX`0+DM{cCEl_3F9Kd z2_jFPyfMp=0x4a%L}yYJ%E^6?&dHyena;(dM%XkHyFgLk8r$86Mx6YRqcW)ieM z=|3!Uv}IMUYPWzEoSWmEaT;cNSYs$=S5LUzr}7%c9BTBQ^g5$_AE7X4KG(yqSKQVp zjJmd6)oKm@fyu7dKMWwVt@|yFJoWrBTCEQNDUEGbo5iP2Da^&h$oPkm5g&5yZraSA z9M_R-8k&mP@2>}*jYZiy;*s*(4d*6_^M0H6;=P86GaPSDZGY(-woW0Vx>oiPe1nKJ z4|63I)rkT=zuur8fU&ecKe&P{g!`gO&m*M)QO0F^v0gaU<#BTP!CXU~3bRZ8yK2Z`AD5xc45F}G2BIHMvH>Z_#CNLhhz$7ziXAQs#(o|Xo!Tac-6th3m z?#2*P6UsE4hlJhbGG-KTh8&kaQ|MFOwvd;qfXsw0ApQbQDs)HORxzO_MKvBIn}{Ol z$(!irS!7mJK&T0S7K5X0xXAjcfa8Ln@z*;k&FuWZH7Z$+Hlali5(9%C{}|E~^Tj)g ze$C3S-9-0-YQ6_IMu=U!J+6A`tJd|^%p!=K;WZ3;I_03WRN=(-+*%sP=nBu245}3R zS*7ywr0k8InYuu<^}T>w>b?m9OvCT1tTNS9vo-&8i3(hz_ktbvF9YuCdO1HbG6t#M z8%+h7Q|7e9B=SGVh1H~_h345Scz8r*sDF6!=~q6%?P<_g#w7&x%(jVdqi^7|wYLQ* zT@LQbgiNe8Eq5>+!L+H{toNQOE1tIQD_9KbJm!{*UgTRXa{QeWRAS~gWJbEMVzj{l zH1_fwCDPw z1!TsP)XQ8K;6`{F8_w1%_zfeZ)R#G15NgPh@g}@tZFtW>PklG38WCf zdcv^}EOr2uEeCoj8v}jHsE2y9L8aI7=dF-IN)y#n;y$FxVB4_p>O_EdJ4D%~l`8BcNZ)d(zc3q; z%$_vxd@Q8`=fh-&6K9-BgUbr03OINY)k5(ky@6OjfXcj^`An`SEeA(mWEtcxq=v*5 zune%raM9ygrACPGJ{NMM`Oro6K0;|J_wh*?GwQO*GUqAv2(O#fwRRD{Y%egOCABX~ zTM$I%jgtUVHIxZ{sIWU6s9>EsntuB&y*qaR`Z(9b0-C&h_U%2TOe&Zb76e7k++4q= z!(E+=&^s(vQa+vgz_N{VqvZr~l(JyY{j}JG)7(6-EE$6o$4n_^Wnqb#7#{FX>WP z`k(S6tx$J(t@0v;em0s8%mbEXn?7b_li{?B{v7xrf)}JC-2X^>P>t)Z7tmrt#)^;ov=q#(aKYze(8Wm>V?nma4bs^S+2rhGEy0FJl zHHdOeHe_Hohd?`ET|`(!cCH-e{L*?7iP=sI_2K)Kw4pXbQ%ZKNLN*Kg7a+a-AbB{DKD@7HwZ%4DUw8boC?v{X#YQPoYnqEYN* zt9^By&higmQgfFYl9Ec!0M}|%s4TkZ;qTcl_E=SI(w*ZwX?*Ae^^aR`XrE<5X4u3= zM@c&CM$y_HBRkhKV$sRt(!ZT(X>Y>(A?N&3hQ_yiG^VmC~xuuI5>hMgu0H@r=)O@{ELH1B6rE zoQ|8;+)_f?FSI#cJKR#%4BZ2bIp|xd|G97#p!YQA1Vx=(D7(*kh*d$LlYzshRRgRV zlDeEqp74HKSL?uSJ*8j9IiJ#-wjgDWEJw@jf}k0;RhiBzy^M6}Zz-FVG zi^@I?ELEf%N7AdUlENj<(?u*Om6ZDk^^;u2 zJRHCzBQ-KilB1aIB`@@*vFwn3;QCE4rE(^5f)*vB{X`0 zF|e>yvdbHql{`u3knCrevDwQP1s${eKo2g2Mg(-v&Cv2OLw9#|jj14Oc-$+J8lce* z#usXw3J4;N4G|rl&o=hp+njGG&&dqAa9zrGf3NKNo?1R8sMk-I?JVzTd)FYnXa+Q$ zu*FsMw!T+bPuOoR2Nv|EOyGDfcU?t%({?*oAv%gGl$~t;S234>52WTRr!imm{_MGN zO{=Ix8PB1$7}Oz2oseky9*RF2HF&q16e2c{6XTRjU*vj>)&ImjOb}}5bMKI+Qu5R7 ztDoltd7!4>JJS0Ze~#olIV%6WmhR>uuYZ@1p>uf-!%CtcRGjx@JF6YS3yQ_*&Bs3K z5^!AhxHKJ_qsq#qA0SY!=9$d+ipbK@TK5YJ9~I6;a+c4AfoIwoaha;aN%tE)ldOBS z#%|YrEFbR<+tx$Ay`RLAEWV5RwQa!NKY{fNb;P!dR&gvEL5}L&t^y#&K)D}EN#nZx zlp9p|XPyd6c>w2JiKnSL9(Wmx`nexpgJo3+4yeI{rTlw{sVaF=2#7SZd?-~})WBGN zS)uevJyoopKj_wTu4T@{y6fyjrC$EUU1%$h}JhN)K(jT`}m65=;!JvoD&32)G1R)0?ppk7RJ-l7CkDKuP-VMsviIP zWi7OPj+$TRj35pBrd9A*pDn#<22J zUX#Ci)8k=*Qi8%3q4S2($+;}vu$(%Fk~rjiMmEyKwnj>fIE;;Bh5I$M~O;h7Dwzq z(GoW^vBH^RLOi#ash*f@XEXqB+L4))$!d_e>KQ5Q$jndPdL|N1HM-+UI6o;3(fCj@ z6vS%?52*)R7;Wm2FN|%sv`(kvp(KaJqm)yZ+49}Pf*eyx!FSUsr5>tLdF&dJ$Ihr) zVWQH9qR7(lnLd#ITBX(bi98amG&h=Ld7TcFLG`-G#BxTfVM9oqK$(;gO;mSm?^W0sCrJ$m!nUr+7C%-2HpX~MmKF^Q^yroQ0ppanQz(ec$%*5u{~Qw%9JJ> z8x%71%9o=V2m20xg^&TmtHx+m;g={;ww{A)o4smUZTt@VNe>b`*CInNq!alEGoyx& z$mGq`Iak}{7o_#fRgNBEQB#Yi9LOo)_Ha%O=9qUz?roy)Qz>i1E%752&|Ei`)i5 z=jioTRt)O*Bzdt7*ywZh%!P<>cDk?YBpd8|5m0GE7N`Cq&r)fNa{Bc z1ICw(qG!*URXf3-ra}}W;Yxs|FT}uXR0-SAHk`NHUH5(h)%i#KOTUU=e|WRKNcSIm zwOO6wbw*9>Cst26zq)w&6Gu+`zY%96duls}g+kl>qHIY;0y=%X3%9L6&5W0vhtsj4 z-8z3{T3SMtEIzoD!V);%8_SJ1Whzzg;mU;nsD(7_Nb3|;UH7PVI+XomDqgvvuFJ^( zX>Fe`Mow5MdBt?P2Dsr7yFuv&`|9*^Iq~Ty?>tX^h@FVlcugvjl{Timb3Qi1QpZWE z{D*YIoidD8j6-wzL7>LlmeJ)jqS`z6%x@?QB$Q!Q+xe z?At@LH$viL+PXX%)ZA4Q(-xh8rCL96xZBRp`mRu=nv8|_(|N{x3{k4ReS{f#&%Wk? zVVKlatmU$pe}n2?Y-$3lFH#rEvLTY*{WdOnc?M_L;vj67oTMTb?QX=w*>${avM}`l z-iCT8?3kOG;DtH>T;;YqJ!)qx z+fFho)axoI<+%-@VJ9eb%#QXxtkN@snz^WGrWawdPKxvV=5td$wXT*B{J?Z}Zcj|= z3Nj$0`6}eaB_mDS-}V}mSnNM?)AR9_`i-P*su(SJ1c!adpGC%n^>$68Ur0(uS~;NB z%wWpeD6MoFvkks5{CK<~-DKs);nKUT*+{H(P045Mul_O#F%Mn^Vczb7N40lYYK)J7 z=IR2=N@pheD63~k|8sM?1die?vg=E2`?;K6Bm-pa zZW1McLykt$pO^g&6A@{^7bT)wIZG;PkTV?aMo*fae4ui;VJX~9_+>!vOm0k4a}x%* zLq>3r#eeS$th&L~H`l&y&I;3T)v5+PbZ%IRv9?q~A>Wihm*Pzva@|V8pY?MJ3me|b zGjY6ji4L8fdu2tr*6vHq+AwqWA(@HI zq2uW6QR~aOnG$Vm0GWc>yj9(U+ovOWs-bZ&2sA!!ny&^bX%+~Ce$fl}96t#TirF2-XvMz(tBQw$8{@6JgB8L* z{0ijEooD6P?W`w-V3x4x3caDkxgtk#c0WVCOd8OwC~NHbxw&|f$hv0I@b{tzARol( zE&}3*{ngG`tF=Inm*Ib*I-H&|K*ftyEFV

J66l>z7T20v&BrMUK^shcTlD$)2!& z5`D6QGPrTQv=zN=2yoX7`b0JZ<7eEw>PtzYQ0@7e|0=C)kk-{g3ZpRi)C^3!!<8%g z!&_L7FU-EiMfl4VX6cSGvAF_JKmka$3~V94p7gGvDe+H2qRvIIESKyy_tU1ewy`ntT`-A zm_UBg*vY8U?MMe_7Uy?s^ax1=NK1B&p>4w-mJ22v;h` zu6QF1;AMlTUPtg=DTq5Q4ilqNvW=7qt*aJlMz>MUgUQ|C^x8!=q3pL5qt;qSa6ad= zO~+w8y<)J~iAeegu7H(*wqRl~a#k}@5H;HBI|}R4%EV;+88jwmJneP3u)fGM!qqW~9qqvtrMC&5c@Vt-e><%2E{hc8W2j9nXpE zSeH}W-kCKTdV}0yR39>}Kj3j%KR=+&h_lT)O_fwROc@d*hRVb_S5>QM+7Fd`d0WkU zYwk~22hfs(={SaTx6~nW4$)&#akaWUu{^j#??-=lh&%1B9Tmi?Z=fS_4Vsrt>)O59 z(LXb<{S*b=v_Zhs##=bP*@r*%*c99$tFP!umy=wW6}C$c1}V&X+N4!1zOXU8d#7Sy zTY_c3cEAD5At+4#?VD99TXwoMbh4~9#SLl7Sa{=TE3&XMF^Dhvyb&h<=CsQ!QKXo7 z)18YnNKVe3!*{@?Mh=0pG? z`$qW`Q?+c*<3ErSX;!QKZb{JbFFErH3D zj84IjTP*^;F8_m-4kk8NGcMkh@6x~Jv1Z5Z5QTXzljP=KpH2 zs+M92lq;2S`>B4&>(!q4eW~aEBGP3_XWh&_3}83oV_ovBHkN(0P8GDi2Ap?$kAmne7&I#_)dNHkkzC{`1%QPk&O1rsd zUbhYT{U*5;6!oAJ38VB$mkvCFoSE$Dd6qQtqX7PHDcW&uOp6fXlPXNO43~`fl_`%8p2gNG+Bi8I#aoU!$;gt*+9#YT8Lt%Mj$frL@qyQWp1$eUI&FU ze1rAjve}byr0*pzW%4Rg-!i*vJm2ifV=Dd0aPS=|A}iy!sbiv(CI2I=Hs=uX-PPz$ zlFdIxZX*5aE14im+3%Ez0I_ zg_#CFg@mxU5*9Wi5TY>-3vrm)vKN*JUX+L;;71CMeL>KL)bn(n~$&c=3jD za3-?Ms&--&a5M0zm(DpMlZ5kVx^`mSa_lGDkSwh-?F@$Abk^E-=GOCA05hX#pa3%p zrC8^9ivn)qmU@PNx)%1^kQ#kxY^DyTVh)(y_=At=1RqduHofQ-E?SZ1P>V4A(w84*aYEy*c3EQAqM&)!X&O#eqbu6~>NBPS*(f#gxmnZonDAlT>-6WKU(Vsw zpi^(1Q-Y}}%XdLX4<=j6FeZ3bGdK_Er}*3P@eI(R$3$O9W8~BtiH-}avO?vrx$W_7 zUkHC|-A`Jvve^`S+&tYujkF4Sy2850t0hR+Sv=)jdh64}?q0c<)G*EY_ePG4zP0Iq z6S8g1Pu?ZGF+Ya)etxd7N-wN`67WW|q(LC++VKoN=9NRkgE#kv8=fdCYjT3c>9Tt& zM@w7ghcd7N1wTn1S$&*166H7Ewi2vq4~S@ld|7??Jm9T-nf&Z?daOPf2GUI^tp@%! zV_#=tFS2<_e3bO?4#$s2_X5z>D|eC$%Z!hhcINds;Q=J*b4MR-2boaks@af1Whr~& zfH#1-vPG*=u9`2oYT^1vdp2t5QrY`QRiV{ok1LK(4AhzovP+e3&Py@gg2}{0*OX?; zpxM^GX)zSf)^`{;@rd23Wy|}+SV&YU_N!3Lj*|i39F6C$nBa zjtTIE#>iX-9NsXR2@P6$ged;3@Rk)mnoyfL>%o+jp6T;_1WRMqrK>;=EZt7}eXz|# z#AjAm*Di{jKz64%R#cmv9p+$quw6B-RPfQ!=O2hZtEm5h7ZE!VYLH1uj=%4SM&$i2F09$2^!%e zl&?%SzAd>eqtriRrXM%I0*iWEOrSA96CpoD^X9$E-;7<>z;Ke{+&wqN?e!v2b+g}7 zqg!)1wd(NQ=8MjU(<+Om8Gi0o?=gN99wG>UgAvz9*JpOa0y)-y^A3#QCbFSYc_T7{=A+G4@;e?D5J zxRJ*n9^@=^$3}-?_Y4pN&n1q(p!;ayjnjwQF+S>lHhA`CC$=JQkA30}HIy1l=cT+{B?K5AHkwM0C(Fxu;Uvm{ZT)>Dr_!t&ny4`v-_d;l%`#h$k z(YI9UeOilLnMf0;Z%E+f;v~ApgDKQ~MSZp+ZPU!stbnAcNN2dcPsH_`MNG`}^=$Fv zETYL^qMl2^O)+1iFHbOTW~3%Za(5iw-r!=Y;jMyHE}(1omIm;#^Y5x@1y#PliW{+n zqHG7kzD=Nql3A>l?}DHgn0hOyXhof)FRi}notWEA73*y0;kx^wNKkhmiZCN)gZ)q` z%$toGlJJmof;l&gJ^I*|6efEdpQ z(|O%PXVZWqVjC0u`+Vz2q=4*@uNp9GovPL=lH@O-5%Z>_rD2lW>!3DVX&q%iQQ5Q= zSDG#xL(&CZ8x5PpPYkrv?B&c!Tu2h={NB!Z)*o0~<)g}v87iZz4QWBPdh^1eI@vlU zZB&IT?+nj4cGV&s5Lpdz_fES)GDIyQZX?bpQ7cmglO5YKhyn=+dLg!MlSAyl64MEY zM51nBpp_o1s8Ff&WN1LlQeigh-L9!#0ecPo3(uB_=D;7EdH|iMG5T6ixFHprOeR9v z5Br+uJAs#>toh^AH}2N=v^fgbXAEFXnNi$#7SrV#k8u|r8ae#jR~O#SpQbpocUmZ(m+J5 zps3yn=iyuxxKB}+jxD#@@-G0_uVLkFrtkcTF^RY^^=xDLL4NXg8u)cgg-B=rOO|v6 zNFCaOLXQH3z05wL(ucPkebf$6)&9Z8_Ci8GyvZO0$>G_jlN8UF48edo`V65!&qr** ztj;Qn`iUkkBwk8&qOAG7k5&}6b&yFezm(ljnI5jxNPA^rq?a{D$KKD#j^6fn^!$cGmB1SQBA0;IBVtO@mRTyK&}sj z1ejVmIoSId3SLz!DLuCMC>R;PTUmiIdQY0B3vWqTqAs2ohoT zO8UVZAHR=3HS0XqaznX zjj3E$t^Kqitc?nKUg1L2E5Q^+Yh)wxbbPa&;jpXNs;mDEO3l1*l^PT|YMfFcyf@k~ zz5J_WT5l;yYek?!_gJ~X2BA^)7r>y?BDd0KWX`6SdUus8>*{Pea{9xJ1>33G;fdT! zac3&X_NJkZ3;64=<5Ki-37>#_e<=+d>ra@}#QKh!;TvA<3KKw>?Tp0^M? zPpZk4Apnx3VvYK8dq%a2PsV-V3AKK%PhO5r z9t*!O?=kZ9#A5FUs2y1$G;T0_Zi%+Q_dK^e!D2z?|1ifgq>&*BjtT=&$IwA9Ki_P zklm8QjZtK4-!b?&Y@sD9Qdy7xSkOaGMcuVx>mUGNcybKx(D^n@mzv_%^@R94r`hX= z_E^0aD=(rPYyH}KlN()p#%dTosPes7FhKkDpCwD}n7y--W+dAut=iUgVEV6I=IBH7 zxrTj0TGe6sBbzTqw6xiZaDtmR?log za$=*l(hJ!Od1!{x-iU-82TO`G^$@*cTUryzYp0&2#}~~z=bR;-JLn;fglS~&LgtRi zM*jksz0H1$UE}}$hUJNY{tM6aCqjfbtecK;6~dnCiZX+jqN8UY%W4bQ-)q3_)k6h` zL`@dUoJgER1)?!nHB6NHRJ_3C$N|VkLfXvw)NU+rW%+`L?nmLCrmj%HR1osUh9Exf zUW+rCCFRw*&o}gy%HLF_{slZ82r0WOm|k}|!p964R;vC%$HkmYk{TMEe+CxKZ9O*I zX$LP#fUb-ct|Ev+T0I-kfz#Ykt( z`#?|X3~!uaii`*9b`N|%f9t>1>b~{Zuxc}|fk(!%wJDx4=r3S`5Jh;Ih5t0LQ-5Bj zqFuo`-gTkv#ymw_^nLp$(^7*sZsNKr<^3{I$jLzl@-Kj05&Y2Z{BEU$FXyutp2QGM zkdcWqObxhjqUyHOvssc&1ilIv>#G@DQ48z}Ot^)Fl|Q5i+d?())`DM3wEH|9Y~sJy zifcAvq=lw@REoHNSFGY^ME5Jy`_F`z?)o;8ikU6!b&%cDOw3=vFgWi?zynY{8Zv`K zk38=fGV8poQuwpkFsO`~wcgV?Tn0DK{?x<_fGt&66N;beY+ks|tguPMS5Rd)>aUyC;{(Ln?hWj7HDc=ZuN_-mQU+wx-f2z_I}bhU61tdcC1XqF8H zwp1IPWUkSk$>8L}tlAtj5RA``#Mr1`yV5ZRt~iaS8#9iEW5vXyyD!<2+^~W&4I8rF{lS^Bb8iFU>0M7|U!-|5Ekl z0qaw1WJYYY?!W->5;Mwr`|S%+(O!tZ+9E=ff=3hCn`UD zgbkQyZZK=tuXSVEE0nNQ*49bW2|E0$QlPf9R5?6X07f*G#pXnD_T$Ku*WfuN)uMWX zhqV#~31j25nne?I5$1`Oivk%JcPn2hD<(=!%uvG#bA|VZ5KOA@QURF$O3{qn0BT|; zfE;9GED$!Hzp?^x6LXHFyBYNJ;RsRJcTz=6y*S3SG~G0#leSMR^>BL$ZoS9fx`2#A zq$6NcmrpAou!mWwj%UBO-&dXGq!5;puFY(OzqX&s7IZUBz)-t#3W|Za9#<5sS2w-W z^Oe29YQm6_nJHUkIe50^o--G6gs@#^)YN z9lWT?3;7ctFNXFPj`#AHEbYqiW;h8gK)Qn5;frpjt;w;6oZinTc&{&r8qBhu%eMf; z>(ig)j_ivN$CCrOQZ_#CjV#vb{h?{PO9)!&$4;cyI|#a^vwo4(LmqBiiUZixglS0l zR&u|nNc^$Aao%>U(C#5Y(OaY79b7qty7_EkX_{P;n(J->b9Ay$tF$W1pNqeMuJ6P> zkq*OO$Fc>9yd0rcdcG4sjtKlTI#2W5l*~;5eeo3@g2Fy=Q3Xf?hj8e%()ZJ+8`ur~ zw#GS8_kL?pXAPZ2jXaB?7OpsPN8X0=ilSSe^Lu{`{M`nQ{Kg_JW! zAQhoTrY9_k(1Oc}P#XtNwO8IbQ@@IGUH~NsYdK?OB#YSkX@ZUqBO(8X3Q}>~oa3!& z!RTFmC4+GOps4d9-Te~%H2&rXeY1kKx@Nx8u&jHC!CKc#`gMjO8ILaLlm1s)8PsBKHh+^7ey)}GC!d06-kUGNBF)fQC^g!SSzVtvD znPpsnZ?>V)FNwGu3H&TOvz+{-XD~xvEPd8<3+)Q#7YHJErm_wC`XpuFZ4yPiYu8V4b!IjVv!YD+v#z1E@apxts32jk4fim5j!oB=(FZO?ou!UZY}d

hPAkbbTr(^R-tMK2 zKN0>_fp0py#L1`FRr&*5;-DDr4LLD|sj;tZQ3zKN@*cwl?-923bSUpcacn1JL{nYv zZtft-cac}hZN}n*gHxCHDFv?0CxyQfV9AWc=f(?@7if+U5rSJOQNp>2UTVfCY}SbF zqaxj|>0{M`iOe$Rt9u3RW`~8f`k|HzD zF^%opSyX9+jYUn z&%KiTR$^Vi!830A>~toy)LJOz!?S86`uLBg2Ua4rSk58ug@-Kly(2<+7h z$4T(;B=Ewob?78uEe=6);xm2g<$@*#FTG96J_{Tve(iRE2dF!0pqX(Mqorx|Jm0x? zn;B>Y0=^EuFo+Rx8l(n?tgXnM=cN6S#$h3#tgPlvKPyb0qr#omugkC3hSZIPlx`Qm zvZDfran~yU0tQVc^qq>L(4W@B9>`>is0P||d53}ZJl!nJr>ead>u!vlnM1|`e*Xms zx4$4%27eqmmG(+M4H<{0|CsP_*Ls={{hFIIzEq{!#hDC7*rt6dNA6FgHCE_&kKXSgyWf0T6hkPnzXgD zq4eo@6Lk4bK|I1#K+_75c0QPlwJo>K*xk=|54;NMt!uuNsV{Bfxmf#738k{MdPf2n z`^po}=l7!Cwd+$7`6niNYFh^%l#vXpwrONCp6VgnR?Ol9Ha@A@J)5MHD>i}aKH!!% z8#3+sK|2b=6#N{XdVRy>j|Qf&G6?aq;T^Z7iC#t(IdC_9tV-0pTutTGQtR4vtZYDU zbIrRyLZ|hq*bmh6^7hgK0&R>QPR;_(NVA&k=7q^Y0SYmUS;0Q<;zexTndS5N4(}la z?oNwrZo7FJt+-SD9b^q4b~Arrav(?Sd4|`mhi?prte4`S9DS(ncm7Ez)ZbnkPRDnN z0ZZ*Hz~hg!|1fTZ(Cb@S&`1mgxM1n)e3!5{XI)|f`M`@)BCM(`Rb!v<;VKU$ z$y09BH;tcS-v<~`(RqLOVg;Ue;R4Tq$+Q0{8%JPe!goy?$K<(N@mC(*`}#p zCa7nmM7qGa9Ny3?c8=&X=-Bn#I2G@oWbv&XBjuz)PFBW!xPRCuGn6O?cdqhk+}9xLsyJI|)dcQ7%PI=(?BoP&&tt2<9Udg<0YlLFU&$)A<=b{O_@mAR-C8?(@HZo= z(<%gUV5$Ho8siaRU9!-@v!|ZZgpF6OhegI_&2r5?H0q;?3?zPIjLab8y=$MZJVo$*EQ>u`JXy`1j@@{YN;(mL1~|&H)xKF6gTz{uF;2> zlL#8K8u5TZ0Y4w1R;%MP2d{s-sr_9T9M0c8=4u2ZH8R6T&m+xfBR;|_pJ)&h*J^~O zA3t*oxnV-%*;pn5=I7j(4pyT8EBv%daJZN26^m<-IHEHPgRlr}1a6>nI=lTzgH>Xc zOdpn?p1n9%*+jR1DYQF-!RDYza>CX={rqF7ujf0siB^p)mLvf$KbC5e62j9Ry>T1; z08;FyJIURCr|gsXCQG-TRi;$ESu*DyDUV6fUXMNCGxnuI~dAUJzO8#6NUx;9&QfT9Mg;f}KN> znh8dNpshkl)u9(Ip%;NKf)ibmGjBCzt7M008mx$5cwqO_a4>(~d%E{CXU4r{SJDYB zyeL}rZmBX_5Md{4x5A6TX2(EleG_d{KzzFdYm@zcX9t%pt~iB6`_&7G0X4U-Bv*V^6#fu1z zWdqkq}J(~TaA&G>H&zxkVtNSv~L%nAXbjFuN~x6wrrARi#y%DRM#cF&C30@RoX-OezPAI8PYw!62fhHh{vM*0a-Kb=qX zru|7^P<>Cusc1?9(gQ3jLoUl~7(+VlXa_QMZEj86vN^jo;`*-J{PSPH2ki$=5Nw1a zxE`6k2HIl{8h{;DTdr`ywHuEG-ZJ!*${#kbS(nO&=2@%7i8@Bp?AW?WM5r$g9`gnW;+R)l-b*$*?EEt+alshA9LgB)?M*w zf9@(tQX6ps&?#fF19D!_^*p!4%}b#H#$9h;Mb!u2&V;1zypoCKfEJ% zbO@y~wwB;gtTT%a*8C!&i{$GVDoP8d_7>nFTB&ozHZwfTzEOOZ_zVk;%)UjjsvD}@ zU{1Z*C8Ij0eMcHyrmyJBykxXmnjFunS85YV>3CPW`wQ&ppe3s(g7m@?rp) zS)Xb0-APx0`>mn4LACDGYF7Du?;nN)X?{)DQ~EDXrByui*{ z2sB27i8Tm!H%;3&KAn269Dy&8Y;MXN<=io=eRRnX@*BPLYTbS*yZC7Dt7}SVi-J!! zN}z)fG-JmBMF7`0w^N%u7UNsM8&kDkO=Zo4LA(tk&5q zy;B30AVhVuPE+@ur8Kji8QTsPMLWa+-2QmOXlZf69?_@PS#jb8~fa*B$!H1=7l9ZCcOK6$(h$eWkcsgxh z>1*)FVwr#6)D-kDU^;ur)hW^$JX%hAP5vK$`%)zg`bvwjA_aefac zvCVT?nnH>hhRyjjTdq&XTer1AP=FXePj_*vm4%#BO9!`FpUnN7EFzM5MIxRw>mHW+ zvj&&&6O*DsSdX_A4|#Ke?$v>(n>xQr)5gnOauaC2wsd*Jj5iTU$FY3j5`?6}N!E~j z%D_Z44H*9)0HHu$ze|FNdv0)OCl!N`MkyMhdM2Q}Mlw$IYPCvGBL=9DW zloCM%Qku;!(K}8ut;c$~+O9fqb$ixXY@P-@!_ZZ#-Et!XIQ!8-HH%%$5Io-+2c}*TocSG^Q>?ABv4?;0`?1MU8g{0INTGkZS4({6ik1m{yCD zXa4|;^{10K9rX{+stCGw3a8O?2YlABQEx=;@D)y?aQGa%(byl!ZY7>X8 zDO3Ld#Z`~mDIBVXe)5LFA<7mSEEfx)Z3eTRtG^u!4V)`}4Lfw|6(|&?!}c{xyGn9M zQ6%7ZpxDV2`?S}j&CsPSNm93MESEL?vN83|YaiLxS4mrMS4)dUah{KEac-m>rvs3s zg(wk?@!GN(jK+Dwi7|qi2ltEluM2GZ_63r!VG|aWhbR{V{ z3IPGWGmiP7*U~yv#{Mt$E6aFJ>6`p(a8OIBQd%oZs5=pm2`AVL8lkZw`EhMcLFya4 zr{re}jKi*!H~e}^lcW>4{J`R^eHnS_nYNPTE|tVMsv)I4DW^3k^0ctWh_=tVmXqz; ztq=Tn=a`jQWC9?09WDr-pxp$6sDQ>HrT)gLmTKaY=UZ)t9*o zMzv^za%)n6;W`*gWAj<9DZwwKon&w7C3=cSS)wP($La}fg%A<*YA8YyJ7KpFQ}G0E^UDb;imX{gB&>N>Vq|hcu*toazZ2@ljl* zx+z?ATkW$S>n0ty+~l&kPpwWxkbs{ROZ-c3$_h6bQa{96HljQsVd0okMgh+wvq)5U*02a9kg>+#xoX0b$qefu*59d5o2Dl)!R z=9aG$!cjW~9DWiBKwog64i3kB_;nuBZtCl0_SFo=L$m%yU@2M{OJ+H3dONb>Tqq3aXaOvB#sJFXxgqLH=@eT$Qq@=A!a)hjG;M^qPK!I0FO{L~9Qf4I_{#AaQh+H+ zu$&Rx6(t~UHaXnySZBjjT`tmBYLwaehHwV5b+}Rq(~+E=haJLwYw2%_HydA5baI>$ zVvB@CONjh-jx#siuwNlr!6;&4_3u+H0HNdRNBBg0+OKSMov!S zoPVts`dtSfdo@m-wxzoqj*Gd(L^a%GG^R^R%Sw!t7YYi21T4Ad83S~Y-l`o+r+R?e z;kLscwB%e~>3Odqpnz0`1x8xH<1DF1ZgihYyxAQV+x+{4>1E*!r^FN}1f|DR;$KT> zC!gLWzz3ltgIVWNNtGY3t<&v64@S5_i+qk2bdx?i%2-Q@7(2DCvY*5Pxvt+!CsuRj zNwZ=2UkW|<;cdp@c9@w@+?$^I8Cgj1U5zCLwPYNEqUNj|WOg-_b#3Y+vfMCU`PE0) z4Kf@9QcsShX0qI{V zLYr}!bdO?f(1|l8w$c!_X$ez{01|$R&2=e0$wgAOT9I=y%c3}OPfG#9p^wA~)wiJp z{RiHqrgc@h_dZ?R2HJ=V2xVWpTGF(Flb_+xSApE(p!!bnYJ=34%VegO!qhL>PQv7Pw9Ay=o$8w8f`sA2mRVBI!}hnk>Uit0#7Mfkfno+D1dY7 z7fpOTvUE>VUEj1xTUPsWx@{4h8dSGlh?%cGgFehW$;z(w_c!| zVYKuevGlZ=n%3DZig6$#OI%ta#gkjLlrA^ z3v@Dbp*oaGQJzLqfPIa5`La1WM`kD`vZ}tV7p|ph!PCllw&oUKsh|p!Ji1O*?USEx zTKWdlDbAbtTuq`u`N{9bw0`#**6K<;RRj2Pp^#OObfEV&iF`HbbMz(SiD>&1oDM3mzwxuZJ%UN6toO{Ht z+A?Ake$RCdmKP~3$B31jl>@@xJ83EBggcKl!UAqEUWhiHsj81;(30 zUSirXe|mgkfPaLkWR&PNgohcQaFWavKy8SByDC^KZ7W*Bm6QBSP)W+WBm}3qt7C*# zJ6(~sM^x7oUyF}X10+g;6tsSY(R)>)Sw zc^Pf?T2>YnjX}m1q@Bt^&XEbxy;R%@ETY>c-7`XplO4GaLbwLAf)X7{!=oI8sc75y zM;f!e8PneoVy6E9Ze^)UbssG1SzByO0RALvLW@gQMpKmX2mIC9iN;zvWpH--GI@>l zEQpdmc`_wPN{+da-;T6gLNHU|$w>)JgbZN5&?g(@f@(hN5*JvgmZrSVQ6WKUYCuE< z&RWsu3oYtw7g9XMAOf&R!Agp)mmYxlzta;-{SVc#CdG6Tt9AO?xKjd~-xwez)cXYm zf=-nbkSUig&visODZBLi2p701E=aYh_L56^8Fn+SfTZC;Ckq;P2O#Wd<61+)OSE8< zEH7GRWxI{lO)DX@G1SUV`2__@C0Pe4K3>$jb-vTd zHt!ADw%90s_YjjRJ4^`ET^P!P@rfvLc)`@1>c)H3KFqe_hRcj4hnD74*wv>TZj-tX z({F0>Jx)Jrc`8mG^JN1EIRxYU>*s&%P}f?^_^)@B z)Sx*TE}xTIZwFEt&J&Z8QhkZz7_X*&70A8V`fhJfD%T~}BSUfa(-jGj>eY?6Dgzkx zP_K;orYrYL^!wY*mV~#~@)IG%m$Z|R2+2EDeKTv*?)G>;>dkxHSAw13t2+Z;4G>15 zQ-2}b6oUi4nihEt5ka0dz@p>CQq%^V`cd)iOK^-Nnw+=T)N$neg(RBrG-G5`dt%lx zq|xnJ)u(jTrX7gj5|7TMnXzDF!^rwl6;LQf#(B|Axn*8IyZhFZn+KQheqxaAgkY#Q z`cX)37txkIXIeoTXeVJTa+=d-SjY#+Dgt$!VE*qPYBVc>NO6KTl?=j;=6h_E*8lsl9-dVC3(;NU}B=%|tNgn0{3#$)C;~!d=alCe*-ETQ3*1n+jlOmjKjj4qo=QSeg1q$PShZL4F0x31% zlby3f7QyZBUhs(oID zLU1xFh+77cs1kouK{`pvsTWbUJMw4;k%CThSXRcGbxo^0VARAp;UEm)gHTRwk_Bk({mXCb|4>sRg$0PL_+xN987pXWNCPgv< zuD5$n^PkR%YW19DRhBW-VB-#cw0k{6fIRryn#R$mvw0X8P{-DbMbr_EXZ-6jV*Uxo zh)2?-qu+K1#LWQqUsrWd8y-FC8s~Ue1Ov@Oi_!{nuOfw!a%s`A$@ihrg)Zu%?gF_K z6#2lQk2dt$NK%UNkbd-uEEgLpIZ35#bgZad*0ngIr00FJS_k%ZhULeuoVP3~EXso+ zl>r$}f>eT>5&;Msoyi~NR9mV;e~TI4nwL)|%RB197BsZXiM~!%-s}ZJPUAuDTvYjk}ATJ0S>Ehh2Tv+*TWN#86Qw#zK7AI5|#9^{w^WEH>SGz&1)&E8AF z8hk`UVND@m<0(*az!W=qnQ*T}~p@rPsp zwf$1q9`$I)f~GvAvO8tH{-eJK-=EQN>@Gn_D{$J(ha|EO1=inMo$SznNK;GZGV zq-zN|@iwl77&U1e^P5^IRw(^8!pq~R?TtuuDJoMkS1G}Plj0>hqpSY_mg2wat=IcJ zU;1Lg^~%9}hQk(X>e-0>&MbzP9yG8?W#z<5))10+Iv3dCKAGQ9bQGIo&0OtmH+YQG z{tA9ou^ky!OKrTDKvPOr;Q?Mns{smGHDvfoi)>4q-A~g!0V+Mt4rBElqGH{7^}tXI zIuPQD7C;5W;1I1F5TK@&MUG4MIh^9w%}e6P!-zd04Od>Ev8E(Yr_`U9I^#0pP9JZ^ z8>J4mpyO&7NlKNe0Zr9*Erq*7@NJXtGFn&4i-bFXT3l{qc#&Kzc%RIqvb2IfiLkH< z(Tdw$l-=W@jkW1Au2La3WlX2DoOLC)+2K1bXa3vtQ{=k z1ARRvRQn&Cywi}@*06>#f;4!_C9~zilt!GK%4sj98ZEuD)zgpIZ#J0n9JL^!W9R%` zO8xsK#D|mS+$lOi0bGYwQj(y$ZmwI@l!?;})VS+yfTWm_)Jk-y?p2{fZ$M2^EsK$L zd2m@vFrivTQietpRii$N-_oyt6K~ebC8}ih zBRa#*-eU*6OzPB?6@{xA&IE-3PrtQ%+QstWH{wA@%)1iohmf`(Nb^QpCsK|F7;Ply zw})Mr<58qdzE0V_{{Uz<=g8^%W$VXGseM5aH#trr1NW}7*NZ9TD}oAS0kFrD9O9_I z55HwxI$x-IfY_4t#SxoC2RTqu+(RxnpK+9s2L*ohru;y*C#bqAP0nzNi<8!;8MLys z97Hcd4=F0ZAvr=pDFCO)S04}_3 zx|TgFDLF^(Z6-#>Tqk<^H-jU=Oje7YAzrMy%VA+4o&3v2Gs(`>&&3uW>D%mUqM;@@ zx-Cz%>OsSaZKj(fpI;RZZGd?vIjHwwy=&=bZgODn1=ue$Icw76?z%sEfI9#ZoO`Q1 z>f9r<@i(RHod?@cPVO0Kxx>nI8wmS}QP%bO7-Uz~3ffm{yZD9%E7as^G9HeJ(1$^=F zR^blk))v+;&M7wOueTCv3xPR~>d+U39{Att1xt;_*`n2-T-z^DM^m$4xJhxxWkYf4 zO(VrPk}?TD;oNGf?^X02 z(Q;d#5@4BeOqW6nSD&Dl2aK;We&EaE5ld!E;fZFu%b8fHUh9enC;%9 zde5nthiQe?-9Tc;xX8JvIVELIGadsB68h9vooXl!0Ca(^cmSMIX7#PHH<_2o*C&j7 zW39|^vW-boib53Qji)#&Q#oxTI|c2Hs*`qRFIZh<^%d&XA|FU}i|$<~q4x-CT*W$xdixZQnbS&~Ev|6Y2eB1v3S4#MM5!rFrw%U-?+k&HnDldwV zw+O+&KfEDW8+d3)8nNMnTX^zsAt(5A<2^{D7Ql0k} z(`@*X{pRN?+~g~5&Bmd|F`FTGSM()r6el{LC<-G=e1T4HsEwh%DRmRQJq_Q6{{Rha z;@;aTDZAVnPBgl-oW}}l%;TgFF~|e}k%Z+&-q!R-+;v7|!zZK`s3>{qGJyX8ew51! zR-ZTb0ci?I1BGeIlz){w84G_~^fOm5i;|-?af$#ANQzSIbd$mqk_g}UicU5gRSqG4 z!<5HNPFK#P_)_Ow<0?mhjD8fYg`l!de-gRutZrdniPaFnRqk!oFj!7om1~w?2qa)> zFCkSrwc){Yd@hNqoqc*!fr|I8`-8FTFyPORz9l$4~n0UWofLp&sGnwpPs>#mud@)sn`a%>jD)o+m6jjiMydE&`R zlR}YPn2@ga_{mrAV7!Sj7poyjTw|}n$mc@9--`Iprhm1^#7lIvS~}~cZC7}0BgoA5 zi^;}tmA2_6dyHg-xGO}n{Ac)0)IqDFpmkSB-qeL4Na>62QH{s`?)J)1&dDTa81rpi zD3jlAcggRL-R6Ter=k zcZUk#!a!M;!CGHd1^`kV30~*;Navqgwb{DYr25X-!)y45XzIsetL2d@d_~!>ER1C# zdJY^p1QHTA^{#clP2FC!1;K7r`4PW-BCABlkPi=g&zIRR!kGXT+luQ;XTnm#zk~u#kaJ%h77`Rf5E46A(|v1nzjUXgQcF3_D5sz<5d6+&Sz#m5dZYay< z*vInr^s3F$&7)V6wfWYxfN+n`imF}hX~8-`$m12bf>82ex4T&xP)Qi)ZR(#oI-6cF zd98Z=Z2>Dfd-_&+dV&EY3Ur7{1yyP6GAX4ed}ALii+)ij`5x&4u zGKqo|0qASG2cY7fDDU}VxKKtxAFUv4V*xwq#X8=@Z>=R`JbKrf=bB(K^m-bRia6VP zk8cG@+r3IfP6B-?tU%WG98-x%$l{p?1A0(&0x6IvK0XF0v5a%IMkVl`4HVP*)11{% zjach|VBk;^5;KegMk_eT=7xA@YHAUYj*d?M07?zscI`*V8b|f0_3m&^$F&M1*~Ca& zV>vWqS3-fwpb7`lnRg*42OmmFAX`kD{j-6)nETU=-}Ec-ezj40LXnY@2fYC0$!)9g zdsMEOOEn1nMZfo)d(rXwlS$cilQVoHZAbH?n7$r-Pu8fNg&O>RrpO+By>rb;vDAZM z;(lharZ41U_mw9XsJO$2erdyI#>Y^ed`+rJi`v}b%zf)3jnqjv^X_V5CsDW*07+Q( zJWys~yHC-{cjzQVI5_x*bfjeYd?a zJ|hgsN|urU^5=d*{*)EP=F9m5JMU69xoSg*TS)UHg#*|hZ%VI$6Lx)d)KeLrmSkJo z@mTOig=tX)5rQ_x-)vXT9}v+ccRAlGsID$SIk((e3qb&RwC7?;0D!EIL&a$w9n}{J z$qSs*h*57spDu?QQ6tdqKEzcot!>dIDnyICHm0nVArcvP3qo*kr6e4jdvEKJUp?xR zSr1PIBz%+A##hdn%aeFZZjs`{@my&PHh=KI!Af!2I}CA0o|A4(lr8q;LW43}P?IT5 z6$Twn=~IVep=2X-woY@tOZ{~Lgvoihw|@Y5o@Ki|LO4NANYSL69&iV9wS5)+p6)ie zeJs_VtzC?N){=#UoL~?Xl_y|DYRr5;=y2;7--vxRGAzjDUq0evpq5=) zbQjg5`^HoO1!&czoTvC<;lBs7MKUTCS%l$gQcO(quSGdnhRhc}D{n)Cvh8WO1oE+Nzqz(Fsaf9k45>z`Jvn6#l}L%g7C< zrWD?$QnfS`vH?nx(aJ(qry)o>c2FP@Sx;T{ z!?zpKE>;QGG&gaG&yNpKC|8bD60IkP8&YwoY_I}Vns0e@Bymdi4}bW`__xB3Wwyq) z^*mV3++j4*RAa3H#Vz)8&Im5Jlq>?31%bY#(jwlVS*`+mEJLGFqP9qG(Q5qg( z9Oo%?v~kad;z-I>q)khHwbLFX8mX}}qD_lr|)T=V4 z+TzEOZP6h)9uwuvw&aJ~@mT->Hp)VO=5a%{Uu+3w-jqj8rtamOSbCD!_1X6x327{} z#DADbKf;B!9S4vy6Tztdn3&z?qHQ+|laB`Nw@2SWQ#eXAy8NJAN#6}R(ndkpI0s>q z`o7g>wI&^w$8C9N>lR|fej!ha7EsvwZfYc;PhLA%%CZ5A+@Q^hBBlnDv!Q7Q1v3J+J-;e3+jE74L0b`e9B7XCm278 zC$&!WUrW8$cl(vn@UPjkEmk=%E^d(L%e2xMOK8$g>se@nz5?7=(zR}ZgKv{}wavB| ztXuAOFZOvZqspk!(IVnnZ6IX$g;CL|Kx6lS5;>>IJ}IOuV%xHj@#n5OcdI&r>(P;| zZNZNMBJHu-OPFq3Sq``pqi-^aM#Ex|dsoi=A^RpLTiUKJvK)gh0a0x$C=JSCK?+82 z2+EL?#@;CU)#Lj_^p)^l6VijHM&8n@V+igr3z^AEQEDI%nP&E~&Iy-nZW!nwF%^jTzYu z6`^jn>KR+wRtdsgB09kYlAU!_z2DxE8o|`PM{kxIgK=rqLx_@3%wVkzA$Z*&A)=GQ zd^3P|6&=&<*}CFOjzV=Xo4#L@F{tBCrkx8)T2gRw3fxK)jtU6pQKx#I@p@IED}g1Z z=Ank1@S?V%8T6_|h-?rOpW-1)^yiA_;F$K(q7SGnuE(@Yi6OKpl4|0I`cBV3tU7J!@-J5vTq9aguo0=ZLroV#kC7>Gy!y(r1`=15 z6QsL*Yn+>`X9Q(8G)`r!dns^)FykSl$du}pokU>1oRTmUui+V;>DZkW8L3@QbcEE1 zw&gi)>HA!l6j0<=;s+ZJl@JzEg!qe3a^qxzNfhwV`yLFW7 zYfgjbOj|XfASmfS@S9LcIsWm|Qb`HiR-tq=-w~dXV}o>JJ(?s%E%l*~4Wx&OQr&m( zB_+h<=qgi%v>+`&Y6c$H)l=pOPxT~8jH>LnZa3c%M^w2AD8R_UEVkE*tvqRM9o zpBe2oOEi-z(jT?NWyuVil9!pU6{L9>NZlZ52g(AKfH56HDF_8=0R#^=3u&|bMCo}-g51T2<;ZCU+=dvqC&D>W zl=hLM%ruQWl6TF0hMukKkBe}jbj&M=wZf5YYDC$qj*?t))2l;qf|M50H2_PeO4c&3 z6954vbQev+YJHcY`lI$+ExZ*zU&x{P4?e7FenYyDTy;uNT3aMG+DJYb+bzm6Ku%FU zS-R%cX-;qm&IM^d8onK?sqPl8fwB}r>FY#9Ou9}eNpX}P z?((#R;lkQhu#lXRuymZ7%x~Qta7BP$8b!w2V}P{Bv_ewWkmAltQGvhV%9N$#lB}F$ zR*Fdv7Tcn}j;FgL#CqS=x0!b39e=YfvKHw|iOKNZ*4VB zP*Eg>IKqh8D+6T?Frl)&0m!9Iha%wPCRyc#k>4Z{w|d5Y7JN917I1N^QSf3k9Z*E1 zdCouLx!8Y&Cw?pH9*Ik@!C~n#nr)_(l%ce2pTawBx1qoT70(NEq)XIIaVQ_!XQ>%( z&td7O7R0Fxu%8g2B@ZJB0H~9cC>(K&``67YQ=TC>UjG2i>00OZfW4-7TR$1cK4DRl zp(-c|DMukW0B7s(St0QpR&YQV{cBnLHMvG0N5QH4!BvYPyG$QCeNJ0r+XC;?JO@=V719xTOk0 zm&QFi)Y;D@Iz4?U=8taP;+R3tU@7*8eg3qplM6`$1M;Gpa~q5us5A}vr`iNyjke83 zNESu90sjCh4b=C?){k{K&P53FK^)i7Gu4YD+)3Xw!jLnZedxzi*kYL}89Q@G(MYgW zCw<0g3P#*fD@Q+{(vVTMPTbNA*dmdiT1s0W=ZY^t+PG1;q6Tq>RHbJdpK6wj%7FV& zkkSe7QtaTQ4slG7sA#sV9gQ;LPI=mnKcS{pfJxe&L?bO6jFK~)QSnqb%>n>P-`9ap zvf@I48~tjL><%P}$QzyLXR;1+k9v4{hH00G!5QP$s*sL91bm#})N`csQV)`H1q8O# zPShFCIE@7XkbkW-?mJV9LWbj-X+c=r;EG1YS|a8k?gteIEwTy5)n$;zpiZ7DB3xoK zw-p&tM9~IRq3SEil;TGx=}x-N2_RDoY=O8m6oX^QxhXu8MX`7w{vtm04zvTcAR%cv z6-zeDqzkZF8CM?lY_iVQ;2xEg#apO22N|m!nfab70uD~xR@hO$bj#75)33;VNUOv| zwyoP(AbM73(a|LqQny|uKg-^)uvu}Hr+^P}No7*eYHTOL$ymUsJFK$abSUrdQZ6Yi zM$}dLnyOsqqs9YYC-bI`#+45@48500B>Eb;biJ7ql9V9;IgEPMe!>wTgt#{6aaw0g zL20$6*1}VcO*?cV?381pt~R(0-yL6skQ5T6x$le=t8ROWqV?;-U8Pc^v`f$zt8aIwjD zIU(g3uOVTTXUtQPuYW=^2-wxfr}_usUs7A2LM6^>#e$?^p2OD{{TW?BPLC@7*n?EqjE5aq+!Og5L|4d@iwqYNE_|7 zVUaqA;v$gZBf9K}6ZA}N#@udW4wNWiTAL+$c?C(_9D)XLXF7)E)K@h^YpCx9*|w)r zyI)c=+LJL2qjvmk>RM7pbqu(Jm1D|jR_l{q1D!v9k#39?bJZ0{9u#KWaVa{Gw#iFM z)T9p)wFD?Qm47m5@<{@NXWTlS`;-9KVf!@ThupPJXvjlsD32Jn=5msS6=VW3k`J+R zOIGA{1-@%9NsiQ~T(`ds)hTId+@nu|<}#3=Tv9z zbIhsv9{vC)i~8=4Op5A$LFvwyLJ1xkl(N`q#o&2P_#EytxT{t3rsj1sFEMVBF!efC z%QM?b^s4t6rj+l_wd(RPmR`Zin@DtIrF9u^izBEYe1-ikMsXyv)9QapWU}c?-UGQF zVt|vS!RAh!`GML_Me5yQHduGq_c)iHl4TRwkje_)Ub5?He<|#etb>v=Qa2lhjJUe( zeo(D8j`ch#SBVxm4&~bqQpJO6>Iiz!!VOwECeM-)xcsh@YkTx2WG?F&SNynPE zPMNv2)v}}8rJ+iO4&jN!ytlvYvRv@*T&Dpp#td@*->uI zy>FJwYe+B0aXXJO8B$76v|!{Z0~yVIukg`}H^b{ao>PnYrpo?(_X<4tURE4H-1$pO zLbI?gBaP`ZW8%kz?G?_rVZM;~WoYX-j4k?@?eN0A%Wyea@mdaF?HLE)j zkoO!NWaAj22Nr_cx+`A|?bjAMC*p5Xbk|cdtK#FO$d46%?ZiIv-+8n*z!_6$!5bY| zHQt~2gK@N8Q4-rZw=GaFjzwkFq$Mj2$Y?m?fZM|HR4oG{{ZH9-lw`7;o2>|OwUr@5~r<=!I>Hju}B5K0u*JH zzO@06ubFX50Y@iEC$%%8lc+QyvLVi78$Z zSPwsq&ZpCoPD%2o03ZS*CbbZ>p4n3PASQLBewmAHakvf(t>oMqVUG>hm}Qn0G&pu1 z8Au5~@YX@^R~BKwhQ)^S9z4e4;7oB3Ct6vz{L7A{g%gFLBq_nGB>3nX5m8?WM@@A2 zf`4r|wW;TA<=E>_gjFC&aqzK_NC-}~{{R-CgpI}t*1nsa)*U46rgYntzDHEtU!3#u zn^P{^`hF?UrrT=+!3%Y!nQgJUu-L%P*=|+i{{W)POJeQwFZOiLyeYul-h9U)*i3>% zcA^9&Z%kTF{u6QAN;0fuH2(l5t5;k7rtc=-)3zBh-eOcXQ)FT1JWRHyb(dG14Os=( z3Xa6AZ~zvMh!*)bXm-w>vATvEx})Xix|hnt3lt@xpxJlvWzxfnQ=278ZPfrno7evU zXa3H%*`>D5FBb@3B!p!6%8ulR6qKZ3k>VsQ>c(`Se>G9HLM7zo=c*f+^nRPRnJ99= z(Q)EN9G1(gTFx*J_lnY`e+X=)CYfD^={7qkk#hNKbI458!)c%r>rE*DscQUKQrF~4 zyt&+rBnmI1m__zw=ch|;N4i`SHQ92Jr&3yEYdXlrkV(K#a@o$wz=%bbW6)>XS95k$ zsS}w>173Ga5!9dEB?Ek^Wwhk^Q;;+GjZHn$9<*3`FROkbvuSkOuaXwK%0$PI%H1oJ z2_c4*r2t!Pv@0Yhk^G>OT9-|~WVb+y%>Jih;`?l-NJ~U07j&q&@k%mMl0m|p#z4s6 zDznF-TP+qrV7JXBnT*PiyOXip2$ZA{(NOav{oTBTpdarvq~HQG2R`Dv1V-#_B6Foq zh^T=*%?>!0LP=0z8j5#L$s;4WhXknd6G-|`r&zYhj_tbW-L1(=Sep7KyMkU&3PxKQ z*e4&4QqBQ57y`ZVzR_Tn)9aHcyK-2I;l@L4t%%W*4p!nD1R!sp-nvK_*yjm3=fiJ{ zms=-=G{&;DWtB4?)XOPfjUb#Y=@0pwBO`>AgQZ^7N5=mEjonVgzr?qA_jjf&G?gX8 z9s>^ny1y1zft(}~Kjq$xewRk$Zqf6nP(IbF?DSYDGi6P{FG>ngKs;c%q@A|}gzh*S zRlniIg5>L;v?jUZ&bK^SM071Jq1`;IK4mC@u>m_1&1P3y@?_q`xw4&pV+Cn&tbwV) zp8N#)j=*-}weN%yXI?H5Zf-_mL1ivA9mbThl9EyswQiJxZ~)YOcFjZ7N!25-9VNcc zq`nzJyWAUH-u9^{=A7}Ff*B(K0urYZqp}A2QMVa2KWDkR$5cs6w?pBCFc|lKW~0xw)WQQLXqOQ>1-)rdIPdS*lpUg6#PVqV`6RIOHCQ~Ql$8K@c>XK zAm;}aEL~`v_V>{V@e$9P>np4O05n#WK1vsWay>TZ>)xuQYIG#H7me~pKRWaB-kS2B zWhp>$MByN5-?!43LxB$Nq?4cF?N_6cTXtEk2uLXy`qk#n+D?;}tH)4eLuRN`KiVv z4LsG5loaOTqUK{5qt^Jr-l6U?01DoTh)a^v0nZgny||JU?^YYMu$<)8BJ(>|q7M}| z_BB~h>LrXT2D1L4SOsA3S_S$}vafp1@0d_l{Blh+Xh|9M-9);g2Hw>SK_ru$b5=W? zZEI1+NT~A&o)Sf87e_SXWJ-qnn&y=dFe+7WGq@P0%oV4EoK$2&Q3GowTmFPUevrl>h)0BEws6`qY~yR~-Y}y$Iq+Ab>_F z*E{-9%NgdrjP#-g;&~X&Fi?KfV$qMTDZK|fQZlK6sk7;dMvPHq2apXUwtYFI88A&9 zx%*cN4mjqD@W~?pd)JW4iN-4e&cS;qIxEwBm~MVxLQzoZ&S-idh7v$TW}uASh98vOypwhEr-Csnmj){B$6kovO)a zb+(;KjtCpoYB*3k=B91r_^BSdaZH%$(YSO+Q^C@f9?}jrC;e+ljU}lmNolZhIUTDK z==hj<3P2o;1s$h1wE^l z80m1GK#+X}JQ<`*bmg)2AQp(uz>E-aSG!~jvod2fZkn$Z;t*T;ojV?RPx;b%TG*ct z0U;UOX1jHIMYc*7r&vR2BL@q~KWZ;#iK=x)<#tY;jT+?L%3O3ZH6*wkPDbZZ0YHFw z&0`mSpoMar<7i>Tpn?{qqab8{6@+CV@A$UItzNpS^k>*3&TihPVg!U2SaHOy2OE_E zl%IUn8Vsv_!SvdgmV`)cUPnsG0*5+D$DbpyA46VGsBO-Uyh$Zw0JO`t%WOls8HJ@J zswxl?^KE1CC&M6+KbRekGwD{zy+L+Gp~Xw-L3w5?XXTe7D@l~n$R2jzK+hfbJPksf zYR?!rBHQjV7<9N@Ojyka769<{_UuWCyvn^#QLgBjChWiO8PtM0)>HujW#HS?$ z9YaY5t+t+(xgzYjJu>uLf}uxTt6Ygpr6FiY0c?kW!jZVtf;8amgUyGcA-i|zSn#d) z*jJWe$Wsv(^hCD%ZRjd#w;n;%Qlpfi%{}xeCz_Ip(qXhFS4PH+W8BoQoawo83f9zu z4!;bb6samstQ28sP~#esLwC`ZyD0H`j>US8^c#v-CRL{G8RKhkCAmgA(&9@2Ux2!d zl%sT&DYWTP){#^8Xx9_g_GBdtbpv)7NL)LJK#r-f-!3b`aU>0GsYxK^AhhsC!z38_ zjP#UQk*gEOjdpCmYiJ;;ECnURyvh*NfD|}!9s{@s8B2f+ zqzbup2TVwlW1ZAChoVQ4-O4S-Gj2PTq0p^sV{l3wNiGyPPvHTzgsCLg&$Ip@l)o9f z-O4Mdi25vd0`4;PzEad#dBlu>rX&!R4gz!(k&3E=m(ov??G+w|_)XK-JFvFJY1GbC zhW=J-MJ_BlhJ&oEdJh!r!5b;}sDmq`skjK{X1Na-z z;?;xbJw;r;2lT`{z2Z!jxa4tZMxceDt#M^G;PF@_l1`F^Cnxw->5N(aIrJpnsp!6< zm2FO|q_>!Y(*eK03^c}ZEY@!_KIUVruGz+fV~ePjRhrmRu=b&G$Y;C?`46PI<4UQD(f{V0ARF z4l?3Hfc&d`xz85-1-#o~Lv<1Rv@K&%{{V^N;~P^+H1sBsj|+_Ze_vRxvykGXwCI&2 zvciI;uv|-MQC0~+R+p8Xu#!i5)_UIeaocmD2k02#UXcVbyDfH19V|)!xD+6;`_r))amrjF$ zeCeBGNQ~t9483V}nBE#trM%%wq~irCZ)6U|YF;vPLJ7}H!KBhB>>YnD4_QmFT;7ry zxCB69E;w8vO*XfWjilr{mzUxxqH-0dSUN}$-`ZQLA8V)jYTX7&ZdL9Sc?>dMEw>6H zJj<5&In#|^8+x}SKIY{}HE8$|@KLY!H>P^7?h$ffyPV0Aknm70NJ7b8H_6CJBpr$t zu5&{Cc=&w=&ueneHhFP{Om_J#*H_{=`whnb0CR}92nPw>eiC*bKmo=>rwB%68uRfpc#CT^7o#!WfqV z;^pJ1UV@^ez=a+HtQTm z5|uOar3KgBP{v&F)Z$cj0I5L($_A~1&C^mkL!~Sq4_TiP9YWywi>$dP#aga!l_<1Q zjo)zyTjxnu3c}o3N)nKx2)g9_nr&YcReE-O85b_5oqLGemYbv~p~w-`W47j}63j>q ztw1a#O~z7A;I|YwwH<2|CF8Et)Aufd(+mr-8}TWLs8Q0k%@9Nyr|8iQtpyf<4}UYMJ}mph}wY-hLV7WThzB4eO@3_i#k^3%1AR-_xplBU>Pk!==HVJ zy)d^CbDq>e!EL-{)ta^H47j%*%G?X}*O*yN> zLr8HskEvQqgso#E1xr?}fB^<$tDtz@MQ^${bdCE~ET0?{f{VyJSu)2Z;$-b!hN8b)l2zgLEM$)V`87d)eq%RttJE&{Y5(!X~*F!Dn zg}imN9-Ml_^oYrN@TDc;9Z||(ZZC*d*jONj&H&P#&Z4ydkc4737gKdVQoh$n+2=@YhFVjpXXMshxM?Q}0idANbp9d<^AVR8 za1trTx)J@%X4K8<$nxK}^u5ws(p(|9tbl^<`bw4%Jcvp_N>CN0ARWmh*0Fu*Tc!U1 zrQT($YtS3;v*E*vqT}vhX>A1g+!^5OLJ0(&fTeYV7JituL7UROO53;h)Bft@>#GTl z(hkaeAt@OrASDSICz5xgI&-OKHGONR(KgA?zT)qY0*@oVz#cN>DZUcgA<}9g*2}to;*_>EU!5WR>T91^zYBE)g3#cH8v*O z_(>*y0h== zwL^SP=%y|GJj9E5Y0m{j0);09B^;NG7BqsBO4T_ zkJ6ZpaRGi&fx$WF>t48!>I;r6B`19$q0anwsDB+ubR-o4wn3uN6GyU5*QW|UonLJ7 z5_FvVb6EYBZj}!;XckH2=^BTvO{P*T)h%*{5;OJ9TrG1jK*`#(D_rd9IsIzcZn)_K zCwiZf>WwB0%p@F&n__U%Jk>&jaoZ(TGR4RQ?y7Ym8Z9o*0Y8eNE;j%M=74_bfsE8F z`<|eok@?Yb0z(&h`hoL)wM@H7$vf1Y%6_bne_E^DWa`ci)QL70(O$c~t5*EyV^r&+ z3d(>RRjzp|Bx63Jpk+rcDEsYKNRwqr_=QV5AIi9kW=|O%jad_HQq+9Ad(my$PCpGZ zS|-(%GidVcpVFCj(yV?CYV?aiC-DB6t~+GF&WvWc=BP&mXAjzr{!>CC-mvJ{?Y9-6 z$+PLm2;5Xtmd!3?pbq~4YJ`@Kc|!UkBf=mLE`8|cVT6pS8T!=JTX-lQUFflFDg|0b zYnirkxa`T+GtD-%_9BT=H^wvml#z%^`8X#?*c?}}w?oNZFdjurXSYm5d? zF;wJ><76X@jm{`il9D#S^{KaGk4lezImjSj6In>Q=onk&HsY(4hv zUJRUUIjOlEQ;H+qV>}JLDmp}Bb|S1Lf_9-^iIKsm%48mjq=*A@Y93q;a-Y_&k3<8J z?NPEIg>QkqMaP6$EoN=cT0#dQ)KsB#f$T1Rhgl)h%tlOg?Je8wT zQd5rOiqo2H5o3)=*p5M}r)NwE%j=Dl;Ac6ax<*10vXyU8*L8TxA-w(1CZ~EL++L7? z{>jeuR2dpAmDeAAJ{ZDJJ*d8+5zI9t;4e-Bi6c+mr6IL!g4$G-q$>ieJzXR`(u*oi z2jV%~+irg$LwB=8_e(Cqh%#81>;5~5!)r^(TX`AfK!TsRrtl&>89{xu=PkO!=tv;? zo(Lwr(KD4H=NYwIVT7Rr#gXCos4Dj^0i5r)KJ>fgx)XN+Jy8_X>GNu5LfTh+08f-{ zzBlx*FV(NA+1-ZdJ;?XV#le)zA)Ap`io}K#rAkV1_?%ct@lZWRahw{kxeDt1CtJQY zsIysdd3WU;a7jXvHwsRjlr|wd?ZK#CoVvBucNpltNKzS!vRe6dG?ErElz^h)+#c!0 zXjjN$-x)>?ruO4HJR(`OB12CS0Z4Qw0X~6V+#SVB8gdNUj*Noo*7B`ZsWNQVYqUk3 zY{GR0L#*l^Qd5$U&`{th3C0f~;;MZ@<*%q6Y5t|XrVRUoi1FPFINOoc{5knxhnr9s z0ZMI2SSP({(`NOBvDVhb$EVLz4eGyU=(3aZ>n6!l-??;4` zxU{I`mfS|%g(XzaTl5W{)g7p;b*;JEjm4odGf%T8Q;SIet(P?nF8~tS(vN)-@RE94$($wC&xq!l)K1Wi`&NE5a zp|!BIB0- zHFv*s_ruh@;{IaPt58F(Eep4)))TEKTG=5=QgCyrwT0kr5(x&edo`ASL-kB4&>M$u zwaS*J;Jy;<%Ot{~%3W{wtQ8{!rv)cYbf*N4h)Kn4n@U|DMqaGV!H0Lb!ffTlEZ^Y1 z+FN}D>2bMiDGDvRk1n93Y38mREgJw zNV~D&lH$^+<&`_DxN}=hE3hPyoZx*ccelNDu*I}nTTy-Y&AAFwX(>onu;YsCbm78A zdyMl~hxSzIxv+X>(PNa_K4vQ^NA468;xhIY(q05*Nz^pz17H)_*8K9_QMgJWOvGt& zJRLYn9PwN!3dZL-B}cF-$<9S_MqH&Q_b-ujesBDGwc4RwtQT7w%6YqVnJH>0N|4Kp zG*aA&t}Q?fDp*4efChXb5)RdTjq$Zid^;9vg6A#CscGqGYB*VS)*S>ftZa~_wvQ28 zkbsnp*NSWVRoE?%I(x!)EVtWs;}s|^C0{Wq)V5q-h@}tRlH-dhNf}BKwQ0)9s((OO z+5Z69gytqT`%>hkF10D$+hE9ZOHGb5;ipp6zv5c4{1s&=@n5*DREX|?>PPMot`|Ae zd-`U&Y)y{v# z`SIk#dQ1|D_Q}D*2?^Y5QBLX!Ep8LObnY`w z{SfHhoA`!{G^ZjrZA`gsVJ^KXNn39^GN%^dL0Snvz=W0ga52d1p5~~JQ8K%?M>E#< z_0OD`_In|gwwG=bOLH&N0S-I!4y7f;w2XqLk_z$?6dMRg6>gxt^h_DjdPdy>G9raJ zn2p?qQqyS|irWaoY_tFxmgAqsqC(W8LjM40sqesu*Gb6uy30}$(Ta;(EijE9UXh$BZK?L7bZf|b&W(Q>IqVgkenx(&Zmpi z7dt#q44pJSsOc{bZ5%coCXrnHh{vv;@$ za&d)~HshWUc;O@|(!A-wNx%Z<5-(KQS!PxF_;+U<%MhE6^IssyCk0NGr@=z>fD(kQ z!lROKHmOVM`i=T@jCAxki2^a7wFPugR$h@U*6S7N*eRw#jWPP$wtQ zeJi&Zks(SB%9845B{^+^^GZt`=}USOr)=;Bt@)}o(%~@|C9{8&kuAVNO#C5BPIekj zb$eo^#a@TERc3+H*DJ(l79q-u;4tPA6P;=s{{RZoG#(E1eCjKUZhSMl-Y?d-kjr>2 zWxo)%;TYAeD#lKEAd1LrZvic(yviCw%Su5F2K4^`<~xtIXrC0P!GqBa+ime$dC3VY zd%ty%L0*h~pBS}l<7=_r!f$od12eT`R~ktNZk#qqy98kv(Z z!s5c2E5YAeVLwhP9#T-JO1D8LI!MU;=tP5S#N^lkSp;R@N_pZZ9^BRYlbQ_Icz3LJBzS@emsEJoyM2%VA zia>HDCfq`aIr~&Ri8}MW1mxngoaT@mD&K!fv{5$BJT>;Fk#AF{AbqIlvCGGU_Nn+5 zu%Hf{?YF&EAv+->*^b^%(vND)0l-PCR9is)UHxi7yE)hdewDi#tfwN*Mn;3Zc~*Q3 zXsgm~8-5W@xoQ9$Bz{#nM$JZZD$1S6N8i?<<=Ki?l&2W&T8yhu9H;*P%}}##ARK@z zno+G2akf#|x2Xg0??k3Lf_SR|w*-^0A8Hm%aD^NVzLm!y>YcVBkh7JQkoe^j}gXq2A)K2!8CA;amF{N z*^Rd1qVf&M4H1FCz|S00sfslI6TL_xHvAfSF~%@Qed&F`a8;oAvW9@{!o#c?h9wg!$2?TX?UI)G5!!o2kT2Lz7AQDVT|vC=}C4j zMsZUqh^%VC_O7lXvA733*F3Tuk}A(d=gUDo7di5u%B>GVIXh(WnhkJ?BoVbiy6{X+uMq(+HFf={YX!gsR=)QJ{yia5K&`S@&Eymx(}cPxSu)S{JadlfMrOHwTpp zd`6xK!AMlM8943bN#7OQf^l~cpi&SKutxzsvDbS z>INhLpBhU`j2R#S<{HvLD#mvl58=&hx5=F;aI?QGLo?$(G`Zr$`Nrd9=Tg>$1*JLQ ztn7Eqe328X?)Jynyg1XyLqzCX(zU4P@gYPcD;eyeMl=1Qp_uMDbz<1bXDuk1GCXIG z8PeLb^f>wu4JJ6tq>VJ6X5!aQ%1LSY_WMj&vICwK-rH_Zi50K_8E`X|XWaeG4{+(J zc4iVKK)6n_q-8%c(@AST#I);98_uDyImsBuHIc;l`4aJHofQ_PyJu{r)fp4k$_AY3 z0mW&?Nf=4aPaxECem~f{owm@cauXqf6A~zn%f*Zxma-5MJLv;fPUlZGr;dLHi(e+& zU}vXovf5h+>Q5QvtHUtR?1>0K!3`H$ASoSya4#9}+faupWP5x_mZT&);PIM|axXC; zC@b-(w!k?~dkmlE2C;sc_>Xas8R-z`-mPh6Ki;k6wzrfA;te!{vz_@V2OQ&SrO*D; z9}BGCFd0?bsXC8XOod#eB(~dOSqf2cejAD;0u%etFbQo$U^WHIt%E9Q(>GqF==+t< z_M64)rEXSYwJ4(D9zxKQg#HEih+154Cm<~<89M>Fr{{b!v|V8v)EKdpw1utd7RUS_ z2m!vPRFW2VI}%5*8&}3%bMbSj`hxJQr%qhDo=w9@2%6>P_Srscg{80>(}Tv8B|8r; zC_6vJU+o1YZ(eNHHm)}dQjsQ2&N*?p#0BY2rx2v2%%~7GA!S?S5yg1&`cfxccG>n% zPv2z3LSBl>E>M_nN>zYPoTJ-gUp4;#XfBSn+xpntBijC6ETuN< zR|xL-iHx#$3rHAJPI3Ih1D+48f7_+m3T=;w{TSqVtsM?dfSpQE&V22a44i?u)87KL zE}iR7hF=?D{>Y0dhsu$qEV$x<3nyYm#RoXS;1HaR^ISRSlhZqrbai2ba`g6X-wh)& zlOAV_6qKO|WpS34(E104vY>(dszA~>^AX=QZbk0bY=a%U>fD!bag@3tv=p_r+P?;r z5w=_*I>^RUH~`}~={}o(rC=f(M5XA-W;+UaYQv#v)U)ZVX&D;|ExxNPGTZWH>K8e3 zJVz4w{o`S2OMZL3AsCNfpdv*Okp4<{WVJqOl$zM z+BA+Af`FxY)I7vv7|Cun)!n`wUN8MaW?i1d#HFn-IaUV z5Se|pnZlcG(4?}Kk&Nn7XI8b0<0~1?anlyLDiOC-W*-6iP08}5E*p=-i++k>C7DN4 zo*`jDD?)pymqrP|2}s=5*YIPbru5fWKW}~%c;V}Wx1i5$(613CYFnfMwxo~}b$M}& z9kW$`hnEQb94#(iVW;+$jh3RZH7nQiPI&kX5yq_qDJvP+0o!};Nc9B!=S$iqwGg6x zy|A^y()01ac!_?;5xHZmR!=0;3?E|uUczNBP@CtAFtVdMo^ z0|)A^y@7n{nfCsgu*Xz}p(N!oUSVzxK!TON`<@F21f6JU#c~i6_^Tu6D}9&Y{fQRI zH!={@Npj-y){v)_&VoQXwI4ChlmMS8$OeA%{h~j!_d>MlBtL4nO&KAjT$)HjgdZ}> zjtaNnAw&;P!&Jwc+|vx;)8vT-d>*lMOU_(ww>v$NX-Yf^R|)29w^B2fS!gB2D;<)7 zw)`E$=S@k!^(=I(S?;%6bCDOQ+$~xm#%5|kH z(5gp;zX?bwK3o8Ju9zq#S4TXfk-KWmK3`6Bdunp@rF5H>)2)uE1=x|*;9xfFcE~tVlD4os7LF7WalQ@-&MNs6;&;QouY(MA zeTfm{OhR6RB16bQ(g7d==52fvq%5ya`bG{1>RXM8HiI_795{y%rkZgpdH4>Xa@l#M zVL98#>Jmml0E(kZv`Rd#q>A;nRfnnCL9=3jAN2B zRqLPa_${NS<+IXKZLhLg1iL9-I!dstJlmzDWB_reZuzI#lW@C6a@ix{q$gm=N|&MA zv!*d#z@timEpNY-AY*)IG*y<*)AkuJ+^lFs3xhG#+AZ-WL~$W!gee8&sbFLH^n;wA zO6F7K&W=1*i92fN~(>s_O!tmz-dY=9NE z9CKRvD>s?j-o7s4fjDQDH;vm8*TTi zA{PcPO)b{mY?l$C2}?w&Iq#m?{ObM@m-lY8CdypR6mJo{{SICA^fOh=8=fWINXgnsfcHB zzV!zcK}Q7bQ;@;P$KMq;z=`-ysXBM5_>#1MGrd4VNjTUHcBxqDQ8?3{{L(anPPK4S ze~5uZe$-M^l7A|QLR2>u(vys23N#xeS9H~|S85T-gy#qKpp~d_bMKnsNhi!{k`=JD zJ(XaZl!no8=_=-8%}FLO#&AC>>5LBer1x3{k)9b(+w`w#{{Rh17|PBKaL2|lKTau@S~ZcYZ#e`| zuE#r&Fa=B&FgV-vrk23Y^+@Iln;_g{{52aYHl6XGTCzI9PI1T5fRMo>7^=rT9hmYm z05lFpJLaEeKBJ8^X^b<14G`3Dk_a2)nxgd@D)J366-0g-kY+akjwwvGI3kT{p`41# z)DfKQLL+BzGf#+sk)A%3V+cvYig*-a8b;Wx^BnS#8X;h310%H|$fE}dCZ-hFBS`JV zGFnd@=bDuAk#5LGA_Z@c(wG+VPU#zZ)ahU-@%dL%7!$CrXOJ-=k!}MA6tCH7Jf|kC z4Z_a)M*h_4hqgB3(ucS@R8|*i1abOP4qEUrq*buP17Ik3p<|J^(y1JjG9hi2@S+e7 z6;hI|#kK}ie?93;=w(?0j`T|3OX(>I;BlJteKuPQHNF$2S`aQD3Uwhw;1R!`tdnCt z+o#5G=^Nsz_X%m2G)8eaNauk-bPdvWxHl!Z+~D@49HYWf3M6;iWBF12Qymuw4v1=P z#^R4LBsAtv8d9U?r;#Qtsn&{HVn|5}(wB7U`(vNI9n>9D7Uu~LL5L#CXca5Yf`jN(+7mlYMZ4yP1b3r+^<)YcR91RC$crP`$2V$6}#_Jx$bmQ^84 zr7L$(&Xp~*qfqRk5PMYNCkQr*O(`@~dQ;%F`u^*zqBxN3R|ACs#$HlA2yD0xah_5~ zt}E#8guem$TGMc0i{wj-swq%H-)RUrUdl-f>rqawFhYKuRjruTo|t&7Hi&llX!182 z5?%4VRFV>(IOk4IeR&nEbjMQk)t1(j+pe+Qj^Ng{7u;pdB<`glSRo^xPAi)xX=axh zJrs^Ljy$5a6yL)>owjuK$}B5vR$DMug)6}1gdGP~axihgHTj9vy%y_TNn`4V;uM>m zu)6A6I+R^Y%Vl65#1NeS0L&}(>dL5BSjv$O=K^~YTP>;9W-WLa#s&wJobF0%^TXp` zN4wIpZ#_QrH>1hvxzi;u;q$olvQbLXusGDqt+a4C8nc65pP|OLiFsN$^GW`<;D@nv zyQpM#q`#h*>Xm~HjTSS}^K?>X46Wi(Fti0Y~)| z7!Yg^D%l)Qb|7Uw3oWFm`7)I_q=SK_Dp+h_6+6dz6585bpDto1I~7sVqFCcz31m$Yy%yf@H z>GGjk;>hlav|R2iMrFw8mej>KsEAu(3WAhIWwaef%%8-Pq5&sK@0yRFe7N+?w(N0k zQdAg>%xMonP)f_JIzV+G{uL;UC;{A^ytQ|;Po9?Le8f1@Au#jG(HXx1#}e=uY-%|H zDNBb>Do%oT=Cf;oBK_CV;=d9*vTl+PjHHYtHWZzCJd#{GhTA6_k(y-H#Q^l(A~N%$}0}75u4VK+iocQZp( z(iD>Bq~k{5q=Uf}M^tpAIM)Fqqs;iNMN&}>{FE#xek}zgtqqgDl>$9LtZU&7wNd)6 zY@B8rOt#Q2Em$gAfyVrh0*~IlsflA|d;uX!LKoqZm8&`c;O86f+iI`+m#KG2B)-e- z{YiQ#admC}?8UC15jvK{;$M*(8dUb7a(XLC@aHy^5}%liKxM;_qO1fGe90}jmle5u z9k_KLSi_%rwOgICGW#rormje73wKYMD?`9|$=nQ_x|{%|xy@wv4xF{#y5b#@I)c*L zR_w*NtTxy|)5sB(pxU0cKecLAOA_0jY- z^GC)%gsN+#nK_NNY0Y@Yb_^sYA#QvD-t7B8*-kXm(NJ^CzprkaE zf^?^CIA5VTPC(+nJbKFMZM929zSm@3sm7E^9tsd#17M-3c%Hf7RF_BmM!ZCIq_-tX z#!^zS3KgEpPB3excy)0mvR;|?KgA2=exQQMcjj*HFar`Jt+Y0@0CEc4@dmNn0H8VB zH9+YbG)Y&MS%D?yn$~uEDAAtYLVvAPU%kqm3&amP(pH_)!%0hc$8`FANgqnT{47e| zWphkGP|)EFrA0({&}uWa5u#dp^~elFMX9MQ$CTS(1uT`NUbym;`JYP1Zpf0$YQ%&q zPWVblKc>}w>T9R%NeV;XGL=RJ4YsK5wh|8`%4;^i$dHmkrRhl+(v=aB?geJJ#Lic; zyCvnV%&7?{JM)j$nq*~r(=M%UJ)9g5QcXz1jHQ4``sSk{8~F@?!BHD#tgs@Dwg+*| zM_b{Lp|K*ZcF0-MHlzXxiyH#~f2B&tkAdabQ_&!#Bm+`R!brfzDQLh+kKdE;O}fzo zH8b*u8x81b5y==SKT10_5N{2;(`Y$W9Q1Gw>HR6Tc%iJRtsZA zBx5_&R3s<=07j?n=*a-~JXI2)7CTA?pbcE@Bqtl@srLT>FMrakmgn@5zZFr7)w$Gw z1x%sDY2;N>1c87-six!tk>;ad4SB$GKu93u*PVQ0ni1&Fq=zy||}a zY<3jtTgLdVFLzr!-vc3{2NdEO$-o&D@*Bv;1w5AZRArbcv>~=}+jH$iw$TGLrrRTt z^rG5lSMc%eNV3>$Lv3fa^zVbe6jL!2NZ8Rr8Ne7k(BLj*xh#-^Yq*7>jphn{03=OH_7#?xONTMDF zam6|EqqR3CQlEk(!x-F9l4D^cV~%tBRfNaD;LxweAH)Z>JR4&osJQDc=}HfpkB@Fr zzY}1C>qZs|To^~^UVZ6pqX__vSFPz@&WI`)a?pe+E7UjMq^;7G3>0MXwLtju+OhtA9tsQk_$a-10IxsK1Es zFoIL92sr2P(sFiDX%!oLZb+5#&CH71s>XbGG^CHND)#S8v~0?VyR@$nO_eREZ6vfw z8){fkJNwpEWuH1k{{RK1*m1VisH?_XYAMeNR^vZv`Uj=DX}6-Hd}-=zV+z)^zNHb^ zsHhS40+Th-NNJxQb^idv`}BLtybElUG6_&a%2qZQ$x=WE?fKM~!`_~~+x!<~O|!Wk zQOcuh!|6J=0a{CeM<30D`d8K6MbH!O0ZNRZ@mTVqN%05QNhAuO-8wTu!;P^GwYG7o z=a7`6d=QX?rw7<%``4cH$v!P;>R8j*b-fWAkd~$1Os5%W1xJhEIWdm<9Nj7-7~G8i z09v!!y3#B;X|p(DA|jRI;q@eysBxVNDs5h>&fcc7gC64*GoOT^mq$;lby;-xsrZ4{3erf^G8k!0vk!l0~tsg4DYp0bT7vKyXnh{ zEi>fDy4jo>5E^x1Nk|@CD`biBhtxhW{VO%H>pmG2h3dX|#;{es#KgNE8>U6f+{$?YpNCqUf!j#zq@TI_Tq~{y% zYw3Q!_zCd);`|mmucha)Yw9;z3Pgxbq@mn)T0qjQZ@NZ%8u`!S7xrQJlhZ4HOum+f zdW#rpT)1E|;**VvkdwNPY>My5{V%O*b9^WLnavfK57QOOKiw7wJ}2~#OWGQCWz)ex z%ZMzsl@*kflpJR%!9BgrWY2XS^RI61Pd+2WS$;P#)Y?5<jq(^z zlpKGDBQ$Y(Ur*_uxhba2DdoxZ#x#SvU#sFk_~+A?IBu8OwM;Oikdfgh#2P$3c_BpM zN7Ey0<2CwXBsFsBf1X;vLYqp`NWmZ=AH(mrKBs|SmkOT)p}IyFMk+f^OTau93PK_fuJEgn<*a4h+dh9&!^-Pgp z`94)0UXL8-uZC4Fkmh{-(TF}ANUbaw=}E$zfA^YIfq()^%9Ft=NLQe$z0O9fJZYvA zF^poE?YJ0Ef|ggQVZ8E;nQAwf7%{5Ti`U_AUeKAoL;F|Krq<;EjS zw5F`dY+NKSEvar4G}4kn8fcxtQjmv^LCGn_N_1o?i1?|~kBKp#M5kSF4-AhGtt6Et zJf!8-0(Ss~cBz+ba_!bwRwoh+ZAmI=andxolC4Ww?Uby9@}iF%kuUniv| zJ-H64wKUldX*k~$CNmx$GCuE%GCmWvQn*8k+M0F*l`^a?`>y4sZfMz7X#v8m%1~9YPvHP= zR0Vq<>DeQZxqKdfs>LjEQGQJa;744wVpr@t!>DzpR?|peyi$BnqNKZ#z6inR8w&ad zsw}ZDmdA^PHzCrJoG-@6T6YII7|w8>PIs@4JZp?=nxyqTr7CrCl9r!>Q*L!3Ng7r| zXj4uMauk7`@!kfW=TM>*r4YO(dr*4NR!KFlc(6Fo^~sPF@REQu>~ z@E|siF(mf%8TYJ$C&a&q_ooayZ1(KPRucN(Uy9ntBzeG2c-*LQR$hg%{KB7aqbuFTswN zVQG7;dgb30_mEN>fo{mw;y^vY(2RaBIT;QEU!a!mn}yL`Cj+OgF~mkKaZwGIT^g1W zl!T2bISJz=>^bJK?}UF3<9sB8E3JCA`wCxDU4MBpWXF>nwgN^HHLDFJBVmEC$I^r9 z--|Ze)PyX)TY%9~MrNTbNm7Oa{%oZQZ*2EbJ8_z}T=8jA`d37uiQRd9m#(GiNmkIL z-z7qf9Z7AaGj5WE6^)Lk6f}dl-zV%V9qSK>_L&Sj5-dr{kdRwT%t*oVk*N7l;Xa&k zRr{w|!RgtEW(@nNOweDkNd#vFK_j*{ ztb)%D99yGoNlS}xAN)W9PuOqzR*fN zDwqqIdH_}LPxE7^Kj`73Io${YVs|WfIwS4Q-7fBasEIAe+aZf8sw%ljx2lG3M=4z!(veL6Nz0Hd{bFM^!<0w+jTO)pU864uO zciU`qFczlJl`DK4X0OxYH!(-RC|;bCDsySH%4xd z+v+MA-+n4CKs=05&M4`{0V_L@c>cA+lcSzJ#S4-zV0kp-0q^aY4n7R&WJat?<=>k+G`c ze7Xu&afq46mw$ROxIOgqQgJQiKoSlhhX zr61wr+J`ohKZ3M%SaaKo9d;eR4|*T%*|?eUvy;L|q#II!$rYfg!;Q)~q$15PM= z3@K4WZq`OJj`ZqnXBa3ym0d-MpWxf+PNv2&_&*_A*kMsA3ALPl98;*Z?}D6fwOxN= zV15pJ;*^U%4oY$Kt?V`qET*R0ry5hYu%}T4Tol#7+46Qrw)M~0;DO{nTvC6xVc=EU zw{M&pDX1z00Fp6Q1lmac98>Mu$T-uQKETQhfUNM7L18FCbOgn8TABR4a@xwzTQI~EdUcR-}TQw&tJ5`!Y5L7@$2==J? z5u_!x5~2nxzXUkDDwP$bGa4iNr#s_*X!)%NA?s-%Z9qQ4%#0Dis|-7A+oWYiw#8HH zLUk;RB@L(fmbGChPr8T|=yG&XXs+4sYlQh{&CWAWop%m(?bJuN#eQsuPwv=h3R{jJ zK$L<{+OP0>H>ct$6K~feDGNSS#&_l76Y29SNn77W8&~||r24n1a}- zDJ-_8T37tTQFZVBtA#({tzRbP@MV1?@VBX(Z7+35vvll4I0Dx$V~UWB`n1s@KSv5r zwQP1Cr|1rtjq*0B^Dgf;Fyx6#G1*uAxdB9bWQzIH*>j)NokoFjxj!YwloHBZAU4Q9 z!rN&92p{r+T0E|rCfZqQ=(ws^D{*Hz_N2)w(nTP-ZL>_c^?YlbcXT1o45I-EkcMOF z_tF6Sl4~-)-kEV4U49cwa|mWSA8Gybx0TET$id_5Qe8aD%*S#A1yT|bhb20nDuy;b zbztEC0EV-!v4?rI$dDtdN@6ICx?E@_N8bTBA8*dQ%)hGScENo#jV8r6sBI0kZjo>k z{$qcZVmnR@_6r1{Ap8DawE@*}qg%W;WZ31}+6yjdinkE8C>@j=LX-jLjFa1X%HTM} zCS1BiR2eGC(x+hSLKU6ztb`o;l?t?3`irFBf&=zh3!SA!*Wuh?G|IF1fIuqJIQ}9@ z?NhDUN+p}M+2u@vmhSCGo?QuJ!hOU^aB`#;(`m>wHm{DnJ(Of=z2IBN>p*>{n1L)K-)Q2 z)3r%jFVgyDO|^2f^uwqT40sZ*&Ce1`qzx^%p)HoicilwqkWE(>g~@(F`ZKQYT{PUN z?Dc4Ax00n4DfS!kqzo-6DoTR1gPbiu5Dw(l#qj&%lM#WG$$=G5CP>Kx zQt{z~@2rzpwEm%k)M$80mu_NGG$KkZ;ciD$gE)B&tpm)Vk(8?>gOGEYkAC5Cf*HBY zn|HZH0WCb_)}2xr2{;}Sf^?vqfrVg>{*=;bx)DbEKE1R2Rrq6iNRJVldmb}MQX5c5 zh*EzMI6AOB>pA%2eChLJTXvSx+=pB!Y%QdrF6487HtaPW_xc*gVoqXgUk=`=vRqSb zC2dGlW^j{}mt1izww8e0ft9HL01!Ae$$e;|!}jK_(3_GHmK$;mbviLOPGCwFI_^@Lp0>g)KTpPBFIHv+7UYwx5O8 zz}mvnOBw-bBr9};x%5O+QZvKEdeV0nTauh>vQF z1ZK-AZ3!w`Op>i?BL^c;JY*2Ae%KdFgovxPqSVaQwBbq;mk^9N0Mt&Bk_p)=PW)#) z*Po6{gG{90`KF3xy3pgWSeo2fDp?9chC^Yr5LB%V>eYo5lC>u}0373UQQbti@7r96 zqar)ewRm78lC4R?&=jPcCzG7_BDD0mY?Ekh*QdB~YcSzg=xfmPP zL)LF5(ENyY_Fqsa@YJI44U&+Y0#xP{p`G`}TOb^nlY8O&yzkn-ES@1oU+{% z@5{Dx8DU;j60?E;I7!j~$m9y3SU+jaLs8>IOoA2TXnCOEQifnK&4SA9s}7R16)d6L@STn^zS?=L)1qfP7U12=&woryfo;sTnF&_D zE7CTIk>jN{^oA$Ss#r z8*_9mDH_sFoQ^6o;d_Gg*OfD+MYJi9w|+_*qq6ZMNpoL_3RQ%R6(`r6=DJ*ydwqQD z=Hlpq>OY7bZPZuRAYQtuaidA_-Yqe2T4Ow<`I3byLc9+ElO|zL?+#8IAz{$6 zGx$v%+A5c8Ght}7qBC+IdnHQ?2z3DQg|ecPo_Idupg;Ck0&~i4dRlR0 zoN{8I@T7990C*4A=ww7bZK_EEF6XB3K1uESq{{S!0Ri$Y{S@GSD6{P7d zke4(cMIFwcQ^@o*lgT9M1~FB#6X7SqJA4<|l>@_0h3s9Bk~I(E!mvM6>}zwrS^APJ z9|5GzWsM)o5|!=zD%=o#Gg18m5$LeoT*zo?r47)eY0`H5KzUEpbDYtWXWk%4b#~=V zf)ANrv!({}J#YZe(B_*GQg-_nCC65065>8mTAvv#xCfQTeERqH9+ev9v9(|}TXjHS zlmm~}v{iLElH!Hh*Jrwn;F)m)GBfVBi~vv;yPz|s*uKj-_mK>I4G zC?-PLoyN0ec__|JO~SF`PnUYT#^^*nwVwHCxRbbT5^d5j~S&T5$s68`x;+w>>EXT6`YJ6CZ{6XFamT`lvqFy!%~sr3=ODp84(D!r_!Hp z+%x!d_NlbQ0B(^>5yzztA+jkJ@Blh^q`d(+kJWPzUhQky{sY*FHB9g4tC+f=I& z(g!pX1ScF~q#9VG#KCPN0*q-pXYWBF+;^fHdm9x}@c>Xj?kTmWBWmx8F}*b7*v>Kc zrE&NGJchHdH5Dc(!60r$OD({$oQe_Q*bD3kBz)C;>Z;JPQ zC#P*}c%UejzyQK(UNyHk%703eiv$g{;-%w28+mKr!+@$(M6+t(Y5gdqTJiil)cY_} z0Lo~V;HT2N5nM1wprG&?iixwQjTE719-jcyKrFwdT8ts+ zKPusc+$2`^4Mkt}Kt!mYjYc(z*LF9n<<%m*mIt z@$Fs_2cZ0@dmm#_TK$wUvTLH*zv1WFtY#QHX|H^qPH|h)>K+cNTAwxL_{HgRb=gF^0CvFL$-(%Gj-k7n8 zooFLlTL>XQt$l@ScZjp>tn0Rj+L8uV@!eldv-YYdPO|N`>}0f7l2lFyCarEmpFUHv zF0Xic##;(bcJ>uHzXM~@TXbBcfcx>-QoO_~!ql#L8ncS0bz4`9(pien&L!k*RHZf$ z4tC%vK>LweTqOP`B+be=8rlon*j0Y_I@;8Qwt%-%kKtP=&^^Kc-|t*mBAMBaTufR` zj^?M-+_hZgsRweUwxj|(`O}Z{se2uSNOh!7ea1^(WUUs&4FmpmlrnMaT3VX6U6}`4 ziE|!T9%&(IKA|HzXsaJXS`(^U5hBcRkTUzL)K%(2Ml<@?BhM&ybo**(sQjnww-a); z-|kVv4X6Ui#-e`~ol8nUJ;uMT!l9#e9fDNVLz>IcBlutMOLjt=QqDZ=#-mkC~aA9jdwBolGGhtE_ zCDpo=ZLrIUXdze|4PX#918TD|@DHcb>uthq;Gp1HX(5#%D($0E0^8>vQ<||#>Dw?>7TC!+Bphc>K%&yf?IS5DF3g3d?enC@W*i~#0A*aMxb{{RviVmjQi^vhQW47L*CM8zReJVcUI^Ti7#z<-BP zR@oypyWROI{`?1Fm2?fLkR?GDGES1R&tT4TalFZ&L#8_arqJ4d^)uh zx~R|^Y-?JMuM9MuKmeR)&Pqppsiu@&kA>f7&lXEWi)xiB>guExoKn^h6{t1Fl#PKW zow>*ct^Ff?m1`F0*9%kWeo{$Fn^KT+yaAPNMiLI=a7Y{0vp3<3whLFz%D0^A3scT% zcS2M_&nh`cCv_blas_2wQ5toPF&(|IGV{xIuMbK>aIeIHoPa$BKD7ztkEnlht|{V4 z_%|Mvy0s;dAt`yaINO1=f(l%0DL;n=L?~l?ng&I8MPm zj!DVef30F232=zpEV$!MAt^{uQb-GMa6v!B0KhxsVAaiu-X+yWG`MaMu$GjCD6Pjq zI$Bmo;Lb+FvC66t#VG7)&N9<4MpGg26T%@%QdSr!)U7B4D?Z0yduJ7ud{pZTbXae> z4W^wS5TvO?Iyt}~jX5OozDeG!^6u@m8cAtL&Z6=ZwE|a!A1Fe%DFZm*V;$=j_``X9 z>v4vudZjwBoRh0Y2mpD3&@gg9!OD(!uBc*iXy$WBt6aLK<8ik+JX%Dgkn)2FN}N{X z@TXu$aezGr{pwRJQK#7=bv3mpisFz|!)he2Ac3Hc5<$+LvPe8uk?`lHZZFME@!8C? z0%8*!r%DuD3BfB$9KuN786@+vPT18O_&;cp+V34M#CIgTrFaW5QKuyS5wK2Fa7YQj zPQx1terptykAIp69&xF3Xa4{QCd-(n;9TR)eYGo6j94gJPC7sscZUW-0R-$yLB^!- za(>ThX~<05`lU_Dwy9nRooa}n6|2J~$FSH0t1Yb~2O|T^RfakVeKFLI#=1mPjwL~n zsV^ZRE*ynN@ZC;)`{_D&$E{k!QqggZ9&CA?M;$12D?>5X!;xD;ib{}QZwfk-g>R?~ zCm?~Il+>Na$;mY1*b;8HMj^D^?yHSxpW--tD_y`%Y^6&it3Y3f!8`1bq=G=kDwiP_ zmT!xc$&q+*DpJ~H#&IfbymFAFw1N-Oe5+&4Snto@Al_EbNm;`!f~S<`A%FsaDJXLa zI7s$E=Q^`k-NsK>i^8T!4o?LqGGt0p!-HN%>On#j4`GaYZ%+%0**P@VWs>REKM+?k zQfGAYa+`FPn_Dpw!hjhFNpA}Z#`xnQBXLs}uAQ|_3hF!0PuykC3?w@L0L-)_FrWYl z($lMO9D(If81(H>cHW`tGRmDrhSwqGf`x*^DocYH^D7}rBXTz%oic}Jl7&4PxQ?Yj z>TWKT9rnt~$^QUe^-1T7jbHR79B%Ams~@|q#m9?lgoYL5*A1mK>wiO?{^_dinkDKi z#IyTICA8p_$$6}e_8~yz9sP}5uDxq~wnuFUeqm)u^7v?H83Sx;2Pf(JRVq#Pbm$KP=`)xXKCswM=tTjf&#aqeHZHt1vDKRneB*lF9={Qczi2i35VPjSh4C zX;yuE)yJZxE2u6ikS%4l+F5M4&`3+D?d_jYpUbv0V(3e??&IR6wL@rtqy@M&0qL*+ zkG9pl^evL%)5=6qZfR=gQe5!$D0c@sPJYIw+MB42$BMRW_U4&|YiioTaytAXmXh*z zPy=Sk_4M`_^r|$weEXBdrN{xmbNiP_C+Ym(l}X(0@)FV+S|me#pa>w8IN2M56}Tgg z#*y+3+!|{09fXG5SMS?w1o&@bay0FwUx!8E}3jI8P)mbNSQgXaV#cskn&FHytYgDIkwRGAeS>cf4AW;M?w!S!D9hiK(OY zZCOG5sx`b0-KqA9L86r!iz62Z6A1mKWsj)x^ds`?`sv-X`3jR}+i#Eow_TenJTomm z1D`;wlq>%LFsi(Fa6lA$Pk=cUpWGE)64I^Gp({+d&N6#zP67HJ4KSnxjfHY0=42%J zQKh(s)ysLyXX=xY`cl_(i6KBqj+T^v;xL>Kx;Xr?K}81Nkaeg5x93bPzcObEuIqEudf& zRP-Bg)Coz+^`e-!okPpP%}m3IjBQVg(eI&@ens1%07sNnmJB{Np&-Qoi~+S+Au)gk zE7DE!(6jA{6 zu9SiXV)n&0sGij)pc?UP8r)1nsu}JnwL|u)l{7sm3S@T_IE02VHsX?E)T){SkXjy; zD8yln?MeA>K~iP0TrGjxw-gdA#?IKJ7{fWmNv!~OriuZf#K0df>t6X$&eV5?4JacX zmAIIX`EOBO5rO1;RF8%aYEWy`P~s9eMQ0JzuN<2(Pn9-6p zjHMq{uW!f?^QIqd#U!BtEhPT{3u)7zspS53GvEW-nst$+?LvT&Q<((`Nx}#DRa*5F zoTPE+D*13=kZ>xcc|kxs5^8c(hmU4?H7IG)l@W~8Wx?p>IZ8%-sE(vlh;)&jY9*Fd z>t|Ny-nrX4BApSg$5f|sk%WWIKCc2Y0||2~R@D6)n4 zn{^FaBQ}6G30#8OLO|dqxt=$d!+)Q#bv_-n`TqbVhTdO_?}m#hLr!29Buil`Qgg-% zvPty)D(P*TEFkHC^9a-eP|{ieaq6%=^H~&oC4$u@Nx1w*=*W%}rAmSGS4V;Gg)qrv zeV#vBA_qwH71iAq7rO7NrWIhDH%zmP`#AHi1b@t=ZA%|Yqb;re>;C|wODmH90H^-| z!{~cd6x`N=A}PlZHDT`{2R!OXKc_vbw`dk8QTrnE1xZV*IMWomganL-6PC`BJCdXN z;;KIlBFO2<$hIDlMXAC8D!p|~SEealvePJUNk05miFUHexR+mM?8_1sgSEUV#W$15 zSyDjHAB#29?bLL?Ke5fr>hJUWn6Fu}(y%q!;o5pp z>ohs>l8I1KUTh@*KHEu73XogxosObN{(P*DDdFq{#n z;4CRboj4@p9QLZ>dIzum4D@RjE{}%Q+FO#smYEU+e+i`@Hrfz00lrjLcmM%hv&5fr z{{RPMGA@!dPO`MmzCvOQRwlCupffP#D=U*ThCVGH2_xcETP$QS z+v;{g76SAf4>#h-J7gWW=Lfx8Z2tfntonvy%`0S+wO(;*Glf@%^UmA2hZa@LP&fmQrw3>Q|5nQndHA6+Cs_PJcrg)^mZC*aLqS8#l`TVAR&o4XdR1|#EPR}5{`>xfb3PN| zrFHf+%Bw{4b~&6v?@D`S#K%Mq_{DRV2>&Cf)$Kyn%K-~R%mS{qV!1d z$XXQSu(0ZXn@C8)a!$%4-0fE>)@Ljh0|ws>w%m^yC}<(Xl&eS@TS{_~gN=bYM{i2Z zWx4Txs5)ljDT_KRQkqJQxlF7$*;w-a@TFJ;@8X86jt%x)dFJm#Nlc<04yoP-pd0iIRH z-1e)lO7wrm$=xFN`(Y*v*A`eUtcc@69M9s=Fx#a=azP{>IjIhe_%rbXqk3XuWy`l- zuFs*xGcr;nC(KS#j3v?vaoD77^s2@Wd|&b;vPFKPmd8t6>}@7QzM8d8MAvi&u52R& z9mxtwSBMp{Bk8qQEt#Vpk20hv}=_hY}RrcwIyCt~hwY66tK`BVaT1 z&0aWVB~AYTCnT~*mXU-x)9_*?FD5D;PMm7g0*C%#zvoeM3?&sXWiZOY)1hceNIw4n z_N$LkUphi7%5SZH1$DPa$j;78e&X9JAK`fg#He%s0101uiGgeQTYVwBY+K}nugn^W z*AP~;dj_Xl*k9QysY}u|{{ZO}j!j)Z?fyzc`#rqK@TEjS*M)+N9bWlQKT;`8^ixvQ zhStN#3Lv!Z6M{X+l${(p5v&$KDRPJ{5 zOe<`aDJb7Ufdpf?P^T}StHIdDiP%~f}=q!Z`= z04?YTud2><3MkfFm2pmg6(LtNox-@PQtxE%u9;QrjNy65tq@sua^gAFz@(<)Kbolu zz8svdk9z36wdH?M_YrD#zZlX0N9phcmye8rM*qX;Ykc&1wyrA}$CR z^8rsHdwl92oo9&ZEO4x5iEpS1j&*I#0~g<(MsgMU*FJD4ZmOheBk?TOEaz;0E8)E z(lfs_F@N9nr1hmK69nDj^g{^5&B9>hSCP#e)twVA_ z^r+YfQ6w)Uind(s?dnk_KSF9U6~0ri6vMX4l0n=@j<-KOAlD4Lqp5UP?yyjzJVE6& zixfL}m=Z!%2?uJn%jo{Byf~N;x^7{Rv8fFm2w5JwRG>f}ybkn^p6P21iEl5fddk-s z_Eb`+R>gkMm!9CMpjmIzb*TRUhMhu9bO$7~3)NAfTcXCB9j_fgf)EY?J+&Ug)`(9| z^;x{Jb6)#_t5wmhS_ z(!B9i3unZxs_K|avpac{37|6jP?DHTY4q^e2ub$HHC&Nsb*qz-qOc0c(%M^20#Z8+ zbLvkd0M=YvbNwK7+~33U-yjkH%$Dxh{qo>rZ#rq zmlsuG)E^J#NL|8bPha|qKshryo2V73N7uuSkfHwo-z)1?`|O5oP|BpZmA>yK!^hgG zf%jy(x7qTr*vgzfRF7JgPgN-@ul@f3XQI%<71#Lw4QZaW{hWnA)15if7Kb;|VA+~= z<$Yh3<2(5gQF8j%uCDEPy;0V;sWG2)Hq=om=TfqEOKr3XN|bmBN-6$<_)FJaM-X1F zO{&La@Vs=&nATl+pXX(j6y*p102S4=&;I}uwP>9?d+956(XN{KX58D=+reJ9tq-6{ zTVQ_iZ{3uTh|Gcb#5U4^HdK`j=_8Lk)ydRZu?jXDN00#Sf6bInmlSbRX}`FA$u*qEIH6QO8z z^%OX;2P*R@=LBIR+dI{zpNKbW;|5KxA`CR7DQYKDmPx`gdDKoeIX{&uRm%S4H{-c( zW-DzdHl?csrPY2V41W?vLQXOcrDtr{25+hTCtfGV_3W>)@%ChhZal7xk@GF6c#O8o zZ@SUkrLqG50K7hhNyc|p#~LS1d_=h3bjM?+BtStxFUXejX+wD%r6D=W^RXj1<29b! zZoU}Mo||+b4mgB|7lo->K{!G{8|+R_*-lTan;m9iup3R0Ek{WhaY`g86M}i)f(LJE zhAgs-xi(WQAt>=kE8_nEK=kj#Yg*#mn+~lAP-VomkkRne0z*_aQtTOx3?WF=3X*mhDJMzW8Q_}z0_#4!xZD|i7RQ~AQqt6)6|dcF zG86`ND0zGHfym8#XIB<2b_Qj+yb-d$}{O+-=)dQS1yk; zG+s6IvbX(`eifgI9dxd<>UFgm&0q-d+J7NWy1heK2ZW6~k`I4M{SCF4mbih7a|lrl z0+$F5cz_NFPvSm=bL*P;$KkvbTx`rH-0MkoJ{!fxmK#uT;YAy8Hd0Q<8qUWX?cF;! zsI4tVh^cK#X$ZhBtcO;E=15QnN{HNNVhGrfK&_f`Qqt%Uik90n`JH*zD@>+Yu{K0R z>OvIxNRZw#trA;Qj|PG7d9V z74`M?FQOz!DU{YTC4MPRj187Ex`-TujE?(M9nxP?+$W{Tkz+d*u!K7u}B<0Y=AKfDD9SDhz2Xz>Wncg6%fNK#=)OzG%!$%eGDR~!Yp z$SD9UAzoLLk1CQ>l2fNi%5Z1yvwEJ@W=B!F+zY*`HAw(QKz&!clxCP~DG$rTk%rq9Je1|5{{T21_^dnPO`_G*)0>6Y z@uc*1&PEji=C(w;HOVA?FBPCp{{YUA;2Q;CR+^uUZOG~2arvjw9IxdQaPgFnlzfba z-cSlHr9f&)Qe13+a+Ls}D@(sYZFd_pDmJ%}HQphyZA7SUp+iz+PO_4wooPuR{L}?- zb$~O0-wbicyZ)yOl%V(hiWD19#7~9s*^)2UyHp3xn3r2M-PK!U{{V-?Wo43mlmdrw z=_H=F>IpiL)pqL-Q#O!F&DO@kq!jvxqlTS7T}0ITOf%CBvwrEGqmR+7-am5O#%3z% z(pQiEAIuc3OdnPzHoZO8kxbnMnlth3zYuylUqnxklH@C`m9aL@IlufwG8S@=@kMoM zANWIxe`ax7Zod6~e(3?_Pqq2^@W$Q zny}jKeI3*-w-VXZwlyWy7bEMQ?d!&)?RvTYZ-uiAMQSDttJ{d}IK3+oGN&282 zKc*=?2P*w+Kx!S@WM^61EWV_NTOF2>(6K)J%trnbT#K>q-t%9XxbdNtL$#CzkmNoha(z2zUeC!7gPf)n*|w2$#t zdDT{H-%h;JuNK!MGc0P0Bg*7?DLvYG1h(h>Dn9tDCj5(0@L2_es%`8ZqO`Eh`qvzi z?At@aTM16!rR1mgtmI@L6$r*^8dSzxaV|u2#EkkwtS>p#gTOfMI6mU5ur70LliIr5 zCrN`nkM88fc}%>6?6eS}^vztNb*vtz)eEkyNS`4a9(|1%~EbL&s`%M;P!p{O!a!Q)^SSx7u zUUsN2no!|il`B+bvT#ko$xyB;j7YBN;3Vp%9J?z{>-MPEUO3qO=vNk_zH>z&GC|9#xk}9h^~q{K zi|f4v${!uYCXpZCnQkLdgiqP$wgiWBxblTOVzrz zs6}cgpU%9T_rdR>Wp zSucs3)gwh_>!~`mY~LqKvdUaVtBKr?Z`#S(@`r8<-faVZ{~)2N`FX|O5u;C7^O#dHHqv|;qDaZe@9 z^O_JPM}8^54;#~1C^@%>8K=_qMx|s^sB_zjHfki2@4YL;CWAtWr^jl*p~AZ1a+8g* zTq!4N#-odKYGrPArxcS-mi=o4ZAzw=s(iLbB9L0h$E6DhHotRS1e47;rU>myODEES zhW9+zfIQI&XFCvSVV`Oy8A#uXbpZ2AOFQvS2CzS^Ycb@VyVA3tt#d{zpd4{qW;zL_ zjGk#b*FmFTqsv2OB&jMV&=5abuU=H!Xi-Es-T*4&i| z4ui*ut0|6?>ZFyBe5n2*XuVuhXi|ldDiU}%2UCYr{{YEP4S&tMo`pPCdojN{+*l<< zl@gvh7>pCFuH!K*#G<%cCfY(KcqXa~0U1`+Js3<$3 zu@)nP`7##ML?7bBhrK>sTY@j27-inAc$JG?p$!(%6)qt^xgWx^grNZ+{5H~thw_t^ z7`csf&7M0KUbcj{Uy#xyTl#eiQ!mJALnwAqhnh5X3=o~3bRQ@}4~bKZmN`~6Z{_sA z}Z5=r+_L_Pua+x;0*o_9-j*aw2j>>$Dw4KQV005<^lc`lJ)px6ZO4?oM&V#zQIpD(J z+^?)3Bf)s_*oLmnIG_WM4c$r*+Hlf;c%JH&&9`1#?fsX+8-tBOE&l*ybErCi!*Q;G z=wz_-?yKbFIc+9GdYmB&i4l0z`j&L1=_kzku-Y`R0#+AMO^?%#tD=~cnI<}r?eXZu zI3-I#%3-#o=Tj`Jk~VeiL}e&$o3b?T!`l>opi=a8A5cbL?2rZ(ZnrBaa&P!udkSff z zQmwWKC`+O{gz~VLRQts4xAm27lTnwBw0<8^T&1b7({^aGZxWD3&CHUprkE$yDt;?i z^p2s$bWSm@2n(>|1;;VC zX*`^NF)Hn_$8(B{!vxzZvP!7AOK+%xA5$!)3M)!l2~Gy*d;!V8$DqinHP52&*Tugg z;KG#J-au`=T2GYWI6bm;9lcMjY&LuR8SaS1Ne{Asr8Y>)bH-0@DETbIZ8}y*mr8)n zH`@dXXemMcB;_RgG2Wf{e{iuRrHsb#3oa$lai9UGQTlm)!mLo?b!D;SCSREea0R7) zV3UEt?l#BTwEKHdq#;DOg(b$yR5{rH0Nj4H6L)clK_T`2?I=h&86yCIe^2XFe44hz z^&(MJ1Hn_ML-`>f%${lq8HVkHL+3}Rxpr|t`4J>1d-T< zf;}ov8g9xd$tq@v8trgQhNre%kcr`K?sqHD;VqHahZ%W3#RnT=phv~K_fExks_d;s z@gqq@S&XSayOzVKTAL1_`BaqzxP<~RNcF}O(%oRhxV=juw1WMHRDP*t19DqpOKwX@ z9)*=SulkmI)iz8gPhIs#!b}Z!bWOI}mUuJSTAzlK{{TALEf4rw30z3JGK-bYbaBViU-24j5g~H%@$kz z=3T1Eba|IoagCDwl5=2)*EWZBGE}6vj1cP!d9n|e8^L)!MLVeNeL>Rj(bTM^4E3i@ zLwPO5y|9cHZg5;aR5$~PafiqEWz?n3QQy|zbQE$cT4y|zpz8Ey&D{PYbeB?GZ7D#q zbz`AFGYq!V*3PG4Nz{<*Bq}q`Ek0)5NG-ew(o*3CcF%{sSrY3V);dm1PMLmdnVHhJ zQ}Z1;b9C+dw7yTtyiUM4p}xp{2MDOSNVIgU3wJ?vyytHn4LP?Y^q*3*6<;-MY`8;# zEgv?7ItpaKQ3`3$r8H8kc!2enOy2DZMZ#f>?8h5IoS6~8>|4B7K@87fz%c3@(xfk7 zb*Ib{wJ22FvPZAa&*;XezW)F}K0F$Ym0dj5;ZwRw=_ld$RWh|O^a*noh>kc8l{Va^ z`I_7j;K~IZyhB7@E)W%TcEj-xrlj=s=VdN+raDO|V$q6ysC-|FbM|Ud2@c#g*Ztw8=tFU@d7x)Tg*?rwEddU;@2S-h{Z!>F%Hh z_JT#Ceo92>j1b9Jqw&-DcjZlS-3F@|sJ}WZ@ov-PRWUBxUM@7EB)pYl853*{{ZI4a4Dxgr8)E>v>%ApKB2O?8c)2(t`(Guwa?;0!44fKWJ6gW98xDc`3J?aQ#aCfOo z4^dsZb@$%3Jl}^c0jaShHXe&LvBPh&e6A?S2pi1?^!=e=sYTi=5=F~iJ0jVne*LC0`L;EaL}K_hw%PGYB^bwh4jnCb4QjP1(O!gRz^ z2oo)!{{Z^4irAw&mKJ<7{5~4I)g67aJ!lL`YFkt7vvn#Q_5lsP6Zn!+e9gF=jcqH; zl1Ctg5ZAzLjasrXs)s4xf6koMk&o#~JfG5mlJYiDk?ZuP98WmS6h_P*^Ms$N#IIQG? zp||t3=tP9Z#PP6o5upKXs(Poa-$WDr!e zPik>2o@l=SdQ!tf(ue@VL{|z;7+dzFmU4KkU_AuWt+uT3PlkP|*4qPYRtSV7fGN}k zZLy~ijP|FJ-?d;3fNQ!*{ORfWX^fwQ!`9#TVdvQ;3Y6J|mga*bY>EJ%_zy07OOw z1=miN!PA^>qv(MwYkBm|n*{t6AzRa&oMv^Q_CLJQ%A{Bm-V|s9QTQy#PmG z_=wMQ&TB-3B0_}Qg?QU0ve9xXPz||&E0_MImm1PoY2_6WjlasLifKA2IN7L5y+>c* zl9HceOkzFhl}K_zV?k-eu;i7YP6Fz`3oW5OXP#H^w2jXvNZzsTm3HkdEl<8W`e4d& z#^A+e-raDU8 ztA)-|l!ha+q^D%0sY)tQ2|^H%RHdkdp(!d+024*G@lwOmF`T#bwe6YL7m`|^eq^X2 zNl?O=gOC)Kak8HvE#1OG8#rGy@#LP~qQ5_>cuFwR+bk~hqkRz|IP0+PeNNi)&{|OZ z8zRT?X*ooxVDX(yvU@tQnvv{PnzIjELH&2wPQE~fB*!Na1xM7AOdQ~bL$S8wMCKAT`_WE zdy`bxKve;%bN>D&j!01jn z>CdTKxiVUnBI2@tn59<~{{YfGt1Rjp%-MZAHcYdb{YiaxOqKQ_hJ%G8*gG%(0KguV zZD#1?Z-&z^Q{Er6dfxo}*#|f}Lzz(eYD|&za8*wE3^Q!$_?YvC=E?&c_>z>^KXDvX z@<}DHi~hg#2q$~}zo-3`Lj0T1aMQqGfcY^l`{O-N|MM=Cj^vZ^v-D)zOH)ZB`>u4 zv^^b4KjAtv`2{CGb5)X}rdASo0B7G6G$okqElU|o4Y&X!v2A1g=&X;SeYMy%%50WH zvl=JHaDvx;EaZ5pKjB)El>Ie!G*iAbYav4p5*?DcNm74_8ZeSR>N|Soqg+eUd?J9S z9B_DHBa(On{Hj@?(TZX=;#@=|h=F5;r&uLJ{E zMRk`X$zeq+NK>t-e+tvONc(|T&+Q1NM^&cs!vQH>!s8v#ld(*X0^*O>;Q0Rlrn@lC z@k{K^dBrS??u-xPA56q&;!VcyVUDQQUaV?ARN4?YWr^tV%On2)l<|F^V$Eqk4En17 z0MtD(3M@Gej=T7A4as&%2?1_g>`et1;S`;naw-(!(1V{6DxYfvG>};88#{eN@UGQy zV+((!F0w64Kg1zT)h;nU$&iW39-7CkQfxPiw@=^g{Xx^Rmv6k;ab;Vk*&GE?qD&zxlWTNy;icfytgG<` zU1@1Lka25|($aJ3T3=TfE`_B%ks&7-OZ#;>-q(N6 z{XgV_jcN7#fAB@BD?J(1P~EyT-1`;cBG9R@fmCayvBDDK+s@%>AS*(ETW++FKqGm% zT{#}lCPk&^lBc4qwj4>>)bf?2{{SkIQladjLG)v-^x9?g&F!fYY_g;mortu6+D265bpSqb zl#R(q4xoNC5$85X=iGXG=Ao_(OO2{(f~H$!l`1rnI9HVFZPENLpoOQ@jHJ?flc$@4 z!g|-DC(T{9B!7F9nQk!`TLx?2{LcvOs zpm0*4cLWeY2e>qUV5jmP)R$Rx53=ixI?~ePZZ@)&wzQn3X;2`gK!8C9Al0kIXRnl1 zp|(1$M5y>_N*cW$_(;NAm)`>jd$3EJE|cRAT5$_W7zgQtjD7o1uxu$xBRu47#twd+ zsM|g9cE1`Vh^d`yno=i0TJ#5W{udCf{5O-6p~1M|g(RsYsZXA%jP1=wRq8TJvqslfD{{Y2E{`7wk{ovoks0F&5@+fpQl9!txwxOBOC4B9wX;;ciD9}ky;%U_9_irO3 zQaIcxB&35;dD!NTvtQF`ThXP)Gc9wpqI5laONmZ&xP>3U@-j*n?ox7{VlP_17HU=+H7srU)TZbRo42$;N&PCx6*xf1A4-Y0 zwv{Cy?TVXWDFEQ)W18{(MAe+LXW0Tbd~Zuj0CG5^!&%tZd=F~&k=6*7y{W`Cj?_w7 z7^S7JYGVxsfv+4C%GF}(Oq8aY{hWxur>1!U0f3E z*Bc-llg)DGha;mCqN5(=B+3!n)JgB2YeTejqYv7J`>5D)*eN7OhL;K#J=F4mp}HFf z@whi1g+61H#q>u@M!2};=I9oSY*i#YTR3s|2+Cu$f1N-QRl*aMAK?nsQ3h4Xg-x1< z+rkplOtPFb!zma;NC$+ZeaR%8l1)iLEtPU~VZRWhUEdPp!&+UDHI~?&EtdQyiwMG- zgB830)JDi2TxG>16XgIL#uvlRqU#%jcF3%$a$YJ8x2zxx0;MTSSCt4z1xi|ul%x_! z0=}?UM@G6VG~9JMwJ0G811eIEzz=g#y-U*YtxzAZS@i0W<+)32{{ZheAchiu@DiX2 z{{X~L3O-bzRPyAPZ=k_0muJeIS7S#?+GTWSM9EXpC!`|%dI>3V^MwTn?Sq9i0SW+y z&J_BR9Cu*bZPH@&mr&j&vj*P+Oqmj0LlN0w1x`HT)B;e1j#Nni4|@7i>C#N)UMndf zKg4(*^ogstd@_op_-al5;K>ds3@X_X#Qp*NvK>M59$Rf%c2d=+pZ(0TKe8?DDC^|N z@0NMp180-a-6btAUT#eo-A{aBVQDBQ&1;6>{{ZaiAP3$b->hgKxj^wM?2GSDBUTjS zfOFV;)`Ksp>?&V|P==D6X-Yr<3YY0+^!w;#9r7Iq!@HPVgU;$SD0EC@CZD#8ysOJST%rvS44xr&_dFCBlhZ1`! zm@P|KJ(ji)aY(YfnHD&zMI2Nh=HALr#Y8o3wOD=_^rP!~%V}k|Q1DI+meb+}9#{CJ zE!mTZJSJ>({{X@mOa`V}x(4ZX!nLyz`y*=3;mMA+w@4npD;@Ob{{V?oGh`zF0D6^k zkr}q0nChM_pkXa%@U}e?goL3Z(ITdmXL0LqOZ37ckN*HoTo*OpMJ30R5wFP&JbsQ; z=j!E`E0S8OU&0E)p#-00cj85+WxgLtbeM4Xwo6J(fH*HZq`CF;rO*7;A9ronquAuE z;Zdy7nm_X>DMbGO<}M$oH8=4h=O*XXChvlzy&5}!j@r@U+uZ~a;ky*} z!jR!{ztPe^I!u#pa!ar9Xr!YXD~-17a^q`8rkntS=tusI7>p&RkgblBkD*bc8e}ab zDInx`G}_uvh!UizB_l}o-2VWrBx2WNxi16O<)>Ln&I#D>&%f5BTy0GyL0l0~%4NvT z4B({h4r);njEI^Ra#As$YJMVEhNUGbQ9LNnlcgaspKUz3r67Isf29v4YH1kH98jus zI8;21S|o7PMZN~fT0zJ1sU#ohKPsw0d2d{`5>Dc- zuvUlSNRCHzAJ_i$i?O7rZKjhVIcSYbI7k@hCm-ilUy3a%?Pa;m)TKC?mU*ew=;V2^ z$^7FsKEk6cF#Jo@1{L^zX{ETX-Wn7ZvVDS5qxPfjZ+z{JLvEip>-uff>x{A>vJanD}=-=;=cJ(V^ZIAwO9qPK>iHU_0 zqkxr=6c4B)Bl8tKV13@1>Igkk5g3tamdxUj{6&?A2j&t--lRI_)f=aJs`X%k<{N`G z)Wuh2$5GbF`?jPXy%@f5UvJOyD_ms${{WxhnD|$AwpuPuboWt95v+H&qb5tr84xBk zyu|?aKncH{kNz5zq%Rz@>!Yx%0gom#H{OAq%H!Q^@+D7)82%)_$i&Fw8=Xxy9Zoq^ ze2>LgJNGN%XEZ_zu<8=Dox*`o1QFh~#%@FO9hBSL7crsy9qQ{>&BhR-juw6zpoz#VlQ++dVdL7#7WR}B=f+oDvt)$?M4k;*c9*Y2Mcq3!6 z+G6P$aMokG=ykQEEvG>8xY^-D`VrWI4;1PsOT8Jr_R`YA++{^Rsm^OYb<2-WT|a4C zJRs(Uvg=PH#AZt>#>wMtMVM|E7O3c*t^ZLJ`KhY+DhGbMi&3vi{ng!Q^n zqBNDLC;g!b9#&l^Z=L{Q4jg#k1grPDRI-@3P3dcklOx1tTzM?2Ecg;L^7Epog0>yT z;1U9oq2@x;m8S(FtG!#(Q7y<~`P1?qnbWr=sSw_K9z1ta0bEHMc1!K#9YX~3I1DN2X#;!o zR@0R}kU5yz23!Mokdol@rOi|$m0+iEcLW`=^{rzi@=CFFdL!*ii2B)mm4_R32MR)m z1b^%FqwWSZ(i@lCTP?6!Df#$l*f6CK1FepH6&qeo96#nJFHa!dwIKBi#w8fYP?ujCZFJ!nUEv z4OsNHW!vSw0&89ri0du5jOPj-r7YumY_@s92iB#&LmTO&E*aw$y})oHf;Z!OQC2wL zYBa-9Ja0`o6+;_V zlCH;W*Upmc@_LV`Q*VgtZR*zR&%2)rGbtLJd3gR5Cx23Kk-CLzpAqk^w9BPVfB+ zaM^GTcu~sTa~pWB5EP@^%8}(jtIFC}hw{it8)Bk*bw4Qqu!Iqj%~`E+S&-tMlA;)B zc&{jO5IYZYNJ#fS)i~@jbTpWWvfF-W9Wo^(hR^;RMfIyIYCXkK?^kH3Qkx|? z;8MOM{6ddwOU}OP5~U504gx?JN=Kjo0rVAw#rTE()b#>jMU+})C~`!(Sk#D-n;M;X zj`~x;{v{M7ZlO@kA93+Tas4Zljdg@Zg2CzuNNr^cEy7{Jl)U)HR-xUt@(CZrqLlHf zs69_-wq71wi;SS_?x{{V%Kqs6AEj8uxpi;^4%3EGEzyi5*8JmQ$T6 zKC1pD9_mt!K({NYXQx}LaXV~sO>pJJmXkWM&GyNAufwKpGh^DU%!yOxzKGG8O0nWP zwFIPRu*n%chDqY0z9L#*nu~W$^mMIu9-KnsMR@}sIQrBTDsdjKIFYB(me^15)Rl$% zy@Jtp>Zvce^O7@SwC+Jd-EUOK$GRh^G!JD<2b9q9Z!27O(w4$PHrRHhZdjv9_A*N( zwq+efD)G{B)bo$Kge6NPWbiU;QaxdC>04#jq$MbP&qZ{CziORWKew*ZA|^^ctgy62 z{{Z}Fed=n>Vzf)2{?uO~wJzP$ZGPB&mq00f#@3Mq-;uoi=Xuv{y8l&}8)9u@bmioP9Nu2&C< zy)k5vhY-};+$it*cqQ}~GwtDp*3aNt4n> zKU80p{f!*%T4ssVV-v#>@Z)HQ2{`8fbT@S(L6>}Jx?OysRy^O~NEE9KvK=OIBO?Wj~N z6`>E66R-xIdU_Cz^yOp@YFr;>6uwK3&W53>5f!Rr^r|9A{{V)C;V1s3k`wHl{{S}9 zT|uMfy6I8Zskp4eipd;!apthXasL2HDhH#SYDZ979Fn2(@n8Ap$vXz4q`t4wtbOWI z+NHT*zP})9T!vvZ;V1Z)BshM%v}&+!zpwHC0A%Eq>+Ssi0MS!L#5k~`09tSqlfsAn z#acSLDZfVP>*H!6`L-y@S?;GIsc{$g#B^utREwcZIZ5#GjPLZTY>DYtKv?IpwFO(Q z*C;^t0mMg;k@>0L%++uv(?qiArY6nPzdW|w1t^fxhi}D=#a#M9q0+Z;GF&qv8|^s` ze01&e5gZtGvHmTB(t9lkJ*t47rA#-PxiGR4tZPrFtzRwuOA6%`AH6_QfYPjuzO~Vd z9J?VoBUguIMXKE0sJh&w!Jg;J%0h6NF{Cov%{rfDgcW14Q5}s*$GE_^q4*b+EjHPx z2#~a=C6x^5d=BLVlkSqV_oz;kxIb{{mtzoC_fE`?u)OY=Zx>P!eL@~yN1%9)!lz&P zYrIs%aUeM68wZS&wm(|*VyUK%8NF!+{Ao^AB$f;js70@`ti;W?Y9Bkl)at&23j3Hp10DbEFHaWXT@+(xxh&p{RV^ zAo)hkyQv?AP3C$GBwqS){+g1c36`Ms!qByxYHa!CM9KbOJaiA|L!fTpsH+>#S*MAO zM@)uXZBgY$Q6UneFCHT*^a=nFe}zdxhUpYN%Ke$RU+#)`$nW?~EfB^Jmado1mu}TD zD2V#xz?RZwL6EfIyuyh9tI=s8NLcK&D0)yIvq=6N9MZYgm{f0M3Sr~`Qho7Gw*(wv zJ5=f|#EfXoKHao}HWa?-coDE8qBH4nXewxehY~-DwQQmJZau)KU9@-(siH!HPRavi zr;x9>!TZ$P7Tz+R^m^dz+q-RPPg@OTvU+k$yWp!2%*($+XW_@-az)sJGS)=vHDX&+@h@D zlk}^jmVHM$4)mm1JLx-nQ+vSRR2sIBc%u<%ByI&-k!DJF(@(W)#>FO;<~IVMZA5Yj z`%v!NKp%&xoXnltwG-N}&dl zq?1ns`ikg4CV;^D52bfUwGrW0ygJ4P(d}Lc_M!`Tr1)pjv4+A(?Ow$>;h&{*PAC{o z_3lM?9+kogqG1XmkV(eWvXVV(pIXKSbw`RBWaoM{#=^K9(+XyRfy#w*C$&4^9w|v- zUX_Azutzli0NRq4jXajm6ijdvN*VO7=sl@nCV&nBj(DyfaZYK-;*q3^CJ?QuvZGF@ zW|>gM24N~Y^GPZmw3UK4rj(;b%|h0_BOm;4oe7vvL$8a}Bj8k9egwbTblEpu*0Wak$l z_ncF%y1mrniaAn`xg?sebw%ywUWwC+P!6)8HhNDYOQ{7dpLW^MoOV*(s|QR>L~tEb zhyYhUT`DV&do}ZPVm-SV!4((Q9eB@7^!ss?y$?FrmiW%&Q2o+A$U3p==e1W`yvLt1 zJZUd!jg>heWB6D?O5HzXC0~42JJ!8baO*3a$IyinA>}xvjuWWlC+G$M`cp&nno&uW z4z=oA_gdXuN(+f+RyOZfUW)j5^|z;5g0z&&>JJKR(mxU*Az6&i{c8pF4@9x)RzHWo z4N-FWf15@;bpW%D6Z}U$$;qvL4W=sPyD?26CBu}b0|`I@C`kGwBz>!&Ea^TaA#Lo+ z&%>z8DG6-uF`RdyuKgz(*989Lt3b|2e`?h(R|{>mId#U;bL&;dx{}8(^9>KW+SrU& zIq*jZb-G?A6{{fbjFVsz^$A-g1B0eVKpQ-{Tpudj%$A4Zeu zS>s5!Ta>mQWT^#v5G!6sR9Ym^?Gz< zpQcikexj+}D!^*r)y--krp(ic3hj%OERWW@px*?)bDio;(t2&hxoow}=dBWL)3WQl|Kup z+D}VMa67b{#8{(#%}0I3ey=xQJ;P@5w^Z|KR;M{2d)Ao^sZsi0r)}>8;!K%yxw*ga zm9rVM{{Z@rRU?ibeg%}}-~+5WklXId3qQIfV|tmibwZ|mGrO|VQn1HTTb31_(q4r% zIQ#e`@~ZD$^!t;Nmy^DvCvodWbbXQbdJn53s2?ob1cB^Q~aC@4a*SHsSrhi_6@pC~mJ=^+-0YFe-abvC_f&#P*6moxqfnzD5x(XMiA_jH!0 z+7{rFk}#w^(u#QQJWLAta^dkYn~N+*Z3{x;rD4PlRHdA)M0ydVZP_)WJ=D`Wj@4xN zg$?yPCioF-x&S=Lp8+XLs!;y`<21NWd#OwGHPWE?{>MAqjn}3xR|`DGpLbN2+c;VW z4z!_#p#K2Ol1H|4+O4lC5hN}vd3CHMY&w79G@+>q`=pHf=hCyznwqC#vb7+jl#NOB zKGmQ_@F!Z3px`cBy-xlD)&BtGte>-<)uYhOQ#{JvL&%WAi)(E`Xi()U2V+j%Ev&i| zK09Hy-zQUP;kO6kNcP_&*pEsfhLBP;kaO)u#$hTeisl1yFp^IRSo|vc6NBtcYP}n+ zqKiJxmkA{`T)A}mPDw^{T4alI(xoM6AwZBuI3tl!vTeyyPE%ajB3cZ`E*4D+Id3D} zP=edO$<1i@Yf1r82&($v43JY^blZzMBCP=1VT+Ze?w@fQ-j1ctw{5Tmvqg(v6*XFgH=jH67dqS`nE^rw+;18qO0Pa@xdFgyBF>9=piP0V!y2@t&#L^B`5 z40V&7YrHpO=015o* z#^UD#PTrWJvWnnCCJ5t6rHCW}<%kwopbw zc&k%Z%y60|HuRCe{HRzZ0)XwC93uzmL$1}6jXwVXTE0lOq#ii^YB}3}GBNoINj+%i zo-&+mz#a(tiYb;fU=dRCAQYWJG{}sk`U>Tgju}IXKFdxiM-(TDplhMiUtwsc5RB52 zMGU7;Cc1|+Ld`*7?~GH40QRO3plQ^fYQ~{jL9P@ESQ~e)5NMchM|$CJ+MY^9D5kX- z6t+EU+7q5B@{x`y1ORphia;7#0qso`Z^aTu+*blqS)hX!^SvcZjPXW<3gK-hwF3br zIP|56^h#R6;+;!Fy<-6Uxf`A7VX82-?MN-2!m)w$54|T&4Lqa~%`#5>&FILfd@IL%e1tx%86HM!z5KYvg8EB^qdpGs8ijMJj5yZp)52ksN47oUr zmG>l?wApV@xTf1m)xW-KiEDPe+FJstjErN$`~Zp4nQ+&;C>< ztTGVXD{)d16QmQcsi&$LfJ9Bdr|t4qjVKK@;Uxb6iDw`F=v`2HpWCN zuId%0D>0jKMROl@?0Zb`>KdeJYDW6roP zHm@|R9HAxQ*(TKI<9c9juJj1*9;g{QXSw~vQE6O^??jc|8l;RimL+x;t#)Ovm! ziELSU>Go@Pdm`ubZ%_u(ONn9`7#x8pAM1+A`gU83{{U%iavH~9u2l!(lw-Oq6ka5o-*4Sev=^V{Q^3p$kp6-A@ydUDn;vN407jukr zpgMMCo>J|TK@uUjsP5!s>p>}3>YCVX;n?X)kfkALAxTnB2qV5}Cs7(I%Ejrj*!uUX zqsm%Jw>6{n1XivM7n*Gi%Kjt)gs=Qa^ggNly#)q*MP)z(kaq-Pp>)e7@z4((q>_}H**ccqU2LFc zst2Y5m7N1BJK$Agkt8dFWj&^xmFHL`AnjAuTLZF7X)T{Ox2;;OeHE>3>TAoKn!VZj zVVLeAMW6yXspQXkiAdtDvI5!B>1l?N)J<-y32m@9QjW<_)RG6@tQY>8jMd1-kA*dA zWxH!j8-@5Fg{*(f>YZ2pRdcmP5uR1y2pVmTDoO$4#v~=5{{R@ItNw#glbA$kC`)4? zjDDQgGHjzt%2bOelNh$7=NSB`yG^wb>{m-Vq3bslL_Q{e2Y|J#ZQnZmcE@_KTbpfo z@c`e}r|p)=s+0l8n1ROGtHM%EGBd7>)>lcpEk0Bctacsf7dlqs6_S^l)ufc2eP2ch(@M--YkrhMxwqKnc}q%@|M50s8`>03v^j-QzC zPh2`~djZggP4YHIKYqbOPLKNIEqV3uQ|U^L)5(1PDYaZ zVQsX%E8B9~fl9~SDphr?kD@i?L@RuD5`cshq;fqCPFm!~gE^-j$&9dCkTL%N31dqC z03qfc=@f0o#<;FK6(d)IwKtL92Os58egSD(7x@~=yB|!NX}93ZcHbj+W4NZ4M+Ae& z_B@YcQ?Z&$sNSIi!Q?`kvz4uo!-V%+UO;~6@ao5IT2|Nsop(ZPT&SV+CP5x!sL2oxG80x-SxU7wcv$^=AN_jJ@)%AC#ZM$`F?3Ka(69(1s&XPEI7jrYD(M+I zRg~2-d|W$_JJ*+<6m{ZcGGc?;k7_V}(&Gx5QD>oK7*g!SC#8V5c0?F^?gHfyH@D z4gjU4c;38KkxJNTYvqzp0Ml!XIK>nWPc_Bq!KIT!A~8|MbAh!swm}DKkx~IUq$!;3 zik*%7kzyuKA9^9ULqGc7jYA_mQBAUwz~Y%PUVz$)2Wl<2V2#BR+dyzB^q>z)su8ec z#m4xjQxs#0d}BDpDF;k8OL_WYtBCHRdb5XJo2*}5!D>GI=$!R4dU}u^I zY=TBd{{Wh$rMH8ypjMHq1XnIaOytWNKA$e1THtfg(Ec#sisrrcudN~wrw^?<%88)@ zv)_s(kWMQYIjSKXP|a;mp;X8X1SY&eq@iQIbhsgNzT>~3e0pEJ=p;*JADaCjJN!5W|)&*k@fCVTk9jTD&abEF3tYPA? zYl`2D(>ld@PJlV1W;pHu#RBVzN{;mV&R}m*a+XheRun#Eqt?Ci0bTP^3Rx@U)9kN2 z&>F@6F7=xed2Unzy022qgai;U@||cRWRZ z)kAe{{ZI8suII7ua-lE!t4d^iu}U+xMk?iO_)8s&5oP9) zwaqd%PjbOe`BnDS(h={R{Rx8Uh>0b`* zPNXunQNGy#RBu`Q8@D}Tvs*-F%S=I%;KFt;JuTI!{YXn={*hWk(xa_OaX1+5Qxn~f zF4=F$8Tm2KZ{9gjj#iVVQNr3kKfWUW0B6~jpzQm5Bo^?XF|iez$B8=6H%ic?qh%!H zA6oq`xOCi0wY_Y-%2pJd9H+H>@9{U`1Q;wkEfC5^qvlB4e&(%<)j53=B|IuGlt6w9 z^-y%D5#9L%rXiyHyD3tim z2*C`b{{YffzxvZq9Z7MA8Kxrzf~6hBXU%oY+lPyKp7~njtxGE4D3jQB9)qv|Re9d8 zcIA9R3W}11twZg#ReD>IH=2>A#k|BXp^$Nb-1_lT%-*85fa+ZtNY(!UugboBM_4?L zq)bhC1SJH1RbZ2Tl*0v0x=x&&XOW%7bjQ;r+0QJtZBshN8;Eh0tYO69FF637GHMDJ zP(LPHkyoq5N!SzkhqY!_yX)?_li?TQpDU?PAt$){RqDjUZKyc%JVbJb2d82x+*B5c zHs0B%bn0Avg!to9xZl#X?wE#_VWfkMjBmbcBIz}$_uE5n6@afQx%(Q|pu6(SrKb{7 zR6B86J=h;a)!s^rsgtEYzh$>Rr%siB_(?TcnJZHF0DDn)c`y8fN<*0NoLhWvg#fYo z>#C%gt|@p?&%If*MN-*mizCFfx;w{`hX>JVDL<+aQ&y?OU>;3Utn*Y+GB_vqkruJ; zTPbjUi8VWCm=r)HXV)}9(R+^}OGrA7Y1dsV)S?FU9|gctH_bllxK1-c^kf5Z2~tf{ z?oAD3%D1{|i+!|3LhD(_p>Cmq9s5Cx} z#{5xj;(={oZ`jdDUi{M`SXdxZ!iFeaW7d>i82Z(s$w+*w2X58kyeB!{g)eV_X)AaH z^HfZmJt6PTfLptBT#)|f27z0^pY))`u~HNWGz!ShPkMdVLGM90;K@0ze3VNj&!!w` zfyFXVt_i7Cuy@Eb!lR7i74>H&g0_RsXvTIO>#2>7DbzF*_N;3jWYJ0EiX9q9dUQCS zYGh%|9tCiME1oKQ9qWa_Xn;bNQ?(^d6W+R@4|;5;zH1ovbGB(?RUs!Hl==#BzSV+w zd)KuAwJQld=_^WckyzavQ-d7R6yd=%g%r{ht+q4_c~2VPkg?vIBpPi*gMnLN3lHbL zbi0ww0j!OwV5p6bD;RmlGnxtDPH{wOIL>Hi2K%b9fkSy$G)ru7X(1cti(w$*v4>-_ zeJG;Y18i5CMLM+MG)!0T$tjM!9@X<7#?`OoUjxwB)3?;+g=3so&A%BSI_!6f5>Cg8 zl$h~80QiQp`$f>MbslPv9auRB9+jwkLFu`RxjL^5kWSw9N|`fd^L;?5uA;fz9=Rp|mv>W^w_?@S`_^GP@DS&O+OVAPJu9;uEfJKcwn)EC>M7{ALkUAk&Q!H&LQ*$UlzSy3B%eW0 zom01YH6VKxxLMyJ-OyABRyaQX z)pfSbEGRZe4Xcg#{{Xd0L+Fs?0@BcRBxzPo=sy0UpwniZ3g9i$Qm`^SHDhmK^rnkX zY)NZIlkoZ+#9h8&hlp)hI!7N`+!&U^n@e(10RUjub@0cg-Ap{as1%F4F zkYhcE6-BVs5}h|BF7X+HbK{p8$F8>4pXxY zEM}x&rwR!{;NTx@(V%%W%T>umAZq^rI+8N=C(Lu2r&?TfAS9^o$F*1)C28E6S9BC6 ze1w2-p;cRytgJ-8P~G2=HE5I_WuC)pq_BHyc5kM z$34HTc}(nG60S)~+GzltND2q(oR9OS!=8upphLMEovG9~In|uh+ZiC>;XIM;UMoJd zI#;&mYVN)>O2!EecCPCr{pfVBea$alAPP{}QWu~5Q^{WZ(2c9yQ=wb)Kq0NGenk?6 zPz|rt(I{CQ_oA4gI<~JB9CN)i7m>|*YVz|`ixNokoFbWV)26t!ZM`t#gBzNvAs~4U z9OIGRg;Ky7Bx08(eX=t_y6H$c`%rF(H(NUU-~*B>E$K>9%5%5YyD8JS+;-Z8NGn%= zTI8IYAt|{ZXsilhKxcY)UZJ_B$j^HF6iDVk?_9_`is5W}n&Pkq)I NZ`_ZLr78E zY|;YARnN-W(W$&;+7OWc&1X| z#`WT{*iisNRB>EWyJHm3h(_X)r$1U26U%T@N>Lr@KoedGBtqe zpOz$TSi=QrSJsr5QU)_h`9tYm`BDyOGaU~PTIO3k(`$~QzVwu~q0VbqLXeip+Z0P} zp*w>?Hj|n$wCCEg07_Pric61-Qgob9?j>8C(Pjg<;nWa)D=q6@nimF~8Z(i_YHzZ# zDmq22FPtek%_CM6==lEtss0e6Q|on6!QXAFu;0N?-vVL-pYYe(Qo2otvzCFO6VWk( zfh>SNl|Br9NuM?J{{X?ZmmSv(pbTRa)Mr_TX01=8j zK1`)cMNSokxa{tp)f0SMQ2eUKF$h%uAeB7r$dUs1A$!f zi;-wFi%(Hr8=Z953B~wR+7E=|Y^!AtvciYjt#8?G(*vF_55y!%K}jTbAL&QL>NwXV{G7Cd z*b;NKbmGXanGBfSSa(e!R97tB=PyYb0*9Xvry3*H^laZR-IzQo?1g1oY`8~2YtqG2W zhK2!C2_B%Pp_A3qq*5N9*3X@7T^l)YbS<(5lZ~jZp6L0^zWZuH-*7Rp74!-^PBI@u zEu>(a9k{6o{T&cG5uqt_1aZwldZ{vx3QWg(U#8~UCM{kXl*^ogKM%cMx(JIj%BPFFalq?(*j>m&VTSS;h5*cilW0B&tIVt{O>s5@UAsk}5FE+@g zLrHNfN*_unhGTmgi&XJ;HDDhIS!mm!`Rb`v!j-e(oC@SnYKHplCwcS-R3o0Wql1=FMP!s{>_zVh)qFR&K3p zNx-Yk!Y`K?s)gMFv=f@-&l@j^lpC7#ICjAl8eRA8>rdYo`_6mRV{TWCJu}dcM4DYV z-BkN-9P?4AU%eF4{l3*EG9(baxBRQRjya|Ru${NB7YN3GI$90TQdTpurG<{>giEK_ zYFb>$-;B^|fS7Ab8TLKt;H~I3-1GvUO7q15tTDCca(JQ=p_~o=v=eK_H`xCGT70PD zfr3KwN-7w^p%SWatmA5AD@U9l=LUs!SOl7JZULOo?kK1Y`Kra7*)$IN_T+8-XeGI} z^i+@ncIVoooK^|v)9F)_ws1~+2Y<|}>nx=GX@#jpSR2u?h7qe)^%u4XK1}-PoE@v$ z_uO+!4H9;){ISk)n)?w($;~B7{Hfx?yNc(_sO&~)u!5aL>?yUR<35yUfOCvj6&&M= z1|!a@X-ZbD#YicG!A&H`8~JyvU`EL_LII_sIn|9cN+XU1V1g8@(!2uB!jAcD@+nDU z8{)Bn5QT6>I+jWP8Z8V1+Mh!h?M8qA8Nl3Eybj`|iy0hOK3_bF)M7L>*wT{9hXSMo zap_$V#S;MrGBc*SnEC!1kxYMTSdJ?&A}LXldr<0(0ph0>8`_y%Vcb@r*%Bk~_-KaO zz}+T|OCHqnSQU~30q`<56op0hsZktKOnvsFMzSm9f$d3Eykm-&OnvvIi22}FqcRjg z2eo*>AIthw;EX*hpDXQKiL8Z8B!T#8VkjM|TjhpxUHNg?P@`E2^e~`up4D0E3o2iZ zr0CrHRrzcuABLhXaT2D~H1_nRlx!){iYnBIG77*NfyE7YbxDk+1q`Hjt6xz@3_7EY zvsw37Piibz3fF<4k4p3Mz8X5Qu0|d4F8J%x3rsw9l@$K~$|?h*Iz9Q*(=pkjKqzLt>aj-_RnJfl1KdGGIvvHVi%RnA|?A!^sSPM^}V-kRyFjo#$sNyu3XP*GO- z(kgLDZI0XN7fzdjI`kQk$t5z7G^;rCbNbbe*U;Bvb`GqmD?Ud#!pCD*Ux(cv76b%a zpDGQhK0}Yelm5Tfv^ek!ASL!vTxA(?KyIKvi`>^N?etYiBvox43d@KBOJxfq3OV2F zUc7WH`?`HRA&$q>wNU1sNG-PWZSR zBOq;2!nSiZSetNit^t0AZ0WwI!XLZ#eZM)9XnbqR4x3R*zVqSNb8T}{&~mg%|M#vC8sH=q4;U2&zOnpQx{ zxHS@5l7XQvel+?YTAb+S6+Y?|f=Nlv2>nG=*=W6RZak>vrvVv4{XJ4^=pTmt7Qz^6 zTASCX1FIM!se7a^g41Pf(83Vm-yWOOw;1}+Q{pG-R$*>whCnIPw>4e8#7NXov-Hhf zCQj{@IjrcQq#61}b_(K#VA#y+*CHrCg1vtUZybkF8k7!}F;qYy>DJG>uKP za!C78Fq~=EfCh<~WU~b%gM*s7L{dvYfrC|>LrFSCTH{AjTLzg3$oc9TNjR!a{nRNc z7{++3)y0COn$Nn81+=X3jMqMHiel`|FKa1~pnKGFOXCWnua0YYpL&jE&wc%C!^zp{ zN1{mz%0_6mR0n#GLXHOXOH1^rG(G5$3)=$~?Rne$sf4KMuIs?u)}btYE5~Y3cihmS z&p}=eWZ?SIp=>=4@H0=P-OhLewMMq+2eAg9L!&?CO5znJN;K%$`q4>VF}+1H=f2cp z9PiB|R8cU!LvBSVxG*SSv&L~sQufYhFR*MHUU3ML2zcel>?KVg*tjUb1|7f zV`0TcJU~wsD#0)-mJ zP|zHTa5gK7UY)T>`GG{+AMnSeaHHCihe7X7gP!$)V`Vb9(EJQLcK5Z6Gp&zuco@y2@%$PjeNQBRwCu0%^235#d-dzJ4apz z*--dj(rKG7#9LuRt5TFldepk1NVC8#pC2VoIr ztXj8BL2bb2%VdxKA`c_-{*^+%{7HUqPsuGfTYW0R6gkd&`+u!a*y;As3d@)oMXLG zdXCDaYSN68?OGGaTznzGagB`uHX3Co1v~nIU6G2EPF`GDh+H~(*9ir@o%>WPofNkf zkkJ|6AY!*Cx_!5Q6MObG);e9R?%q3bMmSqwc~UW0T@th`E5bjOT5Y{PVK=O!aqY0J zMUIqcVOj>>)DKa^jcbCnl%*#X)$CHXL*2JX$}di(prokbTvY<~czL-V3Qj!Rf2~@t z62tNqM*y5v2LAvZ^*E=o<>tKB?Va&jC*j}POMYc=nwz&yW=jPsUZJ26JDSDOcLMGs zsm~^KpRV<8iGD0O7CS(cs05Nb8ZRB~8On`19jY|=YG~uju=*PFJuoH1Q!OneKosx& z4GP`ihFU|PhgtwVx2dj+y+wB9{9<0o17_qug;;OJ%vY z8bMN0r6gd2IrghmhoDGku(FayKt2A3p`x&cW#&4;N!!}2pySa8J5)#AdIsNieVGij z1A##7Fl$}t%PF_OLuEO_am7RQ>}F)Vj}g2mW850jVMYmMBVmE*U09=T$Vy%adz5le za5xoBx<^`+cdZI~Q>2B8sodkEB|`?T&=azgDU4*E1w%_f1dZy`IVn1TLGcJIsgtaZOl0|3M z7t#twuWG+rRD#-Ycf~wJXQsAF57Mb!MGY%ydB$;9xko7?shvcXcxuLKd1Gj3Ix%jh zhFW=QN3VLXHtrRS_p6Umq=m0idCxUMb&?s|$_{BPxGJDcA6v-x>$s5s0a|iK#sx_7HGnyqU;}telF!Vk4 zr5AS@B7hEaOYs34aoT`vd2Wsl{`9muQk?3})gJTMJb|Cik@G4Ez*f|<=;Y28`~Iea zanz{ep7hJIbqs^f#)U5cc0ZL(v&qW#78kxg)bnc6GI#bg2aE{XpN`>BIO3W=Cr$|; zbp(O9t}JKXyp@ji(nWoc2EupZloP)+ps%GXn#K}EHdBsi-0w(GqRb>$Kpwl-G~oBH z;L?QdoYz7Cr6Yq&3Ga$10zjmxIqy#P;Xi-Sm zQwT{E@CUzoY5=tyjj6IuE4jxsg0V_udb|j(1&a1TG#at2R-s2HD^Z1jDVLF`5GmY>;R6+g=%aOvQmD(o?O!@| zb*Rajw*LT_@3nnD)lH>QR&kw;V7JJL7fnQe7Wu9(Qxm5PB06@~n`OU35&>7?{{T8& z!do`B3$Lp|&Qbu})Xad=7)Wg+DC{bWer|R0=$6uO$}y~*_C2eG(v>r*bz*M)Upnfz z?LrGnSPIA;{WI-Q-D6~!C&0dMrw86NLpfp}L#K z*!Lclb?K>A&k2bEy+1I{wXI*oqqaXfkwv}du5ZqprLwr_`DG;FrNoo@AL=P2oQC9) zceO|HA~unLvYiU`){&5Y;|KGp87h?0T6qccHg`|4-)e=oIOMxhUAZ!-?;&YRjx=^r zyJxBiPimsI1)HE;og(Nhcnnm1Km7juu^S1uGw6{+rW9K80+y z#*U@hlDR|@oG53w#^BUrOwF`iT9*~&DTVAlpI>UQ+U~a5P5#bOe7m+!^5T^m1OUB2 zLce6oTsIC7Rjs}ArCy&yREm`d+D9YxT&ErlJo=KPZJ$xt{VGy5)d-S_4csuL5uY~M z^dQwAtszgeJS9XLk&LB4>_??mWA#}onn0ZB+1r5O~t_SNb- z0sB)CtpXMb9ej_ zD3i8pdg?1G-%85U$i{KSV_jQgax;rsLD>3Nm!BKEI$@HuLZ)8Bekisj6OS>)S5)tY zbdii2A&H5|%D+ANH)BYy*`!i%}28EKhRU?btPHI%_Gprl4Z=CB~(gO0R$6K@zuE| zjY$-Yst8K9t~f{;PB-*Rh&}(p{5UxGzN)~eE zwuLz<2}v2_^r4j{JDh<*NOa)wLMlNUf@(ZZA1kM_VKPD6R&(h`#e5D`o+`G~jmAl- z7}7STOThSYQ$GF+rFfK$CXk@8afypVn#b%NP;rXEyB5(So#=#<*ig;n z8Y#Lq1Eid172u^BSQ)Mqk-)9kEbvk9OFN1{80NdSD1)*Z{Lox=0y1$}hgL=`%~J8U zYk0aNQ!3A;WZheC@trtEbL~*fth5~!3u7ii$X-fF=8+QL9xR9TAS(w@9)s4bwu>^_ zK|UG!;;P+T?I%$39cNF;`uY866K^+dW))Cm34B<0!D{$K*93?&V8yn z94m~7??|~2h{!tfu1M?;xv3^BP;63Fb(Dm;HPjV6r4HVM(y85H)OT5O)sU!`sO=d} z;Ey;@BkR}?dRnwUMV5Ta6{hJ9G=!*3G2!jcshv2VqnL6 z7*4W5&V4hsed>*S>ZD9zyVS;(!X-2dD0FBlLU#%&JCWNILN7NPjB7+04Mm0XGQe%` z#1Ilvq^SC2=k%p(Lw0QaJJj~qV$HT0YSk&V_zG8)6{D1I_*4kbrgKvLIeKF9(jm=l z#$~XxzZ~aL;MS|j%yli^oQ+<)4tJxvN+JW7qIB?`_5s5IY9Ur;Xp052;c4jvD5pY%1+ z6>8BJrd(a>by^nOpD-&pSN-Zv_tp-ymZ!yDRE;Yr&-aM$kMgcJVuyN+xP>6P%Dk!^ z&k{epJI0mt{Q<|K#^n1@qY4kVjq@LgQucST?^c*P4{d099H$5=o!i9>soptthP0JQ1+X8Ajd;VLQ{d*Y-&?Vv7GH!f~7>8vLa2p zP^h<}6p|F0vpNz+Flbj|YDvmzj8&yF&>OZ%_o&CBg)7S51zKdn&+yO=vWD^pG(ph> zZQ&y&12hbnDcwV0H^oj!*dEmtF+&7re@e#ai_@W75nW2oa1P?UwsL#t=}t8im1#N6 zWwR1C_gy$ji8VIN511L#$f)~eyn+-(TP?9l6Zl3ckgb(_pHC%C9Z2n2H(T_>dK^)2 z7|(7i==o7waf7G~;oI80HIcYi0f?* ztfc-ES>sc3paJ}=#mi8YBC*I?nV5DN+*F)dTDd#ss&S-)$SRPNc$1dQ3| z2+8-MA5K9zr=BN5_M*Edi2V^r_ob9-q1#ufG+Sy&=DxLT?mUJ^aZCAh z*L*>WNqil%MvxB{h}hRg+fz$_o#};0IoVW9*kZGaZ0*f9)8vmeJET4)pJgF^tq9ltIpE=4NXg(M-(;zLYFv9q3<{1zFYhq^e3l z-!;TY3Rc;m=*TUmf@q{|wrF-$sUtM_+icb_=Dj?c=x`cgYvg8{E+`&rT^g(@OHXns zmlQFzGFDA<3OpUCIapLrD3jj2g=ys1h#*oio@fjbmxt23tvqIi)DuZBfOoBCpwMYt zb3&o)jg2qGIHIHmhg3nP9RiZKh&vi+$Qj&<&6y!-(kbLD6N5pQ(s(u0ypg`D7G{YM zKo#tyjpzX|W12{s0l`%jBnl*Qk=~M3?ZpV0N%a-gO2-C`qj4LathPofr7qZIK1ok{ zw7l?fB_Pu*#RYlA1s+V&d{G$|$lEOwq;XllRxuttmt9hP+47ujeT8gtbfK*y^sJAs zp*p3L9BNp~hE4#e`l!@&H>ZDA%yRp8a%|IKZM2tPB}ylK2+y@={X|@d>f$YL$F|1Q zRbe!b>09xTqMQrV>dNL+1R*s)+b$!z7aG4>ze2jTag_jYZ#2Y*AI41)YX>I%O zRY7N{y0Q#d5D@5GCfP@XqMQ(vsQ&;8K{|=fp464Q(X*_dDJgOzPn-h-!dOuAC@V_RIMP+I8)F1jBdI#R z$J5sKI7^n>g`RYyJwYJLdNRvJ8Q`~# zh#AQnW3@+hbW)M0^jlq3Xt%v%zt04cP>icqTEGb-?Su9erC1WL8HZB#LX)9jcTnL& z-mtE$mDJ2U-O31{|z$`Y*QCnJyh z)gdJoiOMcW*XA|gup}uuNKhpEQL$Y?Y%4nvOm$Q3x6`X}vJ_U3Qla&zn_TiEAh@sL z9M`Fd@i-;2H?JSWHvuOY?O!}~04_STgSO`#>wD`dEx#EkA5yrK%5bFY4r_ zx3y}Y8&aE2usq(QJk=9%=uZ18Jv=1DOHYNAr~nRqJu5|n^0$kA9Bn0O*-6C)I&Lb` zw#MmJ_#0sMBAK?wl?BHT(NeS>$MYtS88inZDVtD_DL*QNsUYB>jq&MUM*JwVw%it- zD8V2TSdF=m{FWR_NGl_b_4Kd9_?{K9t%k`Rsj)6oC6UxiJA^g>#}t7vBg#!VB!RJ@ z7UDv-O?nh_hqmsm>13-OS`D{UcA}fyXEmt6@1XO6nu4Abl2wqOTCqNmQg;V61wATO zQ`)r{uF95EkWPNp8sfr^NCPx{=7>?>dWm!`D+&t6c@${YDMW>AGfuF)FC>9MzavUA z4)hYP3PIR= zdU0=@WF4v@m-3&*p8U}sI4M4A97pC5r}o_bdsmD8Pxb9V_@_VjuC45&KWbTZL^4Pk zO3IXIbmtf*o_#=!;~lA-jQ*U|S;3QRSUBzeRO4iUjwy|2JJBtq6qB9>G#AJr9iO5> z1aK)pC|7|b(z|B9tmzQ7C=O|owS{nL#VU}M@4XaZSk>6mN=m`}G)e||rD)9%Rf3?9 zcN^2mUMXq-cBB$BLd9SqA!Lr6hWyK0o>CL zq!F>KVlqOMG6Al{E-3D4Pc}1(b|Xm}={18yjC_RdY0~zcjSd!4FcXSp$4)jU@~qt% zAkAR$N)76$^gc&=Et!KDCwdxYsvRxq+)_a~rd^f+{8Y)94&czBhLr)&6w2K|IMqPO zn}M;Y=Ow85fui(fvTem@gG0Em2F8M7QkF5OntYIbP)|BR$fdIdDpG*vkf}@KRYxuIdCeomQijHr z+ZiH6%2wnPfm{<6t7DpuNprqw#DxKpcA%3prb;4OxF(im;BI-SpBRqy;`uuhOb1Ao zW%9M*B$}3EWagpmXe+@YrW6Vho-3mgrDSI9&^)IUuu`6Ch1L;VLKNvBkOu_n+ zj42*3lxN#Ked={FX2m5Im<%+dk>aQfkEJsv-+CIPPr1Zalw>7Lc#8Uj*NJ=jk5ZFb zC~tHRK-(NHz0(Hb^X}<(QzuwR&cP(|bK4bJyxJzkzDz}{rqKzP8%r^e>O!1Io?NW!v6-mexuhqOqzEjEcwC8C0c z60Pv|kVaB=ByN2O$l9w8u_bCvx)ce>ZZZq5Hj)$K^Q#`Gll2s_)IJ1gw_9z^Z8ZTw ziEOPoN|JvHjy%ac9FOv)w?2mAqH03J8e2=T+}mZT2xz3G1zreH$MO%7k7H+X?bp2)}W%cRj4O@wykHUZ_Uqbw`C=%DY&wd+sbsWOR4RvJAyl(&aRLx z7RRBvX_V{lu=);^zN4+4`cke)^v-cs#=H`nr)aKxE$i!tRP^)KE}?T&#=ij|jHav- zu1>%}Mt=wZ8WCM965S;-;l$Q*yAb>ec0q7q5U$os{~a-%CM zJBrBqjbTnWow9TJR+n^+xbakzk+H1K_@p$YEoX9S8Lo2KTy+#wrll=3rx4l5$=a&d zd!y}Wl)yU-C{^m>lzfrE0IN9r)CBuG5+ect1zh8d)gu;=u1d<~m#F2frwh%MCvo%@ za_O7H^45j2RpK_BDLfSqQR`66bm}31qL7~6x%Cw(3eob6H!=&lleTbi{{ZHpUnX+V zq}XpMb<)Gg8j-=^5$RTTtlHf9#z4{rckVqaC2{+b5}EK@aIFLPfDQ5Mio0|_SzINw zVM=RG{#D@MdlA~Gqw-C=E|w?_eDVuOz}1~6jQdy6T`qhZdtoC3NUSEt`=(gK8cv~{ z9gS{%F0i-=2W*UwYNj>DMCY_ix}vW1K9j3B%^Bez;=vWt+D37WtEiO&YQWn&Q_U3- zPG|&fE1E)cf(2t9S~YsqbhzLs`WhkmVT7Ejqn@~j6V62o3RYL|Fa85m`(H3BT6uMYH73CqU z;2dOgUo3T1oiL`5P(cU1ec#v7P^K0GX~qstE938p@d9o~N->oZ1$lnA7m~Qv%=MN> zru8iqttkL{Q0y`7Qn1>_HFh4A=T#k1R4tI8I5b;pB;`l)uO%tQkSp48xH!cdptcYm z2PrBbQXP^L{9m0gr$TTB*ye(fCpj7Rskdi5=UoPggd7EJ{b+_|hl*m&RoPq?Fy=>tFISy_0=l72cyuU zeQGr?g&Y%0*TxP-ERige332^E=k=taY3xRGO%srEX&{uVwM$1gY%v8n$s;sM6rDTl zPu`iT*|XwP(MeOqE9u%1rx{RODV{u zAQ80{6Ii4xM>OblCWd)hMrpRzrFN}#qflFm9jIlwNp(jZ_Muj_6UnR)q`AH`y(?Mm zK_N;flSQ_LE0bEyNKsOaGUC>O>qLf$?kTrfDi{h&~Nj%U7deW=|nl;jwK?HnXtWhf-0iZqyr~i*4Qa{b}7` ziWOY(O{vEcMgYLA$UH-KJ7Sz|qE9)X+K_Sqz%+9Y4P!W_TM_7;w>Fm04l!Q1PeVY0 zp-duL5Z~g))l=$gs*$=C!K=<{5hn<9kz{4$$zDO&b3$I4ZRn05p4Aayx`n3-S2ZDV zn1wK)oyhdAcRon38fI10R-U7AZ?R*PSSlC=C&1Cx6NK+*$8#oYn8OR`USKAH7 zTt!D}W6uT@d`4Px3R_D4R4beg!19yys2e?{uNDe?ccVVTdKi?Sz~f~86?%Y1G04F7 zt6cj8_)NDY^ft64rAW{YS#)wo_!Kjd$A3{>DIzWSC2r0tG$QMM8>}r}Vw1aMoq?$6 zP9{TSl4(~rR~{>YDJ``jpd3LU;F=yHGaV~IAme{}zBWooul0kT8%ooP$nPzQGL(R@ zG36b`Yews-{{StNE0bA7o4X5;RE3RMDg)N1Dsp8PL=R9;T3b^AT0jSXOjLEds9Rcm z2?PR0Njy`R*v?E#szQh=1SI+zq$M(%aJKqLP&ng}REtt3?K1v)jv7j%%SZ0h_(99>Q$+{d9qN7;Q+%=#H7P(HxT-2>MwzHtBccEb3i6)bm1<_>*hWSUD?PH_ z)igGgsDY0AZ&qX?T7gn=fx!ZrKVoeHb$s@uL|Q@ui3Yw__?K#Rb6k?7;Uu0o8`sox z?at0Z(4Yz9$}1xHiPCSOw+dNSFi#c9>Z8gfF`qoQ89?Bt9+{%rMv}spQ8@i5W?Cy50Pjbz&iiNYSqs61#!(nMWOv0JhqV9)dKtF( z^8IPn&_d3`oYPqGw1SYF?gj-u+eDIf-_TbW9+>>-lp{#um7+4(*+%=1%A0%?0!Yto z=&%NN*m{~`W&?Qj8`9CDtOP!F>CeznjGXVswMM4}c+T}8$U1U8C>MlJ(hmq~+nRJI z0OpoKY-FeJOs({Tj^@6jNYvQsPI;y3NIOvB^y8eGZPyk?Q?+Xi1(w5rns3924CGhIGpv;dr0oxSHnQxp^&bShGBAZrLoTn5RPBMo5Hy-q3X+g)eMXA-atm29zZzqngb4@ZTj;<$yy1lof=Wnh4Kek{=q#2ZP$2LDbnK0bG76bp@S|wJE1i zgy~So?_0=FK82Hv=DMiCTak(=7Sw#BZ(2*rDId~@?qELvr{5IoXaI9u4y=vG>0Bxx z!B3;Y06U(pTq6{OOfLTF$Ll6jWxCkW;8_LW5*ufe)^``$tQ}U^+#AV&`?sp4>fd=Y(vPx zK;EdmQP3{WDk)Y0=7wH!jUn;zi?##P{{Rspc$>Tu0py*k^9!jX-B5%f(S>eLwPa)A zT=cqJKu}0I02;V-?TTfk=2Y`*QW7zbQ&5AN(?v3HdqkTrnKyY8%x2RXI?5Q-cR1x( z{*@r5z`5!1)U)ab;ryxQ;WEpG=bQ;75R_y5sfQ*)Zc}L+YV;V*GL7h+(8cQ5h|m;^ zb6Sv7<_Z22>ME%|8qK`CmOtJI^AItd3>wt!%n3>eLIF9?9jayigNZStmV$K*f&~}p zT(*Jl@zXwPzgyKl$sw3dgh*QlAdkf6Gy2mX4eav2W;*1Ck`km8NI~04&cyp2^ZC}T z)jtenK}!4puL~r$us;ucesvYlaGfklhdHke)VQ=94U{?igUwDjnw2r$MAf2KrL56S zyF`{$veSnsS?~Qf;QHdFrL^O0=!FTw*&0An_WuBs`+adobe+0PGN`YB<2flp$j*`2 zE7d>Jr7rLtODm4hG2i7nPExO?YVQ)F3nt96=EEKy5fLmpwdqoDa+6YVZRu(_Ndr1+ zVrNgE2gh|mOF_!E0Gf@PC&N$f#x~-SEf9I#8Q;*?&0Rr#m2|kcl`TQMk(~Ff6q{jvSw7!W!;>Kq(!tUW3Ws`b z>gAb}^9~S_p`4xbiixwqaZ5&q&JN)E*Lu3#@}M~9gh&bW-y^rusIiFViKKM`WI++F zYQnVOf$2=!EfP(`qrR zqROMBM_!_=I5KgXlf>6!b2JNO+Up(0Nr<-^J-|FxnPtB^(i;jVNg3X&of9?%#gs-{ zU@QPIwAGU7U~#=FYw#-s5OIolDMM+ADJP~_oe2#1@s!NTHD#sj- z^z{6K>_g6}#FQV>qHYn60?X<^9nv?V6w{^Btt%rtW15&|E03}RjD-`yJkYk?29heH zrqMA0Zb(vr>^99qPWXQmEV!0azo^Y=k>H`C3y44>y0(}km4S~+bhIUzLV6@bA;m`u zNjvSmao(S4r;xJ3$nF5GN%TWX2`WzAsvY2$7pbQkSqIcnrhr0OA?R?FMky&ze)VH$ z%qSow#HZV8HqmNZ>q#mA_p1wbkbrfZ{i%~^4Wc&M!FdS|4DHPU)>amz+t#AkBTs$m z+a5DX000hXKB0u;kQ_o%Kq8G=rVFNgS=Y9QWAp zNvz4$V4Rxp^2Vg->rT)!%PGhiBk57kyrZ4)eWVNs%O#@M4#6obVP352N$P{!l$MJcWQH>Ii?f!bt_ zKT25|RtXvHMK;g_KE3P9Xlons??+5JI~bhN=ou6_>nR1?k?L#cT(7~RsAReT*ysGK$$u-w z5dQ#xq7;np&e);dXaW?2LBD~A%RxgRZ(Ktv3P0;eWo{_wD$P9FR)u+v)stqUii#Xi z3eV?7e6T_?38u9uD;w90m0+Z()~%Rn%x4Capn%o~CyE@YZZ89RcLaDkDa8|w zK#}W;HMV?RnmY<<=*wwif30dYiPF*$sGqeCpcnW-#T4scpapMBjLKWWl%9QSH13Rm z($lFx-x<2JJzZN*fTb-4ZIa``cq4W5aBsF zuRl6kAMnPkdyS~HIEAD4hjZ;qac@I3&@-){30gNesPyxD;DAIn_8}lnr(xhPhHZ_|_$uBoypD#c^)D z!Ujg6?McZFqSy?i2+~qrNWzZ9b6k^YdQ0hXxa6frIRc+iT*aj583!Y5){AC}!l1U7 z8%{vpgfThEvO*+~gK|;VIidkb) z*i_s~%XTJWvRp;)R$)8xwz^8NC#{TRZ{JLVZ^w~ z-AGn98RochRkU{FPLn=;_`!8U@3|Q`DnF%UQEkYS;cH%%m2HaDek()?oVfsOt--2q zM_LTan`>8?C4Xv$JF>HSvr*_ihK{HRT$WO;fl0?S=U#L&6Kiy^!a{;FkbC-yyW1X* zZiLYoNJ=*q3GoxEqsOpL?=i|Pxz`+r`VTplG>D^jX;7=;vT;A zty?!pzjT5fj-;*B=^*0+oc$|UVdmyULRZKgt1pTKMI~+=fw}gpeX{J5lqsh&2;zc! zpM?ytSLm3S#^kVo3PIn6$o-$@#fr02L9s5VtAIG+`qCqZy~ zcA`K_n};xaq|q@MG6(Nc;tAtwZd5lLUy(t2eQREWRL=&Il!JlpYsN#Z<+QeNcvdNK z+gZk>eQB}K3u2!3ij_N{`tev+s!w8jcZ(5krP;fv_b6Nw|r$)+boNh8Hry(?ilr4~= zeMJkSRgHAT!L=hVF@kr>s?52OmYktU$E8*ngNJm4gahlfSYp9!M;XR_F;!D^O{7N4 zlDN_}sd^h5MHwYg?f=4<${;l zR-k)iccpUp8;xw3gFZogrQzkq*rMgmf59%tMzVpp2YMOnd15<9#Vc`1*e07_h=#lu z(zJzt5ES=X7QBr))OQCMC5S329N_PoiFWnW@tjJb3L_;d8;Uwue51wi&H%}vt+Hln%a1Um zuXOs0@4X=##|9L_d83i9QpwfWG`${DkWzO^Cx2>Mjr11V?3_z0Pzc7L1q&|M zh4_J3_RV>z&$fWduv{JeXeo=h$P0D1z$o0O=}9TO(j7%r*mDYTpTcr|>9?O$hYLXA zdB**!yBFdey_ZW5Fr_$yf&j?Ku1@Pfw^UlUt)&#|1w#P))iqs;kbT1N{gLL|l-Y4B zI0Ox-_!r2s_yGj=Jc`aPy=d&BTbhBl89UT8j;fNOQNG0l2c2Hh&54k9xZ^I&00TwY88_up*sFGId2}&wjZ{oZz`Y zQd>%}m2wZ(t!i8s(dAYTrx>Q;An019Mo6VJ+8tR5V?qhs_M}Ng4&NYIaY1QLwy=;l zJJd4zg4Eg+mnEeuPTbXo=hM<;M{z95N>it7RzG>@d*q33AuAXMIX&sB9Y|E)XtRr& zUVONj=qLv`EA8~6lh%==Vlt$qOD7l@#Zq8cA2Hh*ixS%+r z_;qp(5219z+@EN<#*p-uJX8#Y4DZ^9ORhH{rNnGb#+4>}EF-~&JTs3dsD)3txZ;~s ze6O)1WA96@>?xK6hMgKywF^173QKK`L~tlSom(1Qe1_Bqm}k8)AWka4QH5>t_gP;n;*d>XbnG){5k zYj92`(oi-mMKyLcma1-nyeMNat~;NC_!Q^r+&A4j_#|LwbXn z9DO?2SPL8A_o&ypkuXwQo`tkW3dTYEiae6+9B-n*agfSdQdF%waak8ubtlNmaw}36 zKkC~a)hpEfaG#Z0BQIL9fJI|Bx$|Ft#c@heaGVl&`c!k};=Z2CVaL0WRp$I&DkVo& z;FDCVjFifE#SiJych7n@=)~956&28}yufaL{*_C-+V3t%NNFwv=X`gsJB+05owpc( zd_iHE_Vp0xE}(k$svkg1O{!yvmT`q`lUC{dJ9(bsnQ67Cft;SzpWb?FsT!2yabqkk z8NmXrjX1kxa!o#sjXg~oLQ@qlCv%gE&Hgd<18Wg2$#4{p#e-Bhtb9eb$y=`5j*RN~bfuk30aGkk#pC`#V|1oM%RQ$0V{Fz#1{Jst3- z+d5ABU>dDZlRWOqVs(iD|~Go@Ydqv3~_gK~*}`Pi*F9l}UBQO1q8``zZ;r zKGRC%sRwU*(q()ZzV!{Ao8i8=+7F+>KAxXSMB{e0GrBcSpX*pozzb#IqDI&`sOz6v z#6*N9OF?03(h1upsvQ&4^7?XWVLqQ8KRG*Kj^9eZKcXVje+5XA3@C&)94$$jl_7e z$kfjG^Ca(@ad+fHQhaEn0&sTiOgkD{RhHGA9H5QpYn9?N5jDt$R563)&h&XTqf%ti z>KkIn3dkqgo-yLi!-&SA`qq5i;5RKUA~DGYK`K_n3Vz#tyxi{YGRs~IEhk_>8~WzA zxSOVp!j+l|VzPwcNCf&3QFj@SMP<~XqvhNmK~PZowqc&*HzoK!3bf-H)O~6fsQRy| z8@YHbHpj|12-KoMH2l|rc)uf&y5@SWD~uw#<6vMEifgE(OSv*t;kwe3!71D6nv97q z_SBCR2#lefif~SSD_7~qAl&U?iLA8a>jf(EV2UwrOO2cKZjYYslG3TVSa1?@KNW zeX&Ytk4#p-2V`<+IuQ*4njyz>ayX#SzSuunS#T$QKJ+i-hDnQgV+X!{Dl$XG3M86+ z`2#sB;-Y0d6`lV8?L`siXPulu$ReMIaZy+SUrL2@T9#6kbL&J#W!&+#Z`hQY@?4?Z z)Di(I`qa!+xhu_rK&s866U!iPT3=6E0$K`i9G&X;Zcm7uh;P{MxRNyEYLA_0eUN;- zR*oVzk%BR`0XhRtAc6*Q=~l^sHQ8KwDW7G;l^r0#kPwuelDuY?%ONRRK+Ak7L(|s1 zZ;xu*=8A%V2>~M|WcH~Bt-&pY>vaSYl(sRP1Kd}nx1+nFDmA7vflevZEDeGFA?aO_ zrYaP%2s!}R4oykRvz52=t(2um9rh%1$9iRLy@2C-U0wzXcii#~8lYX2b@eAw%elJE zvdW0)ZNJ^P>E!3I^{Z8-S0|vNMdg&48nCpWU;*nzx|fjiZK1TOOA5#%2~IXAG`3@0 z8r0!R)RiQqI!06vuhNRJ1he5Sm0$vcum{qGn`5`mU^4r@MV~T2+*1s(cSAZRfHf~j zQz$tn-_nRnGZCseVMjT{)Dy5um5kBuNf1h! zao|86xunRB%%_y8I@RG3q@azDtqm)ul_xF5kmD=-YD&?9K9oA#P<#DM0)J9KkK!o{ zLf&t15_cwn{{Y%sv0{OBwuFTfj4U57trB`Ir8webnCerUtSd^1p$EeIbABS^g}vir z1x^(JDOWYHzC8gi;w7$knT6;mRy>>jbv0?~gj@!;SAsTCP{j_%!dpvBA*hH;X+4p# zKYsM|K8v--35fKUBODZ@DZrh7_pIEXMqGT7v8LOfDpI7Ow^Dq-(k5Q*47IImZAwb9 zoQ>!S-46ybRODF_6y`MQ3&;b$MN&WZbJU2r%U$x}^OlCsh{~HdQc3J_=|Rd-@nK0n zv7*PgB6`y%D%x5xf|RIaRHhPJ@fD3Y+*LNu(>*a{Vm-n`&cj|p5T&|EDm-Sa(rw?p zWhk6R6^tivywgSA&>gPF7dk7-`Bh0vdU)R%r+;o)t#Mp-M~1XCGOhAzHch75?6#PA zEw|84vPQ@2NRw-W5ou}Qg(*PkZ11f;wAR+r1t-{3w>xK01V3wN=xNTE zwY-vIp$#~)PP}(D8#dh;nT^}-4k5)If|K5-T9q;*FA1@CWcgB&sYm^4q}Rn;a~HV~ zFWo6{h=#O;K4X<7L0$4VrKqs0M7nwwz{8EKwj3%thONajcKCm3nIRGzX{PbWN&RS+ zx|gPI5>TkqooJxop=rQ5U#xc74*bofh%dfwVz0Gnc!4g?E$bqhj6 zU;?eNMn`lQf#~iuweko%8lv?)l=U2B8)fzjR^3V}@SMPK)p@(aHrPtHwJu3{k`@#j zR-K4A)yEZOjnOAGp3P*pCRhuQlGsn7r7S$o5YCmfp~|&$xBGw0!n~B^f=dJU%>iXli?M%g%pjH0iYDja-}-^M4sDXrXk32 zX>&r8fwub(y?W_36c}GOa#~wLQdFbdVy5Hf##?leO>*CIb;Z2fEg)yO^ASikIWM0W zqaY7TwL!kyCM7CmI?|vBI&+YCKSM)au5HF|a)@DjB~G0nn$8|>!ldqqMqw7#kuEqS z{{VKbHxvcR>kiuYg*C)9zf9ER*z>2_()F6>4cExlQlY6oP(@|;FNR%XdY{;uHc7xLJ%(|;1&-asXDd|AHOYNF zF{dIq2?|yUK>q+btRtWx6+I$N$;Q&=N*hYglu<6uh7{VHVZ)u$ruPa5y+1A^$b3KZ zn$o0$pqwAdr)s*k6&9btS#I9D8vg*@<~Zs#V=6ovm5bFfW0IFi)9XP@OC&u10EYU? zHdlmuA8M^m>L_-13v!!#LawukPZe0YF^!IVoj_jYiL}6cX27)p82aw{Uvd^JsP!+R9DgTR2w9745VCvjKq znuiRj`-RHp;AHI6Xh86rRxnQciX!U{N?G~4Y=pSkSn(F5fE0tVDbJ+`QdSzKRom%H zVwnw?%MT?WWhgs)=hBO}^rV7IanOg6f(XvvdY^@Hj{HaLrxDw42wbL~3R=cJKgy-L zHq&m3;-Lp5w%9tgtYoX-dNFERXcA64B3iROZ;jzUZ)qqeK2f+dMX;^6{3qc!{6YAV z*y64dm1&0=^~6+((h!2Qwj1+oI|`@UdZpB`<;mD$HV+4E4&t=pgBIBB`qON23b;aL z)^^W^r0Rxx~a*Ed4`>3yy^P}3j z;*~(+W)#nHGr-x|IY@Y37bUyU5Oe zp{apY*6(Z^YJa4LG@e zGkU>NToH~6efP*eN}7$kvH582%l6cF%I3~*L!^R3eq@o~J?JT2Lc4%0tqsX(r06Tt z=09qW>prdOCLzU(4m<#j1i&wN$K;rGHtTF#55-dvf*Bgb_QYeR$+f}D>| zcIK%F)9jq4t3~>KwB7AqB&K}IPMiasK>aJ>KkW(d_DzER_0}D!^g?mNsqYMg=j&g6 zIEC5cr7;uYAwE?HK=SYS_Qt}nzwJY*9}VXG!ces-#3>6wI0Msuf6ANHVp(yv`&&6{ zlcV8~l7cy>kZ?u?IFzLCkAG_DXef<2RbaDtDH&fhV-dGitP08inhVQfT}Jtk;MTrDFBp=nwoY?C0Gg@ z)Me6R4Kk%_8OGEyQrOXs&#GFw2bLtF$WoZV>RN~%=K%6{sTStMNNMKOLdx}Fc^J;> z89vkipKE#F3NjQ>+EfRNsk5kdIUMhjYAx3uVYcGAa1Y2+p-!!~kB6ZrJ{Tvq<0Ny% zS}%{>^_su2W>wxAirW_yq`8L>($kgaAA}LLHRD`U6}MftlaQdW&?h_g$6_iuj;@Iu zE;DfmWwofS<;DRiJ^aU!xZb9%y=vO!s@oUB)!`+;ogicrwh89}v@1*0{^G3~eo3_B z;BBd|I_@2}wuB=9XbL-?%2DI7B8%7JG^NC4Db#g1q^n>%k@-}2?a7lNtx{TXl}K9K zOK1hp3?%u~t+AZ#o;En3(H{29b+IPv9W1Fn8)|uihcHOSNFZY!@sevfS3ptuC6yXn zcF_79b@Z!s1IqpJ+PQM+#NAYPT_Hj}Yer1Ji0M z=i*3h4Y@U>ty+?Yd}5M)h6X4Z3w=@Ao`#=7m8C$2fpDZ8lc$V!;-O|iyPz^-+g?yw zlpAmrBsNGSWFGk(4CbcpG0^F!EtWBDVTlo;+}v(ks*bk;pFmfKr9&B3*e5@wBlgdN zr%`V365srWxG9y&PSVMd`K%(#zLIR65sA^lbZ}&Vu*WRRCSfY_-wZ> zn3EPG>pCTQdtH5{L_DgOAEV%pV4dqHw@ZO+!(kOeo#?;~=TcgU53>y1-OrEP$&VljCnxs^{LzK#j8EATrMd@WU!EbbCsx{K#(${^r#!-y=m3Y{>isR zx?1BQ;AFiS2}0d$BO$F$Y=BQH0Y?B0vS_5)I;9MzTh}ht8e931K)~1O^m-PHO&&g-ogqH zLVSvEAfLCmsA<+_Ww|ANOVqaL(4IzLW*9jw={?-fhBA9|M0EAfqbxSYI(pwd;>;$T zZM8n?oJkyjRISE$#{+y-rx zY^7;A!R!DuJlA3645DaZz%?J;l_yZ_4l$g6N`BXMmKkYJjcAkSL0e}yJRfB7liGr= zfbEV&wlOM0nxhzsQo@4Pqm>+ICZcX}82RZboeh;V+QADe+02c`YGJj`W;2uIN-Il_ zD6N;U5Z};t7~Z2MrqgnemdqKgI#rKFsf_>MsvesxX=o0BOGd@kwc;l^q_WCohY zgq-s4YsA+TB)?wKJh4`<(4Zyt`v7ObPD=uSNpZg#t%!erdCUMNNpgup(6n&_=(Ow{`7s% zsUwuBxwc0_6OCs=FrQ4C7LpFzDswOIu@O$rwntI|f(TkQC}iU}#}$!vx5B8se1%;_ zurUR)RPU1C znGs(xXl>;_l9s%=DObO0+HH61>K_K%ku_%qZGNAaf!GM6Q4P(C8iT1h(*fuHoNB!7lBxOP;| z>NKI$u%8iXKv{4B$n`vNTfA#zn2Tm(5rer{Z6VQ(3G7q>-yGCe#ICBi*xI+VVd&G; zQtm5TOJ$s;q=AFyN_@j#0G~rhqSD(Q)R{rB^wp040Mlxj7F(!s4i=e&{6#dj(l!Uq z27PKi4@bvj_!2IGD)9hP+Kjb?kaM;+C)%Hr@nfy6Eh6Q6_(A&~+Rm`zB!`Jtn*m63 z7|7zaE}fNRz1Si3N9_0Q%}y!9t`Zwr#@a|br_hl?SlpGSGLnaAt}4^h@3yDy@*8{Q zj#O6BN!ELTjEvO9yQksJj$z7E#rX~DLQv5u#y85lVx(u<^g?yVCxoGuha7N(q$mTj zb>olpp;j(0Ee}MP!>oj*X>7iR{6rIwpsn?7`R!Vtk!^S(Bw21S-1$XA`Q~FB_*(R- ztPF($ds7WG{G~7#;kBW#;`BSLpyzDk*NU^;AqbkT7dF-J!5&@k4R#H( z*B)EnvXoo#)17RTgs%i+R!-wN+qNiD=@~zfrQR+woO%;oK2){krz*xa$2(MQ<~6=e zyge}wIUU!Vc}q!INeD(Y0)du_v${`uX**Xgz`NH)YIHeoT-k;a9&^E4o(rHB!5odZ zCZ!W*m2VJNSp`M7`mhRCf15q` z^u-*~Ubwc}%vm2Ub*#9{%Slp+%8FB{1B2;NaJJ~8b=a##sgWa7P@}3-g+a!=EDkov zmiQM43M9mzydU_d!!Z zrVEY$dt_}!Na@R6t6bP|V>2PQf*g?MgoU^{2@VtM>%9|iw@Q%X>1}R945+?b2|+<# z2}n+o6^wTGrcrN97NfKTh&SkR{&L-LRA$&A9s}|64tx2qM)Wk+7MP|>mewZRE8OH;ELNm7Xsnac!87TSZRPLsf zg+^^kSSrp6w44;`6dG5ctF45x?XqgyX_dr;meZGyiW0VzdDIVYtwVJ+@-$!n07O~j zL%2K}{6Hxh#&-ck^Xv^uTDpEbYn;2a&N7@#rcm3>1eX)7CuNOXWO6-fBdD!*dpwY- zw#m;#xDeEI!fCOoTM!OYf%WIm&~2r36;1YMm-|)bEwSTgzx=t9qPOEZrMNWk0ZR5N z2Vi`swHrR-%+8vHXzF`Zcm7&39*=RhGQ2pBQWTVr5eJ@lJ?iF;qh?*%7W=R5II~fy zQQUdPmg3qfz|SM`4*kYQHO2IWsvUx2?Jc?#JQ#?|xnhSxURd0+cQax$VOVv^g*g8IWKDd4!2k+|F*af-5hKkF`~gK&vr=(&!w z4fQ}gn$adk}`@8!-m`cmy+Met#9$@4 zB#ilRdmm6w^{J`%yK@L)E0PdjeJWZTTZH(DNXaS2GC<@F!1k`1Oog#-GLokw_&_1V zge64esaWA&n-9H5O)aGjHubHiSxE?5Uv0G^dD!4DW9dOny)?`QWtkCMLyJgpjb%z& z4u1&)bq;a$s=rs<7beOjvh=|fw)G^XwXL@rBpqrA1zMB7K;E;GqbRRE9~}PxYAX*< z*uF1Zp!CmAHD!)jX~!cn-iE`f1gS|#*aNo7+OkaxN#oMB{{ZbB0^=XzOS0v@!|`Rq zW!P^rwJ8cf*x;+p&tftwCvY74*Ny68mRTtDcH&9F4W-{Mk|_>kf}%IB>ECMe#3eZU z*O`@W$;}}d8guDQtmJL^Qgvr@T?p6TA8O^B*~d7Z@{Ne!+P#CQC{uXz5$I^PThC)g z6K&BI3uS|*-%3`UTW?w%ILx}*9ce?{R%LDqXy+Ii+cjy0b#b<|>&6D#*Q@F0s%KU_ z8nWaRhJgxD`cRUtYiTO*$=i+Uz58k6cUHoX_04YWwm7b-WJyvT;UrD^LQbN*V~S#3 z=}$C5oF_f14qTP$!c7KN=?6QGcCIP6BSQ)L2D8+YXLzWzuw(dvy&>>)HV#UPkK$G^ za0WYOr@v=1rI9Imkl{K&KsX(Tpab=;w&ELF&3M|yz`WDT{qKo@ zlpuNv_L7S1qjHvzWR9A)&p?FiH8~h7P*&grw&UMF(v>@;*pSOnU_F4+w4{NoyYp$CNRj1O+~RA9KxsiY}MF6ALeIOo!xDE1?A8HsI`5tkV8guO%s z6yqr)vBxJo(9+WaEW)|7&|XkNv^j-iAA}#FB#Jf1B}8FJVL>`lMv<|}_okNtE-)NZ zZ8ml)2?J7z_4Tact42-U!R95`VQE_3eWffE4Py!*=Wq0;U^5g>b#BB_a|10cN?U1H z0FK!n)hJ=1wu5SYL4S!m4TpWZQzitj3D5VS2ILHGN%x`0$axX(E^JwXN}6BGIG}{3 zCt8Wd{P2*FHXX_KqLB5*VplB^-D3$M0mON|j{9##n`#t-+wtAXz;DMZ^J5$1(uR)0 znjwhtlC&^Y`1T~7vPtyCZFx2P49@BV+Z}1Eq-R}0bHhs72?;7YW7~pBG+36~>+w|6 zjHSXbLbUmPN%sE$N)gy|ZeA_EN63L7jSgU_j2z_Vj{$In6zGR~#W${{Vt1{-Bzo zx89V)VdJS@3mGK#$;R|VOZC~;K4LB9)sS#QfayPD>+MIP(bIEK1QqH~PIIqw>S{S>odidt?UcqHcl_5&nizG)>HCa4Ngr?IyMj#IzyjkKp$vEn+GvQ@fx8}c-P z^rJ)83r)FA+HF27!P2GaWhY+brzJ{jbWd3Wo(t7YW3(ej>`sXy-V-zz{*C0QdFPC@rM98;Emh|JxZ z(dSN4kaL@ae;zH4l|R!^`eV4t)N4TdLdrQB>9d{Vt4OQ&h%eO!j`#BMM-rB zDp(2yY8>OSr*_vsQe70wkL)SZN>MgLQ(jsS17xftCnIC@JNwXgc2#F@Lb`?M;l&cB z(v_?YKqM3Vpb|L8>slNb78Jaawp%ZS0EL~m9DQre+oIi^ek*d2zZ@rCmPT@&!QYde z{RRaUIYbR6R;(zLy$m&Tf5OqNZ#>ePDuTSuzW>&`{JyrjC}4k3C96QgVZ zK|ARi=Clv?W$4iS7tkGHr0GgRcE&MP3yYC0i*cmB+ersxsNf+Uhi|CG9#R6Q(1qXb zO{fOhjIsgMl^t%WSOob`+XK>?W2kPi(#!9%r<)Hg)hMk&PI+<=2LKOBIWu}%LyWf~ zeYhPsO0p6Zka7oXeWZfHvv&DJ0 zp`aAEB;##GCw|0iK8CL|SX9^00HSitJ4knM`WsGa-igv z*(eE8R)-VE@TDN~HpkMU-kg?nIO`H7A(W{_rm&Q_NhM)9!Oq;Bw#I5^Yf@7kGT6D0 zK-8AjfW1h@3C9V@-xV3u5~Rgy8+OQzsX-1rveHOEQb_WpImjF5(!HihLe+E)DMCmeSfr>Dn^w7S7vdpvb`Hq$8@+Jbdw;CQ@8R8*K>%NgyQWAaY0owZGgnyFGIWwMA*D z2vXDn8wm*jEBs2|IKesgpb)O8i2KUAp@}h^bm?0_0HlmzBmw22E;HqY!GUWNC+K`4 zKqx410HSyH8*`2*vi0?roVWhRbxg-}INDS<3mM5O2aU+bxb&rW;44dE4O!pti?ZP* z<&fjr7nEfj;NyS+7*$BfL{FD)Sy*#AT2o<>fsKIA{!#YcyKiaM8jTs|08E4^FF-hK zl$?MG-z4K1?M*(^`0<;zMRk-Z!+K*GR3n`e=Tu}`WT#(L6a6F^{Gmt$!sAyJkB6MZBJ>?b>w2dpme9F#9`c#Y1 zB!;r<wbra70>T)b6VlQ-OLEDFwKTcnG*={ z4vi~l05p#_*ubM0wkkB|B{v~@p9l{v3IVW$j3)r@HzXbC_TWY`o0A}=Dnd(P#VgEG za85wmw?AqrxqZOt6-F-Cq2ot3-`XgYgslunQeQ(zC+HGRaq4#VqHRO0Zr1r-KJ*}% zt&o?QX&@zp?4S}ewn6PvW2e&@9xb`ThEz)N_*+OPH~{WPYGTPQY)g!{ZMO*SA=idk zQZQAGv!Ay#A3M-j7QW>zuAS=Gw^MXt?7(3$$AtmFml`SubrNz;Kmg-$n!2-aQcF!q zf#Dz^Eoku2RLYb#8O91m<8Nw{op^yU#OPrm=!6oMl8|>J&Y$KVr!@h7yveeq7U(e_ zQ;!`%_fS-jNF?NgfO2@_w|XgY1-VjS9ZhMEcbb$vbf0Stv*D^kZK2XZ<5HA110!M0 z1#P#xZMB;)>fhQCwKX0)veMU-i~`q?1rIPjyB>Sgn-inBpO>EVD9T2sT`I$h0|1@& zBB(bxH&jfTfXhvmkRB3Z!`V%za8kSw4&)Gdthqjd)fyC5Au!rLPC^_FtxX>Z3Ri%z z0N*)YoyHHgC=R2Oc6L&tTkX;=_V!fTq$##^w(1Tt*&$k$zJG*c1DsT4t?S27!AmyQ zqgzZ>mmPTuSb4>T94iUi8)tu|NUF`$w#&oP?|n?ll_9jGhGGGh^>BS~PwHsXUrP}* z-H^AK9Za<{EapX~-s+aO5~zYg##8{(0ZB>JNx{x)JEv9EQsz9&`7$HNg(pNgYfgd2 z4l;441n1P8cB{Mh_)*e}EXZ@;EbAdnwpI|XkIf(E?H+gqKhx6gYLHEa2oRy@}r&?hQz)G$^W!#hAv21-zLI zw9-zMV`U?JkapPdiW2b#)RNL+Jh-WBmy*K|C@IC0xd4NYTyQDl7OueMszf`G`jY1v zHrqY>_GEtuw#xi;2?T(ZYC2M_N*Px9!PtydzTat^Z*nSbcR!i|r3RdF(}bx+5{)2| z6R?KWB*&)ZCb`K4NL{ye}evdA#G|Pz4av@wtPnN>0O02pqa}MYkHBMh9+c`AUuD(XB{r8lGRw_vfG~IdoYZxa z4bsnYo^1>@COIAvY3~smfvF=pcfs2{;L~jMZMsY8Ri5ftWy~c>)2S^aZ>2rK=LeDR zML|kpt~7Hx?ECQ+wV^UlQXbMgrALJUw>#q%R@GctHFvaeQ$)uS8jXiz5BJ8CDJx4Z z6!GD>HykG*6pNHAv$GhNZdhtiY^13Tv3r-8iKM(9E@ihAEiW_L>rT>%~O&jA~@a05TJ~xk_ZQJr(vEwYAP)I>{&6;&7;n+ zzZWiLB_t>Pa~q`{t(Rnxe6f z7EUV8)CV1DxTuo3ua^{B!Es7Q;oOms4y=whrYtfp5M?S`_Tv@FGCUVsX_S$tI~8)I z0CT?mxHaha#UeB27f!z&D|7}>$jX%2<$L4;JC69L?kS$t!W?->E+M?Fmk;rIch8~Y zYAJjQ_1Wv>x7~>v)Y}poU=Zu!%YL}-#mNe5DIrK-2U2*dGyM%3Y&A7%Y8u~l_T)CT z%34t2M*%?KYAG5Hz~ejmT&7$dacP8saZie%U1kjIGpcP$ zISL6<)O6q|9rT^)q?!U~i?m;^GTvMVIvCrZL#{fh3`}Wp(#}Xi0NH14lt=^-+-_&A zoh3F)?xN{@THRssDl(h_N@RkqBLti%B`vkVa_W;-lS=?zeykU=1dw#GykT~lS1xY9t% zin2#xiheX0@rqLgT3b>{DM|p!yOZgk>s=l6LhG~TFYQh6&#W%;&s5tjZO?}RhaQyk zLl3Zke8WjPl-SOD>EE{X^2*VXo@?|=s+PsX4!yt7O;|82< zU?&?;Za=4DcFt>veObpf$&v>=kaI&*n~wD@8aOy43=Y*94=PZ=sm2J>J66RCV`IjL z+CVD61J;Vpl;bqhL(7n9(ZBHxtI&p#+X!lpC2B~d1t_bYX~zI4+L>$>L zskqSGg1|x$R6y@Z64cVN>pN$9_Y{6kPEWxO*>W;lN~5GD0|O-UM+*a(U#psb#h2sN$Xiu7XxW!J5OCxYSXp)sA zsE3XR2^1UakjhG$Z^R>f8XYru7#F#%AQ>$jr|=^r@7pzBUA}yo#!QQwr^HcFYzzPo zYLgc!0PB>}QgfVQjcP)k{rZ{+N$-wmxG5@-RNP&ml)Cv~#%VbyBP;$OJ*b#79F;XT z#JsXqzL0hwS{TS#dBO<7h5!`&M$*Uo(>hS_{vbCrwAVq?J$UGL9+assIHx?M6zaeO z+*MBOu9&mJlM+)ThZT~Blag^(ONIL8j}u8J#2621(3aosL57%cEjMrYy<<1KvbokL|BZi zu;-c0@K1VnlXVhq(zXdUJLrC0S})C%JxLE|7%P9gW#hn~x4k&0xrBYPq)3vyG9Or0rZEu?5{55x7Z zPJTMrN|hyODf|GOfDbh|?XJ}GJ{m@q5HfSkN8CDEG?9S4cpE1>QB8i9X4Biz32t&5 zPb4!dTFE4Mz#aQyoQz=Gt}Z(+VUV>eDo#cQPaAfoWOP!N>R)K7ryDp<4&Rj&&qluF zZxL^xyU94_mBQ5_o<(+DVqThiN(0CV=NYIjw~cJNDmwx*YI)^jP#=VPnutY((J`=O zl#no@I|?H0X?i;a__CB#k({fVdXrs~`ct%ub*(Oquc5T%RagD`49_Ky65ajnkR@!xQLf%xS9ZH@GDkG40syHt!%2VjJLAS!T zKuj2jM}Uph}uqb-$dkh0^cSC|E3x1gZl#!^YpY&iE9BDz$| zno80V6p}~yjY%;H^{EWEAHz|=1+;0@alX{VFNqdQgfyZv=|-{wytpbk;;iwl@-8T5 zY7}yv(2%SEDR{V#?q6kTrM(8=q$T%KmldIB83z?54m^fuxiOhV#;~l32W65!7CWBQ zW#TjJge|WDT27!Ddq~&5D)F5GHx>v>5kGgsd~~KR_Y#=Hkb*1Dpe8Cf>I6?ov~F* zoV&%kYNJMm+R{IZj(^g!sO_0PT12#HvXTjQC}jmo-2;8;O1E3tOV3P>9L^k0dCoW{ zg|^&fy2xf@psAs;Pj!3!sn>1}w;jhGM~*RMZbDZ;-oCxqj*J+C5u!iQAa8Ik)wSAZ0j$U}&48c8|9^%*>7Gq7TKu9^%SK5Vpq#9y6P#;kNc{n*3tvJSU1Z3kbh-U?= zYhD@=t>*zz+sKjhG?}*ub1m*$V<^K>K44lX4gfKv_NlpjDxlI(!cLMis2mYOLFsup zLlTv!DB!EmR;YFxZ%mcwtId`Ly)fi>4aoyju7v(9_RqCUyVPAa^SWPbD3gTc1A46v zl7NX1u0SOT7)T1iKK1KMeJqJ34>IX-uym7?y=igq1e@&%yWLv(sV+r;rAH{#*eJ+S z`wvm|rmc4OWwWnBV93;!tteKu)~tRU`|xPV4gBNmn2ZFaCkjbC`;U5-W&~KsQlt`e z`GP>(H7eNmNQ1gsX16HIi7iQ55}dc;qX3S5>OWI$rKc{ef})@Uji>O8{t$f)6@8iW zPOZqWB&p4b$RQ^KZRj>FcI&Hg+A`!YwK=J5g`8>~y{J7s47%BAdD!|A@*Yx@?vQ-g zC`sSSMG16%)Z6jdFw*O*)ZsbFPuJYgeMIHPD-6Z9+w~BfI^wv-Hak=dAKPVX(7kRs zk1<9ORbruC_Nmj$5z20?MW#w{{ST;jn>ByxVGj}(u*y$ zP7o30!1wo|ZfzMMiBY9K`n4z_q>na$GH`uPDT~}>+?-`@L`Q&=k`ws7shb_K#?>G( zA>hhRwzV&G9^iXYa#If;9!YCN%RQz~&!PgN>p*2*Fw}F%KGb6RvAd8=1Vc-W$Wp>y zX$J#I-%;(J)B>P2p~ob;>w^HODCaxWi;K0zjM7_dG?i{bM{lKCH_<2ZD%RDRFTZX~ z2G^taiv>El?~~0#KW%9;*CNisQtPCnLh>3`HyIwHr(wBnN!HMD)e-zX^IQvn+=f|w zm9nKP0Vn#=tx^}wkv9m}$&E&C5sTuof=aRgKj9yKCMt&>SgGky`SS zpP=L3qpsIkv2F>QaDAmjcm~8kfC@ny9QssM-s>{ZhZ>UVFb-Ql-9S_X%k7foi4se$ zx`d2?qB4*-7@@YbGF7%+=HKGPf90Bt;uKN?j-^A(>ACi$T`m)%MO51qS0;SQ0+r&s z`@gCG0JUeAA#Cc1iD9SZ#f;hW1T2D~{HMMuwIoiky7jq>thS&?1R(+9cRjtUe&MBB zHr|=I&zm7hmda4#8&8O>I0W_y&m$C+N}F?@^|_ZCEAc$Z5y(cLbHP61unAXgyXhAp zT&?pJAv(iu1v*u&8QUP#>;C`}di@A)Q)@&xs&Hybhwn4ok5N=$*N^VmjC&ONq z1N^|$bO>EV@a+t#cP|~4CqjWonS1h2uJrbhotBQT)76$mTr945&mjs4XsD#2c*b+= zM}9%C&a0x`jc}67>C)qGEh|XdN`j6?KDGKiW$FtYxc>k)&X}lH6r`Z;Ro`y)@E^w* z@gaOs>IR?@^6QkR5rKi`+~8LauHH2M27{|9tCBe1Gz;(JB<8wTf%QJLE5}*;*O1d? z5^`fMKdoEs=})zdhS@YPhM=qfcRZSXuEQJsDmf!TaqvvRR*{^Fm1X?nd}5%OBmqYu z>;OqQ_ol}Nwo(zIWw|4a?N5gQ?@;YLLBXQK!%sP@ts?87)jl!K1u)#82?rczgj?Cj z-v?~d0q~QJTl!F)Bu05SlFE^*kN*HQH{%kWhQ@&sPIo(` - - -You can view the logs for Test execution & the Multi-model-server in the /tmp dir. +`./regression_tests.sh ` +### Running the test manually local environment. ``` -cat /tmp/test_exec.log -cat /tmp/mms.log +git clone https://github.com/awslabs/multi-model-server.git +cd multi-model-server/test ``` +To execute tests on master run: + +`./regression_tests.sh ` + +To execute tests on different run: + +`./regression_tests.sh ` + +You can view the logs for Test execution & the Multi-model-server in the /tmp/MMS_regression folder. + ### Adding tests @@ -49,4 +57,4 @@ Specifically to test for inference against a new model ![POSTMAN UI](screenshot/postman.png) Afterwards, export the collection as a v2.1 collection and replace the existing exported collection. -To add a new suite of tests, add a new collection to /postman and update regression_tests.sh to run the new collection and buldsepc.yml to keep track of the report. +To add a new suite of tests, add a new collection to /postman and update regression_tests.sh to run the new collection and buldsepc.yml to keep track of the report. \ No newline at end of file diff --git a/test/postman/environment.json b/tests/api/postman/environment.json similarity index 100% rename from test/postman/environment.json rename to tests/api/postman/environment.json diff --git a/test/postman/https_test_collection.json b/tests/api/postman/https_test_collection.json similarity index 99% rename from test/postman/https_test_collection.json rename to tests/api/postman/https_test_collection.json index 175f1bc42..5e21dc48d 100644 --- a/test/postman/https_test_collection.json +++ b/tests/api/postman/https_test_collection.json @@ -273,7 +273,7 @@ "body": { "mode": "file", "file": { - "src": "../examples/image_classifier/kitten.jpg" + "src": "resources/kitten.jpg" }, "options": { "raw": { diff --git a/test/postman/inference_api_test_collection.json b/tests/api/postman/inference_api_test_collection.json similarity index 100% rename from test/postman/inference_api_test_collection.json rename to tests/api/postman/inference_api_test_collection.json diff --git a/test/postman/inference_data.json b/tests/api/postman/inference_data.json similarity index 95% rename from test/postman/inference_data.json rename to tests/api/postman/inference_data.json index b4eb7fb8a..9cfd15ef5 100644 --- a/test/postman/inference_data.json +++ b/tests/api/postman/inference_data.json @@ -4,7 +4,7 @@ "model_name":"alexnet", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -36,7 +36,7 @@ "model_name":"caffenet", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -68,7 +68,7 @@ "model_name":"inception_v1", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -100,7 +100,7 @@ "model_name":"inception-bn", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -132,7 +132,7 @@ "model_name":"mobilenet", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -164,7 +164,7 @@ "model_name":"nin", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -196,7 +196,7 @@ "model_name":"resnet-152", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -228,7 +228,7 @@ "model_name":"resnet-18", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -260,7 +260,7 @@ "model_name":"resnext101", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -292,7 +292,7 @@ "model_name":"resnet18-v1", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -324,7 +324,7 @@ "model_name":"resnet34-v1", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -356,7 +356,7 @@ "model_name":"resnet50-v1", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -388,7 +388,7 @@ "model_name":"resnet101-v1", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -420,7 +420,7 @@ "model_name":"resnet152-v1", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -452,7 +452,7 @@ "model_name":"resnet18-v2", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -484,7 +484,7 @@ "model_name":"resnet34-v2", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -516,7 +516,7 @@ "model_name":"resnet50-v2", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -548,7 +548,7 @@ "model_name":"resnet101-v2", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -580,7 +580,7 @@ "model_name":"resnet152-v2", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -612,7 +612,7 @@ "model_name":"shufflenet", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -644,7 +644,7 @@ "model_name":"onnx-squeezenet", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -676,7 +676,7 @@ "model_name":"squeezenet_v1.1", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -708,7 +708,7 @@ "model_name":"vgg16", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -740,7 +740,7 @@ "model_name":"onnx-vgg16", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -772,7 +772,7 @@ "model_name":"onnx-vgg16_bn", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -804,7 +804,7 @@ "model_name":"vgg19", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -836,7 +836,7 @@ "model_name":"onnx-vgg19", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ @@ -868,7 +868,7 @@ "model_name":"onnx-vgg19_bn", "worker":1, "synchronous":"true", - "file":"test/resources/kitten.jpg", + "file":"resources/kitten.jpg", "content-type":"application/json", "validator":"image_classification", "expected":[ diff --git a/test/postman/management_api_test_collection.json b/tests/api/postman/management_api_test_collection.json similarity index 100% rename from test/postman/management_api_test_collection.json rename to tests/api/postman/management_api_test_collection.json diff --git a/tests/api/regression_tests.sh b/tests/api/regression_tests.sh new file mode 100644 index 000000000..b9f15383d --- /dev/null +++ b/tests/api/regression_tests.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -x +set -e + +MMS_REPO="https://github.com/awslabs/multi-model-server.git" +BRANCH=${1:-master} +ROOT_DIR="/workspace/" +CODEBUILD_WD=$(pwd) +MODEL_STORE=$ROOT_DIR"/model_store" +TEST_EXECUTION_LOG_FILE="/tmp/test_exec.log" +ARTIFACTS_DIR="tests/api/artifacts" +OUTPUT_DIR=/tmp/MMS_regression + +install_mms_from_source() { + echo "Cloning & Building Multi Model Server Repo from " $1 + sudo apt-get -y install nodejs-dev node-gyp libssl1.0-dev + sudo apt-get -y install npm + sudo npm install -g n + sudo n latest + export PATH="$PATH" + sudo npm install -g newman newman-reporter-html + pip install mxnet-mkl + # Clone & Build MMS + echo "Installing MMS from source" + git clone -b $2 $1 + cd multi-model-server + pip install . + echo "MMS Branch : " "$(git rev-parse --abbrev-ref HEAD)" >> $3 + echo "MMS Branch Commit Id : " "$(git rev-parse HEAD)" >> $3 + echo "Build date : " "$(date)" >> $3 + echo "MMS Succesfully installed" +} + +sudo rm -rf $ROOT_DIR $OUTPUT_DIR && sudo mkdir $ROOT_DIR +sudo chown -R $USER:$USER $ROOT_DIR +cd $ROOT_DIR +mkdir $MODEL_STORE + +sudo rm -f $TEST_EXECUTION_LOG_FILE + +echo "** Execuing MMS Regression Test Suite executon for " $MMS_REPO " **" + +install_mms_from_source $MMS_REPO $BRANCH $TEST_EXECUTION_LOG_FILE +ci/scripts/linux_test_api.sh ALL >> $TEST_EXECUTION_LOG_FILE +mv $TEST_EXECUTION_LOG_FILE $ARTIFACTS_DIR +mv $ARTIFACTS_DIR $OUTPUT_DIR +echo "** Tests Complete ** " +exit 0 diff --git a/test/resources/certs.pem b/tests/api/resources/certs.pem similarity index 100% rename from test/resources/certs.pem rename to tests/api/resources/certs.pem diff --git a/test/resources/config.properties b/tests/api/resources/config.properties similarity index 50% rename from test/resources/config.properties rename to tests/api/resources/config.properties index ce6e2f3b9..ed79a84c2 100644 --- a/test/resources/config.properties +++ b/tests/api/resources/config.properties @@ -1,4 +1,4 @@ inference_address=https://127.0.0.1:8443 management_address=https://127.0.0.1:8444 -private_key_file=test/resources/key.pem -certificate_file=test/resources/certs.pem +private_key_file=resources/key.pem +certificate_file=resources/certs.pem diff --git a/test/resources/key.pem b/tests/api/resources/key.pem similarity index 100% rename from test/resources/key.pem rename to tests/api/resources/key.pem diff --git a/test/screenshot/postman.png b/tests/api/screenshot/postman.png similarity index 100% rename from test/screenshot/postman.png rename to tests/api/screenshot/postman.png From 002091a80531497beda4fd97f6da5ad6c7e3eca1 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Fri, 7 Aug 2020 16:22:10 +0530 Subject: [PATCH 23/42] added python shebang --- tests/performance/run_performance_suite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/performance/run_performance_suite.py b/tests/performance/run_performance_suite.py index 4e0d05bfb..aa4d391e6 100755 --- a/tests/performance/run_performance_suite.py +++ b/tests/performance/run_performance_suite.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. # Licensed under the Apache License, Version 2.0 (the "License"). # You may not use this file except in compliance with the License. From 3534eb11a2e789b129da6b792a45aace4c072633 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Fri, 7 Aug 2020 16:32:43 +0530 Subject: [PATCH 24/42] checked test kitten image --- tests/api/resources/kitten.jpg | Bin 0 -> 110969 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/api/resources/kitten.jpg diff --git a/tests/api/resources/kitten.jpg b/tests/api/resources/kitten.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ffcd2be2c674fddc4886117c12f3b29e4367aba8 GIT binary patch literal 110969 zcmb5VcUY4_vo{=iuY!VvW@w=(-9YFiK#%|dsftwTCG;X4M37EEg@j%Oq(}fokRnwD z450`}7YH4s34$*^=R4e-{AEaF`wpKypRmB;o+T z-))jgBNuOfZ$B4rA3mAek^qQ?o&h;I;EG6+fh1M{GA=uj~Ml`T+!8{QZ3t@7(dU3-8h z{zoFI%UEPz8L7bZSc` zmSbApr)s;zbVXyiE1kDM91J%TuHXL{3(26dP%uz`P`Gd{IlLP2%DGe6u!Y^SZJG^T zO#?M;)PjBaIdLeSw|#OI>0G(pkwX`0+DM{cCEl_3F9Kd z2_jFPyfMp=0x4a%L}yYJ%E^6?&dHyena;(dM%XkHyFgLk8r$86Mx6YRqcW)ieM z=|3!Uv}IMUYPWzEoSWmEaT;cNSYs$=S5LUzr}7%c9BTBQ^g5$_AE7X4KG(yqSKQVp zjJmd6)oKm@fyu7dKMWwVt@|yFJoWrBTCEQNDUEGbo5iP2Da^&h$oPkm5g&5yZraSA z9M_R-8k&mP@2>}*jYZiy;*s*(4d*6_^M0H6;=P86GaPSDZGY(-woW0Vx>oiPe1nKJ z4|63I)rkT=zuur8fU&ecKe&P{g!`gO&m*M)QO0F^v0gaU<#BTP!CXU~3bRZ8yK2Z`AD5xc45F}G2BIHMvH>Z_#CNLhhz$7ziXAQs#(o|Xo!Tac-6th3m z?#2*P6UsE4hlJhbGG-KTh8&kaQ|MFOwvd;qfXsw0ApQbQDs)HORxzO_MKvBIn}{Ol z$(!irS!7mJK&T0S7K5X0xXAjcfa8Ln@z*;k&FuWZH7Z$+Hlali5(9%C{}|E~^Tj)g ze$C3S-9-0-YQ6_IMu=U!J+6A`tJd|^%p!=K;WZ3;I_03WRN=(-+*%sP=nBu245}3R zS*7ywr0k8InYuu<^}T>w>b?m9OvCT1tTNS9vo-&8i3(hz_ktbvF9YuCdO1HbG6t#M z8%+h7Q|7e9B=SGVh1H~_h345Scz8r*sDF6!=~q6%?P<_g#w7&x%(jVdqi^7|wYLQ* zT@LQbgiNe8Eq5>+!L+H{toNQOE1tIQD_9KbJm!{*UgTRXa{QeWRAS~gWJbEMVzj{l zH1_fwCDPw z1!TsP)XQ8K;6`{F8_w1%_zfeZ)R#G15NgPh@g}@tZFtW>PklG38WCf zdcv^}EOr2uEeCoj8v}jHsE2y9L8aI7=dF-IN)y#n;y$FxVB4_p>O_EdJ4D%~l`8BcNZ)d(zc3q; z%$_vxd@Q8`=fh-&6K9-BgUbr03OINY)k5(ky@6OjfXcj^`An`SEeA(mWEtcxq=v*5 zune%raM9ygrACPGJ{NMM`Oro6K0;|J_wh*?GwQO*GUqAv2(O#fwRRD{Y%egOCABX~ zTM$I%jgtUVHIxZ{sIWU6s9>EsntuB&y*qaR`Z(9b0-C&h_U%2TOe&Zb76e7k++4q= z!(E+=&^s(vQa+vgz_N{VqvZr~l(JyY{j}JG)7(6-EE$6o$4n_^Wnqb#7#{FX>WP z`k(S6tx$J(t@0v;em0s8%mbEXn?7b_li{?B{v7xrf)}JC-2X^>P>t)Z7tmrt#)^;ov=q#(aKYze(8Wm>V?nma4bs^S+2rhGEy0FJl zHHdOeHe_Hohd?`ET|`(!cCH-e{L*?7iP=sI_2K)Kw4pXbQ%ZKNLN*Kg7a+a-AbB{DKD@7HwZ%4DUw8boC?v{X#YQPoYnqEYN* zt9^By&higmQgfFYl9Ec!0M}|%s4TkZ;qTcl_E=SI(w*ZwX?*Ae^^aR`XrE<5X4u3= zM@c&CM$y_HBRkhKV$sRt(!ZT(X>Y>(A?N&3hQ_yiG^VmC~xuuI5>hMgu0H@r=)O@{ELH1B6rE zoQ|8;+)_f?FSI#cJKR#%4BZ2bIp|xd|G97#p!YQA1Vx=(D7(*kh*d$LlYzshRRgRV zlDeEqp74HKSL?uSJ*8j9IiJ#-wjgDWEJw@jf}k0;RhiBzy^M6}Zz-FVG zi^@I?ELEf%N7AdUlENj<(?u*Om6ZDk^^;u2 zJRHCzBQ-KilB1aIB`@@*vFwn3;QCE4rE(^5f)*vB{X`0 zF|e>yvdbHql{`u3knCrevDwQP1s${eKo2g2Mg(-v&Cv2OLw9#|jj14Oc-$+J8lce* z#usXw3J4;N4G|rl&o=hp+njGG&&dqAa9zrGf3NKNo?1R8sMk-I?JVzTd)FYnXa+Q$ zu*FsMw!T+bPuOoR2Nv|EOyGDfcU?t%({?*oAv%gGl$~t;S234>52WTRr!imm{_MGN zO{=Ix8PB1$7}Oz2oseky9*RF2HF&q16e2c{6XTRjU*vj>)&ImjOb}}5bMKI+Qu5R7 ztDoltd7!4>JJS0Ze~#olIV%6WmhR>uuYZ@1p>uf-!%CtcRGjx@JF6YS3yQ_*&Bs3K z5^!AhxHKJ_qsq#qA0SY!=9$d+ipbK@TK5YJ9~I6;a+c4AfoIwoaha;aN%tE)ldOBS z#%|YrEFbR<+tx$Ay`RLAEWV5RwQa!NKY{fNb;P!dR&gvEL5}L&t^y#&K)D}EN#nZx zlp9p|XPyd6c>w2JiKnSL9(Wmx`nexpgJo3+4yeI{rTlw{sVaF=2#7SZd?-~})WBGN zS)uevJyoopKj_wTu4T@{y6fyjrC$EUU1%$h}JhN)K(jT`}m65=;!JvoD&32)G1R)0?ppk7RJ-l7CkDKuP-VMsviIP zWi7OPj+$TRj35pBrd9A*pDn#<22J zUX#Ci)8k=*Qi8%3q4S2($+;}vu$(%Fk~rjiMmEyKwnj>fIE;;Bh5I$M~O;h7Dwzq z(GoW^vBH^RLOi#ash*f@XEXqB+L4))$!d_e>KQ5Q$jndPdL|N1HM-+UI6o;3(fCj@ z6vS%?52*)R7;Wm2FN|%sv`(kvp(KaJqm)yZ+49}Pf*eyx!FSUsr5>tLdF&dJ$Ihr) zVWQH9qR7(lnLd#ITBX(bi98amG&h=Ld7TcFLG`-G#BxTfVM9oqK$(;gO;mSm?^W0sCrJ$m!nUr+7C%-2HpX~MmKF^Q^yroQ0ppanQz(ec$%*5u{~Qw%9JJ> z8x%71%9o=V2m20xg^&TmtHx+m;g={;ww{A)o4smUZTt@VNe>b`*CInNq!alEGoyx& z$mGq`Iak}{7o_#fRgNBEQB#Yi9LOo)_Ha%O=9qUz?roy)Qz>i1E%752&|Ei`)i5 z=jioTRt)O*Bzdt7*ywZh%!P<>cDk?YBpd8|5m0GE7N`Cq&r)fNa{Bc z1ICw(qG!*URXf3-ra}}W;Yxs|FT}uXR0-SAHk`NHUH5(h)%i#KOTUU=e|WRKNcSIm zwOO6wbw*9>Cst26zq)w&6Gu+`zY%96duls}g+kl>qHIY;0y=%X3%9L6&5W0vhtsj4 z-8z3{T3SMtEIzoD!V);%8_SJ1Whzzg;mU;nsD(7_Nb3|;UH7PVI+XomDqgvvuFJ^( zX>Fe`Mow5MdBt?P2Dsr7yFuv&`|9*^Iq~Ty?>tX^h@FVlcugvjl{Timb3Qi1QpZWE z{D*YIoidD8j6-wzL7>LlmeJ)jqS`z6%x@?QB$Q!Q+xe z?At@LH$viL+PXX%)ZA4Q(-xh8rCL96xZBRp`mRu=nv8|_(|N{x3{k4ReS{f#&%Wk? zVVKlatmU$pe}n2?Y-$3lFH#rEvLTY*{WdOnc?M_L;vj67oTMTb?QX=w*>${avM}`l z-iCT8?3kOG;DtH>T;;YqJ!)qx z+fFho)axoI<+%-@VJ9eb%#QXxtkN@snz^WGrWawdPKxvV=5td$wXT*B{J?Z}Zcj|= z3Nj$0`6}eaB_mDS-}V}mSnNM?)AR9_`i-P*su(SJ1c!adpGC%n^>$68Ur0(uS~;NB z%wWpeD6MoFvkks5{CK<~-DKs);nKUT*+{H(P045Mul_O#F%Mn^Vczb7N40lYYK)J7 z=IR2=N@pheD63~k|8sM?1die?vg=E2`?;K6Bm-pa zZW1McLykt$pO^g&6A@{^7bT)wIZG;PkTV?aMo*fae4ui;VJX~9_+>!vOm0k4a}x%* zLq>3r#eeS$th&L~H`l&y&I;3T)v5+PbZ%IRv9?q~A>Wihm*Pzva@|V8pY?MJ3me|b zGjY6ji4L8fdu2tr*6vHq+AwqWA(@HI zq2uW6QR~aOnG$Vm0GWc>yj9(U+ovOWs-bZ&2sA!!ny&^bX%+~Ce$fl}96t#TirF2-XvMz(tBQw$8{@6JgB8L* z{0ijEooD6P?W`w-V3x4x3caDkxgtk#c0WVCOd8OwC~NHbxw&|f$hv0I@b{tzARol( zE&}3*{ngG`tF=Inm*Ib*I-H&|K*ftyEFV

J66l>z7T20v&BrMUK^shcTlD$)2!& z5`D6QGPrTQv=zN=2yoX7`b0JZ<7eEw>PtzYQ0@7e|0=C)kk-{g3ZpRi)C^3!!<8%g z!&_L7FU-EiMfl4VX6cSGvAF_JKmka$3~V94p7gGvDe+H2qRvIIESKyy_tU1ewy`ntT`-A zm_UBg*vY8U?MMe_7Uy?s^ax1=NK1B&p>4w-mJ22v;h` zu6QF1;AMlTUPtg=DTq5Q4ilqNvW=7qt*aJlMz>MUgUQ|C^x8!=q3pL5qt;qSa6ad= zO~+w8y<)J~iAeegu7H(*wqRl~a#k}@5H;HBI|}R4%EV;+88jwmJneP3u)fGM!qqW~9qqvtrMC&5c@Vt-e><%2E{hc8W2j9nXpE zSeH}W-kCKTdV}0yR39>}Kj3j%KR=+&h_lT)O_fwROc@d*hRVb_S5>QM+7Fd`d0WkU zYwk~22hfs(={SaTx6~nW4$)&#akaWUu{^j#??-=lh&%1B9Tmi?Z=fS_4Vsrt>)O59 z(LXb<{S*b=v_Zhs##=bP*@r*%*c99$tFP!umy=wW6}C$c1}V&X+N4!1zOXU8d#7Sy zTY_c3cEAD5At+4#?VD99TXwoMbh4~9#SLl7Sa{=TE3&XMF^Dhvyb&h<=CsQ!QKXo7 z)18YnNKVe3!*{@?Mh=0pG? z`$qW`Q?+c*<3ErSX;!QKZb{JbFFErH3D zj84IjTP*^;F8_m-4kk8NGcMkh@6x~Jv1Z5Z5QTXzljP=KpH2 zs+M92lq;2S`>B4&>(!q4eW~aEBGP3_XWh&_3}83oV_ovBHkN(0P8GDi2Ap?$kAmne7&I#_)dNHkkzC{`1%QPk&O1rsd zUbhYT{U*5;6!oAJ38VB$mkvCFoSE$Dd6qQtqX7PHDcW&uOp6fXlPXNO43~`fl_`%8p2gNG+Bi8I#aoU!$;gt*+9#YT8Lt%Mj$frL@qyQWp1$eUI&FU ze1rAjve}byr0*pzW%4Rg-!i*vJm2ifV=Dd0aPS=|A}iy!sbiv(CI2I=Hs=uX-PPz$ zlFdIxZX*5aE14im+3%Ez0I_ zg_#CFg@mxU5*9Wi5TY>-3vrm)vKN*JUX+L;;71CMeL>KL)bn(n~$&c=3jD za3-?Ms&--&a5M0zm(DpMlZ5kVx^`mSa_lGDkSwh-?F@$Abk^E-=GOCA05hX#pa3%p zrC8^9ivn)qmU@PNx)%1^kQ#kxY^DyTVh)(y_=At=1RqduHofQ-E?SZ1P>V4A(w84*aYEy*c3EQAqM&)!X&O#eqbu6~>NBPS*(f#gxmnZonDAlT>-6WKU(Vsw zpi^(1Q-Y}}%XdLX4<=j6FeZ3bGdK_Er}*3P@eI(R$3$O9W8~BtiH-}avO?vrx$W_7 zUkHC|-A`Jvve^`S+&tYujkF4Sy2850t0hR+Sv=)jdh64}?q0c<)G*EY_ePG4zP0Iq z6S8g1Pu?ZGF+Ya)etxd7N-wN`67WW|q(LC++VKoN=9NRkgE#kv8=fdCYjT3c>9Tt& zM@w7ghcd7N1wTn1S$&*166H7Ewi2vq4~S@ld|7??Jm9T-nf&Z?daOPf2GUI^tp@%! zV_#=tFS2<_e3bO?4#$s2_X5z>D|eC$%Z!hhcINds;Q=J*b4MR-2boaks@af1Whr~& zfH#1-vPG*=u9`2oYT^1vdp2t5QrY`QRiV{ok1LK(4AhzovP+e3&Py@gg2}{0*OX?; zpxM^GX)zSf)^`{;@rd23Wy|}+SV&YU_N!3Lj*|i39F6C$nBa zjtTIE#>iX-9NsXR2@P6$ged;3@Rk)mnoyfL>%o+jp6T;_1WRMqrK>;=EZt7}eXz|# z#AjAm*Di{jKz64%R#cmv9p+$quw6B-RPfQ!=O2hZtEm5h7ZE!VYLH1uj=%4SM&$i2F09$2^!%e zl&?%SzAd>eqtriRrXM%I0*iWEOrSA96CpoD^X9$E-;7<>z;Ke{+&wqN?e!v2b+g}7 zqg!)1wd(NQ=8MjU(<+Om8Gi0o?=gN99wG>UgAvz9*JpOa0y)-y^A3#QCbFSYc_T7{=A+G4@;e?D5J zxRJ*n9^@=^$3}-?_Y4pN&n1q(p!;ayjnjwQF+S>lHhA`CC$=JQkA30}HIy1l=cT+{B?K5AHkwM0C(Fxu;Uvm{ZT)>Dr_!t&ny4`v-_d;l%`#h$k z(YI9UeOilLnMf0;Z%E+f;v~ApgDKQ~MSZp+ZPU!stbnAcNN2dcPsH_`MNG`}^=$Fv zETYL^qMl2^O)+1iFHbOTW~3%Za(5iw-r!=Y;jMyHE}(1omIm;#^Y5x@1y#PliW{+n zqHG7kzD=Nql3A>l?}DHgn0hOyXhof)FRi}notWEA73*y0;kx^wNKkhmiZCN)gZ)q` z%$toGlJJmof;l&gJ^I*|6efEdpQ z(|O%PXVZWqVjC0u`+Vz2q=4*@uNp9GovPL=lH@O-5%Z>_rD2lW>!3DVX&q%iQQ5Q= zSDG#xL(&CZ8x5PpPYkrv?B&c!Tu2h={NB!Z)*o0~<)g}v87iZz4QWBPdh^1eI@vlU zZB&IT?+nj4cGV&s5Lpdz_fES)GDIyQZX?bpQ7cmglO5YKhyn=+dLg!MlSAyl64MEY zM51nBpp_o1s8Ff&WN1LlQeigh-L9!#0ecPo3(uB_=D;7EdH|iMG5T6ixFHprOeR9v z5Br+uJAs#>toh^AH}2N=v^fgbXAEFXnNi$#7SrV#k8u|r8ae#jR~O#SpQbpocUmZ(m+J5 zps3yn=iyuxxKB}+jxD#@@-G0_uVLkFrtkcTF^RY^^=xDLL4NXg8u)cgg-B=rOO|v6 zNFCaOLXQH3z05wL(ucPkebf$6)&9Z8_Ci8GyvZO0$>G_jlN8UF48edo`V65!&qr** ztj;Qn`iUkkBwk8&qOAG7k5&}6b&yFezm(ljnI5jxNPA^rq?a{D$KKD#j^6fn^!$cGmB1SQBA0;IBVtO@mRTyK&}sj z1ejVmIoSId3SLz!DLuCMC>R;PTUmiIdQY0B3vWqTqAs2ohoT zO8UVZAHR=3HS0XqaznX zjj3E$t^Kqitc?nKUg1L2E5Q^+Yh)wxbbPa&;jpXNs;mDEO3l1*l^PT|YMfFcyf@k~ zz5J_WT5l;yYek?!_gJ~X2BA^)7r>y?BDd0KWX`6SdUus8>*{Pea{9xJ1>33G;fdT! zac3&X_NJkZ3;64=<5Ki-37>#_e<=+d>ra@}#QKh!;TvA<3KKw>?Tp0^M? zPpZk4Apnx3VvYK8dq%a2PsV-V3AKK%PhO5r z9t*!O?=kZ9#A5FUs2y1$G;T0_Zi%+Q_dK^e!D2z?|1ifgq>&*BjtT=&$IwA9Ki_P zklm8QjZtK4-!b?&Y@sD9Qdy7xSkOaGMcuVx>mUGNcybKx(D^n@mzv_%^@R94r`hX= z_E^0aD=(rPYyH}KlN()p#%dTosPes7FhKkDpCwD}n7y--W+dAut=iUgVEV6I=IBH7 zxrTj0TGe6sBbzTqw6xiZaDtmR?log za$=*l(hJ!Od1!{x-iU-82TO`G^$@*cTUryzYp0&2#}~~z=bR;-JLn;fglS~&LgtRi zM*jksz0H1$UE}}$hUJNY{tM6aCqjfbtecK;6~dnCiZX+jqN8UY%W4bQ-)q3_)k6h` zL`@dUoJgER1)?!nHB6NHRJ_3C$N|VkLfXvw)NU+rW%+`L?nmLCrmj%HR1osUh9Exf zUW+rCCFRw*&o}gy%HLF_{slZ82r0WOm|k}|!p964R;vC%$HkmYk{TMEe+CxKZ9O*I zX$LP#fUb-ct|Ev+T0I-kfz#Ykt( z`#?|X3~!uaii`*9b`N|%f9t>1>b~{Zuxc}|fk(!%wJDx4=r3S`5Jh;Ih5t0LQ-5Bj zqFuo`-gTkv#ymw_^nLp$(^7*sZsNKr<^3{I$jLzl@-Kj05&Y2Z{BEU$FXyutp2QGM zkdcWqObxhjqUyHOvssc&1ilIv>#G@DQ48z}Ot^)Fl|Q5i+d?())`DM3wEH|9Y~sJy zifcAvq=lw@REoHNSFGY^ME5Jy`_F`z?)o;8ikU6!b&%cDOw3=vFgWi?zynY{8Zv`K zk38=fGV8poQuwpkFsO`~wcgV?Tn0DK{?x<_fGt&66N;beY+ks|tguPMS5Rd)>aUyC;{(Ln?hWj7HDc=ZuN_-mQU+wx-f2z_I}bhU61tdcC1XqF8H zwp1IPWUkSk$>8L}tlAtj5RA``#Mr1`yV5ZRt~iaS8#9iEW5vXyyD!<2+^~W&4I8rF{lS^Bb8iFU>0M7|U!-|5Ekl z0qaw1WJYYY?!W->5;Mwr`|S%+(O!tZ+9E=ff=3hCn`UD zgbkQyZZK=tuXSVEE0nNQ*49bW2|E0$QlPf9R5?6X07f*G#pXnD_T$Ku*WfuN)uMWX zhqV#~31j25nne?I5$1`Oivk%JcPn2hD<(=!%uvG#bA|VZ5KOA@QURF$O3{qn0BT|; zfE;9GED$!Hzp?^x6LXHFyBYNJ;RsRJcTz=6y*S3SG~G0#leSMR^>BL$ZoS9fx`2#A zq$6NcmrpAou!mWwj%UBO-&dXGq!5;puFY(OzqX&s7IZUBz)-t#3W|Za9#<5sS2w-W z^Oe29YQm6_nJHUkIe50^o--G6gs@#^)YN z9lWT?3;7ctFNXFPj`#AHEbYqiW;h8gK)Qn5;frpjt;w;6oZinTc&{&r8qBhu%eMf; z>(ig)j_ivN$CCrOQZ_#CjV#vb{h?{PO9)!&$4;cyI|#a^vwo4(LmqBiiUZixglS0l zR&u|nNc^$Aao%>U(C#5Y(OaY79b7qty7_EkX_{P;n(J->b9Ay$tF$W1pNqeMuJ6P> zkq*OO$Fc>9yd0rcdcG4sjtKlTI#2W5l*~;5eeo3@g2Fy=Q3Xf?hj8e%()ZJ+8`ur~ zw#GS8_kL?pXAPZ2jXaB?7OpsPN8X0=ilSSe^Lu{`{M`nQ{Kg_JW! zAQhoTrY9_k(1Oc}P#XtNwO8IbQ@@IGUH~NsYdK?OB#YSkX@ZUqBO(8X3Q}>~oa3!& z!RTFmC4+GOps4d9-Te~%H2&rXeY1kKx@Nx8u&jHC!CKc#`gMjO8ILaLlm1s)8PsBKHh+^7ey)}GC!d06-kUGNBF)fQC^g!SSzVtvD znPpsnZ?>V)FNwGu3H&TOvz+{-XD~xvEPd8<3+)Q#7YHJErm_wC`XpuFZ4yPiYu8V4b!IjVv!YD+v#z1E@apxts32jk4fim5j!oB=(FZO?ou!UZY}d

hPAkbbTr(^R-tMK2 zKN0>_fp0py#L1`FRr&*5;-DDr4LLD|sj;tZQ3zKN@*cwl?-923bSUpcacn1JL{nYv zZtft-cac}hZN}n*gHxCHDFv?0CxyQfV9AWc=f(?@7if+U5rSJOQNp>2UTVfCY}SbF zqaxj|>0{M`iOe$Rt9u3RW`~8f`k|HzD zF^%opSyX9+jYUn z&%KiTR$^Vi!830A>~toy)LJOz!?S86`uLBg2Ua4rSk58ug@-Kly(2<+7h z$4T(;B=Ewob?78uEe=6);xm2g<$@*#FTG96J_{Tve(iRE2dF!0pqX(Mqorx|Jm0x? zn;B>Y0=^EuFo+Rx8l(n?tgXnM=cN6S#$h3#tgPlvKPyb0qr#omugkC3hSZIPlx`Qm zvZDfran~yU0tQVc^qq>L(4W@B9>`>is0P||d53}ZJl!nJr>ead>u!vlnM1|`e*Xms zx4$4%27eqmmG(+M4H<{0|CsP_*Ls={{hFIIzEq{!#hDC7*rt6dNA6FgHCE_&kKXSgyWf0T6hkPnzXgD zq4eo@6Lk4bK|I1#K+_75c0QPlwJo>K*xk=|54;NMt!uuNsV{Bfxmf#738k{MdPf2n z`^po}=l7!Cwd+$7`6niNYFh^%l#vXpwrONCp6VgnR?Ol9Ha@A@J)5MHD>i}aKH!!% z8#3+sK|2b=6#N{XdVRy>j|Qf&G6?aq;T^Z7iC#t(IdC_9tV-0pTutTGQtR4vtZYDU zbIrRyLZ|hq*bmh6^7hgK0&R>QPR;_(NVA&k=7q^Y0SYmUS;0Q<;zexTndS5N4(}la z?oNwrZo7FJt+-SD9b^q4b~Arrav(?Sd4|`mhi?prte4`S9DS(ncm7Ez)ZbnkPRDnN z0ZZ*Hz~hg!|1fTZ(Cb@S&`1mgxM1n)e3!5{XI)|f`M`@)BCM(`Rb!v<;VKU$ z$y09BH;tcS-v<~`(RqLOVg;Ue;R4Tq$+Q0{8%JPe!goy?$K<(N@mC(*`}#p zCa7nmM7qGa9Ny3?c8=&X=-Bn#I2G@oWbv&XBjuz)PFBW!xPRCuGn6O?cdqhk+}9xLsyJI|)dcQ7%PI=(?BoP&&tt2<9Udg<0YlLFU&$)A<=b{O_@mAR-C8?(@HZo= z(<%gUV5$Ho8siaRU9!-@v!|ZZgpF6OhegI_&2r5?H0q;?3?zPIjLab8y=$MZJVo$*EQ>u`JXy`1j@@{YN;(mL1~|&H)xKF6gTz{uF;2> zlL#8K8u5TZ0Y4w1R;%MP2d{s-sr_9T9M0c8=4u2ZH8R6T&m+xfBR;|_pJ)&h*J^~O zA3t*oxnV-%*;pn5=I7j(4pyT8EBv%daJZN26^m<-IHEHPgRlr}1a6>nI=lTzgH>Xc zOdpn?p1n9%*+jR1DYQF-!RDYza>CX={rqF7ujf0siB^p)mLvf$KbC5e62j9Ry>T1; z08;FyJIURCr|gsXCQG-TRi;$ESu*DyDUV6fUXMNCGxnuI~dAUJzO8#6NUx;9&QfT9Mg;f}KN> znh8dNpshkl)u9(Ip%;NKf)ibmGjBCzt7M008mx$5cwqO_a4>(~d%E{CXU4r{SJDYB zyeL}rZmBX_5Md{4x5A6TX2(EleG_d{KzzFdYm@zcX9t%pt~iB6`_&7G0X4U-Bv*V^6#fu1z zWdqkq}J(~TaA&G>H&zxkVtNSv~L%nAXbjFuN~x6wrrARi#y%DRM#cF&C30@RoX-OezPAI8PYw!62fhHh{vM*0a-Kb=qX zru|7^P<>Cusc1?9(gQ3jLoUl~7(+VlXa_QMZEj86vN^jo;`*-J{PSPH2ki$=5Nw1a zxE`6k2HIl{8h{;DTdr`ywHuEG-ZJ!*${#kbS(nO&=2@%7i8@Bp?AW?WM5r$g9`gnW;+R)l-b*$*?EEt+alshA9LgB)?M*w zf9@(tQX6ps&?#fF19D!_^*p!4%}b#H#$9h;Mb!u2&V;1zypoCKfEJ% zbO@y~wwB;gtTT%a*8C!&i{$GVDoP8d_7>nFTB&ozHZwfTzEOOZ_zVk;%)UjjsvD}@ zU{1Z*C8Ij0eMcHyrmyJBykxXmnjFunS85YV>3CPW`wQ&ppe3s(g7m@?rp) zS)Xb0-APx0`>mn4LACDGYF7Du?;nN)X?{)DQ~EDXrByui*{ z2sB27i8Tm!H%;3&KAn269Dy&8Y;MXN<=io=eRRnX@*BPLYTbS*yZC7Dt7}SVi-J!! zN}z)fG-JmBMF7`0w^N%u7UNsM8&kDkO=Zo4LA(tk&5q zy;B30AVhVuPE+@ur8Kji8QTsPMLWa+-2QmOXlZf69?_@PS#jb8~fa*B$!H1=7l9ZCcOK6$(h$eWkcsgxh z>1*)FVwr#6)D-kDU^;ur)hW^$JX%hAP5vK$`%)zg`bvwjA_aefac zvCVT?nnH>hhRyjjTdq&XTer1AP=FXePj_*vm4%#BO9!`FpUnN7EFzM5MIxRw>mHW+ zvj&&&6O*DsSdX_A4|#Ke?$v>(n>xQr)5gnOauaC2wsd*Jj5iTU$FY3j5`?6}N!E~j z%D_Z44H*9)0HHu$ze|FNdv0)OCl!N`MkyMhdM2Q}Mlw$IYPCvGBL=9DW zloCM%Qku;!(K}8ut;c$~+O9fqb$ixXY@P-@!_ZZ#-Et!XIQ!8-HH%%$5Io-+2c}*TocSG^Q>?ABv4?;0`?1MU8g{0INTGkZS4({6ik1m{yCD zXa4|;^{10K9rX{+stCGw3a8O?2YlABQEx=;@D)y?aQGa%(byl!ZY7>X8 zDO3Ld#Z`~mDIBVXe)5LFA<7mSEEfx)Z3eTRtG^u!4V)`}4Lfw|6(|&?!}c{xyGn9M zQ6%7ZpxDV2`?S}j&CsPSNm93MESEL?vN83|YaiLxS4mrMS4)dUah{KEac-m>rvs3s zg(wk?@!GN(jK+Dwi7|qi2ltEluM2GZ_63r!VG|aWhbR{V{ z3IPGWGmiP7*U~yv#{Mt$E6aFJ>6`p(a8OIBQd%oZs5=pm2`AVL8lkZw`EhMcLFya4 zr{re}jKi*!H~e}^lcW>4{J`R^eHnS_nYNPTE|tVMsv)I4DW^3k^0ctWh_=tVmXqz; ztq=Tn=a`jQWC9?09WDr-pxp$6sDQ>HrT)gLmTKaY=UZ)t9*o zMzv^za%)n6;W`*gWAj<9DZwwKon&w7C3=cSS)wP($La}fg%A<*YA8YyJ7KpFQ}G0E^UDb;imX{gB&>N>Vq|hcu*toazZ2@ljl* zx+z?ATkW$S>n0ty+~l&kPpwWxkbs{ROZ-c3$_h6bQa{96HljQsVd0okMgh+wvq)5U*02a9kg>+#xoX0b$qefu*59d5o2Dl)!R z=9aG$!cjW~9DWiBKwog64i3kB_;nuBZtCl0_SFo=L$m%yU@2M{OJ+H3dONb>Tqq3aXaOvB#sJFXxgqLH=@eT$Qq@=A!a)hjG;M^qPK!I0FO{L~9Qf4I_{#AaQh+H+ zu$&Rx6(t~UHaXnySZBjjT`tmBYLwaehHwV5b+}Rq(~+E=haJLwYw2%_HydA5baI>$ zVvB@CONjh-jx#siuwNlr!6;&4_3u+H0HNdRNBBg0+OKSMov!S zoPVts`dtSfdo@m-wxzoqj*Gd(L^a%GG^R^R%Sw!t7YYi21T4Ad83S~Y-l`o+r+R?e z;kLscwB%e~>3Odqpnz0`1x8xH<1DF1ZgihYyxAQV+x+{4>1E*!r^FN}1f|DR;$KT> zC!gLWzz3ltgIVWNNtGY3t<&v64@S5_i+qk2bdx?i%2-Q@7(2DCvY*5Pxvt+!CsuRj zNwZ=2UkW|<;cdp@c9@w@+?$^I8Cgj1U5zCLwPYNEqUNj|WOg-_b#3Y+vfMCU`PE0) z4Kf@9QcsShX0qI{V zLYr}!bdO?f(1|l8w$c!_X$ez{01|$R&2=e0$wgAOT9I=y%c3}OPfG#9p^wA~)wiJp z{RiHqrgc@h_dZ?R2HJ=V2xVWpTGF(Flb_+xSApE(p!!bnYJ=34%VegO!qhL>PQv7Pw9Ay=o$8w8f`sA2mRVBI!}hnk>Uit0#7Mfkfno+D1dY7 z7fpOTvUE>VUEj1xTUPsWx@{4h8dSGlh?%cGgFehW$;z(w_c!| zVYKuevGlZ=n%3DZig6$#OI%ta#gkjLlrA^ z3v@Dbp*oaGQJzLqfPIa5`La1WM`kD`vZ}tV7p|ph!PCllw&oUKsh|p!Ji1O*?USEx zTKWdlDbAbtTuq`u`N{9bw0`#**6K<;RRj2Pp^#OObfEV&iF`HbbMz(SiD>&1oDM3mzwxuZJ%UN6toO{Ht z+A?Ake$RCdmKP~3$B31jl>@@xJ83EBggcKl!UAqEUWhiHsj81;(30 zUSirXe|mgkfPaLkWR&PNgohcQaFWavKy8SByDC^KZ7W*Bm6QBSP)W+WBm}3qt7C*# zJ6(~sM^x7oUyF}X10+g;6tsSY(R)>)Sw zc^Pf?T2>YnjX}m1q@Bt^&XEbxy;R%@ETY>c-7`XplO4GaLbwLAf)X7{!=oI8sc75y zM;f!e8PneoVy6E9Ze^)UbssG1SzByO0RALvLW@gQMpKmX2mIC9iN;zvWpH--GI@>l zEQpdmc`_wPN{+da-;T6gLNHU|$w>)JgbZN5&?g(@f@(hN5*JvgmZrSVQ6WKUYCuE< z&RWsu3oYtw7g9XMAOf&R!Agp)mmYxlzta;-{SVc#CdG6Tt9AO?xKjd~-xwez)cXYm zf=-nbkSUig&visODZBLi2p701E=aYh_L56^8Fn+SfTZC;Ckq;P2O#Wd<61+)OSE8< zEH7GRWxI{lO)DX@G1SUV`2__@C0Pe4K3>$jb-vTd zHt!ADw%90s_YjjRJ4^`ET^P!P@rfvLc)`@1>c)H3KFqe_hRcj4hnD74*wv>TZj-tX z({F0>Jx)Jrc`8mG^JN1EIRxYU>*s&%P}f?^_^)@B z)Sx*TE}xTIZwFEt&J&Z8QhkZz7_X*&70A8V`fhJfD%T~}BSUfa(-jGj>eY?6Dgzkx zP_K;orYrYL^!wY*mV~#~@)IG%m$Z|R2+2EDeKTv*?)G>;>dkxHSAw13t2+Z;4G>15 zQ-2}b6oUi4nihEt5ka0dz@p>CQq%^V`cd)iOK^-Nnw+=T)N$neg(RBrG-G5`dt%lx zq|xnJ)u(jTrX7gj5|7TMnXzDF!^rwl6;LQf#(B|Axn*8IyZhFZn+KQheqxaAgkY#Q z`cX)37txkIXIeoTXeVJTa+=d-SjY#+Dgt$!VE*qPYBVc>NO6KTl?=j;=6h_E*8lsl9-dVC3(;NU}B=%|tNgn0{3#$)C;~!d=alCe*-ETQ3*1n+jlOmjKjj4qo=QSeg1q$PShZL4F0x31% zlby3f7QyZBUhs(oID zLU1xFh+77cs1kouK{`pvsTWbUJMw4;k%CThSXRcGbxo^0VARAp;UEm)gHTRwk_Bk({mXCb|4>sRg$0PL_+xN987pXWNCPgv< zuD5$n^PkR%YW19DRhBW-VB-#cw0k{6fIRryn#R$mvw0X8P{-DbMbr_EXZ-6jV*Uxo zh)2?-qu+K1#LWQqUsrWd8y-FC8s~Ue1Ov@Oi_!{nuOfw!a%s`A$@ihrg)Zu%?gF_K z6#2lQk2dt$NK%UNkbd-uEEgLpIZ35#bgZad*0ngIr00FJS_k%ZhULeuoVP3~EXso+ zl>r$}f>eT>5&;Msoyi~NR9mV;e~TI4nwL)|%RB197BsZXiM~!%-s}ZJPUAuDTvYjk}ATJ0S>Ehh2Tv+*TWN#86Qw#zK7AI5|#9^{w^WEH>SGz&1)&E8AF z8hk`UVND@m<0(*az!W=qnQ*T}~p@rPsp zwf$1q9`$I)f~GvAvO8tH{-eJK-=EQN>@Gn_D{$J(ha|EO1=inMo$SznNK;GZGV zq-zN|@iwl77&U1e^P5^IRw(^8!pq~R?TtuuDJoMkS1G}Plj0>hqpSY_mg2wat=IcJ zU;1Lg^~%9}hQk(X>e-0>&MbzP9yG8?W#z<5))10+Iv3dCKAGQ9bQGIo&0OtmH+YQG z{tA9ou^ky!OKrTDKvPOr;Q?Mns{smGHDvfoi)>4q-A~g!0V+Mt4rBElqGH{7^}tXI zIuPQD7C;5W;1I1F5TK@&MUG4MIh^9w%}e6P!-zd04Od>Ev8E(Yr_`U9I^#0pP9JZ^ z8>J4mpyO&7NlKNe0Zr9*Erq*7@NJXtGFn&4i-bFXT3l{qc#&Kzc%RIqvb2IfiLkH< z(Tdw$l-=W@jkW1Au2La3WlX2DoOLC)+2K1bXa3vtQ{=k z1ARRvRQn&Cywi}@*06>#f;4!_C9~zilt!GK%4sj98ZEuD)zgpIZ#J0n9JL^!W9R%` zO8xsK#D|mS+$lOi0bGYwQj(y$ZmwI@l!?;})VS+yfTWm_)Jk-y?p2{fZ$M2^EsK$L zd2m@vFrivTQietpRii$N-_oyt6K~ebC8}ih zBRa#*-eU*6OzPB?6@{xA&IE-3PrtQ%+QstWH{wA@%)1iohmf`(Nb^QpCsK|F7;Ply zw})Mr<58qdzE0V_{{Uz<=g8^%W$VXGseM5aH#trr1NW}7*NZ9TD}oAS0kFrD9O9_I z55HwxI$x-IfY_4t#SxoC2RTqu+(RxnpK+9s2L*ohru;y*C#bqAP0nzNi<8!;8MLys z97Hcd4=F0ZAvr=pDFCO)S04}_3 zx|TgFDLF^(Z6-#>Tqk<^H-jU=Oje7YAzrMy%VA+4o&3v2Gs(`>&&3uW>D%mUqM;@@ zx-Cz%>OsSaZKj(fpI;RZZGd?vIjHwwy=&=bZgODn1=ue$Icw76?z%sEfI9#ZoO`Q1 z>f9r<@i(RHod?@cPVO0Kxx>nI8wmS}QP%bO7-Uz~3ffm{yZD9%E7as^G9HeJ(1$^=F zR^blk))v+;&M7wOueTCv3xPR~>d+U39{Att1xt;_*`n2-T-z^DM^m$4xJhxxWkYf4 zO(VrPk}?TD;oNGf?^X02 z(Q;d#5@4BeOqW6nSD&Dl2aK;We&EaE5ld!E;fZFu%b8fHUh9enC;%9 zde5nthiQe?-9Tc;xX8JvIVELIGadsB68h9vooXl!0Ca(^cmSMIX7#PHH<_2o*C&j7 zW39|^vW-boib53Qji)#&Q#oxTI|c2Hs*`qRFIZh<^%d&XA|FU}i|$<~q4x-CT*W$xdixZQnbS&~Ev|6Y2eB1v3S4#MM5!rFrw%U-?+k&HnDldwV zw+O+&KfEDW8+d3)8nNMnTX^zsAt(5A<2^{D7Ql0k} z(`@*X{pRN?+~g~5&Bmd|F`FTGSM()r6el{LC<-G=e1T4HsEwh%DRmRQJq_Q6{{Rha z;@;aTDZAVnPBgl-oW}}l%;TgFF~|e}k%Z+&-q!R-+;v7|!zZK`s3>{qGJyX8ew51! zR-ZTb0ci?I1BGeIlz){w84G_~^fOm5i;|-?af$#ANQzSIbd$mqk_g}UicU5gRSqG4 z!<5HNPFK#P_)_Ow<0?mhjD8fYg`l!de-gRutZrdniPaFnRqk!oFj!7om1~w?2qa)> zFCkSrwc){Yd@hNqoqc*!fr|I8`-8FTFyPORz9l$4~n0UWofLp&sGnwpPs>#mud@)sn`a%>jD)o+m6jjiMydE&`R zlR}YPn2@ga_{mrAV7!Sj7poyjTw|}n$mc@9--`Iprhm1^#7lIvS~}~cZC7}0BgoA5 zi^;}tmA2_6dyHg-xGO}n{Ac)0)IqDFpmkSB-qeL4Na>62QH{s`?)J)1&dDTa81rpi zD3jlAcggRL-R6Ter=k zcZUk#!a!M;!CGHd1^`kV30~*;Navqgwb{DYr25X-!)y45XzIsetL2d@d_~!>ER1C# zdJY^p1QHTA^{#clP2FC!1;K7r`4PW-BCABlkPi=g&zIRR!kGXT+luQ;XTnm#zk~u#kaJ%h77`Rf5E46A(|v1nzjUXgQcF3_D5sz<5d6+&Sz#m5dZYay< z*vInr^s3F$&7)V6wfWYxfN+n`imF}hX~8-`$m12bf>82ex4T&xP)Qi)ZR(#oI-6cF zd98Z=Z2>Dfd-_&+dV&EY3Ur7{1yyP6GAX4ed}ALii+)ij`5x&4u zGKqo|0qASG2cY7fDDU}VxKKtxAFUv4V*xwq#X8=@Z>=R`JbKrf=bB(K^m-bRia6VP zk8cG@+r3IfP6B-?tU%WG98-x%$l{p?1A0(&0x6IvK0XF0v5a%IMkVl`4HVP*)11{% zjach|VBk;^5;KegMk_eT=7xA@YHAUYj*d?M07?zscI`*V8b|f0_3m&^$F&M1*~Ca& zV>vWqS3-fwpb7`lnRg*42OmmFAX`kD{j-6)nETU=-}Ec-ezj40LXnY@2fYC0$!)9g zdsMEOOEn1nMZfo)d(rXwlS$cilQVoHZAbH?n7$r-Pu8fNg&O>RrpO+By>rb;vDAZM z;(lharZ41U_mw9XsJO$2erdyI#>Y^ed`+rJi`v}b%zf)3jnqjv^X_V5CsDW*07+Q( zJWys~yHC-{cjzQVI5_x*bfjeYd?a zJ|hgsN|urU^5=d*{*)EP=F9m5JMU69xoSg*TS)UHg#*|hZ%VI$6Lx)d)KeLrmSkJo z@mTOig=tX)5rQ_x-)vXT9}v+ccRAlGsID$SIk((e3qb&RwC7?;0D!EIL&a$w9n}{J z$qSs*h*57spDu?QQ6tdqKEzcot!>dIDnyICHm0nVArcvP3qo*kr6e4jdvEKJUp?xR zSr1PIBz%+A##hdn%aeFZZjs`{@my&PHh=KI!Af!2I}CA0o|A4(lr8q;LW43}P?IT5 z6$Twn=~IVep=2X-woY@tOZ{~Lgvoihw|@Y5o@Ki|LO4NANYSL69&iV9wS5)+p6)ie zeJs_VtzC?N){=#UoL~?Xl_y|DYRr5;=y2;7--vxRGAzjDUq0evpq5=) zbQjg5`^HoO1!&czoTvC<;lBs7MKUTCS%l$gQcO(quSGdnhRhc}D{n)Cvh8WO1oE+Nzqz(Fsaf9k45>z`Jvn6#l}L%g7C< zrWD?$QnfS`vH?nx(aJ(qry)o>c2FP@Sx;T{ z!?zpKE>;QGG&gaG&yNpKC|8bD60IkP8&YwoY_I}Vns0e@Bymdi4}bW`__xB3Wwyq) z^*mV3++j4*RAa3H#Vz)8&Im5Jlq>?31%bY#(jwlVS*`+mEJLGFqP9qG(Q5qg( z9Oo%?v~kad;z-I>q)khHwbLFX8mX}}qD_lr|)T=V4 z+TzEOZP6h)9uwuvw&aJ~@mT->Hp)VO=5a%{Uu+3w-jqj8rtamOSbCD!_1X6x327{} z#DADbKf;B!9S4vy6Tztdn3&z?qHQ+|laB`Nw@2SWQ#eXAy8NJAN#6}R(ndkpI0s>q z`o7g>wI&^w$8C9N>lR|fej!ha7EsvwZfYc;PhLA%%CZ5A+@Q^hBBlnDv!Q7Q1v3J+J-;e3+jE74L0b`e9B7XCm278 zC$&!WUrW8$cl(vn@UPjkEmk=%E^d(L%e2xMOK8$g>se@nz5?7=(zR}ZgKv{}wavB| ztXuAOFZOvZqspk!(IVnnZ6IX$g;CL|Kx6lS5;>>IJ}IOuV%xHj@#n5OcdI&r>(P;| zZNZNMBJHu-OPFq3Sq``pqi-^aM#Ex|dsoi=A^RpLTiUKJvK)gh0a0x$C=JSCK?+82 z2+EL?#@;CU)#Lj_^p)^l6VijHM&8n@V+igr3z^AEQEDI%nP&E~&Iy-nZW!nwF%^jTzYu z6`^jn>KR+wRtdsgB09kYlAU!_z2DxE8o|`PM{kxIgK=rqLx_@3%wVkzA$Z*&A)=GQ zd^3P|6&=&<*}CFOjzV=Xo4#L@F{tBCrkx8)T2gRw3fxK)jtU6pQKx#I@p@IED}g1Z z=Ank1@S?V%8T6_|h-?rOpW-1)^yiA_;F$K(q7SGnuE(@Yi6OKpl4|0I`cBV3tU7J!@-J5vTq9aguo0=ZLroV#kC7>Gy!y(r1`=15 z6QsL*Yn+>`X9Q(8G)`r!dns^)FykSl$du}pokU>1oRTmUui+V;>DZkW8L3@QbcEE1 zw&gi)>HA!l6j0<=;s+ZJl@JzEg!qe3a^qxzNfhwV`yLFW7 zYfgjbOj|XfASmfS@S9LcIsWm|Qb`HiR-tq=-w~dXV}o>JJ(?s%E%l*~4Wx&OQr&m( zB_+h<=qgi%v>+`&Y6c$H)l=pOPxT~8jH>LnZa3c%M^w2AD8R_UEVkE*tvqRM9o zpBe2oOEi-z(jT?NWyuVil9!pU6{L9>NZlZ52g(AKfH56HDF_8=0R#^=3u&|bMCo}-g51T2<;ZCU+=dvqC&D>W zl=hLM%ruQWl6TF0hMukKkBe}jbj&M=wZf5YYDC$qj*?t))2l;qf|M50H2_PeO4c&3 z6954vbQev+YJHcY`lI$+ExZ*zU&x{P4?e7FenYyDTy;uNT3aMG+DJYb+bzm6Ku%FU zS-R%cX-;qm&IM^d8onK?sqPl8fwB}r>FY#9Ou9}eNpX}P z?((#R;lkQhu#lXRuymZ7%x~Qta7BP$8b!w2V}P{Bv_ewWkmAltQGvhV%9N$#lB}F$ zR*Fdv7Tcn}j;FgL#CqS=x0!b39e=YfvKHw|iOKNZ*4VB zP*Eg>IKqh8D+6T?Frl)&0m!9Iha%wPCRyc#k>4Z{w|d5Y7JN917I1N^QSf3k9Z*E1 zdCouLx!8Y&Cw?pH9*Ik@!C~n#nr)_(l%ce2pTawBx1qoT70(NEq)XIIaVQ_!XQ>%( z&td7O7R0Fxu%8g2B@ZJB0H~9cC>(K&``67YQ=TC>UjG2i>00OZfW4-7TR$1cK4DRl zp(-c|DMukW0B7s(St0QpR&YQV{cBnLHMvG0N5QH4!BvYPyG$QCeNJ0r+XC;?JO@=V719xTOk0 zm&QFi)Y;D@Iz4?U=8taP;+R3tU@7*8eg3qplM6`$1M;Gpa~q5us5A}vr`iNyjke83 zNESu90sjCh4b=C?){k{K&P53FK^)i7Gu4YD+)3Xw!jLnZedxzi*kYL}89Q@G(MYgW zCw<0g3P#*fD@Q+{(vVTMPTbNA*dmdiT1s0W=ZY^t+PG1;q6Tq>RHbJdpK6wj%7FV& zkkSe7QtaTQ4slG7sA#sV9gQ;LPI=mnKcS{pfJxe&L?bO6jFK~)QSnqb%>n>P-`9ap zvf@I48~tjL><%P}$QzyLXR;1+k9v4{hH00G!5QP$s*sL91bm#})N`csQV)`H1q8O# zPShFCIE@7XkbkW-?mJV9LWbj-X+c=r;EG1YS|a8k?gteIEwTy5)n$;zpiZ7DB3xoK zw-p&tM9~IRq3SEil;TGx=}x-N2_RDoY=O8m6oX^QxhXu8MX`7w{vtm04zvTcAR%cv z6-zeDqzkZF8CM?lY_iVQ;2xEg#apO22N|m!nfab70uD~xR@hO$bj#75)33;VNUOv| zwyoP(AbM73(a|LqQny|uKg-^)uvu}Hr+^P}No7*eYHTOL$ymUsJFK$abSUrdQZ6Yi zM$}dLnyOsqqs9YYC-bI`#+45@48500B>Eb;biJ7ql9V9;IgEPMe!>wTgt#{6aaw0g zL20$6*1}VcO*?cV?381pt~R(0-yL6skQ5T6x$le=t8ROWqV?;-U8Pc^v`f$zt8aIwjD zIU(g3uOVTTXUtQPuYW=^2-wxfr}_usUs7A2LM6^>#e$?^p2OD{{TW?BPLC@7*n?EqjE5aq+!Og5L|4d@iwqYNE_|7 zVUaqA;v$gZBf9K}6ZA}N#@udW4wNWiTAL+$c?C(_9D)XLXF7)E)K@h^YpCx9*|w)r zyI)c=+LJL2qjvmk>RM7pbqu(Jm1D|jR_l{q1D!v9k#39?bJZ0{9u#KWaVa{Gw#iFM z)T9p)wFD?Qm47m5@<{@NXWTlS`;-9KVf!@ThupPJXvjlsD32Jn=5msS6=VW3k`J+R zOIGA{1-@%9NsiQ~T(`ds)hTId+@nu|<}#3=Tv9z zbIhsv9{vC)i~8=4Op5A$LFvwyLJ1xkl(N`q#o&2P_#EytxT{t3rsj1sFEMVBF!efC z%QM?b^s4t6rj+l_wd(RPmR`Zin@DtIrF9u^izBEYe1-ikMsXyv)9QapWU}c?-UGQF zVt|vS!RAh!`GML_Me5yQHduGq_c)iHl4TRwkje_)Ub5?He<|#etb>v=Qa2lhjJUe( zeo(D8j`ch#SBVxm4&~bqQpJO6>Iiz!!VOwECeM-)xcsh@YkTx2WG?F&SNynPE zPMNv2)v}}8rJ+iO4&jN!ytlvYvRv@*T&Dpp#td@*->uI zy>FJwYe+B0aXXJO8B$76v|!{Z0~yVIukg`}H^b{ao>PnYrpo?(_X<4tURE4H-1$pO zLbI?gBaP`ZW8%kz?G?_rVZM;~WoYX-j4k?@?eN0A%Wyea@mdaF?HLE)j zkoO!NWaAj22Nr_cx+`A|?bjAMC*p5Xbk|cdtK#FO$d46%?ZiIv-+8n*z!_6$!5bY| zHQt~2gK@N8Q4-rZw=GaFjzwkFq$Mj2$Y?m?fZM|HR4oG{{ZH9-lw`7;o2>|OwUr@5~r<=!I>Hju}B5K0u*JH zzO@06ubFX50Y@iEC$%%8lc+QyvLVi78$Z zSPwsq&ZpCoPD%2o03ZS*CbbZ>p4n3PASQLBewmAHakvf(t>oMqVUG>hm}Qn0G&pu1 z8Au5~@YX@^R~BKwhQ)^S9z4e4;7oB3Ct6vz{L7A{g%gFLBq_nGB>3nX5m8?WM@@A2 zf`4r|wW;TA<=E>_gjFC&aqzK_NC-}~{{R-CgpI}t*1nsa)*U46rgYntzDHEtU!3#u zn^P{^`hF?UrrT=+!3%Y!nQgJUu-L%P*=|+i{{W)POJeQwFZOiLyeYul-h9U)*i3>% zcA^9&Z%kTF{u6QAN;0fuH2(l5t5;k7rtc=-)3zBh-eOcXQ)FT1JWRHyb(dG14Os=( z3Xa6AZ~zvMh!*)bXm-w>vATvEx})Xix|hnt3lt@xpxJlvWzxfnQ=278ZPfrno7evU zXa3H%*`>D5FBb@3B!p!6%8ulR6qKZ3k>VsQ>c(`Se>G9HLM7zo=c*f+^nRPRnJ99= z(Q)EN9G1(gTFx*J_lnY`e+X=)CYfD^={7qkk#hNKbI458!)c%r>rE*DscQUKQrF~4 zyt&+rBnmI1m__zw=ch|;N4i`SHQ92Jr&3yEYdXlrkV(K#a@o$wz=%bbW6)>XS95k$ zsS}w>173Ga5!9dEB?Ek^Wwhk^Q;;+GjZHn$9<*3`FROkbvuSkOuaXwK%0$PI%H1oJ z2_c4*r2t!Pv@0Yhk^G>OT9-|~WVb+y%>Jih;`?l-NJ~U07j&q&@k%mMl0m|p#z4s6 zDznF-TP+qrV7JXBnT*PiyOXip2$ZA{(NOav{oTBTpdarvq~HQG2R`Dv1V-#_B6Foq zh^T=*%?>!0LP=0z8j5#L$s;4WhXknd6G-|`r&zYhj_tbW-L1(=Sep7KyMkU&3PxKQ z*e4&4QqBQ57y`ZVzR_Tn)9aHcyK-2I;l@L4t%%W*4p!nD1R!sp-nvK_*yjm3=fiJ{ zms=-=G{&;DWtB4?)XOPfjUb#Y=@0pwBO`>AgQZ^7N5=mEjonVgzr?qA_jjf&G?gX8 z9s>^ny1y1zft(}~Kjq$xewRk$Zqf6nP(IbF?DSYDGi6P{FG>ngKs;c%q@A|}gzh*S zRlniIg5>L;v?jUZ&bK^SM071Jq1`;IK4mC@u>m_1&1P3y@?_q`xw4&pV+Cn&tbwV) zp8N#)j=*-}weN%yXI?H5Zf-_mL1ivA9mbThl9EyswQiJxZ~)YOcFjZ7N!25-9VNcc zq`nzJyWAUH-u9^{=A7}Ff*B(K0urYZqp}A2QMVa2KWDkR$5cs6w?pBCFc|lKW~0xw)WQQLXqOQ>1-)rdIPdS*lpUg6#PVqV`6RIOHCQ~Ql$8K@c>XK zAm;}aEL~`v_V>{V@e$9P>np4O05n#WK1vsWay>TZ>)xuQYIG#H7me~pKRWaB-kS2B zWhp>$MByN5-?!43LxB$Nq?4cF?N_6cTXtEk2uLXy`qk#n+D?;}tH)4eLuRN`KiVv z4LsG5loaOTqUK{5qt^Jr-l6U?01DoTh)a^v0nZgny||JU?^YYMu$<)8BJ(>|q7M}| z_BB~h>LrXT2D1L4SOsA3S_S$}vafp1@0d_l{Blh+Xh|9M-9);g2Hw>SK_ru$b5=W? zZEI1+NT~A&o)Sf87e_SXWJ-qnn&y=dFe+7WGq@P0%oV4EoK$2&Q3GowTmFPUevrl>h)0BEws6`qY~yR~-Y}y$Iq+Ab>_F z*E{-9%NgdrjP#-g;&~X&Fi?KfV$qMTDZK|fQZlK6sk7;dMvPHq2apXUwtYFI88A&9 zx%*cN4mjqD@W~?pd)JW4iN-4e&cS;qIxEwBm~MVxLQzoZ&S-idh7v$TW}uASh98vOypwhEr-Csnmj){B$6kovO)a zb+(;KjtCpoYB*3k=B91r_^BSdaZH%$(YSO+Q^C@f9?}jrC;e+ljU}lmNolZhIUTDK z==hj<3P2o;1s$h1wE^l z80m1GK#+X}JQ<`*bmg)2AQp(uz>E-aSG!~jvod2fZkn$Z;t*T;ojV?RPx;b%TG*ct z0U;UOX1jHIMYc*7r&vR2BL@q~KWZ;#iK=x)<#tY;jT+?L%3O3ZH6*wkPDbZZ0YHFw z&0`mSpoMar<7i>Tpn?{qqab8{6@+CV@A$UItzNpS^k>*3&TihPVg!U2SaHOy2OE_E zl%IUn8Vsv_!SvdgmV`)cUPnsG0*5+D$DbpyA46VGsBO-Uyh$Zw0JO`t%WOls8HJ@J zswxl?^KE1CC&M6+KbRekGwD{zy+L+Gp~Xw-L3w5?XXTe7D@l~n$R2jzK+hfbJPksf zYR?!rBHQjV7<9N@Ojyka769<{_UuWCyvn^#QLgBjChWiO8PtM0)>HujW#HS?$ z9YaY5t+t+(xgzYjJu>uLf}uxTt6Ygpr6FiY0c?kW!jZVtf;8amgUyGcA-i|zSn#d) z*jJWe$Wsv(^hCD%ZRjd#w;n;%Qlpfi%{}xeCz_Ip(qXhFS4PH+W8BoQoawo83f9zu z4!;bb6samstQ28sP~#esLwC`ZyD0H`j>US8^c#v-CRL{G8RKhkCAmgA(&9@2Ux2!d zl%sT&DYWTP){#^8Xx9_g_GBdtbpv)7NL)LJK#r-f-!3b`aU>0GsYxK^AhhsC!z38_ zjP#UQk*gEOjdpCmYiJ;;ECnURyvh*NfD|}!9s{@s8B2f+ zqzbup2TVwlW1ZAChoVQ4-O4S-Gj2PTq0p^sV{l3wNiGyPPvHTzgsCLg&$Ip@l)o9f z-O4Mdi25vd0`4;PzEad#dBlu>rX&!R4gz!(k&3E=m(ov??G+w|_)XK-JFvFJY1GbC zhW=J-MJ_BlhJ&oEdJh!r!5b;}sDmq`skjK{X1Na-z z;?;xbJw;r;2lT`{z2Z!jxa4tZMxceDt#M^G;PF@_l1`F^Cnxw->5N(aIrJpnsp!6< zm2FO|q_>!Y(*eK03^c}ZEY@!_KIUVruGz+fV~ePjRhrmRu=b&G$Y;C?`46PI<4UQD(f{V0ARF z4l?3Hfc&d`xz85-1-#o~Lv<1Rv@K&%{{V^N;~P^+H1sBsj|+_Ze_vRxvykGXwCI&2 zvciI;uv|-MQC0~+R+p8Xu#!i5)_UIeaocmD2k02#UXcVbyDfH19V|)!xD+6;`_r))amrjF$ zeCeBGNQ~t9483V}nBE#trM%%wq~irCZ)6U|YF;vPLJ7}H!KBhB>>YnD4_QmFT;7ry zxCB69E;w8vO*XfWjilr{mzUxxqH-0dSUN}$-`ZQLA8V)jYTX7&ZdL9Sc?>dMEw>6H zJj<5&In#|^8+x}SKIY{}HE8$|@KLY!H>P^7?h$ffyPV0Aknm70NJ7b8H_6CJBpr$t zu5&{Cc=&w=&ueneHhFP{Om_J#*H_{=`whnb0CR}92nPw>eiC*bKmo=>rwB%68uRfpc#CT^7o#!WfqV z;^pJ1UV@^ez=a+HtQTm z5|uOar3KgBP{v&F)Z$cj0I5L($_A~1&C^mkL!~Sq4_TiP9YWywi>$dP#aga!l_<1Q zjo)zyTjxnu3c}o3N)nKx2)g9_nr&YcReE-O85b_5oqLGemYbv~p~w-`W47j}63j>q ztw1a#O~z7A;I|YwwH<2|CF8Et)Aufd(+mr-8}TWLs8Q0k%@9Nyr|8iQtpyf<4}UYMJ}mph}wY-hLV7WThzB4eO@3_i#k^3%1AR-_xplBU>Pk!==HVJ zy)d^CbDq>e!EL-{)ta^H47j%*%G?X}*O*yN> zLr8HskEvQqgso#E1xr?}fB^<$tDtz@MQ^${bdCE~ET0?{f{VyJSu)2Z;$-b!hN8b)l2zgLEM$)V`87d)eq%RttJE&{Y5(!X~*F!Dn zg}imN9-Ml_^oYrN@TDc;9Z||(ZZC*d*jONj&H&P#&Z4ydkc4737gKdVQoh$n+2=@YhFVjpXXMshxM?Q}0idANbp9d<^AVR8 za1trTx)J@%X4K8<$nxK}^u5ws(p(|9tbl^<`bw4%Jcvp_N>CN0ARWmh*0Fu*Tc!U1 zrQT($YtS3;v*E*vqT}vhX>A1g+!^5OLJ0(&fTeYV7JituL7UROO53;h)Bft@>#GTl z(hkaeAt@OrASDSICz5xgI&-OKHGONR(KgA?zT)qY0*@oVz#cN>DZUcgA<}9g*2}to;*_>EU!5WR>T91^zYBE)g3#cH8v*O z_(>*y0h== zwL^SP=%y|GJj9E5Y0m{j0);09B^;NG7BqsBO4T_ zkJ6ZpaRGi&fx$WF>t48!>I;r6B`19$q0anwsDB+ubR-o4wn3uN6GyU5*QW|UonLJ7 z5_FvVb6EYBZj}!;XckH2=^BTvO{P*T)h%*{5;OJ9TrG1jK*`#(D_rd9IsIzcZn)_K zCwiZf>WwB0%p@F&n__U%Jk>&jaoZ(TGR4RQ?y7Ym8Z9o*0Y8eNE;j%M=74_bfsE8F z`<|eok@?Yb0z(&h`hoL)wM@H7$vf1Y%6_bne_E^DWa`ci)QL70(O$c~t5*EyV^r&+ z3d(>RRjzp|Bx63Jpk+rcDEsYKNRwqr_=QV5AIi9kW=|O%jad_HQq+9Ad(my$PCpGZ zS|-(%GidVcpVFCj(yV?CYV?aiC-DB6t~+GF&WvWc=BP&mXAjzr{!>CC-mvJ{?Y9-6 z$+PLm2;5Xtmd!3?pbq~4YJ`@Kc|!UkBf=mLE`8|cVT6pS8T!=JTX-lQUFflFDg|0b zYnirkxa`T+GtD-%_9BT=H^wvml#z%^`8X#?*c?}}w?oNZFdjurXSYm5d? zF;wJ><76X@jm{`il9D#S^{KaGk4lezImjSj6In>Q=onk&HsY(4hv zUJRUUIjOlEQ;H+qV>}JLDmp}Bb|S1Lf_9-^iIKsm%48mjq=*A@Y93q;a-Y_&k3<8J z?NPEIg>QkqMaP6$EoN=cT0#dQ)KsB#f$T1Rhgl)h%tlOg?Je8wT zQd5rOiqo2H5o3)=*p5M}r)NwE%j=Dl;Ac6ax<*10vXyU8*L8TxA-w(1CZ~EL++L7? z{>jeuR2dpAmDeAAJ{ZDJJ*d8+5zI9t;4e-Bi6c+mr6IL!g4$G-q$>ieJzXR`(u*oi z2jV%~+irg$LwB=8_e(Cqh%#81>;5~5!)r^(TX`AfK!TsRrtl&>89{xu=PkO!=tv;? zo(Lwr(KD4H=NYwIVT7Rr#gXCos4Dj^0i5r)KJ>fgx)XN+Jy8_X>GNu5LfTh+08f-{ zzBlx*FV(NA+1-ZdJ;?XV#le)zA)Ap`io}K#rAkV1_?%ct@lZWRahw{kxeDt1CtJQY zsIysdd3WU;a7jXvHwsRjlr|wd?ZK#CoVvBucNpltNKzS!vRe6dG?ErElz^h)+#c!0 zXjjN$-x)>?ruO4HJR(`OB12CS0Z4Qw0X~6V+#SVB8gdNUj*Noo*7B`ZsWNQVYqUk3 zY{GR0L#*l^Qd5$U&`{th3C0f~;;MZ@<*%q6Y5t|XrVRUoi1FPFINOoc{5knxhnr9s z0ZMI2SSP({(`NOBvDVhb$EVLz4eGyU=(3aZ>n6!l-??;4` zxU{I`mfS|%g(XzaTl5W{)g7p;b*;JEjm4odGf%T8Q;SIet(P?nF8~tS(vN)-@RE94$($wC&xq!l)K1Wi`&NE5a zp|!BIB0- zHFv*s_ruh@;{IaPt58F(Eep4)))TEKTG=5=QgCyrwT0kr5(x&edo`ASL-kB4&>M$u zwaS*J;Jy;<%Ot{~%3W{wtQ8{!rv)cYbf*N4h)Kn4n@U|DMqaGV!H0Lb!ffTlEZ^Y1 z+FN}D>2bMiDGDvRk1n93Y38mREgJw zNV~D&lH$^+<&`_DxN}=hE3hPyoZx*ccelNDu*I}nTTy-Y&AAFwX(>onu;YsCbm78A zdyMl~hxSzIxv+X>(PNa_K4vQ^NA468;xhIY(q05*Nz^pz17H)_*8K9_QMgJWOvGt& zJRLYn9PwN!3dZL-B}cF-$<9S_MqH&Q_b-ujesBDGwc4RwtQT7w%6YqVnJH>0N|4Kp zG*aA&t}Q?fDp*4efChXb5)RdTjq$Zid^;9vg6A#CscGqGYB*VS)*S>ftZa~_wvQ28 zkbsnp*NSWVRoE?%I(x!)EVtWs;}s|^C0{Wq)V5q-h@}tRlH-dhNf}BKwQ0)9s((OO z+5Z69gytqT`%>hkF10D$+hE9ZOHGb5;ipp6zv5c4{1s&=@n5*DREX|?>PPMot`|Ae zd-`U&Y)y{v# z`SIk#dQ1|D_Q}D*2?^Y5QBLX!Ep8LObnY`w z{SfHhoA`!{G^ZjrZA`gsVJ^KXNn39^GN%^dL0Snvz=W0ga52d1p5~~JQ8K%?M>E#< z_0OD`_In|gwwG=bOLH&N0S-I!4y7f;w2XqLk_z$?6dMRg6>gxt^h_DjdPdy>G9raJ zn2p?qQqyS|irWaoY_tFxmgAqsqC(W8LjM40sqesu*Gb6uy30}$(Ta;(EijE9UXh$BZK?L7bZf|b&W(Q>IqVgkenx(&Zmpi z7dt#q44pJSsOc{bZ5%coCXrnHh{vv;@$ za&d)~HshWUc;O@|(!A-wNx%Z<5-(KQS!PxF_;+U<%MhE6^IssyCk0NGr@=z>fD(kQ z!lROKHmOVM`i=T@jCAxki2^a7wFPugR$h@U*6S7N*eRw#jWPP$wtQ zeJi&Zks(SB%9845B{^+^^GZt`=}USOr)=;Bt@)}o(%~@|C9{8&kuAVNO#C5BPIekj zb$eo^#a@TERc3+H*DJ(l79q-u;4tPA6P;=s{{RZoG#(E1eCjKUZhSMl-Y?d-kjr>2 zWxo)%;TYAeD#lKEAd1LrZvic(yviCw%Su5F2K4^`<~xtIXrC0P!GqBa+ime$dC3VY zd%ty%L0*h~pBS}l<7=_r!f$od12eT`R~ktNZk#qqy98kv(Z z!s5c2E5YAeVLwhP9#T-JO1D8LI!MU;=tP5S#N^lkSp;R@N_pZZ9^BRYlbQ_Icz3LJBzS@emsEJoyM2%VA zia>HDCfq`aIr~&Ri8}MW1mxngoaT@mD&K!fv{5$BJT>;Fk#AF{AbqIlvCGGU_Nn+5 zu%Hf{?YF&EAv+->*^b^%(vND)0l-PCR9is)UHxi7yE)hdewDi#tfwN*Mn;3Zc~*Q3 zXsgm~8-5W@xoQ9$Bz{#nM$JZZD$1S6N8i?<<=Ki?l&2W&T8yhu9H;*P%}}##ARK@z zno+G2akf#|x2Xg0??k3Lf_SR|w*-^0A8Hm%aD^NVzLm!y>YcVBkh7JQkoe^j}gXq2A)K2!8CA;amF{N z*^Rd1qVf&M4H1FCz|S00sfslI6TL_xHvAfSF~%@Qed&F`a8;oAvW9@{!o#c?h9wg!$2?TX?UI)G5!!o2kT2Lz7AQDVT|vC=}C4j zMsZUqh^%VC_O7lXvA733*F3Tuk}A(d=gUDo7di5u%B>GVIXh(WnhkJ?BoVbiy6{X+uMq(+HFf={YX!gsR=)QJ{yia5K&`S@&Eymx(}cPxSu)S{JadlfMrOHwTpp zd`6xK!AMlM8943bN#7OQf^l~cpi&SKutxzsvDbS z>INhLpBhU`j2R#S<{HvLD#mvl58=&hx5=F;aI?QGLo?$(G`Zr$`Nrd9=Tg>$1*JLQ ztn7Eqe328X?)Jynyg1XyLqzCX(zU4P@gYPcD;eyeMl=1Qp_uMDbz<1bXDuk1GCXIG z8PeLb^f>wu4JJ6tq>VJ6X5!aQ%1LSY_WMj&vICwK-rH_Zi50K_8E`X|XWaeG4{+(J zc4iVKK)6n_q-8%c(@AST#I);98_uDyImsBuHIc;l`4aJHofQ_PyJu{r)fp4k$_AY3 z0mW&?Nf=4aPaxECem~f{owm@cauXqf6A~zn%f*Zxma-5MJLv;fPUlZGr;dLHi(e+& zU}vXovf5h+>Q5QvtHUtR?1>0K!3`H$ASoSya4#9}+faupWP5x_mZT&);PIM|axXC; zC@b-(w!k?~dkmlE2C;sc_>Xas8R-z`-mPh6Ki;k6wzrfA;te!{vz_@V2OQ&SrO*D; z9}BGCFd0?bsXC8XOod#eB(~dOSqf2cejAD;0u%etFbQo$U^WHIt%E9Q(>GqF==+t< z_M64)rEXSYwJ4(D9zxKQg#HEih+154Cm<~<89M>Fr{{b!v|V8v)EKdpw1utd7RUS_ z2m!vPRFW2VI}%5*8&}3%bMbSj`hxJQr%qhDo=w9@2%6>P_Srscg{80>(}Tv8B|8r; zC_6vJU+o1YZ(eNHHm)}dQjsQ2&N*?p#0BY2rx2v2%%~7GA!S?S5yg1&`cfxccG>n% zPv2z3LSBl>E>M_nN>zYPoTJ-gUp4;#XfBSn+xpntBijC6ETuN< zR|xL-iHx#$3rHAJPI3Ih1D+48f7_+m3T=;w{TSqVtsM?dfSpQE&V22a44i?u)87KL zE}iR7hF=?D{>Y0dhsu$qEV$x<3nyYm#RoXS;1HaR^ISRSlhZqrbai2ba`g6X-wh)& zlOAV_6qKO|WpS34(E104vY>(dszA~>^AX=QZbk0bY=a%U>fD!bag@3tv=p_r+P?;r z5w=_*I>^RUH~`}~={}o(rC=f(M5XA-W;+UaYQv#v)U)ZVX&D;|ExxNPGTZWH>K8e3 zJVz4w{o`S2OMZL3AsCNfpdv*Okp4<{WVJqOl$zM z+BA+Af`FxY)I7vv7|Cun)!n`wUN8MaW?i1d#HFn-IaUV z5Se|pnZlcG(4?}Kk&Nn7XI8b0<0~1?anlyLDiOC-W*-6iP08}5E*p=-i++k>C7DN4 zo*`jDD?)pymqrP|2}s=5*YIPbru5fWKW}~%c;V}Wx1i5$(613CYFnfMwxo~}b$M}& z9kW$`hnEQb94#(iVW;+$jh3RZH7nQiPI&kX5yq_qDJvP+0o!};Nc9B!=S$iqwGg6x zy|A^y()01ac!_?;5xHZmR!=0;3?E|uUczNBP@CtAFtVdMo^ z0|)A^y@7n{nfCsgu*Xz}p(N!oUSVzxK!TON`<@F21f6JU#c~i6_^Tu6D}9&Y{fQRI zH!={@Npj-y){v)_&VoQXwI4ChlmMS8$OeA%{h~j!_d>MlBtL4nO&KAjT$)HjgdZ}> zjtaNnAw&;P!&Jwc+|vx;)8vT-d>*lMOU_(ww>v$NX-Yf^R|)29w^B2fS!gB2D;<)7 zw)`E$=S@k!^(=I(S?;%6bCDOQ+$~xm#%5|kH z(5gp;zX?bwK3o8Ju9zq#S4TXfk-KWmK3`6Bdunp@rF5H>)2)uE1=x|*;9xfFcE~tVlD4os7LF7WalQ@-&MNs6;&;QouY(MA zeTfm{OhR6RB16bQ(g7d==52fvq%5ya`bG{1>RXM8HiI_795{y%rkZgpdH4>Xa@l#M zVL98#>Jmml0E(kZv`Rd#q>A;nRfnnCL9=3jAN2B zRqLPa_${NS<+IXKZLhLg1iL9-I!dstJlmzDWB_reZuzI#lW@C6a@ix{q$gm=N|&MA zv!*d#z@timEpNY-AY*)IG*y<*)AkuJ+^lFs3xhG#+AZ-WL~$W!gee8&sbFLH^n;wA zO6F7K&W=1*i92fN~(>s_O!tmz-dY=9NE z9CKRvD>s?j-o7s4fjDQDH;vm8*TTi zA{PcPO)b{mY?l$C2}?w&Iq#m?{ObM@m-lY8CdypR6mJo{{SICA^fOh=8=fWINXgnsfcHB zzV!zcK}Q7bQ;@;P$KMq;z=`-ysXBM5_>#1MGrd4VNjTUHcBxqDQ8?3{{L(anPPK4S ze~5uZe$-M^l7A|QLR2>u(vys23N#xeS9H~|S85T-gy#qKpp~d_bMKnsNhi!{k`=JD zJ(XaZl!no8=_=-8%}FLO#&AC>>5LBer1x3{k)9b(+w`w#{{Rh17|PBKaL2|lKTau@S~ZcYZ#e`| zuE#r&Fa=B&FgV-vrk23Y^+@Iln;_g{{52aYHl6XGTCzI9PI1T5fRMo>7^=rT9hmYm z05lFpJLaEeKBJ8^X^b<14G`3Dk_a2)nxgd@D)J366-0g-kY+akjwwvGI3kT{p`41# z)DfKQLL+BzGf#+sk)A%3V+cvYig*-a8b;Wx^BnS#8X;h310%H|$fE}dCZ-hFBS`JV zGFnd@=bDuAk#5LGA_Z@c(wG+VPU#zZ)ahU-@%dL%7!$CrXOJ-=k!}MA6tCH7Jf|kC z4Z_a)M*h_4hqgB3(ucS@R8|*i1abOP4qEUrq*buP17Ik3p<|J^(y1JjG9hi2@S+e7 z6;hI|#kK}ie?93;=w(?0j`T|3OX(>I;BlJteKuPQHNF$2S`aQD3Uwhw;1R!`tdnCt z+o#5G=^Nsz_X%m2G)8eaNauk-bPdvWxHl!Z+~D@49HYWf3M6;iWBF12Qymuw4v1=P z#^R4LBsAtv8d9U?r;#Qtsn&{HVn|5}(wB7U`(vNI9n>9D7Uu~LL5L#CXca5Yf`jN(+7mlYMZ4yP1b3r+^<)YcR91RC$crP`$2V$6}#_Jx$bmQ^84 zr7L$(&Xp~*qfqRk5PMYNCkQr*O(`@~dQ;%F`u^*zqBxN3R|ACs#$HlA2yD0xah_5~ zt}E#8guem$TGMc0i{wj-swq%H-)RUrUdl-f>rqawFhYKuRjruTo|t&7Hi&llX!182 z5?%4VRFV>(IOk4IeR&nEbjMQk)t1(j+pe+Qj^Ng{7u;pdB<`glSRo^xPAi)xX=axh zJrs^Ljy$5a6yL)>owjuK$}B5vR$DMug)6}1gdGP~axihgHTj9vy%y_TNn`4V;uM>m zu)6A6I+R^Y%Vl65#1NeS0L&}(>dL5BSjv$O=K^~YTP>;9W-WLa#s&wJobF0%^TXp` zN4wIpZ#_QrH>1hvxzi;u;q$olvQbLXusGDqt+a4C8nc65pP|OLiFsN$^GW`<;D@nv zyQpM#q`#h*>Xm~HjTSS}^K?>X46Wi(Fti0Y~)| z7!Yg^D%l)Qb|7Uw3oWFm`7)I_q=SK_Dp+h_6+6dz6585bpDto1I~7sVqFCcz31m$Yy%yf@H z>GGjk;>hlav|R2iMrFw8mej>KsEAu(3WAhIWwaef%%8-Pq5&sK@0yRFe7N+?w(N0k zQdAg>%xMonP)f_JIzV+G{uL;UC;{A^ytQ|;Po9?Le8f1@Au#jG(HXx1#}e=uY-%|H zDNBb>Do%oT=Cf;oBK_CV;=d9*vTl+PjHHYtHWZzCJd#{GhTA6_k(y-H#Q^l(A~N%$}0}75u4VK+iocQZp( z(iD>Bq~k{5q=Uf}M^tpAIM)Fqqs;iNMN&}>{FE#xek}zgtqqgDl>$9LtZU&7wNd)6 zY@B8rOt#Q2Em$gAfyVrh0*~IlsflA|d;uX!LKoqZm8&`c;O86f+iI`+m#KG2B)-e- z{YiQ#admC}?8UC15jvK{;$M*(8dUb7a(XLC@aHy^5}%liKxM;_qO1fGe90}jmle5u z9k_KLSi_%rwOgICGW#rormje73wKYMD?`9|$=nQ_x|{%|xy@wv4xF{#y5b#@I)c*L zR_w*NtTxy|)5sB(pxU0cKecLAOA_0jY- z^GC)%gsN+#nK_NNY0Y@Yb_^sYA#QvD-t7B8*-kXm(NJ^CzprkaE zf^?^CIA5VTPC(+nJbKFMZM929zSm@3sm7E^9tsd#17M-3c%Hf7RF_BmM!ZCIq_-tX z#!^zS3KgEpPB3excy)0mvR;|?KgA2=exQQMcjj*HFar`Jt+Y0@0CEc4@dmNn0H8VB zH9+YbG)Y&MS%D?yn$~uEDAAtYLVvAPU%kqm3&amP(pH_)!%0hc$8`FANgqnT{47e| zWphkGP|)EFrA0({&}uWa5u#dp^~elFMX9MQ$CTS(1uT`NUbym;`JYP1Zpf0$YQ%&q zPWVblKc>}w>T9R%NeV;XGL=RJ4YsK5wh|8`%4;^i$dHmkrRhl+(v=aB?geJJ#Lic; zyCvnV%&7?{JM)j$nq*~r(=M%UJ)9g5QcXz1jHQ4``sSk{8~F@?!BHD#tgs@Dwg+*| zM_b{Lp|K*ZcF0-MHlzXxiyH#~f2B&tkAdabQ_&!#Bm+`R!brfzDQLh+kKdE;O}fzo zH8b*u8x81b5y==SKT10_5N{2;(`Y$W9Q1Gw>HR6Tc%iJRtsZA zBx5_&R3s<=07j?n=*a-~JXI2)7CTA?pbcE@Bqtl@srLT>FMrakmgn@5zZFr7)w$Gw z1x%sDY2;N>1c87-six!tk>;ad4SB$GKu93u*PVQ0ni1&Fq=zy||}a zY<3jtTgLdVFLzr!-vc3{2NdEO$-o&D@*Bv;1w5AZRArbcv>~=}+jH$iw$TGLrrRTt z^rG5lSMc%eNV3>$Lv3fa^zVbe6jL!2NZ8Rr8Ne7k(BLj*xh#-^Yq*7>jphn{03=OH_7#?xONTMDF zam6|EqqR3CQlEk(!x-F9l4D^cV~%tBRfNaD;LxweAH)Z>JR4&osJQDc=}HfpkB@Fr zzY}1C>qZs|To^~^UVZ6pqX__vSFPz@&WI`)a?pe+E7UjMq^;7G3>0MXwLtju+OhtA9tsQk_$a-10IxsK1Es zFoIL92sr2P(sFiDX%!oLZb+5#&CH71s>XbGG^CHND)#S8v~0?VyR@$nO_eREZ6vfw z8){fkJNwpEWuH1k{{RK1*m1VisH?_XYAMeNR^vZv`Uj=DX}6-Hd}-=zV+z)^zNHb^ zsHhS40+Th-NNJxQb^idv`}BLtybElUG6_&a%2qZQ$x=WE?fKM~!`_~~+x!<~O|!Wk zQOcuh!|6J=0a{CeM<30D`d8K6MbH!O0ZNRZ@mTVqN%05QNhAuO-8wTu!;P^GwYG7o z=a7`6d=QX?rw7<%``4cH$v!P;>R8j*b-fWAkd~$1Os5%W1xJhEIWdm<9Nj7-7~G8i z09v!!y3#B;X|p(DA|jRI;q@eysBxVNDs5h>&fcc7gC64*GoOT^mq$;lby;-xsrZ4{3erf^G8k!0vk!l0~tsg4DYp0bT7vKyXnh{ zEi>fDy4jo>5E^x1Nk|@CD`biBhtxhW{VO%H>pmG2h3dX|#;{es#KgNE8>U6f+{$?YpNCqUf!j#zq@TI_Tq~{y% zYw3Q!_zCd);`|mmucha)Yw9;z3Pgxbq@mn)T0qjQZ@NZ%8u`!S7xrQJlhZ4HOum+f zdW#rpT)1E|;**VvkdwNPY>My5{V%O*b9^WLnavfK57QOOKiw7wJ}2~#OWGQCWz)ex z%ZMzsl@*kflpJR%!9BgrWY2XS^RI61Pd+2WS$;P#)Y?5<jq(^z zlpKGDBQ$Y(Ur*_uxhba2DdoxZ#x#SvU#sFk_~+A?IBu8OwM;Oikdfgh#2P$3c_BpM zN7Ey0<2CwXBsFsBf1X;vLYqp`NWmZ=AH(mrKBs|SmkOT)p}IyFMk+f^OTau93PK_fuJEgn<*a4h+dh9&!^-Pgp z`94)0UXL8-uZC4Fkmh{-(TF}ANUbaw=}E$zfA^YIfq()^%9Ft=NLQe$z0O9fJZYvA zF^poE?YJ0Ef|ggQVZ8E;nQAwf7%{5Ti`U_AUeKAoL;F|Krq<;EjS zw5F`dY+NKSEvar4G}4kn8fcxtQjmv^LCGn_N_1o?i1?|~kBKp#M5kSF4-AhGtt6Et zJf!8-0(Ss~cBz+ba_!bwRwoh+ZAmI=andxolC4Ww?Uby9@}iF%kuUniv| zJ-H64wKUldX*k~$CNmx$GCuE%GCmWvQn*8k+M0F*l`^a?`>y4sZfMz7X#v8m%1~9YPvHP= zR0Vq<>DeQZxqKdfs>LjEQGQJa;744wVpr@t!>DzpR?|peyi$BnqNKZ#z6inR8w&ad zsw}ZDmdA^PHzCrJoG-@6T6YII7|w8>PIs@4JZp?=nxyqTr7CrCl9r!>Q*L!3Ng7r| zXj4uMauk7`@!kfW=TM>*r4YO(dr*4NR!KFlc(6Fo^~sPF@REQu>~ z@E|siF(mf%8TYJ$C&a&q_ooayZ1(KPRucN(Uy9ntBzeG2c-*LQR$hg%{KB7aqbuFTswN zVQG7;dgb30_mEN>fo{mw;y^vY(2RaBIT;QEU!a!mn}yL`Cj+OgF~mkKaZwGIT^g1W zl!T2bISJz=>^bJK?}UF3<9sB8E3JCA`wCxDU4MBpWXF>nwgN^HHLDFJBVmEC$I^r9 z--|Ze)PyX)TY%9~MrNTbNm7Oa{%oZQZ*2EbJ8_z}T=8jA`d37uiQRd9m#(GiNmkIL z-z7qf9Z7AaGj5WE6^)Lk6f}dl-zV%V9qSK>_L&Sj5-dr{kdRwT%t*oVk*N7l;Xa&k zRr{w|!RgtEW(@nNOweDkNd#vFK_j*{ ztb)%D99yGoNlS}xAN)W9PuOqzR*fN zDwqqIdH_}LPxE7^Kj`73Io${YVs|WfIwS4Q-7fBasEIAe+aZf8sw%ljx2lG3M=4z!(veL6Nz0Hd{bFM^!<0w+jTO)pU864uO zciU`qFczlJl`DK4X0OxYH!(-RC|;bCDsySH%4xd z+v+MA-+n4CKs=05&M4`{0V_L@c>cA+lcSzJ#S4-zV0kp-0q^aY4n7R&WJat?<=>k+G`c ze7Xu&afq46mw$ROxIOgqQgJQiKoSlhhX zr61wr+J`ohKZ3M%SaaKo9d;eR4|*T%*|?eUvy;L|q#II!$rYfg!;Q)~q$15PM= z3@K4WZq`OJj`ZqnXBa3ym0d-MpWxf+PNv2&_&*_A*kMsA3ALPl98;*Z?}D6fwOxN= zV15pJ;*^U%4oY$Kt?V`qET*R0ry5hYu%}T4Tol#7+46Qrw)M~0;DO{nTvC6xVc=EU zw{M&pDX1z00Fp6Q1lmac98>Mu$T-uQKETQhfUNM7L18FCbOgn8TABR4a@xwzTQI~EdUcR-}TQw&tJ5`!Y5L7@$2==J? z5u_!x5~2nxzXUkDDwP$bGa4iNr#s_*X!)%NA?s-%Z9qQ4%#0Dis|-7A+oWYiw#8HH zLUk;RB@L(fmbGChPr8T|=yG&XXs+4sYlQh{&CWAWop%m(?bJuN#eQsuPwv=h3R{jJ zK$L<{+OP0>H>ct$6K~feDGNSS#&_l76Y29SNn77W8&~||r24n1a}- zDJ-_8T37tTQFZVBtA#({tzRbP@MV1?@VBX(Z7+35vvll4I0Dx$V~UWB`n1s@KSv5r zwQP1Cr|1rtjq*0B^Dgf;Fyx6#G1*uAxdB9bWQzIH*>j)NokoFjxj!YwloHBZAU4Q9 z!rN&92p{r+T0E|rCfZqQ=(ws^D{*Hz_N2)w(nTP-ZL>_c^?YlbcXT1o45I-EkcMOF z_tF6Sl4~-)-kEV4U49cwa|mWSA8Gybx0TET$id_5Qe8aD%*S#A1yT|bhb20nDuy;b zbztEC0EV-!v4?rI$dDtdN@6ICx?E@_N8bTBA8*dQ%)hGScENo#jV8r6sBI0kZjo>k z{$qcZVmnR@_6r1{Ap8DawE@*}qg%W;WZ31}+6yjdinkE8C>@j=LX-jLjFa1X%HTM} zCS1BiR2eGC(x+hSLKU6ztb`o;l?t?3`irFBf&=zh3!SA!*Wuh?G|IF1fIuqJIQ}9@ z?NhDUN+p}M+2u@vmhSCGo?QuJ!hOU^aB`#;(`m>wHm{DnJ(Of=z2IBN>p*>{n1L)K-)Q2 z)3r%jFVgyDO|^2f^uwqT40sZ*&Ce1`qzx^%p)HoicilwqkWE(>g~@(F`ZKQYT{PUN z?Dc4Ax00n4DfS!kqzo-6DoTR1gPbiu5Dw(l#qj&%lM#WG$$=G5CP>Kx zQt{z~@2rzpwEm%k)M$80mu_NGG$KkZ;ciD$gE)B&tpm)Vk(8?>gOGEYkAC5Cf*HBY zn|HZH0WCb_)}2xr2{;}Sf^?vqfrVg>{*=;bx)DbEKE1R2Rrq6iNRJVldmb}MQX5c5 zh*EzMI6AOB>pA%2eChLJTXvSx+=pB!Y%QdrF6487HtaPW_xc*gVoqXgUk=`=vRqSb zC2dGlW^j{}mt1izww8e0ft9HL01!Ae$$e;|!}jK_(3_GHmK$;mbviLOPGCwFI_^@Lp0>g)KTpPBFIHv+7UYwx5O8 zz}mvnOBw-bBr9};x%5O+QZvKEdeV0nTauh>vQF z1ZK-AZ3!w`Op>i?BL^c;JY*2Ae%KdFgovxPqSVaQwBbq;mk^9N0Mt&Bk_p)=PW)#) z*Po6{gG{90`KF3xy3pgWSeo2fDp?9chC^Yr5LB%V>eYo5lC>u}0373UQQbti@7r96 zqar)ewRm78lC4R?&=jPcCzG7_BDD0mY?Ekh*QdB~YcSzg=xfmPP zL)LF5(ENyY_Fqsa@YJI44U&+Y0#xP{p`G`}TOb^nlY8O&yzkn-ES@1oU+{% z@5{Dx8DU;j60?E;I7!j~$m9y3SU+jaLs8>IOoA2TXnCOEQifnK&4SA9s}7R16)d6L@STn^zS?=L)1qfP7U12=&woryfo;sTnF&_D zE7CTIk>jN{^oA$Ss#r z8*_9mDH_sFoQ^6o;d_Gg*OfD+MYJi9w|+_*qq6ZMNpoL_3RQ%R6(`r6=DJ*ydwqQD z=Hlpq>OY7bZPZuRAYQtuaidA_-Yqe2T4Ow<`I3byLc9+ElO|zL?+#8IAz{$6 zGx$v%+A5c8Ght}7qBC+IdnHQ?2z3DQg|ecPo_Idupg;Ck0&~i4dRlR0 zoN{8I@T7990C*4A=ww7bZK_EEF6XB3K1uESq{{S!0Ri$Y{S@GSD6{P7d zke4(cMIFwcQ^@o*lgT9M1~FB#6X7SqJA4<|l>@_0h3s9Bk~I(E!mvM6>}zwrS^APJ z9|5GzWsM)o5|!=zD%=o#Gg18m5$LeoT*zo?r47)eY0`H5KzUEpbDYtWXWk%4b#~=V zf)ANrv!({}J#YZe(B_*GQg-_nCC65065>8mTAvv#xCfQTeERqH9+ev9v9(|}TXjHS zlmm~}v{iLElH!Hh*Jrwn;F)m)GBfVBi~vv;yPz|s*uKj-_mK>I4G zC?-PLoyN0ec__|JO~SF`PnUYT#^^*nwVwHCxRbbT5^d5j~S&T5$s68`x;+w>>EXT6`YJ6CZ{6XFamT`lvqFy!%~sr3=ODp84(D!r_!Hp z+%x!d_NlbQ0B(^>5yzztA+jkJ@Blh^q`d(+kJWPzUhQky{sY*FHB9g4tC+f=I& z(g!pX1ScF~q#9VG#KCPN0*q-pXYWBF+;^fHdm9x}@c>Xj?kTmWBWmx8F}*b7*v>Kc zrE&NGJchHdH5Dc(!60r$OD({$oQe_Q*bD3kBz)C;>Z;JPQ zC#P*}c%UejzyQK(UNyHk%703eiv$g{;-%w28+mKr!+@$(M6+t(Y5gdqTJiil)cY_} z0Lo~V;HT2N5nM1wprG&?iixwQjTE719-jcyKrFwdT8ts+ zKPusc+$2`^4Mkt}Kt!mYjYc(z*LF9n<<%m*mIt z@$Fs_2cZ0@dmm#_TK$wUvTLH*zv1WFtY#QHX|H^qPH|h)>K+cNTAwxL_{HgRb=gF^0CvFL$-(%Gj-k7n8 zooFLlTL>XQt$l@ScZjp>tn0Rj+L8uV@!eldv-YYdPO|N`>}0f7l2lFyCarEmpFUHv zF0Xic##;(bcJ>uHzXM~@TXbBcfcx>-QoO_~!ql#L8ncS0bz4`9(pien&L!k*RHZf$ z4tC%vK>LweTqOP`B+be=8rlon*j0Y_I@;8Qwt%-%kKtP=&^^Kc-|t*mBAMBaTufR` zj^?M-+_hZgsRweUwxj|(`O}Z{se2uSNOh!7ea1^(WUUs&4FmpmlrnMaT3VX6U6}`4 ziE|!T9%&(IKA|HzXsaJXS`(^U5hBcRkTUzL)K%(2Ml<@?BhM&ybo**(sQjnww-a); z-|kVv4X6Ui#-e`~ol8nUJ;uMT!l9#e9fDNVLz>IcBlutMOLjt=QqDZ=#-mkC~aA9jdwBolGGhtE_ zCDpo=ZLrIUXdze|4PX#918TD|@DHcb>uthq;Gp1HX(5#%D($0E0^8>vQ<||#>Dw?>7TC!+Bphc>K%&yf?IS5DF3g3d?enC@W*i~#0A*aMxb{{RviVmjQi^vhQW47L*CM8zReJVcUI^Ti7#z<-BP zR@oypyWROI{`?1Fm2?fLkR?GDGES1R&tT4TalFZ&L#8_arqJ4d^)uh zx~R|^Y-?JMuM9MuKmeR)&Pqppsiu@&kA>f7&lXEWi)xiB>guExoKn^h6{t1Fl#PKW zow>*ct^Ff?m1`F0*9%kWeo{$Fn^KT+yaAPNMiLI=a7Y{0vp3<3whLFz%D0^A3scT% zcS2M_&nh`cCv_blas_2wQ5toPF&(|IGV{xIuMbK>aIeIHoPa$BKD7ztkEnlht|{V4 z_%|Mvy0s;dAt`yaINO1=f(l%0DL;n=L?~l?ng&I8MPm zj!DVef30F232=zpEV$!MAt^{uQb-GMa6v!B0KhxsVAaiu-X+yWG`MaMu$GjCD6Pjq zI$Bmo;Lb+FvC66t#VG7)&N9<4MpGg26T%@%QdSr!)U7B4D?Z0yduJ7ud{pZTbXae> z4W^wS5TvO?Iyt}~jX5OozDeG!^6u@m8cAtL&Z6=ZwE|a!A1Fe%DFZm*V;$=j_``X9 z>v4vudZjwBoRh0Y2mpD3&@gg9!OD(!uBc*iXy$WBt6aLK<8ik+JX%Dgkn)2FN}N{X z@TXu$aezGr{pwRJQK#7=bv3mpisFz|!)he2Ac3Hc5<$+LvPe8uk?`lHZZFME@!8C? z0%8*!r%DuD3BfB$9KuN786@+vPT18O_&;cp+V34M#CIgTrFaW5QKuyS5wK2Fa7YQj zPQx1terptykAIp69&xF3Xa4{QCd-(n;9TR)eYGo6j94gJPC7sscZUW-0R-$yLB^!- za(>ThX~<05`lU_Dwy9nRooa}n6|2J~$FSH0t1Yb~2O|T^RfakVeKFLI#=1mPjwL~n zsV^ZRE*ynN@ZC;)`{_D&$E{k!QqggZ9&CA?M;$12D?>5X!;xD;ib{}QZwfk-g>R?~ zCm?~Il+>Na$;mY1*b;8HMj^D^?yHSxpW--tD_y`%Y^6&it3Y3f!8`1bq=G=kDwiP_ zmT!xc$&q+*DpJ~H#&IfbymFAFw1N-Oe5+&4Snto@Al_EbNm;`!f~S<`A%FsaDJXLa zI7s$E=Q^`k-NsK>i^8T!4o?LqGGt0p!-HN%>On#j4`GaYZ%+%0**P@VWs>REKM+?k zQfGAYa+`FPn_Dpw!hjhFNpA}Z#`xnQBXLs}uAQ|_3hF!0PuykC3?w@L0L-)_FrWYl z($lMO9D(If81(H>cHW`tGRmDrhSwqGf`x*^DocYH^D7}rBXTz%oic}Jl7&4PxQ?Yj z>TWKT9rnt~$^QUe^-1T7jbHR79B%Ams~@|q#m9?lgoYL5*A1mK>wiO?{^_dinkDKi z#IyTICA8p_$$6}e_8~yz9sP}5uDxq~wnuFUeqm)u^7v?H83Sx;2Pf(JRVq#Pbm$KP=`)xXKCswM=tTjf&#aqeHZHt1vDKRneB*lF9={Qczi2i35VPjSh4C zX;yuE)yJZxE2u6ikS%4l+F5M4&`3+D?d_jYpUbv0V(3e??&IR6wL@rtqy@M&0qL*+ zkG9pl^evL%)5=6qZfR=gQe5!$D0c@sPJYIw+MB42$BMRW_U4&|YiioTaytAXmXh*z zPy=Sk_4M`_^r|$weEXBdrN{xmbNiP_C+Ym(l}X(0@)FV+S|me#pa>w8IN2M56}Tgg z#*y+3+!|{09fXG5SMS?w1o&@bay0FwUx!8E}3jI8P)mbNSQgXaV#cskn&FHytYgDIkwRGAeS>cf4AW;M?w!S!D9hiK(OY zZCOG5sx`b0-KqA9L86r!iz62Z6A1mKWsj)x^ds`?`sv-X`3jR}+i#Eow_TenJTomm z1D`;wlq>%LFsi(Fa6lA$Pk=cUpWGE)64I^Gp({+d&N6#zP67HJ4KSnxjfHY0=42%J zQKh(s)ysLyXX=xY`cl_(i6KBqj+T^v;xL>Kx;Xr?K}81Nkaeg5x93bPzcObEuIqEudf& zRP-Bg)Coz+^`e-!okPpP%}m3IjBQVg(eI&@ens1%07sNnmJB{Np&-Qoi~+S+Au)gk zE7DE!(6jA{6 zu9SiXV)n&0sGij)pc?UP8r)1nsu}JnwL|u)l{7sm3S@T_IE02VHsX?E)T){SkXjy; zD8yln?MeA>K~iP0TrGjxw-gdA#?IKJ7{fWmNv!~OriuZf#K0df>t6X$&eV5?4JacX zmAIIX`EOBO5rO1;RF8%aYEWy`P~s9eMQ0JzuN<2(Pn9-6p zjHMq{uW!f?^QIqd#U!BtEhPT{3u)7zspS53GvEW-nst$+?LvT&Q<((`Nx}#DRa*5F zoTPE+D*13=kZ>xcc|kxs5^8c(hmU4?H7IG)l@W~8Wx?p>IZ8%-sE(vlh;)&jY9*Fd z>t|Ny-nrX4BApSg$5f|sk%WWIKCc2Y0||2~R@D6)n4 zn{^FaBQ}6G30#8OLO|dqxt=$d!+)Q#bv_-n`TqbVhTdO_?}m#hLr!29Buil`Qgg-% zvPty)D(P*TEFkHC^9a-eP|{ieaq6%=^H~&oC4$u@Nx1w*=*W%}rAmSGS4V;Gg)qrv zeV#vBA_qwH71iAq7rO7NrWIhDH%zmP`#AHi1b@t=ZA%|Yqb;re>;C|wODmH90H^-| z!{~cd6x`N=A}PlZHDT`{2R!OXKc_vbw`dk8QTrnE1xZV*IMWomganL-6PC`BJCdXN z;;KIlBFO2<$hIDlMXAC8D!p|~SEealvePJUNk05miFUHexR+mM?8_1sgSEUV#W$15 zSyDjHAB#29?bLL?Ke5fr>hJUWn6Fu}(y%q!;o5pp z>ohs>l8I1KUTh@*KHEu73XogxosObN{(P*DDdFq{#n z;4CRboj4@p9QLZ>dIzum4D@RjE{}%Q+FO#smYEU+e+i`@Hrfz00lrjLcmM%hv&5fr z{{RPMGA@!dPO`MmzCvOQRwlCupffP#D=U*ThCVGH2_xcETP$QS z+v;{g76SAf4>#h-J7gWW=Lfx8Z2tfntonvy%`0S+wO(;*Glf@%^UmA2hZa@LP&fmQrw3>Q|5nQndHA6+Cs_PJcrg)^mZC*aLqS8#l`TVAR&o4XdR1|#EPR}5{`>xfb3PN| zrFHf+%Bw{4b~&6v?@D`S#K%Mq_{DRV2>&Cf)$Kyn%K-~R%mS{qV!1d z$XXQSu(0ZXn@C8)a!$%4-0fE>)@Ljh0|ws>w%m^yC}<(Xl&eS@TS{_~gN=bYM{i2Z zWx4Txs5)ljDT_KRQkqJQxlF7$*;w-a@TFJ;@8X86jt%x)dFJm#Nlc<04yoP-pd0iIRH z-1e)lO7wrm$=xFN`(Y*v*A`eUtcc@69M9s=Fx#a=azP{>IjIhe_%rbXqk3XuWy`l- zuFs*xGcr;nC(KS#j3v?vaoD77^s2@Wd|&b;vPFKPmd8t6>}@7QzM8d8MAvi&u52R& z9mxtwSBMp{Bk8qQEt#Vpk20hv}=_hY}RrcwIyCt~hwY66tK`BVaT1 z&0aWVB~AYTCnT~*mXU-x)9_*?FD5D;PMm7g0*C%#zvoeM3?&sXWiZOY)1hceNIw4n z_N$LkUphi7%5SZH1$DPa$j;78e&X9JAK`fg#He%s0101uiGgeQTYVwBY+K}nugn^W z*AP~;dj_Xl*k9QysY}u|{{ZO}j!j)Z?fyzc`#rqK@TEjS*M)+N9bWlQKT;`8^ixvQ zhStN#3Lv!Z6M{X+l${(p5v&$KDRPJ{5 zOe<`aDJb7Ufdpf?P^T}StHIdDiP%~f}=q!Z`= z04?YTud2><3MkfFm2pmg6(LtNox-@PQtxE%u9;QrjNy65tq@sua^gAFz@(<)Kbolu zz8svdk9z36wdH?M_YrD#zZlX0N9phcmye8rM*qX;Ykc&1wyrA}$CR z^8rsHdwl92oo9&ZEO4x5iEpS1j&*I#0~g<(MsgMU*FJD4ZmOheBk?TOEaz;0E8)E z(lfs_F@N9nr1hmK69nDj^g{^5&B9>hSCP#e)twVA_ z^r+YfQ6w)Uind(s?dnk_KSF9U6~0ri6vMX4l0n=@j<-KOAlD4Lqp5UP?yyjzJVE6& zixfL}m=Z!%2?uJn%jo{Byf~N;x^7{Rv8fFm2w5JwRG>f}ybkn^p6P21iEl5fddk-s z_Eb`+R>gkMm!9CMpjmIzb*TRUhMhu9bO$7~3)NAfTcXCB9j_fgf)EY?J+&Ug)`(9| z^;x{Jb6)#_t5wmhS_ z(!B9i3unZxs_K|avpac{37|6jP?DHTY4q^e2ub$HHC&Nsb*qz-qOc0c(%M^20#Z8+ zbLvkd0M=YvbNwK7+~33U-yjkH%$Dxh{qo>rZ#rq zmlsuG)E^J#NL|8bPha|qKshryo2V73N7uuSkfHwo-z)1?`|O5oP|BpZmA>yK!^hgG zf%jy(x7qTr*vgzfRF7JgPgN-@ul@f3XQI%<71#Lw4QZaW{hWnA)15if7Kb;|VA+~= z<$Yh3<2(5gQF8j%uCDEPy;0V;sWG2)Hq=om=TfqEOKr3XN|bmBN-6$<_)FJaM-X1F zO{&La@Vs=&nATl+pXX(j6y*p102S4=&;I}uwP>9?d+956(XN{KX58D=+reJ9tq-6{ zTVQ_iZ{3uTh|Gcb#5U4^HdK`j=_8Lk)ydRZu?jXDN00#Sf6bInmlSbRX}`FA$u*qEIH6QO8z z^%OX;2P*R@=LBIR+dI{zpNKbW;|5KxA`CR7DQYKDmPx`gdDKoeIX{&uRm%S4H{-c( zW-DzdHl?csrPY2V41W?vLQXOcrDtr{25+hTCtfGV_3W>)@%ChhZal7xk@GF6c#O8o zZ@SUkrLqG50K7hhNyc|p#~LS1d_=h3bjM?+BtStxFUXejX+wD%r6D=W^RXj1<29b! zZoU}Mo||+b4mgB|7lo->K{!G{8|+R_*-lTan;m9iup3R0Ek{WhaY`g86M}i)f(LJE zhAgs-xi(WQAt>=kE8_nEK=kj#Yg*#mn+~lAP-VomkkRne0z*_aQtTOx3?WF=3X*mhDJMzW8Q_}z0_#4!xZD|i7RQ~AQqt6)6|dcF zG86`ND0zGHfym8#XIB<2b_Qj+yb-d$}{O+-=)dQS1yk; zG+s6IvbX(`eifgI9dxd<>UFgm&0q-d+J7NWy1heK2ZW6~k`I4M{SCF4mbih7a|lrl z0+$F5cz_NFPvSm=bL*P;$KkvbTx`rH-0MkoJ{!fxmK#uT;YAy8Hd0Q<8qUWX?cF;! zsI4tVh^cK#X$ZhBtcO;E=15QnN{HNNVhGrfK&_f`Qqt%Uik90n`JH*zD@>+Yu{K0R z>OvIxNRZw#trA;Qj|PG7d9V z74`M?FQOz!DU{YTC4MPRj187Ex`-TujE?(M9nxP?+$W{Tkz+d*u!K7u}B<0Y=AKfDD9SDhz2Xz>Wncg6%fNK#=)OzG%!$%eGDR~!Yp z$SD9UAzoLLk1CQ>l2fNi%5Z1yvwEJ@W=B!F+zY*`HAw(QKz&!clxCP~DG$rTk%rq9Je1|5{{T21_^dnPO`_G*)0>6Y z@uc*1&PEji=C(w;HOVA?FBPCp{{YUA;2Q;CR+^uUZOG~2arvjw9IxdQaPgFnlzfba z-cSlHr9f&)Qe13+a+Ls}D@(sYZFd_pDmJ%}HQphyZA7SUp+iz+PO_4wooPuR{L}?- zb$~O0-wbicyZ)yOl%V(hiWD19#7~9s*^)2UyHp3xn3r2M-PK!U{{V-?Wo43mlmdrw z=_H=F>IpiL)pqL-Q#O!F&DO@kq!jvxqlTS7T}0ITOf%CBvwrEGqmR+7-am5O#%3z% z(pQiEAIuc3OdnPzHoZO8kxbnMnlth3zYuylUqnxklH@C`m9aL@IlufwG8S@=@kMoM zANWIxe`ax7Zod6~e(3?_Pqq2^@W$Q zny}jKeI3*-w-VXZwlyWy7bEMQ?d!&)?RvTYZ-uiAMQSDttJ{d}IK3+oGN&282 zKc*=?2P*w+Kx!S@WM^61EWV_NTOF2>(6K)J%trnbT#K>q-t%9XxbdNtL$#CzkmNoha(z2zUeC!7gPf)n*|w2$#t zdDT{H-%h;JuNK!MGc0P0Bg*7?DLvYG1h(h>Dn9tDCj5(0@L2_es%`8ZqO`Eh`qvzi z?At@aTM16!rR1mgtmI@L6$r*^8dSzxaV|u2#EkkwtS>p#gTOfMI6mU5ur70LliIr5 zCrN`nkM88fc}%>6?6eS}^vztNb*vtz)eEkyNS`4a9(|1%~EbL&s`%M;P!p{O!a!Q)^SSx7u zUUsN2no!|il`B+bvT#ko$xyB;j7YBN;3Vp%9J?z{>-MPEUO3qO=vNk_zH>z&GC|9#xk}9h^~q{K zi|f4v${!uYCXpZCnQkLdgiqP$wgiWBxblTOVzrz zs6}cgpU%9T_rdR>Wp zSucs3)gwh_>!~`mY~LqKvdUaVtBKr?Z`#S(@`r8<-faVZ{~)2N`FX|O5u;C7^O#dHHqv|;qDaZe@9 z^O_JPM}8^54;#~1C^@%>8K=_qMx|s^sB_zjHfki2@4YL;CWAtWr^jl*p~AZ1a+8g* zTq!4N#-odKYGrPArxcS-mi=o4ZAzw=s(iLbB9L0h$E6DhHotRS1e47;rU>myODEES zhW9+zfIQI&XFCvSVV`Oy8A#uXbpZ2AOFQvS2CzS^Ycb@VyVA3tt#d{zpd4{qW;zL_ zjGk#b*FmFTqsv2OB&jMV&=5abuU=H!Xi-Es-T*4&i| z4ui*ut0|6?>ZFyBe5n2*XuVuhXi|ldDiU}%2UCYr{{YEP4S&tMo`pPCdojN{+*l<< zl@gvh7>pCFuH!K*#G<%cCfY(KcqXa~0U1`+Js3<$3 zu@)nP`7##ML?7bBhrK>sTY@j27-inAc$JG?p$!(%6)qt^xgWx^grNZ+{5H~thw_t^ z7`csf&7M0KUbcj{Uy#xyTl#eiQ!mJALnwAqhnh5X3=o~3bRQ@}4~bKZmN`~6Z{_sA z}Z5=r+_L_Pua+x;0*o_9-j*aw2j>>$Dw4KQV005<^lc`lJ)px6ZO4?oM&V#zQIpD(J z+^?)3Bf)s_*oLmnIG_WM4c$r*+Hlf;c%JH&&9`1#?fsX+8-tBOE&l*ybErCi!*Q;G z=wz_-?yKbFIc+9GdYmB&i4l0z`j&L1=_kzku-Y`R0#+AMO^?%#tD=~cnI<}r?eXZu zI3-I#%3-#o=Tj`Jk~VeiL}e&$o3b?T!`l>opi=a8A5cbL?2rZ(ZnrBaa&P!udkSff z zQmwWKC`+O{gz~VLRQts4xAm27lTnwBw0<8^T&1b7({^aGZxWD3&CHUprkE$yDt;?i z^p2s$bWSm@2n(>|1;;VC zX*`^NF)Hn_$8(B{!vxzZvP!7AOK+%xA5$!)3M)!l2~Gy*d;!V8$DqinHP52&*Tugg z;KG#J-au`=T2GYWI6bm;9lcMjY&LuR8SaS1Ne{Asr8Y>)bH-0@DETbIZ8}y*mr8)n zH`@dXXemMcB;_RgG2Wf{e{iuRrHsb#3oa$lai9UGQTlm)!mLo?b!D;SCSREea0R7) zV3UEt?l#BTwEKHdq#;DOg(b$yR5{rH0Nj4H6L)clK_T`2?I=h&86yCIe^2XFe44hz z^&(MJ1Hn_ML-`>f%${lq8HVkHL+3}Rxpr|t`4J>1d-T< zf;}ov8g9xd$tq@v8trgQhNre%kcr`K?sqHD;VqHahZ%W3#RnT=phv~K_fExks_d;s z@gqq@S&XSayOzVKTAL1_`BaqzxP<~RNcF}O(%oRhxV=juw1WMHRDP*t19DqpOKwX@ z9)*=SulkmI)iz8gPhIs#!b}Z!bWOI}mUuJSTAzlK{{TALEf4rw30z3JGK-bYbaBViU-24j5g~H%@$kz z=3T1Eba|IoagCDwl5=2)*EWZBGE}6vj1cP!d9n|e8^L)!MLVeNeL>Rj(bTM^4E3i@ zLwPO5y|9cHZg5;aR5$~PafiqEWz?n3QQy|zbQE$cT4y|zpz8Ey&D{PYbeB?GZ7D#q zbz`AFGYq!V*3PG4Nz{<*Bq}q`Ek0)5NG-ew(o*3CcF%{sSrY3V);dm1PMLmdnVHhJ zQ}Z1;b9C+dw7yTtyiUM4p}xp{2MDOSNVIgU3wJ?vyytHn4LP?Y^q*3*6<;-MY`8;# zEgv?7ItpaKQ3`3$r8H8kc!2enOy2DZMZ#f>?8h5IoS6~8>|4B7K@87fz%c3@(xfk7 zb*Ib{wJ22FvPZAa&*;XezW)F}K0F$Ym0dj5;ZwRw=_ld$RWh|O^a*noh>kc8l{Va^ z`I_7j;K~IZyhB7@E)W%TcEj-xrlj=s=VdN+raDO|V$q6ysC-|FbM|Ud2@c#g*Ztw8=tFU@d7x)Tg*?rwEddU;@2S-h{Z!>F%Hh z_JT#Ceo92>j1b9Jqw&-DcjZlS-3F@|sJ}WZ@ov-PRWUBxUM@7EB)pYl853*{{ZI4a4Dxgr8)E>v>%ApKB2O?8c)2(t`(Guwa?;0!44fKWJ6gW98xDc`3J?aQ#aCfOo z4^dsZb@$%3Jl}^c0jaShHXe&LvBPh&e6A?S2pi1?^!=e=sYTi=5=F~iJ0jVne*LC0`L;EaL}K_hw%PGYB^bwh4jnCb4QjP1(O!gRz^ z2oo)!{{Z^4irAw&mKJ<7{5~4I)g67aJ!lL`YFkt7vvn#Q_5lsP6Zn!+e9gF=jcqH; zl1Ctg5ZAzLjasrXs)s4xf6koMk&o#~JfG5mlJYiDk?ZuP98WmS6h_P*^Ms$N#IIQG? zp||t3=tP9Z#PP6o5upKXs(Poa-$WDr!e zPik>2o@l=SdQ!tf(ue@VL{|z;7+dzFmU4KkU_AuWt+uT3PlkP|*4qPYRtSV7fGN}k zZLy~ijP|FJ-?d;3fNQ!*{ORfWX^fwQ!`9#TVdvQ;3Y6J|mga*bY>EJ%_zy07OOw z1=miN!PA^>qv(MwYkBm|n*{t6AzRa&oMv^Q_CLJQ%A{Bm-V|s9QTQy#PmG z_=wMQ&TB-3B0_}Qg?QU0ve9xXPz||&E0_MImm1PoY2_6WjlasLifKA2IN7L5y+>c* zl9HceOkzFhl}K_zV?k-eu;i7YP6Fz`3oW5OXP#H^w2jXvNZzsTm3HkdEl<8W`e4d& z#^A+e-raDU8 ztA)-|l!ha+q^D%0sY)tQ2|^H%RHdkdp(!d+024*G@lwOmF`T#bwe6YL7m`|^eq^X2 zNl?O=gOC)Kak8HvE#1OG8#rGy@#LP~qQ5_>cuFwR+bk~hqkRz|IP0+PeNNi)&{|OZ z8zRT?X*ooxVDX(yvU@tQnvv{PnzIjELH&2wPQE~fB*!Na1xM7AOdQ~bL$S8wMCKAT`_WE zdy`bxKve;%bN>D&j!01jn z>CdTKxiVUnBI2@tn59<~{{YfGt1Rjp%-MZAHcYdb{YiaxOqKQ_hJ%G8*gG%(0KguV zZD#1?Z-&z^Q{Er6dfxo}*#|f}Lzz(eYD|&za8*wE3^Q!$_?YvC=E?&c_>z>^KXDvX z@<}DHi~hg#2q$~}zo-3`Lj0T1aMQqGfcY^l`{O-N|MM=Cj^vZ^v-D)zOH)ZB`>u4 zv^^b4KjAtv`2{CGb5)X}rdASo0B7G6G$okqElU|o4Y&X!v2A1g=&X;SeYMy%%50WH zvl=JHaDvx;EaZ5pKjB)El>Ie!G*iAbYav4p5*?DcNm74_8ZeSR>N|Soqg+eUd?J9S z9B_DHBa(On{Hj@?(TZX=;#@=|h=F5;r&uLJ{E zMRk`X$zeq+NK>t-e+tvONc(|T&+Q1NM^&cs!vQH>!s8v#ld(*X0^*O>;Q0Rlrn@lC z@k{K^dBrS??u-xPA56q&;!VcyVUDQQUaV?ARN4?YWr^tV%On2)l<|F^V$Eqk4En17 z0MtD(3M@Gej=T7A4as&%2?1_g>`et1;S`;naw-(!(1V{6DxYfvG>};88#{eN@UGQy zV+((!F0w64Kg1zT)h;nU$&iW39-7CkQfxPiw@=^g{Xx^Rmv6k;ab;Vk*&GE?qD&zxlWTNy;icfytgG<` zU1@1Lka25|($aJ3T3=TfE`_B%ks&7-OZ#;>-q(N6 z{XgV_jcN7#fAB@BD?J(1P~EyT-1`;cBG9R@fmCayvBDDK+s@%>AS*(ETW++FKqGm% zT{#}lCPk&^lBc4qwj4>>)bf?2{{SkIQladjLG)v-^x9?g&F!fYY_g;mortu6+D265bpSqb zl#R(q4xoNC5$85X=iGXG=Ao_(OO2{(f~H$!l`1rnI9HVFZPENLpoOQ@jHJ?flc$@4 z!g|-DC(T{9B!7F9nQk!`TLx?2{LcvOs zpm0*4cLWeY2e>qUV5jmP)R$Rx53=ixI?~ePZZ@)&wzQn3X;2`gK!8C9Al0kIXRnl1 zp|(1$M5y>_N*cW$_(;NAm)`>jd$3EJE|cRAT5$_W7zgQtjD7o1uxu$xBRu47#twd+ zsM|g9cE1`Vh^d`yno=i0TJ#5W{udCf{5O-6p~1M|g(RsYsZXA%jP1=wRq8TJvqslfD{{Y2E{`7wk{ovoks0F&5@+fpQl9!txwxOBOC4B9wX;;ciD9}ky;%U_9_irO3 zQaIcxB&35;dD!NTvtQF`ThXP)Gc9wpqI5laONmZ&xP>3U@-j*n?ox7{VlP_17HU=+H7srU)TZbRo42$;N&PCx6*xf1A4-Y0 zwv{Cy?TVXWDFEQ)W18{(MAe+LXW0Tbd~Zuj0CG5^!&%tZd=F~&k=6*7y{W`Cj?_w7 z7^S7JYGVxsfv+4C%GF}(Oq8aY{hWxur>1!U0f3E z*Bc-llg)DGha;mCqN5(=B+3!n)JgB2YeTejqYv7J`>5D)*eN7OhL;K#J=F4mp}HFf z@whi1g+61H#q>u@M!2};=I9oSY*i#YTR3s|2+Cu$f1N-QRl*aMAK?nsQ3h4Xg-x1< z+rkplOtPFb!zma;NC$+ZeaR%8l1)iLEtPU~VZRWhUEdPp!&+UDHI~?&EtdQyiwMG- zgB830)JDi2TxG>16XgIL#uvlRqU#%jcF3%$a$YJ8x2zxx0;MTSSCt4z1xi|ul%x_! z0=}?UM@G6VG~9JMwJ0G811eIEzz=g#y-U*YtxzAZS@i0W<+)32{{ZheAchiu@DiX2 z{{X~L3O-bzRPyAPZ=k_0muJeIS7S#?+GTWSM9EXpC!`|%dI>3V^MwTn?Sq9i0SW+y z&J_BR9Cu*bZPH@&mr&j&vj*P+Oqmj0LlN0w1x`HT)B;e1j#Nni4|@7i>C#N)UMndf zKg4(*^ogstd@_op_-al5;K>ds3@X_X#Qp*NvK>M59$Rf%c2d=+pZ(0TKe8?DDC^|N z@0NMp180-a-6btAUT#eo-A{aBVQDBQ&1;6>{{ZaiAP3$b->hgKxj^wM?2GSDBUTjS zfOFV;)`Ksp>?&V|P==D6X-Yr<3YY0+^!w;#9r7Iq!@HPVgU;$SD0EC@CZD#8ysOJST%rvS44xr&_dFCBlhZ1`! zm@P|KJ(ji)aY(YfnHD&zMI2Nh=HALr#Y8o3wOD=_^rP!~%V}k|Q1DI+meb+}9#{CJ zE!mTZJSJ>({{X@mOa`V}x(4ZX!nLyz`y*=3;mMA+w@4npD;@Ob{{V?oGh`zF0D6^k zkr}q0nChM_pkXa%@U}e?goL3Z(ITdmXL0LqOZ37ckN*HoTo*OpMJ30R5wFP&JbsQ; z=j!E`E0S8OU&0E)p#-00cj85+WxgLtbeM4Xwo6J(fH*HZq`CF;rO*7;A9ronquAuE z;Zdy7nm_X>DMbGO<}M$oH8=4h=O*XXChvlzy&5}!j@r@U+uZ~a;ky*} z!jR!{ztPe^I!u#pa!ar9Xr!YXD~-17a^q`8rkntS=tusI7>p&RkgblBkD*bc8e}ab zDInx`G}_uvh!UizB_l}o-2VWrBx2WNxi16O<)>Ln&I#D>&%f5BTy0GyL0l0~%4NvT z4B({h4r);njEI^Ra#As$YJMVEhNUGbQ9LNnlcgaspKUz3r67Isf29v4YH1kH98jus zI8;21S|o7PMZN~fT0zJ1sU#ohKPsw0d2d{`5>Dc- zuvUlSNRCHzAJ_i$i?O7rZKjhVIcSYbI7k@hCm-ilUy3a%?Pa;m)TKC?mU*ew=;V2^ z$^7FsKEk6cF#Jo@1{L^zX{ETX-Wn7ZvVDS5qxPfjZ+z{JLvEip>-uff>x{A>vJanD}=-=;=cJ(V^ZIAwO9qPK>iHU_0 zqkxr=6c4B)Bl8tKV13@1>Igkk5g3tamdxUj{6&?A2j&t--lRI_)f=aJs`X%k<{N`G z)Wuh2$5GbF`?jPXy%@f5UvJOyD_ms${{WxhnD|$AwpuPuboWt95v+H&qb5tr84xBk zyu|?aKncH{kNz5zq%Rz@>!Yx%0gom#H{OAq%H!Q^@+D7)82%)_$i&Fw8=Xxy9Zoq^ ze2>LgJNGN%XEZ_zu<8=Dox*`o1QFh~#%@FO9hBSL7crsy9qQ{>&BhR-juw6zpoz#VlQ++dVdL7#7WR}B=f+oDvt)$?M4k;*c9*Y2Mcq3!6 z+G6P$aMokG=ykQEEvG>8xY^-D`VrWI4;1PsOT8Jr_R`YA++{^Rsm^OYb<2-WT|a4C zJRs(Uvg=PH#AZt>#>wMtMVM|E7O3c*t^ZLJ`KhY+DhGbMi&3vi{ng!Q^n zqBNDLC;g!b9#&l^Z=L{Q4jg#k1grPDRI-@3P3dcklOx1tTzM?2Ecg;L^7Epog0>yT z;1U9oq2@x;m8S(FtG!#(Q7y<~`P1?qnbWr=sSw_K9z1ta0bEHMc1!K#9YX~3I1DN2X#;!o zR@0R}kU5yz23!Mokdol@rOi|$m0+iEcLW`=^{rzi@=CFFdL!*ii2B)mm4_R32MR)m z1b^%FqwWSZ(i@lCTP?6!Df#$l*f6CK1FepH6&qeo96#nJFHa!dwIKBi#w8fYP?ujCZFJ!nUEv z4OsNHW!vSw0&89ri0du5jOPj-r7YumY_@s92iB#&LmTO&E*aw$y})oHf;Z!OQC2wL zYBa-9Ja0`o6+;_V zlCH;W*Upmc@_LV`Q*VgtZR*zR&%2)rGbtLJd3gR5Cx23Kk-CLzpAqk^w9BPVfB+ zaM^GTcu~sTa~pWB5EP@^%8}(jtIFC}hw{it8)Bk*bw4Qqu!Iqj%~`E+S&-tMlA;)B zc&{jO5IYZYNJ#fS)i~@jbTpWWvfF-W9Wo^(hR^;RMfIyIYCXkK?^kH3Qkx|? z;8MOM{6ddwOU}OP5~U504gx?JN=Kjo0rVAw#rTE()b#>jMU+})C~`!(Sk#D-n;M;X zj`~x;{v{M7ZlO@kA93+Tas4Zljdg@Zg2CzuNNr^cEy7{Jl)U)HR-xUt@(CZrqLlHf zs69_-wq71wi;SS_?x{{V%Kqs6AEj8uxpi;^4%3EGEzyi5*8JmQ$T6 zKC1pD9_mt!K({NYXQx}LaXV~sO>pJJmXkWM&GyNAufwKpGh^DU%!yOxzKGG8O0nWP zwFIPRu*n%chDqY0z9L#*nu~W$^mMIu9-KnsMR@}sIQrBTDsdjKIFYB(me^15)Rl$% zy@Jtp>Zvce^O7@SwC+Jd-EUOK$GRh^G!JD<2b9q9Z!27O(w4$PHrRHhZdjv9_A*N( zwq+efD)G{B)bo$Kge6NPWbiU;QaxdC>04#jq$MbP&qZ{CziORWKew*ZA|^^ctgy62 z{{Z}Fed=n>Vzf)2{?uO~wJzP$ZGPB&mq00f#@3Mq-;uoi=Xuv{y8l&}8)9u@bmioP9Nu2&C< zy)k5vhY-};+$it*cqQ}~GwtDp*3aNt4n> zKU80p{f!*%T4ssVV-v#>@Z)HQ2{`8fbT@S(L6>}Jx?OysRy^O~NEE9KvK=OIBO?Wj~N z6`>E66R-xIdU_Cz^yOp@YFr;>6uwK3&W53>5f!Rr^r|9A{{V)C;V1s3k`wHl{{S}9 zT|uMfy6I8Zskp4eipd;!apthXasL2HDhH#SYDZ979Fn2(@n8Ap$vXz4q`t4wtbOWI z+NHT*zP})9T!vvZ;V1Z)BshM%v}&+!zpwHC0A%Eq>+Ssi0MS!L#5k~`09tSqlfsAn z#acSLDZfVP>*H!6`L-y@S?;GIsc{$g#B^utREwcZIZ5#GjPLZTY>DYtKv?IpwFO(Q z*C;^t0mMg;k@>0L%++uv(?qiArY6nPzdW|w1t^fxhi}D=#a#M9q0+Z;GF&qv8|^s` ze01&e5gZtGvHmTB(t9lkJ*t47rA#-PxiGR4tZPrFtzRwuOA6%`AH6_QfYPjuzO~Vd z9J?VoBUguIMXKE0sJh&w!Jg;J%0h6NF{Cov%{rfDgcW14Q5}s*$GE_^q4*b+EjHPx z2#~a=C6x^5d=BLVlkSqV_oz;kxIb{{mtzoC_fE`?u)OY=Zx>P!eL@~yN1%9)!lz&P zYrIs%aUeM68wZS&wm(|*VyUK%8NF!+{Ao^AB$f;js70@`ti;W?Y9Bkl)at&23j3Hp10DbEFHaWXT@+(xxh&p{RV^ zAo)hkyQv?AP3C$GBwqS){+g1c36`Ms!qByxYHa!CM9KbOJaiA|L!fTpsH+>#S*MAO zM@)uXZBgY$Q6UneFCHT*^a=nFe}zdxhUpYN%Ke$RU+#)`$nW?~EfB^Jmado1mu}TD zD2V#xz?RZwL6EfIyuyh9tI=s8NLcK&D0)yIvq=6N9MZYgm{f0M3Sr~`Qho7Gw*(wv zJ5=f|#EfXoKHao}HWa?-coDE8qBH4nXewxehY~-DwQQmJZau)KU9@-(siH!HPRavi zr;x9>!TZ$P7Tz+R^m^dz+q-RPPg@OTvU+k$yWp!2%*($+XW_@-az)sJGS)=vHDX&+@h@D zlk}^jmVHM$4)mm1JLx-nQ+vSRR2sIBc%u<%ByI&-k!DJF(@(W)#>FO;<~IVMZA5Yj z`%v!NKp%&xoXnltwG-N}&dl zq?1ns`ikg4CV;^D52bfUwGrW0ygJ4P(d}Lc_M!`Tr1)pjv4+A(?Ow$>;h&{*PAC{o z_3lM?9+kogqG1XmkV(eWvXVV(pIXKSbw`RBWaoM{#=^K9(+XyRfy#w*C$&4^9w|v- zUX_Azutzli0NRq4jXajm6ijdvN*VO7=sl@nCV&nBj(DyfaZYK-;*q3^CJ?QuvZGF@ zW|>gM24N~Y^GPZmw3UK4rj(;b%|h0_BOm;4oe7vvL$8a}Bj8k9egwbTblEpu*0Wak$l z_ncF%y1mrniaAn`xg?sebw%ywUWwC+P!6)8HhNDYOQ{7dpLW^MoOV*(s|QR>L~tEb zhyYhUT`DV&do}ZPVm-SV!4((Q9eB@7^!ss?y$?FrmiW%&Q2o+A$U3p==e1W`yvLt1 zJZUd!jg>heWB6D?O5HzXC0~42JJ!8baO*3a$IyinA>}xvjuWWlC+G$M`cp&nno&uW z4z=oA_gdXuN(+f+RyOZfUW)j5^|z;5g0z&&>JJKR(mxU*Az6&i{c8pF4@9x)RzHWo z4N-FWf15@;bpW%D6Z}U$$;qvL4W=sPyD?26CBu}b0|`I@C`kGwBz>!&Ea^TaA#Lo+ z&%>z8DG6-uF`RdyuKgz(*989Lt3b|2e`?h(R|{>mId#U;bL&;dx{}8(^9>KW+SrU& zIq*jZb-G?A6{{fbjFVsz^$A-g1B0eVKpQ-{Tpudj%$A4Zeu zS>s5!Ta>mQWT^#v5G!6sR9Ym^?Gz< zpQcikexj+}D!^*r)y--krp(ic3hj%OERWW@px*?)bDio;(t2&hxoow}=dBWL)3WQl|Kup z+D}VMa67b{#8{(#%}0I3ey=xQJ;P@5w^Z|KR;M{2d)Ao^sZsi0r)}>8;!K%yxw*ga zm9rVM{{Z@rRU?ibeg%}}-~+5WklXId3qQIfV|tmibwZ|mGrO|VQn1HTTb31_(q4r% zIQ#e`@~ZD$^!t;Nmy^DvCvodWbbXQbdJn53s2?ob1cB^Q~aC@4a*SHsSrhi_6@pC~mJ=^+-0YFe-abvC_f&#P*6moxqfnzD5x(XMiA_jH!0 z+7{rFk}#w^(u#QQJWLAta^dkYn~N+*Z3{x;rD4PlRHdA)M0ydVZP_)WJ=D`Wj@4xN zg$?yPCioF-x&S=Lp8+XLs!;y`<21NWd#OwGHPWE?{>MAqjn}3xR|`DGpLbN2+c;VW z4z!_#p#K2Ol1H|4+O4lC5hN}vd3CHMY&w79G@+>q`=pHf=hCyznwqC#vb7+jl#NOB zKGmQ_@F!Z3px`cBy-xlD)&BtGte>-<)uYhOQ#{JvL&%WAi)(E`Xi()U2V+j%Ev&i| zK09Hy-zQUP;kO6kNcP_&*pEsfhLBP;kaO)u#$hTeisl1yFp^IRSo|vc6NBtcYP}n+ zqKiJxmkA{`T)A}mPDw^{T4alI(xoM6AwZBuI3tl!vTeyyPE%ajB3cZ`E*4D+Id3D} zP=edO$<1i@Yf1r82&($v43JY^blZzMBCP=1VT+Ze?w@fQ-j1ctw{5Tmvqg(v6*XFgH=jH67dqS`nE^rw+;18qO0Pa@xdFgyBF>9=piP0V!y2@t&#L^B`5 z40V&7YrHpO=015o* z#^UD#PTrWJvWnnCCJ5t6rHCW}<%kwopbw zc&k%Z%y60|HuRCe{HRzZ0)XwC93uzmL$1}6jXwVXTE0lOq#ii^YB}3}GBNoINj+%i zo-&+mz#a(tiYb;fU=dRCAQYWJG{}sk`U>Tgju}IXKFdxiM-(TDplhMiUtwsc5RB52 zMGU7;Cc1|+Ld`*7?~GH40QRO3plQ^fYQ~{jL9P@ESQ~e)5NMchM|$CJ+MY^9D5kX- z6t+EU+7q5B@{x`y1ORphia;7#0qso`Z^aTu+*blqS)hX!^SvcZjPXW<3gK-hwF3br zIP|56^h#R6;+;!Fy<-6Uxf`A7VX82-?MN-2!m)w$54|T&4Lqa~%`#5>&FILfd@IL%e1tx%86HM!z5KYvg8EB^qdpGs8ijMJj5yZp)52ksN47oUr zmG>l?wApV@xTf1m)xW-KiEDPe+FJstjErN$`~Zp4nQ+&;C>< ztTGVXD{)d16QmQcsi&$LfJ9Bdr|t4qjVKK@;Uxb6iDw`F=v`2HpWCN zuId%0D>0jKMROl@?0Zb`>KdeJYDW6roP zHm@|R9HAxQ*(TKI<9c9juJj1*9;g{QXSw~vQE6O^??jc|8l;RimL+x;t#)Ovm! ziELSU>Go@Pdm`ubZ%_u(ONn9`7#x8pAM1+A`gU83{{U%iavH~9u2l!(lw-Oq6ka5o-*4Sev=^V{Q^3p$kp6-A@ydUDn;vN407jukr zpgMMCo>J|TK@uUjsP5!s>p>}3>YCVX;n?X)kfkALAxTnB2qV5}Cs7(I%Ejrj*!uUX zqsm%Jw>6{n1XivM7n*Gi%Kjt)gs=Qa^ggNly#)q*MP)z(kaq-Pp>)e7@z4((q>_}H**ccqU2LFc zst2Y5m7N1BJK$Agkt8dFWj&^xmFHL`AnjAuTLZF7X)T{Ox2;;OeHE>3>TAoKn!VZj zVVLeAMW6yXspQXkiAdtDvI5!B>1l?N)J<-y32m@9QjW<_)RG6@tQY>8jMd1-kA*dA zWxH!j8-@5Fg{*(f>YZ2pRdcmP5uR1y2pVmTDoO$4#v~=5{{R@ItNw#glbA$kC`)4? zjDDQgGHjzt%2bOelNh$7=NSB`yG^wb>{m-Vq3bslL_Q{e2Y|J#ZQnZmcE@_KTbpfo z@c`e}r|p)=s+0l8n1ROGtHM%EGBd7>)>lcpEk0Bctacsf7dlqs6_S^l)ufc2eP2ch(@M--YkrhMxwqKnc}q%@|M50s8`>03v^j-QzC zPh2`~djZggP4YHIKYqbOPLKNIEqV3uQ|U^L)5(1PDYaZ zVQsX%E8B9~fl9~SDphr?kD@i?L@RuD5`cshq;fqCPFm!~gE^-j$&9dCkTL%N31dqC z03qfc=@f0o#<;FK6(d)IwKtL92Os58egSD(7x@~=yB|!NX}93ZcHbj+W4NZ4M+Ae& z_B@YcQ?Z&$sNSIi!Q?`kvz4uo!-V%+UO;~6@ao5IT2|Nsop(ZPT&SV+CP5x!sL2oxG80x-SxU7wcv$^=AN_jJ@)%AC#ZM$`F?3Ka(69(1s&XPEI7jrYD(M+I zRg~2-d|W$_JJ*+<6m{ZcGGc?;k7_V}(&Gx5QD>oK7*g!SC#8V5c0?F^?gHfyH@D z4gjU4c;38KkxJNTYvqzp0Ml!XIK>nWPc_Bq!KIT!A~8|MbAh!swm}DKkx~IUq$!;3 zik*%7kzyuKA9^9ULqGc7jYA_mQBAUwz~Y%PUVz$)2Wl<2V2#BR+dyzB^q>z)su8ec z#m4xjQxs#0d}BDpDF;k8OL_WYtBCHRdb5XJo2*}5!D>GI=$!R4dU}u^I zY=TBd{{Wh$rMH8ypjMHq1XnIaOytWNKA$e1THtfg(Ec#sisrrcudN~wrw^?<%88)@ zv)_s(kWMQYIjSKXP|a;mp;X8X1SY&eq@iQIbhsgNzT>~3e0pEJ=p;*JADaCjJN!5W|)&*k@fCVTk9jTD&abEF3tYPA? zYl`2D(>ld@PJlV1W;pHu#RBVzN{;mV&R}m*a+XheRun#Eqt?Ci0bTP^3Rx@U)9kN2 z&>F@6F7=xed2Unzy022qgai;U@||cRWRZ z)kAe{{ZI8suII7ua-lE!t4d^iu}U+xMk?iO_)8s&5oP9) zwaqd%PjbOe`BnDS(h={R{Rx8Uh>0b`* zPNXunQNGy#RBu`Q8@D}Tvs*-F%S=I%;KFt;JuTI!{YXn={*hWk(xa_OaX1+5Qxn~f zF4=F$8Tm2KZ{9gjj#iVVQNr3kKfWUW0B6~jpzQm5Bo^?XF|iez$B8=6H%ic?qh%!H zA6oq`xOCi0wY_Y-%2pJd9H+H>@9{U`1Q;wkEfC5^qvlB4e&(%<)j53=B|IuGlt6w9 z^-y%D5#9L%rXiyHyD3tim z2*C`b{{YffzxvZq9Z7MA8Kxrzf~6hBXU%oY+lPyKp7~njtxGE4D3jQB9)qv|Re9d8 zcIA9R3W}11twZg#ReD>IH=2>A#k|BXp^$Nb-1_lT%-*85fa+ZtNY(!UugboBM_4?L zq)bhC1SJH1RbZ2Tl*0v0x=x&&XOW%7bjQ;r+0QJtZBshN8;Eh0tYO69FF637GHMDJ zP(LPHkyoq5N!SzkhqY!_yX)?_li?TQpDU?PAt$){RqDjUZKyc%JVbJb2d82x+*B5c zHs0B%bn0Avg!to9xZl#X?wE#_VWfkMjBmbcBIz}$_uE5n6@afQx%(Q|pu6(SrKb{7 zR6B86J=h;a)!s^rsgtEYzh$>Rr%siB_(?TcnJZHF0DDn)c`y8fN<*0NoLhWvg#fYo z>#C%gt|@p?&%If*MN-*mizCFfx;w{`hX>JVDL<+aQ&y?OU>;3Utn*Y+GB_vqkruJ; zTPbjUi8VWCm=r)HXV)}9(R+^}OGrA7Y1dsV)S?FU9|gctH_bllxK1-c^kf5Z2~tf{ z?oAD3%D1{|i+!|3LhD(_p>Cmq9s5Cx} z#{5xj;(={oZ`jdDUi{M`SXdxZ!iFeaW7d>i82Z(s$w+*w2X58kyeB!{g)eV_X)AaH z^HfZmJt6PTfLptBT#)|f27z0^pY))`u~HNWGz!ShPkMdVLGM90;K@0ze3VNj&!!w` zfyFXVt_i7Cuy@Eb!lR7i74>H&g0_RsXvTIO>#2>7DbzF*_N;3jWYJ0EiX9q9dUQCS zYGh%|9tCiME1oKQ9qWa_Xn;bNQ?(^d6W+R@4|;5;zH1ovbGB(?RUs!Hl==#BzSV+w zd)KuAwJQld=_^WckyzavQ-d7R6yd=%g%r{ht+q4_c~2VPkg?vIBpPi*gMnLN3lHbL zbi0ww0j!OwV5p6bD;RmlGnxtDPH{wOIL>Hi2K%b9fkSy$G)ru7X(1cti(w$*v4>-_ zeJG;Y18i5CMLM+MG)!0T$tjM!9@X<7#?`OoUjxwB)3?;+g=3so&A%BSI_!6f5>Cg8 zl$h~80QiQp`$f>MbslPv9auRB9+jwkLFu`RxjL^5kWSw9N|`fd^L;?5uA;fz9=Rp|mv>W^w_?@S`_^GP@DS&O+OVAPJu9;uEfJKcwn)EC>M7{ALkUAk&Q!H&LQ*$UlzSy3B%eW0 zom01YH6VKxxLMyJ-OyABRyaQX z)pfSbEGRZe4Xcg#{{Xd0L+Fs?0@BcRBxzPo=sy0UpwniZ3g9i$Qm`^SHDhmK^rnkX zY)NZIlkoZ+#9h8&hlp)hI!7N`+!&U^n@e(10RUjub@0cg-Ap{as1%F4F zkYhcE6-BVs5}h|BF7X+HbK{p8$F8>4pXxY zEM}x&rwR!{;NTx@(V%%W%T>umAZq^rI+8N=C(Lu2r&?TfAS9^o$F*1)C28E6S9BC6 ze1w2-p;cRytgJ-8P~G2=HE5I_WuC)pq_BHyc5kM z$34HTc}(nG60S)~+GzltND2q(oR9OS!=8upphLMEovG9~In|uh+ZiC>;XIM;UMoJd zI#;&mYVN)>O2!EecCPCr{pfVBea$alAPP{}QWu~5Q^{WZ(2c9yQ=wb)Kq0NGenk?6 zPz|rt(I{CQ_oA4gI<~JB9CN)i7m>|*YVz|`ixNokoFbWV)26t!ZM`t#gBzNvAs~4U z9OIGRg;Ky7Bx08(eX=t_y6H$c`%rF(H(NUU-~*B>E$K>9%5%5YyD8JS+;-Z8NGn%= zTI8IYAt|{ZXsilhKxcY)UZJ_B$j^HF6iDVk?_9_`is5W}n&Pkq)I NZ`_ZLr78E zY|;YARnN-W(W$&;+7OWc&1X| z#`WT{*iisNRB>EWyJHm3h(_X)r$1U26U%T@N>Lr@KoedGBtqe zpOz$TSi=QrSJsr5QU)_h`9tYm`BDyOGaU~PTIO3k(`$~QzVwu~q0VbqLXeip+Z0P} zp*w>?Hj|n$wCCEg07_Pric61-Qgob9?j>8C(Pjg<;nWa)D=q6@nimF~8Z(i_YHzZ# zDmq22FPtek%_CM6==lEtss0e6Q|on6!QXAFu;0N?-vVL-pYYe(Qo2otvzCFO6VWk( zfh>SNl|Br9NuM?J{{X?ZmmSv(pbTRa)Mr_TX01=8j zK1`)cMNSokxa{tp)f0SMQ2eUKF$h%uAeB7r$dUs1A$!f zi;-wFi%(Hr8=Z953B~wR+7E=|Y^!AtvciYjt#8?G(*vF_55y!%K}jTbAL&QL>NwXV{G7Cd z*b;NKbmGXanGBfSSa(e!R97tB=PyYb0*9Xvry3*H^laZR-IzQo?1g1oY`8~2YtqG2W zhK2!C2_B%Pp_A3qq*5N9*3X@7T^l)YbS<(5lZ~jZp6L0^zWZuH-*7Rp74!-^PBI@u zEu>(a9k{6o{T&cG5uqt_1aZwldZ{vx3QWg(U#8~UCM{kXl*^ogKM%cMx(JIj%BPFFalq?(*j>m&VTSS;h5*cilW0B&tIVt{O>s5@UAsk}5FE+@g zLrHNfN*_unhGTmgi&XJ;HDDhIS!mm!`Rb`v!j-e(oC@SnYKHplCwcS-R3o0Wql1=FMP!s{>_zVh)qFR&K3p zNx-Yk!Y`K?s)gMFv=f@-&l@j^lpC7#ICjAl8eRA8>rdYo`_6mRV{TWCJu}dcM4DYV z-BkN-9P?4AU%eF4{l3*EG9(baxBRQRjya|Ru${NB7YN3GI$90TQdTpurG<{>giEK_ zYFb>$-;B^|fS7Ab8TLKt;H~I3-1GvUO7q15tTDCca(JQ=p_~o=v=eK_H`xCGT70PD zfr3KwN-7w^p%SWatmA5AD@U9l=LUs!SOl7JZULOo?kK1Y`Kra7*)$IN_T+8-XeGI} z^i+@ncIVoooK^|v)9F)_ws1~+2Y<|}>nx=GX@#jpSR2u?h7qe)^%u4XK1}-PoE@v$ z_uO+!4H9;){ISk)n)?w($;~B7{Hfx?yNc(_sO&~)u!5aL>?yUR<35yUfOCvj6&&M= z1|!a@X-ZbD#YicG!A&H`8~JyvU`EL_LII_sIn|9cN+XU1V1g8@(!2uB!jAcD@+nDU z8{)Bn5QT6>I+jWP8Z8V1+Mh!h?M8qA8Nl3Eybj`|iy0hOK3_bF)M7L>*wT{9hXSMo zap_$V#S;MrGBc*SnEC!1kxYMTSdJ?&A}LXldr<0(0ph0>8`_y%Vcb@r*%Bk~_-KaO zz}+T|OCHqnSQU~30q`<56op0hsZktKOnvsFMzSm9f$d3Eykm-&OnvvIi22}FqcRjg z2eo*>AIthw;EX*hpDXQKiL8Z8B!T#8VkjM|TjhpxUHNg?P@`E2^e~`up4D0E3o2iZ zr0CrHRrzcuABLhXaT2D~H1_nRlx!){iYnBIG77*NfyE7YbxDk+1q`Hjt6xz@3_7EY zvsw37Piibz3fF<4k4p3Mz8X5Qu0|d4F8J%x3rsw9l@$K~$|?h*Iz9Q*(=pkjKqzLt>aj-_RnJfl1KdGGIvvHVi%RnA|?A!^sSPM^}V-kRyFjo#$sNyu3XP*GO- z(kgLDZI0XN7fzdjI`kQk$t5z7G^;rCbNbbe*U;Bvb`GqmD?Ud#!pCD*Ux(cv76b%a zpDGQhK0}Yelm5Tfv^ek!ASL!vTxA(?KyIKvi`>^N?etYiBvox43d@KBOJxfq3OV2F zUc7WH`?`HRA&$q>wNU1sNG-PWZSR zBOq;2!nSiZSetNit^t0AZ0WwI!XLZ#eZM)9XnbqR4x3R*zVqSNb8T}{&~mg%|M#vC8sH=q4;U2&zOnpQx{ zxHS@5l7XQvel+?YTAb+S6+Y?|f=Nlv2>nG=*=W6RZak>vrvVv4{XJ4^=pTmt7Qz^6 zTASCX1FIM!se7a^g41Pf(83Vm-yWOOw;1}+Q{pG-R$*>whCnIPw>4e8#7NXov-Hhf zCQj{@IjrcQq#61}b_(K#VA#y+*CHrCg1vtUZybkF8k7!}F;qYy>DJG>uKP za!C78Fq~=EfCh<~WU~b%gM*s7L{dvYfrC|>LrFSCTH{AjTLzg3$oc9TNjR!a{nRNc z7{++3)y0COn$Nn81+=X3jMqMHiel`|FKa1~pnKGFOXCWnua0YYpL&jE&wc%C!^zp{ zN1{mz%0_6mR0n#GLXHOXOH1^rG(G5$3)=$~?Rne$sf4KMuIs?u)}btYE5~Y3cihmS z&p}=eWZ?SIp=>=4@H0=P-OhLewMMq+2eAg9L!&?CO5znJN;K%$`q4>VF}+1H=f2cp z9PiB|R8cU!LvBSVxG*SSv&L~sQufYhFR*MHUU3ML2zcel>?KVg*tjUb1|7f zV`0TcJU~wsD#0)-mJ zP|zHTa5gK7UY)T>`GG{+AMnSeaHHCihe7X7gP!$)V`Vb9(EJQLcK5Z6Gp&zuco@y2@%$PjeNQBRwCu0%^235#d-dzJ4apz z*--dj(rKG7#9LuRt5TFldepk1NVC8#pC2VoIr ztXj8BL2bb2%VdxKA`c_-{*^+%{7HUqPsuGfTYW0R6gkd&`+u!a*y;As3d@)oMXLG zdXCDaYSN68?OGGaTznzGagB`uHX3Co1v~nIU6G2EPF`GDh+H~(*9ir@o%>WPofNkf zkkJ|6AY!*Cx_!5Q6MObG);e9R?%q3bMmSqwc~UW0T@th`E5bjOT5Y{PVK=O!aqY0J zMUIqcVOj>>)DKa^jcbCnl%*#X)$CHXL*2JX$}di(prokbTvY<~czL-V3Qj!Rf2~@t z62tNqM*y5v2LAvZ^*E=o<>tKB?Va&jC*j}POMYc=nwz&yW=jPsUZJ26JDSDOcLMGs zsm~^KpRV<8iGD0O7CS(cs05Nb8ZRB~8On`19jY|=YG~uju=*PFJuoH1Q!OneKosx& z4GP`ihFU|PhgtwVx2dj+y+wB9{9<0o17_qug;;OJ%vY z8bMN0r6gd2IrghmhoDGku(FayKt2A3p`x&cW#&4;N!!}2pySa8J5)#AdIsNieVGij z1A##7Fl$}t%PF_OLuEO_am7RQ>}F)Vj}g2mW850jVMYmMBVmE*U09=T$Vy%adz5le za5xoBx<^`+cdZI~Q>2B8sodkEB|`?T&=azgDU4*E1w%_f1dZy`IVn1TLGcJIsgtaZOl0|3M z7t#twuWG+rRD#-Ycf~wJXQsAF57Mb!MGY%ydB$;9xko7?shvcXcxuLKd1Gj3Ix%jh zhFW=QN3VLXHtrRS_p6Umq=m0idCxUMb&?s|$_{BPxGJDcA6v-x>$s5s0a|iK#sx_7HGnyqU;}telF!Vk4 zr5AS@B7hEaOYs34aoT`vd2Wsl{`9muQk?3})gJTMJb|Cik@G4Ez*f|<=;Y28`~Iea zanz{ep7hJIbqs^f#)U5cc0ZL(v&qW#78kxg)bnc6GI#bg2aE{XpN`>BIO3W=Cr$|; zbp(O9t}JKXyp@ji(nWoc2EupZloP)+ps%GXn#K}EHdBsi-0w(GqRb>$Kpwl-G~oBH z;L?QdoYz7Cr6Yq&3Ga$10zjmxIqy#P;Xi-Sm zQwT{E@CUzoY5=tyjj6IuE4jxsg0V_udb|j(1&a1TG#at2R-s2HD^Z1jDVLF`5GmY>;R6+g=%aOvQmD(o?O!@| zb*Rajw*LT_@3nnD)lH>QR&kw;V7JJL7fnQe7Wu9(Qxm5PB06@~n`OU35&>7?{{T8& z!do`B3$Lp|&Qbu})Xad=7)Wg+DC{bWer|R0=$6uO$}y~*_C2eG(v>r*bz*M)Upnfz z?LrGnSPIA;{WI-Q-D6~!C&0dMrw86NLpfp}L#K z*!Lclb?K>A&k2bEy+1I{wXI*oqqaXfkwv}du5ZqprLwr_`DG;FrNoo@AL=P2oQC9) zceO|HA~unLvYiU`){&5Y;|KGp87h?0T6qccHg`|4-)e=oIOMxhUAZ!-?;&YRjx=^r zyJxBiPimsI1)HE;og(Nhcnnm1Km7juu^S1uGw6{+rW9K80+y z#*U@hlDR|@oG53w#^BUrOwF`iT9*~&DTVAlpI>UQ+U~a5P5#bOe7m+!^5T^m1OUB2 zLce6oTsIC7Rjs}ArCy&yREm`d+D9YxT&ErlJo=KPZJ$xt{VGy5)d-S_4csuL5uY~M z^dQwAtszgeJS9XLk&LB4>_??mWA#}onn0ZB+1r5O~t_SNb- z0sB)CtpXMb9ej_ zD3i8pdg?1G-%85U$i{KSV_jQgax;rsLD>3Nm!BKEI$@HuLZ)8Bekisj6OS>)S5)tY zbdii2A&H5|%D+ANH)BYy*`!i%}28EKhRU?btPHI%_Gprl4Z=CB~(gO0R$6K@zuE| zjY$-Yst8K9t~f{;PB-*Rh&}(p{5UxGzN)~eE zwuLz<2}v2_^r4j{JDh<*NOa)wLMlNUf@(ZZA1kM_VKPD6R&(h`#e5D`o+`G~jmAl- z7}7STOThSYQ$GF+rFfK$CXk@8afypVn#b%NP;rXEyB5(So#=#<*ig;n z8Y#Lq1Eid172u^BSQ)Mqk-)9kEbvk9OFN1{80NdSD1)*Z{Lox=0y1$}hgL=`%~J8U zYk0aNQ!3A;WZheC@trtEbL~*fth5~!3u7ii$X-fF=8+QL9xR9TAS(w@9)s4bwu>^_ zK|UG!;;P+T?I%$39cNF;`uY866K^+dW))Cm34B<0!D{$K*93?&V8yn z94m~7??|~2h{!tfu1M?;xv3^BP;63Fb(Dm;HPjV6r4HVM(y85H)OT5O)sU!`sO=d} z;Ey;@BkR}?dRnwUMV5Ta6{hJ9G=!*3G2!jcshv2VqnL6 z7*4W5&V4hsed>*S>ZD9zyVS;(!X-2dD0FBlLU#%&JCWNILN7NPjB7+04Mm0XGQe%` z#1Ilvq^SC2=k%p(Lw0QaJJj~qV$HT0YSk&V_zG8)6{D1I_*4kbrgKvLIeKF9(jm=l z#$~XxzZ~aL;MS|j%yli^oQ+<)4tJxvN+JW7qIB?`_5s5IY9Ur;Xp052;c4jvD5pY%1+ z6>8BJrd(a>by^nOpD-&pSN-Zv_tp-ymZ!yDRE;Yr&-aM$kMgcJVuyN+xP>6P%Dk!^ z&k{epJI0mt{Q<|K#^n1@qY4kVjq@LgQucST?^c*P4{d099H$5=o!i9>soptthP0JQ1+X8Ajd;VLQ{d*Y-&?Vv7GH!f~7>8vLa2p zP^h<}6p|F0vpNz+Flbj|YDvmzj8&yF&>OZ%_o&CBg)7S51zKdn&+yO=vWD^pG(ph> zZQ&y&12hbnDcwV0H^oj!*dEmtF+&7re@e#ai_@W75nW2oa1P?UwsL#t=}t8im1#N6 zWwR1C_gy$ji8VIN511L#$f)~eyn+-(TP?9l6Zl3ckgb(_pHC%C9Z2n2H(T_>dK^)2 z7|(7i==o7waf7G~;oI80HIcYi0f?* ztfc-ES>sc3paJ}=#mi8YBC*I?nV5DN+*F)dTDd#ss&S-)$SRPNc$1dQ3| z2+8-MA5K9zr=BN5_M*Edi2V^r_ob9-q1#ufG+Sy&=DxLT?mUJ^aZCAh z*L*>WNqil%MvxB{h}hRg+fz$_o#};0IoVW9*kZGaZ0*f9)8vmeJET4)pJgF^tq9ltIpE=4NXg(M-(;zLYFv9q3<{1zFYhq^e3l z-!;TY3Rc;m=*TUmf@q{|wrF-$sUtM_+icb_=Dj?c=x`cgYvg8{E+`&rT^g(@OHXns zmlQFzGFDA<3OpUCIapLrD3jj2g=ys1h#*oio@fjbmxt23tvqIi)DuZBfOoBCpwMYt zb3&o)jg2qGIHIHmhg3nP9RiZKh&vi+$Qj&<&6y!-(kbLD6N5pQ(s(u0ypg`D7G{YM zKo#tyjpzX|W12{s0l`%jBnl*Qk=~M3?ZpV0N%a-gO2-C`qj4LathPofr7qZIK1ok{ zw7l?fB_Pu*#RYlA1s+V&d{G$|$lEOwq;XllRxuttmt9hP+47ujeT8gtbfK*y^sJAs zp*p3L9BNp~hE4#e`l!@&H>ZDA%yRp8a%|IKZM2tPB}ylK2+y@={X|@d>f$YL$F|1Q zRbe!b>09xTqMQrV>dNL+1R*s)+b$!z7aG4>ze2jTag_jYZ#2Y*AI41)YX>I%O zRY7N{y0Q#d5D@5GCfP@XqMQ(vsQ&;8K{|=fp464Q(X*_dDJgOzPn-h-!dOuAC@V_RIMP+I8)F1jBdI#R z$J5sKI7^n>g`RYyJwYJLdNRvJ8Q`~# zh#AQnW3@+hbW)M0^jlq3Xt%v%zt04cP>icqTEGb-?Su9erC1WL8HZB#LX)9jcTnL& z-mtE$mDJ2U-O31{|z$`Y*QCnJyh z)gdJoiOMcW*XA|gup}uuNKhpEQL$Y?Y%4nvOm$Q3x6`X}vJ_U3Qla&zn_TiEAh@sL z9M`Fd@i-;2H?JSWHvuOY?O!}~04_STgSO`#>wD`dEx#EkA5yrK%5bFY4r_ zx3y}Y8&aE2usq(QJk=9%=uZ18Jv=1DOHYNAr~nRqJu5|n^0$kA9Bn0O*-6C)I&Lb` zw#MmJ_#0sMBAK?wl?BHT(NeS>$MYtS88inZDVtD_DL*QNsUYB>jq&MUM*JwVw%it- zD8V2TSdF=m{FWR_NGl_b_4Kd9_?{K9t%k`Rsj)6oC6UxiJA^g>#}t7vBg#!VB!RJ@ z7UDv-O?nh_hqmsm>13-OS`D{UcA}fyXEmt6@1XO6nu4Abl2wqOTCqNmQg;V61wATO zQ`)r{uF95EkWPNp8sfr^NCPx{=7>?>dWm!`D+&t6c@${YDMW>AGfuF)FC>9MzavUA z4)hYP3PIR= zdU0=@WF4v@m-3&*p8U}sI4M4A97pC5r}o_bdsmD8Pxb9V_@_VjuC45&KWbTZL^4Pk zO3IXIbmtf*o_#=!;~lA-jQ*U|S;3QRSUBzeRO4iUjwy|2JJBtq6qB9>G#AJr9iO5> z1aK)pC|7|b(z|B9tmzQ7C=O|owS{nL#VU}M@4XaZSk>6mN=m`}G)e||rD)9%Rf3?9 zcN^2mUMXq-cBB$BLd9SqA!Lr6hWyK0o>CL zq!F>KVlqOMG6Al{E-3D4Pc}1(b|Xm}={18yjC_RdY0~zcjSd!4FcXSp$4)jU@~qt% zAkAR$N)76$^gc&=Et!KDCwdxYsvRxq+)_a~rd^f+{8Y)94&czBhLr)&6w2K|IMqPO zn}M;Y=Ow85fui(fvTem@gG0Em2F8M7QkF5OntYIbP)|BR$fdIdDpG*vkf}@KRYxuIdCeomQijHr z+ZiH6%2wnPfm{<6t7DpuNprqw#DxKpcA%3prb;4OxF(im;BI-SpBRqy;`uuhOb1Ao zW%9M*B$}3EWagpmXe+@YrW6Vho-3mgrDSI9&^)IUuu`6Ch1L;VLKNvBkOu_n+ zj42*3lxN#Ked={FX2m5Im<%+dk>aQfkEJsv-+CIPPr1Zalw>7Lc#8Uj*NJ=jk5ZFb zC~tHRK-(NHz0(Hb^X}<(QzuwR&cP(|bK4bJyxJzkzDz}{rqKzP8%r^e>O!1Io?NW!v6-mexuhqOqzEjEcwC8C0c z60Pv|kVaB=ByN2O$l9w8u_bCvx)ce>ZZZq5Hj)$K^Q#`Gll2s_)IJ1gw_9z^Z8ZTw ziEOPoN|JvHjy%ac9FOv)w?2mAqH03J8e2=T+}mZT2xz3G1zreH$MO%7k7H+X?bp2)}W%cRj4O@wykHUZ_Uqbw`C=%DY&wd+sbsWOR4RvJAyl(&aRLx z7RRBvX_V{lu=);^zN4+4`cke)^v-cs#=H`nr)aKxE$i!tRP^)KE}?T&#=ij|jHav- zu1>%}Mt=wZ8WCM965S;-;l$Q*yAb>ec0q7q5U$os{~a-%CM zJBrBqjbTnWow9TJR+n^+xbakzk+H1K_@p$YEoX9S8Lo2KTy+#wrll=3rx4l5$=a&d zd!y}Wl)yU-C{^m>lzfrE0IN9r)CBuG5+ect1zh8d)gu;=u1d<~m#F2frwh%MCvo%@ za_O7H^45j2RpK_BDLfSqQR`66bm}31qL7~6x%Cw(3eob6H!=&lleTbi{{ZHpUnX+V zq}XpMb<)Gg8j-=^5$RTTtlHf9#z4{rckVqaC2{+b5}EK@aIFLPfDQ5Mio0|_SzINw zVM=RG{#D@MdlA~Gqw-C=E|w?_eDVuOz}1~6jQdy6T`qhZdtoC3NUSEt`=(gK8cv~{ z9gS{%F0i-=2W*UwYNj>DMCY_ix}vW1K9j3B%^Bez;=vWt+D37WtEiO&YQWn&Q_U3- zPG|&fE1E)cf(2t9S~YsqbhzLs`WhkmVT7Ejqn@~j6V62o3RYL|Fa85m`(H3BT6uMYH73CqU z;2dOgUo3T1oiL`5P(cU1ec#v7P^K0GX~qstE938p@d9o~N->oZ1$lnA7m~Qv%=MN> zru8iqttkL{Q0y`7Qn1>_HFh4A=T#k1R4tI8I5b;pB;`l)uO%tQkSp48xH!cdptcYm z2PrBbQXP^L{9m0gr$TTB*ye(fCpj7Rskdi5=UoPggd7EJ{b+_|hl*m&RoPq?Fy=>tFISy_0=l72cyuU zeQGr?g&Y%0*TxP-ERige332^E=k=taY3xRGO%srEX&{uVwM$1gY%v8n$s;sM6rDTl zPu`iT*|XwP(MeOqE9u%1rx{RODV{u zAQ80{6Ii4xM>OblCWd)hMrpRzrFN}#qflFm9jIlwNp(jZ_Muj_6UnR)q`AH`y(?Mm zK_N;flSQ_LE0bEyNKsOaGUC>O>qLf$?kTrfDi{h&~Nj%U7deW=|nl;jwK?HnXtWhf-0iZqyr~i*4Qa{b}7` ziWOY(O{vEcMgYLA$UH-KJ7Sz|qE9)X+K_Sqz%+9Y4P!W_TM_7;w>Fm04l!Q1PeVY0 zp-duL5Z~g))l=$gs*$=C!K=<{5hn<9kz{4$$zDO&b3$I4ZRn05p4Aayx`n3-S2ZDV zn1wK)oyhdAcRon38fI10R-U7AZ?R*PSSlC=C&1Cx6NK+*$8#oYn8OR`USKAH7 zTt!D}W6uT@d`4Px3R_D4R4beg!19yys2e?{uNDe?ccVVTdKi?Sz~f~86?%Y1G04F7 zt6cj8_)NDY^ft64rAW{YS#)wo_!Kjd$A3{>DIzWSC2r0tG$QMM8>}r}Vw1aMoq?$6 zP9{TSl4(~rR~{>YDJ``jpd3LU;F=yHGaV~IAme{}zBWooul0kT8%ooP$nPzQGL(R@ zG36b`Yews-{{StNE0bA7o4X5;RE3RMDg)N1Dsp8PL=R9;T3b^AT0jSXOjLEds9Rcm z2?PR0Njy`R*v?E#szQh=1SI+zq$M(%aJKqLP&ng}REtt3?K1v)jv7j%%SZ0h_(99>Q$+{d9qN7;Q+%=#H7P(HxT-2>MwzHtBccEb3i6)bm1<_>*hWSUD?PH_ z)igGgsDY0AZ&qX?T7gn=fx!ZrKVoeHb$s@uL|Q@ui3Yw__?K#Rb6k?7;Uu0o8`sox z?at0Z(4Yz9$}1xHiPCSOw+dNSFi#c9>Z8gfF`qoQ89?Bt9+{%rMv}spQ8@i5W?Cy50Pjbz&iiNYSqs61#!(nMWOv0JhqV9)dKtF( z^8IPn&_d3`oYPqGw1SYF?gj-u+eDIf-_TbW9+>>-lp{#um7+4(*+%=1%A0%?0!Yto z=&%NN*m{~`W&?Qj8`9CDtOP!F>CeznjGXVswMM4}c+T}8$U1U8C>MlJ(hmq~+nRJI z0OpoKY-FeJOs({Tj^@6jNYvQsPI;y3NIOvB^y8eGZPyk?Q?+Xi1(w5rns3924CGhIGpv;dr0oxSHnQxp^&bShGBAZrLoTn5RPBMo5Hy-q3X+g)eMXA-atm29zZzqngb4@ZTj;<$yy1lof=Wnh4Kek{=q#2ZP$2LDbnK0bG76bp@S|wJE1i zgy~So?_0=FK82Hv=DMiCTak(=7Sw#BZ(2*rDId~@?qELvr{5IoXaI9u4y=vG>0Bxx z!B3;Y06U(pTq6{OOfLTF$Ll6jWxCkW;8_LW5*ufe)^``$tQ}U^+#AV&`?sp4>fd=Y(vPx zK;EdmQP3{WDk)Y0=7wH!jUn;zi?##P{{Rspc$>Tu0py*k^9!jX-B5%f(S>eLwPa)A zT=cqJKu}0I02;V-?TTfk=2Y`*QW7zbQ&5AN(?v3HdqkTrnKyY8%x2RXI?5Q-cR1x( z{*@r5z`5!1)U)ab;ryxQ;WEpG=bQ;75R_y5sfQ*)Zc}L+YV;V*GL7h+(8cQ5h|m;^ zb6Sv7<_Z22>ME%|8qK`CmOtJI^AItd3>wt!%n3>eLIF9?9jayigNZStmV$K*f&~}p zT(*Jl@zXwPzgyKl$sw3dgh*QlAdkf6Gy2mX4eav2W;*1Ck`km8NI~04&cyp2^ZC}T z)jtenK}!4puL~r$us;ucesvYlaGfklhdHke)VQ=94U{?igUwDjnw2r$MAf2KrL56S zyF`{$veSnsS?~Qf;QHdFrL^O0=!FTw*&0An_WuBs`+adobe+0PGN`YB<2flp$j*`2 zE7d>Jr7rLtODm4hG2i7nPExO?YVQ)F3nt96=EEKy5fLmpwdqoDa+6YVZRu(_Ndr1+ zVrNgE2gh|mOF_!E0Gf@PC&N$f#x~-SEf9I#8Q;*?&0Rr#m2|kcl`TQMk(~Ff6q{jvSw7!W!;>Kq(!tUW3Ws`b z>gAb}^9~S_p`4xbiixwqaZ5&q&JN)E*Lu3#@}M~9gh&bW-y^rusIiFViKKM`WI++F zYQnVOf$2=!EfP(`qrR zqROMBM_!_=I5KgXlf>6!b2JNO+Up(0Nr<-^J-|FxnPtB^(i;jVNg3X&of9?%#gs-{ zU@QPIwAGU7U~#=FYw#-s5OIolDMM+ADJP~_oe2#1@s!NTHD#sj- z^z{6K>_g6}#FQV>qHYn60?X<^9nv?V6w{^Btt%rtW15&|E03}RjD-`yJkYk?29heH zrqMA0Zb(vr>^99qPWXQmEV!0azo^Y=k>H`C3y44>y0(}km4S~+bhIUzLV6@bA;m`u zNjvSmao(S4r;xJ3$nF5GN%TWX2`WzAsvY2$7pbQkSqIcnrhr0OA?R?FMky&ze)VH$ z%qSow#HZV8HqmNZ>q#mA_p1wbkbrfZ{i%~^4Wc&M!FdS|4DHPU)>amz+t#AkBTs$m z+a5DX000hXKB0u;kQ_o%Kq8G=rVFNgS=Y9QWAp zNvz4$V4Rxp^2Vg->rT)!%PGhiBk57kyrZ4)eWVNs%O#@M4#6obVP352N$P{!l$MJcWQH>Ii?f!bt_ zKT25|RtXvHMK;g_KE3P9Xlons??+5JI~bhN=ou6_>nR1?k?L#cT(7~RsAReT*ysGK$$u-w z5dQ#xq7;np&e);dXaW?2LBD~A%RxgRZ(Ktv3P0;eWo{_wD$P9FR)u+v)stqUii#Xi z3eV?7e6T_?38u9uD;w90m0+Z()~%Rn%x4Capn%o~CyE@YZZ89RcLaDkDa8|w zK#}W;HMV?RnmY<<=*wwif30dYiPF*$sGqeCpcnW-#T4scpapMBjLKWWl%9QSH13Rm z($lFx-x<2JJzZN*fTb-4ZIa``cq4W5aBsF zuRl6kAMnPkdyS~HIEAD4hjZ;qac@I3&@-){30gNesPyxD;DAIn_8}lnr(xhPhHZ_|_$uBoypD#c^)D z!Ujg6?McZFqSy?i2+~qrNWzZ9b6k^YdQ0hXxa6frIRc+iT*aj583!Y5){AC}!l1U7 z8%{vpgfThEvO*+~gK|;VIidkb) z*i_s~%XTJWvRp;)R$)8xwz^8NC#{TRZ{JLVZ^w~ z-AGn98RochRkU{FPLn=;_`!8U@3|Q`DnF%UQEkYS;cH%%m2HaDek()?oVfsOt--2q zM_LTan`>8?C4Xv$JF>HSvr*_ihK{HRT$WO;fl0?S=U#L&6Kiy^!a{;FkbC-yyW1X* zZiLYoNJ=*q3GoxEqsOpL?=i|Pxz`+r`VTplG>D^jX;7=;vT;A zty?!pzjT5fj-;*B=^*0+oc$|UVdmyULRZKgt1pTKMI~+=fw}gpeX{J5lqsh&2;zc! zpM?ytSLm3S#^kVo3PIn6$o-$@#fr02L9s5VtAIG+`qCqZy~ zcA`K_n};xaq|q@MG6(Nc;tAtwZd5lLUy(t2eQREWRL=&Il!JlpYsN#Z<+QeNcvdNK z+gZk>eQB}K3u2!3ij_N{`tev+s!w8jcZ(5krP;fv_b6Nw|r$)+boNh8Hry(?ilr4~= zeMJkSRgHAT!L=hVF@kr>s?52OmYktU$E8*ngNJm4gahlfSYp9!M;XR_F;!D^O{7N4 zlDN_}sd^h5MHwYg?f=4<${;l zR-k)iccpUp8;xw3gFZogrQzkq*rMgmf59%tMzVpp2YMOnd15<9#Vc`1*e07_h=#lu z(zJzt5ES=X7QBr))OQCMC5S329N_PoiFWnW@tjJb3L_;d8;Uwue51wi&H%}vt+Hln%a1Um zuXOs0@4X=##|9L_d83i9QpwfWG`${DkWzO^Cx2>Mjr11V?3_z0Pzc7L1q&|M zh4_J3_RV>z&$fWduv{JeXeo=h$P0D1z$o0O=}9TO(j7%r*mDYTpTcr|>9?O$hYLXA zdB**!yBFdey_ZW5Fr_$yf&j?Ku1@Pfw^UlUt)&#|1w#P))iqs;kbT1N{gLL|l-Y4B zI0Ox-_!r2s_yGj=Jc`aPy=d&BTbhBl89UT8j;fNOQNG0l2c2Hh&54k9xZ^I&00TwY88_up*sFGId2}&wjZ{oZz`Y zQd>%}m2wZ(t!i8s(dAYTrx>Q;An019Mo6VJ+8tR5V?qhs_M}Ng4&NYIaY1QLwy=;l zJJd4zg4Eg+mnEeuPTbXo=hM<;M{z95N>it7RzG>@d*q33AuAXMIX&sB9Y|E)XtRr& zUVONj=qLv`EA8~6lh%==Vlt$qOD7l@#Zq8cA2Hh*ixS%+r z_;qp(5219z+@EN<#*p-uJX8#Y4DZ^9ORhH{rNnGb#+4>}EF-~&JTs3dsD)3txZ;~s ze6O)1WA96@>?xK6hMgKywF^173QKK`L~tlSom(1Qe1_Bqm}k8)AWka4QH5>t_gP;n;*d>XbnG){5k zYj92`(oi-mMKyLcma1-nyeMNat~;NC_!Q^r+&A4j_#|LwbXn z9DO?2SPL8A_o&ypkuXwQo`tkW3dTYEiae6+9B-n*agfSdQdF%waak8ubtlNmaw}36 zKkC~a)hpEfaG#Z0BQIL9fJI|Bx$|Ft#c@heaGVl&`c!k};=Z2CVaL0WRp$I&DkVo& z;FDCVjFifE#SiJych7n@=)~956&28}yufaL{*_C-+V3t%NNFwv=X`gsJB+05owpc( zd_iHE_Vp0xE}(k$svkg1O{!yvmT`q`lUC{dJ9(bsnQ67Cft;SzpWb?FsT!2yabqkk z8NmXrjX1kxa!o#sjXg~oLQ@qlCv%gE&Hgd<18Wg2$#4{p#e-Bhtb9eb$y=`5j*RN~bfuk30aGkk#pC`#V|1oM%RQ$0V{Fz#1{Jst3- z+d5ABU>dDZlRWOqVs(iD|~Go@Ydqv3~_gK~*}`Pi*F9l}UBQO1q8``zZ;r zKGRC%sRwU*(q()ZzV!{Ao8i8=+7F+>KAxXSMB{e0GrBcSpX*pozzb#IqDI&`sOz6v z#6*N9OF?03(h1upsvQ&4^7?XWVLqQ8KRG*Kj^9eZKcXVje+5XA3@C&)94$$jl_7e z$kfjG^Ca(@ad+fHQhaEn0&sTiOgkD{RhHGA9H5QpYn9?N5jDt$R563)&h&XTqf%ti z>KkIn3dkqgo-yLi!-&SA`qq5i;5RKUA~DGYK`K_n3Vz#tyxi{YGRs~IEhk_>8~WzA zxSOVp!j+l|VzPwcNCf&3QFj@SMP<~XqvhNmK~PZowqc&*HzoK!3bf-H)O~6fsQRy| z8@YHbHpj|12-KoMH2l|rc)uf&y5@SWD~uw#<6vMEifgE(OSv*t;kwe3!71D6nv97q z_SBCR2#lefif~SSD_7~qAl&U?iLA8a>jf(EV2UwrOO2cKZjYYslG3TVSa1?@KNW zeX&Ytk4#p-2V`<+IuQ*4njyz>ayX#SzSuunS#T$QKJ+i-hDnQgV+X!{Dl$XG3M86+ z`2#sB;-Y0d6`lV8?L`siXPulu$ReMIaZy+SUrL2@T9#6kbL&J#W!&+#Z`hQY@?4?Z z)Di(I`qa!+xhu_rK&s866U!iPT3=6E0$K`i9G&X;Zcm7uh;P{MxRNyEYLA_0eUN;- zR*oVzk%BR`0XhRtAc6*Q=~l^sHQ8KwDW7G;l^r0#kPwuelDuY?%ONRRK+Ak7L(|s1 zZ;xu*=8A%V2>~M|WcH~Bt-&pY>vaSYl(sRP1Kd}nx1+nFDmA7vflevZEDeGFA?aO_ zrYaP%2s!}R4oykRvz52=t(2um9rh%1$9iRLy@2C-U0wzXcii#~8lYX2b@eAw%elJE zvdW0)ZNJ^P>E!3I^{Z8-S0|vNMdg&48nCpWU;*nzx|fjiZK1TOOA5#%2~IXAG`3@0 z8r0!R)RiQqI!06vuhNRJ1he5Sm0$vcum{qGn`5`mU^4r@MV~T2+*1s(cSAZRfHf~j zQz$tn-_nRnGZCseVMjT{)Dy5um5kBuNf1h! zao|86xunRB%%_y8I@RG3q@azDtqm)ul_xF5kmD=-YD&?9K9oA#P<#DM0)J9KkK!o{ zLf&t15_cwn{{Y%sv0{OBwuFTfj4U57trB`Ir8webnCerUtSd^1p$EeIbABS^g}vir z1x^(JDOWYHzC8gi;w7$knT6;mRy>>jbv0?~gj@!;SAsTCP{j_%!dpvBA*hH;X+4p# zKYsM|K8v--35fKUBODZ@DZrh7_pIEXMqGT7v8LOfDpI7Ow^Dq-(k5Q*47IImZAwb9 zoQ>!S-46ybRODF_6y`MQ3&;b$MN&WZbJU2r%U$x}^OlCsh{~HdQc3J_=|Rd-@nK0n zv7*PgB6`y%D%x5xf|RIaRHhPJ@fD3Y+*LNu(>*a{Vm-n`&cj|p5T&|EDm-Sa(rw?p zWhk6R6^tivywgSA&>gPF7dk7-`Bh0vdU)R%r+;o)t#Mp-M~1XCGOhAzHch75?6#PA zEw|84vPQ@2NRw-W5ou}Qg(*PkZ11f;wAR+r1t-{3w>xK01V3wN=xNTE zwY-vIp$#~)PP}(D8#dh;nT^}-4k5)If|K5-T9q;*FA1@CWcgB&sYm^4q}Rn;a~HV~ zFWo6{h=#O;K4X<7L0$4VrKqs0M7nwwz{8EKwj3%thONajcKCm3nIRGzX{PbWN&RS+ zx|gPI5>TkqooJxop=rQ5U#xc74*bofh%dfwVz0Gnc!4g?E$bqhj6 zU;?eNMn`lQf#~iuweko%8lv?)l=U2B8)fzjR^3V}@SMPK)p@(aHrPtHwJu3{k`@#j zR-K4A)yEZOjnOAGp3P*pCRhuQlGsn7r7S$o5YCmfp~|&$xBGw0!n~B^f=dJU%>iXli?M%g%pjH0iYDja-}-^M4sDXrXk32 zX>&r8fwub(y?W_36c}GOa#~wLQdFbdVy5Hf##?leO>*CIb;Z2fEg)yO^ASikIWM0W zqaY7TwL!kyCM7CmI?|vBI&+YCKSM)au5HF|a)@DjB~G0nn$8|>!ldqqMqw7#kuEqS z{{VKbHxvcR>kiuYg*C)9zf9ER*z>2_()F6>4cExlQlY6oP(@|;FNR%XdY{;uHc7xLJ%(|;1&-asXDd|AHOYNF zF{dIq2?|yUK>q+btRtWx6+I$N$;Q&=N*hYglu<6uh7{VHVZ)u$ruPa5y+1A^$b3KZ zn$o0$pqwAdr)s*k6&9btS#I9D8vg*@<~Zs#V=6ovm5bFfW0IFi)9XP@OC&u10EYU? zHdlmuA8M^m>L_-13v!!#LawukPZe0YF^!IVoj_jYiL}6cX27)p82aw{Uvd^JsP!+R9DgTR2w9745VCvjKq znuiRj`-RHp;AHI6Xh86rRxnQciX!U{N?G~4Y=pSkSn(F5fE0tVDbJ+`QdSzKRom%H zVwnw?%MT?WWhgs)=hBO}^rV7IanOg6f(XvvdY^@Hj{HaLrxDw42wbL~3R=cJKgy-L zHq&m3;-Lp5w%9tgtYoX-dNFERXcA64B3iROZ;jzUZ)qqeK2f+dMX;^6{3qc!{6YAV z*y64dm1&0=^~6+((h!2Qwj1+oI|`@UdZpB`<;mD$HV+4E4&t=pgBIBB`qON23b;aL z)^^W^r0Rxx~a*Ed4`>3yy^P}3j z;*~(+W)#nHGr-x|IY@Y37bUyU5Oe zp{apY*6(Z^YJa4LG@e zGkU>NToH~6efP*eN}7$kvH582%l6cF%I3~*L!^R3eq@o~J?JT2Lc4%0tqsX(r06Tt z=09qW>prdOCLzU(4m<#j1i&wN$K;rGHtTF#55-dvf*Bgb_QYeR$+f}D>| zcIK%F)9jq4t3~>KwB7AqB&K}IPMiasK>aJ>KkW(d_DzER_0}D!^g?mNsqYMg=j&g6 zIEC5cr7;uYAwE?HK=SYS_Qt}nzwJY*9}VXG!ces-#3>6wI0Msuf6ANHVp(yv`&&6{ zlcV8~l7cy>kZ?u?IFzLCkAG_DXef<2RbaDtDH&fhV-dGitP08inhVQfT}Jtk;MTrDFBp=nwoY?C0Gg@ z)Me6R4Kk%_8OGEyQrOXs&#GFw2bLtF$WoZV>RN~%=K%6{sTStMNNMKOLdx}Fc^J;> z89vkipKE#F3NjQ>+EfRNsk5kdIUMhjYAx3uVYcGAa1Y2+p-!!~kB6ZrJ{Tvq<0Ny% zS}%{>^_su2W>wxAirW_yq`8L>($kgaAA}LLHRD`U6}MftlaQdW&?h_g$6_iuj;@Iu zE;DfmWwofS<;DRiJ^aU!xZb9%y=vO!s@oUB)!`+;ogicrwh89}v@1*0{^G3~eo3_B z;BBd|I_@2}wuB=9XbL-?%2DI7B8%7JG^NC4Db#g1q^n>%k@-}2?a7lNtx{TXl}K9K zOK1hp3?%u~t+AZ#o;En3(H{29b+IPv9W1Fn8)|uihcHOSNFZY!@sevfS3ptuC6yXn zcF_79b@Z!s1IqpJ+PQM+#NAYPT_Hj}Yer1Ji0M z=i*3h4Y@U>ty+?Yd}5M)h6X4Z3w=@Ao`#=7m8C$2fpDZ8lc$V!;-O|iyPz^-+g?yw zlpAmrBsNGSWFGk(4CbcpG0^F!EtWBDVTlo;+}v(ks*bk;pFmfKr9&B3*e5@wBlgdN zr%`V365srWxG9y&PSVMd`K%(#zLIR65sA^lbZ}&Vu*WRRCSfY_-wZ> zn3EPG>pCTQdtH5{L_DgOAEV%pV4dqHw@ZO+!(kOeo#?;~=TcgU53>y1-OrEP$&VljCnxs^{LzK#j8EATrMd@WU!EbbCsx{K#(${^r#!-y=m3Y{>isR zx?1BQ;AFiS2}0d$BO$F$Y=BQH0Y?B0vS_5)I;9MzTh}ht8e931K)~1O^m-PHO&&g-ogqH zLVSvEAfLCmsA<+_Ww|ANOVqaL(4IzLW*9jw={?-fhBA9|M0EAfqbxSYI(pwd;>;$T zZM8n?oJkyjRISE$#{+y-rx zY^7;A!R!DuJlA3645DaZz%?J;l_yZ_4l$g6N`BXMmKkYJjcAkSL0e}yJRfB7liGr= zfbEV&wlOM0nxhzsQo@4Pqm>+ICZcX}82RZboeh;V+QADe+02c`YGJj`W;2uIN-Il_ zD6N;U5Z};t7~Z2MrqgnemdqKgI#rKFsf_>MsvesxX=o0BOGd@kwc;l^q_WCohY zgq-s4YsA+TB)?wKJh4`<(4Zyt`v7ObPD=uSNpZg#t%!erdCUMNNpgup(6n&_=(Ow{`7s% zsUwuBxwc0_6OCs=FrQ4C7LpFzDswOIu@O$rwntI|f(TkQC}iU}#}$!vx5B8se1%;_ zurUR)RPU1C znGs(xXl>;_l9s%=DObO0+HH61>K_K%ku_%qZGNAaf!GM6Q4P(C8iT1h(*fuHoNB!7lBxOP;| z>NKI$u%8iXKv{4B$n`vNTfA#zn2Tm(5rer{Z6VQ(3G7q>-yGCe#ICBi*xI+VVd&G; zQtm5TOJ$s;q=AFyN_@j#0G~rhqSD(Q)R{rB^wp040Mlxj7F(!s4i=e&{6#dj(l!Uq z27PKi4@bvj_!2IGD)9hP+Kjb?kaM;+C)%Hr@nfy6Eh6Q6_(A&~+Rm`zB!`Jtn*m63 z7|7zaE}fNRz1Si3N9_0Q%}y!9t`Zwr#@a|br_hl?SlpGSGLnaAt}4^h@3yDy@*8{Q zj#O6BN!ELTjEvO9yQksJj$z7E#rX~DLQv5u#y85lVx(u<^g?yVCxoGuha7N(q$mTj zb>olpp;j(0Ee}MP!>oj*X>7iR{6rIwpsn?7`R!Vtk!^S(Bw21S-1$XA`Q~FB_*(R- ztPF($ds7WG{G~7#;kBW#;`BSLpyzDk*NU^;AqbkT7dF-J!5&@k4R#H( z*B)EnvXoo#)17RTgs%i+R!-wN+qNiD=@~zfrQR+woO%;oK2){krz*xa$2(MQ<~6=e zyge}wIUU!Vc}q!INeD(Y0)du_v${`uX**Xgz`NH)YIHeoT-k;a9&^E4o(rHB!5odZ zCZ!W*m2VJNSp`M7`mhRCf15q` z^u-*~Ubwc}%vm2Ub*#9{%Slp+%8FB{1B2;NaJJ~8b=a##sgWa7P@}3-g+a!=EDkov zmiQM43M9mzydU_d!!Z zrVEY$dt_}!Na@R6t6bP|V>2PQf*g?MgoU^{2@VtM>%9|iw@Q%X>1}R945+?b2|+<# z2}n+o6^wTGrcrN97NfKTh&SkR{&L-LRA$&A9s}|64tx2qM)Wk+7MP|>mewZRE8OH;ELNm7Xsnac!87TSZRPLsf zg+^^kSSrp6w44;`6dG5ctF45x?XqgyX_dr;meZGyiW0VzdDIVYtwVJ+@-$!n07O~j zL%2K}{6Hxh#&-ck^Xv^uTDpEbYn;2a&N7@#rcm3>1eX)7CuNOXWO6-fBdD!*dpwY- zw#m;#xDeEI!fCOoTM!OYf%WIm&~2r36;1YMm-|)bEwSTgzx=t9qPOEZrMNWk0ZR5N z2Vi`swHrR-%+8vHXzF`Zcm7&39*=RhGQ2pBQWTVr5eJ@lJ?iF;qh?*%7W=R5II~fy zQQUdPmg3qfz|SM`4*kYQHO2IWsvUx2?Jc?#JQ#?|xnhSxURd0+cQax$VOVv^g*g8IWKDd4!2k+|F*af-5hKkF`~gK&vr=(&!w z4fQ}gn$adk}`@8!-m`cmy+Met#9$@4 zB#ilRdmm6w^{J`%yK@L)E0PdjeJWZTTZH(DNXaS2GC<@F!1k`1Oog#-GLokw_&_1V zge64esaWA&n-9H5O)aGjHubHiSxE?5Uv0G^dD!4DW9dOny)?`QWtkCMLyJgpjb%z& z4u1&)bq;a$s=rs<7beOjvh=|fw)G^XwXL@rBpqrA1zMB7K;E;GqbRRE9~}PxYAX*< z*uF1Zp!CmAHD!)jX~!cn-iE`f1gS|#*aNo7+OkaxN#oMB{{ZbB0^=XzOS0v@!|`Rq zW!P^rwJ8cf*x;+p&tftwCvY74*Ny68mRTtDcH&9F4W-{Mk|_>kf}%IB>ECMe#3eZU z*O`@W$;}}d8guDQtmJL^Qgvr@T?p6TA8O^B*~d7Z@{Ne!+P#CQC{uXz5$I^PThC)g z6K&BI3uS|*-%3`UTW?w%ILx}*9ce?{R%LDqXy+Ii+cjy0b#b<|>&6D#*Q@F0s%KU_ z8nWaRhJgxD`cRUtYiTO*$=i+Uz58k6cUHoX_04YWwm7b-WJyvT;UrD^LQbN*V~S#3 z=}$C5oF_f14qTP$!c7KN=?6QGcCIP6BSQ)L2D8+YXLzWzuw(dvy&>>)HV#UPkK$G^ za0WYOr@v=1rI9Imkl{K&KsX(Tpab=;w&ELF&3M|yz`WDT{qKo@ zlpuNv_L7S1qjHvzWR9A)&p?FiH8~h7P*&grw&UMF(v>@;*pSOnU_F4+w4{NoyYp$CNRj1O+~RA9KxsiY}MF6ALeIOo!xDE1?A8HsI`5tkV8guO%s z6yqr)vBxJo(9+WaEW)|7&|XkNv^j-iAA}#FB#Jf1B}8FJVL>`lMv<|}_okNtE-)NZ zZ8ml)2?J7z_4Tact42-U!R95`VQE_3eWffE4Py!*=Wq0;U^5g>b#BB_a|10cN?U1H z0FK!n)hJ=1wu5SYL4S!m4TpWZQzitj3D5VS2ILHGN%x`0$axX(E^JwXN}6BGIG}{3 zCt8Wd{P2*FHXX_KqLB5*VplB^-D3$M0mON|j{9##n`#t-+wtAXz;DMZ^J5$1(uR)0 znjwhtlC&^Y`1T~7vPtyCZFx2P49@BV+Z}1Eq-R}0bHhs72?;7YW7~pBG+36~>+w|6 zjHSXbLbUmPN%sE$N)gy|ZeA_EN63L7jSgU_j2z_Vj{$In6zGR~#W${{Vt1{-Bzo zx89V)VdJS@3mGK#$;R|VOZC~;K4LB9)sS#QfayPD>+MIP(bIEK1QqH~PIIqw>S{S>odidt?UcqHcl_5&nizG)>HCa4Ngr?IyMj#IzyjkKp$vEn+GvQ@fx8}c-P z^rJ)83r)FA+HF27!P2GaWhY+brzJ{jbWd3Wo(t7YW3(ej>`sXy-V-zz{*C0QdFPC@rM98;Emh|JxZ z(dSN4kaL@ae;zH4l|R!^`eV4t)N4TdLdrQB>9d{Vt4OQ&h%eO!j`#BMM-rB zDp(2yY8>OSr*_vsQe70wkL)SZN>MgLQ(jsS17xftCnIC@JNwXgc2#F@Lb`?M;l&cB z(v_?YKqM3Vpb|L8>slNb78Jaawp%ZS0EL~m9DQre+oIi^ek*d2zZ@rCmPT@&!QYde z{RRaUIYbR6R;(zLy$m&Tf5OqNZ#>ePDuTSuzW>&`{JyrjC}4k3C96QgVZ zK|ARi=Clv?W$4iS7tkGHr0GgRcE&MP3yYC0i*cmB+ersxsNf+Uhi|CG9#R6Q(1qXb zO{fOhjIsgMl^t%WSOob`+XK>?W2kPi(#!9%r<)Hg)hMk&PI+<=2LKOBIWu}%LyWf~ zeYhPsO0p6Zka7oXeWZfHvv&DJ0 zp`aAEB;##GCw|0iK8CL|SX9^00HSitJ4knM`WsGa-igv z*(eE8R)-VE@TDN~HpkMU-kg?nIO`H7A(W{_rm&Q_NhM)9!Oq;Bw#I5^Yf@7kGT6D0 zK-8AjfW1h@3C9V@-xV3u5~Rgy8+OQzsX-1rveHOEQb_WpImjF5(!HihLe+E)DMCmeSfr>Dn^w7S7vdpvb`Hq$8@+Jbdw;CQ@8R8*K>%NgyQWAaY0owZGgnyFGIWwMA*D z2vXDn8wm*jEBs2|IKesgpb)O8i2KUAp@}h^bm?0_0HlmzBmw22E;HqY!GUWNC+K`4 zKqx410HSyH8*`2*vi0?roVWhRbxg-}INDS<3mM5O2aU+bxb&rW;44dE4O!pti?ZP* z<&fjr7nEfj;NyS+7*$BfL{FD)Sy*#AT2o<>fsKIA{!#YcyKiaM8jTs|08E4^FF-hK zl$?MG-z4K1?M*(^`0<;zMRk-Z!+K*GR3n`e=Tu}`WT#(L6a6F^{Gmt$!sAyJkB6MZBJ>?b>w2dpme9F#9`c#Y1 zB!;r<wbra70>T)b6VlQ-OLEDFwKTcnG*={ z4vi~l05p#_*ubM0wkkB|B{v~@p9l{v3IVW$j3)r@HzXbC_TWY`o0A}=Dnd(P#VgEG za85wmw?AqrxqZOt6-F-Cq2ot3-`XgYgslunQeQ(zC+HGRaq4#VqHRO0Zr1r-KJ*}% zt&o?QX&@zp?4S}ewn6PvW2e&@9xb`ThEz)N_*+OPH~{WPYGTPQY)g!{ZMO*SA=idk zQZQAGv!Ay#A3M-j7QW>zuAS=Gw^MXt?7(3$$AtmFml`SubrNz;Kmg-$n!2-aQcF!q zf#Dz^Eoku2RLYb#8O91m<8Nw{op^yU#OPrm=!6oMl8|>J&Y$KVr!@h7yveeq7U(e_ zQ;!`%_fS-jNF?NgfO2@_w|XgY1-VjS9ZhMEcbb$vbf0Stv*D^kZK2XZ<5HA110!M0 z1#P#xZMB;)>fhQCwKX0)veMU-i~`q?1rIPjyB>Sgn-inBpO>EVD9T2sT`I$h0|1@& zBB(bxH&jfTfXhvmkRB3Z!`V%za8kSw4&)Gdthqjd)fyC5Au!rLPC^_FtxX>Z3Ri%z z0N*)YoyHHgC=R2Oc6L&tTkX;=_V!fTq$##^w(1Tt*&$k$zJG*c1DsT4t?S27!AmyQ zqgzZ>mmPTuSb4>T94iUi8)tu|NUF`$w#&oP?|n?ll_9jGhGGGh^>BS~PwHsXUrP}* z-H^AK9Za<{EapX~-s+aO5~zYg##8{(0ZB>JNx{x)JEv9EQsz9&`7$HNg(pNgYfgd2 z4l;441n1P8cB{Mh_)*e}EXZ@;EbAdnwpI|XkIf(E?H+gqKhx6gYLHEa2oRy@}r&?hQz)G$^W!#hAv21-zLI zw9-zMV`U?JkapPdiW2b#)RNL+Jh-WBmy*K|C@IC0xd4NYTyQDl7OueMszf`G`jY1v zHrqY>_GEtuw#xi;2?T(ZYC2M_N*Px9!PtydzTat^Z*nSbcR!i|r3RdF(}bx+5{)2| z6R?KWB*&)ZCb`K4NL{ye}evdA#G|Pz4av@wtPnN>0O02pqa}MYkHBMh9+c`AUuD(XB{r8lGRw_vfG~IdoYZxa z4bsnYo^1>@COIAvY3~smfvF=pcfs2{;L~jMZMsY8Ri5ftWy~c>)2S^aZ>2rK=LeDR zML|kpt~7Hx?ECQ+wV^UlQXbMgrALJUw>#q%R@GctHFvaeQ$)uS8jXiz5BJ8CDJx4Z z6!GD>HykG*6pNHAv$GhNZdhtiY^13Tv3r-8iKM(9E@ihAEiW_L>rT>%~O&jA~@a05TJ~xk_ZQJr(vEwYAP)I>{&6;&7;n+ zzZWiLB_t>Pa~q`{t(Rnxe6f z7EUV8)CV1DxTuo3ua^{B!Es7Q;oOms4y=whrYtfp5M?S`_Tv@FGCUVsX_S$tI~8)I z0CT?mxHaha#UeB27f!z&D|7}>$jX%2<$L4;JC69L?kS$t!W?->E+M?Fmk;rIch8~Y zYAJjQ_1Wv>x7~>v)Y}poU=Zu!%YL}-#mNe5DIrK-2U2*dGyM%3Y&A7%Y8u~l_T)CT z%34t2M*%?KYAG5Hz~ejmT&7$dacP8saZie%U1kjIGpcP$ zISL6<)O6q|9rT^)q?!U~i?m;^GTvMVIvCrZL#{fh3`}Wp(#}Xi0NH14lt=^-+-_&A zoh3F)?xN{@THRssDl(h_N@RkqBLti%B`vkVa_W;-lS=?zeykU=1dw#GykT~lS1xY9t% zin2#xiheX0@rqLgT3b>{DM|p!yOZgk>s=l6LhG~TFYQh6&#W%;&s5tjZO?}RhaQyk zLl3Zke8WjPl-SOD>EE{X^2*VXo@?|=s+PsX4!yt7O;|82< zU?&?;Za=4DcFt>veObpf$&v>=kaI&*n~wD@8aOy43=Y*94=PZ=sm2J>J66RCV`IjL z+CVD61J;Vpl;bqhL(7n9(ZBHxtI&p#+X!lpC2B~d1t_bYX~zI4+L>$>L zskqSGg1|x$R6y@Z64cVN>pN$9_Y{6kPEWxO*>W;lN~5GD0|O-UM+*a(U#psb#h2sN$Xiu7XxW!J5OCxYSXp)sA zsE3XR2^1UakjhG$Z^R>f8XYru7#F#%AQ>$jr|=^r@7pzBUA}yo#!QQwr^HcFYzzPo zYLgc!0PB>}QgfVQjcP)k{rZ{+N$-wmxG5@-RNP&ml)Cv~#%VbyBP;$OJ*b#79F;XT z#JsXqzL0hwS{TS#dBO<7h5!`&M$*Uo(>hS_{vbCrwAVq?J$UGL9+assIHx?M6zaeO z+*MBOu9&mJlM+)ThZT~Blag^(ONIL8j}u8J#2621(3aosL57%cEjMrYy<<1KvbokL|BZi zu;-c0@K1VnlXVhq(zXdUJLrC0S})C%JxLE|7%P9gW#hn~x4k&0xrBYPq)3vyG9Or0rZEu?5{55x7Z zPJTMrN|hyODf|GOfDbh|?XJ}GJ{m@q5HfSkN8CDEG?9S4cpE1>QB8i9X4Biz32t&5 zPb4!dTFE4Mz#aQyoQz=Gt}Z(+VUV>eDo#cQPaAfoWOP!N>R)K7ryDp<4&Rj&&qluF zZxL^xyU94_mBQ5_o<(+DVqThiN(0CV=NYIjw~cJNDmwx*YI)^jP#=VPnutY((J`=O zl#no@I|?H0X?i;a__CB#k({fVdXrs~`ct%ub*(Oquc5T%RagD`49_Ky65ajnkR@!xQLf%xS9ZH@GDkG40syHt!%2VjJLAS!T zKuj2jM}Uph}uqb-$dkh0^cSC|E3x1gZl#!^YpY&iE9BDz$| zno80V6p}~yjY%;H^{EWEAHz|=1+;0@alX{VFNqdQgfyZv=|-{wytpbk;;iwl@-8T5 zY7}yv(2%SEDR{V#?q6kTrM(8=q$T%KmldIB83z?54m^fuxiOhV#;~l32W65!7CWBQ zW#TjJge|WDT27!Ddq~&5D)F5GHx>v>5kGgsd~~KR_Y#=Hkb*1Dpe8Cf>I6?ov~F* zoV&%kYNJMm+R{IZj(^g!sO_0PT12#HvXTjQC}jmo-2;8;O1E3tOV3P>9L^k0dCoW{ zg|^&fy2xf@psAs;Pj!3!sn>1}w;jhGM~*RMZbDZ;-oCxqj*J+C5u!iQAa8Ik)wSAZ0j$U}&48c8|9^%*>7Gq7TKu9^%SK5Vpq#9y6P#;kNc{n*3tvJSU1Z3kbh-U?= zYhD@=t>*zz+sKjhG?}*ub1m*$V<^K>K44lX4gfKv_NlpjDxlI(!cLMis2mYOLFsup zLlTv!DB!EmR;YFxZ%mcwtId`Ly)fi>4aoyju7v(9_RqCUyVPAa^SWPbD3gTc1A46v zl7NX1u0SOT7)T1iKK1KMeJqJ34>IX-uym7?y=igq1e@&%yWLv(sV+r;rAH{#*eJ+S z`wvm|rmc4OWwWnBV93;!tteKu)~tRU`|xPV4gBNmn2ZFaCkjbC`;U5-W&~KsQlt`e z`GP>(H7eNmNQ1gsX16HIi7iQ55}dc;qX3S5>OWI$rKc{ef})@Uji>O8{t$f)6@8iW zPOZqWB&p4b$RQ^KZRj>FcI&Hg+A`!YwK=J5g`8>~y{J7s47%BAdD!|A@*Yx@?vQ-g zC`sSSMG16%)Z6jdFw*O*)ZsbFPuJYgeMIHPD-6Z9+w~BfI^wv-Hak=dAKPVX(7kRs zk1<9ORbruC_Nmj$5z20?MW#w{{ST;jn>ByxVGj}(u*y$ zP7o30!1wo|ZfzMMiBY9K`n4z_q>na$GH`uPDT~}>+?-`@L`Q&=k`ws7shb_K#?>G( zA>hhRwzV&G9^iXYa#If;9!YCN%RQz~&!PgN>p*2*Fw}F%KGb6RvAd8=1Vc-W$Wp>y zX$J#I-%;(J)B>P2p~ob;>w^HODCaxWi;K0zjM7_dG?i{bM{lKCH_<2ZD%RDRFTZX~ z2G^taiv>El?~~0#KW%9;*CNisQtPCnLh>3`HyIwHr(wBnN!HMD)e-zX^IQvn+=f|w zm9nKP0Vn#=tx^}wkv9m}$&E&C5sTuof=aRgKj9yKCMt&>SgGky`SS zpP=L3qpsIkv2F>QaDAmjcm~8kfC@ny9QssM-s>{ZhZ>UVFb-Ql-9S_X%k7foi4se$ zx`d2?qB4*-7@@YbGF7%+=HKGPf90Bt;uKN?j-^A(>ACi$T`m)%MO51qS0;SQ0+r&s z`@gCG0JUeAA#Cc1iD9SZ#f;hW1T2D~{HMMuwIoiky7jq>thS&?1R(+9cRjtUe&MBB zHr|=I&zm7hmda4#8&8O>I0W_y&m$C+N}F?@^|_ZCEAc$Z5y(cLbHP61unAXgyXhAp zT&?pJAv(iu1v*u&8QUP#>;C`}di@A)Q)@&xs&Hybhwn4ok5N=$*N^VmjC&ONq z1N^|$bO>EV@a+t#cP|~4CqjWonS1h2uJrbhotBQT)76$mTr945&mjs4XsD#2c*b+= zM}9%C&a0x`jc}67>C)qGEh|XdN`j6?KDGKiW$FtYxc>k)&X}lH6r`Z;Ro`y)@E^w* z@gaOs>IR?@^6QkR5rKi`+~8LauHH2M27{|9tCBe1Gz;(JB<8wTf%QJLE5}*;*O1d? z5^`fMKdoEs=})zdhS@YPhM=qfcRZSXuEQJsDmf!TaqvvRR*{^Fm1X?nd}5%OBmqYu z>;OqQ_ol}Nwo(zIWw|4a?N5gQ?@;YLLBXQK!%sP@ts?87)jl!K1u)#82?rczgj?Cj z-v?~d0q~QJTl!F)Bu05SlFE^*kN*HQH{%kWhQ@&sPIo( Date: Fri, 7 Aug 2020 18:05:37 +0530 Subject: [PATCH 25/42] changed environment file name for PRT --- ci/scripts/linux_test_perf_regression.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/linux_test_perf_regression.sh b/ci/scripts/linux_test_perf_regression.sh index 070b10a07..5fa0f7271 100755 --- a/ci/scripts/linux_test_perf_regression.sh +++ b/ci/scripts/linux_test_perf_regression.sh @@ -18,7 +18,7 @@ pip install -r requirements.txt pip install bzt # Execute performance test suite and store exit code -./run_performance_suite.py -j $JMETER_PATH -e ci_linux_medium -x example* --no-compare-local --no-monit +./run_performance_suite.py -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit EXIT_CODE=$? # Exit with the same error code as that of test execution From 2c1d363f1007c9af46282cbbfeb4360f6b13e61a Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Mon, 10 Aug 2020 12:21:30 +0530 Subject: [PATCH 26/42] changed S3 bucket name for test --- tests/performance/agents/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index aacbe97e1..7e7caa67c 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -6,4 +6,4 @@ HOST = PORT = 9009 [suite] -s3_bucket = mms-performance-regression-reports \ No newline at end of file +s3_bucket = shivam-codebuild-test \ No newline at end of file From efc1347ebfe9085db3f94d185f83df0a09882964 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Mon, 10 Aug 2020 20:08:34 +0530 Subject: [PATCH 27/42] updated buildspec yamls --- ci/buildspec-PRT.yml | 10 +++++++--- ci/buildspec-nightly.yml | 20 ++++++++++++++------ ci/buildspec-smoke.yml | 13 +++++++++---- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/ci/buildspec-PRT.yml b/ci/buildspec-PRT.yml index 9fcd60892..1e49a93d1 100644 --- a/ci/buildspec-PRT.yml +++ b/ci/buildspec-PRT.yml @@ -10,8 +10,12 @@ phases: - pip install dist/* # run PRT - ci/scripts/linux_test_perf_regression.sh + post_build: + commands: + - mkdir -p prt_artifacts/logs; mkdir -p prt_artifacts/run_artifacts; + - mv tests/performance/logs/* prt_artifacts/logs + - mv tests/performance/run_artifacts/* prt_artifacts/run_artifacts artifacts: - files: - - tests/performance/logs/**/* - - tests/performance/run_artifacts/**/* + files: + - prt_artifacts/**/* diff --git a/ci/buildspec-nightly.yml b/ci/buildspec-nightly.yml index ce916c8e1..50a2a1ce4 100644 --- a/ci/buildspec-nightly.yml +++ b/ci/buildspec-nightly.yml @@ -17,10 +17,18 @@ phases: - ci/scripts/linux_test_api.sh inference - ci/scripts/linux_test_api.sh https + post_build: + commands: + - mkdir -p nightly_artifacts/build_artifacts; mkdir -p nightly_artifacts/model_archiver_tests; + - mkdir -p nightly_artifacts/mms_python_units; mkdir -p nightly_artifacts/api_tests + - mv frontend/server/build/reports/tests/test/* nightly_artifacts/build_artifacts + - mv model-archiver/result_units/* nightly_artifacts/model_archiver_tests + - mv result_units/* nightly_artifacts/mms_python_units + - mv tests/api/artifacts/* nightly_artifacts/api_tests + + artifacts: - files: - - frontend/server/build/reports/tests/test/**/* - - model-archiver/results_units/**/* - - results_units/**/* - - tests/api/artifacts/**/* - name: MMS-NIGHTLY-$(date +%Y-%m-%d) + files: + - '**/*' + name: MMS-NIGHTLY-$(date +%Y-%m-%d) + base-directory: nightly_artifacts diff --git a/ci/buildspec-smoke.yml b/ci/buildspec-smoke.yml index 2c2f477a2..1a4467c7c 100644 --- a/ci/buildspec-smoke.yml +++ b/ci/buildspec-smoke.yml @@ -12,9 +12,14 @@ phases: - ci/scripts/linux_test_modelarchiver.sh # run python tests - ci/scripts/linux_test_python.sh + post_build: + commands: + - mkdir -p smoke_artifacts/build_artifacts; mkdir -p smoke_artifacts/model_archiver_tests; + - mkdir -p smoke_artifacts/mms_python_units; + - mv frontend/server/build/reports/tests/test/* smoke_artifacts/build_artifacts + - mv model-archiver/result_units/* smoke_artifacts/model_archiver_tests + - mv result_units/* smoke_artifacts/mms_python_units artifacts: - files: - - frontend/server/build/reports/tests/test/**/* - - model-archiver/results_units/**/* - - results_units/**/* + files: + - smoke_artifacts/**/* From e6c7abdd0cd0096f8d9acb6eb99db826faa4b95d Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 01:34:16 +0530 Subject: [PATCH 28/42] removed tests/performance/logs/ --- ci/buildspec-PRT.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/buildspec-PRT.yml b/ci/buildspec-PRT.yml index 1e49a93d1..41d50a6aa 100644 --- a/ci/buildspec-PRT.yml +++ b/ci/buildspec-PRT.yml @@ -12,8 +12,7 @@ phases: - ci/scripts/linux_test_perf_regression.sh post_build: commands: - - mkdir -p prt_artifacts/logs; mkdir -p prt_artifacts/run_artifacts; - - mv tests/performance/logs/* prt_artifacts/logs + - mkdir -p prt_artifacts/run_artifacts; - mv tests/performance/run_artifacts/* prt_artifacts/run_artifacts artifacts: From 827f38f33259db0fa677caa2c4207ded9c3746d4 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 12:55:17 +0530 Subject: [PATCH 29/42] added changes to store comparisons artifacts in a specific folder --- ci/scripts/linux_test_perf_regression.sh | 2 +- tests/performance/agents/config.ini | 3 ++- tests/performance/runs/storage.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/scripts/linux_test_perf_regression.sh b/ci/scripts/linux_test_perf_regression.sh index 5fa0f7271..02184cc81 100755 --- a/ci/scripts/linux_test_perf_regression.sh +++ b/ci/scripts/linux_test_perf_regression.sh @@ -18,7 +18,7 @@ pip install -r requirements.txt pip install bzt # Execute performance test suite and store exit code -./run_performance_suite.py -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit +./run_performance_suite.py -p api_description -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit EXIT_CODE=$? # Exit with the same error code as that of test execution diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index 7e7caa67c..28827140a 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -6,4 +6,5 @@ HOST = PORT = 9009 [suite] -s3_bucket = shivam-codebuild-test \ No newline at end of file +s3_bucket = shivam-codebuild-test +comparison_artifacts_dir = perf_comparison_artifacts \ No newline at end of file diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 6098cb4b6..7b4df811f 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout, format="%(message)s", level=logging.INFO) S3_BUCKET = configuration.get('suite', 's3_bucket') - +S3_COMPARE_DIR = configuration.get('suite', 'comparison_artifacts_dir') class Storage(): """Class to store and retrieve artifacts""" @@ -107,7 +107,7 @@ def get_dir_to_compare(self): os.makedirs(comp_data_path) tgt_path = os.path.join(comp_data_path, latest_run) - run_process("aws s3 cp s3://{}/{} {} --recursive".format(bucket.name, latest_run, tgt_path)) + run_process("aws s3 cp s3://{}/{}/{} {} --recursive".format(bucket.name, S3_COMPARE_DIR, latest_run, tgt_path)) return tgt_path, latest_run @@ -117,5 +117,5 @@ def store_results(self): if os.path.exists(comp_data_path): shutil.rmtree(comp_data_path) - run_process("aws s3 cp {} s3://{}/{} --recursive".format(self.artifacts_dir, S3_BUCKET, + run_process("aws s3 cp {} s3://{}/{}/{} --recursive".format(self.artifacts_dir, S3_BUCKET, S3_COMPARE_DIR, self.current_run_name)) From e42796c2e523fad03e54ec7ac82d70befa53cfe8 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 14:06:43 +0530 Subject: [PATCH 30/42] updated to read compare from specific s3 folder --- tests/performance/runs/storage.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 7b4df811f..2d3a398a3 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -32,6 +32,7 @@ S3_BUCKET = configuration.get('suite', 's3_bucket') S3_COMPARE_DIR = configuration.get('suite', 'comparison_artifacts_dir') + class Storage(): """Class to store and retrieve artifacts""" @@ -60,7 +61,7 @@ def get_latest(names, env_name, exclude_name, compare_with): latest_run = '' for run_name in names: run_name_list = run_name.split('__') - if env_name == run_name_list[0] and compare_with == run_name_list[1]\ + if env_name == run_name_list[0] and compare_with == run_name_list[1] \ and run_name != exclude_name: if int(run_name_list[2]) > max_ts: max_ts = int(run_name_list[2]) @@ -93,10 +94,12 @@ def get_dir_to_compare(self): s3 = boto3.resource('s3') bucket = s3.Bucket(S3_BUCKET) result = bucket.meta.client.list_objects(Bucket=bucket.name, + Prefix=S3_COMPARE_DIR, Delimiter='/') run_names = [] for o in result.get('CommonPrefixes'): - run_names.append(o.get('Prefix')[:-1]) + prefix_list = o.get('Prefix').split('/') + run_names.append(prefix_list[len(prefix_list) - 2]) latest_run = self.get_latest(run_names, self.env_name, self.current_run_name, self.compare_with) if not latest_run: @@ -118,4 +121,4 @@ def store_results(self): shutil.rmtree(comp_data_path) run_process("aws s3 cp {} s3://{}/{}/{} --recursive".format(self.artifacts_dir, S3_BUCKET, S3_COMPARE_DIR, - self.current_run_name)) + self.current_run_name)) From d4cd95fb965450cb4a37359193098acae4a1e50d Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 15:17:38 +0530 Subject: [PATCH 31/42] add / handling for comparison result retrival --- tests/performance/runs/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 2d3a398a3..0c8754ecf 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -93,8 +93,9 @@ def get_dir_to_compare(self): comp_data_path = os.path.join(self.artifacts_dir, "comp_data") s3 = boto3.resource('s3') bucket = s3.Bucket(S3_BUCKET) + prefix = S3_COMPARE_DIR+"/" if not S3_COMPARE_DIR.endswith('/') else S3_COMPARE_DIR result = bucket.meta.client.list_objects(Bucket=bucket.name, - Prefix=S3_COMPARE_DIR, + Prefix=prefix, Delimiter='/') run_names = [] for o in result.get('CommonPrefixes'): From 0d9023a8b82418d230fcad02a82321759ee75339 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 15:33:06 +0530 Subject: [PATCH 32/42] changes to run full PRT suite --- ci/scripts/linux_test_perf_regression.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/linux_test_perf_regression.sh b/ci/scripts/linux_test_perf_regression.sh index 02184cc81..5c86bc4d8 100755 --- a/ci/scripts/linux_test_perf_regression.sh +++ b/ci/scripts/linux_test_perf_regression.sh @@ -18,7 +18,7 @@ pip install -r requirements.txt pip install bzt # Execute performance test suite and store exit code -./run_performance_suite.py -p api_description -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit +./run_performance_suite.py -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit EXIT_CODE=$? # Exit with the same error code as that of test execution From ec6598776095ec1feb09e1ab784ac6c0d99cdb85 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 15:46:40 +0530 Subject: [PATCH 33/42] fix error --- tests/performance/runs/storage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 0c8754ecf..91612d997 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -98,9 +98,10 @@ def get_dir_to_compare(self): Prefix=prefix, Delimiter='/') run_names = [] - for o in result.get('CommonPrefixes'): - prefix_list = o.get('Prefix').split('/') - run_names.append(prefix_list[len(prefix_list) - 2]) + if result.get('CommonPrefixes') is not None: + for o in result.get('CommonPrefixes'): + prefix_list = o.get('Prefix').split('/') + run_names.append(prefix_list[len(prefix_list) - 2]) latest_run = self.get_latest(run_names, self.env_name, self.current_run_name, self.compare_with) if not latest_run: From 9c3ed414ca62f81e58908e110cd7597e0bb383e5 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 17:19:10 +0530 Subject: [PATCH 34/42] Dummy checkin for comparsion with previous PRT run --- tests/performance/agents/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index 28827140a..c5be6f0d3 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -7,4 +7,4 @@ PORT = 9009 [suite] s3_bucket = shivam-codebuild-test -comparison_artifacts_dir = perf_comparison_artifacts \ No newline at end of file +comparison_artifacts_dir = perf_comparison_artifacts/ \ No newline at end of file From 9d8484230007e517d2add7f1caaa29d0c59e2406 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 19:03:21 +0530 Subject: [PATCH 35/42] PRT_2 changes --- .../batch_and_single_inference/environments/xlarge.yaml | 4 ++-- .../tests/batch_inference/environments/xlarge.yaml | 2 +- .../inference_multiple_models/environments/xlarge.yaml | 4 ++-- .../tests/inference_single_worker/environments/xlarge.yaml | 2 +- .../tests/register_unregister/environments/xlarge.yaml | 2 +- .../tests/register_unregister/register_unregister.yaml | 7 +++++++ .../tests/scale_up_workers/environments/xlarge.yaml | 2 +- 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml b/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml index d42bbd30a..e4ab23643 100644 --- a/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml +++ b/tests/performance/tests/batch_and_single_inference/environments/xlarge.yaml @@ -3,10 +3,10 @@ settings: env: API_LABEL : Inference1 API_SUCCESS : 80% - API_AVG_RT : 30ms + API_AVG_RT : 2.5s INF2_SUCC: 80% - INF2_AVG_RT: 30ms + INF2_AVG_RT: 50ms TOTAL_WORKERS: 6 TOTAL_WORKERS_MEM: 999686400 diff --git a/tests/performance/tests/batch_inference/environments/xlarge.yaml b/tests/performance/tests/batch_inference/environments/xlarge.yaml index c1f8df5df..73b2d8379 100644 --- a/tests/performance/tests/batch_inference/environments/xlarge.yaml +++ b/tests/performance/tests/batch_inference/environments/xlarge.yaml @@ -3,7 +3,7 @@ settings: env: API_LABEL : Inference API_SUCCESS : 80% - API_AVG_RT : 30ms + API_AVG_RT : 2.5s TOTAL_WORKERS: 3 TOTAL_WORKERS_MEM: 3000000000 diff --git a/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml b/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml index b68f5c08e..38e33a727 100644 --- a/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml +++ b/tests/performance/tests/inference_multiple_models/environments/xlarge.yaml @@ -4,10 +4,10 @@ settings: env: API_LABEL : Inference1 API_SUCCESS : 80% - API_AVG_RT : 30ms + API_AVG_RT : 200ms INFR2_SUCC: 100% - INFR2_RT: 450ms + INFR2_RT: 550ms TOTAL_WORKERS: 4 TOTAL_WORKERS_MEM: 600000000 diff --git a/tests/performance/tests/inference_single_worker/environments/xlarge.yaml b/tests/performance/tests/inference_single_worker/environments/xlarge.yaml index f70048f05..1dbeafb0f 100644 --- a/tests/performance/tests/inference_single_worker/environments/xlarge.yaml +++ b/tests/performance/tests/inference_single_worker/environments/xlarge.yaml @@ -4,7 +4,7 @@ settings: env: API_LABEL : Inference API_SUCCESS : 80% - API_AVG_RT : 140ms + API_AVG_RT : 330ms TOTAL_WORKERS: 2 TOTAL_WORKERS_MEM: 300000000 diff --git a/tests/performance/tests/register_unregister/environments/xlarge.yaml b/tests/performance/tests/register_unregister/environments/xlarge.yaml index cbe49892b..87137002b 100644 --- a/tests/performance/tests/register_unregister/environments/xlarge.yaml +++ b/tests/performance/tests/register_unregister/environments/xlarge.yaml @@ -44,7 +44,7 @@ settings: CONCURRENCY : 1 RAMP-UP : 1s - HOLD-FOR : 300s + ITERATIONS : 1400 #approximately runs for 5 mins SCRIPT : register_unregister.jmx STOP : false #possible values true, false. Bug in bzt so for false use '' diff --git a/tests/performance/tests/register_unregister/register_unregister.yaml b/tests/performance/tests/register_unregister/register_unregister.yaml index 35778217d..0bf11cab7 100644 --- a/tests/performance/tests/register_unregister/register_unregister.yaml +++ b/tests/performance/tests/register_unregister/register_unregister.yaml @@ -1,4 +1,10 @@ --- +~execution: +- concurrency: ${CONCURRENCY} + ramp-up: ${RAMP-UP} + iterations: ${ITERATIONS} + scenario: scenario_0 + scenarios: scenario_0: script: register_unregister.jmx @@ -9,3 +15,4 @@ reporting: # Inbuilt Criteria - success of UnregisterModel<${UNREG_SUCC} for 10s, ${STOP_ALIAS} as failed - avg-rt of UnregisterModel>${UNREG_RT} for 10s, ${STOP_ALIAS} as failed + diff --git a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml index 5beca12dc..0b9082b0c 100644 --- a/tests/performance/tests/scale_up_workers/environments/xlarge.yaml +++ b/tests/performance/tests/scale_up_workers/environments/xlarge.yaml @@ -8,7 +8,7 @@ settings: TOTAL_WRKRS_AFTR_SCL_UP : 5 TOTAL_WRKRS_B4_SCL_UP : 3 FRNTEND_FDS : 88 - TOTAL_WRKRS_FDS_AFTR_SCL_UP : 38 + TOTAL_WRKRS_FDS_AFTR_SCL_UP : 43 FRNTEND_MEM : 1000000000 TOTAL_WRKRS_MEM_AFTR_SCL_UP : 796492032 TOTAL_WRKRS_MEM_B4_SCL_UP : 115000000 #115MB From e76ff40cfe5b692f5a879cb7add2aad8d63f9146 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Tue, 11 Aug 2020 20:09:00 +0530 Subject: [PATCH 36/42] reverted test changes --- tests/performance/agents/config.ini | 2 +- tests/performance/runs/storage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index c5be6f0d3..28827140a 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -7,4 +7,4 @@ PORT = 9009 [suite] s3_bucket = shivam-codebuild-test -comparison_artifacts_dir = perf_comparison_artifacts/ \ No newline at end of file +comparison_artifacts_dir = perf_comparison_artifacts \ No newline at end of file diff --git a/tests/performance/runs/storage.py b/tests/performance/runs/storage.py index 91612d997..5e6f04ac6 100644 --- a/tests/performance/runs/storage.py +++ b/tests/performance/runs/storage.py @@ -93,7 +93,7 @@ def get_dir_to_compare(self): comp_data_path = os.path.join(self.artifacts_dir, "comp_data") s3 = boto3.resource('s3') bucket = s3.Bucket(S3_BUCKET) - prefix = S3_COMPARE_DIR+"/" if not S3_COMPARE_DIR.endswith('/') else S3_COMPARE_DIR + prefix = S3_COMPARE_DIR+"/" result = bucket.meta.client.list_objects(Bucket=bucket.name, Prefix=prefix, Delimiter='/') From 036eb2743ca01f10fe76ee19d355132021093abe Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Thu, 13 Aug 2020 20:47:49 +0530 Subject: [PATCH 37/42] updated register_unregister_multiple test case --- .../register_unregister_multiple/environments/xlarge.yaml | 4 ++-- .../register_unregister_multiple.yaml | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml b/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml index eb4d0e024..1793349e5 100644 --- a/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml +++ b/tests/performance/tests/register_unregister_multiple/environments/xlarge.yaml @@ -35,7 +35,7 @@ settings: TOTAL_ORPHANS_PREV_DIFF: 0 TOTAL_ZOMBIES_PREV_DIFF: 0 - TOTAL_WORKERS_RUN_DIFF: 200 + TOTAL_WORKERS_RUN_DIFF: 0 TOTAL_WORKERS_MEM_RUN_DIFF: 200 TOTAL_WORKERS_FDS_RUN_DIFF: 200 TOTAL_MEM_RUN_DIFF: 200 @@ -48,7 +48,7 @@ settings: CONCURRENCY : 1 RAMP-UP : 1s - HOLD-FOR : 300s + ITERATIONS : 360 #approximately runs for 5 mins SCRIPT : register_unregister_multiple.jmx STOP : '' #possible values true, false. Bug in bzt so for false use '' diff --git a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml index b87aa88a9..b6bbfb8dc 100644 --- a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml +++ b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml @@ -1,4 +1,10 @@ --- +~execution: +- concurrency: ${CONCURRENCY} + ramp-up: ${RAMP-UP} + iterations: ${ITERATIONS} + scenario: scenario_0 + scenarios: scenario_0: script: register_unregister_multiple.jmx From 15ea1a4fe6158932c4ba5f6c6ee51a790cdcdd06 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Fri, 14 Aug 2020 14:49:03 +0530 Subject: [PATCH 38/42] test changes for enhanced logs --- ci/scripts/linux_test_perf_regression.sh | 2 +- tests/performance/agents/metrics/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/linux_test_perf_regression.sh b/ci/scripts/linux_test_perf_regression.sh index 5c86bc4d8..940042bbb 100755 --- a/ci/scripts/linux_test_perf_regression.sh +++ b/ci/scripts/linux_test_perf_regression.sh @@ -18,7 +18,7 @@ pip install -r requirements.txt pip install bzt # Execute performance test suite and store exit code -./run_performance_suite.py -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit +./run_performance_suite.py -j $JMETER_PATH -p register_unregister -e xlarge --no-compare-local --no-monit EXIT_CODE=$? # Exit with the same error code as that of test execution diff --git a/tests/performance/agents/metrics/__init__.py b/tests/performance/agents/metrics/__init__.py index 21266ffd2..4348a2484 100644 --- a/tests/performance/agents/metrics/__init__.py +++ b/tests/performance/agents/metrics/__init__.py @@ -95,7 +95,7 @@ def get_metrics(server_process, child_processes, logger): """ result = {} children.update(child_processes) - logger.debug("children : {0}".format(",".join([str(c.pid) for c in children]))) + logger.info("children : {0}".format(",".join([str(c.pid) for c in children]))) def update_metric(metric_name, proc_type, stats): stats = list(filter(lambda x: isinstance(x, (float, int)), stats)) @@ -129,12 +129,12 @@ def update_metric(metric_name, proc_type, stats): processes_stats.append({'type': ProcessType.WORKER, 'stats': child.as_dict()}) else: reclaimed_pids.append(child) - logger.debug('child {0} no longer available'.format(child.pid)) + logger.info('child {0} no longer available'.format(child.pid)) except ZombieProcess: zombie_children.add(child) except NoSuchProcess: reclaimed_pids.append(child) - logger.debug('child {0} no longer available'.format(child.pid)) + logger.info('child {0} no longer available'.format(child.pid)) for p in reclaimed_pids: if p in children: From 7c5ee7736b8389428f00a5cd8425011deaa741f4 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Fri, 14 Aug 2020 15:53:45 +0530 Subject: [PATCH 39/42] Get Last data read before shutdown. --- ci/scripts/linux_test_perf_regression.sh | 2 +- tests/performance/runs/taurus/override/metrics_monitoring.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/scripts/linux_test_perf_regression.sh b/ci/scripts/linux_test_perf_regression.sh index 940042bbb..5c86bc4d8 100755 --- a/ci/scripts/linux_test_perf_regression.sh +++ b/ci/scripts/linux_test_perf_regression.sh @@ -18,7 +18,7 @@ pip install -r requirements.txt pip install bzt # Execute performance test suite and store exit code -./run_performance_suite.py -j $JMETER_PATH -p register_unregister -e xlarge --no-compare-local --no-monit +./run_performance_suite.py -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit EXIT_CODE=$? # Exit with the same error code as that of test execution diff --git a/tests/performance/runs/taurus/override/metrics_monitoring.py b/tests/performance/runs/taurus/override/metrics_monitoring.py index 859b2ea3a..483ef7317 100644 --- a/tests/performance/runs/taurus/override/metrics_monitoring.py +++ b/tests/performance/runs/taurus/override/metrics_monitoring.py @@ -59,6 +59,11 @@ def __init__(self, parent_log, label, config, engine=None): else: self.label = 'ServerLocalClient' + def disconnect(self): + self.log.info("Last metric values before shutdown") + self.interval = 0 + self.get_data() + def connect(self): exc = TaurusConfigError('Metric is required in Local monitoring client') metric_names = self.config.get('metrics', exc) From 3bf7effe27ba0eba3c1efe0ff2ef9a25c3cca8c1 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Fri, 14 Aug 2020 19:37:19 +0530 Subject: [PATCH 40/42] added a dummy commit for PRT test runs --- tests/performance/agents/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index 28827140a..84d5d2dd6 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -7,4 +7,4 @@ PORT = 9009 [suite] s3_bucket = shivam-codebuild-test -comparison_artifacts_dir = perf_comparison_artifacts \ No newline at end of file +comparison_artifacts_dir = perf_comparison_artifacts From 61fe44f0d3546328f424ba297a07dd1fbf040f01 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Sun, 16 Aug 2020 01:33:53 +0530 Subject: [PATCH 41/42] dummy checkin --- tests/performance/agents/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/agents/config.ini b/tests/performance/agents/config.ini index 84d5d2dd6..28827140a 100644 --- a/tests/performance/agents/config.ini +++ b/tests/performance/agents/config.ini @@ -7,4 +7,4 @@ PORT = 9009 [suite] s3_bucket = shivam-codebuild-test -comparison_artifacts_dir = perf_comparison_artifacts +comparison_artifacts_dir = perf_comparison_artifacts \ No newline at end of file From d3e016753d46e4290c4eaa288d24899c7ba12383 Mon Sep 17 00:00:00 2001 From: Shivam Shriwas Date: Mon, 17 Aug 2020 18:56:58 +0530 Subject: [PATCH 42/42] rt check to avg-rt. --- .../api_description/api_description.yaml | 4 +-- .../batch_and_single_inference.yaml | 4 +-- tests/performance/tests/global_config.yaml | 4 +-- .../inference_multiple_models.yaml | 4 +-- .../multiple_inference_and_scaling.yaml | 28 +++++++++---------- .../register_unregister.yaml | 5 ++-- .../register_unregister_multiple.yaml | 12 ++++---- .../scale_down_workers.yaml | 4 +-- .../scale_up_workers/scale_up_workers.yaml | 4 +-- 9 files changed, 34 insertions(+), 35 deletions(-) diff --git a/tests/performance/tests/api_description/api_description.yaml b/tests/performance/tests/api_description/api_description.yaml index 873104ba6..88546be71 100644 --- a/tests/performance/tests/api_description/api_description.yaml +++ b/tests/performance/tests/api_description/api_description.yaml @@ -3,8 +3,8 @@ reporting: - module: passfail criteria: # Inbuilt Criteria - - success of ManagementAPIDescription<${MGMT_DESC_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ManagementAPIDescription>${MGMT_DESC_AVG_RT} for 10s, ${STOP_ALIAS} as failed + - success of ManagementAPIDescription<${MGMT_DESC_SUCC} + - avg-rt of ManagementAPIDescription>${MGMT_DESC_AVG_RT} # # Custom Criteria # - class: bzt.modules.monitoring.MonitoringCriteria # subject: ServerLocalClient/total_processes diff --git a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml index 60373dcde..62f4af8ad 100644 --- a/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml +++ b/tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml @@ -18,5 +18,5 @@ reporting: - module: passfail criteria: # Inbuilt Criteria - - success of ManagementAPIDescription<${INF2_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ManagementAPIDescription>${INF2_AVG_RT} for 10s, ${STOP_ALIAS} as failed \ No newline at end of file + - success of ManagementAPIDescription<${INF2_SUCC} + - avg-rt of ManagementAPIDescription>${INF2_AVG_RT} \ No newline at end of file diff --git a/tests/performance/tests/global_config.yaml b/tests/performance/tests/global_config.yaml index 98c343d86..51de8a03d 100644 --- a/tests/performance/tests/global_config.yaml +++ b/tests/performance/tests/global_config.yaml @@ -76,8 +76,8 @@ reporting: - module: passfail criteria: # API requests KPI crieteria - - success of ${API_LABEL}<${API_SUCCESS} for 10s, stop as failed - - avg-rt of ${API_LABEL}>${API_AVG_RT} for 10s, ${STOP_ALIAS} as failed + - success of ${API_LABEL}<${API_SUCCESS} + - avg-rt of ${API_LABEL}>${API_AVG_RT} # # # Monitoring metrics criteria # - class: bzt.modules.monitoring.MonitoringCriteria diff --git a/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml b/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml index 92a8d155c..87da67e2c 100644 --- a/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml +++ b/tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml @@ -16,8 +16,8 @@ reporting: - module: passfail criteria: # Inbuilt Criteria - - success of Inference2<${INFR2_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of Inference2>${INFR2_RT} for 10s, ${STOP_ALIAS} as failed + - success of Inference2<${INFR2_SUCC} + - avg-rt of Inference2>${INFR2_RT} - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes condition: '<' diff --git a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml index d5a38242b..6b7e4891a 100644 --- a/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml +++ b/tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml @@ -16,18 +16,18 @@ reporting: - module: passfail criteria: # Inbuilt Criteria - - success of Inference2<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of Inference2>${INFR2_RT} for 10s, ${STOP_ALIAS} as failed - - success of Inference11<${INFR1_SUCC} for 10s, stop as failed - - success of Inference21<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of Inference11>${INFR1_RT} for 10s, ${STOP_ALIAS} as failed - - avg-rt of Inference21>${INFR2_RT} for 10s, ${STOP_ALIAS} as failed - - success of ScaleUp1<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleUp1>${SCALEUP1_RT} for 10s, ${STOP_ALIAS} as failed - - success of ScaleUp2<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleUp2>${SCALEUP2_RT} for 10s, ${STOP_ALIAS} as failed - - success of ScaleDown1<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleDown1>${SCALEDOWN1_RT} for 10s, ${STOP_ALIAS} as failed - - success of ScaleDown2<${INFR2_SUCC} for 10s, stop as failed - - avg-rt of ScaleDown2>${SCALEDOWN2_RT} for 10s, ${STOP_ALIAS} as failed + - success of Inference2<${INFR2_SUCC} + - avg-rt of Inference2>${INFR2_RT} + - success of Inference11<${INFR1_SUCC} + - success of Inference21<${INFR2_SUCC} + - avg-rt of Inference11>${INFR1_RT} + - avg-rt of Inference21>${INFR2_RT} + - success of ScaleUp1<${INFR2_SUCC} + - avg-rt of ScaleUp1>${SCALEUP1_RT} + - success of ScaleUp2<${INFR2_SUCC} + - avg-rt of ScaleUp2>${SCALEUP2_RT} + - success of ScaleDown1<${INFR2_SUCC} + - avg-rt of ScaleDown1>${SCALEDOWN1_RT} + - success of ScaleDown2<${INFR2_SUCC} + - avg-rt of ScaleDown2>${SCALEDOWN2_RT} diff --git a/tests/performance/tests/register_unregister/register_unregister.yaml b/tests/performance/tests/register_unregister/register_unregister.yaml index 0bf11cab7..296870996 100644 --- a/tests/performance/tests/register_unregister/register_unregister.yaml +++ b/tests/performance/tests/register_unregister/register_unregister.yaml @@ -13,6 +13,5 @@ reporting: - module: passfail criteria: # Inbuilt Criteria - - success of UnregisterModel<${UNREG_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of UnregisterModel>${UNREG_RT} for 10s, ${STOP_ALIAS} as failed - + - success of UnregisterModel<${UNREG_SUCC} + - avg-rt of UnregisterModel>${UNREG_RT} diff --git a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml index b6bbfb8dc..e7c30a660 100644 --- a/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml +++ b/tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml @@ -26,9 +26,9 @@ services: - module: passfail criteria: # Inbuilt Criteria - - success of ${API_LABEL}<${API_SUCCESS} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ${API_LABEL}>${API_AVG_RT} for 10s, ${STOP_ALIAS} as failed - - success of ScaleUp<${SCL_UP_SUCC} for 10s, ${STOP_ALIAS} as failed - - success of UnregisterModel<${UNREG_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ScaleUp>${SCL_UP_RT} for 10s, ${STOP_ALIAS} as failed - - avg-rt of UnregisterModel>${UNREG_RT} for 10s, ${STOP_ALIAS} as failed + - success of ${API_LABEL}<${API_SUCCESS} + - avg-rt of ${API_LABEL}>${API_AVG_RT} + - success of ScaleUp<${SCL_UP_SUCC} + - success of UnregisterModel<${UNREG_SUCC} + - avg-rt of ScaleUp>${SCL_UP_RT} + - avg-rt of UnregisterModel>${UNREG_RT} diff --git a/tests/performance/tests/scale_down_workers/scale_down_workers.yaml b/tests/performance/tests/scale_down_workers/scale_down_workers.yaml index 13924e484..c5f21b61e 100644 --- a/tests/performance/tests/scale_down_workers/scale_down_workers.yaml +++ b/tests/performance/tests/scale_down_workers/scale_down_workers.yaml @@ -23,8 +23,8 @@ services: - module: passfail criteria: # Inbuilt Criteria - - success of ScaleDown<${SCL_DWN_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ScaleDown>${SCL_DWN_RT} for 10s, ${STOP_ALIAS} as failed + - success of ScaleDown<${SCL_DWN_SUCC} + - avg-rt of ScaleDown>${SCL_DWN_RT} # Custom Criteria - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes diff --git a/tests/performance/tests/scale_up_workers/scale_up_workers.yaml b/tests/performance/tests/scale_up_workers/scale_up_workers.yaml index da139df6d..2404a7f82 100644 --- a/tests/performance/tests/scale_up_workers/scale_up_workers.yaml +++ b/tests/performance/tests/scale_up_workers/scale_up_workers.yaml @@ -23,8 +23,8 @@ services: - module: passfail criteria: # Inbuilt Criteria - - success of ScaleUp<${SCL_UP_SUCC} for 10s, ${STOP_ALIAS} as failed - - avg-rt of ScaleUp>${SCL_UP_RT} for 10s, ${STOP_ALIAS} as failed + - success of ScaleUp<${SCL_UP_SUCC} + - avg-rt of ScaleUp>${SCL_UP_RT} # Custom Criteria - class: bzt.modules.monitoring.MonitoringCriteria subject: ServerLocalClient/total_processes