Skip to content

Commit 152b418

Browse files
KamiKami
authored
Merge pull request #4691 from StackStorm/301_additional_changes
Additional changes for v3.0.1
2 parents 5fa408d + 70424b6 commit 152b418

File tree

13 files changed

+121
-87
lines changed

13 files changed

+121
-87
lines changed

‎CHANGELOG.rst‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ In development
55
--------------
66

77

8-
3.0.1 - May 20, 2019
8+
3.0.1 - May 24, 2019
99
--------------------
1010

1111
Fixed
@@ -39,6 +39,15 @@ Fixed
3939
``service`` or ``action`` parameter. (bug fix) #4675
4040

4141
Reported by James Robinson (Netskope and Veracode).
42+
* Replace ``sseclient`` library on which CLI depends on with ``sseclient-py``. ``sseclient`` has
43+
various issue which cause client to sometimes hang and keep the connection open which also causes
44+
``st2 execution tail`` command to sometimes hang for a long time. (improvement)
45+
* Truncate some database index names so they are less than 65 characters long in total. This way it
46+
also works with AWS DocumentDB which doesn't support longer index name at the moment.
47+
48+
NOTE: AWS DocumentDB is not officially supported. Use at your own risk. (improvement) #4688 #4690
49+
50+
Reported by Guillaume Truchot (@GuiTeK)
4251

4352
3.0.0 - April 26, 2019
4453
----------------------

‎Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,8 @@ flake8: requirements .flake8
281281
chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
282282

283283
$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip>=9.0,<9.1"
284+
# NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions
285+
$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==41.0.1"
284286
$(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd ..
285287
$(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version
286288
$(VIRTUALENV_ST2CLIENT_DIR)/bin/python -c "import st2client"

‎conf/HA/nginx/st2.conf.blueprint.sample‎

Lines changed: 9 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ server {
1313
add_header X-Content-Type-Options nosniff;
1414

1515
if ($ssl_protocol = "") {
16-
return 301 https://$host$request_uri;
16+
return 308 https://$host$request_uri;
1717
}
1818

19-
index index.html index.htm index.php;
19+
index index.html;
2020

2121
access_log /var/log/nginx/st2webui.access.log combined;
2222
error_log /var/log/nginx/st2webui.error.log;
@@ -32,10 +32,10 @@ server {
3232
ssl_session_cache shared:SSL:10m;
3333
ssl_session_timeout 5m;
3434
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
35-
ssl_ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:ECDHE-RSA-AES128-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA128:DHE-RSA-AES128-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA128:ECDHE-RSA-AES128-SHA384:ECDHE-RSA-AES128-SHA128:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA384:AES128-GCM-SHA128:AES128-SHA128:AES128-SHA128:AES128-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4;
35+
ssl_ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:ECDHE-RSA-AES128-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA128:DHE-RSA-AES128-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA128:ECDHE-RSA-AES128-SHA384:ECDHE-RSA-AES128-SHA128:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:AES128-GCM-SHA384:AES128-GCM-SHA128:AES128-SHA128:AES128-SHA128:AES128-SHA:AES128-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4;
3636
ssl_prefer_server_ciphers on;
3737

38-
index index.html index.htm index.php;
38+
index index.html;
3939

4040
access_log /var/log/nginx/ssl-st2webui.access.log combined;
4141
error_log /var/log/nginx/ssl-st2webui.error.log;
@@ -62,58 +62,14 @@ server {
6262
proxy_set_header Host $host;
6363
}
6464

65-
location /stream/ {
66-
rewrite ^/stream/(.*) /$1 break;
67-
68-
proxy_pass http://127.0.0.1:9102/;
69-
proxy_read_timeout 90;
70-
proxy_connect_timeout 90;
71-
proxy_redirect off;
72-
73-
proxy_set_header Host $host;
74-
proxy_set_header X-Real-IP $remote_addr;
75-
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
76-
proxy_pass_header Authorization;
77-
78-
sendfile on;
79-
tcp_nopush on;
80-
tcp_nodelay on;
81-
82-
# Disable buffering and chunked encoding.
83-
# In the stream case we want to receive the whole payload at once, we don't
84-
# want multiple chunks.
85-
proxy_set_header Connection '';
86-
chunked_transfer_encoding off;
87-
proxy_buffering off;
88-
proxy_cache off;
89-
proxy_set_header Host $host;
90-
}
91-
9265
# For backward compatibility reasons, rewrite requests from "/api/stream"
9366
# to "/stream/v1/stream" and "/api/v1/stream" to "/stream/v1/stream"
94-
location /api/stream/ {
95-
rewrite ^/api/stream/?(.*)$ /v1/stream/$1 break;
96-
proxy_pass http://127.0.0.1:9102;
97-
proxy_set_header Host $host;
98-
proxy_set_header X-Real-IP $remote_addr;
99-
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
100-
101-
sendfile on;
102-
tcp_nopush on;
103-
tcp_nodelay on;
104-
105-
# Disable buffering and chunked encoding.
106-
# In the stream case we want to receive the whole payload at once, we don't
107-
# want multiple chunks.
108-
proxy_set_header Connection '';
109-
chunked_transfer_encoding off;
110-
proxy_buffering off;
111-
proxy_cache off;
112-
}
67+
rewrite ^/api/stream/?$ /stream/v1/stream break;
68+
rewrite ^/api/(v\d)/stream/?$ /stream/$1/stream break;
69+
location /stream/ {
70+
rewrite ^/stream/(.*) /$1 break;
11371

114-
location /api/v1/stream/ {
115-
rewrite ^/api/v1/stream/?(.*)$ /v1/stream/$1 break;
116-
proxy_pass http://127.0.0.1:9102;
72+
proxy_pass http://127.0.0.1:9102/;
11773
proxy_set_header Host $host;
11874
proxy_set_header X-Real-IP $remote_addr;
11975
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -170,5 +126,4 @@ server {
170126
proxy_cache off;
171127
proxy_set_header Host $host;
172128
}
173-
174129
}

‎conf/HA/nginx/st2.conf.controller.sample‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ server {
1818
add_header X-Content-Type-Options nosniff;
1919

2020
if ($ssl_protocol = "") {
21-
return 301 https://$host$request_uri;
21+
return 308 https://$host$request_uri;
2222
}
2323

24-
index index.html index.htm index.php;
24+
index index.html;
2525

2626
access_log /var/log/nginx/st2webui.access.log combined;
2727
error_log /var/log/nginx/st2webui.error.log;
@@ -37,21 +37,27 @@ server {
3737
ssl_session_cache shared:SSL:10m;
3838
ssl_session_timeout 5m;
3939
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
40-
ssl_ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:ECDHE-RSA-AES128-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA128:DHE-RSA-AES128-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA128:ECDHE-RSA-AES128-SHA384:ECDHE-RSA-AES128-SHA128:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA384:AES128-GCM-SHA128:AES128-SHA128:AES128-SHA128:AES128-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4;
40+
ssl_ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:ECDHE-RSA-AES128-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA128:DHE-RSA-AES128-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA128:ECDHE-RSA-AES128-SHA384:ECDHE-RSA-AES128-SHA128:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:AES128-GCM-SHA384:AES128-GCM-SHA128:AES128-SHA128:AES128-SHA128:AES128-SHA:AES128-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4;
4141
ssl_prefer_server_ciphers on;
4242

43-
index index.html index.htm index.php;
43+
index index.html;
4444

4545
access_log /var/log/nginx/ssl-st2webui.access.log combined;
4646
error_log /var/log/nginx/ssl-st2webui.error.log;
4747

4848
add_header Front-End-Https on;
4949
add_header X-Content-Type-Options nosniff;
5050

51+
location @apiError {
52+
add_header Content-Type application/json always;
53+
return 503 '{ "faultstring": "Nginx is unable to reach st2api. Make sure service is running." }';
54+
}
55+
5156
location /api/ {
52-
rewrite ^/api/(.*) /api/$1 break;
57+
error_page 502 = @apiError;
5358

5459
proxy_pass https://st2/api/;
60+
proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
5561
proxy_read_timeout 90;
5662
proxy_connect_timeout 90;
5763
proxy_redirect off;
@@ -67,7 +73,7 @@ server {
6773
proxy_set_header Host $host;
6874
}
6975

70-
location @streamError {
76+
location @streamError {
7177
add_header Content-Type text/event-stream;
7278
return 200 "retry: 1000\n\n";
7379
}
@@ -79,9 +85,8 @@ server {
7985
location /stream/ {
8086
error_page 502 = @streamError;
8187

82-
rewrite ^/stream/(.*) /$1 break;
83-
8488
proxy_pass https://st2/stream/;
89+
proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
8590
proxy_set_header Host $host;
8691
proxy_set_header X-Real-IP $remote_addr;
8792
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -99,10 +104,16 @@ server {
99104
proxy_cache off;
100105
}
101106

107+
location @authError {
108+
add_header Content-Type application/json always;
109+
return 503 '{ "faultstring": "Nginx is unable to reach st2auth. Make sure service is running." }';
110+
}
111+
102112
location /auth/ {
103-
rewrite ^/auth/(.*) /auth/$1 break;
113+
error_page 502 = @authError;
104114

105115
proxy_pass https://st2/auth/;
116+
proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
106117
proxy_read_timeout 90;
107118
proxy_connect_timeout 90;
108119
proxy_redirect off;
@@ -118,11 +129,10 @@ server {
118129
proxy_cache off;
119130
proxy_set_header Host $host;
120131
}
121-
122-
location /mistral/ {
123-
rewrite ^/mistral/(.*) /mistral/$1 break;
124132

133+
location /mistral/ {
125134
proxy_pass https://st2/mistral/;
135+
proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
126136
proxy_read_timeout 90;
127137
proxy_connect_timeout 90;
128138
proxy_redirect off;
@@ -141,6 +151,10 @@ server {
141151

142152
location / {
143153
root /opt/stackstorm/static/webui/;
144-
index index.html index.htm index.php;
154+
index index.html;
155+
156+
sendfile on;
157+
tcp_nopush on;
158+
tcp_nodelay on;
145159
}
146160
}

‎fixed-requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ cryptography==2.6.1
3535
retrying==1.3.3
3636
# Note: We use latest version of virtualenv which uses pip 9.0
3737
virtualenv==15.1.0
38-
sseclient==0.0.19
38+
# NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc.
39+
sseclient-py==1.7
3940
python-editor==1.0.4
4041
prompt-toolkit==1.0.15
4142
tooz==1.64.2

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ retrying==1.3.3
5050
routes==2.4.1
5151
semver==2.8.1
5252
six==1.12.0
53-
sseclient==0.0.19
53+
sseclient-py==1.7
5454
stevedore==1.30.1
5555
tooz==1.64.2
5656
ujson==1.35

‎st2client/in-requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ jsonschema
88
jsonpath-rw
99
requests
1010
six
11-
sseclient
11+
sseclient-py
1212
python-editor
1313
prompt-toolkit
1414
cryptography

‎st2client/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@ pytz==2019.1
1111
pyyaml==5.1
1212
requests[security]<2.15,>=2.14.1
1313
six==1.12.0
14-
sseclient==0.0.19
14+
sseclient-py==1.7

‎st2client/st2client/models/core.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
# limitations under the License.
1515

1616
from __future__ import absolute_import
17+
1718
import os
1819
import json
1920
import logging
@@ -22,6 +23,7 @@
2223
import six
2324
from six.moves import urllib
2425
from six.moves import http_client
26+
import requests
2527

2628
from st2client.utils import httpclient
2729

@@ -632,8 +634,10 @@ def listen(self, events=None, **kwargs):
632634
query_string = '?' + urllib.parse.urlencode(query_params)
633635
url = url + query_string
634636

635-
for message in SSEClient(url, **request_params):
637+
response = requests.get(url, stream=True, **request_params)
638+
client = SSEClient(response)
636639

640+
for message in client.events():
637641
# If the execution on the API server takes too long, the message
638642
# can be empty. In this case, rerun the query.
639643
if not message.data:

‎st2client/tests/unit/test_models.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -257,32 +257,41 @@ def test_resource_delete_failed(self):
257257
instance = mgr.get_by_name('abc')
258258
self.assertRaises(Exception, mgr.delete, instance)
259259

260+
@mock.patch('requests.get')
260261
@mock.patch('sseclient.SSEClient')
261-
def test_stream_resource_listen(self, mock):
262+
def test_stream_resource_listen(self, mock_sseclient, mock_requests):
262263
mock_msg = mock.Mock()
263264
mock_msg.data = json.dumps(base.RESOURCES)
264265

265266
# checking the case to specify valid 'cacert' parameter to the StreamManager
266-
def side_effect_checking_verify_parameter_is(endpoint_url, **kwargs):
267-
self.assertEqual(endpoint_url, 'https://example.com/stream?events=foo%2Cbar')
268-
self.assertEqual(kwargs['verify'], '/path/ca.crt')
267+
def side_effect_checking_verify_parameter_is():
269268
return [mock_msg]
270269

271-
mock.side_effect = side_effect_checking_verify_parameter_is
270+
mock_sseclient.return_value.events.side_effect = side_effect_checking_verify_parameter_is
272271
mgr = models.StreamManager('https://example.com', cacert='/path/ca.crt')
273272

274273
resp = mgr.listen(events=['foo', 'bar'])
275274
self.assertEqual(list(resp), [base.RESOURCES])
276275

276+
call_args = tuple(['https://example.com/stream?events=foo%2Cbar'])
277+
call_kwargs = {'stream': True, 'verify': '/path/ca.crt'}
278+
279+
self.assertEqual(mock_requests.call_args_list[0][0], call_args)
280+
self.assertEqual(mock_requests.call_args_list[0][1], call_kwargs)
281+
277282
# checking the case not to specify valid 'cacert' parameter to the StreamManager
278-
def side_effect_checking_verify_parameter_is_not(endpoint_url, **kwargs):
279-
# checking endpoint_url in case of no event specification
280-
self.assertEqual(endpoint_url, 'https://example.com/stream?')
281-
self.assertFalse('verify' in kwargs)
283+
def side_effect_checking_verify_parameter_is_not():
282284
return [mock_msg]
283285

284-
mock.side_effect = side_effect_checking_verify_parameter_is_not
286+
mock_sseclient.return_value.events.side_effect = \
287+
side_effect_checking_verify_parameter_is_not
285288
mgr = models.StreamManager('https://example.com')
286289

287290
resp = mgr.listen()
288291
self.assertEqual(list(resp), [base.RESOURCES])
292+
293+
call_args = tuple(['https://example.com/stream?'])
294+
call_kwargs = {'stream': True}
295+
296+
self.assertEqual(mock_requests.call_args_list[1][0], call_args)
297+
self.assertEqual(mock_requests.call_args_list[1][1], call_kwargs)

0 commit comments

Comments
 (0)