Prep 2.2.2 (#288)

* updated Python Connector reqs to branch

* updated Python Connector reqs to branch master

* SNOW-139744 bump idna and requests

* updated Python Connector reqs to branch master

* SNOW-141285 Fix a bug about not retrying in chunkdownloader

* SNOW-132958 SNOW-104556 copied connector scripts into sub-directory and added py38 support

* updated Python Connector reqs to branch master

* SNOW-132958 SNOW-104556 fixed merge conflict

* clean up

* fixed script call

* Bump up version to 2.2.2

* removed orig file

* reverting build_inside_docker.sh removal

* fixed permission issue

* updated script call

* removed period removal from install.sh

Co-authored-by: Shige Takeda <[email protected]>
Co-authored-by: Mark Keller <[email protected]>

Author: 3 people

DESCRIPTION.rst

@@ -9,6 +9,11 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne
 Release Notes
 -------------------------------------------------------------------------------
 
+- v2.2.2(March 9,2020)
+
+    - Fix retry with chunck_downloader.py for stability.
+    - Support Python 3.8 for Linux and Mac.
+
 - v2.2.1(February 18,2020)
 
     - Fix use DictCursor with execute_string #248

arrow_iterator.pyx

@@ -162,7 +162,7 @@ cdef class PyArrowIterator(EmptyPyArrowIterator):
                 cursor,
                 OperationalError,
                 {
-                    u'msg': u'Failed to open arrow stream: ' + ret.message(),
+                    u'msg': u'Failed to open arrow stream: ' + str(ret.message()),
                     u'errno': ER_FAILED_TO_READ_ARROW_STREAM
                 })
 
@@ -174,7 +174,7 @@ cdef class PyArrowIterator(EmptyPyArrowIterator):
                     cursor,
                     OperationalError,
                     {
-                        u'msg': u'Failed to read next arrow batch: ' + ret.message(),
+                        u'msg': u'Failed to read next arrow batch: ' + str(ret.message()),
                         u'errno': ER_FAILED_TO_READ_ARROW_STREAM
                     })
 

chunk_downloader.py

@@ -5,6 +5,7 @@
 #
 
 import json
+import time
 from collections import namedtuple
 from gzip import GzipFile
 from io import BytesIO
@@ -18,9 +19,9 @@
 from .arrow_context import ArrowConverterContext
 from .errorcode import ER_CHUNK_DOWNLOAD_FAILED
 from .errors import Error, OperationalError
-from .time_util import get_time_millis
+from .time_util import get_time_millis, DecorrelateJitterBackoff
 
-DEFAULT_REQUEST_TIMEOUT = 3600
+DEFAULT_REQUEST_TIMEOUT = 7
 
 DEFAULT_CLIENT_PREFETCH_THREADS = 4
 MAX_CLIENT_PREFETCH_THREADS = 10
@@ -113,41 +114,50 @@ def _download_chunk(self, idx):
         """
         logger.debug(u'downloading chunk %s/%s', idx + 1, self._chunk_size)
         headers = {}
-        try:
-            if self._chunk_headers is not None:
-                headers = self._chunk_headers
-                logger.debug(u'use chunk headers from result')
-            elif self._qrmk is not None:
-                headers[SSE_C_ALGORITHM] = SSE_C_AES
-                headers[SSE_C_KEY] = self._qrmk
-
-            logger.debug(u"started getting the result set %s: %s",
-                         idx + 1, self._chunks[idx].url)
-            result_data = self._fetch_chunk(self._chunks[idx].url, headers)
-            logger.debug(u"finished getting the result set %s: %s",
-                         idx + 1, self._chunks[idx].url)
-
-            if isinstance(result_data, ResultIterWithTimings):
-                metrics = result_data.get_timings()
-                with self._downloading_chunks_lock:
-                    self._total_millis_downloading_chunks += metrics[
-                        ResultIterWithTimings.DOWNLOAD]
-                    self._total_millis_parsing_chunks += metrics[
-                        ResultIterWithTimings.PARSE]
-
-            with self._chunk_cond:
-                self._chunks[idx] = self._chunks[idx]._replace(
-                    result_data=result_data,
-                    ready=True)
-                self._chunk_cond.notify_all()
-                logger.debug(
-                    u'added chunk %s/%s to a chunk list.', idx + 1,
-                    self._chunk_size)
-        except Exception as e:
-            logger.exception(
-                u'Failed to fetch the large result set chunk %s/%s',
-                idx + 1, self._chunk_size)
-            self._downloader_error = e
+        if self._chunk_headers is not None:
+            headers = self._chunk_headers
+            logger.debug(u'use chunk headers from result')
+        elif self._qrmk is not None:
+            headers[SSE_C_ALGORITHM] = SSE_C_AES
+            headers[SSE_C_KEY] = self._qrmk
+
+        last_error = None
+        backoff = DecorrelateJitterBackoff(1, 16)
+        sleep_timer = 1
+        for retry in range(10):
+            try:
+                logger.debug(u"started getting the result set %s: %s",
+                             idx + 1, self._chunks[idx].url)
+                result_data = self._fetch_chunk(self._chunks[idx].url, headers)
+                logger.debug(u"finished getting the result set %s: %s",
+                             idx + 1, self._chunks[idx].url)
+
+                if isinstance(result_data, ResultIterWithTimings):
+                    metrics = result_data.get_timings()
+                    with self._downloading_chunks_lock:
+                        self._total_millis_downloading_chunks += metrics[
+                            ResultIterWithTimings.DOWNLOAD]
+                        self._total_millis_parsing_chunks += metrics[
+                            ResultIterWithTimings.PARSE]
+
+                with self._chunk_cond:
+                    self._chunks[idx] = self._chunks[idx]._replace(
+                        result_data=result_data,
+                        ready=True)
+                    self._chunk_cond.notify_all()
+                    logger.debug(
+                        u'added chunk %s/%s to a chunk list.', idx + 1,
+                        self._chunk_size)
+                break
+            except Exception as e:
+                last_error = e
+                sleep_timer = backoff.next_sleep(1, sleep_timer)
+                logger.exception(
+                    u'Failed to fetch the large result set chunk %s/%s for the %s th time, backing off for %s s',
+                    idx + 1, self._chunk_size, retry + 1, sleep_timer)
+                time.sleep(sleep_timer)
+        else:
+            self._downloader_error = last_error
 
     def next_chunk(self):
         """

docker/manylinux2010/Dockerfile-x86_64_base

@@ -11,7 +11,7 @@ RUN chmod +x /usr/local/bin/entrypoint.sh
 
 WORKDIR /home/user
 RUN chmod 777 /home/user
-RUN git clone https://github.com/matthew-brett/multibuild.git && cd /home/user/multibuild && git checkout 1a7f31be677185f2dface2643284846e14130c3f
+RUN git clone https://github.com/matthew-brett/multibuild.git && cd /home/user/multibuild && git checkout b943f33a92772fd52c2fa38c03d08aac974c53bd
 
 ADD scripts/build_virtualenvs.sh /home/user
 RUN /home/user/build_virtualenvs.sh

docker/manylinux2010/scripts/build_virtualenvs.sh

@@ -19,23 +19,27 @@
 # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
 # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
 
-PYTHON_VERSIONS="${PYTHON_VERSIONS:-35 36 37}"
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-3.5 3.6 3.7 3.8}"
 
 source /home/user/multibuild/manylinux_utils.sh
 
 for PYTHON in ${PYTHON_VERSIONS}; do
     U_WIDTH=16
-    PYTHON_INTERPRETER="$(cpython_path $PYTHON {U_WIDTH})/bin/python"
+    PYTHON_INTERPRETER="$(cpython_path $PYTHON ${U_WIDTH})/bin/python"
     PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip"
     PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})"
 
+    echo "=== Updating pip ==="
+    $PIP install -U "pip"
+
     echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ==="
-    $PIP install "virtualenv==16.3.0"
+    $PIP install "virtualenv"
 
     echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for build ==="
     "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /home/user/venv-build-${PYTHON}
     source /home/user/venv-build-${PYTHON}/bin/activate
-    pip install "cython==0.29.8" "setuptools" "flake8" "wheel" "pyarrow==0.15.1"
+    pip install -U pip
+    pip install "cython==0.29.15" "setuptools" "flake8" "wheel" "pyarrow==0.16.0"
     deactivate
 done
 

docker/python_connector_tests/Dockerfile-x86_64_base

@@ -0,0 +1,16 @@
+# We use manylinux1 base image because pyarrow_manylinux2010 has a bug and wheel failed to be audited
+FROM quay.io/pypa/manylinux2010_x86_64
+
+# This is to solve permission issue, read https://denibertovic.com/posts/handling-permissions-with-docker-volumes/
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64"
+RUN chmod +x /usr/local/bin/gosu
+
+COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+
+WORKDIR /home/user
+RUN chmod 777 /home/user
+ENV PATH="${PATH}:/opt/python/cp35-cp35m/bin:/opt/python/cp36-cp36m/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin/"
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/python_connector_tests/scripts/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Add local user
+# Either use the LOCAL_USER_ID if passed in at runtime or
+# fallback
+
+USER_ID=${LOCAL_USER_ID:-9001}
+
+echo "Starting with UID : $USER_ID"
+useradd --shell /bin/bash -u $USER_ID -o -c "" -m user
+export HOME=/home/user
+
+/usr/local/bin/gosu user "$@"

scripts/anaconda/bld.bat

@@ -0,0 +1 @@
+$PYTHON setup.py install

scripts/anaconda/build.sh

@@ -0,0 +1 @@
+$PYTHON setup.py install --single-version-externally-managed --record=record.txt

scripts/anaconda/meta.yaml

@@ -0,0 +1,29 @@
+package:
+    name: snowflake_connector_python
+    version: "1.2.3"
+ 
+source:
+    path: /tmp/anaconda_workspace/src
+ 
+requirements:
+    build:
+        - python
+        - setuptools
+
+    run:
+        - python
+        - boto3 ==1.3.1
+        - botocore ==1.4.26
+        - future
+        - six
+        - pytz
+        - pycrypto ==2.6.1
+        - pyopenssl ==0.15.1
+        - cryptography ==1.2.3
+        - cffi ==1.6.0
+ 
+about:
+    home: https://www.snowflake.net/
+    license: Apache 2.0
+    license_file: /tmp/anaconda_workspace/src/LICENSE.txt
+    summary: Snowflake Connector for Python