Update to Python 3.10, fix GitHub workflow and automated tests

.github/workflows/im-build.yml

@@ -8,12 +8,12 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
-    
+
     services:
       # Label used to access the service container
       postgres:
-        image: postgres:11
-        env: 
+        image: postgres:14
+        env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -22,24 +22,35 @@ jobs:
           --health-timeout 5s
           --health-retries 5
         ports:
-          - 5432/tcp
-    env:
-      TESTMODEL_URL: http://localhost:8080/intermine-demo
-      TESTMODEL_PATH: intermine-demo
+          - 5432:5432
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up python 3.7
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v2
+      with:
+        distribution: 'temurin'
+        java-version: '11'
+    - name: Set up python 3.10
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.7'
+        python-version: '3.10'
     - name: Install dependencies
       run: |
+        set -euo pipefail
+        sudo apt-get install libxml2-dev libxslt-dev python3-dev
         python -m pip install --upgrade pip
+        pip install -U setuptools
         pip install -r requirements.txt
     - name: Install PostgreSQL client
       run: |
+        set -euo pipefail
         sudo apt-get update -y
         sudo apt-get install -y libpq-dev postgresql-client
         sudo service postgresql start
     - name: Run unit tests
-      run: ./config/ci/init-solr.sh && ./config/ci/init.sh && python setup.py test && python setup.py livetest
+      run: |
+        set -euo pipefail
+        ./config/ci/init-solr.sh ${GITHUB_WORKSPACE}
+        ./config/ci/init.sh ${GITHUB_WORKSPACE} http://localhost:8080/intermine-demo
+        python setup.py test
+        python setup.py livetest

.gitignore

@@ -8,3 +8,10 @@ intermine-docs-1.00.03.zip
 build
 MANIFEST
 .idea
+
+# CI
+
+server/
+
+solr-*.tgz
+solr-*/

config/ci/init-solr.sh

@@ -1,7 +1,46 @@
+#!/bin/bash
+
 # set up solr for testmine
 # testmine's setup script populates these empty indexes
 
-wget http://archive.apache.org/dist/lucene/solr/8.6.2/solr-8.6.2.tgz
-tar xzf solr-8.6.2.tgz && ./solr-8.6.2/bin/solr start
-./solr-8.6.2/bin/solr create -c intermine-search
-./solr-8.6.2/bin/solr create -c intermine-autocomplete
+set -euo pipefail
+
+if [ "$#" != "1" ]; then
+   echo "Usage: $0 <workspace_dir>"
+   exit 2
+fi
+
+WORKSPACE_DIR=$1
+
+cd "$WORKSPACE_DIR"
+
+SOLR_VERSION=8.6.2
+SOLR_PACKAGE=solr-${SOLR_VERSION}.tgz
+SOLR_DIR=solr-${SOLR_VERSION}
+SOLR=${SOLR_DIR}/bin/solr
+
+create_solr_core() {
+    local core_name=$1
+
+    local status
+
+    status=$(curl -s "http://localhost:8983/solr/admin/cores?action=STATUS&core=${core_name}" | jq --arg core_name "${core_name}" '.status[$core_name]')
+
+    if [ "$status" = "{}" ]; then
+        ${SOLR} create -c "${core_name}"
+    else
+        echo "Solr core ${core_name} already exists"
+    fi
+}
+
+if [ ! -d $SOLR_DIR ]; then
+    if [ ! -f $SOLR_PACKAGE ]; then
+        wget http://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/${SOLR_PACKAGE}
+    fi
+
+    tar xzf $SOLR_PACKAGE
+fi
+
+${SOLR} restart
+create_solr_core intermine-search
+create_solr_core intermine-autocomplete

config/ci/init.sh

@@ -1,44 +1,61 @@
 #!/bin/bash
 
-set -e
+set -euo pipefail
 
-if [ -z $(which wget) ]; then
-    # use curl
-    GET='curl'
-else
-    GET='wget -O -'
+if [ "$#" != "2" ]; then
+   echo "Usage: $0 <workspace_dir> <testmodel_url>"
+   exit 2
 fi
 
+WORKSPACE_DIR=$1
+TESTMODEL_URL=$2
+
+INTERMINE_DIR=${WORKSPACE_DIR}/server
+TESTMINE_DIR=${INTERMINE_DIR}/testmine
+
+cd "${WORKSPACE_DIR}"
+
 # Pull in the server code.
-git clone --single-branch --depth 1 https://github.com/intermine/intermine.git server
+
+if [ ! -d "${INTERMINE_DIR}" ]; then
+    git clone --single-branch --depth 1 -b dev https://github.com/ucam-department-of-psychiatry/intermine.git "${INTERMINE_DIR}"
+fi
 
 export PSQL_USER=test
 export PSQL_PWD=test
+export PSQL_HOST=localhost
+export PGPASSWORD=${PGPASSWORD:-postgres}
 export KEYSTORE=${PWD}/keystore.jks
 
 echo "#---> Running unit tests"
 
-sudo -u postgres createuser test
-sudo -u postgres psql -c "alter user test with encrypted password 'test';"
+sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists intermine-demo
+sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists userprofile-demo
+
+sudo -E -u postgres dropuser -h "${PSQL_HOST}" --if-exists test
+sudo -E -u postgres createuser -h "${PSQL_HOST}" test
+sudo -E -u postgres psql -h "${PSQL_HOST}" -c "alter user test with encrypted password 'test';"
 
 # Set up properties
-PROPDIR=$HOME/.intermine
-TESTMODEL_PROPS=$PROPDIR/testmodel.properties
-SED_SCRIPT='s/PSQL_USER/test/'
+PROPDIR=${HOME}/.intermine
+TESTMODEL_PROPS=${PROPDIR}/testmodel.properties
+
+mkdir -p "${PROPDIR}"
 
-mkdir -p $PROPDIR
+echo "#--- creating ${TESTMODEL_PROPS}"
+cp "${INTERMINE_DIR}"/config/testmodel.properties "${TESTMODEL_PROPS}"
+sed -i -e "s/PSQL_HOST/${PSQL_HOST}/" "$TESTMODEL_PROPS"
+sed -i -e "s/PSQL_USER/${PSQL_USER}/" "$TESTMODEL_PROPS"
+sed -i -e "s/PSQL_PWD/${PSQL_PWD}/" "$TESTMODEL_PROPS"
 
-echo "#--- creating $TESTMODEL_PROPS"
-cp server/config/testmodel.properties $TESTMODEL_PROPS
-sed -i -e $SED_SCRIPT $TESTMODEL_PROPS
 
 # We will need a fully operational web-application
 echo '#---> Building and releasing web application to test against'
-(cd server/testmine && ./setup.sh)
+(cd "${TESTMINE_DIR}" && ./setup.sh "${INTERMINE_DIR}")
 # Travis is so slow
 sleep 90 # let webapp startup
 
 # Warm up the keyword search by requesting results, but ignoring the results
-$GET "$TESTMODEL_URL/service/search" > /dev/null
+wget -O - "${TESTMODEL_URL}/service/search" > /dev/null
 # Start any list upgrades
-$GET "$TESTMODEL_URL/service/lists?token=test-user-token" > /dev/null
+wget -O - "${TESTMODEL_URL}/service/lists?token=test-user-token" > /dev/null

intermine/results.py

@@ -603,7 +603,7 @@ def __init__(self, credentials=None, token=None):
         self.token = token
         if credentials and len(credentials) == 2:
             encoded = '{0}:{1}'.format(*credentials).encode('utf8')
-            base64string = 'Basic {0}'.format(base64.encodestring(encoded)[:-1].decode('ascii'))
+            base64string = 'Basic {0}'.format(base64.encodebytes(encoded)[:-1].decode('ascii'))
             self.auth_header = base64string
             self.using_authentication = True
         elif self.token is not None:

intermine/webservice.py

@@ -5,16 +5,10 @@
 
 import requests
 
-try:
-    from urlparse import urlparse
-    from UserDict import DictMixin
-    from urllib import urlopen
-    from urllib import urlencode
-except ImportError:
-    from urllib.parse import urlparse
-    from urllib.parse import urlencode
-    from collections import MutableMapping as DictMixin
-    from urllib.request import urlopen
+from urllib.parse import urlparse
+from urllib.parse import urlencode
+from collections.abc import MutableMapping as DictMixin
+from urllib.request import urlopen
 
 try:
     import simplejson as json  # Prefer this as it is faster

requirements.txt

@@ -1,17 +1,17 @@
-certifi==2020.4.5.1
-chardet==3.0.4
-cycler==0.10.0
-Cython==0.29.19
-idna==2.9
-kiwisolver==1.2.0
-lxml==4.5.1
-matplotlib==3.2.1
-numpy==1.18.4
-pandas==1.0.3
-pyparsing==2.4.7
-python-dateutil==2.8.1
-pytz==2020.1
-requests==2.23.0
-simplejson==3.17.0
-six==1.15.0
-urllib3==1.25.9
+certifi==2024.8.30
+chardet==5.2.0
+cycler==0.12.1
+Cython==3.0.11
+idna==3.8
+kiwisolver==1.4.5
+lxml==5.3.0
+matplotlib==3.9.2
+numpy==2.1.0
+pandas==2.2.2
+pyparsing==3.1.4
+python-dateutil==2.9.0
+pytz==2024.1
+requests==2.32.4
+simplejson==3.19.3
+six==1.16.0
+urllib3==2.5.0