Merge pull request #2 from open-data-rescue/create-docker-image

Build docker image logic and some typo fixes

Author: balen

.github/workflows/deploy.yml

@@ -0,0 +1,95 @@
+---
+name: "Build a Docker image for deployment"
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+        type: choice
+        options:
+        - info
+        - warning
+        - debug
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    # No env...
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          swap-size-mb: 4096
+          temp-reserve-mb: 4096
+          root-reserve-mb: 16384
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+
+      - uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern=latest
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Builder instance name
+        run: echo ${{ steps.buildx.outputs.name }}
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      #
+      - name: Build and Push the image
+        id: docker_push
+        uses: docker/build-push-action@v4
+        with:
+          context: ./
+          file: ./Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      # Needed to prevent cache from growing forever (see https://github.com/docker/build-push-action/issues/252)
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.gitignore

@@ -0,0 +1,18 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile ~/.gitignore_global
+
+/log
+/tmp
+tmp/*
+rerun.txt
+*.DS_Store
+.project
+.docker-sync
+.envrc
+.vscode
+
+# Ignore pow environment settings
+.powenv

DRAW-post-processing/database_connection.py

@@ -10,23 +10,27 @@
 phase_2_errors=[]
 duplicateless=[]
 
+#
 db_user=os.environ.get('DRAW_local_db_user')
 db_passwd=os.environ.get('DRAW_local_db_pass')
 db_name=os.environ.get('DRAW_local_db_name')
-db_host=os.environ.get('DRAW_db_host')
+db_host=os.environ.get('DRAW_db_host',"127.0.0.1")
+db_port=os.environ.get('DRAW_db_port',"3306")
 
 # connection to copy of database on local machine
 conn = mysql.connector.connect(
     #####   FOLLOWING 3 VARIABLES TO BE CONFIGURED AS NECESSARY FOR LOCAL MACHINE:   #####
     user=db_user,
     password=db_passwd,
     database=db_name,
-    host=db_host
+    host=db_host,
+    port=db_port
 )
 
 cursor = conn.cursor()
 
-url = "mysql+mysqlconnector://"+db_user+":"+db_passwd+"@l"+db_host+"/"+db_name
+# Problem? dialect+driver://username:password@host:port/database
+url = "mysql+mysqlconnector://"+db_user+":"+db_passwd+"@"+db_host+":"+db_port+"/"+db_name
 engine = sqlalchemy.create_engine(url)
 
 

DRAW-post-processing/requirements.txt

@@ -0,0 +1,6 @@
+##### Required packages for the draw python code
+mysql-connector-python
+sqlalchemy
+matplotlib
+Numpy
+pandas

DRAW-post-processing/sef_gen.py

@@ -33,7 +33,8 @@ def getPeriod(sef_type):
 
 
 def getFilename(sef_type,type_result_set):
-    filename=os.environ.get('DRAW_sef_folder')+os.sep+"McGill_DRAW_1491_"
+    # FileNotFoundError: [Errno 2] No such file or directory: 'SEF/McGill_DRAW_1491_1874-10_1934-5-atb.tsv'
+    filename=os.environ.get('DRAW_sef_folder', 'SEF')+os.sep+"McGill_DRAW_1491_"
 
     if len(type_result_set)>0:
         sorted_type_results=sorted(type_result_set)
@@ -106,7 +107,7 @@ def generateSEF(sef_type):
         f.write ("Lat\t45.5\n")
         f.write ("Lon\t-73.59\n")
         f.write ("Alt\t49\n")
-        f.write ("Source\McGill\n")
+        f.write ("Source\tMcGill\n")
         f.write ("Link\thttps://draw.geog.mcgill.ca/\n")
         f.write ("Vbl\t" + config.sef_type_to_unit[sef_type]+"\n")
         f.write ("Stat\t")

Dockerfile

@@ -0,0 +1,16 @@
+# Dockerfile
+# To run the Python post processing
+# 
+FROM python:3
+
+# WORKDIR /setup
+ADD ./DRAW-post-processing /opt/draw-post-processing
+WORKDIR /opt/draw-post-processing
+
+RUN mkdir /opt/draw-post-processing/SEF
+
+# Install the packages that are needed
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Run the post process script
+CMD [ "python", "./execute_post_process.py" ]