Log directly to GitHub

Author: thriqon

invfile.lua.md

@@ -70,7 +70,7 @@ We also determine where the results will be stored:
 
     quay_prefix = 'mulled'
     namespace = 'quay.io/' .. quay_prefix
-    github_repo = 'mulled/api'
+    github_repo = 'mulled/mulled'
 
     current_build_id =
       'https://travis-ci.org/mulled/mulled/builds/'
@@ -519,41 +519,23 @@ contains one line, and the combination of `--raw-input` and `--slurp` makes
 `jq` convert this to a big string, with newlines as separators.  This array is
 split and indexed with positional indexes.
 
-Similar to the Quay.io repository description, the description and homepage are
-taken from the latest revision.
-
-However, we have to make sure that the file containing 'old' revisions exists in
-the package information directory. If any old revisions exist, they are copied over,
-otherwise a new array is used.
-
-    .using(busybox)
-      .withHostConfig({binds = {
-        builddir .. ':/pkg',
-        './data:/data'
-      }})
-      .run('/bin/sh', '-c',
-        '(tail -n +3 /data/api/_images/' .. package .. '.json'
-        .. '|| echo "{}") > /pkg/info/github.json')
-
     .using(jq)
       .withHostConfig({binds = {
         builddir .. ':/pkg',
         "./data:/data"
       }})
-      .run('/jq-linux64 --raw-output --raw-input --slurp \'.|split("\n") as $i'
-        .. '| ("---\n---\n" + ({'
-          .. 'image: "' .. package .. '",'
+      .run('/jq-linux64 -c --raw-input --slurp \'.|split("\n") as $i'
+        .. '| {image: "' .. package .. '", '
           .. 'packager: "' .. packager .. '", '
           .. 'homepage: $i[0], description: $i[1], '
-          .. 'versions: ($i[3] | fromjson | (.versions // []) | '
-          .. '  map(select(.revision != "' .. new_revision
-          .. '" )) + [{version: $i[2], revision: "' .. new_revision
-          .. '", size: $i[4], date: (now | todate)}]),'
-          .. 'buildurl: "' .. current_build_id .. '"}'
-          .. '|tostring))\''
+          .. 'travisid: ' .. ENV.TRAVIS_BUILD_ID .. ','
+          .. 'travisslug: "' .. ENV.TRAVIS_REPO_SLUG .. '",'
+          .. 'date: "\'$(date -Iseconds)\'", '
+          .. 'version: $i[2], revision: "'
+          .. new_revision .. '", size: $i[3]}\''
           .. ' /pkg/info/homepage /pkg/info/description '
-          .. '/pkg/info/version /pkg/info/github.json  /pkg/info/size '
-          .. ' > /data/api/_images/' .. package .. '.json')
+          .. '/pkg/info/version /pkg/info/size '
+          .. ' >> /data/loglines.jsonl')
     end
 
 # The Build Tasks
@@ -613,7 +595,7 @@ attach all predefined, package specific tasks to the overall tasks:
         end
       end
       deploy
-        .runTask('main:commit_api_database')
+        .runTask('main:upload_build_log')
     end
 
 # Determining What To Build
@@ -622,55 +604,21 @@ It is highly inefficient to build every single package every time a build is
 invoked. In this tool, we restrict ourselves to building packages in versions
 that are not already registered in our API database.
 
-Firstly, we fetch the current API 'database' from GitHub. This database is
-comprised of several JSON files containing common and revision specific
-information for each package. Each file may look like the following example
-(note the YAML frontmatter for Jekyll):
-
-```json
----
----
-{
- "image": "bc",
- "packager": "alpine",
- "homepage": "http://www.gnu.org/software/bc/bc.html",
- "description": "An arbitrary precision numeric processing language (calculator)",
- "versions": [
-   {
-   "revision": "1",
-   "version": "1.06.95-r2",
-   "size": 5000,
-   "date": "2016-01-26T11:00:02Z",
-   "buildurl": "https://travis.ci-org./mulled/mulled/builds/203830"
-   },
-   {
-   "revision": "2",
-   "version": "1.06.95-r3",
-   "size": 4000,
-   "date": "2016-01-26T11:02:02Z",
-   "buildurl": "https://travis.ci-org./mulled/mulled/builds/203831"
-  }
- ]
-}
-```
-
-We clone the repository into a subdirectory called `data/api`:
-
     inv.task('main:versions:clone_from_github')
       .using(git)
-        .run('clone', 'https://github.com/mulled/api.git', 'data/api')
-
-These data files still contain the YAML frontmatter, and have to be transformed
-to make easy use of them in later steps. A `jq` program is used to read them in,
-and store a single JSON object assinging a list of versions to each package.
-
-    local parseGithubImages = 'map({(.image): (.versions | map(.revision))}) | add // {}'
+        .run('fetch', '--unshallow')
+        .run('config', 'remote.origin.fetch',
+          '+refs/heads/*:refs/remotes/origin/*')
+        .run('fetch', 'origin', 'gh-pages')
+        .run('checkout', 'origin/gh-pages',
+          '--', '_builds')
 
     inv.task('main:versions:process_versions_from_github')
-      .using(busybox)
-        .run('/bin/sh', '-c', 'tail -q -n +3 data/api/_images/*json > data/github_jsons.jsonl || touch data/github_jsons.jsonl')
       .using(jq)
-        .run('/jq-linux64 --slurp \'' .. parseGithubImages .. '\' data/github_jsons.jsonl > data/github_versions')
+        .run('cat _builds/* | grep -v -- "---" | /jq-linux64 --slurp --raw-output '
+          .. '\'map(.packages)|flatten|group_by(.image)|'
+          .. 'map({key: .[0].image, value: map(.revision)})|from_entries\' '
+          .. ' > data/github_versions')
 
 As the next step, we parse the list of locally defined packages and intersect
 this with the list of remotely available images. The `packages.tsv` is split
@@ -735,22 +683,39 @@ duplicates and tells the user.
 
 # Final Steps
 
-If we are doing a production build on Travis, we have to commit/push the new
-API database.
+If we are doing a production build on Travis, we have to upload the build log
+to GitHub.
 
-    inv.task('main:commit_api_database')
-      .using(git)
-        .withHostConfig({
-          binds = {
-            "./data/git/.gitconfig:/root/.gitconfig",
-            "./data/netrc:/root/.netrc",
-            "./data/api:/source"
-          }})
-        .run('add', '_images')
-        .withConfig({entrypoint = {'/bin/sh', '-c'}})
-        .run('git commit -m "Build '
-          .. ENV.TRAVIS_BUILD_NUMBER .. '" || true')
-        .run('git push origin gh-pages')
+    inv.task('main:upload_build_log')
+      .using(busybox)
+        .run('touch', 'data/loglines.jsonl')
+      .using(jq)
+        .run('/jq-linux64 --slurp '
+          .. '\'{title: "' .. ENV.TRAVIS_BUILD_NUMBER .. '",'
+          .. 'packages: .}\' '
+          .. 'data/loglines.jsonl > data/log.json')
+      .using(busybox)
+        .withHostConfig({binds = {
+          './data:/data'
+        }})
+        .run('/bin/sh', '-c', 'cat /data/log.json')
+        .run('/bin/sh', '-c',
+          '(echo "---"; '
+          .. 'cat /data/log.json; '
+          .. 'echo "---") > /data/build_log')
+      .using(curl)
+        .withConfig({
+          env = {"TOKEN=" .. ENV.GITHUB_TOKEN}
+        })
+        .run('/bin/sh', '-c',
+          'curl -X PUT '
+          .. 'https://api.github.com/repos/'
+           .. github_repo .. '/contents/_builds/'
+           .. ENV.TRAVIS_BUILD_NUMBER .. '-$(date +%s).json '
+          .. '-d \'{"message":"no ' .. ENV.TRAVIS_BUILD_NUMBER
+           .. '", "branch": "gh-pages", "content": "'
+           .. '\'$(base64 data/build_log | tr -d "\n")\'"}\' '
+          .. '-HAuthorization:Bearer\\ $TOKEN ')
 
 # Appendix