Add a bunch of scripts

.gitlab-ci.yml

@@ -1,7 +1,12 @@
+stages:
+  - build
+  - deploy
+  - publish
+
 cache:
   key: ${CI_COMMIT_REF_SLUG}
   paths:
-  - lib/
+    - lib/
 
 before_script:
   - echo Current working directory is $(pwd)
@@ -53,7 +58,7 @@ build linux x86_64:
     # Build
     - qmake -spec linux-clang
     - make -j4
-    
+
     # Post-processing
     - upx --lzma -9 --force bin/Attorney_Online
   artifacts:
@@ -101,7 +106,7 @@ build windows i686:
     cp -a ../README.md README.md.txt
     cp -a ../LICENSE.MIT LICENSE.txt
 
-deploy linux:
+deploy linux x86_64:
   stage: deploy
   dependencies:
     - build linux x86_64
@@ -124,13 +129,13 @@ deploy linux:
     # Zipping
     # zip -r -9 -l Attorney_Online_$(git describe --tags)_linux_x86_64.zip .
     - mkdir ../zip
-    - tar cavf ../zip/Attorney_Online_$(git describe --tags)_x64.tar.xz *
+    - tar cavf ../zip/Attorney_Online_$(git describe --tags)_linux_x64.tar.xz *
     - sha1sum ../zip/*
   artifacts:
     paths:
       - zip/
 
-deploy windows:
+deploy windows i686:
   image: ubuntu
   stage: deploy
   dependencies:
@@ -154,8 +159,34 @@ deploy windows:
     # Zipping
     # -r: recursive; -9: max compression; -l: convert to CR LF
     - mkdir ../zip
-    - zip -r -9 -l ../zip/Attorney_Online_$(git describe --tags)_windows_i386.zip .
+    - zip -r -9 -l ../zip/Attorney_Online_$(git describe --tags)_windows_x86.zip .
     - sha1sum ../zip/*
   artifacts:
     paths:
-      - zip/
+      - zip/
+
+publish linux x86_64:
+  image: ubuntu
+  stage: publish
+  dependencies:
+    - deploy linux x86_64
+  when: manual
+  script:
+    - cd zip
+    - ../scripts/wasabi.sh
+  variables:
+    MANIFEST: program_linux_x86_64.json
+    ARTIFACT_SUFFIX: _linux_x64.tar.xz
+
+publish windows i686:
+  image: ubuntu
+  stage: publish
+  dependencies:
+    - deploy windows i686
+  when: manual
+  script:
+    - cd zip
+    - ../scripts/wasabi.sh
+  variables:
+    MANIFEST: program_winnt_i386.json
+    ARTIFACT_SUFFIX: _windows_x86.zip

.travis.yml

@@ -7,6 +7,5 @@ addons:
             - qt5
 
 script:
-    # XXX: please chmod +x the shell script.
+    - ./scripts/macos_build.sh
-    - bash ./scripts/macos_build.sh
+    - ./scripts/macos_post_build.sh
-    - bash ./scripts/macos_post_build.sh

scripts/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "ao-ci-scripts",
+  "version": "1.0.0",
+  "main": "update_manifest.js",
+  "dependencies": {
+    "argparse": "^1.0.10"
+  },
+  "license": "ISC"
+}

scripts/update_manifest.js

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const crypto = require("crypto");
+const ArgumentParser = require("argparse").ArgumentParser;
+
+function isFile(file) {
+    if (!fs.existsSync(file)) {
+        console.error(`File '${file}' not found. Try again.`);
+        throw Error();
+    }
+    return file;
+}
+
+const argParser = new ArgumentParser({
+    addHelp: true,
+    description: "Adds a new latest version to the manifest file based on the " +
+        "provided zip file, including an incremental update."
+});
+argParser.addArgument("manifestFile", {
+    metavar: "<manifest file>", type: isFile
+});
+argParser.addArgument("version", {
+    metavar: "<version>"
+});
+argParser.addArgument([ "-f", "--full" ], {
+    metavar: "<full zip file>", type: isFile, nargs: 1,
+    dest: "fullZipFile"
+});
+argParser.addArgument([ "-i", "--incremental" ], {
+    type: isFile, nargs: 2, dest: "incremental",
+    metavar: ["<incremental zip file>", "<file containing list of deleted files>"]
+});
+argParser.addArgument([ "-e", "--executable" ], {
+    metavar: "[executable file]", nargs: 1,
+    dest: "executable"
+});
+
+const {
+    manifestFile,
+    version,
+    fullZipFile,
+    incremental: [incrementalZipFile, deletionsFile],
+    executable
+} = argParser.parseArgs();
+
+const manifest = JSON.parse(fs.readFileSync(manifestFile));
+
+const deleteActions = deletionsFile ? fs.readFileSync(deletionsFile)
+    .split("\n").map(file => {
+        // XXX: This does not delete empty directories. Finding them would
+        // actually be a substantial amount of work because Git does not
+        // give us a good way of finding directories that were deleted.
+        return { action: "delete", target: file };
+    }) : [];
+
+const urlBase = "https://s3.wasabisys.com/ao-downloads/";
+
+manifest.versions = [{
+    version,
+    executable,
+    prev: manifest.version[0] ? manifest.version[0].version : undefined,
+    full: fullZipFile ? [
+        {
+            action: "dl",
+            url: urlBase + encodeURIComponent(fullZipFile),
+            hash: crypto.createHash("sha1")
+                .update(fs.readFileSync(fullZipFile))
+                .digest("hex")
+        }
+    ] : undefined,
+    update: incremental ? [
+        ...deleteActions,
+        {
+            action: "dl",
+            url: urlBase + encodeURIComponent(incrementalZipFile),
+            hash: crypto.createHash("sha1")
+                .update(fs.readFileSync(incrementalZipFile))
+                .digest("hex")
+        }
+    ] : undefined
+}, ...manifest.versions];
+
+fs.writeFileSync(manifestFile, JSON.stringify(manifest, null, 4));

scripts/update_program_manifest.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const crypto = require("crypto");
+
+const [ _nodeExe, _jsPath, manifestFile, version, zipFile ] = process.argv;
+
+if (!manifestFile || !version || !zipFile) {
+    console.log(`Usage: update_program_manifest <manifest file> <version> <zip file>`);
+    console.log(`Adds a new latest version to the manifest file based on the ` +
+                `provided zip file.`);
+    process.exit(1);
+}
+
+if (!fs.existsSync(manifestFile)) {
+    console.error(`Manifest file '${manifestFile}' not found. Try again.`);
+    process.exit(2);
+}
+
+if (!fs.existsSync(zipFile)) {
+    console.error(`Zip file '${zipFile}' not found. Try again.`);
+    process.exit(2);
+}
+
+const manifest = JSON.parse(fs.readFileSync(manifestFile));
+
+manifest.versions = [{
+    version,
+    executable: "Attorney_Online.exe",
+    full: [
+        {
+            action: "dl",
+            url: "https://s3.wasabisys.com/ao-downloads/" + encodeURIComponent(zipFile),
+            hash: crypto.createHash("sha1").update(fs.readFileSync(zipFile)).digest("hex")
+        }
+    ]
+}, ...manifest.versions];
+
+fs.writeFileSync(manifestFile, JSON.stringify(manifest, null, 4));

scripts/wasabi_asset.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Updates the specified program manifest to a new archive and version
+# and uploads the new archive and manifest to S3/Wasabi.
+#
+# Requires:
+#   MANIFEST: name of the manifest file
+#   ARTIFACT_SUFFIX: suffix of the archive to be uploaded (including extension)
+#   S3_ACCESS_KEY and S3_SECRET_KEY
+
+
+# -E: inherit ERR trap by shell functions
+# -e: stop script on ERR trap
+# -u: stop script on unbound variables
+# -x: print command before running it
+# -o pipefail: fail if any command in a pipeline fails
+set -Eeuxo pipefail
+
+aws configure set aws_access_key_id ${S3_ACCESS_KEY}
+aws configure set aws_secret_access_key ${S3_SECRET_KEY}
+aws configure set default.region us-east-1
+
+export S3_COPY="aws s3 cp --endpoint-url=https://s3.wasabisys.com"
+export S3_MANIFESTS="s3://ao-manifests"
+export S3_ARCHIVES="s3://ao-downloads"
+
+export VERSION=$(git describe --tags)
+export ARCHIVE_FULL="vanilla_full_${VERSION}_${ARTIFACT_SUFFIX}"
+export ARCHIVE_INCR="vanilla_update_${VERSION}_${ARTIFACT_SUFFIX}"
+
+git log --diff-filter=D --summary $(git rev-list --tags --skip=1 --max-count=1)..HEAD | \
+    grep "delete mode" | cut -d' ' -f 5- > deletions.txt
+
+${S3_COPY} ${S3_MANIFESTS}/${MANIFEST} .
+if [[ -n FULL_ZIP && FULL_ZIP != '0' ]]; then
+    node $(dirname $0)/update_manifest.js ${MANIFEST} ${VERSION}
+        -f ${ARCHIVE_FULL} -i ${ARCHIVE_INCR} deletions.txt
+    ${S3_COPY} ${ARCHIVE_FULL} ${S3_ARCHIVES}
+else
+    node $(dirname $0)/update_manifest.js ${MANIFEST} ${VERSION}
+        -i ${ARCHIVE_INCR} deletions.txt
+fi
+${S3_COPY} ${ARCHIVE_INCR} ${S3_ARCHIVES}
+${S3_COPY} ${MANIFEST} ${S3_MANIFESTS}
+
+rm -f ${MANIFEST}

scripts/wasabi_program.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Updates the specified program manifest to a new archive and version
+# and uploads the new archive and manifest to S3/Wasabi.
+#
+# Requires:
+#   MANIFEST: name of the manifest file
+#   ARTIFACT_SUFFIX: suffix of the archive to be uploaded (including extension)
+#   S3_ACCESS_KEY and S3_SECRET_KEY
+
+
+# -E: inherit ERR trap by shell functions
+# -e: stop script on ERR trap
+# -u: stop script on unbound variables
+# -x: print command before running it
+# -o pipefail: fail if any command in a pipeline fails
+set -Eeuxo pipefail
+
+aws configure set aws_access_key_id ${S3_ACCESS_KEY}
+aws configure set aws_secret_access_key ${S3_SECRET_KEY}
+aws configure set default.region us-east-1
+
+export S3_COPY="aws s3 cp --endpoint-url=https://s3.wasabisys.com"
+export S3_MANIFESTS="s3://ao-manifests"
+export S3_ARCHIVES="s3://ao-downloads"
+
+export VERSION=$(git describe --tags)
+export ARCHIVE="Attorney_Online_${VERSION}_${ARTIFACT_SUFFIX}"
+
+${S3_COPY} ${S3_MANIFESTS}/${MANIFEST} .
+node $(dirname $0)/update_manifest.js ${MANIFEST} ${VERSION} \
+    -f ${ARCHIVE} -e Attorney_Online.exe
+${S3_COPY} ${ARCHIVE} ${S3_ARCHIVES}
+${S3_COPY} ${MANIFEST} ${S3_MANIFESTS}
+
+rm -f ${MANIFEST}

scripts/windows/Dockerfile

@@ -0,0 +1,17 @@
+FROM oldmud0/mxe-qt:5.12.1-win32-static-posix
+#FROM fffaraz/qt:windows
+
+ENV TARGET_SPEC i686-w64-mingw32.static.posix
+
+# Build Discord RPC statically
+RUN git clone https://github.com/discordapp/discord-rpc
+WORKDIR discord-rpc/build
+RUN /opt/mxe/usr/bin/${TARGET_SPEC}-cmake .. -DCMAKE_INSTALL_PREFIX=/opt/mxe/usr/${TARGET_SPEC}
+RUN /opt/mxe/usr/bin/${TARGET_SPEC}-cmake --build . --config Release --target install
+WORKDIR ../..
+
+# Build QtApng statically
+RUN git clone https://github.com/Skycoder42/QtApng
+WORKDIR QtApng
+RUN /opt/mxe/usr/${TARGET_SPEC}/qt5/bin/qmake && make qmake_all && make && make install
+WORKDIR ..

scripts/windows/Dockerfile-mxe

@@ -0,0 +1,44 @@
+FROM ubuntu:18.04
+
+RUN apt-get update
+RUN apt-get install -y \
+    autoconf \
+    automake \
+    autopoint \
+    bash \
+    bison \
+    bzip2 \
+    flex \
+    g++ \
+    g++-multilib \
+    gettext \
+    git \
+    gperf \
+    intltool \
+    libc6-dev-i386 \
+    libgdk-pixbuf2.0-dev \
+    libltdl-dev \
+    libssl-dev \
+    libtool-bin \
+    libxml-parser-perl \
+    lzip \
+    make \
+    openssl \
+    p7zip-full \
+    patch \
+    perl \
+    pkg-config \
+    python \
+    ruby \
+    sed \
+    unzip \
+    wget \
+    xz-utils
+
+RUN git clone https://github.com/mxe/mxe.git
+RUN mv mxe /opt/mxe
+WORKDIR /opt/mxe
+RUN make -j4 MXE_TARGETS="i686-w64-mingw32.static.posix" qtbase qtmultimedia
+ENV PATH=/opt/mxe/usr/bin:$PATH
+
+WORKDIR /

scripts/windows/how-to-push.md

@@ -0,0 +1,19 @@
+When you want to build a new version of Qt:
+```docker
+docker build -t mxe-windows-static . -f Dockerfile-mxe
+docker tag mxe-windows-static oldmud0/mxe-qt:5.12.1-win32-static-posix
+docker push oldmud0/mxe-qt:5.12.1-win32-static-posix
+```
+
+Remember to log into Docker Hub before attempting to push.
+
+When you want to build a new version of any dependency required for building AO:
+```docker
+docker build -t mxe-windows-static-ao . -f Dockerfile
+docker tag mxe-windows-static-ao registry.gitlab.com/attorneyonline/ao2-client/builder-windows-i686
+docker push registry.gitlab.com/attorneyonline/ao2-client/builder-windows-i686
+```
+
+Remember to create an access token in GitLab before attempting to push.
+
+GitLab CI depends on `builder-windows-i686` image to be present in the repository's registry in order for the Windows build to succeed.