diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 928352f..d05a337 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,12 @@ +stages: + - build + - deploy + - publish + cache: key: ${CI_COMMIT_REF_SLUG} paths: - - lib/ + - lib/ before_script: - echo Current working directory is $(pwd) @@ -53,7 +58,7 @@ build linux x86_64: # Build - qmake -spec linux-clang - make -j4 - + # Post-processing - upx --lzma -9 --force bin/Attorney_Online artifacts: @@ -101,7 +106,7 @@ build windows i686: cp -a ../README.md README.md.txt cp -a ../LICENSE.MIT LICENSE.txt -deploy linux: +deploy linux x86_64: stage: deploy dependencies: - build linux x86_64 @@ -124,13 +129,13 @@ deploy linux: # Zipping # zip -r -9 -l Attorney_Online_$(git describe --tags)_linux_x86_64.zip . - mkdir ../zip - - tar cavf ../zip/Attorney_Online_$(git describe --tags)_x64.tar.xz * + - tar cavf ../zip/Attorney_Online_$(git describe --tags)_linux_x64.tar.xz * - sha1sum ../zip/* artifacts: paths: - zip/ -deploy windows: +deploy windows i686: image: ubuntu stage: deploy dependencies: @@ -154,8 +159,34 @@ deploy windows: # Zipping # -r: recursive; -9: max compression; -l: convert to CR LF - mkdir ../zip - - zip -r -9 -l ../zip/Attorney_Online_$(git describe --tags)_windows_i386.zip . + - zip -r -9 -l ../zip/Attorney_Online_$(git describe --tags)_windows_x86.zip . - sha1sum ../zip/* artifacts: paths: - - zip/ \ No newline at end of file + - zip/ + +publish linux x86_64: + image: ubuntu + stage: publish + dependencies: + - deploy linux x86_64 + when: manual + script: + - cd zip + - ../scripts/wasabi.sh + variables: + MANIFEST: program_linux_x86_64.json + ARTIFACT_SUFFIX: _linux_x64.tar.xz + +publish windows i686: + image: ubuntu + stage: publish + dependencies: + - deploy windows i686 + when: manual + script: + - cd zip + - ../scripts/wasabi.sh + variables: + MANIFEST: program_winnt_i386.json + ARTIFACT_SUFFIX: _windows_x86.zip diff --git a/.travis.yml b/.travis.yml index dfde416..4243b3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,5 @@ addons: - qt5 script: - # XXX: please chmod +x the shell script. - - bash ./scripts/macos_build.sh - - bash ./scripts/macos_post_build.sh + - ./scripts/macos_build.sh + - ./scripts/macos_post_build.sh diff --git a/scripts/macos_build.sh b/scripts/macos_build.sh old mode 100644 new mode 100755 diff --git a/scripts/macos_post_build.sh b/scripts/macos_post_build.sh old mode 100644 new mode 100755 diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 0000000..126a392 --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,9 @@ +{ + "name": "ao-ci-scripts", + "version": "1.0.0", + "main": "update_manifest.js", + "dependencies": { + "argparse": "^1.0.10" + }, + "license": "ISC" +} diff --git a/scripts/update_manifest.js b/scripts/update_manifest.js new file mode 100755 index 0000000..5f64cfb --- /dev/null +++ b/scripts/update_manifest.js @@ -0,0 +1,84 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const crypto = require("crypto"); +const ArgumentParser = require("argparse").ArgumentParser; + +function isFile(file) { + if (!fs.existsSync(file)) { + console.error(`File '${file}' not found. Try again.`); + throw Error(); + } + return file; +} + +const argParser = new ArgumentParser({ + addHelp: true, + description: "Adds a new latest version to the manifest file based on the " + + "provided zip file, including an incremental update." +}); +argParser.addArgument("manifestFile", { + metavar: "", type: isFile +}); +argParser.addArgument("version", { + metavar: "" +}); +argParser.addArgument([ "-f", "--full" ], { + metavar: "", type: isFile, nargs: 1, + dest: "fullZipFile" +}); +argParser.addArgument([ "-i", "--incremental" ], { + type: isFile, nargs: 2, dest: "incremental", + metavar: ["", ""] +}); +argParser.addArgument([ "-e", "--executable" ], { + metavar: "[executable file]", nargs: 1, + dest: "executable" +}); + +const { + manifestFile, + version, + fullZipFile, + incremental: [incrementalZipFile, deletionsFile], + executable +} = argParser.parseArgs(); + +const manifest = JSON.parse(fs.readFileSync(manifestFile)); + +const deleteActions = deletionsFile ? fs.readFileSync(deletionsFile) + .split("\n").map(file => { + // XXX: This does not delete empty directories. Finding them would + // actually be a substantial amount of work because Git does not + // give us a good way of finding directories that were deleted. + return { action: "delete", target: file }; + }) : []; + +const urlBase = "https://s3.wasabisys.com/ao-downloads/"; + +manifest.versions = [{ + version, + executable, + prev: manifest.version[0] ? manifest.version[0].version : undefined, + full: fullZipFile ? [ + { + action: "dl", + url: urlBase + encodeURIComponent(fullZipFile), + hash: crypto.createHash("sha1") + .update(fs.readFileSync(fullZipFile)) + .digest("hex") + } + ] : undefined, + update: incremental ? [ + ...deleteActions, + { + action: "dl", + url: urlBase + encodeURIComponent(incrementalZipFile), + hash: crypto.createHash("sha1") + .update(fs.readFileSync(incrementalZipFile)) + .digest("hex") + } + ] : undefined +}, ...manifest.versions]; + +fs.writeFileSync(manifestFile, JSON.stringify(manifest, null, 4)); diff --git a/scripts/update_program_manifest.js b/scripts/update_program_manifest.js new file mode 100755 index 0000000..9efc814 --- /dev/null +++ b/scripts/update_program_manifest.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const crypto = require("crypto"); + +const [ _nodeExe, _jsPath, manifestFile, version, zipFile ] = process.argv; + +if (!manifestFile || !version || !zipFile) { + console.log(`Usage: update_program_manifest `); + console.log(`Adds a new latest version to the manifest file based on the ` + + `provided zip file.`); + process.exit(1); +} + +if (!fs.existsSync(manifestFile)) { + console.error(`Manifest file '${manifestFile}' not found. Try again.`); + process.exit(2); +} + +if (!fs.existsSync(zipFile)) { + console.error(`Zip file '${zipFile}' not found. Try again.`); + process.exit(2); +} + +const manifest = JSON.parse(fs.readFileSync(manifestFile)); + +manifest.versions = [{ + version, + executable: "Attorney_Online.exe", + full: [ + { + action: "dl", + url: "https://s3.wasabisys.com/ao-downloads/" + encodeURIComponent(zipFile), + hash: crypto.createHash("sha1").update(fs.readFileSync(zipFile)).digest("hex") + } + ] +}, ...manifest.versions]; + +fs.writeFileSync(manifestFile, JSON.stringify(manifest, null, 4)); \ No newline at end of file diff --git a/scripts/wasabi_asset.sh b/scripts/wasabi_asset.sh new file mode 100755 index 0000000..d424183 --- /dev/null +++ b/scripts/wasabi_asset.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Updates the specified program manifest to a new archive and version +# and uploads the new archive and manifest to S3/Wasabi. +# +# Requires: +# MANIFEST: name of the manifest file +# ARTIFACT_SUFFIX: suffix of the archive to be uploaded (including extension) +# S3_ACCESS_KEY and S3_SECRET_KEY + + +# -E: inherit ERR trap by shell functions +# -e: stop script on ERR trap +# -u: stop script on unbound variables +# -x: print command before running it +# -o pipefail: fail if any command in a pipeline fails +set -Eeuxo pipefail + +aws configure set aws_access_key_id ${S3_ACCESS_KEY} +aws configure set aws_secret_access_key ${S3_SECRET_KEY} +aws configure set default.region us-east-1 + +export S3_COPY="aws s3 cp --endpoint-url=https://s3.wasabisys.com" +export S3_MANIFESTS="s3://ao-manifests" +export S3_ARCHIVES="s3://ao-downloads" + +export VERSION=$(git describe --tags) +export ARCHIVE_FULL="vanilla_full_${VERSION}_${ARTIFACT_SUFFIX}" +export ARCHIVE_INCR="vanilla_update_${VERSION}_${ARTIFACT_SUFFIX}" + +git log --diff-filter=D --summary $(git rev-list --tags --skip=1 --max-count=1)..HEAD | \ + grep "delete mode" | cut -d' ' -f 5- > deletions.txt + +${S3_COPY} ${S3_MANIFESTS}/${MANIFEST} . +if [[ -n FULL_ZIP && FULL_ZIP != '0' ]]; then + node $(dirname $0)/update_manifest.js ${MANIFEST} ${VERSION} + -f ${ARCHIVE_FULL} -i ${ARCHIVE_INCR} deletions.txt + ${S3_COPY} ${ARCHIVE_FULL} ${S3_ARCHIVES} +else + node $(dirname $0)/update_manifest.js ${MANIFEST} ${VERSION} + -i ${ARCHIVE_INCR} deletions.txt +fi +${S3_COPY} ${ARCHIVE_INCR} ${S3_ARCHIVES} +${S3_COPY} ${MANIFEST} ${S3_MANIFESTS} + +rm -f ${MANIFEST} diff --git a/scripts/wasabi_program.sh b/scripts/wasabi_program.sh new file mode 100755 index 0000000..41e2e35 --- /dev/null +++ b/scripts/wasabi_program.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Updates the specified program manifest to a new archive and version +# and uploads the new archive and manifest to S3/Wasabi. +# +# Requires: +# MANIFEST: name of the manifest file +# ARTIFACT_SUFFIX: suffix of the archive to be uploaded (including extension) +# S3_ACCESS_KEY and S3_SECRET_KEY + + +# -E: inherit ERR trap by shell functions +# -e: stop script on ERR trap +# -u: stop script on unbound variables +# -x: print command before running it +# -o pipefail: fail if any command in a pipeline fails +set -Eeuxo pipefail + +aws configure set aws_access_key_id ${S3_ACCESS_KEY} +aws configure set aws_secret_access_key ${S3_SECRET_KEY} +aws configure set default.region us-east-1 + +export S3_COPY="aws s3 cp --endpoint-url=https://s3.wasabisys.com" +export S3_MANIFESTS="s3://ao-manifests" +export S3_ARCHIVES="s3://ao-downloads" + +export VERSION=$(git describe --tags) +export ARCHIVE="Attorney_Online_${VERSION}_${ARTIFACT_SUFFIX}" + +${S3_COPY} ${S3_MANIFESTS}/${MANIFEST} . +node $(dirname $0)/update_manifest.js ${MANIFEST} ${VERSION} \ + -f ${ARCHIVE} -e Attorney_Online.exe +${S3_COPY} ${ARCHIVE} ${S3_ARCHIVES} +${S3_COPY} ${MANIFEST} ${S3_MANIFESTS} + +rm -f ${MANIFEST} diff --git a/scripts/windows/Dockerfile b/scripts/windows/Dockerfile new file mode 100644 index 0000000..90d6c27 --- /dev/null +++ b/scripts/windows/Dockerfile @@ -0,0 +1,17 @@ +FROM oldmud0/mxe-qt:5.12.1-win32-static-posix +#FROM fffaraz/qt:windows + +ENV TARGET_SPEC i686-w64-mingw32.static.posix + +# Build Discord RPC statically +RUN git clone https://github.com/discordapp/discord-rpc +WORKDIR discord-rpc/build +RUN /opt/mxe/usr/bin/${TARGET_SPEC}-cmake .. -DCMAKE_INSTALL_PREFIX=/opt/mxe/usr/${TARGET_SPEC} +RUN /opt/mxe/usr/bin/${TARGET_SPEC}-cmake --build . --config Release --target install +WORKDIR ../.. + +# Build QtApng statically +RUN git clone https://github.com/Skycoder42/QtApng +WORKDIR QtApng +RUN /opt/mxe/usr/${TARGET_SPEC}/qt5/bin/qmake && make qmake_all && make && make install +WORKDIR .. diff --git a/scripts/windows/Dockerfile-mxe b/scripts/windows/Dockerfile-mxe new file mode 100644 index 0000000..e6caec5 --- /dev/null +++ b/scripts/windows/Dockerfile-mxe @@ -0,0 +1,44 @@ +FROM ubuntu:18.04 + +RUN apt-get update +RUN apt-get install -y \ + autoconf \ + automake \ + autopoint \ + bash \ + bison \ + bzip2 \ + flex \ + g++ \ + g++-multilib \ + gettext \ + git \ + gperf \ + intltool \ + libc6-dev-i386 \ + libgdk-pixbuf2.0-dev \ + libltdl-dev \ + libssl-dev \ + libtool-bin \ + libxml-parser-perl \ + lzip \ + make \ + openssl \ + p7zip-full \ + patch \ + perl \ + pkg-config \ + python \ + ruby \ + sed \ + unzip \ + wget \ + xz-utils + +RUN git clone https://github.com/mxe/mxe.git +RUN mv mxe /opt/mxe +WORKDIR /opt/mxe +RUN make -j4 MXE_TARGETS="i686-w64-mingw32.static.posix" qtbase qtmultimedia +ENV PATH=/opt/mxe/usr/bin:$PATH + +WORKDIR / diff --git a/scripts/windows/how-to-push.md b/scripts/windows/how-to-push.md new file mode 100644 index 0000000..8c1c18d --- /dev/null +++ b/scripts/windows/how-to-push.md @@ -0,0 +1,19 @@ +When you want to build a new version of Qt: +```docker +docker build -t mxe-windows-static . -f Dockerfile-mxe +docker tag mxe-windows-static oldmud0/mxe-qt:5.12.1-win32-static-posix +docker push oldmud0/mxe-qt:5.12.1-win32-static-posix +``` + +Remember to log into Docker Hub before attempting to push. + +When you want to build a new version of any dependency required for building AO: +```docker +docker build -t mxe-windows-static-ao . -f Dockerfile +docker tag mxe-windows-static-ao registry.gitlab.com/attorneyonline/ao2-client/builder-windows-i686 +docker push registry.gitlab.com/attorneyonline/ao2-client/builder-windows-i686 +``` + +Remember to create an access token in GitLab before attempting to push. + +GitLab CI depends on `builder-windows-i686` image to be present in the repository's registry in order for the Windows build to succeed.