diff options
Diffstat (limited to '.ci/scripts')
| -rw-r--r-- | .ci/scripts/common/post-upload.sh | 15 | ||||
| -rw-r--r-- | .ci/scripts/common/pre-upload.sh | 6 | ||||
| -rw-r--r-- | .ci/scripts/format/docker.sh | 6 | ||||
| -rw-r--r-- | .ci/scripts/format/exec.sh | 4 | ||||
| -rw-r--r-- | .ci/scripts/format/script.sh | 37 | ||||
| -rw-r--r-- | .ci/scripts/linux/docker.sh | 14 | ||||
| -rw-r--r-- | .ci/scripts/linux/exec.sh | 5 | ||||
| -rw-r--r-- | .ci/scripts/linux/upload.sh | 14 | ||||
| -rw-r--r-- | .ci/scripts/merge/apply-patches-by-label-private.py | 41 | ||||
| -rw-r--r-- | .ci/scripts/merge/apply-patches-by-label.py | 30 | ||||
| -rw-r--r-- | .ci/scripts/merge/check-label-presence.py | 18 | ||||
| -rw-r--r-- | .ci/scripts/merge/yuzubot-git-config.sh | 2 | ||||
| -rw-r--r-- | .ci/scripts/windows/docker.sh | 50 | ||||
| -rw-r--r-- | .ci/scripts/windows/exec.sh | 5 | ||||
| -rw-r--r-- | .ci/scripts/windows/scan_dll.py | 106 | ||||
| -rw-r--r-- | .ci/scripts/windows/upload.sh | 13 |
16 files changed, 366 insertions, 0 deletions
diff --git a/.ci/scripts/common/post-upload.sh b/.ci/scripts/common/post-upload.sh new file mode 100644 index 000000000..bb4e9d328 --- /dev/null +++ b/.ci/scripts/common/post-upload.sh @@ -0,0 +1,15 @@ +#!/bin/bash -ex + +# Copy documentation +cp license.txt "$REV_NAME" +cp README.md "$REV_NAME" + +tar $COMPRESSION_FLAGS "$ARCHIVE_NAME" "$REV_NAME" + +mv "$REV_NAME" $RELEASE_NAME + +7z a "$REV_NAME.7z" $RELEASE_NAME + +# move the compiled archive into the artifacts directory to be uploaded by travis releases +mv "$ARCHIVE_NAME" artifacts/ +mv "$REV_NAME.7z" artifacts/ diff --git a/.ci/scripts/common/pre-upload.sh b/.ci/scripts/common/pre-upload.sh new file mode 100644 index 000000000..3c2fc79a2 --- /dev/null +++ b/.ci/scripts/common/pre-upload.sh @@ -0,0 +1,6 @@ +#!/bin/bash -ex + +GITDATE="`git show -s --date=short --format='%ad' | sed 's/-//g'`" +GITREV="`git show -s --format='%h'`" + +mkdir -p artifacts diff --git a/.ci/scripts/format/docker.sh b/.ci/scripts/format/docker.sh new file mode 100644 index 000000000..778411e4a --- /dev/null +++ b/.ci/scripts/format/docker.sh @@ -0,0 +1,6 @@ +#!/bin/bash -ex + +# Run clang-format +cd /yuzu +chmod a+x ./.ci/scripts/format/script.sh +./.ci/scripts/format/script.sh diff --git a/.ci/scripts/format/exec.sh b/.ci/scripts/format/exec.sh new file mode 100644 index 000000000..5d6393b38 --- /dev/null +++ b/.ci/scripts/format/exec.sh @@ -0,0 +1,4 @@ +#!/bin/bash -ex + +chmod a+x ./.ci/scripts/format/docker.sh +docker run -v $(pwd):/yuzu yuzuemu/build-environments:linux-clang-format /bin/bash -ex /yuzu/.ci/scripts/format/docker.sh diff --git a/.ci/scripts/format/script.sh b/.ci/scripts/format/script.sh new file mode 100644 index 000000000..5ab828d5e --- /dev/null +++ b/.ci/scripts/format/script.sh @@ -0,0 +1,37 @@ +#!/bin/bash -ex + +if grep -nrI '\s$' src *.yml *.txt *.md Doxyfile .gitignore .gitmodules .ci* dist/*.desktop \ + dist/*.svg dist/*.xml; then + echo Trailing whitespace found, aborting + exit 1 +fi + +# Default clang-format points to default 3.5 version one +CLANG_FORMAT=clang-format-6.0 +$CLANG_FORMAT --version + +if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then + # Get list of every file modified in this pull request + files_to_lint="$(git diff --name-only --diff-filter=ACMRTUXB $TRAVIS_COMMIT_RANGE | grep '^src/[^.]*[.]\(cpp\|h\)$' || true)" +else + # Check everything for branch pushes + files_to_lint="$(find src/ -name '*.cpp' -or -name '*.h')" +fi + +# Turn off tracing for this because it's too verbose +set +x + +for f in $files_to_lint; do + d=$(diff -u "$f" <($CLANG_FORMAT "$f") || true) + if ! [ -z "$d" ]; then + echo "!!! $f not compliant to coding style, here is the fix:" + echo "$d" + fail=1 + fi +done + +set -x + +if [ "$fail" = 1 ]; then + exit 1 +fi diff --git a/.ci/scripts/linux/docker.sh b/.ci/scripts/linux/docker.sh new file mode 100644 index 000000000..f538a4081 --- /dev/null +++ b/.ci/scripts/linux/docker.sh @@ -0,0 +1,14 @@ +#!/bin/bash -ex + +cd /yuzu + +ccache -s + +mkdir build || true && cd build +cmake .. -G Ninja -DYUZU_USE_BUNDLED_UNICORN=ON -DYUZU_USE_QT_WEB_ENGINE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=/usr/lib/ccache/gcc -DCMAKE_CXX_COMPILER=/usr/lib/ccache/g++ -DYUZU_ENABLE_COMPATIBILITY_REPORTING=${ENABLE_COMPATIBILITY_REPORTING:-"OFF"} -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON -DUSE_DISCORD_PRESENCE=ON + +ninja + +ccache -s + +ctest -VV -C Release diff --git a/.ci/scripts/linux/exec.sh b/.ci/scripts/linux/exec.sh new file mode 100644 index 000000000..a5a6c34b9 --- /dev/null +++ b/.ci/scripts/linux/exec.sh @@ -0,0 +1,5 @@ +#!/bin/bash -ex + +mkdir -p "ccache" || true +chmod a+x ./.ci/scripts/linux/docker.sh +docker run -e ENABLE_COMPATIBILITY_REPORTING -e CCACHE_DIR=/yuzu/ccache -v $(pwd):/yuzu yuzuemu/build-environments:linux-fresh /bin/bash /yuzu/.ci/scripts/linux/docker.sh diff --git a/.ci/scripts/linux/upload.sh b/.ci/scripts/linux/upload.sh new file mode 100644 index 000000000..0d131d1dd --- /dev/null +++ b/.ci/scripts/linux/upload.sh @@ -0,0 +1,14 @@ +#!/bin/bash -ex + +. .ci/scripts/common/pre-upload.sh + +REV_NAME="yuzu-linux-${GITDATE}-${GITREV}" +ARCHIVE_NAME="${REV_NAME}.tar.xz" +COMPRESSION_FLAGS="-cJvf" + +mkdir "$REV_NAME" + +cp build/bin/yuzu-cmd "$REV_NAME" +cp build/bin/yuzu "$REV_NAME" + +. .ci/scripts/common/post-upload.sh diff --git a/.ci/scripts/merge/apply-patches-by-label-private.py b/.ci/scripts/merge/apply-patches-by-label-private.py new file mode 100644 index 000000000..11ec60010 --- /dev/null +++ b/.ci/scripts/merge/apply-patches-by-label-private.py @@ -0,0 +1,41 @@ +# Download all pull requests as patches that match a specific label +# Usage: python download-patches-by-label.py <Label to Match> <Root Path Folder to DL to> + +import requests, sys, json, urllib3.request, shutil, subprocess, os, traceback + +org = os.getenv("PrivateMergeOrg".upper(), "yuzu-emu") +repo = os.getenv("PrivateMergeRepo".upper(), "yuzu-private") +tagline = os.getenv("MergeTaglinePrivate".upper(), "") +user = sys.argv[1] + +http = urllib3.PoolManager() +dl_list = {} + +def check_individual(repo_id, pr_id): + url = 'https://%sdev.azure.com/%s/%s/_apis/git/repositories/%s/pullRequests/%s/labels?api-version=5.1-preview.1' % (user, org, repo, repo_id, pr_id) + response = requests.get(url) + if (response.ok): + j = json.loads(response.content) + for tg in j['value']: + if (tg['name'] == sys.argv[2]): + return True + return False + +try: + url = 'https://%sdev.azure.com/%s/%s/_apis/git/pullrequests?api-version=5.1' % (user, org, repo) + response = requests.get(url) + if (response.ok): + j = json.loads(response.content) + for pr in j["value"]: + repo_id = pr['repository']['id'] + pr_id = pr['pullRequestId'] + if (check_individual(repo_id, pr_id)): + pn = pr_id + ref = pr['sourceRefName'] + print("Matched PR# %s" % pn) + print(subprocess.check_output(["git", "fetch", "https://%sdev.azure.com/%s/_git/%s" % (user, org, repo), ref, "-f"])) + print(subprocess.check_output(["git", "merge", "--squash", 'origin/' + ref.replace('refs/heads/','')])) + print(subprocess.check_output(["git", "commit", "-m\"Merge %s PR %s\"" % (tagline, pn)])) +except: + traceback.print_exc(file=sys.stdout) + sys.exit(-1) diff --git a/.ci/scripts/merge/apply-patches-by-label.py b/.ci/scripts/merge/apply-patches-by-label.py new file mode 100644 index 000000000..7f1ea06cf --- /dev/null +++ b/.ci/scripts/merge/apply-patches-by-label.py @@ -0,0 +1,30 @@ +# Download all pull requests as patches that match a specific label +# Usage: python download-patches-by-label.py <Label to Match> <Root Path Folder to DL to> + +import requests, sys, json, urllib3.request, shutil, subprocess, os + +tagline = os.getenv("MergeTaglinePublic".upper(), "") + +http = urllib3.PoolManager() +dl_list = {} + +def check_individual(labels): + for label in labels: + if (label["name"] == sys.argv[1]): + return True + return False + +try: + url = 'https://api.github.com/repos/yuzu-emu/yuzu/pulls' + response = requests.get(url) + if (response.ok): + j = json.loads(response.content) + for pr in j: + if (check_individual(pr["labels"])): + pn = pr["number"] + print("Matched PR# %s" % pn) + print(subprocess.check_output(["git", "fetch", "https://github.com/yuzu-emu/yuzu.git", "pull/%s/head:pr-%s" % (pn, pn), "-f"])) + print(subprocess.check_output(["git", "merge", "--squash", "pr-%s" % pn])) + print(subprocess.check_output(["git", "commit", "-m\"Merge %s PR %s\"" % (tagline, pn)])) +except: + sys.exit(-1) diff --git a/.ci/scripts/merge/check-label-presence.py b/.ci/scripts/merge/check-label-presence.py new file mode 100644 index 000000000..048466d7e --- /dev/null +++ b/.ci/scripts/merge/check-label-presence.py @@ -0,0 +1,18 @@ +# Checks to see if the specified pull request # has the specified tag +# Usage: python check-label-presence.py <Pull Request ID> <Name of Label> + +import requests, json, sys + +try: + url = 'https://api.github.com/repos/yuzu-emu/yuzu/issues/%s' % sys.argv[1] + response = requests.get(url) + if (response.ok): + j = json.loads(response.content) + for label in j["labels"]: + if label["name"] == sys.argv[2]: + print('##vso[task.setvariable variable=enabletesting;]true') + sys.exit() +except: + sys.exit(-1) + +print('##vso[task.setvariable variable=enabletesting;]false') diff --git a/.ci/scripts/merge/yuzubot-git-config.sh b/.ci/scripts/merge/yuzubot-git-config.sh new file mode 100644 index 000000000..d9d595bbc --- /dev/null +++ b/.ci/scripts/merge/yuzubot-git-config.sh @@ -0,0 +1,2 @@ +git config --global user.email "yuzu@yuzu-emu.org" +git config --global user.name "yuzubot"
\ No newline at end of file diff --git a/.ci/scripts/windows/docker.sh b/.ci/scripts/windows/docker.sh new file mode 100644 index 000000000..f7093363b --- /dev/null +++ b/.ci/scripts/windows/docker.sh @@ -0,0 +1,50 @@ +#!/bin/bash -ex + +cd /yuzu + +ccache -s + +# Dirty hack to trick unicorn makefile into believing we are in a MINGW system +mv /bin/uname /bin/uname1 && echo -e '#!/bin/sh\necho MINGW64' >> /bin/uname +chmod +x /bin/uname + +# Dirty hack to trick unicorn makefile into believing we have cmd +echo '' >> /bin/cmd +chmod +x /bin/cmd + +mkdir build || true && cd build +cmake .. -G Ninja -DCMAKE_TOOLCHAIN_FILE="$(pwd)/../CMakeModules/MinGWCross.cmake" -DUSE_CCACHE=ON -DYUZU_USE_BUNDLED_UNICORN=ON -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON -DCMAKE_BUILD_TYPE=Release +ninja + +# Clean up the dirty hacks +rm /bin/uname && mv /bin/uname1 /bin/uname +rm /bin/cmd + +ccache -s + +echo "Tests skipped" +#ctest -VV -C Release + +echo 'Prepare binaries...' +cd .. +mkdir package + +QT_PLATFORM_DLL_PATH='/usr/x86_64-w64-mingw32/lib/qt5/plugins/platforms/' +find build/ -name "yuzu*.exe" -exec cp {} 'package' \; + +# copy Qt plugins +mkdir package/platforms +cp "${QT_PLATFORM_DLL_PATH}/qwindows.dll" package/platforms/ +cp -rv "${QT_PLATFORM_DLL_PATH}/../mediaservice/" package/ +cp -rv "${QT_PLATFORM_DLL_PATH}/../imageformats/" package/ +rm -f package/mediaservice/*d.dll + +for i in package/*.exe; do + # we need to process pdb here, however, cv2pdb + # does not work here, so we just simply strip all the debug symbols + x86_64-w64-mingw32-strip "${i}" +done + +pip3 install pefile +python3 .ci/scripts/windows/scan_dll.py package/*.exe "package/" +python3 .ci/scripts/windows/scan_dll.py package/imageformats/*.dll "package/" diff --git a/.ci/scripts/windows/exec.sh b/.ci/scripts/windows/exec.sh new file mode 100644 index 000000000..d6a994856 --- /dev/null +++ b/.ci/scripts/windows/exec.sh @@ -0,0 +1,5 @@ +#!/bin/bash -ex + +mkdir -p "ccache" || true +chmod a+x ./.ci/scripts/windows/docker.sh +docker run -e CCACHE_DIR=/yuzu/ccache -v $(pwd):/yuzu yuzuemu/build-environments:linux-mingw /bin/bash -ex /yuzu/.ci/scripts/windows/docker.sh diff --git a/.ci/scripts/windows/scan_dll.py b/.ci/scripts/windows/scan_dll.py new file mode 100644 index 000000000..163183f2e --- /dev/null +++ b/.ci/scripts/windows/scan_dll.py @@ -0,0 +1,106 @@ +import pefile +import sys +import re +import os +import queue +import shutil + +# constant definitions +KNOWN_SYS_DLLS = ['WINMM.DLL', 'MSVCRT.DLL', 'VERSION.DLL', 'MPR.DLL', + 'DWMAPI.DLL', 'UXTHEME.DLL', 'DNSAPI.DLL', 'IPHLPAPI.DLL'] +# below is for Ubuntu 18.04 with specified PPA enabled, if you are using +# other distro or different repositories, change the following accordingly +DLL_PATH = [ + '/usr/x86_64-w64-mingw32/bin/', + '/usr/x86_64-w64-mingw32/lib/', + '/usr/lib/gcc/x86_64-w64-mingw32/7.3-posix/' +] + +missing = [] + + +def parse_imports(file_name): + results = [] + pe = pefile.PE(file_name, fast_load=True) + pe.parse_data_directories() + + for entry in pe.DIRECTORY_ENTRY_IMPORT: + current = entry.dll.decode() + current_u = current.upper() # b/c Windows is often case insensitive + # here we filter out system dlls + # dll w/ names like *32.dll are likely to be system dlls + if current_u.upper() not in KNOWN_SYS_DLLS and not re.match(string=current_u, pattern=r'.*32\.DLL'): + results.append(current) + + return results + + +def parse_imports_recursive(file_name, path_list=[]): + q = queue.Queue() # create a FIFO queue + # file_name can be a string or a list for the convience + if isinstance(file_name, str): + q.put(file_name) + elif isinstance(file_name, list): + for i in file_name: + q.put(i) + full_list = [] + while q.qsize(): + current = q.get_nowait() + print('> %s' % current) + deps = parse_imports(current) + # if this dll does not have any import, ignore it + if not deps: + continue + for dep in deps: + # the dependency already included in the list, skip + if dep in full_list: + continue + # find the requested dll in the provided paths + full_path = find_dll(dep) + if not full_path: + missing.append(dep) + continue + full_list.append(dep) + q.put(full_path) + path_list.append(full_path) + return full_list + + +def find_dll(name): + for path in DLL_PATH: + for root, _, files in os.walk(path): + for f in files: + if name.lower() == f.lower(): + return os.path.join(root, f) + + +def deploy(name, dst, dry_run=False): + dlls_path = [] + parse_imports_recursive(name, dlls_path) + for dll_entry in dlls_path: + if not dry_run: + shutil.copy(dll_entry, dst) + else: + print('[Dry-Run] Copy %s to %s' % (dll_entry, dst)) + print('Deploy completed.') + return dlls_path + + +def main(): + if len(sys.argv) < 3: + print('Usage: %s [files to examine ...] [target deploy directory]') + return 1 + to_deploy = sys.argv[1:-1] + tgt_dir = sys.argv[-1] + if not os.path.isdir(tgt_dir): + print('%s is not a directory.' % tgt_dir) + return 1 + print('Scanning dependencies...') + deploy(to_deploy, tgt_dir) + if missing: + print('Following DLLs are not found: %s' % ('\n'.join(missing))) + return 0 + + +if __name__ == '__main__': + main() diff --git a/.ci/scripts/windows/upload.sh b/.ci/scripts/windows/upload.sh new file mode 100644 index 000000000..de73d3541 --- /dev/null +++ b/.ci/scripts/windows/upload.sh @@ -0,0 +1,13 @@ +#!/bin/bash -ex + +. .ci/scripts/common/pre-upload.sh + +REV_NAME="yuzu-windows-mingw-${GITDATE}-${GITREV}" +ARCHIVE_NAME="${REV_NAME}.tar.gz" +COMPRESSION_FLAGS="-czvf" + +mkdir "$REV_NAME" +# get around the permission issues +cp -r package/* "$REV_NAME" + +. .ci/scripts/common/post-upload.sh |