From 1d6692c3a56004252e44ef87c2265b0f13276482 Mon Sep 17 00:00:00 2001 From: Roman Haefeli Date: Thu, 14 Jan 2021 21:54:02 +0100 Subject: [PATCH 001/157] add localdeps.linux.sh script --- pd-lib-builder/localdeps.linux.sh | 156 ++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100755 pd-lib-builder/localdeps.linux.sh diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh new file mode 100755 index 0000000..4769e9c --- /dev/null +++ b/pd-lib-builder/localdeps.linux.sh @@ -0,0 +1,156 @@ +#!/bin/sh +# +# creates local copies of all dependencies (dynamic libraries) +# and sets RUNPATH to $ORIGIN on each so they will find +# each other. +# +# usage: $0 + +case $1 in + "") + echo "Usage: /bin/sh ${0} " + exit + ;; + *) + binary_file=$1 + ;; +esac + +# List of libraries that we do not include into our packaging +# becaue we think they will be installed on any system +LD_EXCLUDE_LIST="linux-gate\.so.* +linux-vdso\.so.* +libarmmem.*\.so.* +libc.so\.* +ld-linux.*\.so.* +libdl\.so.* +libglib-.*\.so.* +libgomp\.so.* +libgthread.*\.so.* +libm\.so.* +libpthread.*\.so.* +libstdc++\.so.* +libgcc_s\.so.* +libpcre\.so.*" + +# Check dependencies +cmdlist="awk grep ldd patchelf uname" +for cmd in $cmdlist +do + if ! which $cmd > /dev/null + then + echo "Could not find ${cmd}. Is it installed?" > /dev/stderr + exit 1 + fi +done + +# Set LD_LIBRARY_PATH depending on arch +ARCH=$(uname -m) +case $ARCH in + "x86_64") + LD_LIBRARY_PATH="$HOME/.local/lib:/usr/local/lib64:/usr/lib/x86_64-linux-gnu" + ;; + "i686") + LD_LIBRARY_PATH="$HOME/.local/lib:/usr/local/lib:/usr/lib/i386-linux-gnu" + ;; + "armv7l") + LD_LIBRARY_PATH="$HOME/.local/lib:/usr/local/lib:/usr/lib/arm-linux-gnueabihf" + ;; + *) + echo "Arch '$ARCH' not (yet) supported. Please file a bug report" + exit 1 +esac + +# Check if we can read from given file +if ! ldd $binary_file > /dev/null 2>&1 +then + echo "Can't read '${binary_file}'. Is it a binary file?" > /dev/stderr + exit 1 +fi + +library_in_exclude_list() { +# arg1: library name +# returns 0 if arg1 is found in exclude list, otherwise 1 + libexname="$1" + skip=1 + set -f + for expat in $(echo "$LD_EXCLUDE_LIST") + do + if echo "$(basename $libexname)" | grep "${expat}" > /dev/null + then + skip=0 + break + fi + done + set +f + return $skip +} + +search_make_local_copy() { + # look for given library in all library paths + # and make a local copy of it + # arg1: name of the library to make a local copy of + found=false + IFSold=$IFS + IFS=: + for path in ${LD_LIBRARY_PATH} + do + [ -f "${path}/$1" ] && cp "${path}/$1" . && found=true && break + done + IFS=$IFSold + if ! $found + then + echo "$1 not found" > /dev/stderr + exit 1 + fi +} + +make_local_copy_and_set_rpath() { + # make a local copy of all linked libraries of given binary + # and set RUNPATH to $ORIGIN (exclude "standard" libraries) + # arg1: binary to check + ldd $1 | while read ldd_line + do + libname=$(echo "$ldd_line" | awk '{ print $1 }') + libpath=$(echo "$ldd_line" | awk '{ print $3 }') + if ! [ -f "$(basename $libname)" ] && ! library_in_exclude_list "$libname" + then + if [ "$libpath" != "" ] + then + cp "$libpath" . + elif echo "$libname" | grep '/' > /dev/null + then + cp "$libname" . + else + echo "Warning: could not make copy of '$libname'. Not found" + fi + fi + if ! library_in_exclude_list "$libname" + then + patchelf --set-rpath \$ORIGIN "$(basename $libname)" + fi + done +} + +find_missing() { + # find libraries that are shown as 'not found' in ldd and + # create a local copy of them. + # arg1: binary file to check for missing links + while true + do + ldd_output=$(ldd ${1}) + if echo "$ldd_output" | grep '=> not found' > /dev/null + then + next_missing=$(echo "$ldd_output" | grep '=> not found' | head -n1 | awk '{print $1}') + search_make_local_copy "$next_missing" + else + break + fi + done +} + +find_missing $binary_file +make_local_copy_and_set_rpath $binary_file + +# clean after ourselves +rm $0 -- GitLab From 91687346aac9ce2f56f71145e3b855d52ef1e9ab Mon Sep 17 00:00:00 2001 From: Roman Haefeli Date: Fri, 15 Jan 2021 21:45:25 +0100 Subject: [PATCH 002/157] do not consider libraries that don't need a lookup; shorten the list of manually excluded libraries (fixes #2 [partially]) --- pd-lib-builder/localdeps.linux.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 4769e9c..dbdb97d 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -18,11 +18,8 @@ esac # List of libraries that we do not include into our packaging # becaue we think they will be installed on any system -LD_EXCLUDE_LIST="linux-gate\.so.* -linux-vdso\.so.* -libarmmem.*\.so.* +LD_EXCLUDE_LIST="libarmmem.*\.so.* libc.so\.* -ld-linux.*\.so.* libdl\.so.* libglib-.*\.so.* libgomp\.so.* @@ -109,7 +106,7 @@ make_local_copy_and_set_rpath() { # make a local copy of all linked libraries of given binary # and set RUNPATH to $ORIGIN (exclude "standard" libraries) # arg1: binary to check - ldd $1 | while read ldd_line + ldd $1 | grep ' => ' | while read ldd_line do libname=$(echo "$ldd_line" | awk '{ print $1 }') libpath=$(echo "$ldd_line" | awk '{ print $3 }') -- GitLab From 2e7b039f267c8734fcc515d2554ad33c2b8c7b41 Mon Sep 17 00:00:00 2001 From: Roman Haefeli Date: Fri, 15 Jan 2021 22:50:20 +0100 Subject: [PATCH 003/157] do not try to fix broken build; skip part that's searching for missing libs; --- pd-lib-builder/localdeps.linux.sh | 54 ------------------------------- 1 file changed, 54 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index dbdb97d..73876c2 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -41,23 +41,6 @@ do fi done -# Set LD_LIBRARY_PATH depending on arch -ARCH=$(uname -m) -case $ARCH in - "x86_64") - LD_LIBRARY_PATH="$HOME/.local/lib:/usr/local/lib64:/usr/lib/x86_64-linux-gnu" - ;; - "i686") - LD_LIBRARY_PATH="$HOME/.local/lib:/usr/local/lib:/usr/lib/i386-linux-gnu" - ;; - "armv7l") - LD_LIBRARY_PATH="$HOME/.local/lib:/usr/local/lib:/usr/lib/arm-linux-gnueabihf" - ;; - *) - echo "Arch '$ARCH' not (yet) supported. Please file a bug report" - exit 1 -esac - # Check if we can read from given file if ! ldd $binary_file > /dev/null 2>&1 then @@ -83,25 +66,6 @@ library_in_exclude_list() { return $skip } -search_make_local_copy() { - # look for given library in all library paths - # and make a local copy of it - # arg1: name of the library to make a local copy of - found=false - IFSold=$IFS - IFS=: - for path in ${LD_LIBRARY_PATH} - do - [ -f "${path}/$1" ] && cp "${path}/$1" . && found=true && break - done - IFS=$IFSold - if ! $found - then - echo "$1 not found" > /dev/stderr - exit 1 - fi -} - make_local_copy_and_set_rpath() { # make a local copy of all linked libraries of given binary # and set RUNPATH to $ORIGIN (exclude "standard" libraries) @@ -129,24 +93,6 @@ make_local_copy_and_set_rpath() { done } -find_missing() { - # find libraries that are shown as 'not found' in ldd and - # create a local copy of them. - # arg1: binary file to check for missing links - while true - do - ldd_output=$(ldd ${1}) - if echo "$ldd_output" | grep '=> not found' > /dev/null - then - next_missing=$(echo "$ldd_output" | grep '=> not found' | head -n1 | awk '{print $1}') - search_make_local_copy "$next_missing" - else - break - fi - done -} - -find_missing $binary_file make_local_copy_and_set_rpath $binary_file # clean after ourselves -- GitLab From 18b5ecee2a9ebfe3aafce4ae85f6fbcbfac8b8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:13:08 +0100 Subject: [PATCH 004/157] drop self-removal --- pd-lib-builder/localdeps.linux.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 73876c2..d42bfbf 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -94,6 +94,3 @@ make_local_copy_and_set_rpath() { } make_local_copy_and_set_rpath $binary_file - -# clean after ourselves -rm $0 -- GitLab From 82b88188ad835a1a63af4bb97f3ccf2f6abfcb29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:13:50 +0100 Subject: [PATCH 005/157] re-indentation --- pd-lib-builder/localdeps.linux.sh | 106 +++++++++++++++--------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index d42bfbf..093eb21 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -7,13 +7,13 @@ # usage: $0 case $1 in - "") - echo "Usage: /bin/sh ${0} " - exit - ;; - *) - binary_file=$1 - ;; + "") + echo "Usage: /bin/sh ${0} " + exit + ;; + *) + binary_file=$1 + ;; esac # List of libraries that we do not include into our packaging @@ -34,63 +34,63 @@ libpcre\.so.*" cmdlist="awk grep ldd patchelf uname" for cmd in $cmdlist do - if ! which $cmd > /dev/null - then - echo "Could not find ${cmd}. Is it installed?" > /dev/stderr - exit 1 - fi + if ! which $cmd > /dev/null + then + echo "Could not find ${cmd}. Is it installed?" > /dev/stderr + exit 1 + fi done # Check if we can read from given file if ! ldd $binary_file > /dev/null 2>&1 then - echo "Can't read '${binary_file}'. Is it a binary file?" > /dev/stderr - exit 1 + echo "Can't read '${binary_file}'. Is it a binary file?" > /dev/stderr + exit 1 fi library_in_exclude_list() { -# arg1: library name -# returns 0 if arg1 is found in exclude list, otherwise 1 - libexname="$1" - skip=1 - set -f - for expat in $(echo "$LD_EXCLUDE_LIST") - do - if echo "$(basename $libexname)" | grep "${expat}" > /dev/null - then - skip=0 - break - fi - done - set +f - return $skip + # arg1: library name + # returns 0 if arg1 is found in exclude list, otherwise 1 + libexname="$1" + skip=1 + set -f + for expat in $(echo "$LD_EXCLUDE_LIST") + do + if echo "$(basename $libexname)" | grep "${expat}" > /dev/null + then + skip=0 + break + fi + done + set +f + return $skip } make_local_copy_and_set_rpath() { - # make a local copy of all linked libraries of given binary - # and set RUNPATH to $ORIGIN (exclude "standard" libraries) - # arg1: binary to check - ldd $1 | grep ' => ' | while read ldd_line - do - libname=$(echo "$ldd_line" | awk '{ print $1 }') - libpath=$(echo "$ldd_line" | awk '{ print $3 }') - if ! [ -f "$(basename $libname)" ] && ! library_in_exclude_list "$libname" - then - if [ "$libpath" != "" ] - then - cp "$libpath" . - elif echo "$libname" | grep '/' > /dev/null - then - cp "$libname" . - else - echo "Warning: could not make copy of '$libname'. Not found" - fi - fi - if ! library_in_exclude_list "$libname" - then - patchelf --set-rpath \$ORIGIN "$(basename $libname)" - fi - done + # make a local copy of all linked libraries of given binary + # and set RUNPATH to $ORIGIN (exclude "standard" libraries) + # arg1: binary to check + ldd $1 | grep ' => ' | while read ldd_line + do + libname=$(echo "$ldd_line" | awk '{ print $1 }') + libpath=$(echo "$ldd_line" | awk '{ print $3 }') + if ! [ -f "$(basename $libname)" ] && ! library_in_exclude_list "$libname" + then + if [ "$libpath" != "" ] + then + cp "$libpath" . + elif echo "$libname" | grep '/' > /dev/null + then + cp "$libname" . + else + echo "Warning: could not make copy of '$libname'. Not found" + fi + fi + if ! library_in_exclude_list "$libname" + then + patchelf --set-rpath \$ORIGIN "$(basename $libname)" + fi + done } make_local_copy_and_set_rpath $binary_file -- GitLab From bdd767f31624dd89daaba9a9f9e606df5577e072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:16:49 +0100 Subject: [PATCH 006/157] use "1>&2" instead of ">/dev/stderr" just my personal style --- pd-lib-builder/localdeps.linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 093eb21..45775c6 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -36,7 +36,7 @@ for cmd in $cmdlist do if ! which $cmd > /dev/null then - echo "Could not find ${cmd}. Is it installed?" > /dev/stderr + echo "Could not find ${cmd}. Is it installed?" 1>&2 exit 1 fi done @@ -44,7 +44,7 @@ done # Check if we can read from given file if ! ldd $binary_file > /dev/null 2>&1 then - echo "Can't read '${binary_file}'. Is it a binary file?" > /dev/stderr + echo "Can't read '${binary_file}'. Is it a binary file?" 1>&2 exit 1 fi -- GitLab From adda4677e5cd8c9714e7f62ce179e90735b4d154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:17:15 +0100 Subject: [PATCH 007/157] quote filenames --- pd-lib-builder/localdeps.linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 45775c6..dfd0bec 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -34,7 +34,7 @@ libpcre\.so.*" cmdlist="awk grep ldd patchelf uname" for cmd in $cmdlist do - if ! which $cmd > /dev/null + if ! which "${cmd}" > /dev/null then echo "Could not find ${cmd}. Is it installed?" 1>&2 exit 1 @@ -42,7 +42,7 @@ do done # Check if we can read from given file -if ! ldd $binary_file > /dev/null 2>&1 +if ! ldd "${binary_file}" > /dev/null 2>&1 then echo "Can't read '${binary_file}'. Is it a binary file?" 1>&2 exit 1 -- GitLab From 5212095254532de5327ce8108f7ab3ea565cf3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:17:30 +0100 Subject: [PATCH 008/157] assume that the script is executable --- pd-lib-builder/localdeps.linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index dfd0bec..e86ec9f 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -8,7 +8,7 @@ case $1 in "") - echo "Usage: /bin/sh ${0} " + echo "Usage: ${0} " exit ;; *) -- GitLab From cf7227bb409e7af481c56f34dbd4c3be6be3a5ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:17:56 +0100 Subject: [PATCH 009/157] slightly re-order the LD_EXCLUDE_LIST, so libc is on top that entry will never change --- pd-lib-builder/localdeps.linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index e86ec9f..28070b4 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -18,8 +18,8 @@ esac # List of libraries that we do not include into our packaging # becaue we think they will be installed on any system -LD_EXCLUDE_LIST="libarmmem.*\.so.* -libc.so\.* +LD_EXCLUDE_LIST="libc\.so\.* +libarmmem.*\.so.* libdl\.so.* libglib-.*\.so.* libgomp\.so.* -- GitLab From cc9926bc0094b53bc5f638067dcb63ea86c3ac2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:18:17 +0100 Subject: [PATCH 010/157] simplify localisation --- pd-lib-builder/localdeps.linux.sh | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 28070b4..7f98487 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -70,26 +70,17 @@ make_local_copy_and_set_rpath() { # make a local copy of all linked libraries of given binary # and set RUNPATH to $ORIGIN (exclude "standard" libraries) # arg1: binary to check - ldd $1 | grep ' => ' | while read ldd_line - do - libname=$(echo "$ldd_line" | awk '{ print $1 }') - libpath=$(echo "$ldd_line" | awk '{ print $3 }') - if ! [ -f "$(basename $libname)" ] && ! library_in_exclude_list "$libname" - then - if [ "$libpath" != "" ] - then - cp "$libpath" . - elif echo "$libname" | grep '/' > /dev/null - then - cp "$libname" . - else - echo "Warning: could not make copy of '$libname'. Not found" - fi + ldd $1 | grep ' => ' | while read _ _ libpath _; do + libname=$(basename "${libpath}") + if library_in_exclude_list "$libname"; then + continue fi - if ! library_in_exclude_list "$libname" - then - patchelf --set-rpath \$ORIGIN "$(basename $libname)" + if [ ! -e "${libpath}" ]; then + echo "Warning: could not make copy of '${libpath}'. Not found" 1>&2 + continue fi + cp "$libpath" . + patchelf --set-rpath \$ORIGIN "${libname}" done } -- GitLab From 281502e437348b4388b0a56b580301d7989a0b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:18:25 +0100 Subject: [PATCH 011/157] use local vars --- pd-lib-builder/localdeps.linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 7f98487..b85f7d8 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -51,7 +51,7 @@ fi library_in_exclude_list() { # arg1: library name # returns 0 if arg1 is found in exclude list, otherwise 1 - libexname="$1" + local libexname="$1" skip=1 set -f for expat in $(echo "$LD_EXCLUDE_LIST") -- GitLab From e0c937de6f58e22d3fe829860aaa2eea94d99f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:18:32 +0100 Subject: [PATCH 012/157] don't search for uname --- pd-lib-builder/localdeps.linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index b85f7d8..3c90a59 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -31,7 +31,7 @@ libgcc_s\.so.* libpcre\.so.*" # Check dependencies -cmdlist="awk grep ldd patchelf uname" +cmdlist="awk grep ldd patchelf" for cmd in $cmdlist do if ! which "${cmd}" > /dev/null -- GitLab From 6c9cfa0e32894f3ea8152c75c910dd4d046bd2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:22:25 +0100 Subject: [PATCH 013/157] lowercase LD_EXCLUDE_LIST not an envvar --- pd-lib-builder/localdeps.linux.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 3c90a59..45d7d82 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -18,7 +18,7 @@ esac # List of libraries that we do not include into our packaging # becaue we think they will be installed on any system -LD_EXCLUDE_LIST="libc\.so\.* +ld_exclude_list="libc\.so\.* libarmmem.*\.so.* libdl\.so.* libglib-.*\.so.* @@ -54,8 +54,7 @@ library_in_exclude_list() { local libexname="$1" skip=1 set -f - for expat in $(echo "$LD_EXCLUDE_LIST") - do + for expat in $(echo "${ld_exclude_list}"); do if echo "$(basename $libexname)" | grep "${expat}" > /dev/null then skip=0 -- GitLab From 3d641767a84355cd49912b0c15ae0f53beb534c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:23:04 +0100 Subject: [PATCH 014/157] no linebreak between "if" and "then" (resp "for" and "do") --- pd-lib-builder/localdeps.linux.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 45d7d82..bb308a5 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -32,18 +32,15 @@ libpcre\.so.*" # Check dependencies cmdlist="awk grep ldd patchelf" -for cmd in $cmdlist -do - if ! which "${cmd}" > /dev/null - then +for cmd in $cmdlist; do + if ! which "${cmd}" > /dev/null; then echo "Could not find ${cmd}. Is it installed?" 1>&2 exit 1 fi done # Check if we can read from given file -if ! ldd "${binary_file}" > /dev/null 2>&1 -then +if ! ldd "${binary_file}" > /dev/null 2>&1; then echo "Can't read '${binary_file}'. Is it a binary file?" 1>&2 exit 1 fi @@ -55,8 +52,7 @@ library_in_exclude_list() { skip=1 set -f for expat in $(echo "${ld_exclude_list}"); do - if echo "$(basename $libexname)" | grep "${expat}" > /dev/null - then + if echo "$(basename $libexname)" | grep "${expat}" > /dev/null; then skip=0 break fi -- GitLab From ac157bf7d6a4803c87b3bbd8517f6de6e5a61045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:29:38 +0100 Subject: [PATCH 015/157] introduce "error" function --- pd-lib-builder/localdeps.linux.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index bb308a5..44bf03e 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -30,11 +30,15 @@ libstdc++\.so.* libgcc_s\.so.* libpcre\.so.*" +error() { + echo "$@" 1>&2 +} + # Check dependencies cmdlist="awk grep ldd patchelf" for cmd in $cmdlist; do if ! which "${cmd}" > /dev/null; then - echo "Could not find ${cmd}. Is it installed?" 1>&2 + error "Could not find ${cmd}. Is it installed?" exit 1 fi done @@ -71,7 +75,7 @@ make_local_copy_and_set_rpath() { continue fi if [ ! -e "${libpath}" ]; then - echo "Warning: could not make copy of '${libpath}'. Not found" 1>&2 + error "Warning: could not make copy of '${libpath}'. Not found" continue fi cp "$libpath" . -- GitLab From 1353cae60b490f2aa84515b337a3ee9a6f8ce9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:29:55 +0100 Subject: [PATCH 016/157] allow multiple binaries on cmdline --- pd-lib-builder/localdeps.linux.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 44bf03e..f35c650 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -43,12 +43,6 @@ for cmd in $cmdlist; do fi done -# Check if we can read from given file -if ! ldd "${binary_file}" > /dev/null 2>&1; then - echo "Can't read '${binary_file}'. Is it a binary file?" 1>&2 - exit 1 -fi - library_in_exclude_list() { # arg1: library name # returns 0 if arg1 is found in exclude list, otherwise 1 @@ -83,4 +77,12 @@ make_local_copy_and_set_rpath() { done } -make_local_copy_and_set_rpath $binary_file + +for binary_file in "$@"; do + # Check if we can read from given file + if ! ldd "${binary_file}" > /dev/null 2>&1; then + error "Skipping '${binary_file}'. Is it a binary file?" + continue + fi + make_local_copy_and_set_rpath $binary_file +done -- GitLab From 674da1428fdc1c41c4136e18e68845cea397d767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:30:06 +0100 Subject: [PATCH 017/157] simplified argcheck --- pd-lib-builder/localdeps.linux.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index f35c650..417639f 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -5,16 +5,15 @@ # each other. # # usage: $0 +usage() { + echo "Usage: ${0} [ ... ]" + echo " copies the local dependencies of all given binaries besides them" +} -case $1 in - "") - echo "Usage: ${0} " - exit - ;; - *) - binary_file=$1 - ;; -esac +if [ $# -lt 1 ]; then + usage + exit 1 +fi # List of libraries that we do not include into our packaging # becaue we think they will be installed on any system -- GitLab From 9b2a09a544aa4abf985f607feae43b3065d882f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:42:36 +0100 Subject: [PATCH 018/157] more "${var}" rather than $var --- pd-lib-builder/localdeps.linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 417639f..8ba4671 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -62,9 +62,9 @@ make_local_copy_and_set_rpath() { # make a local copy of all linked libraries of given binary # and set RUNPATH to $ORIGIN (exclude "standard" libraries) # arg1: binary to check - ldd $1 | grep ' => ' | while read _ _ libpath _; do + ldd "$1" | grep ' => ' | while read _ _ libpath _; do libname=$(basename "${libpath}") - if library_in_exclude_list "$libname"; then + if library_in_exclude_list "${libname}"; then continue fi if [ ! -e "${libpath}" ]; then -- GitLab From 5c1ca1b3c601b28d1208770ae412f0d4d8089266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:43:15 +0100 Subject: [PATCH 019/157] use location of inputfile as outputdir rather than the pwd --- pd-lib-builder/localdeps.linux.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 8ba4671..3be991e 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -62,8 +62,14 @@ make_local_copy_and_set_rpath() { # make a local copy of all linked libraries of given binary # and set RUNPATH to $ORIGIN (exclude "standard" libraries) # arg1: binary to check + local outdir + outdir=$(dirname "$1") + if [ ! -d "${outdir}" ]; then + outdir=. + fi ldd "$1" | grep ' => ' | while read _ _ libpath _; do libname=$(basename "${libpath}") + outfile="${outdir}/${libname}" if library_in_exclude_list "${libname}"; then continue fi @@ -71,8 +77,13 @@ make_local_copy_and_set_rpath() { error "Warning: could not make copy of '${libpath}'. Not found" continue fi - cp "$libpath" . - patchelf --set-rpath \$ORIGIN "${libname}" + if [ -e "${outfile}" ]; then + error "DEP: ${INSTALLDEPS_INDENT} ${libpath} SKIPPED" + else + error "DEP: ${INSTALLDEPS_INDENT} ${libpath} -> ${outdir}/" + cp "${libpath}" "${outfile}" + patchelf --set-rpath \$ORIGIN "${outfile}" + fi done } -- GitLab From f429bab49aa0c096fc74e34d5333cd23b9e04a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:43:49 +0100 Subject: [PATCH 020/157] set rpath of the external itself --- pd-lib-builder/localdeps.linux.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 3be991e..0c57586 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -85,6 +85,7 @@ make_local_copy_and_set_rpath() { patchelf --set-rpath \$ORIGIN "${outfile}" fi done + patchelf --set-rpath \$ORIGIN "${1}" } -- GitLab From 31e51b0a5e16284f32feff98296e9c276d2d6da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 15 Jan 2021 23:52:53 +0100 Subject: [PATCH 021/157] align not-found-error with other output --- pd-lib-builder/localdeps.linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh index 0c57586..7f7eab6 100755 --- a/pd-lib-builder/localdeps.linux.sh +++ b/pd-lib-builder/localdeps.linux.sh @@ -74,7 +74,7 @@ make_local_copy_and_set_rpath() { continue fi if [ ! -e "${libpath}" ]; then - error "Warning: could not make copy of '${libpath}'. Not found" + error "DEP: ${INSTALLDEPS_INDENT} WARNING: could not make copy of '${libpath}'. Not found" continue fi if [ -e "${outfile}" ]; then -- GitLab From 239556328fd92dee5da0ad8b6f685ab87add6c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sat, 16 Jan 2021 21:40:09 +0100 Subject: [PATCH 022/157] fixed typo --- pd-lib-builder/localdeps.win.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 2a23cb7..2068e19 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -11,7 +11,7 @@ # this uses an ugly hack to allow side-by-side installation of 32bit and 64bit # dependencies: # embedded dependencies are renamed from "libfoo.dll" to "libfoo.w32" resp. -# "libfoo.w64", and the files are modified (using 'sed') to reflext this +# "libfoo.w64", and the files are modified (using 'sed') to reflect this # renaming. # this is somewhat brittle and likely to break! -- GitLab From 73e020da8278c73316ffc6572cf15fda998c180e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 9 Feb 2021 16:29:34 +0100 Subject: [PATCH 023/157] allow overriding the output dir --- no-build/iem-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index 83cb0ef..983fa07 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -40,9 +40,10 @@ variables: before_script: - apt-get update && apt-get --no-install-recommends -y install deken git script: + - IEM_CI_PROJECT_DIR=${IEM_CI_PROJECT_DIR:-${IEM_CI_PROJECT_NAME}} - chmod -R go-w . # create source package - - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_NAME}/ HEAD | tar xf - + - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_DIR}/ HEAD | tar xf - - deken package --version="${CI_COMMIT_TAG#v}" "tmp/${IEM_CI_PROJECT_NAME}" # upload deken packages - test -z "${CI_COMMIT_TAG}" || test -z "${DEKEN_USERNAME}" || test -z "${DEKEN_PASSWORD}" || deken upload --no-source-error ./*.dek -- GitLab From dd31e603e5eef3a8a6ff17843f8e06bb88f08160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 9 Feb 2021 17:37:32 +0100 Subject: [PATCH 024/157] support for sub-libraries in the no-build section --- no-build/iem-ci.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index 983fa07..4ef96ee 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -25,7 +25,15 @@ variables: SRCDIR: . IEM_CI_TMPDIR: .git-ci/_build/ IEM_CI_PKGLIBDIR: "" + # the actual name of the library (as used in the deken package). + # you only need to set this if it is different from the repository name IEM_CI_PROJECT_NAME: ${CI_PROJECT_NAME} + # path component where to install the library to (e.g. "foo/bar") + # defaults to ${IEM_CI_PROJECT_NAME} + IEM_CI_PROJECT_INSTALLDIR: "" + # root path component to use for packaging (e.g. "foo") + # defaults to the first path component of IEM_CI_PROJECT_INSTALLDIR + IEM_CI_PROJECT_ROOTDIR: "" ####################################################################### ### create deken packages and (optionally) upload them; @@ -39,12 +47,15 @@ variables: DEKEN_ROOT: "yes" before_script: - apt-get update && apt-get --no-install-recommends -y install deken git + - IEM_CI_PROJECT_INSTALLDIR=${IEM_CI_PROJECT_INSTALLDIR:-${IEM_CI_PROJECT_NAME}} + - IEM_CI_PROJECT_INSTALLROOT=${IEM_CI_PROJECT_INSTALLROOT:-${IEM_CI_PROJECT_INSTALLDIR%%/*}} script: - - IEM_CI_PROJECT_DIR=${IEM_CI_PROJECT_DIR:-${IEM_CI_PROJECT_NAME}} - chmod -R go-w . -# create source package - - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_DIR}/ HEAD | tar xf - - - deken package --version="${CI_COMMIT_TAG#v}" "tmp/${IEM_CI_PROJECT_NAME}" +# create package + - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_INSTALLDIR}/ HEAD "${SRCDIR:-.}" | tar xf - + - deken package --version="${CI_COMMIT_TAG#v}" "tmp/${IEM_CI_PROJECT_INSTALLROOT}" +# fix package name for sub-libraries + - test "${IEM_CI_PROJECT_NAME}" != "${IEM_CI_PROJECT_ROOTDIR}" && for f in "${IEM_CI_PROJECT_ROOTDIR}"*.dek "${IEM_CI_PROJECT_ROOTDIR}"*.dek.*; do mv -b -v "${f}" "${IEM_CI_PROJECT_NAME}${p#${IEM_CI_PROJECT_ROOTDIR}}"; done # upload deken packages - test -z "${CI_COMMIT_TAG}" || test -z "${DEKEN_USERNAME}" || test -z "${DEKEN_PASSWORD}" || deken upload --no-source-error ./*.dek artifacts: -- GitLab From e17caae93e6fd186285a8ff255ec83199b530cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 9 Feb 2021 17:37:52 +0100 Subject: [PATCH 025/157] in-line documentation of no-build variables --- no-build/iem-ci.yml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index 4ef96ee..aba6767 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -3,21 +3,29 @@ ### ### this file contains the template definitions for various build/deploy jobs ### -### the jobs found here can be used for any external with a build system like: -### - 'make' builds the external -### - 'make install' installs the external to be used by Pd -### the following make variables must be honoured: -### - extension: filename extension for externals -### - DESTDIR: base directory for 'make install' -### - pkglibdir: directory to put externals into (relative to DESTDIR) -### some more expectations: -### - 'make install' will put all relevant files into ${DESTDIR}/${pkglibdir}/${IEM_CI_PROJECT_NAME} -### with: -### - IEM_CI_PROJECT_NAME: the name of the project (defaults to ${CI_PROJECT_NAME}, but should match -### the Pd-library name (e.g. pd-lib-builder's "lib.name") +### the jobs found here can be used for any library that does not require building +### (abstractions only libraries) +### the only assumption is that the entire content of the repository +### is to be installed as path/to/pdlibs// +### +### normally this should work out of the box +### +### tweaking: +### +### if you want to package only a subdirectory of the repository +### (e.g. the 'foo' directory), specify this directory in the SRCDIR +### +### if the library name differs from the repository name +### (e.g. the repository is called 'pd-foo' but the library is called 'foo') +### you need to set the IEM_CI_PROJECT_NAME variable to the name of the library +### +### if you are packaging a sub-library +### (e.g. the repository 'foo-bar' should install as 'foo/bar') +### you need to set IEM_CI_PROJECT_INSTALLDIR to the install path component. +### the package will be based on the first element of the path ('foo'), +### but this can be overridden with IEM_CI_PROJECT_ROOTDIR. +### the deken package will be renamed to match the 'library name' ### -### one well known build-system that can be used straight away is -### "pd-lib-builder" -> https://github.com/pure-data/pd-lib-builder/ ####################################################################### variables: -- GitLab From f237d83831856e2ed661504212df656a0d5ee136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 9 Feb 2021 18:23:11 +0100 Subject: [PATCH 026/157] document sub-libraries (for no-build) --- no-build/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/no-build/README.md b/no-build/README.md index e686954..8f7a1ea 100644 --- a/no-build/README.md +++ b/no-build/README.md @@ -101,6 +101,29 @@ variables: IEM_CI_PROJECT_NAME: superlib ~~~ + +### packaging sub-libraries +Sometimes you need to package sub-libraries that live in the namespace (aka: directory) of another library. +Consider a library `foo` that is extended by another library that installs to `foo/bar`. +To package such a library, you need to specify the target (installation) directory with the `IEM_CI_PROJECT_INSTALLDIR` variable. + +~~~yml +--- +include: + - https://git.iem.at/pd/iem-ci/raw/master/no-build/gitlab-iem.yml + +variables: + IEM_CI_PROJECT_INSTALLDIR: foo/bar +~~~ + +The package will be based on the first path component of the `IEM_CI_PROJECT_INSTALLDIR` +(in the example this is the `foo` directory). +If needed, the base directory can be changed with the `IEM_CI_PROJECT_ROOTDIR` variable. + +(The so-generated deken-packages will use `IEM_CI_PROJECT_ROOTDIR` in their package name. +The build-process automatically renames them to the library name (`IEM_CI_PROJECT_NAME`)). + + # who can use it The IEM-CI infrastructure can (currently) only be used for projects hosted on https://git.iem.at/ -- GitLab From 213b7a9b90c858204f02ac99978547f4db256909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 9 Feb 2021 18:32:34 +0100 Subject: [PATCH 027/157] fixed typo in variable-name when renaming dek-packages --- no-build/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index aba6767..fba1396 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -63,7 +63,7 @@ variables: - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_INSTALLDIR}/ HEAD "${SRCDIR:-.}" | tar xf - - deken package --version="${CI_COMMIT_TAG#v}" "tmp/${IEM_CI_PROJECT_INSTALLROOT}" # fix package name for sub-libraries - - test "${IEM_CI_PROJECT_NAME}" != "${IEM_CI_PROJECT_ROOTDIR}" && for f in "${IEM_CI_PROJECT_ROOTDIR}"*.dek "${IEM_CI_PROJECT_ROOTDIR}"*.dek.*; do mv -b -v "${f}" "${IEM_CI_PROJECT_NAME}${p#${IEM_CI_PROJECT_ROOTDIR}}"; done + - test "${IEM_CI_PROJECT_NAME}" != "${IEM_CI_PROJECT_ROOTDIR}" && for f in "${IEM_CI_PROJECT_ROOTDIR}"*.dek "${IEM_CI_PROJECT_ROOTDIR}"*.dek.*; do mv -b -v "${f}" "${IEM_CI_PROJECT_NAME}${f#${IEM_CI_PROJECT_ROOTDIR}}"; done # upload deken packages - test -z "${CI_COMMIT_TAG}" || test -z "${DEKEN_USERNAME}" || test -z "${DEKEN_PASSWORD}" || deken upload --no-source-error ./*.dek artifacts: -- GitLab From 4ec41f421f2cf570c7b6e133859d6f33a2e8bb46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 9 Feb 2021 18:39:20 +0100 Subject: [PATCH 028/157] fixed variable name confusion: it's IEM_CI_PROJECT_ROOTDIR --- no-build/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index fba1396..c12a498 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -56,12 +56,12 @@ variables: before_script: - apt-get update && apt-get --no-install-recommends -y install deken git - IEM_CI_PROJECT_INSTALLDIR=${IEM_CI_PROJECT_INSTALLDIR:-${IEM_CI_PROJECT_NAME}} - - IEM_CI_PROJECT_INSTALLROOT=${IEM_CI_PROJECT_INSTALLROOT:-${IEM_CI_PROJECT_INSTALLDIR%%/*}} + - IEM_CI_PROJECT_ROOTDIR=${IEM_CI_PROJECT_ROOTDIR:-${IEM_CI_PROJECT_INSTALLDIR%%/*}} script: - chmod -R go-w . # create package - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_INSTALLDIR}/ HEAD "${SRCDIR:-.}" | tar xf - - - deken package --version="${CI_COMMIT_TAG#v}" "tmp/${IEM_CI_PROJECT_INSTALLROOT}" + - deken package --version="${CI_COMMIT_TAG#v}" "tmp/${IEM_CI_PROJECT_ROOTDIR}" # fix package name for sub-libraries - test "${IEM_CI_PROJECT_NAME}" != "${IEM_CI_PROJECT_ROOTDIR}" && for f in "${IEM_CI_PROJECT_ROOTDIR}"*.dek "${IEM_CI_PROJECT_ROOTDIR}"*.dek.*; do mv -b -v "${f}" "${IEM_CI_PROJECT_NAME}${f#${IEM_CI_PROJECT_ROOTDIR}}"; done # upload deken packages -- GitLab From a8b7a628ae018fd305ff3f95789d97ac088c0380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 12 Mar 2021 13:02:32 +0100 Subject: [PATCH 029/157] drop _snapshot versions of build-jobs (make all build-jobs 'snapshot') for some time now, gitlab will keep the latest artifacts around (even if expired). and the final 'deken' job, won't expire anyhow. --- pd-lib-builder/pipeline-jobs.yml | 46 +++++--------------------------- 1 file changed, 7 insertions(+), 39 deletions(-) diff --git a/pd-lib-builder/pipeline-jobs.yml b/pd-lib-builder/pipeline-jobs.yml index 79b2584..2e37a9e 100644 --- a/pd-lib-builder/pipeline-jobs.yml +++ b/pd-lib-builder/pipeline-jobs.yml @@ -19,64 +19,32 @@ ### the actual jobs: (linux,macos,windows)*(release,snapshot) ### the job-templates can be found in 'iem-ci.yml' (which must be included beforehand) -### release jobs +### build jobs Linux: - extends: - - .Linux - - .release -Linuxi386: - extends: - - .Linuxi386 - - .release -LinuxARMhf: - extends: - - .LinuxARMhf - - .release -LinuxARM64: - extends: - - .LinuxARM64 - - .release -Darwin: - extends: - - .Darwin - - .release -w32: - extends: - - .w32 - - .release -w64: - extends: - - .w64 - - .release - - - -### snapshot jobs -Linux_snapshot: extends: - .Linux - .snapshot -Linuxi386_snapshot: +Linuxi386: extends: - .Linuxi386 - .snapshot -LinuxARMhf_snapshot: +LinuxARMhf: extends: - .LinuxARMhf - .snapshot -LinuxARM64_snapshot: +LinuxARM64: extends: - .LinuxARM64 - .snapshot -Darwin_snapshot: +Darwin: extends: - .Darwin - .snapshot -w32_snapshot: +w32: extends: - .w32 - .snapshot -w64_snapshot: +w64: extends: - .w64 - .snapshot -- GitLab From e3001f18b9c5124f61625b0806d2acfeaa9d87f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 12 Mar 2021 13:16:02 +0100 Subject: [PATCH 030/157] separate ".artifacts" job for a common artifact definition --- pd-lib-builder/iem-ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 282c4bf..ad29470 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -49,12 +49,7 @@ variables: ####################################################################### ### configuration templates (to be used for snapshot and release builds) -.build:script: - stage: build - script: - - *script_make - - *script_make_check - - *script_make_install +.artifacts: artifacts: name: ${CI_PROJECT_NAME}_${CI_COMMIT_REF_NAME}_${CI_JOB_NAME%_*} paths: @@ -70,6 +65,15 @@ variables: only: - tags + +.build:script: + extends: .artifacts + stage: build + script: + - *script_make + - *script_make_check + - *script_make_install + .build:linux: &build_linux extends: .build:script image: registry.git.iem.at/devtools/docker/debiancross:amd64 -- GitLab From 78f0e3107e22b5a1a400c8385f2d1a9cb412ede8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 12 Mar 2021 13:16:16 +0100 Subject: [PATCH 031/157] codesigning job for macOS --- pd-lib-builder/iem-ci.yml | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index ad29470..90ac3dd 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -182,6 +182,46 @@ variables: <<: *build_w64 +####################################################################### +### sign code +### if you want to enable code-signing, you must provide +### a signing certificate (bundled with the matching key in a PKCS#12 file) +### in the _CERTIFICATE_PFX file-variable, and the matching passphrase +### in the _CERTIFICATE_PWD variable. + +# code signing jobs +.sign:macos: + extends: .artifacts + stage: sign + tags: + - osx + variables: + keychainpass: $CI_BUILD_TOKEN + CODESIGNFLAGS: --timestamp --string --force + only: + variables: + - $MACOS_CERTIFICATE_PFX + - $MACOS_CERTIFICATE_PWD + before_script: + - curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer > /tmp/AppleWWDRCA.cer + - curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer > /tmp/AppleWWDRCAG3.cer + - cat "${MACOS_CERTIFICATE_PFX}" | base64 -D > /tmp/sign.pfx + - md5 /tmp/sign.pfx || true + - security create-keychain -p "${keychainpass}" build.keychain + - security default-keychain -s build.keychain + - security unlock-keychain -p "${keychainpass}" build.keychain + - security import /tmp/AppleWWDRCA.cer -k build.keychain + - security import /tmp/AppleWWDRCAG3.cer -k build.keychain + - security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign + - security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain + - security find-identity -v + - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') + script: + - find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + + after_script: + - find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" + + ####################################################################### ### create deken packages and (optionally) upload them; ### if you want to automatically upload a package, you need to -- GitLab From 2482493043c747a461ca2596680748d1da44d368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 12 Mar 2021 13:20:11 +0100 Subject: [PATCH 032/157] define stages (with 'sign') --- pd-lib-builder/pipeline-jobs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pd-lib-builder/pipeline-jobs.yml b/pd-lib-builder/pipeline-jobs.yml index 2e37a9e..b781d62 100644 --- a/pd-lib-builder/pipeline-jobs.yml +++ b/pd-lib-builder/pipeline-jobs.yml @@ -19,6 +19,12 @@ ### the actual jobs: (linux,macos,windows)*(release,snapshot) ### the job-templates can be found in 'iem-ci.yml' (which must be included beforehand) +stages: + - build + - test + - sign + - deploy + ### build jobs Linux: extends: -- GitLab From c5733708e96e5449c5f9586a736710589b7472e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 12 Mar 2021 13:20:19 +0100 Subject: [PATCH 033/157] enable Darwin_sign stage --- pd-lib-builder/pipeline-jobs.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pd-lib-builder/pipeline-jobs.yml b/pd-lib-builder/pipeline-jobs.yml index b781d62..39cb466 100644 --- a/pd-lib-builder/pipeline-jobs.yml +++ b/pd-lib-builder/pipeline-jobs.yml @@ -55,6 +55,13 @@ w64: - .w64 - .snapshot +Darwin_sign: + extends: + - .sign:macos + stage: sign + dependencies: + - Darwin + ####################################################################### ### create deken packages and (optionally) upload them; ### if you want to automatically upload a package, you need to -- GitLab From dea8dc46890f702b3b88a32f345d89dac9709116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 12 Mar 2021 13:39:31 +0100 Subject: [PATCH 034/157] fixed default flag when codesigning on macOS --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 90ac3dd..b49b911 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -197,7 +197,7 @@ variables: - osx variables: keychainpass: $CI_BUILD_TOKEN - CODESIGNFLAGS: --timestamp --string --force + CODESIGNFLAGS: --timestamp --strict --force only: variables: - $MACOS_CERTIFICATE_PFX -- GitLab From cce22ad9cf852b008ee2cb4ea7898ae82d749132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 12 Mar 2021 21:16:15 +0100 Subject: [PATCH 035/157] allow the signing stage to fail --- pd-lib-builder/pipeline-jobs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/pipeline-jobs.yml b/pd-lib-builder/pipeline-jobs.yml index 39cb466..cc4c61a 100644 --- a/pd-lib-builder/pipeline-jobs.yml +++ b/pd-lib-builder/pipeline-jobs.yml @@ -61,6 +61,7 @@ Darwin_sign: stage: sign dependencies: - Darwin + allow_failure: true ####################################################################### ### create deken packages and (optionally) upload them; -- GitLab From d594d64dc3f84f05fd9c3b9f95776ee500c957e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 22 Mar 2021 15:45:05 +0100 Subject: [PATCH 036/157] no-build: use deken docker-image for building the deken package --- no-build/iem-ci.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index c12a498..9cb4b2e 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -50,23 +50,21 @@ variables: ### (https://git.iem.at/help/ci/variables/README#variables) .deken: stage: deploy - image: debian:buster - variables: - DEKEN_ROOT: "yes" + image: registry.git.iem.at/pd/deken:latest before_script: - - apt-get update && apt-get --no-install-recommends -y install deken git + - apt-get update && apt-get --no-install-recommends -y install git - IEM_CI_PROJECT_INSTALLDIR=${IEM_CI_PROJECT_INSTALLDIR:-${IEM_CI_PROJECT_NAME}} - IEM_CI_PROJECT_ROOTDIR=${IEM_CI_PROJECT_ROOTDIR:-${IEM_CI_PROJECT_INSTALLDIR%%/*}} script: - chmod -R go-w . # create package + - rm -rf tmp/ - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_INSTALLDIR}/ HEAD "${SRCDIR:-.}" | tar xf - - - deken package --version="${CI_COMMIT_TAG#v}" "tmp/${IEM_CI_PROJECT_ROOTDIR}" -# fix package name for sub-libraries - - test "${IEM_CI_PROJECT_NAME}" != "${IEM_CI_PROJECT_ROOTDIR}" && for f in "${IEM_CI_PROJECT_ROOTDIR}"*.dek "${IEM_CI_PROJECT_ROOTDIR}"*.dek.*; do mv -b -v "${f}" "${IEM_CI_PROJECT_NAME}${f#${IEM_CI_PROJECT_ROOTDIR}}"; done + - deken package --version="${CI_COMMIT_TAG#v}" --name "${IEM_CI_PROJECT_NAME}" "tmp/${IEM_CI_PROJECT_ROOTDIR}" # upload deken packages - test -z "${CI_COMMIT_TAG}" || test -z "${DEKEN_USERNAME}" || test -z "${DEKEN_PASSWORD}" || deken upload --no-source-error ./*.dek artifacts: name: deken-package paths: - ./*.dek + - ./*.dek.* -- GitLab From 6ab6b4dcdadc1050f4c5072e048881f9a1a3c595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 08:44:16 +0100 Subject: [PATCH 037/157] integrate macOS code-signing into the .build:macos job until https://gitlab.com/gitlab-org/gitlab/-/issues/324412 is resolved... --- pd-lib-builder/iem-ci.yml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b49b911..261a5e5 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -108,6 +108,9 @@ variables: extends: .build:script tags: - osx + variables: + keychainpass: $CI_BUILD_TOKEN + CODESIGNFLAGS: --timestamp --strict --force before_script: - date - if [ -e .git-ci/requirements.brew ]; then brew bundle --no-upgrade --file=.git-ci/requirements.brew; fi @@ -116,9 +119,29 @@ variables: - tar xvf Pd.tgz -C /Applications/ - rm -f Pd.tgz - export PD=$(find /Applications/Pd*.app/Contents/Resources/bin/ type f -name pd -print -quit) + # setup for code-signing (LATER move this into a separate stage; see 'SIGN CODE' below) + - | + test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || ( + curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer > /tmp/AppleWWDRCA.cer + curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer > /tmp/AppleWWDRCAG3.cer + cat "${MACOS_CERTIFICATE_PFX}" | base64 -D > /tmp/sign.pfx + security create-keychain -p "${keychainpass}" build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p "${keychainpass}" build.keychain + security import /tmp/AppleWWDRCA.cer -k build.keychain + security import /tmp/AppleWWDRCAG3.cer -k build.keychain + security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign + security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain + md5 /tmp/sign.pfx || true + security find-identity -v + ) || true + - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') || true after_script: - *script_fetch_localdeps - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi + # sign the code and verify it (LATER move this into a separate stage; see 'SIGN CODE' below) + - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + || true + - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" || true .build:w32: &build_w32 extends: .build:script @@ -183,12 +206,19 @@ variables: ####################################################################### -### sign code +### SIGN CODE ### if you want to enable code-signing, you must provide ### a signing certificate (bundled with the matching key in a PKCS#12 file) ### in the _CERTIFICATE_PFX file-variable, and the matching passphrase ### in the _CERTIFICATE_PWD variable. +## there's a bug in gitlab-ci that downloads artifacts in the reverse stage order +## - https://gitlab.com/gitlab-org/gitlab/-/issues/324412 +## - https://gitlab.com/gitlab-org/gitlab-runner/-/issues/1568 +## the result of this issue is that all the signed binaries will be overwritten +## by their unsigned versions. +## until this is fixed, we put the code-signing in the build-jobs + # code signing jobs .sign:macos: extends: .artifacts -- GitLab From dc3cd124e2a4f461ccd22e3e8d6924676cb84675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 08:57:14 +0100 Subject: [PATCH 038/157] retry failed .build:macos jobs if there is a system-problem --- pd-lib-builder/iem-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 261a5e5..9e4ccd8 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -108,6 +108,10 @@ variables: extends: .build:script tags: - osx + retry: + max: 1 + when: + - runner_system_failure variables: keychainpass: $CI_BUILD_TOKEN CODESIGNFLAGS: --timestamp --strict --force -- GitLab From a6fe3b10160080da494c4c488e1f771264cabb57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 09:28:05 +0100 Subject: [PATCH 039/157] move setting of sign_id into after_script --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 9e4ccd8..a56befd 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -139,11 +139,11 @@ variables: md5 /tmp/sign.pfx || true security find-identity -v ) || true - - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') || true after_script: - *script_fetch_localdeps - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi # sign the code and verify it (LATER move this into a separate stage; see 'SIGN CODE' below) + - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') || true - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + || true - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" || true -- GitLab From d85a4c4b310a1835295e9787b05fee5dc79c840f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 09:28:14 +0100 Subject: [PATCH 040/157] some debugging printout --- pd-lib-builder/iem-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index a56befd..9c9fd1f 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -144,6 +144,9 @@ variables: - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi # sign the code and verify it (LATER move this into a separate stage; see 'SIGN CODE' below) - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') || true + - test -z "${MACOS_CERTIFICATE_PFX}" || echo "got certificate" + - test -z "${MACOS_CERTIFICATE_PWD}" || echo "got certificate password" + - test -z "${sign_id}" || echo "signing with ${sign_id}" - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + || true - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" || true -- GitLab From 986454fe2614fd511c13834134d0b410a68e94c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 09:28:35 +0100 Subject: [PATCH 041/157] disable Darwin_sign job (this is now handled by Darwin_build) --- pd-lib-builder/pipeline-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/pipeline-jobs.yml b/pd-lib-builder/pipeline-jobs.yml index cc4c61a..1cc7342 100644 --- a/pd-lib-builder/pipeline-jobs.yml +++ b/pd-lib-builder/pipeline-jobs.yml @@ -55,7 +55,7 @@ w64: - .w64 - .snapshot -Darwin_sign: +.Darwin_sign: extends: - .sign:macos stage: sign -- GitLab From fec238ecb2780111070bba56062af60b282dcd6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 09:48:51 +0100 Subject: [PATCH 042/157] use deken:latest Dockerimage for the package job also add the supplementary dek-files to the artifacts --- pd-lib-builder/iem-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 9c9fd1f..f1aefc9 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -266,11 +266,9 @@ variables: ### (https://git.iem.at/help/ci/variables/README#variables) .deken: stage: deploy - image: debian:buster - variables: - DEKEN_ROOT: "yes" + image: registry.git.iem.at/pd/deken:latest before_script: - - apt-get update && apt-get --no-install-recommends -y install deken git + - apt-get update && apt-get --no-install-recommends -y install git script: - chmod -R go-w . # create source package @@ -285,3 +283,4 @@ variables: name: deken-package paths: - ./*.dek + - ./*.dek.* -- GitLab From 997ec7109cb232fe613bbfaf60fd2f389e2f715a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 11:20:42 +0100 Subject: [PATCH 043/157] attempt to build fat-binaries if MACOS_AUTO_FAT is set to "true" simply by setting the extension to "d_fat" (pd-lib-builder does the rest) --- pd-lib-builder/iem-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index f1aefc9..35b15d7 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -139,6 +139,7 @@ variables: md5 /tmp/sign.pfx || true security find-identity -v ) || true + - test -n "${pd_extension}" || case "${MACOS_AUTO_FAT}" in yes|1|true) pd_extension=d_fat ;; esac after_script: - *script_fetch_localdeps - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi -- GitLab From a0180acd7d48a239f9a73c8cf948b9a96b7b8dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 23 Mar 2021 11:23:05 +0100 Subject: [PATCH 044/157] less verbose extraction of Pd.tgz --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 35b15d7..e835007 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -120,7 +120,7 @@ variables: - if [ -e .git-ci/requirements.brew ]; then brew bundle --no-upgrade --file=.git-ci/requirements.brew; fi - wget -q -O Pd.tgz http://msp.ucsd.edu/Software/pd-${PDVERSION}.mac.tar.gz - rm -rf /Applications/Pd*.app/ - - tar xvf Pd.tgz -C /Applications/ + - tar xf Pd.tgz -C /Applications/ - rm -f Pd.tgz - export PD=$(find /Applications/Pd*.app/Contents/Resources/bin/ type f -name pd -print -quit) # setup for code-signing (LATER move this into a separate stage; see 'SIGN CODE' below) -- GitLab From ffd775e12c6568e9f68955b5d19919cbc869b6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 15:20:33 +0100 Subject: [PATCH 045/157] renamed the variable for universal builds to "IEM_CI_MACOS_BUILDFAT" --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index e835007..4fddf79 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -139,7 +139,7 @@ variables: md5 /tmp/sign.pfx || true security find-identity -v ) || true - - test -n "${pd_extension}" || case "${MACOS_AUTO_FAT}" in yes|1|true) pd_extension=d_fat ;; esac + - test -n "${pd_extension}" || case "${IEM_CI_MACOS_BUILDFAT}" in yes|1|true) pd_extension=d_fat ;; esac after_script: - *script_fetch_localdeps - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi -- GitLab From bd526d0679a1377c19b0c609e9de2e2e118d9489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 15:21:59 +0100 Subject: [PATCH 046/157] document how to build fat libraries --- pd-lib-builder/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pd-lib-builder/README.md b/pd-lib-builder/README.md index 57d5653..f982987 100644 --- a/pd-lib-builder/README.md +++ b/pd-lib-builder/README.md @@ -204,6 +204,14 @@ variables: IEM_CI_PROJECT_NAME: superlib ~~~ +### fat libraries + +On macOS, you have the option to build universal binaries (that contain multiple architectures). +This will only work if all the [dependencies](#build-dependencies) are also universal binaries. +This feature relies on the build-system doing the "right thing" if the Pd library extension is `d_fat` (pd-lib-builder does this). + +To enable this feature, set the `IEM_CI_MACOS_BUILDFAT` variable to `1`. +To disable this feature, set the variable to `0`. ### Build Dependencies -- GitLab From 31f02f38f5557874f6daa4c31793729fa105a10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 16:35:27 +0100 Subject: [PATCH 047/157] move signing logic into a YAML-anchor, so we can re-use it better. --- pd-lib-builder/iem-ci.yml | 43 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 4fddf79..fa8e1b6 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -47,6 +47,27 @@ variables: - rm -rf "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" - mv "${IEM_CI_TMPDIR}/${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" . +.script:codesign_macos: &script_codesign_macos + - | + if test -n "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then + keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; + echo "code signing" && + curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer && + curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer && + cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx && + security create-keychain -p "${keychainpass}" build.keychain && security default-keychain -s build.keychain && security unlock-keychain -p "${keychainpass}" build.keychain && + security import /tmp/AppleWWDRCA.cer -k build.keychain && security import /tmp/AppleWWDRCAG3.cer -k build.keychain && + security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign && + security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain && + (md5 /tmp/sign.pfx || true) && security find-identity -v && + sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "signing with ${sign_id}" && + if test -n "${sign_id}"; then + find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + && + (find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" + fi + fi || codesign_error=$? + + ####################################################################### ### configuration templates (to be used for snapshot and release builds) .artifacts: @@ -124,32 +145,12 @@ variables: - rm -f Pd.tgz - export PD=$(find /Applications/Pd*.app/Contents/Resources/bin/ type f -name pd -print -quit) # setup for code-signing (LATER move this into a separate stage; see 'SIGN CODE' below) - - | - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || ( - curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer > /tmp/AppleWWDRCA.cer - curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer > /tmp/AppleWWDRCAG3.cer - cat "${MACOS_CERTIFICATE_PFX}" | base64 -D > /tmp/sign.pfx - security create-keychain -p "${keychainpass}" build.keychain - security default-keychain -s build.keychain - security unlock-keychain -p "${keychainpass}" build.keychain - security import /tmp/AppleWWDRCA.cer -k build.keychain - security import /tmp/AppleWWDRCAG3.cer -k build.keychain - security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign - security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain - md5 /tmp/sign.pfx || true - security find-identity -v - ) || true - test -n "${pd_extension}" || case "${IEM_CI_MACOS_BUILDFAT}" in yes|1|true) pd_extension=d_fat ;; esac after_script: - *script_fetch_localdeps - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi # sign the code and verify it (LATER move this into a separate stage; see 'SIGN CODE' below) - - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') || true - - test -z "${MACOS_CERTIFICATE_PFX}" || echo "got certificate" - - test -z "${MACOS_CERTIFICATE_PWD}" || echo "got certificate password" - - test -z "${sign_id}" || echo "signing with ${sign_id}" - - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + || true - - test -z "${MACOS_CERTIFICATE_PFX}" || test -z "${MACOS_CERTIFICATE_PWD}" || test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" || true + - *script_macsign .build:w32: &build_w32 extends: .build:script -- GitLab From 8e19286890d2fdd206b79095198b7c594e842a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 16:44:38 +0100 Subject: [PATCH 048/157] fixed name of snippet --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index fa8e1b6..45ee23f 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -150,7 +150,7 @@ variables: - *script_fetch_localdeps - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi # sign the code and verify it (LATER move this into a separate stage; see 'SIGN CODE' below) - - *script_macsign + - *script_codesign_macos .build:w32: &build_w32 extends: .build:script -- GitLab From 820c43ffe6213ab6e7a94841a9b6cb311e15aed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 16:44:53 +0100 Subject: [PATCH 049/157] unset ${codesign_error} if there's no error --- pd-lib-builder/iem-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 45ee23f..c324d52 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -66,6 +66,7 @@ variables: (find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" fi fi || codesign_error=$? + - if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; fi ####################################################################### -- GitLab From b5c32d1dc7e600b821fe0841257d2c01b38e6736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 16:45:12 +0100 Subject: [PATCH 050/157] use *script_codesign_macos for standalone signing job --- pd-lib-builder/iem-ci.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index c324d52..5a81620 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -242,22 +242,9 @@ variables: variables: - $MACOS_CERTIFICATE_PFX - $MACOS_CERTIFICATE_PWD - before_script: - - curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer > /tmp/AppleWWDRCA.cer - - curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer > /tmp/AppleWWDRCAG3.cer - - cat "${MACOS_CERTIFICATE_PFX}" | base64 -D > /tmp/sign.pfx - - md5 /tmp/sign.pfx || true - - security create-keychain -p "${keychainpass}" build.keychain - - security default-keychain -s build.keychain - - security unlock-keychain -p "${keychainpass}" build.keychain - - security import /tmp/AppleWWDRCA.cer -k build.keychain - - security import /tmp/AppleWWDRCAG3.cer -k build.keychain - - security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign - - security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain - - security find-identity -v - - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') script: - - find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + + - *script_codesign_macos + - test "x${codesign_error}" != "x" && exit ${codesign_error} after_script: - find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" -- GitLab From e10fdab29e95c97e4b6f77645236ad1df5ee4ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 17:02:13 +0100 Subject: [PATCH 051/157] fixed syntax errors --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 5a81620..80492e4 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -63,8 +63,8 @@ variables: sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "signing with ${sign_id}" && if test -n "${sign_id}"; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + && - (find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" - fi + find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" + fi; fi || codesign_error=$? - if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; fi -- GitLab From 987be66eeb27792b7fd3dcba15dec98db7e1305f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 17:02:27 +0100 Subject: [PATCH 052/157] check if certificate file exists --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 80492e4..55105df 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -49,7 +49,7 @@ variables: .script:codesign_macos: &script_codesign_macos - | - if test -n "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then + if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; echo "code signing" && curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer && -- GitLab From f6cb6a51c64269baf5c9e0de24cdbc87af9fe375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 17:14:17 +0100 Subject: [PATCH 053/157] verbose printout during code-signing multi-line commands get collapsed, so we don't know where something goes wrong... --- pd-lib-builder/iem-ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 55105df..2a83710 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -52,15 +52,19 @@ variables: if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; echo "code signing" && + echo "fetch certificates" && curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer && curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer && - cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx && + cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx && (md5 /tmp/sign.pfx || true) && + echo "create and unlock keychain" && security create-keychain -p "${keychainpass}" build.keychain && security default-keychain -s build.keychain && security unlock-keychain -p "${keychainpass}" build.keychain && + echo "import certificates into keychain" && security import /tmp/AppleWWDRCA.cer -k build.keychain && security import /tmp/AppleWWDRCAG3.cer -k build.keychain && security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign && - security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain && - (md5 /tmp/sign.pfx || true) && security find-identity -v && - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "signing with ${sign_id}" && + echo "enable certificate for codesigning" && + security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain >/dev/null && + security find-identity -v && + sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "sign binaries with ${sign_id}" && if test -n "${sign_id}"; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + && find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" -- GitLab From 59118d51c8a77e84620d69509b30a787dace49ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 17:24:33 +0100 Subject: [PATCH 054/157] make notifications green... --- pd-lib-builder/iem-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 2a83710..9efc937 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -50,21 +50,21 @@ variables: .script:codesign_macos: &script_codesign_macos - | if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then - keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; - echo "code signing" && - echo "fetch certificates" && + keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; TXT_NOTICE="\e[32m"; TXT_CLEAR="\e[0m" + echo "${TXT_NOTICE}code signing${TXT_CLEAR} && + echo "${TXT_NOTICE}fetch certificates${TXT_CLEAR} && curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer && curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer && cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx && (md5 /tmp/sign.pfx || true) && - echo "create and unlock keychain" && + echo "${TXT_NOTICE}create and unlock keychain${TXT_CLEAR} && security create-keychain -p "${keychainpass}" build.keychain && security default-keychain -s build.keychain && security unlock-keychain -p "${keychainpass}" build.keychain && - echo "import certificates into keychain" && + echo "${TXT_NOTICE}import certificates into keychain${TXT_CLEAR} && security import /tmp/AppleWWDRCA.cer -k build.keychain && security import /tmp/AppleWWDRCAG3.cer -k build.keychain && security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign && - echo "enable certificate for codesigning" && + echo "${TXT_NOTICE}enable certificate for codesigning${TXT_CLEAR} && security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain >/dev/null && security find-identity -v && - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "sign binaries with ${sign_id}" && + sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "${TXT_NOTICE}sign binaries with ${sign_id}${TXT_CLEAR} && if test -n "${sign_id}"; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + && find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" -- GitLab From d7a3885e35e6f21badf788db9a7af4dea55e376a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 17:35:05 +0100 Subject: [PATCH 055/157] fixed formatted printout --- pd-lib-builder/iem-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 9efc937..a6701d1 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -51,26 +51,26 @@ variables: - | if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; TXT_NOTICE="\e[32m"; TXT_CLEAR="\e[0m" - echo "${TXT_NOTICE}code signing${TXT_CLEAR} && - echo "${TXT_NOTICE}fetch certificates${TXT_CLEAR} && + echo -e "${TXT_NOTICE}code signing${TXT_CLEAR}" && + echo -e "${TXT_NOTICE}fetch certificates${TXT_CLEAR}" && curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer && curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer && cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx && (md5 /tmp/sign.pfx || true) && - echo "${TXT_NOTICE}create and unlock keychain${TXT_CLEAR} && + echo -e "${TXT_NOTICE}create and unlock keychain${TXT_CLEAR}" && security create-keychain -p "${keychainpass}" build.keychain && security default-keychain -s build.keychain && security unlock-keychain -p "${keychainpass}" build.keychain && - echo "${TXT_NOTICE}import certificates into keychain${TXT_CLEAR} && + echo -e "${TXT_NOTICE}import certificates into keychain${TXT_CLEAR}" && security import /tmp/AppleWWDRCA.cer -k build.keychain && security import /tmp/AppleWWDRCAG3.cer -k build.keychain && security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign && - echo "${TXT_NOTICE}enable certificate for codesigning${TXT_CLEAR} && + echo -e "${TXT_NOTICE}enable certificate for codesigning${TXT_CLEAR}" && security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain >/dev/null && security find-identity -v && - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "${TXT_NOTICE}sign binaries with ${sign_id}${TXT_CLEAR} && + sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo -e "${TXT_NOTICE}sign binaries with ${sign_id}${TXT_CLEAR}" && if test -n "${sign_id}"; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + && find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" fi; fi || codesign_error=$? - - if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; fi + if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; else echo -e "${TXT_NOTICE}code signing returned with ${codesign_error}${TXT_CLEAR}"; fi ####################################################################### -- GitLab From bdb85a1e3b2f62687fe666994c0ad5f8c1b12aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 17:44:55 +0100 Subject: [PATCH 056/157] drop the code-formatting, it doesn't work --- pd-lib-builder/iem-ci.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index a6701d1..23bd052 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -50,27 +50,29 @@ variables: .script:codesign_macos: &script_codesign_macos - | if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then - keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; TXT_NOTICE="\e[32m"; TXT_CLEAR="\e[0m" - echo -e "${TXT_NOTICE}code signing${TXT_CLEAR}" && - echo -e "${TXT_NOTICE}fetch certificates${TXT_CLEAR}" && + keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; TXT_NOTICE="INFO..."; TXT_CLEAR=""; + echo "${TXT_NOTICE}code signing${TXT_CLEAR}" && + echo "${TXT_NOTICE}fetch certificates${TXT_CLEAR}" && curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer && curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer && cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx && (md5 /tmp/sign.pfx || true) && - echo -e "${TXT_NOTICE}create and unlock keychain${TXT_CLEAR}" && + echo "${TXT_NOTICE}create and unlock keychain${TXT_CLEAR}" && security create-keychain -p "${keychainpass}" build.keychain && security default-keychain -s build.keychain && security unlock-keychain -p "${keychainpass}" build.keychain && - echo -e "${TXT_NOTICE}import certificates into keychain${TXT_CLEAR}" && + echo "${TXT_NOTICE}import certificates into keychain${TXT_CLEAR}" && security import /tmp/AppleWWDRCA.cer -k build.keychain && security import /tmp/AppleWWDRCAG3.cer -k build.keychain && security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign && - echo -e "${TXT_NOTICE}enable certificate for codesigning${TXT_CLEAR}" && + echo "${TXT_NOTICE}enable certificate for codesigning${TXT_CLEAR}" && security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain >/dev/null && security find-identity -v && - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo -e "${TXT_NOTICE}sign binaries with ${sign_id}${TXT_CLEAR}" && + sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "${TXT_NOTICE}sign binaries with ${sign_id}${TXT_CLEAR}" && if test -n "${sign_id}"; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + && find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" fi; fi || codesign_error=$? - if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; else echo -e "${TXT_NOTICE}code signing returned with ${codesign_error}${TXT_CLEAR}"; fi + - | + true; + if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; else echo "${TXT_NOTICE}code signing returned with ${codesign_error}${TXT_CLEAR}"; fi ####################################################################### -- GitLab From 11bc13576b5ca772fdb29c8ca661d6d30b6cafbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 24 Mar 2021 17:56:31 +0100 Subject: [PATCH 057/157] codesign_error display --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 23bd052..6d3acb6 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -71,8 +71,8 @@ variables: fi; fi || codesign_error=$? - | - true; - if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; else echo "${TXT_NOTICE}code signing returned with ${codesign_error}${TXT_CLEAR}"; fi + echo "${TXT_NOTICE}code signing returned with ${codesign_error}${TXT_CLEAR}"; + if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; fi ####################################################################### -- GitLab From 11919c7fa46f158d5387e98610a8d234e48caaff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 14 Apr 2021 22:25:38 +0200 Subject: [PATCH 058/157] job-template for Linux_amd64 that uses the .l_amd64 extension --- pd-lib-builder/iem-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 6d3acb6..177ff90 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -114,6 +114,12 @@ variables: - rm -rf "${IEM_CI_TMPDIR}" - export PD=/usr/bin/pd +.build:linux_amd64: &build_linux_amd64 + extends: .build:linux + image: registry.git.iem.at/devtools/docker/debiancross:amd64 + variables: + pd_extension: l_amd64 + .build:linux_i386: &build_linux_i386 extends: .build:linux image: registry.git.iem.at/devtools/docker/debiancross:i386 -- GitLab From d6f5c59053e60ea62b07400aa42d8c0fe1b073ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 13:03:11 +0200 Subject: [PATCH 059/157] use pd_extension (if available) for generating local (macOS) deps --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 177ff90..d80d01e 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -161,7 +161,7 @@ variables: - test -n "${pd_extension}" || case "${IEM_CI_MACOS_BUILDFAT}" in yes|1|true) pd_extension=d_fat ;; esac after_script: - *script_fetch_localdeps - - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.pd_darwin" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi + - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.${pd_extension:-pd_darwin}" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi # sign the code and verify it (LATER move this into a separate stage; see 'SIGN CODE' below) - *script_codesign_macos -- GitLab From 41d1dcb489bc5c22f759a0ae616296d758334ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 13:03:41 +0200 Subject: [PATCH 060/157] base job for notarizing binary on macOS --- pd-lib-builder/iem-ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index d80d01e..a1e4a01 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -165,6 +165,31 @@ variables: # sign the code and verify it (LATER move this into a separate stage; see 'SIGN CODE' below) - *script_codesign_macos +.notarize:macos: + # notarize binaries in ${IEM_CI_PKGLIBDIR} (fallback to ${IEM_CI_PROJECT_NAME}) + tags: + - osx + retry: + max: 1 + when: + - runner_system_failure + allow_failure: true + script: &script_notarize_macos + # either use BUNDLE_ID as bundle_id, or generate one from the CI-settings + - bundle_id=${BUNDLE_ID:-$(echo $(echo ${CI_SERVER_HOST} | tr '.' $'\n' | tac | paste -s -d '.').${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME} | sed -e 's/[^a-zA-Z0-9-]/./g' | tr 'A-Z' 'a-z')} + - echo "bundle_id ${bundle_id}" + # try to switch to the latest and greatest XCode, so we get 'altool' + - sudo xcode-select --switch $(for d in /Applications/Xcode*.app/Contents/Info.plist ; do echo $(defaults read ${d%.plist} CFBundleShortVersionString) ${d%/Contents/Info.plist}; done | sort -t. -k1,1n -k2,2n -k3,3n | tail -1 | sed -e 's/^[^ ]* //' ) || true + # check if altool exists + - xcrun --find altool + # stuff everything into a disk-image + - hdiutil create -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" + - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${DMG}.dmg" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) + artifacts: + name: notarization + paths: + - ./notarize.plist + .build:w32: &build_w32 extends: .build:script tags: -- GitLab From 4a297e9fa8c548fd838ad88150dd1917917bedbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 13:23:51 +0200 Subject: [PATCH 061/157] no 'tac' on macOS; and 'paste' *requires* a file... --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index a1e4a01..e80e21f 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -176,7 +176,7 @@ variables: allow_failure: true script: &script_notarize_macos # either use BUNDLE_ID as bundle_id, or generate one from the CI-settings - - bundle_id=${BUNDLE_ID:-$(echo $(echo ${CI_SERVER_HOST} | tr '.' $'\n' | tac | paste -s -d '.').${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME} | sed -e 's/[^a-zA-Z0-9-]/./g' | tr 'A-Z' 'a-z')} + - bundle_id=${BUNDLE_ID:-$(echo $(echo ${CI_SERVER_HOST} | tr '.' $'\n' | (tac 2>/dev/null || tail -r) | paste -s -d '.' -).${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME} | sed -e 's/[^a-zA-Z0-9-]/./g' | tr 'A-Z' 'a-z')} - echo "bundle_id ${bundle_id}" # try to switch to the latest and greatest XCode, so we get 'altool' - sudo xcode-select --switch $(for d in /Applications/Xcode*.app/Contents/Info.plist ; do echo $(defaults read ${d%.plist} CFBundleShortVersionString) ${d%/Contents/Info.plist}; done | sort -t. -k1,1n -k2,2n -k3,3n | tail -1 | sed -e 's/^[^ ]* //' ) || true -- GitLab From adf8679ca87fb50f13edc8e96259cf1293df498e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 13:42:28 +0200 Subject: [PATCH 062/157] debugging printout when creating DMG --- pd-lib-builder/iem-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index e80e21f..b734847 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -183,7 +183,9 @@ variables: # check if altool exists - xcrun --find altool # stuff everything into a disk-image - - hdiutil create -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" + - echo hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" + - hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" + # and upload to apple... - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${DMG}.dmg" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) artifacts: name: notarization -- GitLab From a9ecf4a95a1f42e41754fa6b03b9f2d9b90d497f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 14:41:37 +0200 Subject: [PATCH 063/157] do the DMG verification before trying to upload it... --- pd-lib-builder/iem-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b734847..623a2f4 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -185,6 +185,7 @@ variables: # stuff everything into a disk-image - echo hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" - hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" + - hdiutil verify "${IEM_CI_PROJECT_NAME}.dmg" # and upload to apple... - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${DMG}.dmg" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) artifacts: -- GitLab From 9202dd97d5ceb4f014cdcd8df9608292e674d6ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 14:57:37 +0200 Subject: [PATCH 064/157] fixed archivename when uploading for macOS-notarization --- pd-lib-builder/iem-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 623a2f4..8e2c823 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -176,6 +176,7 @@ variables: allow_failure: true script: &script_notarize_macos # either use BUNDLE_ID as bundle_id, or generate one from the CI-settings + - archivefile="${IEM_CI_PROJECT_NAME}.dmg" - bundle_id=${BUNDLE_ID:-$(echo $(echo ${CI_SERVER_HOST} | tr '.' $'\n' | (tac 2>/dev/null || tail -r) | paste -s -d '.' -).${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME} | sed -e 's/[^a-zA-Z0-9-]/./g' | tr 'A-Z' 'a-z')} - echo "bundle_id ${bundle_id}" # try to switch to the latest and greatest XCode, so we get 'altool' @@ -183,11 +184,10 @@ variables: # check if altool exists - xcrun --find altool # stuff everything into a disk-image - - echo hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" - - hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${IEM_CI_PROJECT_NAME}.dmg" - - hdiutil verify "${IEM_CI_PROJECT_NAME}.dmg" + - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${archivefile}" + - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil verify "${archivefile}" # and upload to apple... - - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${DMG}.dmg" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) + - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) artifacts: name: notarization paths: -- GitLab From 277a96f08903a0b3800871275d41ed7358180eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 14:57:53 +0200 Subject: [PATCH 065/157] handle ZIP-archives as containers for apple uploads --- pd-lib-builder/iem-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 8e2c823..55fbe92 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -186,6 +186,8 @@ variables: # stuff everything into a disk-image - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${archivefile}" - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil verify "${archivefile}" + # or a ZIP-fil , if you really want + - test "${archivefile}" = "${archivefile%.zip}" || zip -r -y "${archivefile}" "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" # and upload to apple... - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) artifacts: -- GitLab From 59b4bf481f9c03b1ca564602de2ddf5a1735f0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 18:03:21 +0200 Subject: [PATCH 066/157] (optionally) wait for notarization result and staple --- pd-lib-builder/iem-ci.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 55fbe92..3e789b0 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -167,6 +167,9 @@ variables: .notarize:macos: # notarize binaries in ${IEM_CI_PKGLIBDIR} (fallback to ${IEM_CI_PROJECT_NAME}) + variables: + NOTARIZE_TIMEOUT: 0 + ignore_staple_errors: "true" tags: - osx retry: @@ -186,14 +189,33 @@ variables: # stuff everything into a disk-image - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${archivefile}" - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil verify "${archivefile}" - # or a ZIP-fil , if you really want + # or a ZIP-file , if you really want - test "${archivefile}" = "${archivefile%.zip}" || zip -r -y "${archivefile}" "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" # and upload to apple... - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) + # read back the UUID of the notarization request + - notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's/^[^"]*"//' -e 's/".*//') + # if NOTARIZE_TIMEOUT is set, wait (at most) that long to see whether notarization succeeded + - end=0 + - test -z "${notarize_uuid}" test 0 -le ${NOTARIZE_TIMEOUT} || end=$(($(date +%s) + ${NOTARIZE_TIMEOUT})) + - | + while [ ${end} -gt $(date +%s) ]; do + xcrun altool -u "${APPLE_ID}" -p "${APPLE_PWD}" --output-format xml --notarization-info "${notarize_uuid}" > notarization-info.plist; + defaults read $(pwd)/notarization-info.plist notarization-info; + if [ -z $(defaults read $(pwd)/notarization-info.plist notarization-info | egrep '^ *Status *=') ]; then + break; + fi; + logfile=$(defaults read $(pwd)/notarization-info.plist notarization-info | egrep '^ *LogFileURL *=' | sed -e 's/^[^"]*"//' -e 's/".*//'); + if [ -n "${logfile}" ]; then curl "${logfile}" | tee notarization.log.json; then + break; + fi; + done + # attempt to staple the notarization ticket + - xcrun stapler staple "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" || ${ignore_staple_errors} artifacts: name: notarization paths: - - ./notarize.plist + - ./notariz*.* .build:w32: &build_w32 extends: .build:script -- GitLab From 1074404cc47891a79496662bf8fae4d25fdafeed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 18:17:18 +0200 Subject: [PATCH 067/157] fixed syntax error in if-clause double "then"... --- pd-lib-builder/iem-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 3e789b0..b71c2fe 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -206,7 +206,8 @@ variables: break; fi; logfile=$(defaults read $(pwd)/notarization-info.plist notarization-info | egrep '^ *LogFileURL *=' | sed -e 's/^[^"]*"//' -e 's/".*//'); - if [ -n "${logfile}" ]; then curl "${logfile}" | tee notarization.log.json; then + if [ -n "${logfile}" ]; then + curl "${logfile}" | tee notarization.log.json; break; fi; done -- GitLab From 996c214229d70315663a42845ed42e6ea37c7169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 18:17:30 +0200 Subject: [PATCH 068/157] less verbose DMG-building --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b71c2fe..dbf9d67 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -187,8 +187,8 @@ variables: # check if altool exists - xcrun --find altool # stuff everything into a disk-image - - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil create -verbose -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${archivefile}" - - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil verify "${archivefile}" + - test "${archivefile}" = "${archivefile%.dmg}" || hdiutil create -volname "${IEM_CI_PROJECT_NAME}" -format UDZO -srcfolder "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" "${archivefile}" + #- test "${archivefile}" = "${archivefile%.dmg}" || hdiutil verify "${archivefile}" # or a ZIP-file , if you really want - test "${archivefile}" = "${archivefile%.zip}" || zip -r -y "${archivefile}" "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" # and upload to apple... -- GitLab From d8ef11c7bc90437f9f4de283c4a22bf9584e17ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 18:17:47 +0200 Subject: [PATCH 069/157] try a zip --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index dbf9d67..094c472 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -179,7 +179,7 @@ variables: allow_failure: true script: &script_notarize_macos # either use BUNDLE_ID as bundle_id, or generate one from the CI-settings - - archivefile="${IEM_CI_PROJECT_NAME}.dmg" + - archivefile="${IEM_CI_PROJECT_NAME}.zip" - bundle_id=${BUNDLE_ID:-$(echo $(echo ${CI_SERVER_HOST} | tr '.' $'\n' | (tac 2>/dev/null || tail -r) | paste -s -d '.' -).${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME} | sed -e 's/[^a-zA-Z0-9-]/./g' | tr 'A-Z' 'a-z')} - echo "bundle_id ${bundle_id}" # try to switch to the latest and greatest XCode, so we get 'altool' -- GitLab From fe120b087dac37c653b1e641d0edc8a94031eb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 18:23:33 +0200 Subject: [PATCH 070/157] proper macOS:notarize job --- pd-lib-builder/iem-ci.yml | 1 - pd-lib-builder/pipeline-jobs.yml | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 094c472..2591aa4 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -176,7 +176,6 @@ variables: max: 1 when: - runner_system_failure - allow_failure: true script: &script_notarize_macos # either use BUNDLE_ID as bundle_id, or generate one from the CI-settings - archivefile="${IEM_CI_PROJECT_NAME}.zip" diff --git a/pd-lib-builder/pipeline-jobs.yml b/pd-lib-builder/pipeline-jobs.yml index 1cc7342..dbe35ec 100644 --- a/pd-lib-builder/pipeline-jobs.yml +++ b/pd-lib-builder/pipeline-jobs.yml @@ -63,6 +63,16 @@ w64: - Darwin allow_failure: true +macOS:notarize: + stage: deploy + extends: + - .notarize:macos + dependencies: + - Darwin + needs: + - Darwin + allow_failure: true + ####################################################################### ### create deken packages and (optionally) upload them; ### if you want to automatically upload a package, you need to -- GitLab From 2c14e61ccb2d994821cd6b241ae4ea2a60641f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 18:30:14 +0200 Subject: [PATCH 071/157] don't tee te plist --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 2591aa4..23440d0 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -191,7 +191,7 @@ variables: # or a ZIP-file , if you really want - test "${archivefile}" = "${archivefile%.zip}" || zip -r -y "${archivefile}" "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" # and upload to apple... - - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml | tee notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) + - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml > notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) # read back the UUID of the notarization request - notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's/^[^"]*"//' -e 's/".*//') # if NOTARIZE_TIMEOUT is set, wait (at most) that long to see whether notarization succeeded -- GitLab From 4eb7eb5abe407a4565e4ab65e0f5f109c051f8e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 22 Apr 2021 18:30:39 +0200 Subject: [PATCH 072/157] only calculate the notarization_uuid if there actually *should* be one... --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 23440d0..b2a33bb 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -193,7 +193,7 @@ variables: # and upload to apple... - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml > notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) # read back the UUID of the notarization request - - notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's/^[^"]*"//' -e 's/".*//') + - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's/^[^"]*"//' -e 's/".*//') # if NOTARIZE_TIMEOUT is set, wait (at most) that long to see whether notarization succeeded - end=0 - test -z "${notarize_uuid}" test 0 -le ${NOTARIZE_TIMEOUT} || end=$(($(date +%s) + ${NOTARIZE_TIMEOUT})) -- GitLab From 28b09b10dbc5024f86699f5ea9401a5c5e676822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 23 Apr 2021 11:45:45 +0200 Subject: [PATCH 073/157] simplify the wait-loop for the apple notarization to finish --- pd-lib-builder/iem-ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b2a33bb..75d0905 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -200,16 +200,11 @@ variables: - | while [ ${end} -gt $(date +%s) ]; do xcrun altool -u "${APPLE_ID}" -p "${APPLE_PWD}" --output-format xml --notarization-info "${notarize_uuid}" > notarization-info.plist; - defaults read $(pwd)/notarization-info.plist notarization-info; - if [ -z $(defaults read $(pwd)/notarization-info.plist notarization-info | egrep '^ *Status *=') ]; then - break; - fi; - logfile=$(defaults read $(pwd)/notarization-info.plist notarization-info | egrep '^ *LogFileURL *=' | sed -e 's/^[^"]*"//' -e 's/".*//'); - if [ -n "${logfile}" ]; then - curl "${logfile}" | tee notarization.log.json; - break; - fi; + defaults read $(pwd)/notarization-info.plist notarization-info | tee /dev/stderr | egrep "Status.*in progress" >/dev/null && continue || break; done + # check whether there's a logfile to report + - test ! -e notarization-info.plist || logfile=$(defaults read $(pwd)/notarization-info.plist notarization-info | egrep '^ *LogFileURL *=' | sed -e 's|.*"\(.*\)";|\1|') + - test -z "${logfile}" || curl "${logfile}" | tee notarization.log.json # attempt to staple the notarization ticket - xcrun stapler staple "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" || ${ignore_staple_errors} artifacts: -- GitLab From 60143965094099df69172e959b5f8f99eb38aabc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 23 Apr 2021 11:45:59 +0200 Subject: [PATCH 074/157] simplify sed-expression --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 75d0905..7c0efc2 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -193,7 +193,7 @@ variables: # and upload to apple... - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml > notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) # read back the UUID of the notarization request - - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's/^[^"]*"//' -e 's/".*//') + - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's|.*"\(.*\)";|\1|') # if NOTARIZE_TIMEOUT is set, wait (at most) that long to see whether notarization succeeded - end=0 - test -z "${notarize_uuid}" test 0 -le ${NOTARIZE_TIMEOUT} || end=$(($(date +%s) + ${NOTARIZE_TIMEOUT})) -- GitLab From 8c2846641f519e7ae7a23f6b9bac9603404c0ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 23 Apr 2021 12:03:39 +0200 Subject: [PATCH 075/157] convert multiline script into multiple lines --- pd-lib-builder/iem-ci.yml | 49 +++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 7c0efc2..2dc25fb 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -47,33 +47,27 @@ variables: - rm -rf "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" - mv "${IEM_CI_TMPDIR}/${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" . -.script:codesign_macos: &script_codesign_macos - - | - if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then - keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}}; TXT_NOTICE="INFO..."; TXT_CLEAR=""; - echo "${TXT_NOTICE}code signing${TXT_CLEAR}" && - echo "${TXT_NOTICE}fetch certificates${TXT_CLEAR}" && - curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer && - curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer && - cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx && (md5 /tmp/sign.pfx || true) && - echo "${TXT_NOTICE}create and unlock keychain${TXT_CLEAR}" && - security create-keychain -p "${keychainpass}" build.keychain && security default-keychain -s build.keychain && security unlock-keychain -p "${keychainpass}" build.keychain && - echo "${TXT_NOTICE}import certificates into keychain${TXT_CLEAR}" && - security import /tmp/AppleWWDRCA.cer -k build.keychain && security import /tmp/AppleWWDRCAG3.cer -k build.keychain && - security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign && - echo "${TXT_NOTICE}enable certificate for codesigning${TXT_CLEAR}" && - security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain >/dev/null && - security find-identity -v && - sign_id=$(security find-identity -v | head -1 | awk '{print $2}') && echo "${TXT_NOTICE}sign binaries with ${sign_id}${TXT_CLEAR}" && - if test -n "${sign_id}"; then - find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + && - find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" - fi; - fi || codesign_error=$? - - | - echo "${TXT_NOTICE}code signing returned with ${codesign_error}${TXT_CLEAR}"; - if [ "x${codesign_error}" = "x0" ]; then codesign_error=""; fi - +.script:codesign:macos: &script_codesign_macos + # setup a keychain with the signing certificate + - skipchain="true" + - if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then skipchain="false"; fi + - ${skipchain} || keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}} + - ${skipchain} || curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer + - ${skipchain} || curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer + - ${skipchain} || cat "${MACOS_CERTIFICATE_PFX}" | base64 -D >/tmp/sign.pfx + - test ! -e /tmp/sign.pfx || shasum /tmp/sign.pfx + - ${skipchain} || security create-keychain -p "${keychainpass}" build.keychain + - ${skipchain} || security default-keychain -s build.keychain + - ${skipchain} || security unlock-keychain -p "${keychainpass}" build.keychain + - ${skipchain} || security import /tmp/AppleWWDRCA.cer -k build.keychain + - ${skipchain} || security import /tmp/AppleWWDRCAG3.cer -k build.keychain + - ${skipchain} || security import /tmp/sign.pfx -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign + - ${skipchain} || security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${keychainpass}" build.keychain >/dev/null + - security find-identity -v + - ${skipchain} || sign_id=$(security find-identity -v | head -1 | awk '{print $2}') + # use the keychain to sign whatever is there + - test -z "${sign_id}" || echo "${TXT_NOTICE}sign binaries with ${sign_id}${TXT_CLEAR}" + - test -z "${sign_id}" || find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verbose --sign "${sign_id}" ${CODESIGNFLAGS} {} + ####################################################################### ### configuration templates (to be used for snapshot and release builds) @@ -303,7 +297,6 @@ variables: - $MACOS_CERTIFICATE_PWD script: - *script_codesign_macos - - test "x${codesign_error}" != "x" && exit ${codesign_error} after_script: - find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f -exec codesign --verify {} ";" -- GitLab From c60e3b6550a3a8186fc15d5a7d2029c822c75e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 23 Apr 2021 12:04:10 +0200 Subject: [PATCH 076/157] drop deken_snapshot job (and add some comments) --- pd-lib-builder/pipeline-jobs.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pd-lib-builder/pipeline-jobs.yml b/pd-lib-builder/pipeline-jobs.yml index dbe35ec..91d2ea5 100644 --- a/pd-lib-builder/pipeline-jobs.yml +++ b/pd-lib-builder/pipeline-jobs.yml @@ -30,31 +30,38 @@ Linux: extends: - .Linux - .snapshot + Linuxi386: extends: - .Linuxi386 - .snapshot + LinuxARMhf: extends: - .LinuxARMhf - .snapshot + LinuxARM64: extends: - .LinuxARM64 - .snapshot + Darwin: extends: - .Darwin - .snapshot + w32: extends: - .w32 - .snapshot + w64: extends: - .w64 - .snapshot +### signing job (disabled for now) .Darwin_sign: extends: - .sign:macos @@ -63,6 +70,7 @@ w64: - Darwin allow_failure: true +## macOS notarize binaries macOS:notarize: stage: deploy extends: @@ -79,11 +87,6 @@ macOS:notarize: ### set DEKEN_USERNAME/DEKEN_PASSWORD in the CI-project settings. ### (https://git.iem.at/help/ci/variables/README#variables) deken: - extends: - - .deken - - .release - -deken_snapshot: extends: - .deken - .snapshot -- GitLab From d4a538d8305b2c8fa0597f13c860afe280a61d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 26 Apr 2021 19:19:14 +0200 Subject: [PATCH 077/157] debugging printout for x-compilation --- pd-lib-builder/iem-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 2dc25fb..891bc69 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -37,6 +37,11 @@ variables: ## build snippets .script:make: &script_make + # some info on the build-system + - make --version || true + - ${CC:-cc} -dumpmachine || true + - echo target architecture ${TARGETARCH} + # do the actual build - make -C "${SRCDIR}" ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} .script:make_check: &script_make_check # the 'make --version' invocation is used to detect cross-compilation -- GitLab From 686ef153a33ac52f435de388eddfe867049acfb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 26 Apr 2021 19:21:23 +0200 Subject: [PATCH 078/157] trying to fix test for notarization timeout --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 891bc69..861db6b 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -195,7 +195,7 @@ variables: - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's|.*"\(.*\)";|\1|') # if NOTARIZE_TIMEOUT is set, wait (at most) that long to see whether notarization succeeded - end=0 - - test -z "${notarize_uuid}" test 0 -le ${NOTARIZE_TIMEOUT} || end=$(($(date +%s) + ${NOTARIZE_TIMEOUT})) + - test -z "${notarize_uuid}" || test 0 -gt ${NOTARIZE_TIMEOUT} || end=$(($(date +%s) + ${NOTARIZE_TIMEOUT})) - | while [ ${end} -gt $(date +%s) ]; do xcrun altool -u "${APPLE_ID}" -p "${APPLE_PWD}" --output-format xml --notarization-info "${notarize_uuid}" > notarization-info.plist; -- GitLab From ab981f573939b67875410952c6d7846a50f0e598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 10:17:11 +0200 Subject: [PATCH 079/157] skip code-signing if NOSIGN is set to 1 --- pd-lib-builder/iem-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 861db6b..5bd2ed5 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -56,6 +56,7 @@ variables: # setup a keychain with the signing certificate - skipchain="true" - if test -e "${MACOS_CERTIFICATE_PFX}" && test -n "${MACOS_CERTIFICATE_PWD}"; then skipchain="false"; fi + - if test "x${NOSIGN}" != x1 && test "x${NOSIGN}" != xyes ; then :; else skipchain="true"; fi - ${skipchain} || keychainpass=${keychainpass:-${CI_BUILD_TOKEN:-$(date +%s)}} - ${skipchain} || curl https://developer.apple.com/certificationauthority/AppleWWDRCA.cer >/tmp/AppleWWDRCA.cer - ${skipchain} || curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer >/tmp/AppleWWDRCAG3.cer -- GitLab From cae9c80bd51282fb016d1111992b986fd771d052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 10:18:24 +0200 Subject: [PATCH 080/157] check whether we can run compiled binaries (and if not, skip 'make check') that should be a more robust test, than the whacko arch-triplet comparision --- pd-lib-builder/iem-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 5bd2ed5..633a9f0 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -44,8 +44,11 @@ variables: # do the actual build - make -C "${SRCDIR}" ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} .script:make_check: &script_make_check - # the 'make --version' invocation is used to detect cross-compilation - - if make --version | egrep $(echo ${TARGETARCH:-$(${CC:-cc} -dumpmachine)} | sed -e 's|-.*-|-.*-|') && make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 ; then make -C "${SRCDIR}" check ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH}; else echo "no 'check' target or cross-compiling...skipping"; fi + # check if there's a 'make check' target (no use to run it, if it is not there) + - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi + # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) + - if [ "x${NOCHECK}" = "x" ]; then crosscheckcc=${CC:-${TARGETARCH}${TARGETARCH:+-cc}}; mkdir -p .git-ci/crosscheck/; echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || true; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected"; fi + - if test "x${NOCHECK}" != x1 && test "x${NOCHECK}" != xyes ; then make -C "${SRCDIR}" check ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH}; else echo "skipping 'make check'"; fi .script:make_install: &script_make_install - rm -rf "${IEM_CI_TMPDIR}" - make -C "${SRCDIR}" install ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} DESTDIR="$(pwd)" pkglibdir="/${IEM_CI_TMPDIR}/${IEM_CI_PKGLIBDIR}${IEM_CI_PKGLIBDIR:+/${CI_JOB_NAME}/}" -- GitLab From 0cb650089c63584745f0a6418cb2d5b7c3f1676b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 10:49:13 +0200 Subject: [PATCH 081/157] better detection of cross-compiler --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 633a9f0..0a6c741 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -47,7 +47,7 @@ variables: # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) - - if [ "x${NOCHECK}" = "x" ]; then crosscheckcc=${CC:-${TARGETARCH}${TARGETARCH:+-cc}}; mkdir -p .git-ci/crosscheck/; echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || true; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected"; fi + - if [ "x${NOCHECK}" = "x" ]; then crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} | head -1); mkdir -p .git-ci/crosscheck/; echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || true; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected"; fi - if test "x${NOCHECK}" != x1 && test "x${NOCHECK}" != xyes ; then make -C "${SRCDIR}" check ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH}; else echo "skipping 'make check'"; fi .script:make_install: &script_make_install - rm -rf "${IEM_CI_TMPDIR}" -- GitLab From 0fb49bbbc5193b87dae8b5042a30153a5a061d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 11:13:40 +0200 Subject: [PATCH 082/157] suppress errors/print steps when detecting cross-compilation --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 0a6c741..d4e57b4 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -47,7 +47,7 @@ variables: # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) - - if [ "x${NOCHECK}" = "x" ]; then crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} | head -1); mkdir -p .git-ci/crosscheck/; echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || true; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected"; fi + - if [ "x${NOCHECK}" = "x" ]; then echo "checking for cross-compiler"; crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} 2>/dev/null | head -1); echo "crosscompiler: ${crosscheckcc}"; mkdir -p .git-ci/crosscheck/; echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || echo "crosscheck failed to compile"; ls .git-ci/crosscheck/; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected!!!"; fi - if test "x${NOCHECK}" != x1 && test "x${NOCHECK}" != xyes ; then make -C "${SRCDIR}" check ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH}; else echo "skipping 'make check'"; fi .script:make_install: &script_make_install - rm -rf "${IEM_CI_TMPDIR}" -- GitLab From a372cc91e4003b2b35ab9860a119228d7736c5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 11:19:59 +0200 Subject: [PATCH 083/157] split cross-compilation check into multiple lines --- pd-lib-builder/iem-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index d4e57b4..5067afe 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -47,7 +47,10 @@ variables: # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) - - if [ "x${NOCHECK}" = "x" ]; then echo "checking for cross-compiler"; crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} 2>/dev/null | head -1); echo "crosscompiler: ${crosscheckcc}"; mkdir -p .git-ci/crosscheck/; echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || echo "crosscheck failed to compile"; ls .git-ci/crosscheck/; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected!!!"; fi + - crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} 2>/dev/null | head -1) + - mkdir -p .git-ci/crosscheck/ + - echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c + - if [ "x${NOCHECK}" = "x" ]; then echo "checking for cross-compiler ${crosscheck}"; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || echo "crosscheck failed to compile"; ls .git-ci/crosscheck/; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected!!!"; fi - if test "x${NOCHECK}" != x1 && test "x${NOCHECK}" != xyes ; then make -C "${SRCDIR}" check ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH}; else echo "skipping 'make check'"; fi .script:make_install: &script_make_install - rm -rf "${IEM_CI_TMPDIR}" -- GitLab From c26b215b1988813d930f8eda22fa39ce6c408fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 11:23:42 +0200 Subject: [PATCH 084/157] don't fail if no cross-compiler can be detected --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 5067afe..ec4a251 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -47,7 +47,7 @@ variables: # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) - - crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} 2>/dev/null | head -1) + - crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} 2>/dev/null || true | head -1) - mkdir -p .git-ci/crosscheck/ - echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c - if [ "x${NOCHECK}" = "x" ]; then echo "checking for cross-compiler ${crosscheck}"; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || echo "crosscheck failed to compile"; ls .git-ci/crosscheck/; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected!!!"; fi -- GitLab From c489b73282215855083ad3a0be6e4cf63a7b677f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 11:55:38 +0200 Subject: [PATCH 085/157] use IEM_CI_TMPDIR for crosscheck --- pd-lib-builder/iem-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index ec4a251..6213fcd 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -47,10 +47,12 @@ variables: # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) + - rm -rf "${IEM_CI_TMPDIR}" + - mkdir -p "${IEM_CI_TMPDIR}" - crosscheckcc=$(which ${CC} ${TARGETARCH}${TARGETARCH:+-cc} ${TARGETARCH}${TARGETARCH:+-gcc} 2>/dev/null || true | head -1) - - mkdir -p .git-ci/crosscheck/ - - echo "int main(){return 0;}" > .git-ci/crosscheck/crosscheck.c - - if [ "x${NOCHECK}" = "x" ]; then echo "checking for cross-compiler ${crosscheck}"; make -C .git-ci/crosscheck/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || echo "crosscheck failed to compile"; ls .git-ci/crosscheck/; .git-ci/crosscheck/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected!!!"; fi + - echo "int main(){return 0;}" > ${IEM_CI_TMPDIR}/crosscheck.c + - if [ "x${NOCHECK}" = "x" ]; then echo "checking for cross-compiler ${crosscheck}"; make -C ${IEM_CI_TMPDIR}/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || echo "crosscheck failed to compile"; ls ${IEM_CI_TMPDIR}/; ${IEM_CI_TMPDIR}/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected!!!"; fi + - rm -rf "${IEM_CI_TMPDIR}" - if test "x${NOCHECK}" != x1 && test "x${NOCHECK}" != xyes ; then make -C "${SRCDIR}" check ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH}; else echo "skipping 'make check'"; fi .script:make_install: &script_make_install - rm -rf "${IEM_CI_TMPDIR}" -- GitLab From b16b223686c233f5adab01a6fc794f25e92b2e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 11:56:54 +0200 Subject: [PATCH 086/157] split apt-requirements code into multiple lines --- pd-lib-builder/iem-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 6213fcd..4dd708d 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -118,7 +118,11 @@ variables: - apt-get update - rm -rf "${IEM_CI_TMPDIR}" - mkdir -p "${IEM_CI_TMPDIR}" - - if [ -e .git-ci/requirements.apt ]; then apt-get install -y --no-install-recommends equivs; equivs-control "${IEM_CI_TMPDIR}"/builddeps; sed -e "s|^\(Package:\) .*|\1 builddeps-${CI_PROJECT_NAME}|" -e '/^Depends/d' -i "${IEM_CI_TMPDIR}"/builddeps; (echo "Depends:"" puredata-dev, puredata-core,"; cat .git-ci/requirements.apt | sed -e 's|#.*||' -e 's|[[:space:],]*$||' -e '/^$/d' -e 's|$|,|' -e 's|^[[:space:]]*| |') >> "${IEM_CI_TMPDIR}"/builddeps; (cd ${IEM_CI_TMPDIR}/; equivs-build ${TARGETDEBARCH:+-a} ${TARGETDEBARCH} builddeps; find . -name "*.deb" -exec dpkg -i {} + || apt-get -f install -y --no-install-recommends); else apt-get install -y --no-install-recommends build-essential puredata-dev puredata-core; fi + - if [ -e .git-ci/requirements.apt ]; then apt-get install -y --no-install-recommends equivs; equivs-control "${IEM_CI_TMPDIR}"/builddeps; fi + - if [ -e .git-ci/requirements.apt ]; then sed -e "s|^\(Package:\) .*|\1 builddeps-${CI_PROJECT_NAME}|" -e '/^Depends/d' -i "${IEM_CI_TMPDIR}"/builddeps; fi + - if [ -e .git-ci/requirements.apt ]; then (echo "Depends:"" puredata-dev, puredata-core,"; cat .git-ci/requirements.apt | sed -e 's|#.*||' -e 's|[[:space:],]*$||' -e '/^$/d' -e 's|$|,|' -e 's|^[[:space:]]*| |') >> "${IEM_CI_TMPDIR}"/builddeps; fi + - if [ -e .git-ci/requirements.apt ]; then (cd ${IEM_CI_TMPDIR}/; equivs-build ${TARGETDEBARCH:+-a} ${TARGETDEBARCH} builddeps; find . -name "*.deb" -exec dpkg -i {} + || apt-get -f install -y --no-install-recommends); fi + - if [ ! -e .git-ci/requirements.apt ]; then apt-get install -y --no-install-recommends build-essential$ puredata-dev puredata-core; fi - rm -rf "${IEM_CI_TMPDIR}" - export PD=/usr/bin/pd -- GitLab From 2c3302884543a2f9f684895682aab05b295ce3bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 11:58:01 +0200 Subject: [PATCH 087/157] when cross-compiling, install the cross-compilation puredata-* and build-essential packages --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 4dd708d..204228b 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -122,7 +122,7 @@ variables: - if [ -e .git-ci/requirements.apt ]; then sed -e "s|^\(Package:\) .*|\1 builddeps-${CI_PROJECT_NAME}|" -e '/^Depends/d' -i "${IEM_CI_TMPDIR}"/builddeps; fi - if [ -e .git-ci/requirements.apt ]; then (echo "Depends:"" puredata-dev, puredata-core,"; cat .git-ci/requirements.apt | sed -e 's|#.*||' -e 's|[[:space:],]*$||' -e '/^$/d' -e 's|$|,|' -e 's|^[[:space:]]*| |') >> "${IEM_CI_TMPDIR}"/builddeps; fi - if [ -e .git-ci/requirements.apt ]; then (cd ${IEM_CI_TMPDIR}/; equivs-build ${TARGETDEBARCH:+-a} ${TARGETDEBARCH} builddeps; find . -name "*.deb" -exec dpkg -i {} + || apt-get -f install -y --no-install-recommends); fi - - if [ ! -e .git-ci/requirements.apt ]; then apt-get install -y --no-install-recommends build-essential$ puredata-dev puredata-core; fi + - if [ ! -e .git-ci/requirements.apt ]; then apt-get install -y --no-install-recommends ${TARGETDEBARCH:+cross}build-essential${TARGETDEBARCH:+-}${TARGETDEBARCH} puredata-dev${TARGETDEBARCH:+:}${TARGETDEBARCH} puredata-core${TARGETDEBARCH:+:}${TARGETDEBARCH}; fi - rm -rf "${IEM_CI_TMPDIR}" - export PD=/usr/bin/pd -- GitLab From b31bcf7e1936060012a108cd3822c8f40933884b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 27 Apr 2021 12:07:26 +0200 Subject: [PATCH 088/157] add check whether ${PD} is usable for running tests... --- pd-lib-builder/iem-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 204228b..c31b2ce 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -46,6 +46,8 @@ variables: .script:make_check: &script_make_check # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi + # if PD is defined and exists, check whether we can run it + - if test "x${NOCHECK}" = "x" && test -x "${PD}"; then "${PD}" -version 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "${PD} is not usable!"; fi # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) - rm -rf "${IEM_CI_TMPDIR}" - mkdir -p "${IEM_CI_TMPDIR}" @@ -53,6 +55,7 @@ variables: - echo "int main(){return 0;}" > ${IEM_CI_TMPDIR}/crosscheck.c - if [ "x${NOCHECK}" = "x" ]; then echo "checking for cross-compiler ${crosscheck}"; make -C ${IEM_CI_TMPDIR}/ crosscheck ${crosscheckcc:+CC=}${crosscheckcc} || echo "crosscheck failed to compile"; ls ${IEM_CI_TMPDIR}/; ${IEM_CI_TMPDIR}/crosscheck || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "cross-compilation detected!!!"; fi - rm -rf "${IEM_CI_TMPDIR}" + # run 'make check' - if test "x${NOCHECK}" != x1 && test "x${NOCHECK}" != xyes ; then make -C "${SRCDIR}" check ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH}; else echo "skipping 'make check'"; fi .script:make_install: &script_make_install - rm -rf "${IEM_CI_TMPDIR}" -- GitLab From ca32067487b29375b89bd3449bab15a68175a01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 28 Apr 2021 09:33:44 +0200 Subject: [PATCH 089/157] only run notarize-job if APPLE_ID and APPLE_PWD are non-empty --- pd-lib-builder/iem-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index c31b2ce..2a1c145 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -187,6 +187,9 @@ variables: ignore_staple_errors: "true" tags: - osx + only: + variables: + - $APPLE_ID != "" && $APPLE_PWD != "" retry: max: 1 when: @@ -206,9 +209,9 @@ variables: # or a ZIP-file , if you really want - test "${archivefile}" = "${archivefile%.zip}" || zip -r -y "${archivefile}" "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" # and upload to apple... - - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml > notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) + - xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml > notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload # read back the UUID of the notarization request - - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's|.*"\(.*\)";|\1|') + - notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's|.*"\(.*\)";|\1|') # if NOTARIZE_TIMEOUT is set, wait (at most) that long to see whether notarization succeeded - end=0 - test -z "${notarize_uuid}" || test 0 -gt ${NOTARIZE_TIMEOUT} || end=$(($(date +%s) + ${NOTARIZE_TIMEOUT})) -- GitLab From 8a4dd7985a60e43686dabb65c76737fb89e2e756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 28 Apr 2021 10:43:44 +0200 Subject: [PATCH 090/157] only run notarize-job if APPLE_ID and APPLE_PWD are non-empty (another try) --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 2a1c145..c259d3f 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -187,9 +187,9 @@ variables: ignore_staple_errors: "true" tags: - osx - only: + except: variables: - - $APPLE_ID != "" && $APPLE_PWD != "" + - $APPLE_ID == "" || $APPLE_PWD == "" retry: max: 1 when: -- GitLab From a5ba10f7e1fb96e97bd1eb4af5779f7d7bf11f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 28 Apr 2021 11:32:22 +0200 Subject: [PATCH 091/157] Revert "only run notarize-job if APPLE_ID and APPLE_PWD are non-empty (another try)" This reverts commit 8a4dd7985a60e43686dabb65c76737fb89e2e756. --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index c259d3f..2a1c145 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -187,9 +187,9 @@ variables: ignore_staple_errors: "true" tags: - osx - except: + only: variables: - - $APPLE_ID == "" || $APPLE_PWD == "" + - $APPLE_ID != "" && $APPLE_PWD != "" retry: max: 1 when: -- GitLab From 24d91ebde86d3598685f5793ae3d72e456cbc968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 28 Apr 2021 11:32:29 +0200 Subject: [PATCH 092/157] Revert "only run notarize-job if APPLE_ID and APPLE_PWD are non-empty" This reverts commit ca32067487b29375b89bd3449bab15a68175a01e. --- pd-lib-builder/iem-ci.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 2a1c145..c31b2ce 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -187,9 +187,6 @@ variables: ignore_staple_errors: "true" tags: - osx - only: - variables: - - $APPLE_ID != "" && $APPLE_PWD != "" retry: max: 1 when: @@ -209,9 +206,9 @@ variables: # or a ZIP-file , if you really want - test "${archivefile}" = "${archivefile%.zip}" || zip -r -y "${archivefile}" "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" # and upload to apple... - - xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml > notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload + - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || (xcrun altool --notarize-app --primary-bundle-id "${bundle_id}" --username "${APPLE_ID}" --password "${APPLE_PWD}" --file "${archivefile}" --verbose --output-format xml > notarize.plist && defaults read $(pwd)/notarize.plist notarization-upload) # read back the UUID of the notarization request - - notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's|.*"\(.*\)";|\1|') + - test -z "${APPLE_ID}" || test -z "${APPLE_PWD}" || notarize_uuid=$(defaults read $(pwd)/notarize.plist notarization-upload | grep RequestUUID | sed -e 's|.*"\(.*\)";|\1|') # if NOTARIZE_TIMEOUT is set, wait (at most) that long to see whether notarization succeeded - end=0 - test -z "${notarize_uuid}" || test 0 -gt ${NOTARIZE_TIMEOUT} || end=$(($(date +%s) + ${NOTARIZE_TIMEOUT})) -- GitLab From 26ee7a82790685cfe6442418414df719fbeb99a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 30 Apr 2021 14:04:36 +0200 Subject: [PATCH 093/157] drop the deken_snapshot job --- no-build/gitlab-iem.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/no-build/gitlab-iem.yml b/no-build/gitlab-iem.yml index 2401ca1..d9c22bb 100644 --- a/no-build/gitlab-iem.yml +++ b/no-build/gitlab-iem.yml @@ -28,16 +28,7 @@ include: # tagged releases never expire deken: - only: - - tags extends: - .deken - -# untagged snapshot builds expire soon -deken_snapshot: - except: - - tags artifacts: expire_in: 1 week - extends: - - .deken -- GitLab From c00b4d1a47f6ed394f5d1c3cf7f97961b296341d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 30 Apr 2021 14:22:22 +0200 Subject: [PATCH 094/157] allow to include artifacts from previous build-jobs --- no-build/iem-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index 9cb4b2e..3c4409a 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -56,9 +56,13 @@ variables: - IEM_CI_PROJECT_INSTALLDIR=${IEM_CI_PROJECT_INSTALLDIR:-${IEM_CI_PROJECT_NAME}} - IEM_CI_PROJECT_ROOTDIR=${IEM_CI_PROJECT_ROOTDIR:-${IEM_CI_PROJECT_INSTALLDIR%%/*}} script: - - chmod -R go-w . -# create package - rm -rf tmp/ + # just in case a previous job added stuff, we include that here + - git add -A || true + - git commit -m "dummy commit" || true + # cleanup + - chmod -R go-w . + # create package - git archive --format=tar --prefix=tmp/${IEM_CI_PROJECT_INSTALLDIR}/ HEAD "${SRCDIR:-.}" | tar xf - - deken package --version="${CI_COMMIT_TAG#v}" --name "${IEM_CI_PROJECT_NAME}" "tmp/${IEM_CI_PROJECT_ROOTDIR}" # upload deken packages -- GitLab From b3481d0746a188dd61f008830c15e5abb41bb678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 3 May 2021 11:28:47 +0200 Subject: [PATCH 095/157] fixed spelling errors --- README.md | 6 +++--- no-build/README.md | 6 +++--- pd-lib-builder/README.md | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c1a79e5..87bf375 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Continuous Integration for Pd ============================= -Continous Integraton (CI) will automatically build your software whenever you +Continuous Integraton (CI) will automatically build your software whenever you `git push` your code to a (CI-enabled) git-server. This project targets building externals for Pure Data ([Pd][]), using the @@ -84,7 +84,7 @@ git push ## build products -(Successfull) builds will create a ZIP-file (so called *artifacts*). +(Successful) builds will create a ZIP-file (so called *artifacts*). These ZIP-files are available for download e.g. from `https://git.iem.at///pipelines/latest` (on the *Jobs*-Tab) @@ -94,7 +94,7 @@ The `deken` job (which is run as a final stage for tagged releases) will produce the `.dek`-files generated by the build. ## how to automatically upload tagged releases to deken -If you want you can automatically upload successfull builds for tagged releases. +If you want you can automatically upload successful builds for tagged releases. On `https://git.iem.at///settings/ci_cd` in the *Variables* section, set two variables diff --git a/no-build/README.md b/no-build/README.md index 8f7a1ea..58d4405 100644 --- a/no-build/README.md +++ b/no-build/README.md @@ -1,7 +1,7 @@ Continuous Integration for Pd - no-build variant ================================================ -Continous Integraton (CI) will automatically build your software whenever you +Continuous Integraton (CI) will automatically build your software whenever you `git push` your code to a (CI-enabled) git-server. This project targets building externals for Pure Data ([Pd][]), using the @@ -56,7 +56,7 @@ git push ## build products -(Successfull) builds will create a ZIP-file (so called *artifacts*) +(Successful) builds will create a ZIP-file (so called *artifacts*) with the compiled binaries (as created by a `make install` step). These ZIP-files are available for download e.g. from `https://git.iem.at///pipelines/latest` (on the *Jobs*-Tab) @@ -67,7 +67,7 @@ The `deken` job (which is run as a final stage for tagged releases) will produce the `.dek`-files generated by the build. ## how to automatically upload tagged releases to deken -If you want you can automatically upload successfull builds for tagged releases. +If you want you can automatically upload successful builds for tagged releases. On `https://git.iem.at///settings/ci_cd` in the *Variables* section, set two variables diff --git a/pd-lib-builder/README.md b/pd-lib-builder/README.md index f982987..759f39a 100644 --- a/pd-lib-builder/README.md +++ b/pd-lib-builder/README.md @@ -1,7 +1,7 @@ Continuous Integration for Pd - pd-lib-builder variant ====================================================== -Continous Integraton (CI) will automatically build your software whenever you +Continuous Integraton (CI) will automatically build your software whenever you `git push` your code to a (CI-enabled) git-server. This project targets building externals for Pure Data ([Pd][]), using the @@ -122,7 +122,7 @@ git push ## build products -(Successfull) builds will create a ZIP-file (so called *artifacts*) +(Successful) builds will create a ZIP-file (so called *artifacts*) with the compiled binaries (as created by a `make install` step). These ZIP-files are available for download e.g. from `https://git.iem.at///pipelines/latest` (on the *Jobs*-Tab) @@ -133,7 +133,7 @@ The `deken` job (which is run as a final stage for tagged releases) will produce the `.dek`-files generated by the build. ## how to automatically upload tagged releases to deken -If you want you can automatically upload successfull builds for tagged releases. +If you want you can automatically upload successful builds for tagged releases. On `https://git.iem.at///settings/ci_cd` in the *Variables* section, set two variables -- GitLab From a2de9dbffc3fd36196d4e3abf2a5e6b040b2bda2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 3 May 2021 11:29:58 +0200 Subject: [PATCH 096/157] no binaries for the "no-build" action --- no-build/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/no-build/README.md b/no-build/README.md index 58d4405..77046e8 100644 --- a/no-build/README.md +++ b/no-build/README.md @@ -57,7 +57,6 @@ git push ## build products (Successful) builds will create a ZIP-file (so called *artifacts*) -with the compiled binaries (as created by a `make install` step). These ZIP-files are available for download e.g. from `https://git.iem.at///pipelines/latest` (on the *Jobs*-Tab) -- GitLab From 9a08501916723bf3cd9fefa0ee97c0bb88d4bde3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 3 May 2021 11:30:37 +0200 Subject: [PATCH 097/157] 'iembot' uploads --- README.md | 7 +++++++ no-build/README.md | 7 +++++++ pd-lib-builder/README.md | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/README.md b/README.md index 87bf375..c03f384 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,13 @@ section, set two variables Please do not use any valuable username/password! +### `iembot` uploads from https://git.iem.at/pd/... + +Projects hosted under https://git.iem.at/pd/ already have these values set to automatically upload +new releases as the `iembot` user. +You are of course free to override these (or set them to an empty value to prevent automatic uploads altogether). + + ## customizing builds The CI-configuration should work out of the box. diff --git a/no-build/README.md b/no-build/README.md index 77046e8..89e3643 100644 --- a/no-build/README.md +++ b/no-build/README.md @@ -78,6 +78,13 @@ section, set two variables Please do not use any valuable username/password! +### `iembot` uploads from https://git.iem.at/pd/... + +Projects hosted under https://git.iem.at/pd/ already have these values set to automatically upload +new releases as the `iembot` user. +You are of course free to override these (or set them to an empty value to prevent automatic uploads altogether). + + ## customizing builds The CI-configuration should work out of the box. diff --git a/pd-lib-builder/README.md b/pd-lib-builder/README.md index 759f39a..308c19d 100644 --- a/pd-lib-builder/README.md +++ b/pd-lib-builder/README.md @@ -145,6 +145,13 @@ section, set two variables Please do not use any valuable username/password! +### `iembot` uploads from https://git.iem.at/pd/... + +Projects hosted under https://git.iem.at/pd/ already have these values set to automatically upload +new releases as the `iembot` user. +You are of course free to override these (or set them to an empty value to prevent automatic uploads altogether). + + ## customizing builds The CI-configuration should work out of the box. -- GitLab From e2d5a11b0858cdc748cb9ec5ca3535569d2eb3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 3 May 2021 12:09:24 +0200 Subject: [PATCH 098/157] document macOS codesigning --- pd-lib-builder/README.md | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pd-lib-builder/README.md b/pd-lib-builder/README.md index 308c19d..727d7fb 100644 --- a/pd-lib-builder/README.md +++ b/pd-lib-builder/README.md @@ -132,6 +132,61 @@ Download one of the ZIP-files, and extract it to a place where Pd can find it to The `deken` job (which is run as a final stage for tagged releases) will produce a ZIP-file containing all the `.dek`-files generated by the build. +## codesigning et al. +Recent versions of `macOS` require executables to be be [signed](https://developer.apple.com/support/code-signing/) +and [notarized](https://developer.apple.com/documentation/security/notarizing_macos_software_before_distribution). + +This is mostly important for *Applications* and less so for *plugins* (such as: Pd-externals). +In any case, there is basic support for both code-signing and notarization of macOS binaries. + +Recent versions of Windows push in the same direction, but there is no integration with the CI yet. + +### macOS: codesigning + +Code-Signing is done with a certificate that has been cross-signed by Apple (which they only +do if you have enrolled for their paid subscription of the Apple Developer program). + +The Darwin build jobs (that is: jobs inheriting the `before_script` and `after_script` from `.build:macos`) +will automatically sign any files in the installation directory +(`${IEM_CI_PKGLIBDIR}/`, `${IEM_CI_PROJECT_NAME}/` or `${CI_PROJECT_NAME}/`), +using the certificate found in the file named by the `MACOS_CERTIFICATE_PFX` variable. +The certificate must be base64-encoded and password protected: + +| variable name | description | +|-------------------------|----------------------------------------------------------------------| +| `MACOS_CERTIFICATE_PFX` | filename of a base64-encoded certificate (joined public+private key) | +| `MACOS_CERTIFICATE_PWD` | password to unlock the certificate | + +When adding `MACOS_CERTIFICATE_PFX` via the Gitlab CI/CD-Settings of your project, +make sure to set the type to `File`. + +Due to a [bug in Gitlab](https://gitlab.com/gitlab-org/gitlab/-/issues/324412), +code-signing currently needs to be integrated into the build job (rather than a standalone job). + +### macOS: notarization + +*Notarization* is the process of submitting an already signed binary to an Apple webservice +for automated security scans. +Applications that have been successfully notarized, +can be started by the user with only minor annoyances (they are asked whether they want to start an Application obtained from the internet once) +rather than the major annoyances imposed on Applications that are not notarized (where users have to manually whitelist applications). +Only Applications that are distributed through Apple's AppStore can be started by the user without annoyances (unless you thinl AppStore is an annoyance). + +The binaries must be signed first. +Uploading also requires an *Apple ID* plus password (A so-called *app-specific password* can be created after enabling two-factor authentication for your Apple ID) + +| variable name | description | +|---------------|--------------------------------------------------------------------------| +| `APPLE_ID` | Apple ID (typically an email-address) to use for submitting the binaries | +| `APPLE_PWD` | app-specific password assiciated with your Apple ID | + + +#### `IEM Developer ID Application` + +Projects hosted under https://git.iem.at/pd/ already have these values set to automatically sign and notarize macOS binaries. +As a security measure, this only happens on **`protected`** `branches & tags`. +You are of course free to override these (or set them to an empty value to prevent automatic signing/notarization altogether). + ## how to automatically upload tagged releases to deken If you want you can automatically upload successful builds for tagged releases. -- GitLab From dfade4112c6ea55fa8d6b2d697b66a1bf9b65ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig=20=28w10=40xenakis=29?= Date: Mon, 10 May 2021 14:46:22 +0200 Subject: [PATCH 099/157] try to include/exclude search paths for dependencies (should close the following, only the filtering doesn't work yet) Related: https://git.iem.at/pd/iem-ci/-/issues/4 also: recursively call `list_deps()` (should close the following, but needs more testing) Related: https://git.iem.at/pd/iem-ci/-/issues/1 --- pd-lib-builder/localdeps.win.sh | 130 ++++++++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 8 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 2068e19..8891223 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ## puts dependencies besides the binary @@ -20,19 +20,84 @@ error() { } NTLDD=$(which ntldd 2>/dev/null) +CYGPATH=$(which cygpath 2>/dev/null) +if [ -z "${CYGPATH}" ]; then + CYGPATH=echo +fi if [ "x${NTLDD}" = "x" ]; then error "no 'ntldd' binary found" exit 0 fi +normalize_path() { + # convert to unix-format (C:\foo\bar\ --> /c/foo/bar/) + # and lower-case everything (because on microsoft-fs, paths are case-insensitive) + ${CYGPATH} $1 | tr "[A-Z]" "[a-z]" +} + +exclude_paths="/c/msys64/usr/binx/:/usr/binx" +include_paths="/c/msys64/mingw64/bin/:/c/msys64/usr/bin/:/mingw64/bin/" + +list_dirs() { + local _ifs="$IFS" + local w + IFS=":" + for w in $@; do + IFS=${_ifs} + echo "$w" + done + IFS="${_ifs}" +} + +check_includedep() { + local path=$(normalize_path $1) + local p + local result=0 + error "" + error "check_includedep $@" + # exclude non-existing files + if [ ! -e "${path}" ]; then + exit + fi + + # exclude files in path that are excluded + list_dirs "${exclude_paths}" |while read p; do + if [ "x${path}" != "x${path#$p}" ]; then + exit + fi + done + # by default, we always include "mingw" stuff + if echo "${path}" | grep mingw >/dev/null; then + echo "${path}" + exit + fi + + # include only files in paths that are included + list_dirs "${include_paths}" |while read p; do + if [ "x${path}" != "x${path#$p}" ]; then + echo "${path}" + exit + fi + done + + exit +} + list_deps() { - ${NTLDD} -R "$1" \ - | grep -i mingw \ - | awk '{print $3}' \ - | grep -i mingw \ - | sed -e 's|\\|/|g' + local path + local inc + "${NTLDD}" "$1" \ + | grep ' => ' \ + | sed -e 's|\\|/|g' -e 's|.* => ||' -e 's| (0.*||' \ + | while read path; do + path=$(echo $path |sed -e 's|/|\\|g') + inc=$(check_includedep "${path}") + if [ "x${inc}" != "x" ]; then + echo "${inc}" + fi + done } file2arch() { @@ -77,15 +142,64 @@ list_deps "$1" | while read dep; do else error "DEP: ${INSTALLDEPS_INDENT} ${dep} -> ${outdir}/${odepfile}" cp "${dep}" "${outdir}/${odepfile}" - chmod a-x "${outdir}/${depfile}" + chmod a-x "${outdir}/${odepfile}" fi if [ "x${archext}" != "x" ]; then - sed -b -e "s|${idepfile}|${odepfile}|g" -i "${outdir}/${odepfile}" "${outdir}"/*."${archext}" "$1" + sed -b -e "s|${idepfile}|${odepfile}|g" -i "${outdir}/${odepfile}" "${dep}" "$1" fi + #recursively resolve dependencies + install_deps "${outdir}/${odepfile}" done } +usage() { + error "usage: $0 [-I ] [-X ] [ ...]" + error " recursively includes all dependencies of the given binaries" + error "" + error " -I : adds one include path entry" + error " -X : adds one exclude path entry" + error " only dependencies are added that live in a path that starts with any of the includepath-items" + error " but do not start with any of the excludepath-items" + error "" + error " dependencies are renamed from .dll to .w64 (resp .w32)" + exit 1 +} + +include_paths= +exclude_paths= +while getopts "hI:X:" arg; do + case $arg in + h) + usage + ;; + I) + p=$(normalize_path "${OPTARG}") + if [ "x${p}" != "x" ]; then + include_paths="${p}:${include_paths}" + fi + ;; + X) + p=$(normalize_path "${OPTARG}") + if [ "x${p}" != "x" ]; then + exclude_paths="${p}:${exclude_paths}" + fi + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) +include_paths=${include_paths%:} +exclude_paths=${exclude_paths%:} + +echo "include: ${include_paths}" +echo "exclude: ${exclude_paths}" + +#echo "binaris: $@" +#exit 0 +echo for f in "$@"; do if [ -e "${f}" ]; then -- GitLab From 136c7453fcdef7fd179537041c2f311ed5e4e7d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 10 May 2021 15:46:11 +0200 Subject: [PATCH 100/157] trying to get loops with subshells right --- pd-lib-builder/localdeps.win.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 8891223..2c8e2e7 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -58,30 +58,31 @@ check_includedep() { error "check_includedep $@" # exclude non-existing files if [ ! -e "${path}" ]; then - exit + return 0 fi - # exclude files in path that are excluded - list_dirs "${exclude_paths}" |while read p; do + list_dirs "${exclude_paths}" | while read p; do + # exclude files in paths that are excluded if [ "x${path}" != "x${path#$p}" ]; then - exit + return 1 fi - done - # by default, we always include "mingw" stuff + done || return 1 + if echo "${path}" | grep mingw >/dev/null; then + # by default, we always include "mingw" stuff echo "${path}" - exit + return 0 fi # include only files in paths that are included list_dirs "${include_paths}" |while read p; do if [ "x${path}" != "x${path#$p}" ]; then echo "${path}" - exit + return 0 fi - done + done && return 0 - exit + return 1 } -- GitLab From f9d00563c9c272f054982aa312ba5eb8fac3ec95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 10 May 2021 15:48:15 +0200 Subject: [PATCH 101/157] less verbose --- pd-lib-builder/localdeps.win.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 2c8e2e7..f303e2f 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -54,8 +54,6 @@ check_includedep() { local path=$(normalize_path $1) local p local result=0 - error "" - error "check_includedep $@" # exclude non-existing files if [ ! -e "${path}" ]; then return 0 @@ -195,12 +193,12 @@ shift $((OPTIND-1)) include_paths=${include_paths%:} exclude_paths=${exclude_paths%:} -echo "include: ${include_paths}" -echo "exclude: ${exclude_paths}" +error "include: ${include_paths}" +error "exclude: ${exclude_paths}" #echo "binaris: $@" #exit 0 -echo +error for f in "$@"; do if [ -e "${f}" ]; then -- GitLab From 750dec134c85e939827d2bf1d03bb3e46637a384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 11 May 2021 18:12:27 +0200 Subject: [PATCH 102/157] udpate to Pd-0.51.3 --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index c31b2ce..b04a64a 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -21,7 +21,7 @@ ####################################################################### variables: - PDVERSION: 0.50-2 + PDVERSION: 0.51-3 SRCDIR: . IEM_CI_TMPDIR: .git-ci/_build/ IEM_CI_PKGLIBDIR: "" -- GitLab From 589ba6fe9e1f4235ffee0797c0213ad4234dc2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 26 May 2021 17:40:05 +0200 Subject: [PATCH 103/157] print CI_COMMIT_TAG and DEKEN_USERNAME --- pd-lib-builder/iem-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b04a64a..050e8cf 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -332,6 +332,8 @@ variables: image: registry.git.iem.at/pd/deken:latest before_script: - apt-get update && apt-get --no-install-recommends -y install git + - echo "${CI_COMMIT_TAG}" + - echo "${DEKEN_USERNAME}" script: - chmod -R go-w . # create source package -- GitLab From 155016bf145c384e79e09fb8fc63607b88f9543b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 26 May 2021 17:40:05 +0200 Subject: [PATCH 104/157] print CI_COMMIT_TAG and DEKEN_USERNAME --- pd-lib-builder/iem-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index c31b2ce..b17b383 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -332,6 +332,8 @@ variables: image: registry.git.iem.at/pd/deken:latest before_script: - apt-get update && apt-get --no-install-recommends -y install git + - echo "${CI_COMMIT_TAG}" + - echo "${DEKEN_USERNAME}" script: - chmod -R go-w . # create source package -- GitLab From e6cfccc26e8b1ed15c899c5a2e4529ccefda08d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 26 May 2021 17:53:54 +0200 Subject: [PATCH 105/157] snapshots not limited to no-tags --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b17b383..14c03aa 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -95,8 +95,8 @@ variables: - "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" .snapshot: - except: - - tags + # except: + # - tags artifacts: expire_in: 1 week -- GitLab From 40fb540832f450c9d217152380eb430d115fc0eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 08:54:57 +0200 Subject: [PATCH 106/157] add "-v/-q" flags and only print include/excludepaths if verbosity>0 --- pd-lib-builder/localdeps.win.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index f303e2f..11ae236 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -19,6 +19,7 @@ error() { echo "$@" 1>&2 } +verbose=0 NTLDD=$(which ntldd 2>/dev/null) CYGPATH=$(which cygpath 2>/dev/null) if [ -z "${CYGPATH}" ]; then @@ -167,7 +168,7 @@ usage() { include_paths= exclude_paths= -while getopts "hI:X:" arg; do +while getopts "hqvI:X:" arg; do case $arg in h) usage @@ -184,6 +185,12 @@ while getopts "hI:X:" arg; do exclude_paths="${p}:${exclude_paths}" fi ;; + q) + verbose=$((verbose-1)) + ;; + v) + verbose=$((verbose+1)) + ;; *) usage ;; @@ -193,15 +200,15 @@ shift $((OPTIND-1)) include_paths=${include_paths%:} exclude_paths=${exclude_paths%:} -error "include: ${include_paths}" -error "exclude: ${exclude_paths}" - -#echo "binaris: $@" -#exit 0 -error +if [ ${verbose} -gt 0 ]; then + error "EXCLUDEPATHs: ${exclude_paths}" + error "INCLUDEPATHs: ${include_paths}" + error ' *mingw*' +fi for f in "$@"; do if [ -e "${f}" ]; then + error install_deps "${f}" fi done -- GitLab From 7ef155713af09342baa037c8537585ad87b2a6fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 08:56:45 +0200 Subject: [PATCH 107/157] use heredoc in usage() and clarify that deps that are excluded AND included will be excluded. --- pd-lib-builder/localdeps.win.sh | 36 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 11ae236..0236444 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -154,15 +154,33 @@ done } usage() { - error "usage: $0 [-I ] [-X ] [ ...]" - error " recursively includes all dependencies of the given binaries" - error "" - error " -I : adds one include path entry" - error " -X : adds one exclude path entry" - error " only dependencies are added that live in a path that starts with any of the includepath-items" - error " but do not start with any of the excludepath-items" - error "" - error " dependencies are renamed from .dll to .w64 (resp .w32)" + cat >/dev/stderr <] [-X ] [ ...] + recursively includes all dependencies of the given binaries + + -I : adds one include path entry + -X : adds one exclude path entry + -v: raise verbosity + -q: lower verbosity + + dependencies are renamed from .dll to .w64 (resp .w32) + +EXCLUDING/INCLUDING +------------------- +When traversing the runtime dependencies of a binary, dependencies are filtered +out based on their location (this is mainly to exclude system libraries that +can be very large and which are to be found on the target systems anyhow). + +Only dependencies (and sub-dependencies) that live in a path that +do NOT match any of the EXCLUDEPATHs and match at least one of the INCLUDEPATHs +are considered for inclusion (in this order. a dependency that matches both +EXCLUDEPATHs and INCLUDEPATHs is dropped). + +Matching is done on the start of the paths, so a pattern '/foo/bar' matches the +dependencies '/foo/bar.dll', '/foo/bartender.dll' and '/foo/bar/pizza.dll', +whereas a pattern '/foo/bar/' only matches '/foo/bar/pizza.dll'. + +EOF exit 1 } -- GitLab From 6f097d0bb3b13a6b79f388f9a7145209998aa460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 08:59:25 +0200 Subject: [PATCH 108/157] grouped internal state variables at the beginning --- pd-lib-builder/localdeps.win.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 0236444..d30a4f2 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -20,6 +20,11 @@ error() { } verbose=0 +exclude_paths="/c/msys64/usr/binx/:/usr/binx" +include_paths="/c/msys64/mingw64/bin/:/c/msys64/usr/bin/:/mingw64/bin/" +include_paths= +exclude_paths= + NTLDD=$(which ntldd 2>/dev/null) CYGPATH=$(which cygpath 2>/dev/null) if [ -z "${CYGPATH}" ]; then @@ -37,9 +42,6 @@ normalize_path() { ${CYGPATH} $1 | tr "[A-Z]" "[a-z]" } -exclude_paths="/c/msys64/usr/binx/:/usr/binx" -include_paths="/c/msys64/mingw64/bin/:/c/msys64/usr/bin/:/mingw64/bin/" - list_dirs() { local _ifs="$IFS" local w @@ -184,8 +186,6 @@ EOF exit 1 } -include_paths= -exclude_paths= while getopts "hqvI:X:" arg; do case $arg in h) -- GitLab From dcf531b6cb618da5b0d0d86995ed4436b29b7360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 08:59:33 +0200 Subject: [PATCH 109/157] quoting --- pd-lib-builder/localdeps.win.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index d30a4f2..9854890 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -39,7 +39,7 @@ fi normalize_path() { # convert to unix-format (C:\foo\bar\ --> /c/foo/bar/) # and lower-case everything (because on microsoft-fs, paths are case-insensitive) - ${CYGPATH} $1 | tr "[A-Z]" "[a-z]" + ${CYGPATH} "$1" | tr "[A-Z]" "[a-z]" } list_dirs() { -- GitLab From 89e618cee513fd1bda1ec1211a5d8cbe9a183078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 09:00:40 +0200 Subject: [PATCH 110/157] remove exactly matching paths from the include/exclude paths if they are added to the adversary --- pd-lib-builder/localdeps.win.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 9854890..3fddef2 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -182,6 +182,9 @@ Matching is done on the start of the paths, so a pattern '/foo/bar' matches the dependencies '/foo/bar.dll', '/foo/bartender.dll' and '/foo/bar/pizza.dll', whereas a pattern '/foo/bar/' only matches '/foo/bar/pizza.dll'. +You can remove an element from the INCLUDEPATHs by excluding it (exactly), +and vice versa. + EOF exit 1 } @@ -196,12 +199,14 @@ while getopts "hqvI:X:" arg; do if [ "x${p}" != "x" ]; then include_paths="${p}:${include_paths}" fi + exclude_paths=$(echo :${exclude_paths}: | sed -e "s|:${p}:|:|" -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; X) p=$(normalize_path "${OPTARG}") if [ "x${p}" != "x" ]; then exclude_paths="${p}:${exclude_paths}" fi + include_paths=$(echo :${include_paths}: | sed -e "s|:${p}:|:|" -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; q) verbose=$((verbose-1)) -- GitLab From a94b5f355f2cc7176c654179a16ba3960c0e9085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 10:07:42 +0200 Subject: [PATCH 111/157] helper-function for replacing literals https://stackoverflow.com/q/27782548/1169096 --- pd-lib-builder/localdeps.win.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 3fddef2..343e6d1 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -19,6 +19,13 @@ error() { echo "$@" 1>&2 } +substitute() { + # substitutes literal strings + # usage: echo foo | substitute foo bar g + sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" +} + + verbose=0 exclude_paths="/c/msys64/usr/binx/:/usr/binx" include_paths="/c/msys64/mingw64/bin/:/c/msys64/usr/bin/:/mingw64/bin/" -- GitLab From 655ac567f47241adf464c392d4eee0ecf469b14c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 10:07:52 +0200 Subject: [PATCH 112/157] use helper-function --- pd-lib-builder/localdeps.win.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 343e6d1..270be0d 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -206,14 +206,14 @@ while getopts "hqvI:X:" arg; do if [ "x${p}" != "x" ]; then include_paths="${p}:${include_paths}" fi - exclude_paths=$(echo :${exclude_paths}: | sed -e "s|:${p}:|:|" -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') + exclude_paths=$(echo :${exclude_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; X) p=$(normalize_path "${OPTARG}") if [ "x${p}" != "x" ]; then exclude_paths="${p}:${exclude_paths}" fi - include_paths=$(echo :${include_paths}: | sed -e "s|:${p}:|:|" -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') + include_paths=$(echo :${include_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; q) verbose=$((verbose-1)) -- GitLab From 803bb8e97f53cabebdbe2f38f23c7f926271c712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Thu, 27 May 2021 10:09:21 +0200 Subject: [PATCH 113/157] moved NTLDD check to end of script (before its actually used) so we can get the help and everything, even if it is not present --- pd-lib-builder/localdeps.win.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh index 270be0d..9f979c3 100755 --- a/pd-lib-builder/localdeps.win.sh +++ b/pd-lib-builder/localdeps.win.sh @@ -38,11 +38,6 @@ if [ -z "${CYGPATH}" ]; then CYGPATH=echo fi -if [ "x${NTLDD}" = "x" ]; then - error "no 'ntldd' binary found" - exit 0 -fi - normalize_path() { # convert to unix-format (C:\foo\bar\ --> /c/foo/bar/) # and lower-case everything (because on microsoft-fs, paths are case-insensitive) @@ -236,6 +231,11 @@ if [ ${verbose} -gt 0 ]; then error ' *mingw*' fi +if [ "x${NTLDD}" = "x" ]; then + error "no 'ntldd' binary found" + exit 0 +fi + for f in "$@"; do if [ -e "${f}" ]; then error -- GitLab From 8bbe277329334bd722711cb52f187b1bf1030874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 08:21:11 +0200 Subject: [PATCH 114/157] re-organized localdeps-scripts with common utilities. TODO: a build-script that automatically includes the sourced-files (for easier deployment from a single-file download) --- localdeps/localdeps.linux.sh | 262 +++++++++++++++++++++++++ localdeps/localdeps.macos.sh | 261 +++++++++++++++++++++++++ localdeps/localdeps.utilities.source | 155 +++++++++++++++ localdeps/localdeps.win.sh | 281 +++++++++++++++++++++++++++ 4 files changed, 959 insertions(+) create mode 100755 localdeps/localdeps.linux.sh create mode 100755 localdeps/localdeps.macos.sh create mode 100644 localdeps/localdeps.utilities.source create mode 100755 localdeps/localdeps.win.sh diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh new file mode 100755 index 0000000..4f568b1 --- /dev/null +++ b/localdeps/localdeps.linux.sh @@ -0,0 +1,262 @@ +#!/bin/sh +# +# creates local copies of all dependencies (dynamic libraries) +# and sets RUNPATH to $ORIGIN on each so they will find +# each other. +# +# usage: $0 + + + +verbose=0 +include_paths= +exclude_paths= + + + +#default exclude/include paths +exclude_paths="*/libc.so.*:*/libarmmem.*.so.*:*/libdl.so.*:*/libglib-.*.so.*:*/libgomp.so.*:*/libgthread.*.so.*:*/libm.so.*:*/libpthread.*.so.*:*/libpthread.so.*:*/libstdc++.so.*:*/libgcc_s.so.*:*/libpcre.so.*:*/libz.so.*" +include_paths="/lib/*:/usr/lib/*:/lib64/*:/usr/lib64/*" + +# UTILITIES +if [ -e "${0%/*}/localdeps.utilities.source" ]; then +. "${0%/*}/localdeps.utilities.source" +else + # the following section (from @BEGIN_UTILITIES@ to @END_UTILITIES@) + # was copied from 'localdeps.utilities.source'. + # changes you make to this section will be lost. +#@BEGIN_UTILITIES@ +verbose=${verbose:-0} + +error() { + echo "$@" 1>&2 +} + +substitute() { + # substitutes literal strings + # usage: echo foo | substitute foo bar g + sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" +} + + +normalize_path() { + # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" + # on most system this doesn't do anything, but override it to your needs... + # e.g. on Windows use: ${CYGPATH} "$1" | tr "[A-Z]" "[a-z]" + echo "$1" +} + +list_dirs() { + # + local IN="$@" + local iter + while [ "$IN" ] ;do + iter=${IN%%:*} + echo "${iter}" + [ "$IN" = "$iter" ] && IN='' || IN="${IN#*:}" + done +} + +check_in_path() { + local needle=$1 + local p + local patterns + shift + patterns="$@" + while [ "${patterns}" ]; do + p=${patterns%%:*} + [ "$patterns" = "$p" ] && patterns='' || patterns="${patterns#*:}" + + case "${needle}" in + ${p}) + echo "${needle}" + break + ;; + esac + done | grep . >/dev/null +} + +check_includedep() { + local path=$(normalize_path "$1") + local p + local result=0 + # exclude non-existing files + if [ ! -e "${path}" ]; then + return 0 + fi + + # skip paths that match one of the patterns in ${exclude_paths} + if check_in_path "${path}" "${exclude_paths}"; then + return 1 + fi + # only include paths that match one of the patterns in ${include_paths} + if check_in_path "${path}" "${include_paths}"; then + echo "${path}" + return 0 + fi + # skip the rest + return 1 +} + +usage() { + cat >/dev/stderr <] [-X ] [ ...] + recursively includes all dependencies of the given binaries + + -I : adds one include path entry + -X : adds one exclude path entry + -v: raise verbosity + -q: lower verbosity + +EOF + + case "$0" in + *win*) + cat >/dev/stderr </dev/stderr < /dev/null; then + skip=0 + break + fi + done + set +f + return $skip +} + +list_deps() { + local libpath + local inc + ldd "$1" \ + | grep ' => ' \ + | while read _ _ libpath _; do + inc=$(check_includedep "${libpath}") + if [ "x${inc}" != "x" ]; then + echo "${inc}" + fi + done +} + +install_deps () { + # make a local copy of all linked libraries of given binary + # and set RUNPATH to $ORIGIN (exclude "standard" libraries) + # arg1: binary to check + local outdir=$(dirname "$1") + local outfile + if [ ! -d "${outdir}" ]; then + outdir=. + fi + list_deps "$1" | while read libpath; do + libname=$(basename "${libpath}") + if [ ! -e "${libpath}" ]; then + error "DEP: ${INSTALLDEPS_INDENT} WARNING: could not make copy of '${libpath}'. Not found" + continue + fi + outfile="${outdir}/$(basename ${libpath})" + if [ -e "${outfile}" ]; then + error "DEP: ${INSTALLDEPS_INDENT} ${libpath} SKIPPED" + else + error "DEP: ${INSTALLDEPS_INDENT} ${libpath} -> ${outdir}/" + cp "${libpath}" "${outfile}" + patchelf --set-rpath \$ORIGIN "${outfile}" + fi + done + patchelf --set-rpath \$ORIGIN "${1}" +} + + + +# Check dependencies +cmdlist="awk grep ldd patchelf" +for cmd in $cmdlist; do + if ! which "${cmd}" > /dev/null; then + error "Could not find ${cmd}. Is it installed?" + exit 1 + fi +done + +for f in "$@"; do + # Check if we can read from given file + if ! ldd "${f}" > /dev/null 2>&1; then + error "Skipping '${f}'. Is it a binary file?" + continue + fi + install_deps "${f}" +done diff --git a/localdeps/localdeps.macos.sh b/localdeps/localdeps.macos.sh new file mode 100755 index 0000000..63ae729 --- /dev/null +++ b/localdeps/localdeps.macos.sh @@ -0,0 +1,261 @@ +#!/bin/sh + +## puts dependencies besides the binary +# LATER: put dependencies into a separate folder + +## usage: $0 [...] + +#default exclude/include paths +exclude_paths="/usr/lib/:/System/Library/Frameworks" +include_paths= + +# UTILITIES +if [ -e "${0%/*}/localdeps.utilities.source" ]; then +. "${0%/*}/localdeps.utilities.source" +else + # the following section (from @BEGIN_UTILITIES@ to @END_UTILITIES@) + # was copied from 'localdeps.utilities.source'. + # changes you make to this section will be lost. +#@BEGIN_UTILITIES@ +verbose=${verbose:-0} + +error() { + echo "$@" 1>&2 +} + +substitute() { + # substitutes literal strings + # usage: echo foo | substitute foo bar g + sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" +} + + +normalize_path() { + # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" + # on most system this doesn't do anything, but override it to your needs... + # e.g. on Windows use: ${CYGPATH} "$1" | tr "[A-Z]" "[a-z]" + echo "$1" +} + +list_dirs() { + # + local IN="$@" + local iter + while [ "$IN" ] ;do + iter=${IN%%:*} + echo "${iter}" + [ "$IN" = "$iter" ] && IN='' || IN="${IN#*:}" + done +} + +check_in_path() { + local needle=$1 + local p + local patterns + shift + patterns="$@" + while [ "${patterns}" ]; do + p=${patterns%%:*} + [ "$patterns" = "$p" ] && patterns='' || patterns="${patterns#*:}" + + case "${needle}" in + ${p}) + echo "${needle}" + break + ;; + esac + done | grep . >/dev/null +} + +check_includedep() { + local path=$(normalize_path "$1") + local p + local result=0 + # exclude non-existing files + if [ ! -e "${path}" ]; then + return 0 + fi + + # skip paths that match one of the patterns in ${exclude_paths} + if check_in_path "${path}" "${exclude_paths}"; then + return 1 + fi + # only include paths that match one of the patterns in ${include_paths} + if check_in_path "${path}" "${include_paths}"; then + echo "${path}" + return 0 + fi + # skip the rest + return 1 +} + +usage() { + cat >/dev/stderr <] [-X ] [ ...] + recursively includes all dependencies of the given binaries + + -I : adds one include path entry + -X : adds one exclude path entry + -v: raise verbosity + -q: lower verbosity + +EOF + + case "$0" in + *win*) + cat >/dev/stderr </dev/stderr < ${outdir}" + cp "${dep}" "${outdir}" + chmod u+w "${outdir}/${depfile}" + + # make sure the dependency announces itself with the local path + install_name_tool -id "@loader_path/${depfile}" "${outdir}/${depfile}" + # recursively call ourselves, to resolve higher-order dependencies + INSTALLDEPS_INDENT="${INSTALLDEPS_INDENT} " $0 "${outdir}/${depfile}" + fi + done +} + +if [ "x${otool}" = "x" ]; then + error "no 'otool' binary found" + exit 0 +fi + + +for f in "$@"; do + if [ -e "${f}" ]; then + error + install_deps "${f}" + fi +done diff --git a/localdeps/localdeps.utilities.source b/localdeps/localdeps.utilities.source new file mode 100644 index 0000000..7dabe59 --- /dev/null +++ b/localdeps/localdeps.utilities.source @@ -0,0 +1,155 @@ +verbose=${verbose:-0} + +error() { + echo "$@" 1>&2 +} + +substitute() { + # substitutes literal strings + # usage: echo foo | substitute foo bar g + sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" +} + + +normalize_path() { + # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" + # on most system this doesn't do anything, but override it to your needs... + # e.g. on Windows use: ${CYGPATH} "$1" | tr "[A-Z]" "[a-z]" + echo "$1" +} + +list_dirs() { + # + local IN="$@" + local iter + while [ "$IN" ] ;do + iter=${IN%%:*} + echo "${iter}" + [ "$IN" = "$iter" ] && IN='' || IN="${IN#*:}" + done +} + +check_in_path() { + local needle=$1 + local p + local patterns + shift + patterns="$@" + while [ "${patterns}" ]; do + p=${patterns%%:*} + [ "$patterns" = "$p" ] && patterns='' || patterns="${patterns#*:}" + + case "${needle}" in + ${p}) + echo "${needle}" + break + ;; + esac + done | grep . >/dev/null +} + +check_includedep() { + local path=$(normalize_path "$1") + local p + local result=0 + # exclude non-existing files + if [ ! -e "${path}" ]; then + return 0 + fi + + # skip paths that match one of the patterns in ${exclude_paths} + if check_in_path "${path}" "${exclude_paths}"; then + return 1 + fi + # only include paths that match one of the patterns in ${include_paths} + if check_in_path "${path}" "${include_paths}"; then + echo "${path}" + return 0 + fi + # skip the rest + return 1 +} + +usage() { + cat >/dev/stderr <] [-X ] [ ...] + recursively includes all dependencies of the given binaries + + -I : adds one include path entry + -X : adds one exclude path entry + -v: raise verbosity + -q: lower verbosity + +EOF + + case "$0" in + *win*) + cat >/dev/stderr </dev/stderr < [...] + + +########################################### +# WARNING +# +# this uses an ugly hack to allow side-by-side installation of 32bit and 64bit +# dependencies: +# embedded dependencies are renamed from "libfoo.dll" to "libfoo.w32" resp. +# "libfoo.w64", and the files are modified (using 'sed') to reflect this +# renaming. +# this is somewhat brittle and likely to break! + +#default exclude/include paths +exclude_paths="" +include_paths="*mingw*" + +# UTILITIES +if [ -e "${0%/*}/localdeps.utilities.source" ]; then +. "${0%/*}/localdeps.utilities.source" +else + # the following section (from @BEGIN_UTILITIES@ to @END_UTILITIES@) + # was copied from 'localdeps.utilities.source'. + # changes you make to this section will be lost. +#@BEGIN_UTILITIES@ +verbose=${verbose:-0} + +error() { + echo "$@" 1>&2 +} + +substitute() { + # substitutes literal strings + # usage: echo foo | substitute foo bar g + sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" +} + + +normalize_path() { + # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" + # on most system this doesn't do anything, but override it to your needs... + # e.g. on Windows use: ${CYGPATH} "$1" | tr "[A-Z]" "[a-z]" + echo "$1" +} + +list_dirs() { + # + local IN="$@" + local iter + while [ "$IN" ] ;do + iter=${IN%%:*} + echo "${iter}" + [ "$IN" = "$iter" ] && IN='' || IN="${IN#*:}" + done +} + +check_in_path() { + local needle=$1 + local p + local patterns + shift + patterns="$@" + while [ "${patterns}" ]; do + p=${patterns%%:*} + [ "$patterns" = "$p" ] && patterns='' || patterns="${patterns#*:}" + + case "${needle}" in + ${p}) + echo "${needle}" + break + ;; + esac + done | grep . >/dev/null +} + +check_includedep() { + local path=$(normalize_path "$1") + local p + local result=0 + # exclude non-existing files + if [ ! -e "${path}" ]; then + return 0 + fi + + # skip paths that match one of the patterns in ${exclude_paths} + if check_in_path "${path}" "${exclude_paths}"; then + return 1 + fi + # only include paths that match one of the patterns in ${include_paths} + if check_in_path "${path}" "${include_paths}"; then + echo "${path}" + return 0 + fi + # skip the rest + return 1 +} + +usage() { + cat >/dev/stderr <] [-X ] [ ...] + recursively includes all dependencies of the given binaries + + -I : adds one include path entry + -X : adds one exclude path entry + -v: raise verbosity + -q: lower verbosity + +EOF + + case "$0" in + *win*) + cat >/dev/stderr </dev/stderr </dev/null) +CYGPATH=$(which cygpath 2>/dev/null) +if [ -z "${CYGPATH}" ]; then + CYGPATH=echo +fi + +normalize_path() { + # convert to unix-format (C:\foo\bar\ --> /c/foo/bar/) + # and lower-case everything (because on microsoft-fs, paths are case-insensitive) + ${CYGPATH} "$1" | tr "[A-Z]" "[a-z]" +} + +list_deps() { + local path + local inc + "${NTLDD}" "$1" \ + | grep ' => ' \ + | sed -e 's|\\|/|g' -e 's|.* => ||' -e 's| (0.*||' \ + | while read path; do + path=$(echo $path |sed -e 's|/|\\|g') + inc=$(check_includedep "${path}") + if [ "x${inc}" != "x" ]; then + echo "${inc}" + fi + done +} + +file2arch() { + if file "$1" | grep -w "PE32+" >/dev/null; then + echo "w64" + return + fi + if file "$1" | grep -w "PE32" >/dev/null; then + echo "w32" + return + fi +} + +install_deps () { + local outdir="$2" + local idepfile + local odepfile + local archext + local dep + error "DEP: ${INSTALLDEPS_INDENT}'$1' '$2'" + + if [ "x${outdir}" = "x" ]; then + outdir=${1%/*} + fi + if [ ! -d "${outdir}" ]; then + outdir=. + fi + + list_deps "$1" | while read dep; do + idepfile=$(basename "${dep}") + odepfile=${idepfile} + archext=$(file2arch "${dep}") + if [ "x${archext}" != "x" ]; then + odepfile=$(echo ${idepfile} | sed -e "s|\.dll|.${archext}|") + fi + if [ "x${idepfile}" = "x${odepfile}" ]; then + archext="" + fi + if [ -e "${outdir}/${odepfile}" ]; then + error "DEP: ${INSTALLDEPS_INDENT} ${dep} SKIPPED" + else + error "DEP: ${INSTALLDEPS_INDENT} ${dep} -> ${outdir}/${odepfile}" + cp "${dep}" "${outdir}/${odepfile}" + chmod a-x "${outdir}/${odepfile}" + fi + + if [ "x${archext}" != "x" ]; then + sed -b \ + -e "s|${idepfile}|${odepfile}|g" \ + -i \ + "${outdir}/${odepfile}" "${dep}" "$1" + fi + #recursively resolve dependencies + INSTALLDEPS_INDENT="${INSTALLDEPS_INDENT} " install_deps "${outdir}/${odepfile}" + done +} + +if [ "x${NTLDD}" = "x" ]; then + error "no 'ntldd' binary found" + exit 0 +fi + +for f in "$@"; do + if [ -e "${f}" ]; then + error + install_deps "${f}" + fi +done -- GitLab From 45da6e98cc26b20aade033b83961e14e3dfa9216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 09:24:47 +0200 Subject: [PATCH 115/157] fetch localdeps script from 'main' branch (and localdeps/ directory) --- pd-lib-builder/iem-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 4dfca76..434acf8 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -32,8 +32,9 @@ variables: ####################################################################### ### script snippets .script:fetch_localdeps: &script_fetch_localdeps - - test -e .git-ci/localdeps.win.sh || (wget -q -O .git-ci/localdeps.win.sh "https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/localdeps.win.sh" && chmod +x .git-ci/localdeps.win.sh) || true - - test -e .git-ci/localdeps.macos.sh || (wget -q -O .git-ci/localdeps.macos.sh "https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/localdeps.macos.sh" && chmod +x .git-ci/localdeps.macos.sh) || true + - test -e .git-ci/localdeps.win.sh || (wget -q -O .git-ci/localdeps.win.sh "https://git.iem.at/pd/iem-ci/raw/main/localdeps/localdeps.win.sh" && chmod +x .git-ci/localdeps.win.sh) || true + - test -e .git-ci/localdeps.macos.sh || (wget -q -O .git-ci/localdeps.macos.sh "https://git.iem.at/pd/iem-ci/raw/main/localdeps/localdeps.macos.sh" && chmod +x .git-ci/localdeps.macos.sh) || true + - test -e .git-ci/localdeps.linux.sh || (wget -q -O .git-ci/localdeps.linux.sh "https://git.iem.at/pd/iem-ci/raw/main/localdeps/localdeps.linux.sh" && chmod +x .git-ci/localdeps.linux.sh) || true ## build snippets .script:make: &script_make -- GitLab From 3337cc242149ac5b438648b56d88d929c35019b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 09:25:06 +0200 Subject: [PATCH 116/157] gitignore backup files --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56968f1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# backup files +*~ +\#*\# +.\#* + +# MacOS +*.DS_STORE -- GitLab From 003b2123683468e63eeecdd2d8ac8c6a39baf805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 09:25:32 +0200 Subject: [PATCH 117/157] don't include repo-config in the archives --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..29b65a9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +.git* export-ignore +.travis.yml export-ignore -- GitLab From 33de8ab91b3acd0ed6dc3e8d15326f63b1c71837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 09:35:41 +0200 Subject: [PATCH 118/157] script to update localdeps.sh scripts with utilities --- localdeps/update_scripts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 localdeps/update_scripts diff --git a/localdeps/update_scripts b/localdeps/update_scripts new file mode 100755 index 0000000..75bb80a --- /dev/null +++ b/localdeps/update_scripts @@ -0,0 +1,29 @@ +#!/bin/sh + +# run this script to include the 'localdeps.utilities.source' snippet +# in the localdeps.*.sh scripts + +# running without arguments will (silently) update all localdeps.*.sh +# scripts in this folder. +# ./update_scripts +# running with arguments, you can check what would happen (or run +# selectively on a single localdeps.sh script +# print the new script to stdout: +# ./update_scripts path/to/localdeps.linux.sh +# update a single script: +# ./update_scripts -i path/to/localdeps.linux.sh + +localdepsdir="${0%/*}" + +update() { +sed \ + -e '/^#@BEGIN_UTILITIES@$/,/^#@END_UTILITIES@$/{//!d;};' \ + -e "/^#@BEGIN_UTILITIES@$/r ${localdepsdir}/localdeps.utilities.source" \ + "$@" +} + +if [ $# -gt 0 ]; then + update "$@" +else + update -i "${localdepsdir}/"localdeps.*.sh +fi -- GitLab From 5d32fea68775a4ddaad17a6fb1060a396b75bae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig=20=28w10=40xenakis=29?= Date: Fri, 28 May 2021 10:04:46 +0200 Subject: [PATCH 119/157] localdeps.win.sh: return unmangled dependency well, almost unmangled: backslashes are replaced by forwardslashes this fixes an issue where dependencies with upper-case characters were not properly included. actually, the files were included but the dependent would not be updated, as the sed-expression was case-sensitive. rather than doing a case-insensitive replace, we keep the cases. --- localdeps/localdeps.win.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/localdeps/localdeps.win.sh b/localdeps/localdeps.win.sh index 492c573..8779e2e 100755 --- a/localdeps/localdeps.win.sh +++ b/localdeps/localdeps.win.sh @@ -200,15 +200,16 @@ normalize_path() { list_deps() { local path + local path0 local inc "${NTLDD}" "$1" \ | grep ' => ' \ | sed -e 's|\\|/|g' -e 's|.* => ||' -e 's| (0.*||' \ | while read path; do - path=$(echo $path |sed -e 's|/|\\|g') - inc=$(check_includedep "${path}") + path0=$(echo $path |sed -e 's|/|\\|g') + inc=$(check_includedep "${path0}") if [ "x${inc}" != "x" ]; then - echo "${inc}" + echo "${path}" fi done } -- GitLab From 4d841cc413acd284d422ef6ce23e3e6fb83b5d60 Mon Sep 17 00:00:00 2001 From: IOhannes m zmoelnig Date: Fri, 28 May 2021 10:56:03 +0200 Subject: [PATCH 120/157] localdeps.macos.sh: use wildcards for includes/excludes --- localdeps/localdeps.macos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localdeps/localdeps.macos.sh b/localdeps/localdeps.macos.sh index 63ae729..4f14c4f 100755 --- a/localdeps/localdeps.macos.sh +++ b/localdeps/localdeps.macos.sh @@ -6,8 +6,8 @@ ## usage: $0 [...] #default exclude/include paths -exclude_paths="/usr/lib/:/System/Library/Frameworks" -include_paths= +exclude_paths="/usr/lib/*:/System/Library/Frameworks/*" +include_paths="/*" # UTILITIES if [ -e "${0%/*}/localdeps.utilities.source" ]; then -- GitLab From 73cd443df6552f33a4ad0b71e3d4e2ccea3cbdd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 10:58:58 +0200 Subject: [PATCH 121/157] localdeps.linux.sh: drop dependency on 'awk' it's not used --- localdeps/localdeps.linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh index 4f568b1..68dc2db 100755 --- a/localdeps/localdeps.linux.sh +++ b/localdeps/localdeps.linux.sh @@ -244,7 +244,7 @@ install_deps () { # Check dependencies -cmdlist="awk grep ldd patchelf" +cmdlist="grep ldd patchelf" for cmd in $cmdlist; do if ! which "${cmd}" > /dev/null; then error "Could not find ${cmd}. Is it installed?" -- GitLab From 1e8971dd04a039eb9abb2d5f061c949c8401bc3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 11:07:03 +0200 Subject: [PATCH 122/157] localdeps: check_binaries helper --- localdeps/localdeps.linux.sh | 8 +------- localdeps/localdeps.macos.sh | 13 ++++--------- localdeps/localdeps.utilities.source | 10 ++++++++++ localdeps/localdeps.win.sh | 5 ++--- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh index 68dc2db..dc553bc 100755 --- a/localdeps/localdeps.linux.sh +++ b/localdeps/localdeps.linux.sh @@ -244,13 +244,7 @@ install_deps () { # Check dependencies -cmdlist="grep ldd patchelf" -for cmd in $cmdlist; do - if ! which "${cmd}" > /dev/null; then - error "Could not find ${cmd}. Is it installed?" - exit 1 - fi -done +check_binaries grep ldd patchelf for f in "$@"; do # Check if we can read from given file diff --git a/localdeps/localdeps.macos.sh b/localdeps/localdeps.macos.sh index 4f14c4f..233c049 100755 --- a/localdeps/localdeps.macos.sh +++ b/localdeps/localdeps.macos.sh @@ -175,10 +175,6 @@ fi #@END_UTILITIES@ fi -if [ "x${otool}" = "x" ]; then - otool="otool -L" -fi - basename () { local x=${1##*/} if [ "x$x" = "x" ]; then @@ -199,7 +195,7 @@ dirname () { list_deps() { local path local inc - ${otool} "$1" \ + ${OTOOL} "$1" \ | grep -v ":$" \ | grep compatibility \ | awk '{print $1}' \ @@ -247,12 +243,11 @@ install_deps () { done } -if [ "x${otool}" = "x" ]; then - error "no 'otool' binary found" - exit 0 +if [ "x${OTOOL}" = "x" ]; then + check_binaries otool + OTOOL="otool -L" fi - for f in "$@"; do if [ -e "${f}" ]; then error diff --git a/localdeps/localdeps.utilities.source b/localdeps/localdeps.utilities.source index 7dabe59..7d6cd25 100644 --- a/localdeps/localdeps.utilities.source +++ b/localdeps/localdeps.utilities.source @@ -10,6 +10,16 @@ substitute() { sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" } +check_binaries() { + local cmd + for cmd in "$@"; do + if ! which "${cmd}" > /dev/null; then + error "Could not find '${cmd}'. Is it installed?" + exit 0 + fi + done +} + normalize_path() { # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" diff --git a/localdeps/localdeps.win.sh b/localdeps/localdeps.win.sh index 8779e2e..d43a5bb 100755 --- a/localdeps/localdeps.win.sh +++ b/localdeps/localdeps.win.sh @@ -186,7 +186,6 @@ fi #@END_UTILITIES@ fi -NTLDD=$(which ntldd 2>/dev/null) CYGPATH=$(which cygpath 2>/dev/null) if [ -z "${CYGPATH}" ]; then CYGPATH=echo @@ -270,8 +269,8 @@ install_deps () { } if [ "x${NTLDD}" = "x" ]; then - error "no 'ntldd' binary found" - exit 0 + check_binaries ntldd + NTLDD="ntldd" fi for f in "$@"; do -- GitLab From bc4305afa9cef9a208d1e200d76c3e1d04ab8548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 11:09:42 +0200 Subject: [PATCH 123/157] update scripts to new utilities --- localdeps/localdeps.linux.sh | 10 ++++++++++ localdeps/localdeps.macos.sh | 10 ++++++++++ localdeps/localdeps.win.sh | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh index dc553bc..327ba44 100755 --- a/localdeps/localdeps.linux.sh +++ b/localdeps/localdeps.linux.sh @@ -38,6 +38,16 @@ substitute() { sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" } +check_binaries() { + local cmd + for cmd in "$@"; do + if ! which "${cmd}" > /dev/null; then + error "Could not find '${cmd}'. Is it installed?" + exit 0 + fi + done +} + normalize_path() { # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" diff --git a/localdeps/localdeps.macos.sh b/localdeps/localdeps.macos.sh index 233c049..381c337 100755 --- a/localdeps/localdeps.macos.sh +++ b/localdeps/localdeps.macos.sh @@ -29,6 +29,16 @@ substitute() { sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" } +check_binaries() { + local cmd + for cmd in "$@"; do + if ! which "${cmd}" > /dev/null; then + error "Could not find '${cmd}'. Is it installed?" + exit 0 + fi + done +} + normalize_path() { # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" diff --git a/localdeps/localdeps.win.sh b/localdeps/localdeps.win.sh index d43a5bb..d76ae83 100755 --- a/localdeps/localdeps.win.sh +++ b/localdeps/localdeps.win.sh @@ -40,6 +40,16 @@ substitute() { sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" } +check_binaries() { + local cmd + for cmd in "$@"; do + if ! which "${cmd}" > /dev/null; then + error "Could not find '${cmd}'. Is it installed?" + exit 0 + fi + done +} + normalize_path() { # normalize a path specification, e.g. on Windows turn C:\Foo\Bar\ into /c/foo/bar/" -- GitLab From 3a6b2da19d966d71bf4fb230af45959b219a141b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 11:12:36 +0200 Subject: [PATCH 124/157] switch 'master' to 'main' --- README.md | 6 +++--- no-build/README.md | 6 +++--- no-build/gitlab-iem.yml | 2 +- pd-lib-builder/README.md | 8 ++++---- pd-lib-builder/gitlab-iem.yml | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c03f384..49eb9d4 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Pd package download server). ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml ~~~ If your project doesn't need to build anything and has on `Makefile` (e.g. because @@ -69,7 +69,7 @@ include: ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/no-build/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/no-build/gitlab-iem.yml ~~~ - Push the new file to the repository: @@ -130,7 +130,7 @@ you would do something like this: ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml variables: IEM_CI_PROJECT_NAME: superlib diff --git a/no-build/README.md b/no-build/README.md index 89e3643..54cdcc0 100644 --- a/no-build/README.md +++ b/no-build/README.md @@ -41,7 +41,7 @@ Pd package download server). ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/no-build/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/no-build/gitlab-iem.yml ~~~ - Push the new file to the repository: @@ -101,7 +101,7 @@ E.g. if your repository has the name `pd-superlib`, but the library is really ca ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/no-build/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/no-build/gitlab-iem.yml variables: IEM_CI_PROJECT_NAME: superlib @@ -116,7 +116,7 @@ To package such a library, you need to specify the target (installation) directo ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/no-build/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/no-build/gitlab-iem.yml variables: IEM_CI_PROJECT_INSTALLDIR: foo/bar diff --git a/no-build/gitlab-iem.yml b/no-build/gitlab-iem.yml index d9c22bb..a516187 100644 --- a/no-build/gitlab-iem.yml +++ b/no-build/gitlab-iem.yml @@ -17,7 +17,7 @@ include: # define job-templates: - - https://git.iem.at/pd/iem-ci/raw/master/no-build/iem-ci.yml + - https://git.iem.at/pd/iem-ci/raw/main/no-build/iem-ci.yml ####################################################################### diff --git a/pd-lib-builder/README.md b/pd-lib-builder/README.md index 727d7fb..9e60e23 100644 --- a/pd-lib-builder/README.md +++ b/pd-lib-builder/README.md @@ -107,7 +107,7 @@ yet. ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml ~~~ - Push the new file to the repository: @@ -222,7 +222,7 @@ For example, to force the Pd-version to `0.49-0` use something like this: ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml variables: PDVERSION: 0.49-0 @@ -241,7 +241,7 @@ If this is not the case, you can specify an alternate directory using the `SRCDI ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml variables: SRCDIR: pd/ @@ -260,7 +260,7 @@ E.g. if your repository has the name `pd-superlib`, but the library is really ca ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml variables: IEM_CI_PROJECT_NAME: superlib diff --git a/pd-lib-builder/gitlab-iem.yml b/pd-lib-builder/gitlab-iem.yml index fc55ec5..af16f81 100644 --- a/pd-lib-builder/gitlab-iem.yml +++ b/pd-lib-builder/gitlab-iem.yml @@ -17,6 +17,6 @@ include: # define job-templates: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/iem-ci.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/iem-ci.yml # turn templates into real jobs: - - https://git.iem.at/pd/iem-ci/raw/master/pd-lib-builder/pipeline-jobs.yml + - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/pipeline-jobs.yml -- GitLab From 4b7cc3344b5a725ad7a6f17c289b48f1ca997d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Fri, 28 May 2021 11:54:11 +0200 Subject: [PATCH 125/157] drop localdeps.*.sh scripts in pd-lib-builder/ in favour of localdeps/ --- pd-lib-builder/localdeps.linux.sh | 99 ------------ pd-lib-builder/localdeps.macos.sh | 80 ---------- pd-lib-builder/localdeps.win.sh | 244 ------------------------------ 3 files changed, 423 deletions(-) delete mode 100755 pd-lib-builder/localdeps.linux.sh delete mode 100755 pd-lib-builder/localdeps.macos.sh delete mode 100755 pd-lib-builder/localdeps.win.sh diff --git a/pd-lib-builder/localdeps.linux.sh b/pd-lib-builder/localdeps.linux.sh deleted file mode 100755 index 7f7eab6..0000000 --- a/pd-lib-builder/localdeps.linux.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/sh -# -# creates local copies of all dependencies (dynamic libraries) -# and sets RUNPATH to $ORIGIN on each so they will find -# each other. -# -# usage: $0 -usage() { - echo "Usage: ${0} [ ... ]" - echo " copies the local dependencies of all given binaries besides them" -} - -if [ $# -lt 1 ]; then - usage - exit 1 -fi - -# List of libraries that we do not include into our packaging -# becaue we think they will be installed on any system -ld_exclude_list="libc\.so\.* -libarmmem.*\.so.* -libdl\.so.* -libglib-.*\.so.* -libgomp\.so.* -libgthread.*\.so.* -libm\.so.* -libpthread.*\.so.* -libstdc++\.so.* -libgcc_s\.so.* -libpcre\.so.*" - -error() { - echo "$@" 1>&2 -} - -# Check dependencies -cmdlist="awk grep ldd patchelf" -for cmd in $cmdlist; do - if ! which "${cmd}" > /dev/null; then - error "Could not find ${cmd}. Is it installed?" - exit 1 - fi -done - -library_in_exclude_list() { - # arg1: library name - # returns 0 if arg1 is found in exclude list, otherwise 1 - local libexname="$1" - skip=1 - set -f - for expat in $(echo "${ld_exclude_list}"); do - if echo "$(basename $libexname)" | grep "${expat}" > /dev/null; then - skip=0 - break - fi - done - set +f - return $skip -} - -make_local_copy_and_set_rpath() { - # make a local copy of all linked libraries of given binary - # and set RUNPATH to $ORIGIN (exclude "standard" libraries) - # arg1: binary to check - local outdir - outdir=$(dirname "$1") - if [ ! -d "${outdir}" ]; then - outdir=. - fi - ldd "$1" | grep ' => ' | while read _ _ libpath _; do - libname=$(basename "${libpath}") - outfile="${outdir}/${libname}" - if library_in_exclude_list "${libname}"; then - continue - fi - if [ ! -e "${libpath}" ]; then - error "DEP: ${INSTALLDEPS_INDENT} WARNING: could not make copy of '${libpath}'. Not found" - continue - fi - if [ -e "${outfile}" ]; then - error "DEP: ${INSTALLDEPS_INDENT} ${libpath} SKIPPED" - else - error "DEP: ${INSTALLDEPS_INDENT} ${libpath} -> ${outdir}/" - cp "${libpath}" "${outfile}" - patchelf --set-rpath \$ORIGIN "${outfile}" - fi - done - patchelf --set-rpath \$ORIGIN "${1}" -} - - -for binary_file in "$@"; do - # Check if we can read from given file - if ! ldd "${binary_file}" > /dev/null 2>&1; then - error "Skipping '${binary_file}'. Is it a binary file?" - continue - fi - make_local_copy_and_set_rpath $binary_file -done diff --git a/pd-lib-builder/localdeps.macos.sh b/pd-lib-builder/localdeps.macos.sh deleted file mode 100755 index b53e96b..0000000 --- a/pd-lib-builder/localdeps.macos.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -## puts dependencies besides the binary -# LATER: put dependencies into a separate folder - -## usage: $0 [...] - -error() { - echo "$@" 1>&2 -} -basename () { - local x=${1##*/} - if [ "x$x" = "x" ]; then - echo $1 - else - echo $x - fi -} -dirname () { - local x=${1%/*} - if [ "x$x" = "x" ]; then - echo . - else - echo $x - fi -} - -if [ "x${otool}" = "x" ]; then - otool="otool -L" -fi -if [ "x${otool}" = "x" ]; then - error "no 'otool' binary found" - exit 0 -fi - -list_deps() { - $otool "$1" \ - | grep -v ":$" \ - | grep compatibility \ - | awk '{print $1}' \ - | egrep '^/' \ - | egrep -v '^/usr/lib/' \ - | egrep -v '^/System/Library/Frameworks' -} - -install_deps () { -error "DEP: ${INSTALLDEPS_INDENT}$1" -outdir=$2 -if [ "x${outdir}" = "x" ]; then - outdir=$(dirname "$1") -fi -if [ ! -d "${outdir}" ]; then - outdir=. -fi - -list_deps "$1" | while read dep; do - infile=$(basename "$1") - depfile=$(basename "${dep}") - install_name_tool -change "${dep}" "@loader_path/${depfile}" "$1" - - if [ -e "${outdir}/${depfile}" ]; then - error "DEP: ${INSTALLDEPS_INDENT} ${dep} SKIPPED" - else - error "DEP: ${INSTALLDEPS_INDENT} ${dep} -> ${outdir}" - cp "${dep}" "${outdir}" - chmod u+w "${outdir}/${depfile}" - install_name_tool -id "@loader_path/${depfile}" "${outdir}/${depfile}" - # recursively call ourselves, to resolve higher-order dependencies - INSTALLDEPS_INDENT="${INSTALLDEPS_INDENT} " $0 "${outdir}/${depfile}" - fi -done - -} - - -for f in "$@"; do - if [ -e "${f}" ]; then - install_deps "${f}" - fi -done diff --git a/pd-lib-builder/localdeps.win.sh b/pd-lib-builder/localdeps.win.sh deleted file mode 100755 index 9f979c3..0000000 --- a/pd-lib-builder/localdeps.win.sh +++ /dev/null @@ -1,244 +0,0 @@ -#!/bin/bash - -## puts dependencies besides the binary - -## usage: $0 [...] - - -########################################### -# WARNING -# -# this uses an ugly hack to allow side-by-side installation of 32bit and 64bit -# dependencies: -# embedded dependencies are renamed from "libfoo.dll" to "libfoo.w32" resp. -# "libfoo.w64", and the files are modified (using 'sed') to reflect this -# renaming. -# this is somewhat brittle and likely to break! - -error() { - echo "$@" 1>&2 -} - -substitute() { - # substitutes literal strings - # usage: echo foo | substitute foo bar g - sed "s/$(echo $1 | sed 's:[]\[^$.*/&]:\\&:g')/$(echo $2 | sed 's:[]\[^$.*/&]:\\&:g')/$3" -} - - -verbose=0 -exclude_paths="/c/msys64/usr/binx/:/usr/binx" -include_paths="/c/msys64/mingw64/bin/:/c/msys64/usr/bin/:/mingw64/bin/" -include_paths= -exclude_paths= - -NTLDD=$(which ntldd 2>/dev/null) -CYGPATH=$(which cygpath 2>/dev/null) -if [ -z "${CYGPATH}" ]; then - CYGPATH=echo -fi - -normalize_path() { - # convert to unix-format (C:\foo\bar\ --> /c/foo/bar/) - # and lower-case everything (because on microsoft-fs, paths are case-insensitive) - ${CYGPATH} "$1" | tr "[A-Z]" "[a-z]" -} - -list_dirs() { - local _ifs="$IFS" - local w - IFS=":" - for w in $@; do - IFS=${_ifs} - echo "$w" - done - IFS="${_ifs}" -} - -check_includedep() { - local path=$(normalize_path $1) - local p - local result=0 - # exclude non-existing files - if [ ! -e "${path}" ]; then - return 0 - fi - - list_dirs "${exclude_paths}" | while read p; do - # exclude files in paths that are excluded - if [ "x${path}" != "x${path#$p}" ]; then - return 1 - fi - done || return 1 - - if echo "${path}" | grep mingw >/dev/null; then - # by default, we always include "mingw" stuff - echo "${path}" - return 0 - fi - - # include only files in paths that are included - list_dirs "${include_paths}" |while read p; do - if [ "x${path}" != "x${path#$p}" ]; then - echo "${path}" - return 0 - fi - done && return 0 - - return 1 -} - - -list_deps() { - local path - local inc - "${NTLDD}" "$1" \ - | grep ' => ' \ - | sed -e 's|\\|/|g' -e 's|.* => ||' -e 's| (0.*||' \ - | while read path; do - path=$(echo $path |sed -e 's|/|\\|g') - inc=$(check_includedep "${path}") - if [ "x${inc}" != "x" ]; then - echo "${inc}" - fi - done -} - -file2arch() { - if file "$1" | grep -w "PE32+" >/dev/null; then - echo "w64" - return - fi - if file "$1" | grep -w "PE32" >/dev/null; then - echo "w32" - return - fi -} - -install_deps () { -local outdir -local idepfile -local odepfile -local archext -local dep -error "DEP: ${INSTALLDEPS_INDENT}'$1' '$2'" - -outdir=$2 -if [ "x${outdir}" = "x" ]; then - outdir=${1%/*} -fi -if [ ! -d "${outdir}" ]; then - outdir=. -fi - -list_deps "$1" | while read dep; do - idepfile=$(basename "${dep}") - odepfile=${idepfile} - archext=$(file2arch "${dep}") - if [ "x${archext}" != "x" ]; then - odepfile=$(echo ${idepfile} | sed -e "s|\.dll|.${archext}|") - fi - if [ "x${idepfile}" = "x${odepfile}" ]; then - archext="" - fi - if [ -e "${outdir}/${odepfile}" ]; then - error "DEP: ${INSTALLDEPS_INDENT} ${dep} SKIPPED" - else - error "DEP: ${INSTALLDEPS_INDENT} ${dep} -> ${outdir}/${odepfile}" - cp "${dep}" "${outdir}/${odepfile}" - chmod a-x "${outdir}/${odepfile}" - fi - - if [ "x${archext}" != "x" ]; then - sed -b -e "s|${idepfile}|${odepfile}|g" -i "${outdir}/${odepfile}" "${dep}" "$1" - fi - #recursively resolve dependencies - install_deps "${outdir}/${odepfile}" -done -} - -usage() { - cat >/dev/stderr <] [-X ] [ ...] - recursively includes all dependencies of the given binaries - - -I : adds one include path entry - -X : adds one exclude path entry - -v: raise verbosity - -q: lower verbosity - - dependencies are renamed from .dll to .w64 (resp .w32) - -EXCLUDING/INCLUDING -------------------- -When traversing the runtime dependencies of a binary, dependencies are filtered -out based on their location (this is mainly to exclude system libraries that -can be very large and which are to be found on the target systems anyhow). - -Only dependencies (and sub-dependencies) that live in a path that -do NOT match any of the EXCLUDEPATHs and match at least one of the INCLUDEPATHs -are considered for inclusion (in this order. a dependency that matches both -EXCLUDEPATHs and INCLUDEPATHs is dropped). - -Matching is done on the start of the paths, so a pattern '/foo/bar' matches the -dependencies '/foo/bar.dll', '/foo/bartender.dll' and '/foo/bar/pizza.dll', -whereas a pattern '/foo/bar/' only matches '/foo/bar/pizza.dll'. - -You can remove an element from the INCLUDEPATHs by excluding it (exactly), -and vice versa. - -EOF - exit 1 -} - -while getopts "hqvI:X:" arg; do - case $arg in - h) - usage - ;; - I) - p=$(normalize_path "${OPTARG}") - if [ "x${p}" != "x" ]; then - include_paths="${p}:${include_paths}" - fi - exclude_paths=$(echo :${exclude_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') - ;; - X) - p=$(normalize_path "${OPTARG}") - if [ "x${p}" != "x" ]; then - exclude_paths="${p}:${exclude_paths}" - fi - include_paths=$(echo :${include_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') - ;; - q) - verbose=$((verbose-1)) - ;; - v) - verbose=$((verbose+1)) - ;; - *) - usage - ;; - esac -done -shift $((OPTIND-1)) -include_paths=${include_paths%:} -exclude_paths=${exclude_paths%:} - -if [ ${verbose} -gt 0 ]; then - error "EXCLUDEPATHs: ${exclude_paths}" - error "INCLUDEPATHs: ${include_paths}" - error ' *mingw*' -fi - -if [ "x${NTLDD}" = "x" ]; then - error "no 'ntldd' binary found" - exit 0 -fi - -for f in "$@"; do - if [ -e "${f}" ]; then - error - install_deps "${f}" - fi -done -- GitLab From 12b36191b2596344132f09aecc7bfbb11e2088a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sun, 30 May 2021 23:38:56 +0200 Subject: [PATCH 126/157] localdeps: relax include-paths on windows & linux --- localdeps/localdeps.linux.sh | 2 +- localdeps/localdeps.win.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh index 327ba44..7152688 100755 --- a/localdeps/localdeps.linux.sh +++ b/localdeps/localdeps.linux.sh @@ -16,7 +16,7 @@ exclude_paths= #default exclude/include paths exclude_paths="*/libc.so.*:*/libarmmem.*.so.*:*/libdl.so.*:*/libglib-.*.so.*:*/libgomp.so.*:*/libgthread.*.so.*:*/libm.so.*:*/libpthread.*.so.*:*/libpthread.so.*:*/libstdc++.so.*:*/libgcc_s.so.*:*/libpcre.so.*:*/libz.so.*" -include_paths="/lib/*:/usr/lib/*:/lib64/*:/usr/lib64/*" +include_paths="/*" # UTILITIES if [ -e "${0%/*}/localdeps.utilities.source" ]; then diff --git a/localdeps/localdeps.win.sh b/localdeps/localdeps.win.sh index d76ae83..a279b1c 100755 --- a/localdeps/localdeps.win.sh +++ b/localdeps/localdeps.win.sh @@ -18,7 +18,7 @@ #default exclude/include paths exclude_paths="" -include_paths="*mingw*" +include_paths="*mingw*:*/msys/*" # UTILITIES if [ -e "${0%/*}/localdeps.utilities.source" ]; then -- GitLab From 8aab01c1fba12cf5e43dbc8e2e20206c6b49f02b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sun, 30 May 2021 23:42:47 +0200 Subject: [PATCH 127/157] clarify that EXCLUDES take precedence over INCLUDES --- localdeps/localdeps.utilities.source | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/localdeps/localdeps.utilities.source b/localdeps/localdeps.utilities.source index 7d6cd25..2b4d3fe 100644 --- a/localdeps/localdeps.utilities.source +++ b/localdeps/localdeps.utilities.source @@ -117,6 +117,11 @@ Matching is done with globbing patterns, so a pattern '/foo/bar*' matches the dependencies '/foo/bar.dll', '/foo/bartender.dll' and '/foo/bar/pizza.dll', whereas a pattern '/foo/bar/*' only matches '/foo/bar/pizza.dll'. +Only paths that are not excluded, will be considered for inclusion. +Thus if there are both an exclude pattern '/usr/*' and an include pattern +'/usr/lib/*', then a path '/usr/lib/libfoo.so' will be omitted (and the include +pattern is practically useless). + You can remove an element from the INCLUDEPATHs by excluding it (exactly), and vice versa. -- GitLab From e5fd5047e0d90857eedd27b6de03c4dbe196f2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sun, 30 May 2021 23:45:14 +0200 Subject: [PATCH 128/157] update scripts --- localdeps/localdeps.linux.sh | 5 +++++ localdeps/localdeps.macos.sh | 5 +++++ localdeps/localdeps.win.sh | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh index 7152688..24852dd 100755 --- a/localdeps/localdeps.linux.sh +++ b/localdeps/localdeps.linux.sh @@ -145,6 +145,11 @@ Matching is done with globbing patterns, so a pattern '/foo/bar*' matches the dependencies '/foo/bar.dll', '/foo/bartender.dll' and '/foo/bar/pizza.dll', whereas a pattern '/foo/bar/*' only matches '/foo/bar/pizza.dll'. +Only paths that are not excluded, will be considered for inclusion. +Thus if there are both an exclude pattern '/usr/*' and an include pattern +'/usr/lib/*', then a path '/usr/lib/libfoo.so' will be omitted (and the include +pattern is practically useless). + You can remove an element from the INCLUDEPATHs by excluding it (exactly), and vice versa. diff --git a/localdeps/localdeps.macos.sh b/localdeps/localdeps.macos.sh index 381c337..205b1f4 100755 --- a/localdeps/localdeps.macos.sh +++ b/localdeps/localdeps.macos.sh @@ -136,6 +136,11 @@ Matching is done with globbing patterns, so a pattern '/foo/bar*' matches the dependencies '/foo/bar.dll', '/foo/bartender.dll' and '/foo/bar/pizza.dll', whereas a pattern '/foo/bar/*' only matches '/foo/bar/pizza.dll'. +Only paths that are not excluded, will be considered for inclusion. +Thus if there are both an exclude pattern '/usr/*' and an include pattern +'/usr/lib/*', then a path '/usr/lib/libfoo.so' will be omitted (and the include +pattern is practically useless). + You can remove an element from the INCLUDEPATHs by excluding it (exactly), and vice versa. diff --git a/localdeps/localdeps.win.sh b/localdeps/localdeps.win.sh index a279b1c..b4723c8 100755 --- a/localdeps/localdeps.win.sh +++ b/localdeps/localdeps.win.sh @@ -147,6 +147,11 @@ Matching is done with globbing patterns, so a pattern '/foo/bar*' matches the dependencies '/foo/bar.dll', '/foo/bartender.dll' and '/foo/bar/pizza.dll', whereas a pattern '/foo/bar/*' only matches '/foo/bar/pizza.dll'. +Only paths that are not excluded, will be considered for inclusion. +Thus if there are both an exclude pattern '/usr/*' and an include pattern +'/usr/lib/*', then a path '/usr/lib/libfoo.so' will be omitted (and the include +pattern is practically useless). + You can remove an element from the INCLUDEPATHs by excluding it (exactly), and vice versa. -- GitLab From f79b44233ee1319a2848eba49c3d0400e6111261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 9 Jun 2021 16:47:40 +0200 Subject: [PATCH 129/157] Exit with 127 if helpers cannot be found Closes: https://git.iem.at/pd/iem-ci/-/issues/4 --- localdeps/localdeps.utilities.source | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localdeps/localdeps.utilities.source b/localdeps/localdeps.utilities.source index 2b4d3fe..2859834 100644 --- a/localdeps/localdeps.utilities.source +++ b/localdeps/localdeps.utilities.source @@ -15,7 +15,7 @@ check_binaries() { for cmd in "$@"; do if ! which "${cmd}" > /dev/null; then error "Could not find '${cmd}'. Is it installed?" - exit 0 + exit 127 fi done } -- GitLab From 6862346eea6c44eb876841007b35624fb82465c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 9 Jun 2021 16:48:37 +0200 Subject: [PATCH 130/157] update scripts --- localdeps/localdeps.linux.sh | 2 +- localdeps/localdeps.macos.sh | 2 +- localdeps/localdeps.win.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh index 24852dd..7aec1f6 100755 --- a/localdeps/localdeps.linux.sh +++ b/localdeps/localdeps.linux.sh @@ -43,7 +43,7 @@ check_binaries() { for cmd in "$@"; do if ! which "${cmd}" > /dev/null; then error "Could not find '${cmd}'. Is it installed?" - exit 0 + exit 127 fi done } diff --git a/localdeps/localdeps.macos.sh b/localdeps/localdeps.macos.sh index 205b1f4..b332947 100755 --- a/localdeps/localdeps.macos.sh +++ b/localdeps/localdeps.macos.sh @@ -34,7 +34,7 @@ check_binaries() { for cmd in "$@"; do if ! which "${cmd}" > /dev/null; then error "Could not find '${cmd}'. Is it installed?" - exit 0 + exit 127 fi done } diff --git a/localdeps/localdeps.win.sh b/localdeps/localdeps.win.sh index b4723c8..f05a566 100755 --- a/localdeps/localdeps.win.sh +++ b/localdeps/localdeps.win.sh @@ -45,7 +45,7 @@ check_binaries() { for cmd in "$@"; do if ! which "${cmd}" > /dev/null; then error "Could not find '${cmd}'. Is it installed?" - exit 0 + exit 127 fi done } -- GitLab From cc4610079e582ba1a8d623b6bc2e459b7a4160bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 21 Jun 2021 13:24:33 +0200 Subject: [PATCH 131/157] enable SAST checks for pd-lib-builder projects --- pd-lib-builder/gitlab-iem.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pd-lib-builder/gitlab-iem.yml b/pd-lib-builder/gitlab-iem.yml index af16f81..c599e3f 100644 --- a/pd-lib-builder/gitlab-iem.yml +++ b/pd-lib-builder/gitlab-iem.yml @@ -20,3 +20,5 @@ include: - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/iem-ci.yml # turn templates into real jobs: - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/pipeline-jobs.yml + # and run some static code analysis + - template: Security/SAST.gitlab-ci.yml -- GitLab From f828f8ecd226c3867a366f4c0a2346acdb6aab52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Mon, 27 Sep 2021 12:48:41 +0200 Subject: [PATCH 132/157] describe PDVERSION --- pd-lib-builder/iem-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 434acf8..d4e5588 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -21,7 +21,9 @@ ####################################################################### variables: - PDVERSION: 0.51-3 + PDVERSION: + value: "0.51-3" + description: "The Pd-version against which to build" SRCDIR: . IEM_CI_TMPDIR: .git-ci/_build/ IEM_CI_PKGLIBDIR: "" -- GitLab From e1f6805c9858c03c62198948b40092828ec7e015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 12 Oct 2021 22:19:59 +0200 Subject: [PATCH 133/157] print compiler version --- pd-lib-builder/iem-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index d4e5588..a283dfc 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -42,6 +42,7 @@ variables: .script:make: &script_make # some info on the build-system - make --version || true + - ${CC:-cc} --version || true - ${CC:-cc} -dumpmachine || true - echo target architecture ${TARGETARCH} # do the actual build -- GitLab From 4a67762add5f5cbca85a56448a81b1a6af6c747b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 18 Jan 2022 17:23:54 +0100 Subject: [PATCH 134/157] simplified test whether PD is runnable (for tests) --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index a283dfc..b79c730 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -51,7 +51,7 @@ variables: # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi # if PD is defined and exists, check whether we can run it - - if test "x${NOCHECK}" = "x" && test -x "${PD}"; then "${PD}" -version 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "${PD} is not usable!"; fi + - if test "x${NOCHECK}" = "x" && test "x${PD}" != "x"; then "${PD}" -version 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "${PD} is not usable!"; fi # check if we can run compiled binaries (no use to run tests, if they are CPU-incompatible) - rm -rf "${IEM_CI_TMPDIR}" - mkdir -p "${IEM_CI_TMPDIR}" -- GitLab From daf3c46f51915ac5d90d8049a7ef4b914f9bbfcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 18 Jan 2022 17:44:39 +0100 Subject: [PATCH 135/157] pd-lib-builder: honor PDVERSION on linux (for the headers) --- pd-lib-builder/iem-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b79c730..5854a1c 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -132,6 +132,12 @@ variables: - if [ ! -e .git-ci/requirements.apt ]; then apt-get install -y --no-install-recommends ${TARGETDEBARCH:+cross}build-essential${TARGETDEBARCH:+-}${TARGETDEBARCH} puredata-dev${TARGETDEBARCH:+:}${TARGETDEBARCH} puredata-core${TARGETDEBARCH:+:}${TARGETDEBARCH}; fi - rm -rf "${IEM_CI_TMPDIR}" - export PD=/usr/bin/pd + # force Pd-version: this might break tests + - mkdir -p /usr/local/pd + - test -z "${PDVERSION}" || wget -q -O Pd.tgz http://msp.ucsd.edu/Software/pd-${PDVERSION}.src.tar.gz + - test -z "${PDVERSION}" || tar --extract --strip-components=1 --file=Pd.tgz --directory /usr/local/pd/ + - tmp_pdinc=$(find /usr/local/pd/ -name "m_pd.h" -exec dirname {} + -quit) + - test ! -d "${tmp_pdinc}" || export PDINCLUDEDIR="${tmp_pdinc}" .build:linux_amd64: &build_linux_amd64 extends: .build:linux -- GitLab From fdcb5e523cbccddb211a47499624d942f9dbeeeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 18 Jan 2022 17:56:48 +0100 Subject: [PATCH 136/157] use 'curl' rather than 'wget' to fetch Pd-sources no wget on the CI --- pd-lib-builder/iem-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 5854a1c..5dc5977 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -134,8 +134,7 @@ variables: - export PD=/usr/bin/pd # force Pd-version: this might break tests - mkdir -p /usr/local/pd - - test -z "${PDVERSION}" || wget -q -O Pd.tgz http://msp.ucsd.edu/Software/pd-${PDVERSION}.src.tar.gz - - test -z "${PDVERSION}" || tar --extract --strip-components=1 --file=Pd.tgz --directory /usr/local/pd/ + - test -z "${PDVERSION}" || curl -sL http://msp.ucsd.edu/Software/pd-${PDVERSION}.src.tar.gz | tar --extract --gzip --strip-components=1 --directory /usr/local/pd/ - tmp_pdinc=$(find /usr/local/pd/ -name "m_pd.h" -exec dirname {} + -quit) - test ! -d "${tmp_pdinc}" || export PDINCLUDEDIR="${tmp_pdinc}" -- GitLab From 832085cbc9230fd3049ec6c4a9edd7e06777a0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 18 Jan 2022 18:00:14 +0100 Subject: [PATCH 137/157] pd-lib-builder: install 'curl' --- pd-lib-builder/iem-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 5dc5977..460a07a 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -134,6 +134,7 @@ variables: - export PD=/usr/bin/pd # force Pd-version: this might break tests - mkdir -p /usr/local/pd + - apt-get install -y --no-install-recommends curl ca-certificates - test -z "${PDVERSION}" || curl -sL http://msp.ucsd.edu/Software/pd-${PDVERSION}.src.tar.gz | tar --extract --gzip --strip-components=1 --directory /usr/local/pd/ - tmp_pdinc=$(find /usr/local/pd/ -name "m_pd.h" -exec dirname {} + -quit) - test ! -d "${tmp_pdinc}" || export PDINCLUDEDIR="${tmp_pdinc}" -- GitLab From 57cfa7aa08645132125677ddb0af4ce2becb7707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 18 Jan 2022 21:44:54 +0100 Subject: [PATCH 138/157] newer versions of Pd-for-macOS come as a .dmg that is wrapped into a zip... --- pd-lib-builder/iem-ci.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 460a07a..25b5046 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -177,10 +177,27 @@ variables: before_script: - date - if [ -e .git-ci/requirements.brew ]; then brew bundle --no-upgrade --file=.git-ci/requirements.brew; fi - - wget -q -O Pd.tgz http://msp.ucsd.edu/Software/pd-${PDVERSION}.mac.tar.gz - rm -rf /Applications/Pd*.app/ - - tar xf Pd.tgz -C /Applications/ - - rm -f Pd.tgz + - rm -rf "${IEM_CI_TMPDIR}" + # get Pd tarball + - curl -sL http://msp.ucsd.edu/Software/pd-${PDVERSION}.mac.tar.gz | tar --extract --gzip --directory /Applications/ || mkdir -p "${IEM_CI_TMPDIR}" + # if there was no tarball, try to get a zip-file + - test ! -d "${IEM_CI_TMPDIR}" || wget -q -O "${IEM_CI_TMPDIR}"/Pd.zip http://msp.ucsd.edu/Software/pd-${PDVERSION}.macos.zip + - test ! -f "${IEM_CI_TMPDIR}"/Pd.zip || unzip "${IEM_CI_TMPDIR}"/Pd.zip -d "${IEM_CI_TMPDIR}" + # the ZIP-file contains a .dmg containing Pd + - | + for dmg in "${IEM_CI_TMPDIR}"/Pd*.dmg; do break; done + - pddisk="" + - test ! -f "${dmg}" || pddisk=$(hdiutil attach "${dmg}" 2>/dev/null | egrep "^/.*/Volumes/" | tail -1 | awk '{print $NF}') + - rm -rf "${dmg}" + - | + for app in "${pddisk}"/Pd*.app "${IEM_CI_TMPDIR}"/Pd*.app; do if test -d "${app}"; then cp -r "${app}" /Applications/; break; fi; done + - test -d "${pddisk}" || umount "${pddisk}" + - | + rm -rf "${IEM_CI_TMPDIR}" + dmg="" + pddisk="" + app="" - export PD=$(find /Applications/Pd*.app/Contents/Resources/bin/ type f -name pd -print -quit) # setup for code-signing (LATER move this into a separate stage; see 'SIGN CODE' below) - test -n "${pd_extension}" || case "${IEM_CI_MACOS_BUILDFAT}" in yes|1|true) pd_extension=d_fat ;; esac -- GitLab From 7ced3e0366233ac8d983c81d93f13a98f5a12a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sat, 22 Jan 2022 13:40:40 +0100 Subject: [PATCH 139/157] fix test whether we should unmount the pddisk (on macOS) --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 25b5046..bb43abf 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -192,7 +192,7 @@ variables: - rm -rf "${dmg}" - | for app in "${pddisk}"/Pd*.app "${IEM_CI_TMPDIR}"/Pd*.app; do if test -d "${app}"; then cp -r "${app}" /Applications/; break; fi; done - - test -d "${pddisk}" || umount "${pddisk}" + - test ! -d "${pddisk}" || umount "${pddisk}" - | rm -rf "${IEM_CI_TMPDIR}" dmg="" -- GitLab From 5ce19d09b2eaa64957fb08b96c0c7ceb8a359c37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 1 Feb 2022 14:30:46 +0100 Subject: [PATCH 140/157] pd-lib-builder: build on BigSur --- pd-lib-builder/iem-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index bb43abf..fd355d6 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -166,7 +166,7 @@ variables: .build:macos: &build_macos extends: .build:script tags: - - osx + - bigsur retry: max: 1 when: @@ -213,7 +213,7 @@ variables: NOTARIZE_TIMEOUT: 0 ignore_staple_errors: "true" tags: - - osx + - bigsur retry: max: 1 when: @@ -335,7 +335,7 @@ variables: extends: .artifacts stage: sign tags: - - osx + - bigsur variables: keychainpass: $CI_BUILD_TOKEN CODESIGNFLAGS: --timestamp --strict --force -- GitLab From fa1e5b06921d9ec5089eef9a668f583f53837e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 1 Feb 2022 15:21:40 +0100 Subject: [PATCH 141/157] autodetect supported architectures depending on Xcode-version --- pd-lib-builder/iem-ci.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index fd355d6..c363fa1 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -46,7 +46,7 @@ variables: - ${CC:-cc} -dumpmachine || true - echo target architecture ${TARGETARCH} # do the actual build - - make -C "${SRCDIR}" ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} + - make -C "${SRCDIR}" ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} ${archs:+archs="${archs}"} .script:make_check: &script_make_check # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi @@ -201,6 +201,35 @@ variables: - export PD=$(find /Applications/Pd*.app/Contents/Resources/bin/ type f -name pd -print -quit) # setup for code-signing (LATER move this into a separate stage; see 'SIGN CODE' below) - test -n "${pd_extension}" || case "${IEM_CI_MACOS_BUILDFAT}" in yes|1|true) pd_extension=d_fat ;; esac + - | + if test "x${pd_extension}" = "xd_fat" && test "x${archs}" = "x"; then + # detect macOS SDK from Xcode toolchain version & deduce archs + XCODE_VERSION=$(pkgutil --pkg-info=com.apple.pkg.CLTools_Executables | grep "^version" | awk '{print $2}') + if [ "$XCODE_VER" = "" ] ; then + # no CLTools, try xcodebuild + XCODE_VERSION=$(xcodebuild -version | grep "^Xcode" | awk '{print $2}') + fi + XCODE_VERSION_MAJOR=$((${XCODE_VERSION%%.*})) + if [ $XCODE_VERSION_MAJOR -gt 11 ] ; then + # Xcode 12+: 11.0+ + archs="x86_64 arm64" + elif [ $XCODE_VERSION_MAJOR -gt 9 ] ; then + # Xcode 10 - 11: 10.14 - 10.15 + archs="x86_64" + echo "warning: Xcode version $XCODE_VERSION only builds ${archs}" 1>&2 + elif [ $XCODE_VERSION_MAJOR -gt 3 ] ; then + # Xcode 4 - 9: 10.7 - 10.13 + archs="i386 x86_64" + elif [ $XCODE_VERSION_MAJOR = 3 ] ; then + # Xcode 3: 10.6 + archs="ppc i386 x86_64" + else + archs="i386 x86_64" + echo "warning: unknown or unsupported Xcode version $XCODE_VERSION, trying ${archs}" 1>&2 + fi + fi + echo "detected Xcode-$XCODE_VERSION builds ${archs}" + after_script: - *script_fetch_localdeps - if [ -x .git-ci/localdeps.macos.sh ]; then find "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" -type f "(" -name "*.${pd_extension:-pd_darwin}" -o -name "*.so" ")" -exec .git-ci/localdeps.macos.sh {} +; fi -- GitLab From 59c1b57af9f0a463250034aceee13ec3a94cf057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 1 Feb 2022 15:30:59 +0100 Subject: [PATCH 142/157] Fix typo: pd-lib-builder wants an 'arch' variable rather than 'archs' --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index c363fa1..6d3db5f 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -44,9 +44,9 @@ variables: - make --version || true - ${CC:-cc} --version || true - ${CC:-cc} -dumpmachine || true - - echo target architecture ${TARGETARCH} + - echo target architecture ${TARGETARCH} - ${arch} # do the actual build - - make -C "${SRCDIR}" ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} ${archs:+archs="${archs}"} + - make -C "${SRCDIR}" ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} ${archs:+arch="${archs}"} .script:make_check: &script_make_check # check if there's a 'make check' target (no use to run it, if it is not there) - if [ "x${NOCHECK}" = "x" ]; then make -C "${SRCDIR}" check -n ${pd_extension:+extension=}${pd_extension} ${TARGETARCH:+PLATFORM=}${TARGETARCH} >/dev/null 2>&1 || NOCHECK=1; test "x${NOCHECK}" != "x1" || echo "no 'check' target"; fi -- GitLab From a0317bf2c4bf97f2f41c851b70cafbea7193c301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 10:53:00 +0100 Subject: [PATCH 143/157] document variables for manual pipelines --- pd-lib-builder/iem-ci.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 6d3db5f..11f9de2 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -24,10 +24,19 @@ variables: PDVERSION: value: "0.51-3" description: "The Pd-version against which to build" - SRCDIR: . + SRCDIR: + value: "." + description: "relative path where the Makefile lives" + IEM_CI_PKGLIBDIR: + value: "override the library component for the installation path (if it differs from the library name)" + description: "" + IEM_CI_PROJECT_NAME: + value: ${CI_PROJECT_NAME} + description: "override the library name (as used in installation directories and deken packages) if it is not the same as the repository name" + IEM_CI_MACOS_BUILDFAT: + value: "0" + description: "set to '1' if you would like to create universal binaries on macOS" IEM_CI_TMPDIR: .git-ci/_build/ - IEM_CI_PKGLIBDIR: "" - IEM_CI_PROJECT_NAME: ${CI_PROJECT_NAME} -- GitLab From 2a0ff4f7e2203b0f563d4640adf3d7f5b952929e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 10:58:26 +0100 Subject: [PATCH 144/157] job template that only runs if the job is listed in IEM_CI_JOBS --- pd-lib-builder/iem-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 11f9de2..139a7c3 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -36,6 +36,9 @@ variables: IEM_CI_MACOS_BUILDFAT: value: "0" description: "set to '1' if you would like to create universal binaries on macOS" + IEM_CI_JOBS: + value: "" + description: "space-separated list of jobs to run (leave empty to select ALL jobs)" IEM_CI_TMPDIR: .git-ci/_build/ @@ -107,6 +110,15 @@ variables: paths: - "${IEM_CI_PKGLIBDIR:-${IEM_CI_PROJECT_NAME}}" +.run-selected: + rules: + # run if IEM_CI_JOBS list is empty + - if: $IEM_CI_JOBS == "" + # run if CI_JOB_NAME is in the IEM_CI_JOBS list + - if: $CI_JOB_NAME =~ $IEM_CI_JOBS + # otherwise, don't run + - when: never + .snapshot: # except: # - tags -- GitLab From 2d9a62a3f3bb51713d1cf4e5e4efeb7586270e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 10:58:50 +0100 Subject: [PATCH 145/157] extend jobs to only run if selected --- pd-lib-builder/iem-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 139a7c3..da127b6 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -131,7 +131,9 @@ variables: .build:script: - extends: .artifacts + extends: + - .artifacts + - .run-selected stage: build script: - *script_make @@ -259,6 +261,8 @@ variables: .notarize:macos: # notarize binaries in ${IEM_CI_PKGLIBDIR} (fallback to ${IEM_CI_PROJECT_NAME}) + extends: + - .run-selected variables: NOTARIZE_TIMEOUT: 0 ignore_staple_errors: "true" @@ -382,7 +386,9 @@ variables: # code signing jobs .sign:macos: - extends: .artifacts + extends: + - .artifacts + - .run-selected stage: sign tags: - bigsur @@ -405,6 +411,8 @@ variables: ### set DEKEN_USERNAME/DEKEN_PASSWORD in the CI-project settings. ### (https://git.iem.at/help/ci/variables/README#variables) .deken: + extends: + - .run-selected stage: deploy image: registry.git.iem.at/pd/deken:latest before_script: -- GitLab From d982225bfcd4b381e4227b137f124fd83edf79bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 11:01:48 +0100 Subject: [PATCH 146/157] drop description from variables where it doesn't make much sense to set them in the webinterface --- pd-lib-builder/iem-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index da127b6..533e126 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -26,13 +26,13 @@ variables: description: "The Pd-version against which to build" SRCDIR: value: "." - description: "relative path where the Makefile lives" + #description: "relative path where the Makefile lives" IEM_CI_PKGLIBDIR: - value: "override the library component for the installation path (if it differs from the library name)" - description: "" + value: "" + #description: "override the library component for the installation path (if it differs from the library name)" IEM_CI_PROJECT_NAME: value: ${CI_PROJECT_NAME} - description: "override the library name (as used in installation directories and deken packages) if it is not the same as the repository name" + #description: "override the library name (as used in installation directories and deken packages) if it is not the same as the repository name" IEM_CI_MACOS_BUILDFAT: value: "0" description: "set to '1' if you would like to create universal binaries on macOS" -- GitLab From f6b1cfc8592133c1bb23a90e5526a3d17276ba14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 11:04:00 +0100 Subject: [PATCH 147/157] wipe the descriptions for those pesky variables --- pd-lib-builder/iem-ci.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 533e126..cad85e3 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -24,21 +24,15 @@ variables: PDVERSION: value: "0.51-3" description: "The Pd-version against which to build" - SRCDIR: - value: "." - #description: "relative path where the Makefile lives" - IEM_CI_PKGLIBDIR: - value: "" - #description: "override the library component for the installation path (if it differs from the library name)" - IEM_CI_PROJECT_NAME: - value: ${CI_PROJECT_NAME} - #description: "override the library name (as used in installation directories and deken packages) if it is not the same as the repository name" IEM_CI_MACOS_BUILDFAT: value: "0" description: "set to '1' if you would like to create universal binaries on macOS" IEM_CI_JOBS: value: "" description: "space-separated list of jobs to run (leave empty to select ALL jobs)" + SRCDIR: "." + IEM_CI_PKGLIBDIR: "" + IEM_CI_PROJECT_NAME: ${CI_PROJECT_NAME} IEM_CI_TMPDIR: .git-ci/_build/ -- GitLab From 5779017111cd7c34f9276f10f27348fe4545eac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 12:08:46 +0100 Subject: [PATCH 148/157] run macOS signing jobs on Mojave (rather than BigSur) the later does weird things (and the job fails) --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index cad85e3..14b4995 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -385,7 +385,7 @@ variables: - .run-selected stage: sign tags: - - bigsur + - mojave variables: keychainpass: $CI_BUILD_TOKEN CODESIGNFLAGS: --timestamp --strict --force -- GitLab From 7198b10c2674395dee0e5cceca82c3ebbf71f67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 13:19:13 +0100 Subject: [PATCH 149/157] macOS:notarize should run on mojave for now... --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 14b4995..2bb2b24 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -261,7 +261,7 @@ variables: NOTARIZE_TIMEOUT: 0 ignore_staple_errors: "true" tags: - - bigsur + - mojave retry: max: 1 when: @@ -385,7 +385,7 @@ variables: - .run-selected stage: sign tags: - - mojave + - osx variables: keychainpass: $CI_BUILD_TOKEN CODESIGNFLAGS: --timestamp --strict --force -- GitLab From b0c2190a37e99fa90cecead786e783a363457804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 13:28:45 +0100 Subject: [PATCH 150/157] macOS: automatically build universal binaries if there are no requirements --- pd-lib-builder/iem-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 2bb2b24..f971331 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -216,7 +216,7 @@ variables: pddisk="" app="" - export PD=$(find /Applications/Pd*.app/Contents/Resources/bin/ type f -name pd -print -quit) - # setup for code-signing (LATER move this into a separate stage; see 'SIGN CODE' below) + - test -n "${IEM_CI_MACOS_BUILDFAT}" || test -e .git-ci/requirements.brew || IEM_CI_MACOS_BUILDFAT=1 - test -n "${pd_extension}" || case "${IEM_CI_MACOS_BUILDFAT}" in yes|1|true) pd_extension=d_fat ;; esac - | if test "x${pd_extension}" = "xd_fat" && test "x${archs}" = "x"; then -- GitLab From 1ac529a7eb7dd1fa3b5114a0472d2e0cceba4b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 13:33:58 +0100 Subject: [PATCH 151/157] document fat arch autodetection --- pd-lib-builder/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pd-lib-builder/README.md b/pd-lib-builder/README.md index 9e60e23..9ca9f19 100644 --- a/pd-lib-builder/README.md +++ b/pd-lib-builder/README.md @@ -266,14 +266,20 @@ variables: IEM_CI_PROJECT_NAME: superlib ~~~ -### fat libraries +### fat libraries (macOS only) On macOS, you have the option to build universal binaries (that contain multiple architectures). -This will only work if all the [dependencies](#build-dependencies) are also universal binaries. -This feature relies on the build-system doing the "right thing" if the Pd library extension is `d_fat` (pd-lib-builder does this). +This will *only* work if all the [dependencies](#build-dependencies) are also universal binaries. + +If universal binaries are enabled, the `arch` buildvariable is set to a list of architectures depending +on the detected Xcode version. +Otherwise this feature relies on the build-system doing the "right thing" if the Pd library extension is +`d_fat` (pd-lib-builder does this). To enable this feature, set the `IEM_CI_MACOS_BUILDFAT` variable to `1`. To disable this feature, set the variable to `0`. +If `IEM_CI_MACOS_BUILDFAT` is unset **and** there is no `.git-ci/requirements.brew` dependency file, +universal binaries are enabled automatically. ### Build Dependencies -- GitLab From f2013d94fc2d60f918ceec55bcf95fe718c2ceb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 13:43:51 +0100 Subject: [PATCH 152/157] fix URLs to raw files --- no-build/README.md | 6 +++--- no-build/gitlab-iem.yml | 2 +- pd-lib-builder/README.md | 10 +++++----- pd-lib-builder/gitlab-iem.yml | 4 ++-- pd-lib-builder/iem-ci.yml | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/no-build/README.md b/no-build/README.md index 54cdcc0..b3a7007 100644 --- a/no-build/README.md +++ b/no-build/README.md @@ -41,7 +41,7 @@ Pd package download server). ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/main/no-build/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/no-build/gitlab-iem.yml ~~~ - Push the new file to the repository: @@ -101,7 +101,7 @@ E.g. if your repository has the name `pd-superlib`, but the library is really ca ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/main/no-build/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/no-build/gitlab-iem.yml variables: IEM_CI_PROJECT_NAME: superlib @@ -116,7 +116,7 @@ To package such a library, you need to specify the target (installation) directo ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/main/no-build/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/no-build/gitlab-iem.yml variables: IEM_CI_PROJECT_INSTALLDIR: foo/bar diff --git a/no-build/gitlab-iem.yml b/no-build/gitlab-iem.yml index a516187..5849b27 100644 --- a/no-build/gitlab-iem.yml +++ b/no-build/gitlab-iem.yml @@ -17,7 +17,7 @@ include: # define job-templates: - - https://git.iem.at/pd/iem-ci/raw/main/no-build/iem-ci.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/no-build/iem-ci.yml ####################################################################### diff --git a/pd-lib-builder/README.md b/pd-lib-builder/README.md index 9ca9f19..d618a9f 100644 --- a/pd-lib-builder/README.md +++ b/pd-lib-builder/README.md @@ -107,7 +107,7 @@ yet. ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/pd-lib-builder/gitlab-iem.yml ~~~ - Push the new file to the repository: @@ -222,7 +222,7 @@ For example, to force the Pd-version to `0.49-0` use something like this: ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/pd-lib-builder/gitlab-iem.yml variables: PDVERSION: 0.49-0 @@ -241,7 +241,7 @@ If this is not the case, you can specify an alternate directory using the `SRCDI ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/pd-lib-builder/gitlab-iem.yml variables: SRCDIR: pd/ @@ -260,7 +260,7 @@ E.g. if your repository has the name `pd-superlib`, but the library is really ca ~~~yml --- include: - - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/gitlab-iem.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/pd-lib-builder/gitlab-iem.yml variables: IEM_CI_PROJECT_NAME: superlib @@ -345,4 +345,4 @@ https://git.iem.at/ [IEM]: https://iem.at/ [pd-lib-builder]: https://github.com/pure-data/pd-lib-builder/ [deken]: https://deken.puredata.info/ -[gitlab-iem.yml]: https://git.iem.at/pd/iem-ci/gitlab-iem.yml +[gitlab-iem.yml]: https://git.iem.at/pd/iem-ci/-/raw/main/pd-lib-builder/gitlab-iem.yml diff --git a/pd-lib-builder/gitlab-iem.yml b/pd-lib-builder/gitlab-iem.yml index c599e3f..f4c2fcb 100644 --- a/pd-lib-builder/gitlab-iem.yml +++ b/pd-lib-builder/gitlab-iem.yml @@ -17,8 +17,8 @@ include: # define job-templates: - - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/iem-ci.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/pd-lib-builder/iem-ci.yml # turn templates into real jobs: - - https://git.iem.at/pd/iem-ci/raw/main/pd-lib-builder/pipeline-jobs.yml + - https://git.iem.at/pd/iem-ci/-/raw/main/pd-lib-builder/pipeline-jobs.yml # and run some static code analysis - template: Security/SAST.gitlab-ci.yml diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index f971331..17b323b 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -40,9 +40,9 @@ variables: ####################################################################### ### script snippets .script:fetch_localdeps: &script_fetch_localdeps - - test -e .git-ci/localdeps.win.sh || (wget -q -O .git-ci/localdeps.win.sh "https://git.iem.at/pd/iem-ci/raw/main/localdeps/localdeps.win.sh" && chmod +x .git-ci/localdeps.win.sh) || true - - test -e .git-ci/localdeps.macos.sh || (wget -q -O .git-ci/localdeps.macos.sh "https://git.iem.at/pd/iem-ci/raw/main/localdeps/localdeps.macos.sh" && chmod +x .git-ci/localdeps.macos.sh) || true - - test -e .git-ci/localdeps.linux.sh || (wget -q -O .git-ci/localdeps.linux.sh "https://git.iem.at/pd/iem-ci/raw/main/localdeps/localdeps.linux.sh" && chmod +x .git-ci/localdeps.linux.sh) || true + - test -e .git-ci/localdeps.win.sh || (wget -q -O .git-ci/localdeps.win.sh "https://git.iem.at/pd/iem-ci/-/raw/main/localdeps/localdeps.win.sh" && chmod +x .git-ci/localdeps.win.sh) || true + - test -e .git-ci/localdeps.macos.sh || (wget -q -O .git-ci/localdeps.macos.sh "https://git.iem.at/pd/iem-ci/-/raw/main/localdeps/localdeps.macos.sh" && chmod +x .git-ci/localdeps.macos.sh) || true + - test -e .git-ci/localdeps.linux.sh || (wget -q -O .git-ci/localdeps.linux.sh "https://git.iem.at/pd/iem-ci/-/raw/main/localdeps/localdeps.linux.sh" && chmod +x .git-ci/localdeps.linux.sh) || true ## build snippets .script:make: &script_make -- GitLab From b15b67e59511b2c5b97fef040abeb681e444b861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 13:55:20 +0100 Subject: [PATCH 153/157] only print auto-detected Xcode if we did attempt to detect it --- pd-lib-builder/iem-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 17b323b..26621bc 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -244,8 +244,10 @@ variables: archs="i386 x86_64" echo "warning: unknown or unsupported Xcode version $XCODE_VERSION, trying ${archs}" 1>&2 fi + echo "detected Xcode-$XCODE_VERSION builds ${archs}" + else + echo "building native arch${pd_extension:+ for $pd_extension}" fi - echo "detected Xcode-$XCODE_VERSION builds ${archs}" after_script: - *script_fetch_localdeps -- GitLab From 545ff13556981a9157aee4392521a2c983750b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Wed, 2 Feb 2022 13:55:32 +0100 Subject: [PATCH 154/157] set IEM_CI_MACOS_BUILDFAT to empty by default --- pd-lib-builder/iem-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index 26621bc..a1ccd91 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -25,8 +25,8 @@ variables: value: "0.51-3" description: "The Pd-version against which to build" IEM_CI_MACOS_BUILDFAT: - value: "0" - description: "set to '1' if you would like to create universal binaries on macOS" + value: "" + description: "set to '1' to create universal binaries on macOS, to '0' to prevent it and leave it empty for auto-detection" IEM_CI_JOBS: value: "" description: "space-separated list of jobs to run (leave empty to select ALL jobs)" -- GitLab From b6544ef3c988b74d13f69e75fe887069c42f9b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 10 May 2022 10:49:01 +0200 Subject: [PATCH 155/157] do not gpg-sign deken packages at least with newer versions of deken --- no-build/iem-ci.yml | 2 ++ pd-lib-builder/iem-ci.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index 3c4409a..82052ec 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -52,6 +52,8 @@ variables: stage: deploy image: registry.git.iem.at/pd/deken:latest before_script: + - : ${DEKEN_SIGN_GPG:+False} + - export DEKEN_SIGN_GPG - apt-get update && apt-get --no-install-recommends -y install git - IEM_CI_PROJECT_INSTALLDIR=${IEM_CI_PROJECT_INSTALLDIR:-${IEM_CI_PROJECT_NAME}} - IEM_CI_PROJECT_ROOTDIR=${IEM_CI_PROJECT_ROOTDIR:-${IEM_CI_PROJECT_INSTALLDIR%%/*}} diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index a1ccd91..b8c1c54 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -412,9 +412,12 @@ variables: stage: deploy image: registry.git.iem.at/pd/deken:latest before_script: + - : ${DEKEN_SIGN_GPG:+False} + - export DEKEN_SIGN_GPG - apt-get update && apt-get --no-install-recommends -y install git - echo "${CI_COMMIT_TAG}" - echo "${DEKEN_USERNAME}" + - echo "${DEKEN_SIGN_GPG}" script: - chmod -R go-w . # create source package -- GitLab From e5a5ee3bec09c3d3375830815e4753492478c129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Tue, 10 May 2022 10:52:14 +0200 Subject: [PATCH 156/157] reduce colons in yaml yaml and me... --- no-build/iem-ci.yml | 3 +-- pd-lib-builder/iem-ci.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/no-build/iem-ci.yml b/no-build/iem-ci.yml index 82052ec..bbcc47f 100644 --- a/no-build/iem-ci.yml +++ b/no-build/iem-ci.yml @@ -52,8 +52,7 @@ variables: stage: deploy image: registry.git.iem.at/pd/deken:latest before_script: - - : ${DEKEN_SIGN_GPG:+False} - - export DEKEN_SIGN_GPG + - export DEKEN_SIGN_GPG="${DEKEN_SIGN_GPG:-0}" - apt-get update && apt-get --no-install-recommends -y install git - IEM_CI_PROJECT_INSTALLDIR=${IEM_CI_PROJECT_INSTALLDIR:-${IEM_CI_PROJECT_NAME}} - IEM_CI_PROJECT_ROOTDIR=${IEM_CI_PROJECT_ROOTDIR:-${IEM_CI_PROJECT_INSTALLDIR%%/*}} diff --git a/pd-lib-builder/iem-ci.yml b/pd-lib-builder/iem-ci.yml index b8c1c54..e0ac74b 100644 --- a/pd-lib-builder/iem-ci.yml +++ b/pd-lib-builder/iem-ci.yml @@ -412,8 +412,7 @@ variables: stage: deploy image: registry.git.iem.at/pd/deken:latest before_script: - - : ${DEKEN_SIGN_GPG:+False} - - export DEKEN_SIGN_GPG + - export DEKEN_SIGN_GPG="${DEKEN_SIGN_GPG:-0}" - apt-get update && apt-get --no-install-recommends -y install git - echo "${CI_COMMIT_TAG}" - echo "${DEKEN_USERNAME}" -- GitLab From a3425ea6eef7314bb22ffcf07b835d358568b28f Mon Sep 17 00:00:00 2001 From: Roman Haefeli Date: Fri, 13 May 2022 15:09:21 +0200 Subject: [PATCH 157/157] add flags -d, -s to localdeps scripts: * -d: put localdeps into subfolder named $arch * -s: codesign compiled binaries and localdeps --- localdeps/localdeps.linux.sh | 73 ++++++++++++++++++---------- localdeps/localdeps.macos.sh | 65 ++++++++++++++++++++++--- localdeps/localdeps.utilities.source | 33 +++++++++++-- localdeps/localdeps.win.sh | 33 +++++++++++-- 4 files changed, 166 insertions(+), 38 deletions(-) diff --git a/localdeps/localdeps.linux.sh b/localdeps/localdeps.linux.sh index 7aec1f6..e66e3ef 100755 --- a/localdeps/localdeps.linux.sh +++ b/localdeps/localdeps.linux.sh @@ -26,10 +26,30 @@ else # was copied from 'localdeps.utilities.source'. # changes you make to this section will be lost. #@BEGIN_UTILITIES@ +sign=false +subdir=false verbose=${verbose:-0} error() { - echo "$@" 1>&2 + echo "$@" 1>&2 +} + +print_arch() { + # conflation of arch names follows rules + # from deken-plugin.tcl + arch=$(uname -m) + case $arch in + x86_64) + arch=amd64 + ;; + i486 | i586 | i686) + arch=i386 + ;; + armv6 | arm6l | arm7 | arm7l) + arch=arm + ;; + esac + echo $arch } substitute() { @@ -115,6 +135,8 @@ usage: $0 [-I ] [-X ] [ ...] -I : adds one include path entry -X : adds one exclude path entry + -d: put localdeps into subdirectory named \$arch (Linux and macOS) + -s: codesign resulting binaries (macOS only) -v: raise verbosity -q: lower verbosity @@ -157,8 +179,7 @@ EOF exit 1 } - -while getopts "hqvI:X:" arg; do +while getopts "dhqsvI:X:" arg; do case $arg in h) usage @@ -177,9 +198,15 @@ while getopts "hqvI:X:" arg; do fi include_paths=$(echo :${include_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; + d) + subdir=true + ;; q) verbose=$((verbose-1)) ;; + s) + sign=true + ;; v) verbose=$((verbose+1)) ;; @@ -199,23 +226,6 @@ fi #@END_UTILITIES@ fi - -library_in_exclude_list() { - # arg1: library name - # returns 0 if arg1 is found in exclude list, otherwise 1 - local libexname="$1" - skip=1 - set -f - for expat in $(echo "${ld_exclude_list}"); do - if echo "$(basename $libexname)" | grep "${expat}" > /dev/null; then - skip=0 - break - fi - done - set +f - return $skip -} - list_deps() { local libpath local inc @@ -223,18 +233,22 @@ list_deps() { | grep ' => ' \ | while read _ _ libpath _; do inc=$(check_includedep "${libpath}") - if [ "x${inc}" != "x" ]; then - echo "${inc}" - fi + if [ "x${inc}" != "x" ]; then + echo "${inc}" + fi done } - install_deps () { # make a local copy of all linked libraries of given binary # and set RUNPATH to $ORIGIN (exclude "standard" libraries) # arg1: binary to check - local outdir=$(dirname "$1") + local outdir local outfile + if $subdir; then + outdir="$(dirname "$1")/$(print_arch)" + else + outdir="$(dirname "$1")" + fi if [ ! -d "${outdir}" ]; then outdir=. fi @@ -253,7 +267,11 @@ install_deps () { patchelf --set-rpath \$ORIGIN "${outfile}" fi done - patchelf --set-rpath \$ORIGIN "${1}" + if $subdir; then + patchelf --set-rpath "\$ORIGIN/$(print_arch)" "${1}" + else + patchelf --set-rpath \$ORIGIN "${1}" + fi } @@ -267,5 +285,8 @@ for f in "$@"; do error "Skipping '${f}'. Is it a binary file?" continue fi + if $subdir; then + mkdir -p "$(dirname ${f})/$(print_arch)" + fi install_deps "${f}" done diff --git a/localdeps/localdeps.macos.sh b/localdeps/localdeps.macos.sh index b332947..2068db9 100755 --- a/localdeps/localdeps.macos.sh +++ b/localdeps/localdeps.macos.sh @@ -17,10 +17,30 @@ else # was copied from 'localdeps.utilities.source'. # changes you make to this section will be lost. #@BEGIN_UTILITIES@ +sign=false +subdir=false verbose=${verbose:-0} error() { - echo "$@" 1>&2 + echo "$@" 1>&2 +} + +print_arch() { + # conflation of arch names follows rules + # from deken-plugin.tcl + arch=$(uname -m) + case $arch in + x86_64) + arch=amd64 + ;; + i486 | i586 | i686) + arch=i386 + ;; + armv6 | arm6l | arm7 | arm7l) + arch=arm + ;; + esac + echo $arch } substitute() { @@ -106,6 +126,8 @@ usage: $0 [-I ] [-X ] [ ...] -I : adds one include path entry -X : adds one exclude path entry + -d: put localdeps into subdirectory named \$arch (Linux and macOS) + -s: codesign resulting binaries (macOS only) -v: raise verbosity -q: lower verbosity @@ -148,8 +170,7 @@ EOF exit 1 } - -while getopts "hqvI:X:" arg; do +while getopts "dhqsvI:X:" arg; do case $arg in h) usage @@ -168,9 +189,15 @@ while getopts "hqvI:X:" arg; do fi include_paths=$(echo :${include_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; + d) + subdir=true + ;; q) verbose=$((verbose-1)) ;; + s) + sign=true + ;; v) verbose=$((verbose+1)) ;; @@ -236,12 +263,20 @@ install_deps () { if [ ! -d "${outdir}" ]; then outdir=. fi - + if $subdir; then + outdir="${outdir}/$(print_arch)" + mkdir -p "${outdir}" + fi list_deps "$1" | while read dep; do infile=$(basename "$1") depfile=$(basename "${dep}") + if $subdir; then + loaderpath="@loader_path/$(print_arch)/${depfile}" + else + loaderpath="@loader_path/${depfile}" + fi # make sure the binary looks for the dependency in the local path - install_name_tool -change "${dep}" "@loader_path/${depfile}" "$1" + install_name_tool -change "${dep}" "${loaderpath}" "$1" if [ -e "${outdir}/${depfile}" ]; then error "DEP: ${INSTALLDEPS_INDENT} ${dep} SKIPPED" @@ -251,7 +286,7 @@ install_deps () { chmod u+w "${outdir}/${depfile}" # make sure the dependency announces itself with the local path - install_name_tool -id "@loader_path/${depfile}" "${outdir}/${depfile}" + install_name_tool -id "${loaderpath}" "${outdir}/${depfile}" # recursively call ourselves, to resolve higher-order dependencies INSTALLDEPS_INDENT="${INSTALLDEPS_INDENT} " $0 "${outdir}/${depfile}" fi @@ -269,3 +304,21 @@ for f in "$@"; do install_deps "${f}" fi done + +# Code signing +# On Monterey, binaries are automatically codesigned. Modifying them with this script renders the signature +# invalid. When Pd loads an external with an invalid signature, it exits immediately. Thus, we need to make sure +# that we codesign them again _after_ the localdeps process + +# This needs to be the absolutely last step. We don't do it while we're still inside a recursion. +if $sign; then + echo -n "Code signing in progress... " + if $subdir; then + outdir="$(dirname "$1")/$(print_arch)" + else + outdir="$(dirname "$1")" + fi + codesign --remove-signature "${ARGS[@]}" ${outdir}/*.dylib + codesign -s - "${ARGS[@]}" ${outdir}/*.dylib + echo "Done" +fi diff --git a/localdeps/localdeps.utilities.source b/localdeps/localdeps.utilities.source index 2859834..5d3e58f 100644 --- a/localdeps/localdeps.utilities.source +++ b/localdeps/localdeps.utilities.source @@ -1,7 +1,27 @@ +sign=false +subdir=false verbose=${verbose:-0} error() { - echo "$@" 1>&2 + echo "$@" 1>&2 +} + +print_arch() { + # conflation of arch names follows rules + # from deken-plugin.tcl + arch=$(uname -m) + case $arch in + x86_64) + arch=amd64 + ;; + i486 | i586 | i686) + arch=i386 + ;; + armv6 | arm6l | arm7 | arm7l) + arch=arm + ;; + esac + echo $arch } substitute() { @@ -87,6 +107,8 @@ usage: $0 [-I ] [-X ] [ ...] -I : adds one include path entry -X : adds one exclude path entry + -d: put localdeps into subdirectory named \$arch (Linux and macOS) + -s: codesign resulting binaries (macOS only) -v: raise verbosity -q: lower verbosity @@ -129,8 +151,7 @@ EOF exit 1 } - -while getopts "hqvI:X:" arg; do +while getopts "dhqsvI:X:" arg; do case $arg in h) usage @@ -149,9 +170,15 @@ while getopts "hqvI:X:" arg; do fi include_paths=$(echo :${include_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; + d) + subdir=true + ;; q) verbose=$((verbose-1)) ;; + s) + sign=true + ;; v) verbose=$((verbose+1)) ;; diff --git a/localdeps/localdeps.win.sh b/localdeps/localdeps.win.sh index f05a566..233a859 100755 --- a/localdeps/localdeps.win.sh +++ b/localdeps/localdeps.win.sh @@ -28,10 +28,30 @@ else # was copied from 'localdeps.utilities.source'. # changes you make to this section will be lost. #@BEGIN_UTILITIES@ +sign=false +subdir=false verbose=${verbose:-0} error() { - echo "$@" 1>&2 + echo "$@" 1>&2 +} + +print_arch() { + # conflation of arch names follows rules + # from deken-plugin.tcl + arch=$(uname -m) + case $arch in + x86_64) + arch=amd64 + ;; + i486 | i586 | i686) + arch=i386 + ;; + armv6 | arm6l | arm7 | arm7l) + arch=arm + ;; + esac + echo $arch } substitute() { @@ -117,6 +137,8 @@ usage: $0 [-I ] [-X ] [ ...] -I : adds one include path entry -X : adds one exclude path entry + -d: put localdeps into subdirectory named \$arch (Linux and macOS) + -s: codesign resulting binaries (macOS only) -v: raise verbosity -q: lower verbosity @@ -159,8 +181,7 @@ EOF exit 1 } - -while getopts "hqvI:X:" arg; do +while getopts "dhqsvI:X:" arg; do case $arg in h) usage @@ -179,9 +200,15 @@ while getopts "hqvI:X:" arg; do fi include_paths=$(echo :${include_paths}: | substitute ":${p}:" ":" | sed -e 's|^:*||' -e 's|:*$||' -e 's|::*|:|g') ;; + d) + subdir=true + ;; q) verbose=$((verbose-1)) ;; + s) + sign=true + ;; v) verbose=$((verbose+1)) ;; -- GitLab