From 2b89215eecb669ad97d18a92f4613d3e6a9e4d21 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 00:18:50 +0200 Subject: [PATCH 1/8] Extract function for launching httpbin into its own file Allow reusing it between GitHub and Travis CI. Also show httpbin log in Travis CI logs too, not just on GitHub. --- .github/workflows/ci.yml | 16 ++++------- build/tools/httpbin.sh | 44 ++++++++++++++++++++++++++++++ build/tools/travis-ci.sh | 58 +++++++++++++--------------------------- 3 files changed, 67 insertions(+), 51 deletions(-) create mode 100644 build/tools/httpbin.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5ad24e7bd..b553ffbe96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,13 +189,9 @@ jobs: if: matrix.skip_testing != true working-directory: tests run: | - echo "Launching httpbin" - # Current decorator (>= 5) is incompatible with Python 2 which we - # still use under in some builds, so explicitly use a version which - # is known to work. - pip install decorator==4.4.2 httpbin - python -m httpbin.core 2>&1 >httpbin.log & - export WX_TEST_WEBREQUEST_URL=http://localhost:5000 + . ../build/tools/httpbin.sh + + httpbin_launch export ASAN_OPTIONS=fast_unwind_on_malloc=0 @@ -210,11 +206,9 @@ jobs: ./test || rc=$? fi fi + if [ -n "$rc" ]; then - echo '*** Tests failed, contents of httpbin.log follows: ***' - echo '-----------------------------------------------------------' - cat httpbin.log - echo '-----------------------------------------------------------' + httpbin_show_log exit $rc fi diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh new file mode 100644 index 0000000000..f24e37ef39 --- /dev/null +++ b/build/tools/httpbin.sh @@ -0,0 +1,44 @@ +# This script is sourced by CI scripts to launch httpbin. +# +# Do not run it directly. + +httpbin_launch() { + echo 'Launching httpbin...' + + case "$(uname -s)" in + Linux) + dist_codename=$(lsb_release --codename --short) + ;; + esac + + # We need to disable SSL certificate checking under Trusty because Python + # version there is too old to support SNI. + case "$dist_codename" in + trusty) + # Here "decorator==4.4.2" doesn't work neither for some reason ("no + # matching distribution found"), so use an explicit URL. + pip_decorator_arg='https://files.pythonhosted.org/packages/ed/1b/72a1821152d07cf1d8b6fce298aeb06a7eb90f4d6d41acec9861e7cc6df0/decorator-4.4.2-py2.py3-none-any.whl' + pip_options='--trusted-host files.pythonhosted.org' + ;; + + *) + # Current decorator (>= 5) is incompatible with Python 2 which we + # still use under in some builds, so explicitly use a version which + # is known to work. + pip_decorator_arg='decorator==4.4.2' + ;; + esac + + pip install $pip_decorator_arg httpbin --user $pip_options + python -m httpbin.core 2>&1 >httpbin.log & + WX_TEST_WEBREQUEST_URL="http://localhost:5000" + + export WX_TEST_WEBREQUEST_URL +} + +httpbin_show_log() { + echo '*** Tests failed, contents of httpbin.log follows: ***' + echo '-----------------------------------------------------------' + cat httpbin.log + echo '-----------------------------------------------------------' +} diff --git a/build/tools/travis-ci.sh b/build/tools/travis-ci.sh index 4e46db6ca1..7c3b085944 100755 --- a/build/tools/travis-ci.sh +++ b/build/tools/travis-ci.sh @@ -4,6 +4,8 @@ set -e +. ./build/tools/httpbin.sh + wxPROC_COUNT=`getconf _NPROCESSORS_ONLN` ((wxPROC_COUNT++)) wxBUILD_ARGS="-j$wxPROC_COUNT" @@ -12,42 +14,6 @@ wxBUILD_ARGS="-j$wxPROC_COUNT" # messages from WebKit tests that we're not interested in. export NO_AT_BRIDGE=1 -launch_httpbin() { - echo 'travis_fold:start:httpbin' - echo 'Running httpbin...' - - case "$(uname -s)" in - Linux) - dist_codename=$(lsb_release --codename --short) - ;; - esac - - # We need to disable SSL certificate checking under Trusty because Python - # version there is too old to support SNI. - case "$dist_codename" in - trusty) - # Here "decorator==4.4.2" doesn't work neither for some reason ("no - # matching distribution found"), so use an explicit URL. - pip_decorator_arg='https://files.pythonhosted.org/packages/ed/1b/72a1821152d07cf1d8b6fce298aeb06a7eb90f4d6d41acec9861e7cc6df0/decorator-4.4.2-py2.py3-none-any.whl' - pip_options='--trusted-host files.pythonhosted.org' - ;; - - *) - # Current decorator (>= 5) is incompatible with Python 2 which we - # still use under in some builds, so explicitly use a version which - # is known to work. - pip_decorator_arg='decorator==4.4.2' - ;; - esac - - pip install $pip_decorator_arg httpbin --user $pip_options - python -m httpbin.core & - WX_TEST_WEBREQUEST_URL="http://localhost:5000" - - export WX_TEST_WEBREQUEST_URL - echo 'travis_fold:end:httpbin' -} - case $wxTOOLSET in cmake) if [ -z $wxCMAKE_TESTS ]; then wxCMAKE_TESTS=CONSOLE_ONLY; fi @@ -78,12 +44,18 @@ case $wxTOOLSET in echo 'travis_fold:end:install' if [ "$wxCMAKE_TESTS" != "OFF" ]; then - launch_httpbin + echo 'travis_fold:start:httpbin' + httpbin_launch + echo 'travis_fold:end:httpbin' echo 'travis_fold:start:testing' echo 'Testing...' - ctest -V -C Debug -E "test_drawing" --output-on-failure --interactive-debug-mode 0 . + ctest -V -C Debug -E "test_drawing" --output-on-failure --interactive-debug-mode 0 . || rc=$? echo 'travis_fold:end:testing' + if [ -n "$rc" ]; then + httpbin_show_log + exit $rc + fi fi echo 'travis_fold:start:testinstall' @@ -162,14 +134,20 @@ case $wxTOOLSET in exit 0 fi - launch_httpbin + echo 'travis_fold:start:httpbin' + httpbin_launch + echo 'travis_fold:end:httpbin' echo 'travis_fold:start:testing' echo 'Testing...' pushd tests - ./test + ./test || rc=$? popd echo 'travis_fold:end:testing' + if [ -n "$rc" ]; then + httpbin_show_log + exit $rc + fi if [ "$wxSKIP_GUI" = 1 ]; then echo 'Skipping the rest of tests for non-GUI build.' From da0f6bd2a6265f405b5b7b26a21e2e618cbf8b2a Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 00:29:27 +0200 Subject: [PATCH 2/8] Use Python 3 for running httpbin in non-Trusty Linux CI builds Even Ubuntu 16.04 has Python 3 and pip3, so just use them instead of trying to work around problems with Python 2. --- build/tools/httpbin.sh | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh index f24e37ef39..bc09d9dd54 100644 --- a/build/tools/httpbin.sh +++ b/build/tools/httpbin.sh @@ -9,28 +9,38 @@ httpbin_launch() { Linux) dist_codename=$(lsb_release --codename --short) ;; + + Darwin) + dist_codename='macOS' + ;; esac - # We need to disable SSL certificate checking under Trusty because Python - # version there is too old to support SNI. case "$dist_codename" in trusty) - # Here "decorator==4.4.2" doesn't work neither for some reason ("no - # matching distribution found"), so use an explicit URL. + # Current decorator (>= 5) is incompatible with Python 2 which we + # still use in some builds, so explicitly use a version which + # is known to work. Just specifying "decorator==4.4.2" doesn't work + # neither for some reason ("no matching distribution found"), so + # use an explicit URL. pip_decorator_arg='https://files.pythonhosted.org/packages/ed/1b/72a1821152d07cf1d8b6fce298aeb06a7eb90f4d6d41acec9861e7cc6df0/decorator-4.4.2-py2.py3-none-any.whl' + + # We need to disable SSL certificate checking under Trusty because + # Python version there is too old to support SNI. pip_options='--trusted-host files.pythonhosted.org' ;; - *) - # Current decorator (>= 5) is incompatible with Python 2 which we - # still use under in some builds, so explicitly use a version which - # is known to work. + macOS) pip_decorator_arg='decorator==4.4.2' ;; + + *) + # Elsewhere just use Python 3. + PY3=3 + ;; esac - pip install $pip_decorator_arg httpbin --user $pip_options - python -m httpbin.core 2>&1 >httpbin.log & + python$PY3 -m pip install $pip_decorator_arg httpbin --user $pip_options + python$PY3 -m httpbin.core 2>&1 >httpbin.log & WX_TEST_WEBREQUEST_URL="http://localhost:5000" export WX_TEST_WEBREQUEST_URL From d5d28e639e4ad3952508c226691e7e4f9a41a981 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 01:29:29 +0200 Subject: [PATCH 3/8] Attempt to make pip install actually work by upgrading it first Old pip versions don't work when used with python3, so upgrade pip before trying to use it. --- build/tools/httpbin.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh index bc09d9dd54..6926979923 100644 --- a/build/tools/httpbin.sh +++ b/build/tools/httpbin.sh @@ -36,6 +36,11 @@ httpbin_launch() { *) # Elsewhere just use Python 3. PY3=3 + + # Running pip install fails with weird errors out of the box when + # using old pip version because it attempts to use python rather + # than python3, so upgrade it to fix this. + python3 -m pip install --upgrade pip setuptools wheel ;; esac From c7961fa666c08cd90d3dc387f1f868d08a170472 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 01:34:45 +0200 Subject: [PATCH 4/8] Don't upgrade pip to a version which is too new Amazingly, updating a package has no problems with installing a newer version incompatible with the current Python version, such as installing pip 21 when using Python 3.5. --- build/tools/httpbin.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh index 6926979923..f4387842d1 100644 --- a/build/tools/httpbin.sh +++ b/build/tools/httpbin.sh @@ -40,7 +40,10 @@ httpbin_launch() { # Running pip install fails with weird errors out of the box when # using old pip version because it attempts to use python rather # than python3, so upgrade it to fix this. - python3 -m pip install --upgrade pip setuptools wheel + # + # However don't upgrade to a version which is too new because then + # it may not support Python version that we actually have. + python3 -m pip install --upgrade pip==20 ;; esac From 54986685140021f68e57e5fefc1eeffd688df3f9 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 01:38:03 +0200 Subject: [PATCH 5/8] Don't use broken pip version 20.0 Pip continues to impress with version 20 being completely broken and not running at all. Try the next one, which is supposed to fix this bug. --- build/tools/httpbin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh index f4387842d1..834e7df0fb 100644 --- a/build/tools/httpbin.sh +++ b/build/tools/httpbin.sh @@ -43,7 +43,7 @@ httpbin_launch() { # # However don't upgrade to a version which is too new because then # it may not support Python version that we actually have. - python3 -m pip install --upgrade pip==20 + python3 -m pip install --upgrade pip==20.0.1 ;; esac From 21c97d155a7777de0e90d7420dd10746a1a6f9e8 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 01:45:01 +0200 Subject: [PATCH 6/8] Show pip version used for installing httpbin in the logs --- build/tools/httpbin.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh index 834e7df0fb..8ea0587a7b 100644 --- a/build/tools/httpbin.sh +++ b/build/tools/httpbin.sh @@ -47,6 +47,8 @@ httpbin_launch() { ;; esac + echo "Installing using `python$PY3 -m pip --version`" + python$PY3 -m pip install $pip_decorator_arg httpbin --user $pip_options python$PY3 -m httpbin.core 2>&1 >httpbin.log & WX_TEST_WEBREQUEST_URL="http://localhost:5000" From ee5a8096bdbd443fb00442d4b054a5e8dda1bf91 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 01:53:29 +0200 Subject: [PATCH 7/8] Install pip if it's not available at all This is not supposed to happen, but somehow it does on Travis CI, so try to remedy this by installing pip via apt if it's missing. --- build/tools/httpbin.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh index 8ea0587a7b..b64609a411 100644 --- a/build/tools/httpbin.sh +++ b/build/tools/httpbin.sh @@ -37,6 +37,11 @@ httpbin_launch() { # Elsewhere just use Python 3. PY3=3 + # Ensure that we have at least some version of pip. + if ! python3 -m pip; then + sudo apt-get -q -o=Dpkg::Use-Pty=0 install python3-pip + fi + # Running pip install fails with weird errors out of the box when # using old pip version because it attempts to use python rather # than python3, so upgrade it to fix this. From 0bc1dbf218a56f834deb23e3cc29b00b8c968c25 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sun, 4 Apr 2021 13:33:23 +0200 Subject: [PATCH 8/8] Use user mode for pip upgrade This is default for some platforms, but not everywhere, so use it explicitly. This should fix the error in Travis CI S/390 build. --- build/tools/httpbin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tools/httpbin.sh b/build/tools/httpbin.sh index b64609a411..b4b04e32f9 100644 --- a/build/tools/httpbin.sh +++ b/build/tools/httpbin.sh @@ -48,7 +48,7 @@ httpbin_launch() { # # However don't upgrade to a version which is too new because then # it may not support Python version that we actually have. - python3 -m pip install --upgrade pip==20.0.1 + python3 -m pip install --user --upgrade pip==20.0.1 ;; esac