From 7bdb39668bd2efd579d94e7d69a0b429e026499b Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Sat, 6 Oct 2018 18:09:56 +0100 Subject: [PATCH 1/5] Add an azure-pipelines.yml v1: Add an azure-pipelines.yml Don't check source line endings if autocrlf is on Handle origin-only refs in skip_ci Add .py to PATHEXT for the benefit of test_find_program() Publish logs as build artifacts and publish test results v2: Use .gitattributes to override autocrlf Move tmpdir, so it's not a subdir of source directory, otherwise it gets included in line-ending checks. Use serial build numbers, rather than date.dailybuildnumber Workaround for #3239 is no longer needed now a fix has been commited Tweak test results and artefact naming Wait for MS-MPI installers to complete Publish test results even if tests had an error --- .gitattributes | 2 +- azure-pipelines.yml | 49 +++++++++++++++++++++++++++ ci/azure-steps.yml | 82 +++++++++++++++++++++++++++++++++++++++++++++ skip_ci.py | 4 +++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 azure-pipelines.yml create mode 100644 ci/azure-steps.yml diff --git a/.gitattributes b/.gitattributes index de66d2ba7..5c53af99c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ .gitignore export-ignore .gitattributes export-ignore - +* text eol=lf diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..8c36abd90 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,49 @@ +name: $(BuildID) + +trigger: + branches: + include: + - 'master' + +variables: + MESON_FIXED_NINJA: 1 + CI: 1 + +jobs: +#- job: vs2015 +# pool: +# vmImage: vs2015-win2012r2 +# +# strategy: +# maxParallel: 10 +# matrix: +# vc2015x86ninja: +# arch: x86 +# compiler: msvc2015 +# backend: ninja +# vc2015x86vs: +# arch: x86 +# compiler: msvc2015 +# backend: vs2015 +# +# steps: +# - template: ci/azure-steps.yml + +- job: vs2017 + pool: + vmImage: VS2017-Win2016 + + strategy: + maxParallel: 10 + matrix: + vc2017x64ninja: + arch: x64 + compiler: msvc2017 + backend: ninja + vc2017x64vs: + arch: x64 + compiler: msvc2017 + backend: vs2017 + + steps: + - template: ci/azure-steps.yml diff --git a/ci/azure-steps.yml b/ci/azure-steps.yml new file mode 100644 index 000000000..6126115b6 --- /dev/null +++ b/ci/azure-steps.yml @@ -0,0 +1,82 @@ +steps: +- powershell: | + # test_find_program exercises some behaviour which relies on .py being in PATHEXT + $env:PATHEXT += ';.py' + + where.exe python + + python ./skip_ci.py --base-branch-env=SYSTEM_PULLREQUEST_TARGETBRANCH --is-pull-env=SYSTEM_PULLREQUEST_PULLREQUESTID --base-branch-origin + if ($LastExitCode -ne 0) { + throw ('error in skip_ci.py') + } + + # remove MinGW from path, so we don't find gfortran and try to use it + $env:Path = ($env:Path.Split(';') | Where-Object { $_ -notlike '*mingw*' }) -join ';' + + # download and install prerequisites + function DownloadFile([String] $Source, [String] $Destination) { + $retries = 10 + for ($i = 1; $i -le $retries; $i++) { + try { + (New-Object net.webclient).DownloadFile($Source, $Destination) + break # succeeded + } catch [net.WebException] { + if ($i -eq $retries) { + throw # fail on last retry + } + $backoff = (10 * $i) # backoff 10s, 20s, 30s... + echo ('{0}: {1}' -f $Source, $_.Exception.Message) + echo ('Retrying in {0}s...' -f $backoff) + Start-Sleep -m ($backoff * 1000) + } + } + } + + DownloadFile -Source 'https://github.com/mesonbuild/cidata/raw/master/ninja.exe' -Destination $(System.WorkFolder)\ninja.exe + DownloadFile -Source 'http://nirbheek.in/files/binaries/pkg-config/win32/pkg-config.exe' -Destination $(System.WorkFolder)\pkg-config.exe + DownloadFile -Source 'https://download.microsoft.com/download/D/B/B/DBB64BA1-7B51-43DB-8BF1-D1FB45EACF7A/msmpisdk.msi' -Destination msmpisdk.msi + DownloadFile -Source 'https://download.microsoft.com/download/D/B/B/DBB64BA1-7B51-43DB-8BF1-D1FB45EACF7A/MSMpiSetup.exe' -Destination MSMpiSetup.exe + Start-Process msiexec.exe -ArgumentList '/i msmpisdk.msi /quiet' -Wait + Start-Process .\MSMpiSetup.exe -ArgumentList '-unattend -full' -Wait + + # add downloads to PATH + $env:Path = "$env:SYSTEM_WORKFOLDER;$env:Path" + + # import visual studio variables + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module Pscx -Scope CurrentUser -AllowClobber + Install-Module VSSetup -Scope CurrentUser + $vsver = $env:compiler.Replace('msvc', '') + Import-VisualStudioVars -VisualStudioVersion $vsver -Architecture $(arch) + + if ($env:backend -eq 'ninja') { + ninja --version + } else { + MSBuild /version + } + + python run_tests.py --backend $(backend) + + echo "##vso[task.setvariable variable=test_status]$LastExitCode" + + continueOnError: true + +- task: PublishTestResults@2 + inputs: + testResultsFiles: meson-test-run.xml + testRunTitle: $(System.JobName) + publishRunAttachments: true + +- task: CopyFiles@2 + inputs: + contents: 'meson-test-run.*' + targetFolder: $(Build.ArtifactStagingDirectory) + +- task: PublishBuildArtifacts@1 + inputs: + artifactName: $(System.JobName) + +- powershell: | + # after publishing test results, even if some failed + # exit with the test status + exit $(test_status) diff --git a/skip_ci.py b/skip_ci.py index 752dfdc01..5e0a7aa72 100755 --- a/skip_ci.py +++ b/skip_ci.py @@ -54,9 +54,13 @@ def main(): help='Branch push is targeted to') parser.add_argument('--is-pull-env', required=True, help='Variable set if it is a PR') + parser.add_argument('--base-branch-origin', action='store_true', + help='Base branch reference is only in origin remote') args = parser.parse_args() check_pr(args.is_pull_env) base = get_base_branch(args.base_branch_env) + if args.base_branch_origin: + base = 'origin/' + base if all(is_documentation(f) for f in get_git_files(base)): print("Don't run CI for documentation-only changes, add '[skip ci]' to commit title.") print('See http://mesonbuild.com/Contributing.html#skipping-integration-tests') From 459b81de4644106fabd410870f577848bef99fe6 Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Sun, 7 Oct 2018 13:12:49 +0100 Subject: [PATCH 2/5] Use 'CI' environment variable to detect CI environment Use the 'CI' environment variable to detect CI environment, rather than a collection of CI-specific ones. --- run_cross_test.py | 2 +- run_project_tests.py | 2 +- run_tests.py | 2 +- run_unittests.py | 8 ++++---- test cases/frameworks/17 mpi/meson.build | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/run_cross_test.py b/run_cross_test.py index 71914028f..2b640ec86 100755 --- a/run_cross_test.py +++ b/run_cross_test.py @@ -38,7 +38,7 @@ def runtests(cross_file): print('\nTotal passed cross tests:', passing_tests) print('Total failed cross tests:', failing_tests) print('Total skipped cross tests:', skipped_tests) - if failing_tests > 0 and ('TRAVIS' in os.environ or 'APPVEYOR' in os.environ): + if failing_tests > 0 and ('CI' in os.environ): print('\nMesonlogs of failing tests\n') for l in failing_logs: print(l, '\n') diff --git a/run_project_tests.py b/run_project_tests.py index 876d1359a..a9c7eca7a 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -81,7 +81,7 @@ class AutoDeletedDir: failing_logs = [] print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ -under_ci = not {'TRAVIS', 'APPVEYOR'}.isdisjoint(os.environ) +under_ci = 'CI' in os.environ do_debug = under_ci or print_debug no_meson_log_msg = 'No meson-log.txt found.' diff --git a/run_tests.py b/run_tests.py index af926eab3..242338825 100755 --- a/run_tests.py +++ b/run_tests.py @@ -245,7 +245,7 @@ if __name__ == '__main__': elif arg == '--cross=arm': cross = 'arm' # Running on a developer machine? Be nice! - if not mesonlib.is_windows() and not mesonlib.is_haiku() and 'TRAVIS' not in os.environ: + if not mesonlib.is_windows() and not mesonlib.is_haiku() and 'CI' not in os.environ: os.nice(20) # Appveyor sets the `platform` environment variable which completely messes # up building with the vs2010 and vs2015 backends. diff --git a/run_unittests.py b/run_unittests.py index 9ffce52f4..e02bc6c51 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -81,7 +81,7 @@ def is_tarball(): return False def is_ci(): - if 'TRAVIS' in os.environ or 'APPVEYOR' in os.environ: + if 'CI' in os.environ: return True return False @@ -98,8 +98,8 @@ def _git_init(project_dir): def skipIfNoPkgconfig(f): ''' - Skip this test if no pkg-config is found, unless we're on Travis or - Appveyor CI. This allows users to run our test suite without having + Skip this test if no pkg-config is found, unless we're on CI. + This allows users to run our test suite without having pkg-config installed on, f.ex., macOS, while ensuring that our CI does not silently skip the test because of misconfiguration. @@ -1671,7 +1671,7 @@ class AllPlatformTests(BasePlatformTests): self.assertIsInstance(linker, lib) self.assertEqual(cc.id, 'msvc') self.assertTrue(hasattr(cc, 'is_64')) - # If we're in the appveyor CI, we know what the compiler will be + # If we're on Windows CI, we know what the compiler will be if 'arch' in os.environ: if os.environ['arch'] == 'x64': self.assertTrue(cc.is_64) diff --git a/test cases/frameworks/17 mpi/meson.build b/test cases/frameworks/17 mpi/meson.build index 1085d40c8..b2f9fb853 100644 --- a/test cases/frameworks/17 mpi/meson.build +++ b/test cases/frameworks/17 mpi/meson.build @@ -17,7 +17,7 @@ exec = executable('exec', test('MPI C', exec) if build_machine.system() != 'windows' - # C++ MPI not supported by MS-MPI used on AppVeyor. + # C++ MPI not supported by MS-MPI mpicpp = dependency('mpi', language : 'cpp') execpp = executable('execpp', 'main.cpp', From 3582108fe77d06e476ab832cd802676b7757ad05 Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Thu, 11 Oct 2018 13:58:55 +0100 Subject: [PATCH 3/5] Add mising files to list of installed files for csharp test Omitted from 81e7a098 as csharp tests are skipped in Appveyor CI --- test cases/csharp/1 basic/installed_files.txt | 1 + test cases/csharp/2 library/installed_files.txt | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test cases/csharp/1 basic/installed_files.txt b/test cases/csharp/1 basic/installed_files.txt index f64c68c66..5022d2870 100644 --- a/test cases/csharp/1 basic/installed_files.txt +++ b/test cases/csharp/1 basic/installed_files.txt @@ -1 +1,2 @@ usr/bin/prog.exe +?msvc:usr/bin/prog.pdb diff --git a/test cases/csharp/2 library/installed_files.txt b/test cases/csharp/2 library/installed_files.txt index 4ebea55d1..73e77a211 100644 --- a/test cases/csharp/2 library/installed_files.txt +++ b/test cases/csharp/2 library/installed_files.txt @@ -1,2 +1,5 @@ usr/bin/prog.exe -usr/lib/helper.dll +?msvc:usr/bin/prog.pdb +?msvc:usr/bin/helper.dll +?msvc:usr/bin/helper.pdb +?gcc:usr/lib/helper.dll From 9717fb81646f0e6a17abd1a5e0ed8d3ccde64412 Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Wed, 10 Oct 2018 20:32:08 +0100 Subject: [PATCH 4/5] Add resgen to PATH so csharp tests are run --- ci/azure-steps.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/azure-steps.yml b/ci/azure-steps.yml index 6126115b6..984d05d74 100644 --- a/ci/azure-steps.yml +++ b/ci/azure-steps.yml @@ -49,6 +49,12 @@ steps: $vsver = $env:compiler.Replace('msvc', '') Import-VisualStudioVars -VisualStudioVersion $vsver -Architecture $(arch) + # add .NET framework tools to path for resgen for C# tests + # (always use 32-bit tool, as there doesn't seem to be a 64-bit tool) + if ((Get-Command "resgen.exe" -ErrorAction SilentlyContinue) -eq $null) { + $env:Path = "$env:WindowsSDK_ExecutablePath_x86;$env:Path" + } + if ($env:backend -eq 'ninja') { ninja --version } else { From efa55c9e1c19621290ddf226002feed9710185e8 Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Thu, 11 Oct 2018 17:43:55 +0100 Subject: [PATCH 5/5] Set env vars so MS-MPI tests are run Import MS-MPI env vars set by installer in registry to environment Skip testing we can apply a version constraint to MPI when version isn't known (as is the case for MS-MPI) (These tests have never worked in appveyor) --- ci/azure-steps.yml | 6 ++++++ test cases/frameworks/17 mpi/meson.build | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/azure-steps.yml b/ci/azure-steps.yml index 984d05d74..4cb6ac8ba 100644 --- a/ci/azure-steps.yml +++ b/ci/azure-steps.yml @@ -39,6 +39,12 @@ steps: Start-Process msiexec.exe -ArgumentList '/i msmpisdk.msi /quiet' -Wait Start-Process .\MSMpiSetup.exe -ArgumentList '-unattend -full' -Wait + # import ms-mpi env vars (set by installer) + foreach ($p in "MSMPI_INC", "MSMPI_LIB32", "MSMPI_LIB64") { + $v = [Environment]::GetEnvironmentVariable($p, "Machine") + Set-Content "env:$p" "$v" + } + # add downloads to PATH $env:Path = "$env:SYSTEM_WORKFOLDER;$env:Path" diff --git a/test cases/frameworks/17 mpi/meson.build b/test cases/frameworks/17 mpi/meson.build index b2f9fb853..ae3f08afc 100644 --- a/test cases/frameworks/17 mpi/meson.build +++ b/test cases/frameworks/17 mpi/meson.build @@ -44,4 +44,6 @@ if uburesult.returncode() != 0 and add_languages('fortran', required : false) endif # Check we can apply a version constraint -dependency('mpi', version: '>=@0@'.format(mpic.version())) +if mpic.version() != 'unknown' + dependency('mpi', version: '>=@0@'.format(mpic.version())) +endif