ci: Use self hosted CI data

pull/6901/head
Daniel Mensinger 5 years ago committed by Jussi Pakkanen
parent 217c92b48a
commit 1a741e0439
  1. 172
      ci/azure-steps.yml
  2. 75
      ci/install-dmd.ps1
  3. 71
      ci/run.ps1

@ -1,172 +1,8 @@
steps:
- powershell: |
echo "=== Check skip CI ==="
python ./skip_ci.py --base-branch-env=SYSTEM_PULLREQUEST_TARGETBRANCH --is-pull-env=SYSTEM_PULLREQUEST_PULLREQUESTID --base-branch-origin
if ($LastExitCode -ne 0) {
exit 0
}
echo "=== BEGIN CI setup ==="
# remove Chocolately, MinGW, Strawberry Perl from path, so we don't find gcc/gfortran and try to use it
$env:Path = ($env:Path.Split(';') | Where-Object { $_ -notmatch 'mingw|Strawberry|Chocolatey' }) -join ';'
# download and install prerequisites
function DownloadFile([String] $Source, [String] $Destination) {
$retries = 10
echo ('Downloading {0} ...' -f $Source)
for ($i = 1; $i -le $retries; $i++) {
try {
(New-Object net.webclient).DownloadFile($Source, $Destination)
echo '... DONE'
break # succeeded
} catch [net.WebException] {
if ($i -eq $retries) {
throw # fail on last retry
}
$backoff = (10 * $i) # backoff 10s, 20s, 30s...
echo ('{0}: {1}' -f $Source, $_.Exception.Message)
echo ('Retrying in {0}s...' -f $backoff)
Start-Sleep -m ($backoff * 1000)
}
}
}
DownloadFile -Source 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip' -Destination $(System.WorkFolder)\ninja-win.zip
# ExpandArchive is only available in Powershell 5+
Add-Type -AssemblyName System.IO.Compression.FileSystem
function unzip {
param( [string]$ziparchive, [string]$extractpath )
[System.IO.Compression.ZipFile]::ExtractToDirectory( $ziparchive, $extractpath )
}
unzip $(System.WorkFolder)\ninja-win.zip $(System.WorkFolder)
DownloadFile -Source 'https://github.com/mesonbuild/cidata/raw/master/win32/pkg-config.exe' -Destination $(System.WorkFolder)\pkg-config.exe
DownloadFile -Source 'https://download.microsoft.com/download/D/B/B/DBB64BA1-7B51-43DB-8BF1-D1FB45EACF7A/msmpisdk.msi' -Destination msmpisdk.msi
DownloadFile -Source 'https://download.microsoft.com/download/D/B/B/DBB64BA1-7B51-43DB-8BF1-D1FB45EACF7A/MSMpiSetup.exe' -Destination MSMpiSetup.exe
Start-Process msiexec.exe -ArgumentList '/i msmpisdk.msi /quiet' -Wait
Start-Process .\MSMpiSetup.exe -ArgumentList '-unattend -full' -Wait
# import ms-mpi env vars (set by installer)
foreach ($p in "MSMPI_INC", "MSMPI_LIB32", "MSMPI_LIB64") {
$v = [Environment]::GetEnvironmentVariable($p, "Machine")
Set-Content "env:$p" "$v"
}
# install boost (except for clang-cl)
if ($env:arch -eq 'x86') { $boost_bitness = '32' } else { $boost_bitness = '64' }
if ($env:compiler -eq 'msvc2017' -Or $env:compiler -eq 'msvc2019' -Or $env:compiler -eq 'clang-cl') {
$boost_version = '1.72.0' ; $boost_abi_tag = '14.1'
}
if ($boost_version) {
$boost_filename = $boost_version.Replace('.', '_')
Downloadfile -Source "https://sourceforge.net/projects/boost/files/boost-binaries/$boost_version/boost_$boost_filename-msvc-$boost_abi_tag-$boost_bitness.exe" -Destination boost_$boost_filename-msvc-$boost_abi_tag-$boost_bitness.exe
Start-Process "boost_$boost_filename-msvc-$boost_abi_tag-$boost_bitness.exe" -ArgumentList "/dir=$(System.WorkFolder)\boost_$boost_filename /silent" -Wait
$env:BOOST_ROOT = "$(System.WorkFolder)\boost_$boost_filename"
$env:Path = "$env:Path;$env:BOOST_ROOT\lib$boost_bitness-msvc-$boost_abi_tag"
} else {
$env:BOOST_ROOT = ""
}
# install D compiler and dub packages
if ($env:backend -eq 'ninja') {
& .\ci\install-dmd.ps1
$arch = 'x86_mscoff'
if ($Env:arch -eq 'x64') {
$arch = 'x86_64'
}
& dub fetch urld
& dub build urld --compiler=dmd --arch=$arch
& dub fetch dubtestproject
& dub build dubtestproject:test1 --compiler=dmd --arch=$arch
& dub build dubtestproject:test2 --compiler=dmd --arch=$arch
}
# test_find_program exercises some behaviour which relies on .py being in PATHEXT
$env:PATHEXT += ';.py'
# add downloads to PATH
$env:Path = "$env:SYSTEM_WORKFOLDER;$env:Path"
$origPath = $env:Path
# import visual studio variables
echo "source vcvarsall.bat"
if ($env:compiler -eq 'msvc2019') {
$vcvars = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat"
} else {
# Note: this is also for clangcl
$vcvars = "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat"
}
## A multiline commit message containing "=" can interact badly with this
## hack to extract the environment from vcvarsall.bat
Remove-Item env:BUILD_SOURCEVERSIONMESSAGE
## ask cmd.exe to output the environment table after the batch file completes
$tempFile = [IO.Path]::GetTempFileName()
cmd /c " `"$vcvars`" $env:arch && set > `"$tempFile`" "
## go through the environment variables in the temp file.
## for each of them, set the variable in our local environment.
Get-Content $tempFile | Foreach-Object {
if($_ -match "^(.*?)=(.*)$") {
Set-Content "env:\$($matches[1])" $matches[2]
}
}
Remove-Item $tempFile
if ($env:compiler -eq 'clang-cl') {
# drop visual studio from PATH
# (but leave INCLUDE, LIB and WindowsSdkDir environment variables set)
$env:Path = $origPath
# install llvm for clang-cl builds
DownloadFile -Source 'http://releases.llvm.org/9.0.0/LLVM-9.0.0-win64.exe' -Destination LLVM-9.0.0-win64.exe
Start-Process .\LLVM-9.0.0-win64.exe -ArgumentList '/S' -Wait
$env:Path = "C:\Program Files\LLVM\bin;$env:Path"
$env:CC = "clang-cl"
$env:CXX = "clang-cl"
# and use Windows SDK tools
$env:Path = "$env:WindowsSdkDir\bin\$env:Arch;$env:Path"
}
# add .NET framework tools to path for resgen for C# tests
# (always use 32-bit tool, as there doesn't seem to be a 64-bit tool)
if ((Get-Command "resgen.exe" -ErrorAction SilentlyContinue) -eq $null) {
$env:Path = "$env:WindowsSDK_ExecutablePath_x86;$env:Path"
}
echo "=== END CI setup ==="
if ($env:backend -eq 'ninja') {
ninja --version
} else {
MSBuild /version
}
echo "=== PATH BEGIN ==="
echo ($env:Path).Replace(';',"`n")
echo "=== PATH END ==="
echo ""
echo "Locating Python:"
where.exe python
python --version
# Needed for running unit tests in parallel.
python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist
echo ""
echo "Locating cl, rc, link:"
where.exe cl
where.exe rc
where.exe link
echo ""
echo "=== Start running tests ==="
# Starting from VS2019 Powershell(?) will fail the test run
# if it prints anything to stderr. Python's test runner
# does that by default so we need to forward it.
cmd /c 'python 2>&1' run_tests.py --backend $(backend)
- task: PowerShell@2
inputs:
targetType: 'filePath'
filePath: .\ci\run.ps1
- task: PublishTestResults@2
inputs:

@ -1,75 +0,0 @@
param (
[string]$Version = $null
)
Set-StrictMode -Version latest
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
echo "=== Installing DMD ==="
# default installation directory
$dmd_install = "C:\D"
$dmd_version_file = "C:\cache\DMD_LATEST"
if (!$Version) {
#echo "Fetching latest DMD version..."
$dmd_latest_url = "http://downloads.dlang.org/releases/LATEST"
$retries = 10
echo ('Downloading {0} ...' -f $dmd_latest_url)
for ($i = 1; $i -le $retries; $i++) {
try {
[system.io.directory]::CreateDirectory((Split-Path -parent $dmd_version_file)) > $null
Invoke-WebRequest -URI $dmd_latest_url -OutFile $dmd_version_file
echo '... DONE'
break
} catch [net.WebException] {
if ($i -eq $retries) {
break
}
$backoff = (10 * $i) # backoff 10s, 20s, 30s...
echo ('{0}: {1}' -f $dmd_latest_url, $_.Exception.Message)
echo ('Retrying in {0}s...' -f $backoff)
Start-Sleep -m ($backoff * 1000)
} catch {
throw
}
}
if (Test-Path $dmd_version_file) {
$dmd_version = Get-Content -Path $dmd_version_file
} else {
throw "Failed to resolve latest DMD version"
}
} else {
$dmd_version = $Version
}
$dmd_url = "http://downloads.dlang.org/releases/2.x/$dmd_version/dmd.$dmd_version.windows.zip"
$dmd_filename = [System.IO.Path]::GetFileName($dmd_url)
$dmd_archive = Join-Path ($env:temp) $dmd_filename
echo "Downloading $dmd_filename..."
$retries = 10
for ($i = 1; $i -le $retries; $i++) {
try {
(New-Object net.webclient).DownloadFile($dmd_url, $dmd_archive)
break
} catch [net.WebException] {
if ($i -eq $retries) {
throw # fail on last retry
}
$backoff = (10 * $i) # backoff 10s, 20s, 30s...
echo ('{0}: {1}' -f $dmd_url, $_.Exception.Message)
echo ('Retrying in {0}s...' -f $backoff)
Start-Sleep -m ($backoff * 1000)
}
}
echo "Extracting $dmd_filename..."
Expand-Archive $dmd_archive -Force -DestinationPath $dmd_install
# add to environment path
echo "Installing DMD..."
$dmd_bin = Join-Path $dmd_install "dmd2\windows\bin"
$Env:Path = $Env:Path + ";" + $dmd_bin
echo "Testing DMD..."
& dmd.exe --version

@ -0,0 +1,71 @@
python ./skip_ci.py --base-branch-env=SYSTEM_PULLREQUEST_TARGETBRANCH --is-pull-env=SYSTEM_PULLREQUEST_PULLREQUESTID --base-branch-origin
if ($LastExitCode -ne 0) {
exit 0
}
# remove Chocolately, MinGW, Strawberry Perl from path, so we don't find gcc/gfortran and try to use it
$env:Path = ($env:Path.Split(';') | Where-Object { $_ -notmatch 'mingw|Strawberry|Chocolatey' }) -join ';'
# download and install prerequisites
function DownloadFile([String] $Source, [String] $Destination) {
$retries = 10
echo "Downloading $Source"
for ($i = 1; $i -le $retries; $i++) {
try {
(New-Object net.webclient).DownloadFile($Source, $Destination)
break # succeeded
} catch [net.WebException] {
if ($i -eq $retries) {
throw # fail on last retry
}
$backoff = (10 * $i) # backoff 10s, 20s, 30s...
echo ('{0}: {1}' -f $Source, $_.Exception.Message)
echo ('Retrying in {0}s...' -f $backoff)
Start-Sleep -m ($backoff * 1000)
}
}
}
if ($env:backend -eq 'ninja') { $dmd = $true } else { $dmd = $false }
DownloadFile -Source https://github.com/mesonbuild/cidata/releases/download/ci1/ci_data.zip -Destination $env:AGENT_WORKFOLDER\ci_data.zip
echo "Extracting ci_data.zip"
Expand-Archive $env:AGENT_WORKFOLDER\ci_data.zip -DestinationPath $env:AGENT_WORKFOLDER\ci_data
& "$env:AGENT_WORKFOLDER\ci_data\install.ps1" -Arch $env:arch -Compiler $env:compiler -Boost $true -DMD $dmd
echo "=== PATH BEGIN ==="
echo ($env:Path).Replace(';',"`n")
echo "=== PATH END ==="
echo ""
$progs = @("python","ninja","pkg-config","cl","rc","link")
foreach ($prog in $progs) {
echo ""
echo "Locating ${prog}:"
where.exe $prog
}
echo ""
echo "Ninja / MSBuld version:"
if ($env:backend -eq 'ninja') {
ninja --version
} else {
MSBuild /version
}
echo ""
echo "Python version:"
python --version
# Needed for running unit tests in parallel.
echo ""
python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist
echo ""
echo "=== Start running tests ==="
# Starting from VS2019 Powershell(?) will fail the test run
# if it prints anything to stderr. Python's test runner
# does that by default so we need to forward it.
cmd /c 'python 2>&1' run_tests.py --backend $env:backend
Loading…
Cancel
Save