Revision control
Copy as Markdown
#!/usr/bin/env perl
# Copyright (C) 2017 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
=head1 NAME
qt-jenkins-deploy.pl - deploy Jenkins configuration from templates
=head1 SYNOPSIS
# jenkins.conf contains template for various Jenkins jobs and nodes...
$ ./qt-jenkins-deploy.pl --conf jenkins.conf --user admin
Jenkins API token or password for admin: ******
job QtBase_master_Integration: no changes required
job QtDeclarative_master_Integration: updated:
- this
- that
+ newthis
+ newthat
node ubuntu-builder-01: created
node ubuntu-builder-02: created
(...)
Quickly deploy or update a set of Jenkins jobs and nodes, based on some
local configuration files.
This script provides a means for maintaining Jenkins configuration as
flat files (e.g. under source control with code review), and deploying
various Jenkins jobs or nodes from a single template (which is otherwise
cumbersome to accomplish).
=head2 OPTIONS
=over
=item --conf <file>
Mandatory. Use the specified configuration <file>.
See L<CONFIGURATION FILE> for details on the configuration format.
=item --user <username>
Use the specified username for authentication with Jenkins.
This may be omitted if the Jenkins instance does not require authentication.
=item --password <password>
Use the specified API token or password for authentication with Jenkins.
If omitted, and --user is specified, the script will prompt.
=item --dry-run
If set, no Jenkins data will be modified; the script will instead print
the changes it would make, then exit.
=back
=head2 TEMPLATE FILES
The Jenkins job and node settings are stored in XML files. This script
loads XML templates using the Template Toolkit.
Here is a complete example of a templated node XML:
<slave>
<name>[% name %]</name>
<description>CI node. Contact: [% contact %] [automatically created]</description>
<remoteFS>[% root %]</remoteFS>
<label>[% labels %]</label>
<numExecutors>1</numExecutors>
<mode>EXCLUSIVE</mode>
<retentionStrategy class="hudson.slaves.RetentionStrategy$Always"/>
<launcher class="hudson.slaves.JNLPLauncher"/>
</slave>
When the template is processed, directives within '[% ... %]' blocks are replaced
with values calculated from the configuration file. For more details on template
=head2 CONFIGURATION FILE
This script uses an INI-style configuration file to declare the jobs and
nodes to be managed.
Unless otherwise stated, each variable mentioned here may also be used from within
a template file.
The following sections are handled:
=over
=item [job.<job_name>]
Manage a job of the given name.
Within a job template, this is accessible as the 'name' variable.
The following attributes may be configured on a job:
=over
=item enabled
If 0, the Jenkins job is disabled. Defaults to 1.
=item pretend
If 1, the Jenkins job should operate in pretend mode.
The semantics of pretend mode depends on the job content, but generally
implies that no destructive/permanent effects result from the job.
Defaults to 0.
=item gerrit_host
Hostname of the Gerrit server.
=item gerrit_port
Port number of the Gerrit server. Defaults to 29418.
=item gerrit_project
Project name on the Gerrit server (e.g. "qt/qtbase").
The default is derived from the first portion of the job name;
for example, a job name of "QtBase_master_Integration" results
in "qt/qtbase".
=item custom_repository
Custom repository if needed.
Defaults to empty.
=item testconfig_project
Project name used for the qtqa/testconfig repository
(e.g. "QtBase master Integration"). Note that underscores and
whitespace are treated as equivalent.
The default is equal to the job name.
=item branch
Short name of the branch to test (e.g. "master").
The default is derived from the second portion of the job name;
for example, a job name of "QtBase_master_Integration" results
in "master".
=item poll_cron
cron-style line for SCM polling (see SCM poll documentation in
Jenkins for syntax details).
If omitted, SCM polling is disabled.
=item trigger_cron
cron-style line for periodic triggering of the job.
If omitted, the job is not triggered periodically.
=item publish_xunit
Parameter to enable xUnit jenkins plugin.
Default is not to publish.
=item publish_build
Parameter to publish build results via another Jenkins dashboar,
Default is not to publish.
=item run_remote_job
Parameter to enable additional build step after default job config.
Default is not to run
=item remote_jenkins
Parameter to configure remote Jenkins' URL, where to run the remote job.
=item remote_jenkins_job
Parameter to configure the remote job to be run.
=item log_days_to_keep
=item log_num_to_keep
Variables to configure how jenkins should store the log items.
=item configurations
A list of space-separated test/build configurations, e.g.
"linux-g++-32_Ubuntu_10.04_x86 win32-msvc2010_Windows_7".
It is possible to set a top-level list of configurations,
then add or subtract from that list for specific jobs by
adding a 'configuration += ...' or 'configuration -= ...' line,
as in the following example:
# default tests on each major desktop platform...
configurations = linux mac win
# ...but this job is not supported on windows
[job.Some_Unix_Thing]
configurations -= win
Note that only one 'configurations +=' or 'configurations -=' directive
may appear in each block.
=item jenkins_url
URL of the Jenkins server on which the job should be managed.
=item log_upload_url
URL to which CI logs should be uploaded (e.g. via ssh).
=item log_download_url
URL from which CI logs should be downloaded (e.g. via http, to be
linked to from gerrit).
=item job_template
Path to the template XML file to be POSTed to Jenkins.
If a relative path, it is interpreted relative to the configuration
file.
See L<TEMPLATE FILES> for more information.
=item qt_version
Qt version number, used in packaging.
=item qt_license
Qt license type (commercial/opensource), used in packaging.
=item build_type
Used for assigning build jobs to dedicated label.
=item notarize
Enable (yes) or disable (no) notarization.
=item prod_addr
Production server address
=item prod_srv_repo_base_path
Production server repository base path
=item prod_srv_repo_pending_area_dir
Production server directory for updated repository content
=item repo_staging_server_test_repo_dist_work
Production server temporary directory for online repository update process
=item cores
Number of cores in virtual machine
=item repositories
repository sources to be copied into virtual machine
=item label_with_branch
Information if branch is added into label
=item artifacts_download_url
The url where to download the build artifacts, typically via http.
=item artifacts_upload_host
The host server to be used when uploading build artifacts, should be
accessible with SSH/SCP.
=item artifacts_upload_path
The base path on host server where to upload build artifacts.
=item send_status_mail
Parameter to send status e-mail from jenkins by using extended e-mail plugin.
=item boot_script
File that contains Groovy script for booting the machine down after the build.
=item build_trigger
Contains next build job to start if build succeeds.
=item build_flow
Contains the information about the build flow e.g. 5.4.1
=item qt5sha1
Contains sha1 for qt5
=item use_create_nodes_and_vms
Set to '1' if vm-cloner is to be used instead of default cloner
=item url_trigger
Contains url to be polled to start specific build flow compilation
=item module_name
Contains module name e.g. add-on, tech-preview or some other separate qt module not part of qt5.git
=item module_version
Contains module version
=item module_sha1
Contains module sha1
=item git_module_repo
Contains module repository
=item git_module_repo_branch
Contains module repository branch
=item release_description_file
Contains addons application release description file name
=item do_update_staging_repository
Contains information whether we want to update staging repository
=item do_update_production_repository
Contains information whether we want to update prodcution repository
=item force_version_number_increase
Contains information whether we want to force version number increase for an online repository build
=item create_maintenance_tool_resource_file
Contains information wheater maintenance tool resource file is created or not
=item execute_postbuild_action
If set then execute post build action for the build
=item module_doc_build_qt_dependency_package_uri
Contains information whether module needs libs outside essentials package
=item build_combination
Contains combination Filter for build. Filtering build
targets for different releases.
=item squish_test_suite_path
Path to Squish installation
=item squish_config_path
Path to job specifig configurations, needed by Squish Plugin.
=item squish_test_suites
Space separated list of test suites to run, combined with squish_test_suite_path should form valid path to test suite
=item auth_token
Authorization token to be used when triggering the build from other scripts
=item job_description
Description for the job.
=item test_script
Test script that is run in shell/batch without Squish
=item cores
Number of cores to be requested for a job, defaults to 2.
=item installer_type
Type of Qt installer to use in RTA test. Online/Offline/Monolite/Pyside.
=item irc_channel
IRC channel to notify
=item pip_path
Path to Qt for Python wheels.
=item python_version
Python version to be used. Version 2 used as default.
=item ts_product
Product name when logging test data to Squish Team Server.
=item ts_labels
Labels for the job data sent to Squish Team Server.
=item ts_batch
Batch info string for the data sent to Squish Team Server.
=item ts_enabled
Whether to enable sending logs to Squish Team Server.
=item qt_creator
QtCreator version preview/released.
=back
=item [node.<node_basename>]
Manage a node or set of nodes of the given basename.
In this context, the name of a node contains a trailing number,
while the basename of a node is the part of the name prior
to the trailing '-' and number:
basename: ubuntu-builder
name: ubuntu-builder-10
'basename' and 'name' may be used from within template files, with
the above semantics.
The following attributes may be configured on a node:
=over
=item basename
Overrides the default basename of the node (described above).
=item contact
A contact email address for this node, e.g. someone who can
be contacted if the node is unexpectedly offline.
=item node_root
The path to the Jenkins root directory on the node,
e.g. "/work", "c:\work". It is recommended to keep this very
short and with no spaces.
=item labels
The labels to be set on the node (space-separated).
In the Qt CI setup, the labels are primarily used as the test
configurations a node is able to participate in. Windows nodes
should also have the 'windows' label.
Example:
labels = win32-msvc2010_Windows_7 win32-msvc2010_developer-build_qtnamespace_Windows_7 windows
=item range
Numeric range of identically configured nodes to manage.
May be a single integer or an (x..y) style range.
The numeric suffix of managed nodes will always be zero-padded to at
least two digits, even if the range only requires a single digit.
For example:
[node.ubuntu-builder]
Range = 11..20
# creates ubuntu-builder-11, ubuntu-builder-12, ..., ubuntu-builder-20
[node.mac-builder]
Range = 1
# creates mac-builder-01 only
=item git_location
The path to the git command to be used on this node.
This is generally required only on Windows nodes, as Jenkins seems unable
to find git.cmd from PATH (and it is undesirable to have git.exe in PATH).
Example:
git_location = c:\Program Files\Git\cmd\git.cmd
=item node_template
Path to the template XML file to be POSTed to Jenkins.
If a relative path, it is interpreted relative to the configuration
file.
See L<TEMPLATE FILES> for more information.
=back
=item [node.<node_basename>.environment]
Set environment variables on the node.
All keys in this block will be applied as environment variables.
For example, it is appropriate to set Visual Studio environment variables here,
if the node will be used exclusively for Visual Studio builds:
[node.windows-builder.environment]
INCLUDE = C:\Program Files\Microsoft Visual Studio 10.0\VC\INCLUDE;c:\openssl\include;...
LIB = C:\Program Files\Microsoft Visual Studio 10.0\VC\LIB;c:\openssl\lib;
(...)
The entire contents of this block are available as the 'environment' variable
from within templates.
To append to an environment variable rather than override it, use the key
<VARIABLE>+<SOME_ID>, where VARIABLE is the variable to append to and
SOME_ID is any unique identifier. For example, to append to PATH:
PATH+MSVC = C:\Program Files\Microsoft Visual Studio 10.0\VSTSDB\Deploy;C:\Program Files\Microsoft SDKs\Windows\v7.0A\bin
Note that this is a virtually undocumented Jenkins feature, and not a feature
implemented by this script, so take care when using it.
=back
Default values for jobs and nodes can be set at the top of the configuration
file, outside of any block. For example, if all jobs use the same gerrit server,
gerrit_host should be specified once near the top of the configuration file,
and omitted from each job block.
=cut
package QtQA::App::JenkinsDeploy;
use strict;
use warnings;
use AnyEvent::HTTP;
use AnyEvent;
use Carp qw(confess croak);
use Config::Tiny;
use Coro;
use Data::Compare;
use English qw( -no_match_vars );
use File::Basename;
use File::Spec::Functions qw( :ALL );
use FindBin;
use Getopt::Long qw( GetOptionsFromArray );
use HTTP::Headers;
use IO::Prompt;
use Pod::Usage qw( pod2usage );
use Readonly;
use Template;
use Text::Diff;
use URI;
use XML::Simple;
use lib catfile( $FindBin::Bin, qw(.. lib perl5) );
use QtQA::WWW::Util qw(:all);
# User-Agent used for HTTP requests/responses
Readonly my $USERAGENT => __PACKAGE__;
#============================== static ========================================
# Returns 1 iff xml1, xml2 are identical after parsing
# (i.e. if they are _semantically_ identical - comments, whitespace ignored).
sub compare_xml
{
my ($xml1, $xml2) = @_;
my $d1 = XMLin( $xml1 );
my $d2 = XMLin( $xml2 );
return Compare( $d1, $d2 );
}
#=============================== instance =====================================
# Calculates and returns an HTTP BASIC authorization string using
# the configured username and password.
#
# If a username is known, but no password is known, prompts for password on
# first use.
sub http_basic_auth_string
{
my ($self) = @_;
if ($self->{ auth }) {
return $self->{ auth };
}
my $user = $self->{ user } || confess 'internal error: called with no username';
# prompt for password on first use, and don't store unnecessarily
my $password = $self->{ password } || prompt( "Jenkins API token or password for $user: ", '-echo', '*' );
my $h = HTTP::Headers->new();
$h->authorization_basic( $user, $password );
return $self->{ auth } = $h->header( 'Authorization' );
}
# Returns a hashref of appropriate HTTP headers.
# If $headers are provided, the contents are included in the returned hashref.
sub http_headers
{
my ($self, %in_headers) = @_;
return {
'User-Agent' => $USERAGENT,
($self->{ user } ? (Authorization => $self->http_basic_auth_string()) : ()),
%in_headers,
};
}
# Returns the config value for the specified $key.
#
# If the value is set under $block, it is returned from there;
# otherwise, if it is set under the default block, it is returned from there;
# otherwise, a fatal error occurs.
sub cfg
{
my ($self, $block, $key) = @_;
my $cfg = $self->{ cfg };
if (exists($cfg->{ $block }) && exists($cfg->{ $block }{ $key })) {
return $cfg->{ $block }{ $key };
}
if (exists($cfg->{ _ }{ $key })) {
return $cfg->{ _ }{ $key };
}
croak "configuration error: $key is not set in $block block or in default block\n";
}
# Returns the desired XML for a job, based on the configured job_template.
sub desired_job_xml
{
my ($self, $name) = @_;
my $job_template = $self->cfg( "job.$name", 'job_template' );
my $template_file = $job_template;
if (!file_name_is_absolute( $template_file )) {
$template_file = rel2abs( $template_file, dirname( $self->{ ini } ) );
}
my $tt = Template->new(
ABSOLUTE => 1
);
# Automatically deteremined gerrit project and branch, used if
# the job name matches a common pattern and if not otherwise configured.
my ($auto_gerrit_project, $auto_branch);
if (
$name =~ m{
\A
([^_]+)
_
(.+)
_Integration
\z
}xms
) {
$auto_gerrit_project = 'qt/'.lc($1);
$auto_branch = $2;
}
my $gerrit_project
= eval { $self->cfg( "job.$name", 'gerrit_project' ) }
|| $auto_gerrit_project
|| confess "job $name: can't determine gerrit project";
my $branch
= eval { $self->cfg( "job.$name", 'branch' ) }
|| $auto_branch
|| confess "job $name: can't determine branch";
# configurations start from the 'configuration' key, but may be added or subtracted by
# configurations += or configurations -=, which are simply parsed as 'configuration +' and
# 'configuration -' keys
my %configurations = map { $_ => 1 } split( /[ ,]+/, $self->cfg( "job.$name", 'configurations' ) );
foreach my $add_cfg (eval { split( /[ ,]+/, $self->cfg( "job.$name", 'configurations +' ) ) } ) {
$configurations{ $add_cfg } = 1;
}
foreach my $remove_cfg (eval { split( /[ ,]+/, $self->cfg( "job.$name", 'configurations -' ) ) } ) {
delete $configurations{ $remove_cfg };
}
my @configurations = sort keys %configurations;
# get squish test suites
my @squish_test_suites;
@squish_test_suites = eval { split( /[ ,]+/, $self->cfg( "job.$name", 'squish_test_suites' )) };
# get boot_script from ini
my $boot_script_file = eval { $self->cfg( "job.$name", 'boot_script' ) } || q{};
my $file_contents = '';
# if file is found read contents to $file_contents variable
if ($boot_script_file) {
if (!file_name_is_absolute( $boot_script_file )) {
$boot_script_file = rel2abs( $boot_script_file, dirname( $self->{ ini } ) );
}
open my $file, "<", $boot_script_file or die "Can't read file $boot_script_file\n";
$file_contents = do { local $/; <$file> };
close ($file);
}
my $data;
$tt->process(
$template_file,
{
name => $name,
gerrit_host => eval { $self->cfg( "job.$name", 'gerrit_host' ) } || q{},
gerrit_port => eval { $self->cfg( "job.$name", 'gerrit_port' ) } // 29418,
build_trigger => eval { $self->cfg( "job.$name", 'build_trigger' ) } || q{},
build_flow => eval { $self->cfg( "job.$name", 'build_flow' ) } || q{},
qt5sha1 => eval { $self->cfg( "job.$name", 'qt5sha1' ) } || q{},
url_trigger => eval { $self->cfg( "job.$name", 'url_trigger' ) } || q{},
module_name => eval { $self->cfg( "job.$name", 'module_name' ) } || q{},
module_version => eval { $self->cfg( "job.$name", 'module_version' ) } || q{},
module_sha1 => eval { $self->cfg( "job.$name", 'module_sha1' ) } || q{},
git_module_repo => eval { $self->cfg( "job.$name", 'git_module_repo' ) } || q{},
git_module_repo_branch => eval { $self->cfg( "job.$name", 'git_module_repo_branch' ) } || q{},
release_description_file => eval { $self->cfg( "job.$name", 'release_description_file' ) } || q{},
do_update_staging_repository => eval { $self->cfg( "job.$name", 'do_update_staging_repository' ) } || q{},
do_update_production_repository => eval { $self->cfg( "job.$name", 'do_update_production_repository' ) } || q{},
force_version_number_increase => eval { $self->cfg( "job.$name", 'force_version_number_increase' ) } || q{},
create_maintenance_tool_resource_file => eval { $self->cfg( "job.$name", 'create_maintenance_tool_resource_file' ) } || q{},
execute_postbuild_action => eval { $self->cfg( "job.$name", 'execute_postbuild_action' ) } || q{},
module_doc_build_qt_dependency_package_uri => eval { $self->cfg( "job.$name", 'module_doc_build_qt_dependency_package_uri' ) } || q{},
build_combination => eval { $self->cfg( "job.$name", 'build_combination' ) } || q{},
gerrit_project => $gerrit_project,
testconfig_project => eval { $self->cfg( "job.$name", 'testconfig_project' ) } // $name,
job_template => $job_template,
branch => $branch,
log_upload_url => eval { $self->cfg( "job.$name", 'log_upload_url' ) } || q{},
log_download_url => eval { $self->cfg( "job.$name", 'log_download_url' ) } || q{},
log_days_to_keep => eval {$self->cfg( "job.$name", 'log_days_to_keep' )} || q{1},
log_num_to_keep => eval {$self->cfg( "job.$name", 'log_num_to_keep' )} || q{4},
enabled => eval { $self->cfg( "job.$name", 'enabled' ) } // 1,
pretend => eval { $self->cfg( "job.$name", 'pretend' ) } // 0,
poll_cron => eval { $self->cfg( "job.$name", 'poll_cron' ) } || q{},
on_demand => eval { $self->cfg( "job.$name", 'on_demand' ) } // 0,
use_create_nodes_and_vms => eval { $self->cfg( "job.$name", 'use_create_nodes_and_vms' ) } // 0,
template_ini_prefix => eval { $self->cfg( "job.$name", 'template_ini_prefix' ) } // $branch,
trigger_cron => eval { $self->cfg( "job.$name", 'trigger_cron' ) } || q{},
publish_xunit => eval { $self->cfg( "job.$name", 'publish_xunit' ) } || q{0},
publish_build => eval { $self->cfg( "job.$name", 'publish_build' ) } || q{0},
run_remote_job => eval { $self->cfg( "job.$name", 'run_remote_job' ) } || q{0},
remote_jenkins => eval { $self->cfg( "job.$name", 'remote_jenkins' ) } || q{},
remote_jenkins_job => eval { $self->cfg( "job.$name", 'remote_jenkins_job' ) } || q{},
configurations => \@configurations,
qt_version => eval { $self->cfg( "job.$name", 'qt_version' ) } || q{},
qt_license => eval { $self->cfg( "job.$name", 'qt_license' ) } || q{},
build_type => eval { $self->cfg( "job.$name", 'build_type' ) } || q{},
notarize => eval { $self->cfg( "job.$name", 'notarize' ) } || q{},
prod_addr => eval { $self->cfg( "job.$name", 'prod_addr' ) } || q{},
prod_srv_repo_base_path => eval { $self->cfg( "job.$name", 'prod_srv_repo_base_path' ) } || q{},
prod_srv_repo_pending_area_dir => eval { $self->cfg( "job.$name", 'prod_srv_repo_pending_area_dir' ) } || q{},
repo_staging_server_test_repo_dist_work => eval { $self->cfg( "job.$name", 'repo_staging_server_test_repo_dist_work' ) } || q{},
cores => eval { $self->cfg( "job.$name", 'cores' ) } || q{},
repositories => eval { $self->cfg( "job.$name", 'repositories' ) } || q{},
label_with_branch => eval { $self->cfg( "job.$name", 'label_with_branch' ) } || q{},
artifacts_download_url => eval { $self->cfg( "job.$name", 'artifacts_download_url' ) } || q{},
artifacts_upload_host => eval { $self->cfg( "job.$name", 'artifacts_upload_host' ) } || q{},
artifacts_upload_path => eval { $self->cfg( "job.$name", 'artifacts_upload_path' ) } || q{},
send_status_mail => eval { $self->cfg( "job.$name", 'send_status_mail' ) } || q{},
boot_script => $file_contents,
squish_test_suite_path => eval { $self->cfg( "job.$name", 'squish_test_suite_path' ) } || q{},
squish_config_path => eval { $self->cfg( "job.$name", 'squish_config_path' ) } || q{},
squish_test_suites => \@squish_test_suites,
auth_token => eval { $self->cfg( "job.$name", 'auth_token' ) } || q{},
job_description => eval { $self->cfg( "job.$name", 'job_description' ) } || q{},
suffix_labels => eval { $self->cfg( "job.$name", 'suffix_labels' ) } || q{0},
test_script => eval { $self->cfg( "job.$name", 'test_script' ) } || q{},
custom_repository => eval { $self->cfg( "job.$name", 'custom_repository' ) } // 0,
cores => eval { $self->cfg( "job.$name", 'cores' ) } || q{2},
irc_channel => eval { $self->cfg( "job.$name", 'irc_channel' ) } || q{},
installer_type => eval { $self->cfg( "job.$name", 'installer_type' ) } || q{},
pip_path => eval { $self->cfg( "job.$name", 'pip_path' ) } || q{},
python_version => eval { $self->cfg( "job.$name", 'python_version' ) } || q{2},
ts_product => eval { $self->cfg( "job.$name", 'ts_product' ) } || q{},
ts_labels => eval { $self->cfg( "job.$name", 'ts_labels' ) } || q{},
ts_batch => eval { $self->cfg( "job.$name", 'ts_batch' ) } || q{},
ts_enabled => eval { $self->cfg( "job.$name", 'ts_enabled' ) } || q{},
qt_creator => eval { $self->cfg( "job.$name", 'qt_creator' ) } || q{},
},
\$data
) || die "job $name: while parsing template: ".$tt->error();
my $suffix_labels = eval { $self->cfg( "job.$name", 'suffix_labels' ) } || q{0};
if ($suffix_labels) {
# Add the configurations to the combination filter and after that replace all
# instances of the configuration with the suffix of $ini_branch
my $ini_branch = eval { $self->cfg( "job.$name", 'template_ini_prefix' ) } // $branch;
foreach my $cfg (@configurations) {
$data =~ s/<\/combinationFilter>/(cfg == \"$cfg-$ini_branch\") || <\/combinationFilter>/;
$data =~ s/\<string\>\Q$cfg\E\<\/string\>/<string>$cfg-$ini_branch<\/string>/g;
}
# Remove the excess " || " from the end of the combinationFilter string
$data =~ s/ \|\| <\/combinationFilter/<\/combinationFilter/;
}
chomp $data;
return $data;
}
# Returns the desired XML for a node, based on the configured node_template.
sub desired_node_xml
{
my ($self, %args) = @_;
my $cfg_basename = $args{ cfg_basename } || confess;
my $template_basename = $args{ template_basename } || confess;
my $name = $args{ name } || confess;
my $cfg_item = sub {
my ($key) = @_;
return $self->cfg( "node.$cfg_basename", $key );
};
my $node_template = $cfg_item->( 'node_template' );
my $template_file = $node_template;
if (!file_name_is_absolute( $template_file )) {
$template_file = rel2abs( $template_file, dirname( $self->{ ini } ) );
}
my $tt = Template->new(
ABSOLUTE => 1
);
my $data;
$tt->process(
$template_file,
{
name => $name,
node_template => $node_template,
basename => $template_basename,
contact => $cfg_item->( 'contact' ),
labels => $cfg_item->( 'labels' ),
root => $cfg_item->( 'node_root' ),
environment => $self->{ cfg }{ "node.$cfg_basename.environment" } || {},
git_location => (eval { $cfg_item->( 'git_location' ) } || q{}),
},
\$data
) || die "node $name: while parsing template: ".$tt->error();
chomp $data;
return $data;
}
# Returns dummy data used to create a new Jenkins slave with the given $name.
#
# The primary method of remotely updating objects in Jenkins is to POST an updated config.xml.
# When creating jobs, Jenkins allows the same config.xml to be sent.
# However, the node API does not support this for some reason. Instead, a form
# must be posted with embedded JSON.
#
# To simplify the creation of nodes, we create a dummy node using the data returned from
# this function, then POST an update afterwards.
sub dummy_node_postdata
{
my ($name) = @_;
my $pkg = __PACKAGE__;
my $json = <<"END_JSON";
{
"name": "$name",
"nodeDescription": "temporary node from $pkg",
"numExecutors": "1",
"remoteFS": "/",
"labelString": "",
"mode": "EXCLUSIVE",
"": ["hudson.slaves.JNLPLauncher", "hudson.slaves.RetentionStrategy\$Always"],
"launcher": {"stapler-class": "hudson.slaves.JNLPLauncher", "tunnel": "", "vmargs": ""},
"retentionStrategy": {"stapler-class": "hudson.slaves.RetentionStrategy\$Always"},
"nodeProperties": {"stapler-class-bag": "true"},
"type": "hudson.slaves.DumbSlave"
}
END_JSON
$json =~ s{(?:^|\n) *}{}msg;
return www_form_urlencoded(
name => $name,
type => 'hudson.slaves.DumbSlave',
json => $json,
);
}
# Issue an HTTP POST.
# Dies on error (any response other than 200 OK)
sub post
{
my ($self, %args) = @_;
my $data = $args{ data } || confess;
my $headers = $args{ headers } || confess;
my $label = $args{ label } || confess;
my $query = $args{ query };
my $url = URI->new( $args{ url } ) || confess;
if ($query) {
$url->query_form( %{ $query } );
}
if ($self->{ dry_run }) {
print "$label: if not in dry run mode, would POST to $url:\n$data\n\n";
return;
}
my (undef, $response_headers) = blocking_http_request(
POST => $url,
body => $data,
headers => $headers,
);
my $status = $response_headers->{ Status };
if ($status != 200) {
die "$label: failed to update: $status $response_headers->{ Reason }\n";
}
return;
}
# Ensure the specified jenkins object exists and has the given content.
sub ensure_jenkins_object
{
my ($self, %args) = @_;
my $update_url = $args{ update_url } || confess;
my $create_url = $args{ create_url } || confess;
my $update_data = $args{ update_data } || confess;
my $create_data = $args{ create_data } || $update_data;
my $update_type = $args{ update_type } || 'text/xml';
my $create_type = $args{ create_type } || $update_type;
my $name = $args{ name } || confess;
my $type = $args{ type } || confess;
my $create_then_update = $args{ create_then_update };
my $request_headers = $self->http_headers( );
my ($data, $headers) = blocking_http_request( GET => $update_url, headers => $request_headers );
my $status = $headers->{ Status };
if ($status == 404) {
print "$type $name: $update_url -> $status $headers->{ Reason }\n";
$self->post(
label => "$type $name",
url => $create_url,
query => { name => $name },
data => $create_data,
headers => $self->http_headers(
'Content-Type' => $create_type
),
);
if (!$create_then_update) {
# that's it, no more to do
return;
}
# in 'create then update' mode, we've created an object; now fetch and
# update it
print "$type $name: created a dummy object for updating\n";
($data, $headers) = blocking_http_request( GET => $update_url, headers => $request_headers );
$status = $headers->{ Status };
}
if ($status == 200) {
# object exists, see if it matches expected.
if (compare_xml( $data, $update_data )) {
print "$type $name: no changes required\n";
return;
}
$self->post(
label => "$type $name",
url => $update_url,
data => $update_data,
headers => $self->http_headers(
'Content-Type' => $update_type
),
);
my $diff = diff( \$data, \$update_data );
chomp $diff;
$diff =~ s{\n}{\n }msg;
print "$type $name: updated:\n $diff\n\n";
return;
}
die "$type $name: unexpected response when fetching $update_url: $status $headers->{ Reason }\n";
}
# Set up a job.
sub do_job
{
my ($self, $name) = @_;
local $Coro::current->{ desc } = "job $name";
my $expected_xml = $self->desired_job_xml( $name );
my $jenkins = $self->cfg( "job.$name", 'jenkins_url' );
$self->die_if_insecure( $jenkins );
# Remove any trailing /
$jenkins =~ s{/\z}{};
$self->ensure_jenkins_object(
name => $name,
type => 'job',
update_url => "$jenkins/job/$name/config.xml",
update_data => $expected_xml,
create_url => "$jenkins/createItem",
);
return;
}
sub die_if_insecure
{
my ($self, $url) = @_;
return if $self->{ allow_insecure };
return if !$self->{ user }; # no security in use
my $scheme = URI->new( $url )->scheme();
if ($scheme ne 'https') {
die "ERROR: authentication was requested, but Jenkins server ($url) is using an "
."unsecured connection.\n"
." Your username and password could be intercepted.\n"
." This check may be disabled with the '--allow-insecure' option.\n";
}
return;
}
# Set up a single node.
sub do_single_node
{
my ($self, %args) = @_;
my $cfg_basename = $args{ cfg_basename };
my $template_basename = $args{ template_basename };
my $name = $args{ name };
local $Coro::current->{ desc } = "node $name";
my $expected_xml = $self->desired_node_xml( %args );
my $jenkins = $self->cfg( "node.$cfg_basename", 'jenkins_url' );
$self->die_if_insecure( $jenkins );
# Remove any trailing /
$jenkins =~ s{/\z}{};
$self->ensure_jenkins_object(
name => $name,
type => 'node',
update_url => "$jenkins/computer/$name/config.xml",
update_data => $expected_xml,
create_url => "$jenkins/computer/doCreateItem",
create_data => dummy_node_postdata( $name ),
create_type => 'application/x-www-form-urlencoded',
create_then_update => 1,
);
return;
}
# Set up a node range.
sub do_node_range
{
my ($self, $name) = @_;
local $Coro::current->{ desc } = "node $name";
my $range = $self->cfg( "node.$name", 'range' );
my $basename = eval { $self->cfg( "node.$name", 'basename' ) } || $name;
# $range should be a number range understood by perl
# e.g. "1", "1, 2", "1..10"
if ($range !~ m{\A[0-9 ,.]+\z}) {
die "node $name: invalid range '$range'\n";
}
my @range = eval $range; ## no critic - $range already checked for correctness
if (my $error = $EVAL_ERROR) {
die "node $name: error evaluating range '$range': $error\n";
}
my @coro;
foreach my $i (@range) {
my $fullname = sprintf( "%s-%02d", $basename, $i );
push @coro, async {
$self->do_single_node(
name => $fullname,
cfg_basename => $name,
template_basename => $basename,
);
};
}
# wait for all nodes to finish
map { $_->join() } @coro;
my $count = @range;
print "node $name: all done ($count node(s))\n";
return;
}
sub new
{
my ($class) = @_;
return bless {}, $class;
}
sub run
{
my ($self, @args) = @_;
GetOptions(
'conf=s' => \$self->{ ini },
'dry-run' => \$self->{ dry_run },
'user=s' => \$self->{ user },
'password=s' => \$self->{ password },
'allow-insecure' => \$self->{ allow_insecure },
'help|h' => sub { pod2usage( 2 ) },
);
$self->{ ini } || die "Missing mandatory --conf=<file> option.\n";
$self->{ cfg } = Config::Tiny->new()->read( $self->{ ini } )
|| die "read $self->{ ini }: ".Config::Tiny->errstr();
local $OUTPUT_AUTOFLUSH = 1;
my @coro;
foreach my $block (keys %{ $self->{ cfg } }) {
if ($block =~ qr{\Ajob\.(.+)\z}) {
my $job = $1;
push @coro, async {
$self->do_job( $job );
};
} elsif ($block =~ qr{\Anode\.([^.]+)\z} ) {
my $node = $1;
push @coro, async {
$self->do_node_range( $node );
};
}
}
if (!@coro) {
die "no configured jobs or nodes.\n";
}
# wait for everything to finish
map { $_->join() } @coro;
return;
}
__PACKAGE__->new( )->run( @ARGV ) unless caller;
1;