From af2f8122e3295f9bc0809f4006b388d5f14650dc Mon Sep 17 00:00:00 2001 From: Joel Rivera Date: Thu, 13 Apr 2017 19:10:53 -0500 Subject: [PATCH 1/2] Allow to configure the timeout value for the GitInput plugin in different places. 1. From the hydra configuration file. The configuration is loaded from the "git-input" block. Currently only the "timeout" variable is been looked up in the file. # general timeout timeout = 400 # specific timeout for a particular input name timeout = 400 # use quotes when the input name has spaces <"foot with spaces"> # specific timeout for a particular input name timeout = 400 2. As an argument in the input value after the repo url and branch (and after the deepClone if is defined) "timeout=" The preference on which value is used: 1. input value 2. Block with the name of the input in the block 3. "timeout" inside the block 4. Default value of 600 seconds. (original hard-coded value) The code is generalized for more values to be configured, it might be too much for a single value on a single plugin. --- src/lib/Hydra/Plugin/GitInput.pm | 103 +++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 6 deletions(-) diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm index 1be98361..d97b4f9a 100644 --- a/src/lib/Hydra/Plugin/GitInput.pm +++ b/src/lib/Hydra/Plugin/GitInput.pm @@ -8,6 +8,11 @@ use Hydra::Helper::Nix; use Nix::Store; use Encode; use Fcntl qw(:flock); +use Env; +use Data::Dumper; + +my $CONFIG_SECTION = "git-input"; + sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -21,9 +26,84 @@ sub _isHash { sub _parseValue { my ($value) = @_; - (my $uri, my $branch, my $deepClone) = split ' ', $value; + my @parts = split ' ', $value; + (my $uri, my $branch, my $deepClone) = @parts; $branch = defined $branch ? $branch : "master"; - return ($uri, $branch, $deepClone); + my $options = {}; + my $start_options = 3; + # if deepClone has "=" then is considered an option + # and not the enabling of deepClone + if (index($deepClone, "=") != -1) { + undef $deepClone; + $start_options = 2; + } + foreach my $option (@parts[$start_options .. $#parts]) { + (my $key, my $value) = split('=', $option); + $options->{$key} = $value; + } + return ($uri, $branch, $deepClone, $options); +} + +sub _printIfDebug { + my ($msg) = @_; + print STDERR "GitInput: $msg" if $ENV{'HYDRA_DEBUG'}; +} + +=item _pluginConfig($main_config, $input_name) + +Read the configuration from the main hydra config file. + +The configuration is loaded from the "git-input" block. + +Currently only the "timeout" variable is been looked up in the file. + +The variables defined directly in the input value will override +the ones on the configuration file, to define the variables +as an input value use: = without spaces and +specify at least he repo url and branch. + + + # general timeout + timeout = 400 + + # specific timeout for a particular input name + timeout = 400 + + + # use quotes when the input name has spaces + <"foot with spaces"> + # specific timeout for a particular input name + timeout = 400 + + +=cut +sub _pluginConfig { + my ($main_config, $input_name) = @_; + my $cfg = $main_config->{$CONFIG_SECTION}; + # default values + my $values = { + timeout => 600, + }; + unless (defined $cfg) { + _printIfDebug "Unable to load $CONFIG_SECTION section\n"; + _printIfDebug "Using default values\n"; + return $values; + } else { + _printIfDebug "Parsing plugin configuration: "; + _printIfDebug Dumper($cfg); + } + if (defined $cfg->{$input_name} and %{$cfg->{$input_name}}) { + _printIfDebug "Merging sections for $input_name\n"; + $cfg = {%{$cfg}, %{$cfg->{$input_name}}}; # merge with precedense to the input name + } + if (exists $cfg->{timeout}) { + $values->{timeout} = int($cfg->{timeout}); + _printIfDebug "Using custom timeout for $input_name:"; + } else { + _printIfDebug "Using default timeout for $input_name:"; + } + _printIfDebug "$values->{timeout}\n"; + return $values; } sub fetchInput { @@ -31,7 +111,17 @@ sub fetchInput { return undef if $type ne "git"; - my ($uri, $branch, $deepClone) = _parseValue($value); + my ($uri, $branch, $deepClone, $options) = _parseValue($value); + my $cfg = _pluginConfig($self->{config}, $name); + + # give preference to the options from the input value + while (my ($opt_name, $opt_value) = each %{$options}) { + if ($opt_value =~ /\d+/) { + $opt_value = int($opt_value); + } + $cfg->{$opt_name} = $opt_value; + _printIfDebug "'$name': override '$opt_name' with input value: $opt_value\n"; + } # Clone or update a branch of the repository into our SCM cache. my $cacheDir = getSCMCacheDir . "/git"; @@ -53,8 +143,9 @@ sub fetchInput { # the remote branch for whatever the repository state is. This command mirrors # only one branch of the remote repository. my $localBranch = _isHash($branch) ? "_hydra_tmp" : $branch; - $res = run(cmd => ["git", "fetch", "-fu", "origin", "+$branch:$localBranch"], dir => $clonePath, timeout => 600); - $res = run(cmd => ["git", "fetch", "-fu", "origin"], dir => $clonePath, timeout => 600) if $res->{status}; + $res = run(cmd => ["git", "fetch", "-fu", "origin", "+$branch:$localBranch"], dir => $clonePath, + timeout => $cfg->{timeout}); + $res = run(cmd => ["git", "fetch", "-fu", "origin"], dir => $clonePath, timeout => $cfg->{timeout}) if $res->{status}; die "error fetching latest change from git repo at `$uri':\n$res->{stderr}" if $res->{status}; # If deepClone is defined, then we look at the content of the repository @@ -71,7 +162,7 @@ sub fetchInput { # This is a TopGit branch. Fetch all the topic branches so # that builders can run "tg patch" and similar. - $res = run(cmd => ["tg", "remote", "--populate", "origin"], dir => $clonePath, timeout => 600); + $res = run(cmd => ["tg", "remote", "--populate", "origin"], dir => $clonePath, timeout => $cfg->{timeout}); print STDERR "warning: `tg remote --populate origin' failed:\n$res->{stderr}" if $res->{status}; } } From a39ffba0427d7a328729de4ec4e4322bf7d984a9 Mon Sep 17 00:00:00 2001 From: Joel Rivera Date: Fri, 28 Apr 2017 21:43:35 -0500 Subject: [PATCH 2/2] Include the project and jobset names in the configuration blocks to set the timeout for a specific input in the GitInput plugin. Remove the comments about using inputs with spaces. --- src/lib/Hydra/Plugin/GitInput.pm | 50 +++++++++++++++++--------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm index d97b4f9a..250defce 100644 --- a/src/lib/Hydra/Plugin/GitInput.pm +++ b/src/lib/Hydra/Plugin/GitInput.pm @@ -49,7 +49,7 @@ sub _printIfDebug { print STDERR "GitInput: $msg" if $ENV{'HYDRA_DEBUG'}; } -=item _pluginConfig($main_config, $input_name) +=item _pluginConfig($main_config, $project_name, $jobset_name, $input_name) Read the configuration from the main hydra config file. @@ -62,58 +62,60 @@ the ones on the configuration file, to define the variables as an input value use: = without spaces and specify at least he repo url and branch. - - # general timeout - timeout = 400 - - # specific timeout for a particular input name - timeout = 400 - +Expected configuration format in the hydra config file: + + # general timeout + timeout = 400 - # use quotes when the input name has spaces - <"foot with spaces"> - # specific timeout for a particular input name - timeout = 400 - - + + # specific timeout for a particular input + timeout = 400 + + + =cut sub _pluginConfig { - my ($main_config, $input_name) = @_; + my ($main_config, $project_name, $jobset_name, $input_name) = @_; my $cfg = $main_config->{$CONFIG_SECTION}; # default values my $values = { timeout => 600, }; + my $input_block = "$project_name:$jobset_name:$input_name"; + unless (defined $cfg) { _printIfDebug "Unable to load $CONFIG_SECTION section\n"; _printIfDebug "Using default values\n"; return $values; } else { - _printIfDebug "Parsing plugin configuration: "; + _printIfDebug "Parsing plugin configuration:\n"; _printIfDebug Dumper($cfg); } - if (defined $cfg->{$input_name} and %{$cfg->{$input_name}}) { - _printIfDebug "Merging sections for $input_name\n"; - $cfg = {%{$cfg}, %{$cfg->{$input_name}}}; # merge with precedense to the input name + if (defined $cfg->{$input_block} and %{$cfg->{$input_block}}) { + _printIfDebug "Merging sections from $input_block\n"; + # merge with precedense to the input block + $cfg = {%{$cfg}, %{$cfg->{$input_block}}}; } if (exists $cfg->{timeout}) { $values->{timeout} = int($cfg->{timeout}); - _printIfDebug "Using custom timeout for $input_name:"; + _printIfDebug "Using custom timeout for $input_block:\n"; } else { - _printIfDebug "Using default timeout for $input_name:"; + _printIfDebug "Using default timeout for $input_block:\n"; } _printIfDebug "$values->{timeout}\n"; return $values; } sub fetchInput { - my ($self, $type, $name, $value) = @_; + my ($self, $type, $name, $value, $project, $jobset) = @_; return undef if $type ne "git"; my ($uri, $branch, $deepClone, $options) = _parseValue($value); - my $cfg = _pluginConfig($self->{config}, $name); - + my $cfg = _pluginConfig($self->{config}, + $project->get_column('name'), + $jobset->get_column('name'), + $name); # give preference to the options from the input value while (my ($opt_name, $opt_value) = each %{$options}) { if ($opt_value =~ /\d+/) {