From 50b723148c7ffafe8b925516e7619408c34d574b Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 22 Feb 2011 02:07:26 -0800 Subject: Teach jitterbug about reusing git repos to save bandwidth --- lib/jitterbug/Builder.pm | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index deb4c5b..14a7bc4 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -12,6 +12,7 @@ use Getopt::Long qw/:config no_ignore_case/; use File::Basename; use Git::Repository; use jitterbug::Schema; +use Cwd; #use Data::Dumper; local $| = 1; @@ -85,6 +86,7 @@ sub run_task { my ($task) = @_; my $desc = JSON::decode_json( $task->commit->content ); my $conf = $self->{'conf'}; + my $buildconf = $conf->{'jitterbug'}{'build_process'}; my $dt = DateTime->now(); $task->update({started_when => $dt}); @@ -101,16 +103,26 @@ sub run_task { $conf->{'jitterbug'}{'build'}{'dir'}, $task->project->name, ); - - debug("Removing $build_dir"); - rmtree($build_dir, { error => \my $err } ); - warn @$err if @$err; - + my ($r, $repo); + unless ($buildconf->{reuse_repo}) { + debug("Removing $build_dir"); + rmtree($build_dir, { error => \my $err } ); + warn @$err if @$err; + $repo = $task->project->url . '.git'; + $r = Git::Repository->create( clone => $repo => $build_dir ); + } else { + my $pwd = getcwd; + chdir $build_dir; + # TODO: Error Checking + debug("Cleaning git repo"); + system("git clean -dfx"); + debug("Rebasing new commits into $repo"); + system("git pull --rebase"); + chdir $pwd; + $r = Git::Repository->new( work_tree => $build_dir ); + } $self->sleep(1); # avoid race conditions - my $repo = $task->project->url . '.git'; - my $r = Git::Repository->create( clone => $repo => $build_dir ); - debug("Checking out " . $task->commit->sha256 . " from $repo into $build_dir\n"); $r->run( 'checkout', $task->commit->sha256 ); -- cgit 1.4.1 From d18aadd5b5e02a82935ef0040710087db4fb4849 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 22 Feb 2011 19:01:15 +0000 Subject: We really only need to fetch new commits, not pull or rebase them. rbuels++ --- lib/jitterbug/Builder.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 14a7bc4..dfb3f99 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -116,8 +116,8 @@ sub run_task { # TODO: Error Checking debug("Cleaning git repo"); system("git clean -dfx"); - debug("Rebasing new commits into $repo"); - system("git pull --rebase"); + debug("Fetching new commits into $repo"); + system("git fetch"); chdir $pwd; $r = Git::Repository->new( work_tree => $build_dir ); } -- cgit 1.4.1 From f92fcb669859ae4a00ef0ba9827f889d80c5197b Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 22 Feb 2011 19:13:58 +0000 Subject: Create a new repo for the first time, even if reuse_repo is true --- lib/jitterbug/Builder.pm | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index dfb3f99..1e71e2e 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -103,23 +103,28 @@ sub run_task { $conf->{'jitterbug'}{'build'}{'dir'}, $task->project->name, ); - my ($r, $repo); + my $r; + my $repo = $task->project->url . '.git'; unless ($buildconf->{reuse_repo}) { debug("Removing $build_dir"); rmtree($build_dir, { error => \my $err } ); warn @$err if @$err; - $repo = $task->project->url . '.git'; $r = Git::Repository->create( clone => $repo => $build_dir ); } else { - my $pwd = getcwd; - chdir $build_dir; - # TODO: Error Checking - debug("Cleaning git repo"); - system("git clean -dfx"); - debug("Fetching new commits into $repo"); - system("git fetch"); - chdir $pwd; - $r = Git::Repository->new( work_tree => $build_dir ); + # If this is the first time, the repo won't exist yet + if( -e $build_dir ){ + my $pwd = getcwd; + chdir $build_dir; + # TODO: Error Checking + debug("Cleaning git repo"); + system("git clean -dfx"); + debug("Fetching new commits into $repo"); + system("git fetch"); + chdir $pwd; + $r = Git::Repository->new( work_tree => $build_dir ); + } else { + $r = Git::Repository->create( clone => $repo => $build_dir ); + } } $self->sleep(1); # avoid race conditions -- cgit 1.4.1 From 3dc954db25b482074bb8abff5aaa51de002d3b5d Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 23 Feb 2011 00:15:10 +0000 Subject: Make reusing git repos actually work This code is not pretty, but it works. Currently, Git::Repository is eschewed because it was giving odd errors. --- lib/jitterbug/Builder.pm | 24 ++++++++++++++++-------- scripts/capsule.sh | 2 ++ 2 files changed, 18 insertions(+), 8 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 1e71e2e..50df80c 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -98,11 +98,11 @@ sub run_task { $task->project->name, $task->commit->sha256, ); + my $dir = $conf->{'jitterbug'}{'build'}{'dir'}; + mkdir $dir unless -d $dir; + + my $build_dir = dir($dir, $task->project->name); - my $build_dir = dir( - $conf->{'jitterbug'}{'build'}{'dir'}, - $task->project->name, - ); my $r; my $repo = $task->project->url . '.git'; unless ($buildconf->{reuse_repo}) { @@ -112,24 +112,32 @@ sub run_task { $r = Git::Repository->create( clone => $repo => $build_dir ); } else { # If this is the first time, the repo won't exist yet - if( -e $build_dir ){ + debug("build_dir = $build_dir"); + if( -d $build_dir ){ my $pwd = getcwd; chdir $build_dir; # TODO: Error Checking debug("Cleaning git repo"); system("git clean -dfx"); debug("Fetching new commits into $repo"); - system("git fetch"); + system("git pull --rebase"); chdir $pwd; $r = Git::Repository->new( work_tree => $build_dir ); } else { - $r = Git::Repository->create( clone => $repo => $build_dir ); + debug("Creating new repo"); + my $pwd = getcwd; + debug("pwd=$pwd"); + chdir $build_dir; + system("git clone $repo $build_dir"); + #$r = Git::Repository->create( clone => $repo => $build_dir ); + chdir $pwd; } } $self->sleep(1); # avoid race conditions debug("Checking out " . $task->commit->sha256 . " from $repo into $build_dir\n"); - $r->run( 'checkout', $task->commit->sha256 ); + # $r->run( 'checkout', $task->commit->sha256 ); + system("git checkout " . $task->commit->sha256 ); my $builder = $conf->{'jitterbug'}{'build_process'}{'builder'}; diff --git a/scripts/capsule.sh b/scripts/capsule.sh index 30c3877..9fca79e 100755 --- a/scripts/capsule.sh +++ b/scripts/capsule.sh @@ -20,6 +20,8 @@ function jitterbug_build () { perl Build.PL >> $logfile 2>&1 # ./Build installdeps is not available in older Module::Build's cpanm --installdeps . >> $logfile 2>&1 + # Run this again in case our Build is out of date (suboptimal) + perl Build.PL >> $logfile 2>&1 HARNESS_VERBOSE=1 ./Build test --verbose >> $logfile 2>&1 elif [ -f 'Makefile.PL' ]; then echo "Found Makefile.PL" -- cgit 1.4.1 From bf1966e4a3160a0fc31430e4caea8d87d69b5937 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 23 Feb 2011 00:25:30 +0000 Subject: Turn off debug statements by default --- lib/jitterbug/Builder.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 50df80c..c597a65 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -16,7 +16,7 @@ use Cwd; #use Data::Dumper; local $| = 1; -use constant DEBUG => 1; +use constant DEBUG => 0; sub new { my $self = bless {} => shift; -- cgit 1.4.1 From 87de41db2a0083c13eec2a38153b2f83d2c75ae4 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 23 Feb 2011 01:10:40 +0000 Subject: Run the git commands in the correct repo --- lib/jitterbug/Builder.pm | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index c597a65..9c7dc2f 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -137,7 +137,10 @@ sub run_task { debug("Checking out " . $task->commit->sha256 . " from $repo into $build_dir\n"); # $r->run( 'checkout', $task->commit->sha256 ); + my $pwd = getcwd; + chdir $build_dir; system("git checkout " . $task->commit->sha256 ); + chdir $pwd; my $builder = $conf->{'jitterbug'}{'build_process'}{'builder'}; -- cgit 1.4.1 From ddfce3ec926451b3b3d8bc2302dab15c5e99a250 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 23 Feb 2011 12:41:17 -0800 Subject: Add a useful comment --- lib/jitterbug/Builder.pm | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 9c7dc2f..9abef14 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -167,6 +167,8 @@ sub run_task { while (<$fh>){ $lines .= $_; } + # if $result is undefined, either there was a build failure + # or the test output is not from a TAP harness ($result) = $lines =~ /Result:\s(\w+)/; my ( $name, ) = basename($version); $name =~ s/\.txt//; -- cgit 1.4.1 From 999793d6a7bd86c4191e38bc04dcac5da97b5fbb Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 23 Feb 2011 21:03:08 +0000 Subject: Explicity fetch and rebase to avoid configuration issues --- lib/jitterbug/Builder.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 9abef14..150eae7 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -120,7 +120,9 @@ sub run_task { debug("Cleaning git repo"); system("git clean -dfx"); debug("Fetching new commits into $repo"); - system("git pull --rebase"); + system("git fetch"); + debug("Rebasing onto origin/master"); + system("git rebase origin/master"); chdir $pwd; $r = Git::Repository->new( work_tree => $build_dir ); } else { -- cgit 1.4.1 From 2c96015078dc417b20ed8f4df734c438c2d0cb8f Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 23 Feb 2011 21:25:22 +0000 Subject: Use checkout instead of rebase, since we don't know which branch name to rebase against --- lib/jitterbug/Builder.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 150eae7..2af15d2 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -16,7 +16,7 @@ use Cwd; #use Data::Dumper; local $| = 1; -use constant DEBUG => 0; +use constant DEBUG => $ENV{DEBUG} || 0; sub new { my $self = bless {} => shift; @@ -121,8 +121,8 @@ sub run_task { system("git clean -dfx"); debug("Fetching new commits into $repo"); system("git fetch"); - debug("Rebasing onto origin/master"); - system("git rebase origin/master"); + debug("Checking out correct commit"); + system("git checkout " . $task->commit->sha256 ); chdir $pwd; $r = Git::Repository->new( work_tree => $build_dir ); } else { -- cgit 1.4.1 From fcbcff656199a42e9966ddaf84c9e89291d2a3f3 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Thu, 21 Apr 2011 12:44:36 -0400 Subject: Allow the config file to specify per-project builder and builder_variables, which override the global values --- lib/jitterbug/Builder.pm | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index f6fb41e..461945c 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -81,25 +81,27 @@ sub sleep { } sub run_task { - my $self = shift; - my ($task) = @_; - my $desc = JSON::decode_json( $task->commit->content ); - my $conf = $self->{'conf'}; + my ($self,$task) = @_; + + my $desc = JSON::decode_json( $task->commit->content ); + my $conf = $self->{'conf'}; + my $project = $task->project; my $dt = DateTime->now(); $task->update({started_when => $dt}); $desc->{'build'}{'start_time'} = $dt->epoch; debug("Build Start"); + my $report_path = dir( $conf->{'jitterbug'}{'reports'}{'dir'}, - $task->project->name, + $project->name, $task->commit->sha256, ); my $build_dir = dir( $conf->{'jitterbug'}{'build'}{'dir'}, - $task->project->name, + $project->name, ); debug("Removing $build_dir"); @@ -114,7 +116,8 @@ sub run_task { debug("Checking out " . $task->commit->sha256 . " from $repo into $build_dir\n"); $r->run( 'checkout', $task->commit->sha256 ); - my $builder = $conf->{'jitterbug'}{'build_process'}{'builder'}; + my $builder = $conf->{'jitterbug'}{'projects'}{$project->name}{'builder'} + || $conf->{'jitterbug'}{'build_process'}{'builder'}; my $perlbrew = $conf->{'jitterbug'}{'options'}{'perlbrew'}; my $email_on_pass = $conf->{'jitterbug'}{'options'}{'email_on_pass'}; @@ -122,7 +125,9 @@ sub run_task { debug("email_on_pass = $email_on_pass"); debug("perlbrew = $perlbrew"); - my $builder_variables = $conf->{'jitterbug'}{'build_process'}{'builder_variables'}; + # If the project has custom builder variables, use those. Otherwise, use the global setting + my $builder_variables = $conf->{'jitterbug'}{'projects'}{$project->name}{'builder_variables'} + || $conf->{'jitterbug'}{'build_process'}{'builder_variables'}; my $builder_command = "$builder_variables $builder $build_dir $report_path $perlbrew"; -- cgit 1.4.1 From e80883cf3934400f27c5e11b8f2c0d5373454699 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 26 Apr 2011 14:52:30 -0700 Subject: Get rid of Git::Repository calls --- lib/jitterbug/Builder.pm | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 02c07bd..7bfe908 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -132,14 +132,12 @@ sub run_task { debug("Checking out correct commit"); system("git checkout " . $task->commit->sha256 ); chdir $pwd; - $r = Git::Repository->new( work_tree => $build_dir ); } else { debug("Creating new repo"); my $pwd = getcwd; debug("pwd=$pwd"); chdir $build_dir; system("git clone $repo $build_dir"); - #$r = Git::Repository->create( clone => $repo => $build_dir ); chdir $pwd; } } -- cgit 1.4.1 From 917e9f2fa279d3e295f9af3c0360b24bb8cb79a6 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 26 Apr 2011 15:07:09 -0700 Subject: Hopefully unbreak the reusing git repo feature --- lib/jitterbug/Builder.pm | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index 7bfe908..cfb579a 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -105,10 +105,6 @@ sub run_task { my $build_dir = dir($dir, $project->name); - debug("Removing $build_dir"); - rmtree($build_dir, { error => \my $err } ); - warn @$err if @$err; - $self->sleep(1); # avoid race conditions my $r; -- cgit 1.4.1 From 6f1d9af240693eeba2eec2fba8c32e08e59798ae Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 26 Apr 2011 15:07:59 -0700 Subject: Remove an unnecessary sleep --- lib/jitterbug/Builder.pm | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index cfb579a..fb4ccf7 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -94,7 +94,6 @@ sub run_task { $desc->{'build'}{'start_time'} = $dt->epoch; debug("Build Start"); - my $report_path = dir( $conf->{'jitterbug'}{'reports'}{'dir'}, $project->name, @@ -105,8 +104,6 @@ sub run_task { my $build_dir = dir($dir, $project->name); - $self->sleep(1); # avoid race conditions - my $r; my $repo = $task->project->url . '.git'; unless ($buildconf->{reuse_repo}) { -- cgit 1.4.1 From 7c631d168f0359be93289c6706ac376379b2d04f Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 27 Apr 2011 20:42:50 -0700 Subject: Prevent a use of uninitialized value warning --- lib/jitterbug/Builder.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/jitterbug/Builder.pm') diff --git a/lib/jitterbug/Builder.pm b/lib/jitterbug/Builder.pm index fb4ccf7..366878f 100644 --- a/lib/jitterbug/Builder.pm +++ b/lib/jitterbug/Builder.pm @@ -154,7 +154,7 @@ sub run_task { # If the project has custom builder variables, use those. Otherwise, use the global setting my $builder_variables = $conf->{'jitterbug'}{'projects'}{$project->name}{'builder_variables'} - || $conf->{'jitterbug'}{'build_process'}{'builder_variables'}; + || $conf->{'jitterbug'}{'build_process'}{'builder_variables'} || ''; my $builder_command = "$builder_variables $builder $build_dir $report_path $perlbrew"; -- cgit 1.4.1