diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/githubexplorer.pm | 39 | ||||
| -rw-r--r-- | lib/githubexplorer/Gexf.pm | 36 | ||||
| -rw-r--r-- | lib/githubexplorer/Profile.pm | 81 |
3 files changed, 86 insertions, 70 deletions
diff --git a/lib/githubexplorer.pm b/lib/githubexplorer.pm index 52a4ec5..a3d27c9 100644 --- a/lib/githubexplorer.pm +++ b/lib/githubexplorer.pm @@ -5,20 +5,26 @@ use YAML::Syck; use Moose; use githubexplorer::Schema; use githubexplorer::Gexf; +use IO::All; with qw/githubexplorer::Profile githubexplorer::Repositorie/; -has seed => ( isa => 'ArrayRef', is => 'rw', required => 1, lazy =>1, default => -sub { -my $self = shift; -my $profiles = $self->schema->resultset('Profiles')->search({done => {'!=', 1}}, {order_by => - 'login desc'}); -my @seeds; -while (my $p = $profiles->next) { - push @seeds, $p->login; -} -return \@seeds; -}); +has seed => ( + isa => 'ArrayRef', + is => 'rw', + required => 1, + lazy => 1, + default => sub { + my $self = shift; + my $profiles = $self->schema->resultset('Profiles') + ->search( { done => { '!=', 1 } }, { order_by => 'login desc' } ); + my @seeds; + while ( my $p = $profiles->next ) { + push @seeds, $p->login; + } + return \@seeds; + } +); has api_login => ( isa => 'Str', is => 'ro', required => 1 ); has api_token => ( isa => 'Str', is => 'ro', required => 1 ); has connect_info => ( isa => 'ArrayRef', is => 'ro', required => 1 ); @@ -42,11 +48,11 @@ sub _connect { } sub harvest_profiles { - my ( $self, $depth) = @_; + my ( $self, $depth ) = @_; $self->_connect() unless $self->has_schema; $depth //= 1; foreach my $login ( @{ $self->seed } ) { - $self->fetch_profile($login, $depth); + $self->fetch_profile( $login, $depth ); } } @@ -54,7 +60,7 @@ sub harvest_repo { my $self = shift; $self->_connect unless $self->has_schema; my $profiles = $self->schema->resultset('Profiles')->search(); - while (my $p = $profiles->next) { + while ( my $p = $profiles->next ) { $self->fetch_repo($p); } } @@ -62,8 +68,9 @@ sub harvest_repo { sub gen_graph { my $self = shift; $self->_connect unless $self->has_schema; - my $graph = githubexplorer::Gexf->new(schema => $self->schema); - $graph->profiles; + my $graph = githubexplorer::Gexf->new( schema => $self->schema ); + my $xml = $graph->profiles; + $xml > io('crawl.gexf'); } 1; diff --git a/lib/githubexplorer/Gexf.pm b/lib/githubexplorer/Gexf.pm index 0d814f0..652800e 100644 --- a/lib/githubexplorer/Gexf.pm +++ b/lib/githubexplorer/Gexf.pm @@ -22,17 +22,17 @@ has graph => ( { id => 0, type => 'string', - title => 'totalrepo' + title => 'name' }, { id => 1, type => 'string', - title => 'accountlogin' + title => 'followers_count' }, { id => 2, type => 'string', - title => 'forkedrepo' + title => 'following_count' }, ] } @@ -48,18 +48,30 @@ sub profiles { while ( my $profile = $profiles->next ) { my $node = { - id => $profile->name, - label => $profile->name, - attvalues => [ - { id => 0, value => 'total' }, - { id => 1, $profile->name }, - { id => 2, 'forked' } - ] + id => $profile->id, + label => $profile->login, + name => $profile->name, + followers_count => $profile->followers_count, + following_count => $profile->following_count, }; push @{ $self->graph->{gexf}->{graph}->{nodes}->{node} }, $node; } - use YAML::Syck; - warn Dump $self->graph; + + my $edges = $self->schema->resultset('Follow')->search(); + my $id = 0; + while ( my $edge = $edges->next ) { + my $e = { + cardinal => 1, + source => $edge->origin, + target => $edge->source, + type => 'dir', + id => $id++, + }; + push @{ $self->graph->{gexf}->{graph}->{eges}->{edge} }, $e; + } + + my $xml_out = XMLout( $self->graph, AttrIndent => 1, keepRoot => 1 ); + return $xml_out; } 1; diff --git a/lib/githubexplorer/Profile.pm b/lib/githubexplorer/Profile.pm index a2a0b2b..1af64cc 100644 --- a/lib/githubexplorer/Profile.pm +++ b/lib/githubexplorer/Profile.pm @@ -27,7 +27,7 @@ sub fetch_profile { if ( !$profile ) { my $followers = $github->followers(); sleep(1); - if (!$followers || ref $followers ne 'ARRAY') { + if ( !$followers || ref $followers ne 'ARRAY' ) { sleep(60); return; } @@ -35,9 +35,9 @@ sub fetch_profile { say "fetch profile for $login ($depth) ..."; sleep(1); my $desc = $github->show; - if (!$desc || ($desc && exists $desc->{error})) { + if ( !$desc || ( $desc && exists $desc->{error} ) ) { sleep(60); - $self->fetch_profile($login, $depth); + $self->fetch_profile( $login, $depth ); } $profile = $self->_create_profile( $login, $github->show, $depth ); return if !$profile; @@ -47,37 +47,32 @@ sub fetch_profile { } } - if ( !$profile->done ) { - my $local_depth = $depth + 1; -# my $followers = $github->followpers(); - sleep(1); - my $following = $github->following(); - - if (!$following || ref $following ne 'ARRAY') { - sleep(60); - return; - } - - # foreach my $f (@$followers) { - # say $to->login . " is followed by " . $from; - # $self->_create_relation($f, $profile, $local_depth); - # } - foreach my $f (@$following) { - # say $profile->login . " follow " . $f; - $self->_create_relation($profile, $f, $local_depth); - } - say "update profile for $login: done"; - $profile->update( { done => 1 } ); - } - - sleep(1); - $profile; + if ( !$profile->done ) { + my $local_depth = $depth + 1; + + sleep(1); + my $following = $github->following(); + + if ( !$following || ref $following ne 'ARRAY' ) { + sleep(60); + return; + } + + foreach my $f (@$following) { + $self->_create_relation( $profile, $f, $local_depth ); + } + say "update profile for $login: done"; + $profile->update( { done => 1 } ); + } + + sleep(1); + $profile; } sub _create_relation { my ( $self, $from, $to, $depth ) = @_; - say "-> create a relation from ".$from->login." to $to"; + say "-> create a relation from " . $from->login . " to $to"; if ( my $p = $self->_profile_exists($to) ) { if ( !$self->_relation_exists( $from->id, $p->id ) ) { $self->schema->txn_do( @@ -125,21 +120,23 @@ sub _create_profile { $profile->{depth} = $depth; - my $profile_rs; my $err; + my $profile_rs; + my $err; try { - $self->schema->txn_do( - sub { - $profile_rs - = $self->schema->resultset('Profiles')->create($profile); - } - ); -}catch{ - warn $_; - $err = 1; -}; -return if $err; - say '-> '.$profile_rs->login . "'s profile created"; + $self->schema->txn_do( + sub { + $profile_rs + = $self->schema->resultset('Profiles')->create($profile); + } + ); + } + catch { + warn $_; + $err = 1; + }; + return if $err; + say '-> ' . $profile_rs->login . "'s profile created"; return $profile_rs; } |