1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
package MooseX::UserAgent::Async;
use Moose::Role;
with qw/MooseX::UserAgent::Content MooseX::UserAgent::Cache/;
use AnyEvent::HTTP;
use HTTP::Response;
sub fetch {
my ( $self, $url ) = @_;
my $status = AnyEvent->condvar;
$AnyEvent::HTTP::USERAGENT = $self->useragent_conf->{name};
my $last_modified = $self->get_ua_cache($url);
my $request_headers = { 'Accept-Encoding' => 'gzip', };
$request_headers->{'If-Modified-Since'} = $last_modified
if $last_modified;
$request_headers->{'Content-Length'} = $self->useragent_conf->{max_size}
if $self->useragent_conf->{max_size};
$request_headers->{'From'} = $self->useragent_conf->{mail}
if $self->useragent_conf->{mail};
http_request
GET => $url,
headers => $request_headers,
timeout => $self->useragent_conf->{timeout},
sub {
my ( $data, $headers ) = @_;
my $response = HTTP::Response->new;
$response->content($data);
$response->code( delete $headers->{Status} );
foreach my $header ( keys %$headers ) {
$response->header( $header => $headers->{$header} );
}
$self->store_ua_cache( $url, $response );
$status->send($response);
};
return $status->recv;
}
1;
__END__
=head1 NAME
RTGI::Role::UserAgent::Async - Fetch an url using AnyEvent::HTTP
=head1 SYNOPSIS
package Foo;
use Moose;
with qw/MooseX::UserAgent::Async/;
has useragent_conf => (
isa => 'HashRef',
default => sub {
{ name => 'myownbot', };
}
);
my $res = $self->fetch($url, $cache);
...
my $content = $self->get_content($res);
=head1 DESCRIPTION
=head2 METHODS
=over 4
=item B<fetch>
This method will fetch a given URL. This method handle only the http
protocol.
If there is a cache configuration, the url will be checked in the cache,
and if there is a match, a 304 HTTP code will be returned.
Return a HTTP::Response object. This headers are different from a
HTTP::Response from LWP::UserAgent.
=back
=head1 BUGS AND LIMITATIONS
=head1 AUTHOR
franck cuny C<< <franck.cuny@rtgi.fr> >>
=head1 LICENCE AND COPYRIGHT
Copyright (c) 2009, RTGI
All rights reserved.
L<http://rtgi.fr/>
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself. See L<perlartistic>.
|
