Errors while debugging code using HTML::TreeBuilder::XPath - apache

Question

I am getting some errors when I try to debug the following code.
Note that it fetches the data from approx 6,000 fields from the http://europa.eu/youth/volunteering/evs-organisation#open
After parsing each page, check for the existence of the next › link at the bottom.
View-source is a browser based command. It tells the browser to output the response in plain text rather than render it based on its actual content type, HTML in this case. You should not need to include view-source in your URL.
Here we have a script that extracts the data out of each block and cleans it up a little. The browse function is generic. It takes an input reference which contains the URL and XPaths of the parent and children in order to construct the output ref. It is just an approach: it does not yet navigate across each page,
In a rough script I tested, I fetched the total results using //span[#class="ey_badge"] then the max page using
my $page_max = $results / 21;
$page_max = int( $page_max ) == $page_max ? $page_max-- : int( $page_max ) ;
See the errors
martin#linux-3645:~/dev/perl> perl eu.pl
syntax error at eu.pl line 81, near "our "
Global symbol "$iterator_organizations" requires explicit package name at eu.pl line 81.
Can't use global #_ in "my" at eu.pl line 84, near "= #_"
Missing right curly or square bracket at eu.pl line 197, at end of line
Execution of eu.pl aborted due to compilation errors.
martin#linux-3645:~/dev/perl> ^C
martin#linux-3645:~/dev/perl>
It fetches the data from approx 6,000 fields from http://europa.eu/youth/volunteering/evs-organisation#open
See the code
use strict;
use warnings FATAL => qw#all#;
use LWP::UserAgent;
use HTML::TreeBuilder::XPath;
use Data::Dumper;
my $handler_relurl = sub { q#https://europa.eu# . $_[0] };
my $handler_trim = sub { $_[0] =~ s#^\s*(.+?)\s*$#$1#r };
my $handler_val = sub { $_[0] =~ s#^[^:]+:\s*##r };
my $handler_split = sub { [ split $_[0], $_[1] ] };
my $handler_split_colon = sub { $handler_split->( qr#; #, $_[0] ) };
my $handler_split_comma = sub { $handler_split->( qr#, #, $_[0] ) };
my $conf = {
url => q#https://europa.eu/youth/volunteering/evs-organisation_en#,
parent => q#//div[#class="vp ey_block block-is-flex"]#,
children => {
internal_url => [ q#//a/#href#, [ $handler_relurl ] ],
external_url => [ q#//i[#class="fa fa-external-link fa-lg"]/parent::p//a/#href#, [ $handler_trim ] ],
title => [ q#//h4# ],
topics => [ q#//div[#class="org_cord"]#, [ $handler_val, $handler_split_colon ] ],
location => [ q#//i[#class="fa fa-location-arrow fa-lg"]/parent::p#, [ $handler_trim ] ],
hand => [ q#//i[#class="fa fa-hand-o-right fa-lg"]/parent::p#, [ $handler_trim, $handler_split_comma ] ],
pic_number => [ q#//p[contains(.,'PIC no')]#, [ $handler_val ] ],
}
};
print Dumper browse( $conf );
sub browse {
my $conf = shift;
my $ref = [ ];
my $lwp_useragent = LWP::UserAgent->new( agent => q#IE 6#, timeout => 10 );
my $response = $lwp_useragent->get( $conf->{url} );
die $response->status_line unless $response->is_success;
my $content = $response->decoded_content;
my $html_treebuilder_xpath = HTML::TreeBuilder::XPath->new_from_content( $content );
my #nodes = $html_treebuilder_xpath->findnodes( $conf->{parent} );
for my $node ( #nodes ) {
push #$ref, { };
while ( my ( $key, $val ) = each %{ $conf->{children} } ) {
my $xpath = $val->[0];
my $handlers = $val->[1] // [ ];
$val = ( $node->findvalues( qq#.$xpath# ) )[0] // next;
$val = $_->( $val ) for #$handlers;
$ref->[-1]->{$key} = $val;
}
}
return $ref;
}
{
'internal_url' => 'https://europa.eu/youth/volunteering/organisation/948417016_en',
'external_url' => 'http://www.apd.ge',
'location' => 'Tbilisi, Georgia',
'title' => '"Academy for Peace and Development" Union',
'topics' => [
'Access for disadvantaged',
'Youth (Participation, Youth Work, Youth Policy)',
'Intercultural/intergenerational education and (lifelong)learning'
],
'pic_number' => '948417016',
'hand' => [
'Receiving',
'Sending'
]
}
our $iterator_organizations = sub {
my ( $browser, $parent ) = #_;
my $url = q#https://europa.eu/youth/volunteering/evs-organisation_en#;
my $nodes = $browser->nodes( url => $url );
my $iterator = sub {
return shift #$nodes;
};
return ( $iterator, 1 );
our $iterator_organizations_b = sub {
my ( $browser, $parent ) = #_;
my $url = q#https://europa.eu/youth/volunteering/evs-organisation_en#;
my $uri = URI->new( $url );
my $xpath = q#//div[#class="vp ey_block block-is-flex"]#;
my $nodes = [ ];
my $page = 0;
my $results = $parent->{results};
my $page_max = $results / 21;
$page_max = int($page_max) == $page_max ? $page_max-- : int($page_max);
my $iterator_uri = sub {
$uri->query_form( page => $page++ );
return $page > 2 ? undef : $uri ; # $page_max;
};
my $iterator_node = sub {
unless ( #$nodes ) {
my $uri = $iterator_uri->( ) // return undef;
my $options = $page == 1 ? { tree => $parent->{_node} } : { url => $uri->as_string };
$nodes = $browser->nodes( %$options, xpath => $xpath );
}
return shift #$nodes;
};
return ( $iterator_node, 0 );
};
our $iterator_organization = sub {
my ( $browser, $parent ) = #_;
my $url = $parent->{internal_url};
my $nodes = $browser->nodes( url => $url );
my $iterator = sub {
return shift #$nodes;
};
return ( $iterator, 1 );
};
sub organizations {
my ( $self, $options ) = ( shift, { #_ } );
my $map = [
$Massweb::Browser::Europa::iterator_organizations,
results => q#.//span[#class="ey_badge"]#,
organizations => [
$Massweb::Browser::Europa::iterator_organizations_b,
internal_url => [ q#.//a/#href#, $Massweb::Browser::Europa::handler_url ],
external_url => [ q#.//i[#class="fa fa-external-link fa-lg"]/parent::p//a/#href#, $Massweb::Browser::handler_trim ],
title => q#.//h4#,
topics => [ q#.//div[#class="org_cord"]#, $Massweb::Browser::handler_val, $Massweb::Browser::handler_list_colon ],
location => [ q#.//i[#class="fa fa-location-arrow fa-lg"]/parent::p#, $Massweb::Browser::handler_trim ],
hand => [ q#.//i[#class="fa fa-hand-o-right fa-lg"]/parent::p#, $Massweb::Browser::handler_trim, $Massweb::Browser::handler_list_comma ],
pic_number => [ q#.//p[contains(.,'PIC no')]#, $Massweb::Browser::handler_val ],
recruiting => [ q#boolean(.//i[#class="fa fa-user-times fa-lg"])#, $Massweb::Browser::handler_bool_rev ],
_ => \&organization,
],
];
my $organizations = $self->browse( map => $map );
return $organizations;
}
sub organization {
my ( $self, $options ) = ( shift, { #_ } );
my $map = [
sub { $Massweb::Browser::Europa::iterator_organization->( $_[0], $options ) },
#title => q#.//h1#,
description => q#.//div[#class="ey_vp_detail_page"]/p#,
];
my $organization = $self->browse( map => $map );
return $organization;
}

Answer 1

The problem appears to be the block/anonymous hash starting 'internal_url'. I can't imagine what you intend there but it is a syntax error and would have no effect if you fixed it
Why are you declaring so many subroutine references like our $iterator_organizations = sub { ... } instead of using standard subroutines? It is a very strange approach