spider.pl
application/x-perl
#!/usr/bin/perl -w
#*-------------------------------------------------------------------
#*- spider.pl:
#*- Author: Manu Konchady
#*- License: GPL
#*- Input: A query
#*- Output: A list of the hubs and authorities for the query
#*- Module Dependencies: HTML::Parser 3.0, LWP::UserAgent,
#*- URI::URL, HTTP::Request, DBI and Posix.
#*-
#*- Description: We assume that two tables have been created for this
#*- particular query to store URLs and associated links
#*- prior to the execution of the spider. An entry has also
#*- been made in a queries table for this query.
#*-
#*- An initial collection of URLs are stored in the URL table
#*- for the query followed by a parallel execution of spiders
#*- to selectively generate the web tree for the query.
#*- After all spiders have terminated, the hubs and authorities
#*- for the query are listed. The list of all URLs sorted by
#*- relevancy can also be viewed.
#*-
#*- Notes: This code is a sample and should be modified as
#*- needed. For example, relevancy can be computed using several
#*- options as described in the article. Links can be pruned or
#*- followed depending on user criteria. Priorities can be
#*- set for certain domains. Entity tables can be used to track
#*- entities across web pages.
#*-
#*- RAL hacked to do postgres instead of MySQL
#*- tables are utab, queries, ltab
#*- ( url varchar(120), status varchar(10), site varchar(120), relevancy float4, level int4, spider int4)
#*- ( query varchar(250), qword varchar(250), url_limit int4, url_count int4, status varchar(10))
#*- the "query" is the text to match, the qword is a name for reference and
#*- which was used to set the tables up
#*- ( url varchar(250), link varchar(250))
#*-------------------------------------------------------------------
use POSIX ":sys_wait_h";
use PgDBI;
use MyWeb;
use strict;
my $dbname = 'querynet1';
my ($i, @pid, $done, $num_spiders, $kid, $query, %urls, $count);
my ($qword, $utab, $ltab, $dbh, $sth, $command, $url_limit, $url);
#*-- The first argument is a table prefix associated with the query.
#*-- It is used to assign the url and link tables for the query.
#*-- The database name for the db connection must be entered.
$qword = $ARGV[0];
$utab = "$qword" . "_u"; $ltab = "$qword" . "_l";
$dbh = PgDBI::connect_db("dbname=$dbname");
#*-- get the full query and maximum number of urls for the query
$command = "select query, url_limit from queries where qword = '$qword'";
$sth = PgDBI::execute_stmt($dbh, $command);
($query, $url_limit) = PgDBI::fetch_row($sth);
#*-- get an initial set of urls from Google for the query
&call_google();
#*-- close the database in ccase google has crashed it
PgDBI::disconnect_db($dbh, $sth);
#*-- start the parallel operation with 4 spiders, this
#*-- works on Linux. On a Windows platform, you may need
#*-- to use the Win32 module to start individual processes
#$num_spiders = 1;
$num_spiders = 4;
for $i (1..$num_spiders)
{
if ($pid[$i] = fork()) #*-- the parent code continues
{ next; }
else #*-- the child code continues
{ defined($pid[$i]) or die "fork: $!"; &fetch_text($i); }
}
####################################################################
#*-- wait till all the spiders are complete. Every two seconds,
#*-- check the status of the spiders
$done = 0;
while (!($done))
{
for $i (1..$num_spiders)
{
next if ($pid[$i] == 0);
$kid = waitpid($pid[$i], &WNOHANG);
$pid[$i] = 0 if ($kid == -1);
}
$done = 1;
for $i (1..$num_spiders) { $done = 0 if ($pid[$i] != 0); }
sleep(2);
}
#*-- clean up after the spiders, find all URLs which were
#*-- not processed and delete them from the link table
$dbh = PgDBI::connect_db("dbname=$dbname");
$command = "select url from $utab where status != 'd'";
$sth = PgDBI::execute_stmt($dbh, $command);
while ( ($url) = PgDBI::fetch_row($sth) ) { $urls{$url}++; }
foreach $url (keys %urls)
{ $command = "delete from $ltab where url = '$url'";
$sth = PgDBI::execute_stmt($dbh, $command);
}
#*-- delete the unprocessed urls from the url table
$command = "delete from $utab where status != 'd'";
$sth = PgDBI::execute_stmt($dbh, $command);
$command = "update queries set status = 'done' where qword = '$qword'";
$sth = PgDBI::execute_stmt($dbh, $command);
#*-- print the top 10 hubs and authorities
print ("List of top 10 Hubs\n"); $i = 1;
$command = "select count(*), url from $ltab group by url order by 1 desc";
$sth = PgDBI::execute_stmt($dbh, $command);
while ( ($count, $url) = PgDBI::fetch_row($sth) )
{ print ("$i. $url\n"); $i++; last if ($i == 11); }
print ("List of top 10 Authorities\n"); $i = 1;
$command = "select count(*), link from $ltab group by link order by 1 desc";
$sth = PgDBI::execute_stmt($dbh, $command);
while ( ($count, $url) = PgDBI::fetch_row($sth) )
{ print ("$i". "- "." $url\n"); $i++; last if ($i == 11); }
PgDBI::disconnect_db($dbh, $sth);
exit(0);
#*------------------------------------------------------------
#*- Each of the spiders run the following code and then terminate.
#*- An unprocessed URL is selected and traversed. New links
#*- are added to the tables. The spider number is passed
#*- as a parameter
#*------------------------------------------------------------
sub fetch_text()
{
my ($snum) = @_;
my ($dbh, $sth, $done, $command, $count, $links, $ref_text, $usite);
my ($ref_links, $url, $level, $link, %links, $site, $url_temp);
#*-- create a handle for the database
$dbh = PgDBI::connect_db("dbname=$dbname");
$done = 0;
while (!($done))
{
$link = '';
## $command = "lock tables $utab write, queries write"; # no good in postgres
# $command = "begin work;"; # ral new
# $sth = PgDBI::execute_stmt($dbh, $command);
# $command = "lock table $utab"; # ral new
# $sth = PgDBI::execute_stmt($dbh, $command);
# $command = "lock table queries"; # ral new
# $sth = PgDBI::execute_stmt($dbh, $command);
#*-- check if there are any unprocessed URLs
$command = "select url, level, site from $utab where status = ''"; # Pg
$sth = PgDBI::execute_stmt($dbh, $command);
($url, $level, $usite) = PgDBI::fetch_row($sth);
$command = "select count(*) from $utab where status = ''"; # Pg
$sth = PgDBI::execute_stmt($dbh, $command);
($count) = PgDBI::fetch_row($sth);
if ($count == 0)
{ $done = 1; next; } # Pg
#*-- update the number of URLs processed in the queries table
$command = "select count(*) from $utab where status = 'd'";
$sth = PgDBI::execute_stmt($dbh, $command);
($count) = PgDBI::fetch_row($sth);
$command = "update queries set url_count = $count where qword = '$qword'";
$sth = PgDBI::execute_stmt($dbh, $command);
#*-- check if the number of processed URLs exceeds the limit
if ($count > $url_limit)
# { $done = 1; $sth = PgDBI::execute_stmt($dbh, "unlock tables"); next; } # no good in postgres
{ $done = 1;
# $sth = PgDBI::execute_stmt($dbh, "commit work"); #
next; } # Pg
#*-- set the status of the URL and unlock the tables
$url_temp = $url;
$url =~ s/'/\\'/g; $url =~ s/\$/\\\$/;
$command = "update $utab set status = 'v' where url = '$url'";
$sth = PgDBI::execute_stmt($dbh, $command);
#*-- if this is an irrelevant URL, then remove it
$url = $url_temp;
($ref_text, $ref_links) = MyWeb::parse_URL($url);
if (!(&match($$ref_text, $query) ) )
{
$command = "delete from $utab where url = '$url'";
$sth = PgDBI::execute_stmt($dbh, $command);
if ($link eq '') { next; } else {
$command = "delete from $ltab where url = '$url'";
$sth = PgDBI::execute_stmt($dbh, $command);
next;}
}
#*-- add all links for this relevant URL to $ltab
$level++; %links = %$ref_links;
foreach $link (keys %links)
{
($site = $link) =~ s#/.*$##i;
#*-- skip links to ads and other formatted documents
next if ($link =~ /doubleclick/i);
next if ($link =~ /affiliates\./i);
next if ($link =~ /\.(jpg|gif|pdf|ps|doc|xls|ppt|mpg|mpeg)$/i);
#*-- skip links to the same site, except when one of the
#*-- query keywords matches the link
next if ( ($site eq $usite) && (!(&match_link($link, $query))) );
$link =~ s/'/\\'/g; $link =~ s/\$/\\\$/;
$link = lc($link);
#*-- check for duplicates
$command = "select count(*) from $utab where url = '$link'";
$sth = PgDBI::execute_stmt($dbh, $command);
($count) = PgDBI::fetch_row($sth);
next if ($count != 0);
#*-- check for too many URLs from the same site
$command = "select count(*) from $utab where site = '$site'";
$sth = PgDBI::execute_stmt($dbh, $command);
($count) = PgDBI::fetch_row($sth);
next if ($count > 200);
#*-- create entries in the link and url tables
$command = "insert into $ltab values ('$url', '$link')";
$sth = PgDBI::execute_stmt($dbh, $command);
$command = "insert into $utab values ('$link', '','$site', 0.0, $level, $snum)";
$sth = PgDBI::execute_stmt($dbh, $command);
}
#*-- set the status and relevancy of the URL, text from
#*-- the web page can be used to set relevancy values
$command = "update $utab set status = 'd' where url = '$url'";
$sth = PgDBI::execute_stmt($dbh, $command);
$command = "update $utab set relevancy = 0.2 where url = '$url'";
$sth = PgDBI::execute_stmt($dbh, $command);
}
PgDBI::disconnect_db($dbh, $sth);
sleep (3);
exit(0);
}
#*------------------------------------------------------
#*- check if query satisfies the boolean query.
#*------------------------------------------------------
sub match()
{
my ( $text, $query_terms) = @_;
my ($word, @words, $quotes, $retval);
$quotes = '"\'';
#*-- split a query with the AND operator and recursively
#*-- parse the query
if ($query_terms =~ /\bAND\b/i)
{
@words = split(/\bAND\b/i, $query_terms);
foreach $word (@words)
{
if ($word =~ /\((.*?)\)/)
{ $retval = &match($text, $1);
if ($retval == 0) { return(0); } else { next; }
}
$word =~ s/[$quotes]//g; $word =~ s/^\s+//; $word =~ s/\s+$//;
return(0) if ($text !~ /\b$word\b/i);
}
return(1);
}
#*-- split a query with the OR operator and recursively
#*-- parse the query
if ($query_terms =~ /\bOR\b/i)
{
@words = split(/\bOR\b/i, $query_terms);
foreach $word (@words)
{
if ($word =~ /\((.*?)\)/)
{ $retval = &match($text, $1);
if ($retval == 1) { return(1); } else { next; }
}
$word =~ s/[$quotes]//g; $word =~ s/^\s+//; $word =~ s/\s+$//;
return(1) if ($text =~ /\b$word\b/i);
}
return(0);
}
#*-- assume an AND operator for a query with multiple terms
$query_terms =~ s/[$quotes]//g; $query_terms =~ s/^\s+//; $query_terms =~ s/\s+$//;
$query_terms =~ s/([()])/\\$1/g;
@words = split(/\s+/, $query_terms);
foreach $word (@words)
{ return(0) if ($text !~ /\b$word\b/i); }
return(1);
}
#*------------------------------------------------------
#*- check if any words in the query match the query text
#*------------------------------------------------------
sub match_link()
{
my ($link, $query) = @_;
my (@words, $word, $match);
$match = 0;
@words = split(/\s+/, $query);
foreach $word (@words)
{
next if ( ($word =~ /\bAND\b/i) || ($word =~ /\bOR\b/i) || ($word =~ /\bNOT\b/i) );
$match = 1 if ($link =~ /$word/i);
}
return($match);
}
#*-------------------------------------------
#*- load the initial set of URLs from Google
#*-------------------------------------------
sub call_google()
{
my ($sql, $query, $url_string, $reg, $j, $script, $ref_html);
my ($output, $site, %urls, $url);
my ($count) = 0;
$sth = PgDBI::execute_stmt($dbh, "delete from $utab");
$sth = PgDBI::execute_stmt($dbh, "delete from $ltab");
#*-- get the full query
$sql = "select query from queries where qword = '$qword'";
$sth = PgDBI::execute_stmt($dbh, "$sql");
($query) = PgDBI::fetch_row($sth);
#*-- get urls from Google
$query =~ s/(\W)/sprintf("%%%x", ord($1))/eg;
$url_string = "http://www.google.com/search?num=100&safe=off&btnG=Google+Search";
$url_string .= "&q=$query&filter=0&start=";
$reg = '<p><a href=http://(.*?)>';
undef $/;
for ($j = 0; $j < 46; $j += 50)
{
$script = "$url_string$j";
($ref_html) = MyWeb::get_URL($script);
$output = $$ref_html; #*-- get all the data
while ($output =~ /$reg/sgi)
{ ($url = $1) =~ tr/A-Z/a-z/; $urls{$url}++; }
sleep(3);
}
#*-- put the urls in a table
foreach $url (keys %urls)
{
$count++;
($site = $url) =~ s#/.*$##i; $url =~ s/'/\\'/g; $url =~ s/\$/\\\$/;
$command = "insert into $utab values ('$url', '', '$site', 0.0, 0, 0)";
$sth = PgDBI::execute_stmt($dbh, $command);
}
}