[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Koha-cvs] CVS: koha/acqui.simple processz3950queue,1.3.2.2,1.3.2.3 z395
|
From: |
Steve Tonnesen |
|
Subject: |
[Koha-cvs] CVS: koha/acqui.simple processz3950queue,1.3.2.2,1.3.2.3 z3950-daemon-shell.sh,1.1.2.2,1.1.2.3 |
|
Date: |
Fri, 28 Jun 2002 10:45:41 -0700 |
Update of /cvsroot/koha/koha/acqui.simple
In directory usw-pr-cvs1:/tmp/cvs-serv26334/acqui.simple
Modified Files:
Tag: rel-1-2
processz3950queue z3950-daemon-shell.sh
Log Message:
z3950queue now listens for a -HUP signal before processing the queue. Z3950.pm
sends the -HUP signal when queries are added to the queue.
Index: processz3950queue
===================================================================
RCS file: /cvsroot/koha/koha/acqui.simple/processz3950queue,v
retrieving revision 1.3.2.2
retrieving revision 1.3.2.3
diff -C2 -r1.3.2.2 -r1.3.2.3
*** processz3950queue 26 Jun 2002 22:29:23 -0000 1.3.2.2
--- processz3950queue 28 Jun 2002 17:45:39 -0000 1.3.2.3
***************
*** 13,226 ****
$sth->finish;
$SIG{CHLD}='reap';
my $reapcounter=0;
my $forkcounter=0;
my $pid=$$;
my $lastrun=0;
while (1) {
if ((time-$lastrun)>5) {
! my $sth=$dbh->prepare("select id,term,type,servers from z3950queue
order by id");
! $sth->execute;
! while (my ($id, $term, $type, $servers) = $sth->fetchrow) {
! if ($forkcounter<12) {
! my $now=time();
! $stk=$dbh->prepare("select
id,server,startdate,enddate,numrecords,active from z3950results where
queryid=$id");
! ($stk->execute) || (next);
! my %serverdone;
! unless ($stk->rows) {
! my $sti=$dbh->prepare("update z3950queue set
done=-1,startdate=$now where id=$id");
! $sti->execute;
! }
! while (my ($r_id,
$r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) {
! if ($r_enddate >0) {
! $serverdone{$r_server}=1;
! } elsif ($active) {
! $serverdone{$r_server}=1;
! } else {
! $serverdone{$r_server}=-1;
}
- }
! $stk->finish;
! my $attr='';
! if ($type eq 'isbn') {
! $attr='1=7';
! } elsif ($type eq 'title') {
! $attr='1=4';
! } elsif ($type eq 'author') {
! $attr='1=1003';
! } elsif ($type eq 'lccn') {
! $attr='1=9';
! } elsif ($type eq 'keyword') {
! $attr='1=1016';
! }
! $term='"'.$term.'"';
! $query="address@hidden $attr $term";
! my $totalrecords=0;
! my $serverinfo;
! my $stillprocessing=0;
! foreach $serverinfo (split(/\s+/, $servers)) {
! (next) if ($serverdone{$serverinfo} == 1);
! my $stillprocessing=1;
! if (my $pid=fork()) {
! $forkcounter++;
! } else {
! #$sth->finish;
! #$sti->finish;
! #$dbh->disconnect;
! my $dbi=C4Connect;
! my ($name, $server, $database, $user, $password) =
split(/\//, $serverinfo, 5);
! $server=~/(.*)\:(\d+)/;
! my $servername=$1;
! my $port=$2;
! print "Processing $type=$term at $name $server
$database (".($forkcounter+1)." forks)\n";
! $now=time();
! my $q_serverinfo=$dbi->quote($serverinfo);
! my $resultsid;
! if ($serverdone{$serverinfo}==-1) {
! my $stj=$dbi->prepare("select id from z3950results
where server=$q_serverinfo and queryid=$id");
! $stj->execute;
! ($resultsid) = $stj->fetchrow;
} else {
! my $stj=$dbi->prepare("select id from z3950results
where server=$q_serverinfo and queryid=$id");
! $stj->execute;
! ($resultsid) = $stj->fetchrow;
! unless ($resultsid) {
! my $stj=$dbi->prepare("insert into z3950results
(server, queryid, startdate) values ($q_serverinfo, $id, $now)");
$stj->execute;
! $resultsid=$dbi->{'mysql_insertid'};
! }
! }
! my $stj=$dbh->prepare("update z3950results set active=1
where id=$resultsid");
! $stj->execute;
! my $conn;
! my $noconnection=0;
! my $error=0;
! if ($user) {
! eval { $conn= new
Net::Z3950::Connection($servername, $port, databaseName => $database, user =>
$user, password => $password); };
! if ($@) {
! $noconnection=1;
} else {
! $error=pe();
}
! } else {
! eval { $conn= new
Net::Z3950::Connection($servername, $port, databaseName => $database); };
! if ($@) {
! $noconnection=1;
} else {
! $error=pe();
}
! }
! if ($noconnection || $error) {
! } else {
! print "Q: $query\n";
! my $rs=$conn->search($query);
! pe();
! eval { $rs->option(preferredRecordSyntax =>
Net::Z3950::RecordSyntax::USMARC);};
! if ($@) {
! print "ERROR: address@hidden";
} else {
pe();
! my $numresults=$rs->size();
! pe();
! my $i;
! my $result='';
! my $scantimerstart=time();
! for ($i=1; $i<=(($numresults<80) ?
($numresults) : (80)); $i++) {
! my $rec=$rs->record($i);
! my $marcdata=$rec->rawdata();
! $result.=$marcdata;
! }
! my $scantimerend=time();
! my $numrecords;
! ($numresults<80) ? ($numrecords=$numresults) :
($numrecords=80);
! my $elapsed=$scantimerend-$scantimerstart;
! if ($elapsed) {
! my $speed=int($numresults/$elapsed*100)/100;
! print " SPEED: $speed $server done
$numrecords\n";
! }
! my $q_result=$dbi->quote($result);
! ($q_result) || ($q_result='""');
! $now=time();
! my $task="update z3950results set
numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now
where id=$resultsid";
! my $stj=$dbi->prepare($task);
! $stj->execute;
! my $counter=0;
! while ($counter<60 && $numrecords<$numresults) {
! $counter++;
! my $stj=$dbi->prepare("select highestseen
from z3950results where id=$resultsid");
$stj->execute;
! my ($highestseen) = $stj->fetchrow;
! if ($highestseen>($numrecords-30)) {
! $counter=0;
! print " $server rescanning\n";
! my $scantimerstart=time();
! for ($i=$numrecords+1;
$i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) {
! my $rec=$rs->record($i);
! my $marcdata=$rec->rawdata();
! $result.=$marcdata;
! }
! my $scantimerend=time();
! ($numresults<$numrecords+40) ?
($numrecords=$numresults) : ($numrecords=$numrecords+40);
! my
$elapsed=$scantimerend-$scantimerstart;
! if ($elapsed) {
! my
$speed=int($numresults/$elapsed*100)/100;
! print " SPEED: $speed $server
done $numrecords\n";
! }
!
! my $q_result=$dbi->quote($result);
! ($q_result) || ($q_result='""');
! $now=time();
! my $task="update z3950results set
numdownloaded=$numrecords,results=$q_result where id=$resultsid";
! my $stj=$dbi->prepare($task);
$stj->execute;
}
- sleep 5;
}
}
! }
! my $stj=$dbi->prepare("update z3950results set active=0
where id=$resultsid");
! $stj->execute;
! eval {$stj->finish};
! $dbi->disconnect;
! print " $server done.\n";
! exit;
! sub pe {
! return 0;
! my $code=$conn->errcode();
! my $msg=$conn->errmsg();
! my $ai=$conn->addinfo();
! print << "EOF";
! CODE: $code
! MSG: $msg
! ADDTL: $ai
EOF
! # if ($msg =~/not yet available/) {
! # return 1;
! # }
! return 0;
}
}
! } unless ($stillprocessing) {
! #my $sti=$dbh->prepare("select enddate from z3950queue
where id=$id");
! #$sti->execute;
! #my ($enddate) = $sti->fetchrow;
! #unless ($enddate) {
! # my $now=time;
! # $sti=$dbh->prepare("update z3950queue set
done=1,numrecords=$totalrecords,enddate=$now where id=$id");
! # $sti->execute;
! # }
}
- } else {
- # my $q_serverinfo=$dbh->quote($serverinfo);
- # my $stj=$dbh->prepare("insert into z3950results (server, queryid,
startdate) values ($q_serverinfo, $id, 0)");
- # $stj->execute;
}
}
! $lastrun=time();
}
- sleep 10;
}
--- 13,227 ----
$sth->finish;
$SIG{CHLD}='reap';
+ $SIG{HUP}='checkqueue';
+
+ my $logdir=$ARGV[0];
+
+ open PID, ">$logdir/processz3950queue.pid";
+ print PID $$."\n";
+ close PID;
my $reapcounter=0;
my $forkcounter=0;
+ my $checkqueue=1;
my $pid=$$;
my $lastrun=0;
while (1) {
if ((time-$lastrun)>5) {
! if ($checkqueue) {
! $checkqueue=0;
! my $sth=$dbh->prepare("select id,term,type,servers from z3950queue
order by id");
! $sth->execute;
! while (my ($id, $term, $type, $servers) = $sth->fetchrow) {
! if ($forkcounter<12) {
! my $now=time();
! $stk=$dbh->prepare("select
id,server,startdate,enddate,numrecords,active from z3950results where
queryid=$id");
! ($stk->execute) || (next);
! my %serverdone;
! unless ($stk->rows) {
! my $sti=$dbh->prepare("update z3950queue set
done=-1,startdate=$now where id=$id");
! $sti->execute;
! }
! while (my ($r_id,
$r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) {
! if ($r_enddate >0) {
! $serverdone{$r_server}=1;
! } elsif ($active) {
! $serverdone{$r_server}=1;
! } else {
! $serverdone{$r_server}=-1;
! }
}
! $stk->finish;
! my $attr='';
! if ($type eq 'isbn') {
! $attr='1=7';
! } elsif ($type eq 'title') {
! $attr='1=4';
! } elsif ($type eq 'author') {
! $attr='1=1003';
! } elsif ($type eq 'lccn') {
! $attr='1=9';
! } elsif ($type eq 'keyword') {
! $attr='1=1016';
! }
! $term='"'.$term.'"';
! $query="address@hidden $attr $term";
! my $totalrecords=0;
! my $serverinfo;
! my $stillprocessing=0;
! foreach $serverinfo (split(/\s+/, $servers)) {
! (next) if ($serverdone{$serverinfo} == 1);
! my $stillprocessing=1;
! if (my $pid=fork()) {
! $forkcounter++;
} else {
! #$sth->finish;
! #$sti->finish;
! #$dbh->disconnect;
! my $dbi=C4Connect;
! my ($name, $server, $database, $user, $password) =
split(/\//, $serverinfo, 5);
! $server=~/(.*)\:(\d+)/;
! my $servername=$1;
! my $port=$2;
! print "Processing $type=$term at $name $server
$database (".($forkcounter+1)." forks)\n";
! $now=time();
! my $q_serverinfo=$dbi->quote($serverinfo);
! my $resultsid;
! if ($serverdone{$serverinfo}==-1) {
! my $stj=$dbi->prepare("select id from
z3950results where server=$q_serverinfo and queryid=$id");
$stj->execute;
! ($resultsid) = $stj->fetchrow;
} else {
! my $stj=$dbi->prepare("select id from
z3950results where server=$q_serverinfo and queryid=$id");
! $stj->execute;
! ($resultsid) = $stj->fetchrow;
! unless ($resultsid) {
! my $stj=$dbi->prepare("insert into
z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)");
! $stj->execute;
! $resultsid=$dbi->{'mysql_insertid'};
! }
}
! my $stj=$dbh->prepare("update z3950results set
active=1 where id=$resultsid");
! $stj->execute;
! my $conn;
! my $noconnection=0;
! my $error=0;
! if ($user) {
! eval { $conn= new
Net::Z3950::Connection($servername, $port, databaseName => $database, user =>
$user, password => $password); };
! if ($@) {
! $noconnection=1;
! } else {
! $error=pe();
! }
} else {
! eval { $conn= new
Net::Z3950::Connection($servername, $port, databaseName => $database); };
! if ($@) {
! $noconnection=1;
! } else {
! $error=pe();
! }
}
! if ($noconnection || $error) {
} else {
+ print "Q: $query\n";
+ my $rs=$conn->search($query);
pe();
! eval { $rs->option(preferredRecordSyntax =>
Net::Z3950::RecordSyntax::USMARC);};
! if ($@) {
! print "ERROR: address@hidden";
! } else {
! pe();
! my $numresults=$rs->size();
! pe();
! my $i;
! my $result='';
! my $scantimerstart=time();
! for ($i=1; $i<=(($numresults<80) ?
($numresults) : (80)); $i++) {
! my $rec=$rs->record($i);
! my $marcdata=$rec->rawdata();
! $result.=$marcdata;
! }
! my $scantimerend=time();
! my $numrecords;
! ($numresults<80) ?
($numrecords=$numresults) : ($numrecords=80);
! my $elapsed=$scantimerend-$scantimerstart;
! if ($elapsed) {
! my
$speed=int($numresults/$elapsed*100)/100;
! print " SPEED: $speed $server done
$numrecords\n";
! }
! my $q_result=$dbi->quote($result);
! ($q_result) || ($q_result='""');
! $now=time();
! my $task="update z3950results set
numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now
where id=$resultsid";
! my $stj=$dbi->prepare($task);
$stj->execute;
! my $counter=0;
! while ($counter<60 &&
$numrecords<$numresults) {
! $counter++;
! my $stj=$dbi->prepare("select
highestseen from z3950results where id=$resultsid");
$stj->execute;
+ my ($highestseen) = $stj->fetchrow;
+ if ($highestseen>($numrecords-30)) {
+ $counter=0;
+ print " $server rescanning\n";
+ my $scantimerstart=time();
+ for ($i=$numrecords+1;
$i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) {
+ my $rec=$rs->record($i);
+ my $marcdata=$rec->rawdata();
+ $result.=$marcdata;
+ }
+ my $scantimerend=time();
+ ($numresults<$numrecords+40) ?
($numrecords=$numresults) : ($numrecords=$numrecords+40);
+ my
$elapsed=$scantimerend-$scantimerstart;
+ if ($elapsed) {
+ my
$speed=int($numresults/$elapsed*100)/100;
+ print " SPEED: $speed $server
done $numrecords\n";
+ }
+
+ my $q_result=$dbi->quote($result);
+ ($q_result) || ($q_result='""');
+ $now=time();
+ my $task="update z3950results set
numdownloaded=$numrecords,results=$q_result where id=$resultsid";
+ my $stj=$dbi->prepare($task);
+ $stj->execute;
+ }
+ sleep 5;
}
}
}
! my $stj=$dbi->prepare("update z3950results set
active=0 where id=$resultsid");
! $stj->execute;
! eval {$stj->finish};
! $dbi->disconnect;
! print " $server done.\n";
! exit;
! sub pe {
! return 0;
! my $code=$conn->errcode();
! my $msg=$conn->errmsg();
! my $ai=$conn->addinfo();
! print << "EOF";
! CODE: $code
! MSG: $msg
! ADDTL: $ai
EOF
! return 0;
! }
}
+ } unless ($stillprocessing) {
+ #my $sti=$dbh->prepare("select enddate from z3950queue
where id=$id");
+ #$sti->execute;
+ #my ($enddate) = $sti->fetchrow;
+ #unless ($enddate) {
}
! } else {
}
}
+ $lastrun=time();
}
! sleep 10;
}
}
***************
*** 260,263 ****
--- 261,268 ----
}
+
+ sub checkqueue {
+ $checkqueue=1;
+ }
Index: z3950-daemon-shell.sh
===================================================================
RCS file: /cvsroot/koha/koha/acqui.simple/z3950-daemon-shell.sh,v
retrieving revision 1.1.2.2
retrieving revision 1.1.2.3
diff -C2 -r1.1.2.2 -r1.1.2.3
*** z3950-daemon-shell.sh 26 Jun 2002 16:25:51 -0000 1.1.2.2
--- z3950-daemon-shell.sh 28 Jun 2002 17:45:39 -0000 1.1.2.3
***************
*** 38,45 ****
export PERL5LIB
! exec $KohaZ3950Script >>$LOGFILE 2>&1
#-------------------
# $Log$
# Revision 1.1.2.2 2002/06/26 16:25:51 amillar
# Make directory variable name more explanatory
--- 38,49 ----
export PERL5LIB
! exec $KohaZ3950Script $LogDir >>$LOGFILE 2>&1
#-------------------
# $Log$
+ # Revision 1.1.2.3 2002/06/28 17:45:39 tonnesen
+ # z3950queue now listens for a -HUP signal before processing the queue.
Z3950.pm
+ # sends the -HUP signal when queries are added to the queue.
+ #
# Revision 1.1.2.2 2002/06/26 16:25:51 amillar
# Make directory variable name more explanatory
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Koha-cvs] CVS: koha/acqui.simple processz3950queue,1.3.2.2,1.3.2.3 z3950-daemon-shell.sh,1.1.2.2,1.1.2.3,
Steve Tonnesen <=