[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Koha-devel] CVS: koha/acqui.simple marcimport.pl,1.4,1.5 processz3950qu
|
From: |
Steve Tonnesen |
|
Subject: |
[Koha-devel] CVS: koha/acqui.simple marcimport.pl,1.4,1.5 processz3950queue,1.1,1.2 |
|
Date: |
Tue Nov 6 10:15:02 2001 |
Update of /cvsroot/koha/koha/acqui.simple
In directory usw-pr-cvs1:/tmp/cvs-serv6588
Modified Files:
marcimport.pl processz3950queue
Log Message:
Continuing work on Z39.50 search tool. Daemon now forks up to 12 processes
to do Z39.50 searches. Daemon will also wait to see if the user looks
beyond the first couple of pages of results, and will download more results
if necessary.
Index: marcimport.pl
===================================================================
RCS file: /cvsroot/koha/koha/acqui.simple/marcimport.pl,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -r1.4 -r1.5
*** marcimport.pl 2001/11/03 07:28:20 1.4
--- marcimport.pl 2001/11/06 18:13:59 1.5
***************
*** 134,139 ****
chop $serverlist;
my $q_serverlist=$dbh->quote($serverlist);
! my $sth=$dbh->prepare("insert into z3950queue (term,type,servers)
values ($q_term, '$type', $q_serverlist)");
$sth->execute;
}
}
--- 134,145 ----
chop $serverlist;
my $q_serverlist=$dbh->quote($serverlist);
! my $rand=$input->param('rand');
! my $sth=$dbh->prepare("select identifier from z3950queue where
! identifier=$rand");
$sth->execute;
+ unless ($sth->rows) {
+ $sth=$dbh->prepare("insert into z3950queue (term,type,servers,
identifier) values ($q_term, '$type', $q_serverlist, '$rand')");
+ $sth->execute;
+ }
}
}
***************
*** 696,707 ****
my ($servers) = $sth->fetchrow;
my $serverstring;
foreach $serverstring (split(/\s+/, $servers)) {
my ($name, $server, $database, $auth) = split(/\//,
$serverstring, 4);
if ($name eq 'MAN') {
print "$server/$database<br>\n";
- } elsif ($name eq 'LOC') {
- print "Library of Congress<br>\n";
- } elsif ($name eq 'NLC') {
- print "National Library of Canada<br>\n";
} else {
my $sti=$dbh->prepare("select name from
--- 702,710 ----
my ($servers) = $sth->fetchrow;
my $serverstring;
+ my $starttimer=time();
foreach $serverstring (split(/\s+/, $servers)) {
my ($name, $server, $database, $auth) = split(/\//,
$serverstring, 4);
if ($name eq 'MAN') {
print "$server/$database<br>\n";
} else {
my $sti=$dbh->prepare("select name from
***************
*** 709,719 ****
$sti->execute;
my ($longname)=$sti->fetchrow;
! print "$longname<br>\n";
}
- print "<ul>\n";
my $q_server=$dbh->quote($serverstring);
my $sti=$dbh->prepare("select
numrecords,id,results,startdate,enddate from z3950results where queryid=$id and
server=$q_server");
$sti->execute;
($numrecords,$resultsid,$data,$startdate,$enddate) =
$sti->fetchrow;
if ($sti->rows == 0) {
print "pending...";
--- 712,755 ----
$sti->execute;
my ($longname)=$sti->fetchrow;
! print "<a name=SERVER-$name></a>\n";
! if ($longname) {
! print "$longname \n";
! } else {
! print "$server/$database \n";
! }
}
my $q_server=$dbh->quote($serverstring);
+ my $startrecord=$input->param("ST-$name");
+ ($startrecord) || ($startrecord='0');
my $sti=$dbh->prepare("select
numrecords,id,results,startdate,enddate from z3950results where queryid=$id and
server=$q_server");
$sti->execute;
($numrecords,$resultsid,$data,$startdate,$enddate) =
$sti->fetchrow;
+ my $serverplaceholder='';
+ foreach ($input->param) {
+ (next) unless (/ST-(.+)/);
+ my $serverid=$1;
+ (next) if ($serverid eq $name);
+ my $place=$input->param("ST-$serverid");
+ $serverplaceholder.="\&ST-$serverid=$place";
+ }
+ if ($numrecords) {
+ my $previous='';
+ my $next='';
+ if ($startrecord>0) {
+ $previous="<a
href=".$ENV{'SCRIPT_NAME'}."?file=Z-$id&menu=z3950$serverplaceholder\&ST-$name=".($startrecord-10)."#SERVER-$name>Previous</a>";
+ }
+ my $highest;
+ $highest=$startrecord+10;
+ ($highest>$numrecords) && ($highest=$numrecords);
+ if ($numrecords>$startrecord+10) {
+ $next="<a
href=".$ENV{'SCRIPT_NAME'}."?file=Z-$id&menu=z3950$serverplaceholder\&ST-$name=$highest#SERVER-$name>Next</a>";
+ }
+ print "<font size=-1>[Viewing ".($startrecord+1)." to
".$highest." of $numrecords records] $previous | $next </font><br>\n";
+ } else {
+ print "<br>\n";
+ }
+ print "<ul>\n";
+ my $stj=$dbh->prepare("update z3950results set
highestseen=".($startrecord+10)." where id=$resultsid");
+ $stj->execute;
if ($sti->rows == 0) {
print "pending...";
***************
*** 729,734 ****
print "<font color=red>processing... ($elapsedtime)</font>";
} elsif ($numrecords) {
! my @records=parsemarcdata($data);
foreach $record (@records) {
my ($lccn, $isbn, $issn, $dewey, $author, $title,
$place, $publisher, $publicationyear, $volume, $number, @subjects, $note,
$controlnumber);
foreach $field (@$record) {
--- 765,779 ----
print "<font color=red>processing... ($elapsedtime)</font>";
} elsif ($numrecords) {
! my $splitchar=chr(29);
! my @records=split(/$splitchar/, $data);
! $data='';
! for ($i=$startrecord; $i<$startrecord+10; $i++) {
! $data.=$records[$i].$splitchar;
! }
! @records=parsemarcdata($data);
! my $counter=0;
foreach $record (@records) {
+ $counter++;
+ #(next) unless ($counter>=$startrecord &&
$counter<=$startrecord+10);
my ($lccn, $isbn, $issn, $dewey, $author, $title,
$place, $publisher, $publicationyear, $volume, $number, @subjects, $note,
$controlnumber);
foreach $field (@$record) {
***************
*** 801,804 ****
--- 846,851 ----
print "</ul>\n";
}
+ my $elapsed=time()-$starttimer;
+ print "<hr>It took $elapsed seconds to process this page.\n";
} else {
my $sth=$dbh->prepare("select marc,name from uploadedmarc where
id=$file");
***************
*** 908,935 ****
$term=~s/</</g;
$term=~s/>/>/g;
! if ($done == 1) {
! my $elapsed=$enddate-$startdate;
! my $elapsedtime='';
! if ($elapsed>60) {
! $elapsedtime=sprintf "%d minutes",($elapsed/60);
! } else {
! $elapsedtime=sprintf "%d seconds",$elapsed;
! }
! if ($numrecords) {
! print "<li><a
href=$ENV{'SCRIPT_NAME'}?file=Z-$id&menu=$menu>$type=$term</a> <font
size=-1>Done. $numrecords records found in $elapsedtime.</font><br>\n";
! } else {
! print "<li><a
href=$ENV{'SCRIPT_NAME'}?file=Z-$id&menu=$menu>$type=$term</a> <font
size=-1>Done. No records found. Search took $elapsedtime.</font><br>\n";
}
! } elsif ($done == -1) {
! my $elapsed=time()-$startdate;
! my $elapsedtime='';
! if ($elapsed>60) {
! $elapsedtime=sprintf "%d minutes",($elapsed/60);
} else {
! $elapsedtime=sprintf "%d seconds",$elapsed;
}
- print "<li><a
href=$ENV{'SCRIPT_NAME'}?file=Z-$id&menu=$menu>$type=$term</a> <font color=red
size=-1>Processing ($elapsedtime)</font><br>\n";
} else {
! print "<li><a
href=$ENV{'SCRIPT_NAME'}?file=Z-$id&menu=$menu>$type=$term</a> $done <font
size=-1>Pending</font><br>\n";
}
}
--- 955,1006 ----
$term=~s/</</g;
$term=~s/>/>/g;
! my $sti=$dbh->prepare("select id,server,startdate,enddate,numrecords
from z3950results where queryid=$id");
! $sti->execute;
! if ($sti->rows) {
! my $processing=0;
! my $realenddate=0;
! my $totalrecords=0;
! while (my ($r_id,$r_server,$r_startdate,$r_enddate,$r_numrecords) =
$sti->fetchrow) {
! if ($r_enddate==0) {
! $processing=1;
! } else {
! if ($r_enddate>$realenddate) {
! $realenddate=$r_enddate;
! }
! }
!
! $totalrecords+=$r_numrecords;
}
! if ($processing) {
! my $elapsed=time()-$startdate;
! my $elapsedtime='';
! if ($elapsed>60) {
! $elapsedtime=sprintf "%d minutes",($elapsed/60);
! } else {
! $elapsedtime=sprintf "%d seconds",$elapsed;
! }
! if ($totalrecords) {
! $totalrecords="$totalrecords found.";
! } else {
! $totalrecords='';
! }
! print "<li><a
href=$ENV{'SCRIPT_NAME'}?file=Z-$id&menu=$menu>$type=$term</a> <font size=-1
color=red>Processing... $totalrecords ($elapsedtime)</font><br>\n";
} else {
! my $elapsed=$realenddate-$startdate;
! my $elapsedtime='';
! if ($elapsed>60) {
! $elapsedtime=sprintf "%d minutes",($elapsed/60);
! } else {
! $elapsedtime=sprintf "%d seconds",$elapsed;
! }
! if ($totalrecords) {
! $totalrecords="$totalrecords found.";
! } else {
! $totalrecords='';
! }
! print "<li><a
href=$ENV{'SCRIPT_NAME'}?file=Z-$id&menu=$menu>$type=$term</a> <font
size=-1>Done. $totalrecords ($elapsedtime)</font><br>\n";
}
} else {
! print "<li><a
href=$ENV{'SCRIPT_NAME'}?file=Z-$id&menu=$menu>$type=$term</a> <font
size=-1>Pending</font><br>\n";
}
}
***************
*** 945,948 ****
--- 1016,1020 ----
$serverlist.="<input type=checkbox name=S-MAN> <input
name=manualz3950server size=25 value=otherserver:210/DATABASE>\n";
+ my $rand=rand(1000000000);
print << "EOF";
<form action=$ENV{'SCRIPT_NAME'} method=GET>
***************
*** 951,957 ****
<p>
<input type=hidden name=test value=testvalue>
<table border=1 bgcolor=#dddddd><tr><th bgcolor=#bbbbbb colspan=2>Search
for MARC records</th></tr>
<tr><td>Query Term</td><td><input name=query></td></tr>
! <tr><td colspan=2 align=center><input type=radio name=type value=isbn
checked> ISBN <input type=radio name=type value=lccn> LCCN <input type=radio
name=type value=title> Title</td></tr>
<tr><td colspan=2>
$serverlist
--- 1023,1030 ----
<p>
<input type=hidden name=test value=testvalue>
+ <input type=hidden name=rand value=$rand>
<table border=1 bgcolor=#dddddd><tr><th bgcolor=#bbbbbb colspan=2>Search
for MARC records</th></tr>
<tr><td>Query Term</td><td><input name=query></td></tr>
! <tr><td colspan=2 align=center><input type=radio name=type value=isbn
checked> ISBN <input type=radio name=type value=lccn> LCCN<br><input
type=radio name=type value=author> Author <input type=radio name=type
value=title> Title <input type=radio name=type
value=keyword> Keyword</td></tr>
<tr><td colspan=2>
$serverlist
Index: processz3950queue
===================================================================
RCS file: /cvsroot/koha/koha/acqui.simple/processz3950queue,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -r1.1 -r1.2
*** processz3950queue 2001/11/02 21:40:01 1.1
--- processz3950queue 2001/11/06 18:13:59 1.2
***************
*** 5,58 ****
use C4::Acquisitions;
use C4::Output;
my $dbh=C4Connect;
!
while (1) {
! my $sth=$dbh->prepare("select id,term,type,servers from z3950queue where
! isnull(done) || done=-1");
! $sth->execute;
! while (my ($id, $term, $type, $servers) = $sth->fetchrow) {
! my $now=time();
! my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now
where id=$id");
! $sti->execute;
! my $attr='';
! if ($type eq 'isbn') {
! $attr='1=7';
! } elsif ($type eq 'title') {
! $attr='1=4';
! } elsif ($type eq 'lccn') {
! $attr='1=9';
! }
! $term='"'.$term.'"';
! $query="f address@hidden $attr $term";
! my $totalrecords=0;
! my $serverinfo;
! foreach $serverinfo (split(/\s+/, $servers)) {
! my ($name, $server, $database, $auth) = split(/\//, $serverinfo, 4);
! ($auth eq '/') && ($auth='');
! print "Processing $type=$term at $name $server $database $auth\n";
! $now=time();
! my $q_serverinfo=$dbh->quote($serverinfo);
! my $sti=$dbh->prepare("insert into z3950results (server, queryid,
startdate) values ($q_serverinfo, $id, $now)");
! $sti->execute;
! my $resultsid=$dbh->{'mysql_insertid'};
! getrecord($server, $database, $query, $auth);
! my $result=`cat yaz.mrc`;
! unlink ('yaz.mrc');
! my $splitchar=chr(29);
! my @records=split(/$splitchar/, $result);
! my $numrecords=$#records+1;
! $totalrecords+=$numrecords;
! my $q_result=$dbh->quote($result);
! ($q_result) || ($q_result='""');
! $now=time();
! $sti=$dbh->prepare("update z3950results set
numrecords=$numrecords,results=$q_result,enddate=$now where id=$resultsid");
! $sti->execute;
}
! $sti=$dbh->prepare("update z3950queue set
done=1,numrecords=$totalrecords,enddate=$now where id=$id");
! $sti->execute;
}
! sleep 15;
}
--- 5,208 ----
use C4::Acquisitions;
use C4::Output;
+ use Net::Z3950;
my $dbh=C4Connect;
+ my $sth=$dbh->prepare("update z3950results set active=0");
+ $sth->execute;
+ $sth->finish;
+ $SIG{CHLD}='reap';
! my $reapcounter=0;
! my $forkcounter=0;
! my $pid=$$;
! my $lastrun=0;
while (1) {
! if ((time-$lastrun)>5) {
! my $sth=$dbh->prepare("select id,term,type,servers from z3950queue
order by id");
! $sth->execute;
! while (my ($id, $term, $type, $servers) = $sth->fetchrow) {
! if ($forkcounter<12) {
! my $now=time();
! $stk=$dbh->prepare("select
id,server,startdate,enddate,numrecords,active from z3950results where
queryid=$id");
! $stk->execute;
! my %serverdone;
! unless ($stk->rows) {
! my $sti=$dbh->prepare("update z3950queue set
done=-1,startdate=$now where id=$id");
! $sti->execute;
! }
! while (my ($r_id,
$r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) {
! if ($r_enddate >0) {
! $serverdone{$r_server}=1;
! } elsif ($active) {
! $serverdone{$r_server}=1;
! } else {
! $serverdone{$r_server}=-1;
! }
! }
!
! $stk->finish;
! my $attr='';
! if ($type eq 'isbn') {
! $attr='1=7';
! } elsif ($type eq 'title') {
! $attr='1=4';
! } elsif ($type eq 'author') {
! $attr='1=1003';
! } elsif ($type eq 'lccn') {
! $attr='1=9';
! } elsif ($type eq 'keyword') {
! $attr='1=1016';
! }
! $term='"'.$term.'"';
! $query="address@hidden $attr $term";
! my $totalrecords=0;
! my $serverinfo;
! my $stillprocessing=0;
! foreach $serverinfo (split(/\s+/, $servers)) {
! (next) if ($serverdone{$serverinfo} == 1);
! my $stillprocessing=1;
! if (my $pid=fork()) {
! $forkcounter++;
! } else {
! #$sth->finish;
! #$sti->finish;
! #$dbh->disconnect;
! my $dbi=C4Connect;
! my ($name, $server, $database, $user, $password) =
split(/\//, $serverinfo, 5);
! $server=~/(.*)\:(\d+)/;
! my $servername=$1;
! my $port=$2;
! print "Processing $type=$term at $name $server
$database (".($forkcounter+1)." forks)\n";
! $now=time();
! my $q_serverinfo=$dbi->quote($serverinfo);
! my $resultsid;
! if ($serverdone{$serverinfo}==-1) {
! my $stj=$dbi->prepare("select id from z3950results
where server=$q_serverinfo and queryid=$id");
! $stj->execute;
! ($resultsid) = $stj->fetchrow;
! } else {
! my $stj=$dbi->prepare("insert into z3950results
(server, queryid, startdate) values ($q_serverinfo, $id, $now)");
! $stj->execute;
! $resultsid=$dbi->{'mysql_insertid'};
! }
! my $stj=$dbh->prepare("update z3950results set active=1
where id=$resultsid");
! $stj->execute;
! my $conn;
! my $noconnection=0;
! if ($user) {
! eval { $conn= new
Net::Z3950::Connection($servername, $port, databaseName => $database, user =>
$user, password => $password); };
! if ($@) {
! $noconnection=1;
! }
! pe();
! } else {
! eval { $conn= new
Net::Z3950::Connection($servername, $port, databaseName => $database); };
! if ($@) {
! $noconnection=1;
! }
! pe();
! }
! if ($noconnection) {
! } else {
! my $rs=$conn->search($query);
! pe();
! $rs->option(preferredRecordSyntax =>
Net::Z3950::RecordSyntax::USMARC);
! pe();
! my $numresults=$rs->size();
! pe();
! my $i;
! my $result='';
! my $scantimerstart=time();
! for ($i=1; $i<=(($numresults<80) ? ($numresults) :
(80)); $i++) {
! my $rec=$rs->record($i);
! my $marcdata=$rec->rawdata();
! $result.=$marcdata;
! }
! my $scantimerend=time();
! my $numrecords;
! ($numresults<80) ? ($numrecords=$numresults) :
($numrecords=80);
! my $elapsed=$scantimerend-$scantimerstart;
! if ($elapsed) {
! my $speed=int($numresults/$elapsed*100)/100;
! print " SPEED: $speed $server done
$numrecords\n";
! }
!
! my $q_result=$dbi->quote($result);
! ($q_result) || ($q_result='""');
! $now=time();
! my $task="update z3950results set
numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now
where id=$resultsid";
! my $stj=$dbi->prepare($task);
! $stj->execute;
! my $counter=0;
! while ($counter<60 && $numrecords<$numresults) {
! $counter++;
! my $stj=$dbi->prepare("select highestseen from
z3950results where id=$resultsid");
! $stj->execute;
! my ($highestseen) = $stj->fetchrow;
! if ($highestseen>($numrecords-30)) {
! $counter=0;
! print " $server rescanning\n";
! my $scantimerstart=time();
! for ($i=$numrecords+1;
$i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) {
! my $rec=$rs->record($i);
! my $marcdata=$rec->rawdata();
! $result.=$marcdata;
! }
! my $scantimerend=time();
! ($numresults<$numrecords+40) ?
($numrecords=$numresults) : ($numrecords=$numrecords+40);
! my $elapsed=$scantimerend-$scantimerstart;
! if ($elapsed) {
! my
$speed=int($numresults/$elapsed*100)/100;
! print " SPEED: $speed $server done
$numrecords\n";
! }
!
! my $q_result=$dbi->quote($result);
! ($q_result) || ($q_result='""');
! $now=time();
! my $task="update z3950results set
numdownloaded=$numrecords,results=$q_result where id=$resultsid";
! my $stj=$dbi->prepare($task);
! $stj->execute;
! }
! sleep 5;
! }
! }
! my $stj=$dbi->prepare("update z3950results set active=0
where id=$resultsid");
! $stj->execute;
! eval {$stj->finish};
! $dbi->disconnect;
! print " $server done.\n";
! exit;
! sub pe {
! (return) unless ($code);
! my $code=$conn->errcode();
! my $msg=$conn->errmsg();
! my $ai=$conn->addinfo();
! print << "EOF";
! CODE: $code
! MSG: $msg
! ADDTL: $ai
!
! EOF
! }
! }
! } unless ($stillprocessing) {
! #my $sti=$dbh->prepare("select enddate from z3950queue
where id=$id");
! #$sti->execute;
! #my ($enddate) = $sti->fetchrow;
! #unless ($enddate) {
! # my $now=time;
! # $sti=$dbh->prepare("update z3950queue set
done=1,numrecords=$totalrecords,enddate=$now where id=$id");
! # $sti->execute;
! # }
! }
! } else {
! # my $q_serverinfo=$dbh->quote($serverinfo);
! # my $stj=$dbh->prepare("insert into z3950results (server, queryid,
startdate) values ($q_serverinfo, $id, 0)");
! # $stj->execute;
! }
}
! $lastrun=time();
}
! sleep 1;
}
***************
*** 62,89 ****
my $query=shift;
my $auth=shift;
! open (M, "|yaz-client -m yaz.mrc >>yaz.out 2>>yaz.err");
select M;
$|=1;
select STDOUT;
($auth) && ($auth="authentication $auth\n");
- print << "EOF";
- $auth\open $server
- base $base
- $query
- s
- s
- s
- s
- s
- s
- s
- s
- s
- s
- quit
- EOF
print M << "EOF";
$auth\open $server
base $base
$query
s
--- 212,225 ----
my $query=shift;
my $auth=shift;
! my $id=shift;
! open (M, "|yaz-client -m yaz-$id.mrc >>yaz.out 2>>yaz.err");
select M;
$|=1;
select STDOUT;
($auth) && ($auth="authentication $auth\n");
print M << "EOF";
$auth\open $server
base $base
+ setnames
$query
s
***************
*** 101,102 ****
--- 237,244 ----
close M;
}
+ sub reap {
+ $forkcounter--;
+ }
+
+
+
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Koha-devel] CVS: koha/acqui.simple marcimport.pl,1.4,1.5 processz3950queue,1.1,1.2,
Steve Tonnesen <=