Skip to content

Instantly share code, notes, and snippets.

@phluid61

phluid61/generate_views_parallel

Last active June 6, 2019 01:31
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save phluid61/8817db6a20d217b44cc128ef41e5bd42 to your computer and use it in GitHub Desktop.
Save phluid61/8817db6a20d217b44cc128ef41e5bd42 to your computer and use it in GitHub Desktop.
Download ZIP
Run generate_views in parallel: `MAX_PARALLEL_CHILDREN=4 bin/generate_views_parallel myrepo`
Raw
generate_views_parallel
#!/usr/bin/perl -w
use FindBin;
use lib "$FindBin::Bin/../perl_lib";
use EPrints;
use strict;
use Getopt::Long;
use Pod::Usage;
use POSIX qw(strftime);
use Carp;
my $DEBUG = 1;
#local $SIG{__WARN__} = sub { confess @_; };
sub now {
return strftime '%F %T', localtime(time);
}
# $childprocs : HASHREF of { $pid => _, ... }
# $limit : int - wait until $childprocs has fewer entries than this
sub flush_childprocs {
my( $childprocs, $limit ) = @_;
$limit //= 1;
while( $limit && ( scalar keys %$childprocs >= $limit ))
{
my $pid = wait;
if( $pid == -1 )
{
print STDERR "fork(): expected ",(scalar keys %$childprocs)," child processes, but wait() returned -1\n";
delete $childprocs->{ keys %$childprocs };
}
else
{
delete $childprocs->{$pid};
}
}
}
my $verbose = 0;
my $quiet = 0;
my $help = 0;
my $man = 0;
my $generate_opt;
my $lang_opt;
Getopt::Long::Configure("permute");
GetOptions(
'help|?' => \$help,
'man' => \$man,
'verbose+' => \$verbose,
'silent' => \$quiet,
'quiet' => \$quiet,
'generate=s' => \$generate_opt,
'lang=s' => \$lang_opt,
) || pod2usage(2);
pod2usage(1) if $help;
pod2usage(-exitstatus => 0, -verbose => 2) if $man;
pod2usage(2) if(scalar @ARGV != 1);
my $noise = 1;
$noise = 0 if($quiet);
$noise = 1+$verbose if($verbose);
my $do_menus = 1;
my $do_lists = 1;
if (defined $generate_opt) {
if ($generate_opt ne 'menus' and $generate_opt ne 'lists') {
print STDERR "--generate must be either menus or lists.\n";
exit 1;
}
}
$|=1;
binmode(STDOUT, ':utf8');
my $repoid = $ARGV[0];
if (!defined $repoid) {
print STDERR "Usage: $0 <repoid>\n";
exit 1;
}
my $session = new EPrints::Session(1 , $repoid, $noise);
if (!defined $session) {
print STDERR "Failed to load repository: $repoid\n";
exit 1;
}
my $repository = $session->get_repository;
my $views = $repository->get_conf('browse_views');
my $MAX_PARALLEL_CHILDREN = 1;
if( defined $ENV{MAX_PARALLEL_CHILDREN} && $ENV{MAX_PARALLEL_CHILDREN} >= 0 )
{
$MAX_PARALLEL_CHILDREN = 0 + $ENV{MAX_PARALLEL_CHILDREN};
}
my %childprocs = ();
VIEW: foreach my $view (@{$views}) {
my $pid = fork();
if( $pid )
{
$childprocs{$pid} = 1;
# if there are enough child processes running, wait for some to
# finish before spawning more
flush_childprocs( \%childprocs, $MAX_PARALLEL_CHILDREN );
}
else
{
print '[', now, '] beginning ', $view->{id}, "\n" if $DEBUG || $noise > 1;
my @args = ('/opt/eprints3/bin/generate_views', $repoid, '--view', $view->{id});
if ($generate_opt) {
push @args, '--generate', $generate_opt;
}
if ($noise < 1) {
push @args, '--silent';
} else {
my $v = $verbose;
while ($v) {
push @args, '--verbose';
$v --;
}
}
if ($lang_opt) {
push @args, '--lang', $lang_opt;
}
print '[', now, '] > ', (join ' ', @args), "\n" if $DEBUG || $noise > 2;
system @args;
print '[', now, '] finished ', $view->{id}, "\n" if $DEBUG || $noise > 1;
exit;
}
}
# wait for any remaining child processes to terminate
flush_childprocs( \%childprocs );
$session->terminate();
exit;
=pod
=for Pod2Wiki
=head1 NAME
B<generate_views_parallel> - Generate static browse pages for an EPrint repository
=head1 SYNOPSIS
B<generate_views_parallel> I<repository_id> [B<options>]
=head1 DESCRIPTION
This script invokes the standard EPrints B<generate_views> script, but does so once per view, to work around a possible memory leak in that script.
B<generate_views> creates some or all of the pages used in the /view/ section of the website. Since 3.1 these pages update themselves if they get only than a certain age, but this can cause delays for the viewer, so this script is still provided to pre-prepare them to provide a smoother experience.
Note: Since EPrints 3.1 it is not essential to run generate_views periodically, but it is still recommended.
=head1 ARGUMENTS
=over 8
=item B<repository_id>
The ID of the eprint repository to use.
=back
=head1 OPTIONS
=over 8
=item B<--generate menus>
Only generate the menu pages, not the pages with links to records.
=item B<--generate lists>
Only generate the pages with lists of records, not the menu pages.
=item B<--lang> I<lang_id>
Generate only pages for the language with this ID.
=item B<--help>
Print a brief help message and exit.
=item B<--man>
Print the full manual page and then exit.
=item B<--quiet>
Be vewwy vewwy quiet. This option will suppress all output unless an error occurs.
=item B<--verbose>
Explain in detail what is going on.
May be repeated for greater effect.
=back
=head1 COPYRIGHT
=for COPYRIGHT BEGIN
Copyright 2019 Queensland University of Technology.
=for COPYRIGHT END
=for LICENSE BEGIN
This file is part of EPrints L<http://www.eprints.org/>.
EPrints is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
EPrints is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
You should have received a copy of the GNU General Public License
along with EPrints. If not, see L<http://www.gnu.org/licenses/>.
=for LICENSE END
=cut
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment