tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Deal with disk full when starting.

Browse source
This commit is contained in:
Ole Tange 2015-03-17 21:36:12 +01:00
parent 815d00ac38
commit 8eae0edbbb
2 changed files with 52 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
src/parallel
View file

@ -1473,6 +1473,7 @@ sub read_args_from_command_line {
# Put args into argfile
print $outfh map { $_,$/ } @group;
seek $outfh, 0, 0;
exit_if_disk_full();
# Append filehandle to -a
push @opt::a, $outfh;
} else {
@ -3384,7 +3385,7 @@ sub reap_usleep {
kill_youngster_if_not_enough_mem();
}
usleep($ms);
Job::exit_if_disk_full();
exit_if_disk_full();
if($opt::linebuffer) {
for my $job (values %Global::running) {
$job->print();
@ -3510,6 +3511,46 @@ sub tmpfile {
return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_);
}
{
my ($disk_full_fh, $b8193, $name);
sub exit_if_disk_full {
# Checks if $TMPDIR is full by writing 8kb to a tmpfile
# If the disk is full: Exit immediately.
# Returns:
# N/A
if(not $disk_full_fh) {
($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df");
unlink $name;
$b8193 = "x"x8193;
}
# Linux does not discover if a disk is full if writing <= 8192
# Tested on:
# bfs btrfs cramfs ext2 ext3 ext4 ext4dev jffs2 jfs minix msdos
# ntfs reiserfs tmpfs ubifs vfat xfs
# TODO this should be tested on different OS similar to this:
#
# doit() {
# sudo mount /dev/ram0 /mnt/loop; sudo chmod 1777 /mnt/loop
# seq 100000 | parallel --tmpdir /mnt/loop/ true &
# seq 6900000 > /mnt/loop/i && echo seq OK
# seq 6980868 > /mnt/loop/i
# seq 10000 > /mnt/loop/ii
# sleep 3
# sudo umount /mnt/loop/ || sudo umount -l /mnt/loop/
# echo >&2
# }
print $disk_full_fh $b8193;
if(not $disk_full_fh
or
tell $disk_full_fh == 0) {
::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
::wait_and_exit(255);
}
truncate $disk_full_fh, 0;
seek($disk_full_fh, 0, 0) || die;
}
}
sub __DEBUGGING__ {}
sub debug {
@ -6826,7 +6867,7 @@ sub print {
}
# Check for disk full
exit_if_disk_full();
::exit_if_disk_full();
if(($opt::dryrun or $Global::verbose)
and
@ -7103,46 +7144,7 @@ sub set_exitsignal {
$self->{'exitsignal'} = $exitsignal;
}
{
my ($disk_full_fh, $b8193, $name);
sub exit_if_disk_full {
# Checks if $TMPDIR is full by writing 8kb to a tmpfile
# If the disk is full: Exit immediately.
# Returns:
# N/A
if(not $disk_full_fh) {
($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df");
unlink $name;
$b8193 = "x"x8193;
}
# Linux does not discover if a disk is full if writing <= 8192
# Tested on:
# bfs btrfs cramfs ext2 ext3 ext4 ext4dev jffs2 jfs minix msdos
# ntfs reiserfs tmpfs ubifs vfat xfs
# TODO this should be tested on different OS similar to this:
#
# doit() {
# sudo mount /dev/ram0 /mnt/loop; sudo chmod 1777 /mnt/loop
# seq 100000 | parallel --tmpdir /mnt/loop/ true &
# seq 6900000 > /mnt/loop/i && echo seq OK
# seq 6980868 > /mnt/loop/i
# seq 10000 > /mnt/loop/ii
# sleep 3
# sudo umount /mnt/loop/ || sudo umount -l /mnt/loop/
# echo >&2
# }
print $disk_full_fh $b8193;
if(not $disk_full_fh
or
tell $disk_full_fh == 0) {
::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
::wait_and_exit(255);
}
truncate $disk_full_fh, 0;
seek($disk_full_fh, 0, 0) || die;
}
}
sub should_we_halt {
# Should we halt? Immediately? Gracefully?

8
testsuite/wanted-results/parallel-local-0.3s
View file

@ -82,6 +82,8 @@ echo '### bug #43817: Some JP char cause problems in positional replacement stri
<EFBFBD><<3C>>
echo '**'
**
echo '### --rpl % that is a substring of longer --rpl %D'
### --rpl % that is a substring of longer --rpl %D
parallel --plus --rpl '%' --rpl '%D $_=::shell_quote(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::' 'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f
a.b/c.d/e.f=a.b/c.d/e.f
a.b/c.d=a.b/c.d
@ -90,3 +92,9 @@ f=f
a.b/c.d/e.f=a.b/c.d/e.f
echo '**'
**
echo '### Disk full'
### Disk full
cat /dev/zero >/mnt/ram/out; parallel --tmpdir /mnt/ram echo ::: OK; rm /mnt/ram/out
cat: write error: No space left on device
parallel: Error: Output is incomplete. Cannot append to buffer file in /mnt/ram. Is the disk full?
parallel: Error: Change $TMPDIR with --tmpdir or use --compress.