mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-26 07:57:58 +00:00
parallel: Deal with disk full when starting.
This commit is contained in:
parent
815d00ac38
commit
8eae0edbbb
86
src/parallel
86
src/parallel
|
@ -1473,6 +1473,7 @@ sub read_args_from_command_line {
|
||||||
# Put args into argfile
|
# Put args into argfile
|
||||||
print $outfh map { $_,$/ } @group;
|
print $outfh map { $_,$/ } @group;
|
||||||
seek $outfh, 0, 0;
|
seek $outfh, 0, 0;
|
||||||
|
exit_if_disk_full();
|
||||||
# Append filehandle to -a
|
# Append filehandle to -a
|
||||||
push @opt::a, $outfh;
|
push @opt::a, $outfh;
|
||||||
} else {
|
} else {
|
||||||
|
@ -3384,7 +3385,7 @@ sub reap_usleep {
|
||||||
kill_youngster_if_not_enough_mem();
|
kill_youngster_if_not_enough_mem();
|
||||||
}
|
}
|
||||||
usleep($ms);
|
usleep($ms);
|
||||||
Job::exit_if_disk_full();
|
exit_if_disk_full();
|
||||||
if($opt::linebuffer) {
|
if($opt::linebuffer) {
|
||||||
for my $job (values %Global::running) {
|
for my $job (values %Global::running) {
|
||||||
$job->print();
|
$job->print();
|
||||||
|
@ -3510,6 +3511,46 @@ sub tmpfile {
|
||||||
return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_);
|
return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
my ($disk_full_fh, $b8193, $name);
|
||||||
|
sub exit_if_disk_full {
|
||||||
|
# Checks if $TMPDIR is full by writing 8kb to a tmpfile
|
||||||
|
# If the disk is full: Exit immediately.
|
||||||
|
# Returns:
|
||||||
|
# N/A
|
||||||
|
if(not $disk_full_fh) {
|
||||||
|
($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df");
|
||||||
|
unlink $name;
|
||||||
|
$b8193 = "x"x8193;
|
||||||
|
}
|
||||||
|
# Linux does not discover if a disk is full if writing <= 8192
|
||||||
|
# Tested on:
|
||||||
|
# bfs btrfs cramfs ext2 ext3 ext4 ext4dev jffs2 jfs minix msdos
|
||||||
|
# ntfs reiserfs tmpfs ubifs vfat xfs
|
||||||
|
# TODO this should be tested on different OS similar to this:
|
||||||
|
#
|
||||||
|
# doit() {
|
||||||
|
# sudo mount /dev/ram0 /mnt/loop; sudo chmod 1777 /mnt/loop
|
||||||
|
# seq 100000 | parallel --tmpdir /mnt/loop/ true &
|
||||||
|
# seq 6900000 > /mnt/loop/i && echo seq OK
|
||||||
|
# seq 6980868 > /mnt/loop/i
|
||||||
|
# seq 10000 > /mnt/loop/ii
|
||||||
|
# sleep 3
|
||||||
|
# sudo umount /mnt/loop/ || sudo umount -l /mnt/loop/
|
||||||
|
# echo >&2
|
||||||
|
# }
|
||||||
|
print $disk_full_fh $b8193;
|
||||||
|
if(not $disk_full_fh
|
||||||
|
or
|
||||||
|
tell $disk_full_fh == 0) {
|
||||||
|
::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
|
||||||
|
::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
|
||||||
|
::wait_and_exit(255);
|
||||||
|
}
|
||||||
|
truncate $disk_full_fh, 0;
|
||||||
|
seek($disk_full_fh, 0, 0) || die;
|
||||||
|
}
|
||||||
|
}
|
||||||
sub __DEBUGGING__ {}
|
sub __DEBUGGING__ {}
|
||||||
|
|
||||||
sub debug {
|
sub debug {
|
||||||
|
@ -6826,7 +6867,7 @@ sub print {
|
||||||
}
|
}
|
||||||
|
|
||||||
# Check for disk full
|
# Check for disk full
|
||||||
exit_if_disk_full();
|
::exit_if_disk_full();
|
||||||
|
|
||||||
if(($opt::dryrun or $Global::verbose)
|
if(($opt::dryrun or $Global::verbose)
|
||||||
and
|
and
|
||||||
|
@ -7103,46 +7144,7 @@ sub set_exitsignal {
|
||||||
$self->{'exitsignal'} = $exitsignal;
|
$self->{'exitsignal'} = $exitsignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
|
||||||
my ($disk_full_fh, $b8193, $name);
|
|
||||||
sub exit_if_disk_full {
|
|
||||||
# Checks if $TMPDIR is full by writing 8kb to a tmpfile
|
|
||||||
# If the disk is full: Exit immediately.
|
|
||||||
# Returns:
|
|
||||||
# N/A
|
|
||||||
if(not $disk_full_fh) {
|
|
||||||
($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df");
|
|
||||||
unlink $name;
|
|
||||||
$b8193 = "x"x8193;
|
|
||||||
}
|
|
||||||
# Linux does not discover if a disk is full if writing <= 8192
|
|
||||||
# Tested on:
|
|
||||||
# bfs btrfs cramfs ext2 ext3 ext4 ext4dev jffs2 jfs minix msdos
|
|
||||||
# ntfs reiserfs tmpfs ubifs vfat xfs
|
|
||||||
# TODO this should be tested on different OS similar to this:
|
|
||||||
#
|
|
||||||
# doit() {
|
|
||||||
# sudo mount /dev/ram0 /mnt/loop; sudo chmod 1777 /mnt/loop
|
|
||||||
# seq 100000 | parallel --tmpdir /mnt/loop/ true &
|
|
||||||
# seq 6900000 > /mnt/loop/i && echo seq OK
|
|
||||||
# seq 6980868 > /mnt/loop/i
|
|
||||||
# seq 10000 > /mnt/loop/ii
|
|
||||||
# sleep 3
|
|
||||||
# sudo umount /mnt/loop/ || sudo umount -l /mnt/loop/
|
|
||||||
# echo >&2
|
|
||||||
# }
|
|
||||||
print $disk_full_fh $b8193;
|
|
||||||
if(not $disk_full_fh
|
|
||||||
or
|
|
||||||
tell $disk_full_fh == 0) {
|
|
||||||
::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
|
|
||||||
::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
|
|
||||||
::wait_and_exit(255);
|
|
||||||
}
|
|
||||||
truncate $disk_full_fh, 0;
|
|
||||||
seek($disk_full_fh, 0, 0) || die;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sub should_we_halt {
|
sub should_we_halt {
|
||||||
# Should we halt? Immediately? Gracefully?
|
# Should we halt? Immediately? Gracefully?
|
||||||
|
|
|
@ -82,6 +82,8 @@ echo '### bug #43817: Some JP char cause problems in positional replacement stri
|
||||||
<EFBFBD><<3C>>
|
<EFBFBD><<3C>>
|
||||||
echo '**'
|
echo '**'
|
||||||
**
|
**
|
||||||
|
echo '### --rpl % that is a substring of longer --rpl %D'
|
||||||
|
### --rpl % that is a substring of longer --rpl %D
|
||||||
parallel --plus --rpl '%' --rpl '%D $_=::shell_quote(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::' 'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f
|
parallel --plus --rpl '%' --rpl '%D $_=::shell_quote(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::' 'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f
|
||||||
a.b/c.d/e.f=a.b/c.d/e.f
|
a.b/c.d/e.f=a.b/c.d/e.f
|
||||||
a.b/c.d=a.b/c.d
|
a.b/c.d=a.b/c.d
|
||||||
|
@ -90,3 +92,9 @@ f=f
|
||||||
a.b/c.d/e.f=a.b/c.d/e.f
|
a.b/c.d/e.f=a.b/c.d/e.f
|
||||||
echo '**'
|
echo '**'
|
||||||
**
|
**
|
||||||
|
echo '### Disk full'
|
||||||
|
### Disk full
|
||||||
|
cat /dev/zero >/mnt/ram/out; parallel --tmpdir /mnt/ram echo ::: OK; rm /mnt/ram/out
|
||||||
|
cat: write error: No space left on device
|
||||||
|
parallel: Error: Output is incomplete. Cannot append to buffer file in /mnt/ram. Is the disk full?
|
||||||
|
parallel: Error: Change $TMPDIR with --tmpdir or use --compress.
|
||||||
|
|
Loading…
Reference in a new issue