Parallel scripting, processing and execution

Reference:
http://www.gnu.org/software/parallel/
http://www.sunfreeware.com/programlistsparc10.html#parallel
# List compressed file content
gzip -l file
# Do the same with parallel [multiple files]
parallel gzip -l *
# unzip files
gunzip file
# Do the same with parallel [multiple files]
parallel gunzip *
# run as many jobs in parallel as possible on all *.Z files
# uncompress files and search for text ‘icons’
# {} reads arguments  passed to parallel
ls *.Z | parallel -j+0 ‘zcat {} | grep icons ‘
# run as many jobs in parallel as possible on all *.Z files
# clock the execution; dump results to templ
# {} reads arguments  passed to parallel
time ls *.Z | parallel -j+0 ‘zcat {} | grep DATE ‘ > temp1
# -k keep sequence in order [queue]
# -j 200% = run two jobs per CPU core
# -n process 100 arguments at time
# search for work DATE in *.Z files
# {} reads arguments  passed to parallel
time ls *.Z | parallel -k -j200% -n 100 ‘zcat {} | grep DATE ‘ > temp1
# using parallel with gunzip
time ls *.gz | parallel -j+0 ‘gunzip  {}’
real     0m2.99s
user     0m1.09s
sys     0m1.48s
# parallel with gzip
time ls *.log | parallel -j+0 ‘gzip  {}’
real     0m3.06s
user     0m1.10s
sys     0m1.55s
# print compressed  files
# then unzip them all in parallel using xargs
time ls -l *.gz | awk ‘{print $NF }’ | xargs gunzip
real     0m0.07s
user     0m0.01s
sys     0m0.02s
# print log  files
# then compress them all in parallel using xargs
time ls -l *.log | awk ‘{print $NF }’ | xargs gzip
real     0m0.09s
user     0m0.03s
sys     0m0.03s

Leave a Reply

Your email address will not be published. Required fields are marked *