== Creation of lists ==
filter="L"
ls bias*[0-9].fit > bias.list
ls dark600s-00*fit > dark600s.list
ls dark*5sec*fit > dark5s.list
ls flat*${filter}.fit > flat${filter}.list
sed s/^/b/g flat${filter}.list > bflat${filter}.list
sed s/^/bd/g flat${filter}.list > bdflat${filter}.list
ls ngc*${filter}.fit > ${filter}.list
sed s/^/b/g ${filter}.list > b${filter}.list
sed s/^/bd/g ${filter}.list > bd${filter}.list
sed s/^/bdf/g ${filter}.list > bdf${filter}.list
sed s/^/objbdf/g ${filter}.list | sed s/fit/pl/g > objbdf${filter}.list

== Basic data reduction ==

# Below is a bash script that creates lists of all images: biases, flats, darks and targets.
# Special case of dark frames of 600sec that Milankovic generates is used
# Last lines create ascii files with median values in flat frames that will be used
# for their normalization

#!/bin/bash

rm *list
for frame in $(ls *fit); do
filterType=$(gethead filter $frame)
imageType=$(gethead imagetyp $frame)
outFileName=$(echo $imageType | tr '[:upper:]' '[:lower:]')
# test if DARK
if [ ${imageType} = "DARK" ]; then
exptime=$(gethead exptime $frame | cut -d. -f1)
ls $frame >> ${outFileName}${exptime}"s".list
echo "b"$frame >> "b"${outFileName}${exptime}"s".list
else ls $frame >> ${outFileName}${filterType}.list
fi
# test if LIGHT and create additional lists
if [ ${imageType} = "LIGHT" ]; then
exptime=$(gethead exptime $frame | cut -d. -f1)
echo $exptime >> exptime${filterType}.list
echo "b"$frame >> "b"${outFileName}${filterType}.list
echo "bd"$frame >> "bd"${outFileName}${filterType}.list
echo "bdf"$frame >> "bdf"${outFileName}${filterType}.list
else if [ ${imageType} = "FLAT" ]; then
echo "b"$frame >> "b"${outFileName}${filterType}.list
echo "bd"$frame >> "bd"${outFileName}${filterType}.list
echo "nbd"$frame >> "nbd"${outFileName}${filterType}.list
fi
fi
done

# Create dark current corresponding to each light frame using exposure time
for exptime in $(ls exptime*.list); do
filter=$(echo $exptime | sed -e s/exptime//g -e s/\.list//g);
awk -v filter=${filter} '{print "dc"NR"_"filter".fits"}' exptime${filter}.list > dc${filter}.list
done

if [ -e reduce.cl ]; then rm reduce.cl; fi
echo "imdelete master_bias,dc_600s,dc_5s,dc_1s" >> reduce.cl
echo "imcombine @bias.list master_bias comb=med" >> reduce.cl
echo "imarith @dark600s.list - master_bias @bdark600s.list" >> reduce.cl
echo "imarith @dark5s.list - master_bias @bdark5s.list" >> reduce.cl
echo "imcombine @bdark5s.list dc_5s comb=med" >> reduce.cl
echo "imcombine @bdark600s.list dc_600s comb=med" >> reduce.cl
echo "imarith dc_600s / 600 dc_1s" >> reduce.cl

for flats in $(ls flat*list); do
filterType=$(echo $flats | sed s/flat//g | cut -d. -f1)
#echo $filterType
echo "imarith @flat${filterType}.list - master_bias @bflat${filterType}.list" >> reduce.cl
echo "imarith @bflat${filterType}.list - dc_5s @bdflat${filterType}.list" >> reduce.cl
echo "imstat @bdflat${filterType}.list fields=\"image,midpt\" format- > norm${filterType}.txt" >> reduce.cl
done

== BPM & object mask creation ==
# cat bpmask.cl
imstat dc_300s field="image,mean,stddev" for- > hot.txt
imstat master_flat_FILTER fields="image,mean" for- > cold.txt
cl < bpmask.cl
# awk -v filter="L" '{print "imexpr \"((a > b) ? 1 : 0)\" hot"filter".pl a="$1" b="($2+3.*$3)*0.75}' hot.txt > hot.cl
# awk -v filter="L" '{print "imexpr \"((a < b) ? 1 : 0)\" cold"filter".pl a="$1" b="$2*0.9}' cold.txt > cold.cl
cl < hot.cl
cl < cold.cl
# cat bpm.cl
imexpr "max(a,b)" bpmFILTER.pl a=hotFILTER.pl b=coldFILTER.pl
hedit @bdfFILTER.list BPM bpmFILTER.pl add+ ver-
# sed 's/FILTER/'${filter}'/g' bpm.cl > dobpm.cl
cl < dobpm.cl

# Object masks creation
nproto
objmask @bdfFILTER.list @objbdfFILTER.list masks="!BPM" hsig=2 lsig=2

== Astrometry ==
# cat astrometry.sh
gethead OBJCTRA @bdfFILTER.list > name_ra.txt
gethead OBJCTDEC @bdfFILTER.list | awk '{print $2,$3,$4}' > dec.txt
paste -d" " name_ra.txt dec.txt | awk '{print $1,($2+$3/60+$4/3600)*15,$5+$6/60+$7/3600}' > astrometryFILTER.txt
# sed s/FILTER/${filter}/g astrometry.sh > astrometry${filter}.sh

sh astrometry${filter}.sh

# This could be used:
# cat komanda.sh
solve-field img --ra imagera --dec imagedec --radius 0.2 --scale-units arcsecperpix --scale-low 0.38 --scale-high 0.40 --crpix-center --wcs img.wcs
### The next command creates new files instead of just wcs binaries that can later be added to the header. It's a stand-alone command. And should be ignored.
### solve-field img --ra imagera --dec imagedec --radius 0.2 --scale-units arcsecperpix --scale-low 0.38 --scale-high 0.40 --crpix-center -p -N wcsimg.fit \
### -O -I noneI.fits -M none.fits -R none.fits -B none.fits -P none -k none -U none -y -S none --axy noneaxy
#while read line; do img=$(echo $line | awk '{print $1}'); imagera=$(echo $line | awk '{print $2}'); imagedec=$(echo $line | awk '{print $3}'); sed -e 's/img/'$img'/g' \
-e 's/imagera/'${imagera}'/g' -e 's/imagedec/'${imagedec}'/g' komanda.sh; done < astrometry${filter}.txt > doAstrometry.sh
# Or this:
awk '{print "solve-field "$1" --ra "$2" --dec "$3" --radius 0.2 --scale-units arcsecperpix --scale-low 0.38 --scale-high 0.40 --crpix-center -p -N wcs"$1" \
-O -I noneI.fits -M none.fits -R none.fits -B none.fits -P none -k none -U none -y -S none --axy noneaxy --wcs bla.wcs"}' astrometry${filter}.txt > doAstrometry.sh

# /bin/bash doAstrometry.sh
#ls *wcs > wcs${filter}.list
# echo "wcscopy @bdf${filter}.list @wcs${filter}.list ver-" > dowcs.cl
cl < dowcs.cl

== Supersky flat ==
# Creation of super sky flat, normalization to each individual image median background and final stacking
# cat stat.cl
real mini
real sky
imstat("img",fields="min",lower=INDEF,upper=INDEF,for-) | scanf("%g", mini)
imcalc(input="img,msk",output="skyimage",equals="if(im2==0.) then im1 else "//mini-0.001,ver-)
imstat("skyimage",fields="midpt",lower=mini-0.001,upper=INDEF,for-) | scanf("%g", sky)
print("img"," ",sky)
imarith("img","/",sky,"norm")
imdelete("skyimage",yes)
# while read line; do img=$(echo $line | awk '{print $1}'); msk="obj"${img%.*}.pl; rsky="norm"${img}; cat stat.cl | sed -e 's/img/'${img}'/g' \
-e 's/msk/'${msk}'/g' -e 's/norm/'${rsky}'/g'; done < bdf${filter}.list > bla.cl
cl < bla.cl > med_values.txt
# ls normbdf*fit > norm.list
imcomb @norm.list norm.fits comb=med masktype="!OBJMASK" maskval=0
# awk '{print "imarith norm.fits * "$2," n"$1}' med_values.txt > donorm.cl
cl < donorm.cl
# awk '{print "imarith",$1,"-","n"$1,"s"$1}' med_values.txt > do.cl
cl < do.cl

imcomb s@bdfFILTER.list final.fits comb=med offset="wcs"

== Surface brightness limit ==

1. Calibrate the image taking non-saturated stars and not too faint stars: intercept and slope (TOPCAT with SDSS DR12 from Vizier)
2. Calculate statistics inside at least 20 (empty, i.e. without objects) boxes of the size of 10" (for example if 1pix=0.396" than 10"~25 pix so boxes = 25 pix × 25 pix)
3. Take the mean value of the standard deviation inside boxes and it's standard deviation
4. Calculate SBmin and SBmax as:
SBmax =-2.5*log10(<σ>-stdev(<σ>))*slope+intercept
SBmin =-2.5*log10(<σ>+stdev(<σ>))*slope+intercept
where <σ> is the average standard deviation inside boxes and stdev(<σ>) its standard deviation.
5. Finally, 1 sigma surface brightness limit (SBlim) is given as:
SBlim = (SBmin+SBmax)/2 +/- (SBmax-SBmin)/2

LinuxShort

2023-11-10T12:45:06Z

Ana.Lalovic: /* Extract particula pages from pdf file */

== Usefull short commands ==

=== '''Kill a process tree''' ===

When some script is started from a command line in a for-loop and a process tree is made, one should find the parent process and its PID and then simply kill it:

$ pstree username -p 
sshd(19953)---bash(19954)---bash(22238)---idl(22323)-+-{idl}(22326)
|-{idl}(22327)
|-{idl}(22328)
`-{idl}(22329) 
sshd(20096)---bash(20097)
sshd(20762)---bash(20763)
sshd(21087)---bash(21088)-+-grep(21309)
`-top(21308)
sshd(21736)---bash(21737)
sshd(21780)---bash(21781)---pstree(22331)

Or if this tree is to long, one should ''grep'' the program (here:idl) with: $ pstree usrname -p | grep idl
$ kill 19953 
$ pstree username -p 
sshd(20096)---bash(20097)
sshd(20762)---bash(20763)
sshd(21087)---bash(21088)-+-grep(21309)
`-top(21308)
sshd(21736)---bash(21737)
sshd(21780)---bash(21781)---pstree(22332)

=== '''Change sth inside file''' ===

sed -i s/'\-9999'/'NaN'/g filename

This will change -9999 with NaN inside a file, without making some tmp file in between.

=== '''Print rotated picture''' ===

lp -o media=a4 -o orientation-requested=4 -o fitplot figure.ps

=== '''Print in fixed width format with awk''' ===

echo $a $b $c | awk '{for (i=1; i<=NF; i++) printf("%-20s ",$i);printf ("\n")}' > outfile.dat

where strings a,b,c may be entire lines.

=== '''Make a small size pdf out of several pdfs''' ===
gs -sDEVICE=pdfwrite -q -dNOPAUSE -dBATCH -sOutputFile=MajorFile.pdf SmallFiles*.pdf

Combined files can ba a mix of .pdf .ps and .eps files and output file can be a .ps also

=== '''Cut a portion of a pdf file saving resolution and sharpness''' ===
gs -g2700x3500 -sOUTPUTFILE=output.pdf -sDEVICE=pdfwrite -dBATCH -c "<< /PageOffset [-30 -120] >> setpagedevice" -f input.pdf

Where [-30 -120] means - shift the content of the old image to its left corner by (x,y)=(30,120) pixels (change the origin of the image) and 2700x3500 are the dimensions of the new image. Be careful - input.pdf should be a single page. 

=== '''Make a series of single-page EPS files from various inputs''' ===
gs -sDEVICE=epswrite -q -dNOPAUSE -dBATCH -dSAFER -sOutputFile=p%08d.eps 5page-first.pdf 7page-second.ps 1page-third.eps

The resulting files will be nicely named as p00000001.eps .... p00000013.eps

=== ''' Cut a portion of a .pdf file ''' ===
Make a script named boundingBox.sh:
gs \
-q \
-dBATCH \
-dNOPAUSE \
-sDEVICE=bbox \
-dLastPage=1 \
$1 \
2>&1 \
| grep %%BoundingBox

Call the script from the line with file_name as input:
./boundingBox.sh file.pdf

The result will be sth. like: %%BoundingBox: 42 510 305 746, where lower left img coords = (42,510) and upper right = (305,746). Finaly, to cut a box 305-42=263 px wide and 746-510=236 high, excluding 42 pixels from the left and 510 pixels from the bottom, make a script ''crop.sh'':
gs \
-o out.png \
-sDEVICE=pngalpha \
-g$2x$3 \
-dLastPage=1 \
-c "<</Install {$4 $5 translate}>> setpagedevice" \
-f $1

where $1x$2=width x height and $4 $5 are the portion excluded from the left and bootom. In this example:
./crop.sh file.pdf 263 236 -42 -510

To make .pdf out try:
gs -o out.pdf -sDEVICE=pdfwrite -c "[/CropBox [42 510 305 746] /PAGES pdfmark" -f file.eps

=== '''Extract particular pages from pdf file''' ===

#!/bin/sh
#
# pdfsplit [input.pdf] [first_page] [last_page] [output.pdf]
#
# Example: pdfsplit big_file.pdf 10 20 pages_ten_to_twenty.pdf
#
# written by: Westley Weimer, Wed Mar 19 17:58:09 EDT 2008
#
# The trick: ghostscript (gs) will do PDF splitting for you, it's just not
# obvious and the required defines are not listed in the manual page.

if [ $# -lt 4 ]
then
echo "Usage: pdfsplit input.pdf first_page last_page output.pdf"
exit 1
fi
gs -dNOPAUSE -dQUIET -dBATCH -sOutputFile="$4" -dFirstPage=$2 -dLastPage=$3 -sDEVICE=pdfwrite "$1"

=== '''Convert eps to jpg using native gs''' ===
gs -sDEVICE=jpeg -q -dJPEGQ=100 -dNOPAUSE -dBATCH -dSAFER -r100 -sOutputFile=image.jpg image.eps # ''-r100'' should be changed to ''-r50'' for denser image

=== '''Convert (e)ps to png or gif ''' ===

name="imgName"; figdir="/home/..../Figures"; text="my image"
1. convert $name.ps -depth 8 -alpha off -density 600
2. convert $name.ps -depth 8 -alpha off -density 600 ${figdir}/${name}_large.gif
3. convert $name.ps -depth 8 -alpha off -density 600 -resize 320x240 ${figdir}/${name}_large.gif
4. convert $name.ps -depth 8 -alpha off -density 600 -gravity Southeast -font helvetica -pointsize 75 \
-draw "text 150,100 '$text'" ${figdir}/${name}_large.gif

1. Resulting image will be named as .eps and will end up at the same place.

2. Resulting image will be named imgName_large.gif and will be placed in \$figdir.

3. Additionally, image can be resized in the same step.

4. Put text into image

=== '''Resize image''' ===
mogrify -trim -resize 70% image.jpg # or ''-resize 800x600'' for fixed width

=== Screen command ===
screen -S name # name a screen
screen -ls # list all screens (those named will be easily read)
Ctrl+a+d # detach screen
screen -r name # attach to a screen
screen -S name -X quit # kill detached screen and return to a terminal
ls -laR /var/run/screen/ # lists when the screens start running

=== Several images into one ===
montage img1.png img2.png img3.png img4.png img5.png img6.png -tile 2x3 -frame 2 -geometry +0+0 out.png

=== Animation on .gif images ===
convert -delay 50 img1.gif img2.gif -loop 0 anim.gif

== Some cool stuff ==

* Very nice themes for Ubuntu 10.04

sudo add-apt-repository ppa:bisigi
sudo aptitude update
sudo aptitude install bisigi-themes

* Convert youtube .flv to .mp3 ([http://ubuntuforums.org/showthread.php?t=1117283 FORUM])
sudo apt-get install youtube-dl
sudo youtube-dl -U
sudo youtube-dl -U
sudo apt-get install ffmpeg libavcodec-unstripped-52
sudo apt-get install ubuntu-restricted-extras
# Example:
youtube-dl http://www.youtube.com/watch?v=Md7IhGYKncs
ffmpeg -i Md7IhGYKncs.flv -acodec libmp3lame -ab 128 indiana.mp3

== Install IRAF in Ubuntu 16 with Anaconda ==

Source: https://www.anaconda.com/download/#_unix

First install the following libs:

sudo apt-get install libc6:i386 libz1:i386 libncurses5:i386 libbz2-1.0:i386 libuuid1:i386 libxcb1:i386
sudo apt-get install libxmu-dev:i386

Then download Anaconda from https://www.continuum.io/downloads

cd /home/<user_name>/
bash Anaconda2-4.4.0-Linux-x86_64.sh
conda config --add channels http://ssb.stsci.edu/astroconda
conda create -n iraf python=2.7 iraf-all pyraf-all stsci
source activate iraf
mkiraf
<set to xgterm>
cl
...
source deactivate iraf

Pyraf also works in a standard way...

LinuxShort

2023-11-10T12:44:49Z

Ana.Lalovic:

== Usefull short commands ==

=== '''Kill a process tree''' ===

When some script is started from a command line in a for-loop and a process tree is made, one should find the parent process and its PID and then simply kill it:

$ pstree username -p 
sshd(19953)---bash(19954)---bash(22238)---idl(22323)-+-{idl}(22326)
|-{idl}(22327)
|-{idl}(22328)
`-{idl}(22329) 
sshd(20096)---bash(20097)
sshd(20762)---bash(20763)
sshd(21087)---bash(21088)-+-grep(21309)
`-top(21308)
sshd(21736)---bash(21737)
sshd(21780)---bash(21781)---pstree(22331)

Or if this tree is to long, one should ''grep'' the program (here:idl) with: $ pstree usrname -p | grep idl
$ kill 19953 
$ pstree username -p 
sshd(20096)---bash(20097)
sshd(20762)---bash(20763)
sshd(21087)---bash(21088)-+-grep(21309)
`-top(21308)
sshd(21736)---bash(21737)
sshd(21780)---bash(21781)---pstree(22332)

=== '''Change sth inside file''' ===

sed -i s/'\-9999'/'NaN'/g filename

This will change -9999 with NaN inside a file, without making some tmp file in between.

=== '''Print rotated picture''' ===

lp -o media=a4 -o orientation-requested=4 -o fitplot figure.ps

=== '''Print in fixed width format with awk''' ===

echo $a $b $c | awk '{for (i=1; i<=NF; i++) printf("%-20s ",$i);printf ("\n")}' > outfile.dat

where strings a,b,c may be entire lines.

=== '''Make a small size pdf out of several pdfs''' ===
gs -sDEVICE=pdfwrite -q -dNOPAUSE -dBATCH -sOutputFile=MajorFile.pdf SmallFiles*.pdf

Combined files can ba a mix of .pdf .ps and .eps files and output file can be a .ps also

=== '''Cut a portion of a pdf file saving resolution and sharpness''' ===
gs -g2700x3500 -sOUTPUTFILE=output.pdf -sDEVICE=pdfwrite -dBATCH -c "<< /PageOffset [-30 -120] >> setpagedevice" -f input.pdf

Where [-30 -120] means - shift the content of the old image to its left corner by (x,y)=(30,120) pixels (change the origin of the image) and 2700x3500 are the dimensions of the new image. Be careful - input.pdf should be a single page. 

=== '''Make a series of single-page EPS files from various inputs''' ===
gs -sDEVICE=epswrite -q -dNOPAUSE -dBATCH -dSAFER -sOutputFile=p%08d.eps 5page-first.pdf 7page-second.ps 1page-third.eps

The resulting files will be nicely named as p00000001.eps .... p00000013.eps

=== ''' Cut a portion of a .pdf file ''' ===
Make a script named boundingBox.sh:
gs \
-q \
-dBATCH \
-dNOPAUSE \
-sDEVICE=bbox \
-dLastPage=1 \
$1 \
2>&1 \
| grep %%BoundingBox

Call the script from the line with file_name as input:
./boundingBox.sh file.pdf

The result will be sth. like: %%BoundingBox: 42 510 305 746, where lower left img coords = (42,510) and upper right = (305,746). Finaly, to cut a box 305-42=263 px wide and 746-510=236 high, excluding 42 pixels from the left and 510 pixels from the bottom, make a script ''crop.sh'':
gs \
-o out.png \
-sDEVICE=pngalpha \
-g$2x$3 \
-dLastPage=1 \
-c "<</Install {$4 $5 translate}>> setpagedevice" \
-f $1

where $1x$2=width x height and $4 $5 are the portion excluded from the left and bootom. In this example:
./crop.sh file.pdf 263 236 -42 -510

To make .pdf out try:
gs -o out.pdf -sDEVICE=pdfwrite -c "[/CropBox [42 510 305 746] /PAGES pdfmark" -f file.eps

=== '''Extract particula pages from pdf file''' ===

#!/bin/sh
#
# pdfsplit [input.pdf] [first_page] [last_page] [output.pdf]
#
# Example: pdfsplit big_file.pdf 10 20 pages_ten_to_twenty.pdf
#
# written by: Westley Weimer, Wed Mar 19 17:58:09 EDT 2008
#
# The trick: ghostscript (gs) will do PDF splitting for you, it's just not
# obvious and the required defines are not listed in the manual page.

if [ $# -lt 4 ]
then
echo "Usage: pdfsplit input.pdf first_page last_page output.pdf"
exit 1
fi
gs -dNOPAUSE -dQUIET -dBATCH -sOutputFile="$4" -dFirstPage=$2 -dLastPage=$3 -sDEVICE=pdfwrite "$1"

=== '''Convert eps to jpg using native gs''' ===
gs -sDEVICE=jpeg -q -dJPEGQ=100 -dNOPAUSE -dBATCH -dSAFER -r100 -sOutputFile=image.jpg image.eps # ''-r100'' should be changed to ''-r50'' for denser image

=== '''Convert (e)ps to png or gif ''' ===

name="imgName"; figdir="/home/..../Figures"; text="my image"
1. convert $name.ps -depth 8 -alpha off -density 600
2. convert $name.ps -depth 8 -alpha off -density 600 ${figdir}/${name}_large.gif
3. convert $name.ps -depth 8 -alpha off -density 600 -resize 320x240 ${figdir}/${name}_large.gif
4. convert $name.ps -depth 8 -alpha off -density 600 -gravity Southeast -font helvetica -pointsize 75 \
-draw "text 150,100 '$text'" ${figdir}/${name}_large.gif

1. Resulting image will be named as .eps and will end up at the same place.

2. Resulting image will be named imgName_large.gif and will be placed in \$figdir.

3. Additionally, image can be resized in the same step.

4. Put text into image

=== '''Resize image''' ===
mogrify -trim -resize 70% image.jpg # or ''-resize 800x600'' for fixed width

=== Screen command ===
screen -S name # name a screen
screen -ls # list all screens (those named will be easily read)
Ctrl+a+d # detach screen
screen -r name # attach to a screen
screen -S name -X quit # kill detached screen and return to a terminal
ls -laR /var/run/screen/ # lists when the screens start running

=== Several images into one ===
montage img1.png img2.png img3.png img4.png img5.png img6.png -tile 2x3 -frame 2 -geometry +0+0 out.png

=== Animation on .gif images ===
convert -delay 50 img1.gif img2.gif -loop 0 anim.gif

== Some cool stuff ==

* Very nice themes for Ubuntu 10.04

sudo add-apt-repository ppa:bisigi
sudo aptitude update
sudo aptitude install bisigi-themes

* Convert youtube .flv to .mp3 ([http://ubuntuforums.org/showthread.php?t=1117283 FORUM])
sudo apt-get install youtube-dl
sudo youtube-dl -U
sudo youtube-dl -U
sudo apt-get install ffmpeg libavcodec-unstripped-52
sudo apt-get install ubuntu-restricted-extras
# Example:
youtube-dl http://www.youtube.com/watch?v=Md7IhGYKncs
ffmpeg -i Md7IhGYKncs.flv -acodec libmp3lame -ab 128 indiana.mp3

== Install IRAF in Ubuntu 16 with Anaconda ==

Source: https://www.anaconda.com/download/#_unix

First install the following libs:

sudo apt-get install libc6:i386 libz1:i386 libncurses5:i386 libbz2-1.0:i386 libuuid1:i386 libxcb1:i386
sudo apt-get install libxmu-dev:i386

Then download Anaconda from https://www.continuum.io/downloads

cd /home/<user_name>/
bash Anaconda2-4.4.0-Linux-x86_64.sh
conda config --add channels http://ssb.stsci.edu/astroconda
conda create -n iraf python=2.7 iraf-all pyraf-all stsci
source activate iraf
mkiraf
<set to xgterm>
cl
...
source deactivate iraf

Pyraf also works in a standard way...