Useful sed one liners

FILE SPACING:
# double space a file
sed G
# double space a file which already has blank lines in it. Output file should contain no more than one blank line between lines of text.
sed '/^$/d;G'
# triple space a file
sed 'G;G'
# undo double-spacing (assumes even-numbered lines are always blank)
sed 'n;d'
# insert a blank line above every line which matches "regex"
sed '/regex/{x;p;x;}'
# insert a blank line below every line which matches "regex"
sed '/regex/G'
# insert a blank line above and below every line which matches "regex"
sed '/regex/{x;p;x;G;}'
NUMBERING:
# number each line of a file (simple left alignment). Using a tab (see # note on '\t' at end of file) instead of space will preserve margins.
sed = filename sed 'N;s/\n/\t/'
# number each line of a file (number on left, right-aligned)
sed = filename sed 'N; s/^/ /; s/ *\(.\{6,\}\)\n/\1 /'
# number each line of file, but only print numbers if line is not blank
sed '/./=' filename sed '/./N; s/\n/ /'
# count lines (emulates "wc -l")
sed -n '$='

TEXT CONVERSION AND SUBSTITUTION:
# IN UNIX ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format
sed 's/.$//' # assumes that all lines end with CR/LF
sed 's/^M$//' # in bash/tcsh, press Ctrl-V then Ctrl-M
sed 's/\x0D$//' # gsed 3.02.80, but top script is easier

# IN UNIX ENVIRONMENT: convert Unix newlines (LF) to DOS format
sed "s/$/`echo -e \\\r`/" # command line under ksh
sed 's/$'"/`echo \\\r`/" # command line under bash
sed "s/$/`echo \\\r`/" # command line under zsh
sed 's/$/\r/' # gsed 3.02.80

# IN DOS ENVIRONMENT: convert Unix newlines (LF) to DOS format
sed "s/$//" # method 1
sed -n p # method 2
# IN DOS ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format Can only be done with UnxUtils sed, version 4.0.7 or higher. Cannot be done with other DOS versions of sed. Use "tr" instead.
sed "s/\r//" infile >outfile # UnxUtils
sed v4.0.7 or higher tr -d \r outfile # GNU tr version 1.22 or higher
# delete leading whitespace (spaces, tabs) from front of each line # aligns all text flush left
sed 's/^[ \t]*//' # see note on '\t' at end of file
# delete trailing whitespace (spaces, tabs) from end of each line
sed 's/[ \t]*$//' # see note on '\t' at end of file
# delete BOTH leading and trailing whitespace from each line
sed 's/^[ \t]*//;s/[ \t]*$//'
# insert 5 blank spaces at beginning of each line (make page offset)
sed 's/^/ /'
# align all text flush right on a 79-column width
sed -e :a -e 's/^.\{1,78\}$/ &/;ta' # set at 78 plus 1 space
# center all text in the middle of 79-column width. In method 1, spaces at the beginning of the line are significant, and trailing # spaces are appended at the end of the line. In method 2, spaces at # the beginning of the line are discarded in centering the line, and no trailing spaces appear at the end of lines.
sed -e :a -e 's/^.\{1,77\}$/ & /;ta' # method 1
sed -e :a -e 's/^.\{1,77\}$/ &/;ta' -e 's/\( *\)\1/\1/' # method 2

# substitute (find and replace) "foo" with "bar" on each line
sed 's/foo/bar/' # replaces only 1st instance in a line
sed 's/foo/bar/4' # replaces only 4th instance in a line
sed 's/foo/bar/g' # replaces ALL instances in a line
sed 's/\(.*\)foo\(.*foo\)/\1bar\2/' # replace the next-to-last case
sed 's/\(.*\)foo/\1bar/' # replace only the last case
# substitute "foo" with "bar" ONLY for lines which contain "baz"
sed '/baz/s/foo/bar/g'
# substitute "foo" with "bar" EXCEPT for lines which contain "baz"
sed '/baz/!s/foo/bar/g'
# change "scarlet" or "ruby" or "puce" to "red"
sed 's/scarlet/red/g;s/ruby/red/g;s/puce/red/g' # most seds gsed 's/scarlet\ruby\puce/red/g' # GNU sed only
# reverse order of lines (emulates "tac") bug/feature in HHsed v1.5 causes blank lines to be deleted
sed '1!G;h;$!d' # method 1
sed -n '1!G;h;$p' # method 2

# reverse each character on the line (emulates "rev")
sed '/\n/!G;s/\(.\)\(.*\n\)/&\2\1/;//D;s/.//'
# join pairs of lines side-by-side (like "paste")
sed '$!N;s/\n/ /'
# if a line ends with a backslash, append the next line to it
sed -e :a -e '/\\$/N; s/\\\n//; ta'
# if a line begins with an equal sign, append it to the previous line # and replace the "=" with a single space
sed -e :a -e '$!N;s/\n=/ /;ta' -e 'P;D'
# add commas to numeric strings, changing "1234567" to "1,234,567"
gsed ':a;s/\B[0-9]\{3\}\>/,&/;ta' # GNU sed
sed -e :a -e 's/\(.*[0-9]\)\([0-9]\{3\}\)/\1,\2/;ta' # other seds
# add commas to numbers with decimal points and minus signs (GNU sed)
gsed ':a;s/\(^\[^0-9.]\)\([0-9]\+\)\([0-9]\{3\}\)/\1\2,\3/g;ta'
# add a blank line every 5 lines (after lines 5, 10, 15, 20, etc.) gsed '0~5G' # GNU sed only
sed 'n;n;n;n;G;' # other seds

SELECTIVE PRINTING OF CERTAIN LINES:
# print first 10 lines of file (emulates behavior of "head")
sed 10q
# print first line of file (emulates "head -1")
sed q
# print the last 10 lines of a file (emulates "tail")
sed -e :a -e '$q;N;11,$D;ba'
# print the last 2 lines of a file (emulates "tail -2")
sed '$!N;$!D'
# print the last line of a file (emulates "tail -1")
sed '$!d' # method 1
sed -n '$p' # method 2
# print only lines which match regular expression (emulates "grep")
sed -n '/regexp/p' # method 1
sed '/regexp/!d' # method 2
# print only lines which do NOT match regexp (emulates "grep -v")
sed -n '/regexp/!p' # method 1, corresponds to above
sed '/regexp/d' # method 2, simpler syntax
# print the line immediately before a regexp, but not the line # containing the regexp
sed -n '/regexp/{g;1!p;};h'
# print the line immediately after a regexp, but not the line # containing the regexp
sed -n '/regexp/{n;p;}'
# print 1 line of context before and after regexp, with line number indicating where the regexp occurred (similar to "grep -A1 -B1")
sed -n -e '/regexp/{=;x;1!p;g;$!N;p;D;}' -e h
# grep for AAA and BBB and CCC (in any order)
sed '/AAA/!d; /BBB/!d; /CCC/!d'
# grep for AAA and BBB and CCC (in that order)
sed '/AAA.*BBB.*CCC/!d'
# grep for AAA or BBB or CCC (emulates "egrep")
sed -e '/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d # most seds
gsed '/AAA\BBB\CCC/!d' # GNU sed only
# print paragraph if it contains AAA (blank lines separate paragraphs) HHsed v1.5 must insert a 'G;' after 'x;' in the next 3 scripts below
sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;'
# print paragraph if it contains AAA and BBB and CCC (in any order)
sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;/BBB/!d;/CCC/!d'
# print paragraph if it contains AAA or BBB or CCC
sed -e '/./{H;$!d;}' -e 'x;/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d
gsed '/./{H;$!d;};x;/AAA\BBB\CCC/b;d' # GNU sed only
# print only lines of 65 characters or longer
sed -n '/^.\{65\}/p'
# print only lines of less than 65 characters
sed -n '/^.\{65\}/!p' # method 1, corresponds to above
sed '/^.\{65\}/d' # method 2, simpler syntax
# print section of file from regular expression to end of file
sed -n '/regexp/,$p'
# print section of file based on line numbers (lines 8-12, inclusive)
sed -n '8,12p' # method 1
sed '8,12!d' # method 2
# print line number 52 sed -n '52p' # method 1
sed '52!d' # method 2
sed '52q;d' # method 3, efficient on large files
# beginning at line 3, print every 7th line
gsed -n '3~7p' # GNU sed only
sed -n '3,${p;n;n;n;n;n;n;}' # other seds
# print section of file between two regular expressions (inclusive)
sed -n '/Iowa/,/Montana/p' # case sensitive

SELECTIVE DELETION OF CERTAIN LINES:
# print all of file EXCEPT section between 2 regular expressions
sed '/Iowa/,/Montana/d'
# delete duplicate, consecutive lines from a file (emulates "uniq"). First line in a set of duplicate lines is kept, rest are deleted.
sed '$!N; /^\(.*\)\n\1$/!P; D'
# delete duplicate, nonconsecutive lines from a file. Beware not to overflow the buffer size of the hold space, or else use GNU sed.
sed -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P'
# delete all lines except duplicate lines (emulates "uniq -d").
sed '$!N; s/^\(.*\)\n\1$/\1/; t; D'
# delete the first 10 lines of a file
sed '1,10d'
# delete the last line of a file
sed '$d'
# delete the last 2 lines of a file
sed 'N;$!P;$!D;$d'
# delete the last 10 lines of a file
sed -e :a -e '$d;N;2,10ba' -e 'P;D' # method 1
sed -n -e :a -e '1,10!{P;N;D;};N;ba' # method 2
# delete every 8th line
gsed '0~8d' # GNU sed only
sed 'n;n;n;n;n;n;n;d;' # other seds
# delete ALL blank lines from a file (same as "grep '.' ")
sed '/^$/d' # method 1
sed '/./!d' # method 2
# delete all CONSECUTIVE blank lines from file except the first; also deletes all blank lines from top and end of file (emulates "cat -s")
sed '/./,/^$/!d' # method 1, allows 0 blanks at top, 1 at EOF
sed '/^$/N;/\n$/D' # method 2, allows 1 blank at top, 0 at EOF
# delete all CONSECUTIVE blank lines from file except the first 2:
sed '/^$/N;/\n$/N;//D'
# delete all leading blank lines at top of file
sed '/./,$!d'
# delete all trailing blank lines at end of file
sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' # works on all seds
sed -e :a -e '/^\n*$/N;/\n$/ba' # ditto, except for gsed 3.02*
# delete the last line of each paragraph
sed -n '/^$/{p;h;};/./{x;/./p;}'

SPECIAL APPLICATIONS:
# remove nroff overstrikes (char, backspace) from man pages. The 'echo' command may need an -e switch if you use Unix System V or bash shell.
sed "s/.`echo \\\b`//g" # double quotes required for Unix environment
sed 's/.^H//g' # in bash/tcsh, press Ctrl-V and then Ctrl-H
sed 's/.\x08//g' # hex expression for sed v1.5
# get Usenet/e-mail message header
sed '/^$/q' # deletes everything after first blank line
# get Usenet/e-mail message body
sed '1,/^$/d' # deletes everything up to first blank line
# get Subject header, but remove initial "Subject: " portion
sed '/^Subject: */!d; s///;q'
# get return address header
sed '/^Reply-To:/q; /^From:/h; /./d;g;q'
# parse out the address proper. Pulls out the e-mail address by itself from the 1-line return address header (see preceding script)
sed 's/ *(.*)//; s/>.*//; s/.*[:<] *//' # add a leading angle bracket and space to each line (quote a message) sed 's/^/> /'
# delete leading angle bracket & space from each line (unquote a message)
sed 's/^> //'
# remove most HTML tags (accommodates multiple-line tags)
sed -e :a -e 's/<[^>]*>//g;/zipup.bat dir /b *.txt sed "s/^\(.*\)\.TXT/pkzip -mo \1 \1.TXT/">>zipup.bat

TYPICAL USE: Sed takes one or more editing commands and applies all ofthem, in sequence, to each line of input. After all the commands havebeen applied to the first input line, that line is output and a secondinput line is taken for processing, and the cycle repeats. Thepreceding examples assume that input comes from the standard inputdevice (i.e, the console, normally this will be piped input). One ormore filenames can be appended to the command line if the input doesnot come from stdin. Output is sent to stdout (the screen). Thus:
cat filename sed '10q' # uses piped input
sed '10q' filename # same effect, avoids a useless "cat"
sed '10q' filename > newfile # redirects output to disk

On line selection or deletion in which you only need to output linesfrom the first part of the file, a "quit" command (q) in the scriptwill drastically reduce processing time for large files. Thus:
sed -n '45,50p' filename # print line nos. 45-50 of a file
sed -n '51q;45,50p' filename # same, but executes much faster

CREDITS AND THANKS:http://student.northpark.edu/pemente/sed/sed1line.txt

Useful awk oneliners

FILE SPACING:
# double space a file
awk '1;{print ""}' awk 'BEGIN{ORS="\n\n"};1'

# double space a file which already has blank lines in it. Output file should contain no more than one blank line between lines of text. NOTE: On Unix systems, DOS lines which have only CRLF (\r\n) are # often treated as non-blank, and thus 'NF' alone will return TRUE.
awk 'NF{print $0 "\n"}'
# triple space a file
awk '1;{print "\n"}'
NUMBERING AND CALCULATIONS:
# precede each line by its line number FOR THAT FILE (left alignment). Using a tab (\t) instead of space will preserve margins.
awk '{print FNR "\t" $0}' files*

# precede each line by its line number FOR ALL FILES TOGETHER, with tab.
awk '{print NR "\t" $0}' files*
# number each line of a file (number on left, right-aligned) # Double the percent signs if typing from the DOS command prompt.
awk '{printf("%5d : %s\n", NR,$0)}'
# number each line of file, but only print numbers if line is not blank Remember caveats about Unix treatment of \r (mentioned above)
awk 'NF{$0=++a " :" $0};{print}' awk '{print (NF? ++a " :" :"") $0}'
# count lines (emulates "wc -l")
awk 'END{print NR}'

# print the sums of the fields of every line
awk '{s=0; for (i=1; i<=NF; i++) s=s+$i; print s}'
# add all fields in all lines and print the sum
awk '{for (i=1; i<=NF; i++) s=s+$i}; END{print s}' # print every line after replacing each field with its absolute value awk '{for (i=1; i<=NF; i++) if ($i < i =" -$i;" i="1;" i =" ($i">
# print the total number of fields ("words") in all lines
awk '{ total = total + NF }; END {print total}' file

# print the total number of lines that contain "Beth"
awk '/Beth/{n++}; END {print n+0}' file
# print the largest first field and the line that contains it Intended for finding the longest string in field 1
awk '$1 > max {max=$1; maxline=$0}; END{ print max, maxline}'

# print the number of fields in each line, followed by the line
awk '{ print NF ":" $0 } '

# print the last field of each line
awk '{ print $NF }'

# print the last field of the last line
awk '{ field = $NF }; END{ print field }'

# print every line with more than 4 fields
awk 'NF > 4'

# print every line where the value of the last field is > 4
awk '$NF > 4'

TEXT CONVERSION AND SUBSTITUTION:
# IN UNIX ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format
awk '{sub(/\r$/,"");print}' # assumes EACH line ends with Ctrl-M

# IN UNIX ENVIRONMENT: convert Unix newlines (LF) to DOS format
awk '{sub(/$/,"\r");print}

# IN DOS ENVIRONMENT: convert Unix newlines (LF) to DOS format
awk 1

# IN DOS ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format # Cannot be done with DOS versions of awk, other than gawk:
gawk -v BINMODE="w" '1' infile >outfile

# Use "tr" instead.
tr -d \r outfile # GNU tr version 1.22 or higher

# delete leading whitespace (spaces, tabs) from front of each line # aligns all text flush left
awk '{sub(/^[ \t]+/, ""); print}'

# delete trailing whitespace (spaces, tabs) from end of each line
awk '{sub(/[ \t]+$/, "");print}'

# delete BOTH leading and trailing whitespace from each line
awk '{gsub(/^[ \t]+[ \t]+$/,"");print}' awk '{$1=$1;print}' # also removes extra space

# insert 5 blank spaces at beginning of each line (make page offset)
awk '{sub(/^/, " ");print}'

# align all text flush right on a 79-column width
awk '{printf "%79s\n", $0}' file*

# center all text on a 79-character width
awk '{l=length();s=int((79-l)/2); printf "%"(s+l)"s\n",$0}' file*

# substitute (find and replace) "foo" with "bar" on each line
awk '{sub(/foo/,"bar");print}' # replaces only 1st instance
gawk '{$0=gensub(/foo/,"bar",4);print}' # replaces only 4th instance
awk '{gsub(/foo/,"bar");print}' # replaces ALL instances in a line
# substitute "foo" with "bar" ONLY for lines which contain "baz"
awk '/baz/{gsub(/foo/, "bar")};{print}'
# substitute "foo" with "bar" EXCEPT for lines which contain "baz" awk '!/baz/{gsub(/foo/, "bar")};{print}'
# change "scarlet" or "ruby" or "puce" to "red"
awk '{gsub(/scarletrubypuce/, "red"); print}'
# reverse order of lines (emulates "tac")
awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' file*
# if a line ends with a backslash, append the next line to it (fails if there are multiple lines ending with backslash...)
awk '/\\$/ {sub(/\\$/,""); getline t; print $0 t; next}; 1' file*
# print and sort the login names of all users
awk -F ":" '{ print $1 "sort" }' /etc/passwd
# print the first 2 fields, in opposite order, of every line
awk '{print $2, $1}' file
# switch the first 2 fields of every line
awk '{temp = $1; $1 = $2; $2 = temp}' file
# print every line, deleting the second field of that line
awk '{ $2 = ""; print }'
# print in reverse order the fields of every line
awk '{for (i=NF; i>0; i--) printf("%s ",i);printf ("\n")}' file
# remove duplicate, consecutive lines (emulates "uniq")
awk 'a !~ $0; {a=$0}'
# remove duplicate, nonconsecutive lines
awk '! a[$0]++' # most concise script
awk '!($0 in a) {a[$0];print}' # most efficient script
# concatenate every 5 lines of input, using a comma separator between fields
awk 'ORS=%NR%5?",":"\n"' file

SELECTIVE PRINTING OF CERTAIN LINES:
# print first 10 lines of file (emulates behavior of "head")
awk 'NR <>1{exit};1'
# print the last 2 lines of a file (emulates "tail -2")
awk '{y=x "\n" $0; x=$0};END{print y}'
# print the last line of a file (emulates "tail -1")
awk 'END{print}'
# print only lines which match regular expression (emulates "grep")
awk '/regex/'
# print only lines which do NOT match regex (emulates "grep -v")
awk '!/regex/'
# print the line immediately before a regex, but not the line # containing the regex
awk '/regex/{print x};{x=$0}' awk '/regex/{print (x=="" ? "match on line 1" : x)};{x=$0}'
# print the line immediately after a regex, but not the line containing the regex
awk '/regex/{getline;print}'
# grep for AAA and BBB and CCC (in any order)
awk '/AAA/; /BBB/; /CCC/'
# grep for AAA and BBB and CCC (in that order)
awk '/AAA.*BBB.*CCC/'
# print only lines of 65 characters or longer
awk 'length > 64'
# print only lines of less than 65 characters
awk 'length < nr="="8,NR="="12'" nr="="52'" nr="="52">
CREDITS AND THANKS:http://student.northpark.edu/pemente/awk/awk1line.txt

Taking count of all files , where file names were passed as parameters from another file

Taking a count of all files listed in a diirectory
=================================

#! /bin/ksh
cd /home/manu
for i in `cat file1.txt`
do
echo "$i" | cat $i | wc -l
done

Removing a particular prefix from all files listed by ls

Removing a particular prefix from all file names listed by ls -lrt e.g. dbo.manu.sql-->manu.sql ......
------------------------------------------------------------------------

#! /bin/ksh
for i in `ls *.sql`
do
NEW=`echo $i | sed -e 's/^dbo.//g'`
mv ${i} ${NEW}
done

use of sed inside shell script for removing a particular pattern

Use of sed in removing a particular pattern from a file name
=============================================

#! /bin/ksh
M=`echo manu_rpt | sed 's/_rpt//'`
echo "$M"
R="$M"_sync1
echo "$R"

Comparing Partial strings and use of config file inside script

This shell script compares the partial strings and also says how do you use another file as config file inside the script.
========================================================

#! /bin/ksh

. `pwd`/config.txt

export RPT_DB_NAME="$1"
string='_rpt'
echo $RPT_DB_NAME | grep $string >> /dev/null
if [ $? = 0 ]
then
if [[ -e check.txt ]]
then rm check.txt
echo "Manu"
echo "Swami"
echo "File Check.txt is removed"
fi
echo "first loop"
echo "first loop"
echo "first loop"
else
echo "No Match Found"
echo "second loop"
echo "second loop"
echo "second loop"
fi
#print manu from the file config.txt
echo "$MANUSWAMI"

Invisible Password

#!/bin/ksh

/bin/stty -echo echonl
read up?password:
/bin/stty echo

# you should get user input password in this up varialbe now
# print it out to verify
echo password is "$up"

Comparing two columns

#! /bin/ksh

########################
### Created on : 10-sep-2008 ###
### Created By : Manu Swami ###
#########################

############## How to use -- USAGE ###############

usage="$0 File1 File2"

############ Setting up the environment variables #####################

datetime=`date '+%Y%m%d'`
export FILE1="$1"
export FILE2="$2"

########## If either of the mandatory parameters is missing raise error #############

if [[ "$FILE1" = "" ]] || [[ "$FILE2" = "" ]] then
print "Invalid usage, Usage is : "
print "$usage"
exit 1
fi
############################################################

echo "\n"
echo "#### Contents of "$1" which are not present in "$2" ####"
for i in $(cat "$1")
do

cat "$2" | grep "^"$i"$">check.txt

if [ -s check.txt ]
then
cat check.txt>>result.txt
else
echo "Not found --> "$i" : in the file "$2".."
fi

done

echo "\n"
echo "#### Contents of "$2" which are not present in "$1" ####"
for j in $(cat "$2")
do

cat "$1" | grep "^"$j"$">checkf.txt

if [ -s checkf.txt ]
then
cat checkf.txt>>result.txt
else
echo "Not found --> "$j" : in the file "$1".."
fi

done

echo "\n"
echo "Cleaning up temporary files, if there is any...."
rm result.txt
rm check.txt
rm checkf.txt
echo "Temp Section cleaned."

echo "\n"

Replace semicolon with @ in a new line

#!/bin/ksh
for i in `ls *.sql`
do
cat $i | sed '/^$/d' | sed '/;/G' | sed 's/;//g'| sed 's/^$/@/g' >> manu.txt
done