awk

awk split file

awk-split-file

awk csv file replace speical column,remove duplicate line by speicl colums(multi)

数据格式如下

$ cat FILE
1,2,3,4,5
1,2,33,44,55
1,2,333,444,555
11,22,3,4,5
11,22,33,44,55
$cat FILE | awk 'BEGIN {FS=",";OFS=","} {print $1,$2,$3,"update 4",$5}' | sort -u | awk -F "," '!seen[$1,$2]++' > output.csv

output

1,2,3,update 4,5
11,22,3,update 4,5

point:seen[$1,$2] $1,$2 is concat [,]

通过awk计算平均值

数据格式如下

$ cat FILE
Id  ht
510 69
510 67
510 65
510 62
510 59
601 29
601 26
601 21
601 20

awk

$ awk '
    NR>1{
        arr[$1]   += $2
        count[$1] += 1
    }
    END{
        for (a in arr) {
            print "id avg " a " = " arr[a] / count[a]
        }
    }
' FILE

output

id avg 601 = 24
id avg 510 = 64.4

average-rows-with-same-first-column