Wednesday, March 23, 2016

Bash in action: multithreading

hi all,

I hit a bottleneck on a for loop statement, when I have a function that can be carried out concurrently for some numbers of thread. But, I don't know how to code it in bash. I bumped into this site: http://stackoverflow.com/questions/1455695/forking-multi-threaded-processes-bash, which inspired me to code it. So, at the last I manage to code a multithreading function using bash. Here is how it looks like. I hope it helps some of the devops engineers to solve their problems.

[hiuy@nasilemak ~]$ ./test.sh
Waiting for jobs: 1199 1200 1201
Waiting for jobs: 1199 1200 1201
Waiting for jobs: 1199 1200 1201
Waiting for jobs: 1199 1200 1201
Waiting for jobs: 1199 1200 1201
Waiting for jobs: 1199 1200 1201
Job is Done -- 1199
Waiting for jobs: 1200 1201 1445
Waiting for jobs: 1200 1201 1445
Waiting for jobs: 1200 1201 1445
Waiting for jobs: 1200 1201 1445
Waiting for jobs: 1200 1201 1445
Job is Done -- 1200
Waiting for jobs: 1201 1445 2099
Waiting for jobs: 1201 1445 2099
Waiting for jobs: 1201 1445 2099
Waiting for jobs: 1201 1445 2099
Waiting for jobs: 1201 1445 2099
Job is Done -- 1201
Waiting for jobs: 1445 2099 2118
Job is Done -- 2099
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Waiting for jobs: 1445 2118 2121
Job is Done -- 1445
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Waiting for jobs: 2118 2121
Job is Done -- 2118
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Waiting for jobs: 2121
Job is Done -- 2121

Here is the code
#!/bin/bash

TOTAL_THREAD=3
declare -a pids

function wait_pid()
{
        pid=$1
        if [ ! -z $pid ] && kill -0 $pid  &>/dev/null
        then
              #this is a valid pid, then add it
              pids=(${pids[@]} $pid)


                if [ "${#pids[@]}" -lt "$TOTAL_THREAD" ]
                then
                        return 0
                elif [ "${#pids[@]}" -eq "$TOTAL_THREAD" ]
                then
                        while [ "${#pids[@]}" -eq "$TOTAL_THREAD" ]
                        do
                                if [ "${#pids[@]}" -lt $TOTAL_THREAD ]
                                then
                                        break
                                fi
                                echo "Waiting for jobs: ${pids[@]}"
                                local range=$(eval echo {0..$((${#pids[@]}-1))})
                                local i
                                for i in $range; do
                                        if ! kill -0 ${pids[$i]} 2> /dev/null; then
                                                echo "Job is Done -- ${pids[$i]}"
                                                unset pids[$i]
                                        fi
                                done
                                pids=("${pids[@]}") # Expunge nulls created by unset.
                                sleep 1
                        done
                        return 0
                fi
        fi

}

function wait_complete()
{
        while [ "${#pids[@]}" -gt "0" ]
        do
                echo "Waiting for jobs: ${pids[@]}"
                local range=$(eval echo {0..$((${#pids[@]}-1))})
                local i
                for i in $range; do
                        if ! kill -0 ${pids[$i]} 2> /dev/null; then
                                echo "Job is Done -- ${pids[$i]}"
                                unset pids[$i]
                        fi
                done
                pids=("${pids[@]}") # Expunge nulls created by unset.
                sleep 1
        done
}

for s in `echo "5 10 15 25 5 25 60"`
do
        sleep $s &
        wait_pid $!
done


wait_complete

No comments: