Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#!/usr/bin/env python3
# Author: Heresh Fattahi
from queue import Queue
import threading
import os
import argparse
import configparser
helpstr = '''
Parallel processing different run commands
'''
class customArgparseAction(argparse.Action):
def __call__(self, parser, args, values, option_string=None):
'''
The action to be performed.
'''
print(helpstr)
parser.exit()
def createParser():
parser = argparse.ArgumentParser( description='Preparing input rdf and processing workflow for phase unwrapping')
parser.add_argument('-i', '--input', dest='input', type=str, required=True,
help='input run file which contains multiple commands to be run. Each line is assumed to be a command that can be run independent of other commands')
parser.add_argument('-p', '--number_of_processors', dest='processors', type=int, default=8,
help='number of processors')
return parser
def cmdLineParse(iargs = None):
parser = createParser()
inputArgs = parser.parse_args(args=iargs)
return inputArgs
class ThreadRun(threading.Thread):
"""Threaded processing commands """
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
rr, opt_dict = self.queue.get()
os.system(rr)
self.queue.task_done()
def main(iargs=None):
inputArgs = cmdLineParse(iargs)
opt_dict={}
opt_dict['parallel']=inputArgs.processors
queue = Queue()
#spawn a pool of threads, and pass them queue instance
for i in range(opt_dict['parallel']):
t = ThreadRun(queue)
t.setDaemon(True)
t.start()
#populate queue with data
runs = []
for line in open(inputArgs.input):
runs.append(line.strip())
#for d in sorted(len(runs), key=operator.itemgetter('collectionName')):
for rr in runs:
queue.put([rr, opt_dict])
#wait on the queue until everything has been processed
queue.join()
if __name__ == "__main__":
# Main engine
main()