Python for Bioinformatics: Xgrid: simple Python scripts

Friday, November 6, 2009

Xgrid: simple Python scripts

I had a student in my Bioinformatics class who taught himself Java. (Silent cheer). But, of course, that meant he was hooked on classes. I'm not convinced. For standard scripting stuff, the code is so targeted to the problem, reusability is more of a dream than a goal. And complexity is not usually an issue. With that in mind, here are a few simple Python functions to explore a running Xgrid setup.

import os, sys, subprocess, time
password = 'mypw'

h = '-h localhost'
pw = '-p ' + password
s = '/usr/bin/cal'
 
# parse the dicts from submit, attributes
def parse(s,n=1):
    L = s.strip().split('\n')[n:-n]
    D = dict()
    for line in L:
        k,v = line.strip().split(' = ')
        D[k] = v.split(';')[0]
    return D

def launch(cmd):
    p = subprocess.Popen(cmd,
        shell=True,
        stdout=subprocess.PIPE)
    r,e = p.communicate()
    return r,e

# build the command we'll use
def build(kind,arg=None):
    rL = ['xgrid',h,pw,'-job',kind]
    if kind in ['run','submit']:
        bin = arg
        if bin:  rL.append(bin)
    if kind in ['results','attributes']:
        jid = arg
        if jid:
            rL.append('-id ' + jid)
    return ' '.join(rL)

def run(bin):
    cmd = build('run',bin)
    r,e = launch(cmd)
    print r.strip() + '\n'

run(s)

localhost:Desktop te$ python xgrid.py 
November 2009
Su Mo Tu We Th Fr Sa
 1  2  3  4  5  6  7
 8  9 10 11 12 13 14
15 16 17 18 19 20 21
22 23 24 25 26 27 28
29 30

def submit(bin):
    cmd = build('submit',bin)
    r,e = launch(cmd)
    D = parse(r)
    jid = D['jobIdentifier']
    print 'jid', jid

    cmd = build('attributes',jid)
    r,e = launch(cmd)
    D = parse(r,n=2)
    while not D['jobStatus'] == 'Finished':
        time.sleep(0.1)
        r,e = launch(cmd)
        D = parse(r,n=2)
        print 'percentDone',D['percentDone']
    
    cmd = build('results',jid)
    r,e = launch(cmd)
    print r.strip() + '\n'    
   
def testPython():
    cmd = build('run')
    bin = '/usr/bin/python /temp/script.py'
    inDir = '/Users/te/Desktop/temp'
    cmd += ' ' + bin + ' -in ' + inDir
    r,e = launch(cmd)
    print r.strip()
    
submit(s)
testPython()

jid 128
November 2009
Su Mo Tu We Th Fr Sa
 1  2  3  4  5  6  7
 8  9 10 11 12 13 14
15 16 17 18 19 20 21
22 23 24 25 26 27 28
29 30

Hello Python world!

Looks like it works! Notice that we are running a Python script on the Agent (as described here).