[Swift-commit] r2952 - SwiftApps/SIDGrid/uchicago_tools
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Mon Jun 1 15:21:07 CDT 2009
Author: skenny
Date: 2009-06-01 15:21:07 -0500 (Mon, 01 Jun 2009)
New Revision: 2952
Added:
SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari
Log:
script for testing the tgsites used by cnari
Added: SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari
===================================================================
--- SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari (rev 0)
+++ SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari 2009-06-01 20:21:07 UTC (rev 2952)
@@ -0,0 +1,328 @@
+#!/usr/bin/python
+import os,sys
+
+class gridData(dict):
+ def add(self,table,tablename,primary="ResourceID",secondary=""):
+ # make this user opt? mercury == dtf
+ site_list = ["ranger", "viz", "dtf.ncsa.teragrid.org"]
+ for row in table:
+ for site in site_list:
+ if (str(row).find(site)>-1):
+ id=row["ResourceID"]
+ print "id is.........."+id
+ if not id in self.keys():
+ print "adding site!!!!!!!"+str(site)
+ self[id]={}
+ if secondary:
+ if not tablename in self[id].keys():
+ self[id][tablename]={}
+ id2=row[secondary]
+ self[id][tablename][id2]=row
+ else:
+ self[id][tablename]=row
+
+ def display(self):
+ ptable=[]
+ for key in self.keys():
+ row=[]
+ for key2,val in self[key].items():
+ print key,key2,val
+
+
+ def __init__(self):
+ dict.__init__(self)
+
+def ParseTable(table_file_obj):
+ table_vals=[]
+ table=table_file_obj
+ firstline=table.readline()
+ firstline.strip()
+ col_names=firstline.split()
+
+ secondline=table.readline()
+ secondline=secondline.rstrip()
+ col_lengths=[len(x) for x in secondline.split()]
+ numcols=len(col_lengths)
+ dashes=sum(col_lengths)
+
+ divsize=(len(secondline)-dashes)/(numcols-1)
+
+ line=table.readline()
+ while line:
+ i=0
+ column=0
+ tmpdict={}
+ for num in col_lengths:
+ tmpkey=col_names[column]
+ colval=line[i:i+num+divsize]
+ colval=colval.strip()
+ tmpdict[tmpkey]=colval
+ i=i+num+divsize
+ column=column+1
+ table_vals.append(tmpdict)
+ line=table.readline()
+ return table_vals
+
+def tginfo(tginfo_exe="./tginfo",optionstring=""):
+ return os.popen(" ".join([tginfo_exe,optionstring]))
+
+def getResourceInfo(tginfo_exe,resource_file="tgresources"):
+ ResourceTable=tginfo(optionstring="res")
+ res_vals=ParseTable(ResourceTable)
+
+ ResourceTable=tginfo(optionstring="gridftp")
+ gftp_vals=ParseTable(ResourceTable)
+
+ ResourceTable=tginfo(optionstring="login")
+ login_vals=ParseTable(ResourceTable)
+
+ ResourceTable=tginfo(optionstring="serv prews | grep -v fork")
+ gram_vals=ParseTable(ResourceTable)
+
+ dataTable=gridData()
+ dataTable.add(res_vals,tablename="res")
+ dataTable.add(gftp_vals,tablename="gftp",secondary="Name")
+
+ dataTable.add(login_vals,tablename="login")
+ dataTable.add(gram_vals,tablename="gram")
+
+
+ return dataTable
+ #print dataTable
+
+
+def checkProxies(username="",verbose=True):
+ import re,tempfile
+ saveout=sys.stdout
+ if not verbose:
+ handle,name=tempfile.mkstemp(dir='.',prefix='proxylog_')
+ sys.stdout=open(name,'w')
+ if not username:
+ username=os.getlogin()
+ print "Checking for a valid proxy...."
+ infocommand="grid-proxy-info -timeleft"
+ status=os.popen4(infocommand)[1].read().strip()
+
+ error=re.search('not found',status)
+ if error:
+ sys.stderr.write('you do not have a valid proxy running, please initialize one.')
+ os.system("grid-proxy-init")
+
+ else:
+ timeleftstr=status.split(' ')
+ print "time str is "+timeleftstr[4]
+ timeleft = int(timeleftstr[4])
+ if timeleft < 3:
+ print "Your proxy has less 3 hours left, please initialize a new one."
+ os.system("grid-proxy-init")
+ sys.stdout=saveout
+
+def CheckTginfo(tginfo_dir=".",tginfo_url="http://info.teragrid.org/tginfo/tginfo",verbose=True):
+ import urllib2,tempfile
+
+ def getLatest(tginfo_url,destination):
+ try:
+ tginfo_web=urllib2.urlopen(tginfo_url)
+ tginfo_content=tginfo_web.read()
+ except urllib2.HTTPError,err:
+ print "HTTP error (",err,") for the website: ",tginfo_url
+ sys.exit(int(err.code))
+ print "Wrting tginfo to ",destination
+ handle=open(destination,'w')
+ handle.write(tginfo_content)
+ handle.close()
+ os.chmod(destination,0744)
+
+ saveout=sys.stdout
+ if not verbose:
+ handle,name=tempfile.mkstemp(dir='.',prefix='tginfolog_')
+ sys.stdout=open(name,'w')
+
+ tginfo_exe=os.path.join(tginfo_dir,"tginfo")
+ if os.path.exists(tginfo_exe):
+ upgrade_string=os.popen(tginfo_exe + " upgrade ").read()
+ #print upgrade_string
+ if upgrade_string.find("http")==-1:
+ print "Your tginfo executable: ",tginfo_exe," is up to date"
+ else:
+ version_string=os.popen(tginfo_exe + " version").read()
+ version=version_string.split(',')[0]
+ version=version.split()[1]
+ new_exe_path=tginfo_exe+'_'+version
+ print "Moving ",tginfo_exe," to ",new_exe_path
+ os.rename(tginfo_exe,new_exe_path)
+ getLatest(tginfo_url,tginfo_exe)
+ else:
+ getLatest(tginfo_url,"tginfo")
+ tginfo_dir="."
+
+ sys.stdout=saveout
+ return tginfo_dir
+
+def getEnv(siteTable):
+ for key in siteTable.keys():
+ loginhost=siteTable[key]['login']["EndPoint"]
+ print "Attempting to get environment variables from",loginhost,"..."
+ result=os.popen4("gsissh -o NumberOfPasswordPrompts=0 "+loginhost+" env")[1].readlines()
+ if len(result)<=1:
+ failedSites.append( (loginhost,result) )
+ print "...connection failed (or required password)"
+ else:
+ result=[x.rstrip() for x in result]
+ home=dict([tuple(x.split('=')) for x in result if x.find("HOME")>-1])
+ scratch=dict([tuple(x.split('=')) for x in result if x.find("SCRATCH")>-1])
+ gridVars["home"][loginhost]=home
+ gridVars["scratch"][loginhost]=scratch
+
+ return gridVars,failedSites
+
+def createShortNames(siteTable):
+ resourceMapper={}
+ for key in siteTable.keys():
+ try:
+ #loginhost=siteTable[key]['login']["EndPoint"]
+ resid=siteTable[key]['res']["ResourceID"]
+ shortid=resid.split('.teragrid.org')[0]
+ mapid="".join( [x.capitalize() for x in shortid.split('.')] )
+ resourceMapper[resid]=mapid
+ except KeyError,err:
+ pass
+ #print key,"does not have a login host, ignoring..."
+ return resourceMapper
+
+def csvFormat(resourceMapper,siteTable):
+ allSites=[]
+ for resid in resourceMapper.keys():
+ globus_type=""
+ try:
+ gram_type=siteTable[resid]['gram']['Name']
+ globus_type=gram_type.split('-')[-1]
+ globus_endpoint=siteTable[resid]['gram']['EndPoint']
+ locport=globus_endpoint.split('/')[0]
+ except KeyError:
+ pass
+ comma_array=[resourceMapper[resid],locport,"compute","Teragrid"]
+ comma_entry=",".join(comma_array)
+ allSites.append(comma_entry)
+ numberedSites=[str(x)+","+y for x,y in enumerate(allSites)]
+ return "\n".join(numberedSites)
+
+def getXML(resourceMapper,siteTable,verbose=True):
+ import tempfile
+ poolList=[]
+ activeList=[]
+ gftp_default=""
+
+ saveout=sys.stdout
+ if not verbose:
+ handle,name=tempfile.mkstemp(dir='.',prefix='xmlog_')
+ sys.stdout=open(name,'a')
+
+ for resid in resourceMapper.keys():
+ globus_type=""
+ try:
+ short=resourceMapper[resid]
+ gram_type=siteTable[resid]['gram']['Name']
+ globus_type=gram_type.split('-')[-1]
+ globus_endpoint=siteTable[resid]['gram']['EndPoint']
+ locport=globus_endpoint.split('/')[0]
+ gftp_default=siteTable[resid]['gftp']['gridftp-default-server']['EndPoint']
+ except KeyError:
+ pass
+ globus_string='globus-job-run '+locport+' /usr/bin/id '
+ print "Testing",short,"... (",globus_string,")"
+ test=os.popen4(globus_string)[1].read()
+ print test
+
+ pool= """
+ <pool handle="%s" >
+ <jobmanager universe="vanilla" url="%s" major="2" />
+ <gridftp url="%s" />
+ </pool>
+ """ % (short,globus_endpoint,gftp_default)
+ poolList.append(pool)
+ if test.startswith("uid"):
+ activeList.append(pool)
+
+ sys.stdout=saveout
+ return poolList,activeList
+
+def usage():
+ usagestring="""
+ Usage: %s -[hqcxed]
+ -h,--help Print this usage description
+ -q,--quiet Quiet -- prints all messages to tempfiles in the current directory
+ -c,--csv Prints info about teragrid sites in csv format for easy parsing
+ -x,--xml Prints default sites.xml entries for teragrid sites
+ -l,--login You can specify a userid on proxyserver. default is login id on this machine
+ -e Optional -- specify a path to your own tginfo executable
+ -d Optional -- specify a directory for your own tginfo executable
+ """ % (sys.argv[0])
+ print usagestring
+
+def main():
+ import getopt
+ tginfo_dir="."
+ verbose=True
+ csvGen=False
+ xmlGen=False
+ WriteToFile=False
+ OutDict={}
+ username=""
+
+ try:
+ opts,args=getopt.getopt(sys.argv[1:],'hqcxe:d:l:',['help','quiet','csv','xml','login'])
+ except getopt.GetoptError,err:
+ print str(err)
+ usage()
+ sys.exit(22)
+ for opt,arg in opts:
+ if opt in ("-h","--help"):
+ usage()
+ sys.exit()
+ elif opt in ("-q","--quiet"):
+ verbose=False
+ elif opt in ("-e"):
+ tginfo_dir=os.path.dirname(arg)
+ elif opt in ("-d"):
+ tginfo_dir=arg
+ elif opt in ("-c","--csv"):
+ csvGen=True
+ elif opt in ("-x","--xml"):
+ xmlGen=True
+ elif opt in ("-l","--login"):
+ username=arg
+
+ tginfo_dir=CheckTginfo(tginfo_dir,verbose=verbose)
+ tginfo_exe=os.path.join(tginfo_dir,"tginfo")
+
+
+ siteTable=getResourceInfo(tginfo_exe)
+ resourceMapper=createShortNames(siteTable)
+
+ if csvGen:
+ csvout=csvFormat(resourceMapper,siteTable)
+ OutDict['csv']=csvout
+ if xmlGen:
+ WriteToFile=True
+ checkProxies()
+ xmlout,xmlactive=getXML(resourceMapper,siteTable,verbose=verbose)
+ OutDict['xml']="".join(xmlout)
+ OutDict['xml_active']="".join(xmlactive)
+
+
+ if not WriteToFile:
+ for key in OutDict:
+ print key
+ print OutDict[key]
+
+ if WriteToFile:
+ sitesfile = open("sites.xml", "w")
+ for key in OutDict:
+ sitesfile.write(OutDict[key])
+ sitesfile.close()
+
+ #gridVars,failedSites=getEnv(siteTable)
+
+if __name__=="__main__":
+ main()
Property changes on: SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari
___________________________________________________________________
Name: svn:executable
+ *
More information about the Swift-commit
mailing list