From noreply at svn.ci.uchicago.edu Mon Jun 1 13:49:37 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 1 Jun 2009 13:49:37 -0500 (CDT) Subject: [Swift-commit] r2951 - SwiftApps/SIDGrid/config Message-ID: <20090601184937.BAC8B9CC8F@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-01 13:49:37 -0500 (Mon, 01 Jun 2009) New Revision: 2951 Modified: SwiftApps/SIDGrid/config/tc.data Log: new path for R install on BSD cluster Modified: SwiftApps/SIDGrid/config/tc.data =================================================================== --- SwiftApps/SIDGrid/config/tc.data 2009-05-28 20:25:56 UTC (rev 2950) +++ SwiftApps/SIDGrid/config/tc.data 2009-06-01 18:49:37 UTC (rev 2951) @@ -723,7 +723,7 @@ BSD touch /bin/touch INSTALLED INTEL32::LINUX null BSD globus-url-copy /soft/globus-4.0.3-r1/bin/globus-url-copy INSTALLED INTEL64::LINUX env::LD_LIBRARY_PATH=/soft/globus-4.0.3-r1/lib BSD R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/disks/ci-gpfs/fmri/cnari/apps/R_library -BSD RInvoke /disks/ci-gpfs/fmri/cnari/scripts/RInvoke.sh INSTALLED INTEL32::LINUX env::R_LIBS=/disks/ci-gpfs/fmri/cnari/apps/R_library +BSD RInvoke /disks/ci-gpfs/fmri/cnari/scripts/RInvoke.sh INSTALLED INTEL32::LINUX env::PATHPREFIX=/disks/ci-gpfs/fmri/SIDGrid/usr32/bin BSD Rcmd /disks/ci-gpfs/fmri/cnari/scripts/Rcmd.sh INSTALLED INTEL32::LINUX env::R_LIBS=/disks/ci-gpfs/fmri/cnari/apps/R_library BSD Mediator /disks/ci-gpfs/fmri/cnari/scripts/Mediator.py INSTALLED INTEL32::LINUX null BSD env /usr/env INSTALLED INTEL32::LINUX null @@ -2155,6 +2155,14 @@ RANGER env /usr/bin/env INSTALLED INTEL64::LINUX GLOBUS::maxwalltime=10,queue=normal RANGER Mediator /scratch/projects/tg/SIDGrid/scripts/Mediator.py INSTALLED INTEL64::LINUX env::PATHPREFIX=/scratch/projects/tg/SIDGrid/usr/bin,LD_LIBRARY_PATH=/scratch/projects/tg/SIDGrid/usr/lib/mysql +#----freesurfer + +RANGER recon1 /scratch/projects/tg/SIDGrid/freesurfer/bin/recon1.sh INSTALLED INTEL32::LINUX env::FREESURFER_HOME=/scratch/projects/tg/SIDGrid/freesurfer +RANGER recon2L /scratch/projects/tg/SIDGrid/freesurfer/bin/recon2L.sh INSTALLED INTEL32::LINUX env::FREESURFER_HOME=/scratch/projects/tg/SIDGrid/freesurfer +RANGER recon2R /scratch/projects/tg/SIDGrid/freesurfer/bin/recon2R.sh INSTALLED INTEL32::LINUX env::FREESURFER_HOME=/scratch/projects/tg/SIDGrid/freesurfer +RANGER autorecon2_BOTH /scratch/projects/tg/SIDGrid/freesurfer/bin/autorecon2_BOTH.sh INSTALLED INTEL32::LINUX env::FREESURFER_HOME=/scratch/projects/tg/SIDGrid/freesurfer +RANGER reconFINAL /scratch/projects/tg/SIDGrid/freesurfer/bin/reconFINAL.sh INSTALLED INTEL32::LINUX env::FREESURFER_HOME=/scratch/projects/tg/SIDGrid/freesurfer,PATHPREFIX=/scratch/projects/tg/SIDGrid/usr/bin + ####-----------------BIGRED----------------------------- #------AFNI BIGRED AFNI_1dcat /N/u/tg-scommun/BigRed/sidgrid/usr/bin/1dcat INSTALLED INTEL64::LINUX GLOBUS::maxwalltime=100;env::PATHPREFIX=/N/u/tg-scommun/BigRed/sidgrid/usr/bin,LD_LIBRARY_PATH=/N/u/tg-scommun/BigRed/sidgrid/usr/bin @@ -2440,3 +2448,4 @@ localhost RInvoke /disks/ci-gpfs/fmri/cnari/scripts/RInvoke.sh INSTALLED INTEL64::LINUX null localhost echo /bin/echo INSTALLED INTEL64::LINUX null +localhost AFNI_3dDeconvolve /disks/ci-gpfs/fmri/tools/AFNI/linux_gcc32/3dDeconvolve INSTALLED INTEL32::LINUX null From noreply at svn.ci.uchicago.edu Mon Jun 1 15:21:07 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 1 Jun 2009 15:21:07 -0500 (CDT) Subject: [Swift-commit] r2952 - SwiftApps/SIDGrid/uchicago_tools Message-ID: <20090601202107.5110A9CC8F@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-01 15:21:07 -0500 (Mon, 01 Jun 2009) New Revision: 2952 Added: SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari Log: script for testing the tgsites used by cnari Added: SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari =================================================================== --- SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari (rev 0) +++ SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari 2009-06-01 20:21:07 UTC (rev 2952) @@ -0,0 +1,328 @@ +#!/usr/bin/python +import os,sys + +class gridData(dict): + def add(self,table,tablename,primary="ResourceID",secondary=""): + # make this user opt? mercury == dtf + site_list = ["ranger", "viz", "dtf.ncsa.teragrid.org"] + for row in table: + for site in site_list: + if (str(row).find(site)>-1): + id=row["ResourceID"] + print "id is.........."+id + if not id in self.keys(): + print "adding site!!!!!!!"+str(site) + self[id]={} + if secondary: + if not tablename in self[id].keys(): + self[id][tablename]={} + id2=row[secondary] + self[id][tablename][id2]=row + else: + self[id][tablename]=row + + def display(self): + ptable=[] + for key in self.keys(): + row=[] + for key2,val in self[key].items(): + print key,key2,val + + + def __init__(self): + dict.__init__(self) + +def ParseTable(table_file_obj): + table_vals=[] + table=table_file_obj + firstline=table.readline() + firstline.strip() + col_names=firstline.split() + + secondline=table.readline() + secondline=secondline.rstrip() + col_lengths=[len(x) for x in secondline.split()] + numcols=len(col_lengths) + dashes=sum(col_lengths) + + divsize=(len(secondline)-dashes)/(numcols-1) + + line=table.readline() + while line: + i=0 + column=0 + tmpdict={} + for num in col_lengths: + tmpkey=col_names[column] + colval=line[i:i+num+divsize] + colval=colval.strip() + tmpdict[tmpkey]=colval + i=i+num+divsize + column=column+1 + table_vals.append(tmpdict) + line=table.readline() + return table_vals + +def tginfo(tginfo_exe="./tginfo",optionstring=""): + return os.popen(" ".join([tginfo_exe,optionstring])) + +def getResourceInfo(tginfo_exe,resource_file="tgresources"): + ResourceTable=tginfo(optionstring="res") + res_vals=ParseTable(ResourceTable) + + ResourceTable=tginfo(optionstring="gridftp") + gftp_vals=ParseTable(ResourceTable) + + ResourceTable=tginfo(optionstring="login") + login_vals=ParseTable(ResourceTable) + + ResourceTable=tginfo(optionstring="serv prews | grep -v fork") + gram_vals=ParseTable(ResourceTable) + + dataTable=gridData() + dataTable.add(res_vals,tablename="res") + dataTable.add(gftp_vals,tablename="gftp",secondary="Name") + + dataTable.add(login_vals,tablename="login") + dataTable.add(gram_vals,tablename="gram") + + + return dataTable + #print dataTable + + +def checkProxies(username="",verbose=True): + import re,tempfile + saveout=sys.stdout + if not verbose: + handle,name=tempfile.mkstemp(dir='.',prefix='proxylog_') + sys.stdout=open(name,'w') + if not username: + username=os.getlogin() + print "Checking for a valid proxy...." + infocommand="grid-proxy-info -timeleft" + status=os.popen4(infocommand)[1].read().strip() + + error=re.search('not found',status) + if error: + sys.stderr.write('you do not have a valid proxy running, please initialize one.') + os.system("grid-proxy-init") + + else: + timeleftstr=status.split(' ') + print "time str is "+timeleftstr[4] + timeleft = int(timeleftstr[4]) + if timeleft < 3: + print "Your proxy has less 3 hours left, please initialize a new one." + os.system("grid-proxy-init") + sys.stdout=saveout + +def CheckTginfo(tginfo_dir=".",tginfo_url="http://info.teragrid.org/tginfo/tginfo",verbose=True): + import urllib2,tempfile + + def getLatest(tginfo_url,destination): + try: + tginfo_web=urllib2.urlopen(tginfo_url) + tginfo_content=tginfo_web.read() + except urllib2.HTTPError,err: + print "HTTP error (",err,") for the website: ",tginfo_url + sys.exit(int(err.code)) + print "Wrting tginfo to ",destination + handle=open(destination,'w') + handle.write(tginfo_content) + handle.close() + os.chmod(destination,0744) + + saveout=sys.stdout + if not verbose: + handle,name=tempfile.mkstemp(dir='.',prefix='tginfolog_') + sys.stdout=open(name,'w') + + tginfo_exe=os.path.join(tginfo_dir,"tginfo") + if os.path.exists(tginfo_exe): + upgrade_string=os.popen(tginfo_exe + " upgrade ").read() + #print upgrade_string + if upgrade_string.find("http")==-1: + print "Your tginfo executable: ",tginfo_exe," is up to date" + else: + version_string=os.popen(tginfo_exe + " version").read() + version=version_string.split(',')[0] + version=version.split()[1] + new_exe_path=tginfo_exe+'_'+version + print "Moving ",tginfo_exe," to ",new_exe_path + os.rename(tginfo_exe,new_exe_path) + getLatest(tginfo_url,tginfo_exe) + else: + getLatest(tginfo_url,"tginfo") + tginfo_dir="." + + sys.stdout=saveout + return tginfo_dir + +def getEnv(siteTable): + for key in siteTable.keys(): + loginhost=siteTable[key]['login']["EndPoint"] + print "Attempting to get environment variables from",loginhost,"..." + result=os.popen4("gsissh -o NumberOfPasswordPrompts=0 "+loginhost+" env")[1].readlines() + if len(result)<=1: + failedSites.append( (loginhost,result) ) + print "...connection failed (or required password)" + else: + result=[x.rstrip() for x in result] + home=dict([tuple(x.split('=')) for x in result if x.find("HOME")>-1]) + scratch=dict([tuple(x.split('=')) for x in result if x.find("SCRATCH")>-1]) + gridVars["home"][loginhost]=home + gridVars["scratch"][loginhost]=scratch + + return gridVars,failedSites + +def createShortNames(siteTable): + resourceMapper={} + for key in siteTable.keys(): + try: + #loginhost=siteTable[key]['login']["EndPoint"] + resid=siteTable[key]['res']["ResourceID"] + shortid=resid.split('.teragrid.org')[0] + mapid="".join( [x.capitalize() for x in shortid.split('.')] ) + resourceMapper[resid]=mapid + except KeyError,err: + pass + #print key,"does not have a login host, ignoring..." + return resourceMapper + +def csvFormat(resourceMapper,siteTable): + allSites=[] + for resid in resourceMapper.keys(): + globus_type="" + try: + gram_type=siteTable[resid]['gram']['Name'] + globus_type=gram_type.split('-')[-1] + globus_endpoint=siteTable[resid]['gram']['EndPoint'] + locport=globus_endpoint.split('/')[0] + except KeyError: + pass + comma_array=[resourceMapper[resid],locport,"compute","Teragrid"] + comma_entry=",".join(comma_array) + allSites.append(comma_entry) + numberedSites=[str(x)+","+y for x,y in enumerate(allSites)] + return "\n".join(numberedSites) + +def getXML(resourceMapper,siteTable,verbose=True): + import tempfile + poolList=[] + activeList=[] + gftp_default="" + + saveout=sys.stdout + if not verbose: + handle,name=tempfile.mkstemp(dir='.',prefix='xmlog_') + sys.stdout=open(name,'a') + + for resid in resourceMapper.keys(): + globus_type="" + try: + short=resourceMapper[resid] + gram_type=siteTable[resid]['gram']['Name'] + globus_type=gram_type.split('-')[-1] + globus_endpoint=siteTable[resid]['gram']['EndPoint'] + locport=globus_endpoint.split('/')[0] + gftp_default=siteTable[resid]['gftp']['gridftp-default-server']['EndPoint'] + except KeyError: + pass + globus_string='globus-job-run '+locport+' /usr/bin/id ' + print "Testing",short,"... (",globus_string,")" + test=os.popen4(globus_string)[1].read() + print test + + pool= """ + + + + + """ % (short,globus_endpoint,gftp_default) + poolList.append(pool) + if test.startswith("uid"): + activeList.append(pool) + + sys.stdout=saveout + return poolList,activeList + +def usage(): + usagestring=""" + Usage: %s -[hqcxed] + -h,--help Print this usage description + -q,--quiet Quiet -- prints all messages to tempfiles in the current directory + -c,--csv Prints info about teragrid sites in csv format for easy parsing + -x,--xml Prints default sites.xml entries for teragrid sites + -l,--login You can specify a userid on proxyserver. default is login id on this machine + -e Optional -- specify a path to your own tginfo executable + -d Optional -- specify a directory for your own tginfo executable + """ % (sys.argv[0]) + print usagestring + +def main(): + import getopt + tginfo_dir="." + verbose=True + csvGen=False + xmlGen=False + WriteToFile=False + OutDict={} + username="" + + try: + opts,args=getopt.getopt(sys.argv[1:],'hqcxe:d:l:',['help','quiet','csv','xml','login']) + except getopt.GetoptError,err: + print str(err) + usage() + sys.exit(22) + for opt,arg in opts: + if opt in ("-h","--help"): + usage() + sys.exit() + elif opt in ("-q","--quiet"): + verbose=False + elif opt in ("-e"): + tginfo_dir=os.path.dirname(arg) + elif opt in ("-d"): + tginfo_dir=arg + elif opt in ("-c","--csv"): + csvGen=True + elif opt in ("-x","--xml"): + xmlGen=True + elif opt in ("-l","--login"): + username=arg + + tginfo_dir=CheckTginfo(tginfo_dir,verbose=verbose) + tginfo_exe=os.path.join(tginfo_dir,"tginfo") + + + siteTable=getResourceInfo(tginfo_exe) + resourceMapper=createShortNames(siteTable) + + if csvGen: + csvout=csvFormat(resourceMapper,siteTable) + OutDict['csv']=csvout + if xmlGen: + WriteToFile=True + checkProxies() + xmlout,xmlactive=getXML(resourceMapper,siteTable,verbose=verbose) + OutDict['xml']="".join(xmlout) + OutDict['xml_active']="".join(xmlactive) + + + if not WriteToFile: + for key in OutDict: + print key + print OutDict[key] + + if WriteToFile: + sitesfile = open("sites.xml", "w") + for key in OutDict: + sitesfile.write(OutDict[key]) + sitesfile.close() + + #gridVars,failedSites=getEnv(siteTable) + +if __name__=="__main__": + main() Property changes on: SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Wed Jun 3 14:30:07 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 3 Jun 2009 14:30:07 -0500 (CDT) Subject: [Swift-commit] r2953 - SwiftApps/SIDGrid/uchicago_tools Message-ID: <20090603193007.6EADD9CC94@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-03 14:30:07 -0500 (Wed, 03 Jun 2009) New Revision: 2953 Modified: SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari Log: checks for proxy and probes cnari tg sites Modified: SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari =================================================================== --- SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari 2009-06-01 20:21:07 UTC (rev 2952) +++ SwiftApps/SIDGrid/uchicago_tools/tgsites_cnari 2009-06-03 19:30:07 UTC (rev 2953) @@ -1,6 +1,10 @@ #!/usr/bin/python import os,sys +#------------------------------global variables------------ + +SWIFT_HOME = "/disks/ci-gpfs/fmri/cnari/swift/swift/bin/" + class gridData(dict): def add(self,table,tablename,primary="ResourceID",secondary=""): # make this user opt? mercury == dtf @@ -9,9 +13,7 @@ for site in site_list: if (str(row).find(site)>-1): id=row["ResourceID"] - print "id is.........."+id if not id in self.keys(): - print "adding site!!!!!!!"+str(site) self[id]={} if secondary: if not tablename in self[id].keys(): @@ -100,18 +102,19 @@ if not username: username=os.getlogin() print "Checking for a valid proxy...." - infocommand="grid-proxy-info -timeleft" + infocommand=SWIFT_HOME+"grid-proxy-info -timeleft" status=os.popen4(infocommand)[1].read().strip() - error=re.search('not found',status) - if error: + error=re.search('not found',status) + error2=re.search('ERROR',status) + if error or error2: sys.stderr.write('you do not have a valid proxy running, please initialize one.') os.system("grid-proxy-init") else: timeleftstr=status.split(' ') - print "time str is "+timeleftstr[4] timeleft = int(timeleftstr[4]) + #timeleft = int(status) if timeleft < 3: print "Your proxy has less 3 hours left, please initialize a new one." os.system("grid-proxy-init") @@ -261,11 +264,13 @@ print usagestring def main(): + import getopt tginfo_dir="." verbose=True csvGen=False - xmlGen=False + # defaulting to true...may remove cmdline opt + xmlGen=True WriteToFile=False OutDict={} username="" @@ -308,7 +313,7 @@ checkProxies() xmlout,xmlactive=getXML(resourceMapper,siteTable,verbose=verbose) OutDict['xml']="".join(xmlout) - OutDict['xml_active']="".join(xmlactive) + #OutDict['xml_active']="".join(xmlactive) if not WriteToFile: From noreply at svn.ci.uchicago.edu Sat Jun 13 18:02:16 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 13 Jun 2009 18:02:16 -0500 (CDT) Subject: [Swift-commit] r2954 - SwiftApps/SIDGrid/config Message-ID: <20090613230216.256429CCAF@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-13 18:02:15 -0500 (Sat, 13 Jun 2009) New Revision: 2954 Added: SwiftApps/SIDGrid/config/sites_ranger_short.xml Log: puts ranger jobs in default dev queue Added: SwiftApps/SIDGrid/config/sites_ranger_short.xml =================================================================== --- SwiftApps/SIDGrid/config/sites_ranger_short.xml (rev 0) +++ SwiftApps/SIDGrid/config/sites_ranger_short.xml 2009-06-13 23:02:15 UTC (rev 2954) @@ -0,0 +1,18 @@ + + + + + + + + + + 50 + 10 + TG-DBS080004N + + 16 + + /work/00926/tg459516/sidgrid_out/{username} + + From noreply at svn.ci.uchicago.edu Sat Jun 13 18:19:42 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 13 Jun 2009 18:19:42 -0500 (CDT) Subject: [Swift-commit] r2955 - SwiftApps/SIDGrid/config Message-ID: <20090613231942.B73399CCAF@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-13 18:19:42 -0500 (Sat, 13 Jun 2009) New Revision: 2955 Modified: SwiftApps/SIDGrid/config/tc.data Log: paths for running freesurfer on localhost added Modified: SwiftApps/SIDGrid/config/tc.data =================================================================== --- SwiftApps/SIDGrid/config/tc.data 2009-06-13 23:02:15 UTC (rev 2954) +++ SwiftApps/SIDGrid/config/tc.data 2009-06-13 23:19:42 UTC (rev 2955) @@ -725,7 +725,7 @@ BSD R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/disks/ci-gpfs/fmri/cnari/apps/R_library BSD RInvoke /disks/ci-gpfs/fmri/cnari/scripts/RInvoke.sh INSTALLED INTEL32::LINUX env::PATHPREFIX=/disks/ci-gpfs/fmri/SIDGrid/usr32/bin BSD Rcmd /disks/ci-gpfs/fmri/cnari/scripts/Rcmd.sh INSTALLED INTEL32::LINUX env::R_LIBS=/disks/ci-gpfs/fmri/cnari/apps/R_library -BSD Mediator /disks/ci-gpfs/fmri/cnari/scripts/Mediator.py INSTALLED INTEL32::LINUX null +BSD Mediator /disks/ci-gpfs/fmri/cnari/scripts/Mediator.py INSTALLED INTEL32::LINUX env::PATHPREFIX=/disks/ci-gpfs/fmri/SIDGrid/usr32/bin BSD env /usr/env INSTALLED INTEL32::LINUX null BSD recon1 /disks/ci-gpfs/fmri/cnari/apps/freesurfer/bin/recon1.sh INSTALLED INTEL32::LINUX env::FREESURFER_HOME=/disks/ci-gpfs/fmri/cnari/apps/freesurfer BSD recon2L /disks/ci-gpfs/fmri/cnari/apps/freesurfer/bin/recon2L.sh INSTALLED INTEL32::LINUX env::FREESURFER_HOME=/disks/ci-gpfs/fmri/cnari/apps/freesurfer @@ -2444,8 +2444,13 @@ BIGRED echo /bin/echo INSTALLED INTEL64::LINUX null BIGRED Mediator /N/u/tg-scommun/BigRed/sidgrid/scripts/Mediator.py INSTALLED INTEL64::LINUX GLOBUS::maxwalltime=120;env::PATHPREFIX=/N/u/tg-scommun/BigRed/sidgrid/usr/bin -#----------------------------localhost---------------------- +#----------------------------localhost (will work on all bsd machines)---------------------- localhost RInvoke /disks/ci-gpfs/fmri/cnari/scripts/RInvoke.sh INSTALLED INTEL64::LINUX null localhost echo /bin/echo INSTALLED INTEL64::LINUX null -localhost AFNI_3dDeconvolve /disks/ci-gpfs/fmri/tools/AFNI/linux_gcc32/3dDeconvolve INSTALLED INTEL32::LINUX null +localhost AFNI_3dDeconvolve /disks/ci-gpfs/fmri/tools/AFNI/linux_gcc32/3dDeconvolve INSTALLED INTEL64::LINUX null +localhost recon1 /disks/ci-gpfs/fmri/cnari/apps/freesurfer/bin/recon1.sh INSTALLED INTEL64::LINUX env::FREESURFER_HOME=/disks/ci-gpfs/fmri/cnari/apps/freesurfer +localhost recon2L /disks/ci-gpfs/fmri/cnari/apps/freesurfer/bin/recon2L.sh INSTALLED INTEL64::LINUX env::FREESURFER_HOME=/disks/ci-gpfs/fmri/cnari/apps/freesurfer +localhost recon2R /disks/ci-gpfs/fmri/cnari/apps/freesurfer/bin/recon2R.sh INSTALLED INTEL64::LINUX env::FREESURFER_HOME=/disks/ci-gpfs/fmri/cnari/apps/freesurfer +localhost autorecon2_BOTH /disks/ci-gpfs/fmri/cnari/apps/freesurfer/bin/autorecon2_BOTH.sh INSTALLED INTEL64::LINUX env::FREESURFER_HOME=/disks/ci-gpfs/fmri/cnari/apps/freesurfer +localhost reconFINAL /disks/ci-gpfs/fmri/cnari/apps/freesurfer/bin/reconFINAL.sh INSTALLED INTEL64::LINUX env::FREESURFER_HOME=/disks/ci-gpfs/fmri/cnari/apps/freesurfer From noreply at svn.ci.uchicago.edu Mon Jun 15 09:42:12 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 15 Jun 2009 09:42:12 -0500 (CDT) Subject: [Swift-commit] r2956 - SwiftApps/SIDGrid/swift/swift_scripts/freesurfer Message-ID: <20090615144212.A33369CD19@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-15 09:42:12 -0500 (Mon, 15 Jun 2009) New Revision: 2956 Removed: SwiftApps/SIDGrid/swift/swift_scripts/freesurfer/autorecon1.swift Log: no longer valid Deleted: SwiftApps/SIDGrid/swift/swift_scripts/freesurfer/autorecon1.swift =================================================================== --- SwiftApps/SIDGrid/swift/swift_scripts/freesurfer/autorecon1.swift 2009-06-13 23:19:42 UTC (rev 2955) +++ SwiftApps/SIDGrid/swift/swift_scripts/freesurfer/autorecon1.swift 2009-06-15 14:42:12 UTC (rev 2956) @@ -1,51 +0,0 @@ -type file {} -type mgzfile; - -(file job_00) new_subject (string subj, mgzfile inimage) { - app { - create_subject @arg("user") subj @inimage stdout=@filename(job_00); - } -} - -(file job_01) autorecon1_4 (string subj, file job_00) { - app { - reconall @arg("user") @strcat("-s ", subj, " -nuintensitycor") stdout=@filename(job_01); - } -} - -(file job_02) autorecon1_5 (string subj, file job_01) { - app { - reconall @arg("user") @strcat("-s ", subj, " -talairach") stdout=@filename(job_02); - } -} - -(file job_03) autorecon1_6 (string subj, file job_02) { - app { - reconall @arg("user") @strcat("-s ", subj, " -normalization") stdout=@filename(job_03); - } -} - -(file job_04) autorecon1_7 (string subj, file job_03) { - app { - reconall @arg("user") @strcat("-s ", subj, " -skullstrip") stdout=@filename(job_04); - } -} - -### tars the subject dir on remote site for tranfer back to host - -(file subject_data) get_subject (string subj, file job_04) { - app { - get_subject @arg("subject_path") subj @subject_data; - } -} - -mgzfile inimage ; -string subj = @arg("subj"); - -file sigstring00 = new_subject(subj, inimage); -file sigstring01 = autorecon1_4(subj, sigstring00); -file sigstring02 = autorecon1_5(subj, sigstring01); -file sigstring03 = autorecon1_6(subj, sigstring02); -file sigstring04 = autorecon1_7(subj, sigstring03); -file subject_data ; -subject_data = get_subject(subj, sigstring04); From noreply at svn.ci.uchicago.edu Mon Jun 15 10:06:54 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 15 Jun 2009 10:06:54 -0500 (CDT) Subject: [Swift-commit] r2957 - SwiftApps/SIDGrid/config Message-ID: <20090615150654.DC7109CD19@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-15 10:06:54 -0500 (Mon, 15 Jun 2009) New Revision: 2957 Modified: SwiftApps/SIDGrid/config/sites_ranger.xml Log: goes to long queue Modified: SwiftApps/SIDGrid/config/sites_ranger.xml =================================================================== --- SwiftApps/SIDGrid/config/sites_ranger.xml 2009-06-15 14:42:12 UTC (rev 2956) +++ SwiftApps/SIDGrid/config/sites_ranger.xml 2009-06-15 15:06:54 UTC (rev 2957) @@ -10,7 +10,7 @@ 1 8 TG-DBS080004N - normal + long 16 From noreply at svn.ci.uchicago.edu Tue Jun 16 10:19:51 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 16 Jun 2009 10:19:51 -0500 (CDT) Subject: [Swift-commit] r2958 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20090616151951.B4DE09CCB0@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-16 10:19:51 -0500 (Tue, 16 Jun 2009) New Revision: 2958 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java Log: Make @filenames return a DSHandle representing an array of strings, rather than a (Java level) array containing DSHandles representing strings. Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2009-06-15 15:06:54 UTC (rev 2957) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2009-06-16 15:19:51 UTC (rev 2958) @@ -8,7 +8,10 @@ import org.globus.cog.karajan.workflow.ExecutionException; import org.griphyn.vdl.karajan.lib.VDLFunction; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.mapping.RootArrayDataNode; import org.griphyn.vdl.type.Types; public class FileNames extends VDLFunction { @@ -18,10 +21,16 @@ public Object function(VariableStack stack) throws ExecutionException { String[] f = filename(stack); - DSHandle[] h = new DSHandle[f.length]; - for (int i = 0; i < f.length; i++) { - h[i] = RootDataNode.newNode(Types.STRING, relativize(f[i])); + DSHandle returnArray = new RootArrayDataNode(Types.STRING.arrayType()); + try { + for (int i = 0; i < f.length; i++) { + Path p = parsePath("["+i+"]", stack); + DSHandle h = returnArray.getField(p); + h.setValue(relativize(f[i])); + } + } catch (InvalidPathException e) { + throw new ExecutionException("Unexpected invalid path exception",e); } - return h; + return returnArray; } } From noreply at svn.ci.uchicago.edu Tue Jun 16 10:20:21 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 16 Jun 2009 10:20:21 -0500 (CDT) Subject: [Swift-commit] r2959 - in trunk/src/org/griphyn/vdl/karajan/lib: . swiftscript Message-ID: <20090616152021.EE5209CCB0@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-16 10:20:21 -0500 (Tue, 16 Jun 2009) New Revision: 2959 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: Add logging of provenance information for @functions. Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2009-06-16 15:19:51 UTC (rev 2958) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2009-06-16 15:20:21 UTC (rev 2959) @@ -572,4 +572,19 @@ return tc; } } + + private static int provenanceIDCount = 451000; + + public static synchronized int nextProvenanceID() { + return provenanceIDCount++; + } + + public static void logProvenanceResult(int id, DSHandle result, String name) throws ExecutionException { + logger.info("FUNCTION id="+id+" name="+name+" result="+result.getIdentifier()); + } + + public static void logProvenanceParameter(int id, DSHandle parameter, String paramName) throws ExecutionException { + logger.info("FUNCTIONPARAMETER id="+id+" input="+parameter.getIdentifier()+" name="+paramName); + + } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2009-06-16 15:19:51 UTC (rev 2958) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2009-06-16 15:20:21 UTC (rev 2959) @@ -35,7 +35,11 @@ String str = breader.readLine(); freader.close(); Double i = new Double(str); - return RootDataNode.newNode(Types.FLOAT, i); + DSHandle result = RootDataNode.newNode(Types.FLOAT, i); + int provid = VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, result, "extractint"); + VDLFunction.logProvenanceParameter(provid, handle, "filename"); + return result; } } catch (IOException ioe) { Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2009-06-16 15:19:51 UTC (rev 2958) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2009-06-16 15:20:21 UTC (rev 2959) @@ -6,6 +6,7 @@ import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; import org.griphyn.vdl.karajan.VDL2FutureException; import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Types; @@ -17,7 +18,11 @@ public Object function(VariableStack stack) throws ExecutionException { try { String s = argList(filename(stack), true); - return RootDataNode.newNode(Types.STRING, s); + DSHandle result = RootDataNode.newNode(Types.STRING, s); + int provid = VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceParameter(provid, (DSHandle) PA_VAR.getValue(stack), "input"); + VDLFunction.logProvenanceResult(provid, result, "filename"); + return result; } catch(VDL2FutureException ve) { synchronized(ve.getHandle().getRoot()) { throw new FutureNotYetAvailable(addFutureListener(stack, ve.getHandle())); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2009-06-16 15:19:51 UTC (rev 2958) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2009-06-16 15:20:21 UTC (rev 2959) @@ -31,6 +31,10 @@ } catch (InvalidPathException e) { throw new ExecutionException("Unexpected invalid path exception",e); } + int provid = VDLFunction.nextProvenanceID(); + logProvenanceParameter(provid, (DSHandle) PA_VAR.getValue(stack), "input"); + logProvenanceResult(provid, returnArray, "filenames"); + return returnArray; } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2009-06-16 15:19:51 UTC (rev 2958) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2009-06-16 15:20:21 UTC (rev 2959) @@ -15,9 +15,12 @@ import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.functions.AbstractFunction; import org.griphyn.vdl.karajan.lib.SwiftArg; +import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Types; + public class FnArg extends AbstractFunction { public static final String PARSED_ARGS = "cmdline:named"; @@ -65,7 +68,14 @@ throw new ExecutionException("Missing command line argument: " + name); } else { - return RootDataNode.newNode(Types.STRING, value); + DSHandle result = RootDataNode.newNode(Types.STRING, value); + int provid=VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, result, "arg"); + VDLFunction.logProvenanceParameter(provid, P_NAME.getRawValue(stack), "name"); + if(P_VALUE.getRawValue(stack) != null) { + VDLFunction.logProvenanceParameter(provid, P_VALUE.getRawValue(stack), "value"); + } + return result; } } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2009-06-16 15:19:51 UTC (rev 2958) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2009-06-16 15:20:21 UTC (rev 2959) @@ -1,5 +1,7 @@ package org.griphyn.vdl.karajan.lib.swiftscript; +import java.io.IOException; + import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -18,11 +20,11 @@ import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.NoSuchTypeException; import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.util.VDL2Config; - public class Misc extends FunctionsCollection { - private static final Logger logger = Logger.getLogger(FunctionsCollection.class); + private static final Logger logger = Logger.getLogger(Misc.class); public static final SwiftArg PA_INPUT = new SwiftArg.Positional("input"); public static final SwiftArg PA_PATTERN = new SwiftArg.Positional("regexp"); @@ -58,6 +60,7 @@ public DSHandle swiftscript_strcat(VariableStack stack) throws ExecutionException, NoSuchTypeException, InvalidPathException { Object[] args = SwiftArg.VARGS.asArray(stack); + int provid = VDLFunction.nextProvenanceID(); StringBuffer buf = new StringBuffer(); for (int i = 0; i < args.length; i++) { buf.append(TypeUtil.toString(args[i])); @@ -65,11 +68,23 @@ DSHandle handle = new RootDataNode(Types.STRING); handle.setValue(buf.toString()); handle.closeShallow(); + try { + if(VDL2Config.getConfig().getProvenanceLog()) { + DSHandle[] provArgs = SwiftArg.VARGS.asDSHandleArray(stack); + for (int i = 0; i < provArgs.length; i++) { + VDLFunction.logProvenanceParameter(provid, (DSHandle)provArgs[i], ""+i); + } + VDLFunction.logProvenanceResult(provid, handle, "strcat"); + } + } catch(IOException ioe) { + throw new ExecutionException("When logging provenance for strcat", ioe); + } return handle; } public DSHandle swiftscript_strcut(VariableStack stack) throws ExecutionException, NoSuchTypeException, InvalidPathException { + int provid = VDLFunction.nextProvenanceID(); String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); if (logger.isDebugEnabled()) { @@ -95,6 +110,9 @@ DSHandle handle = new RootDataNode(Types.STRING); handle.setValue(group); handle.closeShallow(); + VDLFunction.logProvenanceResult(provid, handle, "strcut"); + VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "input"); + VDLFunction.logProvenanceParameter(provid, PA_PATTERN.getRawValue(stack), "pattern"); return handle; } @@ -111,6 +129,10 @@ el.setValue(split[i]); } handle.closeDeep(); + int provid=VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, handle, "strsplit"); + VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "input"); + VDLFunction.logProvenanceParameter(provid, PA_PATTERN.getRawValue(stack), "pattern"); return handle; } @@ -142,6 +164,12 @@ DSHandle handle = new RootDataNode(Types.STRING); handle.setValue(group); handle.closeShallow(); + + int provid=VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, handle, "regexp"); + VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "input"); + VDLFunction.logProvenanceParameter(provid, PA_PATTERN.getRawValue(stack), "pattern"); + VDLFunction.logProvenanceParameter(provid, PA_TRANSFORM.getRawValue(stack), "transform"); return handle; } @@ -151,7 +179,9 @@ DSHandle handle = new RootDataNode(Types.INT); handle.setValue(new Double(Integer.parseInt(inputString))); handle.closeShallow(); + int provid=VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, handle, "toint"); + VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "string"); return handle; } } - From noreply at svn.ci.uchicago.edu Tue Jun 16 11:06:23 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 16 Jun 2009 11:06:23 -0500 (CDT) Subject: [Swift-commit] r2960 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20090616160623.8D0009CCC6@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-16 11:06:23 -0500 (Tue, 16 Jun 2009) New Revision: 2960 Modified: trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java Log: provenance information for explicitly constructed arrays made with vdl:createarray Modified: trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java 2009-06-16 15:20:21 UTC (rev 2959) +++ trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java 2009-06-16 16:06:23 UTC (rev 2960) @@ -3,6 +3,8 @@ import java.util.Iterator; import java.util.List; +import org.apache.log4j.Logger; + import org.globus.cog.karajan.arguments.Arg; import org.globus.cog.karajan.stack.VariableStack; import org.globus.cog.karajan.workflow.ExecutionException; @@ -12,6 +14,9 @@ import org.griphyn.vdl.type.Type; public class CreateArray extends VDLFunction { + + public static final Logger logger = Logger.getLogger(CreateArray.class); + public static final Arg PA_VALUE = new Arg.Positional("value"); static { @@ -54,20 +59,24 @@ DSHandle handle = new RootArrayDataNode(type.arrayType()); + logger.info("CREATEARRAY START array="+handle.getIdentifier()); + int index = 0; i = ((List) value).iterator(); while (i.hasNext()) { // TODO check type consistency of elements with // the type of the array - Object n = i.next(); + DSHandle n = (DSHandle) i.next(); + // we know this DSHandle cast will work because we checked + // it in the previous scan of the array contents Path p = Path.EMPTY_PATH.addLast(String.valueOf(index), true); - // we know this DSHandle cast will work because we checked - // it in the previous scan of the array contents - handle.getField(p).set((DSHandle) n); + handle.getField(p).set(n); + logger.info("CREATEARRAY MEMBER array="+handle.getIdentifier()+" index="+index+" member="+n.getIdentifier()); index++; } closeShallow(stack, handle); + logger.info("CREATEARRAY COMPLETED array="+handle.getIdentifier()); return handle; } From noreply at svn.ci.uchicago.edu Tue Jun 16 11:41:54 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 16 Jun 2009 11:41:54 -0500 (CDT) Subject: [Swift-commit] r2961 - trunk/resources Message-ID: <20090616164154.2332E9CCC6@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-16 11:41:53 -0500 (Tue, 16 Jun 2009) New Revision: 2961 Modified: trunk/resources/Karajan.stg Log: provenance for array slices Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-06-16 16:06:23 UTC (rev 2960) +++ trunk/resources/Karajan.stg 2009-06-16 16:41:53 UTC (rev 2961) @@ -459,11 +459,25 @@ >> slicearray(parent, memberchild, datatype) ::= << - - $parent$ - $memberchild$ - $datatype$ - + + $parent$ + + + swift#array + $memberchild$ + $datatype$ + + + + + SLICEARRAY thread={#thread} slice= + + member=$memberchild$ array= + + + + swift#slice + >> From noreply at svn.ci.uchicago.edu Tue Jun 16 16:58:07 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 16 Jun 2009 16:58:07 -0500 (CDT) Subject: [Swift-commit] r2962 - trunk/docs Message-ID: <20090616215807.313DB9CCC6@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-16 16:58:06 -0500 (Tue, 16 Jun 2009) New Revision: 2962 Modified: trunk/docs/tutorial.xml trunk/docs/userguide.xml Log: remove vestigial use of print in documentation Modified: trunk/docs/tutorial.xml =================================================================== --- trunk/docs/tutorial.xml 2009-06-16 16:41:53 UTC (rev 2961) +++ trunk/docs/tutorial.xml 2009-06-16 21:58:06 UTC (rev 2962) @@ -654,7 +654,7 @@ iterate v { a[v+1] = countstep(a[v]); - print("extract int value ", at extractint(a[v+1])); + trace("extract int value ", at extractint(a[v+1])); } until (@extractint(a[v+1]) <= 1); Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-06-16 16:41:53 UTC (rev 2961) +++ trunk/docs/userguide.xml 2009-06-16 21:58:06 UTC (rev 2962) @@ -2046,7 +2046,7 @@ string t = "my name is John and i like puppies."; string name = @strcut(t, "my name is ([^ ]*) "); string out = @strcat("Your name is ",name); -print(out); +trace(out); @@ -2067,7 +2067,7 @@ string t = "my name is John and i like puppies."; string words[] = @strsplit(t, "\\s"); foreach word in words { - print(word); + trace(word); } From noreply at svn.ci.uchicago.edu Wed Jun 17 10:10:41 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Jun 2009 10:10:41 -0500 (CDT) Subject: [Swift-commit] r2963 - SwiftApps/SIDGrid/config Message-ID: <20090617151041.69C079CCB0@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-06-17 10:10:41 -0500 (Wed, 17 Jun 2009) New Revision: 2963 Modified: SwiftApps/SIDGrid/config/sites_ranger.xml Log: spec walltime Modified: SwiftApps/SIDGrid/config/sites_ranger.xml =================================================================== --- SwiftApps/SIDGrid/config/sites_ranger.xml 2009-06-16 21:58:06 UTC (rev 2962) +++ SwiftApps/SIDGrid/config/sites_ranger.xml 2009-06-17 15:10:41 UTC (rev 2963) @@ -11,6 +11,7 @@ 8 TG-DBS080004N long + 1000 16 From noreply at svn.ci.uchicago.edu Fri Jun 19 09:04:44 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Jun 2009 09:04:44 -0500 (CDT) Subject: [Swift-commit] r2964 - trunk/docs Message-ID: <20090619140444.DDE5F9CCA7@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-19 09:04:44 -0500 (Fri, 19 Jun 2009) New Revision: 2964 Modified: trunk/docs/tutorial.xml Log: remove outdated references in tutorial that referred to examples/vdsk rather than examples/swift Modified: trunk/docs/tutorial.xml =================================================================== --- trunk/docs/tutorial.xml 2009-06-17 15:10:41 UTC (rev 2963) +++ trunk/docs/tutorial.xml 2009-06-19 14:04:44 UTC (rev 2964) @@ -44,7 +44,7 @@
Hello World The first example program (found in the file -examples/vdsk/first.swift) +examples/swift/first.swift) outputs a hello world message into a file called hello.txt. @@ -66,7 +66,7 @@ We can run this program as follows: -$ cd examples/vdsk/ +$ cd examples/swift/ $ swift first.swift Swift v0.2 From noreply at svn.ci.uchicago.edu Mon Jun 22 03:08:26 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 03:08:26 -0500 (CDT) Subject: [Swift-commit] r2965 - trunk/resources Message-ID: <20090622080826.C43359CC7E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 03:08:25 -0500 (Mon, 22 Jun 2009) New Revision: 2965 Modified: trunk/resources/Karajan.stg Log: provenance information for range-described arrays Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-06-19 14:04:44 UTC (rev 2964) +++ trunk/resources/Karajan.stg 2009-06-22 08:08:25 UTC (rev 2965) @@ -320,19 +320,38 @@ >> range(from,to,step,datatype) ::= << - - - $from$ - - - $to$ - - $if(step)$ - - $step$ - - $endif$ - + + $from$ + $to$ + $if(step)$ + $step$ + $endif$ + + + swift#from + swift#to + $if(step)$ + swift#step + $endif$ + + + provenance.logtrue + + + ARRAYRANGE thread={#thread} array= + + from= + to= + $if(step)$ + step= + $else$ + step=none + $endif$ + + + + swift#rangeout + >> if(condition,vthen,velse) ::= << From noreply at svn.ci.uchicago.edu Mon Jun 22 04:02:34 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 04:02:34 -0500 (CDT) Subject: [Swift-commit] r2966 - in trunk: libexec resources src/org/griphyn/vdl/engine Message-ID: <20090622090234.1D1309CC8C@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 04:02:33 -0500 (Mon, 22 Jun 2009) New Revision: 2966 Modified: trunk/libexec/vdl.k trunk/resources/Karajan.stg trunk/src/org/griphyn/vdl/engine/Karajan.java trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java Log: Log parameters and return values from inbuild procedures for provenance purposes. This patch maybe also gives the base support removing @function/procedure distinction in SwiftScript whilst allowing the underlying implementation to have multiple forms of calling semantics, by introducing a calling convention field for all procedure definitions. Modified: trunk/libexec/vdl.k =================================================================== --- trunk/libexec/vdl.k 2009-06-22 08:08:25 UTC (rev 2965) +++ trunk/libexec/vdl.k 2009-06-22 09:02:33 UTC (rev 2966) @@ -23,11 +23,11 @@ export( - element(parameterlog, [direction, variable, id], + element(parameterlog, [direction, variable, id, thread], if( vdl:configProperty("provenance.log") == "true" - log("info","PARAM thread={#thread} direction={direction} variable={variable} provenanceid={id}") + log("info","PARAM thread={thread} direction={direction} variable={variable} provenanceid={id}") ) ) Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-06-22 08:08:25 UTC (rev 2965) +++ trunk/resources/Karajan.stg 2009-06-22 09:02:33 UTC (rev 2966) @@ -117,6 +117,7 @@ input $it.name$ +{#thread} >> @@ -125,6 +126,7 @@ output $it.name$ +{#thread} >> @@ -220,9 +222,53 @@ >> -call(func, outputs, inputs) ::= << +// need to log inputs and outputs at the calling stage here because +// they are not +// $outputs:vdl_log_output();separator="\n"$ + +callInternal(func, outputs, inputs) ::= << + + +#thread <$func$> + $outputs:callInternal_log_output();separator="\n"$ + $inputs:callInternal_log_input();separator="\n"$ + + + + +>> + +callInternal_log_input() ::= << + + $it$ + + input + TODO_name_or_pos + + {swift#cs} + + swift#callInternalValue + +>> + +callInternal_log_output() ::= << + + $it$ + + output + TODO_name_or_pos + + {swift#cs} + + swift#callInternalValue + +>> + +callUserDefined(func, outputs, inputs) ::= << +<$func$> + $outputs;separator="\n"$ $inputs;separator="\n"$ Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-06-22 08:08:25 UTC (rev 2965) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-06-22 09:02:33 UTC (rev 2966) @@ -406,8 +406,6 @@ public StringTemplate call(Call call, VariableScope scope, boolean inhibitOutput) throws CompilationException { try { - StringTemplate callST = template("call"); - callST.setAttribute("func", call.getProc().getLocalPart()); // Check is called procedure declared previously String procName = call.getProc().getLocalPart(); if (proceduresMap.get(procName) == null) @@ -419,6 +417,15 @@ Map outArgs = new HashMap(); ProcedureSignature proc = (ProcedureSignature)proceduresMap.get(procName); + StringTemplate callST; + if(proc.getInvocationMode() == ProcedureSignature.INVOCATION_USERDEFINED) { + callST = template("callUserDefined"); + } else if(proc.getInvocationMode() == ProcedureSignature.INVOCATION_INTERNAL) { + callST = template("callInternal"); + } else { + throw new CompilationException("Unknown procedure invocation mode "+proc.getInvocationMode()); + } + callST.setAttribute("func", call.getProc().getLocalPart()); /* Does number of input arguments match */ for (int i = 0; i < proc.sizeOfInputArray(); i++) { if (proc.getInputArray(i).isOptional()) Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2009-06-22 08:08:25 UTC (rev 2965) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2009-06-22 09:02:33 UTC (rev 2966) @@ -13,14 +13,21 @@ private ArrayList outputArgs; private boolean anyNumOfInputArgs; private boolean anyNumOfOutputArgs; /* this is maybe unnecessary*/ + private int invocationMode; + + /* Procedure is built in to Swift. */ + static public final int INVOCATION_INTERNAL = 600; + + /* Procedure is user defined. */ + static public final int INVOCATION_USERDEFINED = 601; - public ProcedureSignature(String name) { this.name = name; inputArgs = new ArrayList(); outputArgs = new ArrayList(); anyNumOfInputArgs = false; anyNumOfOutputArgs = false; + invocationMode = INVOCATION_USERDEFINED; } public String getName() { @@ -98,6 +105,14 @@ this.addOutputArg(fas); } } + + public void setInvocationMode(int i) { + this.invocationMode = i; + } + + public int getInvocationMode() { + return this.invocationMode; + } public static HashMap makeProcedureSignatures() { HashMap proceduresMap = new HashMap(); @@ -107,6 +122,7 @@ readData.addInputArg(rdInputArg); FormalArgumentSignature rdOutputArg = new FormalArgumentSignature(true); readData.addOutputArg(rdOutputArg); + readData.setInvocationMode(INVOCATION_INTERNAL); proceduresMap.put("readData", readData); ProcedureSignature readData2 = new ProcedureSignature("readData2"); @@ -114,10 +130,12 @@ readData2.addInputArg(rd2InputArg); FormalArgumentSignature rd2OutputArg = new FormalArgumentSignature(true); readData2.addOutputArg(rd2OutputArg); + readData2.setInvocationMode(INVOCATION_INTERNAL); proceduresMap.put("readData2", readData2); ProcedureSignature trace = new ProcedureSignature("trace"); trace.setAnyNumOfInputArgs(); + trace.setInvocationMode(INVOCATION_INTERNAL); proceduresMap.put("trace", trace); return proceduresMap; From noreply at svn.ci.uchicago.edu Mon Jun 22 06:14:00 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 06:14:00 -0500 (CDT) Subject: [Swift-commit] r2967 - in trunk: . libexec/log-processing tests/language-behaviour Message-ID: <20090622111400.716D79CC8C@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 06:14:00 -0500 (Mon, 22 Jun 2009) New Revision: 2967 Added: trunk/libexec/log-processing/log-to-internal-proc-transitions Modified: trunk/build.xml trunk/libexec/log-processing/makefile trunk/tests/language-behaviour/run Log: Log processing for internal procedure provenance logs Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2009-06-22 09:02:33 UTC (rev 2966) +++ trunk/build.xml 2009-06-22 11:14:00 UTC (rev 2967) @@ -154,6 +154,7 @@ + Added: trunk/libexec/log-processing/log-to-internal-proc-transitions =================================================================== --- trunk/libexec/log-processing/log-to-internal-proc-transitions (rev 0) +++ trunk/libexec/log-processing/log-to-internal-proc-transitions 2009-06-22 11:14:00 UTC (rev 2967) @@ -0,0 +1,17 @@ +#!/bin/bash + +# this will take a log file and produce a set of state transition +# events for every internal procedure + +# 2009-05-04 18:02:15,499+0200 INFO vdl:mains INTERNALPROC_START thread=0-1 name=trace + +# 2009-05-04 18:02:15,556+0200 INFO vdl:mains INTERNALPROC_END thread=0-0 + + +grep ' INTERNALPROC_START ' $1 | iso-to-secs | \ +sed 's/^\(.*\) INFO [^ ]* INTERNALPROC_START thread=\([0-9\-]*\) name=\([^ ]*\).*$/\1 \2 START \3/' + +grep ' INTERNALPROC_END ' $1 | iso-to-secs | \ +sed 's/^\(.*\) INFO [^ ]* INTERNALPROC_END thread=\([0-9\-]*\).*$/\1 \2 END/' + + Modified: trunk/libexec/log-processing/makefile =================================================================== --- trunk/libexec/log-processing/makefile 2009-06-22 09:02:33 UTC (rev 2966) +++ trunk/libexec/log-processing/makefile 2009-06-22 11:14:00 UTC (rev 2967) @@ -100,6 +100,9 @@ compound.transitions: $(LOG) log-to-compound-transitions $(LOG) > compound.transitions +internalproc.transitions: $(LOG) + log-to-internal-proc-transitions $(LOG) > internalproc.transitions + ## retry analysis jobs.retrycount.summary: execute2.event Modified: trunk/tests/language-behaviour/run =================================================================== --- trunk/tests/language-behaviour/run 2009-06-22 09:02:33 UTC (rev 2966) +++ trunk/tests/language-behaviour/run 2009-06-22 11:14:00 UTC (rev 2967) @@ -90,9 +90,9 @@ fi # check that all execute and execute2 IDs are unique - swift-plot-log ${t}-*.log execute.transitions compound.transitions - if [ "$( cat compound.transitions execute.transitions | grep ' START ' | cut -d ' ' -f 2 | sort | uniq -d)" != "" ]; then - echo EXECUTE AND COMPOUND IDS NOT UNIQUE + swift-plot-log ${t}-*.log execute.transitions compound.transitions internalproc.transitions + if [ "$( cat compound.transitions execute.transitions internalproc.transitions | grep ' START ' | cut -d ' ' -f 2 | sort | uniq -d)" != "" ]; then + echo EXECUTE AND COMPOUND AND INTERNALPROC IDS NOT UNIQUE exit 3 fi From noreply at svn.ci.uchicago.edu Mon Jun 22 07:01:37 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 07:01:37 -0500 (CDT) Subject: [Swift-commit] r2968 - in trunk: resources tests/language-behaviour Message-ID: <20090622120137.939B49CC7E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 07:01:37 -0500 (Mon, 22 Jun 2009) New Revision: 2968 Added: trunk/tests/language-behaviour/00242-compound-with-vars.swift Modified: trunk/resources/Karajan.stg Log: log internal variables of compound procedures as if they are outputs. these are distinguished by their names, as internal variable names appear in the same namespace as return parameters. Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-06-22 11:14:00 UTC (rev 2967) +++ trunk/resources/Karajan.stg 2009-06-22 12:01:37 UTC (rev 2968) @@ -307,6 +307,12 @@ $endif$ $endif$ + +intermediate +$name$ + +{#thread} + >> vdl_mapping(mapping,file,waitfor) ::= << Added: trunk/tests/language-behaviour/00242-compound-with-vars.swift =================================================================== --- trunk/tests/language-behaviour/00242-compound-with-vars.swift (rev 0) +++ trunk/tests/language-behaviour/00242-compound-with-vars.swift 2009-06-22 12:01:37 UTC (rev 2968) @@ -0,0 +1,20 @@ +type messagefile {} + +(messagefile t) greeting(string m) { + app { + echo m stdout=@filename(t); + } +} + +(messagefile first, messagefile second) compound() { + string third="f"; + first = greeting(third); + second = greeting("s"); +} + +messagefile a <"0024-compound.Q.out">; +messagefile b <"0024-compound.R.out">; + +(a,b) = compound(); + + From noreply at svn.ci.uchicago.edu Mon Jun 22 07:47:08 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 07:47:08 -0500 (CDT) Subject: [Swift-commit] r2969 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20090622124708.4DD699CC8C@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 07:47:07 -0500 (Mon, 22 Jun 2009) New Revision: 2969 Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java Log: FILENAME logs should only be output for file-mapped types, not for externals, arrays, structs or primitive types. Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-06-22 12:01:37 UTC (rev 2968) +++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-06-22 12:47:07 UTC (rev 2969) @@ -447,10 +447,30 @@ // anyway - should perhaps only be trying to map leafnodes? // Mapping // non-leaf stuff is giving wierd paths anyway + +// TODO this is perhaps an unpleasant way of finding if this is a file-backed +// leaf node or not + boolean filemapped = true; + Type type = this.getType(); + if(type.getName().equals("external")) { + filemapped = false; + } + if(type.isPrimitive()) { + filemapped = false; + } + if(type.isArray()) { + filemapped = false; + } + if(handles.size()>0) { + filemapped = false; + } + try { - Object path = m.map(pathFromRoot); - logger.info("FILENAME dataset=" + identifier + " filename=" + if(filemapped) { + Object path = m.map(pathFromRoot); + logger.info("FILENAME dataset=" + identifier + " filename=" + path); + } } catch (Exception e) { logger.info("NOFILENAME dataset=" + identifier); From noreply at svn.ci.uchicago.edu Mon Jun 22 08:40:53 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 08:40:53 -0500 (CDT) Subject: [Swift-commit] r2970 - trunk/libexec Message-ID: <20090622134053.EE3A39CC8C@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 08:40:53 -0500 (Mon, 22 Jun 2009) New Revision: 2970 Modified: trunk/libexec/_swiftwrap Log: Make wrapper use mv to stage out files to shared directory rather than cp. This should be faster when the job directories are on the same filesystem as the site shared cache. (suggested by tuecke) Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2009-06-22 12:47:07 UTC (rev 2969) +++ trunk/libexec/_swiftwrap 2009-06-22 13:40:53 UTC (rev 2970) @@ -342,10 +342,10 @@ fail 254 "The following output files were not created by the application: $MISSING" fi -logstate "COPYING_OUTPUTS" +logstate "MOVING_OUTPUTS" for O in $OUTF ; do - cp "$DIR/$O" "shared/$O" 2>&1 >& "$INFO" - checkError 254 "Failed to copy output file $O to shared directory" + mv "$DIR/$O" "shared/$O" 2>&1 >& "$INFO" + checkError 254 "Failed to move output file $O to shared directory" done logstate "RM_JOBDIR" From noreply at svn.ci.uchicago.edu Mon Jun 22 09:50:26 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 09:50:26 -0500 (CDT) Subject: [Swift-commit] r2971 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20090622145026.EFB229CC7E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 09:50:26 -0500 (Mon, 22 Jun 2009) New Revision: 2971 Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java Log: handle nulls in SwiftArg - this makes misc/ordering-extern-notlazy test work again Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java 2009-06-22 13:40:53 UTC (rev 2970) +++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java 2009-06-22 14:50:26 UTC (rev 2971) @@ -89,6 +89,8 @@ Object v = super.getValue(stack); if(v instanceof DSHandle) { return (DSHandle)v; + } else if(v == null) { + return null; } else { throw new ExecutionException("Expected Swift data, but got some primitive type (" + v + ")"); } From noreply at svn.ci.uchicago.edu Mon Jun 22 11:57:13 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 11:57:13 -0500 (CDT) Subject: [Swift-commit] r2972 - trunk/resources Message-ID: <20090622165713.32C169CC7E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 11:57:12 -0500 (Mon, 22 Jun 2009) New Revision: 2972 Modified: trunk/resources/swiftscript.g Log: Fix whitespace in parser variable declaration rules. Modified: trunk/resources/swiftscript.g =================================================================== --- trunk/resources/swiftscript.g 2009-06-22 14:50:26 UTC (rev 2971) +++ trunk/resources/swiftscript.g 2009-06-22 16:57:12 UTC (rev 2972) @@ -193,15 +193,15 @@ StringTemplate n=null; StringTemplate thisTypeTemplate=null; String thisType = (String) t.getAttribute("name"); - StringTemplate variable=null; - StringTemplate m = null; + StringTemplate variable=null; + StringTemplate m = null; } : n=declarator (LBRACK RBRACK {thisType = thisType + "[]"; } )* { - thisTypeTemplate=template("type"); - thisTypeTemplate.setAttribute("name", thisType); + thisTypeTemplate=template("type"); + thisTypeTemplate.setAttribute("name", thisType); variable = template("variable"); variable.setAttribute("name", n); variable.setAttribute("type", thisTypeTemplate); From noreply at svn.ci.uchicago.edu Mon Jun 22 12:00:46 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 12:00:46 -0500 (CDT) Subject: [Swift-commit] r2973 - in trunk: resources src/org/griphyn/vdl/engine Message-ID: <20090622170046.DA6C29CC7E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 12:00:46 -0500 (Mon, 22 Jun 2009) New Revision: 2973 Modified: trunk/resources/Karajan.stg trunk/resources/swiftscript.xsd trunk/src/org/griphyn/vdl/engine/Karajan.java Log: Remove isArray attribute from variables in the XML representation of SwiftScript, because arrayness is stored in the type attribute now. Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-06-22 16:57:12 UTC (rev 2972) +++ trunk/resources/Karajan.stg 2009-06-22 17:00:46 UTC (rev 2973) @@ -97,7 +97,7 @@ >> typecheck() ::= << - + >> vdl_execute(outputs,inputs,application,name,line) ::= << @@ -291,15 +291,15 @@ >> -variable(name,type,isArray,expr,mapping,nil,file,waitfor,datatype) ::= << +variable(name,type,expr,mapping,nil,file,waitfor,datatype) ::= << $if(mapping)$ - + $vdl_mapping(mapping=mapping,file=file,waitfor=waitfor)$ $else$ $if(file)$ - + $vdl_mapping(mapping=mapping,file=file,waitfor=waitfor)$ $else$ Modified: trunk/resources/swiftscript.xsd =================================================================== --- trunk/resources/swiftscript.xsd 2009-06-22 16:57:12 UTC (rev 2972) +++ trunk/resources/swiftscript.xsd 2009-06-22 17:00:46 UTC (rev 2973) @@ -129,8 +129,6 @@ - - @@ -302,8 +300,6 @@ - - Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-06-22 16:57:12 UTC (rev 2972) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-06-22 17:00:46 UTC (rev 2973) @@ -242,7 +242,6 @@ typeST.setAttribute("name", param.getType().getLocalPart()); typeST.setAttribute("namespace", param.getType().getNamespaceURI()); paramST.setAttribute("type", typeST); - paramST.setAttribute("isArray", new Boolean(param.getIsArray1())); if(!param.isNil()) paramST.setAttribute("default",expressionToKarajan(param.getAbstractExpression(), scope)); return paramST; @@ -252,7 +251,6 @@ StringTemplate variableST = template("variable"); variableST.setAttribute("name", var.getName()); variableST.setAttribute("type", var.getType().getLocalPart()); - variableST.setAttribute("isArray", Boolean.valueOf(var.getIsArray1())); checkIsTypeDefined(var.getType().getLocalPart()); if(!var.isNil()) { From noreply at svn.ci.uchicago.edu Mon Jun 22 16:29:06 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Jun 2009 16:29:06 -0500 (CDT) Subject: [Swift-commit] r2974 - trunk/docs Message-ID: <20090622212906.A42679CC8C@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-22 16:29:06 -0500 (Mon, 22 Jun 2009) New Revision: 2974 Modified: trunk/docs/log-processing.xml Log: remove outdated notes for log-processing Modified: trunk/docs/log-processing.xml =================================================================== --- trunk/docs/log-processing.xml 2009-06-22 17:00:46 UTC (rev 2973) +++ trunk/docs/log-processing.xml 2009-06-22 21:29:06 UTC (rev 2974) @@ -11,17 +11,9 @@
Overview -There is an under-development package of Swift log processing utilties. +There is a package of Swift log processing utilties. - -This code is not distributed in the main Swift release. Instead, it must be -obtained from the Swift SVN repository, in the directory -log-processing/: - -svn co https://svn.ci.uchicago.edu/svn/vdl2/log-processing - -
Prerequisites @@ -32,8 +24,6 @@
Web page about a run -cd log-processing/bin -export PATH=$(pwd):$PATH swift-plot-log /path/to/readData-20080304-0903-xgqf5nhe.log This will create a web page, report-readData-20080304-0903-xgqf5nhe @@ -47,7 +37,7 @@ The log processing tools can output transition streams in CEDPS logging format: -make LOG=/path/to/readData-20080304-0903-xgqf5nhe.log clean execute.cedps +swift-plot-log /path/to/readData-20080304-0903-xgqf5nhe.log execute.cedps
@@ -79,14 +69,14 @@ Streams are generated from their source log files either as .transitions -or .event files, for example by make foo.event. +or .event files, for example by swift-plot-log whatever.log foo.event. Various plots are available based on different streams: - makefile targetDescription + Makefile targetDescriptionfoo.pngPlots the foo event streamfoo-total.pngPlots how many foo events are in progress at any time From noreply at svn.ci.uchicago.edu Wed Jun 24 15:17:20 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Jun 2009 15:17:20 -0500 (CDT) Subject: [Swift-commit] r2975 - trunk/src/org/griphyn/vdl/engine Message-ID: <20090624201720.C54629CC92@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-24 15:17:20 -0500 (Wed, 24 Jun 2009) New Revision: 2975 Modified: trunk/src/org/griphyn/vdl/engine/VariableScope.java Log: public modifier missing in one of the constants in VariableScope. added for consistency. Modified: trunk/src/org/griphyn/vdl/engine/VariableScope.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/VariableScope.java 2009-06-22 21:29:06 UTC (rev 2974) +++ trunk/src/org/griphyn/vdl/engine/VariableScope.java 2009-06-24 20:17:20 UTC (rev 2975) @@ -19,7 +19,7 @@ public class VariableScope { /** permit array up-assignment, but not entire variables */ - final static int ENCLOSURE_LOOP = 301923; + public final static int ENCLOSURE_LOOP = 301923; /** permit all upwards assignments */ public static final int ENCLOSURE_ALL = 301924; From noreply at svn.ci.uchicago.edu Wed Jun 24 15:17:48 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Jun 2009 15:17:48 -0500 (CDT) Subject: [Swift-commit] r2976 - trunk/src/org/griphyn/vdl/engine Message-ID: <20090624201748.642919CC92@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-24 15:17:48 -0500 (Wed, 24 Jun 2009) New Revision: 2976 Modified: trunk/src/org/griphyn/vdl/engine/VariableScope.java Log: variable scope debugging info should be DEBUG not INFO log level Modified: trunk/src/org/griphyn/vdl/engine/VariableScope.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/VariableScope.java 2009-06-24 20:17:20 UTC (rev 2975) +++ trunk/src/org/griphyn/vdl/engine/VariableScope.java 2009-06-24 20:17:48 UTC (rev 2976) @@ -54,9 +54,9 @@ */ public VariableScope(Karajan c, VariableScope parent, int a) { if(parentScope!=null) { - logger.info("New scope "+hashCode()+" with parent scope "+parentScope.hashCode()); + logger.debug("New scope "+hashCode()+" with parent scope "+parentScope.hashCode()); } else { - logger.info("New scope "+hashCode()+" with no parent."); + logger.debug("New scope "+hashCode()+" with no parent."); } compiler = c; parentScope = parent; @@ -75,7 +75,7 @@ declaration already exists. Perhaps error in same scope and warning if it shadows an outer scope? */ public void addVariable(String name, String type) throws CompilationException { - logger.info("Adding variable "+name+" of type "+type+" to scope "+hashCode()); + logger.debug("Adding variable "+name+" of type "+type+" to scope "+hashCode()); if(isVariableDefined(name)) { throw new CompilationException("Variable "+name+" is already defined."); @@ -97,7 +97,7 @@ * an exception if the variable is defined in a parent scope */ public void addInternalVariable(String name, String type) throws CompilationException { - logger.info("Adding internal variable " + name + " of type " + type + " to scope " + hashCode()); + logger.debug("Adding internal variable " + name + " of type " + type + " to scope " + hashCode()); if(isVariableLocallyDefined(name)) { throw new CompilationException("Variable " + name + " is already defined."); @@ -162,7 +162,7 @@ } ld.setAttribute("waitfor", closeID); } else { - logger.info("Variable "+variableName+" is local but has no template."); + logger.debug("Variable "+variableName+" is local but has no template."); } StringTemplate postST = compiler.template("partialclose"); postST.setAttribute("var", variableName); @@ -189,7 +189,7 @@ if(!statementList.contains(closeID)) { statementList.add(closeID); } - logger.info("added "+closeID+" to variable "+variableName+" in scope "+hashCode()); + logger.debug("added "+closeID+" to variable "+variableName+" in scope "+hashCode()); } else { throw new CompilationException("variable "+variableName+" is not writeable in this scope"); } @@ -211,21 +211,21 @@ if(decls instanceof StringTemplate) { StringTemplate declST = (StringTemplate) decls; if(declST.getAttribute("name").equals(name)) { -logger.info("thats the declaration for "+name); + logger.debug("thats the declaration for "+name); return declST; } } else { // assume its a List Iterator it = ((List) decls).iterator(); while(it.hasNext()) { StringTemplate declST = (StringTemplate) it.next(); -logger.info("looking at declaration "+declST); + logger.debug("looking at declaration "+declST); try { if(declST.getAttribute("name").equals(name)) { -logger.info("thats the declaration for "+name); + logger.debug("thats the declaration for "+name); return declST; } } catch(java.util.NoSuchElementException nse) { -logger.info("it so definitely wasn't in that one, we got an exception."); + logger.debug("it so definitely wasn't in that one, we got an exception."); // TODO this is not a nice use of exceptions... } From noreply at svn.ci.uchicago.edu Wed Jun 24 15:18:32 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Jun 2009 15:18:32 -0500 (CDT) Subject: [Swift-commit] r2977 - in trunk: src/org/griphyn/vdl/engine tests/language-behaviour Message-ID: <20090624201832.B35659CC92@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-24 15:18:32 -0500 (Wed, 24 Jun 2009) New Revision: 2977 Added: trunk/tests/language-behaviour/0013-out-of-order.out.expected trunk/tests/language-behaviour/0013-out-of-order.swift Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: split the declaration of variables into two stages: a stage where the name and type are put in the scope symbol tables, and a stage where the mapping expression (if it exists) is processed and variable template is generated. this is needed to allow global variables to be known before procedures are processed. Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-06-24 20:17:48 UTC (rev 2976) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-06-24 20:18:32 UTC (rev 2977) @@ -173,6 +173,8 @@ } } + statementsForSymbols(prog, scope); + // Keep track of declared procedures for (int i = 0; i < prog.sizeOfProcedureArray(); i++) { Procedure proc = prog.getProcedureArray(i); @@ -231,6 +233,7 @@ else { VariableScope compoundScope = new VariableScope(this, innerScope); compoundScope.bodyTemplate = procST; + statementsForSymbols(proc, compoundScope); statements(proc, compoundScope); } } @@ -247,12 +250,17 @@ return paramST; } + public void variableForSymbol(Variable var, VariableScope scope) throws CompilationException { + + checkIsTypeDefined(var.getType().getLocalPart()); + scope.addVariable(var.getName(), var.getType().getLocalPart()); + } + public void variable(Variable var, VariableScope scope) throws CompilationException { StringTemplate variableST = template("variable"); variableST.setAttribute("name", var.getName()); variableST.setAttribute("type", var.getType().getLocalPart()); - - checkIsTypeDefined(var.getType().getLocalPart()); + if(!var.isNil()) { if (var.getFile() != null) { @@ -323,8 +331,8 @@ variableST.setAttribute("mapping", mappingST); variableST.setAttribute("nil", Boolean.TRUE); } + scope.bodyTemplate.setAttribute("declarations", variableST); - scope.addVariable(var.getName(), var.getType().getLocalPart()); } void checkIsTypeDefined(String type) throws CompilationException { @@ -363,6 +371,15 @@ } } + public void statementsForSymbols(XmlObject prog, VariableScope scope) throws CompilationException { + XmlCursor cursor = prog.newCursor(); + cursor.selectPath("*"); + while (cursor.toNextSelection()) { + XmlObject child = cursor.getObject(); + statementForSymbol(child, scope); + } + } + public void statements(XmlObject prog, VariableScope scope) throws CompilationException { XmlCursor cursor = prog.newCursor(); cursor.selectPath("*"); @@ -372,6 +389,26 @@ } } + public void statementForSymbol(XmlObject child, VariableScope scope) throws CompilationException { + if (child instanceof Variable) { + variableForSymbol((Variable) child, scope); + } + else if (child instanceof Assign + || child instanceof Call + || child instanceof Foreach + || child instanceof Iterate + || child instanceof If + || child instanceof Switch + || child instanceof Procedure + || child instanceof Types + || child instanceof FormalParameter) { + // ignore these - they're expected but we don't need to + // do anything for them here + } else { + throw new CompilationException("Unexpected element in XML. Implementing class "+child.getClass()+", content "+child); + } + } + public void statement(XmlObject child, VariableScope scope) throws CompilationException { if (child instanceof Variable) { variable((Variable) child, scope); @@ -402,6 +439,7 @@ } } + public StringTemplate call(Call call, VariableScope scope, boolean inhibitOutput) throws CompilationException { try { // Check is called procedure declared previously @@ -588,6 +626,7 @@ iterateST.setAttribute("var", iterate.getVar()); innerScope.bodyTemplate = iterateST; + statementsForSymbols(iterate.getBody(), innerScope); statements(iterate.getBody(), innerScope); XmlObject cond = iterate.getAbstractExpression(); @@ -627,6 +666,7 @@ innerScope.bodyTemplate = foreachST; + statementsForSymbols(foreach.getBody(), innerScope); statements(foreach.getBody(), innerScope); Object statementID = new Integer(callID++); @@ -656,6 +696,7 @@ innerThenScope.bodyTemplate = template("sub_comp"); ifST.setAttribute("vthen", innerThenScope.bodyTemplate); + statementsForSymbols(thenstat, innerThenScope); statements(thenstat, innerThenScope); Object statementID = new Integer(callID++); @@ -672,6 +713,7 @@ innerElseScope.bodyTemplate = template("sub_comp"); ifST.setAttribute("velse", innerElseScope.bodyTemplate); + statementsForSymbols(elsestat, innerElseScope); statements(elsestat, innerElseScope); Iterator elseScopeIterator = innerElseScope.getVariableIterator(); @@ -714,6 +756,7 @@ VariableScope defaultScope = new VariableScope(this, scope); defaultScope.bodyTemplate = template("sub_comp"); switchST.setAttribute("sdefault", defaultScope.bodyTemplate); + statementsForSymbols(defaultstat, defaultScope); statements(defaultstat, defaultScope); Iterator defaultScopeIterator = defaultScope.getVariableIterator(); while(defaultScopeIterator.hasNext()) { @@ -726,6 +769,7 @@ public void caseStat(Case casestat, VariableScope scope) throws CompilationException { StringTemplate valueST = expressionToKarajan(casestat.getAbstractExpression(), scope); scope.bodyTemplate.setAttribute("value", valueST.toString()); + statementsForSymbols(casestat.getStatements(), scope); statements(casestat.getStatements(), scope); } Added: trunk/tests/language-behaviour/0013-out-of-order.out.expected =================================================================== --- trunk/tests/language-behaviour/0013-out-of-order.out.expected (rev 0) +++ trunk/tests/language-behaviour/0013-out-of-order.out.expected 2009-06-24 20:18:32 UTC (rev 2977) @@ -0,0 +1 @@ +hello Added: trunk/tests/language-behaviour/0013-out-of-order.swift =================================================================== --- trunk/tests/language-behaviour/0013-out-of-order.swift (rev 0) +++ trunk/tests/language-behaviour/0013-out-of-order.swift 2009-06-24 20:18:32 UTC (rev 2977) @@ -0,0 +1,12 @@ +type messagefile; + +(messagefile t) greeting() { + app { + echo "hello" stdout=@filename(t); + } +} + +messagefile outfile <"0013-out-of-order.out">; + +outfile = greeting(); + From noreply at svn.ci.uchicago.edu Wed Jun 24 15:18:51 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Jun 2009 15:18:51 -0500 (CDT) Subject: [Swift-commit] r2978 - trunk/resources Message-ID: <20090624201851.1FC749CC92@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-24 15:18:50 -0500 (Wed, 24 Jun 2009) New Revision: 2978 Modified: trunk/resources/swiftscript.stg Log: Remove unused variable declaration template Modified: trunk/resources/swiftscript.stg =================================================================== --- trunk/resources/swiftscript.stg 2009-06-24 20:18:32 UTC (rev 2977) +++ trunk/resources/swiftscript.stg 2009-06-24 20:18:50 UTC (rev 2978) @@ -120,12 +120,6 @@ >> -vardecl(sourcelocation) ::= << -$if (it.type)$ -$\n$ -$endif$ ->> - returnParam(type,name,bind,sourcelocation) ::= << $name$ >> From noreply at svn.ci.uchicago.edu Wed Jun 24 15:19:20 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Jun 2009 15:19:20 -0500 (CDT) Subject: [Swift-commit] r2979 - trunk/src/org/griphyn/vdl/toolkit Message-ID: <20090624201920.E41B19CC92@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-24 15:19:20 -0500 (Wed, 24 Jun 2009) New Revision: 2979 Modified: trunk/src/org/griphyn/vdl/toolkit/VDLt2VDLx.java Log: Remove a semicolon that was living all by itself unneeded Modified: trunk/src/org/griphyn/vdl/toolkit/VDLt2VDLx.java =================================================================== --- trunk/src/org/griphyn/vdl/toolkit/VDLt2VDLx.java 2009-06-24 20:18:50 UTC (rev 2978) +++ trunk/src/org/griphyn/vdl/toolkit/VDLt2VDLx.java 2009-06-24 20:19:20 UTC (rev 2979) @@ -10,7 +10,6 @@ import org.apache.log4j.Logger; import org.globus.swift.parser.SwiftScriptLexer; import org.globus.swift.parser.SwiftScriptParser; -; /** Commandline tool to convert the textual form of SwiftScript into the XML form. From noreply at svn.ci.uchicago.edu Wed Jun 24 15:19:44 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Jun 2009 15:19:44 -0500 (CDT) Subject: [Swift-commit] r2980 - trunk/tests/language-behaviour Message-ID: <20090624201944.E30549CC92@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-24 15:19:44 -0500 (Wed, 24 Jun 2009) New Revision: 2980 Modified: trunk/tests/language-behaviour/run Log: remove _concurrent directory during test cleanup Modified: trunk/tests/language-behaviour/run =================================================================== --- trunk/tests/language-behaviour/run 2009-06-24 20:19:20 UTC (rev 2979) +++ trunk/tests/language-behaviour/run 2009-06-24 20:19:44 UTC (rev 2980) @@ -33,7 +33,7 @@ rm -rf $1*.out $1*.xml $1*.kml $1*.dot $1*.log $1*.rlog $1*.d elif [ "$RESUME" != "y" ]; then echo Removing files from previous runs - rm -rf *.out *.xml *.kml *.dot *.log *.rlog *.d + rm -rf *.out *.xml *.kml *.dot *.log *.rlog *.d _concurrent fi for testfn in $TESTS; do From noreply at svn.ci.uchicago.edu Tue Jun 30 05:41:17 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 05:41:17 -0500 (CDT) Subject: [Swift-commit] r2982 - provenancedb Message-ID: <20090630104117.382539CC86@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 05:41:15 -0500 (Tue, 30 Jun 2009) New Revision: 2982 Modified: provenancedb/prov-to-sql.sh Log: insert of values into executes table has extra thread field that should not be there Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-25 13:08:16 UTC (rev 2981) +++ provenancedb/prov-to-sql.sh 2009-06-30 10:41:15 UTC (rev 2982) @@ -9,7 +9,7 @@ rm -f tmp-import.sql -while read time duration thread endstate app scratch; do +while read time duration thread localthread endstate app scratch; do echo "INSERT INTO processes (id, type) VALUES ('$thread', 'execute');" >> tmp-import.sql echo "INSERT INTO executes (id, starttime, duration, finalstate, app, scratch) VALUES ('$thread', $time, $duration, '$endstate', '$app', '$scratch');" >> tmp-import.sql done < execute.global.event From noreply at svn.ci.uchicago.edu Tue Jun 30 06:38:26 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 06:38:26 -0500 (CDT) Subject: [Swift-commit] r2983 - trunk/resources Message-ID: <20090630113826.CCC779CC86@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 06:38:26 -0500 (Tue, 30 Jun 2009) New Revision: 2983 Modified: trunk/resources/Karajan.stg Log: More information for provenance, about foreach and if bodies. Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-06-30 10:41:15 UTC (rev 2982) +++ trunk/resources/Karajan.stg 2009-06-30 11:38:26 UTC (rev 2983) @@ -213,12 +213,14 @@ $endif$ - + +SCOPE thread={#thread} + $declarations;separator="\n"$ $if(statements)$ $parallel(statements=statements)$ $endif$ - + >> @@ -410,10 +412,12 @@ $condition$ +SCOPE thread={#thread} $vthen$ $if(velse)$ +SCOPE thread={#thread} $velse$ $endif$ From noreply at svn.ci.uchicago.edu Tue Jun 30 07:33:59 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:33:59 -0500 (CDT) Subject: [Swift-commit] r2984 - in trunk: . bin docs Message-ID: <20090630123359.84B5F9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:33:59 -0500 (Tue, 30 Jun 2009) New Revision: 2984 Modified: trunk/CHANGES.txt trunk/bin/swift-osg-ress-site-catalog trunk/docs/userguide.xml Log: New parameter -condor-g for swift-osg-ress-site-catalog which causes sites files to be generate which will use a local condor-g installation instead of making direct GRAM submissions. Modified: trunk/CHANGES.txt =================================================================== --- trunk/CHANGES.txt 2009-06-30 11:38:26 UTC (rev 2983) +++ trunk/CHANGES.txt 2009-06-30 12:33:59 UTC (rev 2984) @@ -1,3 +1,8 @@ +(06/30/09) +*** New parameter -condor-g for swift-osg-ress-site-catalog which causes + sites files to be generate which will use a local condor-g installation + instead of making direct GRAM submissions. + (06/25/09) *** 'global' keyword to allow top level declarations to be visible in all scopes. Modified: trunk/bin/swift-osg-ress-site-catalog =================================================================== --- trunk/bin/swift-osg-ress-site-catalog 2009-06-30 11:38:26 UTC (rev 2983) +++ trunk/bin/swift-osg-ress-site-catalog 2009-06-30 12:33:59 UTC (rev 2984) @@ -10,6 +10,7 @@ my $opt_vo = 'engage'; my $opt_engage_verified = 0; my $opt_gt4 = 0; +my $opt_condorg = 0; my $opt_out = '&STDOUT'; Getopt::Long::Configure('bundling'); @@ -18,6 +19,7 @@ "vo=s" => \$opt_vo, "engage-verified" => \$opt_engage_verified, "gt4" => \$opt_gt4, + "condor-g" => \$opt_condorg, "out=s" => \$opt_out, ) or pod2usage(1); @@ -94,7 +96,15 @@ print FH " \n"; print FH " \n"; print FH " \n"; - if ($opt_gt4) { + if ($opt_condorg) { + print FH " \n"; + print FH " grid\n"; + if($opt_gt4) { + die("swift-osg-ress-site-catalog cannot generate Condor-G + GRAM4 sites files"); + } + print FH " gt2 $contact\n"; + } + elsif ($opt_gt4) { print FH " \n"; } else { @@ -143,6 +153,11 @@ Write to [filename] instead of stdout +=item B<--condor-g> + +Generates sites files which will submit jobs using a local Condor-G +installation rather than through direct GRAM2 submission. + =back =head1 DESCRIPTION Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-06-30 11:38:26 UTC (rev 2983) +++ trunk/docs/userguide.xml 2009-06-30 12:33:59 UTC (rev 2984) @@ -1564,6 +1564,16 @@ Write to [filename] instead of stdout + + --condor-g + +Generates sites files which will submit jobs using a local Condor-G +installation rather than through direct GRAM2 submission. + + + + + From noreply at svn.ci.uchicago.edu Tue Jun 30 07:37:11 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:37:11 -0500 (CDT) Subject: [Swift-commit] r2985 - in provenancedb: . validator Message-ID: <20090630123711.436599CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:37:10 -0500 (Tue, 30 Jun 2009) New Revision: 2985 Added: provenancedb/validator/ provenancedb/validator/V.java provenancedb/validator/run Log: small java code to validate an OPM against the opm schema Added: provenancedb/validator/V.java =================================================================== --- provenancedb/validator/V.java (rev 0) +++ provenancedb/validator/V.java 2009-06-30 12:37:10 UTC (rev 2985) @@ -0,0 +1,35 @@ +import org.openprovenance.model.v101A.*; + +import java.io.File; +import java.util.Iterator; +import java.util.Collection; +import java.util.ArrayList; +import org.apache.xmlbeans.*; + +public class V { + +public static void main(String args[]) throws Exception { + + XmlOptions options = new XmlOptions(); + Collection errors = new ArrayList(); + options.setErrorListener(errors); + options.setValidateOnSet(); + options.setLoadLineNumbers(); + + OpmGraphDocument opmGraph = OpmGraphDocument.Factory.parse(new File(args[0]), options); + + System.out.println("result of validation = "+opmGraph.validate(options)); + + System.out.println("Validation errors:"); + Iterator i = errors.iterator(); + while(i.hasNext()) { + XmlError error = (XmlError) i.next(); + System.out.println(error.toString()); + } + System.exit(3); + + + +} +} + Added: provenancedb/validator/run =================================================================== --- provenancedb/validator/run (rev 0) +++ provenancedb/validator/run 2009-06-30 12:37:10 UTC (rev 2985) @@ -0,0 +1,15 @@ +#!/bin/bash + +export CLASSPATH=/Users/benc/work/cog/modules/swift/lib/xbean.jar +export CLASSPATH=/Users/benc/work/cog/modules/swift/lib/jsr173_1.0_api.jar:$CLASSPATH + +java org.apache.xmlbeans.impl.tool.SchemaCompiler v1.01.a.xsd + +export CLASSPATH=./xmltypes.jar:$CLASSPATH + +javac V.java + +export CLASSPATH=.:$CLASSPATH + +java V ../tests/provenancedb/opm.xml + Property changes on: provenancedb/validator/run ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Tue Jun 30 07:37:30 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:37:30 -0500 (CDT) Subject: [Swift-commit] r2986 - provenancedb Message-ID: <20090630123730.604EA9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:37:30 -0500 (Tue, 30 Jun 2009) New Revision: 2986 Modified: provenancedb/prepare-provenance-chart provenancedb/prov-to-sql.sh Log: import @functions Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2009-06-30 12:37:10 UTC (rev 2985) +++ provenancedb/prepare-provenance-chart 2009-06-30 12:37:30 UTC (rev 2986) @@ -28,3 +28,14 @@ info-to-extrainfo > extrainfo.txt cat $1 | grep ' OPERATOR ' | sed 's/^.*thread=\([^ ]*\) operator="\([^ ]*\)" lhs=\([^ ]*\) rhs=\([^ ]*\) result=\([^ ]*\).*$/\1 \2 \3 \4 \5/' > operators.txt + + +# 2009-03-19 19:15:35,244+0100 INFO vdl:arguments FUNCTION id=88000-0-4-4 name="f ilename" result=tag:benc at ci.uchicago.edu,2008:swift:dataset:20090319-1915-xj8flg 13:720000000060 +# 2009-03-19 19:15:35,246+0100 INFO vdl:arguments FUNCTIONPARAMETER id=88001-0-4- 4 input=tag:benc at ci.uchicago.edu,2008:swift:dataset:20090319-1915-xj8flg13:72000 0000058 + +cat $1 | grep ' FUNCTION ' | sed "s/^.*id=\([^ ]*\) name=\([^ ]*\) result=\([^ ]*\).*\$/$WFID\1 \2 \3/" > functions.txt +# the IDs in functions.txt should be unique... +# we could even test that... + +cat $1 | grep ' FUNCTIONPARAMETER ' | sed "s/^.*id=\([^ ]*\) input=\([^ ]*\).*\$/$WFID\1 \2/" > function-inputs.txt + Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-30 12:37:10 UTC (rev 2985) +++ provenancedb/prov-to-sql.sh 2009-06-30 12:37:30 UTC (rev 2986) @@ -35,6 +35,17 @@ OPERATORIDCOUNTER=$(( $OPERATORIDCOUNTER + 1)) done < operators.txt +while read id name output; do + echo "INSERT INTO processes (id, type) VALUES ('$id', 'function');" >> tmp-import.sql + echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'O', '$output', 'result');" >> tmp-import.sql + echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$id', '$name');" >> tmp-import.sql +done < functions.txt + +while read id value; do +# TODO need ordering/naming + echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'I', '$value', 'undefined');" >> tmp-import.sql +done < function-inputs.txt + while read thread direction dataset variable rest; do if [ "$direction" == "input" ] ; then dir=I From noreply at svn.ci.uchicago.edu Tue Jun 30 07:37:51 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:37:51 -0500 (CDT) Subject: [Swift-commit] r2987 - provenancedb Message-ID: <20090630123751.D46CF9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:37:51 -0500 (Tue, 30 Jun 2009) New Revision: 2987 Modified: provenancedb/prov-init.sql provenancedb/prov-to-sql.sh Log: rename executes_in_workflows to processes_in_workflows, and implement. Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2009-06-30 12:37:30 UTC (rev 2986) +++ provenancedb/prov-init.sql 2009-06-30 12:37:51 UTC (rev 2987) @@ -9,18 +9,20 @@ DROP TABLE invocation_procedure_names; DROP TABLE dataset_containment; DROP TABLE dataset_filenames; -DROP TABLE executes_in_workflows; +DROP TABLE processes_in_workflows; DROP TABLE dataset_values; DROP TABLE known_workflows; DROP TABLE workflow_events; DROP TABLE extrainfo; --- executes_in_workflow is unused at the moment, but is intended to associate --- each execute with its containing workflow -CREATE TABLE executes_in_workflows +-- associates each process with its containing workflow +-- TODO - perhaps a workflow is itself a big big process? +-- in which case this looks very much like a compound/app +-- containment? +CREATE TABLE processes_in_workflows (workflow_id char(128), - execute_id char(128) + process_id char(128) ); -- processes gives information about each process (in the OPM sense) @@ -155,6 +157,6 @@ -- this GRANT does not work for sqlite; you'll get a syntax error but -- ignore it, as it is not needed in sqlite -grant all on dataset_containment, dataset_filenames, dataset_usage, executes_in_workflows, invocation_procedure_names, known_workflows, workflow_events to public, operators; +grant all on dataset_containment, dataset_filenames, dataset_usage, processes_in_workflows, invocation_procedure_names, known_workflows, workflow_events to public, operators; Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-30 12:37:30 UTC (rev 2986) +++ provenancedb/prov-to-sql.sh 2009-06-30 12:37:51 UTC (rev 2987) @@ -5,12 +5,16 @@ export WFID="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:" +# TODO is there already a URI form for identifying workflows? +export WF="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:run" + echo Generating SQL for $RUNID rm -f tmp-import.sql while read time duration thread localthread endstate app scratch; do echo "INSERT INTO processes (id, type) VALUES ('$thread', 'execute');" >> tmp-import.sql + echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$thread', '$WF');" >> tmp-import.sql echo "INSERT INTO executes (id, starttime, duration, finalstate, app, scratch) VALUES ('$thread', $time, $duration, '$endstate', '$app', '$scratch');" >> tmp-import.sql done < execute.global.event @@ -29,6 +33,7 @@ operatorid="${WFID}operator:$OPERATORIDCOUNTER" echo "INSERT INTO processes (id, type) VALUES ('$operatorid', 'operator');" >> tmp-import.sql echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$operatorid','$name'); " >> tmp-import.sql + echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$operatorid', '$WF');" >> tmp-import.sql echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$lhs', 'lhs');" >> tmp-import.sql echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$rhs', 'rhs');" >> tmp-import.sql echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'O', '$result', 'result');" >> tmp-import.sql @@ -37,6 +42,7 @@ while read id name output; do echo "INSERT INTO processes (id, type) VALUES ('$id', 'function');" >> tmp-import.sql + echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$id', '$WF');" >> tmp-import.sql echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'O', '$output', 'result');" >> tmp-import.sql echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$id', '$name');" >> tmp-import.sql done < functions.txt From noreply at svn.ci.uchicago.edu Tue Jun 30 07:38:12 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:38:12 -0500 (CDT) Subject: [Swift-commit] r2988 - provenancedb Message-ID: <20090630123812.EA30E9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:38:12 -0500 (Tue, 30 Jun 2009) New Revision: 2988 Modified: provenancedb/prepare-provenance-chart provenancedb/prov-init.sql provenancedb/prov-to-sql.sh Log: CREATEARRAY provenance processing Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2009-06-30 12:37:51 UTC (rev 2987) +++ provenancedb/prepare-provenance-chart 2009-06-30 12:38:12 UTC (rev 2988) @@ -39,3 +39,8 @@ cat $1 | grep ' FUNCTIONPARAMETER ' | sed "s/^.*id=\([^ ]*\) input=\([^ ]*\).*\$/$WFID\1 \2/" > function-inputs.txt +cat $1 | grep ' CREATEARRAY START ' | sed 's/^.* array=\([^ ]*\).*$/\1/' > createarray.txt + +cat $1 | grep ' CREATEARRAY MEMBER ' | sed 's/^.* array=\([^ ]*\) index=\([^ ]*\) member=\([^ ]*\).*$/\1 \2 \3/' > createarray-members.txt + + Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2009-06-30 12:37:51 UTC (rev 2987) +++ provenancedb/prov-init.sql 2009-06-30 12:38:12 UTC (rev 2988) @@ -14,6 +14,8 @@ DROP TABLE known_workflows; DROP TABLE workflow_events; DROP TABLE extrainfo; +DROP TABLE createarray; +DROP TABLE createarray_member; -- associates each process with its containing workflow @@ -128,6 +130,22 @@ value char(128) ); +-- The above dataset_* tables are the original containment representation +-- of datasets. +-- Now here are some tables to represent complex datasets differently, in +-- terms of accessors and constructors. +-- It is unclear which is the better representation. + +CREATE TABLE createarray + ( array_id char(128) + ); + +CREATE TABLE createarray_member + ( array_id char(128), + ix char(128), + member_id char(128) + ); + -- known_workflows stores some information about each workflow log that has -- been seen by the importer: the log filename, swift version and import -- status. Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-30 12:37:51 UTC (rev 2987) +++ provenancedb/prov-to-sql.sh 2009-06-30 12:38:12 UTC (rev 2988) @@ -94,6 +94,14 @@ echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$compoundid', '$procname');" >> tmp-import.sql done < compound.event +while read id ; do + echo "INSERT INTO createarray (array_id) VALUES ('$id');" >> tmp-import.sql +done < createarray.txt + +while read arrayid index memberid ; do + echo "INSERT INTO createarray_member (array_id, ix, member_id) VALUES ('$arrayid', '$index', '$memberid');" >> tmp-import.sql +done < createarray-members.txt + echo Sending SQL to DB $SQLCMD < tmp-import.sql From noreply at svn.ci.uchicago.edu Tue Jun 30 07:38:31 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:38:31 -0500 (CDT) Subject: [Swift-commit] r2989 - provenancedb Message-ID: <20090630123831.4C7CB9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:38:31 -0500 (Tue, 30 Jun 2009) New Revision: 2989 Modified: provenancedb/prepare-provenance-chart provenancedb/prov-init.sql Log: import ARRAYRANGE Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2009-06-30 12:38:12 UTC (rev 2988) +++ provenancedb/prepare-provenance-chart 2009-06-30 12:38:31 UTC (rev 2989) @@ -43,4 +43,5 @@ cat $1 | grep ' CREATEARRAY MEMBER ' | sed 's/^.* array=\([^ ]*\) index=\([^ ]*\) member=\([^ ]*\).*$/\1 \2 \3/' > createarray-members.txt +cat $1 | grep ' ARRAYRANGE ' | sed 's/^.* thread=\([^ ]*\).*$/\1/' > arrayrange.txt Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2009-06-30 12:38:12 UTC (rev 2988) +++ provenancedb/prov-init.sql 2009-06-30 12:38:31 UTC (rev 2989) @@ -16,6 +16,7 @@ DROP TABLE extrainfo; DROP TABLE createarray; DROP TABLE createarray_member; +DROP TABLE arrayrange; -- associates each process with its containing workflow @@ -146,6 +147,14 @@ member_id char(128) ); +-- TODO step +CREATE TABLE array_range + ( array_id char(128), + from_id char(128), + to_id char(128), + step_id char(128) -- nullable, if step is unspecified + ); + -- known_workflows stores some information about each workflow log that has -- been seen by the importer: the log filename, swift version and import -- status. From noreply at svn.ci.uchicago.edu Tue Jun 30 07:38:49 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:38:49 -0500 (CDT) Subject: [Swift-commit] r2990 - provenancedb Message-ID: <20090630123849.BD54F9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:38:49 -0500 (Tue, 30 Jun 2009) New Revision: 2990 Modified: provenancedb/prepare-for-import provenancedb/prov-to-sql.sh Log: import internal procedure executions into provenancedb Modified: provenancedb/prepare-for-import =================================================================== --- provenancedb/prepare-for-import 2009-06-30 12:38:31 UTC (rev 2989) +++ provenancedb/prepare-for-import 2009-06-30 12:38:49 UTC (rev 2990) @@ -4,7 +4,7 @@ # workflow has finished and will do everything necessary to import all # of the provenance information into the SQL provenance database. -swift-plot-log $1 execute.global.event execute2.event workflow.event execute2.global.event compound.event +swift-plot-log $1 execute.global.event execute2.event workflow.event execute2.global.event compound.event internalproc.event prepare-provenance-chart $1 Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-30 12:38:31 UTC (rev 2989) +++ provenancedb/prov-to-sql.sh 2009-06-30 12:38:49 UTC (rev 2990) @@ -94,6 +94,12 @@ echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$compoundid', '$procname');" >> tmp-import.sql done < compound.event +while read start duration thread finalstate procname ; do + fqid=$WFID$thread + echo "INSERT INTO processes (id, type) VALUES ('$fqid', 'internal');" >> tmp-import.sql + echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$fqid', '$procname');" >> tmp-import.sql +done < internalproc.event + while read id ; do echo "INSERT INTO createarray (array_id) VALUES ('$id');" >> tmp-import.sql done < createarray.txt From noreply at svn.ci.uchicago.edu Tue Jun 30 07:39:09 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:39:09 -0500 (CDT) Subject: [Swift-commit] r2991 - provenancedb Message-ID: <20090630123909.EBE0E9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:39:09 -0500 (Tue, 30 Jun 2009) New Revision: 2991 Modified: provenancedb/prov-to-sql.sh Log: Make operator import use thread ID (which should be unique now) this is not strictly necessary but fits in with the idea of other processes using thread ID Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-30 12:38:49 UTC (rev 2990) +++ provenancedb/prov-to-sql.sh 2009-06-30 12:39:09 UTC (rev 2991) @@ -27,17 +27,14 @@ echo "INSERT INTO execute2s (id, execute_id, starttime, duration, finalstate, site) VALUES ('$globalid', '$execute_id', '$starttime', '$duration', '$endstate', '$site');" >> tmp-import.sql done < execute2.global.event -# for fabricating operator IDs -OPERATORIDCOUNTER=0 while read thread name lhs rhs result; do - operatorid="${WFID}operator:$OPERATORIDCOUNTER" + operatorid="${WFID}operator:$thread" echo "INSERT INTO processes (id, type) VALUES ('$operatorid', 'operator');" >> tmp-import.sql echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$operatorid','$name'); " >> tmp-import.sql echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$operatorid', '$WF');" >> tmp-import.sql echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$lhs', 'lhs');" >> tmp-import.sql echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$rhs', 'rhs');" >> tmp-import.sql echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'O', '$result', 'result');" >> tmp-import.sql - OPERATORIDCOUNTER=$(( $OPERATORIDCOUNTER + 1)) done < operators.txt while read id name output; do From noreply at svn.ci.uchicago.edu Tue Jun 30 07:39:32 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:39:32 -0500 (CDT) Subject: [Swift-commit] r2992 - provenancedb Message-ID: <20090630123932.54A179CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:39:32 -0500 (Tue, 30 Jun 2009) New Revision: 2992 Modified: provenancedb/prepare-provenance-chart provenancedb/prov-to-sql.sh Log: import arbitrary thread-identified variable scopes as processes in order to maintain key integrity Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2009-06-30 12:39:09 UTC (rev 2991) +++ provenancedb/prepare-provenance-chart 2009-06-30 12:39:32 UTC (rev 2992) @@ -45,3 +45,6 @@ cat $1 | grep ' ARRAYRANGE ' | sed 's/^.* thread=\([^ ]*\).*$/\1/' > arrayrange.txt +# 2009-05-18 21:19:20,295+0200 INFO vdl:mains SCOPE thread=0-6-5-1-5 + +cat $1 | grep ' SCOPE ' | sed 's/^.* thread=\([^ ]*\).*/\1/' > scopes.txt Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-30 12:39:09 UTC (rev 2991) +++ provenancedb/prov-to-sql.sh 2009-06-30 12:39:32 UTC (rev 2992) @@ -105,6 +105,12 @@ echo "INSERT INTO createarray_member (array_id, ix, member_id) VALUES ('$arrayid', '$index', '$memberid');" >> tmp-import.sql done < createarray-members.txt +while read t ; do + thread="${WFID}$t" + echo "INSERT INTO processes (id, type) VALUES ('$thread', 'scope');" >> tmp-import.sql + echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$thread', '$WF');" >> tmp-import.sql +done < scopes.txt + echo Sending SQL to DB $SQLCMD < tmp-import.sql From noreply at svn.ci.uchicago.edu Tue Jun 30 07:40:25 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 30 Jun 2009 07:40:25 -0500 (CDT) Subject: [Swift-commit] r2993 - provenancedb Message-ID: <20090630124025.EAFFF9CC9D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-06-30 07:40:25 -0500 (Tue, 30 Jun 2009) New Revision: 2993 Modified: provenancedb/prov-to-sql.sh Log: record root thread as a process put compounds and internal procedures into the processes_in_workflows table Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-06-30 12:39:32 UTC (rev 2992) +++ provenancedb/prov-to-sql.sh 2009-06-30 12:40:25 UTC (rev 2993) @@ -12,6 +12,11 @@ rm -f tmp-import.sql +# this gives a distinction between the root process for a workflow and the +# workflow itself. perhaps better to model the workflow as a process +echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('${WFID}0', '$WF');" >> tmp-import.sql + echo "INSERT INTO processes (id, type) VALUES ('${WFID}0', 'rootthread');" >> tmp-import.sql + while read time duration thread localthread endstate app scratch; do echo "INSERT INTO processes (id, type) VALUES ('$thread', 'execute');" >> tmp-import.sql echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$thread', '$WF');" >> tmp-import.sql @@ -88,12 +93,14 @@ while read start duration thread finalstate procname ; do compoundid=$WFID$thread echo "INSERT INTO processes (id, type) VALUES ('$compoundid', 'compound');" >> tmp-import.sql + echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$compound', '$WF');" >> tmp-import.sql echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$compoundid', '$procname');" >> tmp-import.sql done < compound.event while read start duration thread finalstate procname ; do fqid=$WFID$thread echo "INSERT INTO processes (id, type) VALUES ('$fqid', 'internal');" >> tmp-import.sql + echo "INSERT INTO processes_in_workflows (process_id, workflow_id) VALUES ('$fqid', '$WF');" >> tmp-import.sql echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$fqid', '$procname');" >> tmp-import.sql done < internalproc.event