From noreply at svn.ci.uchicago.edu Thu Oct 1 09:02:10 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 1 Oct 2009 09:02:10 -0500 (CDT) Subject: [Swift-commit] r3116 - SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations Message-ID: <20091001140210.CEAC79CC9C@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-10-01 09:02:10 -0500 (Thu, 01 Oct 2009) New Revision: 3116 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/clstnodes.swift Log: doing only for node numbers Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/clstnodes.swift =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/clstnodes.swift (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/clstnodes.swift 2009-10-01 14:02:10 UTC (rev 3116) @@ -0,0 +1,63 @@ +##-- coded: 13.Sept.2009 +type file{} +type Rscript; + +## generate permutation brain +app (file permOUT) permJob (Rscript code){ + RInvoke @code @filename(permOUT); +} + +## smooth the permutation brain +app (file surfsmoothOutfile) SurfSmooth (file spec, file smoothwm, file permOUT){ + SurfSmooth "-spec" @spec "-surf_A" @smoothwm "-met" "HEAT_07" "-input" @permOUT + "-fwhm" 4 "-Niter" "-1" "-add_index" "-output" @filename(surfsmoothOutfile); +} + +## cluster the smoothed permutation brain +app (file surfclustOutput) SurfClust (file spec, file pial, file surfsmoothOutfile, float thresh, float rmm, string outPrefix){ + SurfClust "-spec" @spec "-surf_A" @pial "-input" @surfsmoothOutfile "1" "-rmm" rmm "-thresh_col" "1" + "-thresh" thresh "-amm2" "2" "-sort_n_nodes" "-prepend_node_index" "-prefix" outPrefix; +} + +## calculate the cluster mass and keep the biggest one +app (file clusterMassOUT) ClusterMass (file ClstrMassScript, file surfclustOutput){ + python @ClstrMassScript "--input" @filename(surfclustOutput) "--outputname" @filename(clusterMassOUT) "--procedure" "PermNumberNodes"; +} + +(file permOUT, file surfsmoothOutfile, file surfclustOutput, file clusterMassOUT) PermClusterMass (Rscript code, file ClstrMassScript, file spec, file smoothwm, file pial, float thresh, float rmm, string outPrefix){ + permOUT = permJob(code); + surfsmoothOutfile = SurfSmooth(spec, smoothwm, permOUT); + surfclustOutput = SurfClust(spec, pial, surfsmoothOutfile, thresh, rmm, outPrefix); + clusterMassOUT = ClusterMass(ClstrMassScript, surfclustOutput); +} + + +Rscript code; +file ClstrMassScript; + +string idss[] = ["test"]; +string way = "oneway"; +int lag = 0; +string h = "lh"; +float vertexThresh[] = [1.96]; +float rmm_vals[] = [4.1]; +int permbrains[] = [1:1000:1]; +#int permbrains[] = [1]; + +foreach id in idss{ + foreach perm in permbrains{ + foreach thresh in vertexThresh{ + foreach rmm in rmm_vals{ + file permOUT; + file spec; + file smoothwm; + file pial; + string outPrefix=@strcat("perm",perm,id,"_",h,"_lag",lag,way,"Thresh",thresh); + file surfsmoothOutfile; + file surfclustOutput; + file clusterMassOUT; + (permOUT, surfsmoothOutfile, surfclustOutput, clusterMassOUT) = PermClusterMass(code, ClstrMassScript, spec, smoothwm, pial, thresh, rmm, outPrefix); + } + } + } +} From noreply at svn.ci.uchicago.edu Tue Oct 6 14:35:21 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 6 Oct 2009 14:35:21 -0500 (CDT) Subject: [Swift-commit] r3117 - trunk/etc Message-ID: <20091006193521.969859CC87@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-06 14:35:21 -0500 (Tue, 06 Oct 2009) New Revision: 3117 Modified: trunk/etc/swift.properties Log: increased foreach threads to 16k Modified: trunk/etc/swift.properties =================================================================== --- trunk/etc/swift.properties 2009-10-01 14:02:10 UTC (rev 3116) +++ trunk/etc/swift.properties 2009-10-06 19:35:21 UTC (rev 3117) @@ -319,7 +319,7 @@ # Default: 1024 # -foreach.max.threads=1024 +foreach.max.threads=16384 # controls whether the log file will contain provenance information # enabling this will increase the size of log files, sometimes From noreply at svn.ci.uchicago.edu Tue Oct 6 14:41:02 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 6 Oct 2009 14:41:02 -0500 (CDT) Subject: [Swift-commit] r3118 - trunk Message-ID: <20091006194102.AB0919CC87@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-06 14:41:02 -0500 (Tue, 06 Oct 2009) New Revision: 3118 Modified: trunk/launchers.xml Log: clean previous scripts (not doing so seems to cause weirdness) Modified: trunk/launchers.xml =================================================================== --- trunk/launchers.xml 2009-10-06 19:35:21 UTC (rev 3117) +++ trunk/launchers.xml 2009-10-06 19:41:02 UTC (rev 3118) @@ -1,5 +1,7 @@ + + From noreply at svn.ci.uchicago.edu Tue Oct 6 14:42:47 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 6 Oct 2009 14:42:47 -0500 (CDT) Subject: [Swift-commit] r3119 - trunk Message-ID: <20091006194247.9BFF29CC87@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-06 14:42:47 -0500 (Tue, 06 Oct 2009) New Revision: 3119 Modified: trunk/build.xml Log: quote when necessary Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2009-10-06 19:41:02 UTC (rev 3118) +++ trunk/build.xml 2009-10-06 19:42:47 UTC (rev 3119) @@ -250,8 +250,8 @@ set LOCALCLASSPATH=%SWIFT_HOME%\etc;%SWIFT_HOME%\libexec - set COG_OPTS= - set COG_OPTS=-DCOG_INSTALL_PATH=%SWIFT_HOME% -Dswift.home=%SWIFT_HOME% -Dvds.home=%SWIFT_HOME% + set OPTS= + set OPTS=-DCOG_INSTALL_PATH="%SWIFT_HOME%" -Dswift.home="%SWIFT_HOME%" -Dvds.home="%SWIFT_HOME%" @@ -285,6 +285,7 @@ + @@ -292,7 +293,7 @@ - + From noreply at svn.ci.uchicago.edu Tue Oct 6 15:23:56 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 6 Oct 2009 15:23:56 -0500 (CDT) Subject: [Swift-commit] r3120 - trunk/tests/language-behaviour Message-ID: <20091006202356.3E8459CC87@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-06 15:23:55 -0500 (Tue, 06 Oct 2009) New Revision: 3120 Added: trunk/tests/language-behaviour/142-space-and-quotes.1.out.expected trunk/tests/language-behaviour/142-space-and-quotes.2.out.expected trunk/tests/language-behaviour/142-space-and-quotes.3.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.1.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.2.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.3.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.4.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.5.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.6.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.7.out.expected trunk/tests/language-behaviour/generate-odd-files Removed: trunk/tests/language-behaviour/142-space-and-quotes. space .out.expected trunk/tests/language-behaviour/142-space-and-quotes.2" space ".out.expected trunk/tests/language-behaviour/142-space-and-quotes.3' space '.out.expected trunk/tests/language-behaviour/1421-space-and-quotes.''' ' ''' '' '''''' ' """"""""" ' ' ' """' "'".out.expected trunk/tests/language-behaviour/1421-space-and-quotes.2 sp"ace .out.expected trunk/tests/language-behaviour/1421-space-and-quotes.2" space .out.expected trunk/tests/language-behaviour/1421-space-and-quotes.2' sp"ac"e .out.expected trunk/tests/language-behaviour/1421-space-and-quotes.3 sp'ace .out.expected trunk/tests/language-behaviour/1421-space-and-quotes.3' sp'ac'e .out.expected trunk/tests/language-behaviour/1421-space-and-quotes.3' space .out.expected Modified: trunk/tests/language-behaviour/run Log: workaround for checking out on windows Deleted: trunk/tests/language-behaviour/142-space-and-quotes. space .out.expected =================================================================== Added: trunk/tests/language-behaviour/142-space-and-quotes.1.out.expected =================================================================== Deleted: trunk/tests/language-behaviour/142-space-and-quotes.2" space ".out.expected =================================================================== Added: trunk/tests/language-behaviour/142-space-and-quotes.2.out.expected =================================================================== Deleted: trunk/tests/language-behaviour/142-space-and-quotes.3' space '.out.expected =================================================================== Added: trunk/tests/language-behaviour/142-space-and-quotes.3.out.expected =================================================================== Deleted: trunk/tests/language-behaviour/1421-space-and-quotes.''' ' ''' '' '''''' ' """"""""" ' ' ' """' "'".out.expected =================================================================== Added: trunk/tests/language-behaviour/1421-space-and-quotes.1.out.expected =================================================================== Deleted: trunk/tests/language-behaviour/1421-space-and-quotes.2 sp"ace .out.expected =================================================================== Deleted: trunk/tests/language-behaviour/1421-space-and-quotes.2" space .out.expected =================================================================== Deleted: trunk/tests/language-behaviour/1421-space-and-quotes.2' sp"ac"e .out.expected =================================================================== Added: trunk/tests/language-behaviour/1421-space-and-quotes.2.out.expected =================================================================== Deleted: trunk/tests/language-behaviour/1421-space-and-quotes.3 sp'ace .out.expected =================================================================== Deleted: trunk/tests/language-behaviour/1421-space-and-quotes.3' sp'ac'e .out.expected =================================================================== Deleted: trunk/tests/language-behaviour/1421-space-and-quotes.3' space .out.expected =================================================================== Added: trunk/tests/language-behaviour/1421-space-and-quotes.3.out.expected =================================================================== Added: trunk/tests/language-behaviour/1421-space-and-quotes.4.out.expected =================================================================== Added: trunk/tests/language-behaviour/1421-space-and-quotes.5.out.expected =================================================================== Added: trunk/tests/language-behaviour/1421-space-and-quotes.6.out.expected =================================================================== Added: trunk/tests/language-behaviour/1421-space-and-quotes.7.out.expected =================================================================== Added: trunk/tests/language-behaviour/generate-odd-files =================================================================== --- trunk/tests/language-behaviour/generate-odd-files (rev 0) +++ trunk/tests/language-behaviour/generate-odd-files 2009-10-06 20:23:55 UTC (rev 3120) @@ -0,0 +1,14 @@ +#!/bin/bash + +cp 142-space-and-quotes.1.out.expected "142-space-and-quotes. space .out.expected" +cp 142-space-and-quotes.2.out.expected "142-space-and-quotes.2\" space \".out.expected" +cp 142-space-and-quotes.3.out.expected "142-space-and-quotes.3' space '.out.expected" + +cp 1421-space-and-quotes.1.out.expected "1421-space-and-quotes.''' ' ''' '' '''''' ' \"\"\"\"\"\"\"\"\" ' ' ' \"\"\"' \"'\".out.expected" +cp 1421-space-and-quotes.2.out.expected "1421-space-and-quotes.2\" space .out.expected" +cp 1421-space-and-quotes.3.out.expected "1421-space-and-quotes.3' space .out.expected" +cp 1421-space-and-quotes.4.out.expected "1421-space-and-quotes.2 sp\"ace .out.expected" +cp 1421-space-and-quotes.5.out.expected "1421-space-and-quotes.3 sp'ace .out.expected" +cp 1421-space-and-quotes.6.out.expected "1421-space-and-quotes.2' sp\"ac\"e .out.expected" +cp 1421-space-and-quotes.7.out.expected "1421-space-and-quotes.3' sp'ac'e .out.expected" + Property changes on: trunk/tests/language-behaviour/generate-odd-files ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/language-behaviour/run =================================================================== --- trunk/tests/language-behaviour/run 2009-10-06 19:42:47 UTC (rev 3119) +++ trunk/tests/language-behaviour/run 2009-10-06 20:23:55 UTC (rev 3120) @@ -1,5 +1,8 @@ #!/bin/bash +#Some of the space-and-quotes output files break checkouts on Windows +./generate-odd-files + # make up a tc.data file cp $(dirname $(which swift))/../etc/tc.data ./tc.data From noreply at svn.ci.uchicago.edu Tue Oct 6 22:46:08 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 6 Oct 2009 22:46:08 -0500 (CDT) Subject: [Swift-commit] r3121 - trunk Message-ID: <20091007034608.3A47A9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-06 22:46:07 -0500 (Tue, 06 Oct 2009) New Revision: 3121 Modified: trunk/build.xml Log: updated to match cog template Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2009-10-06 20:23:55 UTC (rev 3120) +++ trunk/build.xml 2009-10-07 03:46:07 UTC (rev 3121) @@ -246,8 +246,8 @@ SWIFT_HOME - set LOCALCLASSPATH=%SWIFT_HOME%\etc - set LOCALCLASSPATH=%SWIFT_HOME%\etc;%SWIFT_HOME%\libexec + set LOCALCLASSPATH=%CLASSPATH%;%SWIFT_HOME%\etc + set LOCALCLASSPATH=%CLASSPATH%;%SWIFT_HOME%\etc;%SWIFT_HOME%\libexec set OPTS= From noreply at svn.ci.uchicago.edu Thu Oct 8 12:07:09 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 8 Oct 2009 12:07:09 -0500 (CDT) Subject: [Swift-commit] r3122 - in SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis: . chisqperm Message-ID: <20091008170709.3B93A9CC80@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-10-08 12:07:08 -0500 (Thu, 08 Oct 2009) New Revision: 3122 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/clustermassPerms2.swift SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/swift_ex.sh Log: seperate dir for chisq perms Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/clustermassPerms2.swift =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/clustermassPerms2.swift (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/clustermassPerms2.swift 2009-10-08 17:07:08 UTC (rev 3122) @@ -0,0 +1,66 @@ +##-- coded: 13.Sept.2009 +type file{} +type Rscript; + +## generate permutation brain +app (external permOUT) permJob (Rscript code, string permOUTname){ + RInvoke @code permOUTname; +} + +## smooth the permutation brain +app (external surfsmoothOutfile) SurfSmooth (file spec, file smoothwm, external permOUT, string surfsmoothOutfilename){ + SurfSmooth "-spec" @spec "-surf_A" @smoothwm "-met" "HEAT_07" "-input" @permOUT + "-fwhm" 4 "-Niter" "-1" "-add_index" "-output" surfsmoothOutfilename; +} + +## cluster the smoothed permutation brain +app (external surfclustOutput) SurfClust (file spec, file pial, external surfsmoothOutfile, float thresh, float rmm, string outPrefix){ + SurfClust "-spec" @spec "-surf_A" @pial "-input" @surfsmoothOutfile "1" "-rmm" rmm "-thresh_col" "1" + "-thresh" thresh "-amm2" "2" "-sort_n_nodes" "-prepend_node_index" "-prefix" outPrefix; +} + +## calculate the cluster mass and keep the biggest one +app (file clusterMassOUT) ClusterMass (file ClstrMassScript, external surfclustOutput){ + python @ClstrMassScript "--input" @filename(surfclustOutput) "--outputname" @filename(clusterMassOUT) "--procedure" "PermClusterMass"; +} + +(external permOUT, external surfsmoothOutfile, external surfclustOutput, file clusterMassOUT) PermClusterMass (Rscript code, file ClstrMassScript, file spec, file smoothwm, file pial, float thresh, float rmm, string outPrefix, string permOUTname, string surfsmoothOutfilename){ + permOUT = permJob(code, permOUTname); + surfsmoothOutfile = SurfSmooth(spec, smoothwm, permOUT, surfsmoothOutfilename); + surfclustOutput = SurfClust(spec, pial, surfsmoothOutfile, thresh, rmm, outPrefix); + clusterMassOUT = ClusterMass(ClstrMassScript, surfclustOutput); +} + + +Rscript code; +file ClstrMassScript; + +string idss[] = ["pa_vs_noise"]; +string way = "oneway"; +int lag = 0; +string h = "lh"; +float vertexThresh[] = [3.84]; +float rmm_vals[] = [3.1]; +#int permbrains[] = [1:1000:1]; +int permbrains[] = [1]; + +foreach id in idss{ + foreach perm in permbrains{ + foreach thresh in vertexThresh{ + foreach rmm in rmm_vals{ + external permOUT; + string permOUTname=@strcat("perm",perm,".1D"); + file spec; + file smoothwm; + file pial; + string outPrefix=@strcat("perm",perm,id,"_",h,"_lag",lag,way,"Thresh",thresh); + external surfsmoothOutfile; + string surfsmoothOutfilename=@strcat(outPrefix,"smooth.1D.dset"); + external surfclustOutput; + #string surfclustOutputname=@strcat(outPrefix,"_ClstTable_r",rmm,"_a2.0.1D"); + file clusterMassOUT; + (permOUT, surfsmoothOutfile, surfclustOutput, clusterMassOUT) = PermClusterMass(code, ClstrMassScript, spec, smoothwm, pial, thresh, rmm, outPrefix, permOUTname, surfsmoothOutfilename); + } + } + } +} Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/swift_ex.sh =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/swift_ex.sh (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/swift_ex.sh 2009-10-08 17:07:08 UTC (rev 3122) @@ -0,0 +1,3 @@ +#!/bin/tcsh + +swift clstnodes.swift -sites.file /ci/projects/cnari/config/coaster_mercury.xml -user="andric" Property changes on: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/swift_ex.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Thu Oct 8 12:08:47 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 8 Oct 2009 12:08:47 -0500 (CDT) Subject: [Swift-commit] r3123 - SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/scripts Message-ID: <20091008170847.017949CC80@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-10-08 12:08:46 -0500 (Thu, 08 Oct 2009) New Revision: 3123 Modified: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/scripts/generatePerm.R Log: chisq randoms Modified: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/scripts/generatePerm.R =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/scripts/generatePerm.R 2009-10-08 17:07:08 UTC (rev 3122) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/permutations/scripts/generatePerm.R 2009-10-08 17:08:46 UTC (rev 3123) @@ -3,5 +3,5 @@ print(allinputs) outname <- noquote(strsplit(allinputs," ")[[1]][1]) print(outname) -brain <- rchisq(130934,df=1) +brain <- rchisq(130934,df=1 write.table(round(brain,3),outname,row.names=F,col.names=F,quote=F) From noreply at svn.ci.uchicago.edu Fri Oct 9 15:23:17 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 9 Oct 2009 15:23:17 -0500 (CDT) Subject: [Swift-commit] r3124 - SwiftApps/SEE/trunk Message-ID: <20091009202317.A002F9CC67@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-10-09 15:23:16 -0500 (Fri, 09 Oct 2009) New Revision: 3124 Modified: SwiftApps/SEE/trunk/README SwiftApps/SEE/trunk/ampl.swift Log: Documentation on how to run the workflow Modified: SwiftApps/SEE/trunk/README =================================================================== --- SwiftApps/SEE/trunk/README 2009-10-08 17:08:46 UTC (rev 3123) +++ SwiftApps/SEE/trunk/README 2009-10-09 20:23:16 UTC (rev 3124) @@ -1,36 +1,36 @@ -In order to run ampl.swift, there are two lines of code we need to change. -1) Open template with Emacs, VI, or any other editor -The first line is option ampl_include "/work/00946/zzhang/SEE/static/instances/test/@INSTANCE_ID@\" - Chnage the "/work/00946/zzhang" to your working dir. +AMPL Workflow +Prepared by : Allan Espinosa (aespinosa at cs.uchicago.edu) -2) Open tc.data with Emacs, VI, or any other editor -Change the 3rd coloumn last two entries - localhost run_ampl /work/00946/zzhang/SEE/static/run_ampl INSTALLED INTEL32::LINUX null - tgtacc run_ampl /work/00946/zzhang/SEE/static/run_ampl INSTALLED INTEL32::LINUX null -to - localhost run_ampl /YOUR_WORK_DIR/SEE/static/run_ampl INSTALLED INTEL32::LINUX null - tgtacc run_ampl /YOUR_WORK_DIR/SEE/static/run_ampl INSTALLED INTEL32::LINUX null +This document demonstrates how to run and deploy an AMPL workflow on the +Teraport compute resource. -3) Open tgranger.xml with Emacs, VI, or any other editor -Change the following 3 entries accordingly - TG-CCR080022N - /work/00946/zzhang/work - /tmp/zzhang/jobdir +1. Prepare a site to run the workflow. Copy the site_pack directory in the +site. For this example, let us say we deployed it in +/home/USER/ampl-app/site_pack -project is the project ID on ranger -workdirectory is the shared work place for swift jobs -SWIFT_JOB_DIR: if you don't set this, all swift job dirs will be created in workdirectory; - if you set it to /tmp/$USERANME/jobs, swift job dirs will be created on the ramdisk of the compute nodes. - By $USERNAME, I mean the real user name of you on ranger. +2. Update tc.data entries on TERAPORT about the run_ampl program +TERAPORT run_ampl /home/USER/ampl-app/site_pack/run_ampl INSTALLED INTEL32::LINUX null -4) All instances, 100 or 2000 test cases should be in instances/test/. -Now, we have the 2000 test case in instances/test/ -It looks like run1, run2, run3, ..., run2000. +3. Update the sites.xml file as necessary. This repository contains a sample +file for deployment on TERAPORT -To test ampl.swift local -run "swift -tc.file ./tc.data ampl.swift" +4. Stage your data manually to TERAPORT. A dataset contains directories named +run1, run2, run3, etc. For this example, we copy the dataset in +/home/USER/ampl-data containing 3 run directories. -To test ampl.swift on ranger -run "swift -sites.file ./tgranger-sge-gram2.xml -tc.file ./tc.data ampl.swift" +$ls /home/USER/ampl-data +run1, run2, run3 -Good Luck +5. Update ampl.swift line 47 to reflect the size of the dataset. In this case +we set + +int runs[]=[1:3]; + +6. Update line 45 of ampl.swift to reflect the location of your dataset. In +this example se set + +string data_dir = "/home/USER/ampl-data"; + +7. Run the workflow by invoking swift. You can see example runs in runampl.sh + +swift -tc.file tc.data -sites.file sites.xml ampl.swift Modified: SwiftApps/SEE/trunk/ampl.swift =================================================================== --- SwiftApps/SEE/trunk/ampl.swift 2009-10-08 17:08:46 UTC (rev 3123) +++ SwiftApps/SEE/trunk/ampl.swift 2009-10-09 20:23:16 UTC (rev 3124) @@ -41,8 +41,10 @@ } AmplCmd const_cmd ; +/* Update this to reflect the location of your dataset" */ string data_dir = "/panfs/panasas/CMS/data/SEE/rdraws/all"; +/* Update this to reflect the length of your dataset */ int runs[]=[1:5000]; foreach i in runs { From noreply at svn.ci.uchicago.edu Sat Oct 10 20:30:18 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:30:18 -0500 (CDT) Subject: [Swift-commit] r3125 - trunk/libexec Message-ID: <20091011013018.7B5A49CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:30:18 -0500 (Sat, 10 Oct 2009) New Revision: 3125 Modified: trunk/libexec/vdl.k Log: Replaced vds.home with swift.home Modified: trunk/libexec/vdl.k =================================================================== --- trunk/libexec/vdl.k 2009-10-09 20:23:16 UTC (rev 3124) +++ trunk/libexec/vdl.k 2009-10-11 01:30:18 UTC (rev 3125) @@ -14,8 +14,8 @@ import("java.k") once("vdl.k-print-version" - log("info",sys:file:read("{vds.home}/libexec/version.txt")) - echo(sys:file:read("{vds.home}/libexec/version.txt")) + log("info",sys:file:read("{swift.home}/libexec/version.txt")) + echo(sys:file:read("{swift.home}/libexec/version.txt")) log("info","RUNID id=tag:benc at ci.uchicago.edu,2007:swift:run:{VDL:RUNID}") echo("RunID: {VDL:RUNID}") From noreply at svn.ci.uchicago.edu Sat Oct 10 20:39:23 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:39:23 -0500 (CDT) Subject: [Swift-commit] r3126 - trunk/libexec Message-ID: <20091011013923.A88579CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:39:23 -0500 (Sat, 10 Oct 2009) New Revision: 3126 Added: trunk/libexec/_swiftwrap.vbs Modified: trunk/libexec/vdl-int.k Log: added windows wrapper; allow flexible wrappers Added: trunk/libexec/_swiftwrap.vbs =================================================================== --- trunk/libexec/_swiftwrap.vbs (rev 0) +++ trunk/libexec/_swiftwrap.vbs 2009-10-11 01:39:23 UTC (rev 3126) @@ -0,0 +1,370 @@ +ai = 0 +Set args = WScript.Arguments +Dim info, fs +Set fs = WScript.CreateObject("Scripting.FileSystemObject") +Set shell = WScript.CreateObject("WScript.Shell") +MandatoryArg = True +OptionalArg = False + +'WScript.Echo("-ab-:" + prepareOne("ab")) +'WScript.Echo("-a b-:" + prepareOne("a b")) +'WScript.Echo("-a\b-:" + prepareOne("a\b")) +'WScript.Echo("-a""b-:" + prepareOne("a""b")) +'WScript.Echo("-a\""b-:" + prepareOne("a\""b")) +'WScript.Echo("-a\\b-:" + prepareOne("a\\b")) +'WScript.Echo("-a\\""b-:" + prepareOne("a\\""b")) + +Sub fail(message, code) + log message + If STATUSMODE = "files" Then + errf = WFDIR + "\status\" + JOBDIR + "\" + ID + "-error" + log "Error file: " + errf + Set sf = fs.OpenTextFile(errf, 2, True) + sf.WriteLine(message) + sf.Close + WScript.Quit(0) + Else + WScript.Echo(message) + WScript.Quit(code) + End If +End Sub + +Sub openinfo(name) + Set info = fs.OpenTextFile(name, 8, True) +End Sub + +Sub closeinfo + info.close() +End Sub + +Function getArgVal(cls, name) + arr = getArgVals(cls, name) + If UBound(arr) = 0 Then + getArgVal = Empty + Else + getArgVal = RTrim(Join(arr)) + End If +End Function + +Function getArgVals(cls, name) + Dim val(255) + sz = 0 + again = True + Do While again + done = (ai >= args.count) + If Not(done) Then + If Left(args(ai), 1) = "-" Then + done = True + End If + End If + If done Then + If UBound(val) = 0 and cls = MandatoryArg Then + fail "Expected " + name, 253 + End If + again = False + Else + val(sz) = args(ai) + ai = ai + 1 + sz = sz + 1 + End If + Loop + getArgVals = val +End Function + +Function getOptArg() + getOptArg = getArgVal(OptionalArg, "") +End Function + +Function expectArg(name) + nope = (ai >= args.count) + If Not(nope) Then + If args(ai) <> "-" + name Then + nope = True + End If + End If + If nope Then + fail "Expected argument " + name, 252 + Else + ai = ai + 1 + End If +End Function + +Sub logstate(args) + 'todo: timestamp and whatever else the "standard" format + info.WriteLine("Progress " + args) +End Sub + +Sub log(args) + 'todo: timestamp and whatever else the "standard" format + info.WriteLine(args) +End Sub + + +Function deleteIfExists(name) + If fs.FileExists(name) Then + fs.DeleteFile(name) + End If +End Function + +Sub mkdir(f) + If f = "" Then + fail "mkdir called with empty argument", 249 + End If + If Not fs.FolderExists(f) Then + mkdir fs.GetParentFolderName(f) + fs.CreateFolder(f) + End If +End Sub + +Function prepareOne(v) + 'Arguments with spaces must be quoted with a double quote + 'Literal double quotes must be escaped with a backslash + 'Literal backslashes must be escaped (with a backslash) if they appear before a double quote + start = 1 + Do While start < Len(v) and Not start = 0 + start = InStr(start, v, "\") + If start <> 0 Then + v = Left(v, start) + "\" + Right(v, Len(v) - start) + start = start + 2 + End If + Loop + start = 1 + Do While start < Len(v) and Not start = 0 + start = InStr(start, v, """") + If start <> 0 Then + v = Left(v, start - 1) + "\" + Right(v, Len(v) - start + 1) + start = start + 2 + End If + Loop + If Not InStr(v, " ") = 0 Then + v = """" + v + """" + End If + prepareOne = v +End Function + +Function prepareArgs(args) + For i = 0 To UBound(args) + args(i) = prepareOne(args(i)) + Next + prepareArgs = join(args) +End Function + + +WFDIR = fs.GetAbsolutePathName(".") + +openinfo("wrapper.log") + +ID = getArgVal(MandatoryArg, "job id") +expectArg("jobdir") +JOBDIR = getArgVal(MandatoryArg, "job dir") + +mkdir WFDIR + "\info\" + JOBDIR + +closeinfo +deleteIfExists(WFDIR + "\info\" + JOBDIR + "\" + ID + "-info") +openinfo(WFDIR + "\info\" + JOBDIR + "\" + ID + "-info") + +logstate "LOG_START" + +expectArg("e") +EXEC = getArgVal(MandatoryArg, "executable") + +expectArg("out") +STDOUT=getArgVal(MandatoryArg, "stdout") + +expectArg("err") +STDERR=getArgVal(MandatoryArg, "stderr") + +expectArg("i") +STDIN=getOptArg() + +expectArg("d") +DIRS=getOptArg() + +expectArg("if") +INF=getOptArg() + +expectArg("of") +OUTF=getOptArg() + +expectArg("k") +KICKSTART=getOptArg() + +expectArg("status") +STATUSMODE=getArgVal(MandatoryArg, "status") + +expectArg("a") +Dim ARGS +ARGS=getArgVals(OptionalArg, "args") + +If STATUSMODE = "files" Then + mkdir WFDIR + "\status\" + JOBDIR +End If + +'No linking on Windows +COPYNOTLINK=True +DIR=WFDIR + "\jobs\" + JOBDIR + "\" + ID + +Set env = shell.Environment("PROCESS") +If Not env("PATHPREFIX") = "" Then + env("PATH") = env("PATHPREFIX") + ";" + env("PATH") +End If + +If Not env("SWIFT_EXTRA_INFO") = "" Then + log "EXTRAINFO=" + env("SWIFT_EXTRA_INFO") +End If + + +log "DIR=" + DIR +log "EXEC=" + EXEC +log "STDIN=" + STDIN +log "STDOUT=" + STDOUT +log "STDERR=" + STDERR +log "DIRS=" + DIRS +log "INF=" + INF +log "OUTF=" + OUTF +log "STATUSMODE=" + STATUSMODE +log "KICKSTART=" + KICKSTART +log "ARGS=" + Join(ARGS) + +logstate "CREATE_JOBDIR" +log "Jobdir: " + DIR +mkdir DIR +log "Created job directory: " + DIR + +logstate "CREATE_INPUTDIR" + +For Each D in Split(DIRS, "|") + mkdir DIR + "\" + D + log "Created output directory: " + DIR + "\" + D +Next + +logstate "LINK_INPUTS" +For Each L in Split(INF, "|") + fs.CopyFile WFDIR + "\shared\" + L, DIR + "\" + L + log "Copied input: " + WFDIR + "\shared\" + L + " to " + DIR + "\" + L +Next + +logstate "EXECUTE" +shell.CurrentDirectory = DIR + +If LCase(fs.GetAbsolutePathName(EXEC)) <> LCase(EXEC) Then + 'relative name + If Not fs.FileExists(EXEC) Then + found = False + 'search in path + dirs = split(env("PATH"), ";") + For Each d in dirs + If fs.FileExists(d + "\" + EXEC) Then + found = True + Exit For + End If + Next + If Not Found Then + fail "The executable (" + EXEC + ") was not found in the current directory or search path", 252 + End If + End If +Else + If Not fs.FileExists(EXEC) Then + fail "Executable (" + EXEC + ") does not exist", 251 + End If +End If + +If KICKSTART = "" Then + Set min = Nothing + Set mout = Nothing + Set merr = Nothing + If STDIN <> "" Then + Set min = fs.OpenTextFile(STDIN, 1, False) + End If + If STDOUT <> "" Then + Set mout = fs.OpenTextFile(STDOUT, 2, True) + End If + If STDERR <> "" Then + Set merr = fs.OpenTextFile(STDERR, 2, True) + End If + qargs = prepareArgs(ARGS) + log "Cmd: " + prepareOne(EXEC) + " " + qargs + Set p = shell.exec(prepareOne(EXEC) + " " + qargs) + log "Executable started" + + Do While Not p.StdOut.AtEndOfStream and Not p.StdErr.AtEndOfStream and p.Status = 0 + some = False + If Not min Is Nothing Then + l = min.ReadLine + p.StdIn.WriteLine(l) + some = True + End If + If Not p.StdOut.AtEndOfStream Then + l = p.StdOut.ReadLine + If Not mout Is Nothing Then + mout.WriteLine(l) + End If + some = True + End If + If Not p.StdErr.AtEndOfStream Then + l = p.StdErr.ReadLine + If Not merr Is Nothing Then + merr.WriteLine(l) + End If + some = True + End If + WScript.Sleep(100) + Loop + If Not min Is Nothing Then + min.close() + End If + If Not mout Is Nothing Then + mout.close() + End If + If Not merr Is Nothing Then + merr.close() + End If + If p.ExitCode <> 0 Then + fail "Exit code " + p.ExitCode, p.ExitCode + End If +Else + fail "Kickstart is not supported on Windows", 250 +End If + +shell.CurrentDirectory = WFDIR + +log "Moving back to workflow directory " + WFDIR +logstate "EXECUTE_DONE" +log "Job ran successfully" + +MISSING = "" +For Each O in Split(OUTF, "|") + If Not fs.FileExists(DIR + "\" + O) Then + If MISSING = "" Then + MISSING = O + Else + MISSING = MISSING + ", " + O + End If + End If +Next +If Not MISSING = "" Then + fail "The following output files were not created by the application: " + MISSING, 254 +End If + +logstate "MOVING_OUTPUTS" +For Each O in split(OUTF, "|") + fs.MoveFile DIR + "\" + O, "shared\" + O +Next + +logstate "RM_JOBDIR" +fs.DeleteFolder DIR, True + +If STATUSMODE = "files" Then + logstate "TOUCH_SUCCESS" + Set sf = fs.OpenTextFile("status\" + JOBDIR + "\" + ID + "-success", 2, True) + sf.Close +End If + +logstate "END" + +closeinfo + +'ensure we exit with a 0 after a successful exection +WScript.Quit(0) + Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2009-10-11 01:30:18 UTC (rev 3125) +++ trunk/libexec/vdl-int.k 2009-10-11 01:39:23 UTC (rev 3126) @@ -70,7 +70,7 @@ * the presence of the success file and remove it, all * in one operation. It relies on file:remove() throwing * an exception if the file is not there. - */ + */ file:remove("{wfdir}/status/{jobdir}/{jobid}-success", host=rhost) log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") ) @@ -104,8 +104,9 @@ wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" sharedDir := dircat(wfdir, "shared") dir:make(sharedDir, host=rhost) - transfer(srcdir="{vds.home}/libexec/", srcfile="_swiftwrap", destdir=sharedDir, desthost=rhost) - transfer(srcdir="{vds.home}/libexec/", srcfile="_swiftseq", destdir=sharedDir, desthost=rhost) + transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap", destdir=sharedDir, desthost=rhost) + transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap.vbs", destdir=sharedDir, desthost=rhost) + transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftseq", destdir=sharedDir, desthost=rhost) dir:make(dircat(wfdir, "kickstart"), host=rhost) statusMode := configProperty("status.mode",host=rhost) @@ -416,6 +417,7 @@ stderr := try(stderr, "stderr.txt") kickstart := vdl:kickstart(rhost) fileDirs := fileDirs(stagein, stageout) + os := vdl:tcprofile(tr, rhost, fqn="INTERNAL:OS") if(wrapperMode == "files" sequential( @@ -455,20 +457,29 @@ vdl:setprogress("Submitting") if(wrapperMode == "files" - vdl:execute("/bin/bash", - list("shared/_swiftwrap", jobid, "-p", jobdir) - directory=wfdir - redirect=false - host=rhost - vdl:tcprofile(tr, rhost) //this gets various app params from the tc, such as environment, walltime, etc - replicationGroup=replicationGroup - replicationChannel=replicationChannel - jobid=jobid + vdl:execute( + vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreter"), + list( + vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreterOptions"), + dircat("shared", vdl:tcprofile(tr, rhost, fqn="swift:wrapperScript"), os=os), + jobid, "-p", jobdir + ) + directory=wfdir + redirect=false + host=rhost + vdl:tcprofile(tr, rhost) //this gets various app params from the tc, such as environment, walltime, etc + replicationGroup=replicationGroup + replicationChannel=replicationChannel + jobid=jobid ) ) if(wrapperMode == "args" - vdl:execute("/bin/bash", - list("shared/_swiftwrap", jobid, + vdl:execute( + vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreter"), + list( + vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreterOptions"), + dircat("shared", vdl:tcprofile(tr, rhost, fqn="swift:wrapperScript"), os=os), + jobid, "-jobdir", jobdir, "-e", vdl:executable(tr, rhost), "-out", stdout, @@ -538,7 +549,7 @@ outs := transferStandardFiles(rhost, tmpdir, jobid, stdout, stderr) discard(maybe(transferWrapperLog(rhost, wfdir, jobid, jobdir))) - + kickstartRec := if( kickstart == "" "" else( From noreply at svn.ci.uchicago.edu Sat Oct 10 20:42:02 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:42:02 -0500 (CDT) Subject: [Swift-commit] r3127 - trunk Message-ID: <20091011014202.346B99CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:42:01 -0500 (Sat, 10 Oct 2009) New Revision: 3127 Modified: trunk/build.xml Log: removed vds.home Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2009-10-11 01:39:23 UTC (rev 3126) +++ trunk/build.xml 2009-10-11 01:42:01 UTC (rev 3127) @@ -231,7 +231,6 @@ updateOptions "$SWIFT_HOME" "SWIFT_HOME" updateOptions "$SWIFT_HOME" "COG_INSTALL_PATH" -updateOptions "$SWIFT_HOME" "vds.home" updateOptions "$SWIFT_HOME" "swift.home" #Use /dev/urandom instead of /dev/random for seeding RNGs #This will lower the randomness of the seed, but avoid @@ -251,7 +250,7 @@ set OPTS= - set OPTS=-DCOG_INSTALL_PATH="%SWIFT_HOME%" -Dswift.home="%SWIFT_HOME%" -Dvds.home="%SWIFT_HOME%" + set OPTS=-DCOG_INSTALL_PATH="%SWIFT_HOME%" -Dswift.home="%SWIFT_HOME%" From noreply at svn.ci.uchicago.edu Sat Oct 10 20:46:26 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:46:26 -0500 (CDT) Subject: [Swift-commit] r3129 - trunk/src/org/griphyn/vdl/util Message-ID: <20091011014626.94F889CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:46:26 -0500 (Sat, 10 Oct 2009) New Revision: 3129 Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java Log: replaced vds.home with swift.home Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-10-11 01:43:21 UTC (rev 3128) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-10-11 01:46:26 UTC (rev 3129) @@ -22,9 +22,11 @@ public static final String CONFIG_FILE_NAME = "swift.properties"; public static final String[] CONFIG_FILE_SEARCH_PATH = new String[] { - System.getProperty("vds.home") + File.separator + "etc" + File.separator + System.getProperty("swift.home") + File.separator + "etc" + File.separator + CONFIG_FILE_NAME, System.getProperty("user.home") + File.separator + ".swift" + File.separator + + CONFIG_FILE_NAME, + System.getProperty("vds.home") + File.separator + "etc" + File.separator + CONFIG_FILE_NAME }; private static VDL2Config config; From noreply at svn.ci.uchicago.edu Sat Oct 10 20:47:00 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:47:00 -0500 (CDT) Subject: [Swift-commit] r3130 - trunk/src/org/griphyn/common/util Message-ID: <20091011014700.750CC9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:47:00 -0500 (Sat, 10 Oct 2009) New Revision: 3130 Removed: trunk/src/org/griphyn/common/util/Currently.java Log: not used Deleted: trunk/src/org/griphyn/common/util/Currently.java =================================================================== --- trunk/src/org/griphyn/common/util/Currently.java 2009-10-11 01:46:26 UTC (rev 3129) +++ trunk/src/org/griphyn/common/util/Currently.java 2009-10-11 01:47:00 UTC (rev 3130) @@ -1,313 +0,0 @@ -/* - * This file or a portion of this file is licensed under the terms of - * the Globus Toolkit Public License, found in file GTPL, or at - * http://www.globus.org/toolkit/download/license.html. This notice must - * appear in redistributions of this file, with or without modification. - * - * Redistributions of this Software, with or without modification, must - * reproduce the GTPL in: (1) the Software, or (2) the Documentation or - * some other similar material which is provided with the Software (if - * any). - * - * Copyright 1999-2004 University of Chicago and The University of - * Southern California. All rights reserved. - */ - -package org.griphyn.common.util; - -import java.util.*; -import java.util.regex.*; -import java.text.*; - -/** - * Create a common interface to handle obtaining string timestamps. - * The difficult part is to allow for an ISO 8601 date formatting. - * - * @author Jens-S. V?ckler - * @author Yong Zhao - * @version $Revision: 1.4 $ - * - * @see java.util.Date - * @see java.text.SimpleDateFormat - */ -public class Currently { - /** - * This is used to format the time stamp. - */ - private SimpleDateFormat m_formatter = null; - - /** - * Default time format, which is compact and has a timezone - */ - public static final String DEFAULT_FORMAT = "yyyy.MM.dd HH:mm:ss zzz"; - - /** - * Default ctor: Create a new instance with a default formatting - * string for timestamps. - */ - public Currently() - { - m_formatter = new SimpleDateFormat(DEFAULT_FORMAT); - } - - /** - * Ctor: Create a new instance with a formatting string for time stamps. - * @param formatString complies to {@link java.text.SimpleDateFormat}. - */ - public Currently( String formatString ) - { - m_formatter = new SimpleDateFormat(formatString); - } - - /** - * Ctor: Create a new instance with a formatting string for time stamps. - * @param format is a description of the simple date format to use. - */ - public Currently( SimpleDateFormat format ) - { - m_formatter = (SimpleDateFormat) format.clone(); - } - - /** - * Accessor: Obtains the default timestamp format for all queues. - * - * @return the currently active timestamp prefix format. - * @see #setDateFormat( String ) - * @see #setDateFormat( SimpleDateFormat ) - */ - public SimpleDateFormat getDateFormat() - { - return m_formatter; - } - - /** - * Accessor: Sets the default timestamp format for all queues. - * - * @param format is the new timestamp prefix format. - * @see #setDateFormat( SimpleDateFormat ) - * @see #getDateFormat() - */ - public void setDateFormat( SimpleDateFormat format ) - { - if ( format != null ) - m_formatter = format; - } - - /** - * Accessor: Sets the default timestamp format for all queues. - * - * @param format is the new timestamp prefix format as a string. - * @see #setDateFormat( String ) - * @see #getDateFormat() - */ - public void setDateFormat( String format ) - { - if ( format != null ) - m_formatter = new SimpleDateFormat(format); - } - - /** - * Obtains the current time as formatted string according to - * the format option. - * @return the current time as formatted string. - * @see #now( Date ) - */ - public String now() - { - return this.now( new Date() ); - } - - /** - * Obtains the current time as formatted string according to - * the format option. - * @param then is a timestamp expressed as Date. - * @return the current time as formatted string. - * @see #now() - */ - public String now( Date then ) - { - return this.m_formatter.format( then ); - } - - /** - * Store the regular expressions necessary to parse ISO timestamps. - */ - private static final String c_expression[] = { - "([12][0-9]{3})-([01][0-9])-([0123][0-9])(T([012][0-9]):([0-6][0-9]):([0-6][0-9])?(\\.[0-9]+)?)?(Z|[-+][01][0-9]:?[0-6][0-9])?", - "([12][0-9]{3})([01][0-9])([0123][0-9])(T?([012][0-9])([0-6][0-9])([0-6][0-9])?(\\.[0-9]+)?)?(Z|[-+][01][0-9]:?[0-6][0-9])?" - }; - - /** - * Stores compiled patterns at first use, quasi-Singleton. - */ - private static Pattern c_pattern[] = null; - - /** - * Parses one of the ISO 8601 that it produces. Note, it will not - * parse the full range of ISO timestamps. - * - * @param stamp is the textual timestamp representation. - * @return a time or null, if unparsable. - */ - public static Date parse( String stamp ) - { - // initialize the compiled expressions once - if ( c_pattern == null ) { - c_pattern = new Pattern[ c_expression.length ]; - for ( int i=0; i 0 ) { - c.set( Calendar.HOUR, Integer.parseInt(m.group(5)) ); - c.set( Calendar.MINUTE, Integer.parseInt(m.group(6)) ); - if ( m.group(7) != null && m.group(7).length() > 0 ) - c.set( Calendar.SECOND, Integer.parseInt(m.group(7)) ); - if ( m.group(8) != null && m.group(8).length() > 1 ) { - String millis = m.group(8).substring(1); - while ( millis.length() < 3 ) millis += "0"; - millis = millis.substring(0,3); - c.set( Calendar.MILLISECOND, Integer.parseInt(millis) ); - } - } - - TimeZone z = TimeZone.getDefault(); - if ( m.group(9) != null && m.group(9).length() > 0 ) { - boolean utc = ( Character.toUpperCase( m.group(9).charAt(0) ) == 'Z' ); - if ( ! utc ) z = TimeZone.getTimeZone( "GMT+0" ); - else z = TimeZone.getTimeZone( "GMT" + m.group(9) ); - } - c.setTimeZone(z); - - return c.getTime(); - } - } - - // not found - return null; - } - - /** - * Ignores any internal date format, and tries to show a complete - * date/timp stamp of the current time in extended ISO 8601 format. - * UTC time (Zulu time) or a local timezone will be used. A sample for - * UTC output is 2002-04-23T02:49:58Z A sample for local zone - * (CDT) is 2002-04-22T21:49:58-05:00 - * - * @param zuluTime returns a UTC formatted stamp, if true. Otherwise - * the time will be formatted according to the local zone. - * @return an ISO 8601 formatted date and time representation for the - * current time in extended format without millisecond resolution - * @see #iso8601( boolean, boolean, boolean, Date ) - */ - public static String iso8601( boolean zuluTime ) - { - return Currently.iso8601( zuluTime, true, false, new Date() ); - } - - /** - * Ignores any internal date format, and tries to show a complete - * date/timp stamp in extended ISO 8601 format. UTC time (Zulu time) - * or a local timezone will be used.

- * - * - * - * - * - * - * - * - * - * - * - *
zoneformatfractionexample
localbasicintegral20020523T140427-0500
UTCbasicintegral20020523190427Z
localextd.integral2002-05-23T14:04:27-05:00
UTCextd.integral2002-05-23T19:04:27Z
localbasicmillis20020523T140427.166-0500
UTCbasicmillis20020523190427.166Z
localextd.millis2002-05-23T14:04:27.166-05:00
UTCextd.millis2002-05-23T19:04:27.166Z

- * - * @param zuluTime returns a UTC formatted stamp, if true. Otherwise - * the time will be formatted according to the local zone. Local time - * should be prefixed with the 'T'. - * @param extendedFormat will use the extended ISO 8601 format which - * separates the different timestamp items. If false, the basic - * format will be used. In UTC and basic format, the 'T' separator - * will be omitted. - * @param withMillis will put the millisecond extension into the timestamp. - * If false, the time will be without millisecond fraction. The separator - * is taken from {@link java.text.DecimalFormatSymbols#getMinusSign()}, - * which usually is a period or a comma. - * @param now is a time stamp as Date. - * @return an ISO 8601 formatted date and time representation for - * the given point in time. - */ - public static String iso8601( boolean zuluTime, boolean extendedFormat, - boolean withMillis, Date now ) - { - Calendar c = - Calendar.getInstance( zuluTime ? - TimeZone.getTimeZone("UTC") : - TimeZone.getDefault() ); - c.setTime( now ); - - // set up formattting options - DecimalFormat nf2 = new DecimalFormat("##00"); - DecimalFormat nf3 = new DecimalFormat("###000"); - DecimalFormat nf4 = new DecimalFormat("####0000"); - DecimalFormatSymbols dfs = nf2.getDecimalFormatSymbols(); - - // allocate result string buffer - int size = extendedFormat ? (zuluTime ? 25 : 30) : (zuluTime ? 21 : 25); - if ( ! withMillis ) size -= 4; - StringBuffer result = new StringBuffer(size); - - result.append( nf4.format(c.get(Calendar.YEAR)) ); - if ( extendedFormat ) result.append('-'); // position 5 - result.append( nf2.format(c.get(Calendar.MONTH)+1) ); - if ( extendedFormat ) result.append('-'); // position 8 - result.append( nf2.format(c.get(Calendar.DAY_OF_MONTH)) ); - // generating UTC in basic format may leave out the 'T' separator - if ( extendedFormat || ! zuluTime ) result.append('T'); // position 11 - result.append( nf2.format(c.get(Calendar.HOUR_OF_DAY)) ); - if ( extendedFormat ) result.append(':'); // position 14 - result.append( nf2.format(c.get(Calendar.MINUTE)) ); - if ( extendedFormat ) result.append(':'); // position 17 - result.append( nf2.format(c.get(Calendar.SECOND)) ); - - if ( withMillis ) { - // Though there is no explicit spec which allows a complete - // timestamp with milliseconds, it is implied through two - // levels, sigh. 5.3.4.2 allows decimal fractions with - // time-only stamps that have a timezone. The intro of 5.4.2 - // allows 5.3.1.3. - result.append( dfs.getDecimalSeparator() ); // position 20 - result.append( nf3.format(c.get(Calendar.MILLISECOND)) ); - } - - if ( zuluTime ) { - // this is easy - result.append('Z'); - } else { - // time zone calculations - int zone_offset = c.get(Calendar.ZONE_OFFSET)/1000; - int save_offset = c.get(Calendar.DST_OFFSET)/1000; - int diff = (zone_offset + save_offset) / 60; - result.append( diff < 0 ? dfs.getMinusSign() : '+' ); // position 24 - - if ( diff < 0 ) diff = Math.abs(diff); - result.append( nf2.format( diff / 60 ) ); - if ( extendedFormat ) result.append( ':' ); - result.append( nf2.format( diff % 60 ) ); - } - - return result.toString(); - } - -} - From noreply at svn.ci.uchicago.edu Sat Oct 10 20:47:35 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:47:35 -0500 (CDT) Subject: [Swift-commit] r3131 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20091011014735.98D6F9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:47:35 -0500 (Sat, 10 Oct 2009) New Revision: 3131 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: changed vds.home to swift.home Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-10-11 01:47:00 UTC (rev 3130) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-10-11 01:47:35 UTC (rev 3131) @@ -168,8 +168,8 @@ else { stack.setGlobal(CONST_VDL_OPERATION, VDL_OPERATION_RUN); } - // TODO should not be referring to vds.home any more - stack.setGlobal("vds.home", System.getProperty("vds.home")); + stack.setGlobal("swift.home", System.getProperty("swift.home")); + stack.setGlobal("PATH_SEPARATOR", File.separator); List arguments = ap.getArguments(); if (ap.hasValue(ARG_RESUME)) { From noreply at svn.ci.uchicago.edu Sat Oct 10 20:48:37 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:48:37 -0500 (CDT) Subject: [Swift-commit] r3132 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20091011014837.3983D9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:48:37 -0500 (Sat, 10 Oct 2009) New Revision: 3132 Modified: trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java Log: use backslash if remote site is windows Modified: trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2009-10-11 01:47:35 UTC (rev 3131) +++ trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2009-10-11 01:48:37 UTC (rev 3132) @@ -9,6 +9,7 @@ import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; +import org.griphyn.common.classes.Os; import org.griphyn.vdl.mapping.AbsFile; public class PathUtils extends FunctionsCollection { @@ -67,25 +68,41 @@ } public static final Arg DIR = new Arg.Positional("dir"); + public static final Arg OS = new Arg.Optional("os"); + static { - setArguments("vdl_dircat", new Arg[] { DIR, PATH }); + setArguments("vdl_dircat", new Arg[] { DIR, PATH, OS }); } public String vdl_dircat(VariableStack stack) throws ExecutionException { String dir = TypeUtil.toString(DIR.getValue(stack)); String path = TypeUtil.toString(PATH.getValue(stack)); + boolean windows = false; + if (OS.isPresent(stack)) { + Os os = (Os) OS.getValue(stack); + windows = Os.WINDOWS.equals(os); + } if (dir.equals("")) { - return path; + return windowsify(path, windows); } else if (dir.endsWith("/")) { - return dir + path; + return windowsify(dir + path, windows); } else { - return dir + '/' + path; + return windowsify(dir + '/' + path, windows); } } - public static final Arg FILES = new Arg.Positional("files"); + private String windowsify(String path, boolean windows) { + if (windows) { + return path.replace('/', '\\'); + } + else { + return path; + } + } + + public static final Arg FILES = new Arg.Positional("files"); static { setArguments("vdl_pathnames", new Arg[] { FILES }); } From noreply at svn.ci.uchicago.edu Sat Oct 10 20:50:21 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:50:21 -0500 (CDT) Subject: [Swift-commit] r3133 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20091011015021.3C8589CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:50:20 -0500 (Sat, 10 Oct 2009) New Revision: 3133 Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java Log: a bit of optimization when handling profiles; added wrapper attributes with os-dependent defaults Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2009-10-11 01:48:37 UTC (rev 3132) +++ trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2009-10-11 01:50:20 UTC (rev 3133) @@ -10,6 +10,7 @@ import java.util.Map; import java.util.Set; +import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.execution.WallTime; import org.globus.cog.karajan.arguments.Arg; import org.globus.cog.karajan.arguments.ArgUtil; @@ -21,15 +22,19 @@ import org.globus.cog.karajan.workflow.nodes.grid.GridExec; import org.griphyn.cPlanner.classes.Profile; import org.griphyn.common.catalog.TransformationCatalogEntry; +import org.griphyn.common.classes.Os; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.util.FQN; public class TCProfile extends VDLFunction { + public static final Logger logger = Logger.getLogger(TCProfile.class); + public static final Arg PA_TR = new Arg.Positional("tr"); public static final Arg PA_HOST = new Arg.Positional("host"); + public static final Arg OA_FQN = new Arg.Optional("fqn"); static { - setArguments(TCProfile.class, new Arg[] { PA_TR, PA_HOST }); + setArguments(TCProfile.class, new Arg[] { PA_TR, PA_HOST, OA_FQN }); } private static Map PROFILE_T; @@ -51,11 +56,14 @@ TCCache tc = getTC(stack); String tr = TypeUtil.toString(PA_TR.getValue(stack)); BoundContact bc = (BoundContact) PA_HOST.getValue(stack); + if (OA_FQN.isPresent(stack)) { + return getSingle(tc, tr, bc, new FQN(TypeUtil.toString(OA_FQN.getValue(stack)))); + } NamedArguments named = ArgUtil.getNamedReturn(stack); Map attrs = null; - attrs = attributesFromHost(bc, attrs); + attrs = attributesFromHost(bc, attrs, named); TransformationCatalogEntry tce = getTCE(tc, new FQN(tr), bc); @@ -67,17 +75,102 @@ named.add(GridExec.A_ENVIRONMENT, env); } - attrs = attributesFromTC(tce, attrs); + attrs = attributesFromTC(tce, attrs, named); } - checkWalltime(tr, attrs); + checkWalltime(tr, named); addAttributes(named, attrs); return null; } - private void checkWalltime(String tr, Map attrs) { + public static final FQN SWIFT_WRAPPER_INTERPRETER = new FQN("swift:wrapperInterpreter"); + public static final FQN SWIFT_WRAPPER_INTERPRETER_OPTIONS = new FQN("swift:wrapperInterpreterOptions"); + public static final FQN SWIFT_WRAPPER_SCRIPT = new FQN("swift:wrapperScript"); + public static final FQN INTERNAL_OS = new FQN("INTERNAL:OS"); + + private Object getSingle(TCCache tc, String tr, BoundContact bc, FQN fqn) { + TransformationCatalogEntry tce = getTCE(tc, new FQN(tr), bc); + String value = getProfile(tce, fqn); + if (value == null) { + value = getProfile(bc, fqn); + } + if (value == null) { + if (SWIFT_WRAPPER_INTERPRETER.equals(fqn)) { + if (tce.getSysInfo().getOs().equals(Os.WINDOWS)) { + return "cscript.exe"; + } + else { + return "/bin/bash"; + } + } + else if (SWIFT_WRAPPER_SCRIPT.equals(fqn)) { + if (tce.getSysInfo().getOs().equals(Os.WINDOWS)) { + return "_swiftwrap.vbs"; + } + else { + return "_swiftwrap"; + } + } + else if (SWIFT_WRAPPER_INTERPRETER_OPTIONS.equals(fqn)) { + if (tce.getSysInfo().getOs().equals(Os.WINDOWS)) { + return new String[] {"//Nologo"}; + } + else { + return null; + } + } + else if (INTERNAL_OS.equals(fqn)) { + Os os = tce.getSysInfo().getOs(); + if (os == null) { + return Os.LINUX; + } + else { + return os; + } + } + } + return value; + } + + private String getProfile(BoundContact bc, FQN fqn) { + Object o = bc.getProperty(fqn.toString()); + if (o == null) { + return null; + } + else { + return o.toString(); + } + } + + private String getProfile(TransformationCatalogEntry tce, FQN fqn) { + List profiles = tce.getProfiles(); + if (profiles == null) { + return null; + } + Iterator i = profiles.iterator(); + while (i.hasNext()) { + Profile p = (Profile) i.next(); + if (eq(p.getProfileNamespace(), fqn.getNamespace()) && eq(p.getProfileKey(), fqn.getName())) { + return p.getProfileValue(); + } + } + return null; + } + + private boolean eq(Object o1, Object o2) { + if (o1 == null) { + return o2 == null; + } + else { + return o1.equals(o2); + } + } + + private void checkWalltime(String tr, NamedArguments attrs) { Object walltime = null; if (attrs != null) { - walltime = attrs.get("maxwalltime"); + if (attrs.hasArgument("maxwalltime")) { + walltime = attrs.getArgument("maxwalltime"); + } } if (walltime == null) { return; @@ -130,40 +223,37 @@ } private void addAttributes(NamedArguments named, Map attrs) { + if (logger.isInfoEnabled()) { + logger.info("Attributes: " + attrs); + } if (attrs == null || attrs.size() == 0) { return; } - Iterator i = attrs.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - Arg a = (Arg) PROFILE_T.get(e.getKey()); - if (a != null) { - named.add(a, e.getValue()); - i.remove(); - } - } - if (attrs.size() == 0) { - return; - } named.add(GridExec.A_ATTRIBUTES, attrs); } - private Map attributesFromTC(TransformationCatalogEntry tce, Map attrs) { + private Map attributesFromTC(TransformationCatalogEntry tce, Map attrs, NamedArguments named) { List l = tce.getProfiles(Profile.GLOBUS); if (l != null) { Iterator i = l.iterator(); while (i.hasNext()) { Profile p = (Profile) i.next(); - if (attrs == null) { - attrs = new HashMap(); + Arg a = (Arg) PROFILE_T.get(p.getProfileKey()); + if (a == null) { + if (attrs == null) { + attrs = new HashMap(); + } + attrs.put(p.getProfileKey(), p.getProfileValue()); } - attrs.put(p.getProfileKey(), p.getProfileValue()); + else { + named.add(a, p.getProfileValue()); + } } } return attrs; } - private Map attributesFromHost(BoundContact bc, Map attrs) { + private Map attributesFromHost(BoundContact bc, Map attrs, NamedArguments named) { Map props = bc.getProperties(); if (props != null) { Iterator i = props.entrySet().iterator(); @@ -171,10 +261,16 @@ Map.Entry e = (Map.Entry) i.next(); FQN fqn = new FQN((String) e.getKey()); if (Profile.GLOBUS.equalsIgnoreCase(fqn.getNamespace())) { - if (attrs == null) { - attrs = new HashMap(); + Arg a = (Arg) PROFILE_T.get(fqn.getName()); + if (a == null) { + if (attrs == null) { + attrs = new HashMap(); + } + attrs.put(fqn.getName(), e.getValue()); } - attrs.put(fqn.getName(), e.getValue()); + else { + named.add(a, e.getValue()); + } } } } From noreply at svn.ci.uchicago.edu Sat Oct 10 20:54:15 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:54:15 -0500 (CDT) Subject: [Swift-commit] r3134 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20091011015415.A28129CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:54:15 -0500 (Sat, 10 Oct 2009) New Revision: 3134 Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java Log: check for info being enabled before doing expensive string concatenations - for those who want to reduce data logging verbosity Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-10-11 01:50:20 UTC (rev 3133) +++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-10-11 01:54:15 UTC (rev 3134) @@ -394,7 +394,9 @@ } this.closed = true; notifyListeners(); - logger.info("closed " + this.getIdentifyingString()); + if (logger.isInfoEnabled()) { + logger.info("closed " + this.getIdentifyingString()); + } // so because its closed, we can dump the contents try { @@ -425,11 +427,13 @@ String identifier = this.getIdentifier(); Path pathFromRoot = this.getPathFromRoot(); if (this.getPathFromRoot() != null) { - logger.info("ROOTPATH dataset=" + identifier + " path=" - + pathFromRoot); - if (this.getType().isPrimitive()) { - logger.info("VALUE dataset=" + identifier + " VALUE=" - + this.toString()); + if (logger.isInfoEnabled()) { + logger.info("ROOTPATH dataset=" + identifier + " path=" + + pathFromRoot); + if (this.getType().isPrimitive()) { + logger.info("VALUE dataset=" + identifier + " VALUE=" + + this.toString()); + } } Mapper m; @@ -468,12 +472,16 @@ try { if(filemapped) { Object path = m.map(pathFromRoot); - logger.info("FILENAME dataset=" + identifier + " filename=" - + path); + if (logger.isInfoEnabled()) { + logger.info("FILENAME dataset=" + identifier + " filename=" + + path); + } } } catch (Exception e) { - logger.info("NOFILENAME dataset=" + identifier); + if (logger.isInfoEnabled()) { + logger.info("NOFILENAME dataset=" + identifier); + } } } } @@ -483,8 +491,10 @@ while (i.hasNext()) { Map.Entry e = (Map.Entry) i.next(); AbstractDataNode node = (AbstractDataNode) e.getValue(); - logger.info("CONTAINMENT parent=" + identifier + " child=" - + node.getIdentifier()); + if (logger.isInfoEnabled()) { + logger.info("CONTAINMENT parent=" + identifier + " child=" + + node.getIdentifier()); + } node.logContent(); } } From noreply at svn.ci.uchicago.edu Sat Oct 10 20:55:21 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:55:21 -0500 (CDT) Subject: [Swift-commit] r3135 - trunk Message-ID: <20091011015521.3CA3B9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:55:21 -0500 (Sat, 10 Oct 2009) New Revision: 3135 Modified: trunk/project.properties Log: removed dependency on vds Modified: trunk/project.properties =================================================================== --- trunk/project.properties 2009-10-11 01:54:15 UTC (rev 3134) +++ trunk/project.properties 2009-10-11 01:55:21 UTC (rev 3135) @@ -8,4 +8,4 @@ # stable branch should be a change of version number) version = svn project = Swift workflow system -lib.deps = gvds.*, antlr*.*, jsr173_1.0_api.*, resolver.*, stringtemplate.*, vdldefinitions.*, xbean.*, xbean_xpath.*, jug*.*, jaxrpc*.*, jline*.jar +lib.deps = antlr*.*, jsr173_1.0_api.*, resolver.*, stringtemplate.*, vdldefinitions.*, xbean.*, xbean_xpath.*, jug*.*, jaxrpc*.*, jline*.jar From noreply at svn.ci.uchicago.edu Sat Oct 10 20:55:46 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:55:46 -0500 (CDT) Subject: [Swift-commit] r3136 - trunk/lib Message-ID: <20091011015546.6A9F29CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:55:46 -0500 (Sat, 10 Oct 2009) New Revision: 3136 Removed: trunk/lib/gvds.jar Log: removed vds jar Deleted: trunk/lib/gvds.jar =================================================================== (Binary files differ) From noreply at svn.ci.uchicago.edu Sat Oct 10 20:58:09 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Oct 2009 20:58:09 -0500 (CDT) Subject: [Swift-commit] r3137 - trunk/libexec Message-ID: <20091011015809.43AFF9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-10 20:58:08 -0500 (Sat, 10 Oct 2009) New Revision: 3137 Modified: trunk/libexec/_swiftwrap.vbs Log: removed testing stuff Modified: trunk/libexec/_swiftwrap.vbs =================================================================== --- trunk/libexec/_swiftwrap.vbs 2009-10-11 01:55:46 UTC (rev 3136) +++ trunk/libexec/_swiftwrap.vbs 2009-10-11 01:58:08 UTC (rev 3137) @@ -6,13 +6,6 @@ MandatoryArg = True OptionalArg = False -'WScript.Echo("-ab-:" + prepareOne("ab")) -'WScript.Echo("-a b-:" + prepareOne("a b")) -'WScript.Echo("-a\b-:" + prepareOne("a\b")) -'WScript.Echo("-a""b-:" + prepareOne("a""b")) -'WScript.Echo("-a\""b-:" + prepareOne("a\""b")) -'WScript.Echo("-a\\b-:" + prepareOne("a\\b")) -'WScript.Echo("-a\\""b-:" + prepareOne("a\\""b")) Sub fail(message, code) log message From noreply at svn.ci.uchicago.edu Sun Oct 11 14:15:47 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:15:47 -0500 (CDT) Subject: [Swift-commit] r3138 - trunk/libexec Message-ID: <20091011191547.7A7DD9CC99@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:15:47 -0500 (Sun, 11 Oct 2009) New Revision: 3138 Modified: trunk/libexec/_swiftwrap.vbs Log: fixed broken loop and stream redirection Modified: trunk/libexec/_swiftwrap.vbs =================================================================== --- trunk/libexec/_swiftwrap.vbs 2009-10-11 01:58:08 UTC (rev 3137) +++ trunk/libexec/_swiftwrap.vbs 2009-10-11 19:15:47 UTC (rev 3138) @@ -281,24 +281,24 @@ Set p = shell.exec(prepareOne(EXEC) + " " + qargs) log "Executable started" - Do While Not p.StdOut.AtEndOfStream and Not p.StdErr.AtEndOfStream and p.Status = 0 + Do Until p.StdOut.AtEndOfStream and p.StdErr.AtEndOfStream and p.Status <> 0 some = False If Not min Is Nothing Then l = min.ReadLine - p.StdIn.WriteLine(l) + p.StdIn.Write(l) some = True End If If Not p.StdOut.AtEndOfStream Then l = p.StdOut.ReadLine If Not mout Is Nothing Then - mout.WriteLine(l) + mout.Write(l) End If some = True End If If Not p.StdErr.AtEndOfStream Then l = p.StdErr.ReadLine If Not merr Is Nothing Then - merr.WriteLine(l) + merr.Write(l) End If some = True End If @@ -314,7 +314,7 @@ merr.close() End If If p.ExitCode <> 0 Then - fail "Exit code " + p.ExitCode, p.ExitCode + fail "Exit code " + CStr(p.ExitCode), p.ExitCode End If Else fail "Kickstart is not supported on Windows", 250 From noreply at svn.ci.uchicago.edu Sun Oct 11 14:16:29 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:16:29 -0500 (CDT) Subject: [Swift-commit] r3139 - trunk/libexec Message-ID: <20091011191629.9A19F9CC99@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:16:29 -0500 (Sun, 11 Oct 2009) New Revision: 3139 Modified: trunk/libexec/vdl-lib.xml Log: Added siteprofile element Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2009-10-11 19:15:47 UTC (rev 3138) +++ trunk/libexec/vdl-lib.xml 2009-10-11 19:16:29 UTC (rev 3139) @@ -65,6 +65,7 @@ + From noreply at svn.ci.uchicago.edu Sun Oct 11 14:16:57 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:16:57 -0500 (CDT) Subject: [Swift-commit] r3140 - trunk/libexec Message-ID: <20091011191657.4A7E99CC99@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:16:57 -0500 (Sun, 11 Oct 2009) New Revision: 3140 Modified: trunk/libexec/vdl-sc.k Log: Pass sysinfo profile if specified Modified: trunk/libexec/vdl-sc.k =================================================================== --- trunk/libexec/vdl-sc.k 2009-10-11 19:16:29 UTC (rev 3139) +++ trunk/libexec/vdl-sc.k 2009-10-11 19:16:57 UTC (rev 3140) @@ -12,6 +12,7 @@ to(properties each(properties) maybe(property("gridlaunch", gridlaunch)) + maybe(property("sysinfo", sysinfo)) ) ) ) @@ -31,7 +32,7 @@ major == "4" "GT4" major == "2" "GT2" throw("Unknown job manager version: major = {major}, minor = {minor}, patch = {patch}, url = {url}") - ) + ) service(type="execution", provider=provider, url=url) ) From noreply at svn.ci.uchicago.edu Sun Oct 11 14:19:25 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:19:25 -0500 (CDT) Subject: [Swift-commit] r3141 - trunk/libexec Message-ID: <20091011191925.3E32D9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:19:24 -0500 (Sun, 11 Oct 2009) New Revision: 3141 Modified: trunk/libexec/vdl-int.k Log: Changed to use vdl:siteprofile Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2009-10-11 19:16:57 UTC (rev 3140) +++ trunk/libexec/vdl-int.k 2009-10-11 19:19:24 UTC (rev 3141) @@ -183,7 +183,13 @@ element(cleanup, [dir, host] log(LOG:INFO, "START dir={dir} host={host}") if(vdl:configProperty("sitedir.keep") == "false" - task:execute("/bin/rm", arguments="-rf {dir}", host=host, batch=true) + task:execute( + vdl:siteprofile(host, "swift:cleanupCommand"), + arguments=list( + vdl:siteprofile(host, "swift:cleanupCommandOptions"), + dir + ) + host=host, batch=true) ) log(LOG:INFO, "END dir={dir} host={host}") ) @@ -417,7 +423,7 @@ stderr := try(stderr, "stderr.txt") kickstart := vdl:kickstart(rhost) fileDirs := fileDirs(stagein, stageout) - os := vdl:tcprofile(tr, rhost, fqn="INTERNAL:OS") + os := vdl:siteprofile(rhost, "SYSINFO:OS") if(wrapperMode == "files" sequential( @@ -458,10 +464,10 @@ if(wrapperMode == "files" vdl:execute( - vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreter"), + vdl:siteprofile(rhost, "swift:wrapperInterpreter"), list( - vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreterOptions"), - dircat("shared", vdl:tcprofile(tr, rhost, fqn="swift:wrapperScript"), os=os), + vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), + dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os), jobid, "-p", jobdir ) directory=wfdir @@ -475,10 +481,10 @@ ) if(wrapperMode == "args" vdl:execute( - vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreter"), + vdl:siteprofile(rhost, "swift:wrapperInterpreter"), list( - vdl:tcprofile(tr, rhost, fqn="swift:wrapperInterpreterOptions"), - dircat("shared", vdl:tcprofile(tr, rhost, fqn="swift:wrapperScript"), os=os), + vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), + dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os), jobid, "-jobdir", jobdir, "-e", vdl:executable(tr, rhost), From noreply at svn.ci.uchicago.edu Sun Oct 11 14:23:06 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:23:06 -0500 (CDT) Subject: [Swift-commit] r3142 - trunk/src/org/griphyn Message-ID: <20091011192306.A401B9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:23:06 -0500 (Sun, 11 Oct 2009) New Revision: 3142 Removed: trunk/src/org/griphyn/common/ Log: Moved to different namespace to avoid conflict with vds From noreply at svn.ci.uchicago.edu Sun Oct 11 14:23:39 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:23:39 -0500 (CDT) Subject: [Swift-commit] r3143 - trunk/src/org/griphyn Message-ID: <20091011192339.C980A9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:23:39 -0500 (Sun, 11 Oct 2009) New Revision: 3143 Removed: trunk/src/org/griphyn/cPlanner/ Log: Moved to different package From noreply at svn.ci.uchicago.edu Sun Oct 11 14:27:34 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:27:34 -0500 (CDT) Subject: [Swift-commit] r3145 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20091011192734.00EC29CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:27:33 -0500 (Sun, 11 Oct 2009) New Revision: 3145 Modified: trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java Log: Fixed imports Modified: trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2009-10-11 19:26:42 UTC (rev 3144) +++ trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2009-10-11 19:27:33 UTC (rev 3145) @@ -19,8 +19,8 @@ import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.Contact; import org.globus.cog.karajan.workflow.KarajanRuntimeException; -import org.griphyn.common.catalog.TransformationCatalogEntry; -import org.griphyn.common.classes.TCType; +import org.globus.swift.catalog.TransformationCatalogEntry; +import org.globus.swift.catalog.types.TCType; import org.griphyn.vdl.util.FQN; import org.griphyn.vdl.util.VDL2Config; From noreply at svn.ci.uchicago.edu Sun Oct 11 14:29:02 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:29:02 -0500 (CDT) Subject: [Swift-commit] r3146 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20091011192902.2B1279CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:29:02 -0500 (Sun, 11 Oct 2009) New Revision: 3146 Modified: trunk/src/org/griphyn/vdl/karajan/TCCache.java trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java Log: Fixed imports Modified: trunk/src/org/griphyn/vdl/karajan/TCCache.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/TCCache.java 2009-10-11 19:27:33 UTC (rev 3145) +++ trunk/src/org/griphyn/vdl/karajan/TCCache.java 2009-10-11 19:29:02 UTC (rev 3146) @@ -7,8 +7,8 @@ import java.util.List; import java.util.Map; -import org.griphyn.common.catalog.TransformationCatalog; -import org.griphyn.common.classes.TCType; +import org.globus.swift.catalog.TransformationCatalog; +import org.globus.swift.catalog.types.TCType; import org.griphyn.vdl.util.FQN; public class TCCache { Modified: trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2009-10-11 19:27:33 UTC (rev 3145) +++ trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2009-10-11 19:29:02 UTC (rev 3146) @@ -25,12 +25,12 @@ import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.Contact; import org.globus.cog.karajan.util.TypeUtil; -import org.griphyn.common.catalog.transformation.File; -import org.griphyn.common.classes.TCType; import org.griphyn.vdl.util.FQN; import org.globus.cog.abstraction.impl.common.StatusImpl; import org.globus.cog.abstraction.interfaces.Status; +import org.globus.swift.catalog.transformation.File; +import org.globus.swift.catalog.types.TCType; public class VDSAdaptiveScheduler extends WeightedHostScoreScheduler { From noreply at svn.ci.uchicago.edu Sun Oct 11 14:30:13 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:30:13 -0500 (CDT) Subject: [Swift-commit] r3147 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20091011193013.F02539CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:30:13 -0500 (Sun, 11 Oct 2009) New Revision: 3147 Added: trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java Log: Separated site from tc profile Added: trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java 2009-10-11 19:30:13 UTC (rev 3147) @@ -0,0 +1,136 @@ +/* + * Created on Dec 26, 2006 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.globus.cog.abstraction.impl.common.execution.WallTime; +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.ArgUtil; +import org.globus.cog.karajan.arguments.NamedArguments; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.BoundContact; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.grid.GridExec; +import org.globus.swift.catalog.TransformationCatalogEntry; +import org.globus.swift.catalog.types.Os; +import org.globus.swift.catalog.util.Profile; +import org.griphyn.vdl.karajan.TCCache; +import org.griphyn.vdl.util.FQN; + +public class SiteProfile extends VDLFunction { + public static final Logger logger = Logger.getLogger(SiteProfile.class); + + public static final Arg PA_HOST = new Arg.Positional("host"); + public static final Arg PA_FQN = new Arg.Positional("fqn"); + + static { + setArguments(SiteProfile.class, new Arg[] { PA_HOST, PA_FQN }); + } + + public Object function(VariableStack stack) throws ExecutionException { + BoundContact bc = (BoundContact) PA_HOST.getValue(stack); + return getSingle(bc, new FQN(TypeUtil.toString(PA_FQN.getValue(stack)))); + } + + public static final FQN SWIFT_WRAPPER_INTERPRETER = new FQN("swift:wrapperInterpreter"); + public static final FQN SWIFT_WRAPPER_INTERPRETER_OPTIONS = new FQN("swift:wrapperInterpreterOptions"); + public static final FQN SWIFT_WRAPPER_SCRIPT = new FQN("swift:wrapperScript"); + public static final FQN SWIFT_CLEANUP_COMMAND = new FQN("swift:cleanupCommand"); + public static final FQN SWIFT_CLEANUP_COMMAND_OPTIONS = new FQN("swift:cleanupCommandOptions"); + public static final FQN SYSINFO_OS = new FQN("SYSINFO:OS"); + + private static final Map DEFAULTS; + + private static void addDefault(Os os, FQN fqn, Object value) { + Map osm = (Map) DEFAULTS.get(os); + if (osm == null) { + osm = new HashMap(); + DEFAULTS.put(os, osm); + } + osm.put(fqn, value); + } + + private static boolean hasDefault(Os os, FQN fqn) { + Map osm = (Map) DEFAULTS.get(os); + if (osm == null) { + return false; + } + else { + return osm.containsKey(fqn); + } + } + + private static Object getDefault(Os os, FQN fqn) { + Map osm = (Map) DEFAULTS.get(os); + if (osm == null) { + osm = (Map) DEFAULTS.get(null); + } + return osm.get(fqn); + } + + static { + DEFAULTS = new HashMap(); + addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER, "cscript.exe"); + addDefault(Os.WINDOWS, SWIFT_WRAPPER_SCRIPT, "_swiftwrap.vbs"); + addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER_OPTIONS, new String[] {"//Nologo"}); + addDefault(Os.WINDOWS, SWIFT_CLEANUP_COMMAND, "cmd.exe"); + addDefault(Os.WINDOWS, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"/C", "del", "/Q"}); + addDefault(null, SWIFT_WRAPPER_INTERPRETER, "/bin/bash"); + addDefault(null, SWIFT_WRAPPER_SCRIPT, "_swiftwrap"); + addDefault(null, SWIFT_WRAPPER_INTERPRETER_OPTIONS, null); + addDefault(null, SWIFT_CLEANUP_COMMAND, "/bin/rm"); + addDefault(null, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"-rf"}); + } + + private Object getSingle(BoundContact bc, FQN fqn) { + String value = getProfile(bc, fqn); + if (value == null) { + Os os = getOS(bc); + if ("swift".equals(fqn.getNamespace())) { + return getDefault(os, fqn); + } + else if (SYSINFO_OS.equals(fqn)) { + return os; + } + else { + return null; + } + } + else { + return value; + } + } + + private String getProfile(BoundContact bc, FQN fqn) { + Object o = bc.getProperty(fqn.toString()); + if (o == null) { + return null; + } + else { + return o.toString(); + } + } + + private Os getOS(BoundContact bc) { + Object o = bc.getProperty("sysinfo"); + if (o == null) { + return Os.LINUX; + } + else { + String[] p = o.toString().split("::"); + if (p.length < 2) { + throw new IllegalArgumentException("Invalid sysinfo for " + bc + ": " + o); + } + return Os.fromString(p[1]); + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2009-10-11 19:29:02 UTC (rev 3146) +++ trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2009-10-11 19:30:13 UTC (rev 3147) @@ -20,9 +20,9 @@ import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.grid.GridExec; -import org.griphyn.cPlanner.classes.Profile; -import org.griphyn.common.catalog.TransformationCatalogEntry; -import org.griphyn.common.classes.Os; +import org.globus.swift.catalog.TransformationCatalogEntry; +import org.globus.swift.catalog.types.Os; +import org.globus.swift.catalog.util.Profile; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.util.FQN; @@ -31,10 +31,9 @@ public static final Arg PA_TR = new Arg.Positional("tr"); public static final Arg PA_HOST = new Arg.Positional("host"); - public static final Arg OA_FQN = new Arg.Optional("fqn"); - + static { - setArguments(TCProfile.class, new Arg[] { PA_TR, PA_HOST, OA_FQN }); + setArguments(TCProfile.class, new Arg[] { PA_TR, PA_HOST }); } private static Map PROFILE_T; @@ -56,9 +55,6 @@ TCCache tc = getTC(stack); String tr = TypeUtil.toString(PA_TR.getValue(stack)); BoundContact bc = (BoundContact) PA_HOST.getValue(stack); - if (OA_FQN.isPresent(stack)) { - return getSingle(tc, tr, bc, new FQN(TypeUtil.toString(OA_FQN.getValue(stack)))); - } NamedArguments named = ArgUtil.getNamedReturn(stack); Map attrs = null; @@ -82,89 +78,6 @@ return null; } - public static final FQN SWIFT_WRAPPER_INTERPRETER = new FQN("swift:wrapperInterpreter"); - public static final FQN SWIFT_WRAPPER_INTERPRETER_OPTIONS = new FQN("swift:wrapperInterpreterOptions"); - public static final FQN SWIFT_WRAPPER_SCRIPT = new FQN("swift:wrapperScript"); - public static final FQN INTERNAL_OS = new FQN("INTERNAL:OS"); - - private Object getSingle(TCCache tc, String tr, BoundContact bc, FQN fqn) { - TransformationCatalogEntry tce = getTCE(tc, new FQN(tr), bc); - String value = getProfile(tce, fqn); - if (value == null) { - value = getProfile(bc, fqn); - } - if (value == null) { - if (SWIFT_WRAPPER_INTERPRETER.equals(fqn)) { - if (tce.getSysInfo().getOs().equals(Os.WINDOWS)) { - return "cscript.exe"; - } - else { - return "/bin/bash"; - } - } - else if (SWIFT_WRAPPER_SCRIPT.equals(fqn)) { - if (tce.getSysInfo().getOs().equals(Os.WINDOWS)) { - return "_swiftwrap.vbs"; - } - else { - return "_swiftwrap"; - } - } - else if (SWIFT_WRAPPER_INTERPRETER_OPTIONS.equals(fqn)) { - if (tce.getSysInfo().getOs().equals(Os.WINDOWS)) { - return new String[] {"//Nologo"}; - } - else { - return null; - } - } - else if (INTERNAL_OS.equals(fqn)) { - Os os = tce.getSysInfo().getOs(); - if (os == null) { - return Os.LINUX; - } - else { - return os; - } - } - } - return value; - } - - private String getProfile(BoundContact bc, FQN fqn) { - Object o = bc.getProperty(fqn.toString()); - if (o == null) { - return null; - } - else { - return o.toString(); - } - } - - private String getProfile(TransformationCatalogEntry tce, FQN fqn) { - List profiles = tce.getProfiles(); - if (profiles == null) { - return null; - } - Iterator i = profiles.iterator(); - while (i.hasNext()) { - Profile p = (Profile) i.next(); - if (eq(p.getProfileNamespace(), fqn.getNamespace()) && eq(p.getProfileKey(), fqn.getName())) { - return p.getProfileValue(); - } - } - return null; - } - - private boolean eq(Object o1, Object o2) { - if (o1 == null) { - return o2 == null; - } - else { - return o1.equals(o2); - } - } - private void checkWalltime(String tr, NamedArguments attrs) { Object walltime = null; if (attrs != null) { From noreply at svn.ci.uchicago.edu Sun Oct 11 14:31:33 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:31:33 -0500 (CDT) Subject: [Swift-commit] r3148 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20091011193133.18CD39CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:31:32 -0500 (Sun, 11 Oct 2009) New Revision: 3148 Modified: trunk/src/org/griphyn/vdl/karajan/lib/Executable.java trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: Fixed imports Modified: trunk/src/org/griphyn/vdl/karajan/lib/Executable.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Executable.java 2009-10-11 19:30:13 UTC (rev 3147) +++ trunk/src/org/griphyn/vdl/karajan/lib/Executable.java 2009-10-11 19:31:32 UTC (rev 3148) @@ -8,7 +8,7 @@ import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.common.catalog.TransformationCatalogEntry; +import org.globus.swift.catalog.TransformationCatalogEntry; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.util.FQN; Modified: trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2009-10-11 19:30:13 UTC (rev 3147) +++ trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2009-10-11 19:31:32 UTC (rev 3148) @@ -9,7 +9,7 @@ import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; -import org.griphyn.common.classes.Os; +import org.globus.swift.catalog.types.Os; import org.griphyn.vdl.mapping.AbsFile; public class PathUtils extends FunctionsCollection { Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2009-10-11 19:30:13 UTC (rev 3147) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2009-10-11 19:31:32 UTC (rev 3148) @@ -29,9 +29,9 @@ import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; import org.globus.cog.karajan.workflow.nodes.SequentialWithArguments; import org.globus.cog.karajan.workflow.nodes.restartLog.RestartLog; -import org.griphyn.common.catalog.TransformationCatalogEntry; -import org.griphyn.common.catalog.transformation.File; -import org.griphyn.common.classes.TCType; +import org.globus.swift.catalog.TransformationCatalogEntry; +import org.globus.swift.catalog.transformation.File; +import org.globus.swift.catalog.types.TCType; import org.griphyn.vdl.karajan.InHook; import org.griphyn.vdl.karajan.Monitor; import org.griphyn.vdl.karajan.TCCache; From noreply at svn.ci.uchicago.edu Sun Oct 11 14:48:52 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 14:48:52 -0500 (CDT) Subject: [Swift-commit] r3149 - trunk/libexec Message-ID: <20091011194852.E00219CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 14:48:52 -0500 (Sun, 11 Oct 2009) New Revision: 3149 Modified: trunk/libexec/vdl-int.k Log: Only transfer site specific wrapper script Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2009-10-11 19:31:32 UTC (rev 3148) +++ trunk/libexec/vdl-int.k 2009-10-11 19:48:52 UTC (rev 3149) @@ -104,8 +104,7 @@ wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" sharedDir := dircat(wfdir, "shared") dir:make(sharedDir, host=rhost) - transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap", destdir=sharedDir, desthost=rhost) - transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap.vbs", destdir=sharedDir, desthost=rhost) + transfer(srcdir="{swift.home}/libexec/", vdl:siteprofile(rhost, "swift:wrapperScript"), destdir=sharedDir, desthost=rhost) transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftseq", destdir=sharedDir, desthost=rhost) dir:make(dircat(wfdir, "kickstart"), host=rhost) From noreply at svn.ci.uchicago.edu Sun Oct 11 15:19:42 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 15:19:42 -0500 (CDT) Subject: [Swift-commit] r3150 - trunk/docs Message-ID: <20091011201942.A01159CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 15:19:42 -0500 (Sun, 11 Oct 2009) New Revision: 3150 Modified: trunk/docs/userguide.xml Log: updated user guide with new profiles and some notes for running on windows Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-10-11 19:48:52 UTC (rev 3149) +++ trunk/docs/userguide.xml 2009-10-11 20:19:42 UTC (rev 3150) @@ -3314,6 +3314,31 @@ --> + +

swift namespace + wrapperInterpreter +The wrapper interpreter indicates the command (executable) to be used to run the Swift wrapper +script. The default is "/bin/bash" on Unix sites and "cscript.exe" on Windows sites. + + wrapperInterpreterOptions +Allows specifying additional options to the executable used to run the Swift wrapper. The defaults +are no options on Unix sites and "//Nologo" on Windows sites. + + wrapperScript +Specifies the name of the wrapper script to be used on a site. The defaults are "_swiftwrap" on +Unix sites and "_swiftwrap.vbs" on Windows sites. If you specify a custom wrapper script, it +must be present in the "libexec" directory of the Swift installation. + + cleanupCommand +Indicates the command to be run at the end of a Swift run to clean up the run directories on a +remote site. Defaults are "/bin/rm" on Unix sites and "cmd.exe" on Windows sites + + cleanupCommandOptions +Specifies the options to be passed to the cleanup command above. The options are passed in the +argument list to the cleanup command. After the options, the last argument is the directory +to be deleted. The default on Unix sites is "-rf". The default on Windows sites is ["/C", "del", "/Q"]. + +
env namespace @@ -4003,6 +4028,55 @@ mpirun command should be explicitly invoked.
+
Running on Windows + + + Since 10/11/09, the development version of Swift has the +ability to run on a Windows machine, as well as the ability to submit +jobs to a Windows site (provided that an appropriate provider is used). + + + + +In order to launch Swift on Windows, use the provided batch file +(swift.bat). In certain cases, when a large number of jar libraries are +present in the Swift lib directory and depending on the exact location +of the Swift installation, the classpath environment variable that the +Swift batch launcher tries to create may be larger than what Windows can +handle. In such a case, either install Swift in a directory closer to +the root of the disk (say, c:\swift) or remove non-essential jar files +from the Swift lib directory. + + + + + +Due to the large differences between Windows and Unix environments, +Swift must use environment specific tools to achieve some of its goals. +In particular, each Swift executable is launched using a wrapper script. +This script is a Bourne Shell script. On Windows machines, which have no +Bourne Shell interpreter installed by default, the Windows Scripting +Host is used instead, and the wrapper script is written in VBScript. +Similarly, when cleaning up after a run, the "/bin/rm" command available +in typical Unix environments must be replaced by the "del" shell command. + + + + + +It is important to note that in order to select the proper set of tools +to use, Swift must know when a site runs under Windows. To inform Swift +of this, specify the "sysinfo" attribute for the "pool" element in the +site catalog. For example: + + + + ... + + + + +
From noreply at svn.ci.uchicago.edu Sun Oct 11 23:02:04 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 11 Oct 2009 23:02:04 -0500 (CDT) Subject: [Swift-commit] r3151 - in trunk: . .settings Message-ID: <20091012040204.EE28C9CC99@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-11 23:02:04 -0500 (Sun, 11 Oct 2009) New Revision: 3151 Added: trunk/.classpath trunk/.project trunk/.settings/ trunk/.settings/org.eclipse.jdt.core.prefs trunk/.settings/org.eclipse.jdt.ui.prefs Log: eclipse files Added: trunk/.classpath =================================================================== --- trunk/.classpath (rev 0) +++ trunk/.classpath 2009-10-12 04:02:04 UTC (rev 3151) @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Added: trunk/.project =================================================================== --- trunk/.project (rev 0) +++ trunk/.project 2009-10-12 04:02:04 UTC (rev 3151) @@ -0,0 +1,17 @@ + + + swift + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + Added: trunk/.settings/org.eclipse.jdt.core.prefs =================================================================== --- trunk/.settings/org.eclipse.jdt.core.prefs (rev 0) +++ trunk/.settings/org.eclipse.jdt.core.prefs 2009-10-12 04:02:04 UTC (rev 3151) @@ -0,0 +1,276 @@ +#Thu Sep 03 17:04:31 CDT 2009 +eclipse.preferences.version=1 +org.eclipse.jdt.core.codeComplete.argumentPrefixes= +org.eclipse.jdt.core.codeComplete.argumentSuffixes= +org.eclipse.jdt.core.codeComplete.fieldPrefixes= +org.eclipse.jdt.core.codeComplete.fieldSuffixes= +org.eclipse.jdt.core.codeComplete.localPrefixes= +org.eclipse.jdt.core.codeComplete.localSuffixes= +org.eclipse.jdt.core.codeComplete.staticFieldPrefixes= +org.eclipse.jdt.core.codeComplete.staticFieldSuffixes= +org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes= +org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes= +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.5 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.5 +org.eclipse.jdt.core.formatter.align_type_members_on_columns=false +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=20 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=20 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=20 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=20 +org.eclipse.jdt.core.formatter.alignment_for_assignment=0 +org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 +org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=20 +org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_after_package=1 +org.eclipse.jdt.core.formatter.blank_lines_before_field=0 +org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 +org.eclipse.jdt.core.formatter.blank_lines_before_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=0 +org.eclipse.jdt.core.formatter.blank_lines_before_package=0 +org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 +org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 +org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false +org.eclipse.jdt.core.formatter.comment.format_block_comments=true +org.eclipse.jdt.core.formatter.comment.format_header=false +org.eclipse.jdt.core.formatter.comment.format_html=true +org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true +org.eclipse.jdt.core.formatter.comment.format_line_comments=true +org.eclipse.jdt.core.formatter.comment.format_source_code=true +org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true +org.eclipse.jdt.core.formatter.comment.indent_root_tags=true +org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert +org.eclipse.jdt.core.formatter.comment.line_length=80 +org.eclipse.jdt.core.formatter.compact_else_if=true +org.eclipse.jdt.core.formatter.continuation_indentation=2 +org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true +org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_empty_lines=false +org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true +org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=true +org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=insert +org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=insert +org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=insert +org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert +org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false +org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false +org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false +org.eclipse.jdt.core.formatter.lineSplit=80 +org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false +org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true +org.eclipse.jdt.core.formatter.tabulation.char=space +org.eclipse.jdt.core.formatter.tabulation.size=4 +org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false +org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true Added: trunk/.settings/org.eclipse.jdt.ui.prefs =================================================================== --- trunk/.settings/org.eclipse.jdt.ui.prefs (rev 0) +++ trunk/.settings/org.eclipse.jdt.ui.prefs 2009-10-12 04:02:04 UTC (rev 3151) @@ -0,0 +1,58 @@ +#Sat Feb 07 18:44:35 CST 2009 +eclipse.preferences.version=1 +editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=false +formatter_profile=_spaces +formatter_settings_version=11 +org.eclipse.jdt.ui.exception.name=e +org.eclipse.jdt.ui.gettersetter.use.is=false +org.eclipse.jdt.ui.javadoc=false +org.eclipse.jdt.ui.keywordthis=false +org.eclipse.jdt.ui.overrideannotation=true +org.eclipse.jdt.ui.text.custom_code_templates= +sp_cleanup.add_default_serial_version_id=true +sp_cleanup.add_generated_serial_version_id=false +sp_cleanup.add_missing_annotations=true +sp_cleanup.add_missing_deprecated_annotations=true +sp_cleanup.add_missing_nls_tags=false +sp_cleanup.add_missing_override_annotations=true +sp_cleanup.add_serial_version_id=false +sp_cleanup.always_use_blocks=true +sp_cleanup.always_use_parentheses_in_expressions=false +sp_cleanup.always_use_this_for_non_static_field_access=false +sp_cleanup.always_use_this_for_non_static_method_access=false +sp_cleanup.convert_to_enhanced_for_loop=false +sp_cleanup.format_source_code=false +sp_cleanup.make_local_variable_final=false +sp_cleanup.make_parameters_final=false +sp_cleanup.make_private_fields_final=true +sp_cleanup.make_variable_declarations_final=true +sp_cleanup.never_use_blocks=false +sp_cleanup.never_use_parentheses_in_expressions=true +sp_cleanup.on_save_use_additional_actions=false +sp_cleanup.organize_imports=true +sp_cleanup.qualify_static_field_accesses_with_declaring_class=false +sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_with_declaring_class=false +sp_cleanup.qualify_static_method_accesses_with_declaring_class=false +sp_cleanup.remove_private_constructors=true +sp_cleanup.remove_trailing_whitespaces=false +sp_cleanup.remove_trailing_whitespaces_all=true +sp_cleanup.remove_trailing_whitespaces_ignore_empty=false +sp_cleanup.remove_unnecessary_casts=true +sp_cleanup.remove_unnecessary_nls_tags=false +sp_cleanup.remove_unused_imports=false +sp_cleanup.remove_unused_local_variables=false +sp_cleanup.remove_unused_private_fields=true +sp_cleanup.remove_unused_private_members=false +sp_cleanup.remove_unused_private_methods=true +sp_cleanup.remove_unused_private_types=true +sp_cleanup.sort_members=false +sp_cleanup.sort_members_all=false +sp_cleanup.use_blocks=false +sp_cleanup.use_blocks_only_for_return_and_throw=false +sp_cleanup.use_parentheses_in_expressions=false +sp_cleanup.use_this_for_non_static_field_access=false +sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=true +sp_cleanup.use_this_for_non_static_method_access=false +sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true From noreply at svn.ci.uchicago.edu Mon Oct 12 13:07:27 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 12 Oct 2009 13:07:27 -0500 (CDT) Subject: [Swift-commit] r3152 - trunk/docs Message-ID: <20091012180727.AEA549CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-12 13:07:27 -0500 (Mon, 12 Oct 2009) New Revision: 3152 Modified: trunk/docs/userguide.xml Log: moved to existing swift profile Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-10-12 04:02:04 UTC (rev 3151) +++ trunk/docs/userguide.xml 2009-10-12 18:07:27 UTC (rev 3152) @@ -3198,6 +3198,29 @@ cache will be cleared using the algorithm specified in the caching.algorithm property. + wrapperInterpreter +The wrapper interpreter indicates the command (executable) to be used to run the Swift wrapper +script. The default is "/bin/bash" on Unix sites and "cscript.exe" on Windows sites. + + wrapperInterpreterOptions +Allows specifying additional options to the executable used to run the Swift wrapper. The defaults +are no options on Unix sites and "//Nologo" on Windows sites. + + wrapperScript +Specifies the name of the wrapper script to be used on a site. The defaults are "_swiftwrap" on +Unix sites and "_swiftwrap.vbs" on Windows sites. If you specify a custom wrapper script, it +must be present in the "libexec" directory of the Swift installation. + + cleanupCommand +Indicates the command to be run at the end of a Swift run to clean up the run directories on a +remote site. Defaults are "/bin/rm" on Unix sites and "cmd.exe" on Windows sites + + cleanupCommandOptions +Specifies the options to be passed to the cleanup command above. The options are passed in the +argument list to the cleanup command. After the options, the last argument is the directory +to be deleted. The default on Unix sites is "-rf". The default on Windows sites is ["/C", "del", "/Q"]. + +
Globus namespace maxwalltime specifies a walltime limit for each job, in minutes. @@ -3315,31 +3338,6 @@ -->
-
swift namespace - wrapperInterpreter -The wrapper interpreter indicates the command (executable) to be used to run the Swift wrapper -script. The default is "/bin/bash" on Unix sites and "cscript.exe" on Windows sites. - - wrapperInterpreterOptions -Allows specifying additional options to the executable used to run the Swift wrapper. The defaults -are no options on Unix sites and "//Nologo" on Windows sites. - - wrapperScript -Specifies the name of the wrapper script to be used on a site. The defaults are "_swiftwrap" on -Unix sites and "_swiftwrap.vbs" on Windows sites. If you specify a custom wrapper script, it -must be present in the "libexec" directory of the Swift installation. - - cleanupCommand -Indicates the command to be run at the end of a Swift run to clean up the run directories on a -remote site. Defaults are "/bin/rm" on Unix sites and "cmd.exe" on Windows sites - - cleanupCommandOptions -Specifies the options to be passed to the cleanup command above. The options are passed in the -argument list to the cleanup command. After the options, the last argument is the directory -to be deleted. The default on Unix sites is "-rf". The default on Windows sites is ["/C", "del", "/Q"]. - -
-
env namespace Profile keys set in the env namespace will be set in the unix environment of the From noreply at svn.ci.uchicago.edu Mon Oct 12 14:04:12 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 12 Oct 2009 14:04:12 -0500 (CDT) Subject: [Swift-commit] r3153 - trunk/docs Message-ID: <20091012190412.4E39B9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-12 14:04:11 -0500 (Mon, 12 Oct 2009) New Revision: 3153 Modified: trunk/docs/userguide.xml Log: escape xml stuff Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-10-12 18:07:27 UTC (rev 3152) +++ trunk/docs/userguide.xml 2009-10-12 19:04:11 UTC (rev 3153) @@ -4068,9 +4068,9 @@ site catalog. For example: - + <pool handle="localhost" sysinfo="INTEL32::WINDOWS"> ... - + </pool> From noreply at svn.ci.uchicago.edu Tue Oct 13 12:18:10 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 13 Oct 2009 12:18:10 -0500 (CDT) Subject: [Swift-commit] r3154 - SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm Message-ID: <20091013171810.1FA1E9CC67@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-10-13 12:18:09 -0500 (Tue, 13 Oct 2009) New Revision: 3154 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/ClstrMasstr.py Log: prototype script for clustering and extracting info from permutations Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/ClstrMasstr.py =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/ClstrMasstr.py (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/ClstrMasstr.py 2009-10-13 17:18:09 UTC (rev 3154) @@ -0,0 +1,101 @@ +#!/usr/bin/python + +## Sept.2009: This is for grabbing the cluster information (after running SurfClust). +## Can run 4 types of procedures. Specify after "--procedure" flag: +## 1) "NumberNodes" >>> Writing out the cluster size via number of nodes after SurfClust on a single input. +## 2) "PermNumberNodes" >>> Writing out the largest cluster's number of nodes on clustered permutation output +## 3) "ClusterMass" >>> calculating the cluster mass: the number of nodes times the statistic value on a single clustered input. +## 4) "PermClusterMass" >>> calculating the cluster mass on clustered permutation output, writing out the largest clustermass + +import sys + +class ClstrMasstr: + + def __init__(self): + self.input = "" + self.outname = "" + self.proc_type = "" + + def get_opts (self,allargstr): + print "length of argstr "+str(len(allargstr)) + i = 0 + for o in allargstr: + print "arg is "+o + if o == "--input": + self.input = allargstr[i+1] + elif o == "--outputname": + self.outname = allargstr[i+1] + elif o == "--procedure": + self.proc_type = allargstr[i+1] + print "input: "+self.input + print "outname: "+self.outname + print "procedure: "+self.proc_type + i = i+1 + + def get_clstr(self): + cluster_file = open(self.input,"r").read().split("\n") + cluster_file_length = len(cluster_file)-1 + clstrlist = "" + for i in range(16,cluster_file_length): + numNd = int(cluster_file[i].split()[1]) + clstrlist += ""+cluster_file[i].split()[0]+" "+`numNd`+"\n" + + outfile = open(self.outname,"w") + outfile.write(clstrlist) + outfile.close() + + def get_Perm_clstr(self): + cluster_file = open(self.input,"r").read().split("\n") + cluster_file_length = len(cluster_file)-1 + if len(cluster_file)-1 == 1: + clusterSize = cluster_file[0]+" \n" + else: + numNd = int(cluster_file[16].split()[1]) + clusterSize = `numNd`+" \n" + + outfile = open(self.outname,"w") + outfile.write(clusterSize) + outfile.close() + + def get_clstrmass(self): + cluster_file = open(self.input,"r").read().split("\n") + cluster_file_length = len(cluster_file)-1 + clstrmasslist = "" + for i in range(16,cluster_file_length): + numNd = int(cluster_file[i].split()[1]) + Mean = float(cluster_file[i].split()[3]) + mass = numNd*Mean + massval = "%.3f" % mass + clstrmasslist += ""+cluster_file[i].split()[0]+" "+massval+"\n" + + outfile = open(self.outname,"w") + outfile.write(clstrmasslist) + outfile.close() + + def get_Perm_clstrmass(self): + cluster_file = open(self.input,"r").read().split("\n") + if len(cluster_file)-1 == 1: + clstrmassval = cluster_file[0]+" \n" + else: + numNd = int(cluster_file[16].split()[1]) + Mean = float(cluster_file[16].split()[3]) + mass = numNd*Mean + massval = "%.3f" % mass + clstrmassval = massval+" \n" + + outfile = open(self.outname,"w") + outfile.write(clstrmassval) + outfile.close() + + + +clstr = ClstrMasstr() +clstr.get_opts(sys.argv) +if clstr.proc_type == "NumberNodes": + clstr.get_clstr() +elif clstr.proc_type == "PermNumberNodes": + clstr.get_Perm_clstr() +elif clstr.proc_type == "ClusterMass": + clstr.get_clstrmass() +elif clstr.proc_type == "PermClusterMass": + clstr.get_Perm_clstrmass() Property changes on: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/ClstrMasstr.py ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Tue Oct 13 12:19:07 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 13 Oct 2009 12:19:07 -0500 (CDT) Subject: [Swift-commit] r3155 - SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm Message-ID: <20091013171907.AC01C9CCAB@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-10-13 12:19:07 -0500 (Tue, 13 Oct 2009) New Revision: 3155 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/Alphasurfsim.py Log: for doing randomization, smoothing, clustering, getting largest value on surface data Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/Alphasurfsim.py =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/Alphasurfsim.py (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/Alphasurfsim.py 2009-10-13 17:19:07 UTC (rev 3155) @@ -0,0 +1,159 @@ +#!/usr/bin/python + +import os +import sys + +class AlphaSurfSim: + + def __init__(self): + self.r_swift_args = "" + self.r_script = "" + self.permOut = "" + self.spec = "" + self.smoothwm = "" + self.surfsmoothOut = "" + self.pial = "" + self.maskname = "" + self.thresh = "" + self.pval = "" + self.rmm = "" + self.surfclustOut = "" + self.outname = "" + self.proc_input = "" + self.proc_type = "" + + + def get_opts (self,allargstr): + print "length of argstr "+str(len(allargstr)) + i = 0 + for o in allargstr: + print "arg is "+o + if o == "--r_swift_args": + rstr = "" + j = i + while (allargstr[j+1].find("--")<=-1): + rstr = rstr+allargstr[j+1].strip()+" " + j+=1 + self.r_swift_args= rstr + elif o == "--r_script": + self.r_script = allargstr[i+1] + elif o == "--permOut": + self.permOut = allargstr[i+1] + elif o == "--spec": + self.spec = allargstr[i+1] + elif o == "--smoothwm": + self.smoothwm = allargstr[i+1] + elif o == "--surfsmoothOut": + self.surfsmoothOut = allargstr[i+1] + elif o == "--pial": + self.pial = allargstr[i+1] + elif o == "--maskname": + self.maskname = allargstr[i+1] + elif o == "--thresh": + self.thresh = allargstr[i+1] + elif o == "--pval": + self.pval = allargstr[i+1] + elif o == "--rmm": + self.rmm = allargstr[i+1] + elif o == "--surfclustOut": + self.surfclustOut = allargstr[i+1] + elif o == "--proc_input": + self.proc_input = allargstr[i+1] + elif o == "--outputname": + self.outname = allargstr[i+1] + elif o == "--procedure": + self.proc_type = allargstr[i+1] + i = i+1 + + def get_nodenum(self): + surf = open(self.pial).read().split("\n") + self.numnodes = surf[1].split()[0] + + def get_clstr(self): + cluster_file = open(self.proc_input,"r").read().split("\n") + cluster_file_length = len(cluster_file)-1 + clstrlist = "" + for i in range(16,cluster_file_length): + numNd = int(cluster_file[i].split()[1]) + clstrlist += ""+cluster_file[i].split()[0]+" "+`numNd`+"\n" + + outfile = open(self.outname,"w") + outfile.write(clstrlist) + outfile.close() + + def get_Perm_clstr(self): + cluster_file = open(self.proc_input,"r").read().split("\n") + cluster_file_length = len(cluster_file)-1 + if len(cluster_file)-1 == 1: + clusterSize = cluster_file[0]+" \n" + else: + numNd = int(cluster_file[16].split()[1]) + clusterSize = `numNd`+" \n" + + outfile = open(self.outname,"w") + outfile.write(clusterSize) + outfile.close() + + def get_clstrmass(self): + cluster_file = open(self.proc_input,"r").read().split("\n") + cluster_file_length = len(cluster_file)-1 + clstrmasslist = "" + for i in range(16,cluster_file_length): + numNd = int(cluster_file[i].split()[1]) + Mean = float(cluster_file[i].split()[3]) + mass = numNd*Mean + massval = "%.3f" % mass + clstrmasslist += ""+cluster_file[i].split()[0]+" "+massval+"\n" + + outfile = open(self.outname,"w") + outfile.write(clstrmasslist) + outfile.close() + + def get_Perm_clstrmass(self): + cluster_file = open(self.proc_input,"r").read().split("\n") + if len(cluster_file)-1 == 1: + clstrmassval = cluster_file[0]+" \n" + else: + numNd = int(cluster_file[16].split()[1]) + Mean = float(cluster_file[16].split()[3]) + mass = numNd*Mean + massval = "%.3f" % mass + clstrmassval = massval+" \n" + + outfile = open(self.outname,"w") + outfile.write(clstrmassval) + outfile.close() + + def run_r(self,proc): + os.environ["PROC"] = proc + os.environ["NUM_NODES"] = self.numnodes + os.environ["PERM_OUT"] = self.permOut + os.environ["PVAL"] = self.pval + os.environ["surfsmoothOut"] = self.surfsmoothOut + os.environ["maskname"] = self.maskname + os.system("R CMD BATCH --vanilla "+self.r_script) + pass + + def surfsmooth(self): + os.system("SurfSmooth -spec "+self.spec+" -surf_A "+self.smoothwm+" -met HEAT_07 -input "+self.permOut+" -target_fwhm 4 -Niter -1 -add_index -output "+self.surfsmoothOut) + + def surfclust(self): + os.system("SurfClust -spec "+self.spec+" -surf_A "+self.pial+" -input "+self.maskname+" 0 -rmm "+self.rmm+" -thresh_col 0 -thresh .99 -amm2 2 -sort_n_nodes -prepend_node_index -prefix "+self.surfclustOut) + + + +clstr = AlphaSurfSim() +clstr.get_opts(sys.argv) +clstr.get_nodenum() +clstr.run_r("genperm") +clstr.surfsmooth() +clstr.run_r("cutoff") +clstr.surfclust() +if clstr.proc_type == "NumberNodes": + clstr.get_clstr() +elif clstr.proc_type == "PermNumberNodes": + clstr.get_Perm_clstr() +elif clstr.proc_type == "ClusterMass": + clstr.get_clstrmass() +elif clstr.proc_type == "PermClusterMass": + clstr.get_Perm_clstrmass() From noreply at svn.ci.uchicago.edu Tue Oct 13 12:23:14 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 13 Oct 2009 12:23:14 -0500 (CDT) Subject: [Swift-commit] r3156 - SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm Message-ID: <20091013172314.6DFB89CCAB@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-10-13 12:23:14 -0500 (Tue, 13 Oct 2009) New Revision: 3156 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/alphasurfsim2.swift Log: doing alphasurfsim via swift Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/alphasurfsim2.swift =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/alphasurfsim2.swift (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/turnpointAnalysis/chisqperm/alphasurfsim2.swift 2009-10-13 17:23:14 UTC (rev 3156) @@ -0,0 +1,30 @@ +## pinche AlphaSim surface style +type file{} +type Rscript; + +app (file clusterResult) AlphaSurfSim (file Alphasurf, Rscript code, string permOut, file spec, file smoothwm, file pial, string smoothout, string maskout, float rmm, string surfclustout, string inputclustout){ + python @Alphasurf "--r_script" @code "--permOut" permOut "--spec" @spec "--smoothwm" @smoothwm "--pial" @pial "--surfsmoothOut" smoothout "--maskname" maskout "--rmm" rmm "--pval" 0.05 "--surfclustOut" surfclustout "--proc_input" inputclustout "--procedure" "PermNumberNodes" "--outputname" @filename(clusterResult); +} + +file Alphasurf; +Rscript code; + +string h = "lh"; +float rmm_vals[] = [3.1]; +int permbrains[] = [1:1000]; +#int permbrains[] = [1:2]; + +foreach perm in permbrains{ + foreach rmm in rmm_vals{ + string permOut = @strcat("perm",perm,".1D"); + file spec; + file smoothwm; + file pial; + string smoothout = @strcat("perm",perm,"_",h,"smooth.1D.dset"); + string maskout = @strcat("perm",perm,"_",h,"smoothmask.1D"); + string surfclustout = @strcat("perm",perm,"_",h,"clustout"); + string inputclustout = @strcat("perm",perm,"_",h,"clustout_ClstTable_r",rmm,"_a2.0.1D"); + file clusterResult; + clusterResult = AlphaSurfSim(Alphasurf, code, permOut, spec, smoothwm, pial, smoothout, maskout, rmm, surfclustout, inputclustout); + } +} From noreply at svn.ci.uchicago.edu Tue Oct 13 17:35:54 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 13 Oct 2009 17:35:54 -0500 (CDT) Subject: [Swift-commit] r3157 - trunk/libexec Message-ID: <20091013223554.BD9579CCAB@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-13 17:35:54 -0500 (Tue, 13 Oct 2009) New Revision: 3157 Modified: trunk/libexec/vdl-int.k Log: Oops. Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2009-10-13 17:23:14 UTC (rev 3156) +++ trunk/libexec/vdl-int.k 2009-10-13 22:35:54 UTC (rev 3157) @@ -104,7 +104,7 @@ wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" sharedDir := dircat(wfdir, "shared") dir:make(sharedDir, host=rhost) - transfer(srcdir="{swift.home}/libexec/", vdl:siteprofile(rhost, "swift:wrapperScript"), destdir=sharedDir, desthost=rhost) + transfer(srcdir="{swift.home}/libexec/", srcfile=vdl:siteprofile(rhost, "swift:wrapperScript"), destdir=sharedDir, desthost=rhost) transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftseq", destdir=sharedDir, desthost=rhost) dir:make(dircat(wfdir, "kickstart"), host=rhost) From noreply at svn.ci.uchicago.edu Tue Oct 13 23:53:37 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 13 Oct 2009 23:53:37 -0500 (CDT) Subject: [Swift-commit] r3158 - trunk/docs Message-ID: <20091014045337.A375D9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-13 23:53:36 -0500 (Tue, 13 Oct 2009) New Revision: 3158 Modified: trunk/docs/userguide.xml Log: added mention that the capture group references for the regexp mapper need to be specified with two backslashes instead of one Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-10-13 22:35:54 UTC (rev 3157) +++ trunk/docs/userguide.xml 2009-10-14 04:53:36 UTC (rev 3158) @@ -1220,8 +1220,9 @@ matchRegular expression pattern to match, use () to match whatever regular expression is inside the parentheses, and indicate the start and end of a group; the contents of a -group can be retrieved with the \number special -sequence +group can be retrieved with the \\number special sequence +(two backslashes are needed because the backslash is an escape sequence introducer) + transformThe pattern of the file name to transform to, use \number to reference the group matched. @@ -1235,7 +1236,7 @@ file f <regexp_mapper; source=s, match="(.*)gif", - transform="\1jpg">; + transform="\\1jpg">; This example transforms a string ending gif into one ending jpg and maps that to a file. From noreply at svn.ci.uchicago.edu Wed Oct 14 00:03:07 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 14 Oct 2009 00:03:07 -0500 (CDT) Subject: [Swift-commit] r3159 - in trunk/src/org/griphyn/vdl/mapping: . file Message-ID: <20091014050307.181DF9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-14 00:03:06 -0500 (Wed, 14 Oct 2009) New Revision: 3159 Modified: trunk/src/org/griphyn/vdl/mapping/MappingParam.java trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java Log: complain when mapped data is passed as the source parameter to the regexp mapper; as it is now, it's very unlikely that the user wants to transform some obscure representation of the data handle instead of the filename' Modified: trunk/src/org/griphyn/vdl/mapping/MappingParam.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/MappingParam.java 2009-10-14 04:53:36 UTC (rev 3158) +++ trunk/src/org/griphyn/vdl/mapping/MappingParam.java 2009-10-14 05:03:06 UTC (rev 3159) @@ -144,6 +144,10 @@ } public String toString() { - return "mapping parameter "+name; + return "mapping parameter " + name; } + + public String getName() { + return name; + } } Modified: trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2009-10-14 04:53:36 UTC (rev 3158) +++ trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2009-10-14 05:03:06 UTC (rev 3159) @@ -10,6 +10,7 @@ import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; +import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; @@ -38,6 +39,11 @@ } public PhysicalFormat map(Path path) { + if (PARAM_MATCH.getRawValue(this) instanceof DSHandle) { + DSHandle h = (DSHandle) PARAM_MATCH.getRawValue(this); + throw new IllegalArgumentException("Non-primitive value specified for " + + PARAM_MATCH.getName() + "; maybe you meant @filename(" + h.getPathFromRoot() + ")?"); + } String match = PARAM_MATCH.getStringValue(this); String source = PARAM_SOURCE.getStringValue(this); String transform = PARAM_TRANSFORM.getStringValue(this); From noreply at svn.ci.uchicago.edu Wed Oct 14 00:15:13 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 14 Oct 2009 00:15:13 -0500 (CDT) Subject: [Swift-commit] r3160 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20091014051513.11A459CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-14 00:15:06 -0500 (Wed, 14 Oct 2009) New Revision: 3160 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: added -recompile flag to force recompilation; while I thought we had this figured out, Dr. Tan has demonstrated otherwise today Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-10-14 05:03:06 UTC (rev 3159) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-10-14 05:15:06 UTC (rev 3160) @@ -63,6 +63,7 @@ public static final String ARG_LOGFILE = "logfile"; public static final String ARG_RUNID = "runid"; public static final String ARG_TUI = "tui"; + public static final String ARG_RECOMPILE = "recompile"; public static final String CONST_VDL_OPERATION = "vdl:operation"; public static final String VDL_OPERATION_RUN = "run"; @@ -126,7 +127,7 @@ if (project.endsWith(".swift")) { try { - project = compile(project); + project = compile(project, ap.isPresent(ARG_RECOMPILE)); } catch (ParsingException pe) { // the compiler should have already logged useful @@ -203,10 +204,16 @@ private static void shortUsage() { System.err.print("For usage information: swift -help\n\n"); } - + public static String compile(String project) throws FileNotFoundException, ParsingException, IncorrectInvocationException, CompilationException, IOException { + return compile(project, false); + } + + public static String compile(String project, boolean forceRecompile) throws FileNotFoundException, + ParsingException, IncorrectInvocationException, + CompilationException, IOException { File swiftscript = new File(project); String projectBase = project.substring(0, project.lastIndexOf('.')); File xml = new File(projectBase + ".xml"); @@ -214,7 +221,7 @@ loadBuildVersion(); - boolean recompile = false; + boolean recompile = forceRecompile; if (swiftscript.lastModified() > kml.lastModified()) { logger.info(project + ": source file is new. Recompiling."); @@ -342,6 +349,10 @@ ap.addFlag(ARG_HELP, "Display usage information"); ap.addAlias(ARG_HELP, "h"); + ap.addFlag(ARG_RECOMPILE, + "Forces Swift to re-compile the invoked Swift script. " + + "While Swift is meant to detect when recompilation is necessary, " + + "in some special cases it fails to do so. This flag helps with those special cases."); ap.addFlag(ARG_TYPECHECK, "Does a typecheck instead of executing the SwiftScript program"); From noreply at svn.ci.uchicago.edu Thu Oct 15 14:48:59 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Oct 2009 14:48:59 -0500 (CDT) Subject: [Swift-commit] r3161 - trunk/etc Message-ID: <20091015194859.C69DF9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-15 14:48:58 -0500 (Thu, 15 Oct 2009) New Revision: 3161 Modified: trunk/etc/log4j.properties Log: updated logging configuration Modified: trunk/etc/log4j.properties =================================================================== --- trunk/etc/log4j.properties 2009-10-14 05:15:06 UTC (rev 3160) +++ trunk/etc/log4j.properties 2009-10-15 19:48:58 UTC (rev 3161) @@ -20,9 +20,10 @@ log4j.logger.org.griphyn.vdl.karajan.Loader=DEBUG log4j.logger.org.globus.cog.karajan.workflow.events.WorkerSweeper=WARN log4j.logger.org.globus.cog.karajan.workflow.nodes.FlowNode=WARN -log4j.logger.org.globus.cog.karajan.scheduler.WeightedHostScoreScheduler=DEBUG +log4j.logger.org.globus.cog.karajan.scheduler.WeightedHostScoreScheduler=INFO log4j.logger.org.griphyn.vdl.toolkit.VDLt2VDLx=DEBUG log4j.logger.org.griphyn.vdl.karajan.VDL2ExecutionContext=DEBUG log4j.logger.org.globus.cog.abstraction.impl.common.task.TaskImpl=DEBUG log4j.logger.org.griphyn.vdl.karajan.lib.GetFieldValue=DEBUG log4j.logger.org.griphyn.vdl.engine.Karajan=INFO +log4j.logger.org.globus.cog.abstraction.coaster.rlog=INFO \ No newline at end of file From noreply at svn.ci.uchicago.edu Thu Oct 15 14:49:42 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Oct 2009 14:49:42 -0500 (CDT) Subject: [Swift-commit] r3162 - trunk/bin Message-ID: <20091015194942.CB7339CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-15 14:49:42 -0500 (Thu, 15 Oct 2009) New Revision: 3162 Modified: trunk/bin/swift-plot-log Log: added coaster page Modified: trunk/bin/swift-plot-log =================================================================== --- trunk/bin/swift-plot-log 2009-10-15 19:48:58 UTC (rev 3161) +++ trunk/bin/swift-plot-log 2009-10-15 19:49:42 UTC (rev 3162) @@ -41,7 +41,7 @@ fi if [ "X$2" = "X" ]; then -make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean webpage.kara webpage.weights karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event karatasks.JOB_SUBMISSION.eip $MAKETARGETS webpage distribute +make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean webpage.kara webpage.weights webpage.coasters karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event karatasks.JOB_SUBMISSION.eip $MAKETARGETS webpage distribute else shift make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean $@ From noreply at svn.ci.uchicago.edu Thu Oct 15 14:51:11 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Oct 2009 14:51:11 -0500 (CDT) Subject: [Swift-commit] r3163 - trunk/libexec/log-processing Message-ID: <20091015195111.6051D9CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-15 14:51:11 -0500 (Thu, 15 Oct 2009) New Revision: 3163 Modified: trunk/libexec/log-processing/weights.sh Log: only plot weights if data available Modified: trunk/libexec/log-processing/weights.sh =================================================================== --- trunk/libexec/log-processing/weights.sh 2009-10-15 19:49:42 UTC (rev 3162) +++ trunk/libexec/log-processing/weights.sh 2009-10-15 19:51:11 UTC (rev 3163) @@ -1,8 +1,8 @@ #!/bin/bash -grep 'WeightedHostScoreScheduler Old score:' $1 | iso-to-secs | normalise-event-start-time | sed 's/^\([^ ]*\) .* new score: \(.*\)$/\1 \2/' | sed 's/,//g' > weights.tmp +grep 'WeightedHostScoreScheduler CONTACT_SELECTED' $1 | iso-to-secs | normalise-event-start-time | sed 's/^\([^ ]*\) .* score=\(.*\)$/\1 \2/' | sed 's/,//g' > weights.tmp -gnuplot ${SWIFT_PLOT_HOME}/weights.plot +if [ -s weights.tmp ]; then + gnuplot ${SWIFT_PLOT_HOME}/weights.plot +fi - - From noreply at svn.ci.uchicago.edu Thu Oct 15 14:54:45 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Oct 2009 14:54:45 -0500 (CDT) Subject: [Swift-commit] r3164 - trunk/libexec/log-processing Message-ID: <20091015195445.9C8399CC67@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-10-15 14:54:45 -0500 (Thu, 15 Oct 2009) New Revision: 3164 Added: trunk/libexec/log-processing/coaster-block-timeline.plot trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot trunk/libexec/log-processing/coaster-block-utilization.plot trunk/libexec/log-processing/coaster-qwait-count.plot trunk/libexec/log-processing/coaster-qwait-size.plot trunk/libexec/log-processing/coaster-qwait-wtime.plot trunk/libexec/log-processing/coasters.html.template trunk/libexec/log-processing/extract-block-utilization-vs-count trunk/libexec/log-processing/extract-coaster-block-utilization trunk/libexec/log-processing/extract-coaster-qwait trunk/libexec/log-processing/extract-coaster-timeline trunk/libexec/log-processing/extract-coaster-utilization trunk/libexec/log-processing/makefile.coasters Modified: trunk/libexec/log-processing/falkon.html.template trunk/libexec/log-processing/info.html.template trunk/libexec/log-processing/karajan.html.template trunk/libexec/log-processing/kickstart.html.template trunk/libexec/log-processing/makefile trunk/libexec/log-processing/makefile.webpage trunk/libexec/log-processing/toolbar.html.template Log: added coasters page Added: trunk/libexec/log-processing/coaster-block-timeline.plot =================================================================== --- trunk/libexec/log-processing/coaster-block-timeline.plot (rev 0) +++ trunk/libexec/log-processing/coaster-block-timeline.plot 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,6 @@ +set terminal png +set output "coaster-block-timeline.png" + +set title "Queued/Active Coaster Workers" +set style data steps +plot "coaster-blocks.data" u 1:2 w steps title "Queued Workers", "coaster-blocks.data" u 1:3 w steps title "Running Workers" \ No newline at end of file Added: trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot =================================================================== --- trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot (rev 0) +++ trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,5 @@ +set terminal png +set output "coaster-block-utilization-vs-count.png" + +set title "Block Utilization vs. Worker Count" +plot "coaster-block-utilization-vs-count.data" u 1:2 title "" \ No newline at end of file Added: trunk/libexec/log-processing/coaster-block-utilization.plot =================================================================== --- trunk/libexec/log-processing/coaster-block-utilization.plot (rev 0) +++ trunk/libexec/log-processing/coaster-block-utilization.plot 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,5 @@ +set terminal png +set output "coaster-block-utilization.png" + +set title "Block Utilization (%)" +plot "coaster-block-utilization.data" w linespoints pt 3 title "" \ No newline at end of file Added: trunk/libexec/log-processing/coaster-qwait-count.plot =================================================================== --- trunk/libexec/log-processing/coaster-qwait-count.plot (rev 0) +++ trunk/libexec/log-processing/coaster-qwait-count.plot 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,5 @@ +set terminal png +set output "coaster-qwait-count.png" + +set title "Wait times vs. Block Worker Count" +plot "coaster-qwait-count.data" u 1:2 title "" \ No newline at end of file Added: trunk/libexec/log-processing/coaster-qwait-size.plot =================================================================== --- trunk/libexec/log-processing/coaster-qwait-size.plot (rev 0) +++ trunk/libexec/log-processing/coaster-qwait-size.plot 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,5 @@ +set terminal png +set output "coaster-qwait-size.png" + +set title "Wait times vs. Block Size (walltime x worker count)" +plot "coaster-qwait-size.data" u 1:2 title "" \ No newline at end of file Added: trunk/libexec/log-processing/coaster-qwait-wtime.plot =================================================================== --- trunk/libexec/log-processing/coaster-qwait-wtime.plot (rev 0) +++ trunk/libexec/log-processing/coaster-qwait-wtime.plot 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,5 @@ +set terminal png +set output "coaster-qwait-wtime.png" + +set title "Wait times vs. Block Walltime" +plot "coaster-qwait-wtime.data" u 1:2 title "" \ No newline at end of file Added: trunk/libexec/log-processing/coasters.html.template =================================================================== --- trunk/libexec/log-processing/coasters.html.template (rev 0) +++ trunk/libexec/log-processing/coasters.html.template 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,30 @@ + +Swift log analysis + +

Swift log analysis

+ +include(`toolbar.html.template') + +

Coasters

+

The block timeline

+

This shows how many workers were requested and how many workers were running at each time:

+ + +

Queue wait times

+

How much a block waited in the queue based on it's total size (CPU seconds):

+ + +

How much a block waited in the queue based on the number of workers:

+ + +

How much a block waited in the queue based on it's walltime:

+ + +

Worker utilization

+

Sorted block utilization (average (time spent running jobs / total time) for each worker in the block):

+ +

Utilization vs. Number of workers:

+ + +

end

+ Added: trunk/libexec/log-processing/extract-block-utilization-vs-count =================================================================== --- trunk/libexec/log-processing/extract-block-utilization-vs-count (rev 0) +++ trunk/libexec/log-processing/extract-block-utilization-vs-count 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,15 @@ +#!/bin/bash + +LOG=$1 + +rm -f coaster-block-utilization-vs-count.data + +IFS=$'\n' +for LINE in $(cat coasters.tmp); do + ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'` + U=`echo $LINE | sed -n 's/^.*BLOCK_UTILIZATION.*u=\([^ ]*\)$/\1/p'` + if [ "$U" != "" ]; then + COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2` + echo "$COUNT $U" >> coaster-block-utilization-vs-count.data + fi +done Property changes on: trunk/libexec/log-processing/extract-block-utilization-vs-count ___________________________________________________________________ Name: svn:executable + * Added: trunk/libexec/log-processing/extract-coaster-block-utilization =================================================================== --- trunk/libexec/log-processing/extract-coaster-block-utilization (rev 0) +++ trunk/libexec/log-processing/extract-coaster-block-utilization 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,7 @@ +#!/bin/bash + +LOG=$1 + +rm -f coaster-block-utilization.data + +cat coasters.tmp | sed -n 's/^.*BLOCK_UTILIZATION.*u=\([^ ]*\)$/\1/p' |sort > coaster-block-utilization.data Property changes on: trunk/libexec/log-processing/extract-coaster-block-utilization ___________________________________________________________________ Name: svn:executable + * Added: trunk/libexec/log-processing/extract-coaster-qwait =================================================================== --- trunk/libexec/log-processing/extract-coaster-qwait (rev 0) +++ trunk/libexec/log-processing/extract-coaster-qwait 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,31 @@ +#!/bin/bash + +LOG=$1 + +rm -f block-walltime-table.tmp +rm -f block-start-time-table.tmp +rm -f coaster-qwait-size.data +rm -f coaster-qwait-wtime.data +rm -f coaster-qwait-count.data + +IFS=$'\n' +for LINE in $(cat coasters.tmp); do + TIME=`echo $LINE | sed 's/^\([^ ]*\) .*$/\1/' ` + ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'` + WALLTIME=`echo $LINE | sed -n 's/^.*BLOCK_REQUESTED.*h=\([^ ]*\)$/\1/p'` + if [ "$WALLTIME" != "" ]; then + echo "$ID,$WALLTIME" >> block-walltime-table.tmp + echo "$ID,$TIME" >> block-start-time-table.tmp + else + if echo $LINE | grep "BLOCK_ACTIVE" >/dev/null; then + COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2` + WALLTIME=`cat block-walltime-table.tmp | grep "$ID" | cut -d , -f 2` + STARTTIME=`cat block-start-time-table.tmp | grep "$ID" | cut -d , -f 2` + QTIME=`perl -e "print $TIME - $STARTTIME"` + SIZE=$(($WALLTIME * $COUNT)) + echo "$SIZE $QTIME" >> coaster-qwait-size.data + echo "$WALLTIME $QTIME" >> coaster-qwait-wtime.data + echo "$COUNT $QTIME" >> coaster-qwait-count.data + fi + fi +done Property changes on: trunk/libexec/log-processing/extract-coaster-qwait ___________________________________________________________________ Name: svn:executable + * Added: trunk/libexec/log-processing/extract-coaster-timeline =================================================================== --- trunk/libexec/log-processing/extract-coaster-timeline (rev 0) +++ trunk/libexec/log-processing/extract-coaster-timeline 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,30 @@ +#!/bin/bash + +LOG=$1 + +grep "RemoteLogHandler BLOCK_.*" $LOG | iso-to-secs | normalise-event-start-time >coasters.tmp +rm -f block-count-table.tmp +rm -f coaster-block.data + +IFS=$'\n' +REQUESTED=0 +RUNNING=0 +for LINE in $(cat coasters.tmp); do + TIME=`echo $LINE | sed 's/^\([^ ]*\) .*$/\1/' ` + ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'` + ADD=`echo $LINE | sed -n 's/^.*BLOCK_REQUESTED.*w=\([^ ]*\),.*$/\1/p'` + if [ "$ADD" != "" ]; then + REQUESTED=$(($REQUESTED + $ADD)) + echo "$ID,$ADD" >> block-count-table.tmp + else + COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2` + if echo $LINE | grep "BLOCK_ACTIVE" >/dev/null; then + RUNNING=$(($RUNNING + $COUNT)) + fi + if echo $LINE | grep "BLOCK_SHUTDOWN" >/dev/null; then + RUNNING=$(($RUNNING - $COUNT)) + REQUESTED=$(($REQUESTED - $COUNT)) + fi + fi + echo $TIME $REQUESTED $RUNNING >> coaster-blocks.data +done Property changes on: trunk/libexec/log-processing/extract-coaster-timeline ___________________________________________________________________ Name: svn:executable + * Added: trunk/libexec/log-processing/extract-coaster-utilization =================================================================== --- trunk/libexec/log-processing/extract-coaster-utilization (rev 0) +++ trunk/libexec/log-processing/extract-coaster-utilization 2009-10-15 19:54:45 UTC (rev 3164) @@ -0,0 +1,27 @@ +#!/bin/bash + +LOG=$1 + +rm -f block-utilization.data + +IFS=$'\n' +for LINE in $(cat coasters.tmp); do + TIME=`echo $LINE | sed 's/^\([^ ]*\) .*$/\1/' ` + ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'` + WALLTIME=`echo $LINE | sed -n 's/^.*BLOCK_REQUESTED.*h=\([^ ]*\)$/\1/p'` + if [ "$WALLTIME" != "" ]; then + echo "$ID,$WALLTIME" >> block-walltime-hash.tmp + echo "$ID,$TIME" >> block-start-time-hash.tmp + else + if echo $LINE | grep "BLOCK_ACTIVE" >/dev/null; then + COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2` + WALLTIME=`cat block-walltime-table.tmp | grep "$ID" | cut -d , -f 2` + STARTTIME=`cat block-start-time-table.tmp | grep "$ID" | cut -d , -f 2` + QTIME=`perl -e "print $TIME - $STARTTIME"` + SIZE=$(($WALLTIME * $COUNT)) + echo "$SIZE $QTIME" >> coaster-qwait-size-dist.data + echo "$WALLTIME $QTIME" >> coaster-qwait-wtime-dist.data + echo "$COUNT $QTIME" >> coaster-qwait-count-dist.data + fi + fi +done Property changes on: trunk/libexec/log-processing/extract-coaster-utilization ___________________________________________________________________ Name: svn:executable + * Modified: trunk/libexec/log-processing/falkon.html.template =================================================================== --- trunk/libexec/log-processing/falkon.html.template 2009-10-15 19:51:11 UTC (rev 3163) +++ trunk/libexec/log-processing/falkon.html.template 2009-10-15 19:54:45 UTC (rev 3164) @@ -8,13 +8,13 @@

falkon

Falkon taskperf log:

- +-

Total jobs that falkon has queued or running:

- +-

Falkon job status trails:

- +-

end

Modified: trunk/libexec/log-processing/info.html.template =================================================================== --- trunk/libexec/log-processing/info.html.template 2009-10-15 19:51:11 UTC (rev 3163) +++ trunk/libexec/log-processing/info.html.template 2009-10-15 19:54:45 UTC (rev 3164) @@ -9,27 +9,27 @@

worker.sh info files

info file plots:

info files:

- +-

info files sorted by start time:

- +-

info trails:

- +-

info duration histogram:

- +-

how wrapper.sh is spending its time:

- +-

total concurrent wrapper+application execution on worker nodes

- +-

Offsets between job submission Active events and start times reported by info.

- +-
- +-

end

Modified: trunk/libexec/log-processing/karajan.html.template =================================================================== --- trunk/libexec/log-processing/karajan.html.template 2009-10-15 19:51:11 UTC (rev 3163) +++ trunk/libexec/log-processing/karajan.html.template 2009-10-15 19:54:45 UTC (rev 3164) @@ -106,15 +106,15 @@

job clusters

-1. +1. -
-2. +2. -
-3. +3. -
-4. +4. -
-5. +5. -

end

Modified: trunk/libexec/log-processing/kickstart.html.template =================================================================== --- trunk/libexec/log-processing/kickstart.html.template 2009-10-15 19:51:11 UTC (rev 3163) +++ trunk/libexec/log-processing/kickstart.html.template 2009-10-15 19:54:45 UTC (rev 3164) @@ -9,14 +9,14 @@

kickstart records. (text file of events) this shows applications which are actually running on a worker node, executing:

- +-

kickstart records total - this represents how many worker nodes were actually executing applications at any time:

- +-

kickstart records sorted by start time:

- +-

kickstart stats: 

@@ -26,7 +26,7 @@
 
 
cumulative histogram(?) of kickstart job durations

 
-
+-
 
 
end

 

Modified: trunk/libexec/log-processing/makefile
===================================================================
--- trunk/libexec/log-processing/makefile	2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/makefile	2009-10-15 19:54:45 UTC (rev 3164)
@@ -8,6 +8,7 @@
 include makefile.webpage
 include makefile.kickstart
 include makefile.errors
+include makefile.coasters
 
 distributable: 
 	rm -f *.tmp log *.transitions tmp-*

Added: trunk/libexec/log-processing/makefile.coasters
===================================================================
--- trunk/libexec/log-processing/makefile.coasters	                        (rev 0)
+++ trunk/libexec/log-processing/makefile.coasters	2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,41 @@
+coaster-block-timeline.png: coaster-block-timeline.data
+	gnuplot $(SWIFT_PLOT_HOME)/coaster-block-timeline.plot
+
+extract-timeline:
+	extract-coaster-timeline $(LOG)
+
+coaster-block-timeline.data: extract-timeline
+coasters.tmp: extract-timeline
+block-count-table.tmp: extract-timeline
+
+extract-coaster-qwait: coasters.tmp
+	extract-coaster-qwait $(LOG)
+
+coaster-qwait-count.data: extract-coaster-qwait
+coaster-qwait-size.data: extract-coaster-qwait
+coaster-qwait-wtime.data: extract-coaster-qwait
+coaster-block-utilization-vs-count.data: extract-block-utilization-vs-count
+
+extract-block-utilization: coasters.tmp
+	extract-coaster-block-utilization $(LOG)
+
+extract-block-utilization-vs-count: coasters.tmp
+	extract-block-utilization-vs-count $(LOG)
+
+coaster-block-utilization.data:
+	extract-coaster-block-utilization
+
+coaster-qwait-count.png: coaster-qwait-count.data
+	gnuplot $(SWIFT_PLOT_HOME)/coaster-qwait-count.plot
+
+coaster-qwait-size.png: coaster-qwait-size.data
+	gnuplot $(SWIFT_PLOT_HOME)/coaster-qwait-size.plot
+
+coaster-qwait-wtime.png: coaster-qwait-wtime.data
+	gnuplot $(SWIFT_PLOT_HOME)/coaster-qwait-wtime.plot
+
+coaster-block-utilization.png: coaster-block-utilization.data
+	gnuplot $(SWIFT_PLOT_HOME)/coaster-block-utilization.plot
+
+coaster-block-utilization-vs-count.png: coaster-block-utilization-vs-count.data
+	gnuplot $(SWIFT_PLOT_HOME)/coaster-block-utilization-vs-count.plot

Modified: trunk/libexec/log-processing/makefile.webpage
===================================================================
--- trunk/libexec/log-processing/makefile.webpage	2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/makefile.webpage	2009-10-15 19:54:45 UTC (rev 3164)
@@ -1,7 +1,7 @@
 webpage: pngs htmls
 
 htmls: index.html execute2.html execute.html karajan.html info.html \
-assorted.html kickstart.html falkon.html scheduler.html
+assorted.html kickstart.html falkon.html scheduler.html coasters.html
 
 webpage.kara: karatasks.png karatasks-coloured.png karatasks-type-counts.txt \
 karatasks.lastsummary karatasks-coloured-sorted.png \
@@ -22,6 +22,9 @@
 karatasks.JOB_SUBMISSION.Queue.stats \
 karatasks.JOB_SUBMISSION.Queue-duration-histogram.png
 
+webpage.coasters: coaster-block-timeline.png coaster-qwait-count.png coaster-qwait-size.png \
+	coaster-qwait-wtime.png coaster-block-utilization.png coaster-block-utilization-vs-count.png
+
 webpage.clusters: karatasks.clusters.png karatasks.clusters-total.png \
 karatasks.clusters.sorted-start.png karatasks.clusters-duration-histogram.png
 

Modified: trunk/libexec/log-processing/toolbar.html.template
===================================================================
--- trunk/libexec/log-processing/toolbar.html.template	2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/toolbar.html.template	2009-10-15 19:54:45 UTC (rev 3164)
@@ -7,6 +7,7 @@
 kickstart |
 info |
 falkon |
+coasters |
 assorted swift library routines
 | Other:
 scheduler



From noreply at svn.ci.uchicago.edu  Fri Oct 16 15:14:21 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 16 Oct 2009 15:14:21 -0500 (CDT)
Subject: [Swift-commit] r3165 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20091016201421.4E3619CC67@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-16 15:14:19 -0500 (Fri, 16 Oct 2009)
New Revision: 3165

Modified:
   trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java
Log:
that looks more like debug than info

Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java	2009-10-15 19:54:45 UTC (rev 3164)
+++ trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java	2009-10-16 20:14:19 UTC (rev 3165)
@@ -136,8 +136,8 @@
 	}
 	
 	private void addAttributes(NamedArguments named, Map attrs) {
-	    if (logger.isInfoEnabled()) {
-	        logger.info("Attributes: " + attrs);
+	    if (logger.isDebugEnabled()) {
+	        logger.debug("Attributes: " + attrs);
 	    }
 	    if (attrs == null || attrs.size() == 0) {
 	        return;



From noreply at svn.ci.uchicago.edu  Fri Oct 16 15:14:48 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 16 Oct 2009 15:14:48 -0500 (CDT)
Subject: [Swift-commit] r3166 - trunk/src/org/griphyn/vdl/karajan
Message-ID: <20091016201448.6BDC89CC67@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-16 15:14:48 -0500 (Fri, 16 Oct 2009)
New Revision: 3166

Modified:
   trunk/src/org/griphyn/vdl/karajan/Loader.java
Log:
added reduced logging flag

Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/Loader.java	2009-10-16 20:14:19 UTC (rev 3165)
+++ trunk/src/org/griphyn/vdl/karajan/Loader.java	2009-10-16 20:14:48 UTC (rev 3166)
@@ -35,13 +35,16 @@
 import org.globus.cog.karajan.workflow.PrintStreamChannel;
 import org.globus.cog.karajan.workflow.nodes.FlowElement;
 import org.globus.cog.karajan.workflow.nodes.grid.AbstractGridNode;
+import org.globus.cog.karajan.workflow.service.channels.AbstractKarajanChannel;
 import org.globus.cog.util.ArgumentParser;
 import org.globus.cog.util.ArgumentParserException;
 import org.griphyn.vdl.engine.Karajan;
 import org.griphyn.vdl.karajan.functions.ConfigProperty;
 import org.griphyn.vdl.karajan.lib.Execute;
 import org.griphyn.vdl.karajan.lib.Log;
+import org.griphyn.vdl.karajan.lib.New;
 import org.griphyn.vdl.karajan.monitor.MonitorAppender;
+import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.toolkit.VDLt2VDLx;
 import org.griphyn.vdl.toolkit.VDLt2VDLx.IncorrectInvocationException;
 import org.griphyn.vdl.toolkit.VDLt2VDLx.ParsingException;
@@ -64,6 +67,7 @@
     public static final String ARG_RUNID = "runid";
     public static final String ARG_TUI = "tui";
     public static final String ARG_RECOMPILE = "recompile";
+    public static final String ARG_REDUCED_LOGGING = "reduced.logging";
 
     public static final String CONST_VDL_OPERATION = "vdl:operation";
     public static final String VDL_OPERATION_RUN = "run";
@@ -397,6 +401,8 @@
                 "Specifies the run identifier. This must be unique for every invocation of a workflow and is used in several places to keep files from different executions cleanly separated. By default, a datestamp and random number are used to generate a run identifier.",
                 "string", ArgumentParser.OPTIONAL);
         ap.addFlag(ARG_TUI);
+        ap.addFlag(ARG_REDUCED_LOGGING, "Makes logging more terse by disabling provenance " +
+        		"information and low-level task messages");
 
         Map desc = VDL2ConfigProperties.getPropertyDescriptions();
         Iterator i = desc.entrySet().iterator();
@@ -457,6 +463,11 @@
                 Level.INFO);
             ca.setThreshold(Level.FATAL);
         }
+        else if (ap.isPresent(ARG_REDUCED_LOGGING)) {
+            Logger.getLogger(AbstractDataNode.class).setLevel(Level.WARN);
+            Logger.getLogger(New.class).setLevel(Level.WARN);
+            Logger.getLogger("org.globus.cog.karajan.workflow.service").setLevel(Level.WARN);
+        }
     }
 
     protected static Appender getAppender(Class cls) {



From noreply at svn.ci.uchicago.edu  Fri Oct 16 16:53:10 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 16 Oct 2009 16:53:10 -0500 (CDT)
Subject: [Swift-commit] r3167 - in trunk/src/org/griphyn/vdl/type: . impl
Message-ID: <20091016215311.04F2E9CC67@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-16 16:53:10 -0500 (Fri, 16 Oct 2009)
New Revision: 3167

Modified:
   trunk/src/org/griphyn/vdl/type/Type.java
   trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java
   trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java
Log:
added isComposite() to distinguish between array or struct and non-array and non-struct types

Modified: trunk/src/org/griphyn/vdl/type/Type.java
===================================================================
--- trunk/src/org/griphyn/vdl/type/Type.java	2009-10-16 20:14:48 UTC (rev 3166)
+++ trunk/src/org/griphyn/vdl/type/Type.java	2009-10-16 21:53:10 UTC (rev 3167)
@@ -52,6 +52,11 @@
 	public URI getNamespaceURI();
 
 	boolean isArray();
+	
+	/**
+	 * Returns true if this type is a composite type (array or struct).
+	 */
+	boolean isComposite();
 
 	/**
 	 * set the full name as a QName

Modified: trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java
===================================================================
--- trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java	2009-10-16 20:14:48 UTC (rev 3166)
+++ trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java	2009-10-16 21:53:10 UTC (rev 3167)
@@ -107,8 +107,13 @@
 	public void setPrimitive() {
 		primitive = true;
 	}
+	
 
-	private static class Array extends TypeImpl {
+    public boolean isComposite() {
+        return isArray() || !fields.isEmpty();
+    }
+
+    private static class Array extends TypeImpl {
 		private Field field;
 
 		/** Constructs an array that will contain elements of the

Modified: trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java
===================================================================
--- trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java	2009-10-16 20:14:48 UTC (rev 3166)
+++ trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java	2009-10-16 21:53:10 UTC (rev 3167)
@@ -86,8 +86,12 @@
 	public boolean isArray() {
 		return this.name.endsWith("[]");
 	}
+	
+	public boolean isComposite() {
+        return isArray();
+    }
 
-	public void setBaseType(Type base) {
+    public void setBaseType(Type base) {
 		throw new UnsupportedOperationException();
 	}
 



From noreply at svn.ci.uchicago.edu  Fri Oct 16 16:53:39 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 16 Oct 2009 16:53:39 -0500 (CDT)
Subject: [Swift-commit] r3168 - trunk/src/org/griphyn/vdl/mapping
Message-ID: <20091016215339.200439CC67@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-16 16:53:38 -0500 (Fri, 16 Oct 2009)
New Revision: 3168

Modified:
   trunk/src/org/griphyn/vdl/mapping/Path.java
Log:
cache toString()

Modified: trunk/src/org/griphyn/vdl/mapping/Path.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/Path.java	2009-10-16 21:53:10 UTC (rev 3167)
+++ trunk/src/org/griphyn/vdl/mapping/Path.java	2009-10-16 21:53:38 UTC (rev 3168)
@@ -19,6 +19,8 @@
 
 	/** True if any element of the Path contains a wildcard. */
 	private boolean wildcard;
+	
+	private String tostrcached;
 
 	public static class EmptyPath extends Path {
 		public EmptyPath() {
@@ -245,7 +247,10 @@
 	 * made that Path.parse(somePath.stringForm()).equals(somePath).
 	 * For a consistent such representation of this path use {@link stringForm}.
 	 */
-	public String toString() {
+	public synchronized String toString() {
+	    if (tostrcached != null) {
+	        return tostrcached;
+	    }
 		StringBuffer sb = new StringBuffer();
 		Iterator i = iterator();
 		while (i.hasNext()) {
@@ -260,7 +265,7 @@
 				sb.append(e.getName());
 			}
 		}
-		return sb.toString();
+		return tostrcached = sb.toString();
 	}
 
 	public Iterator iterator() {



From noreply at svn.ci.uchicago.edu  Fri Oct 16 16:54:09 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 16 Oct 2009 16:54:09 -0500 (CDT)
Subject: [Swift-commit] r3169 - in trunk/src/org/griphyn/vdl/mapping: . file
Message-ID: <20091016215409.7F6BE9CC67@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-16 16:54:09 -0500 (Fri, 16 Oct 2009)
New Revision: 3169

Modified:
   trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java
   trunk/src/org/griphyn/vdl/mapping/Mapper.java
   trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java
Log:
re-mappable mappers

Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java	2009-10-16 21:53:38 UTC (rev 3168)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java	2009-10-16 21:54:09 UTC (rev 3169)
@@ -51,4 +51,11 @@
 		return r;
 	}
 
+	public boolean canBeRemapped(Path path) {
+	    return false;
+	}
+
+    public void remap(Path path, PhysicalFormat file) {
+        throw new UnsupportedOperationException();
+    }
 }

Modified: trunk/src/org/griphyn/vdl/mapping/Mapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/Mapper.java	2009-10-16 21:53:38 UTC (rev 3168)
+++ trunk/src/org/griphyn/vdl/mapping/Mapper.java	2009-10-16 21:54:09 UTC (rev 3169)
@@ -43,4 +43,12 @@
     void setParam(String name, Object value);
 
     Object getParam(String name);
+
+    /**
+     * Specifies whether paths mapped by this mapper can be re-mapped
+     * if needed (such as when aliasing another variable)
+     */
+    boolean canBeRemapped(Path path);
+    
+    void remap(Path path, PhysicalFormat file);
 }

Modified: trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java	2009-10-16 21:53:38 UTC (rev 3168)
+++ trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java	2009-10-16 21:54:09 UTC (rev 3169)
@@ -3,12 +3,20 @@
  */
 package org.griphyn.vdl.mapping.file;
 
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.griphyn.vdl.mapping.MappingParam;
+import org.griphyn.vdl.mapping.Path;
+import org.griphyn.vdl.mapping.PhysicalFormat;
 
 public class ConcurrentMapper extends AbstractFileMapper {
 	public static final MappingParam PARAM_THREAD_PREFIX = new MappingParam("thread_prefix", "");
+	
+	private Map remappedPaths;
 
 	public ConcurrentMapper() {
 		super(new ConcurrentElementMapper());
@@ -20,5 +28,42 @@
 		PARAM_PREFIX.setValue(params, prefix);
 		super.setParams(params);
 	}
+	
+    public synchronized Collection existing() {
+        Collection c = super.existing();
+        if (remappedPaths != null) {
+            Set s = new HashSet(c);
+            s.add(remappedPaths.keySet());
+            return s;
+        }
+        else {
+            return c;
+        }
+    }
+
+    public synchronized PhysicalFormat map(Path path) {
+        if (remappedPaths != null) {
+            Object o = remappedPaths.get(path);
+            if (o != null) {
+                return (PhysicalFormat) o;
+            }
+        }
+        return super.map(path);
+    }
+
+    public Path rmap(String name) {
+        throw new UnsupportedOperationException();
+    }
+
+    public boolean canBeRemapped(Path path) {
+        return true;
+    }
+
+    public synchronized void remap(Path path, PhysicalFormat file) {
+        if (remappedPaths == null) {
+            remappedPaths = new HashMap();
+        }
+        remappedPaths.put(path, file);
+    }
 }
 



From noreply at svn.ci.uchicago.edu  Fri Oct 16 16:54:52 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 16 Oct 2009 16:54:52 -0500 (CDT)
Subject: [Swift-commit] r3170 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20091016215452.1ECC29CC67@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-16 16:54:51 -0500 (Fri, 16 Oct 2009)
New Revision: 3170

Added:
   trunk/src/org/griphyn/vdl/karajan/lib/FileCopier.java
Modified:
   trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
Log:
handle non-composite assignments

Added: trunk/src/org/griphyn/vdl/karajan/lib/FileCopier.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/FileCopier.java	                        (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/FileCopier.java	2009-10-16 21:54:51 UTC (rev 3170)
@@ -0,0 +1,145 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Oct 16, 2009
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.globus.cog.abstraction.impl.common.StatusEvent;
+import org.globus.cog.abstraction.impl.common.task.FileTransferSpecificationImpl;
+import org.globus.cog.abstraction.impl.common.task.FileTransferTask;
+import org.globus.cog.abstraction.impl.common.task.FileTransferTaskHandler;
+import org.globus.cog.abstraction.impl.common.task.IllegalSpecException;
+import org.globus.cog.abstraction.impl.common.task.InvalidSecurityContextException;
+import org.globus.cog.abstraction.impl.common.task.InvalidServiceContactException;
+import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl;
+import org.globus.cog.abstraction.impl.common.task.ServiceImpl;
+import org.globus.cog.abstraction.impl.common.task.TaskSubmissionException;
+import org.globus.cog.abstraction.interfaces.FileTransferSpecification;
+import org.globus.cog.abstraction.interfaces.Service;
+import org.globus.cog.abstraction.interfaces.Status;
+import org.globus.cog.abstraction.interfaces.StatusListener;
+import org.globus.cog.abstraction.interfaces.TaskHandler;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventBus;
+import org.globus.cog.karajan.workflow.events.EventListener;
+import org.globus.cog.karajan.workflow.events.EventTargetPair;
+import org.globus.cog.karajan.workflow.futures.Future;
+import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
+import org.globus.cog.karajan.workflow.futures.FuturesMonitor;
+import org.griphyn.vdl.mapping.AbsFile;
+import org.griphyn.vdl.mapping.PhysicalFormat;
+
+public class FileCopier implements Future, StatusListener {
+    private static final TaskHandler fth = new FileTransferTaskHandler();
+
+    private FileTransferTask task;
+    private List actions;
+    private Exception exception;
+    private boolean closed;
+
+    public FileCopier(PhysicalFormat src, PhysicalFormat dst) {
+        AbsFile fsrc = (AbsFile) src;
+        AbsFile fdst = (AbsFile) dst;
+        FileTransferSpecification fts = new FileTransferSpecificationImpl();
+        fts.setDestinationDirectory(fdst.getDir());
+        fts.setDestinationFile(fdst.getPath());
+        fts.setSourceDirectory(fsrc.getDir());
+        fts.setSourceFile(fsrc.getPath());
+        fts.setThirdPartyIfPossible(true);
+        task = new FileTransferTask();
+        task.setSpecification(fts);
+        task.setService(Service.FILE_TRANSFER_SOURCE_SERVICE, new ServiceImpl(
+            fsrc.getProtocol(), new ServiceContactImpl(fsrc.getHost()), null));
+        task.setService(Service.FILE_TRANSFER_DESTINATION_SERVICE,
+            new ServiceImpl(fdst.getProtocol(), new ServiceContactImpl(fdst
+                .getHost()), null));
+        task.addStatusListener(this);
+    }
+
+    public synchronized void addModificationAction(EventListener target,
+            Event event) {
+        if (actions == null) {
+            actions = new LinkedList();
+        }
+        EventTargetPair etp = new EventTargetPair(event, target);
+        if (FuturesMonitor.debug) {
+            FuturesMonitor.monitor.add(etp, this);
+        }
+        synchronized (actions) {
+            actions.add(etp);
+        }
+        if (closed) {
+            actions();
+        }
+    }
+
+    public List getModificationActions() {
+        return actions;
+    }
+
+    private void actions() {
+        if (actions != null) {
+            synchronized (actions) {
+                java.util.Iterator i = actions.iterator();
+                while (i.hasNext()) {
+                    EventTargetPair etp = (EventTargetPair) i.next();
+                    if (FuturesMonitor.debug) {
+                        FuturesMonitor.monitor.remove(etp);
+                    }
+                    i.remove();
+                    EventBus.post(etp.getTarget(), etp.getEvent());
+                }
+            }
+        }
+    }
+
+    public void fail(FutureEvaluationException e) {
+        this.exception = e;
+        actions();
+    }
+
+    public Object getValue() throws ExecutionException {
+        return null;
+    }
+
+    public boolean isClosed() {
+        return closed;
+    }
+
+    public void start() throws IllegalSpecException,
+            InvalidSecurityContextException, InvalidServiceContactException,
+            TaskSubmissionException {
+        fth.submit(task);
+    }
+
+    public void close() {
+        closed = true;
+        actions();
+    }
+
+    public void statusChanged(StatusEvent event) {
+        Status s = event.getStatus();
+        if (s.isTerminal()) {
+            if (s.getStatusCode() == Status.COMPLETED) {
+                close();
+            }
+            else {
+                this.exception = new Exception(s.getMessage(), s.getException());
+                close();
+            }
+        }
+    }
+    
+    public Exception getException() {
+        return exception;
+    }
+}

Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java	2009-10-16 21:54:09 UTC (rev 3169)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java	2009-10-16 21:54:51 UTC (rev 3170)
@@ -4,16 +4,19 @@
 package org.griphyn.vdl.karajan.lib;
 
 import org.apache.log4j.Logger;
-import org.griphyn.vdl.karajan.Pair;
-import org.griphyn.vdl.karajan.PairIterator;
 import org.globus.cog.karajan.arguments.Arg;
 import org.globus.cog.karajan.stack.VariableStack;
 import org.globus.cog.karajan.workflow.ExecutionException;
 import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
+import org.griphyn.vdl.karajan.Pair;
+import org.griphyn.vdl.karajan.PairIterator;
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.InvalidPathException;
 import org.griphyn.vdl.mapping.Path;
 
+import edu.emory.mathcs.backport.java.util.concurrent.ExecutorService;
+import edu.emory.mathcs.backport.java.util.concurrent.Executors;
+
 public class SetFieldValue extends VDLFunction {
 	public static final Logger logger = Logger.getLogger(SetFieldValue.class);
 
@@ -28,21 +31,21 @@
 		try {
 			Path path = parsePath(OA_PATH.getValue(stack), stack);
 			DSHandle leaf = var.getField(path);
-			DSHandle value = (DSHandle)PA_VALUE.getValue(stack);
+			DSHandle value = (DSHandle) PA_VALUE.getValue(stack);
 			if (logger.isInfoEnabled()) {
 				logger.info("Setting " + leaf + " to " + value);
 			}
 			synchronized (var.getRoot()) {
-// TODO want to do a type check here, for runtime type checking
-// and pull out the appropriate internal value from value if it
-// is a DSHandle. There is no need (I think? maybe numerical casting?)
-// for type conversion here; but would be useful to have
-// type checking.
-				synchronized(value.getRoot()) {
-					if(!value.isClosed()) {
+            // TODO want to do a type check here, for runtime type checking
+            // and pull out the appropriate internal value from value if it
+            // is a DSHandle. There is no need (I think? maybe numerical casting?)
+            // for type conversion here; but would be useful to have
+            // type checking.
+				synchronized (value.getRoot()) {
+					if (!value.isClosed()) {
 						throw new FutureNotYetAvailable(addFutureListener(stack, value));
 					}
-					deepCopy(leaf,value,stack);
+					deepCopy(leaf, value, stack);
 				}
 			}
 			return null;
@@ -58,12 +61,12 @@
 	/** make dest look like source - if its a simple value, copy that
 	    and if its an array then recursively copy */
 	void deepCopy(DSHandle dest, DSHandle source, VariableStack stack) throws ExecutionException {
-		if(source.getType().isPrimitive()) {
+		if (source.getType().isPrimitive()) {
 			dest.setValue(source.getValue());
 		}
-		else if(source.getType().isArray()) {
+		else if (source.getType().isArray()) {
 			PairIterator it = new PairIterator(source.getArrayValue());
-			while(it.hasNext()) {
+			while (it.hasNext()) {
 				Pair pair = (Pair) it.next();
 				Object lhs = pair.get(0);
 				DSHandle rhs = (DSHandle) pair.get(1);
@@ -77,16 +80,56 @@
 				DSHandle field;
 				try {
 					field = dest.getField(memberPath);
-				} catch(InvalidPathException ipe) {
+				} 
+				catch (InvalidPathException ipe) {
 					throw new ExecutionException("Could not get destination field",ipe);
 				}
-				deepCopy(field,rhs,stack);
+				deepCopy(field, rhs, stack);
 			}
 			closeShallow(stack, dest);
-		} else {
-			// TODO implement this
-			throw new RuntimeException("Deep non-array structure copying not implemented, when trying to copy "+source);
+		} 
+		else if (!source.getType().isComposite()) {
+		    Path dpath = dest.getPathFromRoot();
+		    if (dest.getMapper().canBeRemapped(dpath)) {
+		        if (logger.isDebugEnabled()) {
+		            logger.debug("Remapping " + dest + " to " + source);
+		        }
+		        dest.getMapper().remap(dpath, source.getMapper().map(source.getPathFromRoot()));
+		        dest.closeShallow();
+		    }
+		    else {
+		        if (stack.currentFrame().isDefined("fc")) {
+		            FileCopier fc = (FileCopier) stack.currentFrame().getVar("fc");
+		            if (!fc.isClosed()) {
+		                throw new FutureNotYetAvailable(fc);
+		            }
+		            else {
+		                if (fc.getException() != null) {
+		                    throw new ExecutionException("Failed to copy " + source + " to " + dest, fc.getException());
+		                }
+		            }
+		        }
+		        else {
+		            FileCopier fc = new FileCopier(source.getMapper().map(source.getPathFromRoot()), 
+		                dest.getMapper().map(dpath));
+		            stack.setVar("fc", fc);
+		            try {
+		                fc.start();
+		                throw new FutureNotYetAvailable(fc);
+		            }
+		            catch (FutureNotYetAvailable e) {
+		                throw e;
+		            }
+		            catch (Exception e) {
+		                throw new ExecutionException("Failed to start file copy", e);
+		            }
+		        }
+		    }
 		}
+		else {
+		    // TODO implement this
+            //throw new RuntimeException("Deep non-array structure copying not implemented, when trying to copy "+source);
+		}
 	}
 
 }



From noreply at svn.ci.uchicago.edu  Tue Oct 20 15:19:33 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 20 Oct 2009 15:19:33 -0500 (CDT)
Subject: [Swift-commit] r3171 - trunk/libexec
Message-ID: <20091020201933.43BA79CC67@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-20 15:19:32 -0500 (Tue, 20 Oct 2009)
New Revision: 3171

Modified:
   trunk/libexec/vdl-sc.k
Log:
added env element; makes more sense than a profile in the env namespace

Modified: trunk/libexec/vdl-sc.k
===================================================================
--- trunk/libexec/vdl-sc.k	2009-10-16 21:54:51 UTC (rev 3170)
+++ trunk/libexec/vdl-sc.k	2009-10-20 20:19:32 UTC (rev 3171)
@@ -51,6 +51,10 @@
 					property("{namespace}:{key}", value)
 				)
 			)
+			
+			element(env, [name, value]
+				property("env:{name}", value)
+			)
 
 			element(workdirectory, [dir]
 				property(name="workdir", value = dir)



From noreply at svn.ci.uchicago.edu  Fri Oct 23 15:29:15 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 23 Oct 2009 15:29:15 -0500 (CDT)
Subject: [Swift-commit] r3172 - trunk/libexec
Message-ID: <20091023202915.75A1B9CC89@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-23 15:29:15 -0500 (Fri, 23 Oct 2009)
New Revision: 3172

Modified:
   trunk/libexec/vdl-int.k
Log:
pass (site) profile to the cleanup job

Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k	2009-10-20 20:19:32 UTC (rev 3171)
+++ trunk/libexec/vdl-int.k	2009-10-23 20:29:15 UTC (rev 3172)
@@ -188,7 +188,7 @@
 						vdl:siteprofile(host, "swift:cleanupCommandOptions"),
 						dir
 					)
-					host=host, batch=true)
+					host=host, batch=true, tcprofile(host))
 			)
 			log(LOG:INFO, "END dir={dir} host={host}")
 		)
@@ -472,7 +472,7 @@
 								directory=wfdir
 								redirect=false
 								host=rhost
-								vdl:tcprofile(tr, rhost) //this gets various app params from the tc, such as environment, walltime, etc
+								vdl:tcprofile(rhost, tr=tr) //this gets various app params from the tc, such as environment, walltime, etc
 								replicationGroup=replicationGroup
 								replicationChannel=replicationChannel
 								jobid=jobid
@@ -500,7 +500,7 @@
 							directory=wfdir
 							redirect=false
 							host=rhost
-							vdl:tcprofile(tr, rhost) //this gets various app params from the tc, such as environment, walltime, etc
+							vdl:tcprofile(rhost, tr=tr) //this gets various app params from the tc, such as environment, walltime, etc
 							replicationGroup=replicationGroup
 							replicationChannel=replicationChannel
 							jobid=jobid



From noreply at svn.ci.uchicago.edu  Fri Oct 23 15:31:00 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 23 Oct 2009 15:31:00 -0500 (CDT)
Subject: [Swift-commit] r3173 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20091023203100.43E609CC89@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-23 15:30:59 -0500 (Fri, 23 Oct 2009)
New Revision: 3173

Modified:
   trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java
Log:
if tr not specified, return site profile only

Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java	2009-10-23 20:29:15 UTC (rev 3172)
+++ trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java	2009-10-23 20:30:59 UTC (rev 3173)
@@ -29,11 +29,11 @@
 public class TCProfile extends VDLFunction {
     public static final Logger logger = Logger.getLogger(TCProfile.class);
     
-	public static final Arg PA_TR = new Arg.Positional("tr");
+	public static final Arg OA_TR = new Arg.Optional("tr");
 	public static final Arg PA_HOST = new Arg.Positional("host");
 	
 	static {
-		setArguments(TCProfile.class, new Arg[] { PA_TR, PA_HOST });
+		setArguments(TCProfile.class, new Arg[] { PA_HOST, OA_TR });
 	}
 
 	private static Map PROFILE_T;
@@ -53,7 +53,10 @@
 
 	public Object function(VariableStack stack) throws ExecutionException {
 		TCCache tc = getTC(stack);
-		String tr = TypeUtil.toString(PA_TR.getValue(stack));
+		String tr = null;
+		if (OA_TR.isPresent(stack)) {
+		    tr = TypeUtil.toString(OA_TR.getValue(stack));
+		}
 		BoundContact bc = (BoundContact) PA_HOST.getValue(stack);
 		
 		NamedArguments named = ArgUtil.getNamedReturn(stack);
@@ -61,7 +64,10 @@
 		
 		attrs = attributesFromHost(bc, attrs, named);
 
-		TransformationCatalogEntry tce = getTCE(tc, new FQN(tr), bc);
+		TransformationCatalogEntry tce = null;
+		if (tr != null) {
+		    getTCE(tc, new FQN(tr), bc);
+		}
 		
 		Map env = new HashMap();
 		if (tce != null) {



From noreply at svn.ci.uchicago.edu  Sun Oct 25 15:49:27 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 25 Oct 2009 15:49:27 -0500 (CDT)
Subject: [Swift-commit] r3174 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20091025204927.75CC69CC98@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-25 15:49:27 -0500 (Sun, 25 Oct 2009)
New Revision: 3174

Modified:
   trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java
Log:
complain if requested profile is missing

Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java	2009-10-23 20:30:59 UTC (rev 3173)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java	2009-10-25 20:49:27 UTC (rev 3174)
@@ -38,7 +38,7 @@
 
 	public Object function(VariableStack stack) throws ExecutionException {
 		BoundContact bc = (BoundContact) PA_HOST.getValue(stack);
-		return getSingle(bc, new FQN(TypeUtil.toString(PA_FQN.getValue(stack))));
+		return getSingle(stack, bc, new FQN(TypeUtil.toString(PA_FQN.getValue(stack))));
 	}
 	
 	public static final FQN SWIFT_WRAPPER_INTERPRETER = new FQN("swift:wrapperInterpreter");
@@ -91,7 +91,7 @@
 		addDefault(null, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"-rf"});
 	}
 	
-	private Object getSingle(BoundContact bc, FQN fqn) {
+	private Object getSingle(VariableStack stack, BoundContact bc, FQN fqn) throws ExecutionException {
             String value = getProfile(bc, fqn);
             if (value == null) {
             	Os os = getOS(bc);
@@ -102,7 +102,7 @@
                 	return os;
                 }
                 else {
-                	return null;
+                	throw new ExecutionException(stack, "Missing profile: " + fqn);
                 }
             }
             else {



From noreply at svn.ci.uchicago.edu  Sun Oct 25 15:50:06 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 25 Oct 2009 15:50:06 -0500 (CDT)
Subject: [Swift-commit] r3175 - trunk/libexec
Message-ID: <20091025205006.3EAAC9CC98@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-25 15:50:06 -0500 (Sun, 25 Oct 2009)
New Revision: 3175

Modified:
   trunk/libexec/vdl-sc.k
Log:
added scratch dir

Modified: trunk/libexec/vdl-sc.k
===================================================================
--- trunk/libexec/vdl-sc.k	2009-10-25 20:49:27 UTC (rev 3174)
+++ trunk/libexec/vdl-sc.k	2009-10-25 20:50:06 UTC (rev 3175)
@@ -59,6 +59,10 @@
 			element(workdirectory, [dir]
 				property(name="workdir", value = dir)
 			)
+			
+			element(scratch, [dir]
+				property(name="scratch", value = dir)
+			)
 
 			resources(
 				executeFile(file)



From noreply at svn.ci.uchicago.edu  Sun Oct 25 15:51:37 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 25 Oct 2009 15:51:37 -0500 (CDT)
Subject: [Swift-commit] r3176 - trunk/libexec
Message-ID: <20091025205137.5D4D19CC98@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-25 15:51:37 -0500 (Sun, 25 Oct 2009)
New Revision: 3176

Modified:
   trunk/libexec/_swiftwrap
   trunk/libexec/_swiftwrap.vbs
   trunk/libexec/vdl-int.k
Log:
scratch directory

Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap	2009-10-25 20:50:06 UTC (rev 3175)
+++ trunk/libexec/_swiftwrap	2009-10-25 20:51:37 UTC (rev 3176)
@@ -144,11 +144,20 @@
 	shift $SHIFTCOUNT
 fi
 
+getarg "-scratch" "$@"
+SCRATCH=$VALUE
+shift $SHIFTCOUNT
+
+if [ "X$PROGRESSIVE_INFO" == "X" ]; then
+	INFODIR=$SCRATCH/info/$JOBDIR
+else
+	INFODIR=$WFDIR/info/$JOBDIR
+fi
 checkEmpty "$JOBDIR" "Missing job directory prefix"
-mkdir -p $WFDIR/info/$JOBDIR
+mkdir -p $INFODIR
 closeinfo
-rm -f "$WFDIR/info/$JOBDIR/${ID}-info"
-openinfo "$WFDIR/info/$JOBDIR/${ID}-info"
+rm -f "$INFODIR/${ID}-info"
+openinfo "$INFODIR/${ID}-info"
 
 logstate "LOG_START"
 infosection "Wrapper"
@@ -213,9 +222,9 @@
 	mkdir -p $WFDIR/status/$JOBDIR
 fi
 
-if [ "X$SWIFT_JOBDIR_PATH" != "X" ]; then
+if [ "$WFDIR" != "$SCRATCH" ]; then
   log "Job directory mode is: local copy"
-  DIR=${SWIFT_JOBDIR_PATH}/$JOBDIR/$ID
+  DIR=$SCRATCH/$JOBDIR/$ID
   COPYNOTLINK=1
 else
   log "Job directory mode is: link on shared filesystem"
@@ -293,8 +302,18 @@
 fi
 if [ "$KICKSTART" == "" ]; then
 	if [ "$STDIN" == "" ]; then
+		if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+			echo "#!/bin/bash" > run.sh
+			echo "\"$EXEC\" \"${CMDARGS[@]}\" 1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh
+			chmod +x run.sh
+		fi
 		"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR"
 	else
+		if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+			echo "#!/bin/bash" > run.sh
+			echo "\"$EXEC\" \"${CMDARGS[@]}\" 1>\"$STDOUT\" 2>\"$STDERR\" <\"$STDIN\"" >> run.sh
+			chmod +x run.sh
+		fi
 		"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN"
 	fi
 	checkError $? "Exit code $?"
@@ -361,6 +380,10 @@
 
 closeinfo
 
-# ensure we exit with a 0 after a successful exection
+if [ "X$PROGRESSIVE_INFO" == "X" ]; then
+	mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info"
+fi
+
+# ensure we exit with a 0 after a successful execution
 exit 0
 

Modified: trunk/libexec/_swiftwrap.vbs
===================================================================
--- trunk/libexec/_swiftwrap.vbs	2009-10-25 20:50:06 UTC (rev 3175)
+++ trunk/libexec/_swiftwrap.vbs	2009-10-25 20:51:37 UTC (rev 3176)
@@ -159,6 +159,10 @@
 
 logstate "LOG_START"
 
+expectArg("scratch")
+SCRATCH = getArgVal(MandatoryArg, "job dir")
+'though we'll ignore SCRATCH here
+
 expectArg("e")
 EXEC = getArgVal(MandatoryArg, "executable")
 

Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k	2009-10-25 20:50:06 UTC (rev 3175)
+++ trunk/libexec/vdl-int.k	2009-10-25 20:51:37 UTC (rev 3176)
@@ -427,6 +427,10 @@
 				if(wrapperMode == "files"
 					sequential(
 						sys:file:write(wrapfile,
+							"-scratch", try(
+								vdl:siteprofile(rhost, "scratch")
+								vdl:siteprofile(rhost, "workdir")
+							)
 							"-e ",vdl:executable(tr, rhost), nl(),
 							"-out ",stdout,nl(),
 							"-err ",stderr,nl(),
@@ -486,6 +490,10 @@
 								dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os),
 								jobid,
 								"-jobdir", jobdir,
+								"-scratch", try(
+									vdl:siteprofile(rhost, "scratch")
+									vdl:siteprofile(rhost, "workdir")
+								)
 								"-e", vdl:executable(tr, rhost), 
 								"-out", stdout, 
 								"-err", stderr, 



From noreply at svn.ci.uchicago.edu  Sun Oct 25 18:26:40 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 25 Oct 2009 18:26:40 -0500 (CDT)
Subject: [Swift-commit] r3177 - trunk/libexec
Message-ID: <20091025232640.C09B39CC82@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-25 18:26:40 -0500 (Sun, 25 Oct 2009)
New Revision: 3177

Modified:
   trunk/libexec/_swiftwrap
Log:
fixed directories

Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap	2009-10-25 20:51:37 UTC (rev 3176)
+++ trunk/libexec/_swiftwrap	2009-10-25 23:26:40 UTC (rev 3177)
@@ -149,7 +149,7 @@
 shift $SHIFTCOUNT
 
 if [ "X$PROGRESSIVE_INFO" == "X" ]; then
-	INFODIR=$SCRATCH/info/$JOBDIR
+	INFODIR=$SCRATCH/$JOBDIR/$ID
 else
 	INFODIR=$WFDIR/info/$JOBDIR
 fi
@@ -235,20 +235,20 @@
 PATH=$PATH:/bin:/usr/bin
 
 if [ "$PATHPREFIX" != "" ]; then
-export PATH=$PATHPREFIX:$PATH
+	export PATH=$PATHPREFIX:$PATH
 fi
 
 if [ "$SWIFT_EXTRA_INFO" != "" ]; then
-  log "EXTRAINFO=$($SWIFT_EXTRA_INFO)"
+	log "EXTRAINFO=$($SWIFT_EXTRA_INFO)"
 fi
 
 if [ "X${EXEC:0:1}" != "X/" ] ; then
-export ORIGEXEC=$EXEC
-export EXEC=$(which $EXEC)
-if [ "X$EXEC" = "X" ] ; then
-fail 254 "Cannot find executable $ORIGEXEC on site system path"
+	export ORIGEXEC=$EXEC
+	export EXEC=$(which $EXEC)
+	if [ "X$EXEC" = "X" ] ; then
+		fail 254 "Cannot find executable $ORIGEXEC on site system path"
+	fi
 fi
-fi
 
 log "DIR=$DIR"
 log "EXEC=$EXEC"
@@ -279,13 +279,13 @@
 logstate "LINK_INPUTS"
 for L in $INF ; do
 	if [ $COPYNOTLINK = 1 ]; then
-		cp "$PWD/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		cp "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO
 		checkError 254 "Failed to copy input file $L"
-		log "Copied input: $PWD/shared/$L to $DIR/$L"
+		log "Copied input: $WFDIR/shared/$L to $DIR/$L"
 	else
-		ln -s "$PWD/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		ln -s "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO
 		checkError 254 "Failed to link input file $L"
-		log "Linked input: $PWD/shared/$L to $DIR/$L"
+		log "Linked input: $WFDIR/shared/$L to $DIR/$L"
 	fi
 done
 
@@ -363,7 +363,7 @@
 
 logstate "MOVING_OUTPUTS"
 for O in $OUTF ; do
-	mv "$DIR/$O" "shared/$O" 2>&1 >& "$INFO"
+	mv "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >& "$INFO"
 	checkError 254 "Failed to move output file $O to shared directory"
 done
 
@@ -373,7 +373,7 @@
 
 if [ "$STATUSMODE" = "files" ]; then
 	logstate "TOUCH_SUCCESS"
-	touch status/${JOBDIR}/${ID}-success
+	touch $WFDIR/status/${JOBDIR}/${ID}-success
 fi
 
 logstate "END"



From noreply at svn.ci.uchicago.edu  Mon Oct 26 01:22:01 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 26 Oct 2009 01:22:01 -0500 (CDT)
Subject: [Swift-commit] r3178 - trunk/libexec
Message-ID: <20091026062201.D90839CCA5@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-26 01:22:01 -0500 (Mon, 26 Oct 2009)
New Revision: 3178

Modified:
   trunk/libexec/_swiftwrap
   trunk/libexec/vdl-int.k
Log:
fixed info file location

Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap	2009-10-25 23:26:40 UTC (rev 3177)
+++ trunk/libexec/_swiftwrap	2009-10-26 06:22:01 UTC (rev 3178)
@@ -148,8 +148,8 @@
 SCRATCH=$VALUE
 shift $SHIFTCOUNT
 
-if [ "X$PROGRESSIVE_INFO" == "X" ]; then
-	INFODIR=$SCRATCH/$JOBDIR/$ID
+if [ "X$PROGRESSIVE_INFO" == "X" && "X$SCRATCH" != "X" ]; then
+	INFODIR=$SCRATCH
 else
 	INFODIR=$WFDIR/info/$JOBDIR
 fi
@@ -380,7 +380,7 @@
 
 closeinfo
 
-if [ "X$PROGRESSIVE_INFO" == "X" ]; then
+if [ "X$PROGRESSIVE_INFO" == "X" && "X$SCRATCH" != "X" ]; then
 	mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info"
 fi
 

Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k	2009-10-25 23:26:40 UTC (rev 3177)
+++ trunk/libexec/vdl-int.k	2009-10-26 06:22:01 UTC (rev 3178)
@@ -427,10 +427,7 @@
 				if(wrapperMode == "files"
 					sequential(
 						sys:file:write(wrapfile,
-							"-scratch", try(
-								vdl:siteprofile(rhost, "scratch")
-								vdl:siteprofile(rhost, "workdir")
-							)
+							"-scratch", try(vdl:siteprofile(rhost, "scratch"), "")
 							"-e ",vdl:executable(tr, rhost), nl(),
 							"-out ",stdout,nl(),
 							"-err ",stderr,nl(),
@@ -490,10 +487,7 @@
 								dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os),
 								jobid,
 								"-jobdir", jobdir,
-								"-scratch", try(
-									vdl:siteprofile(rhost, "scratch")
-									vdl:siteprofile(rhost, "workdir")
-								)
+								"-scratch", try(vdl:siteprofile(rhost, "scratch"), "")
 								"-e", vdl:executable(tr, rhost), 
 								"-out", stdout, 
 								"-err", stderr, 



From noreply at svn.ci.uchicago.edu  Mon Oct 26 15:42:09 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 26 Oct 2009 15:42:09 -0500 (CDT)
Subject: [Swift-commit] r3179 - trunk/libexec
Message-ID: <20091026204209.316819CCA5@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-26 15:42:08 -0500 (Mon, 26 Oct 2009)
New Revision: 3179

Modified:
   trunk/libexec/vdl-sc.k
Log:
intrepid params

Modified: trunk/libexec/vdl-sc.k
===================================================================
--- trunk/libexec/vdl-sc.k	2009-10-26 06:22:01 UTC (rev 3178)
+++ trunk/libexec/vdl-sc.k	2009-10-26 20:42:08 UTC (rev 3179)
@@ -57,12 +57,29 @@
 			)
 
 			element(workdirectory, [dir]
-				property(name="workdir", value = dir)
+				property("workdir", dir)
 			)
 			
 			element(scratch, [dir]
-				property(name="scratch", value = dir)
+				property("scratch", dir)
 			)
+			
+			element(intrepidCoasterParams, [optional(averageJobTime, blockSize)]
+				property("scratch", "/scratch")
+				property("workersPerNode", "4")
+				maybe(
+					discard(averageJobTime)
+					blockSize := try(blockSize, 512)
+					parallelism := averageJobTime * 100	
+					echo("parallelism: {parallelism}")
+					property("slots", math:round(parallelism / (blockSize * 4)))
+					property("nodeGranularity", blockSize)
+					property("maxNodes", blockSize)
+					jt := parallelism * 2
+					property("jobThrottle", jt / 100)
+					print("For best results, please set the foreach.max.threads property in swift.properties to {jt}") 
+				)
+			)
 
 			resources(
 				executeFile(file)



From noreply at svn.ci.uchicago.edu  Mon Oct 26 15:48:31 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 26 Oct 2009 15:48:31 -0500 (CDT)
Subject: [Swift-commit] r3180 - trunk/libexec
Message-ID: <20091026204831.B47829CCA5@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-26 15:48:31 -0500 (Mon, 26 Oct 2009)
New Revision: 3180

Modified:
   trunk/libexec/vdl-sc.k
Log:
also print out what's being set

Modified: trunk/libexec/vdl-sc.k
===================================================================
--- trunk/libexec/vdl-sc.k	2009-10-26 20:42:08 UTC (rev 3179)
+++ trunk/libexec/vdl-sc.k	2009-10-26 20:48:31 UTC (rev 3180)
@@ -65,18 +65,22 @@
 			)
 			
 			element(intrepidCoasterParams, [optional(averageJobTime, blockSize)]
-				property("scratch", "/scratch")
-				property("workersPerNode", "4")
+				print("Setting Intrepid coaster params:")
+				element(p, [name, value]
+					print("	{name} = {value}")
+					property(name, value)
+				)
+				p("scratch", "/scratch")
+				p("workersPerNode", "4")
 				maybe(
 					discard(averageJobTime)
 					blockSize := try(blockSize, 512)
 					parallelism := averageJobTime * 100	
-					echo("parallelism: {parallelism}")
-					property("slots", math:round(parallelism / (blockSize * 4)))
-					property("nodeGranularity", blockSize)
-					property("maxNodes", blockSize)
+					p("slots", math:round(parallelism / (blockSize * 4)))
+					p("nodeGranularity", blockSize)
+					p("maxNodes", blockSize)
 					jt := parallelism * 2
-					property("jobThrottle", jt / 100)
+					p("jobThrottle", jt / 100)
 					print("For best results, please set the foreach.max.threads property in swift.properties to {jt}") 
 				)
 			)



From noreply at svn.ci.uchicago.edu  Tue Oct 27 13:52:16 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 27 Oct 2009 13:52:16 -0500 (CDT)
Subject: [Swift-commit] r3181 - trunk/libexec
Message-ID: <20091027185216.519339CC82@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-27 13:52:15 -0500 (Tue, 27 Oct 2009)
New Revision: 3181

Modified:
   trunk/libexec/_swiftwrap
Log:
bug

Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap	2009-10-26 20:48:31 UTC (rev 3180)
+++ trunk/libexec/_swiftwrap	2009-10-27 18:52:15 UTC (rev 3181)
@@ -222,7 +222,7 @@
 	mkdir -p $WFDIR/status/$JOBDIR
 fi
 
-if [ "$WFDIR" != "$SCRATCH" ]; then
+if [ "X$SCRATCH" != "X" ]; then
   log "Job directory mode is: local copy"
   DIR=$SCRATCH/$JOBDIR/$ID
   COPYNOTLINK=1



From noreply at svn.ci.uchicago.edu  Thu Oct 29 10:57:05 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 29 Oct 2009 10:57:05 -0500 (CDT)
Subject: [Swift-commit] r3182 - trunk/libexec
Message-ID: <20091029155705.5F5799CCAF@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-29 10:57:04 -0500 (Thu, 29 Oct 2009)
New Revision: 3182

Modified:
   trunk/libexec/vdl-sc.k
Log:
added a few more parameters

Modified: trunk/libexec/vdl-sc.k
===================================================================
--- trunk/libexec/vdl-sc.k	2009-10-27 18:52:15 UTC (rev 3181)
+++ trunk/libexec/vdl-sc.k	2009-10-29 15:57:04 UTC (rev 3182)
@@ -72,6 +72,8 @@
 				)
 				p("scratch", "/scratch")
 				p("workersPerNode", "4")
+				p("kernelprofile", "zeptoos")
+				p("alcfbgpnat", "true")
 				maybe(
 					discard(averageJobTime)
 					blockSize := try(blockSize, 512)



From noreply at svn.ci.uchicago.edu  Thu Oct 29 14:14:50 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 29 Oct 2009 14:14:50 -0500 (CDT)
Subject: [Swift-commit] r3183 - trunk/libexec
Message-ID: <20091029191450.A0EAF9CC82@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-29 14:14:50 -0500 (Thu, 29 Oct 2009)
New Revision: 3183

Modified:
   trunk/libexec/vdl-sc.k
Log:
also added surveyor params

Modified: trunk/libexec/vdl-sc.k
===================================================================
--- trunk/libexec/vdl-sc.k	2009-10-29 15:57:04 UTC (rev 3182)
+++ trunk/libexec/vdl-sc.k	2009-10-29 19:14:50 UTC (rev 3183)
@@ -64,12 +64,13 @@
 				property("scratch", dir)
 			)
 			
+			element(p, [name, value]
+				print("	{name} = {value}")
+				property(name, value)
+			)
+			
 			element(intrepidCoasterParams, [optional(averageJobTime, blockSize)]
 				print("Setting Intrepid coaster params:")
-				element(p, [name, value]
-					print("	{name} = {value}")
-					property(name, value)
-				)
 				p("scratch", "/scratch")
 				p("workersPerNode", "4")
 				p("kernelprofile", "zeptoos")
@@ -86,6 +87,25 @@
 					print("For best results, please set the foreach.max.threads property in swift.properties to {jt}") 
 				)
 			)
+			
+			element(surveyorCoasterParams, [optional(averageJobTime, blockSize)]
+				print("Setting Surveyor coaster params:")	
+				p("scratch", "/scratch")
+				p("workersPerNode", "4")
+				p("kernelprofile", "zeptoos")
+				p("alcfbgpnat", "true")
+				maybe(
+					discard(averageJobTime)
+					blockSize := try(blockSize, 64)
+					parallelism := averageJobTime * 100	
+					p("slots", 12)
+					p("nodeGranularity", blockSize)
+					p("maxTime", 3500)
+					jt := parallelism * 2
+					p("jobThrottle", jt / 100)
+					print("For best results, please set the foreach.max.threads property in swift.properties to {jt}") 
+				)
+			)
 
 			resources(
 				executeFile(file)



From noreply at svn.ci.uchicago.edu  Fri Oct 30 00:45:42 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 30 Oct 2009 00:45:42 -0500 (CDT)
Subject: [Swift-commit] r3184 - trunk/libexec
Message-ID: <20091030054542.114DC9CC82@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-30 00:45:41 -0500 (Fri, 30 Oct 2009)
New Revision: 3184

Modified:
   trunk/libexec/_swiftwrap
Log:
fixed conditional tests

Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap	2009-10-29 19:14:50 UTC (rev 3183)
+++ trunk/libexec/_swiftwrap	2009-10-30 05:45:41 UTC (rev 3184)
@@ -148,7 +148,7 @@
 SCRATCH=$VALUE
 shift $SHIFTCOUNT
 
-if [ "X$PROGRESSIVE_INFO" == "X" && "X$SCRATCH" != "X" ]; then
+if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then
 	INFODIR=$SCRATCH
 else
 	INFODIR=$WFDIR/info/$JOBDIR
@@ -380,7 +380,7 @@
 
 closeinfo
 
-if [ "X$PROGRESSIVE_INFO" == "X" && "X$SCRATCH" != "X" ]; then
+if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then
 	mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info"
 fi
 



From noreply at svn.ci.uchicago.edu  Fri Oct 30 19:48:43 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 30 Oct 2009 19:48:43 -0500 (CDT)
Subject: [Swift-commit] r3185 - trunk/libexec
Message-ID: <20091031004844.1E0079CCB1@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-30 19:48:42 -0500 (Fri, 30 Oct 2009)
New Revision: 3185

Modified:
   trunk/libexec/vdl-int.k
Log:
removed useless check if workflow dir exists

Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k	2009-10-30 05:45:41 UTC (rev 3184)
+++ trunk/libexec/vdl-int.k	2009-10-31 00:48:42 UTC (rev 3185)
@@ -126,6 +126,16 @@
 			)
 		)
 		
+		element(initDDir, []
+			ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d"
+			once(ddir
+				if(sys:not(file:exists(ddir))
+					task:dir:make(ddir)
+				)
+			)
+			ddir
+		)
+		
 		element(inFileDirs, [stageins]
 			for(file, stageins
 		 		reldirname(file)
@@ -360,9 +370,6 @@
 		)
 		
 		element(transferKickstartRec, [rhost, wfdir, jobid, jobdir]
-			if(sys:not(file:exists("{VDL:SCRIPTNAME}-{VDL:RUNID}.d"))
-				task:dir:make("{VDL:SCRIPTNAME}-{VDL:RUNID}.d")
-			)
 			recfile := "{jobid}-kickstart.xml"
 			srcdir := dircat(concat(wfdir, "/kickstart/"), jobdir)
 			try(
@@ -376,9 +383,6 @@
 		)
 
 		element(transferWrapperLog, [rhost, wfdir, jobid, jobdir]
-			if(sys:not(file:exists("{VDL:SCRIPTNAME}-{VDL:RUNID}.d"))
-				task:dir:make("{VDL:SCRIPTNAME}-{VDL:RUNID}.d")
-			)
 			recfile := "{jobid}-info"
 			srcdir := dircat(concat(wfdir, "/info/"), jobdir)
 			try(
@@ -398,6 +402,7 @@
 			stageout := list(unique(each(stageout)))
 			allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein)
 				
+				ddir := initDDir()
 				[wfdir, sharedDir] := try(
 					initSharedDir(rhost)
 					throw(exception("Could not initialize shared directory on {rhost}", exception))
@@ -412,10 +417,6 @@
 				statusMode := configProperty("status.mode",host=rhost)
 				wrapperMode := configProperty("wrapper.parameter.mode",host=rhost)
 
-				ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d"
-				if(sys:not(file:exists(ddir))
-					task:dir:make(ddir)
-				)
 				wrapfile := "{ddir}/param-{jobid}"
 
 				stdout := try(stdout, "stdout.txt")



From noreply at svn.ci.uchicago.edu  Sat Oct 31 14:06:57 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sat, 31 Oct 2009 14:06:57 -0500 (CDT)
Subject: [Swift-commit] r3186 - trunk/libexec
Message-ID: <20091031190658.012A49CCC5@vm-125-59.ci.uchicago.edu>

Author: hategan
Date: 2009-10-31 14:06:57 -0500 (Sat, 31 Oct 2009)
New Revision: 3186

Modified:
   trunk/libexec/_swiftwrap
Log:
fixed info stageout issue

Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap	2009-10-31 00:48:42 UTC (rev 3185)
+++ trunk/libexec/_swiftwrap	2009-10-31 19:06:57 UTC (rev 3186)
@@ -381,6 +381,7 @@
 closeinfo
 
 if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then
+	mkdir -p "$WFDIR/info/$JOBDIR"
 	mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info"
 fi