From ketan at ci.uchicago.edu Mon Dec 1 09:40:06 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 1 Dec 2014 09:40:06 -0600 (CST) Subject: [Swift-commit] r8313 - SwiftApps/ematter Message-ID: <20141201154006.B16D59D83A@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-01 09:40:05 -0600 (Mon, 01 Dec 2014) New Revision: 8313 Modified: SwiftApps/ematter/apps SwiftApps/ematter/sites.blues.xml SwiftApps/ematter/sites.cobalt.xml SwiftApps/ematter/workflow.blues.swift Log: changes in sites to shut worker logging Modified: SwiftApps/ematter/apps =================================================================== --- SwiftApps/ematter/apps 2014-11-25 23:58:38 UTC (rev 8312) +++ SwiftApps/ematter/apps 2014-12-01 15:40:05 UTC (rev 8313) @@ -5,11 +5,12 @@ cluster bgsh /home/ketan/SwiftApps/subjobs/bg.sh null null null cluster2 bgsh /home/ketan/SwiftApps/subjobs/bg.sh null null null # -blues lammps lmp_linux null null null +blues lammps lmp_blues null null null blues smeagol /home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg null null null blues bash /bin/bash null null null blues python /soft/python/2.7.6/bin/python null null null blues genfdf /home/ketan/ematter/smallrun/swift null null null +blues mpiexec mpiexec null null null # localblues lammps lmp_blues null null null localblues smeagol /home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg null null null Modified: SwiftApps/ematter/sites.blues.xml =================================================================== --- SwiftApps/ematter/sites.blues.xml 2014-11-25 23:58:38 UTC (rev 8312) +++ SwiftApps/ematter/sites.blues.xml 2014-12-01 15:40:05 UTC (rev 8313) @@ -3,14 +3,14 @@ - 2300 - 00:20:00 - 16 - 1 - 16 - 1 - 1 - 0.20 + 10400 + 02:40:00 + 1 + 2 + 8 + 4 + 4 + 2.20 10000 /tmp/swift.workdir Modified: SwiftApps/ematter/sites.cobalt.xml =================================================================== --- SwiftApps/ematter/sites.cobalt.xml 2014-11-25 23:58:38 UTC (rev 8312) +++ SwiftApps/ematter/sites.cobalt.xml 2014-12-01 15:40:05 UTC (rev 8313) @@ -15,12 +15,14 @@ 64 64 8 + 8 /home/ketan/swiftwork @@ -42,14 +44,15 @@ 64 64 8 + 8 - /home/ketan/swiftwork Modified: SwiftApps/ematter/workflow.blues.swift =================================================================== --- SwiftApps/ematter/workflow.blues.swift 2014-11-25 23:58:38 UTC (rev 8312) +++ SwiftApps/ematter/workflow.blues.swift 2014-12-01 15:40:05 UTC (rev 8313) @@ -8,7 +8,6 @@ -- Invocation of app functions == */ - type file; @@ -64,7 +63,7 @@ file _inputfdf, file _opsf, file _osicpsf, file _tipsf, file _tisicpsf ) { - mpiexec "/home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg" stdin=@_inputfdf stdout=@_out; + mpiexec "/home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg" stdin=@_inputfdf stdout=@_out; } /* == Generate fdfs app definition == */ @@ -81,7 +80,7 @@ file _opsf, file _osicpsf, file _tipsf, file _tisicpsf ) { - mpiexec "/home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg" stdin=@_inputfdf stdout=@_out; + mpiexec "/home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg" stdin=@_inputfdf stdout=@_out; } /* == Input files definitions: Make sure these files are available in the current directory == */ From ketan at ci.uchicago.edu Mon Dec 1 12:50:54 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 1 Dec 2014 12:50:54 -0600 (CST) Subject: [Swift-commit] r8314 - SwiftApps/ematter Message-ID: <20141201185054.8F7EC9D83A@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-01 12:50:54 -0600 (Mon, 01 Dec 2014) New Revision: 8314 Modified: SwiftApps/ematter/runblues Log: make runblues nohup Modified: SwiftApps/ematter/runblues =================================================================== --- SwiftApps/ematter/runblues 2014-12-01 15:40:05 UTC (rev 8313) +++ SwiftApps/ematter/runblues 2014-12-01 18:50:54 UTC (rev 8314) @@ -1,4 +1,4 @@ #! /bin/sh -/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.blues.xml -config cf -tc.file apps workflow.blues.swift +nohup /home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.blues.xml -config cf -tc.file apps workflow.blues.swift /dev/null & #$HOME/swift-0.95-main/cog/modules/swift/dist/swift-svn/bin/swift -sites.file cobalt-local.xml -config cf -tc.file apps workflow.swift From ketan at ci.uchicago.edu Tue Dec 2 09:23:14 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 2 Dec 2014 09:23:14 -0600 (CST) Subject: [Swift-commit] r8315 - SwiftApps/ematter Message-ID: <20141202152314.C8363178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-02 09:23:14 -0600 (Tue, 02 Dec 2014) New Revision: 8315 Modified: SwiftApps/ematter/genfdf SwiftApps/ematter/workflow.blues.swift Log: floating point related changes Modified: SwiftApps/ematter/genfdf =================================================================== --- SwiftApps/ematter/genfdf 2014-12-01 18:50:54 UTC (rev 8314) +++ SwiftApps/ematter/genfdf 2014-12-02 15:23:14 UTC (rev 8315) @@ -3,5 +3,5 @@ i=$1 infdf=$2 -sed -e "s/VInitial[[:space:]]* 1.D0 eV/VInitial $i eV/" -e "s/VFinal[[:space:]]* 1.D0 eV/VFinal $i eV/" $infdf +sed -e "s/VInitial[[:space:]]* 0.D0 eV/VInitial $i eV/" -e "s/VFinal[[:space:]]* 0.D0 eV/VFinal $i eV/" $infdf Modified: SwiftApps/ematter/workflow.blues.swift =================================================================== --- SwiftApps/ematter/workflow.blues.swift 2014-12-01 18:50:54 UTC (rev 8314) +++ SwiftApps/ematter/workflow.blues.swift 2014-12-02 15:23:14 UTC (rev 8315) @@ -107,8 +107,8 @@ file lammps_dump <"lammps.dump">; file fdf_proc<"input_proc.fdf">; -file smeagol_lead_out <"smeagol_out/out.smeagol.lead">; -file smeagol_transport_out<"smeagol_out/out.smeagol_transport">; +file smeagol_lead_out <"smeagol_out/smeagol.lead.out">; +file smeagol_transport_out<"smeagol_out/smeagol.transport.out">; file au3dm <"Au3.DM">; file au3hst <"Au3.HST">; @@ -118,15 +118,18 @@ file tio2auhm<"TiO2Au.HM">; file tio2audm<"TiO2Au.DM">; -file[float] smeagol_transport_outn; +file[] fdf_proc_vn; +== */ -file[float] fdf_proc_vn; +/* == unused for now +float Vstart=0.1; //== Change this value to change the starting voltage == +float Vend=1.0; //== Change this value to change the ending voltage == +float Vstep=0.1; +== */ -float Vstart=0.1; /*== Change this value to change the starting voltage ==*/ -float Vend=1.0; /*== Change this value to change the ending voltage ==*/ - -float Vstep=0.1; float Vs[]=[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]; /* == End files and variable declaration == */ @@ -165,15 +168,21 @@ In a parallel 'foreach' loop, invoke gen_fdf and smeagol for the n V calculations == */ -foreach i in [Vstart:Vend:Vstep] +foreach v,i in Vs { /* == generate fdfs for different voltages == */ -fdf_proc_vn[i]=gen_fdf(genfdf_src, i,fdf_proc); +string fdfname = strcat("input_", v, ".fdf"); +file fdf_proc_vn ; -(smeagol_transport_outn[i]) = smeagol_vn_app(au3dm, au3hst, +fdf_proc_vn = gen_fdf(genfdf_src, v, fdf_proc); + +string outname = strcat("smeagol_out/bias_", v, ".out"); +file smeagol_transport_outn ; + +smeagol_transport_outn = smeagol_vn_app(au3dm, au3hst, aupsf, ausicpsf, bulklft, bulkrgt, - fdf_proc_vn[i], opsf, + fdf_proc_vn, opsf, osicpsf, tipsf, tisicpsf); } From ketan at ci.uchicago.edu Tue Dec 2 15:37:32 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 2 Dec 2014 15:37:32 -0600 (CST) Subject: [Swift-commit] r8316 - SwiftApps/ematter Message-ID: <20141202213732.853E2178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-02 15:37:32 -0600 (Tue, 02 Dec 2014) New Revision: 8316 Modified: SwiftApps/ematter/genfdf SwiftApps/ematter/sites.local.xml SwiftApps/ematter/workflow.blues.swift Log: get an additional file as output Modified: SwiftApps/ematter/genfdf =================================================================== --- SwiftApps/ematter/genfdf 2014-12-02 15:23:14 UTC (rev 8315) +++ SwiftApps/ematter/genfdf 2014-12-02 21:37:32 UTC (rev 8316) @@ -3,5 +3,5 @@ i=$1 infdf=$2 -sed -e "s/VInitial[[:space:]]* 0.D0 eV/VInitial $i eV/" -e "s/VFinal[[:space:]]* 0.D0 eV/VFinal $i eV/" $infdf +sed -e "s/VInitial[[:space:]]* 0.D0 eV/VInitial $i eV/" -e "s/VFinal[[:space:]]* 0.D0 eV/VFinal $i eV/" -e "s/^SystemLabel.*/SystemLabel TiO2Au_$i /" $infdf Modified: SwiftApps/ematter/sites.local.xml =================================================================== --- SwiftApps/ematter/sites.local.xml 2014-12-02 15:23:14 UTC (rev 8315) +++ SwiftApps/ematter/sites.local.xml 2014-12-02 21:37:32 UTC (rev 8316) @@ -9,7 +9,7 @@ 09:55:00 35000 36000 -/tmp/swiftwork +/home/ketan/swiftwork 0.09 Modified: SwiftApps/ematter/workflow.blues.swift =================================================================== --- SwiftApps/ematter/workflow.blues.swift 2014-12-02 15:23:14 UTC (rev 8315) +++ SwiftApps/ematter/workflow.blues.swift 2014-12-02 21:37:32 UTC (rev 8316) @@ -73,7 +73,7 @@ } /* == Smeagol transport Vn app definition == */ -app (file _out) smeagol_vn_app ( file _au3dm, +app (file _trc_out, file _out) smeagol_vn_app ( file _au3dm, file _au3hst, file _aupsf, file _ausicpsf, file _bulklft, file _bulkrgt, file _inputfdf, @@ -124,6 +124,7 @@ file[] fdf_proc_vn; == */ + /* == unused for now float Vstart=0.1; //== Change this value to change the starting voltage == float Vend=1.0; //== Change this value to change the ending voltage == @@ -179,11 +180,16 @@ string outname = strcat("smeagol_out/bias_", v, ".out"); file smeagol_transport_outn ; -smeagol_transport_outn = smeagol_vn_app(au3dm, au3hst, - aupsf, ausicpsf, - bulklft, bulkrgt, - fdf_proc_vn, opsf, - osicpsf, tipsf, - tisicpsf); +file smeagol_trc_out ; + +(smeagol_trc_out, smeagol_transport_outn) = smeagol_vn_app(au3dm, au3hst, + aupsf, ausicpsf, + bulklft, bulkrgt, + fdf_proc_vn, opsf, + osicpsf, tipsf, + tisicpsf); + + } + From ketan at ci.uchicago.edu Tue Dec 2 19:57:42 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 2 Dec 2014 19:57:42 -0600 (CST) Subject: [Swift-commit] r8317 - SwiftApps/ematter Message-ID: <20141203015742.CA65B178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-02 19:57:42 -0600 (Tue, 02 Dec 2014) New Revision: 8317 Modified: SwiftApps/ematter/workflow.blues.swift Log: _ Modified: SwiftApps/ematter/workflow.blues.swift =================================================================== --- SwiftApps/ematter/workflow.blues.swift 2014-12-02 21:37:32 UTC (rev 8316) +++ SwiftApps/ematter/workflow.blues.swift 2014-12-03 01:57:42 UTC (rev 8317) @@ -131,7 +131,7 @@ float Vstep=0.1; == */ -float Vs[]=[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]; +string Vs[]=["0.1","0.2","0.3","0.4","0.5","0.6","0.7","0.8","0.9","1.0"]; /* == End files and variable declaration == */ @@ -175,7 +175,7 @@ string fdfname = strcat("input_", v, ".fdf"); file fdf_proc_vn ; -fdf_proc_vn = gen_fdf(genfdf_src, v, fdf_proc); +fdf_proc_vn = gen_fdf(genfdf_src, toFloat(v), fdf_proc); string outname = strcat("smeagol_out/bias_", v, ".out"); file smeagol_transport_outn ; From ketan at ci.uchicago.edu Wed Dec 3 15:55:57 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Wed, 3 Dec 2014 15:55:57 -0600 (CST) Subject: [Swift-commit] r8318 - SwiftApps/ematter Message-ID: <20141203215557.5740D9D587@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-03 15:55:57 -0600 (Wed, 03 Dec 2014) New Revision: 8318 Modified: SwiftApps/ematter/runbgq SwiftApps/ematter/sites.cobalt-local.xml SwiftApps/ematter/workflow.bgq.swift Log: worker logging off Modified: SwiftApps/ematter/runbgq =================================================================== --- SwiftApps/ematter/runbgq 2014-12-03 01:57:42 UTC (rev 8317) +++ SwiftApps/ematter/runbgq 2014-12-03 21:55:57 UTC (rev 8318) @@ -1,4 +1,4 @@ #! /bin/sh -/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.cobalt-local.xml -config cf -tc.file apps workflow.swift +/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.cobalt-local.xml -config cf -tc.file apps workflow.bgq.swift #$HOME/swift-0.95-main/cog/modules/swift/dist/swift-svn/bin/swift -sites.file cobalt-local.xml -config cf -tc.file apps workflow.swift Modified: SwiftApps/ematter/sites.cobalt-local.xml =================================================================== --- SwiftApps/ematter/sites.cobalt-local.xml 2014-12-03 01:57:42 UTC (rev 8317) +++ SwiftApps/ematter/sites.cobalt-local.xml 2014-12-03 21:55:57 UTC (rev 8318) @@ -16,12 +16,16 @@ 32 32 8 -DEBUG + + + 4 /home/ketan/swiftwork Modified: SwiftApps/ematter/workflow.bgq.swift =================================================================== --- SwiftApps/ematter/workflow.bgq.swift 2014-12-03 01:57:42 UTC (rev 8317) +++ SwiftApps/ematter/workflow.bgq.swift 2014-12-03 21:55:57 UTC (rev 8318) @@ -1,23 +1,22 @@ type file; // Lammps app definition -app (file _dump, file _out, - file _err) lammps_app (string _lammps_exe, file _in, +app (file _dump) lammps_app (string _lammps_exe, file _in, file _geo, file _eam) { - bgsh _lammps_exe "-in" @_in stdout=@_out stderr=@_err; + bgsh _lammps_exe "-in" @_in; } // Pizza app definition -app (file _xyz, file _out, file _err) pizza_app (file _pizza_src, file _dump) +app (file _xyz) pizza_app (file _pizza_src, file _dump) { - python @_pizza_src @_dump @_xyz stdout=@_out stderr=@_err; + python @_pizza_src @_dump @_xyz; } // Pasta app definition -app (file _fdf, file _out, file _err) pasta_app (file _pasta_src, file _xyz, file _fdf_tmp) +app (file _fdf) pasta_app (file _pasta_src, file _xyz, file _fdf_tmp) { - bash @_pasta_src @_xyz @_fdf_tmp @_fdf stdout=@_out stderr=@_err; + bash @_pasta_src @_xyz @_fdf_tmp @_fdf; } /* smeagol lead inputs @@ -27,12 +26,12 @@ //Smeagol lead app definition app (file _au3dm, file _au3hst, file _bulklft, file _bulkrgt, - file _out, file _err) smeagol_lead_app (string _smeagol_exe, file _aupsf, + file _out) smeagol_lead_app (string _smeagol_exe, file _aupsf, file _ausicpsf, file _inputfdf, file _opsf, file _osicpsf, file _tipsf, file _tisicpsf) { - bgsh _smeagol_exe stdin=@_inputfdf stdout=@_out stderr=@_err; + bgsh _smeagol_exe stdin=@_inputfdf stdout=@_out; } /* smeagol transport V0 inputs @@ -43,89 +42,119 @@ //Smeagol transport V0 app definition app (file _tio2auhm, file _tio2audm, - file _out, file _err) smeagol_v0_app ( string _smeagol_exe, file _au3dm, + file _out) smeagol_v0_app ( string _smeagol_exe, file _au3dm, file _au3hst, file _aupsf, file _ausicpsf, file _bulklft, file _bulkrgt, file _inputfdf, file _opsf, file _osicpsf, file _tipsf, file _tisicpsf ) { - bgsh _smeagol_exe stdin=@_inputfdf stdout=@_out stderr=@_err; + bgsh _smeagol_exe stdin=@_inputfdf stdout=@_out; } -//Smeagol transport Vn app definition -//app () smeagol_vn_app (){} +/* == Generate fdfs app definition == */ +app (file _fdf) gen_fdf(file _genfdf_src, float _r, file _infdf) +{ + bash @_genfdf_src _r @_infdf stdout=@_fdf; +} -// Files +/* == Smeagol transport Vn app definition == */ +app (file _trc_out, file _out) smeagol_vn_app (string _smeagol_exe, file _au3dm, + file _au3hst, file _aupsf, + file _ausicpsf, file _bulklft, + file _bulkrgt, file _inputfdf, + file _opsf, file _osicpsf, + file _tipsf, file _tisicpsf ) +{ + bgsh _smeagol_exe stdin=@_inputfdf stdout=@_out; +} + +/* == Input files definitions: Make sure these files are available in the current directory == */ file lammps_in <"in.lammps">; file lammps_geo <"AuTiO.geo">; file lammps_eam <"Au_u3.eam">; -file lammps_out <"out.lammps">; -file lammps_err <"err.lammps">; -file lammps_dump <"lammps.dump">; +file fdf_leads<"input_leads.fdf">; +file fdf_v0<"input_t0.fdf">; -string lammps_exe="/soft/applications/lammps/24Apr13/lmp_bgq_xlomp"; +file aupsf <"Au.psf">; +file ausicpsf <"Au.sic.psf">; +file opsf <"O.psf">; +file osicpsf <"O.sic.psf">; +file tipsf <"Ti.psf">; +file tisicpsf <"Ti.sic.psf">; -//Invoke lammps -(lammps_dump, lammps_out, lammps_err) = lammps_app (lammps_exe, lammps_in, - lammps_geo, lammps_eam); - file pizza_src<"lastlammpsdump2xyz.py">; +file pasta_src<"pasta">; +file genfdf_src<"genfdf">; -file pizza_out <"out.pizza">; -file pizza_err <"err.pizza">; -file xyz <"forsmeagol.xyz">; +string lammps_exe="/soft/applications/lammps/24Apr13/lmp_bgq_xlomp"; +string smeagol_exe="/gpfs/mira-fs1/projects/Oxygen_defects_ALCC/share/bin/smeagol-1.2_csg"; -//Invoke pizza -(xyz, pizza_out, pizza_err) = pizza_app(pizza_src, lammps_dump); -file fdf_in<"input_leads.fdf">; +/* == Output files definitions == */ +file xyz <"forsmeagol.xyz">; +file lammps_dump <"lammps.dump">; +file fdf_proc<"input_proc.fdf">; -file pasta_src<"pasta">; - -file smeagol_lead_out <"out.smeagol.lead">; -file smeagol_lead_err <"err.smeagol.lead">; +file smeagol_lead_out <"smeagol_out/smeagol.lead.out">; +file smeagol_transport_out<"smeagol_out/smeagol.transport.out">; file au3dm <"Au3.DM">; file au3hst <"Au3.HST">; + file bulklft <"bulklft.DAT">; file bulkrgt <"bulkrgt.DAT">; -file aupsf <"Au.psf">; -file ausicpsf <"Au.sic.psf">; -file opsf <"O.psf">; -file osicpsf <"O.sic.psf">; -file tipsf <"Ti.psf">; -file tisicpsf <"Ti.sic.psf">; +file tio2auhm<"TiO2Au.HM">; +file tio2audm<"TiO2Au.DM">; -string smeagol_exe="/gpfs/mira-fs1/projects/Oxygen_defects_ALCC/share/bin/smeagol-1.2_csg"; +string Vs[]=["0.1","0.2","0.3","0.4","0.5","0.6","0.7","0.8","0.9","1.0"]; +//Invoke lammps +(lammps_dump) = lammps_app (lammps_exe, lammps_in, + lammps_geo, lammps_eam); + +//Invoke pizza +(xyz) = pizza_app(pizza_src, lammps_dump); + //Invoke smeagol lead (au3dm, au3hst, bulklft, bulkrgt, - smeagol_lead_out, - smeagol_lead_err) = smeagol_lead_app (smeagol_exe, aupsf, - ausicpsf, fdf_in, + smeagol_lead_out) = smeagol_lead_app (smeagol_exe, aupsf, + ausicpsf, fdf_leads, opsf, osicpsf, tipsf, tisicpsf); -file fdf_v0<"input_t0.fdf">; -file fdf_proc<"input_proc_t0.fdf">; -file pasta_out<"out.pasta">; -file pasta_err<"err.pasta">; -//Invoke pasta -(fdf_proc, pasta_out, pasta_err) = pasta_app(pasta_src, xyz, fdf_v0); - -file smeagol_transport_out<"out.smeagol_transport">; -file smeagol_transport_err<"err.smeagol_transport">; -file tio2auhm<"TiO2Au.HM">; -file tio2audm<"TiO2Au.DM">; //Invoke smeagol transport V0 (tio2auhm, tio2audm, - smeagol_transport_out, smeagol_transport_err) = smeagol_v0_app(smeagol_exe, au3dm, - au3hst, aupsf, ausicpsf, - bulklft, bulkrgt, - fdf_proc, opsf, osicpsf, - tipsf, tisicpsf); + smeagol_transport_out) = smeagol_v0_app(smeagol_exe, au3dm, + au3hst, aupsf, ausicpsf, + bulklft, bulkrgt, + fdf_v0, opsf, osicpsf, + tipsf, tisicpsf); +//Invoke pasta +(fdf_proc) = pasta_app(pasta_src, xyz, fdf_v0); + +foreach v,i in Vs +{ +/* == generate fdfs for different voltages == */ +string fdfname = strcat("input_", v, ".fdf"); +file fdf_proc_vn ; + +fdf_proc_vn = gen_fdf(genfdf_src, toFloat(v), fdf_proc); + +string outname = strcat("smeagol_out/bias_", v, ".out"); +file smeagol_transport_outn ; + +file smeagol_trc_out ; + +(smeagol_trc_out, smeagol_transport_outn) = smeagol_vn_app(smeagol_exe, au3dm, au3hst, + aupsf, ausicpsf, bulklft, bulkrgt, + fdf_proc_vn, opsf, osicpsf, tipsf, + tisicpsf); + + +} + From ketan at ci.uchicago.edu Mon Dec 8 10:25:24 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 8 Dec 2014 10:25:24 -0600 (CST) Subject: [Swift-commit] r8319 - SwiftApps/subjobs Message-ID: <20141208162525.658A29D09F@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-08 10:25:23 -0600 (Mon, 08 Dec 2014) New Revision: 8319 Modified: SwiftApps/subjobs/bg.sh Log: Modified: SwiftApps/subjobs/bg.sh =================================================================== --- SwiftApps/subjobs/bg.sh 2014-12-03 21:55:57 UTC (rev 8318) +++ SwiftApps/subjobs/bg.sh 2014-12-08 16:25:23 UTC (rev 8319) @@ -53,16 +53,18 @@ #Some logging echo "$0": running BLOCK="$COBALT_PARTNAME" SLOT="$SWIFT_JOB_SLOT" echo "$0": running cmd: "$0" args: "$@" - echo "$0": running runjob --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" --ranks-per-node 1 : "$REMAINING_ARGS" + echo "$0": running runjob --strace none --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" #without timeout - runjob --strace 0 --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$SUBBLOCK_SIZE" : "$@" + runjob --strace none --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" echo "Runjob finished." else # run w/o subblocks if no subblock size provided echo "Running in nonsubblock mode." - runjob --strace 0 --block "$COBALT_PARTNAME" -p 16 : "$@" + echo "$0": running runjob -p 16 --block $COBALT_PARTNAME : "$@" + runjob -p 16 --block $COBALT_PARTNAME : "$@" + echo "Finished Running in nonsubblock mode." fi exit 0 From ketan at ci.uchicago.edu Wed Dec 10 15:40:08 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Wed, 10 Dec 2014 15:40:08 -0600 (CST) Subject: [Swift-commit] r8320 - in SwiftApps: . dsp-swift dsp-swift/galaxy Message-ID: <20141210214008.3F1AA9D583@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-10 15:40:07 -0600 (Wed, 10 Dec 2014) New Revision: 8320 Added: SwiftApps/dsp-swift/ SwiftApps/dsp-swift/apps SwiftApps/dsp-swift/cf SwiftApps/dsp-swift/dsp.blues.swift SwiftApps/dsp-swift/galaxy/ SwiftApps/dsp-swift/galaxy/README SwiftApps/dsp-swift/galaxy/setup.py SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml SwiftApps/dsp-swift/runblues SwiftApps/dsp-swift/runlocal SwiftApps/dsp-swift/sites.blues.xml SwiftApps/dsp-swift/sites.local.xml Log: adding dsp and galaxy app Added: SwiftApps/dsp-swift/apps =================================================================== --- SwiftApps/dsp-swift/apps (rev 0) +++ SwiftApps/dsp-swift/apps 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,22 @@ +localhost cat /bin/cat null null null +localhost python /usr/bin/python null null null +localhost bash /bin/bash null null null +# +cluster bgsh /home/ketan/SwiftApps/subjobs/bg.sh null null null +cluster2 bgsh /home/ketan/SwiftApps/subjobs/bg.sh null null null +# +blues dsp /lcrc/project/NEXTGENOPT/DSP/Dsp null null null +# +blues lammps lmp_blues null null null +blues smeagol /home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg null null null +blues bash /bin/bash null null null +blues python /soft/python/2.7.6/bin/python null null null +blues genfdf /home/ketan/ematter/smallrun/swift null null null +blues mpiexec mpiexec null null null +# +localblues lammps lmp_blues null null null +localblues smeagol /home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg null null null +localblues mpiexec mpiexec null null null +localblues bash /bin/bash null null null +localblues python /soft/python/2.7.6/bin/python null null null +localblues dsp /lcrc/project/NEXTGENOPT/DSP/Dsp null null null Added: SwiftApps/dsp-swift/cf =================================================================== --- SwiftApps/dsp-swift/cf (rev 0) +++ SwiftApps/dsp-swift/cf 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,7 @@ +use.provider.staging=false +wrapperlog.always.transfer=false +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=false +use.wrapper.staging=false + Added: SwiftApps/dsp-swift/dsp.blues.swift =================================================================== --- SwiftApps/dsp-swift/dsp.blues.swift (rev 0) +++ SwiftApps/dsp-swift/dsp.blues.swift 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,45 @@ +/* == +This is a Swift workflow script for the DSP application. The script contains the following parts: + +-- A one-line type declaration +-- App function definition +-- Input files and variables definitions +-- Output files definitions +-- Invocation of app function +== */ + +type file; + +/* == App function definitions == */ + +/* == DSP app definition == */ +app (file _out, file _err) dsp_app (string _filespath, file _sto, file _cor, file _tim, int _a, int _n, int _p, int _I) +{ + dsp "-f" _filespath "-a" _a "-n" _n "-p" _p "-I" _I stdout=@_out stderr=@_err; +} + +/* == Input files definitions: Make sure these files are available in the current directory == */ +file farmer_sto <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.sto">; +file farmer_cor <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.cor">; +file farmer_tim <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.tim">; + +file timfiles[]; + + +int a=1; +int n=1; +int p=1; +int I=10; + +foreach in_tim,i in timfiles{ + + /* == Output files definitions == */ + file out ; + file err ; + string filespath[]=strsplit(filename(in_tim),"\\."); + file in_sto ; + file in_cor ; + + /* == App invocation == */ + (out, err) = dsp_app (filespath[0], in_sto, in_cor, in_tim, a, n, p, I); +} Added: SwiftApps/dsp-swift/galaxy/README =================================================================== --- SwiftApps/dsp-swift/galaxy/README (rev 0) +++ SwiftApps/dsp-swift/galaxy/README 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,196 @@ +The jump-DSP Swift Galaxy Integration +====================================== + +Introduction +------------- + +The Swift-Galaxy integration project aims to build tools which can be used to +leverage the best features of http://swift-lang.org/main[Swift] and +http://galaxyproject.org[Galaxy] combined. We start with building a generic +Galaxy tool, called `foreach` which allows user to compose Galaxy workflows +such that each computation stage in the workflow is executed by Swift. User can +compose Galaxy tool and pick up arbitrary executables and dataset to run the +executable over. User can also run the executable over a list of items such as +string or file parameters. + +NOTE: This document can be found http://swift-lang.org/tutorials/galaxy/tutorial.html[here]. + +Swift-Galaxy `foreach` Mechanism +--------------------------------- + +The basic mechanism is that the tool user-interface will accept parameters and +will pass them along to a backend Swift "generator". This generator program +will run the parameters over Swift source and configuration templates. As a +result, the right source and configuration files are produced in a temporary +directory (`/tmp/tmpname`). This is then executed and the results collected. + +There are two ways the `foreach` Swift tool can be utilized. First, over a range +of numbers. The tool is used to run any arbitrary executable via Swift `foreach` +loop over a range of numbers. The executable will run with number in range as +commandline argument, one at a time. Furthermore, user can add arbitrary string +arguments in addition to the number argument to the call to executable. + +The second way runs the `foreach` loop over a list of items read from an input +file. These items can be file references or simple strings. The tool will pass +one item at a time to the executable as argument parameter and run them all in +parallel. This is similar to the above described number range except that the +iteration takes place over a list of items provided as a file. + + +Installation +------------- +This installation recipe assumes that you have Galaxy preinstalled. If not, +installation instructions for Galaxy can be found +https://wiki.galaxyproject.org/Admin/GetGalaxy[here]. + +Step 0. Checkout the swift-galaxy SVN directory as follows: + +---- +svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy +---- + +Step 1. Create a symbolic link to the +swift-galaxy/swift+ directory from +galaxy's +tools+ directory. For example, if Galaxy top level directory is ++galaxy-dist+ and swift directory is $HOME/swift-galaxy/swift: + +---- +cd galaxy-dist/tools +ln -s $HOME/swift-galaxy/swift swift +---- + +The advantage of symbolic links is that as users update the tool directory via +`svn up`, they need not worry about reinstallation. Updates in installation +gets automatically propagated to Galaxy. + +Step 2. Add the tool definitions of the tools contained in this Swift to +Galaxy's tool_conf.xml. This can be done by copying the text in +`
...
` from the tool_conf.xml in swift-galaxy directory and +paste it to the tool_conf.xml of the Galaxy directory. In the current installation, the text to be copied is: + +---- +
+ + + +
+---- + +Step 3. (Re)start Galaxy. + +---- +sh run.sh +---- + +In the Galaxy Web GUI, you should see the tools in a toolset titled Swift. + +Screenshots +----------- + +The following screenshot shows overall look-and-feel of the Swift-Galaxy tool +in the Galaxy web GUI. + +image::./screenshots/overall.png["swift-galaxy screenshot", width=600, align="center"] + +The rest of this section walks you through each of the components shown in the +above screen and their utility. The following screenshot shows execution +location selector. User can select one or more execution locations from the +list as their target execution site. + +image::./screenshots/execloc.png["execution location selector screenshot", width=200, align="center"] + +The following screenshot shows a dialog box to add Swift-level commandline +arguments to be passed to the script. + +image::./screenshots/swift-level-args.png["swift-args screenshot", width=200, align="center"] + +The following screenshot shows a selection dropdown list to chose the execution +interpreter from. The interpreters currently supported and tested are shell, +java and python. Upcoming are R and matlab. + +image::./screenshots/execution-interpreter.png["interpreter screenshot", width=150, align="center"] + +The following screenshot shows the selection for an executable. User can upload +an executable and select it from the list. + +image::./screenshots/executable.png["executable screenshot", width=200, align="center"] + +The following screenshot shows a selection for range of numbers or list of +items to chose from. User can select one and the corresponding dialog box will +appear below. + +image::./screenshots/rangeorlist.png["select range or list screenshot", width=150, align="center"] + +The following screenshot shows a dialog for string or file type arguments. A +user can add string and filetype arguments to their executable and the +commandline will be built accordingly. + +image::./screenshots/string-or-fileargs.png["string or fileargs screenshot", width=250, align="center"] + +The following screenshot shows a dialog box to enter the directory location for +results of execution. The Swift output files will be placed here: + +image::./screenshots/outlocation.png["output location screenshot", width=250, align="center"] + +An example +---------- + +A simple Hello World Swift tool to run with Galaxy. + +To set up with your Galaxy environment follow the steps below: +Prerequisite: Sun/Oracle java. Should work with IBM java but not tested. To test your java version: + +---- +java -version +---- + +Typical output is: + +---- +java version "1.7.0_17" +Java(TM) SE Runtime Environment (build 1.7.0_17-b02) +Java HotSpot(TM) 64-Bit Server VM (build 23.7-b01, mixed mode) +---- + +Step 1. Download and install Swift where Galaxy server is running. From a +command prompt the following wget will pull the latest release from +http://swift-lang.org/downloads/index.php[here]. + +Step 2. SVN checkout the swift-galaxy directory and copy the Swift tool directory to Galaxy's tools directory: + +---- +svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy +cp -r swift-galaxy/swift /location/of/galaxy-server/tools/ +---- + +Step 3. Edit the +tool_conf.xml+ file present at the top-level of galaxy directory tree to add the following lines between the ++ and ++ xml tags: + +---- +
+ +
+---- + +Step 4. Restart the Galaxy server. The swift-galaxy tool should be available in the tool box of Galaxy GUI. + +Step 5. Invoke it by entering any number in the dialog box or leaving it to default. + +Step 6. For examples of other Galaxy-Swift tools, look into the swift directoty. + +Troubleshoot +------------- + +Each tool generates a set of Swift source and configuration files. These files are placed in a unique directory with a random name generated on the fly. The location of the directory is in `/tmp` and it is prefixed with name `swift-gal.` . The suffix is a four letter random number. You can run Swift from this directory to reproduce possible issues as follows: + +---- +swift -tc.file tc -sites.file sites.xml -config cf script.swift +---- + +Feel free to contact Swift http://swift-lang.org/support/index.php[support] if issues persist. + +Upcoming Features +------------------ + +* Mixing the order of file and string arguments +* Optionally allowing user to not use range number as arguments +* Providing more Swift patterns, eg. `if-else` conditionals + Added: SwiftApps/dsp-swift/galaxy/setup.py =================================================================== --- SwiftApps/dsp-swift/galaxy/setup.py (rev 0) +++ SwiftApps/dsp-swift/galaxy/setup.py 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +""" +Setup script to install swift-galaxy framework to your Galaxy installation. + +Usage: ./setup.py +""" + +import xml.etree.ElementTree as ET +import subprocess +import sys +import os +import distutils.spawn +import traceback +import fnmatch +import tarfile +import urllib + +galaxyloc=sys.argv[1] + +#step 0. Download swift-galaxy.tgz and swift-rel.tgz untar and set path +if not os.path.isfile("/tmp/swift-galaxy.tgz"): + urllib.urlretrieve("http://mcs.anl.gov/~ketan/swift-galaxy.tgz", filename="/tmp/swift-galaxy.tgz") + swiftgaltar = tarfile.open("/tmp/swift-galaxy.tgz") + swiftgaltar.extractall(path="/tmp") + swiftgaltar.close() + +if not os.path.isfile("/tmp/swift-0.94.1-RC3.tar.gz"): + urllib.urlretrieve("http://swift-lang.org/packages/swift-0.94.1-RC3.tar.gz", filename="/tmp/swift-0.94.1-RC3.tar.gz") + swiftar = tarfile.open("/tmp/swift-0.94.1-RC3.tar.gz") + swiftar.extractall(path="/tmp") + swiftar.close() + os.environ["PATH"] = os.environ["PATH"] + ":/tmp/swift-0.94.1-RC3/bin" + +#step 1. Create symlink to the swift directory from the tools directory of galaxyloc if not already present +if os.path.lexists(galaxyloc+"/tools/swift"): + os.remove(galaxyloc+"/tools/swift") + +os.symlink ("/tmp/swift-galaxy/swift", galaxyloc+"/tools/swift"); + +#step 2. update the tool_conf.xml +tree = ET.parse(galaxyloc+"/config/tool_conf.xml.main") +root = tree.getroot() + +if 'swift' not in root.attrib: + root.append((ET.fromstring('
\n \n \n \n
\n '))) + +tree.write (galaxyloc+"/config/tool_conf.xml.main") + +#step 3. Print message +print "Swift/Galaxy Installed." + Property changes on: SwiftApps/dsp-swift/galaxy/setup.py ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py (rev 0) +++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,255 @@ +#!/usr/bin/env python + +""" +This is the Swift foreach driver for Galaxy. It receives inputs from the Swift +"foreach" Galaxy tool and depending on the options specified, builds a +canonical Swift/K or Swift/T script and runs over specified computational +environment. +""" + +import subprocess +import sys +import os +import distutils.spawn +import traceback +import fnmatch +import re + +def setwdir(): + return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) + +def genargs(args): + for a in args: + yield a + +def buildcfg(wdir): + f=open(wdir+"/cf", 'w') + f.write("use.provider.staging=true\n") + f.write("wrapperlog.always.transfer=true\n") + f.write("execution.retries=0\n") + f.write("provider.staging.pin.swiftfiles=false\n") + f.write("sitedir.keep=true\n") + f.write("lazy.errors=false\n") + f.close() + +def main(): + myargs = genargs(sys.argv) + try: + this=next(myargs) + range_or_list = next(myargs) + k_or_t = next(myargs) + sites = next(myargs) + swiftargs = next(myargs) + interpreter = next(myargs) + executable = next(myargs) + stdin = next(myargs) + + if range_or_list == "list": + #list of items + listfile = next(myargs) + n = sum(1 for line in open(listfile)) + else: + # numeric range + rstart = next(myargs) + rend = next(myargs) + stepsize = next(myargs) + n = (int(rend) - int(rstart))/int(stepsize) + + outloc = next(myargs) + outlistfile = next(myargs) + logfile = next(myargs) + stringargs = next(myargs) + except: + traceback.print_exc() + sys.exit(1) + + #which swift + swift = distutils.spawn.find_executable("swift") + + BASEDIR = os.path.dirname(os.path.realpath(__file__)) + + # workout the file args + fileargs=[] + for f in myargs: + fileargs.append(f) + + #workout the file array only if user adds args else make it blank + if fileargs: + if k_or_t == "swiftK": + filearrayexpr="file fileargs[] ;\n" + else: #T + filearrayexpr="file fileargs[]="+str(fileargs)+";\n" + else: + filearrayexpr="file fileargs[];\n" + + # create outloc dir + if not os.path.exists(outloc): + os.makedirs(outloc) + + # workout the stringargs + if stringargs: + stringarrayexpr="string stringargs[]="+re.sub('\'','"',str(stringargs.split()))+";\n" + else: + stringarrayexpr="string stringargs[];\n" + + wdir = setwdir().rstrip() + + #workout standard input file + if stdin != "None": + if k_or_t == "swiftK": + stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" + else: + stdinfilexpr = "file stdinfile = input(\""+stdin+"\");\n" + else: + #print "no stdin" + if k_or_t == "swiftK": + stdinfilexpr = "file stdinfile<\"/dev/null\">;\n" + else: + stdinfilexpr = "file stdinfile = input(\"/dev/null\");\n" + + + #Build Swift source code + f = open(wdir+"/script.swift", "w"); + + if k_or_t == "swiftK": + + f.write("type file; \n") + f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ \n") + f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; \n") + f.write("}\n") + f.write(stringarrayexpr) + f.write(filearrayexpr) + f.write(stdinfilexpr) + f.write("file exec<\""+executable+"\">;\n") + + if range_or_list == "list": + + f.write("file listfile<\""+listfile+"\">;\n") + f.write("string list[] = readData(listfile);\n"); + f.write("foreach l, i in list{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + + else: + + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + + f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") + f.write("}\n") + + #build site + if os.path.dirname(sys.argv[0]): + basedir=os.path.dirname(sys.argv[0]) + else: + basedir='.' + #print "%s %s %s" %(basedir, sites, wdir) + subprocess.call([basedir+"/buildsite.sh", sites, wdir]) + + #build config file + buildcfg(wdir) + + #build tc file + with open(wdir+"/tc", 'w') as tc: + for site in sites.split(): + tc.write(site+' '+interpreter+' '+interpreter) + + else: # T + + f.write("import files;\n"); + f.write("import string;\n"); + f.write("import io;\n"); + f.write("\n"); + f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n"); + f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n"); + f.write("}\n"); + f.write("main{\n"); + f.write(stringarrayexpr + " \n"); + f.write(filearrayexpr + " \n"); + f.write(stdinfilexpr + " \n"); + f.write("file exec = input(\""+executable+"\");\n"); + + if range_or_list == "list": + + f.write(" \n"); + f.write("file listfile=input(\""+listfile+"\");\n") + f.write("string list[] = split(read(listfile),);\n"); + f.write("foreach l, i in list{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + + else: + + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n"); + f.write(" file out ; \n"); + f.write(" file err ; \n"); + + f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") + f.write("}\n"); + f.write("printf(\"%s\", \"Done!\");\n"); + f.write("}\n"); + + f.close() + + #Invocation + if k_or_t == "swiftK": + + with open(logfile,"wb") as errfile: + exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile) + print exitcode + + else: + + #print "Invoking T" + stc=distutils.spawn.find_executable("stc") + java=distutils.spawn.find_executable("java") + turbine= distutils.spawn.find_executable("turbine") + + if sites == "localhost": + + print "localhost" + with open(logfile, "wb") as errfile: + exitcode=subprocess.check_call([stc, "-j", java, wdir+"/script.swift"], stderr=errfile) + + if exitcode == 0: + with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile: + exitcode=subprocess.check_call([turbine, "-n", str(n), wdir+"/script.tcl", swiftargs], stdin=infile, stderr=errfile) + + elif sites == "Vesta": + + print "vesta" + pwd=os.path.getcwd() + turbine_home="/home/wozniak/Public/sfw/turbine" + with open(logfile, "wb") as errfile: + exitcode=subprocess.check_call([stc, "-t", "checkpointing", "-r", pwd, "-j", java, wdir+"/script.swift"], stderr=errfile) + + if exitcode == 0: + with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile: + exitcode=subprocess.check_call([turbine_home+"/scripts/submit/cobalt/turbine-cobalt-run.zsh", "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile) + + elif sites == "Mira": + + print "mira" + pwd=os.path.getcwd() + turbine_home="/home/wozniak/Public/sfw/ppc64/turbine" + stc="/home/wozniak/Public/sfw/ppc64/stc/bin/stc" + with open(logfile, "wb") as errfile: + exitcode=subprocess.check_call([stc, "-t", "checkpointing", "-r", pwd, "-j", java, wdir+"/script.swift"], stderr=errfile) + + if exitcode == 0: + with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile: + exitcode=subprocess.check_call([turbine_home+"/scripts/submit/cobalt/turbine-cobalt-run.zsh", "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile) + + else: + + print "Site not recognized." + + #Populate output file + outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) + f=open(outlistfile, "w") + f.write(outlist); + f.close() + +if __name__=='__main__': + main() Property changes on: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml (rev 0) +++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,194 @@ + + + A generic tool to run executable via Swift foreach parallel construct + + + #if $rangeorlist.rl == "rng" + swiftforeach.py + "range" + "${k_or_t}" + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${stdin}" + "${rangeorlist.rngstart}" + "${rangeorlist.rend}" + "${rangeorlist.stepsize}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #elif $rangeorlist.rl == "lst" + swiftforeach.py + "list" + "${k_or_t}" + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${stdin}" + "${rangeorlist.listfile}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #else + swiftforeach.py + "drytest" + "${k_or_t}" + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${stdin}" + "${rangeorlist.rngstart}" + "${rangeorlist.rend}" + "${rangeorlist.stepsize}" + "${rangeorlist.listfile}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #end if + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: warningmark + +**TIP**. Add args to provide additional arguments to your executable. + +----- + +**What it does** + +This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items from a file. + +Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable uploaded by user. + +Start, end, and stepsize are integer values. Note that the stepsize cannot be +less than 1. Optionally, user can add new arguments to be provided to the tool +executable after the numeric argument. + +----- + +**Example** + +Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello". + +If an executable is echo.sh with following line: + +echo $@ + +The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8: + +echo n hello + +----- + +For more help, see: http://swift-lang.org/tutorials/galaxy/tutorial.html + + + Added: SwiftApps/dsp-swift/runblues =================================================================== --- SwiftApps/dsp-swift/runblues (rev 0) +++ SwiftApps/dsp-swift/runblues 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,3 @@ +#! /bin/sh + +nohup /home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.blues.xml -config cf -tc.file apps dsp.blues.swift /dev/null & Property changes on: SwiftApps/dsp-swift/runblues ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/dsp-swift/runlocal =================================================================== --- SwiftApps/dsp-swift/runlocal (rev 0) +++ SwiftApps/dsp-swift/runlocal 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,3 @@ +#! /bin/sh + +/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.local.xml -config cf -tc.file apps dsp.blues.swift Property changes on: SwiftApps/dsp-swift/runlocal ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/dsp-swift/sites.blues.xml =================================================================== --- SwiftApps/dsp-swift/sites.blues.xml (rev 0) +++ SwiftApps/dsp-swift/sites.blues.xml 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,22 @@ + + + + + + + 370 + 00:05:00 + 1 + 1 + 1 + 1 + 1 + 2.20 + 10000 + /tmp/swift.workdir + + + Added: SwiftApps/dsp-swift/sites.local.xml =================================================================== --- SwiftApps/dsp-swift/sites.local.xml (rev 0) +++ SwiftApps/dsp-swift/sites.local.xml 2014-12-10 21:40:07 UTC (rev 8320) @@ -0,0 +1,17 @@ + + + + + +09:55:00 +35000 +36000 +/home/ketan/swiftwork +0.09 + + + + From ketan at ci.uchicago.edu Wed Dec 10 18:54:00 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Wed, 10 Dec 2014 18:54:00 -0600 (CST) Subject: [Swift-commit] r8321 - SwiftApps/dsp-swift Message-ID: <20141211005400.B3A8F9D583@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-10 18:54:00 -0600 (Wed, 10 Dec 2014) New Revision: 8321 Modified: SwiftApps/dsp-swift/dsp.blues.swift SwiftApps/dsp-swift/sites.local.xml Log: args Modified: SwiftApps/dsp-swift/dsp.blues.swift =================================================================== --- SwiftApps/dsp-swift/dsp.blues.swift 2014-12-10 21:40:07 UTC (rev 8320) +++ SwiftApps/dsp-swift/dsp.blues.swift 2014-12-11 00:54:00 UTC (rev 8321) @@ -19,27 +19,24 @@ } /* == Input files definitions: Make sure these files are available in the current directory == */ -file farmer_sto <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.sto">; -file farmer_cor <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.cor">; -file farmer_tim <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.tim">; +string inloc=arg("loc","/lcrc/project/NEXTGENOPT/DSP_old/examples/smps/dcap"); -file timfiles[]; +file stofiles[]; +file corfiles[]; +file timfiles[]; +int a=toInt(arg("a","1")); +int n=toInt(arg("n","1")); +int p=toInt(arg("p","1")); +int I=toInt(arg("I","10")); -int a=1; -int n=1; -int p=1; -int I=10; - foreach in_tim,i in timfiles{ /* == Output files definitions == */ file out ; file err ; string filespath[]=strsplit(filename(in_tim),"\\."); - file in_sto ; - file in_cor ; /* == App invocation == */ - (out, err) = dsp_app (filespath[0], in_sto, in_cor, in_tim, a, n, p, I); + (out, err) = dsp_app (filespath[0], stofiles[i], corfiles[i], timfiles[i], a, n, p, I); } Modified: SwiftApps/dsp-swift/sites.local.xml =================================================================== --- SwiftApps/dsp-swift/sites.local.xml 2014-12-10 21:40:07 UTC (rev 8320) +++ SwiftApps/dsp-swift/sites.local.xml 2014-12-11 00:54:00 UTC (rev 8321) @@ -5,7 +5,7 @@ --> - + 09:55:00 35000 36000 From ketan at ci.uchicago.edu Wed Dec 10 19:06:49 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Wed, 10 Dec 2014 19:06:49 -0600 (CST) Subject: [Swift-commit] r8322 - SwiftApps/dsp-swift Message-ID: <20141211010649.969319D583@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-10 19:06:49 -0600 (Wed, 10 Dec 2014) New Revision: 8322 Modified: SwiftApps/dsp-swift/dsp.blues.swift Log: Modified: SwiftApps/dsp-swift/dsp.blues.swift =================================================================== --- SwiftApps/dsp-swift/dsp.blues.swift 2014-12-11 00:54:00 UTC (rev 8321) +++ SwiftApps/dsp-swift/dsp.blues.swift 2014-12-11 01:06:49 UTC (rev 8322) @@ -27,7 +27,7 @@ int a=toInt(arg("a","1")); int n=toInt(arg("n","1")); -int p=toInt(arg("p","1")); +int p=toInt(arg("pval","1")); int I=toInt(arg("I","10")); foreach in_tim,i in timfiles{ From ketan at ci.uchicago.edu Wed Dec 10 20:28:05 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Wed, 10 Dec 2014 20:28:05 -0600 (CST) Subject: [Swift-commit] r8323 - SwiftApps/dsp-swift/galaxy Message-ID: <20141211022805.DB4199D58A@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-10 20:28:05 -0600 (Wed, 10 Dec 2014) New Revision: 8323 Modified: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml Log: gal Modified: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py 2014-12-11 01:06:49 UTC (rev 8322) +++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py 2014-12-11 02:28:05 UTC (rev 8323) @@ -5,6 +5,15 @@ "foreach" Galaxy tool and depending on the options specified, builds a canonical Swift/K or Swift/T script and runs over specified computational environment. + +swiftjumpdsp.py + "${inlocation}" + "${a}" + "${n}" + "${p}" + "${I}" + "${outlist}" + """ import subprocess @@ -36,29 +45,14 @@ myargs = genargs(sys.argv) try: this=next(myargs) - range_or_list = next(myargs) - k_or_t = next(myargs) - sites = next(myargs) - swiftargs = next(myargs) - interpreter = next(myargs) - executable = next(myargs) - stdin = next(myargs) - - if range_or_list == "list": - #list of items - listfile = next(myargs) - n = sum(1 for line in open(listfile)) - else: - # numeric range - rstart = next(myargs) - rend = next(myargs) - stepsize = next(myargs) - n = (int(rend) - int(rstart))/int(stepsize) - + inloc = next(myargs) + a = next(myargs) + n = next(myargs) + p = next(myargs) + I = next(myargs) outloc = next(myargs) outlistfile = next(myargs) logfile = next(myargs) - stringargs = next(myargs) except: traceback.print_exc() sys.exit(1) Modified: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml 2014-12-11 01:06:49 UTC (rev 8322) +++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml 2014-12-11 02:28:05 UTC (rev 8323) @@ -8,66 +8,16 @@ 5. Add ability to run Swift/T. 6. Let the localhost option be selected by default --> - A generic tool to run executable via Swift foreach parallel construct + A Galaxy tool to run Dsp parameter sweeps via Swift foreach parallel construct - #if $rangeorlist.rl == "rng" - swiftforeach.py - "range" - "${k_or_t}" - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${stdin}" - "${rangeorlist.rngstart}" - "${rangeorlist.rend}" - "${rangeorlist.stepsize}" - "${outloc}" - "${logfile}" + swiftjumpdsp.py + "${inlocation}" + "${a}" + "${n}" + "${p}" + "${I}" "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #elif $rangeorlist.rl == "lst" - swiftforeach.py - "list" - "${k_or_t}" - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${stdin}" - "${rangeorlist.listfile}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #else - swiftforeach.py - "drytest" - "${k_or_t}" - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${stdin}" - "${rangeorlist.rngstart}" - "${rangeorlist.rend}" - "${rangeorlist.stepsize}" - "${rangeorlist.listfile}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #end if - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - + + +/home/ketan/workerlogs +16 -4 - /home/ketan/swiftwork - Modified: SwiftApps/ematter/workflow.bgq.swift =================================================================== --- SwiftApps/ematter/workflow.bgq.swift 2014-12-11 03:21:32 UTC (rev 8325) +++ SwiftApps/ematter/workflow.bgq.swift 2014-12-11 16:24:00 UTC (rev 8326) @@ -2,7 +2,7 @@ // Lammps app definition app (file _dump) lammps_app (string _lammps_exe, file _in, - file _geo, file _eam) + file _geo, file _eam) { bgsh _lammps_exe "-in" @_in; } @@ -70,7 +70,7 @@ } /* == Input files definitions: Make sure these files are available in the current directory == */ -file lammps_in <"in.lammps">; +file lammps_in <"input.lammps">; file lammps_geo <"AuTiO.geo">; file lammps_eam <"Au_u3.eam">; @@ -95,6 +95,7 @@ /* == Output files definitions == */ file xyz <"forsmeagol.xyz">; file lammps_dump <"lammps.dump">; +file lammps_out <"lammps.stdout">; file fdf_proc<"input_proc.fdf">; file smeagol_lead_out <"smeagol_out/smeagol.lead.out">; @@ -110,13 +111,11 @@ string Vs[]=["0.1","0.2","0.3","0.4","0.5","0.6","0.7","0.8","0.9","1.0"]; + //Invoke lammps (lammps_dump) = lammps_app (lammps_exe, lammps_in, lammps_geo, lammps_eam); -//Invoke pizza -(xyz) = pizza_app(pizza_src, lammps_dump); - //Invoke smeagol lead (au3dm, au3hst, bulklft, bulkrgt, @@ -125,7 +124,10 @@ opsf, osicpsf, tipsf, tisicpsf); +//Invoke pizza +(xyz) = pizza_app(pizza_src, lammps_dump); + //Invoke smeagol transport V0 (tio2auhm, tio2audm, smeagol_transport_out) = smeagol_v0_app(smeagol_exe, au3dm, @@ -139,7 +141,7 @@ foreach v,i in Vs { -/* == generate fdfs for different voltages == */ +// == generate fdfs for different voltages == string fdfname = strcat("input_", v, ".fdf"); file fdf_proc_vn ; @@ -157,4 +159,3 @@ } - Modified: SwiftApps/subjobs/bg.sh =================================================================== --- SwiftApps/subjobs/bg.sh 2014-12-11 03:21:32 UTC (rev 8325) +++ SwiftApps/subjobs/bg.sh 2014-12-11 16:24:00 UTC (rev 8326) @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash mname=$(hostname) @@ -56,14 +56,19 @@ echo "$0": running runjob --strace none --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" #without timeout - runjob --strace none --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" + #strace -o "$HOME/strace.runjob.out" runjob --strace none --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" + #runjob --strace none --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" + runjob --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np 2 : "$@" echo "Runjob finished." else # run w/o subblocks if no subblock size provided echo "Running in nonsubblock mode." echo "$0": running runjob -p 16 --block $COBALT_PARTNAME : "$@" + + #strace -o "$HOME/strace.runjob.out" runjob --strace none -p 16 --block $COBALT_PARTNAME : "$@" runjob -p 16 --block $COBALT_PARTNAME : "$@" + echo "Finished Running in nonsubblock mode." fi exit 0 From ketan at ci.uchicago.edu Thu Dec 11 13:04:26 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 13:04:26 -0600 (CST) Subject: [Swift-commit] r8327 - SwiftApps/dsp-swift Message-ID: <20141211190426.D035C9DFE9@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-11 13:04:26 -0600 (Thu, 11 Dec 2014) New Revision: 8327 Modified: SwiftApps/dsp-swift/sites.blues.xml SwiftApps/dsp-swift/sites.local.xml Log: add user home to sites file Modified: SwiftApps/dsp-swift/sites.blues.xml =================================================================== --- SwiftApps/dsp-swift/sites.blues.xml 2014-12-11 16:24:00 UTC (rev 8326) +++ SwiftApps/dsp-swift/sites.blues.xml 2014-12-11 19:04:26 UTC (rev 8327) @@ -16,7 +16,7 @@ 1 2.20 10000 - /tmp/swift.workdir + {user.home}/swift.workdir Modified: SwiftApps/dsp-swift/sites.local.xml =================================================================== --- SwiftApps/dsp-swift/sites.local.xml 2014-12-11 16:24:00 UTC (rev 8326) +++ SwiftApps/dsp-swift/sites.local.xml 2014-12-11 19:04:26 UTC (rev 8327) @@ -9,7 +9,7 @@ 09:55:00 35000 36000 -/home/ketan/swiftwork +{user.home}/swiftwork 0.09 From ketan at ci.uchicago.edu Thu Dec 11 13:21:36 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 13:21:36 -0600 (CST) Subject: [Swift-commit] r8328 - SwiftApps/dsp-swift/galaxy Message-ID: <20141211192136.8E5629DFE9@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-11 13:21:36 -0600 (Thu, 11 Dec 2014) New Revision: 8328 Removed: SwiftApps/dsp-swift/galaxy/README SwiftApps/dsp-swift/galaxy/setup.py SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml Log: cleanup and refactor Deleted: SwiftApps/dsp-swift/galaxy/README =================================================================== --- SwiftApps/dsp-swift/galaxy/README 2014-12-11 19:04:26 UTC (rev 8327) +++ SwiftApps/dsp-swift/galaxy/README 2014-12-11 19:21:36 UTC (rev 8328) @@ -1,196 +0,0 @@ -The jump-DSP Swift Galaxy Integration -====================================== - -Introduction -------------- - -The Swift-Galaxy integration project aims to build tools which can be used to -leverage the best features of http://swift-lang.org/main[Swift] and -http://galaxyproject.org[Galaxy] combined. We start with building a generic -Galaxy tool, called `foreach` which allows user to compose Galaxy workflows -such that each computation stage in the workflow is executed by Swift. User can -compose Galaxy tool and pick up arbitrary executables and dataset to run the -executable over. User can also run the executable over a list of items such as -string or file parameters. - -NOTE: This document can be found http://swift-lang.org/tutorials/galaxy/tutorial.html[here]. - -Swift-Galaxy `foreach` Mechanism ---------------------------------- - -The basic mechanism is that the tool user-interface will accept parameters and -will pass them along to a backend Swift "generator". This generator program -will run the parameters over Swift source and configuration templates. As a -result, the right source and configuration files are produced in a temporary -directory (`/tmp/tmpname`). This is then executed and the results collected. - -There are two ways the `foreach` Swift tool can be utilized. First, over a range -of numbers. The tool is used to run any arbitrary executable via Swift `foreach` -loop over a range of numbers. The executable will run with number in range as -commandline argument, one at a time. Furthermore, user can add arbitrary string -arguments in addition to the number argument to the call to executable. - -The second way runs the `foreach` loop over a list of items read from an input -file. These items can be file references or simple strings. The tool will pass -one item at a time to the executable as argument parameter and run them all in -parallel. This is similar to the above described number range except that the -iteration takes place over a list of items provided as a file. - - -Installation -------------- -This installation recipe assumes that you have Galaxy preinstalled. If not, -installation instructions for Galaxy can be found -https://wiki.galaxyproject.org/Admin/GetGalaxy[here]. - -Step 0. Checkout the swift-galaxy SVN directory as follows: - ----- -svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy ----- - -Step 1. Create a symbolic link to the +swift-galaxy/swift+ directory from -galaxy's +tools+ directory. For example, if Galaxy top level directory is -+galaxy-dist+ and swift directory is $HOME/swift-galaxy/swift: - ----- -cd galaxy-dist/tools -ln -s $HOME/swift-galaxy/swift swift ----- - -The advantage of symbolic links is that as users update the tool directory via -`svn up`, they need not worry about reinstallation. Updates in installation -gets automatically propagated to Galaxy. - -Step 2. Add the tool definitions of the tools contained in this Swift to -Galaxy's tool_conf.xml. This can be done by copying the text in -`
...
` from the tool_conf.xml in swift-galaxy directory and -paste it to the tool_conf.xml of the Galaxy directory. In the current installation, the text to be copied is: - ----- -
- - - -
----- - -Step 3. (Re)start Galaxy. - ----- -sh run.sh ----- - -In the Galaxy Web GUI, you should see the tools in a toolset titled Swift. - -Screenshots ------------ - -The following screenshot shows overall look-and-feel of the Swift-Galaxy tool -in the Galaxy web GUI. - -image::./screenshots/overall.png["swift-galaxy screenshot", width=600, align="center"] - -The rest of this section walks you through each of the components shown in the -above screen and their utility. The following screenshot shows execution -location selector. User can select one or more execution locations from the -list as their target execution site. - -image::./screenshots/execloc.png["execution location selector screenshot", width=200, align="center"] - -The following screenshot shows a dialog box to add Swift-level commandline -arguments to be passed to the script. - -image::./screenshots/swift-level-args.png["swift-args screenshot", width=200, align="center"] - -The following screenshot shows a selection dropdown list to chose the execution -interpreter from. The interpreters currently supported and tested are shell, -java and python. Upcoming are R and matlab. - -image::./screenshots/execution-interpreter.png["interpreter screenshot", width=150, align="center"] - -The following screenshot shows the selection for an executable. User can upload -an executable and select it from the list. - -image::./screenshots/executable.png["executable screenshot", width=200, align="center"] - -The following screenshot shows a selection for range of numbers or list of -items to chose from. User can select one and the corresponding dialog box will -appear below. - -image::./screenshots/rangeorlist.png["select range or list screenshot", width=150, align="center"] - -The following screenshot shows a dialog for string or file type arguments. A -user can add string and filetype arguments to their executable and the -commandline will be built accordingly. - -image::./screenshots/string-or-fileargs.png["string or fileargs screenshot", width=250, align="center"] - -The following screenshot shows a dialog box to enter the directory location for -results of execution. The Swift output files will be placed here: - -image::./screenshots/outlocation.png["output location screenshot", width=250, align="center"] - -An example ----------- - -A simple Hello World Swift tool to run with Galaxy. - -To set up with your Galaxy environment follow the steps below: -Prerequisite: Sun/Oracle java. Should work with IBM java but not tested. To test your java version: - ----- -java -version ----- - -Typical output is: - ----- -java version "1.7.0_17" -Java(TM) SE Runtime Environment (build 1.7.0_17-b02) -Java HotSpot(TM) 64-Bit Server VM (build 23.7-b01, mixed mode) ----- - -Step 1. Download and install Swift where Galaxy server is running. From a -command prompt the following wget will pull the latest release from -http://swift-lang.org/downloads/index.php[here]. - -Step 2. SVN checkout the swift-galaxy directory and copy the Swift tool directory to Galaxy's tools directory: - ----- -svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy -cp -r swift-galaxy/swift /location/of/galaxy-server/tools/ ----- - -Step 3. Edit the +tool_conf.xml+ file present at the top-level of galaxy directory tree to add the following lines between the ++ and ++ xml tags: - ----- -
- -
----- - -Step 4. Restart the Galaxy server. The swift-galaxy tool should be available in the tool box of Galaxy GUI. - -Step 5. Invoke it by entering any number in the dialog box or leaving it to default. - -Step 6. For examples of other Galaxy-Swift tools, look into the swift directoty. - -Troubleshoot -------------- - -Each tool generates a set of Swift source and configuration files. These files are placed in a unique directory with a random name generated on the fly. The location of the directory is in `/tmp` and it is prefixed with name `swift-gal.` . The suffix is a four letter random number. You can run Swift from this directory to reproduce possible issues as follows: - ----- -swift -tc.file tc -sites.file sites.xml -config cf script.swift ----- - -Feel free to contact Swift http://swift-lang.org/support/index.php[support] if issues persist. - -Upcoming Features ------------------- - -* Mixing the order of file and string arguments -* Optionally allowing user to not use range number as arguments -* Providing more Swift patterns, eg. `if-else` conditionals - Deleted: SwiftApps/dsp-swift/galaxy/setup.py =================================================================== --- SwiftApps/dsp-swift/galaxy/setup.py 2014-12-11 19:04:26 UTC (rev 8327) +++ SwiftApps/dsp-swift/galaxy/setup.py 2014-12-11 19:21:36 UTC (rev 8328) @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -""" -Setup script to install swift-galaxy framework to your Galaxy installation. - -Usage: ./setup.py -""" - -import xml.etree.ElementTree as ET -import subprocess -import sys -import os -import distutils.spawn -import traceback -import fnmatch -import tarfile -import urllib - -galaxyloc=sys.argv[1] - -#step 0. Download swift-galaxy.tgz and swift-rel.tgz untar and set path - -#step 1. Create symlink to the swift directory from the tools directory of galaxyloc if not already present -if os.path.lexists(galaxyloc+"/tools/swift"): - os.remove(galaxyloc+"/tools/swift") - -os.symlink ("/tmp/swift-galaxy/swift", galaxyloc+"/tools/swift"); - -#step 2. update the tool_conf.xml -tree = ET.parse(galaxyloc+"/config/tool_conf.xml.main") -root = tree.getroot() - -if 'swift' not in root.attrib: - root.append((ET.fromstring('
\n \n \n \n
\n '))) - -tree.write (galaxyloc+"/config/tool_conf.xml.main") - -#step 3. Print message -print "Swift/Galaxy Installed." - Deleted: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py 2014-12-11 19:04:26 UTC (rev 8327) +++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py 2014-12-11 19:21:36 UTC (rev 8328) @@ -1,201 +0,0 @@ -#!/usr/bin/env python - -""" -This is the Swift foreach driver for Galaxy. It receives inputs from the Swift -"foreach" Galaxy tool and depending on the options specified, builds a -canonical Swift/K or Swift/T script and runs over specified computational -environment. - -swiftjumpdsp.py - "${inlocation}" - "${a}" - "${n}" - "${p}" - "${I}" - "${outlist}" - -""" - -import subprocess -import sys -import os -import distutils.spawn -import traceback -import fnmatch -import re - -def setwdir(): - return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) - -def genargs(args): - for a in args: - yield a - -def buildcfg(wdir): - f=open(wdir+"/cf", 'w') - f.write("use.provider.staging=true\n") - f.write("wrapperlog.always.transfer=true\n") - f.write("execution.retries=0\n") - f.write("provider.staging.pin.swiftfiles=false\n") - f.write("sitedir.keep=true\n") - f.write("lazy.errors=false\n") - f.close() - -def main(): - myargs = genargs(sys.argv) - try: - this=next(myargs) - inloc = next(myargs) - a = next(myargs) - n = next(myargs) - p = next(myargs) - I = next(myargs) - outloc = next(myargs) - outlistfile = next(myargs) - logfile = next(myargs) - except: - traceback.print_exc() - sys.exit(1) - - #which swift - swift = distutils.spawn.find_executable("swift") - - BASEDIR = os.path.dirname(os.path.realpath(__file__)) - - # workout the file args - fileargs=[] - for f in myargs: - fileargs.append(f) - - #workout the file array only if user adds args else make it blank - if fileargs: - if k_or_t == "swiftK": - filearrayexpr="file fileargs[] ;\n" - else: #T - filearrayexpr="file fileargs[]="+str(fileargs)+";\n" - else: - filearrayexpr="file fileargs[];\n" - - # create outloc dir - if not os.path.exists(outloc): - os.makedirs(outloc) - - # workout the stringargs - if stringargs: - stringarrayexpr="string stringargs[]="+re.sub('\'','"',str(stringargs.split()))+";\n" - else: - stringarrayexpr="string stringargs[];\n" - - wdir = setwdir().rstrip() - - #workout standard input file - if stdin != "None": - if k_or_t == "swiftK": - stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" - else: - stdinfilexpr = "file stdinfile = input(\""+stdin+"\");\n" - else: - #print "no stdin" - if k_or_t == "swiftK": - stdinfilexpr = "file stdinfile<\"/dev/null\">;\n" - else: - stdinfilexpr = "file stdinfile = input(\"/dev/null\");\n" - - - #Build Swift source code - f = open(wdir+"/script.swift", "w"); - - if k_or_t == "swiftK": - - f.write("type file; \n") - f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ \n") - f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; \n") - f.write("}\n") - f.write(stringarrayexpr) - f.write(filearrayexpr) - f.write(stdinfilexpr) - f.write("file exec<\""+executable+"\">;\n") - - if range_or_list == "list": - - f.write("file listfile<\""+listfile+"\">;\n") - f.write("string list[] = readData(listfile);\n"); - f.write("foreach l, i in list{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - - else: - - f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - - f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") - f.write("}\n") - - #build site - if os.path.dirname(sys.argv[0]): - basedir=os.path.dirname(sys.argv[0]) - else: - basedir='.' - #print "%s %s %s" %(basedir, sites, wdir) - subprocess.call([basedir+"/buildsite.sh", sites, wdir]) - - #build config file - buildcfg(wdir) - - #build tc file - with open(wdir+"/tc", 'w') as tc: - for site in sites.split(): - tc.write(site+' '+interpreter+' '+interpreter) - - else: # T - - f.write("import files;\n"); - f.write("import string;\n"); - f.write("import io;\n"); - f.write("\n"); - f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n"); - f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n"); - f.write("}\n"); - f.write("main{\n"); - f.write(stringarrayexpr + " \n"); - f.write(filearrayexpr + " \n"); - f.write(stdinfilexpr + " \n"); - f.write("file exec = input(\""+executable+"\");\n"); - - if range_or_list == "list": - - f.write(" \n"); - f.write("file listfile=input(\""+listfile+"\");\n") - f.write("string list[] = split(read(listfile),);\n"); - f.write("foreach l, i in list{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - - else: - - f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n"); - f.write(" file out ; \n"); - f.write(" file err ; \n"); - - f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") - f.write("}\n"); - f.write("printf(\"%s\", \"Done!\");\n"); - f.write("}\n"); - - f.close() - - #Invocation - with open(logfile,"wb") as errfile: - exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile) - print exitcode - - #Populate output file - outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) - f=open(outlistfile, "w") - f.write(outlist); - f.close() - -if __name__=='__main__': - main() Deleted: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml 2014-12-11 19:04:26 UTC (rev 8327) +++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml 2014-12-11 19:21:36 UTC (rev 8328) @@ -1,59 +0,0 @@ - - - A Galaxy tool to run Dsp parameter sweeps via Swift foreach parallel construct - - - swiftjumpdsp.py - "${inlocation}" - "${a}" - "${n}" - "${p}" - "${I}" - "${outlist}" - - - - - - - - - - - - - - - - - -.. class:: warningmark - -**TIP**. Add args to provide additional arguments to your executable. - ------ - -**What it does** - -This is a tool to run dsp via Swift in parallel over files provided by user. - - - From hategan at ci.uchicago.edu Thu Dec 11 13:23:27 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 13:23:27 -0600 (CST) Subject: [Swift-commit] r8329 - branches/release-0.95/src/org/griphyn/vdl/karajan/lib Message-ID: <20141211192327.1222E9DFE9@svn.ci.uchicago.edu> Author: hategan Date: 2014-12-11 13:23:26 -0600 (Thu, 11 Dec 2014) New Revision: 8329 Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java Log: allow environment variables in sites.xml and check if environment variables or system properties exist when doing substitution Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-12-11 19:21:36 UTC (rev 8328) +++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-12-11 19:23:26 UTC (rev 8329) @@ -336,7 +336,13 @@ break; case '}': if (li != -1) { - sb.append(System.getProperty(v.substring(li + 1, i))); + String propName = v.substring(li + 1, i); + if (propName.startsWith("env.")) { + sb.append(getEnv(propName.substring(4))); + } + else { + sb.append(getSystemProperty(propName)); + } li = -1; } else { @@ -351,4 +357,24 @@ } return sb.toString(); } + + private String getEnv(String name) { + String value = System.getenv(name); + if (value == null) { + throw new IllegalArgumentException("No such environment variable: '" + name + "'"); + } + else { + return value; + } + } + + private String getSystemProperty(String propName) { + String value = System.getProperty(propName); + if (value == null) { + throw new IllegalArgumentException("No such system property: '" + propName + "'"); + } + else { + return value; + } + } } From ketan at ci.uchicago.edu Thu Dec 11 13:28:34 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 13:28:34 -0600 (CST) Subject: [Swift-commit] r8330 - SwiftApps/dsp-swift Message-ID: <20141211192834.9DCA09DFE9@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-11 13:28:34 -0600 (Thu, 11 Dec 2014) New Revision: 8330 Removed: SwiftApps/dsp-swift/galaxy/ Log: remove the galaxy dir From ketan at ci.uchicago.edu Thu Dec 11 13:44:14 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 13:44:14 -0600 (CST) Subject: [Swift-commit] r8331 - SwiftApps/dsp-swift Message-ID: <20141211194414.7530D9D58A@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-11 13:44:14 -0600 (Thu, 11 Dec 2014) New Revision: 8331 Added: SwiftApps/dsp-swift/README SwiftApps/dsp-swift/setup.py SwiftApps/dsp-swift/swiftjumpdsp.py SwiftApps/dsp-swift/swiftjumpdsp.xml Log: readme and setup Added: SwiftApps/dsp-swift/README =================================================================== --- SwiftApps/dsp-swift/README (rev 0) +++ SwiftApps/dsp-swift/README 2014-12-11 19:44:14 UTC (rev 8331) @@ -0,0 +1,196 @@ +The jump-DSP Swift Galaxy Integration +====================================== + +Introduction +------------- + +The Swift-Galaxy integration project aims to build tools which can be used to +leverage the best features of http://swift-lang.org/main[Swift] and +http://galaxyproject.org[Galaxy] combined. We start with building a generic +Galaxy tool, called `foreach` which allows user to compose Galaxy workflows +such that each computation stage in the workflow is executed by Swift. User can +compose Galaxy tool and pick up arbitrary executables and dataset to run the +executable over. User can also run the executable over a list of items such as +string or file parameters. + +NOTE: This document can be found http://swift-lang.org/tutorials/galaxy/tutorial.html[here]. + +Swift-Galaxy `foreach` Mechanism +--------------------------------- + +The basic mechanism is that the tool user-interface will accept parameters and +will pass them along to a backend Swift "generator". This generator program +will run the parameters over Swift source and configuration templates. As a +result, the right source and configuration files are produced in a temporary +directory (`/tmp/tmpname`). This is then executed and the results collected. + +There are two ways the `foreach` Swift tool can be utilized. First, over a range +of numbers. The tool is used to run any arbitrary executable via Swift `foreach` +loop over a range of numbers. The executable will run with number in range as +commandline argument, one at a time. Furthermore, user can add arbitrary string +arguments in addition to the number argument to the call to executable. + +The second way runs the `foreach` loop over a list of items read from an input +file. These items can be file references or simple strings. The tool will pass +one item at a time to the executable as argument parameter and run them all in +parallel. This is similar to the above described number range except that the +iteration takes place over a list of items provided as a file. + + +Installation +------------- +This installation recipe assumes that you have Galaxy preinstalled. If not, +installation instructions for Galaxy can be found +https://wiki.galaxyproject.org/Admin/GetGalaxy[here]. + +Step 0. Checkout the swift-galaxy SVN directory as follows: + +---- +svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy +---- + +Step 1. Create a symbolic link to the +swift-galaxy/swift+ directory from +galaxy's +tools+ directory. For example, if Galaxy top level directory is ++galaxy-dist+ and swift directory is $HOME/swift-galaxy/swift: + +---- +cd galaxy-dist/tools +ln -s $HOME/swift-galaxy/swift swift +---- + +The advantage of symbolic links is that as users update the tool directory via +`svn up`, they need not worry about reinstallation. Updates in installation +gets automatically propagated to Galaxy. + +Step 2. Add the tool definitions of the tools contained in this Swift to +Galaxy's tool_conf.xml. This can be done by copying the text in +`
...
` from the tool_conf.xml in swift-galaxy directory and +paste it to the tool_conf.xml of the Galaxy directory. In the current installation, the text to be copied is: + +---- +
+ + + +
+---- + +Step 3. (Re)start Galaxy. + +---- +sh run.sh +---- + +In the Galaxy Web GUI, you should see the tools in a toolset titled Swift. + +Screenshots +----------- + +The following screenshot shows overall look-and-feel of the Swift-Galaxy tool +in the Galaxy web GUI. + +image::./screenshots/overall.png["swift-galaxy screenshot", width=600, align="center"] + +The rest of this section walks you through each of the components shown in the +above screen and their utility. The following screenshot shows execution +location selector. User can select one or more execution locations from the +list as their target execution site. + +image::./screenshots/execloc.png["execution location selector screenshot", width=200, align="center"] + +The following screenshot shows a dialog box to add Swift-level commandline +arguments to be passed to the script. + +image::./screenshots/swift-level-args.png["swift-args screenshot", width=200, align="center"] + +The following screenshot shows a selection dropdown list to chose the execution +interpreter from. The interpreters currently supported and tested are shell, +java and python. Upcoming are R and matlab. + +image::./screenshots/execution-interpreter.png["interpreter screenshot", width=150, align="center"] + +The following screenshot shows the selection for an executable. User can upload +an executable and select it from the list. + +image::./screenshots/executable.png["executable screenshot", width=200, align="center"] + +The following screenshot shows a selection for range of numbers or list of +items to chose from. User can select one and the corresponding dialog box will +appear below. + +image::./screenshots/rangeorlist.png["select range or list screenshot", width=150, align="center"] + +The following screenshot shows a dialog for string or file type arguments. A +user can add string and filetype arguments to their executable and the +commandline will be built accordingly. + +image::./screenshots/string-or-fileargs.png["string or fileargs screenshot", width=250, align="center"] + +The following screenshot shows a dialog box to enter the directory location for +results of execution. The Swift output files will be placed here: + +image::./screenshots/outlocation.png["output location screenshot", width=250, align="center"] + +An example +---------- + +A simple Hello World Swift tool to run with Galaxy. + +To set up with your Galaxy environment follow the steps below: +Prerequisite: Sun/Oracle java. Should work with IBM java but not tested. To test your java version: + +---- +java -version +---- + +Typical output is: + +---- +java version "1.7.0_17" +Java(TM) SE Runtime Environment (build 1.7.0_17-b02) +Java HotSpot(TM) 64-Bit Server VM (build 23.7-b01, mixed mode) +---- + +Step 1. Download and install Swift where Galaxy server is running. From a +command prompt the following wget will pull the latest release from +http://swift-lang.org/downloads/index.php[here]. + +Step 2. SVN checkout the swift-galaxy directory and copy the Swift tool directory to Galaxy's tools directory: + +---- +svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy +cp -r swift-galaxy/swift /location/of/galaxy-server/tools/ +---- + +Step 3. Edit the +tool_conf.xml+ file present at the top-level of galaxy directory tree to add the following lines between the ++ and ++ xml tags: + +---- +
+ +
+---- + +Step 4. Restart the Galaxy server. The swift-galaxy tool should be available in the tool box of Galaxy GUI. + +Step 5. Invoke it by entering any number in the dialog box or leaving it to default. + +Step 6. For examples of other Galaxy-Swift tools, look into the swift directoty. + +Troubleshoot +------------- + +Each tool generates a set of Swift source and configuration files. These files are placed in a unique directory with a random name generated on the fly. The location of the directory is in `/tmp` and it is prefixed with name `swift-gal.` . The suffix is a four letter random number. You can run Swift from this directory to reproduce possible issues as follows: + +---- +swift -tc.file tc -sites.file sites.xml -config cf script.swift +---- + +Feel free to contact Swift http://swift-lang.org/support/index.php[support] if issues persist. + +Upcoming Features +------------------ + +* Mixing the order of file and string arguments +* Optionally allowing user to not use range number as arguments +* Providing more Swift patterns, eg. `if-else` conditionals + Added: SwiftApps/dsp-swift/setup.py =================================================================== --- SwiftApps/dsp-swift/setup.py (rev 0) +++ SwiftApps/dsp-swift/setup.py 2014-12-11 19:44:14 UTC (rev 8331) @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +""" +Setup script to install swift-galaxy framework to your Galaxy installation. + +Usage: ./setup.py +""" + +import xml.etree.ElementTree as ET +import subprocess +import sys +import os +import distutils.spawn +import traceback +import fnmatch +import tarfile +import urllib + +galaxyloc=sys.argv[1] + +#step 0. Download swift-galaxy.tgz and swift-rel.tgz untar and set path + +#step 1. Create symlink to the swift directory from the tools directory of galaxyloc if not already present +if os.path.lexists(galaxyloc+"/tools/swift"): + os.remove(galaxyloc+"/tools/swift") + +os.symlink ("/tmp/swift-galaxy/swift", galaxyloc+"/tools/swift"); + +#step 2. update the tool_conf.xml +tree = ET.parse(galaxyloc+"/config/tool_conf.xml.main") +root = tree.getroot() + +if 'swift' not in root.attrib: + root.append((ET.fromstring('
\n \n \n \n
\n '))) + +tree.write (galaxyloc+"/config/tool_conf.xml.main") + +#step 3. Print message +print "Swift/Galaxy Installed." + Property changes on: SwiftApps/dsp-swift/setup.py ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/dsp-swift/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.py (rev 0) +++ SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-11 19:44:14 UTC (rev 8331) @@ -0,0 +1,201 @@ +#!/usr/bin/env python + +""" +This is the Swift foreach driver for Galaxy. It receives inputs from the Swift +"foreach" Galaxy tool and depending on the options specified, builds a +canonical Swift/K or Swift/T script and runs over specified computational +environment. + +swiftjumpdsp.py + "${inlocation}" + "${a}" + "${n}" + "${p}" + "${I}" + "${outlist}" + +""" + +import subprocess +import sys +import os +import distutils.spawn +import traceback +import fnmatch +import re + +def setwdir(): + return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) + +def genargs(args): + for a in args: + yield a + +def buildcfg(wdir): + f=open(wdir+"/cf", 'w') + f.write("use.provider.staging=true\n") + f.write("wrapperlog.always.transfer=true\n") + f.write("execution.retries=0\n") + f.write("provider.staging.pin.swiftfiles=false\n") + f.write("sitedir.keep=true\n") + f.write("lazy.errors=false\n") + f.close() + +def main(): + myargs = genargs(sys.argv) + try: + this=next(myargs) + inloc = next(myargs) + a = next(myargs) + n = next(myargs) + p = next(myargs) + I = next(myargs) + outloc = next(myargs) + outlistfile = next(myargs) + logfile = next(myargs) + except: + traceback.print_exc() + sys.exit(1) + + #which swift + swift = distutils.spawn.find_executable("swift") + + BASEDIR = os.path.dirname(os.path.realpath(__file__)) + + # workout the file args + fileargs=[] + for f in myargs: + fileargs.append(f) + + #workout the file array only if user adds args else make it blank + if fileargs: + if k_or_t == "swiftK": + filearrayexpr="file fileargs[] ;\n" + else: #T + filearrayexpr="file fileargs[]="+str(fileargs)+";\n" + else: + filearrayexpr="file fileargs[];\n" + + # create outloc dir + if not os.path.exists(outloc): + os.makedirs(outloc) + + # workout the stringargs + if stringargs: + stringarrayexpr="string stringargs[]="+re.sub('\'','"',str(stringargs.split()))+";\n" + else: + stringarrayexpr="string stringargs[];\n" + + wdir = setwdir().rstrip() + + #workout standard input file + if stdin != "None": + if k_or_t == "swiftK": + stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" + else: + stdinfilexpr = "file stdinfile = input(\""+stdin+"\");\n" + else: + #print "no stdin" + if k_or_t == "swiftK": + stdinfilexpr = "file stdinfile<\"/dev/null\">;\n" + else: + stdinfilexpr = "file stdinfile = input(\"/dev/null\");\n" + + + #Build Swift source code + f = open(wdir+"/script.swift", "w"); + + if k_or_t == "swiftK": + + f.write("type file; \n") + f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ \n") + f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; \n") + f.write("}\n") + f.write(stringarrayexpr) + f.write(filearrayexpr) + f.write(stdinfilexpr) + f.write("file exec<\""+executable+"\">;\n") + + if range_or_list == "list": + + f.write("file listfile<\""+listfile+"\">;\n") + f.write("string list[] = readData(listfile);\n"); + f.write("foreach l, i in list{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + + else: + + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + + f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") + f.write("}\n") + + #build site + if os.path.dirname(sys.argv[0]): + basedir=os.path.dirname(sys.argv[0]) + else: + basedir='.' + #print "%s %s %s" %(basedir, sites, wdir) + subprocess.call([basedir+"/buildsite.sh", sites, wdir]) + + #build config file + buildcfg(wdir) + + #build tc file + with open(wdir+"/tc", 'w') as tc: + for site in sites.split(): + tc.write(site+' '+interpreter+' '+interpreter) + + else: # T + + f.write("import files;\n"); + f.write("import string;\n"); + f.write("import io;\n"); + f.write("\n"); + f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n"); + f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n"); + f.write("}\n"); + f.write("main{\n"); + f.write(stringarrayexpr + " \n"); + f.write(filearrayexpr + " \n"); + f.write(stdinfilexpr + " \n"); + f.write("file exec = input(\""+executable+"\");\n"); + + if range_or_list == "list": + + f.write(" \n"); + f.write("file listfile=input(\""+listfile+"\");\n") + f.write("string list[] = split(read(listfile),);\n"); + f.write("foreach l, i in list{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + + else: + + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n"); + f.write(" file out ; \n"); + f.write(" file err ; \n"); + + f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") + f.write("}\n"); + f.write("printf(\"%s\", \"Done!\");\n"); + f.write("}\n"); + + f.close() + + #Invocation + with open(logfile,"wb") as errfile: + exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile) + print exitcode + + #Populate output file + outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) + f=open(outlistfile, "w") + f.write(outlist); + f.close() + +if __name__=='__main__': + main() Property changes on: SwiftApps/dsp-swift/swiftjumpdsp.py ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml (rev 0) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-11 19:44:14 UTC (rev 8331) @@ -0,0 +1,59 @@ + + + A Galaxy tool to run Dsp parameter sweeps via Swift foreach parallel construct + + + swiftjumpdsp.py + "${inlocation}" + "${a}" + "${n}" + "${p}" + "${I}" + "${outlist}" + + + + + + + + + + + + + + + + + +.. class:: warningmark + +**TIP**. Add args to provide additional arguments to your executable. + +----- + +**What it does** + +This is a tool to run dsp via Swift in parallel over files provided by user. + + + From ketan at ci.uchicago.edu Thu Dec 11 14:31:32 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 14:31:32 -0600 (CST) Subject: [Swift-commit] r8332 - SwiftApps/dsp-swift Message-ID: <20141211203132.37DB89D58A@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-11 14:31:32 -0600 (Thu, 11 Dec 2014) New Revision: 8332 Modified: SwiftApps/dsp-swift/swiftjumpdsp.py SwiftApps/dsp-swift/swiftjumpdsp.xml Log: add field for local vs compute Modified: SwiftApps/dsp-swift/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-11 19:44:14 UTC (rev 8331) +++ SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-11 20:31:32 UTC (rev 8332) @@ -31,16 +31,6 @@ for a in args: yield a -def buildcfg(wdir): - f=open(wdir+"/cf", 'w') - f.write("use.provider.staging=true\n") - f.write("wrapperlog.always.transfer=true\n") - f.write("execution.retries=0\n") - f.write("provider.staging.pin.swiftfiles=false\n") - f.write("sitedir.keep=true\n") - f.write("lazy.errors=false\n") - f.close() - def main(): myargs = genargs(sys.argv) try: @@ -58,134 +48,14 @@ sys.exit(1) #which swift - swift = distutils.spawn.find_executable("swift") + swift = "/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift" BASEDIR = os.path.dirname(os.path.realpath(__file__)) - - # workout the file args - fileargs=[] - for f in myargs: - fileargs.append(f) - - #workout the file array only if user adds args else make it blank - if fileargs: - if k_or_t == "swiftK": - filearrayexpr="file fileargs[] ;\n" - else: #T - filearrayexpr="file fileargs[]="+str(fileargs)+";\n" - else: - filearrayexpr="file fileargs[];\n" # create outloc dir if not os.path.exists(outloc): os.makedirs(outloc) - - # workout the stringargs - if stringargs: - stringarrayexpr="string stringargs[]="+re.sub('\'','"',str(stringargs.split()))+";\n" - else: - stringarrayexpr="string stringargs[];\n" - - wdir = setwdir().rstrip() - #workout standard input file - if stdin != "None": - if k_or_t == "swiftK": - stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" - else: - stdinfilexpr = "file stdinfile = input(\""+stdin+"\");\n" - else: - #print "no stdin" - if k_or_t == "swiftK": - stdinfilexpr = "file stdinfile<\"/dev/null\">;\n" - else: - stdinfilexpr = "file stdinfile = input(\"/dev/null\");\n" - - - #Build Swift source code - f = open(wdir+"/script.swift", "w"); - - if k_or_t == "swiftK": - - f.write("type file; \n") - f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ \n") - f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; \n") - f.write("}\n") - f.write(stringarrayexpr) - f.write(filearrayexpr) - f.write(stdinfilexpr) - f.write("file exec<\""+executable+"\">;\n") - - if range_or_list == "list": - - f.write("file listfile<\""+listfile+"\">;\n") - f.write("string list[] = readData(listfile);\n"); - f.write("foreach l, i in list{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - - else: - - f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - - f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") - f.write("}\n") - - #build site - if os.path.dirname(sys.argv[0]): - basedir=os.path.dirname(sys.argv[0]) - else: - basedir='.' - #print "%s %s %s" %(basedir, sites, wdir) - subprocess.call([basedir+"/buildsite.sh", sites, wdir]) - - #build config file - buildcfg(wdir) - - #build tc file - with open(wdir+"/tc", 'w') as tc: - for site in sites.split(): - tc.write(site+' '+interpreter+' '+interpreter) - - else: # T - - f.write("import files;\n"); - f.write("import string;\n"); - f.write("import io;\n"); - f.write("\n"); - f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n"); - f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n"); - f.write("}\n"); - f.write("main{\n"); - f.write(stringarrayexpr + " \n"); - f.write(filearrayexpr + " \n"); - f.write(stdinfilexpr + " \n"); - f.write("file exec = input(\""+executable+"\");\n"); - - if range_or_list == "list": - - f.write(" \n"); - f.write("file listfile=input(\""+listfile+"\");\n") - f.write("string list[] = split(read(listfile),);\n"); - f.write("foreach l, i in list{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - - else: - - f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n"); - f.write(" file out ; \n"); - f.write(" file err ; \n"); - - f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") - f.write("}\n"); - f.write("printf(\"%s\", \"Done!\");\n"); - f.write("}\n"); - - f.close() - #Invocation with open(logfile,"wb") as errfile: exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile) Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-11 19:44:14 UTC (rev 8331) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-11 20:31:32 UTC (rev 8332) @@ -1,4 +1,4 @@ - + -.. class:: warningmark - -**TIP**. Add args to provide additional arguments to your executable. - ------ - **What it does** This is a tool to run dsp via Swift in parallel over files provided by user. +More description to come. + From ketan at ci.uchicago.edu Thu Dec 11 16:00:00 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 16:00:00 -0600 (CST) Subject: [Swift-commit] r8333 - SwiftApps/dsp-swift Message-ID: <20141211220000.82AC19D58A@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-11 16:00:00 -0600 (Thu, 11 Dec 2014) New Revision: 8333 Modified: SwiftApps/dsp-swift/swiftjumpdsp.py SwiftApps/dsp-swift/swiftjumpdsp.xml Log: add missing set Modified: SwiftApps/dsp-swift/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-11 20:31:32 UTC (rev 8332) +++ SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-11 22:00:00 UTC (rev 8333) @@ -40,15 +40,16 @@ n = next(myargs) p = next(myargs) I = next(myargs) - outloc = next(myargs) - outlistfile = next(myargs) - logfile = next(myargs) + local_or_compute = next(myargs) + outlist = next(myargs) except: traceback.print_exc() sys.exit(1) #which swift swift = "/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift" + outloc = "$HOME/outdir" + swiftargs="-inloc="+inloc+" -a="+a+" -n="+n+" -p="+p+" -I="+I BASEDIR = os.path.dirname(os.path.realpath(__file__)) @@ -58,7 +59,7 @@ #Invocation with open(logfile,"wb") as errfile: - exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile) + exitcode=subprocess.check_call([swift, "-sites.file", "sites.local.xml", "-tc.file", "apps", "-config", "cf", "dsp.blues.swift", swiftargs],stderr=errfile) print exitcode #Populate output file Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-11 20:31:32 UTC (rev 8332) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-11 22:00:00 UTC (rev 8333) @@ -17,6 +17,7 @@ "${n}" "${p}" "${I}" + "${local_or_compute}" "${outlist}" - - - - + + + + - - + + + - - - **What it does** From ketan at ci.uchicago.edu Thu Dec 11 20:51:45 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 11 Dec 2014 20:51:45 -0600 (CST) Subject: [Swift-commit] r8337 - SwiftApps/dsp-swift Message-ID: <20141212025145.E504C9DFE9@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-11 20:51:45 -0600 (Thu, 11 Dec 2014) New Revision: 8337 Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml Log: typo Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 02:41:55 UTC (rev 8336) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 02:51:45 UTC (rev 8337) @@ -31,7 +31,7 @@ - + From ketan at ci.uchicago.edu Fri Dec 12 08:57:37 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 08:57:37 -0600 (CST) Subject: [Swift-commit] r8338 - SwiftApps/dsp-swift Message-ID: <20141212145737.863EE178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-12 08:57:36 -0600 (Fri, 12 Dec 2014) New Revision: 8338 Modified: SwiftApps/dsp-swift/README Log: clean readme Modified: SwiftApps/dsp-swift/README =================================================================== --- SwiftApps/dsp-swift/README 2014-12-12 02:51:45 UTC (rev 8337) +++ SwiftApps/dsp-swift/README 2014-12-12 14:57:36 UTC (rev 8338) @@ -4,193 +4,15 @@ Introduction ------------- -The Swift-Galaxy integration project aims to build tools which can be used to -leverage the best features of http://swift-lang.org/main[Swift] and -http://galaxyproject.org[Galaxy] combined. We start with building a generic -Galaxy tool, called `foreach` which allows user to compose Galaxy workflows -such that each computation stage in the workflow is executed by Swift. User can -compose Galaxy tool and pick up arbitrary executables and dataset to run the -executable over. User can also run the executable over a list of items such as -string or file parameters. +The `foreach` Mechanism +------------------------ -NOTE: This document can be found http://swift-lang.org/tutorials/galaxy/tutorial.html[here]. - -Swift-Galaxy `foreach` Mechanism ---------------------------------- - -The basic mechanism is that the tool user-interface will accept parameters and -will pass them along to a backend Swift "generator". This generator program -will run the parameters over Swift source and configuration templates. As a -result, the right source and configuration files are produced in a temporary -directory (`/tmp/tmpname`). This is then executed and the results collected. - -There are two ways the `foreach` Swift tool can be utilized. First, over a range -of numbers. The tool is used to run any arbitrary executable via Swift `foreach` -loop over a range of numbers. The executable will run with number in range as -commandline argument, one at a time. Furthermore, user can add arbitrary string -arguments in addition to the number argument to the call to executable. - -The second way runs the `foreach` loop over a list of items read from an input -file. These items can be file references or simple strings. The tool will pass -one item at a time to the executable as argument parameter and run them all in -parallel. This is similar to the above described number range except that the -iteration takes place over a list of items provided as a file. - - Installation ------------- -This installation recipe assumes that you have Galaxy preinstalled. If not, -installation instructions for Galaxy can be found -https://wiki.galaxyproject.org/Admin/GetGalaxy[here]. -Step 0. Checkout the swift-galaxy SVN directory as follows: - ----- -svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy ----- - -Step 1. Create a symbolic link to the +swift-galaxy/swift+ directory from -galaxy's +tools+ directory. For example, if Galaxy top level directory is -+galaxy-dist+ and swift directory is $HOME/swift-galaxy/swift: - ----- -cd galaxy-dist/tools -ln -s $HOME/swift-galaxy/swift swift ----- - -The advantage of symbolic links is that as users update the tool directory via -`svn up`, they need not worry about reinstallation. Updates in installation -gets automatically propagated to Galaxy. - -Step 2. Add the tool definitions of the tools contained in this Swift to -Galaxy's tool_conf.xml. This can be done by copying the text in -`
...
` from the tool_conf.xml in swift-galaxy directory and -paste it to the tool_conf.xml of the Galaxy directory. In the current installation, the text to be copied is: - ----- -
- - - -
----- - -Step 3. (Re)start Galaxy. - ----- -sh run.sh ----- - -In the Galaxy Web GUI, you should see the tools in a toolset titled Swift. - Screenshots ------------ +------------ -The following screenshot shows overall look-and-feel of the Swift-Galaxy tool -in the Galaxy web GUI. - -image::./screenshots/overall.png["swift-galaxy screenshot", width=600, align="center"] - -The rest of this section walks you through each of the components shown in the -above screen and their utility. The following screenshot shows execution -location selector. User can select one or more execution locations from the -list as their target execution site. - -image::./screenshots/execloc.png["execution location selector screenshot", width=200, align="center"] - -The following screenshot shows a dialog box to add Swift-level commandline -arguments to be passed to the script. - -image::./screenshots/swift-level-args.png["swift-args screenshot", width=200, align="center"] - -The following screenshot shows a selection dropdown list to chose the execution -interpreter from. The interpreters currently supported and tested are shell, -java and python. Upcoming are R and matlab. - -image::./screenshots/execution-interpreter.png["interpreter screenshot", width=150, align="center"] - -The following screenshot shows the selection for an executable. User can upload -an executable and select it from the list. - -image::./screenshots/executable.png["executable screenshot", width=200, align="center"] - -The following screenshot shows a selection for range of numbers or list of -items to chose from. User can select one and the corresponding dialog box will -appear below. - -image::./screenshots/rangeorlist.png["select range or list screenshot", width=150, align="center"] - -The following screenshot shows a dialog for string or file type arguments. A -user can add string and filetype arguments to their executable and the -commandline will be built accordingly. - -image::./screenshots/string-or-fileargs.png["string or fileargs screenshot", width=250, align="center"] - -The following screenshot shows a dialog box to enter the directory location for -results of execution. The Swift output files will be placed here: - -image::./screenshots/outlocation.png["output location screenshot", width=250, align="center"] - An example ----------- +----------- -A simple Hello World Swift tool to run with Galaxy. - -To set up with your Galaxy environment follow the steps below: -Prerequisite: Sun/Oracle java. Should work with IBM java but not tested. To test your java version: - ----- -java -version ----- - -Typical output is: - ----- -java version "1.7.0_17" -Java(TM) SE Runtime Environment (build 1.7.0_17-b02) -Java HotSpot(TM) 64-Bit Server VM (build 23.7-b01, mixed mode) ----- - -Step 1. Download and install Swift where Galaxy server is running. From a -command prompt the following wget will pull the latest release from -http://swift-lang.org/downloads/index.php[here]. - -Step 2. SVN checkout the swift-galaxy directory and copy the Swift tool directory to Galaxy's tools directory: - ----- -svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy -cp -r swift-galaxy/swift /location/of/galaxy-server/tools/ ----- - -Step 3. Edit the +tool_conf.xml+ file present at the top-level of galaxy directory tree to add the following lines between the ++ and ++ xml tags: - ----- -
- -
----- - -Step 4. Restart the Galaxy server. The swift-galaxy tool should be available in the tool box of Galaxy GUI. - -Step 5. Invoke it by entering any number in the dialog box or leaving it to default. - -Step 6. For examples of other Galaxy-Swift tools, look into the swift directoty. - -Troubleshoot -------------- - -Each tool generates a set of Swift source and configuration files. These files are placed in a unique directory with a random name generated on the fly. The location of the directory is in `/tmp` and it is prefixed with name `swift-gal.` . The suffix is a four letter random number. You can run Swift from this directory to reproduce possible issues as follows: - ----- -swift -tc.file tc -sites.file sites.xml -config cf script.swift ----- - -Feel free to contact Swift http://swift-lang.org/support/index.php[support] if issues persist. - -Upcoming Features ------------------- - -* Mixing the order of file and string arguments -* Optionally allowing user to not use range number as arguments -* Providing more Swift patterns, eg. `if-else` conditionals - From ketan at ci.uchicago.edu Fri Dec 12 09:18:17 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 09:18:17 -0600 (CST) Subject: [Swift-commit] r8339 - SwiftApps/dsp-swift Message-ID: <20141212151817.5B0AB178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-12 09:18:17 -0600 (Fri, 12 Dec 2014) New Revision: 8339 Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml Log: default values Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 14:57:36 UTC (rev 8338) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 15:18:17 UTC (rev 8339) @@ -22,16 +22,16 @@ "${outlist}" - - - - - + + + + + - + @@ -39,7 +39,11 @@ **What it does** -This is a tool to run dsp via Swift in parallel over files provided by user. +Run dsp via Swift in parallel over files provided by user. +The files are chosen from the user provided path. It currently works on LCRC +Blues login and compute nodes. The tool accepts various dsp parameters as +input. User can specify the desired location to put the output files.If the +directory of this location does not exist, it will be created. More description to come. From ketan at ci.uchicago.edu Fri Dec 12 09:26:45 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 09:26:45 -0600 (CST) Subject: [Swift-commit] r8340 - SwiftApps/dsp-swift Message-ID: <20141212152645.63FFD178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-12 09:26:45 -0600 (Fri, 12 Dec 2014) New Revision: 8340 Modified: SwiftApps/dsp-swift/swiftjumpdsp.py Log: cleanup py Modified: SwiftApps/dsp-swift/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-12 15:18:17 UTC (rev 8339) +++ SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-12 15:26:45 UTC (rev 8340) @@ -48,15 +48,18 @@ sys.exit(1) #which swift + baseloc="/home/ketan/SwiftApps/dsp-swift" swift = "/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift" swiftargs="-loc="+inloc+" -a="+a+" -n="+n+" -pval="+p+" -I="+I+" -outloc="+outloc - - BASEDIR = os.path.dirname(os.path.realpath(__file__)) - + siteval=baseloc+"/sites.local.xml" + + if local_or_compute == "compute": + siteval=baseloc+"/sites.blues.xml" + #Invocation logfile="/tmp/std.err" with open(logfile,"wb") as errfile: - exitcode=subprocess.check_call([swift, "-sites.file", "/home/ketan/SwiftApps/dsp-swift/sites.local.xml", "-tc.file", "/home/ketan/SwiftApps/dsp-swift/apps", "-config", "/home/ketan/SwiftApps/dsp-swift/cf", "/home/ketan/SwiftApps/dsp-swift/dsp.blues.swift", swiftargs],stderr=errfile) + exitcode=subprocess.check_call([swift, "-sites.file", siteval, "-tc.file", baseloc+"/apps", "-config", baseloc+"/cf", baseloc+"/dsp.blues.swift", swiftargs],stderr=errfile) print exitcode #Populate output file From ketan at ci.uchicago.edu Fri Dec 12 09:36:20 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 09:36:20 -0600 (CST) Subject: [Swift-commit] r8341 - SwiftApps/dsp-swift Message-ID: <20141212153620.65EF6178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-12 09:36:20 -0600 (Fri, 12 Dec 2014) New Revision: 8341 Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml Log: doc Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 15:26:45 UTC (rev 8340) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 15:36:20 UTC (rev 8341) @@ -37,14 +37,19 @@ -**What it does** + **What is this?** + A tool to run dsp in parallel parameter sweeps. + + **What it does?** + Runs dsp in parallel over a user provided input dataset and parameters. -Run dsp via Swift in parallel over files provided by user. -The files are chosen from the user provided path. It currently works on LCRC -Blues login and compute nodes. The tool accepts various dsp parameters as -input. User can specify the desired location to put the output files.If the -directory of this location does not exist, it will be created. + **How does it do?** + The user provides input to the tool which is then delivered to the + underlying Swift script. The script is then invoked on local/login or + compute nodes as selected by the user. Once the tool has run, the output is + produced in the directory location provided by the user. + More description to come.
From hategan at ci.uchicago.edu Fri Dec 12 15:37:12 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 15:37:12 -0600 (CST) Subject: [Swift-commit] r8342 - branches/release-0.95/bin Message-ID: <20141212213712.652AB178884@svn.ci.uchicago.edu> Author: hategan Date: 2014-12-12 15:37:12 -0600 (Fri, 12 Dec 2014) New Revision: 8342 Modified: branches/release-0.95/bin/swiftrun Log: don't allow abbreviated options in swiftrun/Getopt Modified: branches/release-0.95/bin/swiftrun =================================================================== --- branches/release-0.95/bin/swiftrun 2014-12-12 15:36:20 UTC (rev 8341) +++ branches/release-0.95/bin/swiftrun 2014-12-12 21:37:12 UTC (rev 8342) @@ -9,6 +9,7 @@ use Getopt::Long qw(GetOptionsFromArray); use Cwd; +Getopt::Long::Configure("posix_default"); Getopt::Long::Configure("pass_through"); my %properties = (); # Hash storing all swift properties From hategan at ci.uchicago.edu Fri Dec 12 15:48:06 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 15:48:06 -0600 (CST) Subject: [Swift-commit] r8343 - branches/release-0.95/bin Message-ID: <20141212214806.A0644178884@svn.ci.uchicago.edu> Author: hategan Date: 2014-12-12 15:48:06 -0600 (Fri, 12 Dec 2014) New Revision: 8343 Modified: branches/release-0.95/bin/swiftrun Log: actually make it look like the trunk version Modified: branches/release-0.95/bin/swiftrun =================================================================== --- branches/release-0.95/bin/swiftrun 2014-12-12 21:37:12 UTC (rev 8342) +++ branches/release-0.95/bin/swiftrun 2014-12-12 21:48:06 UTC (rev 8343) @@ -9,8 +9,7 @@ use Getopt::Long qw(GetOptionsFromArray); use Cwd; -Getopt::Long::Configure("posix_default"); -Getopt::Long::Configure("pass_through"); +Getopt::Long::Configure("pass_through", "no_auto_abbrev"); my %properties = (); # Hash storing all swift properties my @property_files = (); # List of swift property files to be read From ketan at ci.uchicago.edu Fri Dec 12 16:56:21 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 16:56:21 -0600 (CST) Subject: [Swift-commit] r8344 - SwiftApps/dsp-swift Message-ID: <20141212225621.3E317178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-12 16:56:21 -0600 (Fri, 12 Dec 2014) New Revision: 8344 Modified: SwiftApps/dsp-swift/apps SwiftApps/dsp-swift/dsp.blues.swift SwiftApps/dsp-swift/swiftjumpdsp.py SwiftApps/dsp-swift/swiftjumpdsp.xml Log: more refinements Modified: SwiftApps/dsp-swift/apps =================================================================== --- SwiftApps/dsp-swift/apps 2014-12-12 21:48:06 UTC (rev 8343) +++ SwiftApps/dsp-swift/apps 2014-12-12 22:56:21 UTC (rev 8344) @@ -6,6 +6,7 @@ cluster2 bgsh /home/ketan/SwiftApps/subjobs/bg.sh null null null # blues dsp /lcrc/project/NEXTGENOPT/DSP/Dsp null null null +blues plotme /home/ketan/SwiftApps/dsp-swift/plotme null null null # blues lammps lmp_blues null null null blues smeagol /home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg null null null Modified: SwiftApps/dsp-swift/dsp.blues.swift =================================================================== --- SwiftApps/dsp-swift/dsp.blues.swift 2014-12-12 21:48:06 UTC (rev 8343) +++ SwiftApps/dsp-swift/dsp.blues.swift 2014-12-12 22:56:21 UTC (rev 8344) @@ -18,6 +18,13 @@ dsp "-f" _filespath "-a" _a "-n" _n "-p" _p "-I" _I stdout=@_out stderr=@_err; } + +/* == Plot app definition == */ +app (file _img) plot_app (file[] _fromdsp) +{ + plotme @_img @_fromdsp; +} + /* == Input files definitions: Make sure these files are available in the current directory == */ string inloc=arg("loc","/lcrc/project/NEXTGENOPT/DSP_old/examples/smps/dcap"); @@ -32,6 +39,8 @@ string outdir=arg("outloc","outdir"); +file outs[]; + foreach in_tim,i in timfiles{ /* == Output files definitions == */ @@ -39,6 +48,15 @@ file err ; string filespath[]=strsplit(filename(in_tim),"\\."); + //tracef(" stofiles[%i] = %s\ncorfiles[%i] = %s\ntimfiles[%i] = %s\n", i, filename(stofiles[i]), i, filename(corfiles[i]), i, filename(timfiles[i])); + /* == App invocation == */ (out, err) = dsp_app (filespath[0], stofiles[i], corfiles[i], timfiles[i], a, n, p, I); -} + outs[i]=out; +} + +file img<"img.png">; + +//img=plot_app(outs); + + Modified: SwiftApps/dsp-swift/swiftjumpdsp.py =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-12 21:48:06 UTC (rev 8343) +++ SwiftApps/dsp-swift/swiftjumpdsp.py 2014-12-12 22:56:21 UTC (rev 8344) @@ -43,6 +43,7 @@ local_or_compute = next(myargs) outloc = next(myargs) outlistfile = next(myargs) + logfile=next(myargs) except: traceback.print_exc() sys.exit(1) @@ -57,10 +58,9 @@ siteval=baseloc+"/sites.blues.xml" #Invocation - logfile="/tmp/std.err" with open(logfile,"wb") as errfile: exitcode=subprocess.check_call([swift, "-sites.file", siteval, "-tc.file", baseloc+"/apps", "-config", baseloc+"/cf", baseloc+"/dsp.blues.swift", swiftargs],stderr=errfile) - print exitcode + #print exitcode #Populate output file outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 21:48:06 UTC (rev 8343) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 22:56:21 UTC (rev 8344) @@ -20,6 +20,7 @@ "${local_or_compute}" "${resloc}" "${outlist}" + "${outlogfile}" @@ -35,6 +36,7 @@ + **What is this?** From ketan at ci.uchicago.edu Fri Dec 12 17:06:20 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 17:06:20 -0600 (CST) Subject: [Swift-commit] r8345 - SwiftApps/dsp-swift Message-ID: <20141212230620.4FA59178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-12 17:06:20 -0600 (Fri, 12 Dec 2014) New Revision: 8345 Modified: SwiftApps/dsp-swift/dsp.blues.swift Log: structured regexp mapper Modified: SwiftApps/dsp-swift/dsp.blues.swift =================================================================== --- SwiftApps/dsp-swift/dsp.blues.swift 2014-12-12 22:56:21 UTC (rev 8344) +++ SwiftApps/dsp-swift/dsp.blues.swift 2014-12-12 23:06:20 UTC (rev 8345) @@ -29,8 +29,8 @@ string inloc=arg("loc","/lcrc/project/NEXTGENOPT/DSP_old/examples/smps/dcap"); file stofiles[]; -file corfiles[]; -file timfiles[]; +file corfiles[] ; +file timfiles[] ; int a=toInt(arg("a","1")); int n=toInt(arg("n","1")); @@ -48,7 +48,7 @@ file err ; string filespath[]=strsplit(filename(in_tim),"\\."); - //tracef(" stofiles[%i] = %s\ncorfiles[%i] = %s\ntimfiles[%i] = %s\n", i, filename(stofiles[i]), i, filename(corfiles[i]), i, filename(timfiles[i])); + tracef(" stofiles[%i] = %s\ncorfiles[%i] = %s\ntimfiles[%i] = %s\n", i, filename(stofiles[i]), i, filename(corfiles[i]), i, filename(timfiles[i])); /* == App invocation == */ (out, err) = dsp_app (filespath[0], stofiles[i], corfiles[i], timfiles[i], a, n, p, I); From ketan at ci.uchicago.edu Fri Dec 12 17:30:23 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Dec 2014 17:30:23 -0600 (CST) Subject: [Swift-commit] r8346 - in SwiftApps: dsp-swift subjobs Message-ID: <20141212233023.E8BF1178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-12-12 17:30:23 -0600 (Fri, 12 Dec 2014) New Revision: 8346 Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml SwiftApps/subjobs/README.txt Log: _ Modified: SwiftApps/dsp-swift/swiftjumpdsp.xml =================================================================== --- SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 23:06:20 UTC (rev 8345) +++ SwiftApps/dsp-swift/swiftjumpdsp.xml 2014-12-12 23:30:23 UTC (rev 8346) @@ -39,19 +39,21 @@ - **What is this?** - A tool to run dsp in parallel parameter sweeps. - - **What it does?** - Runs dsp in parallel over a user provided input dataset and parameters. +**What is this?** - **How does it do?** - The user provides input to the tool which is then delivered to the - underlying Swift script. The script is then invoked on local/login or - compute nodes as selected by the user. Once the tool has run, the output is - produced in the directory location provided by the user. +A tool to run dsp in parallel parameter sweeps. +**What it does?** +Runs dsp in parallel over a user provided input dataset and parameters. + +**How does it do?** + +The user provides input to the tool which is then delivered to the +underlying Swift script. The script is then invoked on local/login or +compute nodes as selected by the user. Once the tool has run, the output is +produced in the directory location provided by the user. + More description to come. Modified: SwiftApps/subjobs/README.txt =================================================================== --- SwiftApps/subjobs/README.txt 2014-12-12 23:06:20 UTC (rev 8345) +++ SwiftApps/subjobs/README.txt 2014-12-12 23:30:23 UTC (rev 8346) @@ -208,6 +208,15 @@ ./runmira.sh #run on mira ---- +GridPack +~~~~~~~~ +GridPack is a simulation package designed for powergrid design application. + +HALO +~~~~ +HALO is a astrophysics application. + + Internals ---------- The key driver of the Swift sub-block jobs is a script called +bg.sh+ that does the sub-block jobs calculations and othe chores for the users. The script looks as follows: From hategan at ci.uchicago.edu Sat Dec 13 15:37:45 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Dec 2014 15:37:45 -0600 (CST) Subject: [Swift-commit] r8347 - branches/release-0.95/src/org/griphyn/vdl/mapping Message-ID: <20141213213745.6EA0B9D919@svn.ci.uchicago.edu> Author: hategan Date: 2014-12-13 15:37:44 -0600 (Sat, 13 Dec 2014) New Revision: 8347 Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/AbstractDataNode.java Log: slightly better toString() for file types Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2014-12-12 23:30:23 UTC (rev 8346) +++ branches/release-0.95/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2014-12-13 21:37:44 UTC (rev 8347) @@ -45,7 +45,11 @@ public abstract class AbstractDataNode implements DSHandle, FutureValue { - public static final Object FILE_VALUE = new Object(); + public static final Object FILE_VALUE = new Object() { + public String toString() { + return ""; + } + }; static final String DATASET_URI_PREFIX = "dataset:"; From hategan at ci.uchicago.edu Sat Dec 13 15:48:36 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Dec 2014 15:48:36 -0600 (CST) Subject: [Swift-commit] r8348 - branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20141213214836.DE3539D919@svn.ci.uchicago.edu> Author: hategan Date: 2014-12-13 15:48:36 -0600 (Sat, 13 Dec 2014) New Revision: 8348 Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java Log: made filename() and filenames() context aware so that remote path translation is only done if invoked from an app body Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2014-12-13 21:37:44 UTC (rev 8347) +++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2014-12-13 21:48:36 UTC (rev 8348) @@ -20,7 +20,11 @@ import k.rt.Stack; import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; @@ -30,6 +34,17 @@ public class FileName extends SwiftFunction { private ArgRef var; + private boolean inAppInvocation; + + @Override + public Node compile(WrapperNode w, Scope scope) throws CompilationException { + Node self = super.compile(w, scope); + // either execute(arguments(this)) or execute(named(stdxxx, this)) + if (hasAncestor(this, "swift:execute")) { + inAppInvocation = true; + } + return self; + } @Override protected Signature getSignature() { @@ -44,7 +59,7 @@ @Override public Object function(Stack stack) { AbstractDataNode var = this.var.getValue(stack); - String s = argList(filename(var), true); + String s = argList(filename(var), inAppInvocation); DSHandle result = new RootDataNode(Types.STRING, s); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2014-12-13 21:37:44 UTC (rev 8347) +++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2014-12-13 21:48:36 UTC (rev 8348) @@ -24,7 +24,11 @@ import k.rt.Stack; import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; @@ -36,6 +40,17 @@ public class FileNames extends SwiftFunction { private ArgRef var; + private boolean inAppInvocation; + + @Override + public Node compile(WrapperNode w, Scope scope) throws CompilationException { + Node self = super.compile(w, scope); + // either execute(arguments(this)) or execute(named(stdxxx, this)) + if (hasAncestor(this, "swift:execute")) { + inAppInvocation = true; + } + return self; + } @Override protected Signature getSignature() { @@ -56,7 +71,12 @@ for (int i = 0; i < f.length; i++) { Path p = parsePath("["+i+"]"); DSHandle h = returnArray.getField(p); - h.setValue(relativize(f[i])); + if (inAppInvocation) { + h.setValue(relativize(f[i])); + } + else { + h.setValue(f[i]); + } } } catch (InvalidPathException e) { throw new ExecutionException("Unexpected invalid path exception",e); From wozniak at ci.uchicago.edu Tue Dec 16 10:40:07 2014 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Tue, 16 Dec 2014 10:40:07 -0600 (CST) Subject: [Swift-commit] r8349 - www/Swift-T Message-ID: <20141216164007.D661F178884@svn.ci.uchicago.edu> Author: wozniak Date: 2014-12-16 10:40:06 -0600 (Tue, 16 Dec 2014) New Revision: 8349 Modified: www/Swift-T/turbine-sites.html Log: Turbine Sites updates Modified: www/Swift-T/turbine-sites.html =================================================================== --- www/Swift-T/turbine-sites.html 2014-12-13 21:48:36 UTC (rev 8348) +++ www/Swift-T/turbine-sites.html 2014-12-16 16:40:06 UTC (rev 8349) @@ -747,7 +747,7 @@ Justin M. Wozniak
<wozniak at mcs.anl.gov>
version 0.7.0, -November 2014 +December 2014
Table of Contents
@@ -1360,8 +1360,9 @@

To run:

-
export QUEUE=batch
-~wozniak/Public/turbine/scripts/submit/pbs/turbine-pbs-run.zsh -n 3 program.tic
+
$ export QUEUE=batch
+$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/software/gcc-4.7.2/lib64
+$ ~wozniak/Public/turbine/scripts/submit/pbs/turbine-pbs-run.zsh -n 3 program.tcl

See the Turbine scheduler variables and Turbine run script options for additional settings.

@@ -1372,6 +1373,39 @@
$ which gcc
+$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/software/gcc-4.7.2/lib64
+
+
+ +
+

Fusion

+

Fusion is a 320-node x86 +cluster at ANL. It uses PBS.

+
+

Public installation

+
    +
  • +

    +STC: ~wozniak/Public/compute/stc/bin/stc +

    +
  • +
+

To run:

+
+
+
export QUEUE=batch
+$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/gcc/4.7.2/lib64
+$ ~wozniak/Public/sfw/compute/turbine/scripts/submit/pbs/turbine-pbs-run.zsh -n 3 program.tic
+
+

See the Turbine scheduler variables and +Turbine run script options for additional settings.

+
+
+

Build procedure

+

Use GCC 4.7.2 and set LD_LIBRARY_PATH:

+
+
+
$ which gcc
 /software/gcc-4.7.2/bin/gcc
 $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/software/gcc-4.7.2/lib64
@@ -1408,7 +1442,7 @@ (ALCF). Other existing Blue Gene/P systems may be configured in a similar way.

-
Public installation
+
Public installation
  • @@ -1428,7 +1462,7 @@

-
Build procedure
+
Build procedure

To run on the login node:

  • @@ -1542,67 +1576,68 @@

    Blue Gene/Q

    -

    Mira/Cetus

    -

    As of: 0.7.0 - 11/20/2014

    -

    Add to PATH:

    +

    ALCF

    • -STC: ~wozniak/Public/ppc64/stc/bin +Run with:

      +
      +
      +
      export MODE=BGQ
      +export PROJECT=<project_name>
      +export QUEUE=<queue_name>
      +swift-t -m cobalt -n 3 program.tic
      +
      +

      or:

      +
      +
      +
      export MODE=BGQ
      +export QUEUE=<queue_name>
      +turbine-cobalt-run.zsh -n 2 program.tic
      +
    • +
    +

    The normal Turbine environment variables are honored, plus the +Turbine scheduler variables.

    +
    +
    Public installation: Vesta
    +

    As of: 0.7.0 - 12/16/2014

    +
    • -Turbine (optional): ~wozniak/Public/ppc64/turbine/scripts/submit/cobalt +STC: ~wozniak/Public/sfw/stc/bin/stc

    • -Run as: +Turbine: ~wozniak/Public/sfw/turbine/scripts/submit/cobalt/turbine-cobalt-run.zsh

      -
      -
      -
      export MODE=BGQ
      -export QUEUE=...
      -export PROJECT=...
      -swift-t -m cobalt -n 3 program.tic
      -
    -
    -

    Vesta

    -

    As of: 0.7.0 - 11/20/2014

    +
    +
    Public installation: Mira/Cetus
    +

    As of: 0.5.0 - 4/4/2014

    • -STC: ~wozniak/Public/sfw/stc/bin +STC: ~wozniak/Public/ppc64/stc/bin/stc

    • -Turbine (optional): ~wozniak/Public/sfw/turbine/scripts/submit/turbine-cobalt-run.zsh +Turbine: +~wozniak/Public/ppc64/turbine/scripts/submit/cobalt/turbine-cobalt-run.zsh

    • -
    • -

      -Run as: -

      -
      -
      -
      export MODE=BGQ
      -export QUEUE=default
      -swift-t -m cobalt -n 2 program.tic
      -
      -
    -

    The normal Turbine environment variables are honored, plus the -Turbine scheduler variables.

    +
    -

    Build procedure

    +

    Build procedure

    As of: 0.7.0 - 11/20/2014

    -

    Tcl:

    +

    Building Tcl:

    The GCC installation does not support shared libraries. Thus, you must compile Tcl with bgxlc. You must modify the Makefile to use bgxlc arguments: -qpic, -qmkshrobj. You must link with @@ -1610,6 +1645,7 @@

    You may get errors that say wrong digit. This is apparently a bgxlc bug when applied to Tcl’s StrToD.c. Compiling this file with -O3 fixes the problem.

    +

    Building Swift/T:

    • @@ -1685,7 +1721,7 @@

      Titan

      Titan is a Cray XK7 at the Oak Ridge Leadership Computing Facility.

      -

      Public installation

      +

      Public installation

      As of: 8/27/2014

      -

      Build procedure

      +

      Build procedure

      As of: 8/27/2014

      Cray systems do not use mpicc. We set CC=gcc and use compiler flags to configure the MPI library.

      @@ -1805,7 +1841,7 @@

      Blue Waters is a Cray XE6/XK7 at the University of Illinois at Urbana-Champaign.

      -

      Build procedure

      +

      Build procedure

      As of: 11/05/2013

      Cray systems do not use mpicc. We set CC=gcc and use compiler flags to configure the MPI library.

      @@ -2023,7 +2059,7 @@ NFS (including home directories). Thus, you must install Turbine and its libraries in /lustre. Also, your data must be in /lustre.

      -

      Public installation

      +

      Public installation

      Login nodes

      This installation is for use on the login node.

      @@ -2085,7 +2121,7 @@
      -

      Build procedure

      +

      Build procedure

      Cray systems do not use mpicc. We set CC=gcc and use compiler flags to configure the MPI library.

        @@ -2144,7 +2180,7 @@

        Raven

        Raven is a Cray XE6/XK7 at Cray.

        -

        Build procedure

        +

        Build procedure

        • @@ -2305,7 +2341,7 @@

          From wozniak at ci.uchicago.edu Wed Dec 17 11:49:09 2014 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Dec 2014 11:49:09 -0600 (CST) Subject: [Swift-commit] r8350 - www/Swift-T Message-ID: <20141217174909.A53279D890@svn.ci.uchicago.edu> Author: wozniak Date: 2014-12-17 11:49:08 -0600 (Wed, 17 Dec 2014) New Revision: 8350 Modified: www/Swift-T/guide.html Log: Swift/T guide updates Modified: www/Swift-T/guide.html =================================================================== --- www/Swift-T/guide.html 2014-12-16 16:40:06 UTC (rev 8349) +++ www/Swift-T/guide.html 2014-12-17 17:49:08 UTC (rev 8350) @@ -744,7 +744,7 @@
        +
        +file_exists(string) → boolean +
        +
        +

        +Attempt to find a file with the given name in the filesystem: return +true if found, else false. +

        +
        +
        +file_lines(file) → string[] +
        +
        +

        +Reads the whole file, returning each line as a separate entry in the +output array. Comments with # are excised, leading and trailing +whitespace is trimmed, and blank lines are omitted. +

        +
        +
        +file_mtime(string) → int +
        +
        +

        +Attempt to find a file with the given name in the filesystem: return +its POSIX modification time in seconds since the Unix epoch. +

        +
        +
        +file_type(file) → string +
        +
        +

        +Returns a string giving the type of file, which will be one of +"file", "directory", "characterSpecial", "blockSpecial", +"fifo", "link", or "socket". +

        +
      @@ -2787,7 +2825,7 @@ ];

      We can also tell the Swift compiler a little about the function so -that it can better optimize your programs. For example, double has +that it can better optimmize your programs. For example, double has no side-effects and produces the same result each time for the same arguments (i.e. is deterministic), so we can annotate it as a @pure function.

      @@ -2897,7 +2935,72 @@
    -

    13.2. App functions

    +

    13.2. Dispatch and work types

    +

    Each worker in Swift/T is devoted to executing a single type of work. +There is a default work type that encompasses Swift/T script logic and +CPU tasks.

    +

    There are two subtypes of the default work type that are treated +differently by the Swift/T optimizer. CONTROL, the default, +is intended for functions that run for a short duration, for example +builtin functions such as strcat(), arithmetic. +WORKER, the second subtype, +is intended for functions that perform more computation and I/O, +which may keep a worker busy for a while. Swift/T may bundle +together multiple CONTROL tasks to reduce overhead, but will +keep WORKER tasks separate to increase parallelism.

    +

    The work type is specified with a @dispatch annotation above +leaf function definitions.

    +
    +
    +
    @dispatch=WORKER
    +(int o) process (int i) "pkg" "0.0.1" [
    +  "set <<o>> [ pkg::process <<i>> ]"
    +];
    +
    +

    Alternative work-types include custom user-defined work types, +along with tasks for external executors such as Coasters or GeMTC +for remote command-line and GPU tasks respectively. +It is also possible to define custom types of CPU leaf functions.

    +
    +

    13.2.1. Custom work types

    +

    For some applications, it is useful to be able to divide up +CPU workers into multiple categories that execute different +kinds of work. For these scenarios, Swift/T provides the +ability to define custom work types.

    +

    This sample program illustrates how to define a custom work +type foo_work and define a function, hello1, which executes +on a foo_work worker. For comparison, we also include hello2, +which uses the default work type.

    +
    +
    +
    pragma worktypedef foo_work;
    +
    + at dispatch=foo_work
    +hello1 (string name) "turbine" "0.0" [
    +  "puts [ format {Hello %s} <<msg>> ]"
    +];
    +
    +hello2 (string name) "turbine" "0.0" [
    +  "puts [ format {Hello %s} <<msg>> ]"
    +]
    +
    +main () {
    +  // Hello Foo will be printed on a foo_work worker
    +  hello1("Foo");
    +
    +  // Hello Bar will be printed on a regular worker
    +  hello2("Bar");
    +}
    +
    +

    In order to run the above script, we need to ensure that +a worker is allocated to execute foo_work tasks. This is +achieved by setting the environment variable TURBINE_FOO_WORK_WORKERS +to the desired number of workers. The Swift/T workers are then divided +between default workers and foo_work workers.

    +
    +
    +
    +

    13.3. App functions

    App functions are functions that are implemented as command-line programs. These command-line programs can be brought into a Swift program as functions with typed inputs and outputs. @@ -2980,9 +3083,119 @@

    -

    13.3. External scripting support

    +

    13.4. Remote job execution with Coasters

    +

    Note: Swift/T and Coasters integration is a work in progress +and is currently best suited for advanced users. +Planned future changes will make it easier to install and use.

    +

    Swift/T supports execution of command-line app functions on +a wide range of clusters, clouds, and grids with the +Coaster executor. In order for an app function to be executed +through coasters, the annotation @dispatch=COASTER must be +added to the app function definition:

    +
    +
    +
    @dispatch=COASTER
    +app (file out) echo (string args[]) {
    +  "echo" args @stdout=out
    +}
    +
    +

    Running a Swift/T script with Coasters function requires +a few steps to set up all components:

    +
      +
    • +

      +Before starting, you must have Swift/T compiled with Coaster + support, and the coaster service from Swift/K installed. +

      +
    • +
    • +

      +Before running your Swift/T script, start a Coaster service + from the shell, for example: +

      +
    • +
    +
    +
    +
    export WORKER_MODE=local
    +export IPADDR=127.0.0.1
    +export SERVICE_PORT=53363
    +export JOBSPERNODE=4
    +export LOGDIR=/home/user/swift-logs
    +export WORKER_LOG_DIR=/home/user/swift-logs
    +
    +coaster-service -nosec -port ${SERVICE_PORT}
    +
    +
      +
    • +

      +Coaster configuration must be set in the the TURBINE_COASTER_CONFIG + environment variable, for example: +

      +
    • +
    +
    +
    +
    export TURBINE_COASTER_CONFIG="jobManager=local,maxParallelTasks=4,coasterServiceURL=${IPADDR}:${SERVICE_PORT}"
    +
    +
      +
    • +

      +Once the Coaster service is running and TURBINE_COASTER_CONFIG is set, + you can run your Swift/T program in the normal way, and any coaster app + tasks will be dispatched to the Coaster service for execution. +

      +
    • +
    +

    Configuration keys include:

    +
    +
    +coasterServiceURL +
    +
    +

    +the url of the coaster service to submit tasks through, + e.g. localhost:63001. Default is 127.0.0.1:53001. +

    +
    +
    +jobManager +
    +
    +

    +the Coaster job manager the service should use to submit tasks. + E.g. to execute jobs locally, the job manager can be set to local + and to execute jobs on resources managed through a batch scheduler such + as PBS or Slurm, the job manager should be set to pbs, slurm, or + the appropriate scheduler. +

    +
    +
    +maxParallelTasks +
    +
    +

    +the maximum number of concurrent tasks per Coaster + worker. Default is 256. +

    +
    +
    +Other settings +
    +
    +

    +additional configuration keys are passed through to the + Coaster service. +

    +
    +
    +

    For more information on configuring and using Coasters, please refer +to the Swift/K documentation.

    +
    +
    +

    13.5. External scripting support

    -

    13.3.1. Calling Python

    +

    13.5.1. Calling Python

    You can evaluate arbitrary Python code from within Swift/T. For example, you can perform processing with a Python library. Once you have that working, you can use Swift/T to coordinate @@ -3073,7 +3286,7 @@

    -

    13.3.2. Calling R

    +

    13.5.2. Calling R

    Consider the following Swift script:

    @@ -3125,7 +3338,7 @@
    -

    13.3.3. Calling Julia

    +

    13.5.3. Calling Julia

    Consider the following Swift script:

    @@ -3177,10 +3390,10 @@
    -

    14. Annotations

    +

    14. Function call annotations

    -

    Swift/T supports many annotations to influence the behavior of the -run.

    +

    Swift/T supports many annotations to influence the behavior of function +calls.

    14.1. Priority

    Leaf tasks resulting from Swift dataflow may be prioritized by using @@ -3220,25 +3433,8 @@ debugged or disabled.

    -
    -

    14.3. Dispatch

    -

    This simply dispatches tasks to the given process type: WORKER or -SERVER. By default, work executes on a SERVER.

    -
    -
    -
    @dispatch=WORKER
    -(int o) process (int i) "pkg" "0.0.1" [
    -  "set <<o>> [ pkg::process <<i>> ]"
    -];
    -
    -

    Swift/T builtin functions (strcat(), arithmetic, etc.) are typically -implemented in this way as tasks that execute on the server. User -work should be performed on workers to keep servers free to process -dataflow.

    -

    This annotation is applied to the leaf task definition.

    -

    15. Optimizations

    @@ -3429,6 +3625,36 @@ Swift (Turbine information).

    +TURBINE_<type>_WORKERS +
    +
    +

    +Number of workers of specified type. +

    +
    +
    +

    Any workers not allocated to specific types are general-purpose workers +that execute Swift/T control code and CPU-based tasks. There must +be at least one leftover worker to serve as a general-purpose worker.

    +

    Valid work types include:

    +
      +
    • +

      +TURBINE_COASTER_WORKERS:: for Coaster workers. +

      +
    • +
    • +

      +A work type defined with pragma worktypedef in Swift. + E.g. if you define pragma worktypedef a_new_work_type;" in Swift, + the environment variable is +TURBINE_A_NEW_WORK_TYPE_WORKERS. +

      +
    • +
    +

    Generally you will need to allocate workers for any work type used in +your Swift program.

    +
    +
    TURBINE_LOG=0
    @@ -4001,7 +4227,7 @@

    From wozniak at ci.uchicago.edu Fri Dec 19 16:26:28 2014 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Fri, 19 Dec 2014 16:26:28 -0600 (CST) Subject: [Swift-commit] r8351 - www Message-ID: <20141219222628.794E0178884@svn.ci.uchicago.edu> Author: wozniak Date: 2014-12-19 16:26:27 -0600 (Fri, 19 Dec 2014) New Revision: 8351 Added: www/gallery.html Log: Initial Swift/T application gallery Added: www/gallery.html =================================================================== --- www/gallery.html (rev 0) +++ www/gallery.html 2014-12-19 22:26:27 UTC (rev 8351) @@ -0,0 +1,965 @@ + + + + + +Swift/T Gallery + + + + + +
    +
    +
    +

    This page demonstrates the ease of use of Swift/T when constructing +common application patterns.

    +

    Links:

    + +

    Post to the ExM +user list with questions about these examples.

    +
    +
    +
    +

    1. Hello world

    +
    +

    File: hello-world/hello-world.swift

    +
    +
    +
    import io;
    +main
    +{
    +  printf("Hello world!");
    +}
    +
    +
    +
    +
    +

    2. Running shell commands

    +
    +

    This script converts itself to octal in mtc.octal.

    +

    File: mtc/mtc1.swift

    +
    +
    +
    app (file o) f(file i)
    +{
    +  "od" "-b" i @stdout=o;
    +}
    +
    +main
    +{
    +  file x = input("mtc1.swift");
    +  file y<"mtc1.octal">;
    +  y = f(x);
    +}
    +
    +

    This script splits itself into lines, where line i is in file out- +i .txt

    +

    File: mtc/mtc2.swift

    +
    +
    +
    import files;
    +import string;
    +
    +app (file o) f(string s)
    +{
    +  "/bin/echo" s @stdout=o;
    +}
    +
    +main
    +{
    +  string lines[] = file_lines(input("mtc2.swift"));
    +  foreach line,i in lines
    +  {
    +    file y <sprintf("out-%i.txt",i)> = f(line);
    +  }
    +}
    +
    +

    Note that each /bin/echo is eligible to run concurrently. See +Invocation for how to run with many +processes.

    +
    +
    +
    +

    3. Reductions

    +
    +

    A simplified version of the MapReduce model is to just compute many +things and assemble them together at the end.

    +

    This script splits itself into lines, then reassembles the original +script.

    +

    File: mtc/mtc3.swift

    +
    +
    +
    import files;
    +import string;
    +import unix;
    +
    +app (file o) g(string s)
    +{
    +  "/bin/echo" s @stdout=o;
    +}
    +
    +main
    +{
    +  string lines[] = file_lines(input("mtc2.swift"));
    +  file fragments[];
    +  foreach line,i in lines
    +  {
    +    file y <sprintf("out-%i.txt",i)> = g(line);
    +    fragments[i] = y;
    +  }
    +  file result <"assembled.txt"> = cat(fragments);
    +}
    +
    +

    Note that leading whitespace is trimmed by file_lines(), and cat() +is part of the Swift/T standard library in module unix.

    +
    +
    +
    +

    4. Recursion

    +
    +

    This script computes the given +Fibonacci number:

    +

    File: fib/fib.swift

    +
    +
    +
    import sys;
    +
    +(int o) fib(int i)
    +{
    +  if (i >= 2)
    +  {
    +    o = fib(i-1) + fib(i-2);
    +  }
    +  else if (i == 1)
    +  {
    +    o = 1;
    +  }
    +  else
    +  {
    +    o = 0;
    +  }
    +}
    +
    +main
    +{
    +  int n = toint(argv("n"));
    +  trace(fib(n));
    +}
    +
    +

    Run it as:

    +

    File: fib/run.sh

    +
    +
    +
    #!/bin/sh -e
    +swift-t fib.swift -n=7
    +
    +

    The sys module provides the argv() function, which provides a +handy key/value interface for input values.

    +

    This script implements a parallel +merge sort. The data +files are generated by

    +
    +
    +
    tclsh make-data.tcl
    +
    +

    Each of the 8 files is already sorted, and a merged result will be +placed in sorted.txt.

    +

    File: merge-sort/merge.swift

    +
    +
    +
    import string;
    +
    +app (file o) sort(file i, file j)
    +{
    +  // This uses the standard sort -m feature (merge)
    +  "sort" "-mn" i j @stdout=o;
    +}
    +
    +(file o) merge(int i, int j)
    +{
    +  if (j-i == 1)
    +  {
    +    file fi = input(sprintf("data-%i.txt",i));
    +    file fj = input(sprintf("data-%i.txt",j));
    +    o = sort(fi, fj);
    +  }
    +  else
    +  {
    +    d = j - i + 1;
    +    m = d %/ 2; // integer divide operator
    +    o = sort(merge(i,i+m-1), merge(i+m,j));
    +  }
    +}
    +
    +main
    +{
    +  file result <"sorted.txt"> = merge(0,7);
    +}
    +
    +

    This code runs the sort invocations concurrently, limited only by +available processors and data dependencies.

    +
    +
    +
    +

    5. Python and Numpy

    +
    +

    See this section for information about calling Python or Numpy: +Swift/T Guide: Python

    +
    +
    +
    +

    + + + From wozniak at ci.uchicago.edu Fri Dec 19 16:29:11 2014 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Fri, 19 Dec 2014 16:29:11 -0600 (CST) Subject: [Swift-commit] r8352 - in www: . Swift-T Message-ID: <20141219222911.25144178884@svn.ci.uchicago.edu> Author: wozniak Date: 2014-12-19 16:29:11 -0600 (Fri, 19 Dec 2014) New Revision: 8352 Added: www/Swift-T/gallery.html Removed: www/gallery.html Log: Move Copied: www/Swift-T/gallery.html (from rev 8351, www/gallery.html) =================================================================== --- www/Swift-T/gallery.html (rev 0) +++ www/Swift-T/gallery.html 2014-12-19 22:29:11 UTC (rev 8352) @@ -0,0 +1,965 @@ + + + + + +Swift/T Gallery + + + + + +
    +
    +
    +

    This page demonstrates the ease of use of Swift/T when constructing +common application patterns.

    +

    Links:

    + +

    Post to the ExM +user list with questions about these examples.

    +
    +
    +
    +

    1. Hello world

    +
    +

    File: hello-world/hello-world.swift

    +
    +
    +
    import io;
    +main
    +{
    +  printf("Hello world!");
    +}
    +
    +
    +
    +
    +

    2. Running shell commands

    +
    +

    This script converts itself to octal in mtc.octal.

    +

    File: mtc/mtc1.swift

    +
    +
    +
    app (file o) f(file i)
    +{
    +  "od" "-b" i @stdout=o;
    +}
    +
    +main
    +{
    +  file x = input("mtc1.swift");
    +  file y<"mtc1.octal">;
    +  y = f(x);
    +}
    +
    +

    This script splits itself into lines, where line i is in file out- +i .txt

    +

    File: mtc/mtc2.swift

    +
    +
    +
    import files;
    +import string;
    +
    +app (file o) f(string s)
    +{
    +  "/bin/echo" s @stdout=o;
    +}
    +
    +main
    +{
    +  string lines[] = file_lines(input("mtc2.swift"));
    +  foreach line,i in lines
    +  {
    +    file y <sprintf("out-%i.txt",i)> = f(line);
    +  }
    +}
    +
    +

    Note that each /bin/echo is eligible to run concurrently. See +Invocation for how to run with many +processes.

    +
    +
    +
    +

    3. Reductions

    +
    +

    A simplified version of the MapReduce model is to just compute many +things and assemble them together at the end.

    +

    This script splits itself into lines, then reassembles the original +script.

    +

    File: mtc/mtc3.swift

    +
    +
    +
    import files;
    +import string;
    +import unix;
    +
    +app (file o) g(string s)
    +{
    +  "/bin/echo" s @stdout=o;
    +}
    +
    +main
    +{
    +  string lines[] = file_lines(input("mtc2.swift"));
    +  file fragments[];
    +  foreach line,i in lines
    +  {
    +    file y <sprintf("out-%i.txt",i)> = g(line);
    +    fragments[i] = y;
    +  }
    +  file result <"assembled.txt"> = cat(fragments);
    +}
    +
    +

    Note that leading whitespace is trimmed by file_lines(), and cat() +is part of the Swift/T standard library in module unix.

    +
    +
    +
    +

    4. Recursion

    +
    +

    This script computes the given +Fibonacci number:

    +

    File: fib/fib.swift

    +
    +
    +
    import sys;
    +
    +(int o) fib(int i)
    +{
    +  if (i >= 2)
    +  {
    +    o = fib(i-1) + fib(i-2);
    +  }
    +  else if (i == 1)
    +  {
    +    o = 1;
    +  }
    +  else
    +  {
    +    o = 0;
    +  }
    +}
    +
    +main
    +{
    +  int n = toint(argv("n"));
    +  trace(fib(n));
    +}
    +
    +

    Run it as:

    +

    File: fib/run.sh

    +
    +
    +
    #!/bin/sh -e
    +swift-t fib.swift -n=7
    +
    +

    The sys module provides the argv() function, which provides a +handy key/value interface for input values.

    +

    This script implements a parallel +merge sort. The data +files are generated by

    +
    +
    +
    tclsh make-data.tcl
    +
    +

    Each of the 8 files is already sorted, and a merged result will be +placed in sorted.txt.

    +

    File: merge-sort/merge.swift

    +
    +
    +
    import string;
    +
    +app (file o) sort(file i, file j)
    +{
    +  // This uses the standard sort -m feature (merge)
    +  "sort" "-mn" i j @stdout=o;
    +}
    +
    +(file o) merge(int i, int j)
    +{
    +  if (j-i == 1)
    +  {
    +    file fi = input(sprintf("data-%i.txt",i));
    +    file fj = input(sprintf("data-%i.txt",j));
    +    o = sort(fi, fj);
    +  }
    +  else
    +  {
    +    d = j - i + 1;
    +    m = d %/ 2; // integer divide operator
    +    o = sort(merge(i,i+m-1), merge(i+m,j));
    +  }
    +}
    +
    +main
    +{
    +  file result <"sorted.txt"> = merge(0,7);
    +}
    +
    +

    This code runs the sort invocations concurrently, limited only by +available processors and data dependencies.

    +
    +
    +
    +

    5. Python and Numpy

    +
    +

    See this section for information about calling Python or Numpy: +Swift/T Guide: Python

    +
    +
    +
    +

    + + + Deleted: www/gallery.html =================================================================== --- www/gallery.html 2014-12-19 22:26:27 UTC (rev 8351) +++ www/gallery.html 2014-12-19 22:29:11 UTC (rev 8352) @@ -1,965 +0,0 @@ - - - - - -Swift/T Gallery - - - - - -
    -
    -
    -

    This page demonstrates the ease of use of Swift/T when constructing -common application patterns.

    -

    Links:

    - -

    Post to the ExM -user list with questions about these examples.

    -
    -
    -
    -

    1. Hello world

    -
    -

    File: hello-world/hello-world.swift

    -
    -
    -
    import io;
    -main
    -{
    -  printf("Hello world!");
    -}
    -
    -
    -
    -
    -

    2. Running shell commands

    -
    -

    This script converts itself to octal in mtc.octal.

    -

    File: mtc/mtc1.swift

    -
    -
    -
    app (file o) f(file i)
    -{
    -  "od" "-b" i @stdout=o;
    -}
    -
    -main
    -{
    -  file x = input("mtc1.swift");
    -  file y<"mtc1.octal">;
    -  y = f(x);
    -}
    -
    -

    This script splits itself into lines, where line i is in file out- -i .txt

    -

    File: mtc/mtc2.swift

    -
    -
    -
    import files;
    -import string;
    -
    -app (file o) f(string s)
    -{
    -  "/bin/echo" s @stdout=o;
    -}
    -
    -main
    -{
    -  string lines[] = file_lines(input("mtc2.swift"));
    -  foreach line,i in lines
    -  {
    -    file y <sprintf("out-%i.txt",i)> = f(line);
    -  }
    -}
    -
    -

    Note that each /bin/echo is eligible to run concurrently. See -Invocation for how to run with many -processes.

    -
    -
    -
    -

    3. Reductions

    -
    -

    A simplified version of the MapReduce model is to just compute many -things and assemble them together at the end.

    -

    This script splits itself into lines, then reassembles the original -script.

    -

    File: mtc/mtc3.swift

    -
    -
    -
    import files;
    -import string;
    -import unix;
    -
    -app (file o) g(string s)
    -{
    -  "/bin/echo" s @stdout=o;
    -}
    -
    -main
    -{
    -  string lines[] = file_lines(input("mtc2.swift"));
    -  file fragments[];
    -  foreach line,i in lines
    -  {
    -    file y <sprintf("out-%i.txt",i)> = g(line);
    -    fragments[i] = y;
    -  }
    -  file result <"assembled.txt"> = cat(fragments);
    -}
    -
    -

    Note that leading whitespace is trimmed by file_lines(), and cat() -is part of the Swift/T standard library in module unix.

    -
    -
    -
    -

    4. Recursion

    -
    -

    This script computes the given -Fibonacci number:

    -

    File: fib/fib.swift

    -
    -
    -
    import sys;
    -
    -(int o) fib(int i)
    -{
    -  if (i >= 2)
    -  {
    -    o = fib(i-1) + fib(i-2);
    -  }
    -  else if (i == 1)
    -  {
    -    o = 1;
    -  }
    -  else
    -  {
    -    o = 0;
    -  }
    -}
    -
    -main
    -{
    -  int n = toint(argv("n"));
    -  trace(fib(n));
    -}
    -
    -

    Run it as:

    -

    File: fib/run.sh

    -
    -
    -
    #!/bin/sh -e
    -swift-t fib.swift -n=7
    -
    -

    The sys module provides the argv() function, which provides a -handy key/value interface for input values.

    -

    This script implements a parallel -merge sort. The data -files are generated by

    -
    -
    -
    tclsh make-data.tcl
    -
    -

    Each of the 8 files is already sorted, and a merged result will be -placed in sorted.txt.

    -

    File: merge-sort/merge.swift

    -
    -
    -
    import string;
    -
    -app (file o) sort(file i, file j)
    -{
    -  // This uses the standard sort -m feature (merge)
    -  "sort" "-mn" i j @stdout=o;
    -}
    -
    -(file o) merge(int i, int j)
    -{
    -  if (j-i == 1)
    -  {
    -    file fi = input(sprintf("data-%i.txt",i));
    -    file fj = input(sprintf("data-%i.txt",j));
    -    o = sort(fi, fj);
    -  }
    -  else
    -  {
    -    d = j - i + 1;
    -    m = d %/ 2; // integer divide operator
    -    o = sort(merge(i,i+m-1), merge(i+m,j));
    -  }
    -}
    -
    -main
    -{
    -  file result <"sorted.txt"> = merge(0,7);
    -}
    -
    -

    This code runs the sort invocations concurrently, limited only by -available processors and data dependencies.

    -
    -
    -
    -

    5. Python and Numpy

    -
    -

    See this section for information about calling Python or Numpy: -Swift/T Guide: Python

    -
    -
    -
    -

    - - - From wozniak at ci.uchicago.edu Fri Dec 19 16:34:18 2014 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Fri, 19 Dec 2014 16:34:18 -0600 (CST) Subject: [Swift-commit] r8353 - www/Swift-T Message-ID: <20141219223418.98BD5178884@svn.ci.uchicago.edu> Author: wozniak Date: 2014-12-19 16:34:18 -0600 (Fri, 19 Dec 2014) New Revision: 8353 Modified: www/Swift-T/index.php Log: Add Gallery and tooltips Modified: www/Swift-T/index.php =================================================================== --- www/Swift-T/index.php 2014-12-19 22:29:11 UTC (rev 8352) +++ www/Swift-T/index.php 2014-12-19 22:34:18 UTC (rev 8353) @@ -29,7 +29,7 @@ -