[Swift-commit] r7434 - in SwiftApps/Swift-MapRed/mapred_combiner_V4: . examples/EPlus
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Tue Dec 17 12:17:42 CST 2013
Author: ketan
Date: 2013-12-17 12:17:42 -0600 (Tue, 17 Dec 2013)
New Revision: 7434
Modified:
SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps
SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/cleanup.sh
SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift
Log:
some corrections to map script
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift 2013-12-17 18:13:18 UTC (rev 7433)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift 2013-12-17 18:17:42 UTC (rev 7434)
@@ -87,6 +87,7 @@
file redsite_out[] <simple_mapper; prefix="reduce", suffix=".out">;
file redsite_err[] <simple_mapper; prefix="reduce", suffix=".err">;
(redsite_out[1], redsite_err[1]) = reduce_mid (red_wrapper, red_func, "local", all_map);
+
// Final global reduce (Not required on a single site)
//file final_out <"final_result">;
//file final_err <"final_errors">;
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps 2013-12-17 18:13:18 UTC (rev 7433)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps 2013-12-17 18:17:42 UTC (rev 7434)
@@ -1,2 +1,16 @@
midway sh /bin/sh
midway RunEP /home/maheshwari/Swift-MapRed/mapred_combiner_V4/examples/EPlus/RunAndReduceEP.sh
+midway bash /bin/bash
+localhost bash /bin/bash
+midway mid_bash /bin/bash
+midway loc_bash /bin/bash
+beagle bgl_bash /bin/bash
+local loc_bash /bin/bash
+local cat /bin/cat
+midway blastmerge /home/maheshwari/parallelblast_2.0.9/blastmerge
+midway blastall /home/maheshwari/blast-2.2.26/bin/blastall
+midway fastasplitn /home/maheshwari/parallelblast_2.0.9/fastasplitn
+midway formatdb /home/maheshwari/blast-2.2.26/bin/formatdb
+localhost fastasplitn /home/maheshwari/parallelblast_2.0.9/fastasplitn
+localhost formatdb /home/maheshwari/blast-2.2.26/bin/formatdb
+
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/cleanup.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/cleanup.sh 2013-12-17 18:13:18 UTC (rev 7433)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/cleanup.sh 2013-12-17 18:17:42 UTC (rev 7434)
@@ -1,9 +1,5 @@
#!/bin/bash
-# rm -rf *.log *.rlog *.d sweep1-*-*-* *.kml *.swiftx *.out output outdir logs hw.* _concurrent .swift/tmp
+rm -rf *.log *.rlog *.d sweep1-*-*-* *.kml *.swiftx *.out output outdir logs hw.* _concurrent .swift/tmp
-echo "stopping coaster service"
-
-PATH=/glusterfs/users/swiftlang/swift-0.94.1/bin:$PATH
-
-stop-coaster-service
+#stop-coaster-service
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift 2013-12-17 18:13:18 UTC (rev 7433)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift 2013-12-17 18:17:42 UTC (rev 7434)
@@ -1,52 +1,20 @@
-
-/*
-
-app (fileptr out, file err) map_data ( script wrapper, script map, query _queryfile, fastaseq _fastafile, string resfilename, string p, string e, string f, database db){
- bash @wrapper @map p filename(_queryfile) filename(_fastafile) resfilename e f stdout=@out stderr=@err;
-}
-
-app (fileptr out) aggregate (script a, fileptr map_results[]) {
- bash @a @map_results stdout=@out;
-}
-
-app (file out, file err) reduce_mid (script wrapper, script reducer, string cmd, fileptr all_map){
- mid_bash @wrapper @reducer cmd @all_map stdout=@out stderr=@err;
-}
-
-app (file out, file err) reduce_local (script wrapper, script reducer, string cmd, file site_results[]){
- loc_bash @wrapper @reducer cmd @site_results stdout=@out stderr=@err;
-}
-
-*/
-
type file;
type fileptr;
type script;
-app (file all, file xml, file json) runEP_debug ( file imf, file epw, string params[] )
-{
- RunEP "--imf" @imf "--epw" @epw "--outall" @all "--outxml" @xml "--outjson" @json "--params" params;
-}
-
app (file json) runEP ( file imf, file epw, string params[] )
{
RunEP "--imf" @imf "--epw" @epw "--outall" "temp.tgz" "--outxml" "temp.xml" "--outjson" @json "--params" params;
}
-/* Swift Reduce stuff */
-app (fileptr out, file err) map_data ( script wrapper, script map, query _queryfile, fastaseq _fastafile, string resfilename, string p, string e, string f, database db){
- bash @wrapper @map p filename(_queryfile) filename(_fastafile) resfilename e f stdout=@out stderr=@err;
+app (fileptr out, file err) map_data (script wrapper, script map, file imf, file epw, string outjson, string params[]){
+ bash @wrapper @map @imf @epw outjson params stdout=@out stderr=@err;
}
-foreach item,i in partition {
- (map_out[i], map_err[i]) = map_data (map_wrapper, map_func, );
-}
-
file epconfig <single_file_mapper; file=@arg("epconfig", "SWEEP_RECT.imf")>;
file epweather <single_file_mapper; file=@arg("epweather", "CHICAGO.epw")>;
string outdir=@arg("outdir","output");
-boolean debug=(@arg("debug","")=="debug");
type param {
string pnum;
@@ -64,8 +32,10 @@
}
}
+/*
fileptr map_out[] <simple_mapper; prefix="map", suffix=".out">;
file map_err[] <simple_mapper; prefix="map", suffix=".err">;
+*/
script map_wrapper <"../../map_wrapper.sh">;
script red_wrapper <"../../reduce_wrapper.sh">;
@@ -82,35 +52,28 @@
foreach v6 in pval[6] {
foreach v7 in pval[7] {
- tracef("parameter set: %s:%s %s:%s %s:%s %s:%s %s:%s %s:%s %s:%s %s:%s\n",
- pset[0].pname, v0, pset[1].pname, v1, pset[2].pname, v2, pset[3].pname, v3,
- pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7);
+ string fileid = @strjoin(["/ep",v0,v1,v2,v3,v4,v5,v6,v7],".");
+ /* file outjson <single_file_mapper; file=@strcat(outdir,fileid,".json")>;*/
+
+ fileptr map_out <single_file_mapper; file=@strcat("map",fileid,".out")>;
+ file map_err <single_file_mapper; file=@strcat("map",fileid,".err")>;
- string fileid = @strjoin(["/ep",v0,v1,v2,v3,v4,v5,v6,v7],".");
+ /* outjson = runEP ( epconfig, epweather,
+ [pset[0].pname, v0, pset[1].pname, v1, pset[2].pname, v2, pset[3].pname, v3,
+ pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7] );
+ */
- if (debug) {
- file outall <single_file_mapper; file=@strcat(outdir,fileid,".tgz")>;
- file outxml <single_file_mapper; file=@strcat(outdir,fileid,".xml")>;
- file outjson <single_file_mapper; file=@strcat(outdir,fileid,".json")>;
- (outall, outxml, outjson) = runEP_debug ( epconfig, epweather,
- [pset[0].pname, v0, pset[1].pname, v1, pset[2].pname, v2, pset[3].pname, v3,
- pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7]);
- }
- else {
- file outjson <single_file_mapper; file=@strcat(outdir,fileid,".json")>;
- outjson = runEP ( epconfig, epweather,
- [pset[0].pname, v0, pset[1].pname, v1, pset[2].pname, v2, pset[3].pname, v3,
- pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7] );
-
- (map_out[i], map_err[i]) = map_data (map_wrapper, map_func, epconfig, epweather, [pset[0].pname, v0, pset[1].pname, v1, pset[2].pname, v2, pset[3].pname, v3, pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7]);
-
- }
+ (map_out, map_err) = map_data (map_wrapper, map_func, epconfig, epweather,
+ "outjson", [pset[0].pname, v0, pset[1].pname, v1, pset[2].pname,
+ v2, pset[3].pname, v3, pset[4].pname, v4, pset[5].pname, v5,
+ pset[6].pname, v6, pset[7].pname, v7]);
}}}}}}}}
+/*
fileptr all_map <"all_map.out">;
all_map = aggregate(aggr, map_out);
file redsite_out[] <simple_mapper; prefix="reduce", suffix=".out">;
file redsite_err[] <simple_mapper; prefix="reduce", suffix=".err">;
(redsite_out[1], redsite_err[1]) = reduce_mid (red_wrapper, red_func, "local", all_map);
-
+*/
More information about the Swift-commit
mailing list