[Swift-commit] r7393 - SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus

[ketan at ci.uchicago.edu]

Author: ketan
Date: 2013-12-09 17:56:32 -0600 (Mon, 09 Dec 2013)
New Revision: 7393

Modified:
   SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps
   SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/run.midway.sh
   SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift
Log:
 

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps	2013-12-09 23:03:51 UTC (rev 7392)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/apps	2013-12-09 23:56:32 UTC (rev 7393)
@@ -1,2 +1,2 @@
 midway sh    /bin/sh
-midway RunEP /home/maheshwari/Swift-MapRed/mapred_combiner_V4/urbanccd-work/EPlus/RunAndReduceEP.sh
+midway RunEP /home/maheshwari/Swift-MapRed/mapred_combiner_V4/examples/EPlus/RunAndReduceEP.sh

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/run.midway.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/run.midway.sh	2013-12-09 23:03:51 UTC (rev 7392)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/run.midway.sh	2013-12-09 23:56:32 UTC (rev 7393)
@@ -27,7 +27,7 @@
 use.wrapper.staging=false
 END
 
-cat >>sites.xml <<END
+cat >sites.xml <<END
 <?xml version="1.0" encoding="UTF-8"?>
 <config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
 <pool handle="midway">

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift	2013-12-09 23:03:51 UTC (rev 7392)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/examples/EPlus/sweep7-reduce.swift	2013-12-09 23:56:32 UTC (rev 7393)
@@ -1,4 +1,27 @@
+
+/*
+
+app (fileptr out, file err) map_data ( script wrapper, script map, query _queryfile, fastaseq _fastafile, string resfilename, string p, string e, string f, database db){
+ bash @wrapper @map p filename(_queryfile) filename(_fastafile) resfilename e f stdout=@out stderr=@err;
+}
+
+app (fileptr out) aggregate (script a, fileptr map_results[]) {
+ bash @a @map_results stdout=@out;
+}
+
+app (file out, file err) reduce_mid (script wrapper, script reducer, string cmd, fileptr all_map){
+ mid_bash @wrapper @reducer cmd @all_map stdout=@out stderr=@err;
+}
+
+app (file out, file err) reduce_local (script wrapper, script reducer, string cmd, file site_results[]){
+ loc_bash @wrapper @reducer cmd @site_results stdout=@out stderr=@err;
+}
+
+*/
+
 type file;
+type fileptr;
+type script;
 
 app (file all, file xml, file json) runEP_debug ( file imf, file epw, string params[] )
 {
@@ -10,10 +33,19 @@
   RunEP "--imf" @imf "--epw" @epw "--outall" "temp.tgz" "--outxml" "temp.xml" "--outjson" @json "--params" params;
 }
 
+/* Swift Reduce stuff */
+app (fileptr out, file err) map_data ( script wrapper, script map, query _queryfile, fastaseq _fastafile, string resfilename, string p, string e, string f, database db){
+ bash @wrapper @map p filename(_queryfile) filename(_fastafile) resfilename e f stdout=@out stderr=@err;
+}
+
+foreach item,i in partition {
+ (map_out[i], map_err[i]) = map_data (map_wrapper, map_func, );
+}
+
 file epconfig  <single_file_mapper; file=@arg("epconfig",  "SWEEP_RECT.imf")>;
 file epweather <single_file_mapper; file=@arg("epweather", "CHICAGO.epw")>;
 
-string  outdir=@arg("outdir","output");
+string outdir=@arg("outdir","output");
 boolean debug=(@arg("debug","")=="debug");
 
 type param {
@@ -32,6 +64,15 @@
   }
 }
 
+fileptr map_out[] <simple_mapper; prefix="map", suffix=".out">;
+file  map_err[] <simple_mapper; prefix="map", suffix=".err">;
+
+script map_wrapper <"../../map_wrapper.sh">;
+script red_wrapper <"../../reduce_wrapper.sh">;
+script aggr <"../../aggregate.sh">;
+script map_func <"map_eplus.sh">;
+script red_func <"reduce_eplus.sh">;
+
 foreach v0 in pval[0] {
 foreach v1 in pval[1] {
 foreach v2 in pval[2] {
@@ -59,7 +100,17 @@
     file outjson <single_file_mapper; file=@strcat(outdir,fileid,".json")>;
     outjson = runEP ( epconfig, epweather,
       [pset[0].pname, v0, pset[1].pname, v1, pset[2].pname, v2, pset[3].pname, v3,
-       pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7]);
+       pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7] );
+    
+    (map_out[i], map_err[i]) = map_data (map_wrapper, map_func, epconfig, epweather, [pset[0].pname, v0, pset[1].pname, v1, pset[2].pname, v2, pset[3].pname, v3, pset[4].pname, v4, pset[5].pname, v5, pset[6].pname, v6, pset[7].pname, v7]);
+    
   }
 
 }}}}}}}}
+
+fileptr all_map <"all_map.out">;
+all_map = aggregate(aggr, map_out);
+file redsite_out[] <simple_mapper; prefix="reduce", suffix=".out">;
+file redsite_err[] <simple_mapper; prefix="reduce", suffix=".err">;
+(redsite_out[1], redsite_err[1]) = reduce_mid (red_wrapper, red_func, "local", all_map);
+