[Swift-commit] r7245 - SwiftApps/Swift-MapRed/mapred_combiner_V4

[ketan at ci.uchicago.edu]

Author: ketan
Date: 2013-10-29 15:45:32 -0500 (Tue, 29 Oct 2013)
New Revision: 7245

Added:
   SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.local.xml
Modified:
   SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
   SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.midway.xml
   SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data
Log:
 

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-10-29 20:32:39 UTC (rev 7244)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-10-29 20:45:32 UTC (rev 7245)
@@ -15,19 +15,6 @@
   seqfile psq;
 }
 
-/* Blast app related declarations
-output blast_output_file <single_file_mapper;file = arg("o", "output.html")>;
-output out[] <ext; exec = "outputmapper.sh", n = num_partitions>;
-
-app (output o) blastapp(query i, fastaseq d, string p, string e, string f, database db){
-  blastall "-p" p "-i" filename(i) "-d" filename(d) "-o" filename(o) "-e" e "-T" "-F" f;
-}
-
-app (output o) blastmerge(output o_frags[]){
-  blastmerge filename(o) filenames(o_frags);
-}
-*/
-
 app (database out) formatdb (fastaseq i){
  formatdb "-i" filename(i);
 }
@@ -37,22 +24,22 @@
 }
 
 app (fileptr out, file err) map_data (script wrapper, script map, query _queryfile, fastaseq _fastafile, string resfilename, string p, string e, string f, database db){
-  bash @wrapper @map p filename(_queryfile) filename(_fastafile) resfilename e f stdout=@out stderr=@err;
+ bash @wrapper @map p filename(_queryfile) filename(_fastafile) resfilename e f stdout=@out stderr=@err;
 }
 
 app (fileptr out) aggregate (script a, fileptr map_results[]) {
-  bash @a @map_results stdout=@out;
+ bash @a @map_results stdout=@out;
 }
 
 app (file out, file err) reduce_mid (script wrapper, script reducer, string cmd, fileptr all_map){
-  mid_bash @wrapper @reducer cmd @all_map stdout=@out stderr=@err;
+ mid_bash @wrapper @reducer cmd @all_map stdout=@out stderr=@err;
 }
 
 app (file out, file err) reduce_local (script wrapper, script reducer, string cmd, file site_results[]){
-  loc_bash @wrapper @reducer cmd @site_results stdout=@out stderr=@err;
+ loc_bash @wrapper @reducer cmd @site_results stdout=@out stderr=@err;
 }
 
-string num_partitions = arg("n", "10");
+string num_partitions = arg("n", "50");
 fastaseq partition[] <ext;exec = "examples/blast/splitmapper.sh", n = num_partitions>;
 database formatdbout[] <ext; exec = "examples/blast/formatdbmapper.sh", n = num_partitions>;
 
@@ -71,8 +58,8 @@
 partition=split_database(dbin, num_partitions);
 
 foreach item,i in partition {
-  formatdbout[i]=formatdb(item);
-  (map_out[i], map_err[i]) = map_data (map_wrapper, map_func, query_file, item, "tmpout", "blastp", "0.1", "F", formatdbout[i]);
+ formatdbout[i]=formatdb(item);
+ (map_out[i], map_err[i]) = map_data (map_wrapper, map_func, query_file, item, "tmpout", "blastp", "0.1", "F", formatdbout[i]);
 }
 
 fileptr all_map <"all_map.out">;

Added: SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.local.xml
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.local.xml	                        (rev 0)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.local.xml	2013-10-29 20:45:32 UTC (rev 7245)
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+   <pool handle="localhost">
+      <execution provider="local" />
+      <profile namespace="karajan" key="jobThrottle">0</profile>
+      <profile namespace="karajan" key="initialScore">10000</profile>
+      <filesystem provider="local"/>
+      <workdirectory>/tmp/swift.work</workdirectory>
+      <profile namespace="swift" key="stagingMethod">local</profile>
+   </pool>
+</config>

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.midway.xml
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.midway.xml	2013-10-29 20:32:39 UTC (rev 7244)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/sites.midway.xml	2013-10-29 20:45:32 UTC (rev 7245)
@@ -17,4 +17,12 @@
     <profile namespace="karajan" key="initialScore">10000</profile>
     <workdirectory>/tmp/maheshwari/swift.work</workdirectory>
   </pool>
+   <pool handle="localhost">
+     <execution provider="local" />
+     <profile namespace="karajan" key="jobThrottle">0</profile>
+     <profile namespace="karajan" key="initialScore">10000</profile>
+     <filesystem provider="local"/>
+     <workdirectory>/tmp/swift.work</workdirectory>
+     <profile namespace="swift" key="stagingMethod">local</profile>
+   </pool>
 </config>

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data	2013-10-29 20:32:39 UTC (rev 7244)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data	2013-10-29 20:45:32 UTC (rev 7245)
@@ -4,14 +4,8 @@
 beagle bgl_bash /bin/bash
 local  loc_bash /bin/bash
 local  cat /bin/cat
-midway blastp /home/maheshwari/ncbi-blast-2.2.28+/bin/blastp
-midway fastasplitn /home/maheshwari/parallelblast_2.0.9/fastasplitn
 midway blastmerge /home/maheshwari/parallelblast_2.0.9/blastmerge
 midway blastall /home/maheshwari/blast-2.2.26/bin/blastall
-midway formatdb /home/maheshwari/blast-2.2.26/bin/formatdb
-localhost blastp /home/maheshwari/ncbi-blast-2.2.28+/bin/blastp
 localhost fastasplitn /home/maheshwari/parallelblast_2.0.9/fastasplitn
-localhost blastmerge /home/maheshwari/parallelblast_2.0.9/blastmerge
-localhost blastall /home/maheshwari/blast-2.2.26/bin/blastall
 localhost formatdb /home/maheshwari/blast-2.2.26/bin/formatdb