[Swift-commit] r7131 - SwiftApps/Swift-MapRed/mapred_combiner_V2
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Fri Oct 4 11:32:50 CDT 2013
Author: yadunandb
Date: 2013-10-04 11:32:50 -0500 (Fri, 04 Oct 2013)
New Revision: 7131
Modified:
SwiftApps/Swift-MapRed/mapred_combiner_V2/LAB_RECORDS
SwiftApps/Swift-MapRed/mapred_combiner_V2/README
SwiftApps/Swift-MapRed/mapred_combiner_V2/beagle.xml
SwiftApps/Swift-MapRed/mapred_combiner_V2/clean.sh
SwiftApps/Swift-MapRed/mapred_combiner_V2/client.c
SwiftApps/Swift-MapRed/mapred_combiner_V2/combiner.sh
SwiftApps/Swift-MapRed/mapred_combiner_V2/node_uniq.sh
SwiftApps/Swift-MapRed/mapred_combiner_V2/reducer.sh
SwiftApps/Swift-MapRed/mapred_combiner_V2/run.sh
SwiftApps/Swift-MapRed/mapred_combiner_V2/teragen.swift
Log:
Minor updates
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/LAB_RECORDS
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/LAB_RECORDS 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/LAB_RECORDS 2013-10-04 16:32:50 UTC (rev 7131)
@@ -2150,3 +2150,327 @@
ACTUAL COUNT : 9000000
EXPECTED COUNT: 9000000
---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:10, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Could not open: 10.xml
+Execution failed:
+
+ Could not find sites file: 10.xml
+
+real 5.65
+user 4.97
+sys 0.18
+chunks 20 filesize 10 site 10 type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131003-1128-v7cy8yba
+Progress: time: Thu, 03 Oct 2013 11:28:19 -0500
+Progress: time: Thu, 03 Oct 2013 11:28:27 -0500 Stage in:1 Submitted:9
+Progress: time: Thu, 03 Oct 2013 11:28:28 -0500 Submitted:1 Active:1 Finished successfully:8
+Progress: time: Thu, 03 Oct 2013 11:28:29 -0500 Stage out:2 Finished successfully:11
+Progress: time: Thu, 03 Oct 2013 11:28:34 -0500 Stage out:1 Finished successfully:13
+Final status: Thu, 03 Oct 2013 11:28:34 -0500 Finished successfully:14
+real 20.29
+user 9.36
+sys 0.42
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131003-1508-xp7ii5qg
+Progress: time: Thu, 03 Oct 2013 15:08:51 -0500
+Progress: time: Thu, 03 Oct 2013 15:09:21 -0500 Submitted:10
+Progress: time: Thu, 03 Oct 2013 15:09:26 -0500 Stage in:1 Submitted:9
+Progress: time: Thu, 03 Oct 2013 15:09:27 -0500 Submitted:5 Stage out:1 Finished successfully:4
+Progress: time: Thu, 03 Oct 2013 15:09:28 -0500 Active:1 Finished successfully:10
+Progress: time: Thu, 03 Oct 2013 15:09:30 -0500 Stage in:1 Finished successfully:11
+Progress: time: Thu, 03 Oct 2013 15:09:31 -0500 Stage out:1 Finished successfully:11
+Final status: Thu, 03 Oct 2013 15:09:31 -0500 Finished successfully:13
+real 51.30
+user 8.94
+sys 0.57
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:30 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1511-29oppdx2
+Progress: time: Thu, 03 Oct 2013 15:12:00 -0500
+Progress: time: Thu, 03 Oct 2013 15:12:30 -0500 Submitted:30
+Progress: time: Thu, 03 Oct 2013 15:13:00 -0500 Submitted:30
+Progress: time: Thu, 03 Oct 2013 15:13:10 -0500 Stage in:1 Submitted:29
+Progress: time: Thu, 03 Oct 2013 15:13:11 -0500 Submitted:25 Stage out:1 Finished successfully:4
+Progress: time: Thu, 03 Oct 2013 15:13:12 -0500 Submitted:19 Stage out:1 Finished successfully:10
+Progress: time: Thu, 03 Oct 2013 15:13:13 -0500 Submitted:13 Stage out:1 Finished successfully:16
+Progress: time: Thu, 03 Oct 2013 15:13:14 -0500 Submitted:9 Finished successfully:21
+Progress: time: Thu, 03 Oct 2013 15:13:15 -0500 Stage in:1 Submitted:3 Finished successfully:26
+Progress: time: Thu, 03 Oct 2013 15:13:16 -0500 Active:1 Stage out:1 Finished successfully:31
+Progress: time: Thu, 03 Oct 2013 15:13:19 -0500 Stage out:1 Finished successfully:32
+Progress: time: Thu, 03 Oct 2013 15:13:24 -0500 Stage out:1 Finished successfully:33
+Final status: Thu, 03 Oct 2013 15:13:24 -0500 Finished successfully:34
+real 91.09
+user 106.43
+sys 56.79
+chunks 30 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 3000000
+EXPECTED COUNT: 3000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:30 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1513-ibzazhq2
+Progress: time: Thu, 03 Oct 2013 15:13:53 -0500
+Progress: time: Thu, 03 Oct 2013 15:14:23 -0500 Submitted:30
+Progress: time: Thu, 03 Oct 2013 15:14:53 -0500 Submitted:30
+Progress: time: Thu, 03 Oct 2013 15:15:23 -0500 Submitted:30
+Progress: time: Thu, 03 Oct 2013 15:15:53 -0500 Submitted:30
+Progress: time: Thu, 03 Oct 2013 15:16:23 -0500 Submitted:30
+Progress: time: Thu, 03 Oct 2013 15:16:29 -0500 Stage in:1 Submitted:29
+Progress: time: Thu, 03 Oct 2013 15:16:30 -0500 Submitted:23 Active:1 Stage out:1 Finished successfully:5
+Progress: time: Thu, 03 Oct 2013 15:16:32 -0500 Submitted:9 Active:2 Stage out:1 Finished successfully:18
+Progress: time: Thu, 03 Oct 2013 15:16:33 -0500 Stage out:1 Finished successfully:30
+Progress: time: Thu, 03 Oct 2013 15:16:34 -0500 Active:1 Stage out:1 Finished successfully:32
+Progress: time: Thu, 03 Oct 2013 15:16:39 -0500 Stage out:1 Finished successfully:34
+Final status: Thu, 03 Oct 2013 15:16:39 -0500 Finished successfully:35
+real 173.73
+user 217.98
+sys 115.87
+chunks 30 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 3000000
+EXPECTED COUNT: 3000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1553-0xq6yv2d
+Progress: time: Thu, 03 Oct 2013 15:53:27 -0500
+Progress: time: Thu, 03 Oct 2013 15:53:36 -0500 Stage in:1 Submitted:9
+Progress: time: Thu, 03 Oct 2013 15:53:37 -0500 Submitted:4 Active:1 Stage out:1 Finished successfully:4
+Progress: time: Thu, 03 Oct 2013 15:53:39 -0500 Active:1 Stage out:1 Finished successfully:11
+Progress: time: Thu, 03 Oct 2013 15:53:44 -0500 Stage out:1 Finished successfully:13
+Final status: Thu, 03 Oct 2013 15:53:44 -0500 Finished successfully:14
+real 21.66
+user 23.61
+sys 8.91
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1558-o2zu4p3b
+Progress: time: Thu, 03 Oct 2013 15:58:58 -0500
+Progress: time: Thu, 03 Oct 2013 15:59:07 -0500 Stage in:1 Submitted:9
+Progress: time: Thu, 03 Oct 2013 15:59:08 -0500 Submitted:7 Active:1 Stage out:1 Finished successfully:1
+Progress: time: Thu, 03 Oct 2013 15:59:09 -0500 Active:2 Stage out:1 Finished successfully:11
+Progress: time: Thu, 03 Oct 2013 15:59:15 -0500 Stage out:1 Finished successfully:14
+Final status: Thu, 03 Oct 2013 15:59:15 -0500 Finished successfully:15
+real 21.91
+user 24.05
+sys 8.90
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 700000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1600-4a9j4vm8
+Progress: time: Thu, 03 Oct 2013 16:00:59 -0500
+Progress: time: Thu, 03 Oct 2013 16:01:08 -0500 Stage in:1 Submitted:9
+Progress: time: Thu, 03 Oct 2013 16:01:09 -0500 Submitted:2 Active:1 Stage out:1 Finished successfully:6
+Progress: time: Thu, 03 Oct 2013 16:01:11 -0500 Active:1 Stage out:1 Finished successfully:11
+Progress: time: Thu, 03 Oct 2013 16:01:16 -0500 Stage out:1 Finished successfully:13
+Final status: Thu, 03 Oct 2013 16:01:16 -0500 Finished successfully:14
+real 21.04
+user 23.17
+sys 8.06
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:30 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1601-5v6tk42f
+Progress: time: Thu, 03 Oct 2013 16:01:39 -0500
+Progress: time: Thu, 03 Oct 2013 16:01:48 -0500 Stage in:1 Submitted:29
+Progress: time: Thu, 03 Oct 2013 16:01:49 -0500 Stage in:1 Submitted:23 Stage out:2 Finished successfully:4
+Progress: time: Thu, 03 Oct 2013 16:01:50 -0500 Submitted:14 Active:1 Stage out:1 Finished successfully:14
+Progress: time: Thu, 03 Oct 2013 16:01:51 -0500 Submitted:6 Stage out:2 Finished successfully:22
+Progress: time: Thu, 03 Oct 2013 16:01:52 -0500 Stage in:1 Submitted:1 Finished successfully:31
+Progress: time: Thu, 03 Oct 2013 16:01:54 -0500 Stage out:1 Finished successfully:32
+Final status: Thu, 03 Oct 2013 16:01:54 -0500 Finished successfully:34
+real 19.68
+user 27.90
+sys 10.13
+chunks 30 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 1600000
+EXPECTED COUNT: 3000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:30 X FILESIZE:30
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1610-nxf9q9c2
+Progress: time: Thu, 03 Oct 2013 16:10:24 -0500
+Progress: time: Thu, 03 Oct 2013 16:10:35 -0500 Stage in:1 Submitted:29
+Progress: time: Thu, 03 Oct 2013 16:10:36 -0500 Submitted:22 Stage out:3 Finished successfully:5
+Progress: time: Thu, 03 Oct 2013 16:10:37 -0500 Submitted:10 Active:2 Stage out:1 Finished successfully:17
+Progress: time: Thu, 03 Oct 2013 16:10:38 -0500 Submitted:4 Stage out:3 Finished successfully:23
+Progress: time: Thu, 03 Oct 2013 16:10:42 -0500 Active:2 Stage out:1 Finished successfully:31
+Progress: time: Thu, 03 Oct 2013 16:10:47 -0500 Stage out:1 Finished successfully:34
+Final status: Thu, 03 Oct 2013 16:10:47 -0500 Finished successfully:35
+real 28.29
+user 34.76
+sys 14.59
+chunks 30 filesize 30 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 9000000
+EXPECTED COUNT: 9000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:100 X FILESIZE:50
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1616-shuamhq9
+Progress: time: Thu, 03 Oct 2013 16:16:18 -0500
+Progress: time: Thu, 03 Oct 2013 16:16:27 -0500 Stage in:1 Submitted:99
+Progress: time: Thu, 03 Oct 2013 16:16:28 -0500 Submitted:97 Stage out:1 Finished successfully:2
+Progress: time: Thu, 03 Oct 2013 16:16:29 -0500 Submitted:94 Stage out:1 Finished successfully:5
+Progress: time: Thu, 03 Oct 2013 16:16:30 -0500 Submitted:91 Stage out:1 Finished successfully:8
+Progress: time: Thu, 03 Oct 2013 16:16:32 -0500 Submitted:90 Finished successfully:10
+Progress: time: Thu, 03 Oct 2013 16:16:33 -0500 Submitted:86 Stage out:1 Finished successfully:13
+Progress: time: Thu, 03 Oct 2013 16:16:34 -0500 Submitted:83 Finished successfully:17
+Progress: time: Thu, 03 Oct 2013 16:16:35 -0500 Stage in:1 Submitted:80 Stage out:1 Finished successfully:18
+Progress: time: Thu, 03 Oct 2013 16:16:36 -0500 Submitted:74 Active:2 Finished successfully:24
+Progress: time: Thu, 03 Oct 2013 16:16:37 -0500 Submitted:64 Active:3 Stage out:1 Finished successfully:32
+Progress: time: Thu, 03 Oct 2013 16:16:38 -0500 Submitted:56 Active:2 Stage out:1 Finished successfully:41
+Progress: time: Thu, 03 Oct 2013 16:16:39 -0500 Submitted:42 Active:3 Stage out:1 Finished successfully:54
+Progress: time: Thu, 03 Oct 2013 16:16:40 -0500 Submitted:30 Active:3 Stage out:1 Finished successfully:66
+Progress: time: Thu, 03 Oct 2013 16:16:42 -0500 Submitted:18 Active:3 Stage out:1 Finished successfully:78
+Progress: time: Thu, 03 Oct 2013 16:16:43 -0500 Submitted:8 Active:3 Stage out:1 Finished successfully:88
+Progress: time: Thu, 03 Oct 2013 16:16:44 -0500 Stage out:4 Finished successfully:96
+Progress: time: Thu, 03 Oct 2013 16:16:48 -0500 Active:4 Finished successfully:101
+Progress: time: Thu, 03 Oct 2013 16:16:52 -0500 Active:3 Stage out:1 Finished successfully:101
+Progress: time: Thu, 03 Oct 2013 16:16:55 -0500 Active:1 Stage out:1 Finished successfully:103
+Progress: time: Thu, 03 Oct 2013 16:17:02 -0500 Stage out:1 Finished successfully:104
+Progress: time: Thu, 03 Oct 2013 16:17:07 -0500 Stage out:1 Finished successfully:105
+Final status: Thu, 03 Oct 2013 16:17:08 -0500 Finished successfully:106
+real 54.25
+user 66.57
+sys 31.16
+chunks 100 filesize 50 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 50000000
+EXPECTED COUNT: 50000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:100 X FILESIZE:50
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 38
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131003-1634-9hc5sugc
+Progress: time: Thu, 03 Oct 2013 16:34:47 -0500
+Progress: time: Thu, 03 Oct 2013 16:35:17 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:35:47 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:36:17 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:36:47 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:37:17 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:37:47 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:38:17 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:38:47 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:39:17 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:39:47 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:40:17 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:40:47 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:41:17 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:41:47 -0500 Submitted:100
+Progress: time: Thu, 03 Oct 2013 16:42:17 -0500 Submitted:100
+chunks 100 filesize 50 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 50000000
+---------------------------------------------
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/README
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/README 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/README 2013-10-04 16:32:50 UTC (rev 7131)
@@ -15,4 +15,16 @@
over 10000 tasks, thus generating in total 1 Tera records.
-./teragen_wrap.sh 10 # Generates 10000000
\ No newline at end of file
+./teragen_wrap.sh 10 # Generates 10000000
+
+
+reduce_core.sh
+==============
+This is the core reduce function.
+The spec for such a function is :
+ARGS: Any number of files
+RESULT: One file
+
+Here reduce_core.sh will perform summation of first two columns of
+it's input files to a result file and echo the absolute path of this
+file.
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/beagle.xml
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/beagle.xml 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/beagle.xml 2013-10-04 16:32:50 UTC (rev 7131)
@@ -8,8 +8,9 @@
<!-- to use a beage reservation, eg:
<profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24;pbs.resource_list=advres=wilde.1768</profile>
-->
- <profile namespace="globus" key="maxtime">3600</profile>
- <profile namespace="globus" key="maxWalltime">00:15:00</profile>
+ <profile namespace="globus" key="maxtime">2400</profile>
+ <profile namespace="globus" key="maxWalltime">00:20:00</profile>
+ <profile namespace="globus" key="queue">batch</profile>
<profile namespace="globus" key="userHomeOverride">/lustre/beagle/yadunandb/swiftwork</profile>
<profile namespace="globus" key="slots">4</profile>
<profile namespace="globus" key="maxnodes">1</profile>
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/clean.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/clean.sh 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/clean.sh 2013-10-04 16:32:50 UTC (rev 7131)
@@ -1,9 +1,8 @@
#!/bin/bash
-rm -rf *log *d &> /dev/null
-#rm -rf *swiftx *kml &> /dev/null
-rm -rf *~ &> /dev/null
-rm *{out,err,imd2} &> /dev/null
-rm final_errors final_result
-rm exec_client exec_server
-rm *swiftx *kml
\ No newline at end of file
+rm -rf *log *d &> /dev/null
+rm -rf *swiftx *kml &> /dev/null
+rm -rf *~ &> /dev/null
+rm *{out,err,imd2} &> /dev/null
+rm final_errors final_result &> /dev/null
+rm exec_client exec_server &> /dev/null
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/client.c
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/client.c 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/client.c 2013-10-04 16:32:50 UTC (rev 7131)
@@ -59,19 +59,6 @@
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv));
return 1;
}
-
- /*
- if ((sockfd = socket(p->ai_family, p->ai_socktype,
- p->ai_protocol)) == -1) {
- perror("client: socket");
- }
-
- if ( (con_res = connect(sockfd, p->ai_addr, p->ai_addrlen)) == -1) {
- close(sockfd);
- printf("Connect failed with %d:%s\n", con_res, strerror(errno));
- perror("client: connect");
- }
- */
// loop through all the results and connect to the first we can
for(p = servinfo; p != NULL; p = p->ai_next) {
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/combiner.sh 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/combiner.sh 2013-10-04 16:32:50 UTC (rev 7131)
@@ -1,14 +1,58 @@
#!/bin/bash
ARGS=($*)
-NODE=${ARGS[0]}
-FILES=(${ARGS[*]:1})
-
SUM=0
COUNT=0
-REMOTE_PORT=29900
-HOSTNAME=$(hostname -f)
+fetch_and_process()
+{
+ TOKEN=($(cat $1))
+ HOST=${TOKEN[0]}
+ FILE=${TOKEN[1]}
+ TARGET=$(basename $FILE)
+ echo "HOST:$HOST, HOSTNAME:$HOSTNAME"
+ if [ "$HOST" != "$HOSTNAME" ]
+ then # Fetch if file is remote
+ echo "$HOST $TARGET $FOLDER/$TARGET.imd : "; cat $FOLDER/$TARGET.imd
+ RES=($(awk '{ sum += $1; count += $2 } END { print sum,count }' $FOLDER/$TARGET.imd))
+ rm $FOLDER/$TARGET.imd &> /dev/null &
+ else # Process directly if file is local
+ RES=($(awk '{ sum += $1; count += $2 } END { print sum,count }' $FILE))
+ fi
+ ID=$RANDOM
+ echo "${RES[0]} ${RES[1]}" > "$CHIRP_FOLDER/$2.$RANDOM.imd2"
+ echo -e "$(hostname -f) $CHIRP_FOLDER/$2.$RANDOM.imd2"
+}
+
+
+if [ -f ${ARGS[0]} ]
+then # When fileptrs are passed to combiner
+ # 1. fetch files.
+ # 2. combine and return a file pointer
+ FILES=(${ARGS[*]})
+ ID=$RANDOM
+ for file_token in ${FILES[*]}
+ do
+ fetch_and_process $file_token $ID &
+ done
+ wait
+
+ TOKEN=($(cat ${FILES[0]}))
+ CHIRP_FOLDER=$(dirname ${TOKEN[1]})
+ cat $ID*imd2
+ FINAL=($(awk '{ sum += $1; count += $2 } END { print sum,count }' $ID*imd2))
+ echo "${FINAL[0]} ${FINAL[1]}" > $CHIRP_FOLDER/tree_comb.$RANDOM.imd3
+
+ rm $ID*{imd,imd2} &> /dev/null &
+ exit 0
+else # When strings in format node files... are passed to combiner
+ # 1. Run a local_combiner on the remote node
+ # 2. Return a file pointer
+ NODE=${ARGS[0]}
+ FILES=(${ARGS[*]:1})
+ REMOTE_PORT=29900
+ HOSTNAME=$(hostname -f)
+
############## LOCAL COMBINER CODE ###################
cat << EOF > $NODE.sh
#!/bin/bash
@@ -31,10 +75,13 @@
echo -e "$(hostname -f) $CHIRP_FOLDER/combine_imd.$ID.out"
#echo -e "DEBUG : $SUM $COUNT"
EOF
-chmod a+x $NODE.sh
+ chmod a+x $NODE.sh
############# END LOCAL COMBINER #####################
-cp /lustre/beagle/yadunandb/bin/exec_client .
-echo "Combiner ${ARGS[*]}" 1>&2
-echo "Combiner [./exec_client $NODE $NODE.sh]" 1>&2
-./exec_client $NODE $NODE.sh
-exit 0
\ No newline at end of file
+ cp /lustre/beagle/yadunandb/bin/exec_client .
+ echo "Combiner ${ARGS[*]}" 1>&2
+ echo "Combiner [./exec_client $NODE $NODE.sh]" 1>&2
+ cat $NODE.sh 1>&2
+ ./exec_client $NODE $NODE.sh
+ exit 0
+
+fi
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/node_uniq.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/node_uniq.sh 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/node_uniq.sh 2013-10-04 16:32:50 UTC (rev 7131)
@@ -11,4 +11,4 @@
MATCH=$(grep $SITE tmp | awk '{print $2}')
echo $SITE ${MATCH[*]}
done
-#rm tmp;
+rm tmp;
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/reducer.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/reducer.sh 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/reducer.sh 2013-10-04 16:32:50 UTC (rev 7131)
@@ -17,14 +17,12 @@
TARGET=$(basename $FILE)
echo "HOST:$HOST, HOSTNAME:$HOSTNAME"
if [ "$HOST" != "$HOSTNAME" ]
- then
- # Fetch if file is remote
+ then # Fetch if file is remote
chirp_get $HOST $TARGET $FOLDER/$TARGET.imd
echo "$HOST $TARGET $FOLDER/$TARGET.imd : "; cat $FOLDER/$TARGET.imd
RES=($(awk '{ sum += $1; count += $2 } END { print sum,count }' $FOLDER/$TARGET.imd))
rm $FOLDER/$TARGET.imd &> /dev/null &
- else
- # Process directly if file is local
+ else # Process directly if file is local
RES=($(awk '{ sum += $1; count += $2 } END { print sum,count }' $FILE))
fi
echo "${RES[0]} ${RES[1]}" > "$2.$RANDOM.imd2"
@@ -40,8 +38,6 @@
TOKEN=($(cat ${FILES[0]}))
CHIRP_FOLDER=$(dirname ${TOKEN[1]})
cat $ID*imd2
-#SUM=$(awk '{ sum += $1 } END { print sum }' $ID*imd2)
-#COUNT=$(awk '{ sum += $2 } END { print sum }' $ID*imd2)
FINAL=($(awk '{ sum += $1; count += $2 } END { print sum,count }' $ID*imd2))
echo "${FINAL[0]} ${FINAL[1]}"
rm $ID*{imd,imd2} &> /dev/null &
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/run.sh 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/run.sh 2013-10-04 16:32:50 UTC (rev 7131)
@@ -17,7 +17,7 @@
if [ "$SITE" == "beagle" ]
then
- ./compile
+# ./compile
cp exec_server /lustre/beagle/yadunandb/bin/
cp exec_client /lustre/beagle/yadunandb/bin/
fi;
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V2/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V2/teragen.swift 2013-10-04 15:50:50 UTC (rev 7130)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V2/teragen.swift 2013-10-04 16:32:50 UTC (rev 7131)
@@ -1,22 +1,31 @@
type file;
type script;
-app (file out, file err) map_data (script run, int recsize)
+// type fileptr will be a regular file to swift, but the contents of which, will be
+// interpreted as globally addressable paths to one or more files.
+// The current format of a fileptr is:
+// Node0 /path/to/file0 /path/to/file1 /path/to/file2 ...
+// ...
+// Noden /path/to/file0 ...
+// Evidently, there is room for improvement and extension.
+type fileptr;
+
+app (fileptr out, file err) map_data (script run, int recsize)
{
bash @run recsize stdout=@out stderr=@err;
}
-app (file out, file err) comb_data (script comb, string node_string)
+app (fileptr out, file err) comb_data (script comb, string node_string)
{
bash @comb node_string stdout=@out stderr=@err;
}
-app (file out, file err) reduce_data (script comb, file array[])
+app (file out, file err) reduce_data (script comb, fileptr array[])
{
bash @comb @array stdout=@out stderr=@err;
}
-app (file out, file err) get_uniq_nodes (script uniq, file filepointers[])
+app (file out, file err) get_uniq_nodes (script uniq, fileptr filepointers[])
{
bash @uniq @filepointers stdout=@out stderr=@err;
}
@@ -30,22 +39,22 @@
script combiner <"combiner.sh">;
script reducer <"reducer.sh">;
-file tgen_out[] <simple_mapper; prefix="tgen", suffix=".out">;
-file tgen_err[] <simple_mapper; prefix="tgen", suffix=".err">;
+fileptr tgen_out[] <simple_mapper; prefix="tgen", suffix=".out">;
+file tgen_err[] <simple_mapper; prefix="tgen", suffix=".err">;
// Map stage. File pointers tgen_out[]
foreach item,i in [0:files-1] {
(tgen_out[i], tgen_err[i]) = map_data(mapper, fsize);
}
// Get nodes and intermediate file local to node
-file node_out <simple_mapper; prefix="node", suffix=".out">;
-file node_err <simple_mapper; prefix="node", suffix=".err">;
+file node_out <simple_mapper; prefix="node", suffix=".out">;
+file node_err <simple_mapper; prefix="node", suffix=".err">;
(node_out, node_err) = get_uniq_nodes (uniq, tgen_out);
string nodes[] = readData(node_out);
// Invoke the remote_combiner on each node involved in map stage stage
-file comb_out[] <simple_mapper; prefix="comb", suffix=".out">;
-file comb_err[] <simple_mapper; prefix="comb", suffix=".err">;
+fileptr comb_out[] <simple_mapper; prefix="comb", suffix=".out">;
+file comb_err[] <simple_mapper; prefix="comb", suffix=".err">;
foreach node_info,i in nodes {
(comb_out[i], comb_err[i]) = comb_data(combiner, node_info);
}
More information about the Swift-commit
mailing list